LLVM  15.0.0git
LegalizeDAG.cpp
Go to the documentation of this file.
1 //===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the SelectionDAG::Legalize method.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/ADT/APFloat.h"
14 #include "llvm/ADT/APInt.h"
15 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/SetVector.h"
18 #include "llvm/ADT/SmallPtrSet.h"
19 #include "llvm/ADT/SmallSet.h"
20 #include "llvm/ADT/SmallVector.h"
33 #include "llvm/IR/CallingConv.h"
34 #include "llvm/IR/Constants.h"
35 #include "llvm/IR/DataLayout.h"
36 #include "llvm/IR/DerivedTypes.h"
37 #include "llvm/IR/Function.h"
38 #include "llvm/IR/Metadata.h"
39 #include "llvm/IR/Type.h"
40 #include "llvm/Support/Casting.h"
41 #include "llvm/Support/Compiler.h"
42 #include "llvm/Support/Debug.h"
49 #include <cassert>
50 #include <cstdint>
51 #include <tuple>
52 #include <utility>
53 
54 using namespace llvm;
55 
56 #define DEBUG_TYPE "legalizedag"
57 
58 namespace {
59 
60 /// Keeps track of state when getting the sign of a floating-point value as an
61 /// integer.
62 struct FloatSignAsInt {
63  EVT FloatVT;
64  SDValue Chain;
65  SDValue FloatPtr;
66  SDValue IntPtr;
67  MachinePointerInfo IntPointerInfo;
68  MachinePointerInfo FloatPointerInfo;
69  SDValue IntValue;
70  APInt SignMask;
71  uint8_t SignBit;
72 };
73 
74 //===----------------------------------------------------------------------===//
75 /// This takes an arbitrary SelectionDAG as input and
76 /// hacks on it until the target machine can handle it. This involves
77 /// eliminating value sizes the machine cannot handle (promoting small sizes to
78 /// large sizes or splitting up large values into small values) as well as
79 /// eliminating operations the machine cannot handle.
80 ///
81 /// This code also does a small amount of optimization and recognition of idioms
82 /// as part of its processing. For example, if a target does not support a
83 /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
84 /// will attempt merge setcc and brc instructions into brcc's.
85 class SelectionDAGLegalize {
86  const TargetMachine &TM;
87  const TargetLowering &TLI;
88  SelectionDAG &DAG;
89 
90  /// The set of nodes which have already been legalized. We hold a
91  /// reference to it in order to update as necessary on node deletion.
92  SmallPtrSetImpl<SDNode *> &LegalizedNodes;
93 
94  /// A set of all the nodes updated during legalization.
95  SmallSetVector<SDNode *, 16> *UpdatedNodes;
96 
97  EVT getSetCCResultType(EVT VT) const {
98  return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
99  }
100 
101  // Libcall insertion helpers.
102 
103 public:
104  SelectionDAGLegalize(SelectionDAG &DAG,
105  SmallPtrSetImpl<SDNode *> &LegalizedNodes,
106  SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
107  : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
108  LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
109 
110  /// Legalizes the given operation.
111  void LegalizeOp(SDNode *Node);
112 
113 private:
114  SDValue OptimizeFloatStore(StoreSDNode *ST);
115 
116  void LegalizeLoadOps(SDNode *Node);
117  void LegalizeStoreOps(SDNode *Node);
118 
119  /// Some targets cannot handle a variable
120  /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
121  /// is necessary to spill the vector being inserted into to memory, perform
122  /// the insert there, and then read the result back.
123  SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
124  const SDLoc &dl);
125  SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
126  const SDLoc &dl);
127 
128  /// Return a vector shuffle operation which
129  /// performs the same shuffe in terms of order or result bytes, but on a type
130  /// whose vector element type is narrower than the original shuffle type.
131  /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
132  SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
133  SDValue N1, SDValue N2,
134  ArrayRef<int> Mask) const;
135 
136  SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
137 
138  void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
140  void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
141  RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
142  RTLIB::Libcall Call_F128,
143  RTLIB::Libcall Call_PPCF128,
145  SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, RTLIB::Libcall Call_I8,
146  RTLIB::Libcall Call_I16, RTLIB::Libcall Call_I32,
147  RTLIB::Libcall Call_I64, RTLIB::Libcall Call_I128,
148  RTLIB::Libcall Call_IEXT);
149  void ExpandArgFPLibCall(SDNode *Node,
150  RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
151  RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
152  RTLIB::Libcall Call_PPCF128,
154  void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
155  void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
156 
157  SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
158  const SDLoc &dl);
159  SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
160  const SDLoc &dl, SDValue ChainIn);
161  SDValue ExpandBUILD_VECTOR(SDNode *Node);
162  SDValue ExpandSPLAT_VECTOR(SDNode *Node);
163  SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
164  void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
166  void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
167  SDValue Value) const;
168  SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
169  SDValue NewIntValue) const;
170  SDValue ExpandFCOPYSIGN(SDNode *Node) const;
171  SDValue ExpandFABS(SDNode *Node) const;
172  SDValue ExpandFNEG(SDNode *Node) const;
173  SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
174  void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
176  void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
178  SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
179 
180  SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
181 
182  SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
183  SDValue ExpandInsertToVectorThroughStack(SDValue Op);
184  SDValue ExpandVectorBuildThroughStack(SDNode* Node);
185 
186  SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
187  SDValue ExpandConstant(ConstantSDNode *CP);
188 
189  // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
190  bool ExpandNode(SDNode *Node);
191  void ConvertNodeToLibcall(SDNode *Node);
192  void PromoteNode(SDNode *Node);
193 
194 public:
195  // Node replacement helpers
196 
197  void ReplacedNode(SDNode *N) {
198  LegalizedNodes.erase(N);
199  if (UpdatedNodes)
200  UpdatedNodes->insert(N);
201  }
202 
203  void ReplaceNode(SDNode *Old, SDNode *New) {
204  LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
205  dbgs() << " with: "; New->dump(&DAG));
206 
207  assert(Old->getNumValues() == New->getNumValues() &&
208  "Replacing one node with another that produces a different number "
209  "of values!");
210  DAG.ReplaceAllUsesWith(Old, New);
211  if (UpdatedNodes)
212  UpdatedNodes->insert(New);
213  ReplacedNode(Old);
214  }
215 
216  void ReplaceNode(SDValue Old, SDValue New) {
217  LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
218  dbgs() << " with: "; New->dump(&DAG));
219 
220  DAG.ReplaceAllUsesWith(Old, New);
221  if (UpdatedNodes)
222  UpdatedNodes->insert(New.getNode());
223  ReplacedNode(Old.getNode());
224  }
225 
226  void ReplaceNode(SDNode *Old, const SDValue *New) {
227  LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
228 
229  DAG.ReplaceAllUsesWith(Old, New);
230  for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
231  LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
232  New[i]->dump(&DAG));
233  if (UpdatedNodes)
234  UpdatedNodes->insert(New[i].getNode());
235  }
236  ReplacedNode(Old);
237  }
238 
239  void ReplaceNodeWithValue(SDValue Old, SDValue New) {
240  LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
241  dbgs() << " with: "; New->dump(&DAG));
242 
243  DAG.ReplaceAllUsesOfValueWith(Old, New);
244  if (UpdatedNodes)
245  UpdatedNodes->insert(New.getNode());
246  ReplacedNode(Old.getNode());
247  }
248 };
249 
250 } // end anonymous namespace
251 
252 /// Return a vector shuffle operation which
253 /// performs the same shuffle in terms of order or result bytes, but on a type
254 /// whose vector element type is narrower than the original shuffle type.
255 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
256 SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
257  EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
258  ArrayRef<int> Mask) const {
259  unsigned NumMaskElts = VT.getVectorNumElements();
260  unsigned NumDestElts = NVT.getVectorNumElements();
261  unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
262 
263  assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
264 
265  if (NumEltsGrowth == 1)
266  return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
267 
268  SmallVector<int, 8> NewMask;
269  for (unsigned i = 0; i != NumMaskElts; ++i) {
270  int Idx = Mask[i];
271  for (unsigned j = 0; j != NumEltsGrowth; ++j) {
272  if (Idx < 0)
273  NewMask.push_back(-1);
274  else
275  NewMask.push_back(Idx * NumEltsGrowth + j);
276  }
277  }
278  assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
279  assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
280  return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
281 }
282 
283 /// Expands the ConstantFP node to an integer constant or
284 /// a load from the constant pool.
285 SDValue
286 SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
287  bool Extend = false;
288  SDLoc dl(CFP);
289 
290  // If a FP immediate is precise when represented as a float and if the
291  // target can do an extending load from float to double, we put it into
292  // the constant pool as a float, even if it's is statically typed as a
293  // double. This shrinks FP constants and canonicalizes them for targets where
294  // an FP extending load is the same cost as a normal load (such as on the x87
295  // fp stack or PPC FP unit).
296  EVT VT = CFP->getValueType(0);
297  ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
298  if (!UseCP) {
299  assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
300  return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
301  (VT == MVT::f64) ? MVT::i64 : MVT::i32);
302  }
303 
304  APFloat APF = CFP->getValueAPF();
305  EVT OrigVT = VT;
306  EVT SVT = VT;
307 
308  // We don't want to shrink SNaNs. Converting the SNaN back to its real type
309  // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
310  if (!APF.isSignaling()) {
311  while (SVT != MVT::f32 && SVT != MVT::f16) {
312  SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
314  // Only do this if the target has a native EXTLOAD instruction from
315  // smaller type.
316  TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
317  TLI.ShouldShrinkFPConstant(OrigVT)) {
318  Type *SType = SVT.getTypeForEVT(*DAG.getContext());
319  LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
320  VT = SVT;
321  Extend = true;
322  }
323  }
324  }
325 
326  SDValue CPIdx =
327  DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
328  Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
329  if (Extend) {
330  SDValue Result = DAG.getExtLoad(
331  ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
332  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
333  Alignment);
334  return Result;
335  }
336  SDValue Result = DAG.getLoad(
337  OrigVT, dl, DAG.getEntryNode(), CPIdx,
338  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
339  return Result;
340 }
341 
342 /// Expands the Constant node to a load from the constant pool.
343 SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
344  SDLoc dl(CP);
345  EVT VT = CP->getValueType(0);
346  SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
347  TLI.getPointerTy(DAG.getDataLayout()));
348  Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
349  SDValue Result = DAG.getLoad(
350  VT, dl, DAG.getEntryNode(), CPIdx,
351  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
352  return Result;
353 }
354 
355 /// Some target cannot handle a variable insertion index for the
356 /// INSERT_VECTOR_ELT instruction. In this case, it
357 /// is necessary to spill the vector being inserted into to memory, perform
358 /// the insert there, and then read the result back.
359 SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
360  SDValue Val,
361  SDValue Idx,
362  const SDLoc &dl) {
363  SDValue Tmp1 = Vec;
364  SDValue Tmp2 = Val;
365  SDValue Tmp3 = Idx;
366 
367  // If the target doesn't support this, we have to spill the input vector
368  // to a temporary stack slot, update the element, then reload it. This is
369  // badness. We could also load the value into a vector register (either
370  // with a "move to register" or "extload into register" instruction, then
371  // permute it into place, if the idx is a constant and if the idx is
372  // supported by the target.
373  EVT VT = Tmp1.getValueType();
374  EVT EltVT = VT.getVectorElementType();
375  SDValue StackPtr = DAG.CreateStackTemporary(VT);
376 
377  int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
378 
379  // Store the vector.
380  SDValue Ch = DAG.getStore(
381  DAG.getEntryNode(), dl, Tmp1, StackPtr,
382  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
383 
384  SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
385 
386  // Store the scalar value.
387  Ch = DAG.getTruncStore(
388  Ch, dl, Tmp2, StackPtr2,
389  MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT);
390  // Load the updated vector.
391  return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
392  DAG.getMachineFunction(), SPFI));
393 }
394 
395 SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
396  SDValue Idx,
397  const SDLoc &dl) {
398  if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
399  // SCALAR_TO_VECTOR requires that the type of the value being inserted
400  // match the element type of the vector being created, except for
401  // integers in which case the inserted value can be over width.
402  EVT EltVT = Vec.getValueType().getVectorElementType();
403  if (Val.getValueType() == EltVT ||
404  (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
405  SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
406  Vec.getValueType(), Val);
407 
408  unsigned NumElts = Vec.getValueType().getVectorNumElements();
409  // We generate a shuffle of InVec and ScVec, so the shuffle mask
410  // should be 0,1,2,3,4,5... with the appropriate element replaced with
411  // elt 0 of the RHS.
412  SmallVector<int, 8> ShufOps;
413  for (unsigned i = 0; i != NumElts; ++i)
414  ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
415 
416  return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
417  }
418  }
419  return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
420 }
421 
422 SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
423  if (!ISD::isNormalStore(ST))
424  return SDValue();
425 
426  LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
427  // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
428  // FIXME: move this to the DAG Combiner! Note that we can't regress due
429  // to phase ordering between legalized code and the dag combiner. This
430  // probably means that we need to integrate dag combiner and legalizer
431  // together.
432  // We generally can't do this one for long doubles.
433  SDValue Chain = ST->getChain();
434  SDValue Ptr = ST->getBasePtr();
435  SDValue Value = ST->getValue();
436  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
437  AAMDNodes AAInfo = ST->getAAInfo();
438  SDLoc dl(ST);
439 
440  // Don't optimise TargetConstantFP
441  if (Value.getOpcode() == ISD::TargetConstantFP)
442  return SDValue();
443 
444  if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) {
445  if (CFP->getValueType(0) == MVT::f32 &&
446  TLI.isTypeLegal(MVT::i32)) {
447  SDValue Con = DAG.getConstant(CFP->getValueAPF().
448  bitcastToAPInt().zextOrTrunc(32),
449  SDLoc(CFP), MVT::i32);
450  return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
451  ST->getOriginalAlign(), MMOFlags, AAInfo);
452  }
453 
454  if (CFP->getValueType(0) == MVT::f64) {
455  // If this target supports 64-bit registers, do a single 64-bit store.
456  if (TLI.isTypeLegal(MVT::i64)) {
457  SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
458  zextOrTrunc(64), SDLoc(CFP), MVT::i64);
459  return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
460  ST->getOriginalAlign(), MMOFlags, AAInfo);
461  }
462 
463  if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
464  // Otherwise, if the target supports 32-bit registers, use 2 32-bit
465  // stores. If the target supports neither 32- nor 64-bits, this
466  // xform is certainly not worth it.
467  const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
468  SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
469  SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
470  if (DAG.getDataLayout().isBigEndian())
471  std::swap(Lo, Hi);
472 
473  Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(),
474  ST->getOriginalAlign(), MMOFlags, AAInfo);
475  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(4), dl);
476  Hi = DAG.getStore(Chain, dl, Hi, Ptr,
477  ST->getPointerInfo().getWithOffset(4),
478  ST->getOriginalAlign(), MMOFlags, AAInfo);
479 
480  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
481  }
482  }
483  }
484  return SDValue();
485 }
486 
487 void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
488  StoreSDNode *ST = cast<StoreSDNode>(Node);
489  SDValue Chain = ST->getChain();
490  SDValue Ptr = ST->getBasePtr();
491  SDLoc dl(Node);
492 
493  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
494  AAMDNodes AAInfo = ST->getAAInfo();
495 
496  if (!ST->isTruncatingStore()) {
497  LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
498  if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
499  ReplaceNode(ST, OptStore);
500  return;
501  }
502 
503  SDValue Value = ST->getValue();
504  MVT VT = Value.getSimpleValueType();
505  switch (TLI.getOperationAction(ISD::STORE, VT)) {
506  default: llvm_unreachable("This action is not supported yet!");
507  case TargetLowering::Legal: {
508  // If this is an unaligned store and the target doesn't support it,
509  // expand it.
510  EVT MemVT = ST->getMemoryVT();
511  const DataLayout &DL = DAG.getDataLayout();
512  if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
513  *ST->getMemOperand())) {
514  LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
515  SDValue Result = TLI.expandUnalignedStore(ST, DAG);
516  ReplaceNode(SDValue(ST, 0), Result);
517  } else
518  LLVM_DEBUG(dbgs() << "Legal store\n");
519  break;
520  }
521  case TargetLowering::Custom: {
522  LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
523  SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
524  if (Res && Res != SDValue(Node, 0))
525  ReplaceNode(SDValue(Node, 0), Res);
526  return;
527  }
529  MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
530  assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
531  "Can only promote stores to same size type");
532  Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
533  SDValue Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
534  ST->getOriginalAlign(), MMOFlags, AAInfo);
535  ReplaceNode(SDValue(Node, 0), Result);
536  break;
537  }
538  }
539  return;
540  }
541 
542  LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
543  SDValue Value = ST->getValue();
544  EVT StVT = ST->getMemoryVT();
545  TypeSize StWidth = StVT.getSizeInBits();
546  TypeSize StSize = StVT.getStoreSizeInBits();
547  auto &DL = DAG.getDataLayout();
548 
549  if (StWidth != StSize) {
550  // Promote to a byte-sized store with upper bits zero if not
551  // storing an integral number of bytes. For example, promote
552  // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
553  EVT NVT = EVT::getIntegerVT(*DAG.getContext(), StSize.getFixedSize());
554  Value = DAG.getZeroExtendInReg(Value, dl, StVT);
555  SDValue Result =
556  DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
557  ST->getOriginalAlign(), MMOFlags, AAInfo);
558  ReplaceNode(SDValue(Node, 0), Result);
559  } else if (!StVT.isVector() && !isPowerOf2_64(StWidth.getFixedSize())) {
560  // If not storing a power-of-2 number of bits, expand as two stores.
561  assert(!StVT.isVector() && "Unsupported truncstore!");
562  unsigned StWidthBits = StWidth.getFixedSize();
563  unsigned LogStWidth = Log2_32(StWidthBits);
564  assert(LogStWidth < 32);
565  unsigned RoundWidth = 1 << LogStWidth;
566  assert(RoundWidth < StWidthBits);
567  unsigned ExtraWidth = StWidthBits - RoundWidth;
568  assert(ExtraWidth < RoundWidth);
569  assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
570  "Store size not an integral number of bytes!");
571  EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
572  EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
573  SDValue Lo, Hi;
574  unsigned IncrementSize;
575 
576  if (DL.isLittleEndian()) {
577  // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
578  // Store the bottom RoundWidth bits.
579  Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
580  RoundVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
581 
582  // Store the remaining ExtraWidth bits.
583  IncrementSize = RoundWidth / 8;
584  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
585  Hi = DAG.getNode(
586  ISD::SRL, dl, Value.getValueType(), Value,
587  DAG.getConstant(RoundWidth, dl,
588  TLI.getShiftAmountTy(Value.getValueType(), DL)));
589  Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr,
590  ST->getPointerInfo().getWithOffset(IncrementSize),
591  ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
592  } else {
593  // Big endian - avoid unaligned stores.
594  // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
595  // Store the top RoundWidth bits.
596  Hi = DAG.getNode(
597  ISD::SRL, dl, Value.getValueType(), Value,
598  DAG.getConstant(ExtraWidth, dl,
599  TLI.getShiftAmountTy(Value.getValueType(), DL)));
600  Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), RoundVT,
601  ST->getOriginalAlign(), MMOFlags, AAInfo);
602 
603  // Store the remaining ExtraWidth bits.
604  IncrementSize = RoundWidth / 8;
605  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
606  DAG.getConstant(IncrementSize, dl,
607  Ptr.getValueType()));
608  Lo = DAG.getTruncStore(Chain, dl, Value, Ptr,
609  ST->getPointerInfo().getWithOffset(IncrementSize),
610  ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
611  }
612 
613  // The order of the stores doesn't matter.
614  SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
615  ReplaceNode(SDValue(Node, 0), Result);
616  } else {
617  switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
618  default: llvm_unreachable("This action is not supported yet!");
619  case TargetLowering::Legal: {
620  EVT MemVT = ST->getMemoryVT();
621  // If this is an unaligned store and the target doesn't support it,
622  // expand it.
623  if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
624  *ST->getMemOperand())) {
625  SDValue Result = TLI.expandUnalignedStore(ST, DAG);
626  ReplaceNode(SDValue(ST, 0), Result);
627  }
628  break;
629  }
630  case TargetLowering::Custom: {
631  SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
632  if (Res && Res != SDValue(Node, 0))
633  ReplaceNode(SDValue(Node, 0), Res);
634  return;
635  }
637  assert(!StVT.isVector() &&
638  "Vector Stores are handled in LegalizeVectorOps");
639 
640  SDValue Result;
641 
642  // TRUNCSTORE:i16 i32 -> STORE i16
643  if (TLI.isTypeLegal(StVT)) {
644  Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
645  Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
646  ST->getOriginalAlign(), MMOFlags, AAInfo);
647  } else {
648  // The in-memory type isn't legal. Truncate to the type it would promote
649  // to, and then do a truncstore.
650  Value = DAG.getNode(ISD::TRUNCATE, dl,
651  TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
652  Value);
653  Result =
654  DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), StVT,
655  ST->getOriginalAlign(), MMOFlags, AAInfo);
656  }
657 
658  ReplaceNode(SDValue(Node, 0), Result);
659  break;
660  }
661  }
662 }
663 
664 void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
665  LoadSDNode *LD = cast<LoadSDNode>(Node);
666  SDValue Chain = LD->getChain(); // The chain.
667  SDValue Ptr = LD->getBasePtr(); // The base pointer.
668  SDValue Value; // The value returned by the load op.
669  SDLoc dl(Node);
670 
671  ISD::LoadExtType ExtType = LD->getExtensionType();
672  if (ExtType == ISD::NON_EXTLOAD) {
673  LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
674  MVT VT = Node->getSimpleValueType(0);
675  SDValue RVal = SDValue(Node, 0);
676  SDValue RChain = SDValue(Node, 1);
677 
678  switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
679  default: llvm_unreachable("This action is not supported yet!");
680  case TargetLowering::Legal: {
681  EVT MemVT = LD->getMemoryVT();
682  const DataLayout &DL = DAG.getDataLayout();
683  // If this is an unaligned load and the target doesn't support it,
684  // expand it.
685  if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
686  *LD->getMemOperand())) {
687  std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
688  }
689  break;
690  }
692  if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
693  RVal = Res;
694  RChain = Res.getValue(1);
695  }
696  break;
697 
699  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
700  assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
701  "Can only promote loads to same size type");
702 
703  SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
704  RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
705  RChain = Res.getValue(1);
706  break;
707  }
708  }
709  if (RChain.getNode() != Node) {
710  assert(RVal.getNode() != Node && "Load must be completely replaced");
711  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
712  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
713  if (UpdatedNodes) {
714  UpdatedNodes->insert(RVal.getNode());
715  UpdatedNodes->insert(RChain.getNode());
716  }
717  ReplacedNode(Node);
718  }
719  return;
720  }
721 
722  LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
723  EVT SrcVT = LD->getMemoryVT();
724  TypeSize SrcWidth = SrcVT.getSizeInBits();
725  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
726  AAMDNodes AAInfo = LD->getAAInfo();
727 
728  if (SrcWidth != SrcVT.getStoreSizeInBits() &&
729  // Some targets pretend to have an i1 loading operation, and actually
730  // load an i8. This trick is correct for ZEXTLOAD because the top 7
731  // bits are guaranteed to be zero; it helps the optimizers understand
732  // that these bits are zero. It is also useful for EXTLOAD, since it
733  // tells the optimizers that those bits are undefined. It would be
734  // nice to have an effective generic way of getting these benefits...
735  // Until such a way is found, don't insist on promoting i1 here.
736  (SrcVT != MVT::i1 ||
737  TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
739  // Promote to a byte-sized load if not loading an integral number of
740  // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
741  unsigned NewWidth = SrcVT.getStoreSizeInBits();
742  EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
743  SDValue Ch;
744 
745  // The extra bits are guaranteed to be zero, since we stored them that
746  // way. A zext load from NVT thus automatically gives zext from SrcVT.
747 
748  ISD::LoadExtType NewExtType =
750 
751  SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0),
752  Chain, Ptr, LD->getPointerInfo(), NVT,
753  LD->getOriginalAlign(), MMOFlags, AAInfo);
754 
755  Ch = Result.getValue(1); // The chain.
756 
757  if (ExtType == ISD::SEXTLOAD)
758  // Having the top bits zero doesn't help when sign extending.
759  Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
760  Result.getValueType(),
761  Result, DAG.getValueType(SrcVT));
762  else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
763  // All the top bits are guaranteed to be zero - inform the optimizers.
764  Result = DAG.getNode(ISD::AssertZext, dl,
765  Result.getValueType(), Result,
766  DAG.getValueType(SrcVT));
767 
768  Value = Result;
769  Chain = Ch;
770  } else if (!isPowerOf2_64(SrcWidth.getKnownMinSize())) {
771  // If not loading a power-of-2 number of bits, expand as two loads.
772  assert(!SrcVT.isVector() && "Unsupported extload!");
773  unsigned SrcWidthBits = SrcWidth.getFixedSize();
774  unsigned LogSrcWidth = Log2_32(SrcWidthBits);
775  assert(LogSrcWidth < 32);
776  unsigned RoundWidth = 1 << LogSrcWidth;
777  assert(RoundWidth < SrcWidthBits);
778  unsigned ExtraWidth = SrcWidthBits - RoundWidth;
779  assert(ExtraWidth < RoundWidth);
780  assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
781  "Load size not an integral number of bytes!");
782  EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
783  EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
784  SDValue Lo, Hi, Ch;
785  unsigned IncrementSize;
786  auto &DL = DAG.getDataLayout();
787 
788  if (DL.isLittleEndian()) {
789  // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
790  // Load the bottom RoundWidth bits.
791  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
792  LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
793  MMOFlags, AAInfo);
794 
795  // Load the remaining ExtraWidth bits.
796  IncrementSize = RoundWidth / 8;
797  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
798  Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
799  LD->getPointerInfo().getWithOffset(IncrementSize),
800  ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
801 
802  // Build a factor node to remember that this load is independent of
803  // the other one.
804  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
805  Hi.getValue(1));
806 
807  // Move the top bits to the right place.
808  Hi = DAG.getNode(
809  ISD::SHL, dl, Hi.getValueType(), Hi,
810  DAG.getConstant(RoundWidth, dl,
811  TLI.getShiftAmountTy(Hi.getValueType(), DL)));
812 
813  // Join the hi and lo parts.
814  Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
815  } else {
816  // Big endian - avoid unaligned loads.
817  // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
818  // Load the top RoundWidth bits.
819  Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
820  LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
821  MMOFlags, AAInfo);
822 
823  // Load the remaining ExtraWidth bits.
824  IncrementSize = RoundWidth / 8;
825  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
826  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
827  LD->getPointerInfo().getWithOffset(IncrementSize),
828  ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
829 
830  // Build a factor node to remember that this load is independent of
831  // the other one.
832  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
833  Hi.getValue(1));
834 
835  // Move the top bits to the right place.
836  Hi = DAG.getNode(
837  ISD::SHL, dl, Hi.getValueType(), Hi,
838  DAG.getConstant(ExtraWidth, dl,
839  TLI.getShiftAmountTy(Hi.getValueType(), DL)));
840 
841  // Join the hi and lo parts.
842  Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
843  }
844 
845  Chain = Ch;
846  } else {
847  bool isCustom = false;
848  switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
849  SrcVT.getSimpleVT())) {
850  default: llvm_unreachable("This action is not supported yet!");
852  isCustom = true;
855  Value = SDValue(Node, 0);
856  Chain = SDValue(Node, 1);
857 
858  if (isCustom) {
859  if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
860  Value = Res;
861  Chain = Res.getValue(1);
862  }
863  } else {
864  // If this is an unaligned load and the target doesn't support it,
865  // expand it.
866  EVT MemVT = LD->getMemoryVT();
867  const DataLayout &DL = DAG.getDataLayout();
868  if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
869  *LD->getMemOperand())) {
870  std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
871  }
872  }
873  break;
874 
875  case TargetLowering::Expand: {
876  EVT DestVT = Node->getValueType(0);
877  if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
878  // If the source type is not legal, see if there is a legal extload to
879  // an intermediate type that we can then extend further.
880  EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
881  if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
882  TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
883  // If we are loading a legal type, this is a non-extload followed by a
884  // full extend.
885  ISD::LoadExtType MidExtType =
886  (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
887 
888  SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
889  SrcVT, LD->getMemOperand());
890  unsigned ExtendOp =
891  ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
892  Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
893  Chain = Load.getValue(1);
894  break;
895  }
896 
897  // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
898  // normal undefined upper bits behavior to allow using an in-reg extend
899  // with the illegal FP type, so load as an integer and do the
900  // from-integer conversion.
901  if (SrcVT.getScalarType() == MVT::f16) {
902  EVT ISrcVT = SrcVT.changeTypeToInteger();
903  EVT IDestVT = DestVT.changeTypeToInteger();
904  EVT ILoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
905 
906  SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain,
907  Ptr, ISrcVT, LD->getMemOperand());
908  Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
909  Chain = Result.getValue(1);
910  break;
911  }
912  }
913 
914  assert(!SrcVT.isVector() &&
915  "Vector Loads are handled in LegalizeVectorOps");
916 
917  // FIXME: This does not work for vectors on most targets. Sign-
918  // and zero-extend operations are currently folded into extending
919  // loads, whether they are legal or not, and then we end up here
920  // without any support for legalizing them.
921  assert(ExtType != ISD::EXTLOAD &&
922  "EXTLOAD should always be supported!");
923  // Turn the unsupported load into an EXTLOAD followed by an
924  // explicit zero/sign extend inreg.
925  SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
926  Node->getValueType(0),
927  Chain, Ptr, SrcVT,
928  LD->getMemOperand());
929  SDValue ValRes;
930  if (ExtType == ISD::SEXTLOAD)
931  ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
932  Result.getValueType(),
933  Result, DAG.getValueType(SrcVT));
934  else
935  ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
936  Value = ValRes;
937  Chain = Result.getValue(1);
938  break;
939  }
940  }
941  }
942 
943  // Since loads produce two values, make sure to remember that we legalized
944  // both of them.
945  if (Chain.getNode() != Node) {
946  assert(Value.getNode() != Node && "Load must be completely replaced");
947  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
948  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
949  if (UpdatedNodes) {
950  UpdatedNodes->insert(Value.getNode());
951  UpdatedNodes->insert(Chain.getNode());
952  }
953  ReplacedNode(Node);
954  }
955 }
956 
957 /// Return a legal replacement for the given operation, with all legal operands.
958 void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
959  LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
960 
961  // Allow illegal target nodes and illegal registers.
962  if (Node->getOpcode() == ISD::TargetConstant ||
963  Node->getOpcode() == ISD::Register)
964  return;
965 
966 #ifndef NDEBUG
967  for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
968  assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
970  "Unexpected illegal type!");
971 
972  for (const SDValue &Op : Node->op_values())
973  assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
975  Op.getOpcode() == ISD::TargetConstant ||
976  Op.getOpcode() == ISD::Register) &&
977  "Unexpected illegal type!");
978 #endif
979 
980  // Figure out the correct action; the way to query this varies by opcode
982  bool SimpleFinishLegalizing = true;
983  switch (Node->getOpcode()) {
986  case ISD::INTRINSIC_VOID:
987  case ISD::STACKSAVE:
988  Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
989  break;
991  Action = TLI.getOperationAction(Node->getOpcode(),
992  Node->getValueType(0));
993  break;
994  case ISD::VAARG:
995  Action = TLI.getOperationAction(Node->getOpcode(),
996  Node->getValueType(0));
997  if (Action != TargetLowering::Promote)
998  Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
999  break;
1000  case ISD::FP_TO_FP16:
1001  case ISD::SINT_TO_FP:
1002  case ISD::UINT_TO_FP:
1004  case ISD::LROUND:
1005  case ISD::LLROUND:
1006  case ISD::LRINT:
1007  case ISD::LLRINT:
1008  Action = TLI.getOperationAction(Node->getOpcode(),
1009  Node->getOperand(0).getValueType());
1010  break;
1014  case ISD::STRICT_LRINT:
1015  case ISD::STRICT_LLRINT:
1016  case ISD::STRICT_LROUND:
1017  case ISD::STRICT_LLROUND:
1018  // These pseudo-ops are the same as the other STRICT_ ops except
1019  // they are registered with setOperationAction() using the input type
1020  // instead of the output type.
1021  Action = TLI.getOperationAction(Node->getOpcode(),
1022  Node->getOperand(1).getValueType());
1023  break;
1024  case ISD::SIGN_EXTEND_INREG: {
1025  EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1026  Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1027  break;
1028  }
1029  case ISD::ATOMIC_STORE:
1030  Action = TLI.getOperationAction(Node->getOpcode(),
1031  Node->getOperand(2).getValueType());
1032  break;
1033  case ISD::SELECT_CC:
1034  case ISD::STRICT_FSETCC:
1035  case ISD::STRICT_FSETCCS:
1036  case ISD::SETCC:
1037  case ISD::VP_SETCC:
1038  case ISD::BR_CC: {
1039  unsigned Opc = Node->getOpcode();
1040  unsigned CCOperand = Opc == ISD::SELECT_CC ? 4
1041  : Opc == ISD::STRICT_FSETCC ? 3
1042  : Opc == ISD::STRICT_FSETCCS ? 3
1043  : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1044  : 1;
1045  unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1046  : Opc == ISD::STRICT_FSETCC ? 1
1047  : Opc == ISD::STRICT_FSETCCS ? 1
1048  : 0;
1049  MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1050  ISD::CondCode CCCode =
1051  cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1052  Action = TLI.getCondCodeAction(CCCode, OpVT);
1053  if (Action == TargetLowering::Legal) {
1054  if (Node->getOpcode() == ISD::SELECT_CC)
1055  Action = TLI.getOperationAction(Node->getOpcode(),
1056  Node->getValueType(0));
1057  else
1058  Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1059  }
1060  break;
1061  }
1062  case ISD::LOAD:
1063  case ISD::STORE:
1064  // FIXME: Model these properly. LOAD and STORE are complicated, and
1065  // STORE expects the unlegalized operand in some cases.
1066  SimpleFinishLegalizing = false;
1067  break;
1068  case ISD::CALLSEQ_START:
1069  case ISD::CALLSEQ_END:
1070  // FIXME: This shouldn't be necessary. These nodes have special properties
1071  // dealing with the recursive nature of legalization. Removing this
1072  // special case should be done as part of making LegalizeDAG non-recursive.
1073  SimpleFinishLegalizing = false;
1074  break;
1075  case ISD::EXTRACT_ELEMENT:
1076  case ISD::FLT_ROUNDS_:
1077  case ISD::MERGE_VALUES:
1078  case ISD::EH_RETURN:
1080  case ISD::EH_DWARF_CFA:
1081  case ISD::EH_SJLJ_SETJMP:
1082  case ISD::EH_SJLJ_LONGJMP:
1084  // These operations lie about being legal: when they claim to be legal,
1085  // they should actually be expanded.
1086  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1087  if (Action == TargetLowering::Legal)
1088  Action = TargetLowering::Expand;
1089  break;
1090  case ISD::INIT_TRAMPOLINE:
1092  case ISD::FRAMEADDR:
1093  case ISD::RETURNADDR:
1094  case ISD::ADDROFRETURNADDR:
1095  case ISD::SPONENTRY:
1096  // These operations lie about being legal: when they claim to be legal,
1097  // they should actually be custom-lowered.
1098  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1099  if (Action == TargetLowering::Legal)
1100  Action = TargetLowering::Custom;
1101  break;
1102  case ISD::READCYCLECOUNTER:
1103  // READCYCLECOUNTER returns an i64, even if type legalization might have
1104  // expanded that to several smaller types.
1105  Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1106  break;
1107  case ISD::READ_REGISTER:
1108  case ISD::WRITE_REGISTER:
1109  // Named register is legal in the DAG, but blocked by register name
1110  // selection if not implemented by target (to chose the correct register)
1111  // They'll be converted to Copy(To/From)Reg.
1112  Action = TargetLowering::Legal;
1113  break;
1114  case ISD::UBSANTRAP:
1115  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1116  if (Action == TargetLowering::Expand) {
1117  // replace ISD::UBSANTRAP with ISD::TRAP
1118  SDValue NewVal;
1119  NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1120  Node->getOperand(0));
1121  ReplaceNode(Node, NewVal.getNode());
1122  LegalizeOp(NewVal.getNode());
1123  return;
1124  }
1125  break;
1126  case ISD::DEBUGTRAP:
1127  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1128  if (Action == TargetLowering::Expand) {
1129  // replace ISD::DEBUGTRAP with ISD::TRAP
1130  SDValue NewVal;
1131  NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1132  Node->getOperand(0));
1133  ReplaceNode(Node, NewVal.getNode());
1134  LegalizeOp(NewVal.getNode());
1135  return;
1136  }
1137  break;
1138  case ISD::SADDSAT:
1139  case ISD::UADDSAT:
1140  case ISD::SSUBSAT:
1141  case ISD::USUBSAT:
1142  case ISD::SSHLSAT:
1143  case ISD::USHLSAT:
1144  case ISD::FP_TO_SINT_SAT:
1145  case ISD::FP_TO_UINT_SAT:
1146  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1147  break;
1148  case ISD::SMULFIX:
1149  case ISD::SMULFIXSAT:
1150  case ISD::UMULFIX:
1151  case ISD::UMULFIXSAT:
1152  case ISD::SDIVFIX:
1153  case ISD::SDIVFIXSAT:
1154  case ISD::UDIVFIX:
1155  case ISD::UDIVFIXSAT: {
1156  unsigned Scale = Node->getConstantOperandVal(2);
1157  Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
1158  Node->getValueType(0), Scale);
1159  break;
1160  }
1161  case ISD::MSCATTER:
1162  Action = TLI.getOperationAction(Node->getOpcode(),
1163  cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
1164  break;
1165  case ISD::MSTORE:
1166  Action = TLI.getOperationAction(Node->getOpcode(),
1167  cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
1168  break;
1169  case ISD::VP_SCATTER:
1170  Action = TLI.getOperationAction(
1171  Node->getOpcode(),
1172  cast<VPScatterSDNode>(Node)->getValue().getValueType());
1173  break;
1174  case ISD::VP_STORE:
1175  Action = TLI.getOperationAction(
1176  Node->getOpcode(),
1177  cast<VPStoreSDNode>(Node)->getValue().getValueType());
1178  break;
1179  case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1180  Action = TLI.getOperationAction(
1181  Node->getOpcode(),
1182  cast<VPStridedStoreSDNode>(Node)->getValue().getValueType());
1183  break;
1184  case ISD::VECREDUCE_FADD:
1185  case ISD::VECREDUCE_FMUL:
1186  case ISD::VECREDUCE_ADD:
1187  case ISD::VECREDUCE_MUL:
1188  case ISD::VECREDUCE_AND:
1189  case ISD::VECREDUCE_OR:
1190  case ISD::VECREDUCE_XOR:
1191  case ISD::VECREDUCE_SMAX:
1192  case ISD::VECREDUCE_SMIN:
1193  case ISD::VECREDUCE_UMAX:
1194  case ISD::VECREDUCE_UMIN:
1195  case ISD::VECREDUCE_FMAX:
1196  case ISD::VECREDUCE_FMIN:
1197  case ISD::IS_FPCLASS:
1198  Action = TLI.getOperationAction(
1199  Node->getOpcode(), Node->getOperand(0).getValueType());
1200  break;
1203  case ISD::VP_REDUCE_FADD:
1204  case ISD::VP_REDUCE_FMUL:
1205  case ISD::VP_REDUCE_ADD:
1206  case ISD::VP_REDUCE_MUL:
1207  case ISD::VP_REDUCE_AND:
1208  case ISD::VP_REDUCE_OR:
1209  case ISD::VP_REDUCE_XOR:
1210  case ISD::VP_REDUCE_SMAX:
1211  case ISD::VP_REDUCE_SMIN:
1212  case ISD::VP_REDUCE_UMAX:
1213  case ISD::VP_REDUCE_UMIN:
1214  case ISD::VP_REDUCE_FMAX:
1215  case ISD::VP_REDUCE_FMIN:
1216  case ISD::VP_REDUCE_SEQ_FADD:
1217  case ISD::VP_REDUCE_SEQ_FMUL:
1218  Action = TLI.getOperationAction(
1219  Node->getOpcode(), Node->getOperand(1).getValueType());
1220  break;
1221  default:
1222  if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1223  Action = TLI.getCustomOperationAction(*Node);
1224  } else {
1225  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1226  }
1227  break;
1228  }
1229 
1230  if (SimpleFinishLegalizing) {
1231  SDNode *NewNode = Node;
1232  switch (Node->getOpcode()) {
1233  default: break;
1234  case ISD::SHL:
1235  case ISD::SRL:
1236  case ISD::SRA:
1237  case ISD::ROTL:
1238  case ISD::ROTR: {
1239  // Legalizing shifts/rotates requires adjusting the shift amount
1240  // to the appropriate width.
1241  SDValue Op0 = Node->getOperand(0);
1242  SDValue Op1 = Node->getOperand(1);
1243  if (!Op1.getValueType().isVector()) {
1244  SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1245  // The getShiftAmountOperand() may create a new operand node or
1246  // return the existing one. If new operand is created we need
1247  // to update the parent node.
1248  // Do not try to legalize SAO here! It will be automatically legalized
1249  // in the next round.
1250  if (SAO != Op1)
1251  NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1252  }
1253  }
1254  break;
1255  case ISD::FSHL:
1256  case ISD::FSHR:
1257  case ISD::SRL_PARTS:
1258  case ISD::SRA_PARTS:
1259  case ISD::SHL_PARTS: {
1260  // Legalizing shifts/rotates requires adjusting the shift amount
1261  // to the appropriate width.
1262  SDValue Op0 = Node->getOperand(0);
1263  SDValue Op1 = Node->getOperand(1);
1264  SDValue Op2 = Node->getOperand(2);
1265  if (!Op2.getValueType().isVector()) {
1266  SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1267  // The getShiftAmountOperand() may create a new operand node or
1268  // return the existing one. If new operand is created we need
1269  // to update the parent node.
1270  if (SAO != Op2)
1271  NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1272  }
1273  break;
1274  }
1275  }
1276 
1277  if (NewNode != Node) {
1278  ReplaceNode(Node, NewNode);
1279  Node = NewNode;
1280  }
1281  switch (Action) {
1282  case TargetLowering::Legal:
1283  LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1284  return;
1286  LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1287  // FIXME: The handling for custom lowering with multiple results is
1288  // a complete mess.
1289  if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1290  if (!(Res.getNode() != Node || Res.getResNo() != 0))
1291  return;
1292 
1293  if (Node->getNumValues() == 1) {
1294  // Verify the new types match the original. Glue is waived because
1295  // ISD::ADDC can be legalized by replacing Glue with an integer type.
1296  assert((Res.getValueType() == Node->getValueType(0) ||
1297  Node->getValueType(0) == MVT::Glue) &&
1298  "Type mismatch for custom legalized operation");
1299  LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1300  // We can just directly replace this node with the lowered value.
1301  ReplaceNode(SDValue(Node, 0), Res);
1302  return;
1303  }
1304 
1305  SmallVector<SDValue, 8> ResultVals;
1306  for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1307  // Verify the new types match the original. Glue is waived because
1308  // ISD::ADDC can be legalized by replacing Glue with an integer type.
1309  assert((Res->getValueType(i) == Node->getValueType(i) ||
1310  Node->getValueType(i) == MVT::Glue) &&
1311  "Type mismatch for custom legalized operation");
1312  ResultVals.push_back(Res.getValue(i));
1313  }
1314  LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1315  ReplaceNode(Node, ResultVals.data());
1316  return;
1317  }
1318  LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1321  if (ExpandNode(Node))
1322  return;
1325  ConvertNodeToLibcall(Node);
1326  return;
1328  PromoteNode(Node);
1329  return;
1330  }
1331  }
1332 
1333  switch (Node->getOpcode()) {
1334  default:
1335 #ifndef NDEBUG
1336  dbgs() << "NODE: ";
1337  Node->dump( &DAG);
1338  dbgs() << "\n";
1339 #endif
1340  llvm_unreachable("Do not know how to legalize this operator!");
1341 
1342  case ISD::CALLSEQ_START:
1343  case ISD::CALLSEQ_END:
1344  break;
1345  case ISD::LOAD:
1346  return LegalizeLoadOps(Node);
1347  case ISD::STORE:
1348  return LegalizeStoreOps(Node);
1349  }
1350 }
1351 
1352 SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1353  SDValue Vec = Op.getOperand(0);
1354  SDValue Idx = Op.getOperand(1);
1355  SDLoc dl(Op);
1356 
1357  // Before we generate a new store to a temporary stack slot, see if there is
1358  // already one that we can use. There often is because when we scalarize
1359  // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1360  // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1361  // the vector. If all are expanded here, we don't want one store per vector
1362  // element.
1363 
1364  // Caches for hasPredecessorHelper
1367  Visited.insert(Op.getNode());
1368  Worklist.push_back(Idx.getNode());
1369  SDValue StackPtr, Ch;
1370  for (SDNode *User : Vec.getNode()->uses()) {
1371  if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1372  if (ST->isIndexed() || ST->isTruncatingStore() ||
1373  ST->getValue() != Vec)
1374  continue;
1375 
1376  // Make sure that nothing else could have stored into the destination of
1377  // this store.
1378  if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1379  continue;
1380 
1381  // If the index is dependent on the store we will introduce a cycle when
1382  // creating the load (the load uses the index, and by replacing the chain
1383  // we will make the index dependent on the load). Also, the store might be
1384  // dependent on the extractelement and introduce a cycle when creating
1385  // the load.
1386  if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1387  ST->hasPredecessor(Op.getNode()))
1388  continue;
1389 
1390  StackPtr = ST->getBasePtr();
1391  Ch = SDValue(ST, 0);
1392  break;
1393  }
1394  }
1395 
1396  EVT VecVT = Vec.getValueType();
1397 
1398  if (!Ch.getNode()) {
1399  // Store the value to a temporary stack slot, then LOAD the returned part.
1400  StackPtr = DAG.CreateStackTemporary(VecVT);
1401  Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1402  MachinePointerInfo());
1403  }
1404 
1405  SDValue NewLoad;
1406 
1407  if (Op.getValueType().isVector()) {
1408  StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1409  Op.getValueType(), Idx);
1410  NewLoad =
1411  DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
1412  } else {
1413  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1414  NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1416  VecVT.getVectorElementType());
1417  }
1418 
1419  // Replace the chain going out of the store, by the one out of the load.
1420  DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1421 
1422  // We introduced a cycle though, so update the loads operands, making sure
1423  // to use the original store's chain as an incoming chain.
1424  SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1425  NewLoad->op_end());
1426  NewLoadOperands[0] = Ch;
1427  NewLoad =
1428  SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1429  return NewLoad;
1430 }
1431 
1432 SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1433  assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1434 
1435  SDValue Vec = Op.getOperand(0);
1436  SDValue Part = Op.getOperand(1);
1437  SDValue Idx = Op.getOperand(2);
1438  SDLoc dl(Op);
1439 
1440  // Store the value to a temporary stack slot, then LOAD the returned part.
1441  EVT VecVT = Vec.getValueType();
1442  EVT SubVecVT = Part.getValueType();
1443  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1444  int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1445  MachinePointerInfo PtrInfo =
1446  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1447 
1448  // First store the whole vector.
1449  SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1450 
1451  // Then store the inserted part.
1452  SDValue SubStackPtr =
1453  TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1454 
1455  // Store the subvector.
1456  Ch = DAG.getStore(
1457  Ch, dl, Part, SubStackPtr,
1458  MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
1459 
1460  // Finally, load the updated vector.
1461  return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1462 }
1463 
1464 SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1465  assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1466  Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1467  "Unexpected opcode!");
1468 
1469  // We can't handle this case efficiently. Allocate a sufficiently
1470  // aligned object on the stack, store each operand into it, then load
1471  // the result as a vector.
1472  // Create the stack frame object.
1473  EVT VT = Node->getValueType(0);
1474  EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType()
1475  : Node->getOperand(0).getValueType();
1476  SDLoc dl(Node);
1477  SDValue FIPtr = DAG.CreateStackTemporary(VT);
1478  int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1479  MachinePointerInfo PtrInfo =
1480  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1481 
1482  // Emit a store of each element to the stack slot.
1483  SmallVector<SDValue, 8> Stores;
1484  unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1485  assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1486 
1487  // If the destination vector element type of a BUILD_VECTOR is narrower than
1488  // the source element type, only store the bits necessary.
1489  bool Truncate = isa<BuildVectorSDNode>(Node) &&
1490  MemVT.bitsLT(Node->getOperand(0).getValueType());
1491 
1492  // Store (in the right endianness) the elements to memory.
1493  for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1494  // Ignore undef elements.
1495  if (Node->getOperand(i).isUndef()) continue;
1496 
1497  unsigned Offset = TypeByteSize*i;
1498 
1499  SDValue Idx = DAG.getMemBasePlusOffset(FIPtr, TypeSize::Fixed(Offset), dl);
1500 
1501  if (Truncate)
1502  Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1503  Node->getOperand(i), Idx,
1504  PtrInfo.getWithOffset(Offset), MemVT));
1505  else
1506  Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1507  Idx, PtrInfo.getWithOffset(Offset)));
1508  }
1509 
1510  SDValue StoreChain;
1511  if (!Stores.empty()) // Not all undef elements?
1512  StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1513  else
1514  StoreChain = DAG.getEntryNode();
1515 
1516  // Result is a load from the stack slot.
1517  return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1518 }
1519 
1520 /// Bitcast a floating-point value to an integer value. Only bitcast the part
1521 /// containing the sign bit if the target has no integer value capable of
1522 /// holding all bits of the floating-point value.
1523 void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1524  const SDLoc &DL,
1525  SDValue Value) const {
1526  EVT FloatVT = Value.getValueType();
1527  unsigned NumBits = FloatVT.getScalarSizeInBits();
1528  State.FloatVT = FloatVT;
1529  EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1530  // Convert to an integer of the same size.
1531  if (TLI.isTypeLegal(IVT)) {
1532  State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1533  State.SignMask = APInt::getSignMask(NumBits);
1534  State.SignBit = NumBits - 1;
1535  return;
1536  }
1537 
1538  auto &DataLayout = DAG.getDataLayout();
1539  // Store the float to memory, then load the sign part out as an integer.
1540  MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
1541  // First create a temporary that is aligned for both the load and store.
1542  SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1543  int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1544  // Then store the float to it.
1545  State.FloatPtr = StackPtr;
1546  MachineFunction &MF = DAG.getMachineFunction();
1547  State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1548  State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1549  State.FloatPointerInfo);
1550 
1551  SDValue IntPtr;
1552  if (DataLayout.isBigEndian()) {
1553  assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1554  // Load out a legal integer with the same sign bit as the float.
1555  IntPtr = StackPtr;
1556  State.IntPointerInfo = State.FloatPointerInfo;
1557  } else {
1558  // Advance the pointer so that the loaded byte will contain the sign bit.
1559  unsigned ByteOffset = (NumBits / 8) - 1;
1560  IntPtr =
1561  DAG.getMemBasePlusOffset(StackPtr, TypeSize::Fixed(ByteOffset), DL);
1562  State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1563  ByteOffset);
1564  }
1565 
1566  State.IntPtr = IntPtr;
1567  State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1568  State.IntPointerInfo, MVT::i8);
1569  State.SignMask = APInt::getOneBitSet(LoadTy.getScalarSizeInBits(), 7);
1570  State.SignBit = 7;
1571 }
1572 
1573 /// Replace the integer value produced by getSignAsIntValue() with a new value
1574 /// and cast the result back to a floating-point type.
1575 SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1576  const SDLoc &DL,
1577  SDValue NewIntValue) const {
1578  if (!State.Chain)
1579  return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1580 
1581  // Override the part containing the sign bit in the value stored on the stack.
1582  SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1583  State.IntPointerInfo, MVT::i8);
1584  return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1585  State.FloatPointerInfo);
1586 }
1587 
1588 SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1589  SDLoc DL(Node);
1590  SDValue Mag = Node->getOperand(0);
1591  SDValue Sign = Node->getOperand(1);
1592 
1593  // Get sign bit into an integer value.
1594  FloatSignAsInt SignAsInt;
1595  getSignAsIntValue(SignAsInt, DL, Sign);
1596 
1597  EVT IntVT = SignAsInt.IntValue.getValueType();
1598  SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1599  SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1600  SignMask);
1601 
1602  // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1603  EVT FloatVT = Mag.getValueType();
1604  if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1605  TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1606  SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1607  SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1608  SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1609  DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1610  return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1611  }
1612 
1613  // Transform Mag value to integer, and clear the sign bit.
1614  FloatSignAsInt MagAsInt;
1615  getSignAsIntValue(MagAsInt, DL, Mag);
1616  EVT MagVT = MagAsInt.IntValue.getValueType();
1617  SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1618  SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1619  ClearSignMask);
1620 
1621  // Get the signbit at the right position for MagAsInt.
1622  int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1623  EVT ShiftVT = IntVT;
1624  if (SignBit.getScalarValueSizeInBits() <
1625  ClearedSign.getScalarValueSizeInBits()) {
1626  SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1627  ShiftVT = MagVT;
1628  }
1629  if (ShiftAmount > 0) {
1630  SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);
1631  SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);
1632  } else if (ShiftAmount < 0) {
1633  SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);
1634  SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);
1635  }
1636  if (SignBit.getScalarValueSizeInBits() >
1637  ClearedSign.getScalarValueSizeInBits()) {
1638  SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1639  }
1640 
1641  // Store the part with the modified sign and convert back to float.
1642  SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1643  return modifySignAsInt(MagAsInt, DL, CopiedSign);
1644 }
1645 
1646 SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1647  // Get the sign bit as an integer.
1648  SDLoc DL(Node);
1649  FloatSignAsInt SignAsInt;
1650  getSignAsIntValue(SignAsInt, DL, Node->getOperand(0));
1651  EVT IntVT = SignAsInt.IntValue.getValueType();
1652 
1653  // Flip the sign.
1654  SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1655  SDValue SignFlip =
1656  DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask);
1657 
1658  // Convert back to float.
1659  return modifySignAsInt(SignAsInt, DL, SignFlip);
1660 }
1661 
1662 SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1663  SDLoc DL(Node);
1664  SDValue Value = Node->getOperand(0);
1665 
1666  // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1667  EVT FloatVT = Value.getValueType();
1668  if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1669  SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1670  return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1671  }
1672 
1673  // Transform value to integer, clear the sign bit and transform back.
1674  FloatSignAsInt ValueAsInt;
1675  getSignAsIntValue(ValueAsInt, DL, Value);
1676  EVT IntVT = ValueAsInt.IntValue.getValueType();
1677  SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1678  SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1679  ClearSignMask);
1680  return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1681 }
1682 
1683 void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1685  Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1686  assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1687  " not tell us which reg is the stack pointer!");
1688  SDLoc dl(Node);
1689  EVT VT = Node->getValueType(0);
1690  SDValue Tmp1 = SDValue(Node, 0);
1691  SDValue Tmp2 = SDValue(Node, 1);
1692  SDValue Tmp3 = Node->getOperand(2);
1693  SDValue Chain = Tmp1.getOperand(0);
1694 
1695  // Chain the dynamic stack allocation so that it doesn't modify the stack
1696  // pointer when other instructions are using the stack.
1697  Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1698 
1699  SDValue Size = Tmp2.getOperand(1);
1700  SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1701  Chain = SP.getValue(1);
1702  Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1703  const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1704  unsigned Opc =
1706  ISD::ADD : ISD::SUB;
1707 
1708  Align StackAlign = TFL->getStackAlign();
1709  Tmp1 = DAG.getNode(Opc, dl, VT, SP, Size); // Value
1710  if (Alignment > StackAlign)
1711  Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1712  DAG.getConstant(-Alignment.value(), dl, VT));
1713  Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1714 
1715  Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
1716  DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
1717 
1718  Results.push_back(Tmp1);
1719  Results.push_back(Tmp2);
1720 }
1721 
1722 /// Emit a store/load combination to the stack. This stores
1723 /// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1724 /// a load from the stack slot to DestVT, extending it if needed.
1725 /// The resultant code need not be legal.
1726 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1727  EVT DestVT, const SDLoc &dl) {
1728  return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1729 }
1730 
1731 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1732  EVT DestVT, const SDLoc &dl,
1733  SDValue Chain) {
1734  unsigned SrcSize = SrcOp.getValueSizeInBits();
1735  unsigned SlotSize = SlotVT.getSizeInBits();
1736  unsigned DestSize = DestVT.getSizeInBits();
1737  Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1738  Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1739 
1740  // Don't convert with stack if the load/store is expensive.
1741  if ((SrcSize > SlotSize &&
1742  !TLI.isTruncStoreLegalOrCustom(SrcOp.getValueType(), SlotVT)) ||
1743  (SlotSize < DestSize &&
1744  !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT)))
1745  return SDValue();
1746 
1747  // Create the stack frame object.
1748  Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1749  SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1750  SDValue FIPtr = DAG.CreateStackTemporary(SlotVT.getStoreSize(), SrcAlign);
1751 
1752  FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1753  int SPFI = StackPtrFI->getIndex();
1754  MachinePointerInfo PtrInfo =
1755  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1756 
1757  // Emit a store to the stack slot. Use a truncstore if the input value is
1758  // later than DestVT.
1759  SDValue Store;
1760 
1761  if (SrcSize > SlotSize)
1762  Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo,
1763  SlotVT, SrcAlign);
1764  else {
1765  assert(SrcSize == SlotSize && "Invalid store");
1766  Store =
1767  DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1768  }
1769 
1770  // Result is a load from the stack slot.
1771  if (SlotSize == DestSize)
1772  return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1773 
1774  assert(SlotSize < DestSize && "Unknown extension!");
1775  return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1776  DestAlign);
1777 }
1778 
1779 SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1780  SDLoc dl(Node);
1781  // Create a vector sized/aligned stack slot, store the value to element #0,
1782  // then load the whole vector back out.
1783  SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1784 
1785  FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1786  int SPFI = StackPtrFI->getIndex();
1787 
1788  SDValue Ch = DAG.getTruncStore(
1789  DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1790  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1791  Node->getValueType(0).getVectorElementType());
1792  return DAG.getLoad(
1793  Node->getValueType(0), dl, Ch, StackPtr,
1794  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1795 }
1796 
1797 static bool
1799  const TargetLowering &TLI, SDValue &Res) {
1800  unsigned NumElems = Node->getNumOperands();
1801  SDLoc dl(Node);
1802  EVT VT = Node->getValueType(0);
1803 
1804  // Try to group the scalars into pairs, shuffle the pairs together, then
1805  // shuffle the pairs of pairs together, etc. until the vector has
1806  // been built. This will work only if all of the necessary shuffle masks
1807  // are legal.
1808 
1809  // We do this in two phases; first to check the legality of the shuffles,
1810  // and next, assuming that all shuffles are legal, to create the new nodes.
1811  for (int Phase = 0; Phase < 2; ++Phase) {
1813  NewIntermedVals;
1814  for (unsigned i = 0; i < NumElems; ++i) {
1815  SDValue V = Node->getOperand(i);
1816  if (V.isUndef())
1817  continue;
1818 
1819  SDValue Vec;
1820  if (Phase)
1821  Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1822  IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1823  }
1824 
1825  while (IntermedVals.size() > 2) {
1826  NewIntermedVals.clear();
1827  for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1828  // This vector and the next vector are shuffled together (simply to
1829  // append the one to the other).
1830  SmallVector<int, 16> ShuffleVec(NumElems, -1);
1831 
1832  SmallVector<int, 16> FinalIndices;
1833  FinalIndices.reserve(IntermedVals[i].second.size() +
1834  IntermedVals[i+1].second.size());
1835 
1836  int k = 0;
1837  for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1838  ++j, ++k) {
1839  ShuffleVec[k] = j;
1840  FinalIndices.push_back(IntermedVals[i].second[j]);
1841  }
1842  for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1843  ++j, ++k) {
1844  ShuffleVec[k] = NumElems + j;
1845  FinalIndices.push_back(IntermedVals[i+1].second[j]);
1846  }
1847 
1848  SDValue Shuffle;
1849  if (Phase)
1850  Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1851  IntermedVals[i+1].first,
1852  ShuffleVec);
1853  else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1854  return false;
1855  NewIntermedVals.push_back(
1856  std::make_pair(Shuffle, std::move(FinalIndices)));
1857  }
1858 
1859  // If we had an odd number of defined values, then append the last
1860  // element to the array of new vectors.
1861  if ((IntermedVals.size() & 1) != 0)
1862  NewIntermedVals.push_back(IntermedVals.back());
1863 
1864  IntermedVals.swap(NewIntermedVals);
1865  }
1866 
1867  assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1868  "Invalid number of intermediate vectors");
1869  SDValue Vec1 = IntermedVals[0].first;
1870  SDValue Vec2;
1871  if (IntermedVals.size() > 1)
1872  Vec2 = IntermedVals[1].first;
1873  else if (Phase)
1874  Vec2 = DAG.getUNDEF(VT);
1875 
1876  SmallVector<int, 16> ShuffleVec(NumElems, -1);
1877  for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1878  ShuffleVec[IntermedVals[0].second[i]] = i;
1879  for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1880  ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1881 
1882  if (Phase)
1883  Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1884  else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1885  return false;
1886  }
1887 
1888  return true;
1889 }
1890 
1891 /// Expand a BUILD_VECTOR node on targets that don't
1892 /// support the operation, but do support the resultant vector type.
1893 SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1894  unsigned NumElems = Node->getNumOperands();
1895  SDValue Value1, Value2;
1896  SDLoc dl(Node);
1897  EVT VT = Node->getValueType(0);
1898  EVT OpVT = Node->getOperand(0).getValueType();
1899  EVT EltVT = VT.getVectorElementType();
1900 
1901  // If the only non-undef value is the low element, turn this into a
1902  // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1903  bool isOnlyLowElement = true;
1904  bool MoreThanTwoValues = false;
1905  bool isConstant = true;
1906  for (unsigned i = 0; i < NumElems; ++i) {
1907  SDValue V = Node->getOperand(i);
1908  if (V.isUndef())
1909  continue;
1910  if (i > 0)
1911  isOnlyLowElement = false;
1912  if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1913  isConstant = false;
1914 
1915  if (!Value1.getNode()) {
1916  Value1 = V;
1917  } else if (!Value2.getNode()) {
1918  if (V != Value1)
1919  Value2 = V;
1920  } else if (V != Value1 && V != Value2) {
1921  MoreThanTwoValues = true;
1922  }
1923  }
1924 
1925  if (!Value1.getNode())
1926  return DAG.getUNDEF(VT);
1927 
1928  if (isOnlyLowElement)
1929  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1930 
1931  // If all elements are constants, create a load from the constant pool.
1932  if (isConstant) {
1934  for (unsigned i = 0, e = NumElems; i != e; ++i) {
1935  if (ConstantFPSDNode *V =
1936  dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1937  CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1938  } else if (ConstantSDNode *V =
1939  dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1940  if (OpVT==EltVT)
1941  CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1942  else {
1943  // If OpVT and EltVT don't match, EltVT is not legal and the
1944  // element values have been promoted/truncated earlier. Undo this;
1945  // we don't want a v16i8 to become a v16i32 for example.
1946  const ConstantInt *CI = V->getConstantIntValue();
1947  CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1948  CI->getZExtValue()));
1949  }
1950  } else {
1951  assert(Node->getOperand(i).isUndef());
1952  Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1953  CV.push_back(UndefValue::get(OpNTy));
1954  }
1955  }
1957  SDValue CPIdx =
1958  DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1959  Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
1960  return DAG.getLoad(
1961  VT, dl, DAG.getEntryNode(), CPIdx,
1962  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1963  Alignment);
1964  }
1965 
1966  SmallSet<SDValue, 16> DefinedValues;
1967  for (unsigned i = 0; i < NumElems; ++i) {
1968  if (Node->getOperand(i).isUndef())
1969  continue;
1970  DefinedValues.insert(Node->getOperand(i));
1971  }
1972 
1973  if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
1974  if (!MoreThanTwoValues) {
1975  SmallVector<int, 8> ShuffleVec(NumElems, -1);
1976  for (unsigned i = 0; i < NumElems; ++i) {
1977  SDValue V = Node->getOperand(i);
1978  if (V.isUndef())
1979  continue;
1980  ShuffleVec[i] = V == Value1 ? 0 : NumElems;
1981  }
1982  if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
1983  // Get the splatted value into the low element of a vector register.
1984  SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1985  SDValue Vec2;
1986  if (Value2.getNode())
1987  Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
1988  else
1989  Vec2 = DAG.getUNDEF(VT);
1990 
1991  // Return shuffle(LowValVec, undef, <0,0,0,0>)
1992  return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1993  }
1994  } else {
1995  SDValue Res;
1996  if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
1997  return Res;
1998  }
1999  }
2000 
2001  // Otherwise, we can't handle this case efficiently.
2002  return ExpandVectorBuildThroughStack(Node);
2003 }
2004 
2005 SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2006  SDLoc DL(Node);
2007  EVT VT = Node->getValueType(0);
2008  SDValue SplatVal = Node->getOperand(0);
2009 
2010  return DAG.getSplatBuildVector(VT, DL, SplatVal);
2011 }
2012 
2013 // Expand a node into a call to a libcall. If the result value
2014 // does not fit into a register, return the lo part and set the hi part to the
2015 // by-reg argument. If it does fit into a single register, return the result
2016 // and leave the Hi part unset.
2017 SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2018  bool isSigned) {
2020  TargetLowering::ArgListEntry Entry;
2021  for (const SDValue &Op : Node->op_values()) {
2022  EVT ArgVT = Op.getValueType();
2023  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2024  Entry.Node = Op;
2025  Entry.Ty = ArgTy;
2026  Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2027  Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2028  Args.push_back(Entry);
2029  }
2030  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2031  TLI.getPointerTy(DAG.getDataLayout()));
2032 
2033  EVT RetVT = Node->getValueType(0);
2034  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2035 
2036  // By default, the input chain to this libcall is the entry node of the
2037  // function. If the libcall is going to be emitted as a tail call then
2038  // TLI.isUsedByReturnOnly will change it to the right chain if the return
2039  // node which is being folded has a non-entry input chain.
2040  SDValue InChain = DAG.getEntryNode();
2041 
2042  // isTailCall may be true since the callee does not reference caller stack
2043  // frame. Check if it's in the right position and that the return types match.
2044  SDValue TCChain = InChain;
2045  const Function &F = DAG.getMachineFunction().getFunction();
2046  bool isTailCall =
2047  TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2048  (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2049  if (isTailCall)
2050  InChain = TCChain;
2051 
2053  bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2054  CLI.setDebugLoc(SDLoc(Node))
2055  .setChain(InChain)
2056  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2057  std::move(Args))
2058  .setTailCall(isTailCall)
2059  .setSExtResult(signExtend)
2060  .setZExtResult(!signExtend)
2061  .setIsPostTypeLegalization(true);
2062 
2063  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2064 
2065  if (!CallInfo.second.getNode()) {
2066  LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2067  // It's a tailcall, return the chain (which is the DAG root).
2068  return DAG.getRoot();
2069  }
2070 
2071  LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2072  return CallInfo.first;
2073 }
2074 
2075 void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2076  RTLIB::Libcall LC,
2078  if (LC == RTLIB::UNKNOWN_LIBCALL)
2079  llvm_unreachable("Can't create an unknown libcall!");
2080 
2081  if (Node->isStrictFPOpcode()) {
2082  EVT RetVT = Node->getValueType(0);
2083  SmallVector<SDValue, 4> Ops(drop_begin(Node->ops()));
2085  // FIXME: This doesn't support tail calls.
2086  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2087  Ops, CallOptions,
2088  SDLoc(Node),
2089  Node->getOperand(0));
2090  Results.push_back(Tmp.first);
2091  Results.push_back(Tmp.second);
2092  } else {
2093  SDValue Tmp = ExpandLibCall(LC, Node, false);
2094  Results.push_back(Tmp);
2095  }
2096 }
2097 
2098 /// Expand the node to a libcall based on the result type.
2099 void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2100  RTLIB::Libcall Call_F32,
2101  RTLIB::Libcall Call_F64,
2102  RTLIB::Libcall Call_F80,
2103  RTLIB::Libcall Call_F128,
2104  RTLIB::Libcall Call_PPCF128,
2106  RTLIB::Libcall LC = RTLIB::getFPLibCall(Node->getSimpleValueType(0),
2107  Call_F32, Call_F64, Call_F80,
2108  Call_F128, Call_PPCF128);
2109  ExpandFPLibCall(Node, LC, Results);
2110 }
2111 
2112 SDValue SelectionDAGLegalize::ExpandIntLibCall(
2113  SDNode *Node, bool isSigned, RTLIB::Libcall Call_I8,
2114  RTLIB::Libcall Call_I16, RTLIB::Libcall Call_I32, RTLIB::Libcall Call_I64,
2115  RTLIB::Libcall Call_I128, RTLIB::Libcall Call_IEXT) {
2116  RTLIB::Libcall LC;
2117  switch (Node->getSimpleValueType(0).SimpleTy) {
2118 
2119  default:
2120  LC = Call_IEXT;
2121  break;
2122 
2123  case MVT::i8: LC = Call_I8; break;
2124  case MVT::i16: LC = Call_I16; break;
2125  case MVT::i32: LC = Call_I32; break;
2126  case MVT::i64: LC = Call_I64; break;
2127  case MVT::i128: LC = Call_I128; break;
2128  }
2129  return ExpandLibCall(LC, Node, isSigned);
2130 }
2131 
2132 /// Expand the node to a libcall based on first argument type (for instance
2133 /// lround and its variant).
2134 void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2135  RTLIB::Libcall Call_F32,
2136  RTLIB::Libcall Call_F64,
2137  RTLIB::Libcall Call_F80,
2138  RTLIB::Libcall Call_F128,
2139  RTLIB::Libcall Call_PPCF128,
2141  EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2143  Call_F32, Call_F64, Call_F80,
2144  Call_F128, Call_PPCF128);
2145  ExpandFPLibCall(Node, LC, Results);
2146 }
2147 
2148 /// Issue libcalls to __{u}divmod to compute div / rem pairs.
2149 void
2150 SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2152  unsigned Opcode = Node->getOpcode();
2153  bool isSigned = Opcode == ISD::SDIVREM;
2154 
2155  RTLIB::Libcall LC;
2156  switch (Node->getSimpleValueType(0).SimpleTy) {
2157 
2158  default:
2159  LC = isSigned ? RTLIB::SDIVREM_IEXT : RTLIB::UDIVREM_IEXT;
2160  break;
2161 
2162  case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2163  case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2164  case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2165  case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2166  case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2167  }
2168 
2169  // The input chain to this libcall is the entry node of the function.
2170  // Legalizing the call will automatically add the previous call to the
2171  // dependence.
2172  SDValue InChain = DAG.getEntryNode();
2173 
2174  EVT RetVT = Node->getValueType(0);
2175  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2176 
2178  TargetLowering::ArgListEntry Entry;
2179  for (const SDValue &Op : Node->op_values()) {
2180  EVT ArgVT = Op.getValueType();
2181  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2182  Entry.Node = Op;
2183  Entry.Ty = ArgTy;
2184  Entry.IsSExt = isSigned;
2185  Entry.IsZExt = !isSigned;
2186  Args.push_back(Entry);
2187  }
2188 
2189  // Also pass the return address of the remainder.
2190  SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2191  Entry.Node = FIPtr;
2192  Entry.Ty = RetTy->getPointerTo();
2193  Entry.IsSExt = isSigned;
2194  Entry.IsZExt = !isSigned;
2195  Args.push_back(Entry);
2196 
2197  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2198  TLI.getPointerTy(DAG.getDataLayout()));
2199 
2200  SDLoc dl(Node);
2202  CLI.setDebugLoc(dl)
2203  .setChain(InChain)
2204  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2205  std::move(Args))
2206  .setSExtResult(isSigned)
2207  .setZExtResult(!isSigned);
2208 
2209  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2210 
2211  // Remainder is loaded back from the stack frame.
2212  SDValue Rem =
2213  DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2214  Results.push_back(CallInfo.first);
2215  Results.push_back(Rem);
2216 }
2217 
2218 /// Return true if sincos libcall is available.
2219 static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2220  RTLIB::Libcall LC;
2221  switch (Node->getSimpleValueType(0).SimpleTy) {
2222  default: llvm_unreachable("Unexpected request for libcall!");
2223  case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2224  case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2225  case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2226  case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2227  case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2228  }
2229  return TLI.getLibcallName(LC) != nullptr;
2230 }
2231 
2232 /// Only issue sincos libcall if both sin and cos are needed.
2233 static bool useSinCos(SDNode *Node) {
2234  unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2235  ? ISD::FCOS : ISD::FSIN;
2236 
2237  SDValue Op0 = Node->getOperand(0);
2238  for (const SDNode *User : Op0.getNode()->uses()) {
2239  if (User == Node)
2240  continue;
2241  // The other user might have been turned into sincos already.
2242  if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2243  return true;
2244  }
2245  return false;
2246 }
2247 
2248 /// Issue libcalls to sincos to compute sin / cos pairs.
2249 void
2250 SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2252  RTLIB::Libcall LC;
2253  switch (Node->getSimpleValueType(0).SimpleTy) {
2254  default: llvm_unreachable("Unexpected request for libcall!");
2255  case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2256  case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2257  case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2258  case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2259  case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2260  }
2261 
2262  // The input chain to this libcall is the entry node of the function.
2263  // Legalizing the call will automatically add the previous call to the
2264  // dependence.
2265  SDValue InChain = DAG.getEntryNode();
2266 
2267  EVT RetVT = Node->getValueType(0);
2268  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2269 
2271  TargetLowering::ArgListEntry Entry;
2272 
2273  // Pass the argument.
2274  Entry.Node = Node->getOperand(0);
2275  Entry.Ty = RetTy;
2276  Entry.IsSExt = false;
2277  Entry.IsZExt = false;
2278  Args.push_back(Entry);
2279 
2280  // Pass the return address of sin.
2281  SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2282  Entry.Node = SinPtr;
2283  Entry.Ty = RetTy->getPointerTo();
2284  Entry.IsSExt = false;
2285  Entry.IsZExt = false;
2286  Args.push_back(Entry);
2287 
2288  // Also pass the return address of the cos.
2289  SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2290  Entry.Node = CosPtr;
2291  Entry.Ty = RetTy->getPointerTo();
2292  Entry.IsSExt = false;
2293  Entry.IsZExt = false;
2294  Args.push_back(Entry);
2295 
2296  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2297  TLI.getPointerTy(DAG.getDataLayout()));
2298 
2299  SDLoc dl(Node);
2301  CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2302  TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2303  std::move(Args));
2304 
2305  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2306 
2307  Results.push_back(
2308  DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2309  Results.push_back(
2310  DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2311 }
2312 
2313 /// This function is responsible for legalizing a
2314 /// INT_TO_FP operation of the specified operand when the target requests that
2315 /// we expand it. At this point, we know that the result and operand types are
2316 /// legal for the target.
2317 SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2318  SDValue &Chain) {
2319  bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2320  Node->getOpcode() == ISD::SINT_TO_FP);
2321  EVT DestVT = Node->getValueType(0);
2322  SDLoc dl(Node);
2323  unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2324  SDValue Op0 = Node->getOperand(OpNo);
2325  EVT SrcVT = Op0.getValueType();
2326 
2327  // TODO: Should any fast-math-flags be set for the created nodes?
2328  LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2329  if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2330  (DestVT.bitsLE(MVT::f64) ||
2331  TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2332  : ISD::FP_EXTEND,
2333  DestVT))) {
2334  LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2335  "expansion\n");
2336 
2337  // Get the stack frame index of a 8 byte buffer.
2338  SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2339 
2340  SDValue Lo = Op0;
2341  // if signed map to unsigned space
2342  if (isSigned) {
2343  // Invert sign bit (signed to unsigned mapping).
2344  Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo,
2345  DAG.getConstant(0x80000000u, dl, MVT::i32));
2346  }
2347  // Initial hi portion of constructed double.
2348  SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2349 
2350  // If this a big endian target, swap the lo and high data.
2351  if (DAG.getDataLayout().isBigEndian())
2352  std::swap(Lo, Hi);
2353 
2354  SDValue MemChain = DAG.getEntryNode();
2355 
2356  // Store the lo of the constructed double.
2357  SDValue Store1 = DAG.getStore(MemChain, dl, Lo, StackSlot,
2358  MachinePointerInfo());
2359  // Store the hi of the constructed double.
2360  SDValue HiPtr = DAG.getMemBasePlusOffset(StackSlot, TypeSize::Fixed(4), dl);
2361  SDValue Store2 =
2362  DAG.getStore(MemChain, dl, Hi, HiPtr, MachinePointerInfo());
2363  MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
2364 
2365  // load the constructed double
2366  SDValue Load =
2367  DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo());
2368  // FP constant to bias correct the final result
2369  SDValue Bias = DAG.getConstantFP(isSigned ?
2370  BitsToDouble(0x4330000080000000ULL) :
2371  BitsToDouble(0x4330000000000000ULL),
2372  dl, MVT::f64);
2373  // Subtract the bias and get the final result.
2374  SDValue Sub;
2375  SDValue Result;
2376  if (Node->isStrictFPOpcode()) {
2377  Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
2378  {Node->getOperand(0), Load, Bias});
2379  Chain = Sub.getValue(1);
2380  if (DestVT != Sub.getValueType()) {
2381  std::pair<SDValue, SDValue> ResultPair;
2382  ResultPair =
2383  DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2384  Result = ResultPair.first;
2385  Chain = ResultPair.second;
2386  }
2387  else
2388  Result = Sub;
2389  } else {
2390  Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2391  Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2392  }
2393  return Result;
2394  }
2395 
2396  if (isSigned)
2397  return SDValue();
2398 
2399  // TODO: Generalize this for use with other types.
2400  if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2401  (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2402  LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2403  // For unsigned conversions, convert them to signed conversions using the
2404  // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2405  // should be valid for i32->f32 as well.
2406 
2407  // More generally this transform should be valid if there are 3 more bits
2408  // in the integer type than the significand. Rounding uses the first bit
2409  // after the width of the significand and the OR of all bits after that. So
2410  // we need to be able to OR the shifted out bit into one of the bits that
2411  // participate in the OR.
2412 
2413  // TODO: This really should be implemented using a branch rather than a
2414  // select. We happen to get lucky and machinesink does the right
2415  // thing most of the time. This would be a good candidate for a
2416  // pseudo-op, or, even better, for whole-function isel.
2417  EVT SetCCVT = getSetCCResultType(SrcVT);
2418 
2419  SDValue SignBitTest = DAG.getSetCC(
2420  dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2421 
2422  EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2423  SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2424  SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst);
2425  SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2426  SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst);
2427  SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr);
2428 
2429  SDValue Slow, Fast;
2430  if (Node->isStrictFPOpcode()) {
2431  // In strict mode, we must avoid spurious exceptions, and therefore
2432  // must make sure to only emit a single STRICT_SINT_TO_FP.
2433  SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest, Or, Op0);
2434  Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2435  { Node->getOperand(0), InCvt });
2436  Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2437  { Fast.getValue(1), Fast, Fast });
2438  Chain = Slow.getValue(1);
2439  // The STRICT_SINT_TO_FP inherits the exception mode from the
2440  // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2441  // never raise any exception.
2442  SDNodeFlags Flags;
2443  Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2444  Fast->setFlags(Flags);
2445  Flags.setNoFPExcept(true);
2446  Slow->setFlags(Flags);
2447  } else {
2448  SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or);
2449  Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt);
2450  Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2451  }
2452 
2453  return DAG.getSelect(dl, DestVT, SignBitTest, Slow, Fast);
2454  }
2455 
2456  // Don't expand it if there isn't cheap fadd.
2457  if (!TLI.isOperationLegalOrCustom(
2458  Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT))
2459  return SDValue();
2460 
2461  // The following optimization is valid only if every value in SrcVT (when
2462  // treated as signed) is representable in DestVT. Check that the mantissa
2463  // size of DestVT is >= than the number of bits in SrcVT -1.
2464  assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2465  SrcVT.getSizeInBits() - 1 &&
2466  "Cannot perform lossless SINT_TO_FP!");
2467 
2468  SDValue Tmp1;
2469  if (Node->isStrictFPOpcode()) {
2470  Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2471  { Node->getOperand(0), Op0 });
2472  } else
2473  Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2474 
2475  SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2476  DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2477  SDValue Zero = DAG.getIntPtrConstant(0, dl),
2478  Four = DAG.getIntPtrConstant(4, dl);
2479  SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2480  SignSet, Four, Zero);
2481 
2482  // If the sign bit of the integer is set, the large number will be treated
2483  // as a negative number. To counteract this, the dynamic code adds an
2484  // offset depending on the data type.
2485  uint64_t FF;
2486  switch (SrcVT.getSimpleVT().SimpleTy) {
2487  default:
2488  return SDValue();
2489  case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2490  case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2491  case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2492  case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2493  }
2494  if (DAG.getDataLayout().isLittleEndian())
2495  FF <<= 32;
2496  Constant *FudgeFactor = ConstantInt::get(
2497  Type::getInt64Ty(*DAG.getContext()), FF);
2498 
2499  SDValue CPIdx =
2500  DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2501  Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2502  CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2503  Alignment = commonAlignment(Alignment, 4);
2504  SDValue FudgeInReg;
2505  if (DestVT == MVT::f32)
2506  FudgeInReg = DAG.getLoad(
2507  MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2508  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2509  Alignment);
2510  else {
2511  SDValue Load = DAG.getExtLoad(
2512  ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2513  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2514  Alignment);
2515  HandleSDNode Handle(Load);
2516  LegalizeOp(Load.getNode());
2517  FudgeInReg = Handle.getValue();
2518  }
2519 
2520  if (Node->isStrictFPOpcode()) {
2521  SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2522  { Tmp1.getValue(1), Tmp1, FudgeInReg });
2523  Chain = Result.getValue(1);
2524  return Result;
2525  }
2526 
2527  return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2528 }
2529 
2530 /// This function is responsible for legalizing a
2531 /// *INT_TO_FP operation of the specified operand when the target requests that
2532 /// we promote it. At this point, we know that the result and operand types are
2533 /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2534 /// operation that takes a larger input.
2535 void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2536  SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) {
2537  bool IsStrict = N->isStrictFPOpcode();
2538  bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2539  N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2540  EVT DestVT = N->getValueType(0);
2541  SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2542  unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
2543  unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
2544 
2545  // First step, figure out the appropriate *INT_TO_FP operation to use.
2546  EVT NewInTy = LegalOp.getValueType();
2547 
2548  unsigned OpToUse = 0;
2549 
2550  // Scan for the appropriate larger type to use.
2551  while (true) {
2552  NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2553  assert(NewInTy.isInteger() && "Ran out of possibilities!");
2554 
2555  // If the target supports SINT_TO_FP of this type, use it.
2556  if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2557  OpToUse = SIntOp;
2558  break;
2559  }
2560  if (IsSigned)
2561  continue;
2562 
2563  // If the target supports UINT_TO_FP of this type, use it.
2564  if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2565  OpToUse = UIntOp;
2566  break;
2567  }
2568 
2569  // Otherwise, try a larger type.
2570  }
2571 
2572  // Okay, we found the operation and type to use. Zero extend our input to the
2573  // desired type then run the operation on it.
2574  if (IsStrict) {
2575  SDValue Res =
2576  DAG.getNode(OpToUse, dl, {DestVT, MVT::Other},
2577  {N->getOperand(0),
2578  DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2579  dl, NewInTy, LegalOp)});
2580  Results.push_back(Res);
2581  Results.push_back(Res.getValue(1));
2582  return;
2583  }
2584 
2585  Results.push_back(
2586  DAG.getNode(OpToUse, dl, DestVT,
2587  DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2588  dl, NewInTy, LegalOp)));
2589 }
2590 
2591 /// This function is responsible for legalizing a
2592 /// FP_TO_*INT operation of the specified operand when the target requests that
2593 /// we promote it. At this point, we know that the result and operand types are
2594 /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2595 /// operation that returns a larger result.
2596 void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
2598  bool IsStrict = N->isStrictFPOpcode();
2599  bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
2600  N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2601  EVT DestVT = N->getValueType(0);
2602  SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2603  // First step, figure out the appropriate FP_TO*INT operation to use.
2604  EVT NewOutTy = DestVT;
2605 
2606  unsigned OpToUse = 0;
2607 
2608  // Scan for the appropriate larger type to use.
2609  while (true) {
2610  NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2611  assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2612 
2613  // A larger signed type can hold all unsigned values of the requested type,
2614  // so using FP_TO_SINT is valid
2615  OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
2616  if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2617  break;
2618 
2619  // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2620  OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
2621  if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2622  break;
2623 
2624  // Otherwise, try a larger type.
2625  }
2626 
2627  // Okay, we found the operation and type to use.
2629  if (IsStrict) {
2630  SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2631  Operation = DAG.getNode(OpToUse, dl, VTs, N->getOperand(0), LegalOp);
2632  } else
2633  Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2634 
2635  // Truncate the result of the extended FP_TO_*INT operation to the desired
2636  // size.
2637  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2638  Results.push_back(Trunc);
2639  if (IsStrict)
2640  Results.push_back(Operation.getValue(1));
2641 }
2642 
2643 /// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
2644 /// the result and operand types are legal and there must be a legal
2645 /// FP_TO_*INT_SAT operation for a larger result type.
2646 SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
2647  const SDLoc &dl) {
2648  unsigned Opcode = Node->getOpcode();
2649 
2650  // Scan for the appropriate larger type to use.
2651  EVT NewOutTy = Node->getValueType(0);
2652  while (true) {
2653  NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
2654  assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2655 
2656  if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2657  break;
2658  }
2659 
2660  // Saturation width is determined by second operand, so we don't have to
2661  // perform any fixup and can directly truncate the result.
2662  SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0),
2663  Node->getOperand(1));
2664  return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result);
2665 }
2666 
2667 /// Open code the operations for PARITY of the specified operation.
2668 SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
2669  EVT VT = Op.getValueType();
2670  EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2671  unsigned Sz = VT.getScalarSizeInBits();
2672 
2673  // If CTPOP is legal, use it. Otherwise use shifts and xor.
2674  SDValue Result;
2675  if (TLI.isOperationLegalOrPromote(ISD::CTPOP, VT)) {
2676  Result = DAG.getNode(ISD::CTPOP, dl, VT, Op);
2677  } else {
2678  Result = Op;
2679  for (unsigned i = Log2_32_Ceil(Sz); i != 0;) {
2680  SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, Result,
2681  DAG.getConstant(1ULL << (--i), dl, ShVT));
2682  Result = DAG.getNode(ISD::XOR, dl, VT, Result, Shift);
2683  }
2684  }
2685 
2686  return DAG.getNode(ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
2687 }
2688 
2689 bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2690  LLVM_DEBUG(dbgs() << "Trying to expand node\n");
2692  SDLoc dl(Node);
2693  SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2694  bool NeedInvert;
2695  switch (Node->getOpcode()) {
2696  case ISD::ABS:
2697  if ((Tmp1 = TLI.expandABS(Node, DAG)))
2698  Results.push_back(Tmp1);
2699  break;
2700  case ISD::CTPOP:
2701  if ((Tmp1 = TLI.expandCTPOP(Node, DAG)))
2702  Results.push_back(Tmp1);
2703  break;
2704  case ISD::CTLZ:
2705  case ISD::CTLZ_ZERO_UNDEF:
2706  if ((Tmp1 = TLI.expandCTLZ(Node, DAG)))
2707  Results.push_back(Tmp1);
2708  break;
2709  case ISD::CTTZ:
2710  case ISD::CTTZ_ZERO_UNDEF:
2711  if ((Tmp1 = TLI.expandCTTZ(Node, DAG)))
2712  Results.push_back(Tmp1);
2713  break;
2714  case ISD::BITREVERSE:
2715  if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG)))
2716  Results.push_back(Tmp1);
2717  break;
2718  case ISD::BSWAP:
2719  if ((Tmp1 = TLI.expandBSWAP(Node, DAG)))
2720  Results.push_back(Tmp1);
2721  break;
2722  case ISD::PARITY:
2723  Results.push_back(ExpandPARITY(Node->getOperand(0), dl));
2724  break;
2725  case ISD::FRAMEADDR:
2726  case ISD::RETURNADDR:
2728  Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2729  break;
2730  case ISD::EH_DWARF_CFA: {
2731  SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2732  TLI.getPointerTy(DAG.getDataLayout()));
2733  SDValue Offset = DAG.getNode(ISD::ADD, dl,
2734  CfaArg.getValueType(),
2735  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
2736  CfaArg.getValueType()),
2737  CfaArg);
2738  SDValue FA = DAG.getNode(
2739  ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
2740  DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2741  Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
2742  FA, Offset));
2743  break;
2744  }
2745  case ISD::FLT_ROUNDS_:
2746  Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2747  Results.push_back(Node->getOperand(0));
2748  break;
2749  case ISD::EH_RETURN:
2750  case ISD::EH_LABEL:
2751  case ISD::PREFETCH:
2752  case ISD::VAEND:
2753  case ISD::EH_SJLJ_LONGJMP:
2754  // If the target didn't expand these, there's nothing to do, so just
2755  // preserve the chain and be done.
2756  Results.push_back(Node->getOperand(0));
2757  break;
2758  case ISD::READCYCLECOUNTER:
2759  // If the target didn't expand this, just return 'zero' and preserve the
2760  // chain.
2761  Results.append(Node->getNumValues() - 1,
2762  DAG.getConstant(0, dl, Node->getValueType(0)));
2763  Results.push_back(Node->getOperand(0));
2764  break;
2765  case ISD::EH_SJLJ_SETJMP:
2766  // If the target didn't expand this, just return 'zero' and preserve the
2767  // chain.
2768  Results.push_back(DAG.getConstant(0, dl, MVT::i32));
2769  Results.push_back(Node->getOperand(0));
2770  break;
2771  case ISD::ATOMIC_LOAD: {
2772  // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2773  SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2774  SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2775  SDValue Swap = DAG.getAtomicCmpSwap(
2776  ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2777  Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2778  cast<AtomicSDNode>(Node)->getMemOperand());
2779  Results.push_back(Swap.getValue(0));
2780  Results.push_back(Swap.getValue(1));
2781  break;
2782  }
2783  case ISD::ATOMIC_STORE: {
2784  // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2785  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2786  cast<AtomicSDNode>(Node)->getMemoryVT(),
2787  Node->getOperand(0),
2788  Node->getOperand(1), Node->getOperand(2),
2789  cast<AtomicSDNode>(Node)->getMemOperand());
2790  Results.push_back(Swap.getValue(1));
2791  break;
2792  }
2794  // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2795  // splits out the success value as a comparison. Expanding the resulting
2796  // ATOMIC_CMP_SWAP will produce a libcall.
2797  SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2798  SDValue Res = DAG.getAtomicCmpSwap(
2799  ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2800  Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2801  Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
2802 
2803  SDValue ExtRes = Res;
2804  SDValue LHS = Res;
2805  SDValue RHS = Node->getOperand(1);
2806 
2807  EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
2808  EVT OuterType = Node->getValueType(0);
2809  switch (TLI.getExtendForAtomicOps()) {
2810  case ISD::SIGN_EXTEND:
2811  LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
2812  DAG.getValueType(AtomicType));
2813  RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
2814  Node->getOperand(2), DAG.getValueType(AtomicType));
2815  ExtRes = LHS;
2816  break;
2817  case ISD::ZERO_EXTEND:
2818  LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
2819  DAG.getValueType(AtomicType));
2820  RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2821  ExtRes = LHS;
2822  break;
2823  case ISD::ANY_EXTEND:
2824  LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2825  RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2826  break;
2827  default:
2828  llvm_unreachable("Invalid atomic op extension");
2829  }
2830 
2831  SDValue Success =
2832  DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
2833 
2834  Results.push_back(ExtRes.getValue(0));
2835  Results.push_back(Success);
2836  Results.push_back(Res.getValue(1));
2837  break;
2838  }
2840  ExpandDYNAMIC_STACKALLOC(Node, Results);
2841  break;
2842  case ISD::MERGE_VALUES:
2843  for (unsigned i = 0; i < Node->getNumValues(); i++)
2844  Results.push_back(Node->getOperand(i));
2845  break;
2846  case ISD::UNDEF: {
2847  EVT VT = Node->getValueType(0);
2848  if (VT.isInteger())
2849  Results.push_back(DAG.getConstant(0, dl, VT));
2850  else {
2851  assert(VT.isFloatingPoint() && "Unknown value type!");
2852  Results.push_back(DAG.getConstantFP(0, dl, VT));
2853  }
2854  break;
2855  }
2856  case ISD::STRICT_FP_ROUND:
2857  // When strict mode is enforced we can't do expansion because it
2858  // does not honor the "strict" properties. Only libcall is allowed.
2859  if (TLI.isStrictFPEnabled())
2860  break;
2861  // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
2862  // since this operation is more efficient than stack operation.
2863  if (TLI.getStrictFPOperationAction(Node->getOpcode(),
2864  Node->getValueType(0))
2866  break;
2867  // We fall back to use stack operation when the FP_ROUND operation
2868  // isn't available.
2869  if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0),
2870  Node->getValueType(0), dl,
2871  Node->getOperand(0)))) {
2872  ReplaceNode(Node, Tmp1.getNode());
2873  LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
2874  return true;
2875  }
2876  break;
2877  case ISD::FP_ROUND:
2878  case ISD::BITCAST:
2879  if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
2880  Node->getValueType(0), dl)))
2881  Results.push_back(Tmp1);
2882  break;
2883  case ISD::STRICT_FP_EXTEND:
2884  // When strict mode is enforced we can't do expansion because it
2885  // does not honor the "strict" properties. Only libcall is allowed.
2886  if (TLI.isStrictFPEnabled())
2887  break;
2888  // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
2889  // since this operation is more efficient than stack operation.
2890  if (TLI.getStrictFPOperationAction(Node->getOpcode(),
2891  Node->getValueType(0))
2893  break;
2894  // We fall back to use stack operation when the FP_EXTEND operation
2895  // isn't available.
2896  if ((Tmp1 = EmitStackConvert(
2897  Node->getOperand(1), Node->getOperand(1).getValueType(),
2898  Node->getValueType(0), dl, Node->getOperand(0)))) {
2899  ReplaceNode(Node, Tmp1.getNode());
2900  LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
2901  return true;
2902  }
2903  break;
2904  case ISD::FP_EXTEND:
2905  if ((Tmp1 = EmitStackConvert(Node->getOperand(0),
2906  Node->getOperand(0).getValueType(),
2907  Node->getValueType(0), dl)))
2908  Results.push_back(Tmp1);
2909  break;
2910  case ISD::SIGN_EXTEND_INREG: {
2911  EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2912  EVT VT = Node->getValueType(0);
2913 
2914  // An in-register sign-extend of a boolean is a negation:
2915  // 'true' (1) sign-extended is -1.
2916  // 'false' (0) sign-extended is 0.
2917  // However, we must mask the high bits of the source operand because the
2918  // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
2919 
2920  // TODO: Do this for vectors too?
2921  if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
2922  SDValue One = DAG.getConstant(1, dl, VT);
2923  SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
2924  SDValue Zero = DAG.getConstant(0, dl, VT);
2925  SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
2926  Results.push_back(Neg);
2927  break;
2928  }
2929 
2930  // NOTE: we could fall back on load/store here too for targets without
2931  // SRA. However, it is doubtful that any exist.
2932  EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2933  unsigned BitsDiff = VT.getScalarSizeInBits() -
2934  ExtraVT.getScalarSizeInBits();
2935  SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
2936  Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
2937  Node->getOperand(0), ShiftCst);
2938  Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
2939  Results.push_back(Tmp1);
2940  break;
2941  }
2942  case ISD::UINT_TO_FP:
2944  if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) {
2945  Results.push_back(Tmp1);
2946  if (Node->isStrictFPOpcode())
2947  Results.push_back(Tmp2);
2948  break;
2949  }
2951  case ISD::SINT_TO_FP:
2953  if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) {
2954  Results.push_back(Tmp1);
2955  if (Node->isStrictFPOpcode())
2956  Results.push_back(Tmp2);
2957  }
2958  break;
2959  case ISD::FP_TO_SINT:
2960  if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
2961  Results.push_back(Tmp1);
2962  break;
2964  if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) {
2965  ReplaceNode(Node, Tmp1.getNode());
2966  LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
2967  return true;
2968  }
2969  break;
2970  case ISD::FP_TO_UINT:
2971  if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG))
2972  Results.push_back(Tmp1);
2973  break;
2975  if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) {
2976  // Relink the chain.
2977  DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2);
2978  // Replace the new UINT result.
2979  ReplaceNodeWithValue(SDValue(Node, 0), Tmp1);
2980  LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
2981  return true;
2982  }
2983  break;
2984  case ISD::FP_TO_SINT_SAT:
2985  case ISD::FP_TO_UINT_SAT:
2986  Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG));
2987  break;
2988  case ISD::VAARG:
2989  Results.push_back(DAG.expandVAArg(Node));
2990  Results.push_back(Results[0].getValue(1));
2991  break;
2992  case ISD::VACOPY:
2993  Results.push_back(DAG.expandVACopy(Node));
2994  break;
2996  if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
2997  // This must be an access of the only element. Return it.
2998  Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
2999  Node->getOperand(0));
3000  else
3001  Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
3002  Results.push_back(Tmp1);
3003  break;
3005  Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
3006  break;
3007  case ISD::INSERT_SUBVECTOR:
3008  Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
3009  break;
3010  case ISD::CONCAT_VECTORS:
3011  Results.push_back(ExpandVectorBuildThroughStack(Node));
3012  break;
3013  case ISD::SCALAR_TO_VECTOR:
3014  Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3015  break;
3017  Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
3018  Node->getOperand(1),
3019  Node->getOperand(2), dl));
3020  break;
3021  case ISD::VECTOR_SHUFFLE: {
3022  SmallVector<int, 32> NewMask;
3023  ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3024 
3025  EVT VT = Node->getValueType(0);
3026  EVT EltVT = VT.getVectorElementType();
3027  SDValue Op0 = Node->getOperand(0);
3028  SDValue Op1 = Node->getOperand(1);
3029  if (!TLI.isTypeLegal(EltVT)) {
3030  EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3031 
3032  // BUILD_VECTOR operands are allowed to be wider than the element type.
3033  // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3034  // it.
3035  if (NewEltVT.bitsLT(EltVT)) {
3036  // Convert shuffle node.
3037  // If original node was v4i64 and the new EltVT is i32,
3038  // cast operands to v8i32 and re-build the mask.
3039 
3040  // Calculate new VT, the size of the new VT should be equal to original.
3041  EVT NewVT =
3042  EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3043  VT.getSizeInBits() / NewEltVT.getSizeInBits());
3044  assert(NewVT.bitsEq(VT));
3045 
3046  // cast operands to new VT
3047  Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3048  Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3049 
3050  // Convert the shuffle mask
3051  unsigned int factor =
3053 
3054  // EltVT gets smaller
3055  assert(factor > 0);
3056 
3057  for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3058  if (Mask[i] < 0) {
3059  for (unsigned fi = 0; fi < factor; ++fi)
3060  NewMask.push_back(Mask[i]);
3061  }
3062  else {
3063  for (unsigned fi = 0; fi < factor; ++fi)
3064  NewMask.push_back(Mask[i]*factor+fi);
3065  }
3066  }
3067  Mask = NewMask;
3068  VT = NewVT;
3069  }
3070  EltVT = NewEltVT;
3071  }
3072  unsigned NumElems = VT.getVectorNumElements();
3074  for (unsigned i = 0; i != NumElems; ++i) {
3075  if (Mask[i] < 0) {
3076  Ops.push_back(DAG.getUNDEF(EltVT));
3077  continue;
3078  }
3079  unsigned Idx = Mask[i];
3080  if (Idx < NumElems)
3081  Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3082  DAG.getVectorIdxConstant(Idx, dl)));
3083  else
3084  Ops.push_back(
3085  DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3086  DAG.getVectorIdxConstant(Idx - NumElems, dl)));
3087  }
3088 
3089  Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3090  // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3091  Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3092  Results.push_back(Tmp1);
3093  break;
3094  }
3095  case ISD::VECTOR_SPLICE: {
3096  Results.push_back(TLI.expandVectorSplice(Node, DAG));
3097  break;
3098  }
3099  case ISD::EXTRACT_ELEMENT: {
3100  EVT OpTy = Node->getOperand(0).getValueType();
3101  if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
3102  // 1 -> Hi
3103  Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3104  DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3105  TLI.getShiftAmountTy(
3106  Node->getOperand(0).getValueType(),
3107  DAG.getDataLayout())));
3108  Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3109  } else {
3110  // 0 -> Lo
3111  Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3112  Node->getOperand(0));
3113  }
3114  Results.push_back(Tmp1);
3115  break;
3116  }
3117  case ISD::STACKSAVE:
3118  // Expand to CopyFromReg if the target set
3119  // StackPointerRegisterToSaveRestore.
3120  if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3121  Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3122  Node->getValueType(0)));
3123  Results.push_back(Results[0].getValue(1));
3124  } else {
3125  Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3126  Results.push_back(Node->getOperand(0));
3127  }
3128  break;
3129  case ISD::STACKRESTORE:
3130  // Expand to CopyToReg if the target set
3131  // StackPointerRegisterToSaveRestore.
3132  if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3133  Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3134  Node->getOperand(1)));
3135  } else {
3136  Results.push_back(Node->getOperand(0));
3137  }
3138  break;
3140  Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3141  Results.push_back(Results[0].getValue(0));
3142  break;
3143  case ISD::FCOPYSIGN:
3144  Results.push_back(ExpandFCOPYSIGN(Node));
3145  break;
3146  case ISD::FNEG:
3147  Results.push_back(ExpandFNEG(Node));
3148  break;
3149  case ISD::FABS:
3150  Results.push_back(ExpandFABS(Node));
3151  break;
3152  case ISD::IS_FPCLASS: {
3153  auto CNode = cast<ConstantSDNode>(Node->getOperand(1));
3154  auto Test = static_cast<FPClassTest>(CNode->getZExtValue());
3155  if (SDValue Expanded =
3156  TLI.expandIS_FPCLASS(Node->getValueType(0), Node->getOperand(0),
3157  Test, Node->getFlags(), SDLoc(Node), DAG))
3158  Results.push_back(Expanded);
3159  break;
3160  }
3161  case ISD::SMIN:
3162  case ISD::SMAX:
3163  case ISD::UMIN:
3164  case ISD::UMAX: {
3165  // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3166  ISD::CondCode Pred;
3167  switch (Node->getOpcode()) {
3168  default: llvm_unreachable("How did we get here?");
3169  case ISD::SMAX: Pred = ISD::SETGT; break;
3170  case ISD::SMIN: Pred = ISD::SETLT; break;
3171  case ISD::UMAX: Pred = ISD::SETUGT; break;
3172  case ISD::UMIN: Pred = ISD::SETULT; break;
3173  }
3174  Tmp1 = Node->getOperand(0);
3175  Tmp2 = Node->getOperand(1);
3176  Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3177  Results.push_back(Tmp1);
3178  break;
3179  }
3180  case ISD::FMINNUM:
3181  case ISD::FMAXNUM: {
3182  if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))
3183  Results.push_back(Expanded);
3184  break;
3185  }
3186  case ISD::FSIN:
3187  case ISD::FCOS: {
3188  EVT VT = Node->getValueType(0);
3189  // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3190  // fcos which share the same operand and both are used.
3191  if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3192  isSinCosLibcallAvailable(Node, TLI))
3193  && useSinCos(Node)) {
3194  SDVTList VTs = DAG.getVTList(VT, VT);
3195  Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3196  if (Node->getOpcode() == ISD::FCOS)
3197  Tmp1 = Tmp1.getValue(1);
3198  Results.push_back(Tmp1);
3199  }
3200  break;
3201  }
3202  case ISD::FMAD:
3203  llvm_unreachable("Illegal fmad should never be formed");
3204 
3205  case ISD::FP16_TO_FP:
3206  if (Node->getValueType(0) != MVT::f32) {
3207  // We can extend to types bigger than f32 in two steps without changing
3208  // the result. Since "f16 -> f32" is much more commonly available, give
3209  // CodeGen the option of emitting that before resorting to a libcall.
3210  SDValue Res =
3211  DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3212  Results.push_back(
3213  DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3214  }
3215  break;
3217  if (Node->getValueType(0) != MVT::f32) {
3218  // We can extend to types bigger than f32 in two steps without changing
3219  // the result. Since "f16 -> f32" is much more commonly available, give
3220  // CodeGen the option of emitting that before resorting to a libcall.
3221  SDValue Res =
3223  {Node->getOperand(0), Node->getOperand(1)});
3224  Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
3225  {Node->getValueType(0), MVT::Other},
3226  {Res.getValue(1), Res});
3227  Results.push_back(Res);
3228  Results.push_back(Res.getValue(1));
3229  }
3230  break;
3231  case ISD::FP_TO_FP16:
3232  LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3233  if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3234  SDValue Op = Node->getOperand(0);
3235  MVT SVT = Op.getSimpleValueType();
3236  if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3237  TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3238  // Under fastmath, we can expand this node into a fround followed by
3239  // a float-half conversion.
3240  SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3241  DAG.getIntPtrConstant(0, dl));
3242  Results.push_back(
3243  DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3244  }
3245  }
3246  break;
3247  case ISD::ConstantFP: {
3248  ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3249  // Check to see if this FP immediate is already legal.
3250  // If this is a legal constant, turn it into a TargetConstantFP node.
3251  if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0),
3252  DAG.shouldOptForSize()))
3253  Results.push_back(ExpandConstantFP(CFP, true));
3254  break;
3255  }
3256  case ISD::Constant: {
3257  ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3258  Results.push_back(ExpandConstant(CP));
3259  break;
3260  }
3261  case ISD::FSUB: {
3262  EVT VT = Node->getValueType(0);
3263  if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3264  TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3265  const SDNodeFlags Flags = Node->getFlags();
3266  Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3267  Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3268  Results.push_back(Tmp1);
3269  }
3270  break;
3271  }
3272  case ISD::SUB: {
3273  EVT VT = Node->getValueType(0);
3274  assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3275  TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3276  "Don't know how to expand this subtraction!");
3277  Tmp1 = DAG.getNOT(dl, Node->getOperand(1), VT);
3278  Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3279  Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3280  break;
3281  }
3282  case ISD::UREM:
3283  case ISD::SREM:
3284  if (TLI.expandREM(Node, Tmp1, DAG))
3285  Results.push_back(Tmp1);
3286  break;
3287  case ISD::UDIV:
3288  case ISD::SDIV: {
3289  bool isSigned = Node->getOpcode() == ISD::SDIV;
3290  unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3291  EVT VT = Node->getValueType(0);
3292  if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3293  SDVTList VTs = DAG.getVTList(VT, VT);
3294  Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3295  Node->getOperand(1));
3296  Results.push_back(Tmp1);
3297  }
3298  break;
3299  }
3300  case ISD::MULHU:
3301  case ISD::MULHS: {
3302  unsigned ExpandOpcode =
3303  Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3304  EVT VT = Node->getValueType(0);
3305  SDVTList VTs = DAG.getVTList(VT, VT);
3306 
3307  Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3308  Node->getOperand(1));
3309  Results.push_back(Tmp1.getValue(1));
3310  break;
3311  }
3312  case ISD::UMUL_LOHI:
3313  case ISD::SMUL_LOHI: {
3314  SDValue LHS = Node->getOperand(0);
3315  SDValue RHS = Node->getOperand(1);
3316  MVT VT = LHS.getSimpleValueType();
3317  unsigned MULHOpcode =
3318  Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3319 
3320  if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3321  Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3322  Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3323  break;
3324  }
3325 
3326  SmallVector<SDValue, 4> Halves;
3327  EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3328  assert(TLI.isTypeLegal(HalfType));
3329  if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3330  HalfType, DAG,
3332  for (unsigned i = 0; i < 2; ++i) {
3333  SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3334  SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3335  SDValue Shift = DAG.getConstant(
3336  HalfType.getScalarSizeInBits(), dl,
3337  TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3338  Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3339  Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3340  }
3341  break;
3342  }
3343  break;
3344  }
3345  case ISD::MUL: {
3346  EVT VT = Node->getValueType(0);
3347  SDVTList VTs = DAG.getVTList(VT, VT);
3348  // See if multiply or divide can be lowered using two-result operations.
3349  // We just need the low half of the multiply; try both the signed
3350  // and unsigned forms. If the target supports both SMUL_LOHI and
3351  // UMUL_LOHI, form a preference by checking which forms of plain
3352  // MULH it supports.
3353  bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3354  bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3355  bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3356  bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3357  unsigned OpToUse = 0;
3358  if (HasSMUL_LOHI && !HasMULHS) {
3359  OpToUse = ISD::SMUL_LOHI;
3360  } else if (HasUMUL_LOHI && !HasMULHU) {
3361  OpToUse = ISD::UMUL_LOHI;
3362  } else if (HasSMUL_LOHI) {
3363  OpToUse = ISD::SMUL_LOHI;
3364  } else if (HasUMUL_LOHI) {
3365  OpToUse = ISD::UMUL_LOHI;
3366  }
3367  if (OpToUse) {
3368  Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3369  Node->getOperand(1)));
3370  break;
3371  }
3372 
3373  SDValue Lo, Hi;
3374  EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3375  if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3376  TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3377  TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3378  TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3379  TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3381  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3382  Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3383  SDValue Shift =
3384  DAG.getConstant(HalfType.getSizeInBits(), dl,
3385  TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3386  Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3387  Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3388  }
3389  break;
3390  }
3391  case ISD::FSHL:
3392  case ISD::FSHR:
3393  if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG))
3394  Results.push_back(Expanded);
3395  break;
3396  case ISD::ROTL:
3397  case ISD::ROTR:
3398  if (SDValue Expanded = TLI.expandROT(Node, true /*AllowVectorOps*/, DAG))
3399  Results.push_back(Expanded);
3400  break;
3401  case ISD::SADDSAT:
3402  case ISD::UADDSAT:
3403  case ISD::SSUBSAT:
3404  case ISD::USUBSAT:
3405  Results.push_back(TLI.expandAddSubSat(Node, DAG));
3406  break;
3407  case ISD::SSHLSAT:
3408  case ISD::USHLSAT:
3409  Results.push_back(TLI.expandShlSat(Node, DAG));
3410  break;
3411  case ISD::SMULFIX:
3412  case ISD::SMULFIXSAT:
3413  case ISD::UMULFIX:
3414  case ISD::UMULFIXSAT:
3415  Results.push_back(TLI.expandFixedPointMul(Node, DAG));
3416  break;
3417  case ISD::SDIVFIX:
3418  case ISD::SDIVFIXSAT:
3419  case ISD::UDIVFIX:
3420  case ISD::UDIVFIXSAT:
3421  if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node),
3422  Node->getOperand(0),
3423  Node->getOperand(1),
3424  Node->getConstantOperandVal(2),
3425  DAG)) {
3426  Results.push_back(V);
3427  break;
3428  }
3429  // FIXME: We might want to retry here with a wider type if we fail, if that
3430  // type is legal.
3431  // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
3432  // <= 128 (which is the case for all of the default Embedded-C types),
3433  // we will only get here with types and scales that we could always expand
3434  // if we were allowed to generate libcalls to division functions of illegal
3435  // type. But we cannot do that.
3436  llvm_unreachable("Cannot expand DIVFIX!");
3437  case ISD::ADDCARRY:
3438  case ISD::SUBCARRY: {
3439  SDValue LHS = Node->getOperand(0);
3440  SDValue RHS = Node->getOperand(1);
3441  SDValue Carry = Node->getOperand(2);
3442 
3443  bool IsAdd = Node->getOpcode() == ISD::ADDCARRY;
3444 
3445  // Initial add of the 2 operands.
3446  unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
3447  EVT VT = LHS.getValueType();
3448  SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS);
3449 
3450  // Initial check for overflow.
3451  EVT CarryType = Node->getValueType(1);
3452  EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3453  ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
3454  SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3455 
3456  // Add of the sum and the carry.
3457  SDValue One = DAG.getConstant(1, dl, VT);
3458  SDValue CarryExt =
3459  DAG.getNode(ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3460  SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt);
3461 
3462  // Second check for overflow. If we are adding, we can only overflow if the
3463  // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
3464  // If we are subtracting, we can only overflow if the initial sum is 0 and
3465  // the carry is set, resulting in a new sum of all 1s.
3466  SDValue Zero = DAG.getConstant(0, dl, VT);
3467  SDValue Overflow2 =
3468  IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ)
3469  : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ);
3470  Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2,
3471  DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3472 
3473  SDValue ResultCarry =
3474  DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2);
3475 
3476  Results.push_back(Sum2);
3477  Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3478  break;
3479  }
3480  case ISD::SADDO:
3481  case ISD::SSUBO: {
3482  SDValue Result, Overflow;
3483  TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
3484  Results.push_back(Result);
3485  Results.push_back(Overflow);
3486  break;
3487  }
3488  case ISD::UADDO:
3489  case ISD::USUBO: {
3490  SDValue Result, Overflow;
3491  TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
3492  Results.push_back(Result);
3493  Results.push_back(Overflow);
3494  break;
3495  }
3496  case ISD::UMULO:
3497  case ISD::SMULO: {
3498  SDValue Result, Overflow;
3499  if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
3500  Results.push_back(Result);
3501  Results.push_back(Overflow);
3502  }
3503  break;
3504  }
3505  case ISD::BUILD_PAIR: {
3506  EVT PairTy = Node->getValueType(0);
3507  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3508  Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3509  Tmp2 = DAG.getNode(
3510  ISD::SHL, dl, PairTy, Tmp2,
3511  DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3512  TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3513  Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3514  break;
3515  }
3516  case ISD::SELECT:
3517  Tmp1 = Node->getOperand(0);
3518  Tmp2 = Node->getOperand(1);
3519  Tmp3 = Node->getOperand(2);
3520  if (Tmp1.getOpcode() == ISD::SETCC) {
3521  Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3522  Tmp2, Tmp3,
3523  cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3524  } else {
3525  Tmp1 = DAG.getSelectCC(dl, Tmp1,
3526  DAG.getConstant(0, dl, Tmp1.getValueType()),
3527  Tmp2, Tmp3, ISD::SETNE);
3528  }
3529  Tmp1->setFlags(Node->getFlags());
3530  Results.push_back(Tmp1);
3531  break;
3532  case ISD::BR_JT: {
3533  SDValue Chain = Node->getOperand(0);
3534  SDValue Table = Node->getOperand(1);
3535  SDValue Index = Node->getOperand(2);
3536 
3537  const DataLayout &TD = DAG.getDataLayout();
3538  EVT PTy = TLI.getPointerTy(TD);
3539 
3540  unsigned EntrySize =
3541  DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3542 
3543  // For power-of-two jumptable entry sizes convert multiplication to a shift.
3544  // This transformation needs to be done here since otherwise the MIPS
3545  // backend will end up emitting a three instruction multiply sequence
3546  // instead of a single shift and MSP430 will call a runtime function.
3547  if (llvm::isPowerOf2_32(EntrySize))
3548  Index = DAG.getNode(
3549  ISD::SHL, dl, Index.getValueType(), Index,
3550  DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
3551  else
3552  Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3553  DAG.getConstant(EntrySize, dl, Index.getValueType()));
3554  SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3555  Index, Table);
3556 
3557  EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3558  SDValue LD = DAG.getExtLoad(
3559  ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3560  MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3561  Addr = LD;
3562  if (TLI.isJumpTableRelative()) {
3563  // For PIC, the sequence is:
3564  // BRIND(load(Jumptable + index) + RelocBase)
3565  // RelocBase can be JumpTable, GOT or some sort of global base.
3566  Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3567  TLI.getPICJumpTableRelocBase(Table, DAG));
3568  }
3569 
3570  Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG);
3571  Results.push_back(Tmp1);
3572  break;
3573  }
3574  case ISD::BRCOND:
3575  // Expand brcond's setcc into its constituent parts and create a BR_CC
3576  // Node.
3577  Tmp1 = Node->getOperand(0);
3578  Tmp2 = Node->getOperand(1);
3579  if (Tmp2.getOpcode() == ISD::SETCC &&
3580  TLI.isOperationLegalOrCustom(ISD::BR_CC,
3581  Tmp2.getOperand(0).getValueType())) {
3582  Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, Tmp2.getOperand(2),
3583  Tmp2.getOperand(0), Tmp2.getOperand(1),
3584  Node->getOperand(2));
3585  } else {
3586  // We test only the i1 bit. Skip the AND if UNDEF or another AND.
3587  if (Tmp2.isUndef() ||
3588  (Tmp2.getOpcode() == ISD::AND &&
3589  isa<ConstantSDNode>(Tmp2.getOperand(1)) &&
3590  cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1))
3591  Tmp3 = Tmp2;
3592  else
3593  Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3594  DAG.getConstant(1, dl, Tmp2.getValueType()));
3595  Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3596  DAG.getCondCode(ISD::SETNE), Tmp3,
3597  DAG.getConstant(0, dl, Tmp3.getValueType()),
3598  Node->getOperand(2));
3599  }
3600  Results.push_back(Tmp1);
3601  break;
3602  case ISD::SETCC:
3603  case ISD::VP_SETCC:
3604  case ISD::STRICT_FSETCC:
3605  case ISD::STRICT_FSETCCS: {
3606  bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
3607  bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
3608  Node->getOpcode() == ISD::STRICT_FSETCCS;
3609  bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
3610  SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
3611  unsigned Offset = IsStrict ? 1 : 0;
3612  Tmp1 = Node->getOperand(0 + Offset);
3613  Tmp2 = Node->getOperand(1 + Offset);
3614  Tmp3 = Node->getOperand(2 + Offset);
3615  SDValue Mask, EVL;
3616  if (IsVP) {
3617  Mask = Node->getOperand(3 + Offset);
3618  EVL = Node->getOperand(4 + Offset);
3619  }
3620  bool Legalized = TLI.LegalizeSetCCCondCode(
3621  DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl,
3622  Chain, IsSignaling);
3623 
3624  if (Legalized) {
3625  // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3626  // condition code, create a new SETCC node.
3627  if (Tmp3.getNode()) {
3628  if (IsStrict) {
3629  Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(),
3630  {Chain, Tmp1, Tmp2, Tmp3}, Node->getFlags());
3631  Chain = Tmp1.getValue(1);
3632  } else if (IsVP) {
3633  Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0),
3634  {Tmp1, Tmp2, Tmp3, Mask, EVL}, Node->getFlags());
3635  } else {
3636  Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Tmp1,
3637  Tmp2, Tmp3, Node->getFlags());
3638  }
3639  }
3640 
3641  // If we expanded the SETCC by inverting the condition code, then wrap
3642  // the existing SETCC in a NOT to restore the intended condition.
3643  if (NeedInvert) {
3644  if (!IsVP)
3645  Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
3646  else
3647  Tmp1 =
3648  DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->getValueType(0));
3649  }
3650 
3651  Results.push_back(Tmp1);
3652  if (IsStrict)
3653  Results.push_back(Chain);
3654 
3655  break;
3656  }
3657 
3658  // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
3659  // understand if this code is useful for strict nodes.
3660  assert(!IsStrict && "Don't know how to expand for strict nodes.");
3661 
3662  // Otherwise, SETCC for the given comparison type must be completely
3663  // illegal; expand it into a SELECT_CC.
3664  // FIXME: This drops the mask/evl for VP_SETCC.
3665  EVT VT = Node->getValueType(0);
3666  EVT Tmp1VT = Tmp1.getValueType();
3667  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3668  DAG.getBoolConstant(true, dl, VT, Tmp1VT),
3669  DAG.getBoolConstant(false, dl, VT, Tmp1VT), Tmp3);
3670  Tmp1->setFlags(Node->getFlags());
3671  Results.push_back(Tmp1);
3672  break;
3673  }
3674  case ISD::SELECT_CC: {
3675  // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
3676  Tmp1 = Node->getOperand(0); // LHS
3677  Tmp2 = Node->getOperand(1); // RHS
3678  Tmp3 = Node->getOperand(2); // True
3679  Tmp4 = Node->getOperand(3); // False
3680  EVT VT = Node->getValueType(0);
3681  SDValue Chain;
3682  SDValue CC = Node->getOperand(4);
3683  ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
3684 
3685  if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
3686  // If the condition code is legal, then we need to expand this
3687  // node using SETCC and SELECT.
3688  EVT CmpVT = Tmp1.getValueType();
3689  assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
3690  "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
3691  "expanded.");
3692  EVT CCVT = getSetCCResultType(CmpVT);
3693  SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags());
3694  Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
3695  break;
3696  }
3697 
3698  // SELECT_CC is legal, so the condition code must not be.
3699  bool Legalized = false;
3700  // Try to legalize by inverting the condition. This is for targets that
3701  // might support an ordered version of a condition, but not the unordered
3702  // version (or vice versa).
3703  ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, Tmp1.getValueType());
3704  if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
3705  // Use the new condition code and swap true and false
3706  Legalized = true;
3707  Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3708  Tmp1->setFlags(Node->getFlags());
3709  } else {
3710  // If The inverse is not legal, then try to swap the arguments using
3711  // the inverse condition code.
3712  ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
3713  if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
3714  // The swapped inverse condition is legal, so swap true and false,
3715  // lhs and rhs.
3716  Legalized = true;
3717  Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3718  Tmp1->setFlags(Node->getFlags());
3719  }
3720  }
3721 
3722  if (!Legalized) {
3723  Legalized = TLI.LegalizeSetCCCondCode(
3724  DAG, getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC,
3725  /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
3726 
3727  assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
3728 
3729  // If we expanded the SETCC by inverting the condition code, then swap
3730  // the True/False operands to match.
3731  if (NeedInvert)
3732  std::swap(Tmp3, Tmp4);
3733 
3734  // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3735  // condition code, create a new SELECT_CC node.
3736  if (CC.getNode()) {
3737  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
3738  Tmp1, Tmp2, Tmp3, Tmp4, CC);
3739  } else {
3740  Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
3741  CC = DAG.getCondCode(ISD::SETNE);
3742  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
3743  Tmp2, Tmp3, Tmp4, CC);
3744  }
3745  Tmp1->setFlags(Node->getFlags());
3746  }
3747  Results.push_back(Tmp1);
3748  break;
3749  }
3750  case ISD::BR_CC: {
3751  // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
3752  SDValue Chain;
3753  Tmp1 = Node->getOperand(0); // Chain
3754  Tmp2 = Node->getOperand(2); // LHS
3755  Tmp3 = Node->getOperand(3); // RHS
3756  Tmp4 = Node->getOperand(1); // CC
3757 
3758  bool Legalized = TLI.LegalizeSetCCCondCode(
3759  DAG, getSetCCResultType(Tmp2.getValueType()), Tmp2, Tmp3, Tmp4,
3760  /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
3761  (void)Legalized;
3762  assert(Legalized && "Can't legalize BR_CC with legal condition!");
3763 
3764  // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
3765  // node.
3766  if (Tmp4.getNode()) {
3767  assert(!NeedInvert && "Don't know how to invert BR_CC!");
3768 
3769  Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
3770  Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3771  } else {
3772  Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
3773  Tmp4 = DAG.getCondCode(NeedInvert ? ISD::SETEQ : ISD::SETNE);
3774  Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
3775  Tmp2, Tmp3, Node->getOperand(4));
3776  }
3777  Results.push_back(Tmp1);
3778  break;
3779  }
3780  case ISD::BUILD_VECTOR:
3781  Results.push_back(ExpandBUILD_VECTOR(Node));
3782  break;
3783  case ISD::SPLAT_VECTOR:
3784  Results.push_back(ExpandSPLAT_VECTOR(Node));
3785  break;
3786  case ISD::SRA:
3787  case ISD::SRL:
3788  case ISD::SHL: {
3789  // Scalarize vector SRA/SRL/SHL.
3790  EVT VT = Node->getValueType(0);
3791  assert(VT.isVector() && "Unable to legalize non-vector shift");
3792  assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
3793  unsigned NumElem = VT.getVectorNumElements();
3794 
3795  SmallVector<SDValue, 8> Scalars;
3796  for (unsigned Idx = 0; Idx < NumElem; Idx++) {
3797  SDValue Ex =
3799  Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl));
3800  SDValue Sh =
3802  Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl));
3803  Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
3804  VT.getScalarType(), Ex, Sh));
3805  }
3806 
3807  SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
3808  Results.push_back(Result);
3809  break;
3810  }
3811  case ISD::VECREDUCE_FADD:
3812  case ISD::VECREDUCE_FMUL:
3813  case ISD::VECREDUCE_ADD:
3814  case ISD::VECREDUCE_MUL:
3815  case ISD::VECREDUCE_AND:
3816  case ISD::VECREDUCE_OR:
3817  case ISD::VECREDUCE_XOR:
3818  case ISD::VECREDUCE_SMAX:
3819  case ISD::VECREDUCE_SMIN:
3820  case ISD::VECREDUCE_UMAX:
3821  case ISD::VECREDUCE_UMIN:
3822  case ISD::VECREDUCE_FMAX:
3823  case ISD::VECREDUCE_FMIN:
3824  Results.push_back(TLI.expandVecReduce(Node, DAG));
3825  break;
3827  case ISD::GlobalAddress:
3828  case ISD::GlobalTLSAddress:
3829  case ISD::ExternalSymbol:
3830  case ISD::ConstantPool:
3831  case ISD::JumpTable:
3834  case ISD::INTRINSIC_VOID:
3835  // FIXME: Custom lowering for these operations shouldn't return null!
3836  // Return true so that we don't call ConvertNodeToLibcall which also won't
3837  // do anything.
3838  return true;
3839  }
3840 
3841  if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
3842  // FIXME: We were asked to expand a strict floating-point operation,
3843  // but there is currently no expansion implemented that would preserve
3844  // the "strict" properties. For now, we just fall back to the non-strict
3845  // version if that is legal on the target. The actual mutation of the
3846  // operation will happen in SelectionDAGISel::DoInstructionSelection.
3847  switch (Node->getOpcode()) {
3848  default:
3849  if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3850  Node->getValueType(0))
3852  return true;
3853  break;
3854  case ISD::STRICT_FSUB: {
3855  if (TLI.getStrictFPOperationAction(
3856  ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal)
3857  return true;
3858  if (TLI.getStrictFPOperationAction(
3859  ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal)
3860  break;
3861 
3862  EVT VT = Node->getValueType(0);
3863  const SDNodeFlags Flags = Node->getFlags();
3864  SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags);
3865  SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(),
3866  {Node->getOperand(0), Node->getOperand(1), Neg},
3867  Flags);
3868 
3869  Results.push_back(Fadd);
3870  Results.push_back(Fadd.getValue(1));
3871  break;
3872  }
3875  case ISD::STRICT_LRINT:
3876  case ISD::STRICT_LLRINT:
3877  case ISD::STRICT_LROUND:
3878  case ISD::STRICT_LLROUND:
3879  // These are registered by the operand type instead of the value
3880  // type. Reflect that here.
3881  if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3882  Node->getOperand(1).getValueType())
3884  return true;
3885  break;
3886  }
3887  }
3888 
3889  // Replace the original node with the legalized result.
3890  if (Results.empty()) {
3891  LLVM_DEBUG(dbgs() << "Cannot expand node\n");
3892  return false;
3893  }
3894 
3895  LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
3896  ReplaceNode(Node, Results.data());
3897  return true;
3898 }
3899 
3900 void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
3901  LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
3903  SDLoc dl(Node);
3904  // FIXME: Check flags on the node to see if we can use a finite call.
3905  unsigned Opc = Node->getOpcode();
3906  switch (Opc) {
3907  case ISD::ATOMIC_FENCE: {
3908  // If the target didn't lower this, lower it to '__sync_synchronize()' call
3909  // FIXME: handle "fence singlethread" more efficiently.
3911 
3913  CLI.setDebugLoc(dl)
3914  .setChain(Node->getOperand(0))
3915  .setLibCallee(
3916  CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3917  DAG.getExternalSymbol("__sync_synchronize",
3918  TLI.getPointerTy(DAG.getDataLayout())),
3919  std::move(Args));
3920 
3921  std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3922 
3923  Results.push_back(CallResult.second);
3924  break;
3925  }
3926  // By default, atomic intrinsics are marked Legal and lowered. Targets
3927  // which don't support them directly, however, may want libcalls, in which
3928  // case they mark them Expand, and we get here.
3929  case ISD::ATOMIC_SWAP:
3930  case ISD::ATOMIC_LOAD_ADD:
3931  case ISD::ATOMIC_LOAD_SUB:
3932  case ISD::ATOMIC_LOAD_AND:
3933  case ISD::ATOMIC_LOAD_CLR:
3934  case ISD::ATOMIC_LOAD_OR:
3935  case ISD::ATOMIC_LOAD_XOR:
3936  case ISD::ATOMIC_LOAD_NAND:
3937  case ISD::ATOMIC_LOAD_MIN:
3938  case ISD::ATOMIC_LOAD_MAX:
3939  case ISD::ATOMIC_LOAD_UMIN:
3940  case ISD::ATOMIC_LOAD_UMAX:
3941  case ISD::ATOMIC_CMP_SWAP: {
3942  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
3943  AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering();
3944  RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
3945  EVT RetVT = Node->getValueType(0);
3948  if (TLI.getLibcallName(LC)) {
3949  // If outline atomic available, prepare its arguments and expand.
3950  Ops.append(Node->op_begin() + 2, Node->op_end());
3951  Ops.push_back(Node->getOperand(1));
3952 
3953  } else {
3954  LC = RTLIB::getSYNC(Opc, VT);
3955  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3956  "Unexpected atomic op or value type!");
3957  // Arguments for expansion to sync libcall
3958  Ops.append(Node->op_begin() + 1, Node->op_end());
3959  }
3960  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
3961  Ops, CallOptions,
3962  SDLoc(Node),
3963  Node->getOperand(0));
3964  Results.push_back(Tmp.first);
3965  Results.push_back(Tmp.second);
3966  break;
3967  }
3968  case ISD::TRAP: {
3969  // If this operation is not supported, lower it to 'abort()' call
3972  CLI.setDebugLoc(dl)
3973  .setChain(Node->getOperand(0))
3974  .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3975  DAG.getExternalSymbol(
3976  "abort", TLI.getPointerTy(DAG.getDataLayout())),
3977  std::move(Args));
3978  std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3979 
3980  Results.push_back(CallResult.second);
3981  break;
3982  }
3983  case ISD::FMINNUM:
3984  case ISD::STRICT_FMINNUM:
3985  ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
3986  RTLIB::FMIN_F80, RTLIB::FMIN_F128,
3987  RTLIB::FMIN_PPCF128, Results);
3988  break;
3989  case ISD::FMAXNUM:
3990  case ISD::STRICT_FMAXNUM:
3991  ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
3992  RTLIB::FMAX_F80, RTLIB::FMAX_F128,
3993  RTLIB::FMAX_PPCF128, Results);
3994  break;
3995  case ISD::FSQRT:
3996  case ISD::STRICT_FSQRT:
3997  ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
3998  RTLIB::SQRT_F80, RTLIB::SQRT_F128,
3999  RTLIB::SQRT_PPCF128, Results);
4000  break;
4001  case ISD::FCBRT:
4002  ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4003  RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4004  RTLIB::CBRT_PPCF128, Results);
4005  break;
4006  case ISD::FSIN:
4007  case ISD::STRICT_FSIN:
4008  ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4009  RTLIB::SIN_F80, RTLIB::SIN_F128,
4010  RTLIB::SIN_PPCF128, Results);
4011  break;
4012  case ISD::FCOS:
4013  case ISD::STRICT_FCOS:
4014  ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
4015  RTLIB::COS_F80, RTLIB::COS_F128,
4016  RTLIB::COS_PPCF128, Results);
4017  break;
4018  case ISD::FSINCOS:
4019  // Expand into sincos libcall.
4020  ExpandSinCosLibCall(Node, Results);
4021  break;
4022  case ISD::FLOG:
4023  case ISD::STRICT_FLOG:
4024  ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
4025  RTLIB::LOG_F128, RTLIB::LOG_PPCF128, Results);
4026  break;
4027  case ISD::FLOG2:
4028  case ISD::STRICT_FLOG2:
4029  ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
4030  RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128, Results);
4031  break;
4032  case ISD::FLOG10:
4033  case ISD::STRICT_FLOG10:
4034  ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4035  RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128, Results);
4036  break;
4037  case ISD::FEXP:
4038  case ISD::STRICT_FEXP:
4039  ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4040  RTLIB::EXP_F128, RTLIB::EXP_PPCF128, Results);
4041  break;
4042  case ISD::FEXP2:
4043  case ISD::STRICT_FEXP2:
4044  ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4045  RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128, Results);
4046  break;
4047  case ISD::FTRUNC:
4048  case ISD::STRICT_FTRUNC:
4049  ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4050  RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4051  RTLIB::TRUNC_PPCF128, Results);
4052  break;
4053  case ISD::FFLOOR:
4054  case ISD::STRICT_FFLOOR:
4055  ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4056  RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4057  RTLIB::FLOOR_PPCF128, Results);
4058  break;
4059  case ISD::FCEIL:
4060  case ISD::STRICT_FCEIL:
4061  ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4062  RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4063  RTLIB::CEIL_PPCF128, Results);
4064  break;
4065  case ISD::FRINT:
4066  case ISD::STRICT_FRINT:
4067  ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4068  RTLIB::RINT_F80, RTLIB::RINT_F128,
4069  RTLIB::RINT_PPCF128, Results);
4070  break;
4071  case ISD::FNEARBYINT:
4073  ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4074  RTLIB::NEARBYINT_F64,
4075  RTLIB::NEARBYINT_F80,
4076  RTLIB::NEARBYINT_F128,
4077  RTLIB::NEARBYINT_PPCF128, Results);
4078  break;
4079  case ISD::FROUND:
4080  case ISD::STRICT_FROUND:
4081  ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4082  RTLIB::ROUND_F64,
4083  RTLIB::ROUND_F80,
4084  RTLIB::ROUND_F128,
4085  RTLIB::ROUND_PPCF128, Results);
4086  break;
4087  case ISD::FROUNDEVEN:
4089  ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32,
4090  RTLIB::ROUNDEVEN_F64,
4091  RTLIB::ROUNDEVEN_F80,
4092  RTLIB::ROUNDEVEN_F128,
4093  RTLIB::ROUNDEVEN_PPCF128, Results);
4094  break;
4095  case ISD::FPOWI:
4096  case ISD::STRICT_FPOWI: {
4097  RTLIB::Libcall LC = RTLIB::getPOWI(Node->getSimpleValueType(0));
4098  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
4099  if (!TLI.getLibcallName(LC)) {
4100  // Some targets don't have a powi libcall; use pow instead.
4101  if (Node->isStrictFPOpcode()) {
4102  SDValue Exponent =
4103  DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(Node),
4104  {Node->getValueType(0), Node->getValueType(1)},
4105  {Node->getOperand(0), Node->getOperand(2)});
4106  SDValue FPOW =
4107  DAG.getNode(ISD::STRICT_FPOW, SDLoc(Node),
4108  {Node->getValueType(0), Node->getValueType(1)},
4109  {Exponent.getValue(1), Node->getOperand(1), Exponent});
4110  Results.push_back(FPOW);
4111  Results.push_back(FPOW.getValue(1));
4112  } else {
4113  SDValue Exponent =
4114  DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), Node->getValueType(0),
4115  Node->getOperand(1));
4116  Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node),
4117  Node->getValueType(0),
4118  Node->getOperand(0), Exponent));
4119  }
4120  break;
4121  }
4122  unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
4123  bool ExponentHasSizeOfInt =
4124  DAG.getLibInfo().getIntSize() ==
4125  Node->getOperand(1 + Offset).getValueType().getSizeInBits();
4126  if (!ExponentHasSizeOfInt) {
4127  // If the exponent does not match with sizeof(int) a libcall to
4128  // RTLIB::POWI would use the wrong type for the argument.
4129  DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
4130  Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
4131  break;
4132  }
4133  ExpandFPLibCall(Node, LC, Results);
4134  break;
4135  }
4136  case ISD::FPOW:
4137  case ISD::STRICT_FPOW:
4138  ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4139  RTLIB::POW_F128, RTLIB::POW_PPCF128, Results);
4140  break;
4141  case ISD::LROUND:
4142  case ISD::STRICT_LROUND:
4143  ExpandArgFPLibCall(Node, RTLIB::LROUND_F32,
4144  RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4145  RTLIB::LROUND_F128,
4146  RTLIB::LROUND_PPCF128, Results);
4147  break;
4148  case ISD::LLROUND:
4149  case ISD::STRICT_LLROUND:
4150  ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32,
4151  RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4152  RTLIB::LLROUND_F128,
4153  RTLIB::LLROUND_PPCF128, Results);
4154  break;
4155  case ISD::LRINT:
4156  case ISD::STRICT_LRINT:
4157  ExpandArgFPLibCall(Node, RTLIB::LRINT_F32,
4158  RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4159  RTLIB::LRINT_F128,
4160  RTLIB::LRINT_PPCF128, Results);
4161  break;
4162  case ISD::LLRINT:
4163  case ISD::STRICT_LLRINT:
4164  ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32,
4165  RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4166  RTLIB::LLRINT_F128,
4167  RTLIB::LLRINT_PPCF128, Results);
4168  break;
4169  case ISD::FDIV:
4170  case ISD::STRICT_FDIV:
4171  ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4172  RTLIB::DIV_F80, RTLIB::DIV_F128,
4173  RTLIB::DIV_PPCF128, Results);
4174  break;
4175  case ISD::FREM:
4176  case ISD::STRICT_FREM:
4177  ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4178  RTLIB::REM_F80, RTLIB::REM_F128,
4179  RTLIB::REM_PPCF128, Results);
4180  break;
4181  case ISD::FMA:
4182  case ISD::STRICT_FMA:
4183  ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4184  RTLIB::FMA_F80, RTLIB::FMA_F128,
4185  RTLIB::FMA_PPCF128, Results);
4186  break;
4187  case ISD::FADD:
4188  case ISD::STRICT_FADD:
4189  ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4190  RTLIB::ADD_F80, RTLIB::ADD_F128,
4191  RTLIB::ADD_PPCF128, Results);
4192  break;
4193  case ISD::FMUL:
4194  case ISD::STRICT_FMUL:
4195  ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4196  RTLIB::MUL_F80, RTLIB::MUL_F128,
4197  RTLIB::MUL_PPCF128, Results);
4198  break;
4199  case ISD::FP16_TO_FP:
4200  if (Node->getValueType(0) == MVT::f32) {
4201  Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
4202  }
4203  break;
4204  case ISD::STRICT_FP16_TO_FP: {
4205  if (Node->getValueType(0) == MVT::f32) {
4207  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4208  DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions,
4209  SDLoc(Node), Node->getOperand(0));
4210  Results.push_back(Tmp.first);
4211  Results.push_back(Tmp.second);
4212  }
4213  break;
4214  }
4215  case ISD::FP_TO_FP16: {
4216  RTLIB::Libcall LC =
4217  RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4218  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
4219  Results.push_back(ExpandLibCall(LC, Node, false));
4220  break;
4221  }
4224  case ISD::SINT_TO_FP:
4225  case ISD::UINT_TO_FP: {
4226  // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
4227  bool IsStrict = Node->isStrictFPOpcode();
4228  bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
4229  Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
4230  EVT SVT = Node->getOperand(IsStrict ? 1 : 0).getValueType();
4231  EVT RVT = Node->getValueType(0);
4232  EVT NVT = EVT();
4233  SDLoc dl(Node);
4234 
4235  // Even if the input is legal, no libcall may exactly match, eg. we don't
4236  // have i1 -> fp conversions. So, it needs to be promoted to a larger type,
4237  // eg: i13 -> fp. Then, look for an appropriate libcall.
4238  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4239  for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
4240  t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4241  ++t) {
4242  NVT = (MVT::SimpleValueType)t;
4243  // The source needs to big enough to hold the operand.
4244  if (NVT.bitsGE(SVT))
4245  LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT)
4246  : RTLIB::getUINTTOFP(NVT, RVT);
4247  }
4248  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4249 
4250  SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4251  // Sign/zero extend the argument if the libcall takes a larger type.
4252  SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
4253  NVT, Node->getOperand(IsStrict ? 1 : 0));
4255  CallOptions.setSExt(Signed);
4256  std::pair<SDValue, SDValue> Tmp =
4257  TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, dl, Chain);
4258  Results.push_back(Tmp.first);
4259  if (IsStrict)
4260  Results.push_back(Tmp.second);
4261  break;
4262  }
4263  case ISD::FP_TO_SINT:
4264  case ISD::FP_TO_UINT:
4266  case ISD::STRICT_FP_TO_UINT: {
4267  // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
4268  bool IsStrict = Node->isStrictFPOpcode();
4269  bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
4270  Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
4271 
4272  SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
4273  EVT SVT = Op.getValueType();
4274  EVT RVT = Node->getValueType(0);
4275  EVT NVT = EVT();
4276  SDLoc dl(Node);
4277 
4278  // Even if the result is legal, no libcall may exactly match, eg. we don't
4279  // have fp -> i1 conversions. So, it needs to be promoted to a larger type,
4280  // eg: fp -> i32. Then, look for an appropriate libcall.
4281  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4282  for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
4283  IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4284  ++IntVT) {
4285  NVT = (MVT::SimpleValueType)IntVT;
4286  // The type needs to big enough to hold the result.
4287  if (NVT.bitsGE(RVT))
4288  LC = Signed ? RTLIB::getFPTOSINT(SVT, NVT)
4289  : RTLIB::getFPTOUINT(SVT, NVT);
4290  }
4291  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4292 
4293  SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4295  std::pair<SDValue, SDValue> Tmp =
4296  TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
4297 
4298  // Truncate the result if the libcall returns a larger type.
4299  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first));
4300  if (IsStrict)
4301  Results.push_back(Tmp.second);
4302  break;
4303  }
4304 
4305  case ISD::FP_ROUND:
4306  case ISD::STRICT_FP_ROUND: {
4307  // X = FP_ROUND(Y, TRUNC)
4308  // TRUNC is a flag, which is always an integer that is zero or one.
4309  // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
4310  // is known to not change the value of Y.
4311  // We can only expand it into libcall if the TRUNC is 0.
4312  bool IsStrict = Node->isStrictFPOpcode();
4313  SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
4314  SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4315  EVT VT = Node->getValueType(0);
4316  assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
4317  "Unable to expand as libcall if it is not normal rounding");
4318 
4319  RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), VT);
4320  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4321 
4323  std::pair<SDValue, SDValue> Tmp =
4324  TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, SDLoc(Node), Chain);
4325  Results.push_back(Tmp.first);
4326  if (IsStrict)
4327  Results.push_back(Tmp.second);
4328  break;
4329  }
4330  case ISD::FP_EXTEND: {
4331  Results.push_back(
4332  ExpandLibCall(RTLIB::getFPEXT(Node->getOperand(0).getValueType(),
4333  Node->getValueType(0)),
4334  Node, false));
4335  break;
4336  }
4337  case ISD::STRICT_FP_EXTEND:
4338  case ISD::STRICT_FP_TO_FP16: {
4339  RTLIB::Libcall LC =
4340  Node->getOpcode() == ISD::STRICT_FP_TO_FP16
4341  ? RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16)
4342  : RTLIB::getFPEXT(Node->getOperand(1).getValueType(),
4343  Node->getValueType(0));
4344  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4345 
4347  std::pair<SDValue, SDValue> Tmp =
4348  TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1),
4349  CallOptions, SDLoc(Node), Node->getOperand(0));
4350  Results.push_back(Tmp.first);
4351  Results.push_back(Tmp.second);
4352  break;
4353  }
4354  case ISD::FSUB:
4355  case ISD::STRICT_FSUB:
4356  ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4357  RTLIB::SUB_F80, RTLIB::SUB_F128,
4358  RTLIB::SUB_PPCF128, Results);
4359  break;
4360  case ISD::SREM:
4361  Results.push_back(ExpandIntLibCall(
4362  Node, true, RTLIB::SREM_I8, RTLIB::SREM_I16, RTLIB::SREM_I32,
4363  RTLIB::SREM_I64, RTLIB::SREM_I128, RTLIB::SREM_IEXT));
4364  break;
4365  case ISD::UREM:
4366  Results.push_back(ExpandIntLibCall(
4367  Node, false, RTLIB::UREM_I8, RTLIB::UREM_I16, RTLIB::UREM_I32,
4368  RTLIB::UREM_I64, RTLIB::UREM_I128, RTLIB::UREM_IEXT));
4369  break;
4370  case ISD::SDIV:
4371  Results.push_back(ExpandIntLibCall(
4372  Node, true, RTLIB::SDIV_I8, RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4373  RTLIB::SDIV_I64, RTLIB::SDIV_I128, RTLIB::SDIV_IEXT));
4374  break;
4375  case ISD::UDIV:
4376  Results.push_back(ExpandIntLibCall(
4377  Node, false, RTLIB::UDIV_I8, RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4378  RTLIB::UDIV_I64, RTLIB::UDIV_I128, RTLIB::UDIV_IEXT));
4379  break;
4380  case ISD::SDIVREM:
4381  case ISD::UDIVREM:
4382  // Expand into divrem libcall
4383  ExpandDivRemLibCall(Node, Results);
4384  break;
4385  case ISD::MUL:
4386  Results.push_back(ExpandIntLibCall(
4387  Node, false, RTLIB::MUL_I8, RTLIB::MUL_I16, RTLIB::MUL_I32,
4388  RTLIB::MUL_I64, RTLIB::MUL_I128, RTLIB::MUL_IEXT));
4389  break;
4390  case ISD::CTLZ_ZERO_UNDEF:
4391  switch (Node->getSimpleValueType(0).SimpleTy) {
4392  default:
4393  llvm_unreachable("LibCall explicitly requested, but not available");
4394  case MVT::i32:
4395  Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false));
4396  break;
4397  case MVT::i64:
4398  Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false));
4399  break;
4400  case MVT::i128:
4401  Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false));
4402  break;
4403  }
4404  break;
4405  }
4406 
4407  // Replace the original node with the legalized result.
4408  if (!Results.empty()) {
4409  LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
4410  ReplaceNode(Node, Results.data());
4411  } else
4412  LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
4413 }
4414 
4415 // Determine the vector type to use in place of an original scalar element when
4416 // promoting equally sized vectors.
4418  MVT EltVT, MVT NewEltVT) {
4419  unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4420  MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4421  assert(TLI.isTypeLegal(MidVT) && "unexpected");
4422  return MidVT;
4423 }
4424 
4425 void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4426  LLVM_DEBUG(dbgs() << "Trying to promote node\n");
4428  MVT OVT = Node->getSimpleValueType(0);
4429  if (Node->getOpcode() == ISD::UINT_TO_FP ||
4430  Node->getOpcode() == ISD::SINT_TO_FP ||
4431  Node->getOpcode() == ISD::SETCC ||
4432  Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4433  Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4434  OVT = Node->getOperand(0).getSimpleValueType();
4435  }
4436  if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
4437  Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
4438  Node->getOpcode() == ISD::STRICT_FSETCC ||
4439  Node->getOpcode() == ISD::STRICT_FSETCCS)
4440  OVT = Node->getOperand(1).getSimpleValueType();
4441  if (Node->getOpcode() == ISD::BR_CC ||
4442  Node->getOpcode() == ISD::SELECT_CC)
4443  OVT = Node->getOperand(2).getSimpleValueType();
4444  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4445  SDLoc dl(Node);
4446  SDValue Tmp1, Tmp2, Tmp3, Tmp4;
4447  switch (Node->getOpcode()) {
4448  case ISD::CTTZ:
4449  case ISD::CTTZ_ZERO_UNDEF:
4450  case ISD::CTLZ:
4451  case ISD::CTLZ_ZERO_UNDEF:
4452  case ISD::CTPOP:
4453  // Zero extend the argument unless its cttz, then use any_extend.
4454  if (Node->getOpcode() == ISD::CTTZ ||
4455  Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)
4456  Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));
4457  else
4458  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4459 
4460  if (Node->getOpcode() == ISD::CTTZ) {
4461  // The count is the same in the promoted type except if the original
4462  // value was zero. This can be handled by setting the bit just off
4463  // the top of the original type.
4464  auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4465  OVT.getSizeInBits());
4466  Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4467  DAG.getConstant(TopBit, dl, NVT));
4468  }
4469  // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4470  // already the correct result.
4471  Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4472  if (Node->getOpcode() == ISD::CTLZ ||
4473  Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4474  // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4475  Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4476  DAG.getConstant(NVT.getSizeInBits() -
4477  OVT.getSizeInBits(), dl, NVT));
4478  }
4479  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4480  break;
4481  case ISD::BITREVERSE:
4482  case ISD::BSWAP: {
4483  unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4484  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4485  Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4486  Tmp1 = DAG.getNode(
4487  ISD::SRL, dl, NVT, Tmp1,
4488  DAG.getConstant(DiffBits, dl,
4489  TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4490 
4491  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4492  break;
4493  }
4494  case ISD::FP_TO_UINT:
4496  case ISD::FP_TO_SINT:
4498  PromoteLegalFP_TO_INT(Node, dl, Results);
4499  break;
4500  case ISD::FP_TO_UINT_SAT:
4501  case ISD::FP_TO_SINT_SAT:
4502  Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl));
4503  break;
4504  case ISD::UINT_TO_FP:
4506  case ISD::SINT_TO_FP:
4508  PromoteLegalINT_TO_FP(Node, dl, Results);
4509  break;
4510  case ISD::VAARG: {
4511  SDValue Chain = Node->getOperand(0); // Get the chain.
4512  SDValue Ptr = Node->getOperand(1); // Get the pointer.
4513 
4514  unsigned TruncOp;
4515  if (OVT.isVector()) {
4516  TruncOp = ISD::BITCAST;
4517  } else {
4518  assert(OVT.isInteger()
4519  && "VAARG promotion is supported only for vectors or integer types");
4520  TruncOp = ISD::TRUNCATE;
4521  }
4522 
4523  // Perform the larger operation, then convert back
4524  Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4525  Node->getConstantOperandVal(3));
4526  Chain = Tmp1.getValue(1);
4527 
4528  Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
4529 
4530  // Modified the chain result - switch anything that used the old chain to
4531  // use the new one.
4532  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4533  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4534  if (UpdatedNodes) {
4535  UpdatedNodes->insert(Tmp2.getNode());
4536  UpdatedNodes->insert(Chain.getNode());
4537  }
4538  ReplacedNode(Node);
4539  break;
4540  }
4541  case ISD::MUL:
4542  case ISD::SDIV:
4543  case ISD::SREM:
4544  case ISD::UDIV:
4545  case ISD::UREM:
4546  case ISD::AND:
4547  case ISD::OR:
4548  case ISD::XOR: {
4549  unsigned ExtOp, TruncOp;
4550  if (OVT.isVector()) {
4551  ExtOp = ISD::BITCAST;
4552  TruncOp = ISD::BITCAST;
4553  } else {
4554  assert(OVT.isInteger() && "Cannot promote logic operation");
4555 
4556  switch (Node->getOpcode()) {
4557  default:
4558  ExtOp = ISD::ANY_EXTEND;
4559  break;
4560  case ISD::SDIV:
4561  case ISD::SREM:
4562  ExtOp = ISD::SIGN_EXTEND;
4563  break;
4564  case ISD::UDIV:
4565  case ISD::UREM:
4566  ExtOp = ISD::ZERO_EXTEND;
4567  break;
4568  }
4569  TruncOp = ISD::TRUNCATE;
4570  }
4571  // Promote each of the values to the new type.
4572  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4573  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4574  // Perform the larger operation, then convert back
4575  Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4576  Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4577  break;
4578  }
4579  case ISD::UMUL_LOHI:
4580  case ISD::SMUL_LOHI: {
4581  // Promote to a multiply in a wider integer type.
4582  unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
4583  : ISD::SIGN_EXTEND;
4584  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4585  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4586  Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
4587 
4588  auto &DL = DAG.getDataLayout();
4589  unsigned OriginalSize = OVT.getScalarSizeInBits();
4590  Tmp2 = DAG.getNode(
4591  ISD::SRL, dl, NVT, Tmp1,
4592  DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
4593  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4594  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
4595  break;
4596  }
4597  case ISD::SELECT: {
4598  unsigned ExtOp, TruncOp;
4599  if (Node->getValueType(0).isVector() ||
4600  Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
4601  ExtOp = ISD::BITCAST;
4602  TruncOp = ISD::BITCAST;
4603  } else if (Node->getValueType(0).isInteger()) {
4604  ExtOp = ISD::ANY_EXTEND;
4605  TruncOp = ISD::TRUNCATE;
4606  } else {
4607  ExtOp = ISD::FP_EXTEND;
4608  TruncOp = ISD::FP_ROUND;
4609  }
4610  Tmp1 = Node->getOperand(0);
4611  // Promote each of the values to the new type.
4612  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4613  Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4614  // Perform the larger operation, then round down.
4615  Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4616  Tmp1->setFlags(Node->getFlags());
4617  if (TruncOp != ISD::FP_ROUND)
4618  Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4619  else
4620  Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4621  DAG.getIntPtrConstant(0, dl));
4622  Results.push_back(Tmp1);
4623  break;
4624  }
4625  case ISD::VECTOR_SHUFFLE: {
4626  ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4627 
4628  // Cast the two input vectors.
4629  Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4630  Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4631 
4632  // Convert the shuffle mask to the right # elements.
4633  Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
4634  Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
4635  Results.push_back(Tmp1);
4636  break;
4637  }
4638  case ISD::VECTOR_SPLICE: {
4639  Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));
4640  Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1));
4641  Tmp3 = DAG.getNode(ISD::VECTOR_SPLICE, dl, NVT, Tmp1, Tmp2,
4642  Node->getOperand(2));
4643  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp3));
4644  break;
4645  }
4646  case ISD::SELECT_CC: {
4647  SDValue Cond = Node->getOperand(4);
4648  ISD::CondCode CCCode = cast<CondCodeSDNode>(Cond)->get();
4649  // Type of the comparison operands.
4650  MVT CVT = Node->getSimpleValueType(0);
4651  assert(CVT == OVT && "not handled");
4652 
4653  unsigned ExtOp = ISD::FP_EXTEND;
4654  if (NVT.isInteger()) {
4656  }
4657 
4658  // Promote the comparison operands, if needed.
4659  if (TLI.isCondCodeLegal(CCCode, CVT)) {
4660  Tmp1 = Node->getOperand(0);
4661  Tmp2 = Node->getOperand(1);
4662  } else {
4663  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4664  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4665  }
4666  // Cast the true/false operands.
4667  Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4668  Tmp4 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4669 
4670  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, NVT, {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
4671  Node->getFlags());
4672 
4673  // Cast the result back to the original type.
4674  if (ExtOp != ISD::FP_EXTEND)
4675  Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1);
4676  else
4677  Tmp1 = DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp1,
4678  DAG.getIntPtrConstant(0, dl));
4679 
4680  Results.push_back(Tmp1);
4681  break;
4682  }
4683  case ISD::SETCC:
4684  case ISD::STRICT_FSETCC:
4685  case ISD::STRICT_FSETCCS: {
4686  unsigned ExtOp = ISD::FP_EXTEND;
4687  if (NVT.isInteger()) {
4688  ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
4690  }
4691  if (Node->isStrictFPOpcode()) {
4692  SDValue InChain = Node->getOperand(0);
4693  std::tie(Tmp1, std::ignore) =
4694  DAG.getStrictFPExtendOrRound(Node->getOperand(1), InChain, dl, NVT);
4695  std::tie(Tmp2, std::ignore) =
4696  DAG.getStrictFPExtendOrRound(Node->getOperand(2), InChain, dl, NVT);
4697  SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(1), Tmp2.getValue(1)};
4698  SDValue OutChain = DAG.getTokenFactor(dl, TmpChains);
4699  SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
4700  Results.push_back(DAG.getNode(Node->getOpcode(), dl, VTs,
4701  {OutChain, Tmp1, Tmp2, Node->getOperand(3)},
4702  Node->getFlags()));
4703  Results.push_back(Results.back().getValue(1));
4704  break;
4705  }
4706  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4707  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4708  Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1,
4709  Tmp2, Node->getOperand(2), Node->getFlags()));
4710  break;
4711  }
4712  case ISD::BR_CC: {
4713  unsigned ExtOp = ISD::FP_EXTEND;
4714  if (NVT.isInteger()) {
4715  ISD::CondCode CCCode =
4716  cast<CondCodeSDNode>(Node->getOperand(1))->get();
4718  }
4719  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4720  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4721  Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4722  Node->getOperand(0), Node->getOperand(1),
4723  Tmp1, Tmp2, Node->getOperand(4)));
4724  break;
4725  }
4726  case ISD::FADD:
4727  case ISD::FSUB:
4728  case ISD::FMUL:
4729  case ISD::FDIV:
4730  case ISD::FREM:
4731  case ISD::FMINNUM:
4732  case ISD::FMAXNUM:
4733  case ISD::FPOW:
4734  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4735  Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4736  Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4737  Node->getFlags());
4738  Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4739  Tmp3, DAG.getIntPtrConstant(0, dl)));
4740  break;
4741  case ISD::STRICT_FADD:
4742  case ISD::STRICT_FSUB:
4743  case ISD::STRICT_FMUL:
4744  case ISD::STRICT_FDIV:
4745  case ISD::STRICT_FMINNUM:
4746  case ISD::STRICT_FMAXNUM:
4747  case ISD::STRICT_FREM:
4748  case ISD::STRICT_FPOW:
4749  Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4750  {Node->getOperand(0), Node->getOperand(1)});
4751  Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4752  {Node->getOperand(0), Node->getOperand(2)});
4753  Tmp3 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),
4754  Tmp2.getValue(1));
4755  Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
4756  {Tmp3, Tmp1, Tmp2});
4757  Tmp1 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
4758  {Tmp1.getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)});
4759  Results.push_back(Tmp1);
4760  Results.push_back(Tmp1.getValue(1));
4761  break;
4762  case ISD::FMA:
4763  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4764  Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4765  Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4766  Results.push_back(
4767  DAG.getNode(ISD::FP_ROUND, dl, OVT,
4768  DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4769  DAG.getIntPtrConstant(0, dl)));
4770  break;
4771  case ISD::STRICT_FMA:
4772  Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4773  {Node->getOperand(0), Node->getOperand(1)});
4774  Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4775  {Node->getOperand(0), Node->getOperand(2)});
4776  Tmp3 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4777  {Node->getOperand(0), Node->getOperand(3)});
4778  Tmp4 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),
4779  Tmp2.getValue(1), Tmp3.getValue(1));
4780  Tmp4 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
4781  {Tmp4, Tmp1, Tmp2, Tmp3});
4782  Tmp4 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
4783  {Tmp4.getValue(1), Tmp4, DAG.getIntPtrConstant(0, dl)});
4784  Results.push_back(Tmp4);
4785  Results.push_back(Tmp4.getValue(1));
4786  break;
4787  case ISD::FCOPYSIGN:
4788  case ISD::FPOWI: {
4789  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4790  Tmp2 = Node->getOperand(1);
4791  Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4792 
4793  // fcopysign doesn't change anything but the sign bit, so
4794  // (fp_round (fcopysign (fpext a), b))
4795  // is as precise as
4796  // (fp_round (fpext a))
4797  // which is a no-op. Mark it as a TRUNCating FP_ROUND.
4798  const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
4799  Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4800  Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
4801  break;
4802  }
4803  case ISD::STRICT_FPOWI:
4804  Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4805  {Node->getOperand(0), Node->getOperand(1)});
4806  Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
4807  {Tmp1.getValue(1), Tmp1, Node->getOperand(2)});
4808  Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
4809  {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
4810  Results.push_back(Tmp3);
4811  Results.push_back(Tmp3.getValue(1));
4812  break;
4813  case ISD::FFLOOR:
4814  case ISD::FCEIL:
4815  case ISD::FRINT:
4816  case ISD::FNEARBYINT:
4817  case ISD::FROUND:
4818  case ISD::FROUNDEVEN:
4819  case ISD::FTRUNC:
4820  case ISD::FNEG:
4821  case ISD::FSQRT:
4822  case ISD::FSIN:
4823  case ISD::FCOS:
4824  case ISD::FLOG:
4825  case ISD::FLOG2:
4826  case ISD::FLOG10:
4827  case ISD::FABS:
4828  case ISD::FEXP:
4829  case ISD::FEXP2:
4830  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4831  Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4832  Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4833  Tmp2, DAG.getIntPtrConstant(0, dl)));
4834  break;
4835  case ISD::STRICT_FFLOOR:
4836  case ISD::STRICT_FCEIL:
4837  case ISD::STRICT_FRINT:
4839  case ISD::STRICT_FROUND:
4841  case ISD::STRICT_FTRUNC:
4842  case ISD::STRICT_FSQRT:
4843  case ISD::STRICT_FSIN:
4844  case ISD::STRICT_FCOS:
4845  case ISD::STRICT_FLOG:
4846  case ISD::STRICT_FLOG2:
4847  case ISD::STRICT_FLOG10:
4848  case ISD::STRICT_FEXP:
4849  case ISD::STRICT_FEXP2:
4850  Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4851  {Node->getOperand(0), Node->getOperand(1)});
4852  Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
4853  {Tmp1.getValue(1), Tmp1});
4854  Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
4855  {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
4856  Results.push_back(Tmp3);
4857  Results.push_back(Tmp3.getValue(1));
4858  break;
4859  case ISD::BUILD_VECTOR: {
4860  MVT EltVT = OVT.getVectorElementType();
4861  MVT NewEltVT = NVT.getVectorElementType();
4862 
4863  // Handle bitcasts to a different vector type with the same total bit size
4864  //
4865  // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
4866  // =>
4867  // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
4868 
4869  assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4870  "Invalid promote type for build_vector");
4871  assert(NewEltVT.bitsLT(EltVT) && "not handled");
4872 
4873  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4874 
4875  SmallVector<SDValue, 8> NewOps;
4876  for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
4877  SDValue Op = Node->getOperand(I);
4878  NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
4879  }
4880 
4881  SDLoc SL(Node);
4882  SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
4883  SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4884  Results.push_back(CvtVec);
4885  break;
4886  }
4887  case ISD::EXTRACT_VECTOR_ELT: {
4888  MVT EltVT = OVT.getVectorElementType();
4889  MVT NewEltVT = NVT.getVectorElementType();
4890 
4891  // Handle bitcasts to a different vector type with the same total bit size.
4892  //
4893  // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
4894  // =>
4895  // v4i32:castx = bitcast x:v2i64
4896  //
4897  // i64 = bitcast
4898  // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
4899  // (i32 (extract_vector_elt castx, (2 * y + 1)))
4900  //
4901 
4902  assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4903  "Invalid promote type for extract_vector_elt");
4904  assert(NewEltVT.bitsLT(EltVT) && "not handled");
4905 
4906  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4907  unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4908 
4909  SDValue Idx = Node->getOperand(1);
4910  EVT IdxVT = Idx.getValueType();
4911  SDLoc SL(Node);
4912  SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4913  SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4914 
4915  SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4916 
4917  SmallVector<SDValue, 8> NewOps;
4918  for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4919  SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4920  SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4921 
4922  SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4923  CastVec, TmpIdx);
4924  NewOps.push_back(Elt);
4925  }
4926 
4927  SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
4928  Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
4929  break;
4930  }
4931  case ISD::INSERT_VECTOR_ELT: {
4932  MVT EltVT = OVT.getVectorElementType();
4933  MVT NewEltVT = NVT.getVectorElementType();
4934 
4935  // Handle bitcasts to a different vector type with the same total bit size
4936  //
4937  // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
4938  // =>
4939  // v4i32:castx = bitcast x:v2i64
4940  // v2i32:casty = bitcast y:i64
4941  //
4942  // v2i64 = bitcast
4943  // (v4i32 insert_vector_elt
4944  // (v4i32 insert_vector_elt v4i32:castx,
4945  // (extract_vector_elt casty, 0), 2 * z),
4946  // (extract_vector_elt casty, 1), (2 * z + 1))
4947 
4948  assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4949  "Invalid promote type for insert_vector_elt");
4950  assert(NewEltVT.bitsLT(EltVT) && "not handled");
4951 
4952  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4953  unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4954 
4955  SDValue Val = Node->getOperand(1);
4956  SDValue Idx = Node->getOperand(2);
4957  EVT IdxVT = Idx.getValueType();
4958  SDLoc SL(Node);
4959 
4960  SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
4961  SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4962 
4963  SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4964  SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4965 
4966  SDValue NewVec = CastVec;
4967  for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4968  SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4969  SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4970 
4971  SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4972  CastVal, IdxOffset);
4973 
4974  NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
4975  NewVec, Elt, InEltIdx);
4976  }
4977 
4978  Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
4979  break;
4980  }
4981  case ISD::SCALAR_TO_VECTOR: {
4982  MVT EltVT = OVT.getVectorElementType();
4983  MVT NewEltVT = NVT.getVectorElementType();
4984 
4985  // Handle bitcasts to different vector type with the same total bit size.
4986  //
4987  // e.g. v2i64 = scalar_to_vector x:i64
4988  // =>
4989  // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
4990  //
4991 
4992  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4993  SDValue Val = Node->getOperand(0);
4994  SDLoc SL(Node);
4995 
4996  SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4997  SDValue Undef = DAG.getUNDEF(MidVT);
4998 
4999  SmallVector<SDValue, 8> NewElts;
5000  NewElts.push_back(CastVal);
5001  for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
5002  NewElts.push_back(Undef);
5003 
5004  SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
5005  SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
5006  Results.push_back(CvtVec);
5007  break;
5008  }
5009  case ISD::ATOMIC_SWAP: {
5010  AtomicSDNode *AM = cast<AtomicSDNode>(Node);
5011  SDLoc SL(Node);
5012  SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal());
5013  assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
5014  "unexpected promotion type");
5015  assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
5016  "unexpected atomic_swap with illegal type");
5017 
5018  SDValue NewAtomic
5019  = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT,
5020  DAG.getVTList(NVT, MVT::Other),
5021  { AM->getChain(), AM->getBasePtr(), CastVal },
5022  AM->getMemOperand());