LLVM  8.0.0svn
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1 //===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements routines for translating from LLVM IR into SelectionDAG IR.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SelectionDAGBuilder.h"
15 #include "SDNodeDbgValue.h"
16 #include "llvm/ADT/APFloat.h"
17 #include "llvm/ADT/APInt.h"
18 #include "llvm/ADT/ArrayRef.h"
19 #include "llvm/ADT/BitVector.h"
20 #include "llvm/ADT/DenseMap.h"
21 #include "llvm/ADT/None.h"
22 #include "llvm/ADT/Optional.h"
23 #include "llvm/ADT/STLExtras.h"
24 #include "llvm/ADT/SmallPtrSet.h"
25 #include "llvm/ADT/SmallSet.h"
26 #include "llvm/ADT/SmallVector.h"
27 #include "llvm/ADT/StringRef.h"
28 #include "llvm/ADT/Triple.h"
29 #include "llvm/ADT/Twine.h"
34 #include "llvm/Analysis/Loads.h"
39 #include "llvm/CodeGen/Analysis.h"
57 #include "llvm/CodeGen/StackMaps.h"
66 #include "llvm/IR/Argument.h"
67 #include "llvm/IR/Attributes.h"
68 #include "llvm/IR/BasicBlock.h"
69 #include "llvm/IR/CFG.h"
70 #include "llvm/IR/CallSite.h"
71 #include "llvm/IR/CallingConv.h"
72 #include "llvm/IR/Constant.h"
73 #include "llvm/IR/ConstantRange.h"
74 #include "llvm/IR/Constants.h"
75 #include "llvm/IR/DataLayout.h"
77 #include "llvm/IR/DebugLoc.h"
78 #include "llvm/IR/DerivedTypes.h"
79 #include "llvm/IR/Function.h"
81 #include "llvm/IR/InlineAsm.h"
82 #include "llvm/IR/InstrTypes.h"
83 #include "llvm/IR/Instruction.h"
84 #include "llvm/IR/Instructions.h"
85 #include "llvm/IR/IntrinsicInst.h"
86 #include "llvm/IR/Intrinsics.h"
87 #include "llvm/IR/LLVMContext.h"
88 #include "llvm/IR/Metadata.h"
89 #include "llvm/IR/Module.h"
90 #include "llvm/IR/Operator.h"
91 #include "llvm/IR/Statepoint.h"
92 #include "llvm/IR/Type.h"
93 #include "llvm/IR/User.h"
94 #include "llvm/IR/Value.h"
95 #include "llvm/MC/MCContext.h"
96 #include "llvm/MC/MCSymbol.h"
99 #include "llvm/Support/Casting.h"
100 #include "llvm/Support/CodeGen.h"
102 #include "llvm/Support/Compiler.h"
103 #include "llvm/Support/Debug.h"
106 #include "llvm/Support/MathExtras.h"
111 #include <algorithm>
112 #include <cassert>
113 #include <cstddef>
114 #include <cstdint>
115 #include <cstring>
116 #include <iterator>
117 #include <limits>
118 #include <numeric>
119 #include <tuple>
120 #include <utility>
121 #include <vector>
122 
123 using namespace llvm;
124 
125 #define DEBUG_TYPE "isel"
126 
127 /// LimitFloatPrecision - Generate low-precision inline sequences for
128 /// some float libcalls (6, 8 or 12 bits).
129 static unsigned LimitFloatPrecision;
130 
132  LimitFPPrecision("limit-float-precision",
133  cl::desc("Generate low-precision inline sequences "
134  "for some float libcalls"),
135  cl::location(LimitFloatPrecision), cl::Hidden,
136  cl::init(0));
137 
139  "switch-peel-threshold", cl::Hidden, cl::init(66),
140  cl::desc("Set the case probability threshold for peeling the case from a "
141  "switch statement. A value greater than 100 will void this "
142  "optimization"));
143 
144 // Limit the width of DAG chains. This is important in general to prevent
145 // DAG-based analysis from blowing up. For example, alias analysis and
146 // load clustering may not complete in reasonable time. It is difficult to
147 // recognize and avoid this situation within each individual analysis, and
148 // future analyses are likely to have the same behavior. Limiting DAG width is
149 // the safe approach and will be especially important with global DAGs.
150 //
151 // MaxParallelChains default is arbitrarily high to avoid affecting
152 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
153 // sequence over this should have been converted to llvm.memcpy by the
154 // frontend. It is easy to induce this behavior with .ll code such as:
155 // %buffer = alloca [4096 x i8]
156 // %data = load [4096 x i8]* %argPtr
157 // store [4096 x i8] %data, [4096 x i8]* %buffer
158 static const unsigned MaxParallelChains = 64;
159 
160 // Return the calling convention if the Value passed requires ABI mangling as it
161 // is a parameter to a function or a return value from a function which is not
162 // an intrinsic.
164  if (auto *R = dyn_cast<ReturnInst>(V))
165  return R->getParent()->getParent()->getCallingConv();
166 
167  if (auto *CI = dyn_cast<CallInst>(V)) {
168  const bool IsInlineAsm = CI->isInlineAsm();
169  const bool IsIndirectFunctionCall =
170  !IsInlineAsm && !CI->getCalledFunction();
171 
172  // It is possible that the call instruction is an inline asm statement or an
173  // indirect function call in which case the return value of
174  // getCalledFunction() would be nullptr.
175  const bool IsInstrinsicCall =
176  !IsInlineAsm && !IsIndirectFunctionCall &&
177  CI->getCalledFunction()->getIntrinsicID() != Intrinsic::not_intrinsic;
178 
179  if (!IsInlineAsm && !IsInstrinsicCall)
180  return CI->getCallingConv();
181  }
182 
183  return None;
184 }
185 
186 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
187  const SDValue *Parts, unsigned NumParts,
188  MVT PartVT, EVT ValueVT, const Value *V,
190 
191 /// getCopyFromParts - Create a value that contains the specified legal parts
192 /// combined into the value they represent. If the parts combine to a type
193 /// larger than ValueVT then AssertOp can be used to specify whether the extra
194 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
195 /// (ISD::AssertSext).
197  const SDValue *Parts, unsigned NumParts,
198  MVT PartVT, EVT ValueVT, const Value *V,
200  Optional<ISD::NodeType> AssertOp = None) {
201  if (ValueVT.isVector())
202  return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
203  CC);
204 
205  assert(NumParts > 0 && "No parts to assemble!");
206  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
207  SDValue Val = Parts[0];
208 
209  if (NumParts > 1) {
210  // Assemble the value from multiple parts.
211  if (ValueVT.isInteger()) {
212  unsigned PartBits = PartVT.getSizeInBits();
213  unsigned ValueBits = ValueVT.getSizeInBits();
214 
215  // Assemble the power of 2 part.
216  unsigned RoundParts = NumParts & (NumParts - 1) ?
217  1 << Log2_32(NumParts) : NumParts;
218  unsigned RoundBits = PartBits * RoundParts;
219  EVT RoundVT = RoundBits == ValueBits ?
220  ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
221  SDValue Lo, Hi;
222 
223  EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
224 
225  if (RoundParts > 2) {
226  Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
227  PartVT, HalfVT, V);
228  Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
229  RoundParts / 2, PartVT, HalfVT, V);
230  } else {
231  Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
232  Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
233  }
234 
235  if (DAG.getDataLayout().isBigEndian())
236  std::swap(Lo, Hi);
237 
238  Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
239 
240  if (RoundParts < NumParts) {
241  // Assemble the trailing non-power-of-2 part.
242  unsigned OddParts = NumParts - RoundParts;
243  EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
244  Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
245  OddVT, V, CC);
246 
247  // Combine the round and odd parts.
248  Lo = Val;
249  if (DAG.getDataLayout().isBigEndian())
250  std::swap(Lo, Hi);
251  EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
252  Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
253  Hi =
254  DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
255  DAG.getConstant(Lo.getValueSizeInBits(), DL,
256  TLI.getPointerTy(DAG.getDataLayout())));
257  Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
258  Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
259  }
260  } else if (PartVT.isFloatingPoint()) {
261  // FP split into multiple FP parts (for ppcf128)
262  assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
263  "Unexpected split");
264  SDValue Lo, Hi;
265  Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
266  Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
267  if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
268  std::swap(Lo, Hi);
269  Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
270  } else {
271  // FP split into integer parts (soft fp)
272  assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
273  !PartVT.isVector() && "Unexpected split");
274  EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
275  Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
276  }
277  }
278 
279  // There is now one part, held in Val. Correct it to match ValueVT.
280  // PartEVT is the type of the register class that holds the value.
281  // ValueVT is the type of the inline asm operation.
282  EVT PartEVT = Val.getValueType();
283 
284  if (PartEVT == ValueVT)
285  return Val;
286 
287  if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
288  ValueVT.bitsLT(PartEVT)) {
289  // For an FP value in an integer part, we need to truncate to the right
290  // width first.
291  PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
292  Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
293  }
294 
295  // Handle types that have the same size.
296  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
297  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
298 
299  // Handle types with different sizes.
300  if (PartEVT.isInteger() && ValueVT.isInteger()) {
301  if (ValueVT.bitsLT(PartEVT)) {
302  // For a truncate, see if we have any information to
303  // indicate whether the truncated bits will always be
304  // zero or sign-extension.
305  if (AssertOp.hasValue())
306  Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
307  DAG.getValueType(ValueVT));
308  return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
309  }
310  return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
311  }
312 
313  if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
314  // FP_ROUND's are always exact here.
315  if (ValueVT.bitsLT(Val.getValueType()))
316  return DAG.getNode(
317  ISD::FP_ROUND, DL, ValueVT, Val,
318  DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
319 
320  return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
321  }
322 
323  llvm_unreachable("Unknown mismatch!");
324 }
325 
327  const Twine &ErrMsg) {
328  const Instruction *I = dyn_cast_or_null<Instruction>(V);
329  if (!V)
330  return Ctx.emitError(ErrMsg);
331 
332  const char *AsmError = ", possible invalid constraint for vector type";
333  if (const CallInst *CI = dyn_cast<CallInst>(I))
334  if (isa<InlineAsm>(CI->getCalledValue()))
335  return Ctx.emitError(I, ErrMsg + AsmError);
336 
337  return Ctx.emitError(I, ErrMsg);
338 }
339 
340 /// getCopyFromPartsVector - Create a value that contains the specified legal
341 /// parts combined into the value they represent. If the parts combine to a
342 /// type larger than ValueVT then AssertOp can be used to specify whether the
343 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
344 /// ValueVT (ISD::AssertSext).
346  const SDValue *Parts, unsigned NumParts,
347  MVT PartVT, EVT ValueVT, const Value *V,
348  Optional<CallingConv::ID> CallConv) {
349  assert(ValueVT.isVector() && "Not a vector value");
350  assert(NumParts > 0 && "No parts to assemble!");
351  const bool IsABIRegCopy = CallConv.hasValue();
352 
353  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
354  SDValue Val = Parts[0];
355 
356  // Handle a multi-element vector.
357  if (NumParts > 1) {
358  EVT IntermediateVT;
359  MVT RegisterVT;
360  unsigned NumIntermediates;
361  unsigned NumRegs;
362 
363  if (IsABIRegCopy) {
365  *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
366  NumIntermediates, RegisterVT);
367  } else {
368  NumRegs =
369  TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
370  NumIntermediates, RegisterVT);
371  }
372 
373  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
374  NumParts = NumRegs; // Silence a compiler warning.
375  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
376  assert(RegisterVT.getSizeInBits() ==
377  Parts[0].getSimpleValueType().getSizeInBits() &&
378  "Part type sizes don't match!");
379 
380  // Assemble the parts into intermediate operands.
381  SmallVector<SDValue, 8> Ops(NumIntermediates);
382  if (NumIntermediates == NumParts) {
383  // If the register was not expanded, truncate or copy the value,
384  // as appropriate.
385  for (unsigned i = 0; i != NumParts; ++i)
386  Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
387  PartVT, IntermediateVT, V);
388  } else if (NumParts > 0) {
389  // If the intermediate type was expanded, build the intermediate
390  // operands from the parts.
391  assert(NumParts % NumIntermediates == 0 &&
392  "Must expand into a divisible number of parts!");
393  unsigned Factor = NumParts / NumIntermediates;
394  for (unsigned i = 0; i != NumIntermediates; ++i)
395  Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
396  PartVT, IntermediateVT, V);
397  }
398 
399  // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
400  // intermediate operands.
401  EVT BuiltVectorTy =
402  EVT::getVectorVT(*DAG.getContext(), IntermediateVT.getScalarType(),
403  (IntermediateVT.isVector()
404  ? IntermediateVT.getVectorNumElements() * NumParts
405  : NumIntermediates));
406  Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
408  DL, BuiltVectorTy, Ops);
409  }
410 
411  // There is now one part, held in Val. Correct it to match ValueVT.
412  EVT PartEVT = Val.getValueType();
413 
414  if (PartEVT == ValueVT)
415  return Val;
416 
417  if (PartEVT.isVector()) {
418  // If the element type of the source/dest vectors are the same, but the
419  // parts vector has more elements than the value vector, then we have a
420  // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the
421  // elements we want.
422  if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
423  assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
424  "Cannot narrow, it would be a lossy transformation");
425  return DAG.getNode(
426  ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
427  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
428  }
429 
430  // Vector/Vector bitcast.
431  if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
432  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
433 
434  assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
435  "Cannot handle this kind of promotion");
436  // Promoted vector extract
437  return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
438 
439  }
440 
441  // Trivial bitcast if the types are the same size and the destination
442  // vector type is legal.
443  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
444  TLI.isTypeLegal(ValueVT))
445  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
446 
447  if (ValueVT.getVectorNumElements() != 1) {
448  // Certain ABIs require that vectors are passed as integers. For vectors
449  // are the same size, this is an obvious bitcast.
450  if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
451  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
452  } else if (ValueVT.getSizeInBits() < PartEVT.getSizeInBits()) {
453  // Bitcast Val back the original type and extract the corresponding
454  // vector we want.
455  unsigned Elts = PartEVT.getSizeInBits() / ValueVT.getScalarSizeInBits();
456  EVT WiderVecType = EVT::getVectorVT(*DAG.getContext(),
457  ValueVT.getVectorElementType(), Elts);
458  Val = DAG.getBitcast(WiderVecType, Val);
459  return DAG.getNode(
460  ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
461  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
462  }
463 
465  *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
466  return DAG.getUNDEF(ValueVT);
467  }
468 
469  // Handle cases such as i8 -> <1 x i1>
470  EVT ValueSVT = ValueVT.getVectorElementType();
471  if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT)
472  Val = ValueVT.isFloatingPoint() ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
473  : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
474 
475  return DAG.getBuildVector(ValueVT, DL, Val);
476 }
477 
478 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
479  SDValue Val, SDValue *Parts, unsigned NumParts,
480  MVT PartVT, const Value *V,
481  Optional<CallingConv::ID> CallConv);
482 
483 /// getCopyToParts - Create a series of nodes that contain the specified value
484 /// split into legal parts. If the parts contain more bits than Val, then, for
485 /// integers, ExtendKind can be used to specify how to generate the extra bits.
486 static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
487  SDValue *Parts, unsigned NumParts, MVT PartVT,
488  const Value *V,
489  Optional<CallingConv::ID> CallConv = None,
490  ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
491  EVT ValueVT = Val.getValueType();
492 
493  // Handle the vector case separately.
494  if (ValueVT.isVector())
495  return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
496  CallConv);
497 
498  unsigned PartBits = PartVT.getSizeInBits();
499  unsigned OrigNumParts = NumParts;
500  assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
501  "Copying to an illegal type!");
502 
503  if (NumParts == 0)
504  return;
505 
506  assert(!ValueVT.isVector() && "Vector case handled elsewhere");
507  EVT PartEVT = PartVT;
508  if (PartEVT == ValueVT) {
509  assert(NumParts == 1 && "No-op copy with multiple parts!");
510  Parts[0] = Val;
511  return;
512  }
513 
514  if (NumParts * PartBits > ValueVT.getSizeInBits()) {
515  // If the parts cover more bits than the value has, promote the value.
516  if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
517  assert(NumParts == 1 && "Do not know what to promote to!");
518  Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
519  } else {
520  if (ValueVT.isFloatingPoint()) {
521  // FP values need to be bitcast, then extended if they are being put
522  // into a larger container.
523  ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
524  Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
525  }
526  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
527  ValueVT.isInteger() &&
528  "Unknown mismatch!");
529  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
530  Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
531  if (PartVT == MVT::x86mmx)
532  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
533  }
534  } else if (PartBits == ValueVT.getSizeInBits()) {
535  // Different types of the same size.
536  assert(NumParts == 1 && PartEVT != ValueVT);
537  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
538  } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
539  // If the parts cover less bits than value has, truncate the value.
540  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
541  ValueVT.isInteger() &&
542  "Unknown mismatch!");
543  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
544  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
545  if (PartVT == MVT::x86mmx)
546  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
547  }
548 
549  // The value may have changed - recompute ValueVT.
550  ValueVT = Val.getValueType();
551  assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
552  "Failed to tile the value with PartVT!");
553 
554  if (NumParts == 1) {
555  if (PartEVT != ValueVT) {
557  "scalar-to-vector conversion failed");
558  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
559  }
560 
561  Parts[0] = Val;
562  return;
563  }
564 
565  // Expand the value into multiple parts.
566  if (NumParts & (NumParts - 1)) {
567  // The number of parts is not a power of 2. Split off and copy the tail.
568  assert(PartVT.isInteger() && ValueVT.isInteger() &&
569  "Do not know what to expand to!");
570  unsigned RoundParts = 1 << Log2_32(NumParts);
571  unsigned RoundBits = RoundParts * PartBits;
572  unsigned OddParts = NumParts - RoundParts;
573  SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
574  DAG.getIntPtrConstant(RoundBits, DL));
575  getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
576  CallConv);
577 
578  if (DAG.getDataLayout().isBigEndian())
579  // The odd parts were reversed by getCopyToParts - unreverse them.
580  std::reverse(Parts + RoundParts, Parts + NumParts);
581 
582  NumParts = RoundParts;
583  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
584  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
585  }
586 
587  // The number of parts is a power of 2. Repeatedly bisect the value using
588  // EXTRACT_ELEMENT.
589  Parts[0] = DAG.getNode(ISD::BITCAST, DL,
591  ValueVT.getSizeInBits()),
592  Val);
593 
594  for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
595  for (unsigned i = 0; i < NumParts; i += StepSize) {
596  unsigned ThisBits = StepSize * PartBits / 2;
597  EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
598  SDValue &Part0 = Parts[i];
599  SDValue &Part1 = Parts[i+StepSize/2];
600 
601  Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
602  ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
603  Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
604  ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
605 
606  if (ThisBits == PartBits && ThisVT != PartVT) {
607  Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
608  Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
609  }
610  }
611  }
612 
613  if (DAG.getDataLayout().isBigEndian())
614  std::reverse(Parts, Parts + OrigNumParts);
615 }
616 
618  SDValue Val, const SDLoc &DL, EVT PartVT) {
619  if (!PartVT.isVector())
620  return SDValue();
621 
622  EVT ValueVT = Val.getValueType();
623  unsigned PartNumElts = PartVT.getVectorNumElements();
624  unsigned ValueNumElts = ValueVT.getVectorNumElements();
625  if (PartNumElts > ValueNumElts &&
626  PartVT.getVectorElementType() == ValueVT.getVectorElementType()) {
627  EVT ElementVT = PartVT.getVectorElementType();
628  // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
629  // undef elements.
631  DAG.ExtractVectorElements(Val, Ops);
632  SDValue EltUndef = DAG.getUNDEF(ElementVT);
633  for (unsigned i = ValueNumElts, e = PartNumElts; i != e; ++i)
634  Ops.push_back(EltUndef);
635 
636  // FIXME: Use CONCAT for 2x -> 4x.
637  return DAG.getBuildVector(PartVT, DL, Ops);
638  }
639 
640  return SDValue();
641 }
642 
643 /// getCopyToPartsVector - Create a series of nodes that contain the specified
644 /// value split into legal parts.
645 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
646  SDValue Val, SDValue *Parts, unsigned NumParts,
647  MVT PartVT, const Value *V,
648  Optional<CallingConv::ID> CallConv) {
649  EVT ValueVT = Val.getValueType();
650  assert(ValueVT.isVector() && "Not a vector");
651  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
652  const bool IsABIRegCopy = CallConv.hasValue();
653 
654  if (NumParts == 1) {
655  EVT PartEVT = PartVT;
656  if (PartEVT == ValueVT) {
657  // Nothing to do.
658  } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
659  // Bitconvert vector->vector case.
660  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
661  } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
662  Val = Widened;
663  } else if (PartVT.isVector() &&
664  PartEVT.getVectorElementType().bitsGE(
665  ValueVT.getVectorElementType()) &&
666  PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
667 
668  // Promoted vector extract
669  Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
670  } else {
671  if (ValueVT.getVectorNumElements() == 1) {
672  Val = DAG.getNode(
673  ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
674  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
675  } else {
676  assert(PartVT.getSizeInBits() > ValueVT.getSizeInBits() &&
677  "lossy conversion of vector to scalar type");
678  EVT IntermediateType =
679  EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
680  Val = DAG.getBitcast(IntermediateType, Val);
681  Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
682  }
683  }
684 
685  assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
686  Parts[0] = Val;
687  return;
688  }
689 
690  // Handle a multi-element vector.
691  EVT IntermediateVT;
692  MVT RegisterVT;
693  unsigned NumIntermediates;
694  unsigned NumRegs;
695  if (IsABIRegCopy) {
696  NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
697  *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
698  NumIntermediates, RegisterVT);
699  } else {
700  NumRegs =
701  TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
702  NumIntermediates, RegisterVT);
703  }
704 
705  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
706  NumParts = NumRegs; // Silence a compiler warning.
707  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
708 
709  unsigned IntermediateNumElts = IntermediateVT.isVector() ?
710  IntermediateVT.getVectorNumElements() : 1;
711 
712  // Convert the vector to the appropiate type if necessary.
713  unsigned DestVectorNoElts = NumIntermediates * IntermediateNumElts;
714 
715  EVT BuiltVectorTy = EVT::getVectorVT(
716  *DAG.getContext(), IntermediateVT.getScalarType(), DestVectorNoElts);
717  MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
718  if (ValueVT != BuiltVectorTy) {
719  if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy))
720  Val = Widened;
721 
722  Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
723  }
724 
725  // Split the vector into intermediate operands.
726  SmallVector<SDValue, 8> Ops(NumIntermediates);
727  for (unsigned i = 0; i != NumIntermediates; ++i) {
728  if (IntermediateVT.isVector()) {
729  Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
730  DAG.getConstant(i * IntermediateNumElts, DL, IdxVT));
731  } else {
732  Ops[i] = DAG.getNode(
733  ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
734  DAG.getConstant(i, DL, IdxVT));
735  }
736  }
737 
738  // Split the intermediate operands into legal parts.
739  if (NumParts == NumIntermediates) {
740  // If the register was not expanded, promote or copy the value,
741  // as appropriate.
742  for (unsigned i = 0; i != NumParts; ++i)
743  getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
744  } else if (NumParts > 0) {
745  // If the intermediate type was expanded, split each the value into
746  // legal parts.
747  assert(NumIntermediates != 0 && "division by zero");
748  assert(NumParts % NumIntermediates == 0 &&
749  "Must expand into a divisible number of parts!");
750  unsigned Factor = NumParts / NumIntermediates;
751  for (unsigned i = 0; i != NumIntermediates; ++i)
752  getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
753  CallConv);
754  }
755 }
756 
758  EVT valuevt, Optional<CallingConv::ID> CC)
759  : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
760  RegCount(1, regs.size()), CallConv(CC) {}
761 
763  const DataLayout &DL, unsigned Reg, Type *Ty,
765  ComputeValueVTs(TLI, DL, Ty, ValueVTs);
766 
767  CallConv = CC;
768 
769  for (EVT ValueVT : ValueVTs) {
770  unsigned NumRegs =
771  isABIMangled()
772  ? TLI.getNumRegistersForCallingConv(Context, CC.getValue(), ValueVT)
773  : TLI.getNumRegisters(Context, ValueVT);
774  MVT RegisterVT =
775  isABIMangled()
776  ? TLI.getRegisterTypeForCallingConv(Context, CC.getValue(), ValueVT)
777  : TLI.getRegisterType(Context, ValueVT);
778  for (unsigned i = 0; i != NumRegs; ++i)
779  Regs.push_back(Reg + i);
780  RegVTs.push_back(RegisterVT);
781  RegCount.push_back(NumRegs);
782  Reg += NumRegs;
783  }
784 }
785 
787  FunctionLoweringInfo &FuncInfo,
788  const SDLoc &dl, SDValue &Chain,
789  SDValue *Flag, const Value *V) const {
790  // A Value with type {} or [0 x %t] needs no registers.
791  if (ValueVTs.empty())
792  return SDValue();
793 
794  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
795 
796  // Assemble the legal parts into the final values.
797  SmallVector<SDValue, 4> Values(ValueVTs.size());
799  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
800  // Copy the legal parts from the registers.
801  EVT ValueVT = ValueVTs[Value];
802  unsigned NumRegs = RegCount[Value];
803  MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
804  *DAG.getContext(),
805  CallConv.getValue(), RegVTs[Value])
806  : RegVTs[Value];
807 
808  Parts.resize(NumRegs);
809  for (unsigned i = 0; i != NumRegs; ++i) {
810  SDValue P;
811  if (!Flag) {
812  P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
813  } else {
814  P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
815  *Flag = P.getValue(2);
816  }
817 
818  Chain = P.getValue(1);
819  Parts[i] = P;
820 
821  // If the source register was virtual and if we know something about it,
822  // add an assert node.
824  !RegisterVT.isInteger() || RegisterVT.isVector())
825  continue;
826 
828  FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
829  if (!LOI)
830  continue;
831 
832  unsigned RegSize = RegisterVT.getSizeInBits();
833  unsigned NumSignBits = LOI->NumSignBits;
834  unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
835 
836  if (NumZeroBits == RegSize) {
837  // The current value is a zero.
838  // Explicitly express that as it would be easier for
839  // optimizations to kick in.
840  Parts[i] = DAG.getConstant(0, dl, RegisterVT);
841  continue;
842  }
843 
844  // FIXME: We capture more information than the dag can represent. For
845  // now, just use the tightest assertzext/assertsext possible.
846  bool isSExt;
847  EVT FromVT(MVT::Other);
848  if (NumZeroBits) {
849  FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
850  isSExt = false;
851  } else if (NumSignBits > 1) {
852  FromVT =
853  EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
854  isSExt = true;
855  } else {
856  continue;
857  }
858  // Add an assertion node.
859  assert(FromVT != MVT::Other);
860  Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
861  RegisterVT, P, DAG.getValueType(FromVT));
862  }
863 
864  Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
865  RegisterVT, ValueVT, V, CallConv);
866  Part += NumRegs;
867  Parts.clear();
868  }
869 
870  return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
871 }
872 
874  const SDLoc &dl, SDValue &Chain, SDValue *Flag,
875  const Value *V,
876  ISD::NodeType PreferredExtendType) const {
877  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
878  ISD::NodeType ExtendKind = PreferredExtendType;
879 
880  // Get the list of the values's legal parts.
881  unsigned NumRegs = Regs.size();
882  SmallVector<SDValue, 8> Parts(NumRegs);
883  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
884  unsigned NumParts = RegCount[Value];
885 
886  MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
887  *DAG.getContext(),
888  CallConv.getValue(), RegVTs[Value])
889  : RegVTs[Value];
890 
891  if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
892  ExtendKind = ISD::ZERO_EXTEND;
893 
894  getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
895  NumParts, RegisterVT, V, CallConv, ExtendKind);
896  Part += NumParts;
897  }
898 
899  // Copy the parts into the registers.
900  SmallVector<SDValue, 8> Chains(NumRegs);
901  for (unsigned i = 0; i != NumRegs; ++i) {
902  SDValue Part;
903  if (!Flag) {
904  Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
905  } else {
906  Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
907  *Flag = Part.getValue(1);
908  }
909 
910  Chains[i] = Part.getValue(0);
911  }
912 
913  if (NumRegs == 1 || Flag)
914  // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
915  // flagged to it. That is the CopyToReg nodes and the user are considered
916  // a single scheduling unit. If we create a TokenFactor and return it as
917  // chain, then the TokenFactor is both a predecessor (operand) of the
918  // user as well as a successor (the TF operands are flagged to the user).
919  // c1, f1 = CopyToReg
920  // c2, f2 = CopyToReg
921  // c3 = TokenFactor c1, c2
922  // ...
923  // = op c3, ..., f2
924  Chain = Chains[NumRegs-1];
925  else
926  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
927 }
928 
929 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
930  unsigned MatchingIdx, const SDLoc &dl,
931  SelectionDAG &DAG,
932  std::vector<SDValue> &Ops) const {
933  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
934 
935  unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
936  if (HasMatching)
937  Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
938  else if (!Regs.empty() &&
940  // Put the register class of the virtual registers in the flag word. That
941  // way, later passes can recompute register class constraints for inline
942  // assembly as well as normal instructions.
943  // Don't do this for tied operands that can use the regclass information
944  // from the def.
946  const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
947  Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
948  }
949 
950  SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
951  Ops.push_back(Res);
952 
953  if (Code == InlineAsm::Kind_Clobber) {
954  // Clobbers should always have a 1:1 mapping with registers, and may
955  // reference registers that have illegal (e.g. vector) types. Hence, we
956  // shouldn't try to apply any sort of splitting logic to them.
957  assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
958  "No 1:1 mapping from clobbers to regs?");
959  unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
960  (void)SP;
961  for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
962  Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
963  assert(
964  (Regs[I] != SP ||
966  "If we clobbered the stack pointer, MFI should know about it.");
967  }
968  return;
969  }
970 
971  for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
972  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
973  MVT RegisterVT = RegVTs[Value];
974  for (unsigned i = 0; i != NumRegs; ++i) {
975  assert(Reg < Regs.size() && "Mismatch in # registers expected");
976  unsigned TheReg = Regs[Reg++];
977  Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
978  }
979  }
980 }
981 
985  unsigned I = 0;
986  for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
987  unsigned RegCount = std::get<0>(CountAndVT);
988  MVT RegisterVT = std::get<1>(CountAndVT);
989  unsigned RegisterSize = RegisterVT.getSizeInBits();
990  for (unsigned E = I + RegCount; I != E; ++I)
991  OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
992  }
993  return OutVec;
994 }
995 
997  const TargetLibraryInfo *li) {
998  AA = aa;
999  GFI = gfi;
1000  LibInfo = li;
1001  DL = &DAG.getDataLayout();
1002  Context = DAG.getContext();
1003  LPadToCallSiteMap.clear();
1004 }
1005 
1007  NodeMap.clear();
1008  UnusedArgNodeMap.clear();
1009  PendingLoads.clear();
1010  PendingExports.clear();
1011  CurInst = nullptr;
1012  HasTailCall = false;
1013  SDNodeOrder = LowestSDNodeOrder;
1014  StatepointLowering.clear();
1015 }
1016 
1018  DanglingDebugInfoMap.clear();
1019 }
1020 
1022  if (PendingLoads.empty())
1023  return DAG.getRoot();
1024 
1025  if (PendingLoads.size() == 1) {
1026  SDValue Root = PendingLoads[0];
1027  DAG.setRoot(Root);
1028  PendingLoads.clear();
1029  return Root;
1030  }
1031 
1032  // Otherwise, we have to make a token factor node.
1033  SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
1034  PendingLoads);
1035  PendingLoads.clear();
1036  DAG.setRoot(Root);
1037  return Root;
1038 }
1039 
1041  SDValue Root = DAG.getRoot();
1042 
1043  if (PendingExports.empty())
1044  return Root;
1045 
1046  // Turn all of the CopyToReg chains into one factored node.
1047  if (Root.getOpcode() != ISD::EntryToken) {
1048  unsigned i = 0, e = PendingExports.size();
1049  for (; i != e; ++i) {
1050  assert(PendingExports[i].getNode()->getNumOperands() > 1);
1051  if (PendingExports[i].getNode()->getOperand(0) == Root)
1052  break; // Don't add the root if we already indirectly depend on it.
1053  }
1054 
1055  if (i == e)
1056  PendingExports.push_back(Root);
1057  }
1058 
1059  Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
1060  PendingExports);
1061  PendingExports.clear();
1062  DAG.setRoot(Root);
1063  return Root;
1064 }
1065 
1067  // Set up outgoing PHI node register values before emitting the terminator.
1068  if (I.isTerminator()) {
1069  HandlePHINodesInSuccessorBlocks(I.getParent());
1070  }
1071 
1072  // Increase the SDNodeOrder if dealing with a non-debug instruction.
1073  if (!isa<DbgInfoIntrinsic>(I))
1074  ++SDNodeOrder;
1075 
1076  CurInst = &I;
1077 
1078  visit(I.getOpcode(), I);
1079 
1080  if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) {
1081  // Propagate the fast-math-flags of this IR instruction to the DAG node that
1082  // maps to this instruction.
1083  // TODO: We could handle all flags (nsw, etc) here.
1084  // TODO: If an IR instruction maps to >1 node, only the final node will have
1085  // flags set.
1086  if (SDNode *Node = getNodeForIRValue(&I)) {
1087  SDNodeFlags IncomingFlags;
1088  IncomingFlags.copyFMF(*FPMO);
1089  if (!Node->getFlags().isDefined())
1090  Node->setFlags(IncomingFlags);
1091  else
1092  Node->intersectFlagsWith(IncomingFlags);
1093  }
1094  }
1095 
1096  if (!I.isTerminator() && !HasTailCall &&
1097  !isStatepoint(&I)) // statepoints handle their exports internally
1098  CopyToExportRegsIfNeeded(&I);
1099 
1100  CurInst = nullptr;
1101 }
1102 
1103 void SelectionDAGBuilder::visitPHI(const PHINode &) {
1104  llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1105 }
1106 
1107 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1108  // Note: this doesn't use InstVisitor, because it has to work with
1109  // ConstantExpr's in addition to instructions.
1110  switch (Opcode) {
1111  default: llvm_unreachable("Unknown instruction type encountered!");
1112  // Build the switch statement using the Instruction.def file.
1113 #define HANDLE_INST(NUM, OPCODE, CLASS) \
1114  case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1115 #include "llvm/IR/Instruction.def"
1116  }
1117 }
1118 
1120  const DIExpression *Expr) {
1121  auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1122  const DbgValueInst *DI = DDI.getDI();
1123  DIVariable *DanglingVariable = DI->getVariable();
1124  DIExpression *DanglingExpr = DI->getExpression();
1125  if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1126  LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n");
1127  return true;
1128  }
1129  return false;
1130  };
1131 
1132  for (auto &DDIMI : DanglingDebugInfoMap) {
1133  DanglingDebugInfoVector &DDIV = DDIMI.second;
1134  DDIV.erase(remove_if(DDIV, isMatchingDbgValue), DDIV.end());
1135  }
1136 }
1137 
1138 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1139 // generate the debug data structures now that we've seen its definition.
1141  SDValue Val) {
1142  auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1143  if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1144  return;
1145 
1146  DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1147  for (auto &DDI : DDIV) {
1148  const DbgValueInst *DI = DDI.getDI();
1149  assert(DI && "Ill-formed DanglingDebugInfo");
1150  DebugLoc dl = DDI.getdl();
1151  unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1152  unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1153  DILocalVariable *Variable = DI->getVariable();
1154  DIExpression *Expr = DI->getExpression();
1155  assert(Variable->isValidLocationForIntrinsic(dl) &&
1156  "Expected inlined-at fields to agree");
1157  SDDbgValue *SDV;
1158  if (Val.getNode()) {
1159  if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) {
1160  LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="
1161  << DbgSDNodeOrder << "] for:\n " << *DI << "\n");
1162  LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1163  // Increase the SDNodeOrder for the DbgValue here to make sure it is
1164  // inserted after the definition of Val when emitting the instructions
1165  // after ISel. An alternative could be to teach
1166  // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1167  LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1168  << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1169  << ValSDNodeOrder << "\n");
1170  SDV = getDbgValue(Val, Variable, Expr, dl,
1171  std::max(DbgSDNodeOrder, ValSDNodeOrder));
1172  DAG.AddDbgValue(SDV, Val.getNode(), false);
1173  } else
1174  LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DI
1175  << "in EmitFuncArgumentDbgValue\n");
1176  } else
1177  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1178  }
1179  DDIV.clear();
1180 }
1181 
1182 /// getCopyFromRegs - If there was virtual register allocated for the value V
1183 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1185  DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
1186  SDValue Result;
1187 
1188  if (It != FuncInfo.ValueMap.end()) {
1189  unsigned InReg = It->second;
1190 
1191  RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1192  DAG.getDataLayout(), InReg, Ty,
1193  None); // This is not an ABI copy.
1194  SDValue Chain = DAG.getEntryNode();
1195  Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1196  V);
1197  resolveDanglingDebugInfo(V, Result);
1198  }
1199 
1200  return Result;
1201 }
1202 
1203 /// getValue - Return an SDValue for the given Value.
1205  // If we already have an SDValue for this value, use it. It's important
1206  // to do this first, so that we don't create a CopyFromReg if we already
1207  // have a regular SDValue.
1208  SDValue &N = NodeMap[V];
1209  if (N.getNode()) return N;
1210 
1211  // If there's a virtual register allocated and initialized for this
1212  // value, use it.
1213  if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1214  return copyFromReg;
1215 
1216  // Otherwise create a new SDValue and remember it.
1217  SDValue Val = getValueImpl(V);
1218  NodeMap[V] = Val;
1219  resolveDanglingDebugInfo(V, Val);
1220  return Val;
1221 }
1222 
1223 // Return true if SDValue exists for the given Value
1225  return (NodeMap.find(V) != NodeMap.end()) ||
1226  (FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end());
1227 }
1228 
1229 /// getNonRegisterValue - Return an SDValue for the given Value, but
1230 /// don't look in FuncInfo.ValueMap for a virtual register.
1232  // If we already have an SDValue for this value, use it.
1233  SDValue &N = NodeMap[V];
1234  if (N.getNode()) {
1235  if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1236  // Remove the debug location from the node as the node is about to be used
1237  // in a location which may differ from the original debug location. This
1238  // is relevant to Constant and ConstantFP nodes because they can appear
1239  // as constant expressions inside PHI nodes.
1240  N->setDebugLoc(DebugLoc());
1241  }
1242  return N;
1243  }
1244 
1245  // Otherwise create a new SDValue and remember it.
1246  SDValue Val = getValueImpl(V);
1247  NodeMap[V] = Val;
1248  resolveDanglingDebugInfo(V, Val);
1249  return Val;
1250 }
1251 
1252 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
1253 /// Create an SDValue for the given value.
1255  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1256 
1257  if (const Constant *C = dyn_cast<Constant>(V)) {
1258  EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1259 
1260  if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1261  return DAG.getConstant(*CI, getCurSDLoc(), VT);
1262 
1263  if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1264  return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1265 
1266  if (isa<ConstantPointerNull>(C)) {
1267  unsigned AS = V->getType()->getPointerAddressSpace();
1268  return DAG.getConstant(0, getCurSDLoc(),
1269  TLI.getPointerTy(DAG.getDataLayout(), AS));
1270  }
1271 
1272  if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1273  return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1274 
1275  if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1276  return DAG.getUNDEF(VT);
1277 
1278  if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1279  visit(CE->getOpcode(), *CE);
1280  SDValue N1 = NodeMap[V];
1281  assert(N1.getNode() && "visit didn't populate the NodeMap!");
1282  return N1;
1283  }
1284 
1285  if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1287  for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
1288  OI != OE; ++OI) {
1289  SDNode *Val = getValue(*OI).getNode();
1290  // If the operand is an empty aggregate, there are no values.
1291  if (!Val) continue;
1292  // Add each leaf value from the operand to the Constants list
1293  // to form a flattened list of all the values.
1294  for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1295  Constants.push_back(SDValue(Val, i));
1296  }
1297 
1298  return DAG.getMergeValues(Constants, getCurSDLoc());
1299  }
1300 
1301  if (const ConstantDataSequential *CDS =
1302  dyn_cast<ConstantDataSequential>(C)) {
1304  for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1305  SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1306  // Add each leaf value from the operand to the Constants list
1307  // to form a flattened list of all the values.
1308  for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1309  Ops.push_back(SDValue(Val, i));
1310  }
1311 
1312  if (isa<ArrayType>(CDS->getType()))
1313  return DAG.getMergeValues(Ops, getCurSDLoc());
1314  return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1315  }
1316 
1317  if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1318  assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1319  "Unknown struct or array constant!");
1320 
1322  ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1323  unsigned NumElts = ValueVTs.size();
1324  if (NumElts == 0)
1325  return SDValue(); // empty struct
1327  for (unsigned i = 0; i != NumElts; ++i) {
1328  EVT EltVT = ValueVTs[i];
1329  if (isa<UndefValue>(C))
1330  Constants[i] = DAG.getUNDEF(EltVT);
1331  else if (EltVT.isFloatingPoint())
1332  Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1333  else
1334  Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1335  }
1336 
1337  return DAG.getMergeValues(Constants, getCurSDLoc());
1338  }
1339 
1340  if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1341  return DAG.getBlockAddress(BA, VT);
1342 
1343  VectorType *VecTy = cast<VectorType>(V->getType());
1344  unsigned NumElements = VecTy->getNumElements();
1345 
1346  // Now that we know the number and type of the elements, get that number of
1347  // elements into the Ops array based on what kind of constant it is.
1349  if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1350  for (unsigned i = 0; i != NumElements; ++i)
1351  Ops.push_back(getValue(CV->getOperand(i)));
1352  } else {
1353  assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
1354  EVT EltVT =
1355  TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1356 
1357  SDValue Op;
1358  if (EltVT.isFloatingPoint())
1359  Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1360  else
1361  Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1362  Ops.assign(NumElements, Op);
1363  }
1364 
1365  // Create a BUILD_VECTOR node.
1366  return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1367  }
1368 
1369  // If this is a static alloca, generate it as the frameindex instead of
1370  // computation.
1371  if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1373  FuncInfo.StaticAllocaMap.find(AI);
1374  if (SI != FuncInfo.StaticAllocaMap.end())
1375  return DAG.getFrameIndex(SI->second,
1376  TLI.getFrameIndexTy(DAG.getDataLayout()));
1377  }
1378 
1379  // If this is an instruction which fast-isel has deferred, select it now.
1380  if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1381  unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1382 
1383  RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1384  Inst->getType(), getABIRegCopyCC(V));
1385  SDValue Chain = DAG.getEntryNode();
1386  return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1387  }
1388 
1389  llvm_unreachable("Can't get register for value!");
1390 }
1391 
1392 void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1393  auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1394  bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1395  bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1396  bool IsSEH = isAsynchronousEHPersonality(Pers);
1397  bool IsWasmCXX = Pers == EHPersonality::Wasm_CXX;
1398  MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1399  if (!IsSEH)
1400  CatchPadMBB->setIsEHScopeEntry();
1401  // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1402  if (IsMSVCCXX || IsCoreCLR)
1403  CatchPadMBB->setIsEHFuncletEntry();
1404  // Wasm does not need catchpads anymore
1405  if (!IsWasmCXX)
1406  DAG.setRoot(DAG.getNode(ISD::CATCHPAD, getCurSDLoc(), MVT::Other,
1407  getControlRoot()));
1408 }
1409 
1410 void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1411  // Update machine-CFG edge.
1412  MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1413  FuncInfo.MBB->addSuccessor(TargetMBB);
1414 
1415  auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1416  bool IsSEH = isAsynchronousEHPersonality(Pers);
1417  if (IsSEH) {
1418  // If this is not a fall-through branch or optimizations are switched off,
1419  // emit the branch.
1420  if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1421  TM.getOptLevel() == CodeGenOpt::None)
1422  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1423  getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1424  return;
1425  }
1426 
1427  // Figure out the funclet membership for the catchret's successor.
1428  // This will be used by the FuncletLayout pass to determine how to order the
1429  // BB's.
1430  // A 'catchret' returns to the outer scope's color.
1431  Value *ParentPad = I.getCatchSwitchParentPad();
1432  const BasicBlock *SuccessorColor;
1433  if (isa<ConstantTokenNone>(ParentPad))
1434  SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1435  else
1436  SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1437  assert(SuccessorColor && "No parent funclet for catchret!");
1438  MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1439  assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1440 
1441  // Create the terminator node.
1442  SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1443  getControlRoot(), DAG.getBasicBlock(TargetMBB),
1444  DAG.getBasicBlock(SuccessorColorMBB));
1445  DAG.setRoot(Ret);
1446 }
1447 
1448 void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1449  // Don't emit any special code for the cleanuppad instruction. It just marks
1450  // the start of an EH scope/funclet.
1451  FuncInfo.MBB->setIsEHScopeEntry();
1452  auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1453  if (Pers != EHPersonality::Wasm_CXX) {
1454  FuncInfo.MBB->setIsEHFuncletEntry();
1455  FuncInfo.MBB->setIsCleanupFuncletEntry();
1456  }
1457 }
1458 
1459 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
1460 /// many places it could ultimately go. In the IR, we have a single unwind
1461 /// destination, but in the machine CFG, we enumerate all the possible blocks.
1462 /// This function skips over imaginary basic blocks that hold catchswitch
1463 /// instructions, and finds all the "real" machine
1464 /// basic block destinations. As those destinations may not be successors of
1465 /// EHPadBB, here we also calculate the edge probability to those destinations.
1466 /// The passed-in Prob is the edge probability to EHPadBB.
1468  FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1469  BranchProbability Prob,
1470  SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1471  &UnwindDests) {
1472  EHPersonality Personality =
1474  bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1475  bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1476  bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
1477  bool IsSEH = isAsynchronousEHPersonality(Personality);
1478 
1479  while (EHPadBB) {
1480  const Instruction *Pad = EHPadBB->getFirstNonPHI();
1481  BasicBlock *NewEHPadBB = nullptr;
1482  if (isa<LandingPadInst>(Pad)) {
1483  // Stop on landingpads. They are not funclets.
1484  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1485  break;
1486  } else if (isa<CleanupPadInst>(Pad)) {
1487  // Stop on cleanup pads. Cleanups are always funclet entries for all known
1488  // personalities.
1489  UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1490  UnwindDests.back().first->setIsEHScopeEntry();
1491  if (!IsWasmCXX)
1492  UnwindDests.back().first->setIsEHFuncletEntry();
1493  break;
1494  } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1495  // Add the catchpad handlers to the possible destinations.
1496  for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1497  UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1498  // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1499  if (IsMSVCCXX || IsCoreCLR)
1500  UnwindDests.back().first->setIsEHFuncletEntry();
1501  if (!IsSEH)
1502  UnwindDests.back().first->setIsEHScopeEntry();
1503  }
1504  NewEHPadBB = CatchSwitch->getUnwindDest();
1505  } else {
1506  continue;
1507  }
1508 
1509  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1510  if (BPI && NewEHPadBB)
1511  Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1512  EHPadBB = NewEHPadBB;
1513  }
1514 }
1515 
1516 void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1517  // Update successor info.
1519  auto UnwindDest = I.getUnwindDest();
1520  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1521  BranchProbability UnwindDestProb =
1522  (BPI && UnwindDest)
1523  ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1525  findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1526  for (auto &UnwindDest : UnwindDests) {
1527  UnwindDest.first->setIsEHPad();
1528  addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1529  }
1530  FuncInfo.MBB->normalizeSuccProbs();
1531 
1532  // Create the terminator node.
1533  SDValue Ret =
1534  DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1535  DAG.setRoot(Ret);
1536 }
1537 
1538 void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1539  report_fatal_error("visitCatchSwitch not yet implemented!");
1540 }
1541 
1542 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1543  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1544  auto &DL = DAG.getDataLayout();
1545  SDValue Chain = getControlRoot();
1547  SmallVector<SDValue, 8> OutVals;
1548 
1549  // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1550  // lower
1551  //
1552  // %val = call <ty> @llvm.experimental.deoptimize()
1553  // ret <ty> %val
1554  //
1555  // differently.
1557  LowerDeoptimizingReturn();
1558  return;
1559  }
1560 
1561  if (!FuncInfo.CanLowerReturn) {
1562  unsigned DemoteReg = FuncInfo.DemoteRegister;
1563  const Function *F = I.getParent()->getParent();
1564 
1565  // Emit a store of the return value through the virtual register.
1566  // Leave Outs empty so that LowerReturn won't try to load return
1567  // registers the usual way.
1568  SmallVector<EVT, 1> PtrValueVTs;
1569  ComputeValueVTs(TLI, DL,
1572  PtrValueVTs);
1573 
1574  SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
1575  DemoteReg, PtrValueVTs[0]);
1576  SDValue RetOp = getValue(I.getOperand(0));
1577 
1580  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets);
1581  unsigned NumValues = ValueVTs.size();
1582 
1583  SmallVector<SDValue, 4> Chains(NumValues);
1584  for (unsigned i = 0; i != NumValues; ++i) {
1585  // An aggregate return value cannot wrap around the address space, so
1586  // offsets to its parts don't wrap either.
1587  SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr, Offsets[i]);
1588  Chains[i] = DAG.getStore(
1589  Chain, getCurSDLoc(), SDValue(RetOp.getNode(), RetOp.getResNo() + i),
1590  // FIXME: better loc info would be nice.
1592  }
1593 
1594  Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
1595  MVT::Other, Chains);
1596  } else if (I.getNumOperands() != 0) {
1598  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1599  unsigned NumValues = ValueVTs.size();
1600  if (NumValues) {
1601  SDValue RetOp = getValue(I.getOperand(0));
1602 
1603  const Function *F = I.getParent()->getParent();
1604 
1605  ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1606  if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1607  Attribute::SExt))
1608  ExtendKind = ISD::SIGN_EXTEND;
1609  else if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1610  Attribute::ZExt))
1611  ExtendKind = ISD::ZERO_EXTEND;
1612 
1613  LLVMContext &Context = F->getContext();
1614  bool RetInReg = F->getAttributes().hasAttribute(
1615  AttributeList::ReturnIndex, Attribute::InReg);
1616 
1617  for (unsigned j = 0; j != NumValues; ++j) {
1618  EVT VT = ValueVTs[j];
1619 
1620  if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1621  VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1622 
1623  CallingConv::ID CC = F->getCallingConv();
1624 
1625  unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
1626  MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
1627  SmallVector<SDValue, 4> Parts(NumParts);
1628  getCopyToParts(DAG, getCurSDLoc(),
1629  SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1630  &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
1631 
1632  // 'inreg' on function refers to return value
1633  ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1634  if (RetInReg)
1635  Flags.setInReg();
1636 
1637  // Propagate extension type if any
1638  if (ExtendKind == ISD::SIGN_EXTEND)
1639  Flags.setSExt();
1640  else if (ExtendKind == ISD::ZERO_EXTEND)
1641  Flags.setZExt();
1642 
1643  for (unsigned i = 0; i < NumParts; ++i) {
1644  Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
1645  VT, /*isfixed=*/true, 0, 0));
1646  OutVals.push_back(Parts[i]);
1647  }
1648  }
1649  }
1650  }
1651 
1652  // Push in swifterror virtual register as the last element of Outs. This makes
1653  // sure swifterror virtual register will be returned in the swifterror
1654  // physical register.
1655  const Function *F = I.getParent()->getParent();
1656  if (TLI.supportSwiftError() &&
1657  F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
1658  assert(FuncInfo.SwiftErrorArg && "Need a swift error argument");
1659  ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1660  Flags.setSwiftError();
1661  Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/,
1662  EVT(TLI.getPointerTy(DL)) /*argvt*/,
1663  true /*isfixed*/, 1 /*origidx*/,
1664  0 /*partOffs*/));
1665  // Create SDNode for the swifterror virtual register.
1666  OutVals.push_back(
1667  DAG.getRegister(FuncInfo.getOrCreateSwiftErrorVRegUseAt(
1668  &I, FuncInfo.MBB, FuncInfo.SwiftErrorArg).first,
1669  EVT(TLI.getPointerTy(DL))));
1670  }
1671 
1672  bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
1673  CallingConv::ID CallConv =
1675  Chain = DAG.getTargetLoweringInfo().LowerReturn(
1676  Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
1677 
1678  // Verify that the target's LowerReturn behaved as expected.
1679  assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
1680  "LowerReturn didn't return a valid chain!");
1681 
1682  // Update the DAG with the new chain value resulting from return lowering.
1683  DAG.setRoot(Chain);
1684 }
1685 
1686 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
1687 /// created for it, emit nodes to copy the value into the virtual
1688 /// registers.
1690  // Skip empty types
1691  if (V->getType()->isEmptyTy())
1692  return;
1693 
1694  DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
1695  if (VMI != FuncInfo.ValueMap.end()) {
1696  assert(!V->use_empty() && "Unused value assigned virtual registers!");
1697  CopyValueToVirtualRegister(V, VMI->second);
1698  }
1699 }
1700 
1701 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
1702 /// the current basic block, add it to ValueMap now so that we'll get a
1703 /// CopyTo/FromReg.
1705  // No need to export constants.
1706  if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
1707 
1708  // Already exported?
1709  if (FuncInfo.isExportedInst(V)) return;
1710 
1711  unsigned Reg = FuncInfo.InitializeRegForValue(V);
1712  CopyValueToVirtualRegister(V, Reg);
1713 }
1714 
1716  const BasicBlock *FromBB) {
1717  // The operands of the setcc have to be in this block. We don't know
1718  // how to export them from some other block.
1719  if (const Instruction *VI = dyn_cast<Instruction>(V)) {
1720  // Can export from current BB.
1721  if (VI->getParent() == FromBB)
1722  return true;
1723 
1724  // Is already exported, noop.
1725  return FuncInfo.isExportedInst(V);
1726  }
1727 
1728  // If this is an argument, we can export it if the BB is the entry block or
1729  // if it is already exported.
1730  if (isa<Argument>(V)) {
1731  if (FromBB == &FromBB->getParent()->getEntryBlock())
1732  return true;
1733 
1734  // Otherwise, can only export this if it is already exported.
1735  return FuncInfo.isExportedInst(V);
1736  }
1737 
1738  // Otherwise, constants can always be exported.
1739  return true;
1740 }
1741 
1742 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
1744 SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
1745  const MachineBasicBlock *Dst) const {
1746  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1747  const BasicBlock *SrcBB = Src->getBasicBlock();
1748  const BasicBlock *DstBB = Dst->getBasicBlock();
1749  if (!BPI) {
1750  // If BPI is not available, set the default probability as 1 / N, where N is
1751  // the number of successors.
1752  auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
1753  return BranchProbability(1, SuccSize);
1754  }
1755  return BPI->getEdgeProbability(SrcBB, DstBB);
1756 }
1757 
1758 void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
1759  MachineBasicBlock *Dst,
1760  BranchProbability Prob) {
1761  if (!FuncInfo.BPI)
1762  Src->addSuccessorWithoutProb(Dst);
1763  else {
1764  if (Prob.isUnknown())
1765  Prob = getEdgeProbability(Src, Dst);
1766  Src->addSuccessor(Dst, Prob);
1767  }
1768 }
1769 
1770 static bool InBlock(const Value *V, const BasicBlock *BB) {
1771  if (const Instruction *I = dyn_cast<Instruction>(V))
1772  return I->getParent() == BB;
1773  return true;
1774 }
1775 
1776 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
1777 /// This function emits a branch and is used at the leaves of an OR or an
1778 /// AND operator tree.
1779 void
1781  MachineBasicBlock *TBB,
1782  MachineBasicBlock *FBB,
1783  MachineBasicBlock *CurBB,
1784  MachineBasicBlock *SwitchBB,
1785  BranchProbability TProb,
1786  BranchProbability FProb,
1787  bool InvertCond) {
1788  const BasicBlock *BB = CurBB->getBasicBlock();
1789 
1790  // If the leaf of the tree is a comparison, merge the condition into
1791  // the caseblock.
1792  if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
1793  // The operands of the cmp have to be in this block. We don't know
1794  // how to export them from some other block. If this is the first block
1795  // of the sequence, no exporting is needed.
1796  if (CurBB == SwitchBB ||
1797  (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
1798  isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
1799  ISD::CondCode Condition;
1800  if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
1801  ICmpInst::Predicate Pred =
1802  InvertCond ? IC->getInversePredicate() : IC->getPredicate();
1803  Condition = getICmpCondCode(Pred);
1804  } else {
1805  const FCmpInst *FC = cast<FCmpInst>(Cond);
1806  FCmpInst::Predicate Pred =
1807  InvertCond ? FC->getInversePredicate() : FC->getPredicate();
1808  Condition = getFCmpCondCode(Pred);
1809  if (TM.Options.NoNaNsFPMath)
1810  Condition = getFCmpCodeWithoutNaN(Condition);
1811  }
1812 
1813  CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
1814  TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
1815  SwitchCases.push_back(CB);
1816  return;
1817  }
1818  }
1819 
1820  // Create a CaseBlock record representing this branch.
1821  ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
1822  CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
1823  nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
1824  SwitchCases.push_back(CB);
1825 }
1826 
1827 /// FindMergedConditions - If Cond is an expression like
1829  MachineBasicBlock *TBB,
1830  MachineBasicBlock *FBB,
1831  MachineBasicBlock *CurBB,
1832  MachineBasicBlock *SwitchBB,
1834  BranchProbability TProb,
1835  BranchProbability FProb,
1836  bool InvertCond) {
1837  // Skip over not part of the tree and remember to invert op and operands at
1838  // next level.
1839  if (BinaryOperator::isNot(Cond) && Cond->hasOneUse()) {
1840  const Value *CondOp = BinaryOperator::getNotArgument(Cond);
1841  if (InBlock(CondOp, CurBB->getBasicBlock())) {
1842  FindMergedConditions(CondOp, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
1843  !InvertCond);
1844  return;
1845  }
1846  }
1847 
1848  const Instruction *BOp = dyn_cast<Instruction>(Cond);
1849  // Compute the effective opcode for Cond, taking into account whether it needs
1850  // to be inverted, e.g.
1851  // and (not (or A, B)), C
1852  // gets lowered as
1853  // and (and (not A, not B), C)
1854  unsigned BOpc = 0;
1855  if (BOp) {
1856  BOpc = BOp->getOpcode();
1857  if (InvertCond) {
1858  if (BOpc == Instruction::And)
1859  BOpc = Instruction::Or;
1860  else if (BOpc == Instruction::Or)
1861  BOpc = Instruction::And;
1862  }
1863  }
1864 
1865  // If this node is not part of the or/and tree, emit it as a branch.
1866  if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
1867  BOpc != unsigned(Opc) || !BOp->hasOneUse() ||
1868  BOp->getParent() != CurBB->getBasicBlock() ||
1869  !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
1870  !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
1871  EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
1872  TProb, FProb, InvertCond);
1873  return;
1874  }
1875 
1876  // Create TmpBB after CurBB.
1877  MachineFunction::iterator BBI(CurBB);
1878  MachineFunction &MF = DAG.getMachineFunction();
1880  CurBB->getParent()->insert(++BBI, TmpBB);
1881 
1882  if (Opc == Instruction::Or) {
1883  // Codegen X | Y as:
1884  // BB1:
1885  // jmp_if_X TBB
1886  // jmp TmpBB
1887  // TmpBB:
1888  // jmp_if_Y TBB
1889  // jmp FBB
1890  //
1891 
1892  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1893  // The requirement is that
1894  // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
1895  // = TrueProb for original BB.
1896  // Assuming the original probabilities are A and B, one choice is to set
1897  // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
1898  // A/(1+B) and 2B/(1+B). This choice assumes that
1899  // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
1900  // Another choice is to assume TrueProb for BB1 equals to TrueProb for
1901  // TmpBB, but the math is more complicated.
1902 
1903  auto NewTrueProb = TProb / 2;
1904  auto NewFalseProb = TProb / 2 + FProb;
1905  // Emit the LHS condition.
1906  FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
1907  NewTrueProb, NewFalseProb, InvertCond);
1908 
1909  // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
1910  SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
1911  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1912  // Emit the RHS condition into TmpBB.
1913  FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1914  Probs[0], Probs[1], InvertCond);
1915  } else {
1916  assert(Opc == Instruction::And && "Unknown merge op!");
1917  // Codegen X & Y as:
1918  // BB1:
1919  // jmp_if_X TmpBB
1920  // jmp FBB
1921  // TmpBB:
1922  // jmp_if_Y TBB
1923  // jmp FBB
1924  //
1925  // This requires creation of TmpBB after CurBB.
1926 
1927  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1928  // The requirement is that
1929  // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
1930  // = FalseProb for original BB.
1931  // Assuming the original probabilities are A and B, one choice is to set
1932  // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
1933  // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
1934  // TrueProb for BB1 * FalseProb for TmpBB.
1935 
1936  auto NewTrueProb = TProb + FProb / 2;
1937  auto NewFalseProb = FProb / 2;
1938  // Emit the LHS condition.
1939  FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
1940  NewTrueProb, NewFalseProb, InvertCond);
1941 
1942  // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
1943  SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
1944  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1945  // Emit the RHS condition into TmpBB.
1946  FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1947  Probs[0], Probs[1], InvertCond);
1948  }
1949 }
1950 
1951 /// If the set of cases should be emitted as a series of branches, return true.
1952 /// If we should emit this as a bunch of and/or'd together conditions, return
1953 /// false.
1954 bool
1955 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
1956  if (Cases.size() != 2) return true;
1957 
1958  // If this is two comparisons of the same values or'd or and'd together, they
1959  // will get folded into a single comparison, so don't emit two blocks.
1960  if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
1961  Cases[0].CmpRHS == Cases[1].CmpRHS) ||
1962  (Cases[0].CmpRHS == Cases[1].CmpLHS &&
1963  Cases[0].CmpLHS == Cases[1].CmpRHS)) {
1964  return false;
1965  }
1966 
1967  // Handle: (X != null) | (Y != null) --> (X|Y) != 0
1968  // Handle: (X == null) & (Y == null) --> (X|Y) == 0
1969  if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
1970  Cases[0].CC == Cases[1].CC &&
1971  isa<Constant>(Cases[0].CmpRHS) &&
1972  cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
1973  if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
1974  return false;
1975  if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
1976  return false;
1977  }
1978 
1979  return true;
1980 }
1981 
1982 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
1983  MachineBasicBlock *BrMBB = FuncInfo.MBB;
1984 
1985  // Update machine-CFG edges.
1986  MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
1987 
1988  if (I.isUnconditional()) {
1989  // Update machine-CFG edges.
1990  BrMBB->addSuccessor(Succ0MBB);
1991 
1992  // If this is not a fall-through branch or optimizations are switched off,
1993  // emit the branch.
1994  if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
1995  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
1996  MVT::Other, getControlRoot(),
1997  DAG.getBasicBlock(Succ0MBB)));
1998 
1999  return;
2000  }
2001 
2002  // If this condition is one of the special cases we handle, do special stuff
2003  // now.
2004  const Value *CondVal = I.getCondition();
2005  MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2006 
2007  // If this is a series of conditions that are or'd or and'd together, emit
2008  // this as a sequence of branches instead of setcc's with and/or operations.
2009  // As long as jumps are not expensive, this should improve performance.
2010  // For example, instead of something like:
2011  // cmp A, B
2012  // C = seteq
2013  // cmp D, E
2014  // F = setle
2015  // or C, F
2016  // jnz foo
2017  // Emit:
2018  // cmp A, B
2019  // je foo
2020  // cmp D, E
2021  // jle foo
2022  if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
2023  Instruction::BinaryOps Opcode = BOp->getOpcode();
2024  if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp->hasOneUse() &&
2026  (Opcode == Instruction::And || Opcode == Instruction::Or)) {
2027  FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
2028  Opcode,
2029  getEdgeProbability(BrMBB, Succ0MBB),
2030  getEdgeProbability(BrMBB, Succ1MBB),
2031  /*InvertCond=*/false);
2032  // If the compares in later blocks need to use values not currently
2033  // exported from this block, export them now. This block should always
2034  // be the first entry.
2035  assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2036 
2037  // Allow some cases to be rejected.
2038  if (ShouldEmitAsBranches(SwitchCases)) {
2039  for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
2040  ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
2041  ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
2042  }
2043 
2044  // Emit the branch for this block.
2045  visitSwitchCase(SwitchCases[0], BrMBB);
2046  SwitchCases.erase(SwitchCases.begin());
2047  return;
2048  }
2049 
2050  // Okay, we decided not to do this, remove any inserted MBB's and clear
2051  // SwitchCases.
2052  for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
2053  FuncInfo.MF->erase(SwitchCases[i].ThisBB);
2054 
2055  SwitchCases.clear();
2056  }
2057  }
2058 
2059  // Create a CaseBlock record representing this branch.
2060  CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2061  nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2062 
2063  // Use visitSwitchCase to actually insert the fast branch sequence for this
2064  // cond branch.
2065  visitSwitchCase(CB, BrMBB);
2066 }
2067 
2068 /// visitSwitchCase - Emits the necessary code to represent a single node in
2069 /// the binary search tree resulting from lowering a switch instruction.
2071  MachineBasicBlock *SwitchBB) {
2072  SDValue Cond;
2073  SDValue CondLHS = getValue(CB.CmpLHS);
2074  SDLoc dl = CB.DL;
2075 
2076  // Build the setcc now.
2077  if (!CB.CmpMHS) {
2078  // Fold "(X == true)" to X and "(X == false)" to !X to
2079  // handle common cases produced by branch lowering.
2080  if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2081  CB.CC == ISD::SETEQ)
2082  Cond = CondLHS;
2083  else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2084  CB.CC == ISD::SETEQ) {
2085  SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2086  Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2087  } else
2088  Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
2089  } else {
2090  assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2091 
2092  const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2093  const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2094 
2095  SDValue CmpOp = getValue(CB.CmpMHS);
2096  EVT VT = CmpOp.getValueType();
2097 
2098  if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2099  Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2100  ISD::SETLE);
2101  } else {
2102  SDValue SUB = DAG.getNode(ISD::SUB, dl,
2103  VT, CmpOp, DAG.getConstant(Low, dl, VT));
2104  Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2105  DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2106  }
2107  }
2108 
2109  // Update successor info
2110  addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2111  // TrueBB and FalseBB are always different unless the incoming IR is
2112  // degenerate. This only happens when running llc on weird IR.
2113  if (CB.TrueBB != CB.FalseBB)
2114  addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2115  SwitchBB->normalizeSuccProbs();
2116 
2117  // If the lhs block is the next block, invert the condition so that we can
2118  // fall through to the lhs instead of the rhs block.
2119  if (CB.TrueBB == NextBlock(SwitchBB)) {
2120  std::swap(CB.TrueBB, CB.FalseBB);
2121  SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2122  Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2123  }
2124 
2125  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2126  MVT::Other, getControlRoot(), Cond,
2127  DAG.getBasicBlock(CB.TrueBB));
2128 
2129  // Insert the false branch. Do this even if it's a fall through branch,
2130  // this makes it easier to do DAG optimizations which require inverting
2131  // the branch condition.
2132  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2133  DAG.getBasicBlock(CB.FalseBB));
2134 
2135  DAG.setRoot(BrCond);
2136 }
2137 
2138 /// visitJumpTable - Emit JumpTable node in the current MBB
2140  // Emit the code for the jump table
2141  assert(JT.Reg != -1U && "Should lower JT Header first!");
2143  SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
2144  JT.Reg, PTy);
2145  SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2146  SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
2147  MVT::Other, Index.getValue(1),
2148  Table, Index);
2149  DAG.setRoot(BrJumpTable);
2150 }
2151 
2152 /// visitJumpTableHeader - This function emits necessary code to produce index
2153 /// in the JumpTable from switch case.
2155  JumpTableHeader &JTH,
2156  MachineBasicBlock *SwitchBB) {
2157  SDLoc dl = getCurSDLoc();
2158 
2159  // Subtract the lowest switch case value from the value being switched on and
2160  // conditional branch to default mbb if the result is greater than the
2161  // difference between smallest and largest cases.
2162  SDValue SwitchOp = getValue(JTH.SValue);
2163  EVT VT = SwitchOp.getValueType();
2164  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2165  DAG.getConstant(JTH.First, dl, VT));
2166 
2167  // The SDNode we just created, which holds the value being switched on minus
2168  // the smallest case value, needs to be copied to a virtual register so it
2169  // can be used as an index into the jump table in a subsequent basic block.
2170  // This value may be smaller or larger than the target's pointer type, and
2171  // therefore require extension or truncating.
2172  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2173  SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2174 
2175  unsigned JumpTableReg =
2176  FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
2177  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2178  JumpTableReg, SwitchOp);
2179  JT.Reg = JumpTableReg;
2180 
2181  // Emit the range check for the jump table, and branch to the default block
2182  // for the switch statement if the value being switched on exceeds the largest
2183  // case in the switch.
2184  SDValue CMP = DAG.getSetCC(
2185  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2186  Sub.getValueType()),
2187  Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2188 
2189  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2190  MVT::Other, CopyTo, CMP,
2191  DAG.getBasicBlock(JT.Default));
2192 
2193  // Avoid emitting unnecessary branches to the next block.
2194  if (JT.MBB != NextBlock(SwitchBB))
2195  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2196  DAG.getBasicBlock(JT.MBB));
2197 
2198  DAG.setRoot(BrCond);
2199 }
2200 
2201 /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2202 /// variable if there exists one.
2204  SDValue &Chain) {
2205  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2206  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2207  MachineFunction &MF = DAG.getMachineFunction();
2208  Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());
2209  MachineSDNode *Node =
2210  DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2211  if (Global) {
2212  MachinePointerInfo MPInfo(Global);
2216  MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlignment(PtrTy));
2217  DAG.setNodeMemRefs(Node, {MemRef});
2218  }
2219  return SDValue(Node, 0);
2220 }
2221 
2222 /// Codegen a new tail for a stack protector check ParentMBB which has had its
2223 /// tail spliced into a stack protector check success bb.
2224 ///
2225 /// For a high level explanation of how this fits into the stack protector
2226 /// generation see the comment on the declaration of class
2227 /// StackProtectorDescriptor.
2228 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
2229  MachineBasicBlock *ParentBB) {
2230 
2231  // First create the loads to the guard/stack slot for the comparison.
2232  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2233  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2234 
2235  MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2236  int FI = MFI.getStackProtectorIndex();
2237 
2238  SDValue Guard;
2239  SDLoc dl = getCurSDLoc();
2240  SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2241  const Module &M = *ParentBB->getParent()->getFunction().getParent();
2242  unsigned Align = DL->getPrefTypeAlignment(Type::getInt8PtrTy(M.getContext()));
2243 
2244  // Generate code to load the content of the guard slot.
2245  SDValue GuardVal = DAG.getLoad(
2246  PtrTy, dl, DAG.getEntryNode(), StackSlotPtr,
2249 
2250  if (TLI.useStackGuardXorFP())
2251  GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
2252 
2253  // Retrieve guard check function, nullptr if instrumentation is inlined.
2254  if (const Value *GuardCheck = TLI.getSSPStackGuardCheck(M)) {
2255  // The target provides a guard check function to validate the guard value.
2256  // Generate a call to that function with the content of the guard slot as
2257  // argument.
2258  auto *Fn = cast<Function>(GuardCheck);
2259  FunctionType *FnTy = Fn->getFunctionType();
2260  assert(FnTy->getNumParams() == 1 && "Invalid function signature");
2261 
2264  Entry.Node = GuardVal;
2265  Entry.Ty = FnTy->getParamType(0);
2266  if (Fn->hasAttribute(1, Attribute::AttrKind::InReg))
2267  Entry.IsInReg = true;
2268  Args.push_back(Entry);
2269 
2271  CLI.setDebugLoc(getCurSDLoc())
2272  .setChain(DAG.getEntryNode())
2273  .setCallee(Fn->getCallingConv(), FnTy->getReturnType(),
2274  getValue(GuardCheck), std::move(Args));
2275 
2276  std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2277  DAG.setRoot(Result.second);
2278  return;
2279  }
2280 
2281  // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2282  // Otherwise, emit a volatile load to retrieve the stack guard value.
2283  SDValue Chain = DAG.getEntryNode();
2284  if (TLI.useLoadStackGuardNode()) {
2285  Guard = getLoadStackGuard(DAG, dl, Chain);
2286  } else {
2287  const Value *IRGuard = TLI.getSDagStackGuard(M);
2288  SDValue GuardPtr = getValue(IRGuard);
2289 
2290  Guard =
2291  DAG.getLoad(PtrTy, dl, Chain, GuardPtr, MachinePointerInfo(IRGuard, 0),
2293  }
2294 
2295  // Perform the comparison via a subtract/getsetcc.
2296  EVT VT = Guard.getValueType();
2297  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, GuardVal);
2298 
2299  SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2300  *DAG.getContext(),
2301  Sub.getValueType()),
2302  Sub, DAG.getConstant(0, dl, VT), ISD::SETNE);
2303 
2304  // If the sub is not 0, then we know the guard/stackslot do not equal, so
2305  // branch to failure MBB.
2306  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2307  MVT::Other, GuardVal.getOperand(0),
2308  Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2309  // Otherwise branch to success MBB.
2310  SDValue Br = DAG.getNode(ISD::BR, dl,
2311  MVT::Other, BrCond,
2312  DAG.getBasicBlock(SPD.getSuccessMBB()));
2313 
2314  DAG.setRoot(Br);
2315 }
2316 
2317 /// Codegen the failure basic block for a stack protector check.
2318 ///
2319 /// A failure stack protector machine basic block consists simply of a call to
2320 /// __stack_chk_fail().
2321 ///
2322 /// For a high level explanation of how this fits into the stack protector
2323 /// generation see the comment on the declaration of class
2324 /// StackProtectorDescriptor.
2325 void
2326 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2327  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2328  SDValue Chain =
2329  TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2330  None, false, getCurSDLoc(), false, false).second;
2331  DAG.setRoot(Chain);
2332 }
2333 
2334 /// visitBitTestHeader - This function emits necessary code to produce value
2335 /// suitable for "bit tests"
2337  MachineBasicBlock *SwitchBB) {
2338  SDLoc dl = getCurSDLoc();
2339 
2340  // Subtract the minimum value
2341  SDValue SwitchOp = getValue(B.SValue);
2342  EVT VT = SwitchOp.getValueType();
2343  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2344  DAG.getConstant(B.First, dl, VT));
2345 
2346  // Check range
2347  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2348  SDValue RangeCmp = DAG.getSetCC(
2349  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2350  Sub.getValueType()),
2351  Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
2352 
2353  // Determine the type of the test operands.
2354  bool UsePtrType = false;
2355  if (!TLI.isTypeLegal(VT))
2356  UsePtrType = true;
2357  else {
2358  for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2359  if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2360  // Switch table case range are encoded into series of masks.
2361  // Just use pointer type, it's guaranteed to fit.
2362  UsePtrType = true;
2363  break;
2364  }
2365  }
2366  if (UsePtrType) {
2367  VT = TLI.getPointerTy(DAG.getDataLayout());
2368  Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2369  }
2370 
2371  B.RegVT = VT.getSimpleVT();
2372  B.Reg = FuncInfo.CreateReg(B.RegVT);
2373  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2374 
2375  MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2376 
2377  addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2378  addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2379  SwitchBB->normalizeSuccProbs();
2380 
2381  SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
2382  MVT::Other, CopyTo, RangeCmp,
2383  DAG.getBasicBlock(B.Default));
2384 
2385  // Avoid emitting unnecessary branches to the next block.
2386  if (MBB != NextBlock(SwitchBB))
2387  BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
2388  DAG.getBasicBlock(MBB));
2389 
2390  DAG.setRoot(BrRange);
2391 }
2392 
2393 /// visitBitTestCase - this function produces one "bit test"
2395  MachineBasicBlock* NextMBB,
2396  BranchProbability BranchProbToNext,
2397  unsigned Reg,
2398  BitTestCase &B,
2399  MachineBasicBlock *SwitchBB) {
2400  SDLoc dl = getCurSDLoc();
2401  MVT VT = BB.RegVT;
2402  SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2403  SDValue Cmp;
2404  unsigned PopCount = countPopulation(B.Mask);
2405  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2406  if (PopCount == 1) {
2407  // Testing for a single bit; just compare the shift count with what it
2408  // would need to be to shift a 1 bit in that position.
2409  Cmp = DAG.getSetCC(
2410  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2411  ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2412  ISD::SETEQ);
2413  } else if (PopCount == BB.Range) {
2414  // There is only one zero bit in the range, test for it directly.
2415  Cmp = DAG.getSetCC(
2416  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2417  ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2418  ISD::SETNE);
2419  } else {
2420  // Make desired shift
2421  SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2422  DAG.getConstant(1, dl, VT), ShiftOp);
2423 
2424  // Emit bit tests and jumps
2425  SDValue AndOp = DAG.getNode(ISD::AND, dl,
2426  VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2427  Cmp = DAG.getSetCC(
2428  dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2429  AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2430  }
2431 
2432  // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2433  addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2434  // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2435  addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2436  // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2437  // one as they are relative probabilities (and thus work more like weights),
2438  // and hence we need to normalize them to let the sum of them become one.
2439  SwitchBB->normalizeSuccProbs();
2440 
2441  SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2442  MVT::Other, getControlRoot(),
2443  Cmp, DAG.getBasicBlock(B.TargetBB));
2444 
2445  // Avoid emitting unnecessary branches to the next block.
2446  if (NextMBB != NextBlock(SwitchBB))
2447  BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2448  DAG.getBasicBlock(NextMBB));
2449 
2450  DAG.setRoot(BrAnd);
2451 }
2452 
2453 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2454  MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2455 
2456  // Retrieve successors. Look through artificial IR level blocks like
2457  // catchswitch for successors.
2458  MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2459  const BasicBlock *EHPadBB = I.getSuccessor(1);
2460 
2461  // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2462  // have to do anything here to lower funclet bundles.
2464  {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2465  "Cannot lower invokes with arbitrary operand bundles yet!");
2466 
2467  const Value *Callee(I.getCalledValue());
2468  const Function *Fn = dyn_cast<Function>(Callee);
2469  if (isa<InlineAsm>(Callee))
2470  visitInlineAsm(&I);
2471  else if (Fn && Fn->isIntrinsic()) {
2472  switch (Fn->getIntrinsicID()) {
2473  default:
2474  llvm_unreachable("Cannot invoke this intrinsic");
2475  case Intrinsic::donothing:
2476  // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2477  break;
2478  case Intrinsic::experimental_patchpoint_void:
2479  case Intrinsic::experimental_patchpoint_i64:
2480  visitPatchpoint(&I, EHPadBB);
2481  break;
2482  case Intrinsic::experimental_gc_statepoint:
2483  LowerStatepoint(ImmutableStatepoint(&I), EHPadBB);
2484  break;
2485  }
2487  // Currently we do not lower any intrinsic calls with deopt operand bundles.
2488  // Eventually we will support lowering the @llvm.experimental.deoptimize
2489  // intrinsic, and right now there are no plans to support other intrinsics
2490  // with deopt state.
2491  LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2492  } else {
2493  LowerCallTo(&I, getValue(Callee), false, EHPadBB);
2494  }
2495 
2496  // If the value of the invoke is used outside of its defining block, make it
2497  // available as a virtual register.
2498  // We already took care of the exported value for the statepoint instruction
2499  // during call to the LowerStatepoint.
2500  if (!isStatepoint(I)) {
2501  CopyToExportRegsIfNeeded(&I);
2502  }
2503 
2505  BranchProbabilityInfo *BPI = FuncInfo.BPI;
2506  BranchProbability EHPadBBProb =
2507  BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2509  findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2510 
2511  // Update successor info.
2512  addSuccessorWithProb(InvokeMBB, Return);
2513  for (auto &UnwindDest : UnwindDests) {
2514  UnwindDest.first->setIsEHPad();
2515  addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2516  }
2517  InvokeMBB->normalizeSuccProbs();
2518 
2519  // Drop into normal successor.
2520  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2521  MVT::Other, getControlRoot(),
2522  DAG.getBasicBlock(Return)));
2523 }
2524 
2525 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
2526  llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
2527 }
2528 
2529 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
2530  assert(FuncInfo.MBB->isEHPad() &&
2531  "Call to landingpad not in landing pad!");
2532 
2533  // If there aren't registers to copy the values into (e.g., during SjLj
2534  // exceptions), then don't bother to create these DAG nodes.
2535  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2536  const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
2537  if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2538  TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2539  return;
2540 
2541  // If landingpad's return type is token type, we don't create DAG nodes
2542  // for its exception pointer and selector value. The extraction of exception
2543  // pointer or selector value from token type landingpads is not currently
2544  // supported.
2545  if (LP.getType()->isTokenTy())
2546  return;
2547 
2549  SDLoc dl = getCurSDLoc();
2550  ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
2551  assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
2552 
2553  // Get the two live-in registers as SDValues. The physregs have already been
2554  // copied into virtual registers.
2555  SDValue Ops[2];
2556  if (FuncInfo.ExceptionPointerVirtReg) {
2557  Ops[0] = DAG.getZExtOrTrunc(
2558  DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2559  FuncInfo.ExceptionPointerVirtReg,
2560  TLI.getPointerTy(DAG.getDataLayout())),
2561  dl, ValueVTs[0]);
2562  } else {
2563  Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
2564  }
2565  Ops[1] = DAG.getZExtOrTrunc(
2566  DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2567  FuncInfo.ExceptionSelectorVirtReg,
2568  TLI.getPointerTy(DAG.getDataLayout())),
2569  dl, ValueVTs[1]);
2570 
2571  // Merge into one.
2572  SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
2573  DAG.getVTList(ValueVTs), Ops);
2574  setValue(&LP, Res);
2575 }
2576 
2577 void SelectionDAGBuilder::sortAndRangeify(CaseClusterVector &Clusters) {
2578 #ifndef NDEBUG
2579  for (const CaseCluster &CC : Clusters)
2580  assert(CC.Low == CC.High && "Input clusters must be single-case");
2581 #endif
2582 
2583  llvm::sort(Clusters, [](const CaseCluster &a, const CaseCluster &b) {
2584  return a.Low->getValue().slt(b.Low->getValue());
2585  });
2586 
2587  // Merge adjacent clusters with the same destination.
2588  const unsigned N = Clusters.size();
2589  unsigned DstIndex = 0;
2590  for (unsigned SrcIndex = 0; SrcIndex < N; ++SrcIndex) {
2591  CaseCluster &CC = Clusters[SrcIndex];
2592  const ConstantInt *CaseVal = CC.Low;
2593  MachineBasicBlock *Succ = CC.MBB;
2594 
2595  if (DstIndex != 0 && Clusters[DstIndex - 1].MBB == Succ &&
2596  (CaseVal->getValue() - Clusters[DstIndex - 1].High->getValue()) == 1) {
2597  // If this case has the same successor and is a neighbour, merge it into
2598  // the previous cluster.
2599  Clusters[DstIndex - 1].High = CaseVal;
2600  Clusters[DstIndex - 1].Prob += CC.Prob;
2601  } else {
2602  std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex],
2603  sizeof(Clusters[SrcIndex]));
2604  }
2605  }
2606  Clusters.resize(DstIndex);
2607 }
2608 
2610  MachineBasicBlock *Last) {
2611  // Update JTCases.
2612  for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
2613  if (JTCases[i].first.HeaderBB == First)
2614  JTCases[i].first.HeaderBB = Last;
2615 
2616  // Update BitTestCases.
2617  for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
2618  if (BitTestCases[i].Parent == First)
2619  BitTestCases[i].Parent = Last;
2620 }
2621 
2622 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
2623  MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
2624 
2625  // Update machine-CFG edges with unique successors.
2627  for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
2628  BasicBlock *BB = I.getSuccessor(i);
2629  bool Inserted = Done.insert(BB).second;
2630  if (!Inserted)
2631  continue;
2632 
2633  MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
2634  addSuccessorWithProb(IndirectBrMBB, Succ);
2635  }
2636  IndirectBrMBB->normalizeSuccProbs();
2637 
2638  DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
2639  MVT::Other, getControlRoot(),
2640  getValue(I.getAddress())));
2641 }
2642 
2643 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
2644  if (!DAG.getTarget().Options.TrapUnreachable)
2645  return;
2646 
2647  // We may be able to ignore unreachable behind a noreturn call.
2649  const BasicBlock &BB = *I.getParent();
2650  if (&I != &BB.front()) {
2652  std::prev(BasicBlock::const_iterator(&I));
2653  if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
2654  if (Call->doesNotReturn())
2655  return;
2656  }
2657  }
2658  }
2659 
2660  DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
2661 }
2662 
2663 void SelectionDAGBuilder::visitFSub(const User &I) {
2664  // -0.0 - X --> fneg
2665  Type *Ty = I.getType();
2666  if (isa<Constant>(I.getOperand(0)) &&
2668  SDValue Op2 = getValue(I.getOperand(1));
2669  setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
2670  Op2.getValueType(), Op2));
2671  return;
2672  }
2673 
2674  visitBinary(I, ISD::FSUB);
2675 }
2676 
2677 /// Checks if the given instruction performs a vector reduction, in which case
2678 /// we have the freedom to alter the elements in the result as long as the
2679 /// reduction of them stays unchanged.
2680 static bool isVectorReductionOp(const User *I) {
2681  const Instruction *Inst = dyn_cast<Instruction>(I);
2682  if (!Inst || !Inst->getType()->isVectorTy())
2683  return false;
2684 
2685  auto OpCode = Inst->getOpcode();
2686  switch (OpCode) {
2687  case Instruction::Add:
2688  case Instruction::Mul:
2689  case Instruction::And:
2690  case Instruction::Or:
2691  case Instruction::Xor:
2692  break;
2693  case Instruction::FAdd:
2694  case Instruction::FMul:
2695  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2696  if (FPOp->getFastMathFlags().isFast())
2697  break;
2699  default:
2700  return false;
2701  }
2702 
2703  unsigned ElemNum = Inst->getType()->getVectorNumElements();
2704  // Ensure the reduction size is a power of 2.
2705  if (!isPowerOf2_32(ElemNum))
2706  return false;
2707 
2708  unsigned ElemNumToReduce = ElemNum;
2709 
2710  // Do DFS search on the def-use chain from the given instruction. We only
2711  // allow four kinds of operations during the search until we reach the
2712  // instruction that extracts the first element from the vector:
2713  //
2714  // 1. The reduction operation of the same opcode as the given instruction.
2715  //
2716  // 2. PHI node.
2717  //
2718  // 3. ShuffleVector instruction together with a reduction operation that
2719  // does a partial reduction.
2720  //
2721  // 4. ExtractElement that extracts the first element from the vector, and we
2722  // stop searching the def-use chain here.
2723  //
2724  // 3 & 4 above perform a reduction on all elements of the vector. We push defs
2725  // from 1-3 to the stack to continue the DFS. The given instruction is not
2726  // a reduction operation if we meet any other instructions other than those
2727  // listed above.
2728 
2729  SmallVector<const User *, 16> UsersToVisit{Inst};
2731  bool ReduxExtracted = false;
2732 
2733  while (!UsersToVisit.empty()) {
2734  auto User = UsersToVisit.back();
2735  UsersToVisit.pop_back();
2736  if (!Visited.insert(User).second)
2737  continue;
2738 
2739  for (const auto &U : User->users()) {
2740  auto Inst = dyn_cast<Instruction>(U);
2741  if (!Inst)
2742  return false;
2743 
2744  if (Inst->getOpcode() == OpCode || isa<PHINode>(U)) {
2745  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2746  if (!isa<PHINode>(FPOp) && !FPOp->getFastMathFlags().isFast())
2747  return false;
2748  UsersToVisit.push_back(U);
2749  } else if (const ShuffleVectorInst *ShufInst =
2750  dyn_cast<ShuffleVectorInst>(U)) {
2751  // Detect the following pattern: A ShuffleVector instruction together
2752  // with a reduction that do partial reduction on the first and second
2753  // ElemNumToReduce / 2 elements, and store the result in
2754  // ElemNumToReduce / 2 elements in another vector.
2755 
2756  unsigned ResultElements = ShufInst->getType()->getVectorNumElements();
2757  if (ResultElements < ElemNum)
2758  return false;
2759 
2760  if (ElemNumToReduce == 1)
2761  return false;
2762  if (!isa<UndefValue>(U->getOperand(1)))
2763  return false;
2764  for (unsigned i = 0; i < ElemNumToReduce / 2; ++i)
2765  if (ShufInst->getMaskValue(i) != int(i + ElemNumToReduce / 2))
2766  return false;
2767  for (unsigned i = ElemNumToReduce / 2; i < ElemNum; ++i)
2768  if (ShufInst->getMaskValue(i) != -1)
2769  return false;
2770 
2771  // There is only one user of this ShuffleVector instruction, which
2772  // must be a reduction operation.
2773  if (!U->hasOneUse())
2774  return false;
2775 
2776  auto U2 = dyn_cast<Instruction>(*U->user_begin());
2777  if (!U2 || U2->getOpcode() != OpCode)
2778  return false;
2779 
2780  // Check operands of the reduction operation.
2781  if ((U2->getOperand(0) == U->getOperand(0) && U2->getOperand(1) == U) ||
2782  (U2->getOperand(1) == U->getOperand(0) && U2->getOperand(0) == U)) {
2783  UsersToVisit.push_back(U2);
2784  ElemNumToReduce /= 2;
2785  } else
2786  return false;
2787  } else if (isa<ExtractElementInst>(U)) {
2788  // At this moment we should have reduced all elements in the vector.
2789  if (ElemNumToReduce != 1)
2790  return false;
2791 
2792  const ConstantInt *Val = dyn_cast<ConstantInt>(U->getOperand(1));
2793  if (!Val || !Val->isZero())
2794  return false;
2795 
2796  ReduxExtracted = true;
2797  } else
2798  return false;
2799  }
2800  }
2801  return ReduxExtracted;
2802 }
2803 
2804 void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
2805  SDNodeFlags Flags;
2806  if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
2807  Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
2808  Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
2809  }
2810  if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I)) {
2811  Flags.setExact(ExactOp->isExact());
2812  }
2813  if (isVectorReductionOp(&I)) {
2814  Flags.setVectorReduction(true);
2815  LLVM_DEBUG(dbgs() << "Detected a reduction operation:" << I << "\n");
2816  }
2817 
2818  SDValue Op1 = getValue(I.getOperand(0));
2819  SDValue Op2 = getValue(I.getOperand(1));
2820  SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
2821  Op1, Op2, Flags);
2822  setValue(&I, BinNodeValue);
2823 }
2824 
2825 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
2826  SDValue Op1 = getValue(I.getOperand(0));
2827  SDValue Op2 = getValue(I.getOperand(1));
2828 
2829  EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
2830  Op1.getValueType(), DAG.getDataLayout());
2831 
2832  // Coerce the shift amount to the right type if we can.
2833  if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
2834  unsigned ShiftSize = ShiftTy.getSizeInBits();
2835  unsigned Op2Size = Op2.getValueSizeInBits();
2836  SDLoc DL = getCurSDLoc();
2837 
2838  // If the operand is smaller than the shift count type, promote it.
2839  if (ShiftSize > Op2Size)
2840  Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
2841 
2842  // If the operand is larger than the shift count type but the shift
2843  // count type has enough bits to represent any shift value, truncate
2844  // it now. This is a common case and it exposes the truncate to
2845  // optimization early.
2846  else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))
2847  Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
2848  // Otherwise we'll need to temporarily settle for some other convenient
2849  // type. Type legalization will make adjustments once the shiftee is split.
2850  else
2851  Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
2852  }
2853 
2854  bool nuw = false;
2855  bool nsw = false;
2856  bool exact = false;
2857 
2858  if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
2859 
2860  if (const OverflowingBinaryOperator *OFBinOp =
2861  dyn_cast<const OverflowingBinaryOperator>(&I)) {
2862  nuw = OFBinOp->hasNoUnsignedWrap();
2863  nsw = OFBinOp->hasNoSignedWrap();
2864  }
2865  if (const PossiblyExactOperator *ExactOp =
2866  dyn_cast<const PossiblyExactOperator>(&I))
2867  exact = ExactOp->isExact();
2868  }
2869  SDNodeFlags Flags;
2870  Flags.setExact(exact);
2871  Flags.setNoSignedWrap(nsw);
2872  Flags.setNoUnsignedWrap(nuw);
2873  SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
2874  Flags);
2875  setValue(&I, Res);
2876 }
2877 
2878 void SelectionDAGBuilder::visitSDiv(const User &I) {
2879  SDValue Op1 = getValue(I.getOperand(0));
2880  SDValue Op2 = getValue(I.getOperand(1));
2881 
2882  SDNodeFlags Flags;
2883  Flags.setExact(isa<PossiblyExactOperator>(&I) &&
2884  cast<PossiblyExactOperator>(&I)->isExact());
2885  setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
2886  Op2, Flags));
2887 }
2888 
2889 void SelectionDAGBuilder::visitICmp(const User &I) {
2891  if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
2892  predicate = IC->getPredicate();
2893  else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
2894  predicate = ICmpInst::Predicate(IC->getPredicate());
2895  SDValue Op1 = getValue(I.getOperand(0));
2896  SDValue Op2 = getValue(I.getOperand(1));
2897  ISD::CondCode Opcode = getICmpCondCode(predicate);
2898 
2899  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2900  I.getType());
2901  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
2902 }
2903 
2904 void SelectionDAGBuilder::visitFCmp(const User &I) {
2906  if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
2907  predicate = FC->getPredicate();
2908  else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
2909  predicate = FCmpInst::Predicate(FC->getPredicate());
2910  SDValue Op1 = getValue(I.getOperand(0));
2911  SDValue Op2 = getValue(I.getOperand(1));
2912 
2913  ISD::CondCode Condition = getFCmpCondCode(predicate);
2914  auto *FPMO = dyn_cast<FPMathOperator>(&I);
2915  if ((FPMO && FPMO->hasNoNaNs()) || TM.Options.NoNaNsFPMath)
2916  Condition = getFCmpCodeWithoutNaN(Condition);
2917 
2918  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2919  I.getType());
2920  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
2921 }
2922 
2923 // Check if the condition of the select has one use or two users that are both
2924 // selects with the same condition.
2925 static bool hasOnlySelectUsers(const Value *Cond) {
2926  return llvm::all_of(Cond->users(), [](const Value *V) {
2927  return isa<SelectInst>(V);
2928  });
2929 }
2930 
2931 void SelectionDAGBuilder::visitSelect(const User &I) {
2934  ValueVTs);
2935  unsigned NumValues = ValueVTs.size();
2936  if (NumValues == 0) return;
2937 
2938  SmallVector<SDValue, 4> Values(NumValues);
2939  SDValue Cond = getValue(I.getOperand(0));
2940  SDValue LHSVal = getValue(I.getOperand(1));
2941  SDValue RHSVal = getValue(I.getOperand(2));
2942  auto BaseOps = {Cond};
2943  ISD::NodeType OpCode = Cond.getValueType().isVector() ?
2945 
2946  // Min/max matching is only viable if all output VTs are the same.
2947  if (is_splat(ValueVTs)) {
2948  EVT VT = ValueVTs[0];
2949  LLVMContext &Ctx = *DAG.getContext();
2950  auto &TLI = DAG.getTargetLoweringInfo();
2951 
2952  // We care about the legality of the operation after it has been type
2953  // legalized.
2954  while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal &&
2955  VT != TLI.getTypeToTransformTo(Ctx, VT))
2956  VT = TLI.getTypeToTransformTo(Ctx, VT);
2957 
2958  // If the vselect is legal, assume we want to leave this as a vector setcc +
2959  // vselect. Otherwise, if this is going to be scalarized, we want to see if
2960  // min/max is legal on the scalar type.
2961  bool UseScalarMinMax = VT.isVector() &&
2962  !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
2963 
2964  Value *LHS, *RHS;
2965  auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
2967  switch (SPR.Flavor) {
2968  case SPF_UMAX: Opc = ISD::UMAX; break;
2969  case SPF_UMIN: Opc = ISD::UMIN; break;
2970  case SPF_SMAX: Opc = ISD::SMAX; break;
2971  case SPF_SMIN: Opc = ISD::SMIN; break;
2972  case SPF_FMINNUM:
2973  switch (SPR.NaNBehavior) {
2974  case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
2975  case SPNB_RETURNS_NAN: Opc = ISD::FMINNAN; break;
2976  case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
2977  case SPNB_RETURNS_ANY: {
2978  if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
2979  Opc = ISD::FMINNUM;
2980  else if (TLI.isOperationLegalOrCustom(ISD::FMINNAN, VT))
2981  Opc = ISD::FMINNAN;
2982  else if (UseScalarMinMax)
2983  Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
2985  break;
2986  }
2987  }
2988  break;
2989  case SPF_FMAXNUM:
2990  switch (SPR.NaNBehavior) {
2991  case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
2992  case SPNB_RETURNS_NAN: Opc = ISD::FMAXNAN; break;
2993  case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
2994  case SPNB_RETURNS_ANY:
2995 
2996  if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
2997  Opc = ISD::FMAXNUM;
2998  else if (TLI.isOperationLegalOrCustom(ISD::FMAXNAN, VT))
2999  Opc = ISD::FMAXNAN;
3000  else if (UseScalarMinMax)
3001  Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
3003  break;
3004  }
3005  break;
3006  default: break;
3007  }
3008 
3009  if (Opc != ISD::DELETED_NODE &&
3010  (TLI.isOperationLegalOrCustom(Opc, VT) ||
3011  (UseScalarMinMax &&
3012  TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3013  // If the underlying comparison instruction is used by any other
3014  // instruction, the consumed instructions won't be destroyed, so it is
3015  // not profitable to convert to a min/max.
3016  hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3017  OpCode = Opc;
3018  LHSVal = getValue(LHS);
3019  RHSVal = getValue(RHS);
3020  BaseOps = {};
3021  }
3022  }
3023 
3024  for (unsigned i = 0; i != NumValues; ++i) {
3025  SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3026  Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3027  Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3028  Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
3029  LHSVal.getNode()->getValueType(LHSVal.getResNo()+i),
3030  Ops);
3031  }
3032 
3033  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3034  DAG.getVTList(ValueVTs), Values));
3035 }
3036 
3037 void SelectionDAGBuilder::visitTrunc(const User &I) {
3038  // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3039  SDValue N = getValue(I.getOperand(0));
3040  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3041  I.getType());
3042  setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
3043 }
3044 
3045 void SelectionDAGBuilder::visitZExt(const User &I) {
3046  // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3047  // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3048  SDValue N = getValue(I.getOperand(0));
3049  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3050  I.getType());
3051  setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
3052 }
3053 
3054 void SelectionDAGBuilder::visitSExt(const User &I) {
3055  // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3056  // SExt also can't be a cast to bool for same reason. So, nothing much to do
3057  SDValue N = getValue(I.getOperand(0));
3058  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3059  I.getType());
3060  setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
3061 }
3062 
3063 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3064  // FPTrunc is never a no-op cast, no need to check
3065  SDValue N = getValue(I.getOperand(0));
3066  SDLoc dl = getCurSDLoc();
3067  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3068  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3069  setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3070  DAG.getTargetConstant(
3071  0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3072 }
3073 
3074 void SelectionDAGBuilder::visitFPExt(const User &I) {
3075  // FPExt is never a no-op cast, no need to check
3076  SDValue N = getValue(I.getOperand(0));
3077  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3078  I.getType());
3079  setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
3080 }
3081 
3082 void SelectionDAGBuilder::visitFPToUI(const User &I) {
3083  // FPToUI is never a no-op cast, no need to check
3084  SDValue N = getValue(I.getOperand(0));
3085  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3086  I.getType());
3087  setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
3088 }
3089 
3090 void SelectionDAGBuilder::visitFPToSI(const User &I) {
3091  // FPToSI is never a no-op cast, no need to check
3092  SDValue N = getValue(I.getOperand(0));
3093  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3094  I.getType());
3095  setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
3096 }
3097 
3098 void SelectionDAGBuilder::visitUIToFP(const User &I) {
3099  // UIToFP is never a no-op cast, no need to check
3100  SDValue N = getValue(I.getOperand(0));
3101  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3102  I.getType());
3103  setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
3104 }
3105 
3106 void SelectionDAGBuilder::visitSIToFP(const User &I) {
3107  // SIToFP is never a no-op cast, no need to check
3108  SDValue N = getValue(I.getOperand(0));
3109  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3110  I.getType());
3111  setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
3112 }
3113 
3114 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3115  // What to do depends on the size of the integer and the size of the pointer.
3116  // We can either truncate, zero extend, or no-op, accordingly.
3117  SDValue N = getValue(I.getOperand(0));
3118  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3119  I.getType());
3120  setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
3121 }
3122 
3123 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3124  // What to do depends on the size of the integer and the size of the pointer.
3125  // We can either truncate, zero extend, or no-op, accordingly.
3126  SDValue N = getValue(I.getOperand(0));
3127  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3128  I.getType());
3129  setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
3130 }
3131 
3132 void SelectionDAGBuilder::visitBitCast(const User &I) {
3133  SDValue N = getValue(I.getOperand(0));
3134  SDLoc dl = getCurSDLoc();
3135  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3136  I.getType());
3137 
3138  // BitCast assures us that source and destination are the same size so this is
3139  // either a BITCAST or a no-op.
3140  if (DestVT != N.getValueType())
3141  setValue(&I, DAG.getNode(ISD::BITCAST, dl,
3142  DestVT, N)); // convert types.
3143  // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3144  // might fold any kind of constant expression to an integer constant and that
3145  // is not what we are looking for. Only recognize a bitcast of a genuine
3146  // constant integer as an opaque constant.
3147  else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3148  setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3149  /*isOpaque*/true));
3150  else
3151  setValue(&I, N); // noop cast.
3152 }
3153 
3154 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3155  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3156  const Value *SV = I.getOperand(0);
3157  SDValue N = getValue(SV);
3158  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3159 
3160  unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3161  unsigned DestAS = I.getType()->getPointerAddressSpace();
3162 
3163  if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
3164  N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3165 
3166  setValue(&I, N);
3167 }
3168 
3169 void SelectionDAGBuilder::visitInsertElement(const User &I) {
3170  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3171  SDValue InVec = getValue(I.getOperand(0));
3172  SDValue InVal = getValue(I.getOperand(1));
3173  SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3174  TLI.getVectorIdxTy(DAG.getDataLayout()));
3175  setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
3176  TLI.getValueType(DAG.getDataLayout(), I.getType()),
3177  InVec, InVal, InIdx));
3178 }
3179 
3180 void SelectionDAGBuilder::visitExtractElement(const User &I) {
3181  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3182  SDValue InVec = getValue(I.getOperand(0));
3183  SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3184  TLI.getVectorIdxTy(DAG.getDataLayout()));
3185  setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
3186  TLI.getValueType(DAG.getDataLayout(), I.getType()),
3187  InVec, InIdx));
3188 }
3189 
3190 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3191  SDValue Src1 = getValue(I.getOperand(0));
3192  SDValue Src2 = getValue(I.getOperand(1));
3193  SDLoc DL = getCurSDLoc();
3194 
3196  ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
3197  unsigned MaskNumElts = Mask.size();
3198 
3199  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3200  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3201  EVT SrcVT = Src1.getValueType();
3202  unsigned SrcNumElts = SrcVT.getVectorNumElements();
3203 
3204  if (SrcNumElts == MaskNumElts) {
3205  setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3206  return;
3207  }
3208 
3209  // Normalize the shuffle vector since mask and vector length don't match.
3210  if (SrcNumElts < MaskNumElts) {
3211  // Mask is longer than the source vectors. We can use concatenate vector to
3212  // make the mask and vectors lengths match.
3213 
3214  if (MaskNumElts % SrcNumElts == 0) {
3215  // Mask length is a multiple of the source vector length.
3216  // Check if the shuffle is some kind of concatenation of the input
3217  // vectors.
3218  unsigned NumConcat = MaskNumElts / SrcNumElts;
3219  bool IsConcat = true;
3220  SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3221  for (unsigned i = 0; i != MaskNumElts; ++i) {
3222  int Idx = Mask[i];
3223  if (Idx < 0)
3224  continue;
3225  // Ensure the indices in each SrcVT sized piece are sequential and that
3226  // the same source is used for the whole piece.
3227  if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3228  (ConcatSrcs[i / SrcNumElts] >= 0 &&
3229  ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3230  IsConcat = false;
3231  break;
3232  }
3233  // Remember which source this index came from.
3234  ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3235  }
3236 
3237  // The shuffle is concatenating multiple vectors together. Just emit
3238  // a CONCAT_VECTORS operation.
3239  if (IsConcat) {
3240  SmallVector<SDValue, 8> ConcatOps;
3241  for (auto Src : ConcatSrcs) {
3242  if (Src < 0)
3243  ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3244  else if (Src == 0)
3245  ConcatOps.push_back(Src1);
3246  else
3247  ConcatOps.push_back(Src2);
3248  }
3249  setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3250  return;
3251  }
3252  }
3253 
3254  unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3255  unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3256  EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3257  PaddedMaskNumElts);
3258 
3259  // Pad both vectors with undefs to make them the same length as the mask.
3260  SDValue UndefVal = DAG.getUNDEF(SrcVT);
3261 
3262  SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3263  SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3264  MOps1[0] = Src1;
3265  MOps2[0] = Src2;
3266 
3267  Src1 = Src1.isUndef()
3268  ? DAG.getUNDEF(PaddedVT)
3269  : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3270  Src2 = Src2.isUndef()
3271  ? DAG.getUNDEF(PaddedVT)
3272  : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3273 
3274  // Readjust mask for new input vector length.
3275  SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3276  for (unsigned i = 0; i != MaskNumElts; ++i) {
3277  int Idx = Mask[i];
3278  if (Idx >= (int)SrcNumElts)
3279  Idx -= SrcNumElts - PaddedMaskNumElts;
3280  MappedOps[i] = Idx;
3281  }
3282 
3283  SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3284 
3285  // If the concatenated vector was padded, extract a subvector with the
3286  // correct number of elements.
3287  if (MaskNumElts != PaddedMaskNumElts)
3288  Result = DAG.getNode(
3289  ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3290  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3291 
3292  setValue(&I, Result);
3293  return;
3294  }
3295 
3296  if (SrcNumElts > MaskNumElts) {
3297  // Analyze the access pattern of the vector to see if we can extract
3298  // two subvectors and do the shuffle.
3299  int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
3300  bool CanExtract = true;
3301  for (int Idx : Mask) {
3302  unsigned Input = 0;
3303  if (Idx < 0)
3304  continue;
3305 
3306  if (Idx >= (int)SrcNumElts) {
3307  Input = 1;
3308  Idx -= SrcNumElts;
3309  }
3310 
3311  // If all the indices come from the same MaskNumElts sized portion of
3312  // the sources we can use extract. Also make sure the extract wouldn't
3313  // extract past the end of the source.
3314  int NewStartIdx = alignDown(Idx, MaskNumElts);
3315  if (NewStartIdx + MaskNumElts > SrcNumElts ||
3316  (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3317  CanExtract = false;
3318  // Make sure we always update StartIdx as we use it to track if all
3319  // elements are undef.
3320  StartIdx[Input] = NewStartIdx;
3321  }
3322 
3323  if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3324  setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3325  return;
3326  }
3327  if (CanExtract) {
3328  // Extract appropriate subvector and generate a vector shuffle
3329  for (unsigned Input = 0; Input < 2; ++Input) {
3330  SDValue &Src = Input == 0 ? Src1 : Src2;
3331  if (StartIdx[Input] < 0)
3332  Src = DAG.getUNDEF(VT);
3333  else {
3334  Src = DAG.getNode(
3335  ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3336  DAG.getConstant(StartIdx[Input], DL,
3337  TLI.getVectorIdxTy(DAG.getDataLayout())));
3338  }
3339  }
3340 
3341  // Calculate new mask.
3342  SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());
3343  for (int &Idx : MappedOps) {
3344  if (Idx >= (int)SrcNumElts)
3345  Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3346  else if (Idx >= 0)
3347  Idx -= StartIdx[0];
3348  }
3349 
3350  setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3351  return;
3352  }
3353  }
3354 
3355  // We can't use either concat vectors or extract subvectors so fall back to
3356  // replacing the shuffle with extract and build vector.
3357  // to insert and build vector.
3358  EVT EltVT = VT.getVectorElementType();
3359  EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
3361  for (int Idx : Mask) {
3362  SDValue Res;
3363 
3364  if (Idx < 0) {
3365  Res = DAG.getUNDEF(EltVT);
3366  } else {
3367  SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3368  if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3369 
3370  Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
3371  EltVT, Src, DAG.getConstant(Idx, DL, IdxVT));
3372  }
3373 
3374  Ops.push_back(Res);
3375  }
3376 
3377  setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3378 }
3379 
3380 void SelectionDAGBuilder::visitInsertValue(const User &I) {
3381  ArrayRef<unsigned> Indices;
3382  if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I))
3383  Indices = IV->getIndices();
3384  else
3385  Indices = cast<ConstantExpr>(&I)->getIndices();
3386 
3387  const Value *Op0 = I.getOperand(0);
3388  const Value *Op1 = I.getOperand(1);
3389  Type *AggTy = I.getType();
3390  Type *ValTy = Op1->getType();
3391  bool IntoUndef = isa<UndefValue>(Op0);
3392  bool FromUndef = isa<UndefValue>(Op1);
3393 
3394  unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3395 
3396  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3397  SmallVector<EVT, 4> AggValueVTs;
3398  ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3399  SmallVector<EVT, 4> ValValueVTs;
3400  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3401 
3402  unsigned NumAggValues = AggValueVTs.size();
3403  unsigned NumValValues = ValValueVTs.size();
3404  SmallVector<SDValue, 4> Values(NumAggValues);
3405 
3406  // Ignore an insertvalue that produces an empty object
3407  if (!NumAggValues) {
3408  setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3409  return;
3410  }
3411 
3412  SDValue Agg = getValue(Op0);
3413  unsigned i = 0;
3414  // Copy the beginning value(s) from the original aggregate.
3415  for (; i != LinearIndex; ++i)
3416  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3417  SDValue(Agg.getNode(), Agg.getResNo() + i);
3418  // Copy values from the inserted value(s).
3419  if (NumValValues) {
3420  SDValue Val = getValue(Op1);
3421  for (; i != LinearIndex + NumValValues; ++i)
3422  Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3423  SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3424  }
3425  // Copy remaining value(s) from the original aggregate.
3426  for (; i != NumAggValues; ++i)
3427  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3428  SDValue(Agg.getNode(), Agg.getResNo() + i);
3429 
3430  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3431  DAG.getVTList(AggValueVTs), Values));
3432 }
3433 
3434 void SelectionDAGBuilder::visitExtractValue(const User &I) {
3435  ArrayRef<unsigned> Indices;
3436  if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I))
3437  Indices = EV->getIndices();
3438  else
3439  Indices = cast<ConstantExpr>(&I)->getIndices();
3440 
3441  const Value *Op0 = I.getOperand(0);
3442  Type *AggTy = Op0->getType();
3443  Type *ValTy = I.getType();
3444  bool OutOfUndef = isa<UndefValue>(Op0);
3445 
3446  unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3447 
3448  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3449  SmallVector<EVT, 4> ValValueVTs;
3450  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3451 
3452  unsigned NumValValues = ValValueVTs.size();
3453 
3454  // Ignore a extractvalue that produces an empty object
3455  if (!NumValValues) {
3456  setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3457  return;
3458  }
3459 
3460  SmallVector<SDValue, 4> Values(NumValValues);
3461 
3462  SDValue Agg = getValue(Op0);
3463  // Copy out the selected value(s).
3464  for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3465  Values[i - LinearIndex] =
3466  OutOfUndef ?
3467  DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3468  SDValue(Agg.getNode(), Agg.getResNo() + i);
3469 
3470  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3471  DAG.getVTList(ValValueVTs), Values));
3472 }
3473 
3474 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3475  Value *Op0 = I.getOperand(0);
3476  // Note that the pointer operand may be a vector of pointers. Take the scalar
3477  // element which holds a pointer.
3478  unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3479  SDValue N = getValue(Op0);
3480  SDLoc dl = getCurSDLoc();
3481 
3482  // Normalize Vector GEP - all scalar operands should be converted to the
3483  // splat vector.
3484  unsigned VectorWidth = I.getType()->isVectorTy() ?
3485  cast<VectorType>(I.getType())->getVectorNumElements() : 0;
3486 
3487  if (VectorWidth && !N.getValueType().isVector()) {
3488  LLVMContext &Context = *DAG.getContext();
3489  EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorWidth);
3490  N = DAG.getSplatBuildVector(VT, dl, N);
3491  }
3492 
3493  for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3494  GTI != E; ++GTI) {
3495  const Value *Idx = GTI.getOperand();
3496  if (StructType *StTy = GTI.getStructTypeOrNull()) {
3497  unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3498  if (Field) {
3499  // N = N + Offset
3500  uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
3501 
3502  // In an inbounds GEP with an offset that is nonnegative even when
3503  // interpreted as signed, assume there is no unsigned overflow.
3504  SDNodeFlags Flags;
3505  if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3506  Flags.setNoUnsignedWrap(true);
3507 
3508  N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3509  DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3510  }
3511  } else {
3512  unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
3513  MVT IdxTy = MVT::getIntegerVT(IdxSize);
3514  APInt ElementSize(IdxSize, DL->getTypeAllocSize(GTI.getIndexedType()));
3515 
3516  // If this is a scalar constant or a splat vector of constants,
3517  // handle it quickly.
3518  const auto *CI = dyn_cast<ConstantInt>(Idx);
3519  if (!CI && isa<ConstantDataVector>(Idx) &&
3520  cast<ConstantDataVector>(Idx)->getSplatValue())
3521  CI = cast<ConstantInt>(cast<ConstantDataVector>(Idx)->getSplatValue());
3522 
3523  if (CI) {
3524  if (CI->isZero())
3525  continue;
3526  APInt Offs = ElementSize * CI->getValue().sextOrTrunc(IdxSize);
3527  LLVMContext &Context = *DAG.getContext();
3528  SDValue OffsVal = VectorWidth ?
3529  DAG.getConstant(Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorWidth)) :
3530  DAG.getConstant(Offs, dl, IdxTy);
3531 
3532  // In an inbouds GEP with an offset that is nonnegative even when
3533  // interpreted as signed, assume there is no unsigned overflow.
3534  SDNodeFlags Flags;
3535  if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3536  Flags.setNoUnsignedWrap(true);
3537 
3538  N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3539  continue;
3540  }
3541 
3542  // N = N + Idx * ElementSize;
3543  SDValue IdxN = getValue(Idx);
3544 
3545  if (!IdxN.getValueType().isVector() && VectorWidth) {
3546  EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(), VectorWidth);
3547  IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3548  }
3549 
3550  // If the index is smaller or larger than intptr_t, truncate or extend
3551  // it.
3552  IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3553 
3554  // If this is a multiply by a power of two, turn it into a shl
3555  // immediately. This is a very common case.
3556  if (ElementSize != 1) {
3557  if (ElementSize.isPowerOf2()) {
3558  unsigned Amt = ElementSize.logBase2();
3559  IdxN = DAG.getNode(ISD::SHL, dl,
3560  N.getValueType(), IdxN,
3561  DAG.getConstant(Amt, dl, IdxN.getValueType()));
3562  } else {
3563  SDValue Scale = DAG.getConstant(ElementSize, dl, IdxN.getValueType());
3564  IdxN = DAG.getNode(ISD::MUL, dl,
3565  N.getValueType(), IdxN, Scale);
3566  }
3567  }
3568 
3569  N = DAG.getNode(ISD::ADD, dl,
3570  N.getValueType(), N, IdxN);
3571  }
3572  }
3573 
3574  setValue(&I, N);
3575 }
3576 
3577 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3578  // If this is a fixed sized alloca in the entry block of the function,
3579  // allocate it statically on the stack.
3580  if (FuncInfo.StaticAllocaMap.count(&I))
3581  return; // getValue will auto-populate this.
3582 
3583  SDLoc dl = getCurSDLoc();
3584  Type *Ty = I.getAllocatedType();
3585  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3586  auto &DL = DAG.getDataLayout();
3587  uint64_t TySize = DL.getTypeAllocSize(Ty);
3588  unsigned Align =
3589  std::max((unsigned)DL.getPrefTypeAlignment(Ty), I.getAlignment());
3590 
3591  SDValue AllocSize = getValue(I.getArraySize());
3592 
3593  EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), DL.getAllocaAddrSpace());
3594  if (AllocSize.getValueType() != IntPtr)
3595  AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
3596 
3597  AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
3598  AllocSize,
3599  DAG.getConstant(TySize, dl, IntPtr));
3600 
3601  // Handle alignment. If the requested alignment is less than or equal to
3602  // the stack alignment, ignore it. If the size is greater than or equal to
3603  // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
3604  unsigned StackAlign =
3606  if (Align <= StackAlign)
3607  Align = 0;
3608 
3609  // Round the size of the allocation up to the stack alignment size
3610  // by add SA-1 to the size. This doesn't overflow because we're computing
3611  // an address inside an alloca.
3612  SDNodeFlags Flags;
3613  Flags.setNoUnsignedWrap(true);
3614  AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
3615  DAG.getConstant(StackAlign - 1, dl, IntPtr), Flags);
3616 
3617  // Mask out the low bits for alignment purposes.
3618  AllocSize =
3619  DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
3620  DAG.getConstant(~(uint64_t)(StackAlign - 1), dl, IntPtr));
3621 
3622  SDValue Ops[] = {getRoot(), AllocSize, DAG.getConstant(Align, dl, IntPtr)};
3623  SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
3624  SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
3625  setValue(&I, DSA);
3626  DAG.setRoot(DSA.getValue(1));
3627 
3628  assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
3629 }
3630 
3631 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
3632  if (I.isAtomic())
3633  return visitAtomicLoad(I);
3634 
3635  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3636  const Value *SV = I.getOperand(0);
3637  if (TLI.supportSwiftError()) {
3638  // Swifterror values can come from either a function parameter with
3639  // swifterror attribute or an alloca with swifterror attribute.
3640  if (const Argument *Arg = dyn_cast<Argument>(SV)) {
3641  if (Arg->hasSwiftErrorAttr())
3642  return visitLoadFromSwiftError(I);
3643  }
3644 
3645  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
3646  if (Alloca->isSwiftError())
3647  return visitLoadFromSwiftError(I);
3648  }
3649  }
3650 
3651  SDValue Ptr = getValue(SV);
3652 
3653  Type *Ty = I.getType();
3654 
3655  bool isVolatile = I.isVolatile();
3656  bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
3657  bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
3658  bool isDereferenceable = isDereferenceablePointer(SV, DAG.getDataLayout());
3659  unsigned Alignment = I.getAlignment();
3660 
3661  AAMDNodes AAInfo;
3662  I.getAAMetadata(AAInfo);
3663  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3664 
3667  ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &Offsets);
3668  unsigned NumValues = ValueVTs.size();
3669  if (NumValues == 0)
3670  return;
3671 
3672  SDValue Root;
3673  bool ConstantMemory = false;
3674  if (isVolatile || NumValues > MaxParallelChains)
3675  // Serialize volatile loads with other side effects.
3676  Root = getRoot();
3677  else if (AA && AA->pointsToConstantMemory(MemoryLocation(
3678  SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) {
3679  // Do not serialize (non-volatile) loads of constant memory with anything.
3680  Root = DAG.getEntryNode();
3681  ConstantMemory = true;
3682  } else {
3683  // Do not serialize non-volatile loads against each other.
3684  Root = DAG.getRoot();
3685  }
3686 
3687  SDLoc dl = getCurSDLoc();
3688 
3689  if (isVolatile)
3690  Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
3691 
3692  // An aggregate load cannot wrap around the address space, so offsets to its
3693  // parts don't wrap either.
3694  SDNodeFlags Flags;
3695  Flags.setNoUnsignedWrap(true);
3696 
3697  SmallVector<SDValue, 4> Values(NumValues);
3698  SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3699  EVT PtrVT = Ptr.getValueType();
3700  unsigned ChainI = 0;
3701  for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3702  // Serializing loads here may result in excessive register pressure, and
3703  // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
3704  // could recover a bit by hoisting nodes upward in the chain by recognizing
3705  // they are side-effect free or do not alias. The optimizer should really
3706  // avoid this case by converting large object/array copies to llvm.memcpy
3707  // (MaxParallelChains should always remain as failsafe).
3708  if (ChainI == MaxParallelChains) {
3709  assert(PendingLoads.empty() && "PendingLoads must be serialized first");
3710  SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3711  makeArrayRef(Chains.data(), ChainI));
3712  Root = Chain;
3713  ChainI = 0;
3714  }
3715  SDValue A = DAG.getNode(ISD::ADD, dl,
3716  PtrVT, Ptr,
3717  DAG.getConstant(Offsets[i], dl, PtrVT),
3718  Flags);
3719  auto MMOFlags = MachineMemOperand::MONone;
3720  if (isVolatile)
3721  MMOFlags |= MachineMemOperand::MOVolatile;
3722  if (isNonTemporal)
3724  if (isInvariant)
3725  MMOFlags |= MachineMemOperand::MOInvariant;
3726  if (isDereferenceable)
3728  MMOFlags |= TLI.getMMOFlags(I);
3729 
3730  SDValue L = DAG.getLoad(ValueVTs[i], dl, Root, A,
3731  MachinePointerInfo(SV, Offsets[i]), Alignment,
3732  MMOFlags, AAInfo, Ranges);
3733 
3734  Values[i] = L;
3735  Chains[ChainI] = L.getValue(1);
3736  }
3737 
3738  if (!ConstantMemory) {
3739  SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3740  makeArrayRef(Chains.data(), ChainI));
3741  if (isVolatile)
3742  DAG.setRoot(Chain);
3743  else
3744  PendingLoads.push_back(Chain);
3745  }
3746 
3747  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
3748  DAG.getVTList(ValueVTs), Values));
3749 }
3750 
3751 void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
3753  "call visitStoreToSwiftError when backend supports swifterror");
3754 
3757  const Value *SrcV = I.getOperand(0);
3759  SrcV->getType(), ValueVTs, &Offsets);
3760  assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
3761  "expect a single EVT for swifterror");
3762 
3763  SDValue Src = getValue(SrcV);
3764  // Create a virtual register, then update the virtual register.
3765  unsigned VReg; bool CreatedVReg;
3766  std::tie(VReg, CreatedVReg) = FuncInfo.getOrCreateSwiftErrorVRegDefAt(&I);
3767  // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
3768  // Chain can be getRoot or getControlRoot.
3769  SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
3770  SDValue(Src.getNode(), Src.getResNo()));
3771  DAG.setRoot(CopyNode);
3772  if (CreatedVReg)
3773  FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, I.getOperand(1), VReg);
3774 }
3775 
3776 void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
3778  "call visitLoadFromSwiftError when backend supports swifterror");
3779 
3780  assert(!I.isVolatile() &&
3781  I.getMetadata(LLVMContext::MD_nontemporal) == nullptr &&
3783  "Support volatile, non temporal, invariant for load_from_swift_error");
3784 
3785  const Value *SV = I.getOperand(0);
3786  Type *Ty = I.getType();
3787  AAMDNodes AAInfo;
3788  I.getAAMetadata(AAInfo);
3789  assert((!AA || !AA->pointsToConstantMemory(MemoryLocation(
3790  SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) &&
3791  "load_from_swift_error should not be constant memory");
3792 
3796  ValueVTs, &Offsets);
3797  assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
3798  "expect a single EVT for swifterror");
3799 
3800  // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
3801  SDValue L = DAG.getCopyFromReg(
3802  getRoot(), getCurSDLoc(),
3803  FuncInfo.getOrCreateSwiftErrorVRegUseAt(&I, FuncInfo.MBB, SV).first,
3804  ValueVTs[0]);
3805 
3806  setValue(&I, L);
3807 }
3808 
3809 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
3810  if (I.isAtomic())
3811  return visitAtomicStore(I);
3812 
3813  const Value *SrcV = I.getOperand(0);
3814  const Value *PtrV = I.getOperand(1);
3815 
3816  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3817  if (TLI.supportSwiftError()) {
3818  // Swifterror values can come from either a function parameter with
3819  // swifterror attribute or an alloca with swifterror attribute.
3820  if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
3821  if (Arg->hasSwiftErrorAttr())
3822  return visitStoreToSwiftError(I);
3823  }
3824 
3825  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
3826  if (Alloca->isSwiftError())
3827  return visitStoreToSwiftError(I);
3828  }
3829  }
3830 
3834  SrcV->getType(), ValueVTs, &Offsets);
3835  unsigned NumValues = ValueVTs.size();
3836  if (NumValues == 0)
3837  return;
3838 
3839  // Get the lowered operands. Note that we do this after
3840  // checking if NumResults is zero, because with zero results
3841  // the operands won't have values in the map.
3842  SDValue Src = getValue(SrcV);
3843  SDValue Ptr = getValue(PtrV);
3844 
3845  SDValue Root = getRoot();
3846  SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3847  SDLoc dl = getCurSDLoc();
3848  EVT PtrVT = Ptr.getValueType();
3849  unsigned Alignment = I.getAlignment();
3850  AAMDNodes AAInfo;
3851  I.getAAMetadata(AAInfo);
3852 
3853  auto MMOFlags = MachineMemOperand::MONone;
3854  if (I.isVolatile())
3855  MMOFlags |= MachineMemOperand::MOVolatile;
3856  if (I.getMetadata(LLVMContext::MD_nontemporal) != nullptr)
3858  MMOFlags |= TLI.getMMOFlags(I);
3859 
3860  // An aggregate load cannot wrap around the address space, so offsets to its
3861  // parts don't wrap either.
3862  SDNodeFlags Flags;
3863  Flags.setNoUnsignedWrap(true);
3864 
3865  unsigned ChainI = 0;
3866  for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3867  // See visitLoad comments.
3868  if (ChainI == MaxParallelChains) {
3869  SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3870  makeArrayRef(Chains.data(), ChainI));
3871  Root = Chain;
3872  ChainI = 0;
3873  }
3874  SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
3875  DAG.getConstant(Offsets[i], dl, PtrVT), Flags);
3876  SDValue St = DAG.getStore(
3877  Root, dl, SDValue(Src.getNode(), Src.getResNo() + i), Add,
3878  MachinePointerInfo(PtrV, Offsets[i]), Alignment, MMOFlags, AAInfo);
3879  Chains[ChainI] = St;
3880  }
3881 
3882  SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3883  makeArrayRef(Chains.data(), ChainI));
3884  DAG.setRoot(StoreNode);
3885 }
3886 
3887 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
3888  bool IsCompressing) {
3889  SDLoc sdl = getCurSDLoc();
3890 
3891  auto getMaskedStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3892  unsigned& Alignment) {
3893  // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
3894  Src0 = I.getArgOperand(0);
3895  Ptr = I.getArgOperand(1);
3896  Alignment = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
3897  Mask = I.getArgOperand(3);
3898  };
3899  auto getCompressingStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3900  unsigned& Alignment) {
3901  // llvm.masked.compressstore.*(Src0, Ptr, Mask)
3902  Src0 = I.getArgOperand(0);
3903  Ptr = I.getArgOperand(1);
3904  Mask = I.getArgOperand(2);
3905  Alignment = 0;
3906  };
3907 
3908  Value *PtrOperand, *MaskOperand, *Src0Operand;
3909  unsigned Alignment;
3910  if (IsCompressing)
3911  getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3912  else
3913  getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3914 
3915  SDValue Ptr = getValue(PtrOperand);
3916  SDValue Src0 = getValue(Src0Operand);
3917  SDValue Mask = getValue(MaskOperand);
3918 
3919  EVT VT = Src0.getValueType();
3920  if (!Alignment)
3921  Alignment = DAG.getEVTAlignment(VT);
3922 
3923  AAMDNodes AAInfo;
3924  I.getAAMetadata(AAInfo);
3925 
3926  MachineMemOperand *MMO =
3927  DAG.getMachineFunction().
3928  getMachineMemOperand(MachinePointerInfo(PtrOperand),
3930  Alignment, AAInfo);
3931  SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
3932  MMO, false /* Truncating */,
3933  IsCompressing);
3934  DAG.setRoot(StoreNode);
3935  setValue(&I, StoreNode);
3936 }
3937 
3938 // Get a uniform base for the Gather/Scatter intrinsic.
3939 // The first argument of the Gather/Scatter intrinsic is a vector of pointers.
3940 // We try to represent it as a base pointer + vector of indices.
3941 // Usually, the vector of pointers comes from a 'getelementptr' instruction.
3942 // The first operand of the GEP may be a single pointer or a vector of pointers
3943 // Example:
3944 // %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
3945 // or
3946 // %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
3947 // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
3948 //
3949 // When the first GEP operand is a single pointer - it is the uniform base we
3950 // are looking for. If first operand of the GEP is a splat vector - we
3951 // extract the splat value and use it as a uniform base.
3952 // In all other cases the function returns 'false'.
3953 static bool getUniformBase(const Value* &Ptr, SDValue& Base, SDValue& Index,
3954  SDValue &Scale, SelectionDAGBuilder* SDB) {
3955  SelectionDAG& DAG = SDB->DAG;
3956  LLVMContext &Context = *DAG.getContext();
3957 
3958  assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type");
3960  if (!GEP)
3961  return false;
3962 
3963  const Value *GEPPtr = GEP->getPointerOperand();
3964  if (!GEPPtr->getType()->isVectorTy())
3965  Ptr = GEPPtr;
3966  else if (!(Ptr = getSplatValue(GEPPtr)))
3967  return false;
3968 
3969  unsigned FinalIndex = GEP->getNumOperands() - 1;
3970  Value *IndexVal = GEP->getOperand(FinalIndex);
3971 
3972  // Ensure all the other indices are 0.
3973  for (unsigned i = 1; i < FinalIndex; ++i) {
3974  auto *C = dyn_cast<ConstantInt>(GEP->getOperand(i));
3975  if (!C || !C->isZero())
3976  return false;
3977  }
3978 
3979  // The operands of the GEP may be defined in another basic block.
3980  // In this case we'll not find nodes for the operands.
3981  if (!SDB->findValue(Ptr) || !SDB->findValue(IndexVal))
3982  return false;
3983 
3984  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3985  const DataLayout &DL = DAG.getDataLayout();
3986  Scale = DAG.getTargetConstant(DL.getTypeAllocSize(GEP->getResultElementType()),
3987  SDB->getCurSDLoc(), TLI.getPointerTy(DL));
3988  Base = SDB->getValue(Ptr);
3989  Index = SDB->getValue(IndexVal);
3990 
3991  if (!Index.getValueType().isVector()) {
3992  unsigned GEPWidth = GEP->getType()->getVectorNumElements();
3993  EVT VT = EVT::getVectorVT(Context, Index.getValueType(), GEPWidth);
3994  Index = DAG.getSplatBuildVector(VT, SDLoc(Index), Index);
3995  }
3996  return true;
3997 }
3998 
3999 void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4000  SDLoc sdl = getCurSDLoc();
4001 
4002  // llvm.masked.scatter.*(Src0, Ptrs, alignemt, Mask)
4003  const Value *Ptr = I.getArgOperand(1);
4004  SDValue Src0 = getValue(I.getArgOperand(0));
4005  SDValue Mask = getValue(I.getArgOperand(3));
4006  EVT VT = Src0.getValueType();
4007  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
4008  if (!Alignment)
4009  Alignment = DAG.getEVTAlignment(VT);
4010  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4011 
4012  AAMDNodes AAInfo;
4013  I.getAAMetadata(AAInfo);
4014 
4015  SDValue Base;
4016  SDValue Index;
4017  SDValue Scale;
4018  const Value *BasePtr = Ptr;
4019  bool UniformBase = getUniformBase(BasePtr, Base, Index, Scale, this);
4020 
4021  const Value *MemOpBasePtr = UniformBase ? BasePtr : nullptr;
4023  getMachineMemOperand(MachinePointerInfo(MemOpBasePtr),
4024  MachineMemOperand::MOStore, VT.getStoreSize(),
4025  Alignment, AAInfo);
4026  if (!UniformBase) {
4027  Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4028  Index = getValue(Ptr);
4029  Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4030  }
4031  SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index, Scale };
4032  SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4033  Ops, MMO);
4034  DAG.setRoot(Scatter);
4035  setValue(&I, Scatter);
4036 }
4037 
4038 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4039  SDLoc sdl = getCurSDLoc();
4040 
4041  auto getMaskedLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4042  unsigned& Alignment) {
4043  // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4044  Ptr = I.getArgOperand(0);
4045  Alignment = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
4046  Mask = I.getArgOperand(2);
4047  Src0 = I.getArgOperand(3);
4048  };
4049  auto getExpandingLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4050  unsigned& Alignment) {
4051  // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4052  Ptr = I.getArgOperand(0);
4053  Alignment = 0;
4054  Mask = I.getArgOperand(1);
4055  Src0 = I.getArgOperand(2);
4056  };
4057 
4058  Value *PtrOperand, *MaskOperand, *Src0Operand;
4059  unsigned Alignment;
4060  if (IsExpanding)
4061  getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4062  else
4063  getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4064 
4065  SDValue Ptr = getValue(PtrOperand);
4066  SDValue Src0 = getValue(Src0Operand);
4067  SDValue Mask = getValue(MaskOperand);
4068 
4069  EVT VT = Src0.getValueType();
4070  if (!Alignment)
4071  Alignment = DAG.getEVTAlignment(VT);
4072 
4073  AAMDNodes AAInfo;
4074  I.getAAMetadata(AAInfo);
4075  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4076 
4077  // Do not serialize masked loads of constant memory with anything.
4078  bool AddToChain = !AA || !AA->pointsToConstantMemory(MemoryLocation(
4079  PtrOperand, DAG.getDataLayout().getTypeStoreSize(I.getType()), AAInfo));
4080  SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4081 
4082  MachineMemOperand *MMO =
4083  DAG.getMachineFunction().
4084  getMachineMemOperand(MachinePointerInfo(PtrOperand),
4086  Alignment, AAInfo, Ranges);
4087 
4088  SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
4089  ISD::NON_EXTLOAD, IsExpanding);
4090  if (AddToChain)
4091  PendingLoads.push_back(Load.getValue(1));
4092  setValue(&I, Load);
4093 }
4094 
4095 void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
4096  SDLoc sdl = getCurSDLoc();
4097 
4098  // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
4099  const Value *Ptr = I.getArgOperand(0);
4100  SDValue Src0 = getValue(I.getArgOperand(3));
4101  SDValue Mask = getValue(I.getArgOperand(2));
4102 
4103  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4104  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4105  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
4106  if (!Alignment)
4107  Alignment = DAG.getEVTAlignment(VT);
4108 
4109  AAMDNodes AAInfo;
4110  I.getAAMetadata(AAInfo);
4111  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4112 
4113  SDValue Root = DAG.getRoot();
4114  SDValue Base;
4115  SDValue Index;
4116  SDValue Scale;
4117  const Value *BasePtr = Ptr;
4118  bool UniformBase = getUniformBase(BasePtr, Base, Index, Scale, this);
4119  bool ConstantMemory = false;
4120  if (UniformBase &&
4121  AA && AA->pointsToConstantMemory(MemoryLocation(
4122  BasePtr, DAG.getDataLayout().getTypeStoreSize(I.getType()),
4123  AAInfo))) {
4124  // Do not serialize (non-volatile) loads of constant memory with anything.
4125  Root = DAG.getEntryNode();
4126  ConstantMemory = true;
4127  }
4128 
4129  MachineMemOperand *MMO =
4130  DAG.getMachineFunction().
4131  getMachineMemOperand(MachinePointerInfo(UniformBase ? BasePtr : nullptr),
4133  Alignment, AAInfo, Ranges);
4134 
4135  if (!UniformBase) {
4136  Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4137  Index = getValue(Ptr);
4138  Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4139  }
4140  SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
4141  SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
4142  Ops, MMO);
4143 
4144  SDValue OutChain = Gather.getValue(1);
4145  if (!ConstantMemory)
4146  PendingLoads.push_back(OutChain);
4147  setValue(&I, Gather);
4148 }
4149 
4150 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
4151  SDLoc dl = getCurSDLoc();
4152  AtomicOrdering SuccessOrder = I.getSuccessOrdering();
4153  AtomicOrdering FailureOrder = I.getFailureOrdering();
4154  SyncScope::ID SSID = I.getSyncScopeID();
4155 
4156  SDValue InChain = getRoot();
4157 
4158  MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
4159  SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
4160  SDValue L = DAG.getAtomicCmpSwap(
4161  ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
4162  getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
4164  /*Alignment=*/ 0, SuccessOrder, FailureOrder, SSID);
4165 
4166  SDValue OutChain = L.getValue(2);
4167 
4168  setValue(&I, L);
4169  DAG.setRoot(OutChain);
4170 }
4171 
4172 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
4173  SDLoc dl = getCurSDLoc();
4174  ISD::NodeType NT;
4175  switch (I.getOperation()) {
4176  default: llvm_unreachable("Unknown atomicrmw operation");
4177  case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
4178  case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
4179  case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
4180  case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
4181  case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
4182  case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
4183  case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
4184  case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
4185  case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
4186  case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
4187  case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
4188  }
4189  AtomicOrdering Order = I.getOrdering();
4190  SyncScope::ID SSID = I.getSyncScopeID();
4191 
4192  SDValue InChain = getRoot();
4193 
4194  SDValue L =
4195  DAG.getAtomic(NT, dl,
4196  getValue(I.getValOperand()).getSimpleValueType(),
4197  InChain,
4198  getValue(I.getPointerOperand()),
4199  getValue(I.getValOperand()),
4200  I.getPointerOperand(),
4201  /* Alignment=*/ 0, Order, SSID);
4202 
4203  SDValue OutChain = L.getValue(1);
4204 
4205  setValue(&I, L);
4206  DAG.setRoot(OutChain);
4207 }
4208 
4209 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
4210  SDLoc dl = getCurSDLoc();
4211  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4212  SDValue Ops[3];
4213  Ops[0] = getRoot();
4214  Ops[1] = DAG.getConstant((unsigned)I.getOrdering(), dl,
4215  TLI.getFenceOperandTy(DAG.getDataLayout()));
4216  Ops[2] = DAG.getConstant(I.getSyncScopeID(), dl,
4217  TLI.getFenceOperandTy(DAG.getDataLayout()));
4218  DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
4219 }
4220 
4221 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
4222  SDLoc dl = getCurSDLoc();
4223  AtomicOrdering Order = I.getOrdering();
4224  SyncScope::ID SSID = I.getSyncScopeID();
4225 
4226  SDValue InChain = getRoot();
4227 
4228  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4229  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4230 
4231  if (!TLI.supportsUnalignedAtomics() &&
4232  I.getAlignment() < VT.getStoreSize())
4233  report_fatal_error("Cannot generate unaligned atomic load");
4234 
4235  MachineMemOperand *MMO =
4236  DAG.getMachineFunction().
4237  getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
4240  VT.getStoreSize(),
4241  I.getAlignment() ? I.getAlignment() :
4242  DAG.getEVTAlignment(VT),
4243  AAMDNodes(), nullptr, SSID, Order);
4244 
4245  InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
4246  SDValue L =
4247  DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
4248  getValue(I.getPointerOperand()), MMO);
4249 
4250  SDValue OutChain = L.getValue(1);
4251 
4252  setValue(&I, L);
4253  DAG.setRoot(OutChain);
4254 }
4255 
4256 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
4257  SDLoc dl = getCurSDLoc();
4258 
4259  AtomicOrdering Order = I.getOrdering();
4260  SyncScope::ID SSID = I.getSyncScopeID();
4261 
4262  SDValue InChain = getRoot();
4263 
4264  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4265  EVT VT =
4267 
4268  if (I.getAlignment() < VT.getStoreSize())
4269  report_fatal_error("Cannot generate unaligned atomic store");
4270 
4271  SDValue OutChain =
4272  DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
4273  InChain,
4274  getValue(I.getPointerOperand()),
4275  getValue(I.getValueOperand()),
4277  Order, SSID);
4278 
4279  DAG.setRoot(OutChain);
4280 }
4281 
4282 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4283 /// node.
4284 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4285  unsigned Intrinsic) {
4286  // Ignore the callsite's attributes. A specific call site may be marked with
4287  // readnone, but the lowering code will expect the chain based on the
4288  // definition.
4289  const Function *F = I.getCalledFunction();
4290  bool HasChain = !F->doesNotAccessMemory();
4291  bool OnlyLoad = HasChain && F->onlyReadsMemory();
4292 
4293  // Build the operand list.
4295  if (HasChain) { // If this intrinsic has side-effects, chainify it.
4296  if (OnlyLoad) {
4297  // We don't need to serialize loads against other loads.
4298  Ops.push_back(DAG.getRoot());
4299  } else {
4300  Ops.push_back(getRoot());
4301  }
4302  }
4303 
4304  // Info is set by getTgtMemInstrinsic
4305  TargetLowering::IntrinsicInfo Info;
4306  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4307  bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
4308  DAG.getMachineFunction(),
4309  Intrinsic);
4310 
4311  // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4312  if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
4313  Info.opc == ISD::INTRINSIC_W_CHAIN)
4314  Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4315  TLI.getPointerTy(DAG.getDataLayout())));
4316 
4317  // Add all operands of the call to the operand list.
4318  for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
4319  SDValue Op = getValue(I.getArgOperand(i));
4320  Ops.push_back(Op);
4321  }
4322 
4324  ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4325 
4326  if (HasChain)
4327  ValueVTs.push_back(MVT::Other);
4328 
4329  SDVTList VTs = DAG.getVTList(ValueVTs);
4330 
4331  // Create the node.
4332  SDValue Result;
4333  if (IsTgtIntrinsic) {
4334  // This is target intrinsic that touches memory
4335  Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs,
4336  Ops, Info.memVT,
4337  MachinePointerInfo(Info.ptrVal, Info.offset), Info.align,
4338  Info.flags, Info.size);
4339  } else if (!HasChain) {
4340  Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4341  } else if (!I.getType()->isVoidTy()) {
4342  Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4343  } else {
4344  Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4345  }
4346 
4347  if (HasChain) {
4348  SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4349  if (OnlyLoad)
4350  PendingLoads.push_back(Chain);
4351  else
4352  DAG.setRoot(Chain);
4353  }
4354 
4355  if (!I.getType()->isVoidTy()) {
4356  if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
4357  EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);
4358  Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
4359  } else
4360  Result = lowerRangeToAssertZExt(DAG, I, Result);
4361 
4362  setValue(&I, Result);
4363  }
4364 }
4365 
4366 /// GetSignificand - Get the significand and build it into a floating-point
4367 /// number with exponent of 1:
4368 ///
4369 /// Op = (Op & 0x007fffff) | 0x3f800000;
4370 ///
4371 /// where Op is the hexadecimal representation of floating point value.
4373  SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4374  DAG.getConstant(0x007fffff, dl, MVT::i32));
4375  SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4376  DAG.getConstant(0x3f800000, dl, MVT::i32));
4377  return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4378 }
4379 
4380 /// GetExponent - Get the exponent:
4381 ///
4382 /// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4383 ///
4384 /// where Op is the hexadecimal representation of floating point value.
4386  const TargetLowering &TLI, const SDLoc &dl) {
4387  SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4388  DAG.getConstant(0x7f800000, dl, MVT::i32));
4389  SDValue t1 = DAG.getNode(
4390  ISD::SRL, dl, MVT::i32, t0,
4391  DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));
4392  SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4393  DAG.getConstant(127, dl, MVT::i32));
4394  return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4395 }
4396 
4397 /// getF32Constant - Get 32-bit floating point constant.
4398 static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4399  const SDLoc &dl) {
4400  return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4401  MVT::f32);
4402 }
4403 
4405  SelectionDAG &DAG) {
4406  // TODO: What fast-math-flags should be set on the floating-point nodes?
4407 
4408  // IntegerPartOfX = ((int32_t)(t0);
4409  SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4410 
4411  // FractionalPartOfX = t0 - (float)IntegerPartOfX;
4412  SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4413  SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4414 
4415  // IntegerPartOfX <<= 23;
4416  IntegerPartOfX = DAG.getNode(
4417  ISD::SHL, dl, MVT::i32, IntegerPartOfX,
4419  DAG.getDataLayout())));
4420 
4421  SDValue TwoToFractionalPartOfX;
4422  if (LimitFloatPrecision <= 6) {
4423  // For floating-point precision of 6:
4424  //
4425  // TwoToFractionalPartOfX =
4426  // 0.997535578f +
4427  // (0.735607626f + 0.252464424f * x) * x;
4428  //
4429  // error 0.0144103317, which is 6 bits
4430  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4431  getF32Constant(DAG, 0x3e814304, dl));
4432  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4433  getF32Constant(DAG, 0x3f3c50c8, dl));
4434  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4435  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4436  getF32Constant(DAG, 0x3f7f5e7e, dl));
4437  } else if (LimitFloatPrecision <= 12) {
4438  // For floating-point precision of 12:
4439  //
4440  // TwoToFractionalPartOfX =
4441  // 0.999892986f +
4442  // (0.696457318f +
4443  // (0.224338339f + 0.792043434e-1f * x) * x) * x;
4444  //
4445  // error 0.000107046256, which is 13 to 14 bits
4446  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4447  getF32Constant(DAG, 0x3da235e3, dl));
4448  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4449  getF32Constant(DAG, 0x3e65b8f3, dl));
4450  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4451  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4452  getF32Constant(DAG, 0x3f324b07, dl));
4453  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4454  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4455  getF32Constant(DAG, 0x3f7ff8fd, dl));
4456  } else { // LimitFloatPrecision <= 18
4457  // For floating-point precision of 18:
4458  //
4459  // TwoToFractionalPartOfX =
4460  // 0.999999982f +
4461  // (0.693148872f +
4462  // (0.240227044f +
4463  // (0.554906021e-1f +
4464  // (0.961591928e-2f +
4465  // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
4466  // error 2.47208000*10^(-7), which is better than 18 bits
4467  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4468  getF32Constant(DAG, 0x3924b03e, dl));
4469  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4470  getF32Constant(DAG, 0x3ab24b87, dl));
4471  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4472  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4473  getF32Constant(DAG, 0x3c1d8c17, dl));
4474  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4475  SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4476  getF32Constant(DAG, 0x3d634a1d, dl));
4477  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4478  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4479  getF32Constant(DAG, 0x3e75fe14, dl));
4480  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4481  SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4482  getF32Constant(DAG, 0x3f317234, dl));
4483  SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4484  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
4485  getF32Constant(DAG, 0x3f800000, dl));
4486  }
4487 
4488  // Add the exponent into the result in integer domain.
4489  SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4490  return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4491  DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
4492 }
4493 
4494 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
4495 /// limited-precision mode.
4496 static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4497  const TargetLowering &TLI) {
4498  if (Op.getValueType() == MVT::f32 &&
4499  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4500 
4501  // Put the exponent in the right bit position for later addition to the
4502  // final result:
4503  //
4504  // #define LOG2OFe 1.4426950f
4505  // t0 = Op * LOG2OFe
4506 
4507  // TODO: What fast-math-flags should be set here?
4508  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
4509  getF32Constant(DAG, 0x3fb8aa3b, dl));
4510  return getLimitedPrecisionExp2(t0, dl, DAG);
4511  }
4512 
4513  // No special expansion.
4514  return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
4515 }
4516 
4517 /// expandLog - Lower a log intrinsic. Handles the special sequences for
4518 /// limited-precision mode.
4519 static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4520  const TargetLowering &TLI) {
4521  // TODO: What fast-math-flags should be set on the floating-point nodes?
4522 
4523  if (Op.getValueType() == MVT::f32 &&
4524  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4525  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4526 
4527  // Scale the exponent by log(2) [0.69314718f].
4528  SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4529  SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4530  getF32Constant(DAG, 0x3f317218, dl));
4531 
4532  // Get the significand and build it into a floating-point number with
4533  // exponent of 1.
4534  SDValue X = GetSignificand(DAG, Op1, dl);
4535 
4536  SDValue LogOfMantissa;
4537  if (LimitFloatPrecision <= 6) {
4538  // For floating-point precision of 6:
4539  //
4540  // LogofMantissa =
4541  // -1.1609546f +
4542  // (1.4034025f - 0.23903021f * x) * x;
4543  //
4544  // error 0.0034276066, which is better than 8 bits
4545  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4546  getF32Constant(DAG, 0xbe74c456, dl));
4547  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4548  getF32Constant(DAG, 0x3fb3a2b1, dl));
4549  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4550  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4551  getF32Constant(DAG, 0x3f949a29, dl));
4552  } else if (LimitFloatPrecision <= 12) {
4553  // For floating-point precision of 12:
4554  //
4555  // LogOfMantissa =
4556  // -1.7417939f +
4557  // (2.8212026f +
4558  // (-1.4699568f +
4559  // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
4560  //
4561  // error 0.000061011436, which is 14 bits
4562  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4563  getF32Constant(DAG, 0xbd67b6d6, dl));
4564  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4565  getF32Constant(DAG, 0x3ee4f4b8, dl));
4566  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4567  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4568  getF32Constant(DAG, 0x3fbc278b, dl));
4569  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4570  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4571  getF32Constant(DAG, 0x40348e95, dl));
4572  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4573  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4574  getF32Constant(DAG, 0x3fdef31a, dl));
4575  } else { // LimitFloatPrecision <= 18
4576  // For floating-point precision of 18:
4577  //
4578  // LogOfMantissa =
4579  // -2.1072184f +
4580  // (4.2372794f +
4581  // (-3.7029485f +
4582  // (2.2781945f +
4583  // (-0.87823314f +
4584  // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
4585  //
4586  // error 0.0000023660568, which is better than 18 bits
4587  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4588  getF32Constant(DAG, 0xbc91e5ac, dl));
4589  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4590  getF32Constant(DAG, 0x3e4350aa, dl));
4591  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4592  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4593  getF32Constant(DAG, 0x3f60d3e3, dl));
4594  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4595  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4596  getF32Constant(DAG, 0x4011cdf0, dl));
4597  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4598  SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4599  getF32Constant(DAG, 0x406cfd1c, dl));
4600  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4601  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4602  getF32Constant(DAG, 0x408797cb, dl));
4603  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4604  LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4605  getF32Constant(DAG, 0x4006dcab, dl));
4606  }
4607 
4608  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
4609  }
4610 
4611  // No special expansion.
4612  return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
4613 }
4614 
4615 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
4616 /// limited-precision mode.
4617 static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4618  const TargetLowering &TLI) {
4619  // TODO: What fast-math-flags should be set on the floating-point nodes?
4620 
4621  if (Op.getValueType() == MVT::f32 &&
4622  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4623  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4624 
4625  // Get the exponent.
4626  SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
4627 
4628  // Get the significand and build it into a floating-point number with
4629  // exponent of 1.
4630  SDValue X = GetSignificand(DAG, Op1, dl);
4631 
4632  // Different possible minimax approximations of significand in
4633  // floating-point for various degrees of accuracy over [1,2].
4634  SDValue Log2ofMantissa;
4635  if (LimitFloatPrecision <= 6) {
4636  // For floating-point precision of 6:
4637  //
4638  // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
4639  //
4640  // error 0.0049451742, which is more than 7 bits
4641  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4642  getF32Constant(DAG, 0xbeb08fe0, dl));
4643  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4644  getF32Constant(DAG, 0x40019463, dl));
4645  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4646  Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4647  getF32Constant(DAG, 0x3fd6633d, dl));
4648  } else if (LimitFloatPrecision <= 12) {
4649  // For floating-point precision of 12:
4650  //
4651  // Log2ofMantissa =
4652  // -2.51285454f +
4653  // (4.07009056f +
4654  // (-2.12067489f +
4655  // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
4656  //
4657  // error 0.0000876136000, which is better than 13 bits
4658  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4659  getF32Constant(DAG, 0xbda7262e, dl));
4660  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4661  getF32Constant(DAG, 0x3f25280b, dl));
4662  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4663  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4664  getF32Constant(DAG, 0x4007b923, dl));
4665  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4666  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4667  getF32Constant(DAG, 0x40823e2f, dl));
4668  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4669  Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4670  getF32Constant(DAG, 0x4020d29c, dl));
4671  } else { // LimitFloatPrecision <= 18
4672  // For floating-point precision of 18:
4673  //
4674  // Log2ofMantissa =
4675  // -3.0400495f +
4676  // (6.1129976f +
4677  // (-5.3420409f +
4678  // (3.2865683f +
4679  // (-1.2669343f +
4680  // (0.27515199f -
4681  // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
4682  //
4683  // error 0.0000018516, which is better than 18 bits
4684  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4685  getF32Constant(DAG, 0xbcd2769e, dl));
4686  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4687  getF32Constant(DAG, 0x3e8ce0b9, dl));
4688  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4689  SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4690  getF32Constant(DAG, 0x3fa22ae7, dl));
4691  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4692  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4693  getF32Constant(DAG, 0x40525723, dl));
4694  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4695  SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4696  getF32Constant(DAG, 0x40aaf200, dl));
4697  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4698  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4699  getF32Constant(DAG, 0x40c39dad, dl));
4700  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4701  Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4702  getF32Constant(DAG, 0x4042902c, dl));
4703  }
4704 
4705  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
4706  }
4707 
4708  // No special expansion.
4709  return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
4710 }
4711 
4712 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
4713 /// limited-precision mode.
4715  const TargetLowering &TLI) {
4716  // TODO: What fast-math-flags should be set on the floating-point nodes?
4717 
4718  if (Op.getValueType() == MVT::f32 &&
4719  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4720  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4721 
4722  // Scale the exponent by log10(2) [0.30102999f].
4723  SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4724  SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4725  getF32Constant(DAG, 0x3e9a209a, dl));
4726 
4727  // Get the significand and build it into a floating-point number with
4728  // exponent of 1.
4729  SDValue X = GetSignificand(DAG, Op1, dl);
4730 
4731  SDValue Log10ofMantissa;
4732  if (LimitFloatPrecision <= 6) {
4733  // For floating-point precision of 6:
4734  //
4735  // Log10ofMantissa =
4736  // -0.50419619f +
4737  // (0.60948995f - 0.10380950f * x) * x;
4738  //
4739  // error 0.0014886165, which is 6 bits
4740  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4741  getF32Constant(DAG, 0xbdd49a13, dl));
4742  SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4743  getF32Constant(DAG, 0x3f1c0789, dl));
4744  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4745  Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4746  getF32Constant(DAG, 0x3f011300, dl));
4747  } else if (LimitFloatPrecision <= 12) {
4748  // For floating-point precision of 12:
4749  //
4750  // Log10ofMantissa =
4751  // -0.64831180f +
4752  // (0.91751397f +
4753  // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
4754  //
4755  // error 0.00019228036, which is better than 12 bits
4756  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4757  getF32Constant(DAG, 0x3d431f31, dl));
4758  SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4759  getF32Constant(DAG, 0x3ea21fb2, dl));
4760  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4761  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4762  getF32Constant(DAG, 0x3f6ae232, dl));
4763  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4764  Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4765  getF32Constant(DAG, 0x3f25f7c3, dl));
4766  } else { // LimitFloatPrecision <= 18
4767  // For floating-point precision of 18:
4768  //
4769  // Log10ofMantissa =
4770  // -0.84299375f +
4771  // (1.5327582f +
4772  // (-1.0688956f +
4773  // (0.49102474f +
4774  // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
4775  //
4776  // error 0.0000037995730, which is better than 18 bits
4777  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4778  getF32Constant(DAG, 0x3c5d51ce, dl));
4779  SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4780  getF32Constant(DAG, 0x3e00685a, dl));
4781  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4782  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4783  getF32Constant(DAG, 0x3efb6798, dl));
4784  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4785  SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4786  getF32Constant(DAG, 0x3f88d192, dl));
4787  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4788  SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4789  getF32Constant(DAG, 0x3fc4316c, dl));
4790  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4791  Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
4792  getF32Constant(DAG, 0x3f57ce70, dl));
4793  }
4794 
4795  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
4796  }
4797 
4798  // No special expansion.
4799  return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
4800 }
4801 
4802 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
4803 /// limited-precision mode.
4804 static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4805  const TargetLowering &TLI) {
4806  if (Op.getValueType() == MVT::f32 &&
4807  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
4808  return getLimitedPrecisionExp2(Op, dl, DAG);
4809 
4810  // No special expansion.
4811  return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
4812 }
4813 
4814 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
4815 /// limited-precision mode with x == 10.0f.
4816 static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
4817  SelectionDAG &DAG, const TargetLowering &TLI) {
4818  bool IsExp10 = false;
4819  if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
4820  LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4821  if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
4822  APFloat Ten(10.0f);
4823  IsExp10 = LHSC->isExactlyValue(Ten);
4824  }
4825  }
4826 
4827  // TODO: What fast-math-flags should be set on the FMUL node?
4828  if (IsExp10) {
4829  // Put the exponent in the right bit position for later addition to the
4830  // final result:
4831  //
4832  // #define LOG2OF10 3.3219281f
4833  // t0 = Op * LOG2OF10;
4834  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
4835  getF32Constant(DAG, 0x40549a78, dl));
4836  return getLimitedPrecisionExp2(t0, dl, DAG);
4837  }
4838 
4839  // No special expansion.
4840  return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
4841 }
4842 
4843 /// ExpandPowI - Expand a llvm.powi intrinsic.
4844 static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
4845  SelectionDAG &DAG) {
4846  // If RHS is a constant, we can expand this out to a multiplication tree,
4847  // otherwise we end up lowering to a call to __powidf2 (for example). When
4848  // optimizing for size, we only want to do this if the expansion would produce
4849  // a small number of multiplies, otherwise we do the full expansion.
4850  if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
4851  // Get the exponent as a positive value.
4852  unsigned Val = RHSC->getSExtValue();
4853  if ((int)Val < 0) Val = -Val;
4854 
4855  // powi(x, 0) -> 1.0
4856  if (Val == 0)
4857  return DAG.getConstantFP(1.0, DL, LHS.getValueType());
4858 
4859  const Function &F = DAG.getMachineFunction().getFunction();
4860  if (!F.optForSize() ||
4861  // If optimizing for size, don't insert too many multiplies.
4862  // This inserts up to 5 multiplies.
4863  countPopulation(Val) + Log2_32(Val) < 7) {
4864  // We use the simple binary decomposition method to generate the multiply
4865  // sequence. There are more optimal ways to do this (for example,
4866  // powi(x,15) generates one more multiply than it should), but this has
4867  // the benefit of being both really simple and much better than a libcall.
4868  SDValue Res; // Logically starts equal to 1.0
4869  SDValue CurSquare = LHS;
4870  // TODO: Intrinsics should have fast-math-flags that propagate to these
4871  // nodes.
4872  while (Val) {
4873  if (Val & 1) {
4874  if (Res.getNode())
4875  Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
4876  else
4877  Res = CurSquare; // 1.0*CurSquare.
4878  }
4879 
4880  CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
4881  CurSquare, CurSquare);
4882  Val >>= 1;
4883  }
4884 
4885  // If the original was negative, invert the result, producing 1/(x*x*x).
4886  if (RHSC->getSExtValue() < 0)
4887  Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
4888  DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
4889  return Res;
4890  }
4891  }
4892 
4893  // Otherwise, expand to a libcall.
4894  return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
4895 }
4896 
4897 // getUnderlyingArgReg - Find underlying register used for a truncated or
4898 // bitcasted argument.
4899 static unsigned getUnderlyingArgReg(const SDValue &N) {
4900  switch (N.getOpcode()) {
4901  case ISD::CopyFromReg:
4902  return cast<RegisterSDNode>(N.getOperand(1))->getReg();
4903  case ISD::BITCAST:
4904  case ISD::AssertZext:
4905  case ISD::AssertSext:
4906  case ISD::TRUNCATE:
4907  return getUnderlyingArgReg(N.getOperand(0));
4908  default:
4909  return 0;
4910  }
4911 }
4912 
4913 /// If the DbgValueInst is a dbg_value of a function argument, create the
4914 /// corresponding DBG_VALUE machine instruction for it now. At the end of
4915 /// instruction selection, they will be inserted to the entry BB.
4916 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
4917  const Value *V, DILocalVariable *Variable, DIExpression *Expr,
4918  DILocation *DL, bool IsDbgDeclare, const SDValue &N) {
4919  const Argument *Arg = dyn_cast<Argument>(V);
4920  if (!Arg)
4921  return false;
4922 
4923  MachineFunction &MF = DAG.getMachineFunction();
4924  const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
4925 
4926  bool IsIndirect = false;
4928  // Some arguments' frame index is recorded during argument lowering.
4929  int FI = FuncInfo.getArgumentFrameIndex(Arg);
4930  if (FI != std::numeric_limits<int>::max())
4931  Op = MachineOperand::CreateFI(FI);
4932 
4933  if (!Op && N.getNode()) {
4934  unsigned Reg = getUnderlyingArgReg(N);
4935  if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
4936  MachineRegisterInfo &RegInfo = MF.getRegInfo();
4937  unsigned PR = RegInfo.getLiveInPhysReg(Reg);
4938  if (PR)
4939  Reg = PR;
4940  }
4941  if (Reg) {
4942  Op = MachineOperand::CreateReg(Reg, false);
4943  IsIndirect = IsDbgDeclare;
4944  }
4945  }
4946 
4947  if (!Op && N.getNode())
4948  // Check if frame index is available.
4949  if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
4950  if (FrameIndexSDNode *FINode =
4951  dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
4952  Op = MachineOperand::CreateFI(FINode->getIndex());
4953 
4954  if (!Op) {
4955  // Check if ValueMap has reg number.
4956  DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
4957  if (VMI != FuncInfo.ValueMap.end()) {
4958  const auto &TLI = DAG.getTargetLoweringInfo();
4959  RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
4960  V->getType(), getABIRegCopyCC(V));
4961  if (RFV.occupiesMultipleRegs()) {
4962  unsigned Offset = 0;
4963  for (auto RegAndSize : RFV.getRegsAndSizes()) {
4964  Op = MachineOperand::CreateReg(RegAndSize.first, false);
4965  auto FragmentExpr = DIExpression::createFragmentExpression(
4966  Expr, Offset, RegAndSize.second);
4967  if (!FragmentExpr)
4968  continue;
4969  FuncInfo.ArgDbgValues.push_back(
4970  BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsDbgDeclare,
4971  Op->getReg(), Variable, *FragmentExpr));
4972  Offset += RegAndSize.second;
4973  }
4974  return true;
4975  }
4976  Op = MachineOperand::CreateReg(VMI->second, false);
4977  IsIndirect = IsDbgDeclare;
4978  }
4979  }
4980 
4981  if (!Op)
4982  return false;
4983 
4984  assert(Variable->isValidLocationForIntrinsic(DL) &&
4985  "Expected inlined-at fields to agree");
4986  IsIndirect = (Op->isReg()) ? IsIndirect : true;
4987  FuncInfo.ArgDbgValues.push_back(
4988  BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
4989  *Op, Variable, Expr));
4990