LLVM  13.0.0git
LegalizeVectorTypes.cpp
Go to the documentation of this file.
1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file performs vector type splitting and scalarization for LegalizeTypes.
10 // Scalarization is the act of changing a computation in an illegal one-element
11 // vector type to be a computation in its scalar element type. For example,
12 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
14 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15 // types.
16 // Splitting is the act of changing a computation in an invalid vector type to
17 // be a computation in two vectors of half the size. For example, implementing
18 // <128 x f32> operations in terms of two <64 x f32> operations.
19 //
20 //===----------------------------------------------------------------------===//
21 
22 #include "LegalizeTypes.h"
24 #include "llvm/IR/DataLayout.h"
26 #include "llvm/Support/TypeSize.h"
28 using namespace llvm;
29 
30 #define DEBUG_TYPE "legalize-types"
31 
32 //===----------------------------------------------------------------------===//
33 // Result Vector Scalarization: <1 x ty> -> ty.
34 //===----------------------------------------------------------------------===//
35 
36 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
37  LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
38  dbgs() << "\n");
39  SDValue R = SDValue();
40 
41  switch (N->getOpcode()) {
42  default:
43 #ifndef NDEBUG
44  dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
45  N->dump(&DAG);
46  dbgs() << "\n";
47 #endif
48  report_fatal_error("Do not know how to scalarize the result of this "
49  "operator!\n");
50 
51  case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
52  case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
53  case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
54  case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
55  case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
56  case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
57  case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
58  case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
59  case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
60  case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
61  case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
62  case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
63  case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
64  case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
65  case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
66  case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
70  R = ScalarizeVecRes_VecInregOp(N);
71  break;
72  case ISD::ABS:
73  case ISD::ANY_EXTEND:
74  case ISD::BITREVERSE:
75  case ISD::BSWAP:
76  case ISD::CTLZ:
78  case ISD::CTPOP:
79  case ISD::CTTZ:
81  case ISD::FABS:
82  case ISD::FCEIL:
83  case ISD::FCOS:
84  case ISD::FEXP:
85  case ISD::FEXP2:
86  case ISD::FFLOOR:
87  case ISD::FLOG:
88  case ISD::FLOG10:
89  case ISD::FLOG2:
90  case ISD::FNEARBYINT:
91  case ISD::FNEG:
92  case ISD::FREEZE:
93  case ISD::FP_EXTEND:
94  case ISD::FP_TO_SINT:
95  case ISD::FP_TO_UINT:
96  case ISD::FRINT:
97  case ISD::FROUND:
98  case ISD::FROUNDEVEN:
99  case ISD::FSIN:
100  case ISD::FSQRT:
101  case ISD::FTRUNC:
102  case ISD::SIGN_EXTEND:
103  case ISD::SINT_TO_FP:
104  case ISD::TRUNCATE:
105  case ISD::UINT_TO_FP:
106  case ISD::ZERO_EXTEND:
107  case ISD::FCANONICALIZE:
108  R = ScalarizeVecRes_UnaryOp(N);
109  break;
110 
111  case ISD::ADD:
112  case ISD::AND:
113  case ISD::FADD:
114  case ISD::FCOPYSIGN:
115  case ISD::FDIV:
116  case ISD::FMUL:
117  case ISD::FMINNUM:
118  case ISD::FMAXNUM:
119  case ISD::FMINNUM_IEEE:
120  case ISD::FMAXNUM_IEEE:
121  case ISD::FMINIMUM:
122  case ISD::FMAXIMUM:
123  case ISD::SMIN:
124  case ISD::SMAX:
125  case ISD::UMIN:
126  case ISD::UMAX:
127 
128  case ISD::SADDSAT:
129  case ISD::UADDSAT:
130  case ISD::SSUBSAT:
131  case ISD::USUBSAT:
132  case ISD::SSHLSAT:
133  case ISD::USHLSAT:
134 
135  case ISD::FPOW:
136  case ISD::FREM:
137  case ISD::FSUB:
138  case ISD::MUL:
139  case ISD::OR:
140  case ISD::SDIV:
141  case ISD::SREM:
142  case ISD::SUB:
143  case ISD::UDIV:
144  case ISD::UREM:
145  case ISD::XOR:
146  case ISD::SHL:
147  case ISD::SRA:
148  case ISD::SRL:
149  case ISD::ROTL:
150  case ISD::ROTR:
151  R = ScalarizeVecRes_BinOp(N);
152  break;
153  case ISD::FMA:
154  case ISD::FSHL:
155  case ISD::FSHR:
156  R = ScalarizeVecRes_TernaryOp(N);
157  break;
158 
159 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
160  case ISD::STRICT_##DAGN:
161 #include "llvm/IR/ConstrainedOps.def"
162  R = ScalarizeVecRes_StrictFPOp(N);
163  break;
164 
165  case ISD::FP_TO_UINT_SAT:
166  case ISD::FP_TO_SINT_SAT:
167  R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
168  break;
169 
170  case ISD::UADDO:
171  case ISD::SADDO:
172  case ISD::USUBO:
173  case ISD::SSUBO:
174  case ISD::UMULO:
175  case ISD::SMULO:
176  R = ScalarizeVecRes_OverflowOp(N, ResNo);
177  break;
178  case ISD::SMULFIX:
179  case ISD::SMULFIXSAT:
180  case ISD::UMULFIX:
181  case ISD::UMULFIXSAT:
182  case ISD::SDIVFIX:
183  case ISD::SDIVFIXSAT:
184  case ISD::UDIVFIX:
185  case ISD::UDIVFIXSAT:
186  R = ScalarizeVecRes_FIX(N);
187  break;
188  }
189 
190  // If R is null, the sub-method took care of registering the result.
191  if (R.getNode())
192  SetScalarizedVector(SDValue(N, ResNo), R);
193 }
194 
195 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
196  SDValue LHS = GetScalarizedVector(N->getOperand(0));
197  SDValue RHS = GetScalarizedVector(N->getOperand(1));
198  return DAG.getNode(N->getOpcode(), SDLoc(N),
199  LHS.getValueType(), LHS, RHS, N->getFlags());
200 }
201 
202 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
203  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
204  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
205  SDValue Op2 = GetScalarizedVector(N->getOperand(2));
206  return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
207  Op2, N->getFlags());
208 }
209 
210 SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
211  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
212  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
213  SDValue Op2 = N->getOperand(2);
214  return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
215  Op2, N->getFlags());
216 }
217 
218 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
219  EVT VT = N->getValueType(0).getVectorElementType();
220  unsigned NumOpers = N->getNumOperands();
221  SDValue Chain = N->getOperand(0);
222  EVT ValueVTs[] = {VT, MVT::Other};
223  SDLoc dl(N);
224 
225  SmallVector<SDValue, 4> Opers(NumOpers);
226 
227  // The Chain is the first operand.
228  Opers[0] = Chain;
229 
230  // Now process the remaining operands.
231  for (unsigned i = 1; i < NumOpers; ++i) {
232  SDValue Oper = N->getOperand(i);
233 
234  if (Oper.getValueType().isVector())
235  Oper = GetScalarizedVector(Oper);
236 
237  Opers[i] = Oper;
238  }
239 
240  SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs),
241  Opers, N->getFlags());
242 
243  // Legalize the chain result - switch anything that used the old chain to
244  // use the new one.
245  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
246  return Result;
247 }
248 
249 SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
250  unsigned ResNo) {
251  SDLoc DL(N);
252  EVT ResVT = N->getValueType(0);
253  EVT OvVT = N->getValueType(1);
254 
255  SDValue ScalarLHS, ScalarRHS;
256  if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
257  ScalarLHS = GetScalarizedVector(N->getOperand(0));
258  ScalarRHS = GetScalarizedVector(N->getOperand(1));
259  } else {
260  SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
261  DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
262  DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
263  ScalarLHS = ElemsLHS[0];
264  ScalarRHS = ElemsRHS[0];
265  }
266 
267  SDVTList ScalarVTs = DAG.getVTList(
269  SDNode *ScalarNode = DAG.getNode(
270  N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
271  ScalarNode->setFlags(N->getFlags());
272 
273  // Replace the other vector result not being explicitly scalarized here.
274  unsigned OtherNo = 1 - ResNo;
275  EVT OtherVT = N->getValueType(OtherNo);
276  if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
277  SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
278  } else {
279  SDValue OtherVal = DAG.getNode(
280  ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
281  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
282  }
283 
284  return SDValue(ScalarNode, ResNo);
285 }
286 
287 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
288  unsigned ResNo) {
289  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
290  return GetScalarizedVector(Op);
291 }
292 
293 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
294  SDValue Op = N->getOperand(0);
295  if (Op.getValueType().isVector()
296  && Op.getValueType().getVectorNumElements() == 1
297  && !isSimpleLegalType(Op.getValueType()))
298  Op = GetScalarizedVector(Op);
299  EVT NewVT = N->getValueType(0).getVectorElementType();
300  return DAG.getNode(ISD::BITCAST, SDLoc(N),
301  NewVT, Op);
302 }
303 
304 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
305  EVT EltVT = N->getValueType(0).getVectorElementType();
306  SDValue InOp = N->getOperand(0);
307  // The BUILD_VECTOR operands may be of wider element types and
308  // we may need to truncate them back to the requested return type.
309  if (EltVT.isInteger())
310  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
311  return InOp;
312 }
313 
314 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
315  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
316  N->getValueType(0).getVectorElementType(),
317  N->getOperand(0), N->getOperand(1));
318 }
319 
320 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
321  SDLoc DL(N);
322  SDValue Op = N->getOperand(0);
323  EVT OpVT = Op.getValueType();
324  // The result needs scalarizing, but it's not a given that the source does.
325  // See similar logic in ScalarizeVecRes_UnaryOp.
326  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
327  Op = GetScalarizedVector(Op);
328  } else {
329  EVT VT = OpVT.getVectorElementType();
330  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
331  DAG.getVectorIdxConstant(0, DL));
332  }
333  return DAG.getNode(ISD::FP_ROUND, DL,
334  N->getValueType(0).getVectorElementType(), Op,
335  N->getOperand(1));
336 }
337 
338 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
339  SDValue Op = GetScalarizedVector(N->getOperand(0));
340  return DAG.getNode(ISD::FPOWI, SDLoc(N),
341  Op.getValueType(), Op, N->getOperand(1));
342 }
343 
344 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
345  // The value to insert may have a wider type than the vector element type,
346  // so be sure to truncate it to the element type if necessary.
347  SDValue Op = N->getOperand(1);
348  EVT EltVT = N->getValueType(0).getVectorElementType();
349  if (Op.getValueType() != EltVT)
350  // FIXME: Can this happen for floating point types?
351  Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
352  return Op;
353 }
354 
355 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
356  assert(N->isUnindexed() && "Indexed vector load?");
357 
358  SDValue Result = DAG.getLoad(
359  ISD::UNINDEXED, N->getExtensionType(),
360  N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
361  N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
362  N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
363  N->getOriginalAlign(), N->getMemOperand()->getFlags(), N->getAAInfo());
364 
365  // Legalize the chain result - switch anything that used the old chain to
366  // use the new one.
367  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
368  return Result;
369 }
370 
371 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
372  // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
373  EVT DestVT = N->getValueType(0).getVectorElementType();
374  SDValue Op = N->getOperand(0);
375  EVT OpVT = Op.getValueType();
376  SDLoc DL(N);
377  // The result needs scalarizing, but it's not a given that the source does.
378  // This is a workaround for targets where it's impossible to scalarize the
379  // result of a conversion, because the source type is legal.
380  // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
381  // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
382  // legal and was not scalarized.
383  // See the similar logic in ScalarizeVecRes_SETCC
384  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
385  Op = GetScalarizedVector(Op);
386  } else {
387  EVT VT = OpVT.getVectorElementType();
388  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
389  DAG.getVectorIdxConstant(0, DL));
390  }
391  return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op, N->getFlags());
392 }
393 
394 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
395  EVT EltVT = N->getValueType(0).getVectorElementType();
396  EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
397  SDValue LHS = GetScalarizedVector(N->getOperand(0));
398  return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
399  LHS, DAG.getValueType(ExtVT));
400 }
401 
402 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
403  SDLoc DL(N);
404  SDValue Op = N->getOperand(0);
405 
406  EVT OpVT = Op.getValueType();
407  EVT OpEltVT = OpVT.getVectorElementType();
408  EVT EltVT = N->getValueType(0).getVectorElementType();
409 
410  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
411  Op = GetScalarizedVector(Op);
412  } else {
413  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
414  DAG.getVectorIdxConstant(0, DL));
415  }
416 
417  switch (N->getOpcode()) {
419  return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
421  return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
423  return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
424  }
425 
426  llvm_unreachable("Illegal extend_vector_inreg opcode");
427 }
428 
429 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
430  // If the operand is wider than the vector element type then it is implicitly
431  // truncated. Make that explicit here.
432  EVT EltVT = N->getValueType(0).getVectorElementType();
433  SDValue InOp = N->getOperand(0);
434  if (InOp.getValueType() != EltVT)
435  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
436  return InOp;
437 }
438 
439 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
440  SDValue Cond = N->getOperand(0);
441  EVT OpVT = Cond.getValueType();
442  SDLoc DL(N);
443  // The vselect result and true/value operands needs scalarizing, but it's
444  // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
445  // See the similar logic in ScalarizeVecRes_SETCC
446  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
447  Cond = GetScalarizedVector(Cond);
448  } else {
449  EVT VT = OpVT.getVectorElementType();
451  DAG.getVectorIdxConstant(0, DL));
452  }
453 
454  SDValue LHS = GetScalarizedVector(N->getOperand(1));
455  TargetLowering::BooleanContent ScalarBool =
456  TLI.getBooleanContents(false, false);
457  TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
458 
459  // If integer and float booleans have different contents then we can't
460  // reliably optimize in all cases. There is a full explanation for this in
461  // DAGCombiner::visitSELECT() where the same issue affects folding
462  // (select C, 0, 1) to (xor C, 1).
463  if (TLI.getBooleanContents(false, false) !=
464  TLI.getBooleanContents(false, true)) {
465  // At least try the common case where the boolean is generated by a
466  // comparison.
467  if (Cond->getOpcode() == ISD::SETCC) {
468  EVT OpVT = Cond->getOperand(0).getValueType();
469  ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
470  VecBool = TLI.getBooleanContents(OpVT);
471  } else
473  }
474 
475  EVT CondVT = Cond.getValueType();
476  if (ScalarBool != VecBool) {
477  switch (ScalarBool) {
479  break;
483  // Vector read from all ones, scalar expects a single 1 so mask.
484  Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
485  Cond, DAG.getConstant(1, SDLoc(N), CondVT));
486  break;
490  // Vector reads from a one, scalar from all ones so sign extend.
491  Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
492  Cond, DAG.getValueType(MVT::i1));
493  break;
494  }
495  }
496 
497  // Truncate the condition if needed
498  auto BoolVT = getSetCCResultType(CondVT);
499  if (BoolVT.bitsLT(CondVT))
500  Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
501 
502  return DAG.getSelect(SDLoc(N),
503  LHS.getValueType(), Cond, LHS,
504  GetScalarizedVector(N->getOperand(2)));
505 }
506 
507 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
508  SDValue LHS = GetScalarizedVector(N->getOperand(1));
509  return DAG.getSelect(SDLoc(N),
510  LHS.getValueType(), N->getOperand(0), LHS,
511  GetScalarizedVector(N->getOperand(2)));
512 }
513 
514 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
515  SDValue LHS = GetScalarizedVector(N->getOperand(2));
516  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
517  N->getOperand(0), N->getOperand(1),
518  LHS, GetScalarizedVector(N->getOperand(3)),
519  N->getOperand(4));
520 }
521 
522 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
523  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
524 }
525 
526 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
527  // Figure out if the scalar is the LHS or RHS and return it.
528  SDValue Arg = N->getOperand(2).getOperand(0);
529  if (Arg.isUndef())
530  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
531  unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
532  return GetScalarizedVector(N->getOperand(Op));
533 }
534 
535 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
536  SDValue Src = N->getOperand(0);
537  EVT SrcVT = Src.getValueType();
538  SDLoc dl(N);
539 
540  // Handle case where result is scalarized but operand is not
541  if (getTypeAction(SrcVT) == TargetLowering::TypeScalarizeVector)
542  Src = GetScalarizedVector(Src);
543  else
544  Src = DAG.getNode(
546  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
547 
548  EVT DstVT = N->getValueType(0).getVectorElementType();
549  return DAG.getNode(N->getOpcode(), dl, DstVT, Src, N->getOperand(1));
550 }
551 
552 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
553  assert(N->getValueType(0).isVector() &&
554  N->getOperand(0).getValueType().isVector() &&
555  "Operand types must be vectors");
556  SDValue LHS = N->getOperand(0);
557  SDValue RHS = N->getOperand(1);
558  EVT OpVT = LHS.getValueType();
559  EVT NVT = N->getValueType(0).getVectorElementType();
560  SDLoc DL(N);
561 
562  // The result needs scalarizing, but it's not a given that the source does.
563  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
564  LHS = GetScalarizedVector(LHS);
565  RHS = GetScalarizedVector(RHS);
566  } else {
567  EVT VT = OpVT.getVectorElementType();
568  LHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
569  DAG.getVectorIdxConstant(0, DL));
570  RHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
571  DAG.getVectorIdxConstant(0, DL));
572  }
573 
574  // Turn it into a scalar SETCC.
575  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
576  N->getOperand(2));
577  // Vectors may have a different boolean contents to scalars. Promote the
578  // value appropriately.
579  ISD::NodeType ExtendCode =
581  return DAG.getNode(ExtendCode, DL, NVT, Res);
582 }
583 
584 
585 //===----------------------------------------------------------------------===//
586 // Operand Vector Scalarization <1 x ty> -> ty.
587 //===----------------------------------------------------------------------===//
588 
589 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
590  LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
591  dbgs() << "\n");
592  SDValue Res = SDValue();
593 
594  switch (N->getOpcode()) {
595  default:
596 #ifndef NDEBUG
597  dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
598  N->dump(&DAG);
599  dbgs() << "\n";
600 #endif
601  report_fatal_error("Do not know how to scalarize this operator's "
602  "operand!\n");
603  case ISD::BITCAST:
604  Res = ScalarizeVecOp_BITCAST(N);
605  break;
606  case ISD::ANY_EXTEND:
607  case ISD::ZERO_EXTEND:
608  case ISD::SIGN_EXTEND:
609  case ISD::TRUNCATE:
610  case ISD::FP_TO_SINT:
611  case ISD::FP_TO_UINT:
612  case ISD::SINT_TO_FP:
613  case ISD::UINT_TO_FP:
614  Res = ScalarizeVecOp_UnaryOp(N);
615  break;
620  Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
621  break;
622  case ISD::CONCAT_VECTORS:
623  Res = ScalarizeVecOp_CONCAT_VECTORS(N);
624  break;
626  Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
627  break;
628  case ISD::VSELECT:
629  Res = ScalarizeVecOp_VSELECT(N);
630  break;
631  case ISD::SETCC:
632  Res = ScalarizeVecOp_VSETCC(N);
633  break;
634  case ISD::STORE:
635  Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
636  break;
638  Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
639  break;
640  case ISD::FP_ROUND:
641  Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
642  break;
644  Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
645  break;
646  case ISD::FP_EXTEND:
647  Res = ScalarizeVecOp_FP_EXTEND(N);
648  break;
649  case ISD::VECREDUCE_FADD:
650  case ISD::VECREDUCE_FMUL:
651  case ISD::VECREDUCE_ADD:
652  case ISD::VECREDUCE_MUL:
653  case ISD::VECREDUCE_AND:
654  case ISD::VECREDUCE_OR:
655  case ISD::VECREDUCE_XOR:
656  case ISD::VECREDUCE_SMAX:
657  case ISD::VECREDUCE_SMIN:
658  case ISD::VECREDUCE_UMAX:
659  case ISD::VECREDUCE_UMIN:
660  case ISD::VECREDUCE_FMAX:
661  case ISD::VECREDUCE_FMIN:
662  Res = ScalarizeVecOp_VECREDUCE(N);
663  break;
666  Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
667  break;
668  }
669 
670  // If the result is null, the sub-method took care of registering results etc.
671  if (!Res.getNode()) return false;
672 
673  // If the result is N, the sub-method updated N in place. Tell the legalizer
674  // core about this.
675  if (Res.getNode() == N)
676  return true;
677 
678  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
679  "Invalid operand expansion");
680 
681  ReplaceValueWith(SDValue(N, 0), Res);
682  return false;
683 }
684 
685 /// If the value to convert is a vector that needs to be scalarized, it must be
686 /// <1 x ty>. Convert the element instead.
687 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
688  SDValue Elt = GetScalarizedVector(N->getOperand(0));
689  return DAG.getNode(ISD::BITCAST, SDLoc(N),
690  N->getValueType(0), Elt);
691 }
692 
693 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
694 /// Do the operation on the element instead.
695 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
696  assert(N->getValueType(0).getVectorNumElements() == 1 &&
697  "Unexpected vector type!");
698  SDValue Elt = GetScalarizedVector(N->getOperand(0));
699  SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
700  N->getValueType(0).getScalarType(), Elt);
701  // Revectorize the result so the types line up with what the uses of this
702  // expression expect.
703  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
704 }
705 
706 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
707 /// Do the strict FP operation on the element instead.
708 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
709  assert(N->getValueType(0).getVectorNumElements() == 1 &&
710  "Unexpected vector type!");
711  SDValue Elt = GetScalarizedVector(N->getOperand(1));
712  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
713  { N->getValueType(0).getScalarType(), MVT::Other },
714  { N->getOperand(0), Elt });
715  // Legalize the chain result - switch anything that used the old chain to
716  // use the new one.
717  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
718  // Revectorize the result so the types line up with what the uses of this
719  // expression expect.
720  Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
721 
722  // Do our own replacement and return SDValue() to tell the caller that we
723  // handled all replacements since caller can only handle a single result.
724  ReplaceValueWith(SDValue(N, 0), Res);
725  return SDValue();
726 }
727 
728 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
729 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
730  SmallVector<SDValue, 8> Ops(N->getNumOperands());
731  for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
732  Ops[i] = GetScalarizedVector(N->getOperand(i));
733  return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
734 }
735 
736 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
737 /// so just return the element, ignoring the index.
738 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
739  EVT VT = N->getValueType(0);
740  SDValue Res = GetScalarizedVector(N->getOperand(0));
741  if (Res.getValueType() != VT)
742  Res = VT.isFloatingPoint()
743  ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
744  : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
745  return Res;
746 }
747 
748 /// If the input condition is a vector that needs to be scalarized, it must be
749 /// <1 x i1>, so just convert to a normal ISD::SELECT
750 /// (still with vector output type since that was acceptable if we got here).
751 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
752  SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
753  EVT VT = N->getValueType(0);
754 
755  return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
756  N->getOperand(2));
757 }
758 
759 /// If the operand is a vector that needs to be scalarized then the
760 /// result must be v1i1, so just convert to a scalar SETCC and wrap
761 /// with a scalar_to_vector since the res type is legal if we got here
762 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
763  assert(N->getValueType(0).isVector() &&
764  N->getOperand(0).getValueType().isVector() &&
765  "Operand types must be vectors");
766  assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
767 
768  EVT VT = N->getValueType(0);
769  SDValue LHS = GetScalarizedVector(N->getOperand(0));
770  SDValue RHS = GetScalarizedVector(N->getOperand(1));
771 
772  EVT OpVT = N->getOperand(0).getValueType();
773  EVT NVT = VT.getVectorElementType();
774  SDLoc DL(N);
775  // Turn it into a scalar SETCC.
776  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
777  N->getOperand(2));
778 
779  // Vectors may have a different boolean contents to scalars. Promote the
780  // value appropriately.
781  ISD::NodeType ExtendCode =
783 
784  Res = DAG.getNode(ExtendCode, DL, NVT, Res);
785 
786  return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
787 }
788 
789 /// If the value to store is a vector that needs to be scalarized, it must be
790 /// <1 x ty>. Just store the element.
791 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
792  assert(N->isUnindexed() && "Indexed store of one-element vector?");
793  assert(OpNo == 1 && "Do not know how to scalarize this operand!");
794  SDLoc dl(N);
795 
796  if (N->isTruncatingStore())
797  return DAG.getTruncStore(
798  N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
799  N->getBasePtr(), N->getPointerInfo(),
800  N->getMemoryVT().getVectorElementType(), N->getOriginalAlign(),
801  N->getMemOperand()->getFlags(), N->getAAInfo());
802 
803  return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
804  N->getBasePtr(), N->getPointerInfo(),
805  N->getOriginalAlign(), N->getMemOperand()->getFlags(),
806  N->getAAInfo());
807 }
808 
809 /// If the value to round is a vector that needs to be scalarized, it must be
810 /// <1 x ty>. Convert the element instead.
811 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
812  assert(OpNo == 0 && "Wrong operand for scalarization!");
813  SDValue Elt = GetScalarizedVector(N->getOperand(0));
814  SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
815  N->getValueType(0).getVectorElementType(), Elt,
816  N->getOperand(1));
817  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
818 }
819 
820 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
821  unsigned OpNo) {
822  assert(OpNo == 1 && "Wrong operand for scalarization!");
823  SDValue Elt = GetScalarizedVector(N->getOperand(1));
825  { N->getValueType(0).getVectorElementType(),
826  MVT::Other },
827  { N->getOperand(0), Elt, N->getOperand(2) });
828  // Legalize the chain result - switch anything that used the old chain to
829  // use the new one.
830  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
831 
832  Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
833 
834  // Do our own replacement and return SDValue() to tell the caller that we
835  // handled all replacements since caller can only handle a single result.
836  ReplaceValueWith(SDValue(N, 0), Res);
837  return SDValue();
838 }
839 
840 /// If the value to extend is a vector that needs to be scalarized, it must be
841 /// <1 x ty>. Convert the element instead.
842 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
843  SDValue Elt = GetScalarizedVector(N->getOperand(0));
844  SDValue Res = DAG.getNode(ISD::FP_EXTEND, SDLoc(N),
845  N->getValueType(0).getVectorElementType(), Elt);
846  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
847 }
848 
849 /// If the value to extend is a vector that needs to be scalarized, it must be
850 /// <1 x ty>. Convert the element instead.
851 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
852  SDValue Elt = GetScalarizedVector(N->getOperand(1));
853  SDValue Res =
855  {N->getValueType(0).getVectorElementType(), MVT::Other},
856  {N->getOperand(0), Elt});
857  // Legalize the chain result - switch anything that used the old chain to
858  // use the new one.
859  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
860 
861  Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
862 
863  // Do our own replacement and return SDValue() to tell the caller that we
864  // handled all replacements since caller can only handle a single result.
865  ReplaceValueWith(SDValue(N, 0), Res);
866  return SDValue();
867 }
868 
869 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
870  SDValue Res = GetScalarizedVector(N->getOperand(0));
871  // Result type may be wider than element type.
872  if (Res.getValueType() != N->getValueType(0))
873  Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res);
874  return Res;
875 }
876 
877 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
878  SDValue AccOp = N->getOperand(0);
879  SDValue VecOp = N->getOperand(1);
880 
881  unsigned BaseOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
882 
883  SDValue Op = GetScalarizedVector(VecOp);
884  return DAG.getNode(BaseOpc, SDLoc(N), N->getValueType(0),
885  AccOp, Op, N->getFlags());
886 }
887 
888 //===----------------------------------------------------------------------===//
889 // Result Vector Splitting
890 //===----------------------------------------------------------------------===//
891 
892 /// This method is called when the specified result of the specified node is
893 /// found to need vector splitting. At this point, the node may also have
894 /// invalid operands or may have other results that need legalization, we just
895 /// know that (at least) one result needs vector splitting.
896 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
897  LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
898  SDValue Lo, Hi;
899 
900  // See if the target wants to custom expand this node.
901  if (CustomLowerNode(N, N->getValueType(ResNo), true))
902  return;
903 
904  switch (N->getOpcode()) {
905  default:
906 #ifndef NDEBUG
907  dbgs() << "SplitVectorResult #" << ResNo << ": ";
908  N->dump(&DAG);
909  dbgs() << "\n";
910 #endif
911  report_fatal_error("Do not know how to split the result of this "
912  "operator!\n");
913 
914  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
915  case ISD::VSELECT:
916  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
917  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
918  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
919  case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
920  case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
921  case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
922  case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
923  case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
924  case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
925  case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
926  case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
927  case ISD::SPLAT_VECTOR:
929  SplitVecRes_ScalarOp(N, Lo, Hi);
930  break;
931  case ISD::STEP_VECTOR:
932  SplitVecRes_STEP_VECTOR(N, Lo, Hi);
933  break;
934  case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
935  case ISD::LOAD:
936  SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
937  break;
938  case ISD::MLOAD:
939  SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
940  break;
941  case ISD::MGATHER:
942  SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
943  break;
944  case ISD::SETCC:
945  SplitVecRes_SETCC(N, Lo, Hi);
946  break;
947  case ISD::VECTOR_REVERSE:
948  SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
949  break;
950  case ISD::VECTOR_SHUFFLE:
951  SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
952  break;
953  case ISD::VECTOR_SPLICE:
954  SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
955  break;
956  case ISD::VAARG:
957  SplitVecRes_VAARG(N, Lo, Hi);
958  break;
959 
963  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
964  break;
965 
966  case ISD::ABS:
967  case ISD::BITREVERSE:
968  case ISD::BSWAP:
969  case ISD::CTLZ:
970  case ISD::CTTZ:
973  case ISD::CTPOP:
974  case ISD::FABS:
975  case ISD::FCEIL:
976  case ISD::FCOS:
977  case ISD::FEXP:
978  case ISD::FEXP2:
979  case ISD::FFLOOR:
980  case ISD::FLOG:
981  case ISD::FLOG10:
982  case ISD::FLOG2:
983  case ISD::FNEARBYINT:
984  case ISD::FNEG:
985  case ISD::FREEZE:
986  case ISD::FP_EXTEND:
987  case ISD::FP_ROUND:
988  case ISD::FP_TO_SINT:
989  case ISD::FP_TO_UINT:
990  case ISD::FRINT:
991  case ISD::FROUND:
992  case ISD::FROUNDEVEN:
993  case ISD::FSIN:
994  case ISD::FSQRT:
995  case ISD::FTRUNC:
996  case ISD::SINT_TO_FP:
997  case ISD::TRUNCATE:
998  case ISD::UINT_TO_FP:
999  case ISD::FCANONICALIZE:
1000  SplitVecRes_UnaryOp(N, Lo, Hi);
1001  break;
1002 
1003  case ISD::ANY_EXTEND:
1004  case ISD::SIGN_EXTEND:
1005  case ISD::ZERO_EXTEND:
1006  SplitVecRes_ExtendOp(N, Lo, Hi);
1007  break;
1008 
1009  case ISD::ADD:
1010  case ISD::SUB:
1011  case ISD::MUL:
1012  case ISD::MULHS:
1013  case ISD::MULHU:
1014  case ISD::FADD:
1015  case ISD::FSUB:
1016  case ISD::FMUL:
1017  case ISD::FMINNUM:
1018  case ISD::FMAXNUM:
1019  case ISD::FMINIMUM:
1020  case ISD::FMAXIMUM:
1021  case ISD::SDIV:
1022  case ISD::UDIV:
1023  case ISD::FDIV:
1024  case ISD::FPOW:
1025  case ISD::AND:
1026  case ISD::OR:
1027  case ISD::XOR:
1028  case ISD::SHL:
1029  case ISD::SRA:
1030  case ISD::SRL:
1031  case ISD::UREM:
1032  case ISD::SREM:
1033  case ISD::FREM:
1034  case ISD::SMIN:
1035  case ISD::SMAX:
1036  case ISD::UMIN:
1037  case ISD::UMAX:
1038  case ISD::SADDSAT:
1039  case ISD::UADDSAT:
1040  case ISD::SSUBSAT:
1041  case ISD::USUBSAT:
1042  case ISD::SSHLSAT:
1043  case ISD::USHLSAT:
1044  case ISD::ROTL:
1045  case ISD::ROTR:
1046  SplitVecRes_BinOp(N, Lo, Hi);
1047  break;
1048  case ISD::FMA:
1049  case ISD::FSHL:
1050  case ISD::FSHR:
1051  SplitVecRes_TernaryOp(N, Lo, Hi);
1052  break;
1053 
1054 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1055  case ISD::STRICT_##DAGN:
1056 #include "llvm/IR/ConstrainedOps.def"
1057  SplitVecRes_StrictFPOp(N, Lo, Hi);
1058  break;
1059 
1060  case ISD::FP_TO_UINT_SAT:
1061  case ISD::FP_TO_SINT_SAT:
1062  SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1063  break;
1064 
1065  case ISD::UADDO:
1066  case ISD::SADDO:
1067  case ISD::USUBO:
1068  case ISD::SSUBO:
1069  case ISD::UMULO:
1070  case ISD::SMULO:
1071  SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1072  break;
1073  case ISD::SMULFIX:
1074  case ISD::SMULFIXSAT:
1075  case ISD::UMULFIX:
1076  case ISD::UMULFIXSAT:
1077  case ISD::SDIVFIX:
1078  case ISD::SDIVFIXSAT:
1079  case ISD::UDIVFIX:
1080  case ISD::UDIVFIXSAT:
1081  SplitVecRes_FIX(N, Lo, Hi);
1082  break;
1083  }
1084 
1085  // If Lo/Hi is null, the sub-method took care of registering results etc.
1086  if (Lo.getNode())
1087  SetSplitVector(SDValue(N, ResNo), Lo, Hi);
1088 }
1089 
1090 void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1091  MachinePointerInfo &MPI, SDValue &Ptr,
1092  uint64_t *ScaledOffset) {
1093  SDLoc DL(N);
1094  unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinSize() / 8;
1095 
1096  if (MemVT.isScalableVector()) {
1097  SDNodeFlags Flags;
1098  SDValue BytesIncrement = DAG.getVScale(
1099  DL, Ptr.getValueType(),
1100  APInt(Ptr.getValueSizeInBits().getFixedSize(), IncrementSize));
1101  MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1102  Flags.setNoUnsignedWrap(true);
1103  if (ScaledOffset)
1104  *ScaledOffset += IncrementSize;
1105  Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, BytesIncrement,
1106  Flags);
1107  } else {
1108  MPI = N->getPointerInfo().getWithOffset(IncrementSize);
1109  // Increment the pointer to the other half.
1110  Ptr = DAG.getObjectPtrOffset(DL, Ptr, TypeSize::Fixed(IncrementSize));
1111  }
1112 }
1113 
1114 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
1115  SDValue &Hi) {
1116  SDValue LHSLo, LHSHi;
1117  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1118  SDValue RHSLo, RHSHi;
1119  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1120  SDLoc dl(N);
1121 
1122  const SDNodeFlags Flags = N->getFlags();
1123  unsigned Opcode = N->getOpcode();
1124  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
1125  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
1126 }
1127 
1128 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1129  SDValue &Hi) {
1130  SDValue Op0Lo, Op0Hi;
1131  GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
1132  SDValue Op1Lo, Op1Hi;
1133  GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
1134  SDValue Op2Lo, Op2Hi;
1135  GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
1136  SDLoc dl(N);
1137 
1138  Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(), Op0Lo, Op1Lo,
1139  Op2Lo, N->getFlags());
1140  Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(), Op0Hi, Op1Hi,
1141  Op2Hi, N->getFlags());
1142 }
1143 
1144 void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1145  SDValue LHSLo, LHSHi;
1146  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1147  SDValue RHSLo, RHSHi;
1148  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1149  SDLoc dl(N);
1150  SDValue Op2 = N->getOperand(2);
1151 
1152  unsigned Opcode = N->getOpcode();
1153  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2,
1154  N->getFlags());
1155  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2,
1156  N->getFlags());
1157 }
1158 
1159 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1160  SDValue &Hi) {
1161  // We know the result is a vector. The input may be either a vector or a
1162  // scalar value.
1163  EVT LoVT, HiVT;
1164  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1165  SDLoc dl(N);
1166 
1167  SDValue InOp = N->getOperand(0);
1168  EVT InVT = InOp.getValueType();
1169 
1170  // Handle some special cases efficiently.
1171  switch (getTypeAction(InVT)) {
1179  break;
1182  // A scalar to vector conversion, where the scalar needs expansion.
1183  // If the vector is being split in two then we can just convert the
1184  // expanded pieces.
1185  if (LoVT == HiVT) {
1186  GetExpandedOp(InOp, Lo, Hi);
1187  if (DAG.getDataLayout().isBigEndian())
1188  std::swap(Lo, Hi);
1189  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1190  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1191  return;
1192  }
1193  break;
1195  // If the input is a vector that needs to be split, convert each split
1196  // piece of the input now.
1197  GetSplitVector(InOp, Lo, Hi);
1198  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1199  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1200  return;
1202  report_fatal_error("Scalarization of scalable vectors is not supported.");
1203  }
1204 
1205  // In the general case, convert the input to an integer and split it by hand.
1206  EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
1207  EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
1208  if (DAG.getDataLayout().isBigEndian())
1209  std::swap(LoIntVT, HiIntVT);
1210 
1211  SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
1212 
1213  if (DAG.getDataLayout().isBigEndian())
1214  std::swap(Lo, Hi);
1215  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1216  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1217 }
1218 
1219 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1220  SDValue &Hi) {
1221  EVT LoVT, HiVT;
1222  SDLoc dl(N);
1223  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1224  unsigned LoNumElts = LoVT.getVectorNumElements();
1225  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1226  Lo = DAG.getBuildVector(LoVT, dl, LoOps);
1227 
1228  SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1229  Hi = DAG.getBuildVector(HiVT, dl, HiOps);
1230 }
1231 
1232 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1233  SDValue &Hi) {
1234  assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1235  SDLoc dl(N);
1236  unsigned NumSubvectors = N->getNumOperands() / 2;
1237  if (NumSubvectors == 1) {
1238  Lo = N->getOperand(0);
1239  Hi = N->getOperand(1);
1240  return;
1241  }
1242 
1243  EVT LoVT, HiVT;
1244  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1245 
1246  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1247  Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
1248 
1249  SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1250  Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
1251 }
1252 
1253 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1254  SDValue &Hi) {
1255  SDValue Vec = N->getOperand(0);
1256  SDValue Idx = N->getOperand(1);
1257  SDLoc dl(N);
1258 
1259  EVT LoVT, HiVT;
1260  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1261 
1262  Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
1263  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1264  Hi = DAG.getNode(
1265  ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
1266  DAG.getVectorIdxConstant(IdxVal + LoVT.getVectorMinNumElements(), dl));
1267 }
1268 
1269 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1270  SDValue &Hi) {
1271  SDValue Vec = N->getOperand(0);
1272  SDValue SubVec = N->getOperand(1);
1273  SDValue Idx = N->getOperand(2);
1274  SDLoc dl(N);
1275  GetSplitVector(Vec, Lo, Hi);
1276 
1277  EVT VecVT = Vec.getValueType();
1278  EVT LoVT = Lo.getValueType();
1279  EVT SubVecVT = SubVec.getValueType();
1280  unsigned VecElems = VecVT.getVectorMinNumElements();
1281  unsigned SubElems = SubVecVT.getVectorMinNumElements();
1282  unsigned LoElems = LoVT.getVectorMinNumElements();
1283 
1284  // If we know the index is in the first half, and we know the subvector
1285  // doesn't cross the boundary between the halves, we can avoid spilling the
1286  // vector, and insert into the lower half of the split vector directly.
1287  unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1288  if (IdxVal + SubElems <= LoElems) {
1289  Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1290  return;
1291  }
1292  // Similarly if the subvector is fully in the high half, but mind that we
1293  // can't tell whether a fixed-length subvector is fully within the high half
1294  // of a scalable vector.
1295  if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1296  IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1297  Hi = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, Hi.getValueType(), Hi, SubVec,
1298  DAG.getVectorIdxConstant(IdxVal - LoElems, dl));
1299  return;
1300  }
1301 
1302  // Spill the vector to the stack.
1303  // In cases where the vector is illegal it will be broken down into parts
1304  // and stored in parts - we should use the alignment for the smallest part.
1305  Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1306  SDValue StackPtr =
1307  DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1308  auto &MF = DAG.getMachineFunction();
1309  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1310  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1311 
1312  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1313  SmallestAlign);
1314 
1315  // Store the new subvector into the specified index.
1316  SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1317  Store = DAG.getStore(Store, dl, SubVec, SubVecPtr,
1319 
1320  // Load the Lo part from the stack slot.
1321  Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, PtrInfo,
1322  SmallestAlign);
1323 
1324  // Increment the pointer to the other part.
1325  auto *Load = cast<LoadSDNode>(Lo);
1326  MachinePointerInfo MPI = Load->getPointerInfo();
1327  IncrementPointer(Load, LoVT, MPI, StackPtr);
1328 
1329  // Load the Hi part from the stack slot.
1330  Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MPI, SmallestAlign);
1331 }
1332 
1333 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1334  SDValue &Hi) {
1335  SDLoc dl(N);
1336  GetSplitVector(N->getOperand(0), Lo, Hi);
1337  Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
1338  Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
1339 }
1340 
1341 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1342  SDValue &Hi) {
1343  SDValue LHSLo, LHSHi;
1344  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1345  SDLoc DL(N);
1346 
1347  SDValue RHSLo, RHSHi;
1348  SDValue RHS = N->getOperand(1);
1349  EVT RHSVT = RHS.getValueType();
1350  if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1351  GetSplitVector(RHS, RHSLo, RHSHi);
1352  else
1353  std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1354 
1355 
1356  Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
1357  Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
1358 }
1359 
1360 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1361  SDValue &Hi) {
1362  SDValue LHSLo, LHSHi;
1363  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1364  SDLoc dl(N);
1365 
1366  EVT LoVT, HiVT;
1367  std::tie(LoVT, HiVT) =
1368  DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1369 
1370  Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1371  DAG.getValueType(LoVT));
1372  Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1373  DAG.getValueType(HiVT));
1374 }
1375 
1376 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1377  SDValue &Hi) {
1378  unsigned Opcode = N->getOpcode();
1379  SDValue N0 = N->getOperand(0);
1380 
1381  SDLoc dl(N);
1382  SDValue InLo, InHi;
1383 
1384  if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1385  GetSplitVector(N0, InLo, InHi);
1386  else
1387  std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1388 
1389  EVT InLoVT = InLo.getValueType();
1390  unsigned InNumElements = InLoVT.getVectorNumElements();
1391 
1392  EVT OutLoVT, OutHiVT;
1393  std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1394  unsigned OutNumElements = OutLoVT.getVectorNumElements();
1395  assert((2 * OutNumElements) <= InNumElements &&
1396  "Illegal extend vector in reg split");
1397 
1398  // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1399  // input vector (i.e. we only use InLo):
1400  // OutLo will extend the first OutNumElements from InLo.
1401  // OutHi will extend the next OutNumElements from InLo.
1402 
1403  // Shuffle the elements from InLo for OutHi into the bottom elements to
1404  // create a 'fake' InHi.
1405  SmallVector<int, 8> SplitHi(InNumElements, -1);
1406  for (unsigned i = 0; i != OutNumElements; ++i)
1407  SplitHi[i] = i + OutNumElements;
1408  InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1409 
1410  Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1411  Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1412 }
1413 
1414 void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1415  SDValue &Hi) {
1416  unsigned NumOps = N->getNumOperands();
1417  SDValue Chain = N->getOperand(0);
1418  EVT LoVT, HiVT;
1419  SDLoc dl(N);
1420  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1421 
1422  SmallVector<SDValue, 4> OpsLo(NumOps);
1423  SmallVector<SDValue, 4> OpsHi(NumOps);
1424 
1425  // The Chain is the first operand.
1426  OpsLo[0] = Chain;
1427  OpsHi[0] = Chain;
1428 
1429  // Now process the remaining operands.
1430  for (unsigned i = 1; i < NumOps; ++i) {
1431  SDValue Op = N->getOperand(i);
1432  SDValue OpLo = Op;
1433  SDValue OpHi = Op;
1434 
1435  EVT InVT = Op.getValueType();
1436  if (InVT.isVector()) {
1437  // If the input also splits, handle it directly for a
1438  // compile time speedup. Otherwise split it by hand.
1439  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1440  GetSplitVector(Op, OpLo, OpHi);
1441  else
1442  std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1443  }
1444 
1445  OpsLo[i] = OpLo;
1446  OpsHi[i] = OpHi;
1447  }
1448 
1449  EVT LoValueVTs[] = {LoVT, MVT::Other};
1450  EVT HiValueVTs[] = {HiVT, MVT::Other};
1451  Lo = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo,
1452  N->getFlags());
1453  Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi,
1454  N->getFlags());
1455 
1456  // Build a factor node to remember that this Op is independent of the
1457  // other one.
1458  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1459  Lo.getValue(1), Hi.getValue(1));
1460 
1461  // Legalize the chain result - switch anything that used the old chain to
1462  // use the new one.
1463  ReplaceValueWith(SDValue(N, 1), Chain);
1464 }
1465 
1466 SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1467  SDValue Chain = N->getOperand(0);
1468  EVT VT = N->getValueType(0);
1469  unsigned NE = VT.getVectorNumElements();
1470  EVT EltVT = VT.getVectorElementType();
1471  SDLoc dl(N);
1472 
1473  SmallVector<SDValue, 8> Scalars;
1474  SmallVector<SDValue, 4> Operands(N->getNumOperands());
1475 
1476  // If ResNE is 0, fully unroll the vector op.
1477  if (ResNE == 0)
1478  ResNE = NE;
1479  else if (NE > ResNE)
1480  NE = ResNE;
1481 
1482  //The results of each unrolled operation, including the chain.
1483  EVT ChainVTs[] = {EltVT, MVT::Other};
1484  SmallVector<SDValue, 8> Chains;
1485 
1486  unsigned i;
1487  for (i = 0; i != NE; ++i) {
1488  Operands[0] = Chain;
1489  for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
1490  SDValue Operand = N->getOperand(j);
1491  EVT OperandVT = Operand.getValueType();
1492  if (OperandVT.isVector()) {
1493  EVT OperandEltVT = OperandVT.getVectorElementType();
1494  Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
1495  Operand, DAG.getVectorIdxConstant(i, dl));
1496  } else {
1497  Operands[j] = Operand;
1498  }
1499  }
1500  SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1501  Scalar.getNode()->setFlags(N->getFlags());
1502 
1503  //Add in the scalar as well as its chain value to the
1504  //result vectors.
1505  Scalars.push_back(Scalar);
1506  Chains.push_back(Scalar.getValue(1));
1507  }
1508 
1509  for (; i < ResNE; ++i)
1510  Scalars.push_back(DAG.getUNDEF(EltVT));
1511 
1512  // Build a new factor node to connect the chain back together.
1513  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1514  ReplaceValueWith(SDValue(N, 1), Chain);
1515 
1516  // Create a new BUILD_VECTOR node
1517  EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE);
1518  return DAG.getBuildVector(VecVT, dl, Scalars);
1519 }
1520 
1521 void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1522  SDValue &Lo, SDValue &Hi) {
1523  SDLoc dl(N);
1524  EVT ResVT = N->getValueType(0);
1525  EVT OvVT = N->getValueType(1);
1526  EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1527  std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
1528  std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
1529 
1530  SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1531  if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
1532  GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
1533  GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
1534  } else {
1535  std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
1536  std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
1537  }
1538 
1539  unsigned Opcode = N->getOpcode();
1540  SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
1541  SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
1542  SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
1543  SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
1544  LoNode->setFlags(N->getFlags());
1545  HiNode->setFlags(N->getFlags());
1546 
1547  Lo = SDValue(LoNode, ResNo);
1548  Hi = SDValue(HiNode, ResNo);
1549 
1550  // Replace the other vector result not being explicitly split here.
1551  unsigned OtherNo = 1 - ResNo;
1552  EVT OtherVT = N->getValueType(OtherNo);
1553  if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
1554  SetSplitVector(SDValue(N, OtherNo),
1555  SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1556  } else {
1557  SDValue OtherVal = DAG.getNode(
1558  ISD::CONCAT_VECTORS, dl, OtherVT,
1559  SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1560  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
1561  }
1562 }
1563 
1564 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1565  SDValue &Hi) {
1566  SDValue Vec = N->getOperand(0);
1567  SDValue Elt = N->getOperand(1);
1568  SDValue Idx = N->getOperand(2);
1569  SDLoc dl(N);
1570  GetSplitVector(Vec, Lo, Hi);
1571 
1572  if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1573  unsigned IdxVal = CIdx->getZExtValue();
1574  unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
1575  if (IdxVal < LoNumElts) {
1576  Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1577  Lo.getValueType(), Lo, Elt, Idx);
1578  return;
1579  } else if (!Vec.getValueType().isScalableVector()) {
1580  Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1581  DAG.getVectorIdxConstant(IdxVal - LoNumElts, dl));
1582  return;
1583  }
1584  }
1585 
1586  // See if the target wants to custom expand this node.
1587  if (CustomLowerNode(N, N->getValueType(0), true))
1588  return;
1589 
1590  // Make the vector elements byte-addressable if they aren't already.
1591  EVT VecVT = Vec.getValueType();
1592  EVT EltVT = VecVT.getVectorElementType();
1593  if (VecVT.getScalarSizeInBits() < 8) {
1594  EltVT = MVT::i8;
1595  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1596  VecVT.getVectorElementCount());
1597  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1598  // Extend the element type to match if needed.
1599  if (EltVT.bitsGT(Elt.getValueType()))
1600  Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1601  }
1602 
1603  // Spill the vector to the stack.
1604  // In cases where the vector is illegal it will be broken down into parts
1605  // and stored in parts - we should use the alignment for the smallest part.
1606  Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1607  SDValue StackPtr =
1608  DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1609  auto &MF = DAG.getMachineFunction();
1610  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1611  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1612 
1613  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1614  SmallestAlign);
1615 
1616  // Store the new element. This may be larger than the vector element type,
1617  // so use a truncating store.
1618  SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1619  Store = DAG.getTruncStore(
1620  Store, dl, Elt, EltPtr, MachinePointerInfo::getUnknownStack(MF), EltVT,
1621  commonAlignment(SmallestAlign,
1622  EltVT.getFixedSizeInBits() / 8));
1623 
1624  EVT LoVT, HiVT;
1625  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1626 
1627  // Load the Lo part from the stack slot.
1628  Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo, SmallestAlign);
1629 
1630  // Increment the pointer to the other part.
1631  auto Load = cast<LoadSDNode>(Lo);
1632  MachinePointerInfo MPI = Load->getPointerInfo();
1633  IncrementPointer(Load, LoVT, MPI, StackPtr);
1634 
1635  Hi = DAG.getLoad(HiVT, dl, Store, StackPtr, MPI, SmallestAlign);
1636 
1637  // If we adjusted the original type, we need to truncate the results.
1638  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1639  if (LoVT != Lo.getValueType())
1640  Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1641  if (HiVT != Hi.getValueType())
1642  Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1643 }
1644 
1645 void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
1646  SDValue &Hi) {
1647  EVT LoVT, HiVT;
1648  SDLoc dl(N);
1649  assert(N->getValueType(0).isScalableVector() &&
1650  "Only scalable vectors are supported for STEP_VECTOR");
1651  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1652  SDValue Step = N->getOperand(0);
1653 
1654  Lo = DAG.getNode(ISD::STEP_VECTOR, dl, LoVT, Step);
1655 
1656  // Hi = Lo + (EltCnt * Step)
1657  EVT EltVT = Step.getValueType();
1658  APInt StepVal = cast<ConstantSDNode>(Step)->getAPIntValue();
1659  SDValue StartOfHi =
1660  DAG.getVScale(dl, EltVT, StepVal * LoVT.getVectorMinNumElements());
1661  StartOfHi = DAG.getSExtOrTrunc(StartOfHi, dl, HiVT.getVectorElementType());
1662  StartOfHi = DAG.getNode(ISD::SPLAT_VECTOR, dl, HiVT, StartOfHi);
1663 
1664  Hi = DAG.getNode(ISD::STEP_VECTOR, dl, HiVT, Step);
1665  Hi = DAG.getNode(ISD::ADD, dl, HiVT, Hi, StartOfHi);
1666 }
1667 
1668 void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
1669  SDValue &Hi) {
1670  EVT LoVT, HiVT;
1671  SDLoc dl(N);
1672  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1673  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, N->getOperand(0));
1674  if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
1675  Hi = DAG.getUNDEF(HiVT);
1676  } else {
1677  assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
1678  Hi = Lo;
1679  }
1680 }
1681 
1682 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1683  SDValue &Hi) {
1684  assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1685  EVT LoVT, HiVT;
1686  SDLoc dl(LD);
1687  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1688 
1689  ISD::LoadExtType ExtType = LD->getExtensionType();
1690  SDValue Ch = LD->getChain();
1691  SDValue Ptr = LD->getBasePtr();
1692  SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1693  EVT MemoryVT = LD->getMemoryVT();
1694  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1695  AAMDNodes AAInfo = LD->getAAInfo();
1696 
1697  EVT LoMemVT, HiMemVT;
1698  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1699 
1700  if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
1701  SDValue Value, NewChain;
1702  std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
1703  std::tie(Lo, Hi) = DAG.SplitVector(Value, dl);
1704  ReplaceValueWith(SDValue(LD, 1), NewChain);
1705  return;
1706  }
1707 
1708  Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1709  LD->getPointerInfo(), LoMemVT, LD->getOriginalAlign(),
1710  MMOFlags, AAInfo);
1711 
1712  MachinePointerInfo MPI;
1713  IncrementPointer(LD, LoMemVT, MPI, Ptr);
1714 
1715  Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, MPI,
1716  HiMemVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
1717 
1718  // Build a factor node to remember that this load is independent of the
1719  // other one.
1720  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1721  Hi.getValue(1));
1722 
1723  // Legalize the chain result - switch anything that used the old chain to
1724  // use the new one.
1725  ReplaceValueWith(SDValue(LD, 1), Ch);
1726 }
1727 
1728 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
1729  SDValue &Lo, SDValue &Hi) {
1730  assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
1731  EVT LoVT, HiVT;
1732  SDLoc dl(MLD);
1733  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1734 
1735  SDValue Ch = MLD->getChain();
1736  SDValue Ptr = MLD->getBasePtr();
1737  SDValue Offset = MLD->getOffset();
1738  assert(Offset.isUndef() && "Unexpected indexed masked load offset");
1739  SDValue Mask = MLD->getMask();
1740  SDValue PassThru = MLD->getPassThru();
1741  Align Alignment = MLD->getOriginalAlign();
1742  ISD::LoadExtType ExtType = MLD->getExtensionType();
1743 
1744  // Split Mask operand
1745  SDValue MaskLo, MaskHi;
1746  if (Mask.getOpcode() == ISD::SETCC) {
1747  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1748  } else {
1749  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1750  GetSplitVector(Mask, MaskLo, MaskHi);
1751  else
1752  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1753  }
1754 
1755  EVT MemoryVT = MLD->getMemoryVT();
1756  EVT LoMemVT, HiMemVT;
1757  bool HiIsEmpty = false;
1758  std::tie(LoMemVT, HiMemVT) =
1759  DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
1760 
1761  SDValue PassThruLo, PassThruHi;
1762  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1763  GetSplitVector(PassThru, PassThruLo, PassThruHi);
1764  else
1765  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1766 
1767  unsigned LoSize = MemoryLocation::getSizeOrUnknown(LoMemVT.getStoreSize());
1769  MLD->getPointerInfo(), MachineMemOperand::MOLoad, LoSize, Alignment,
1770  MLD->getAAInfo(), MLD->getRanges());
1771 
1772  Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
1773  MMO, MLD->getAddressingMode(), ExtType,
1774  MLD->isExpandingLoad());
1775 
1776  if (HiIsEmpty) {
1777  // The hi masked load has zero storage size. We therefore simply set it to
1778  // the low masked load and rely on subsequent removal from the chain.
1779  Hi = Lo;
1780  } else {
1781  // Generate hi masked load.
1782  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1783  MLD->isExpandingLoad());
1784  unsigned HiSize = MemoryLocation::getSizeOrUnknown(HiMemVT.getStoreSize());
1785 
1786  MachinePointerInfo MPI;
1787  if (LoMemVT.isScalableVector())
1789  else
1790  MPI = MLD->getPointerInfo().getWithOffset(
1791  LoMemVT.getStoreSize().getFixedSize());
1792 
1794  MPI, MachineMemOperand::MOLoad, HiSize, Alignment, MLD->getAAInfo(),
1795  MLD->getRanges());
1796 
1797  Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
1798  HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
1799  MLD->isExpandingLoad());
1800  }
1801 
1802  // Build a factor node to remember that this load is independent of the
1803  // other one.
1804  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1805  Hi.getValue(1));
1806 
1807  // Legalize the chain result - switch anything that used the old chain to
1808  // use the new one.
1809  ReplaceValueWith(SDValue(MLD, 1), Ch);
1810 
1811 }
1812 
1813 void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
1814  SDValue &Lo, SDValue &Hi) {
1815  EVT LoVT, HiVT;
1816  SDLoc dl(MGT);
1817  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1818 
1819  SDValue Ch = MGT->getChain();
1820  SDValue Ptr = MGT->getBasePtr();
1821  SDValue Mask = MGT->getMask();
1822  SDValue PassThru = MGT->getPassThru();
1823  SDValue Index = MGT->getIndex();
1824  SDValue Scale = MGT->getScale();
1825  EVT MemoryVT = MGT->getMemoryVT();
1826  Align Alignment = MGT->getOriginalAlign();
1827  ISD::LoadExtType ExtType = MGT->getExtensionType();
1828 
1829  // Split Mask operand
1830  SDValue MaskLo, MaskHi;
1831  if (Mask.getOpcode() == ISD::SETCC) {
1832  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1833  } else {
1834  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1835  GetSplitVector(Mask, MaskLo, MaskHi);
1836  else
1837  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1838  }
1839 
1840  EVT LoMemVT, HiMemVT;
1841  // Split MemoryVT
1842  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1843 
1844  SDValue PassThruLo, PassThruHi;
1845  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1846  GetSplitVector(PassThru, PassThruLo, PassThruHi);
1847  else
1848  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1849 
1850  SDValue IndexHi, IndexLo;
1851  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1852  GetSplitVector(Index, IndexLo, IndexHi);
1853  else
1854  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1855 
1858  MemoryLocation::UnknownSize, Alignment, MGT->getAAInfo(),
1859  MGT->getRanges());
1860 
1861  SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
1862  Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, OpsLo,
1863  MMO, MGT->getIndexType(), ExtType);
1864 
1865  SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
1866  Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, OpsHi,
1867  MMO, MGT->getIndexType(), ExtType);
1868 
1869  // Build a factor node to remember that this load is independent of the
1870  // other one.
1871  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1872  Hi.getValue(1));
1873 
1874  // Legalize the chain result - switch anything that used the old chain to
1875  // use the new one.
1876  ReplaceValueWith(SDValue(MGT, 1), Ch);
1877 }
1878 
1879 
1880 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1881  assert(N->getValueType(0).isVector() &&
1882  N->getOperand(0).getValueType().isVector() &&
1883  "Operand types must be vectors");
1884 
1885  EVT LoVT, HiVT;
1886  SDLoc DL(N);
1887  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1888 
1889  // If the input also splits, handle it directly. Otherwise split it by hand.
1890  SDValue LL, LH, RL, RH;
1891  if (getTypeAction(N->getOperand(0).getValueType()) ==
1893  GetSplitVector(N->getOperand(0), LL, LH);
1894  else
1895  std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
1896 
1897  if (getTypeAction(N->getOperand(1).getValueType()) ==
1899  GetSplitVector(N->getOperand(1), RL, RH);
1900  else
1901  std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
1902 
1903  Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
1904  Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
1905 }
1906 
1907 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
1908  SDValue &Hi) {
1909  // Get the dest types - they may not match the input types, e.g. int_to_fp.
1910  EVT LoVT, HiVT;
1911  SDLoc dl(N);
1912  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1913 
1914  // If the input also splits, handle it directly for a compile time speedup.
1915  // Otherwise split it by hand.
1916  unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1917  EVT InVT = N->getOperand(OpNo).getValueType();
1918  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1919  GetSplitVector(N->getOperand(OpNo), Lo, Hi);
1920  else
1921  std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, OpNo);
1922 
1923  if (N->getOpcode() == ISD::FP_ROUND) {
1924  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1),
1925  N->getFlags());
1926  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1),
1927  N->getFlags());
1928  } else {
1929  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getFlags());
1930  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getFlags());
1931  }
1932 }
1933 
1934 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
1935  SDValue &Hi) {
1936  SDLoc dl(N);
1937  EVT SrcVT = N->getOperand(0).getValueType();
1938  EVT DestVT = N->getValueType(0);
1939  EVT LoVT, HiVT;
1940  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
1941 
1942  // We can do better than a generic split operation if the extend is doing
1943  // more than just doubling the width of the elements and the following are
1944  // true:
1945  // - The number of vector elements is even,
1946  // - the source type is legal,
1947  // - the type of a split source is illegal,
1948  // - the type of an extended (by doubling element size) source is legal, and
1949  // - the type of that extended source when split is legal.
1950  //
1951  // This won't necessarily completely legalize the operation, but it will
1952  // more effectively move in the right direction and prevent falling down
1953  // to scalarization in many cases due to the input vector being split too
1954  // far.
1955  if (SrcVT.getVectorElementCount().isKnownEven() &&
1956  SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
1957  LLVMContext &Ctx = *DAG.getContext();
1958  EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
1959  EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
1960 
1961  EVT SplitLoVT, SplitHiVT;
1962  std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
1963  if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
1964  TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
1965  LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
1966  N->dump(&DAG); dbgs() << "\n");
1967  // Extend the source vector by one step.
1968  SDValue NewSrc =
1969  DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
1970  // Get the low and high halves of the new, extended one step, vector.
1971  std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
1972  // Extend those vector halves the rest of the way.
1973  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1974  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1975  return;
1976  }
1977  }
1978  // Fall back to the generic unary operator splitting otherwise.
1979  SplitVecRes_UnaryOp(N, Lo, Hi);
1980 }
1981 
1982 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
1983  SDValue &Lo, SDValue &Hi) {
1984  // The low and high parts of the original input give four input vectors.
1985  SDValue Inputs[4];
1986  SDLoc dl(N);
1987  GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
1988  GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
1989  EVT NewVT = Inputs[0].getValueType();
1990  unsigned NewElts = NewVT.getVectorNumElements();
1991 
1992  // If Lo or Hi uses elements from at most two of the four input vectors, then
1993  // express it as a vector shuffle of those two inputs. Otherwise extract the
1994  // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
1996  for (unsigned High = 0; High < 2; ++High) {
1997  SDValue &Output = High ? Hi : Lo;
1998 
1999  // Build a shuffle mask for the output, discovering on the fly which
2000  // input vectors to use as shuffle operands (recorded in InputUsed).
2001  // If building a suitable shuffle vector proves too hard, then bail
2002  // out with useBuildVector set.
2003  unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
2004  unsigned FirstMaskIdx = High * NewElts;
2005  bool useBuildVector = false;
2006  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
2007  // The mask element. This indexes into the input.
2008  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
2009 
2010  // The input vector this mask element indexes into.
2011  unsigned Input = (unsigned)Idx / NewElts;
2012 
2013  if (Input >= array_lengthof(Inputs)) {
2014  // The mask element does not index into any input vector.
2015  Ops.push_back(-1);
2016  continue;
2017  }
2018 
2019  // Turn the index into an offset from the start of the input vector.
2020  Idx -= Input * NewElts;
2021 
2022  // Find or create a shuffle vector operand to hold this input.
2023  unsigned OpNo;
2024  for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
2025  if (InputUsed[OpNo] == Input) {
2026  // This input vector is already an operand.
2027  break;
2028  } else if (InputUsed[OpNo] == -1U) {
2029  // Create a new operand for this input vector.
2030  InputUsed[OpNo] = Input;
2031  break;
2032  }
2033  }
2034 
2035  if (OpNo >= array_lengthof(InputUsed)) {
2036  // More than two input vectors used! Give up on trying to create a
2037  // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
2038  useBuildVector = true;
2039  break;
2040  }
2041 
2042  // Add the mask index for the new shuffle vector.
2043  Ops.push_back(Idx + OpNo * NewElts);
2044  }
2045 
2046  if (useBuildVector) {
2047  EVT EltVT = NewVT.getVectorElementType();
2049 
2050  // Extract the input elements by hand.
2051  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
2052  // The mask element. This indexes into the input.
2053  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
2054 
2055  // The input vector this mask element indexes into.
2056  unsigned Input = (unsigned)Idx / NewElts;
2057 
2058  if (Input >= array_lengthof(Inputs)) {
2059  // The mask element is "undef" or indexes off the end of the input.
2060  SVOps.push_back(DAG.getUNDEF(EltVT));
2061  continue;
2062  }
2063 
2064  // Turn the index into an offset from the start of the input vector.
2065  Idx -= Input * NewElts;
2066 
2067  // Extract the vector element by hand.
2068  SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
2069  Inputs[Input],
2070  DAG.getVectorIdxConstant(Idx, dl)));
2071  }
2072 
2073  // Construct the Lo/Hi output using a BUILD_VECTOR.
2074  Output = DAG.getBuildVector(NewVT, dl, SVOps);
2075  } else if (InputUsed[0] == -1U) {
2076  // No input vectors were used! The result is undefined.
2077  Output = DAG.getUNDEF(NewVT);
2078  } else {
2079  SDValue Op0 = Inputs[InputUsed[0]];
2080  // If only one input was used, use an undefined vector for the other.
2081  SDValue Op1 = InputUsed[1] == -1U ?
2082  DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
2083  // At least one input vector was used. Create a new shuffle vector.
2084  Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
2085  }
2086 
2087  Ops.clear();
2088  }
2089 }
2090 
2091 void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2092  EVT OVT = N->getValueType(0);
2093  EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext());
2094  SDValue Chain = N->getOperand(0);
2095  SDValue Ptr = N->getOperand(1);
2096  SDValue SV = N->getOperand(2);
2097  SDLoc dl(N);
2098 
2099  const Align Alignment =
2101 
2102  Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment.value());
2103  Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment.value());
2104  Chain = Hi.getValue(1);
2105 
2106  // Modified the chain - switch anything that used the old chain to use
2107  // the new one.
2108  ReplaceValueWith(SDValue(N, 1), Chain);
2109 }
2110 
2111 void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
2112  SDValue &Hi) {
2113  EVT DstVTLo, DstVTHi;
2114  std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(N->getValueType(0));
2115  SDLoc dl(N);
2116 
2117  SDValue SrcLo, SrcHi;
2118  EVT SrcVT = N->getOperand(0).getValueType();
2119  if (getTypeAction(SrcVT) == TargetLowering::TypeSplitVector)
2120  GetSplitVector(N->getOperand(0), SrcLo, SrcHi);
2121  else
2122  std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(N, 0);
2123 
2124  Lo = DAG.getNode(N->getOpcode(), dl, DstVTLo, SrcLo, N->getOperand(1));
2125  Hi = DAG.getNode(N->getOpcode(), dl, DstVTHi, SrcHi, N->getOperand(1));
2126 }
2127 
2128 //===----------------------------------------------------------------------===//
2129 // Operand Vector Splitting
2130 //===----------------------------------------------------------------------===//
2131 
2132 /// This method is called when the specified operand of the specified node is
2133 /// found to need vector splitting. At this point, all of the result types of
2134 /// the node are known to be legal, but other operands of the node may need
2135 /// legalization as well as the specified one.
2136 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
2137  LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
2138  SDValue Res = SDValue();
2139 
2140  // See if the target wants to custom split this node.
2141  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2142  return false;
2143 
2144  switch (N->getOpcode()) {
2145  default:
2146 #ifndef NDEBUG
2147  dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
2148  N->dump(&DAG);
2149  dbgs() << "\n";
2150 #endif
2151  report_fatal_error("Do not know how to split this operator's "
2152  "operand!\n");
2153 
2154  case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
2155  case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
2156  case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
2157  case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
2158  case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
2159  case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
2160  case ISD::TRUNCATE:
2161  Res = SplitVecOp_TruncateHelper(N);
2162  break;
2163  case ISD::STRICT_FP_ROUND:
2164  case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
2165  case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
2166  case ISD::STORE:
2167  Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
2168  break;
2169  case ISD::MSTORE:
2170  Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
2171  break;
2172  case ISD::MSCATTER:
2173  Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
2174  break;
2175  case ISD::MGATHER:
2176  Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
2177  break;
2178  case ISD::VSELECT:
2179  Res = SplitVecOp_VSELECT(N, OpNo);
2180  break;
2183  case ISD::SINT_TO_FP:
2184  case ISD::UINT_TO_FP:
2185  if (N->getValueType(0).bitsLT(
2186  N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType()))
2187  Res = SplitVecOp_TruncateHelper(N);
2188  else
2189  Res = SplitVecOp_UnaryOp(N);
2190  break;
2191  case ISD::FP_TO_SINT_SAT:
2192  case ISD::FP_TO_UINT_SAT:
2193  Res = SplitVecOp_FP_TO_XINT_SAT(N);
2194  break;
2195  case ISD::FP_TO_SINT:
2196  case ISD::FP_TO_UINT:
2199  case ISD::STRICT_FP_EXTEND:
2200  case ISD::FP_EXTEND:
2201  case ISD::SIGN_EXTEND:
2202  case ISD::ZERO_EXTEND:
2203  case ISD::ANY_EXTEND:
2204  case ISD::FTRUNC:
2205  Res = SplitVecOp_UnaryOp(N);
2206  break;
2207 
2211  Res = SplitVecOp_ExtVecInRegOp(N);
2212  break;
2213 
2214  case ISD::VECREDUCE_FADD:
2215  case ISD::VECREDUCE_FMUL:
2216  case ISD::VECREDUCE_ADD:
2217  case ISD::VECREDUCE_MUL:
2218  case ISD::VECREDUCE_AND:
2219  case ISD::VECREDUCE_OR:
2220  case ISD::VECREDUCE_XOR:
2221  case ISD::VECREDUCE_SMAX:
2222  case ISD::VECREDUCE_SMIN:
2223  case ISD::VECREDUCE_UMAX:
2224  case ISD::VECREDUCE_UMIN:
2225  case ISD::VECREDUCE_FMAX:
2226  case ISD::VECREDUCE_FMIN:
2227  Res = SplitVecOp_VECREDUCE(N, OpNo);
2228  break;
2231  Res = SplitVecOp_VECREDUCE_SEQ(N);
2232  break;
2233  }
2234 
2235  // If the result is null, the sub-method took care of registering results etc.
2236  if (!Res.getNode()) return false;
2237 
2238  // If the result is N, the sub-method updated N in place. Tell the legalizer
2239  // core about this.
2240  if (Res.getNode() == N)
2241  return true;
2242 
2243  if (N->isStrictFPOpcode())
2244  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
2245  "Invalid operand expansion");
2246  else
2247  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2248  "Invalid operand expansion");
2249 
2250  ReplaceValueWith(SDValue(N, 0), Res);
2251  return false;
2252 }
2253 
2254 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
2255  // The only possibility for an illegal operand is the mask, since result type
2256  // legalization would have handled this node already otherwise.
2257  assert(OpNo == 0 && "Illegal operand must be mask");
2258 
2259  SDValue Mask = N->getOperand(0);
2260  SDValue Src0 = N->getOperand(1);
2261  SDValue Src1 = N->getOperand(2);
2262  EVT Src0VT = Src0.getValueType();
2263  SDLoc DL(N);
2264  assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
2265 
2266  SDValue Lo, Hi;
2267  GetSplitVector(N->getOperand(0), Lo, Hi);
2268  assert(Lo.getValueType() == Hi.getValueType() &&
2269  "Lo and Hi have differing types");
2270 
2271  EVT LoOpVT, HiOpVT;
2272  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
2273  assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
2274 
2275  SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
2276  std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
2277  std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
2278  std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
2279 
2280  SDValue LoSelect =
2281  DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
2282  SDValue HiSelect =
2283  DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
2284 
2285  return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
2286 }
2287 
2288 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
2289  EVT ResVT = N->getValueType(0);
2290  SDValue Lo, Hi;
2291  SDLoc dl(N);
2292 
2293  SDValue VecOp = N->getOperand(OpNo);
2294  EVT VecVT = VecOp.getValueType();
2295  assert(VecVT.isVector() && "Can only split reduce vector operand");
2296  GetSplitVector(VecOp, Lo, Hi);
2297  EVT LoOpVT, HiOpVT;
2298  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2299 
2300  // Use the appropriate scalar instruction on the split subvectors before
2301  // reducing the now partially reduced smaller vector.
2302  unsigned CombineOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
2303  SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
2304  return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
2305 }
2306 
2307 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
2308  EVT ResVT = N->getValueType(0);
2309  SDValue Lo, Hi;
2310  SDLoc dl(N);
2311 
2312  SDValue AccOp = N->getOperand(0);
2313  SDValue VecOp = N->getOperand(1);
2314  SDNodeFlags Flags = N->getFlags();
2315 
2316  EVT VecVT = VecOp.getValueType();
2317  assert(VecVT.isVector() && "Can only split reduce vector operand");
2318  GetSplitVector(VecOp, Lo, Hi);
2319  EVT LoOpVT, HiOpVT;
2320  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2321 
2322  // Reduce low half.
2323  SDValue Partial = DAG.getNode(N->getOpcode(), dl, ResVT, AccOp, Lo, Flags);
2324 
2325  // Reduce high half, using low half result as initial value.
2326  return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, Hi, Flags);
2327 }
2328 
2329 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
2330  // The result has a legal vector type, but the input needs splitting.
2331  EVT ResVT = N->getValueType(0);
2332  SDValue Lo, Hi;
2333  SDLoc dl(N);
2334  GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
2335  EVT InVT = Lo.getValueType();
2336 
2337  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2338  InVT.getVectorElementCount());
2339 
2340  if (N->isStrictFPOpcode()) {
2341  Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2342  { N->getOperand(0), Lo });
2343  Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2344  { N->getOperand(0), Hi });
2345 
2346  // Build a factor node to remember that this operation is independent
2347  // of the other one.
2348  SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2349  Hi.getValue(1));
2350 
2351  // Legalize the chain result - switch anything that used the old chain to
2352  // use the new one.
2353  ReplaceValueWith(SDValue(N, 1), Ch);
2354  } else {
2355  Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
2356  Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
2357  }
2358 
2359  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
2360 }
2361 
2362 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
2363  // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
2364  // end up being split all the way down to individual components. Convert the
2365  // split pieces into integers and reassemble.
2366  SDValue Lo, Hi;
2367  GetSplitVector(N->getOperand(0), Lo, Hi);
2368  Lo = BitConvertToInteger(Lo);
2369  Hi = BitConvertToInteger(Hi);
2370 
2371  if (DAG.getDataLayout().isBigEndian())
2372  std::swap(Lo, Hi);
2373 
2374  return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
2375  JoinIntegers(Lo, Hi));
2376 }
2377 
2378 SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
2379  unsigned OpNo) {
2380  assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
2381  // We know that the result type is legal.
2382  EVT ResVT = N->getValueType(0);
2383 
2384  SDValue Vec = N->getOperand(0);
2385  SDValue SubVec = N->getOperand(1);
2386  SDValue Idx = N->getOperand(2);
2387  SDLoc dl(N);
2388 
2389  SDValue Lo, Hi;
2390  GetSplitVector(SubVec, Lo, Hi);
2391 
2392  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2393  uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2394 
2395  SDValue FirstInsertion =
2396  DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Lo, Idx);
2397  SDValue SecondInsertion =
2398  DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, FirstInsertion, Hi,
2399  DAG.getVectorIdxConstant(IdxVal + LoElts, dl));
2400 
2401  return SecondInsertion;
2402 }
2403 
2404 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
2405  // We know that the extracted result type is legal.
2406  EVT SubVT = N->getValueType(0);
2407 
2408  SDValue Idx = N->getOperand(1);
2409  SDLoc dl(N);
2410  SDValue Lo, Hi;
2411 
2412  if (SubVT.isScalableVector() !=
2413  N->getOperand(0).getValueType().isScalableVector())
2414  report_fatal_error("Extracting a fixed-length vector from an illegal "
2415  "scalable vector is not yet supported");
2416 
2417  GetSplitVector(N->getOperand(0), Lo, Hi);
2418 
2419  uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2420  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2421 
2422  if (IdxVal < LoElts) {
2423  assert(IdxVal + SubVT.getVectorMinNumElements() <= LoElts &&
2424  "Extracted subvector crosses vector split!");
2425  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
2426  } else {
2427  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
2428  DAG.getVectorIdxConstant(IdxVal - LoElts, dl));
2429  }
2430 }
2431 
2432 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
2433  SDValue Vec = N->getOperand(0);
2434  SDValue Idx = N->getOperand(1);
2435  EVT VecVT = Vec.getValueType();
2436 
2437  if (isa<ConstantSDNode>(Idx)) {
2438  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2439 
2440  SDValue Lo, Hi;
2441  GetSplitVector(Vec, Lo, Hi);
2442 
2443  uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2444 
2445  if (IdxVal < LoElts)
2446  return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
2447  else if (!Vec.getValueType().isScalableVector())
2448  return SDValue(DAG.UpdateNodeOperands(N, Hi,
2449  DAG.getConstant(IdxVal - LoElts, SDLoc(N),
2450  Idx.getValueType())), 0);
2451  }
2452 
2453  // See if the target wants to custom expand this node.
2454  if (CustomLowerNode(N, N->getValueType(0), true))
2455  return SDValue();
2456 
2457  // Make the vector elements byte-addressable if they aren't already.
2458  SDLoc dl(N);
2459  EVT EltVT = VecVT.getVectorElementType();
2460  if (VecVT.getScalarSizeInBits() < 8) {
2461  EltVT = MVT::i8;
2462  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
2463  VecVT.getVectorElementCount());
2464  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
2465  }
2466 
2467  // Store the vector to the stack.
2468  // In cases where the vector is illegal it will be broken down into parts
2469  // and stored in parts - we should use the alignment for the smallest part.
2470  Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
2471  SDValue StackPtr =
2472  DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
2473  auto &MF = DAG.getMachineFunction();
2474  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
2475  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
2476  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
2477  SmallestAlign);
2478 
2479  // Load back the required element.
2480  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
2481 
2482  // FIXME: This is to handle i1 vectors with elements promoted to i8.
2483  // i1 vector handling needs general improvement.
2484  if (N->getValueType(0).bitsLT(EltVT)) {
2485  SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
2487  return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
2488  }
2489 
2490  return DAG.getExtLoad(
2491  ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
2493  commonAlignment(SmallestAlign, EltVT.getFixedSizeInBits() / 8));
2494 }
2495 
2496 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
2497  SDValue Lo, Hi;
2498 
2499  // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
2500  // splitting the result has the same effect as splitting the input operand.
2501  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
2502 
2503  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
2504 }
2505 
2506 SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
2507  unsigned OpNo) {
2508  EVT LoVT, HiVT;
2509  SDLoc dl(MGT);
2510  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
2511 
2512  SDValue Ch = MGT->getChain();
2513  SDValue Ptr = MGT->getBasePtr();
2514  SDValue Index = MGT->getIndex();
2515  SDValue Scale = MGT->getScale();
2516  SDValue Mask = MGT->getMask();
2517  SDValue PassThru = MGT->getPassThru();
2518  Align Alignment = MGT->getOriginalAlign();
2519  ISD::LoadExtType ExtType = MGT->getExtensionType();
2520 
2521  SDValue MaskLo, MaskHi;
2522  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2523  // Split Mask operand
2524  GetSplitVector(Mask, MaskLo, MaskHi);
2525  else
2526  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2527 
2528  EVT MemoryVT = MGT->getMemoryVT();
2529  EVT LoMemVT, HiMemVT;
2530  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2531 
2532  SDValue PassThruLo, PassThruHi;
2533  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2534  GetSplitVector(PassThru, PassThruLo, PassThruHi);
2535  else
2536  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2537 
2538  SDValue IndexHi, IndexLo;
2539  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2540  GetSplitVector(Index, IndexLo, IndexHi);
2541  else
2542  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
2543 
2546  MemoryLocation::UnknownSize, Alignment, MGT->getAAInfo(),
2547  MGT->getRanges());
2548 
2549  SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
2550  SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl,
2551  OpsLo, MMO, MGT->getIndexType(), ExtType);
2552 
2553  SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
2554  SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl,
2555  OpsHi, MMO, MGT->getIndexType(), ExtType);
2556 
2557  // Build a factor node to remember that this load is independent of the
2558  // other one.
2559  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2560  Hi.getValue(1));
2561 
2562  // Legalize the chain result - switch anything that used the old chain to
2563  // use the new one.
2564  ReplaceValueWith(SDValue(MGT, 1), Ch);
2565 
2566  SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
2567  Hi);
2568  ReplaceValueWith(SDValue(MGT, 0), Res);
2569  return SDValue();
2570 }
2571 
2572 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
2573  unsigned OpNo) {
2574  assert(N->isUnindexed() && "Indexed masked store of vector?");
2575  SDValue Ch = N->getChain();
2576  SDValue Ptr = N->getBasePtr();
2577  SDValue Offset = N->getOffset();
2578  assert(Offset.isUndef() && "Unexpected indexed masked store offset");
2579  SDValue Mask = N->getMask();
2580  SDValue Data = N->getValue();
2581  Align Alignment = N->getOriginalAlign();
2582  SDLoc DL(N);
2583 
2584  SDValue DataLo, DataHi;
2585  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2586  // Split Data operand
2587  GetSplitVector(Data, DataLo, DataHi);
2588  else
2589  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2590 
2591  // Split Mask operand
2592  SDValue MaskLo, MaskHi;
2593  if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
2594  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2595  } else {
2596  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2597  GetSplitVector(Mask, MaskLo, MaskHi);
2598  else
2599  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2600  }
2601 
2602  EVT MemoryVT = N->getMemoryVT();
2603  EVT LoMemVT, HiMemVT;
2604  bool HiIsEmpty = false;
2605  std::tie(LoMemVT, HiMemVT) =
2606  DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
2607 
2608  SDValue Lo, Hi, Res;
2609  unsigned LoSize = MemoryLocation::getSizeOrUnknown(LoMemVT.getStoreSize());
2611  N->getPointerInfo(), MachineMemOperand::MOStore, LoSize, Alignment,
2612  N->getAAInfo(), N->getRanges());
2613 
2614  Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
2615  N->getAddressingMode(), N->isTruncatingStore(),
2616  N->isCompressingStore());
2617 
2618  if (HiIsEmpty) {
2619  // The hi masked store has zero storage size.
2620  // Only the lo masked store is needed.
2621  Res = Lo;
2622  } else {
2623 
2624  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
2625  N->isCompressingStore());
2626 
2627  MachinePointerInfo MPI;
2628  if (LoMemVT.isScalableVector()) {
2629  Alignment = commonAlignment(
2630  Alignment, LoMemVT.getSizeInBits().getKnownMinSize() / 8);
2631  MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
2632  } else
2633  MPI = N->getPointerInfo().getWithOffset(
2634  LoMemVT.getStoreSize().getFixedSize());
2635 
2636  unsigned HiSize = MemoryLocation::getSizeOrUnknown(HiMemVT.getStoreSize());
2638  MPI, MachineMemOperand::MOStore, HiSize, Alignment, N->getAAInfo(),
2639  N->getRanges());
2640 
2641  Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
2642  N->getAddressingMode(), N->isTruncatingStore(),
2643  N->isCompressingStore());
2644 
2645  // Build a factor node to remember that this store is independent of the
2646  // other one.
2647  Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2648  }
2649 
2650  return Res;
2651 }
2652 
2653 SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
2654  unsigned OpNo) {
2655  SDValue Ch = N->getChain();
2656  SDValue Ptr = N->getBasePtr();
2657  SDValue Mask = N->getMask();
2658  SDValue Index = N->getIndex();
2659  SDValue Scale = N->getScale();
2660  SDValue Data = N->getValue();
2661  EVT MemoryVT = N->getMemoryVT();
2662  Align Alignment = N->getOriginalAlign();
2663  SDLoc DL(N);
2664 
2665  // Split all operands
2666 
2667  EVT LoMemVT, HiMemVT;
2668  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2669 
2670  SDValue DataLo, DataHi;
2671  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2672  // Split Data operand
2673  GetSplitVector(Data, DataLo, DataHi);
2674  else
2675  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2676 
2677  // Split Mask operand
2678  SDValue MaskLo, MaskHi;
2679  if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
2680  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2681  } else {
2682  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2683  GetSplitVector(Mask, MaskLo, MaskHi);
2684  else
2685  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2686  }
2687 
2688  SDValue IndexHi, IndexLo;
2689  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2690  GetSplitVector(Index, IndexLo, IndexHi);
2691  else
2692  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
2693 
2694  SDValue Lo;
2696  N->getPointerInfo(), MachineMemOperand::MOStore,
2697  MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
2698 
2699  SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale};
2700  Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), LoMemVT,
2701  DL, OpsLo, MMO, N->getIndexType(),
2702  N->isTruncatingStore());
2703 
2704  // The order of the Scatter operation after split is well defined. The "Hi"
2705  // part comes after the "Lo". So these two operations should be chained one
2706  // after another.
2707  SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale};
2708  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), HiMemVT,
2709  DL, OpsHi, MMO, N->getIndexType(),
2710  N->isTruncatingStore());
2711 }
2712 
2713 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
2714  assert(N->isUnindexed() && "Indexed store of vector?");
2715  assert(OpNo == 1 && "Can only split the stored value");
2716  SDLoc DL(N);
2717 
2718  bool isTruncating = N->isTruncatingStore();
2719  SDValue Ch = N->getChain();
2720  SDValue Ptr = N->getBasePtr();
2721  EVT MemoryVT = N->getMemoryVT();
2722  Align Alignment = N->getOriginalAlign();
2723  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2724  AAMDNodes AAInfo = N->getAAInfo();
2725  SDValue Lo, Hi;
2726  GetSplitVector(N->getOperand(1), Lo, Hi);
2727 
2728  EVT LoMemVT, HiMemVT;
2729  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2730 
2731  // Scalarize if the split halves are not byte-sized.
2732  if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
2733  return TLI.scalarizeVectorStore(N, DAG);
2734 
2735  if (isTruncating)
2736  Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
2737  Alignment, MMOFlags, AAInfo);
2738  else
2739  Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2740  AAInfo);
2741 
2742  MachinePointerInfo MPI;
2743  IncrementPointer(N, LoMemVT, MPI, Ptr);
2744 
2745  if (isTruncating)
2746  Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, MPI,
2747  HiMemVT, Alignment, MMOFlags, AAInfo);
2748  else
2749  Hi = DAG.getStore(Ch, DL, Hi, Ptr, MPI, Alignment, MMOFlags, AAInfo);
2750 
2751  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2752 }
2753 
2754 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
2755  SDLoc DL(N);
2756 
2757  // The input operands all must have the same type, and we know the result
2758  // type is valid. Convert this to a buildvector which extracts all the
2759  // input elements.
2760  // TODO: If the input elements are power-two vectors, we could convert this to
2761  // a new CONCAT_VECTORS node with elements that are half-wide.
2763  EVT EltVT = N->getValueType(0).getVectorElementType();
2764  for (const SDValue &Op : N->op_values()) {
2765  for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
2766  i != e; ++i) {
2767  Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
2768  DAG.getVectorIdxConstant(i, DL)));
2769  }
2770  }
2771 
2772  return DAG.getBuildVector(N->getValueType(0), DL, Elts);
2773 }
2774 
2775 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
2776  // The result type is legal, but the input type is illegal. If splitting
2777  // ends up with the result type of each half still being legal, just
2778  // do that. If, however, that would result in an illegal result type,
2779  // we can try to get more clever with power-two vectors. Specifically,
2780  // split the input type, but also widen the result element size, then
2781  // concatenate the halves and truncate again. For example, consider a target
2782  // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
2783  // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
2784  // %inlo = v4i32 extract_subvector %in, 0
2785  // %inhi = v4i32 extract_subvector %in, 4
2786  // %lo16 = v4i16 trunc v4i32 %inlo
2787  // %hi16 = v4i16 trunc v4i32 %inhi
2788  // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
2789  // %res = v8i8 trunc v8i16 %in16
2790  //
2791  // Without this transform, the original truncate would end up being
2792  // scalarized, which is pretty much always a last resort.
2793  unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
2794  SDValue InVec = N->getOperand(OpNo);
2795  EVT InVT = InVec->getValueType(0);
2796  EVT OutVT = N->getValueType(0);
2797  ElementCount NumElements = OutVT.getVectorElementCount();
2798  bool IsFloat = OutVT.isFloatingPoint();
2799 
2800  unsigned InElementSize = InVT.getScalarSizeInBits();
2801  unsigned OutElementSize = OutVT.getScalarSizeInBits();
2802 
2803  // Determine the split output VT. If its legal we can just split dirctly.
2804  EVT LoOutVT, HiOutVT;
2805  std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
2806  assert(LoOutVT == HiOutVT && "Unequal split?");
2807 
2808  // If the input elements are only 1/2 the width of the result elements,
2809  // just use the normal splitting. Our trick only work if there's room
2810  // to split more than once.
2811  if (isTypeLegal(LoOutVT) ||
2812  InElementSize <= OutElementSize * 2)
2813  return SplitVecOp_UnaryOp(N);
2814  SDLoc DL(N);
2815 
2816  // Don't touch if this will be scalarized.
2817  EVT FinalVT = InVT;
2818  while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
2819  FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
2820 
2821  if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector)
2822  return SplitVecOp_UnaryOp(N);
2823 
2824  // Get the split input vector.
2825  SDValue InLoVec, InHiVec;
2826  GetSplitVector(InVec, InLoVec, InHiVec);
2827 
2828  // Truncate them to 1/2 the element size.
2829  //
2830  // This assumes the number of elements is a power of two; any vector that
2831  // isn't should be widened, not split.
2832  EVT HalfElementVT = IsFloat ?
2833  EVT::getFloatingPointVT(InElementSize/2) :
2834  EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
2835  EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
2836  NumElements.divideCoefficientBy(2));
2837 
2838  SDValue HalfLo;
2839  SDValue HalfHi;
2840  SDValue Chain;
2841  if (N->isStrictFPOpcode()) {
2842  HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
2843  {N->getOperand(0), InLoVec});
2844  HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
2845  {N->getOperand(0), InHiVec});
2846  // Legalize the chain result - switch anything that used the old chain to
2847  // use the new one.
2848  Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1),
2849  HalfHi.getValue(1));
2850  } else {
2851  HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
2852  HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
2853  }
2854  // Concatenate them to get the full intermediate truncation result.
2855  EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
2856  SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
2857  HalfHi);
2858  // Now finish up by truncating all the way down to the original result
2859  // type. This should normally be something that ends up being legal directly,
2860  // but in theory if a target has very wide vectors and an annoyingly
2861  // restricted set of legal types, this split can chain to build things up.
2862 
2863  if (N->isStrictFPOpcode()) {
2864  SDValue Res = DAG.getNode(
2865  ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other},
2866  {Chain, InterVec,
2867  DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))});
2868  // Relink the chain
2869  ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
2870  return Res;
2871  }
2872 
2873  return IsFloat
2874  ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
2875  DAG.getTargetConstant(
2876  0, DL, TLI.getPointerTy(DAG.getDataLayout())))
2877  : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
2878 }
2879 
2880 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
2881  assert(N->getValueType(0).isVector() &&
2882  N->getOperand(0).getValueType().isVector() &&
2883  "Operand types must be vectors");
2884  // The result has a legal vector type, but the input needs splitting.
2885  SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
2886  SDLoc DL(N);
2887  GetSplitVector(N->getOperand(0), Lo0, Hi0);
2888  GetSplitVector(N->getOperand(1), Lo1, Hi1);
2889  auto PartEltCnt = Lo0.getValueType().getVectorElementCount();
2890 
2891  LLVMContext &Context = *DAG.getContext();
2892  EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt);
2893  EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2);
2894 
2895  LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
2896  HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
2897  SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
2898 
2899  EVT OpVT = N->getOperand(0).getValueType();
2900  ISD::NodeType ExtendCode =
2902  return DAG.getNode(ExtendCode, DL, N->getValueType(0), Con);
2903 }
2904 
2905 
2906 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
2907  // The result has a legal vector type, but the input needs splitting.
2908  EVT ResVT = N->getValueType(0);
2909  SDValue Lo, Hi;
2910  SDLoc DL(N);
2911  GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
2912  EVT InVT = Lo.getValueType();
2913 
2914  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2915  InVT.getVectorElementCount());
2916 
2917  if (N->isStrictFPOpcode()) {
2918  Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
2919  { N->getOperand(0), Lo, N->getOperand(2) });
2920  Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
2921  { N->getOperand(0), Hi, N->getOperand(2) });
2922  // Legalize the chain result - switch anything that used the old chain to
2923  // use the new one.
2924  SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
2925  Lo.getValue(1), Hi.getValue(1));
2926  ReplaceValueWith(SDValue(N, 1), NewChain);
2927  } else {
2928  Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
2929  Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
2930  }
2931 
2932  return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
2933 }
2934 
2935 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
2936  // The result (and the first input) has a legal vector type, but the second
2937  // input needs splitting.
2938  return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
2939 }
2940 
2941 SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
2942  EVT ResVT = N->getValueType(0);
2943  SDValue Lo, Hi;
2944  SDLoc dl(N);
2945  GetSplitVector(N->getOperand(0), Lo, Hi);
2946  EVT InVT = Lo.getValueType();
2947 
2948  EVT NewResVT =
2950  InVT.getVectorElementCount());
2951 
2952  Lo = DAG.getNode(N->getOpcode(), dl, NewResVT, Lo, N->getOperand(1));
2953  Hi = DAG.getNode(N->getOpcode(), dl, NewResVT, Hi, N->getOperand(1));
2954 
2955  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
2956 }
2957 
2958 //===----------------------------------------------------------------------===//
2959 // Result Vector Widening
2960 //===----------------------------------------------------------------------===//
2961 
2962 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
2963  LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
2964  dbgs() << "\n");
2965 
2966  // See if the target wants to custom widen this node.
2967  if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
2968  return;
2969 
2970  SDValue Res = SDValue();
2971  switch (N->getOpcode()) {
2972  default:
2973 #ifndef NDEBUG
2974  dbgs() << "WidenVectorResult #" << ResNo << ": ";
2975  N->dump(&DAG);
2976  dbgs() << "\n";
2977 #endif
2978  llvm_unreachable("Do not know how to widen the result of this operator!");
2979 
2980  case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
2981  case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
2982  case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
2983  case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
2984  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
2985  case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
2986  case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
2987  case ISD::SPLAT_VECTOR:
2988  case ISD::SCALAR_TO_VECTOR:
2989  Res = WidenVecRes_ScalarOp(N);
2990  break;
2991  case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
2992  case ISD::VSELECT:
2993  case ISD::SELECT: Res = WidenVecRes_SELECT(N); break;
2994  case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
2995  case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
2996  case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
2997  case ISD::VECTOR_SHUFFLE:
2998  Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
2999  break;
3000  case ISD::MLOAD:
3001  Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
3002  break;
3003  case ISD::MGATHER:
3004  Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
3005  break;
3006 
3007  case ISD::ADD:
3008  case ISD::AND:
3009  case ISD::MUL:
3010  case ISD::MULHS:
3011  case ISD::MULHU:
3012  case ISD::OR:
3013  case ISD::SUB:
3014  case ISD::XOR:
3015  case ISD::SHL:
3016  case ISD::SRA:
3017  case ISD::SRL:
3018  case ISD::FMINNUM:
3019  case ISD::FMAXNUM:
3020  case ISD::FMINIMUM:
3021  case ISD::FMAXIMUM:
3022  case ISD::SMIN:
3023  case ISD::SMAX:
3024  case ISD::UMIN:
3025  case ISD::UMAX:
3026  case ISD::UADDSAT:
3027  case ISD::SADDSAT:
3028  case ISD::USUBSAT:
3029  case ISD::SSUBSAT:
3030  case ISD::SSHLSAT:
3031  case ISD::USHLSAT:
3032  case ISD::ROTL:
3033  case ISD::ROTR:
3034  Res = WidenVecRes_Binary(N);
3035  break;
3036 
3037  case ISD::FADD:
3038  case ISD::FMUL:
3039  case ISD::FPOW:
3040  case ISD::FSUB:
3041  case ISD::FDIV:
3042  case ISD::FREM:
3043  case ISD::SDIV:
3044  case ISD::UDIV:
3045  case ISD::SREM:
3046  case ISD::UREM:
3047  Res = WidenVecRes_BinaryCanTrap(N);
3048  break;
3049 
3050  case ISD::SMULFIX:
3051  case ISD::SMULFIXSAT:
3052  case ISD::UMULFIX:
3053  case ISD::UMULFIXSAT:
3054  // These are binary operations, but with an extra operand that shouldn't
3055  // be widened (the scale).
3056  Res = WidenVecRes_BinaryWithExtraScalarOp(N);
3057  break;
3058 
3059 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
3060  case ISD::STRICT_##DAGN:
3061 #include "llvm/IR/ConstrainedOps.def"
3062  Res = WidenVecRes_StrictFP(N);
3063  break;
3064 
3065  case ISD::UADDO:
3066  case ISD::SADDO:
3067  case ISD::USUBO:
3068  case ISD::SSUBO:
3069  case ISD::UMULO:
3070  case ISD::SMULO:
3071  Res = WidenVecRes_OverflowOp(N, ResNo);
3072  break;
3073 
3074  case ISD::FCOPYSIGN:
3075  Res = WidenVecRes_FCOPYSIGN(N);
3076  break;
3077 
3078  case ISD::FPOWI:
3079  Res = WidenVecRes_POWI(N);
3080  break;
3081 
3085  Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
3086  break;
3087 
3088  case ISD::ANY_EXTEND:
3089  case ISD::FP_EXTEND:
3090  case ISD::FP_ROUND:
3091  case ISD::FP_TO_SINT:
3092  case ISD::FP_TO_UINT:
3093  case ISD::SIGN_EXTEND:
3094  case ISD::SINT_TO_FP:
3095  case ISD::TRUNCATE:
3096  case ISD::UINT_TO_FP:
3097  case ISD::ZERO_EXTEND:
3098  Res = WidenVecRes_Convert(N);
3099  break;
3100 
3101  case ISD::FP_TO_SINT_SAT:
3102  case ISD::FP_TO_UINT_SAT:
3103  Res = WidenVecRes_FP_TO_XINT_SAT(N);
3104  break;
3105 
3106  case ISD::FABS:
3107  case ISD::FCEIL:
3108  case ISD::FCOS:
3109  case ISD::FEXP:
3110  case ISD::FEXP2:
3111  case ISD::FFLOOR:
3112  case ISD::FLOG:
3113  case ISD::FLOG10:
3114  case ISD::FLOG2:
3115  case ISD::FNEARBYINT:
3116  case ISD::FRINT:
3117  case ISD::FROUND:
3118  case ISD::FROUNDEVEN:
3119  case ISD::FSIN:
3120  case ISD::FSQRT:
3121  case ISD::FTRUNC: {
3122  // We're going to widen this vector op to a legal type by padding with undef
3123  // elements. If the wide vector op is eventually going to be expanded to
3124  // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
3125  // libcalls on the undef elements.
3126  EVT VT = N->getValueType(0);
3127  EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3128  if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
3129  TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
3130  Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
3131  break;
3132  }
3133  }
3134  // If the target has custom/legal support for the scalar FP intrinsic ops
3135  // (they are probably not destined to become libcalls), then widen those like
3136  // any other unary ops.
3138 
3139  case ISD::ABS:
3140  case ISD::BITREVERSE:
3141  case ISD::BSWAP:
3142  case ISD::CTLZ:
3143  case ISD::CTLZ_ZERO_UNDEF:
3144  case ISD::CTPOP:
3145  case ISD::CTTZ:
3146  case ISD::CTTZ_ZERO_UNDEF:
3147  case ISD::FNEG:
3148  case ISD::FREEZE:
3149  case ISD::FCANONICALIZE:
3150  Res = WidenVecRes_Unary(N);
3151  break;
3152  case ISD::FMA:
3153  case ISD::FSHL:
3154  case ISD::FSHR:
3155  Res = WidenVecRes_Ternary(N);
3156  break;
3157  }
3158 
3159  // If Res is null, the sub-method took care of registering the result.
3160  if (Res.getNode())
3161  SetWidenedVector(SDValue(N, ResNo), Res);
3162 }
3163 
3164 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
3165  // Ternary op widening.
3166  SDLoc dl(N);
3167  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3168  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3169  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3170  SDValue InOp3 = GetWidenedVector(N->getOperand(2));
3171  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
3172 }
3173 
3174 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
3175  // Binary op widening.
3176  SDLoc dl(N);
3177  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3178  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3179  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3180  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags());
3181 }
3182 
3183 SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
3184  // Binary op widening, but with an extra operand that shouldn't be widened.
3185  SDLoc dl(N);
3186  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3187  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3188  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3189  SDValue InOp3 = N->getOperand(2);
3190  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3,
3191  N->getFlags());
3192 }
3193 
3194 // Given a vector of operations that have been broken up to widen, see
3195 // if we can collect them together into the next widest legal VT. This
3196 // implementation is trap-safe.
3198  SmallVectorImpl<SDValue> &ConcatOps,
3199  unsigned ConcatEnd, EVT VT, EVT MaxVT,
3200  EVT WidenVT) {
3201  // Check to see if we have a single operation with the widen type.
3202  if (ConcatEnd == 1) {
3203  VT = ConcatOps[0].getValueType();
3204  if (VT == WidenVT)
3205  return ConcatOps[0];
3206  }
3207 
3208  SDLoc dl(ConcatOps[0]);
3209  EVT WidenEltVT = WidenVT.getVectorElementType();
3210 
3211  // while (Some element of ConcatOps is not of type MaxVT) {
3212  // From the end of ConcatOps, collect elements of the same type and put
3213  // them into an op of the next larger supported type
3214  // }
3215  while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
3216  int Idx = ConcatEnd - 1;
3217  VT = ConcatOps[Idx--].getValueType();
3218  while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
3219  Idx--;
3220 
3221  int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
3222  EVT NextVT;
3223  do {
3224  NextSize *= 2;
3225  NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
3226  } while (!TLI.isTypeLegal(NextVT));
3227 
3228  if (!VT.isVector()) {
3229  // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
3230  SDValue VecOp = DAG.getUNDEF(NextVT);
3231  unsigned NumToInsert = ConcatEnd - Idx - 1;
3232  for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
3233  VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp,
3234  ConcatOps[OpIdx], DAG.getVectorIdxConstant(i, dl));
3235  }
3236  ConcatOps[Idx+1] = VecOp;
3237  ConcatEnd = Idx + 2;
3238  } else {
3239  // Vector type, create a CONCAT_VECTORS of type NextVT
3240  SDValue undefVec = DAG.getUNDEF(VT);
3241  unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
3242  SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
3243  unsigned RealVals = ConcatEnd - Idx - 1;
3244  unsigned SubConcatEnd = 0;
3245  unsigned SubConcatIdx = Idx + 1;
3246  while (SubConcatEnd < RealVals)
3247  SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
3248  while (SubConcatEnd < OpsToConcat)
3249  SubConcatOps[SubConcatEnd++] = undefVec;
3250  ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
3251  NextVT, SubConcatOps);
3252  ConcatEnd = SubConcatIdx + 1;
3253  }
3254  }
3255 
3256  // Check to see if we have a single operation with the widen type.
3257  if (ConcatEnd == 1) {
3258  VT = ConcatOps[0].getValueType();
3259  if (VT == WidenVT)
3260  return ConcatOps[0];
3261  }
3262 
3263  // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
3264  unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
3265  if (NumOps != ConcatEnd ) {
3266  SDValue UndefVal = DAG.getUNDEF(MaxVT);
3267  for (unsigned j = ConcatEnd; j < NumOps; ++j)
3268  ConcatOps[j] = UndefVal;
3269  }
3270  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
3271  makeArrayRef(ConcatOps.data(), NumOps));
3272 }
3273 
3274 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
3275  // Binary op widening for operations that can trap.
3276  unsigned Opcode = N->getOpcode();
3277  SDLoc dl(N);
3278  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3279  EVT WidenEltVT = WidenVT.getVectorElementType();
3280  EVT VT = WidenVT;
3281  unsigned NumElts = VT.getVectorNumElements();
3282  const SDNodeFlags Flags = N->getFlags();
3283  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
3284  NumElts = NumElts / 2;
3285  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3286  }
3287 
3288  if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
3289  // Operation doesn't trap so just widen as normal.
3290  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3291  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3292  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
3293  }
3294 
3295  // No legal vector version so unroll the vector operation and then widen.
3296  if (NumElts == 1)
3297  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
3298 
3299  // Since the operation can trap, apply operation on the original vector.
3300  EVT MaxVT = VT;
3301  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3302  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3303  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
3304 
3305  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3306  unsigned ConcatEnd = 0; // Current ConcatOps index.
3307  int Idx = 0; // Current Idx into input vectors.
3308 
3309  // NumElts := greatest legal vector size (at most WidenVT)
3310  // while (orig. vector has unhandled elements) {
3311  // take munches of size NumElts from the beginning and add to ConcatOps
3312  // NumElts := next smaller supported vector size or 1
3313  // }
3314  while (CurNumElts != 0) {
3315  while (CurNumElts >= NumElts) {
3316  SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
3317  DAG.getVectorIdxConstant(Idx, dl));
3318  SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
3319  DAG.getVectorIdxConstant(Idx, dl));
3320  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
3321  Idx += NumElts;
3322  CurNumElts -= NumElts;
3323  }
3324  do {
3325  NumElts = NumElts / 2;
3326  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3327  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
3328 
3329  if (NumElts == 1) {
3330  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
3331  SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
3332  InOp1, DAG.getVectorIdxConstant(Idx, dl));
3333  SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
3334  InOp2, DAG.getVectorIdxConstant(Idx, dl));
3335  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
3336  EOp1, EOp2, Flags);
3337  }
3338  CurNumElts = 0;
3339  }
3340  }
3341 
3342  return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3343 }
3344 
3345 SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
3346  switch (N->getOpcode()) {
3347  case ISD::STRICT_FSETCC:
3348  case ISD::STRICT_FSETCCS:
3349  return WidenVecRes_STRICT_FSETCC(N);
3350  case ISD::STRICT_FP_EXTEND:
3351  case ISD::STRICT_FP_ROUND:
3356  return WidenVecRes_Convert_StrictFP(N);
3357  default:
3358  break;
3359  }
3360 
3361  // StrictFP op widening for operations that can trap.
3362  unsigned NumOpers = N->getNumOperands();
3363  unsigned Opcode = N->getOpcode();
3364  SDLoc dl(N);
3365  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3366  EVT WidenEltVT = WidenVT.getVectorElementType();
3367  EVT VT = WidenVT;
3368  unsigned NumElts = VT.getVectorNumElements();
3369  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
3370  NumElts = NumElts / 2;
3371  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3372  }
3373 
3374  // No legal vector version so unroll the vector operation and then widen.
3375  if (NumElts == 1)
3376  return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements());
3377 
3378  // Since the operation can trap, apply operation on the original vector.
3379  EVT MaxVT = VT;
3381  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
3382 
3383  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3384  SmallVector<SDValue, 16> Chains;
3385  unsigned ConcatEnd = 0; // Current ConcatOps index.
3386  int Idx = 0; // Current Idx into input vectors.
3387 
3388  // The Chain is the first operand.
3389  InOps.push_back(N->getOperand(0));
3390 
3391  // Now process the remaining operands.
3392  for (unsigned i = 1; i < NumOpers; ++i) {
3393  SDValue Oper = N->getOperand(i);
3394 
3395  if (Oper.getValueType().isVector()) {
3396  assert(Oper.getValueType() == N->getValueType(0) &&
3397  "Invalid operand type to widen!");
3398  Oper = GetWidenedVector(Oper);
3399  }
3400 
3401  InOps.push_back(Oper);
3402  }
3403 
3404  // NumElts := greatest legal vector size (at most WidenVT)
3405  // while (orig. vector has unhandled elements) {
3406  // take munches of size NumElts from the beginning and add to ConcatOps
3407  // NumElts := next smaller supported vector size or 1
3408  // }
3409  while (CurNumElts != 0) {
3410  while (CurNumElts >= NumElts) {
3412 
3413  for (unsigned i = 0; i < NumOpers; ++i) {
3414  SDValue Op = InOps[i];
3415 
3416  if (Op.getValueType().isVector())
3417  Op = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Op,
3418  DAG.getVectorIdxConstant(Idx, dl));
3419 
3420  EOps.push_back(Op);
3421  }
3422 
3423  EVT OperVT[] = {VT, MVT::Other};
3424  SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
3425  ConcatOps[ConcatEnd++] = Oper;
3426  Chains.push_back(Oper.getValue(1));
3427  Idx += NumElts;
3428  CurNumElts -= NumElts;
3429  }
3430  do {
3431  NumElts = NumElts / 2;
3432  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3433  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
3434 
3435  if (NumElts == 1) {
3436  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
3438 
3439  for (unsigned i = 0; i < NumOpers; ++i) {
3440  SDValue Op = InOps[i];
3441 
3442  if (Op.getValueType().isVector())
3443  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op,
3444  DAG.getVectorIdxConstant(Idx, dl));
3445 
3446  EOps.push_back(Op);
3447  }
3448 
3449  EVT WidenVT[] = {WidenEltVT, MVT::Other};
3450  SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
3451  ConcatOps[ConcatEnd++] = Oper;
3452  Chains.push_back(Oper.getValue(1));
3453  }
3454  CurNumElts = 0;
3455  }
3456  }
3457 
3458  // Build a factor node to remember all the Ops that have been created.
3459  SDValue NewChain;
3460  if (Chains.size() == 1)
3461  NewChain = Chains[0];
3462  else
3463  NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
3464  ReplaceValueWith(SDValue(N, 1), NewChain);
3465 
3466  return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3467 }
3468 
3469 SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
3470  SDLoc DL(N);
3471  EVT ResVT = N->getValueType(0);
3472  EVT OvVT = N->getValueType(1);
3473  EVT WideResVT, WideOvVT;
3474  SDValue WideLHS, WideRHS;
3475 
3476  // TODO: This might result in a widen/split loop.
3477  if (ResNo == 0) {
3478  WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
3479  WideOvVT = EVT::getVectorVT(
3480  *DAG.getContext(), OvVT.getVectorElementType(),
3481  WideResVT.getVectorNumElements());
3482 
3483  WideLHS = GetWidenedVector(N->getOperand(0));
3484  WideRHS = GetWidenedVector(N->getOperand(1));
3485  } else {
3486  WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT);
3487  WideResVT = EVT::getVectorVT(
3488  *DAG.getContext(), ResVT.getVectorElementType(),
3489  WideOvVT.getVectorNumElements());
3490 
3491  SDValue Zero = DAG.getVectorIdxConstant(0, DL);
3492  WideLHS = DAG.getNode(
3493  ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3494  N->getOperand(0), Zero);
3495  WideRHS = DAG.getNode(
3496  ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3497  N->getOperand(1), Zero);
3498  }
3499 
3500  SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT);
3501  SDNode *WideNode = DAG.getNode(
3502  N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode();
3503 
3504  // Replace the other vector result not being explicitly widened here.
3505  unsigned OtherNo = 1 - ResNo;
3506  EVT OtherVT = N->getValueType(OtherNo);
3507  if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) {
3508  SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo));
3509  } else {
3510  SDValue Zero = DAG.getVectorIdxConstant(0, DL);
3511  SDValue OtherVal = DAG.getNode(
3512  ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero);
3513  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
3514  }
3515 
3516  return SDValue(WideNode, ResNo);
3517 }
3518 
3519 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
3520  LLVMContext &Ctx = *DAG.getContext();
3521  SDValue InOp = N->getOperand(0);
3522  SDLoc DL(N);
3523 
3524  EVT WidenVT = TLI.getTypeToTransformTo(Ctx, N->getValueType(0));
3525  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3526 
3527  EVT InVT = InOp.getValueType();
3528 
3529  unsigned Opcode = N->getOpcode();
3530  const SDNodeFlags Flags = N->getFlags();
3531 
3532  // Handle the case of ZERO_EXTEND where the promoted InVT element size does
3533  // not equal that of WidenVT.
3534  if (N->getOpcode() == ISD::ZERO_EXTEND &&
3535  getTypeAction(InVT) == TargetLowering::TypePromoteInteger &&
3536  TLI.getTypeToTransformTo(Ctx, InVT).getScalarSizeInBits() !=
3537  WidenVT.getScalarSizeInBits()) {
3538  InOp = ZExtPromotedInteger(InOp);
3539  InVT = InOp.getValueType();
3540  if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
3541  Opcode = ISD::TRUNCATE;
3542  }
3543 
3544  EVT InEltVT = InVT.getVectorElementType();
3545  EVT InWidenVT = EVT::getVectorVT(Ctx, InEltVT, WidenNumElts);
3546  unsigned InVTNumElts = InVT.getVectorNumElements();
3547 
3548  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3549  InOp = GetWidenedVector(N->getOperand(0));
3550  InVT = InOp.getValueType();
3551  InVTNumElts = InVT.getVectorNumElements();
3552  if (InVTNumElts == WidenNumElts) {
3553  if (N->getNumOperands() == 1)
3554  return DAG.getNode(Opcode, DL, WidenVT, InOp);
3555  return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
3556  }
3557  if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
3558  // If both input and result vector types are of same width, extend
3559  // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
3560  // accepts fewer elements in the result than in the input.
3561  if (Opcode == ISD::ANY_EXTEND)
3562  return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3563  if (Opcode == ISD::SIGN_EXTEND)
3564  return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3565  if (Opcode == ISD::ZERO_EXTEND)
3566  return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3567  }
3568  }
3569 
3570  if (TLI.isTypeLegal(InWidenVT)) {
3571  // Because the result and the input are different vector types, widening
3572  // the result could create a legal type but widening the input might make
3573  // it an illegal type that might lead to repeatedly splitting the input
3574  // and then widening it. To avoid this, we widen the input only if
3575  // it results in a legal type.
3576  if (WidenNumElts % InVTNumElts == 0) {
3577  // Widen the input and call convert on the widened input vector.
3578  unsigned NumConcat = WidenNumElts/InVTNumElts;
3579  SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
3580  Ops[0] = InOp;
3581  SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
3582  if (N->getNumOperands() == 1)
3583  return DAG.getNode(Opcode, DL, WidenVT, InVec);
3584  return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
3585  }
3586 
3587  if (InVTNumElts % WidenNumElts == 0) {
3588  SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
3589  DAG.getVectorIdxConstant(0, DL));
3590  // Extract the input and convert the shorten input vector.
3591  if (N->getNumOperands() == 1)
3592  return DAG.getNode(Opcode, DL, WidenVT, InVal);
3593  return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
3594  }
3595  }
3596 
3597  // Otherwise unroll into some nasty scalar code and rebuild the vector.
3598  EVT EltVT = WidenVT.getVectorElementType();
3599  SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3600  // Use the original element count so we don't do more scalar opts than
3601  // necessary.
3602  unsigned MinElts = N->getValueType(0).getVectorNumElements();
3603  for (unsigned i=0; i < MinElts; ++i) {
3604  SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3605  DAG.getVectorIdxConstant(i, DL));
3606  if (N->getNumOperands() == 1)
3607  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
3608  else
3609  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
3610  }
3611 
3612  return DAG.getBuildVector(WidenVT, DL, Ops);
3613 }
3614 
3615 SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
3616  SDLoc dl(N);
3617  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3618  ElementCount WidenNumElts = WidenVT.getVectorElementCount();
3619 
3620  SDValue Src = N->getOperand(0);
3621  EVT SrcVT = Src.getValueType();
3622 
3623  // Also widen the input.
3624  if (getTypeAction(SrcVT) == TargetLowering::TypeWidenVector) {
3625  Src = GetWidenedVector(Src);
3626  SrcVT = Src.getValueType();
3627  }
3628 
3629  // Input and output not widened to the same size, give up.
3630  if (WidenNumElts != SrcVT.getVectorElementCount())
3631  return DAG.UnrollVectorOp(N, WidenNumElts.getKnownMinValue());
3632 
3633  return DAG.getNode(N->getOpcode(), dl, WidenVT, Src, N->getOperand(1));
3634 }
3635 
3636 SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
3637  SDValue InOp = N->getOperand(1);
3638  SDLoc DL(N);
3639  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
3640 
3641  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3642  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3643 
3644  EVT InVT = InOp.getValueType();
3645  EVT InEltVT = InVT.getVectorElementType();
3646 
3647  unsigned Opcode = N->getOpcode();
3648 
3649  // FIXME: Optimizations need to be implemented here.
3650 
3651  // Otherwise unroll into some nasty scalar code and rebuild the vector.
3652  EVT EltVT = WidenVT.getVectorElementType();
3653  std::array<EVT, 2> EltVTs = {{EltVT, MVT::Other}};
3654  SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3655  SmallVector<SDValue, 32> OpChains;
3656  // Use the original element count so we don't do more scalar opts than
3657  // necessary.
3658  unsigned MinElts = N->getValueType(0).getVectorNumElements();
3659  for (unsigned i=0; i < MinElts; ++i) {
3660  NewOps[1] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3661  DAG.getVectorIdxConstant(i, DL));
3662  Ops[i] = DAG.getNode(Opcode, DL, EltVTs, NewOps);
3663  OpChains.push_back(Ops[i].getValue(1));
3664  }
3665  SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains);
3666  ReplaceValueWith(SDValue(N, 1), NewChain);
3667 
3668  return DAG.getBuildVector(WidenVT, DL, Ops);
3669 }
3670 
3671 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
3672  unsigned Opcode = N->getOpcode();
3673  SDValue InOp = N->getOperand(0);
3674  SDLoc DL(N);
3675 
3676  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3677  EVT WidenSVT = WidenVT.getVectorElementType();
3678  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3679 
3680  EVT InVT = InOp.getValueType();
3681  EVT InSVT = InVT.getVectorElementType();
3682  unsigned InVTNumElts = InVT.getVectorNumElements();
3683 
3684  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3685  InOp = GetWidenedVector(InOp);
3686  InVT = InOp.getValueType();
3687  if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
3688  switch (Opcode) {
3692  return DAG.getNode(Opcode, DL, WidenVT, InOp);
3693  }
3694  }
3695  }
3696 
3697  // Unroll, extend the scalars and rebuild the vector.
3699  for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
3700  SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
3701  DAG.getVectorIdxConstant(i, DL));
3702  switch (Opcode) {
3704  Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
3705  break;
3707  Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
3708  break;
3710  Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
3711  break;
3712  default:
3713  llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
3714  }
3715  Ops.push_back(Val);
3716  }
3717 
3718  while (Ops.size() != WidenNumElts)
3719  Ops.push_back(DAG.getUNDEF(WidenSVT));
3720 
3721  return DAG.getBuildVector(WidenVT, DL, Ops);
3722 }
3723 
3724 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
3725  // If this is an FCOPYSIGN with same input types, we can treat it as a
3726  // normal (can trap) binary op.
3727  if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
3728  return WidenVecRes_BinaryCanTrap(N);
3729 
3730  // If the types are different, fall back to unrolling.
3731  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3732  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
3733 }
3734 
3735 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
3736  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3737  SDValue InOp = GetWidenedVector(N->getOperand(0));
3738  SDValue ShOp = N->getOperand(1);
3739  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
3740 }
3741 
3742 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
3743  // Unary op widening.
3744  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3745  SDValue InOp = GetWidenedVector(N->getOperand(0));
3746  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
3747 }
3748 
3749 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
3750  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3751  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
3752  cast<VTSDNode>(N->getOperand(1))->getVT()
3753  .getVectorElementType(),
3754  WidenVT.getVectorNumElements());
3755  SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
3756  return DAG.getNode(N->getOpcode(), SDLoc(N),
3757  WidenVT, WidenLHS, DAG.getValueType(ExtVT));
3758 }
3759 
3760 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
3761  SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
3762  return GetWidenedVector(WidenVec);
3763 }
3764 
3765 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
3766  SDValue InOp = N->getOperand(0);
3767  EVT InVT = InOp.getValueType();
3768  EVT VT = N->getValueType(0);
3769  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3770  SDLoc dl(N);
3771 
3772  switch (getTypeAction(InVT)) {
3774  break;
3776  report_fatal_error("Scalarization of scalable vectors is not supported.");
3778  // If the incoming type is a vector that is being promoted, then
3779  // we know that the elements are arranged differently and that we
3780  // must perform the conversion using a stack slot.
3781  if (InVT.isVector())
3782  break;
3783 
3784  // If the InOp is promoted to the same size, convert it. Otherwise,
3785  // fall out of the switch and widen the promoted input.
3786  SDValue NInOp = GetPromotedInteger(InOp);
3787  EVT NInVT = NInOp.getValueType();
3788  if (WidenVT.bitsEq(NInVT)) {
3789  // For big endian targets we need to shift the input integer or the
3790  // interesting bits will end up at the wrong place.
3791  if (DAG.getDataLayout().isBigEndian()) {
3792  unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
3793  EVT ShiftAmtTy = TLI.getShiftAmountTy(NInVT, DAG.getDataLayout());
3794  assert(ShiftAmt < WidenVT.getSizeInBits() && "Too large shift amount!");
3795  NInOp = DAG.getNode(ISD::SHL, dl, NInVT, NInOp,
3796  DAG.getConstant(ShiftAmt, dl, ShiftAmtTy));
3797  }
3798  return DAG.getNode(ISD::BITCAST, dl, WidenVT, NInOp);
3799  }
3800  InOp = NInOp;
3801  InVT = NInVT;
3802  break;
3803  }
3811  break;
3813  // If the InOp is widened to the same size, convert it. Otherwise, fall
3814  // out of the switch and widen the widened input.
3815  InOp = GetWidenedVector(InOp);
3816  InVT = InOp.getValueType();
3817  if (WidenVT.bitsEq(InVT))
3818  // The input widens to the same size. Convert to the widen value.
3819  return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
3820  break;
3821  }
3822 
3823  unsigned WidenSize = WidenVT.getSizeInBits();
3824  unsigned InSize = InVT.getSizeInBits();
3825  // x86mmx is not an acceptable vector element type, so don't try.
3826  if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
3827  // Determine new input vector type. The new input vector type will use
3828  // the same element type (if its a vector) or use the input type as a
3829  // vector. It is the same size as the type to widen to.
3830  EVT NewInVT;
3831  unsigned NewNumElts = WidenSize / InSize;
3832  if (InVT.isVector()) {
3833  EVT InEltVT = InVT.getVectorElementType();
3834  NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
3835  WidenSize / InEltVT.getSizeInBits());
3836  } else {
3837  NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
3838  }
3839 
3840  if (TLI.isTypeLegal(NewInVT)) {
3841  SDValue NewVec;
3842  if (InVT.isVector()) {
3843  // Because the result and the input are different vector types, widening
3844  // the result could create a legal type but widening the input might make
3845  // it an illegal type that might lead to repeatedly splitting the input
3846  // and then widening it. To avoid this, we widen the input only if
3847  // it results in a legal type.
3848  SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT));
3849  Ops[0] = InOp;
3850 
3851  NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
3852  } else {
3853  NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp);
3854  }
3855  return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
3856  }
3857  }
3858 
3859  return CreateStackStoreLoad(InOp, WidenVT);
3860 }
3861 
3862 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
3863  SDLoc dl(N);
3864  // Build a vector with undefined for the new nodes.
3865  EVT VT = N->getValueType(0);
3866 
3867  // Integer BUILD_VECTOR operands may be larger than the node's vector element
3868  // type. The UNDEFs need to have the same type as the existing operands.
3869  EVT EltVT = N->getOperand(0).getValueType();
3870  unsigned NumElts = VT.getVectorNumElements();
3871 
3872  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3873  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3874 
3875  SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
3876  assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
3877  NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
3878 
3879  return DAG.getBuildVector(WidenVT, dl, NewOps);
3880 }
3881 
3882 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
3883  EVT InVT = N->getOperand(0).getValueType();
3884  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3885  SDLoc dl(N);
3886  unsigned NumOperands = N->getNumOperands();
3887 
3888  bool InputWidened = false; // Indicates we need to widen the input.
3889  if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
3890  unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
3891  unsigned NumInElts = InVT.getVectorMinNumElements();
3892  if (WidenNumElts % NumInElts == 0) {
3893  // Add undef vectors to widen to correct length.
3894  unsigned NumConcat = WidenNumElts / NumInElts;
3895  SDValue UndefVal = DAG.getUNDEF(InVT);
3896  SmallVector<SDValue, 16> Ops(NumConcat);
3897  for (unsigned i=0; i < NumOperands; ++i)
3898  Ops[i] = N->getOperand(i);
3899  for (unsigned i = NumOperands; i != NumConcat; ++i)
3900  Ops[i] = UndefVal;
3901  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
3902  }
3903  } else {
3904  InputWidened = true;
3905  if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
3906  // The inputs and the result are widen to the same value.
3907  unsigned i;
3908  for (i=1; i < NumOperands; ++i)
3909  if (!N->getOperand(i).isUndef())
3910  break;
3911 
3912  if (i == NumOperands)
3913  // Everything but the first operand is an UNDEF so just return the
3914  // widened first operand.
3915  return GetWidenedVector(N->getOperand(0));
3916 
3917  if (NumOperands == 2) {
3918  assert(!WidenVT.isScalableVector() &&
3919  "Cannot use vector shuffles to widen CONCAT_VECTOR result");
3920  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3921  unsigned NumInElts = InVT.getVectorNumElements();
3922 
3923  // Replace concat of two operands with a shuffle.
3924  SmallVector<int, 16> MaskOps(WidenNumElts, -1);
3925  for (unsigned i = 0; i < NumInElts; ++i) {
3926  MaskOps[i] = i;
3927  MaskOps[i + NumInElts] = i + WidenNumElts;
3928  }
3929  return DAG.getVectorShuffle(WidenVT, dl,
3930  GetWidenedVector(N->getOperand(0)),
3931  GetWidenedVector(N->getOperand(1)),
3932  MaskOps);
3933  }
3934  }
3935  }
3936 
3937  assert(!WidenVT.isScalableVector() &&
3938  "Cannot use build vectors to widen CONCAT_VECTOR result");
3939  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3940  unsigned NumInElts = InVT.getVectorNumElements();
3941 
3942  // Fall back to use extracts and build vector.
3943  EVT EltVT = WidenVT.getVectorElementType();
3944  SmallVector<SDValue, 16> Ops(WidenNumElts);
3945  unsigned Idx = 0;
3946  for (unsigned i=0; i < NumOperands; ++i) {
3947  SDValue InOp = N->getOperand(i);
3948  if (InputWidened)
3949  InOp = GetWidenedVector(InOp);
3950  for (unsigned j = 0; j < NumInElts; ++j)
3951  Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3952  DAG.getVectorIdxConstant(j, dl));
3953  }
3954  SDValue UndefVal = DAG.getUNDEF(EltVT);
3955  for (; Idx < WidenNumElts; ++Idx)
3956  Ops[Idx] = UndefVal;
3957  return DAG.getBuildVector(WidenVT, dl, Ops);
3958 }
3959 
3960 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
3961  EVT VT = N->getValueType(0);
3962  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3963  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3964  SDValue InOp = N->getOperand(0);
3965  SDValue Idx = N->getOperand(1);
3966  SDLoc dl(N);
3967 
3968  if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
3969  InOp = GetWidenedVector(InOp);
3970 
3971  EVT InVT = InOp.getValueType();
3972 
3973  // Check if we can just return the input vector after widening.
3974  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3975  if (IdxVal == 0 && InVT == WidenVT)
3976  return InOp;
3977 
3978  // Check if we can extract from the vector.
3979  unsigned InNumElts = InVT.getVectorNumElements();
3980  if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
3981  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
3982 
3983  // We could try widening the input to the right length but for now, extract
3984  // the original elements, fill the rest with undefs and build a vector.
3985  SmallVector<SDValue, 16> Ops(WidenNumElts);
3986  EVT EltVT = VT.getVectorElementType();
3987  unsigned NumElts = VT.getVectorNumElements();
3988  unsigned i;
3989  for (i = 0; i < NumElts; ++i)
3990  Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3991  DAG.getVectorIdxConstant(IdxVal + i, dl));
3992 
3993  SDValue UndefVal = DAG.getUNDEF(EltVT);
3994  for (; i < WidenNumElts; ++i)
3995  Ops[i] = UndefVal;
3996  return DAG.getBuildVector(WidenVT, dl, Ops);
3997 }
3998 
3999 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
4000  SDValue InOp = GetWidenedVector(N->getOperand(0));
4001  return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
4002  InOp.getValueType(), InOp,
4003  N->getOperand(1), N->getOperand(2));
4004 }
4005 
4006 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
4007  LoadSDNode *LD = cast<LoadSDNode>(N);
4008  ISD::LoadExtType ExtType = LD->getExtensionType();
4009 
4010  // A vector must always be stored in memory as-is, i.e. without any padding
4011  // between the elements, since various code depend on it, e.g. in the
4012  // handling of a bitcast of a vector type to int, which may be done with a
4013  // vector store followed by an integer load. A vector that does not have
4014  // elements that are byte-sized must therefore be stored as an integer
4015  // built out of the extracted vector elements.
4016  if (!LD->getMemoryVT().isByteSized()) {
4017  SDValue Value, NewChain;
4018  std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
4019  ReplaceValueWith(SDValue(LD, 0), Value);
4020  ReplaceValueWith(SDValue(LD, 1), NewChain);
4021  return SDValue();
4022  }
4023 
4024  SDValue Result;
4025  SmallVector<SDValue, 16> LdChain; // Chain for the series of load
4026  if (ExtType != ISD::NON_EXTLOAD)
4027  Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
4028  else
4029  Result = GenWidenVectorLoads(LdChain, LD);
4030 
4031  // If we generate a single load, we can use that for the chain. Otherwise,
4032  // build a factor node to remember the multiple loads are independent and
4033  // chain to that.
4034  SDValue NewChain;
4035  if (LdChain.size() == 1)
4036  NewChain = LdChain[0];
4037  else
4038  NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
4039 
4040  // Modified the chain - switch anything that used the old chain to use
4041  // the new one.
4042  ReplaceValueWith(SDValue(N, 1), NewChain);
4043 
4044  return Result;
4045 }
4046 
4047 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
4048 
4049  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
4050  SDValue Mask = N->getMask();
4051  EVT MaskVT = Mask.getValueType();
4052  SDValue PassThru = GetWidenedVector(N->getPassThru());
4053  ISD::LoadExtType ExtType = N->getExtensionType();
4054  SDLoc dl(N);
4055 
4056  // The mask should be widened as well
4057  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4058  MaskVT.getVectorElementType(),
4059  WidenVT.getVectorNumElements());
4060  Mask = ModifyToType(Mask, WideMaskVT, true);
4061 
4062  SDValue Res = DAG.getMaskedLoad(
4063  WidenVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask,
4064  PassThru, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(),
4065  ExtType, N->isExpandingLoad());
4066  // Legalize the chain result - switch anything that used the old chain to
4067  // use the new one.
4068  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4069  return Res;
4070 }
4071 
4072 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
4073 
4074  EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4075  SDValue Mask = N->getMask();
4076  EVT MaskVT = Mask.getValueType();
4077  SDValue PassThru = GetWidenedVector(N->getPassThru());
4078  SDValue Scale = N->getScale();
4079  unsigned NumElts = WideVT.getVectorNumElements();
4080  SDLoc dl(N);
4081 
4082  // The mask should be widened as well
4083  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4084  MaskVT.getVectorElementType(),
4085  WideVT.getVectorNumElements());
4086  Mask = ModifyToType(Mask, WideMaskVT, true);
4087 
4088  // Widen the Index operand
4089  SDValue Index = N->getIndex();
4090  EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
4091  Index.getValueType().getScalarType(),
4092  NumElts);
4093  Index = ModifyToType(Index, WideIndexVT);
4094  SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
4095  Scale };
4096 
4097  // Widen the MemoryType
4098  EVT WideMemVT = EVT::getVectorVT(*DAG.getContext(),
4099  N->getMemoryVT().getScalarType(), NumElts);
4100  SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
4101  WideMemVT, dl, Ops, N->getMemOperand(),
4102  N->getIndexType(), N->getExtensionType());
4103 
4104  // Legalize the chain result - switch anything that used the old chain to
4105  // use the new one.
4106  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4107  return Res;
4108 }
4109 
4110 SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
4111  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4112  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, N->getOperand(0));
4113 }
4114 
4115 // Return true is this is a SETCC node or a strict version of it.
4116 static inline bool isSETCCOp(unsigned Opcode) {
4117  switch (Opcode) {
4118  case ISD::SETCC:
4119  case ISD::STRICT_FSETCC:
4120  case ISD::STRICT_FSETCCS:
4121  return true;
4122  }
4123  return false;
4124 }
4125 
4126 // Return true if this is a node that could have two SETCCs as operands.
4127 static inline bool isLogicalMaskOp(unsigned Opcode) {
4128  switch (Opcode) {
4129  case ISD::AND:
4130  case ISD::OR:
4131  case ISD::XOR:
4132  return true;
4133  }
4134  return false;
4135 }
4136 
4137 // If N is a SETCC or a strict variant of it, return the type
4138 // of the compare operands.
4140  unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
4141  return N->getOperand(OpNo).getValueType();
4142 }
4143 
4144 // This is used just for the assert in convertMask(). Check that this either
4145 // a SETCC or a previously handled SETCC by convertMask().
4146 #ifndef NDEBUG
4147 static inline bool isSETCCorConvertedSETCC(SDValue N) {
4148  if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
4149  N = N.getOperand(0);
4150  else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
4151  for (unsigned i = 1; i < N->getNumOperands(); ++i)
4152  if (!N->getOperand(i)->isUndef())
4153  return false;
4154  N = N.getOperand(0);
4155  }
4156 
4157  if (N.getOpcode() == ISD::TRUNCATE)
4158  N = N.getOperand(0);
4159  else if (N.getOpcode() == ISD::SIGN_EXTEND)
4160  N = N.getOperand(0);
4161 
4162  if (isLogicalMaskOp(N.getOpcode()))
4163  return isSETCCorConvertedSETCC(N.getOperand(0)) &&
4164  isSETCCorConvertedSETCC(N.getOperand(1));
4165 
4166  return (isSETCCOp(N.getOpcode()) ||
4168 }
4169 #endif
4170 
4171 // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
4172 // to ToMaskVT if needed with vector extension or truncation.
4173 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
4174  EVT ToMaskVT) {
4175  // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
4176  // FIXME: This code seems to be too restrictive, we might consider
4177  // generalizing it or dropping it.
4178  assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
4179 
4180  // Make a new Mask node, with a legal result VT.
4181  SDValue Mask;
4183  for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
4184  Ops.push_back(InMask->getOperand(i));
4185  if (InMask->isStrictFPOpcode()) {
4186  Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask),
4187  { MaskVT, MVT::Other }, Ops);
4188  ReplaceValueWith(InMask.getValue(1), Mask.getValue(1));
4189  }
4190  else
4191  Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
4192 
4193  // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
4194  // extend or truncate is needed.
4195  LLVMContext &Ctx = *DAG.getContext();
4196  unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
4197  unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
4198  if (MaskScalarBits < ToMaskScalBits) {
4199  EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
4200  MaskVT.getVectorNumElements());
4201  Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
4202  } else if (MaskScalarBits > ToMaskScalBits) {
4203  EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
4204  MaskVT.getVectorNumElements());
4205  Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
4206  }
4207 
4208  assert(Mask->getValueType(0).getScalarSizeInBits() ==
4209  ToMaskVT.getScalarSizeInBits() &&
4210  "Mask should have the right element size by now.");
4211 
4212  // Adjust Mask to the right number of elements.
4213  unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
4214  if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
4215  SDValue ZeroIdx = DAG.getVectorIdxConstant(0, SDLoc(Mask));
4216  Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
4217  ZeroIdx);
4218  } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
4219  unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
4220  EVT SubVT = Mask->getValueType(0);
4221  SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
4222  SubOps[0] = Mask;
4223  Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
4224  }
4225 
4226  assert((Mask->getValueType(0) == ToMaskVT) &&
4227  "A mask of ToMaskVT should have been produced by now.");
4228 
4229  return Mask;
4230 }
4231 
4232 // This method tries to handle some special cases for the vselect mask
4233 // and if needed adjusting the mask vector type to match that of the VSELECT.
4234 // Without it, many cases end up with scalarization of the SETCC, with many
4235 // unnecessary instructions.
4236 SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
4237  LLVMContext &Ctx = *DAG.getContext();
4238  SDValue Cond = N->getOperand(0);
4239 
4240  if (N->getOpcode() != ISD::VSELECT)
4241  return SDValue();
4242 
4243  if (!isSETCCOp(Cond->getOpcode()) && !isLogicalMaskOp(Cond->getOpcode()))
4244  return SDValue();
4245 
4246  // If this is a splitted VSELECT that was previously already handled, do
4247  // nothing.
4248  EVT CondVT = Cond->getValueType(0);
4249  if (CondVT.getScalarSizeInBits() != 1)
4250  return SDValue();
4251 
4252  EVT VSelVT = N->getValueType(0);
4253  // Only handle vector types which are a power of 2.
4254  if (!isPowerOf2_64(VSelVT.getSizeInBits()))
4255  return SDValue();
4256 
4257  // Don't touch if this will be scalarized.
4258  EVT FinalVT = VSelVT;
4259  while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
4260  FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
4261 
4262  if (FinalVT.getVectorNumElements() == 1)
4263  return SDValue();
4264 
4265  // If there is support for an i1 vector mask, don't touch.
4266  if (isSETCCOp(Cond.getOpcode())) {
4267  EVT SetCCOpVT = getSETCCOperandType(Cond);
4268  while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
4269  SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
4270  EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
4271  if (SetCCResVT.getScalarSizeInBits() == 1)
4272  return SDValue();
4273  } else if (CondVT.getScalarType() == MVT::i1) {
4274  // If there is support for an i1 vector mask (or only scalar i1 conditions),
4275  // don't touch.
4276  while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal)
4277  CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
4278 
4279  if (CondVT.getScalarType() == MVT::i1)
4280  return SDValue();
4281  }
4282 
4283  // Widen the vselect result type if needed.
4284  if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector)
4285  VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
4286 
4287  // The mask of the VSELECT should have integer elements.
4288  EVT ToMaskVT = VSelVT;
4289  if (!ToMaskVT.getScalarType().isInteger())
4290  ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
4291 
4292  SDValue Mask;
4293  if (isSETCCOp(Cond->getOpcode())) {
4294  EVT MaskVT = getSetCCResultType(getSETCCOperandType(Cond));
4295  Mask = convertMask(Cond, MaskVT, ToMaskVT);
4296  } else if (isLogicalMaskOp(Cond->getOpcode()) &&
4297  isSETCCOp(Cond->getOperand(0).getOpcode()) &&
4298  isSETCCOp(Cond->getOperand(1).getOpcode())) {
4299  // Cond is (AND/OR/XOR (SETCC, SETCC))
4300  SDValue SETCC0 = Cond->getOperand(0);
4301  SDValue SETCC1 = Cond->getOperand(1);
4302  EVT VT0 = getSetCCResultType(getSETCCOperandType(SETCC0));
4303  EVT VT1 = getSetCCResultType(getSETCCOperandType(SETCC1));
4304  unsigned ScalarBits0 = VT0.getScalarSizeInBits();
4305  unsigned ScalarBits1 = VT1.getScalarSizeInBits();
4306  unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
4307  EVT MaskVT;
4308  // If the two SETCCs have different VTs, either extend/truncate one of
4309  // them to the other "towards" ToMaskVT, or truncate one and extend the
4310  // other to ToMaskVT.
4311  if (ScalarBits0 != ScalarBits1) {
4312  EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
4313  EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
4314  if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
4315  MaskVT = WideVT;
4316  else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
4317  MaskVT = NarrowVT;
4318  else
4319  MaskVT = ToMaskVT;
4320  } else
4321  // If the two SETCCs have the same VT, don't change it.
4322  MaskVT = VT0;
4323 
4324  // Make new SETCCs and logical nodes.
4325  SETCC0 = convertMask(SETCC0, VT0, MaskVT);
4326  SETCC1 = convertMask(SETCC1, VT1, MaskVT);
4327  Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1);
4328 
4329  // Convert the logical op for VSELECT if needed.
4330  Mask = convertMask(Cond, MaskVT, ToMaskVT);
4331  } else
4332  return SDValue();
4333 
4334  return Mask;
4335 }
4336 
4337 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
4338  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4339  unsigned WidenNumElts = WidenVT.getVectorNumElements();
4340 
4341  SDValue Cond1 = N->getOperand(0);
4342  EVT CondVT = Cond1.getValueType();
4343  if (CondVT.isVector()) {
4344  if (SDValue WideCond = WidenVSELECTMask(N)) {
4345  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
4346  SDValue InOp2 = GetWidenedVector(N->getOperand(2));
4347  assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
4348  return DAG.getNode(N->getOpcode(), SDLoc(N),
4349  WidenVT, WideCond, InOp1, InOp2);
4350  }
4351 
4352  EVT CondEltVT = CondVT.getVectorElementType();
4353  EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(),
4354  CondEltVT, WidenNumElts);
4355  if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
4356  Cond1 = GetWidenedVector(Cond1);
4357 
4358  // If we have to split the condition there is no point in widening the
4359  // select. This would result in an cycle of widening the select ->
4360  // widening the condition operand -> splitting the condition operand ->
4361  // splitting the select -> widening the select. Instead split this select
4362  // further and widen the resulting type.
4363  if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
4364  SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
4365  SDValue Res = ModifyToType(SplitSelect, WidenVT);
4366  return Res;
4367  }
4368 
4369  if (Cond1.getValueType() != CondWidenVT)
4370  Cond1 = ModifyToType(Cond1, CondWidenVT);
4371  }
4372 
4373  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
4374  SDValue InOp2 = GetWidenedVector(N->getOperand(2));
4375  assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
4376  return DAG.getNode(N->getOpcode(), SDLoc(N),
4377  WidenVT, Cond1, InOp1, InOp2);
4378 }
4379 
4380 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
4381  SDValue InOp1 = GetWidenedVector(N->getOperand(2));
4382  SDValue InOp2 = GetWidenedVector(N->getOperand(3));
4383  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
4384  InOp1.getValueType(), N->getOperand(0),
4385  N->getOperand(1), InOp1, InOp2, N->getOperand(4));
4386 }
4387 
4388 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
4389  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4390  return DAG.getUNDEF(WidenVT);
4391 }
4392 
4393 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
4394  EVT VT = N->getValueType(0);
4395  SDLoc dl(N);
4396 
4397  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4398  unsigned NumElts = VT.getVectorNumElements();
4399  unsigned WidenNumElts = WidenVT.getVectorNumElements();
4400 
4401  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4402  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4403 
4404  // Adjust mask based on new input vector length.
4405  SmallVector<int, 16> NewMask;
4406  for (unsigned i = 0; i != NumElts; ++i) {
4407  int Idx = N->getMaskElt(i);
4408  if (Idx < (int)NumElts)
4409  NewMask.push_back(Idx);
4410  else
4411  NewMask.push_back(Idx - NumElts + WidenNumElts);
4412  }
4413  for (unsigned i = NumElts; i != WidenNumElts; ++i)
4414  NewMask.push_back(-1);
4415  return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
4416 }