LLVM  15.0.0git
LegalizeVectorTypes.cpp
Go to the documentation of this file.
1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file performs vector type splitting and scalarization for LegalizeTypes.
10 // Scalarization is the act of changing a computation in an illegal one-element
11 // vector type to be a computation in its scalar element type. For example,
12 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
14 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15 // types.
16 // Splitting is the act of changing a computation in an invalid vector type to
17 // be a computation in two vectors of half the size. For example, implementing
18 // <128 x f32> operations in terms of two <64 x f32> operations.
19 //
20 //===----------------------------------------------------------------------===//
21 
22 #include "LegalizeTypes.h"
26 #include "llvm/IR/DataLayout.h"
28 #include "llvm/Support/TypeSize.h"
30 using namespace llvm;
31 
32 #define DEBUG_TYPE "legalize-types"
33 
34 //===----------------------------------------------------------------------===//
35 // Result Vector Scalarization: <1 x ty> -> ty.
36 //===----------------------------------------------------------------------===//
37 
38 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
39  LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
40  dbgs() << "\n");
41  SDValue R = SDValue();
42 
43  switch (N->getOpcode()) {
44  default:
45 #ifndef NDEBUG
46  dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
47  N->dump(&DAG);
48  dbgs() << "\n";
49 #endif
50  report_fatal_error("Do not know how to scalarize the result of this "
51  "operator!\n");
52 
53  case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
54  case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
55  case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
56  case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
57  case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
58  case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
59  case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
60  case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
61  case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
62  case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
63  case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
64  case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
65  case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
66  case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
67  case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
68  case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
69  case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
73  R = ScalarizeVecRes_VecInregOp(N);
74  break;
75  case ISD::ABS:
76  case ISD::ANY_EXTEND:
77  case ISD::BITREVERSE:
78  case ISD::BSWAP:
79  case ISD::CTLZ:
81  case ISD::CTPOP:
82  case ISD::CTTZ:
84  case ISD::FABS:
85  case ISD::FCEIL:
86  case ISD::FCOS:
87  case ISD::FEXP:
88  case ISD::FEXP2:
89  case ISD::FFLOOR:
90  case ISD::FLOG:
91  case ISD::FLOG10:
92  case ISD::FLOG2:
93  case ISD::FNEARBYINT:
94  case ISD::FNEG:
95  case ISD::FREEZE:
96  case ISD::ARITH_FENCE:
97  case ISD::FP_EXTEND:
98  case ISD::FP_TO_SINT:
99  case ISD::FP_TO_UINT:
100  case ISD::FRINT:
101  case ISD::FROUND:
102  case ISD::FROUNDEVEN:
103  case ISD::FSIN:
104  case ISD::FSQRT:
105  case ISD::FTRUNC:
106  case ISD::SIGN_EXTEND:
107  case ISD::SINT_TO_FP:
108  case ISD::TRUNCATE:
109  case ISD::UINT_TO_FP:
110  case ISD::ZERO_EXTEND:
111  case ISD::FCANONICALIZE:
112  R = ScalarizeVecRes_UnaryOp(N);
113  break;
114 
115  case ISD::ADD:
116  case ISD::AND:
117  case ISD::FADD:
118  case ISD::FCOPYSIGN:
119  case ISD::FDIV:
120  case ISD::FMUL:
121  case ISD::FMINNUM:
122  case ISD::FMAXNUM:
123  case ISD::FMINNUM_IEEE:
124  case ISD::FMAXNUM_IEEE:
125  case ISD::FMINIMUM:
126  case ISD::FMAXIMUM:
127  case ISD::SMIN:
128  case ISD::SMAX:
129  case ISD::UMIN:
130  case ISD::UMAX:
131 
132  case ISD::SADDSAT:
133  case ISD::UADDSAT:
134  case ISD::SSUBSAT:
135  case ISD::USUBSAT:
136  case ISD::SSHLSAT:
137  case ISD::USHLSAT:
138 
139  case ISD::FPOW:
140  case ISD::FREM:
141  case ISD::FSUB:
142  case ISD::MUL:
143  case ISD::OR:
144  case ISD::SDIV:
145  case ISD::SREM:
146  case ISD::SUB:
147  case ISD::UDIV:
148  case ISD::UREM:
149  case ISD::XOR:
150  case ISD::SHL:
151  case ISD::SRA:
152  case ISD::SRL:
153  case ISD::ROTL:
154  case ISD::ROTR:
155  R = ScalarizeVecRes_BinOp(N);
156  break;
157  case ISD::FMA:
158  case ISD::FSHL:
159  case ISD::FSHR:
160  R = ScalarizeVecRes_TernaryOp(N);
161  break;
162 
163 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
164  case ISD::STRICT_##DAGN:
165 #include "llvm/IR/ConstrainedOps.def"
166  R = ScalarizeVecRes_StrictFPOp(N);
167  break;
168 
169  case ISD::FP_TO_UINT_SAT:
170  case ISD::FP_TO_SINT_SAT:
171  R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
172  break;
173 
174  case ISD::UADDO:
175  case ISD::SADDO:
176  case ISD::USUBO:
177  case ISD::SSUBO:
178  case ISD::UMULO:
179  case ISD::SMULO:
180  R = ScalarizeVecRes_OverflowOp(N, ResNo);
181  break;
182  case ISD::SMULFIX:
183  case ISD::SMULFIXSAT:
184  case ISD::UMULFIX:
185  case ISD::UMULFIXSAT:
186  case ISD::SDIVFIX:
187  case ISD::SDIVFIXSAT:
188  case ISD::UDIVFIX:
189  case ISD::UDIVFIXSAT:
190  R = ScalarizeVecRes_FIX(N);
191  break;
192  }
193 
194  // If R is null, the sub-method took care of registering the result.
195  if (R.getNode())
196  SetScalarizedVector(SDValue(N, ResNo), R);
197 }
198 
199 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
200  SDValue LHS = GetScalarizedVector(N->getOperand(0));
201  SDValue RHS = GetScalarizedVector(N->getOperand(1));
202  return DAG.getNode(N->getOpcode(), SDLoc(N),
203  LHS.getValueType(), LHS, RHS, N->getFlags());
204 }
205 
206 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
207  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
208  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
209  SDValue Op2 = GetScalarizedVector(N->getOperand(2));
210  return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
211  Op2, N->getFlags());
212 }
213 
214 SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
215  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
216  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
217  SDValue Op2 = N->getOperand(2);
218  return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
219  Op2, N->getFlags());
220 }
221 
222 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
223  EVT VT = N->getValueType(0).getVectorElementType();
224  unsigned NumOpers = N->getNumOperands();
225  SDValue Chain = N->getOperand(0);
226  EVT ValueVTs[] = {VT, MVT::Other};
227  SDLoc dl(N);
228 
229  SmallVector<SDValue, 4> Opers(NumOpers);
230 
231  // The Chain is the first operand.
232  Opers[0] = Chain;
233 
234  // Now process the remaining operands.
235  for (unsigned i = 1; i < NumOpers; ++i) {
236  SDValue Oper = N->getOperand(i);
237  EVT OperVT = Oper.getValueType();
238 
239  if (OperVT.isVector()) {
240  if (getTypeAction(OperVT) == TargetLowering::TypeScalarizeVector)
241  Oper = GetScalarizedVector(Oper);
242  else
243  Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
244  OperVT.getVectorElementType(), Oper,
245  DAG.getVectorIdxConstant(0, dl));
246  }
247 
248  Opers[i] = Oper;
249  }
250 
251  SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs),
252  Opers, N->getFlags());
253 
254  // Legalize the chain result - switch anything that used the old chain to
255  // use the new one.
256  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
257  return Result;
258 }
259 
260 SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
261  unsigned ResNo) {
262  SDLoc DL(N);
263  EVT ResVT = N->getValueType(0);
264  EVT OvVT = N->getValueType(1);
265 
266  SDValue ScalarLHS, ScalarRHS;
267  if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
268  ScalarLHS = GetScalarizedVector(N->getOperand(0));
269  ScalarRHS = GetScalarizedVector(N->getOperand(1));
270  } else {
271  SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
272  DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
273  DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
274  ScalarLHS = ElemsLHS[0];
275  ScalarRHS = ElemsRHS[0];
276  }
277 
278  SDVTList ScalarVTs = DAG.getVTList(
280  SDNode *ScalarNode = DAG.getNode(
281  N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
282  ScalarNode->setFlags(N->getFlags());
283 
284  // Replace the other vector result not being explicitly scalarized here.
285  unsigned OtherNo = 1 - ResNo;
286  EVT OtherVT = N->getValueType(OtherNo);
287  if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
288  SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
289  } else {
290  SDValue OtherVal = DAG.getNode(
291  ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
292  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
293  }
294 
295  return SDValue(ScalarNode, ResNo);
296 }
297 
298 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
299  unsigned ResNo) {
300  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
301  return GetScalarizedVector(Op);
302 }
303 
304 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
305  SDValue Op = N->getOperand(0);
306  if (Op.getValueType().isVector()
307  && Op.getValueType().getVectorNumElements() == 1
308  && !isSimpleLegalType(Op.getValueType()))
309  Op = GetScalarizedVector(Op);
310  EVT NewVT = N->getValueType(0).getVectorElementType();
311  return DAG.getNode(ISD::BITCAST, SDLoc(N),
312  NewVT, Op);
313 }
314 
315 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
316  EVT EltVT = N->getValueType(0).getVectorElementType();
317  SDValue InOp = N->getOperand(0);
318  // The BUILD_VECTOR operands may be of wider element types and
319  // we may need to truncate them back to the requested return type.
320  if (EltVT.isInteger())
321  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
322  return InOp;
323 }
324 
325 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
326  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
327  N->getValueType(0).getVectorElementType(),
328  N->getOperand(0), N->getOperand(1));
329 }
330 
331 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
332  SDLoc DL(N);
333  SDValue Op = N->getOperand(0);
334  EVT OpVT = Op.getValueType();
335  // The result needs scalarizing, but it's not a given that the source does.
336  // See similar logic in ScalarizeVecRes_UnaryOp.
337  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
338  Op = GetScalarizedVector(Op);
339  } else {
340  EVT VT = OpVT.getVectorElementType();
341  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
342  DAG.getVectorIdxConstant(0, DL));
343  }
344  return DAG.getNode(ISD::FP_ROUND, DL,
345  N->getValueType(0).getVectorElementType(), Op,
346  N->getOperand(1));
347 }
348 
349 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
350  SDValue Op = GetScalarizedVector(N->getOperand(0));
351  return DAG.getNode(ISD::FPOWI, SDLoc(N),
352  Op.getValueType(), Op, N->getOperand(1));
353 }
354 
355 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
356  // The value to insert may have a wider type than the vector element type,
357  // so be sure to truncate it to the element type if necessary.
358  SDValue Op = N->getOperand(1);
359  EVT EltVT = N->getValueType(0).getVectorElementType();
360  if (Op.getValueType() != EltVT)
361  // FIXME: Can this happen for floating point types?
362  Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
363  return Op;
364 }
365 
366 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
367  assert(N->isUnindexed() && "Indexed vector load?");
368 
369  SDValue Result = DAG.getLoad(
370  ISD::UNINDEXED, N->getExtensionType(),
371  N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
372  N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
373  N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
374  N->getOriginalAlign(), N->getMemOperand()->getFlags(), N->getAAInfo());
375 
376  // Legalize the chain result - switch anything that used the old chain to
377  // use the new one.
378  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
379  return Result;
380 }
381 
382 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
383  // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
384  EVT DestVT = N->getValueType(0).getVectorElementType();
385  SDValue Op = N->getOperand(0);
386  EVT OpVT = Op.getValueType();
387  SDLoc DL(N);
388  // The result needs scalarizing, but it's not a given that the source does.
389  // This is a workaround for targets where it's impossible to scalarize the
390  // result of a conversion, because the source type is legal.
391  // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
392  // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
393  // legal and was not scalarized.
394  // See the similar logic in ScalarizeVecRes_SETCC
395  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
396  Op = GetScalarizedVector(Op);
397  } else {
398  EVT VT = OpVT.getVectorElementType();
399  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
400  DAG.getVectorIdxConstant(0, DL));
401  }
402  return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op, N->getFlags());
403 }
404 
405 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
406  EVT EltVT = N->getValueType(0).getVectorElementType();
407  EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
408  SDValue LHS = GetScalarizedVector(N->getOperand(0));
409  return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
410  LHS, DAG.getValueType(ExtVT));
411 }
412 
413 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
414  SDLoc DL(N);
415  SDValue Op = N->getOperand(0);
416 
417  EVT OpVT = Op.getValueType();
418  EVT OpEltVT = OpVT.getVectorElementType();
419  EVT EltVT = N->getValueType(0).getVectorElementType();
420 
421  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
422  Op = GetScalarizedVector(Op);
423  } else {
424  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
425  DAG.getVectorIdxConstant(0, DL));
426  }
427 
428  switch (N->getOpcode()) {
430  return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
432  return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
434  return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
435  }
436 
437  llvm_unreachable("Illegal extend_vector_inreg opcode");
438 }
439 
440 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
441  // If the operand is wider than the vector element type then it is implicitly
442  // truncated. Make that explicit here.
443  EVT EltVT = N->getValueType(0).getVectorElementType();
444  SDValue InOp = N->getOperand(0);
445  if (InOp.getValueType() != EltVT)
446  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
447  return InOp;
448 }
449 
450 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
451  SDValue Cond = N->getOperand(0);
452  EVT OpVT = Cond.getValueType();
453  SDLoc DL(N);
454  // The vselect result and true/value operands needs scalarizing, but it's
455  // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
456  // See the similar logic in ScalarizeVecRes_SETCC
457  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
458  Cond = GetScalarizedVector(Cond);
459  } else {
460  EVT VT = OpVT.getVectorElementType();
462  DAG.getVectorIdxConstant(0, DL));
463  }
464 
465  SDValue LHS = GetScalarizedVector(N->getOperand(1));
466  TargetLowering::BooleanContent ScalarBool =
467  TLI.getBooleanContents(false, false);
468  TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
469 
470  // If integer and float booleans have different contents then we can't
471  // reliably optimize in all cases. There is a full explanation for this in
472  // DAGCombiner::visitSELECT() where the same issue affects folding
473  // (select C, 0, 1) to (xor C, 1).
474  if (TLI.getBooleanContents(false, false) !=
475  TLI.getBooleanContents(false, true)) {
476  // At least try the common case where the boolean is generated by a
477  // comparison.
478  if (Cond->getOpcode() == ISD::SETCC) {
479  EVT OpVT = Cond->getOperand(0).getValueType();
480  ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
481  VecBool = TLI.getBooleanContents(OpVT);
482  } else
484  }
485 
486  EVT CondVT = Cond.getValueType();
487  if (ScalarBool != VecBool) {
488  switch (ScalarBool) {
490  break;
494  // Vector read from all ones, scalar expects a single 1 so mask.
495  Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
496  Cond, DAG.getConstant(1, SDLoc(N), CondVT));
497  break;
501  // Vector reads from a one, scalar from all ones so sign extend.
502  Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
503  Cond, DAG.getValueType(MVT::i1));
504  break;
505  }
506  }
507 
508  // Truncate the condition if needed
509  auto BoolVT = getSetCCResultType(CondVT);
510  if (BoolVT.bitsLT(CondVT))
511  Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
512 
513  return DAG.getSelect(SDLoc(N),
514  LHS.getValueType(), Cond, LHS,
515  GetScalarizedVector(N->getOperand(2)));
516 }
517 
518 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
519  SDValue LHS = GetScalarizedVector(N->getOperand(1));
520  return DAG.getSelect(SDLoc(N),
521  LHS.getValueType(), N->getOperand(0), LHS,
522  GetScalarizedVector(N->getOperand(2)));
523 }
524 
525 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
526  SDValue LHS = GetScalarizedVector(N->getOperand(2));
527  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
528  N->getOperand(0), N->getOperand(1),
529  LHS, GetScalarizedVector(N->getOperand(3)),
530  N->getOperand(4));
531 }
532 
533 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
534  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
535 }
536 
537 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
538  // Figure out if the scalar is the LHS or RHS and return it.
539  SDValue Arg = N->getOperand(2).getOperand(0);
540  if (Arg.isUndef())
541  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
542  unsigned Op = !cast<ConstantSDNode>(Arg)->isZero();
543  return GetScalarizedVector(N->getOperand(Op));
544 }
545 
546 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
547  SDValue Src = N->getOperand(0);
548  EVT SrcVT = Src.getValueType();
549  SDLoc dl(N);
550 
551  // Handle case where result is scalarized but operand is not
552  if (getTypeAction(SrcVT) == TargetLowering::TypeScalarizeVector)
553  Src = GetScalarizedVector(Src);
554  else
555  Src = DAG.getNode(
557  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
558 
559  EVT DstVT = N->getValueType(0).getVectorElementType();
560  return DAG.getNode(N->getOpcode(), dl, DstVT, Src, N->getOperand(1));
561 }
562 
563 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
564  assert(N->getValueType(0).isVector() &&
565  N->getOperand(0).getValueType().isVector() &&
566  "Operand types must be vectors");
567  SDValue LHS = N->getOperand(0);
568  SDValue RHS = N->getOperand(1);
569  EVT OpVT = LHS.getValueType();
570  EVT NVT = N->getValueType(0).getVectorElementType();
571  SDLoc DL(N);
572 
573  // The result needs scalarizing, but it's not a given that the source does.
574  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
575  LHS = GetScalarizedVector(LHS);
576  RHS = GetScalarizedVector(RHS);
577  } else {
578  EVT VT = OpVT.getVectorElementType();
580  DAG.getVectorIdxConstant(0, DL));
582  DAG.getVectorIdxConstant(0, DL));
583  }
584 
585  // Turn it into a scalar SETCC.
586  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
587  N->getOperand(2));
588  // Vectors may have a different boolean contents to scalars. Promote the
589  // value appropriately.
590  ISD::NodeType ExtendCode =
592  return DAG.getNode(ExtendCode, DL, NVT, Res);
593 }
594 
595 SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
596  SDLoc DL(N);
597  SDValue Arg = N->getOperand(0);
598  SDValue Test = N->getOperand(1);
599  EVT ArgVT = Arg.getValueType();
600  EVT ResultVT = N->getValueType(0).getVectorElementType();
601 
602  if (getTypeAction(ArgVT) == TargetLowering::TypeScalarizeVector) {
603  Arg = GetScalarizedVector(Arg);
604  } else {
605  EVT VT = ArgVT.getVectorElementType();
607  DAG.getVectorIdxConstant(0, DL));
608  }
609 
610  SDValue Res =
611  DAG.getNode(ISD::IS_FPCLASS, DL, MVT::i1, {Arg, Test}, N->getFlags());
612  // Vectors may have a different boolean contents to scalars. Promote the
613  // value appropriately.
614  ISD::NodeType ExtendCode =
616  return DAG.getNode(ExtendCode, DL, ResultVT, Res);
617 }
618 
619 //===----------------------------------------------------------------------===//
620 // Operand Vector Scalarization <1 x ty> -> ty.
621 //===----------------------------------------------------------------------===//
622 
623 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
624  LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
625  dbgs() << "\n");
626  SDValue Res = SDValue();
627 
628  switch (N->getOpcode()) {
629  default:
630 #ifndef NDEBUG
631  dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
632  N->dump(&DAG);
633  dbgs() << "\n";
634 #endif
635  report_fatal_error("Do not know how to scalarize this operator's "
636  "operand!\n");
637  case ISD::BITCAST:
638  Res = ScalarizeVecOp_BITCAST(N);
639  break;
640  case ISD::ANY_EXTEND:
641  case ISD::ZERO_EXTEND:
642  case ISD::SIGN_EXTEND:
643  case ISD::TRUNCATE:
644  case ISD::FP_TO_SINT:
645  case ISD::FP_TO_UINT:
646  case ISD::SINT_TO_FP:
647  case ISD::UINT_TO_FP:
648  Res = ScalarizeVecOp_UnaryOp(N);
649  break;
654  Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
655  break;
656  case ISD::CONCAT_VECTORS:
657  Res = ScalarizeVecOp_CONCAT_VECTORS(N);
658  break;
660  Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
661  break;
662  case ISD::VSELECT:
663  Res = ScalarizeVecOp_VSELECT(N);
664  break;
665  case ISD::SETCC:
666  Res = ScalarizeVecOp_VSETCC(N);
667  break;
668  case ISD::STORE:
669  Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
670  break;
672  Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
673  break;
674  case ISD::FP_ROUND:
675  Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
676  break;
678  Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
679  break;
680  case ISD::FP_EXTEND:
681  Res = ScalarizeVecOp_FP_EXTEND(N);
682  break;
683  case ISD::VECREDUCE_FADD:
684  case ISD::VECREDUCE_FMUL:
685  case ISD::VECREDUCE_ADD:
686  case ISD::VECREDUCE_MUL:
687  case ISD::VECREDUCE_AND:
688  case ISD::VECREDUCE_OR:
689  case ISD::VECREDUCE_XOR:
690  case ISD::VECREDUCE_SMAX:
691  case ISD::VECREDUCE_SMIN:
692  case ISD::VECREDUCE_UMAX:
693  case ISD::VECREDUCE_UMIN:
694  case ISD::VECREDUCE_FMAX:
695  case ISD::VECREDUCE_FMIN:
696  Res = ScalarizeVecOp_VECREDUCE(N);
697  break;
700  Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
701  break;
702  }
703 
704  // If the result is null, the sub-method took care of registering results etc.
705  if (!Res.getNode()) return false;
706 
707  // If the result is N, the sub-method updated N in place. Tell the legalizer
708  // core about this.
709  if (Res.getNode() == N)
710  return true;
711 
712  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
713  "Invalid operand expansion");
714 
715  ReplaceValueWith(SDValue(N, 0), Res);
716  return false;
717 }
718 
719 /// If the value to convert is a vector that needs to be scalarized, it must be
720 /// <1 x ty>. Convert the element instead.
721 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
722  SDValue Elt = GetScalarizedVector(N->getOperand(0));
723  return DAG.getNode(ISD::BITCAST, SDLoc(N),
724  N->getValueType(0), Elt);
725 }
726 
727 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
728 /// Do the operation on the element instead.
729 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
730  assert(N->getValueType(0).getVectorNumElements() == 1 &&
731  "Unexpected vector type!");
732  SDValue Elt = GetScalarizedVector(N->getOperand(0));
733  SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
734  N->getValueType(0).getScalarType(), Elt);
735  // Revectorize the result so the types line up with what the uses of this
736  // expression expect.
737  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
738 }
739 
740 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
741 /// Do the strict FP operation on the element instead.
742 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
743  assert(N->getValueType(0).getVectorNumElements() == 1 &&
744  "Unexpected vector type!");
745  SDValue Elt = GetScalarizedVector(N->getOperand(1));
746  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
747  { N->getValueType(0).getScalarType(), MVT::Other },
748  { N->getOperand(0), Elt });
749  // Legalize the chain result - switch anything that used the old chain to
750  // use the new one.
751  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
752  // Revectorize the result so the types line up with what the uses of this
753  // expression expect.
754  Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
755 
756  // Do our own replacement and return SDValue() to tell the caller that we
757  // handled all replacements since caller can only handle a single result.
758  ReplaceValueWith(SDValue(N, 0), Res);
759  return SDValue();
760 }
761 
762 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
763 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
764  SmallVector<SDValue, 8> Ops(N->getNumOperands());
765  for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
766  Ops[i] = GetScalarizedVector(N->getOperand(i));
767  return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
768 }
769 
770 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
771 /// so just return the element, ignoring the index.
772 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
773  EVT VT = N->getValueType(0);
774  SDValue Res = GetScalarizedVector(N->getOperand(0));
775  if (Res.getValueType() != VT)
776  Res = VT.isFloatingPoint()
777  ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
778  : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
779  return Res;
780 }
781 
782 /// If the input condition is a vector that needs to be scalarized, it must be
783 /// <1 x i1>, so just convert to a normal ISD::SELECT
784 /// (still with vector output type since that was acceptable if we got here).
785 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
786  SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
787  EVT VT = N->getValueType(0);
788 
789  return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
790  N->getOperand(2));
791 }
792 
793 /// If the operand is a vector that needs to be scalarized then the
794 /// result must be v1i1, so just convert to a scalar SETCC and wrap
795 /// with a scalar_to_vector since the res type is legal if we got here
796 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
797  assert(N->getValueType(0).isVector() &&
798  N->getOperand(0).getValueType().isVector() &&
799  "Operand types must be vectors");
800  assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
801 
802  EVT VT = N->getValueType(0);
803  SDValue LHS = GetScalarizedVector(N->getOperand(0));
804  SDValue RHS = GetScalarizedVector(N->getOperand(1));
805 
806  EVT OpVT = N->getOperand(0).getValueType();
807  EVT NVT = VT.getVectorElementType();
808  SDLoc DL(N);
809  // Turn it into a scalar SETCC.
810  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
811  N->getOperand(2));
812 
813  // Vectors may have a different boolean contents to scalars. Promote the
814  // value appropriately.
815  ISD::NodeType ExtendCode =
817 
818  Res = DAG.getNode(ExtendCode, DL, NVT, Res);
819 
820  return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
821 }
822 
823 /// If the value to store is a vector that needs to be scalarized, it must be
824 /// <1 x ty>. Just store the element.
825 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
826  assert(N->isUnindexed() && "Indexed store of one-element vector?");
827  assert(OpNo == 1 && "Do not know how to scalarize this operand!");
828  SDLoc dl(N);
829 
830  if (N->isTruncatingStore())
831  return DAG.getTruncStore(
832  N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
833  N->getBasePtr(), N->getPointerInfo(),
834  N->getMemoryVT().getVectorElementType(), N->getOriginalAlign(),
835  N->getMemOperand()->getFlags(), N->getAAInfo());
836 
837  return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
838  N->getBasePtr(), N->getPointerInfo(),
839  N->getOriginalAlign(), N->getMemOperand()->getFlags(),
840  N->getAAInfo());
841 }
842 
843 /// If the value to round is a vector that needs to be scalarized, it must be
844 /// <1 x ty>. Convert the element instead.
845 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
846  assert(OpNo == 0 && "Wrong operand for scalarization!");
847  SDValue Elt = GetScalarizedVector(N->getOperand(0));
848  SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
849  N->getValueType(0).getVectorElementType(), Elt,
850  N->getOperand(1));
851  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
852 }
853 
854 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
855  unsigned OpNo) {
856  assert(OpNo == 1 && "Wrong operand for scalarization!");
857  SDValue Elt = GetScalarizedVector(N->getOperand(1));
859  { N->getValueType(0).getVectorElementType(),
860  MVT::Other },
861  { N->getOperand(0), Elt, N->getOperand(2) });
862  // Legalize the chain result - switch anything that used the old chain to
863  // use the new one.
864  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
865 
866  Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
867 
868  // Do our own replacement and return SDValue() to tell the caller that we
869  // handled all replacements since caller can only handle a single result.
870  ReplaceValueWith(SDValue(N, 0), Res);
871  return SDValue();
872 }
873 
874 /// If the value to extend is a vector that needs to be scalarized, it must be
875 /// <1 x ty>. Convert the element instead.
876 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
877  SDValue Elt = GetScalarizedVector(N->getOperand(0));
878  SDValue Res = DAG.getNode(ISD::FP_EXTEND, SDLoc(N),
879  N->getValueType(0).getVectorElementType(), Elt);
880  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
881 }
882 
883 /// If the value to extend is a vector that needs to be scalarized, it must be
884 /// <1 x ty>. Convert the element instead.
885 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
886  SDValue Elt = GetScalarizedVector(N->getOperand(1));
887  SDValue Res =
889  {N->getValueType(0).getVectorElementType(), MVT::Other},
890  {N->getOperand(0), Elt});
891  // Legalize the chain result - switch anything that used the old chain to
892  // use the new one.
893  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
894 
895  Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
896 
897  // Do our own replacement and return SDValue() to tell the caller that we
898  // handled all replacements since caller can only handle a single result.
899  ReplaceValueWith(SDValue(N, 0), Res);
900  return SDValue();
901 }
902 
903 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
904  SDValue Res = GetScalarizedVector(N->getOperand(0));
905  // Result type may be wider than element type.
906  if (Res.getValueType() != N->getValueType(0))
907  Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res);
908  return Res;
909 }
910 
911 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
912  SDValue AccOp = N->getOperand(0);
913  SDValue VecOp = N->getOperand(1);
914 
915  unsigned BaseOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
916 
917  SDValue Op = GetScalarizedVector(VecOp);
918  return DAG.getNode(BaseOpc, SDLoc(N), N->getValueType(0),
919  AccOp, Op, N->getFlags());
920 }
921 
922 //===----------------------------------------------------------------------===//
923 // Result Vector Splitting
924 //===----------------------------------------------------------------------===//
925 
926 /// This method is called when the specified result of the specified node is
927 /// found to need vector splitting. At this point, the node may also have
928 /// invalid operands or may have other results that need legalization, we just
929 /// know that (at least) one result needs vector splitting.
930 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
931  LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
932  SDValue Lo, Hi;
933 
934  // See if the target wants to custom expand this node.
935  if (CustomLowerNode(N, N->getValueType(ResNo), true))
936  return;
937 
938  switch (N->getOpcode()) {
939  default:
940 #ifndef NDEBUG
941  dbgs() << "SplitVectorResult #" << ResNo << ": ";
942  N->dump(&DAG);
943  dbgs() << "\n";
944 #endif
945  report_fatal_error("Do not know how to split the result of this "
946  "operator!\n");
947 
948  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
949  case ISD::VSELECT:
950  case ISD::SELECT:
951  case ISD::VP_MERGE:
952  case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break;
953  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
954  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
955  case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
956  case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
957  case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
958  case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
959  case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
960  case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
961  case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
962  case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
963  case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
964  case ISD::SPLAT_VECTOR:
966  SplitVecRes_ScalarOp(N, Lo, Hi);
967  break;
968  case ISD::STEP_VECTOR:
969  SplitVecRes_STEP_VECTOR(N, Lo, Hi);
970  break;
971  case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
972  case ISD::LOAD:
973  SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
974  break;
975  case ISD::VP_LOAD:
976  SplitVecRes_VP_LOAD(cast<VPLoadSDNode>(N), Lo, Hi);
977  break;
978  case ISD::MLOAD:
979  SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
980  break;
981  case ISD::MGATHER:
982  case ISD::VP_GATHER:
983  SplitVecRes_Gather(cast<MemSDNode>(N), Lo, Hi, /*SplitSETCC*/ true);
984  break;
985  case ISD::SETCC:
986  SplitVecRes_SETCC(N, Lo, Hi);
987  break;
988  case ISD::VECTOR_REVERSE:
989  SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
990  break;
991  case ISD::VECTOR_SHUFFLE:
992  SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
993  break;
994  case ISD::VECTOR_SPLICE:
995  SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
996  break;
997  case ISD::VAARG:
998  SplitVecRes_VAARG(N, Lo, Hi);
999  break;
1000 
1004  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1005  break;
1006 
1007  case ISD::ABS:
1008  case ISD::BITREVERSE:
1009  case ISD::BSWAP:
1010  case ISD::CTLZ:
1011  case ISD::CTTZ:
1012  case ISD::CTLZ_ZERO_UNDEF:
1013  case ISD::CTTZ_ZERO_UNDEF:
1014  case ISD::CTPOP:
1015  case ISD::FABS:
1016  case ISD::FCEIL:
1017  case ISD::FCOS:
1018  case ISD::FEXP:
1019  case ISD::FEXP2:
1020  case ISD::FFLOOR:
1021  case ISD::FLOG:
1022  case ISD::FLOG10:
1023  case ISD::FLOG2:
1024  case ISD::FNEARBYINT:
1025  case ISD::FNEG: case ISD::VP_FNEG:
1026  case ISD::FREEZE:
1027  case ISD::ARITH_FENCE:
1028  case ISD::FP_EXTEND:
1029  case ISD::FP_ROUND:
1030  case ISD::FP_TO_SINT:
1031  case ISD::FP_TO_UINT:
1032  case ISD::FRINT:
1033  case ISD::FROUND:
1034  case ISD::FROUNDEVEN:
1035  case ISD::FSIN:
1036  case ISD::FSQRT:
1037  case ISD::FTRUNC:
1038  case ISD::SINT_TO_FP:
1039  case ISD::TRUNCATE:
1040  case ISD::UINT_TO_FP:
1041  case ISD::FCANONICALIZE:
1042  SplitVecRes_UnaryOp(N, Lo, Hi);
1043  break;
1044 
1045  case ISD::ANY_EXTEND:
1046  case ISD::SIGN_EXTEND:
1047  case ISD::ZERO_EXTEND:
1048  SplitVecRes_ExtendOp(N, Lo, Hi);
1049  break;
1050 
1051  case ISD::ADD: case ISD::VP_ADD:
1052  case ISD::SUB: case ISD::VP_SUB:
1053  case ISD::MUL: case ISD::VP_MUL:
1054  case ISD::MULHS:
1055  case ISD::MULHU:
1056  case ISD::FADD: case ISD::VP_FADD:
1057  case ISD::FSUB: case ISD::VP_FSUB:
1058  case ISD::FMUL: case ISD::VP_FMUL:
1059  case ISD::FMINNUM:
1060  case ISD::FMAXNUM:
1061  case ISD::FMINIMUM:
1062  case ISD::FMAXIMUM:
1063  case ISD::SDIV: case ISD::VP_SDIV:
1064  case ISD::UDIV: case ISD::VP_UDIV:
1065  case ISD::FDIV: case ISD::VP_FDIV:
1066  case ISD::FPOW:
1067  case ISD::AND: case ISD::VP_AND:
1068  case ISD::OR: case ISD::VP_OR:
1069  case ISD::XOR: case ISD::VP_XOR:
1070  case ISD::SHL: case ISD::VP_SHL:
1071  case ISD::SRA: case ISD::VP_ASHR:
1072  case ISD::SRL: case ISD::VP_LSHR:
1073  case ISD::UREM: case ISD::VP_UREM:
1074  case ISD::SREM: case ISD::VP_SREM:
1075  case ISD::FREM: case ISD::VP_FREM:
1076  case ISD::SMIN:
1077  case ISD::SMAX:
1078  case ISD::UMIN:
1079  case ISD::UMAX:
1080  case ISD::SADDSAT:
1081  case ISD::UADDSAT:
1082  case ISD::SSUBSAT:
1083  case ISD::USUBSAT:
1084  case ISD::SSHLSAT:
1085  case ISD::USHLSAT:
1086  case ISD::ROTL:
1087  case ISD::ROTR:
1088  SplitVecRes_BinOp(N, Lo, Hi);
1089  break;
1090  case ISD::FMA: case ISD::VP_FMA:
1091  case ISD::FSHL:
1092  case ISD::FSHR:
1093  SplitVecRes_TernaryOp(N, Lo, Hi);
1094  break;
1095 
1096 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1097  case ISD::STRICT_##DAGN:
1098 #include "llvm/IR/ConstrainedOps.def"
1099  SplitVecRes_StrictFPOp(N, Lo, Hi);
1100  break;
1101 
1102  case ISD::FP_TO_UINT_SAT:
1103  case ISD::FP_TO_SINT_SAT:
1104  SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1105  break;
1106 
1107  case ISD::UADDO:
1108  case ISD::SADDO:
1109  case ISD::USUBO:
1110  case ISD::SSUBO:
1111  case ISD::UMULO:
1112  case ISD::SMULO:
1113  SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1114  break;
1115  case ISD::SMULFIX:
1116  case ISD::SMULFIXSAT:
1117  case ISD::UMULFIX:
1118  case ISD::UMULFIXSAT:
1119  case ISD::SDIVFIX:
1120  case ISD::SDIVFIXSAT:
1121  case ISD::UDIVFIX:
1122  case ISD::UDIVFIXSAT:
1123  SplitVecRes_FIX(N, Lo, Hi);
1124  break;
1125  }
1126 
1127  // If Lo/Hi is null, the sub-method took care of registering results etc.
1128  if (Lo.getNode())
1129  SetSplitVector(SDValue(N, ResNo), Lo, Hi);
1130 }
1131 
1132 void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1133  MachinePointerInfo &MPI, SDValue &Ptr,
1134  uint64_t *ScaledOffset) {
1135  SDLoc DL(N);
1136  unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinSize() / 8;
1137 
1138  if (MemVT.isScalableVector()) {
1139  SDNodeFlags Flags;
1140  SDValue BytesIncrement = DAG.getVScale(
1141  DL, Ptr.getValueType(),
1142  APInt(Ptr.getValueSizeInBits().getFixedSize(), IncrementSize));
1143  MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1144  Flags.setNoUnsignedWrap(true);
1145  if (ScaledOffset)
1146  *ScaledOffset += IncrementSize;
1147  Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, BytesIncrement,
1148  Flags);
1149  } else {
1150  MPI = N->getPointerInfo().getWithOffset(IncrementSize);
1151  // Increment the pointer to the other half.
1152  Ptr = DAG.getObjectPtrOffset(DL, Ptr, TypeSize::Fixed(IncrementSize));
1153  }
1154 }
1155 
1156 std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1157  return SplitMask(Mask, SDLoc(Mask));
1158 }
1159 
1160 std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1161  const SDLoc &DL) {
1162  SDValue MaskLo, MaskHi;
1163  EVT MaskVT = Mask.getValueType();
1164  if (getTypeAction(MaskVT) == TargetLowering::TypeSplitVector)
1165  GetSplitVector(Mask, MaskLo, MaskHi);
1166  else
1167  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
1168  return std::make_pair(MaskLo, MaskHi);
1169 }
1170 
1171 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1172  SDValue LHSLo, LHSHi;
1173  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1174  SDValue RHSLo, RHSHi;
1175  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1176  SDLoc dl(N);
1177 
1178  const SDNodeFlags Flags = N->getFlags();
1179  unsigned Opcode = N->getOpcode();
1180  if (N->getNumOperands() == 2) {
1181  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
1182  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
1183  return;
1184  }
1185 
1186  assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1187  assert(N->isVPOpcode() && "Expected VP opcode");
1188 
1189  SDValue MaskLo, MaskHi;
1190  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2));
1191 
1192  SDValue EVLLo, EVLHi;
1193  std::tie(EVLLo, EVLHi) =
1194  DAG.SplitEVL(N->getOperand(3), N->getValueType(0), dl);
1195 
1196  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(),
1197  {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1198  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(),
1199  {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1200 }
1201 
1202 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1203  SDValue &Hi) {
1204  SDValue Op0Lo, Op0Hi;
1205  GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
1206  SDValue Op1Lo, Op1Hi;
1207  GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
1208  SDValue Op2Lo, Op2Hi;
1209  GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
1210  SDLoc dl(N);
1211 
1212  const SDNodeFlags Flags = N->getFlags();
1213  unsigned Opcode = N->getOpcode();
1214  if (N->getNumOperands() == 3) {
1215  Lo = DAG.getNode(Opcode, dl, Op0Lo.getValueType(), Op0Lo, Op1Lo, Op2Lo, Flags);
1216  Hi = DAG.getNode(Opcode, dl, Op0Hi.getValueType(), Op0Hi, Op1Hi, Op2Hi, Flags);
1217  return;
1218  }
1219 
1220  assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
1221  assert(N->isVPOpcode() && "Expected VP opcode");
1222 
1223  SDValue MaskLo, MaskHi;
1224  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
1225 
1226  SDValue EVLLo, EVLHi;
1227  std::tie(EVLLo, EVLHi) =
1228  DAG.SplitEVL(N->getOperand(4), N->getValueType(0), dl);
1229 
1230  Lo = DAG.getNode(Opcode, dl, Op0Lo.getValueType(),
1231  {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags);
1232  Hi = DAG.getNode(Opcode, dl, Op0Hi.getValueType(),
1233  {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags);
1234 }
1235 
1236 void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1237  SDValue LHSLo, LHSHi;
1238  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1239  SDValue RHSLo, RHSHi;
1240  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1241  SDLoc dl(N);
1242  SDValue Op2 = N->getOperand(2);
1243 
1244  unsigned Opcode = N->getOpcode();
1245  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2,
1246  N->getFlags());
1247  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2,
1248  N->getFlags());
1249 }
1250 
1251 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1252  SDValue &Hi) {
1253  // We know the result is a vector. The input may be either a vector or a
1254  // scalar value.
1255  EVT LoVT, HiVT;
1256  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1257  SDLoc dl(N);
1258 
1259  SDValue InOp = N->getOperand(0);
1260  EVT InVT = InOp.getValueType();
1261 
1262  // Handle some special cases efficiently.
1263  switch (getTypeAction(InVT)) {
1271  break;
1274  // A scalar to vector conversion, where the scalar needs expansion.
1275  // If the vector is being split in two then we can just convert the
1276  // expanded pieces.
1277  if (LoVT == HiVT) {
1278  GetExpandedOp(InOp, Lo, Hi);
1279  if (DAG.getDataLayout().isBigEndian())
1280  std::swap(Lo, Hi);
1281  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1282  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1283  return;
1284  }
1285  break;
1287  // If the input is a vector that needs to be split, convert each split
1288  // piece of the input now.
1289  GetSplitVector(InOp, Lo, Hi);
1290  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1291  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1292  return;
1294  report_fatal_error("Scalarization of scalable vectors is not supported.");
1295  }
1296 
1297  // In the general case, convert the input to an integer and split it by hand.
1298  EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
1299  EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
1300  if (DAG.getDataLayout().isBigEndian())
1301  std::swap(LoIntVT, HiIntVT);
1302 
1303  SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
1304 
1305  if (DAG.getDataLayout().isBigEndian())
1306  std::swap(Lo, Hi);
1307  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1308  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1309 }
1310 
1311 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1312  SDValue &Hi) {
1313  EVT LoVT, HiVT;
1314  SDLoc dl(N);
1315  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1316  unsigned LoNumElts = LoVT.getVectorNumElements();
1317  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1318  Lo = DAG.getBuildVector(LoVT, dl, LoOps);
1319 
1320  SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1321  Hi = DAG.getBuildVector(HiVT, dl, HiOps);
1322 }
1323 
1324 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1325  SDValue &Hi) {
1326  assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1327  SDLoc dl(N);
1328  unsigned NumSubvectors = N->getNumOperands() / 2;
1329  if (NumSubvectors == 1) {
1330  Lo = N->getOperand(0);
1331  Hi = N->getOperand(1);
1332  return;
1333  }
1334 
1335  EVT LoVT, HiVT;
1336  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1337 
1338  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1339  Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
1340 
1341  SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1342  Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
1343 }
1344 
1345 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1346  SDValue &Hi) {
1347  SDValue Vec = N->getOperand(0);
1348  SDValue Idx = N->getOperand(1);
1349  SDLoc dl(N);
1350 
1351  EVT LoVT, HiVT;
1352  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1353 
1354  Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
1355  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1356  Hi = DAG.getNode(
1357  ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
1358  DAG.getVectorIdxConstant(IdxVal + LoVT.getVectorMinNumElements(), dl));
1359 }
1360 
1361 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1362  SDValue &Hi) {
1363  SDValue Vec = N->getOperand(0);
1364  SDValue SubVec = N->getOperand(1);
1365  SDValue Idx = N->getOperand(2);
1366  SDLoc dl(N);
1367  GetSplitVector(Vec, Lo, Hi);
1368 
1369  EVT VecVT = Vec.getValueType();
1370  EVT LoVT = Lo.getValueType();
1371  EVT SubVecVT = SubVec.getValueType();
1372  unsigned VecElems = VecVT.getVectorMinNumElements();
1373  unsigned SubElems = SubVecVT.getVectorMinNumElements();
1374  unsigned LoElems = LoVT.getVectorMinNumElements();
1375 
1376  // If we know the index is in the first half, and we know the subvector
1377  // doesn't cross the boundary between the halves, we can avoid spilling the
1378  // vector, and insert into the lower half of the split vector directly.
1379  unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1380  if (IdxVal + SubElems <= LoElems) {
1381  Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1382  return;
1383  }
1384  // Similarly if the subvector is fully in the high half, but mind that we
1385  // can't tell whether a fixed-length subvector is fully within the high half
1386  // of a scalable vector.
1387  if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1388  IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1389  Hi = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, Hi.getValueType(), Hi, SubVec,
1390  DAG.getVectorIdxConstant(IdxVal - LoElems, dl));
1391  return;
1392  }
1393 
1394  // Spill the vector to the stack.
1395  // In cases where the vector is illegal it will be broken down into parts
1396  // and stored in parts - we should use the alignment for the smallest part.
1397  Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1398  SDValue StackPtr =
1399  DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1400  auto &MF = DAG.getMachineFunction();
1401  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1402  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1403 
1404  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1405  SmallestAlign);
1406 
1407  // Store the new subvector into the specified index.
1408  SDValue SubVecPtr =
1409  TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1410  Store = DAG.getStore(Store, dl, SubVec, SubVecPtr,
1412 
1413  // Load the Lo part from the stack slot.
1414  Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, PtrInfo,
1415  SmallestAlign);
1416 
1417  // Increment the pointer to the other part.
1418  auto *Load = cast<LoadSDNode>(Lo);
1419  MachinePointerInfo MPI = Load->getPointerInfo();
1420  IncrementPointer(Load, LoVT, MPI, StackPtr);
1421 
1422  // Load the Hi part from the stack slot.
1423  Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MPI, SmallestAlign);
1424 }
1425 
1426 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1427  SDValue &Hi) {
1428  SDLoc dl(N);
1429  GetSplitVector(N->getOperand(0), Lo, Hi);
1430  Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
1431  Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
1432 }
1433 
1434 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1435  SDValue &Hi) {
1436  SDValue LHSLo, LHSHi;
1437  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1438  SDLoc DL(N);
1439 
1440  SDValue RHSLo, RHSHi;
1441  SDValue RHS = N->getOperand(1);
1442  EVT RHSVT = RHS.getValueType();
1443  if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1444  GetSplitVector(RHS, RHSLo, RHSHi);
1445  else
1446  std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1447 
1448 
1449  Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
1450  Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
1451 }
1452 
1453 void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
1454  SDValue &Hi) {
1455  SDLoc DL(N);
1456  SDValue ArgLo, ArgHi;
1457  SDValue Test = N->getOperand(1);
1458  GetSplitVector(N->getOperand(0), ArgLo, ArgHi);
1459  EVT LoVT, HiVT;
1460  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1461 
1462  Lo = DAG.getNode(ISD::IS_FPCLASS, DL, LoVT, ArgLo, Test, N->getFlags());
1463  Hi = DAG.getNode(ISD::IS_FPCLASS, DL, HiVT, ArgHi, Test, N->getFlags());
1464 }
1465 
1466 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1467  SDValue &Hi) {
1468  SDValue LHSLo, LHSHi;
1469  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1470  SDLoc dl(N);
1471 
1472  EVT LoVT, HiVT;
1473  std::tie(LoVT, HiVT) =
1474  DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1475 
1476  Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1477  DAG.getValueType(LoVT));
1478  Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1479  DAG.getValueType(HiVT));
1480 }
1481 
1482 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1483  SDValue &Hi) {
1484  unsigned Opcode = N->getOpcode();
1485  SDValue N0 = N->getOperand(0);
1486 
1487  SDLoc dl(N);
1488  SDValue InLo, InHi;
1489 
1490  if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1491  GetSplitVector(N0, InLo, InHi);
1492  else
1493  std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1494 
1495  EVT InLoVT = InLo.getValueType();
1496  unsigned InNumElements = InLoVT.getVectorNumElements();
1497 
1498  EVT OutLoVT, OutHiVT;
1499  std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1500  unsigned OutNumElements = OutLoVT.getVectorNumElements();
1501  assert((2 * OutNumElements) <= InNumElements &&
1502  "Illegal extend vector in reg split");
1503 
1504  // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1505  // input vector (i.e. we only use InLo):
1506  // OutLo will extend the first OutNumElements from InLo.
1507  // OutHi will extend the next OutNumElements from InLo.
1508 
1509  // Shuffle the elements from InLo for OutHi into the bottom elements to
1510  // create a 'fake' InHi.
1511  SmallVector<int, 8> SplitHi(InNumElements, -1);
1512  for (unsigned i = 0; i != OutNumElements; ++i)
1513  SplitHi[i] = i + OutNumElements;
1514  InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1515 
1516  Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1517  Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1518 }
1519 
1520 void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1521  SDValue &Hi) {
1522  unsigned NumOps = N->getNumOperands();
1523  SDValue Chain = N->getOperand(0);
1524  EVT LoVT, HiVT;
1525  SDLoc dl(N);
1526  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1527 
1528  SmallVector<SDValue, 4> OpsLo(NumOps);
1529  SmallVector<SDValue, 4> OpsHi(NumOps);
1530 
1531  // The Chain is the first operand.
1532  OpsLo[0] = Chain;
1533  OpsHi[0] = Chain;
1534 
1535  // Now process the remaining operands.
1536  for (unsigned i = 1; i < NumOps; ++i) {
1537  SDValue Op = N->getOperand(i);
1538  SDValue OpLo = Op;
1539  SDValue OpHi = Op;
1540 
1541  EVT InVT = Op.getValueType();
1542  if (InVT.isVector()) {
1543  // If the input also splits, handle it directly for a
1544  // compile time speedup. Otherwise split it by hand.
1545  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1546  GetSplitVector(Op, OpLo, OpHi);
1547  else
1548  std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1549  }
1550 
1551  OpsLo[i] = OpLo;
1552  OpsHi[i] = OpHi;
1553  }
1554 
1555  EVT LoValueVTs[] = {LoVT, MVT::Other};
1556  EVT HiValueVTs[] = {HiVT, MVT::Other};
1557  Lo = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo,
1558  N->getFlags());
1559  Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi,
1560  N->getFlags());
1561 
1562  // Build a factor node to remember that this Op is independent of the
1563  // other one.
1564  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1565  Lo.getValue(1), Hi.getValue(1));
1566 
1567  // Legalize the chain result - switch anything that used the old chain to
1568  // use the new one.
1569  ReplaceValueWith(SDValue(N, 1), Chain);
1570 }
1571 
1572 SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1573  SDValue Chain = N->getOperand(0);
1574  EVT VT = N->getValueType(0);
1575  unsigned NE = VT.getVectorNumElements();
1576  EVT EltVT = VT.getVectorElementType();
1577  SDLoc dl(N);
1578 
1579  SmallVector<SDValue, 8> Scalars;
1580  SmallVector<SDValue, 4> Operands(N->getNumOperands());
1581 
1582  // If ResNE is 0, fully unroll the vector op.
1583  if (ResNE == 0)
1584  ResNE = NE;
1585  else if (NE > ResNE)
1586  NE = ResNE;
1587 
1588  //The results of each unrolled operation, including the chain.
1589  EVT ChainVTs[] = {EltVT, MVT::Other};
1590  SmallVector<SDValue, 8> Chains;
1591 
1592  unsigned i;
1593  for (i = 0; i != NE; ++i) {
1594  Operands[0] = Chain;
1595  for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
1596  SDValue Operand = N->getOperand(j);
1597  EVT OperandVT = Operand.getValueType();
1598  if (OperandVT.isVector()) {
1599  EVT OperandEltVT = OperandVT.getVectorElementType();
1600  Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
1601  Operand, DAG.getVectorIdxConstant(i, dl));
1602  } else {
1603  Operands[j] = Operand;
1604  }
1605  }
1606  SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1607  Scalar.getNode()->setFlags(N->getFlags());
1608 
1609  //Add in the scalar as well as its chain value to the
1610  //result vectors.
1611  Scalars.push_back(Scalar);
1612  Chains.push_back(Scalar.getValue(1));
1613  }
1614 
1615  for (; i < ResNE; ++i)
1616  Scalars.push_back(DAG.getUNDEF(EltVT));
1617 
1618  // Build a new factor node to connect the chain back together.
1619  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1620  ReplaceValueWith(SDValue(N, 1), Chain);
1621 
1622  // Create a new BUILD_VECTOR node
1623  EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE);
1624  return DAG.getBuildVector(VecVT, dl, Scalars);
1625 }
1626 
1627 void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1628  SDValue &Lo, SDValue &Hi) {
1629  SDLoc dl(N);
1630  EVT ResVT = N->getValueType(0);
1631  EVT OvVT = N->getValueType(1);
1632  EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1633  std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
1634  std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
1635 
1636  SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1637  if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
1638  GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
1639  GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
1640  } else {
1641  std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
1642  std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
1643  }
1644 
1645  unsigned Opcode = N->getOpcode();
1646  SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
1647  SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
1648  SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
1649  SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
1650  LoNode->setFlags(N->getFlags());
1651  HiNode->setFlags(N->getFlags());
1652 
1653  Lo = SDValue(LoNode, ResNo);
1654  Hi = SDValue(HiNode, ResNo);
1655 
1656  // Replace the other vector result not being explicitly split here.
1657  unsigned OtherNo = 1 - ResNo;
1658  EVT OtherVT = N->getValueType(OtherNo);
1659  if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
1660  SetSplitVector(SDValue(N, OtherNo),
1661  SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1662  } else {
1663  SDValue OtherVal = DAG.getNode(
1664  ISD::CONCAT_VECTORS, dl, OtherVT,
1665  SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1666  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
1667  }
1668 }
1669 
1670 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1671  SDValue &Hi) {
1672  SDValue Vec = N->getOperand(0);
1673  SDValue Elt = N->getOperand(1);
1674  SDValue Idx = N->getOperand(2);
1675  SDLoc dl(N);
1676  GetSplitVector(Vec, Lo, Hi);
1677 
1678  if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1679  unsigned IdxVal = CIdx->getZExtValue();
1680  unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
1681  if (IdxVal < LoNumElts) {
1682  Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1683  Lo.getValueType(), Lo, Elt, Idx);
1684  return;
1685  } else if (!Vec.getValueType().isScalableVector()) {
1686  Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1687  DAG.getVectorIdxConstant(IdxVal - LoNumElts, dl));
1688  return;
1689  }
1690  }
1691 
1692  // See if the target wants to custom expand this node.
1693  if (CustomLowerNode(N, N->getValueType(0), true))
1694  return;
1695 
1696  // Make the vector elements byte-addressable if they aren't already.
1697  EVT VecVT = Vec.getValueType();
1698  EVT EltVT = VecVT.getVectorElementType();
1699  if (VecVT.getScalarSizeInBits() < 8) {
1700  EltVT = MVT::i8;
1701  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1702  VecVT.getVectorElementCount());
1703  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1704  // Extend the element type to match if needed.
1705  if (EltVT.bitsGT(Elt.getValueType()))
1706  Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1707  }
1708 
1709  // Spill the vector to the stack.
1710  // In cases where the vector is illegal it will be broken down into parts
1711  // and stored in parts - we should use the alignment for the smallest part.
1712  Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1713  SDValue StackPtr =
1714  DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1715  auto &MF = DAG.getMachineFunction();
1716  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1717  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1718 
1719  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1720  SmallestAlign);
1721 
1722  // Store the new element. This may be larger than the vector element type,
1723  // so use a truncating store.
1724  SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1725  Store = DAG.getTruncStore(
1726  Store, dl, Elt, EltPtr, MachinePointerInfo::getUnknownStack(MF), EltVT,
1727  commonAlignment(SmallestAlign,
1728  EltVT.getFixedSizeInBits() / 8));
1729 
1730  EVT LoVT, HiVT;
1731  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1732 
1733  // Load the Lo part from the stack slot.
1734  Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo, SmallestAlign);
1735 
1736  // Increment the pointer to the other part.
1737  auto Load = cast<LoadSDNode>(Lo);
1738  MachinePointerInfo MPI = Load->getPointerInfo();
1739  IncrementPointer(Load, LoVT, MPI, StackPtr);
1740 
1741  Hi = DAG.getLoad(HiVT, dl, Store, StackPtr, MPI, SmallestAlign);
1742 
1743  // If we adjusted the original type, we need to truncate the results.
1744  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1745  if (LoVT != Lo.getValueType())
1746  Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1747  if (HiVT != Hi.getValueType())
1748  Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1749 }
1750 
1751 void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
1752  SDValue &Hi) {
1753  EVT LoVT, HiVT;
1754  SDLoc dl(N);
1755  assert(N->getValueType(0).isScalableVector() &&
1756  "Only scalable vectors are supported for STEP_VECTOR");
1757  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1758  SDValue Step = N->getOperand(0);
1759 
1760  Lo = DAG.getNode(ISD::STEP_VECTOR, dl, LoVT, Step);
1761 
1762  // Hi = Lo + (EltCnt * Step)
1763  EVT EltVT = Step.getValueType();
1764  APInt StepVal = cast<ConstantSDNode>(Step)->getAPIntValue();
1765  SDValue StartOfHi =
1766  DAG.getVScale(dl, EltVT, StepVal * LoVT.getVectorMinNumElements());
1767  StartOfHi = DAG.getSExtOrTrunc(StartOfHi, dl, HiVT.getVectorElementType());
1768  StartOfHi = DAG.getNode(ISD::SPLAT_VECTOR, dl, HiVT, StartOfHi);
1769 
1770  Hi = DAG.getNode(ISD::STEP_VECTOR, dl, HiVT, Step);
1771  Hi = DAG.getNode(ISD::ADD, dl, HiVT, Hi, StartOfHi);
1772 }
1773 
1774 void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
1775  SDValue &Hi) {
1776  EVT LoVT, HiVT;
1777  SDLoc dl(N);
1778  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1779  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, N->getOperand(0));
1780  if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
1781  Hi = DAG.getUNDEF(HiVT);
1782  } else {
1783  assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
1784  Hi = Lo;
1785  }
1786 }
1787 
1788 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1789  SDValue &Hi) {
1790  assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1791  EVT LoVT, HiVT;
1792  SDLoc dl(LD);
1793  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1794 
1795  ISD::LoadExtType ExtType = LD->getExtensionType();
1796  SDValue Ch = LD->getChain();
1797  SDValue Ptr = LD->getBasePtr();
1798  SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1799  EVT MemoryVT = LD->getMemoryVT();
1800  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1801  AAMDNodes AAInfo = LD->getAAInfo();
1802 
1803  EVT LoMemVT, HiMemVT;
1804  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1805 
1806  if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
1807  SDValue Value, NewChain;
1808  std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
1809  std::tie(Lo, Hi) = DAG.SplitVector(Value, dl);
1810  ReplaceValueWith(SDValue(LD, 1), NewChain);
1811  return;
1812  }
1813 
1814  Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1815  LD->getPointerInfo(), LoMemVT, LD->getOriginalAlign(),
1816  MMOFlags, AAInfo);
1817 
1818  MachinePointerInfo MPI;
1819  IncrementPointer(LD, LoMemVT, MPI, Ptr);
1820 
1821  Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, MPI,
1822  HiMemVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
1823 
1824  // Build a factor node to remember that this load is independent of the
1825  // other one.
1826  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1827  Hi.getValue(1));
1828 
1829  // Legalize the chain result - switch anything that used the old chain to
1830  // use the new one.
1831  ReplaceValueWith(SDValue(LD, 1), Ch);
1832 }
1833 
1834 void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
1835  SDValue &Hi) {
1836  assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
1837  EVT LoVT, HiVT;
1838  SDLoc dl(LD);
1839  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1840 
1841  ISD::LoadExtType ExtType = LD->getExtensionType();
1842  SDValue Ch = LD->getChain();
1843  SDValue Ptr = LD->getBasePtr();
1844  SDValue Offset = LD->getOffset();
1845  assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
1846  Align Alignment = LD->getOriginalAlign();
1847  SDValue Mask = LD->getMask();
1848  SDValue EVL = LD->getVectorLength();
1849  EVT MemoryVT = LD->getMemoryVT();
1850 
1851  EVT LoMemVT, HiMemVT;
1852  bool HiIsEmpty = false;
1853  std::tie(LoMemVT, HiMemVT) =
1854  DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
1855 
1856  // Split Mask operand
1857  SDValue MaskLo, MaskHi;
1858  if (Mask.getOpcode() == ISD::SETCC) {
1859  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1860  } else {
1861  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1862  GetSplitVector(Mask, MaskLo, MaskHi);
1863  else
1864  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1865  }
1866 
1867  // Split EVL operand
1868  SDValue EVLLo, EVLHi;
1869  std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, LD->getValueType(0), dl);
1870 
1872  LD->getPointerInfo(), MachineMemOperand::MOLoad,
1873  MemoryLocation::UnknownSize, Alignment, LD->getAAInfo(), LD->getRanges());
1874 
1875  Lo =
1876  DAG.getLoadVP(LD->getAddressingMode(), ExtType, LoVT, dl, Ch, Ptr, Offset,
1877  MaskLo, EVLLo, LoMemVT, MMO, LD->isExpandingLoad());
1878 
1879  if (HiIsEmpty) {
1880  // The hi vp_load has zero storage size. We therefore simply set it to
1881  // the low vp_load and rely on subsequent removal from the chain.
1882  Hi = Lo;
1883  } else {
1884  // Generate hi vp_load.
1885  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1886  LD->isExpandingLoad());
1887 
1888  MachinePointerInfo MPI;
1889  if (LoMemVT.isScalableVector())
1890  MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
1891  else
1892  MPI = LD->getPointerInfo().getWithOffset(
1893  LoMemVT.getStoreSize().getFixedSize());
1894 
1897  LD->getAAInfo(), LD->getRanges());
1898 
1899  Hi = DAG.getLoadVP(LD->getAddressingMode(), ExtType, HiVT, dl, Ch, Ptr,
1900  Offset, MaskHi, EVLHi, HiMemVT, MMO,
1901  LD->isExpandingLoad());
1902  }
1903 
1904  // Build a factor node to remember that this load is independent of the
1905  // other one.
1906  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1907  Hi.getValue(1));
1908 
1909  // Legalize the chain result - switch anything that used the old chain to
1910  // use the new one.
1911  ReplaceValueWith(SDValue(LD, 1), Ch);
1912 }
1913 
1914 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
1915  SDValue &Lo, SDValue &Hi) {
1916  assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
1917  EVT LoVT, HiVT;
1918  SDLoc dl(MLD);
1919  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1920 
1921  SDValue Ch = MLD->getChain();
1922  SDValue Ptr = MLD->getBasePtr();
1923  SDValue Offset = MLD->getOffset();
1924  assert(Offset.isUndef() && "Unexpected indexed masked load offset");
1925  SDValue Mask = MLD->getMask();
1926  SDValue PassThru = MLD->getPassThru();
1927  Align Alignment = MLD->getOriginalAlign();
1928  ISD::LoadExtType ExtType = MLD->getExtensionType();
1929 
1930  // Split Mask operand
1931  SDValue MaskLo, MaskHi;
1932  if (Mask.getOpcode() == ISD::SETCC) {
1933  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1934  } else {
1935  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1936  GetSplitVector(Mask, MaskLo, MaskHi);
1937  else
1938  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1939  }
1940 
1941  EVT MemoryVT = MLD->getMemoryVT();
1942  EVT LoMemVT, HiMemVT;
1943  bool HiIsEmpty = false;
1944  std::tie(LoMemVT, HiMemVT) =
1945  DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
1946 
1947  SDValue PassThruLo, PassThruHi;
1948  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1949  GetSplitVector(PassThru, PassThruLo, PassThruHi);
1950  else
1951  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1952 
1955  MemoryLocation::UnknownSize, Alignment, MLD->getAAInfo(),
1956  MLD->getRanges());
1957 
1958  Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
1959  MMO, MLD->getAddressingMode(), ExtType,
1960  MLD->isExpandingLoad());
1961 
1962  if (HiIsEmpty) {
1963  // The hi masked load has zero storage size. We therefore simply set it to
1964  // the low masked load and rely on subsequent removal from the chain.
1965  Hi = Lo;
1966  } else {
1967  // Generate hi masked load.
1968  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1969  MLD->isExpandingLoad());
1970 
1971  MachinePointerInfo MPI;
1972  if (LoMemVT.isScalableVector())
1974  else
1975  MPI = MLD->getPointerInfo().getWithOffset(
1976  LoMemVT.getStoreSize().getFixedSize());
1977 
1980  MLD->getAAInfo(), MLD->getRanges());
1981 
1982  Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
1983  HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
1984  MLD->isExpandingLoad());
1985  }
1986 
1987  // Build a factor node to remember that this load is independent of the
1988  // other one.
1989  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1990  Hi.getValue(1));
1991 
1992  // Legalize the chain result - switch anything that used the old chain to
1993  // use the new one.
1994  ReplaceValueWith(SDValue(MLD, 1), Ch);
1995 
1996 }
1997 
1998 void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
1999  SDValue &Hi, bool SplitSETCC) {
2000  EVT LoVT, HiVT;
2001  SDLoc dl(N);
2002  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2003 
2004  SDValue Ch = N->getChain();
2005  SDValue Ptr = N->getBasePtr();
2006  struct Operands {
2007  SDValue Mask;
2008  SDValue Index;
2009  SDValue Scale;
2010  } Ops = [&]() -> Operands {
2011  if (auto *MSC = dyn_cast<MaskedGatherSDNode>(N)) {
2012  return {MSC->getMask(), MSC->getIndex(), MSC->getScale()};
2013  }
2014  auto *VPSC = cast<VPGatherSDNode>(N);
2015  return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale()};
2016  }();
2017 
2018  EVT MemoryVT = N->getMemoryVT();
2019  Align Alignment = N->getOriginalAlign();
2020 
2021  // Split Mask operand
2022  SDValue MaskLo, MaskHi;
2023  if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2024  SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
2025  } else {
2026  std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, dl);
2027  }
2028 
2029  EVT LoMemVT, HiMemVT;
2030  // Split MemoryVT
2031  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2032 
2033  SDValue IndexHi, IndexLo;
2034  if (getTypeAction(Ops.Index.getValueType()) ==
2036  GetSplitVector(Ops.Index, IndexLo, IndexHi);
2037  else
2038  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, dl);
2039 
2041  N->getPointerInfo(), MachineMemOperand::MOLoad,
2042  MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
2043 
2044  if (auto *MGT = dyn_cast<MaskedGatherSDNode>(N)) {
2045  SDValue PassThru = MGT->getPassThru();
2046  SDValue PassThruLo, PassThruHi;
2047  if (getTypeAction(PassThru.getValueType()) ==
2049  GetSplitVector(PassThru, PassThruLo, PassThruHi);
2050  else
2051  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2052 
2053  ISD::LoadExtType ExtType = MGT->getExtensionType();
2054  ISD::MemIndexType IndexTy = MGT->getIndexType();
2055 
2056  SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2057  Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl,
2058  OpsLo, MMO, IndexTy, ExtType);
2059 
2060  SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2061  Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl,
2062  OpsHi, MMO, IndexTy, ExtType);
2063  } else {
2064  auto *VPGT = cast<VPGatherSDNode>(N);
2065  SDValue EVLLo, EVLHi;
2066  std::tie(EVLLo, EVLHi) =
2067  DAG.SplitEVL(VPGT->getVectorLength(), MemoryVT, dl);
2068 
2069  SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2070  Lo = DAG.getGatherVP(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, OpsLo,
2071  MMO, VPGT->getIndexType());
2072 
2073  SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2074  Hi = DAG.getGatherVP(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, OpsHi,
2075  MMO, VPGT->getIndexType());
2076  }
2077 
2078  // Build a factor node to remember that this load is independent of the
2079  // other one.
2080  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2081  Hi.getValue(1));
2082 
2083  // Legalize the chain result - switch anything that used the old chain to
2084  // use the new one.
2085  ReplaceValueWith(SDValue(N, 1), Ch);
2086 }
2087 
2088 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
2089  assert(N->getValueType(0).isVector() &&
2090  N->getOperand(0).getValueType().isVector() &&
2091  "Operand types must be vectors");
2092 
2093  EVT LoVT, HiVT;
2094  SDLoc DL(N);
2095  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2096 
2097  // If the input also splits, handle it directly. Otherwise split it by hand.
2098  SDValue LL, LH, RL, RH;
2099  if (getTypeAction(N->getOperand(0).getValueType()) ==
2101  GetSplitVector(N->getOperand(0), LL, LH);
2102  else
2103  std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
2104 
2105  if (getTypeAction(N->getOperand(1).getValueType()) ==
2107  GetSplitVector(N->getOperand(1), RL, RH);
2108  else
2109  std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
2110 
2111  Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
2112  Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
2113 }
2114 
2115 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
2116  SDValue &Hi) {
2117  // Get the dest types - they may not match the input types, e.g. int_to_fp.
2118  EVT LoVT, HiVT;
2119  SDLoc dl(N);
2120  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2121 
2122  // If the input also splits, handle it directly for a compile time speedup.
2123  // Otherwise split it by hand.
2124  EVT InVT = N->getOperand(0).getValueType();
2125  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
2126  GetSplitVector(N->getOperand(0), Lo, Hi);
2127  else
2128  std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
2129 
2130  const SDNodeFlags Flags = N->getFlags();
2131  unsigned Opcode = N->getOpcode();
2132  if (N->getNumOperands() <= 2) {
2133  if (Opcode == ISD::FP_ROUND) {
2134  Lo = DAG.getNode(Opcode, dl, LoVT, Lo, N->getOperand(1), Flags);
2135  Hi = DAG.getNode(Opcode, dl, HiVT, Hi, N->getOperand(1), Flags);
2136  } else {
2137  Lo = DAG.getNode(Opcode, dl, LoVT, Lo, Flags);
2138  Hi = DAG.getNode(Opcode, dl, HiVT, Hi, Flags);
2139  }
2140  return;
2141  }
2142 
2143  assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
2144  assert(N->isVPOpcode() && "Expected VP opcode");
2145 
2146  SDValue MaskLo, MaskHi;
2147  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
2148 
2149  SDValue EVLLo, EVLHi;
2150  std::tie(EVLLo, EVLHi) =
2151  DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
2152 
2153  Lo = DAG.getNode(Opcode, dl, LoVT, {Lo, MaskLo, EVLLo}, Flags);
2154  Hi = DAG.getNode(Opcode, dl, HiVT, {Hi, MaskHi, EVLHi}, Flags);
2155 }
2156 
2157 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
2158  SDValue &Hi) {
2159  SDLoc dl(N);
2160  EVT SrcVT = N->getOperand(0).getValueType();
2161  EVT DestVT = N->getValueType(0);
2162  EVT LoVT, HiVT;
2163  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
2164 
2165  // We can do better than a generic split operation if the extend is doing
2166  // more than just doubling the width of the elements and the following are
2167  // true:
2168  // - The number of vector elements is even,
2169  // - the source type is legal,
2170  // - the type of a split source is illegal,
2171  // - the type of an extended (by doubling element size) source is legal, and
2172  // - the type of that extended source when split is legal.
2173  //
2174  // This won't necessarily completely legalize the operation, but it will
2175  // more effectively move in the right direction and prevent falling down
2176  // to scalarization in many cases due to the input vector being split too
2177  // far.
2178  if (SrcVT.getVectorElementCount().isKnownEven() &&
2179  SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
2180  LLVMContext &Ctx = *DAG.getContext();
2181  EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
2182  EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
2183 
2184  EVT SplitLoVT, SplitHiVT;
2185  std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
2186  if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
2187  TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
2188  LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
2189  N->dump(&DAG); dbgs() << "\n");
2190  // Extend the source vector by one step.
2191  SDValue NewSrc =
2192  DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
2193  // Get the low and high halves of the new, extended one step, vector.
2194  std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
2195  // Extend those vector halves the rest of the way.
2196  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
2197  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
2198  return;
2199  }
2200  }
2201  // Fall back to the generic unary operator splitting otherwise.
2202  SplitVecRes_UnaryOp(N, Lo, Hi);
2203 }
2204 
2205 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
2206  SDValue &Lo, SDValue &Hi) {
2207  // The low and high parts of the original input give four input vectors.
2208  SDValue Inputs[4];
2209  SDLoc DL(N);
2210  GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
2211  GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
2212  EVT NewVT = Inputs[0].getValueType();
2213  unsigned NewElts = NewVT.getVectorNumElements();
2214 
2215  auto &&IsConstant = [](const SDValue &N) {
2216  APInt SplatValue;
2217  return N.getResNo() == 0 &&
2218  (ISD::isConstantSplatVector(N.getNode(), SplatValue) ||
2220  };
2221  auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
2222  SDValue &Input2,
2223  ArrayRef<int> Mask) {
2224  assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
2225  Input2->getOpcode() == ISD::BUILD_VECTOR &&
2226  "Expected build vector node.");
2227  EVT EltVT = NewVT.getVectorElementType();
2228  SmallVector<SDValue> Ops(NewElts, DAG.getUNDEF(EltVT));
2229  for (unsigned I = 0; I < NewElts; ++I) {
2230  if (Mask[I] == UndefMaskElem)
2231  continue;
2232  unsigned Idx = Mask[I];
2233  if (Idx >= NewElts)
2234  Ops[I] = Input2.getOperand(Idx - NewElts);
2235  else
2236  Ops[I] = Input1.getOperand(Idx);
2237  // Make the type of all elements the same as the element type.
2238  if (Ops[I].getValueType().bitsGT(EltVT))
2239  Ops[I] = DAG.getNode(ISD::TRUNCATE, DL, EltVT, Ops[I]);
2240  }
2241  return DAG.getBuildVector(NewVT, DL, Ops);
2242  };
2243 
2244  // If Lo or Hi uses elements from at most two of the four input vectors, then
2245  // express it as a vector shuffle of those two inputs. Otherwise extract the
2246  // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
2247  SmallVector<int> OrigMask(N->getMask().begin(), N->getMask().end());
2248  // Try to pack incoming shuffles/inputs.
2249  auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
2251  // Check if all inputs are shuffles of the same operands or non-shuffles.
2253  for (unsigned Idx = 0; Idx < array_lengthof(Inputs); ++Idx) {
2254  SDValue Input = Inputs[Idx];
2255  auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Input.getNode());
2256  if (!Shuffle ||
2257  Input.getOperand(0).getValueType() != Input.getValueType())
2258  continue;
2259  ShufflesIdxs[std::make_pair(Input.getOperand(0), Input.getOperand(1))]
2260  .push_back(Idx);
2261  ShufflesIdxs[std::make_pair(Input.getOperand(1), Input.getOperand(0))]
2262  .push_back(Idx);
2263  }
2264  for (auto &P : ShufflesIdxs) {
2265  if (P.second.size() < 2)
2266  continue;
2267  // Use shuffles operands instead of shuffles themselves.
2268  // 1. Adjust mask.
2269  for (int &Idx : Mask) {
2270  if (Idx == UndefMaskElem)
2271  continue;
2272  unsigned SrcRegIdx = Idx / NewElts;
2273  if (Inputs[SrcRegIdx].isUndef()) {
2274  Idx = UndefMaskElem;
2275  continue;
2276  }
2277  auto *Shuffle =
2278  dyn_cast<ShuffleVectorSDNode>(Inputs[SrcRegIdx].getNode());
2279  if (!Shuffle || !is_contained(P.second, SrcRegIdx))
2280  continue;
2281  int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2282  if (MaskElt == UndefMaskElem) {
2283  Idx = UndefMaskElem;
2284  continue;
2285  }
2286  Idx = MaskElt % NewElts +
2287  P.second[Shuffle->getOperand(MaskElt / NewElts) == P.first.first
2288  ? 0
2289  : 1] *
2290  NewElts;
2291  }
2292  // 2. Update inputs.
2293  Inputs[P.second[0]] = P.first.first;
2294  Inputs[P.second[1]] = P.first.second;
2295  // Clear the pair data.
2296  P.second.clear();
2297  ShufflesIdxs[std::make_pair(P.first.second, P.first.first)].clear();
2298  }
2299  // Check if any concat_vectors can be simplified.
2300  SmallBitVector UsedSubVector(2 * array_lengthof(Inputs));
2301  for (int &Idx : Mask) {
2302  if (Idx == UndefMaskElem)
2303  continue;
2304  unsigned SrcRegIdx = Idx / NewElts;
2305  if (Inputs[SrcRegIdx].isUndef()) {
2306  Idx = UndefMaskElem;
2307  continue;
2308  }
2310  getTypeAction(Inputs[SrcRegIdx].getValueType());
2311  if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
2312  Inputs[SrcRegIdx].getNumOperands() == 2 &&
2313  !Inputs[SrcRegIdx].getOperand(1).isUndef() &&
2314  (TypeAction == TargetLowering::TypeLegal ||
2315  TypeAction == TargetLowering::TypeWidenVector))
2316  UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
2317  }
2318  if (UsedSubVector.count() > 1) {
2320  for (unsigned I = 0; I < array_lengthof(Inputs); ++I) {
2321  if (UsedSubVector.test(2 * I) == UsedSubVector.test(2 * I + 1))
2322  continue;
2323  if (Pairs.empty() || Pairs.back().size() == 2)
2324  Pairs.emplace_back();
2325  if (UsedSubVector.test(2 * I)) {
2326  Pairs.back().emplace_back(I, 0);
2327  } else {
2328  assert(UsedSubVector.test(2 * I + 1) &&
2329  "Expected to be used one of the subvectors.");
2330  Pairs.back().emplace_back(I, 1);
2331  }
2332  }
2333  if (!Pairs.empty() && Pairs.front().size() > 1) {
2334  // Adjust mask.
2335  for (int &Idx : Mask) {
2336  if (Idx == UndefMaskElem)
2337  continue;
2338  unsigned SrcRegIdx = Idx / NewElts;
2339  auto *It = find_if(
2340  Pairs, [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
2341  return Idxs.front().first == SrcRegIdx ||
2342  Idxs.back().first == SrcRegIdx;
2343  });
2344  if (It == Pairs.end())
2345  continue;
2346  Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
2347  (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
2348  }
2349  // Adjust inputs.
2350  for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
2351  Inputs[Idxs.front().first] = DAG.getNode(
2353  Inputs[Idxs.front().first].getValueType(),
2354  Inputs[Idxs.front().first].getOperand(Idxs.front().second),
2355  Inputs[Idxs.back().first].getOperand(Idxs.back().second));
2356  }
2357  }
2358  }
2359  bool Changed;
2360  do {
2361  // Try to remove extra shuffles (except broadcasts) and shuffles with the
2362  // reused operands.
2363  Changed = false;
2364  for (unsigned I = 0; I < array_lengthof(Inputs); ++I) {
2365  auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Inputs[I].getNode());
2366  if (!Shuffle)
2367  continue;
2368  if (Shuffle->getOperand(0).getValueType() != NewVT)
2369  continue;
2370  int Op = -1;
2371  if (!Inputs[I].hasOneUse() && Shuffle->getOperand(1).isUndef() &&
2372  !Shuffle->isSplat()) {
2373  Op = 0;
2374  } else if (!Inputs[I].hasOneUse() &&
2375  !Shuffle->getOperand(1).isUndef()) {
2376  // Find the only used operand, if possible.
2377  for (int &Idx : Mask) {
2378  if (Idx == UndefMaskElem)
2379  continue;
2380  unsigned SrcRegIdx = Idx / NewElts;
2381  if (SrcRegIdx != I)
2382  continue;
2383  int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2384  if (MaskElt == UndefMaskElem) {
2385  Idx = UndefMaskElem;
2386  continue;
2387  }
2388  int OpIdx = MaskElt / NewElts;
2389  if (Op == -1) {
2390  Op = OpIdx;
2391  continue;
2392  }
2393  if (Op != OpIdx) {
2394  Op = -1;
2395  break;
2396  }
2397  }
2398  }
2399  if (Op < 0) {
2400  // Try to check if one of the shuffle operands is used already.
2401  for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
2402  if (Shuffle->getOperand(OpIdx).isUndef())
2403  continue;
2404  auto *It = find(Inputs, Shuffle->getOperand(OpIdx));
2405  if (It == std::end(Inputs))
2406  continue;
2407  int FoundOp = std::distance(std::begin(Inputs), It);
2408  // Found that operand is used already.
2409  // 1. Fix the mask for the reused operand.
2410  for (int &Idx : Mask) {
2411  if (Idx == UndefMaskElem)
2412  continue;
2413  unsigned SrcRegIdx = Idx / NewElts;
2414  if (SrcRegIdx != I)
2415  continue;
2416  int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2417  if (MaskElt == UndefMaskElem) {
2418  Idx = UndefMaskElem;
2419  continue;
2420  }
2421  int MaskIdx = MaskElt / NewElts;
2422  if (OpIdx == MaskIdx)
2423  Idx = MaskElt % NewElts + FoundOp * NewElts;
2424  }
2425  // 2. Set Op to the unused OpIdx.
2426  Op = (OpIdx + 1) % 2;
2427  break;
2428  }
2429  }
2430  if (Op >= 0) {
2431  Changed = true;
2432  Inputs[I] = Shuffle->getOperand(Op);
2433  // Adjust mask.
2434  for (int &Idx : Mask) {
2435  if (Idx == UndefMaskElem)
2436  continue;
2437  unsigned SrcRegIdx = Idx / NewElts;
2438  if (SrcRegIdx != I)
2439  continue;
2440  int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2441  int OpIdx = MaskElt / NewElts;
2442  if (OpIdx != Op)
2443  continue;
2444  Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
2445  }
2446  }
2447  }
2448  } while (Changed);
2449  };
2450  TryPeekThroughShufflesInputs(OrigMask);
2451  // Proces unique inputs.
2452  auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
2453  NewElts](SmallVectorImpl<int> &Mask) {
2454  SetVector<SDValue> UniqueInputs;
2455  SetVector<SDValue> UniqueConstantInputs;
2456  for (unsigned I = 0; I < array_lengthof(Inputs); ++I) {
2457  if (IsConstant(Inputs[I]))
2458  UniqueConstantInputs.insert(Inputs[I]);
2459  else if (!Inputs[I].isUndef())
2460  UniqueInputs.insert(Inputs[I]);
2461  }
2462  // Adjust mask in case of reused inputs. Also, need to insert constant
2463  // inputs at first, otherwise it affects the final outcome.
2464  if (UniqueInputs.size() != array_lengthof(Inputs)) {
2465  auto &&UniqueVec = UniqueInputs.takeVector();
2466  auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
2467  unsigned ConstNum = UniqueConstantVec.size();
2468  for (int &Idx : Mask) {
2469  if (Idx == UndefMaskElem)
2470  continue;
2471  unsigned SrcRegIdx = Idx / NewElts;
2472  if (Inputs[SrcRegIdx].isUndef()) {
2473  Idx = UndefMaskElem;
2474  continue;
2475  }
2476  const auto It = find(UniqueConstantVec, Inputs[SrcRegIdx]);
2477  if (It != UniqueConstantVec.end()) {
2478  Idx = (Idx % NewElts) +
2479  NewElts * std::distance(UniqueConstantVec.begin(), It);
2480  assert(Idx >= 0 && "Expected defined mask idx.");
2481  continue;
2482  }
2483  const auto RegIt = find(UniqueVec, Inputs[SrcRegIdx]);
2484  assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
2485  Idx = (Idx % NewElts) +
2486  NewElts * (std::distance(UniqueVec.begin(), RegIt) + ConstNum);
2487  assert(Idx >= 0 && "Expected defined mask idx.");
2488  }
2489  copy(UniqueConstantVec, std::begin(Inputs));
2490  copy(UniqueVec, std::next(std::begin(Inputs), ConstNum));
2491  }
2492  };
2493  MakeUniqueInputs(OrigMask);
2494  SDValue OrigInputs[4];
2495  copy(Inputs, std::begin(OrigInputs));
2496  for (unsigned High = 0; High < 2; ++High) {
2497  SDValue &Output = High ? Hi : Lo;
2498 
2499  // Build a shuffle mask for the output, discovering on the fly which
2500  // input vectors to use as shuffle operands.
2501  unsigned FirstMaskIdx = High * NewElts;
2502  SmallVector<int> Mask(NewElts * array_lengthof(Inputs), UndefMaskElem);
2503  copy(makeArrayRef(OrigMask).slice(FirstMaskIdx, NewElts), Mask.begin());
2504  assert(!Output && "Expected default initialized initial value.");
2505  TryPeekThroughShufflesInputs(Mask);
2506  MakeUniqueInputs(Mask);
2507  SDValue TmpInputs[4];
2508  copy(Inputs, std::begin(TmpInputs));
2509  // Track changes in the output registers.
2510  int UsedIdx = -1;
2511  bool SecondIteration = false;
2512  auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
2513  if (UsedIdx < 0) {
2514  UsedIdx = Idx;
2515  return false;
2516  }
2517  if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
2518  SecondIteration = true;
2519  return SecondIteration;
2520  };
2522  Mask, array_lengthof(Inputs), array_lengthof(Inputs),
2523  /*NumOfUsedRegs=*/1,
2524  [&Output, &DAG = DAG, NewVT]() { Output = DAG.getUNDEF(NewVT); },
2525  [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
2526  &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
2527  if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
2528  Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
2529  else
2530  Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx],
2531  DAG.getUNDEF(NewVT), Mask);
2532  Inputs[Idx] = Output;
2533  },
2534  [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
2535  &TmpInputs,
2536  &BuildVector](ArrayRef<int> Mask, unsigned Idx1, unsigned Idx2) {
2537  if (AccumulateResults(Idx1)) {
2538  if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
2539  Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
2540  Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
2541  else
2542  Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx1],
2543  Inputs[Idx2], Mask);
2544  } else {
2545  if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
2546  TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
2547  Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
2548  else
2549  Output = DAG.getVectorShuffle(NewVT, DL, TmpInputs[Idx1],
2550  TmpInputs[Idx2], Mask);
2551  }
2552  Inputs[Idx1] = Output;
2553  });
2554  copy(OrigInputs, std::begin(Inputs));
2555  }
2556 }
2557 
2558 void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2559  EVT OVT = N->getValueType(0);
2560  EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext());
2561  SDValue Chain = N->getOperand(0);
2562  SDValue Ptr = N->getOperand(1);
2563  SDValue SV = N->getOperand(2);
2564  SDLoc dl(N);
2565 
2566  const Align Alignment =
2567  DAG.getDataLayout().getABITypeAlign(NVT.getTypeForEVT(*DAG.getContext()));
2568 
2569  Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment.value());
2570  Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment.value());
2571  Chain = Hi.getValue(1);
2572 
2573  // Modified the chain - switch anything that used the old chain to use
2574  // the new one.
2575  ReplaceValueWith(SDValue(N, 1), Chain);
2576 }
2577 
2578 void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
2579  SDValue &Hi) {
2580  EVT DstVTLo, DstVTHi;
2581  std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(N->getValueType(0));
2582  SDLoc dl(N);
2583 
2584  SDValue SrcLo, SrcHi;
2585  EVT SrcVT = N->getOperand(0).getValueType();
2586  if (getTypeAction(SrcVT) == TargetLowering::TypeSplitVector)
2587  GetSplitVector(N->getOperand(0), SrcLo, SrcHi);
2588  else
2589  std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(N, 0);
2590 
2591  Lo = DAG.getNode(N->getOpcode(), dl, DstVTLo, SrcLo, N->getOperand(1));
2592  Hi = DAG.getNode(N->getOpcode(), dl, DstVTHi, SrcHi, N->getOperand(1));
2593 }
2594 
2595 void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
2596  SDValue &Hi) {
2597  SDValue InLo, InHi;
2598  GetSplitVector(N->getOperand(0), InLo, InHi);
2599  SDLoc DL(N);
2600 
2601  Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, InHi.getValueType(), InHi);
2602  Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, InLo.getValueType(), InLo);
2603 }
2604 
2605 void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
2606  SDValue &Hi) {
2607  EVT VT = N->getValueType(0);
2608  SDLoc DL(N);
2609 
2610  EVT LoVT, HiVT;
2611  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
2612 
2613  SDValue Expanded = TLI.expandVectorSplice(N, DAG);
2614  Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, Expanded,
2615  DAG.getVectorIdxConstant(0, DL));
2616  Hi =
2617  DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, Expanded,
2618  DAG.getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL));
2619 }
2620 
2621 //===----------------------------------------------------------------------===//
2622 // Operand Vector Splitting
2623 //===----------------------------------------------------------------------===//
2624 
2625 /// This method is called when the specified operand of the specified node is
2626 /// found to need vector splitting. At this point, all of the result types of
2627 /// the node are known to be legal, but other operands of the node may need
2628 /// legalization as well as the specified one.
2629 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
2630  LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
2631  SDValue Res = SDValue();
2632 
2633  // See if the target wants to custom split this node.
2634  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2635  return false;
2636 
2637  switch (N->getOpcode()) {
2638  default:
2639 #ifndef NDEBUG
2640  dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
2641  N->dump(&DAG);
2642  dbgs() << "\n";
2643 #endif
2644  report_fatal_error("Do not know how to split this operator's "
2645  "operand!\n");
2646 
2647  case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
2648  case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
2649  case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
2650  case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
2651  case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
2652  case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
2653  case ISD::TRUNCATE:
2654  Res = SplitVecOp_TruncateHelper(N);
2655  break;
2656  case ISD::STRICT_FP_ROUND:
2657  case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
2658  case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
2659  case ISD::STORE:
2660  Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
2661  break;
2662  case ISD::VP_STORE:
2663  Res = SplitVecOp_VP_STORE(cast<VPStoreSDNode>(N), OpNo);
2664  break;
2665  case ISD::MSTORE:
2666  Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
2667  break;
2668  case ISD::MSCATTER:
2669  case ISD::VP_SCATTER:
2670  Res = SplitVecOp_Scatter(cast<MemSDNode>(N), OpNo);
2671  break;
2672  case ISD::MGATHER:
2673  case ISD::VP_GATHER:
2674  Res = SplitVecOp_Gather(cast<MemSDNode>(N), OpNo);
2675  break;
2676  case ISD::VSELECT:
2677  Res = SplitVecOp_VSELECT(N, OpNo);
2678  break;
2681  case ISD::SINT_TO_FP:
2682  case ISD::UINT_TO_FP:
2683  if (N->getValueType(0).bitsLT(
2684  N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType()))
2685  Res = SplitVecOp_TruncateHelper(N);
2686  else
2687  Res = SplitVecOp_UnaryOp(N);
2688  break;
2689  case ISD::FP_TO_SINT_SAT:
2690  case ISD::FP_TO_UINT_SAT:
2691  Res = SplitVecOp_FP_TO_XINT_SAT(N);
2692  break;
2693  case ISD::FP_TO_SINT:
2694  case ISD::FP_TO_UINT:
2697  case ISD::STRICT_FP_EXTEND:
2698  case ISD::FP_EXTEND:
2699  case ISD::SIGN_EXTEND:
2700  case ISD::ZERO_EXTEND:
2701  case ISD::ANY_EXTEND:
2702  case ISD::FTRUNC:
2703  Res = SplitVecOp_UnaryOp(N);
2704  break;
2705 
2709  Res = SplitVecOp_ExtVecInRegOp(N);
2710  break;
2711 
2712  case ISD::VECREDUCE_FADD:
2713  case ISD::VECREDUCE_FMUL:
2714  case ISD::VECREDUCE_ADD:
2715  case ISD::VECREDUCE_MUL:
2716  case ISD::VECREDUCE_AND:
2717  case ISD::VECREDUCE_OR:
2718  case ISD::VECREDUCE_XOR:
2719  case ISD::VECREDUCE_SMAX:
2720  case ISD::VECREDUCE_SMIN:
2721  case ISD::VECREDUCE_UMAX:
2722  case ISD::VECREDUCE_UMIN:
2723  case ISD::VECREDUCE_FMAX:
2724  case ISD::VECREDUCE_FMIN:
2725  Res = SplitVecOp_VECREDUCE(N, OpNo);
2726  break;
2729  Res = SplitVecOp_VECREDUCE_SEQ(N);
2730  break;
2731  case ISD::VP_REDUCE_FADD:
2732  case ISD::VP_REDUCE_SEQ_FADD:
2733  case ISD::VP_REDUCE_FMUL:
2734  case ISD::VP_REDUCE_SEQ_FMUL:
2735  case ISD::VP_REDUCE_ADD:
2736  case ISD::VP_REDUCE_MUL:
2737  case ISD::VP_REDUCE_AND:
2738  case ISD::VP_REDUCE_OR:
2739  case ISD::VP_REDUCE_XOR:
2740  case ISD::VP_REDUCE_SMAX:
2741  case ISD::VP_REDUCE_SMIN:
2742  case ISD::VP_REDUCE_UMAX:
2743  case ISD::VP_REDUCE_UMIN:
2744  case ISD::VP_REDUCE_FMAX:
2745  case ISD::VP_REDUCE_FMIN:
2746  Res = SplitVecOp_VP_REDUCE(N, OpNo);
2747  break;
2748  }
2749 
2750  // If the result is null, the sub-method took care of registering results etc.
2751  if (!Res.getNode()) return false;
2752 
2753  // If the result is N, the sub-method updated N in place. Tell the legalizer
2754  // core about this.
2755  if (Res.getNode() == N)
2756  return true;
2757 
2758  if (N->isStrictFPOpcode())
2759  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
2760  "Invalid operand expansion");
2761  else
2762  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2763  "Invalid operand expansion");
2764 
2765  ReplaceValueWith(SDValue(N, 0), Res);
2766  return false;
2767 }
2768 
2769 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
2770  // The only possibility for an illegal operand is the mask, since result type
2771  // legalization would have handled this node already otherwise.
2772  assert(OpNo == 0 && "Illegal operand must be mask");
2773 
2774  SDValue Mask = N->getOperand(0);
2775  SDValue Src0 = N->getOperand(1);
2776  SDValue Src1 = N->getOperand(2);
2777  EVT Src0VT = Src0.getValueType();
2778  SDLoc DL(N);
2779  assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
2780 
2781  SDValue Lo, Hi;
2782  GetSplitVector(N->getOperand(0), Lo, Hi);
2783  assert(Lo.getValueType() == Hi.getValueType() &&
2784  "Lo and Hi have differing types");
2785 
2786  EVT LoOpVT, HiOpVT;
2787  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
2788  assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
2789 
2790  SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
2791  std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
2792  std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
2793  std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
2794 
2795  SDValue LoSelect =
2796  DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
2797  SDValue HiSelect =
2798  DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
2799 
2800  return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
2801 }
2802 
2803 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
2804  EVT ResVT = N->getValueType(0);
2805  SDValue Lo, Hi;
2806  SDLoc dl(N);
2807 
2808  SDValue VecOp = N->getOperand(OpNo);
2809  EVT VecVT = VecOp.getValueType();
2810  assert(VecVT.isVector() && "Can only split reduce vector operand");
2811  GetSplitVector(VecOp, Lo, Hi);
2812  EVT LoOpVT, HiOpVT;
2813  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2814 
2815  // Use the appropriate scalar instruction on the split subvectors before
2816  // reducing the now partially reduced smaller vector.
2817  unsigned CombineOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
2818  SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
2819  return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
2820 }
2821 
2822 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
2823  EVT ResVT = N->getValueType(0);
2824  SDValue Lo, Hi;
2825  SDLoc dl(N);
2826 
2827  SDValue AccOp = N->getOperand(0);
2828  SDValue VecOp = N->getOperand(1);
2829  SDNodeFlags Flags = N->getFlags();
2830 
2831  EVT VecVT = VecOp.getValueType();
2832  assert(VecVT.isVector() && "Can only split reduce vector operand");
2833  GetSplitVector(VecOp, Lo, Hi);
2834  EVT LoOpVT, HiOpVT;
2835  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2836 
2837  // Reduce low half.
2838  SDValue Partial = DAG.getNode(N->getOpcode(), dl, ResVT, AccOp, Lo, Flags);
2839 
2840  // Reduce high half, using low half result as initial value.
2841  return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, Hi, Flags);
2842 }
2843 
2844 SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
2845  assert(N->isVPOpcode() && "Expected VP opcode");
2846  assert(OpNo == 1 && "Can only split reduce vector operand");
2847 
2848  unsigned Opc = N->getOpcode();
2849  EVT ResVT = N->getValueType(0);
2850  SDValue Lo, Hi;
2851  SDLoc dl(N);
2852 
2853  SDValue VecOp = N->getOperand(OpNo);
2854  EVT VecVT = VecOp.getValueType();
2855  assert(VecVT.isVector() && "Can only split reduce vector operand");
2856  GetSplitVector(VecOp, Lo, Hi);
2857 
2858  SDValue MaskLo, MaskHi;
2859  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2));
2860 
2861  SDValue EVLLo, EVLHi;
2862  std::tie(EVLLo, EVLHi) = DAG.SplitEVL(N->getOperand(3), VecVT, dl);
2863 
2864  const SDNodeFlags Flags = N->getFlags();
2865 
2866  SDValue ResLo =
2867  DAG.getNode(Opc, dl, ResVT, {N->getOperand(0), Lo, MaskLo, EVLLo}, Flags);
2868  return DAG.getNode(Opc, dl, ResVT, {ResLo, Hi, MaskHi, EVLHi}, Flags);
2869 }
2870 
2871 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
2872  // The result has a legal vector type, but the input needs splitting.
2873  EVT ResVT = N->getValueType(0);
2874  SDValue Lo, Hi;
2875  SDLoc dl(N);
2876  GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
2877  EVT InVT = Lo.getValueType();
2878 
2879  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2880  InVT.getVectorElementCount());
2881 
2882  if (N->isStrictFPOpcode()) {
2883  Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2884  { N->getOperand(0), Lo });
2885  Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2886  { N->getOperand(0), Hi });
2887 
2888  // Build a factor node to remember that this operation is independent
2889  // of the other one.
2890  SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2891  Hi.getValue(1));
2892 
2893  // Legalize the chain result - switch anything that used the old chain to
2894  // use the new one.
2895  ReplaceValueWith(SDValue(N, 1), Ch);
2896  } else {
2897  Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
2898  Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
2899  }
2900 
2901  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
2902 }
2903 
2904 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
2905  // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
2906  // end up being split all the way down to individual components. Convert the
2907  // split pieces into integers and reassemble.
2908  SDValue Lo, Hi;
2909  GetSplitVector(N->getOperand(0), Lo, Hi);
2910  Lo = BitConvertToInteger(Lo);
2911  Hi = BitConvertToInteger(Hi);
2912 
2913  if (DAG.getDataLayout().isBigEndian())
2914  std::swap(Lo, Hi);
2915 
2916  return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
2917  JoinIntegers(Lo, Hi));
2918 }
2919 
2920 SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
2921  unsigned OpNo) {
2922  assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
2923  // We know that the result type is legal.
2924  EVT ResVT = N->getValueType(0);
2925 
2926  SDValue Vec = N->getOperand(0);
2927  SDValue SubVec = N->getOperand(1);
2928  SDValue Idx = N->getOperand(2);
2929  SDLoc dl(N);
2930 
2931  SDValue Lo, Hi;
2932  GetSplitVector(SubVec, Lo, Hi);
2933 
2934  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2935  uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2936 
2937  SDValue FirstInsertion =
2938  DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Lo, Idx);
2939  SDValue SecondInsertion =
2940  DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, FirstInsertion, Hi,
2941  DAG.getVectorIdxConstant(IdxVal + LoElts, dl));
2942 
2943  return SecondInsertion;
2944 }
2945 
2946 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
2947  // We know that the extracted result type is legal.
2948  EVT SubVT = N->getValueType(0);
2949 
2950  SDValue Idx = N->getOperand(1);
2951  SDLoc dl(N);
2952  SDValue Lo, Hi;
2953 
2954  if (SubVT.isScalableVector() !=
2955  N->getOperand(0).getValueType().isScalableVector())
2956  report_fatal_error("Extracting a fixed-length vector from an illegal "
2957  "scalable vector is not yet supported");
2958 
2959  GetSplitVector(N->getOperand(0), Lo, Hi);
2960 
2961  uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2962  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2963 
2964  if (IdxVal < LoElts) {
2965  assert(IdxVal + SubVT.getVectorMinNumElements() <= LoElts &&
2966  "Extracted subvector crosses vector split!");
2967  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
2968  } else {
2969  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
2970  DAG.getVectorIdxConstant(IdxVal - LoElts, dl));
2971  }
2972 }
2973 
2974 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
2975  SDValue Vec = N->getOperand(0);
2976  SDValue Idx = N->getOperand(1);
2977  EVT VecVT = Vec.getValueType();
2978 
2979  if (isa<ConstantSDNode>(Idx)) {
2980  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2981 
2982  SDValue Lo, Hi;
2983  GetSplitVector(Vec, Lo, Hi);
2984 
2985  uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2986 
2987  if (IdxVal < LoElts)
2988  return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
2989  else if (!Vec.getValueType().isScalableVector())
2990  return SDValue(DAG.UpdateNodeOperands(N, Hi,
2991  DAG.getConstant(IdxVal - LoElts, SDLoc(N),
2992  Idx.getValueType())), 0);
2993  }
2994 
2995  // See if the target wants to custom expand this node.
2996  if (CustomLowerNode(N, N->getValueType(0), true))
2997  return SDValue();
2998 
2999  // Make the vector elements byte-addressable if they aren't already.
3000  SDLoc dl(N);
3001  EVT EltVT = VecVT.getVectorElementType();
3002  if (VecVT.getScalarSizeInBits() < 8) {
3003  EltVT = MVT::i8;
3004  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
3005  VecVT.getVectorElementCount());
3006  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
3007  }
3008 
3009  // Store the vector to the stack.
3010  // In cases where the vector is illegal it will be broken down into parts
3011  // and stored in parts - we should use the alignment for the smallest part.
3012  Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
3013  SDValue StackPtr =
3014  DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
3015  auto &MF = DAG.getMachineFunction();
3016  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
3017  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
3018  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
3019  SmallestAlign);
3020 
3021  // Load back the required element.
3022  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
3023 
3024  // FIXME: This is to handle i1 vectors with elements promoted to i8.
3025  // i1 vector handling needs general improvement.
3026  if (N->getValueType(0).bitsLT(EltVT)) {
3027  SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
3028  MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
3029  return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
3030  }
3031 
3032  return DAG.getExtLoad(
3033  ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
3034  MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT,
3035  commonAlignment(SmallestAlign, EltVT.getFixedSizeInBits() / 8));
3036 }
3037 
3038 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
3039  SDValue Lo, Hi;
3040 
3041  // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
3042  // splitting the result has the same effect as splitting the input operand.
3043  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
3044 
3045  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
3046 }
3047 
3048 SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
3049  (void)OpNo;
3050  SDValue Lo, Hi;
3051  SplitVecRes_Gather(N, Lo, Hi);
3052 
3053  SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, N, N->getValueType(0), Lo, Hi);
3054  ReplaceValueWith(SDValue(N, 0), Res);
3055  return SDValue();
3056 }
3057 
3058 SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
3059  assert(N->isUnindexed() && "Indexed vp_store of vector?");
3060  SDValue Ch = N->getChain();
3061  SDValue Ptr = N->getBasePtr();
3062  SDValue Offset = N->getOffset();
3063  assert(Offset.isUndef() && "Unexpected VP store offset");
3064  SDValue Mask = N->getMask();
3065  SDValue EVL = N->getVectorLength();
3066  SDValue Data = N->getValue();
3067  Align Alignment = N->getOriginalAlign();
3068  SDLoc DL(N);
3069 
3070  SDValue DataLo, DataHi;
3071  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3072  // Split Data operand
3073  GetSplitVector(Data, DataLo, DataHi);
3074  else
3075  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
3076 
3077  // Split Mask operand
3078  SDValue MaskLo, MaskHi;
3079  if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
3080  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
3081  } else {
3082  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
3083  GetSplitVector(Mask, MaskLo, MaskHi);
3084  else
3085  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
3086  }
3087 
3088  EVT MemoryVT = N->getMemoryVT();
3089  EVT LoMemVT, HiMemVT;
3090  bool HiIsEmpty = false;
3091  std::tie(LoMemVT, HiMemVT) =
3092  DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
3093 
3094  // Split EVL
3095  SDValue EVLLo, EVLHi;
3096  std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, Data.getValueType(), DL);
3097 
3098  SDValue Lo, Hi;
3099  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3100  N->getPointerInfo(), MachineMemOperand::MOStore,
3101  MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3102 
3103  Lo = DAG.getStoreVP(Ch, DL, DataLo, Ptr, Offset, MaskLo, EVLLo, LoMemVT, MMO,
3104  N->getAddressingMode(), N->isTruncatingStore(),
3105  N->isCompressingStore());
3106 
3107  // If the hi vp_store has zero storage size, only the lo vp_store is needed.
3108  if (HiIsEmpty)
3109  return Lo;
3110 
3111  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
3112  N->isCompressingStore());
3113 
3114  MachinePointerInfo MPI;
3115  if (LoMemVT.isScalableVector()) {
3116  Alignment = commonAlignment(Alignment,
3117  LoMemVT.getSizeInBits().getKnownMinSize() / 8);
3118  MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
3119  } else
3120  MPI = N->getPointerInfo().getWithOffset(
3121  LoMemVT.getStoreSize().getFixedSize());
3122 
3123  MMO = DAG.getMachineFunction().getMachineMemOperand(
3125  N->getAAInfo(), N->getRanges());
3126 
3127  Hi = DAG.getStoreVP(Ch, DL, DataHi, Ptr, Offset, MaskHi, EVLHi, HiMemVT, MMO,
3128  N->getAddressingMode(), N->isTruncatingStore(),
3129  N->isCompressingStore());
3130 
3131  // Build a factor node to remember that this store is independent of the
3132  // other one.
3133  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3134 }
3135 
3136 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
3137  unsigned OpNo) {
3138  assert(N->isUnindexed() && "Indexed masked store of vector?");
3139  SDValue Ch = N->getChain();
3140  SDValue Ptr = N->getBasePtr();
3141  SDValue Offset = N->getOffset();
3142  assert(Offset.isUndef() && "Unexpected indexed masked store offset");
3143  SDValue Mask = N->getMask();
3144  SDValue Data = N->getValue();
3145  Align Alignment = N->getOriginalAlign();
3146  SDLoc DL(N);
3147 
3148  SDValue DataLo, DataHi;
3149  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3150  // Split Data operand
3151  GetSplitVector(Data, DataLo, DataHi);
3152  else
3153  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
3154 
3155  // Split Mask operand
3156  SDValue MaskLo, MaskHi;
3157  if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
3158  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
3159  } else {
3160  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
3161  GetSplitVector(Mask, MaskLo, MaskHi);
3162  else
3163  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
3164  }
3165 
3166  EVT MemoryVT = N->getMemoryVT();
3167  EVT LoMemVT, HiMemVT;
3168  bool HiIsEmpty = false;
3169  std::tie(LoMemVT, HiMemVT) =
3170  DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
3171 
3172  SDValue Lo, Hi, Res;
3173  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3174  N->getPointerInfo(), MachineMemOperand::MOStore,
3175  MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3176 
3177  Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
3178  N->getAddressingMode(), N->isTruncatingStore(),
3179  N->isCompressingStore());
3180 
3181  if (HiIsEmpty) {
3182  // The hi masked store has zero storage size.
3183  // Only the lo masked store is needed.
3184  Res = Lo;
3185  } else {
3186 
3187  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
3188  N->isCompressingStore());
3189 
3190  MachinePointerInfo MPI;
3191  if (LoMemVT.isScalableVector()) {
3193  Alignment, LoMemVT.getSizeInBits().getKnownMinSize() / 8);
3194  MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
3195  } else
3196  MPI = N->getPointerInfo().getWithOffset(
3197  LoMemVT.getStoreSize().getFixedSize());
3198 
3199  MMO = DAG.getMachineFunction().getMachineMemOperand(
3201  N->getAAInfo(), N->getRanges());
3202 
3203  Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
3204  N->getAddressingMode(), N->isTruncatingStore(),
3205  N->isCompressingStore());
3206 
3207  // Build a factor node to remember that this store is independent of the
3208  // other one.
3209  Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3210  }
3211 
3212  return Res;
3213 }
3214 
3215 SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
3216  SDValue Ch = N->getChain();
3217  SDValue Ptr = N->getBasePtr();
3218  EVT MemoryVT = N->getMemoryVT();
3219  Align Alignment = N->getOriginalAlign();
3220  SDLoc DL(N);
3221  struct Operands {
3222  SDValue Mask;
3223  SDValue Index;
3224  SDValue Scale;
3225  SDValue Data;
3226  } Ops = [&]() -> Operands {
3227  if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
3228  return {MSC->getMask(), MSC->getIndex(), MSC->getScale(),
3229  MSC->getValue()};
3230  }
3231  auto *VPSC = cast<VPScatterSDNode>(N);
3232  return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale(),
3233  VPSC->getValue()};
3234  }();
3235  // Split all operands
3236 
3237  EVT LoMemVT, HiMemVT;
3238  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
3239 
3240  SDValue DataLo, DataHi;
3241  if (getTypeAction(Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
3242  // Split Data operand
3243  GetSplitVector(Ops.Data, DataLo, DataHi);
3244  else
3245  std::tie(DataLo, DataHi) = DAG.SplitVector(Ops.Data, DL);
3246 
3247  // Split Mask operand
3248  SDValue MaskLo, MaskHi;
3249  if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
3250  SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
3251  } else {
3252  std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, DL);
3253  }
3254 
3255  SDValue IndexHi, IndexLo;
3256  if (getTypeAction(Ops.Index.getValueType()) ==
3258  GetSplitVector(Ops.Index, IndexLo, IndexHi);
3259  else
3260  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, DL);
3261 
3262  SDValue Lo;
3263  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3264  N->getPointerInfo(), MachineMemOperand::MOStore,
3265  MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3266 
3267  if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
3268  SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
3269  Lo =
3270  DAG.getMaskedScatter(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO,
3271  MSC->getIndexType(), MSC->isTruncatingStore());
3272 
3273  // The order of the Scatter operation after split is well defined. The "Hi"
3274  // part comes after the "Lo". So these two operations should be chained one
3275  // after another.
3276  SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
3277  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi,
3278  MMO, MSC->getIndexType(),
3279  MSC->isTruncatingStore());
3280  }
3281  auto *VPSC = cast<VPScatterSDNode>(N);
3282  SDValue EVLLo, EVLHi;
3283  std::tie(EVLLo, EVLHi) =
3284  DAG.SplitEVL(VPSC->getVectorLength(), Ops.Data.getValueType(), DL);
3285 
3286  SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
3287  Lo = DAG.getScatterVP(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO,
3288  VPSC->getIndexType());
3289 
3290  // The order of the Scatter operation after split is well defined. The "Hi"
3291  // part comes after the "Lo". So these two operations should be chained one
3292  // after another.
3293  SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
3294  return DAG.getScatterVP(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi, MMO,
3295  VPSC->getIndexType());
3296 }
3297 
3298 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
3299  assert(N->isUnindexed() && "Indexed store of vector?");
3300  assert(OpNo == 1 && "Can only split the stored value");
3301  SDLoc DL(N);
3302 
3303  bool isTruncating = N->isTruncatingStore();
3304  SDValue Ch = N->getChain();
3305  SDValue Ptr = N->getBasePtr();
3306  EVT MemoryVT = N->getMemoryVT();
3307  Align Alignment = N->getOriginalAlign();
3308  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3309  AAMDNodes AAInfo = N->getAAInfo();
3310  SDValue Lo, Hi;
3311  GetSplitVector(N->getOperand(1), Lo, Hi);
3312 
3313  EVT LoMemVT, HiMemVT;
3314  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
3315 
3316  // Scalarize if the split halves are not byte-sized.
3317  if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
3318  return TLI.scalarizeVectorStore(N, DAG);
3319 
3320  if (isTruncating)
3321  Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
3322  Alignment, MMOFlags, AAInfo);
3323  else
3324  Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3325  AAInfo);
3326 
3327  MachinePointerInfo MPI;
3328  IncrementPointer(N, LoMemVT, MPI, Ptr);
3329 
3330  if (isTruncating)
3331  Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, MPI,
3332  HiMemVT, Alignment, MMOFlags, AAInfo);
3333  else
3334  Hi = DAG.getStore(Ch, DL, Hi, Ptr, MPI, Alignment, MMOFlags, AAInfo);
3335 
3336  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3337 }
3338 
3339 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
3340  SDLoc DL(N);
3341 
3342  // The input operands all must have the same type, and we know the result
3343  // type is valid. Convert this to a buildvector which extracts all the
3344  // input elements.
3345  // TODO: If the input elements are power-two vectors, we could convert this to
3346  // a new CONCAT_VECTORS node with elements that are half-wide.
3348  EVT EltVT = N->getValueType(0).getVectorElementType();
3349  for (const SDValue &Op : N->op_values()) {
3350  for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
3351  i != e; ++i) {
3352  Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
3353  DAG.getVectorIdxConstant(i, DL)));
3354  }
3355  }
3356 
3357  return DAG.getBuildVector(N->getValueType(0), DL, Elts);
3358 }
3359 
3360 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
3361  // The result type is legal, but the input type is illegal. If splitting
3362  // ends up with the result type of each half still being legal, just
3363  // do that. If, however, that would result in an illegal result type,
3364  // we can try to get more clever with power-two vectors. Specifically,
3365  // split the input type, but also widen the result element size, then
3366  // concatenate the halves and truncate again. For example, consider a target
3367  // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
3368  // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
3369  // %inlo = v4i32 extract_subvector %in, 0
3370  // %inhi = v4i32 extract_subvector %in, 4
3371  // %lo16 = v4i16 trunc v4i32 %inlo
3372  // %hi16 = v4i16 trunc v4i32 %inhi
3373  // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
3374  // %res = v8i8 trunc v8i16 %in16
3375  //
3376  // Without this transform, the original truncate would end up being
3377  // scalarized, which is pretty much always a last resort.
3378  unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
3379  SDValue InVec = N->getOperand(OpNo);
3380  EVT InVT = InVec->getValueType(0);
3381  EVT OutVT = N->getValueType(0);
3382  ElementCount NumElements = OutVT.getVectorElementCount();
3383  bool IsFloat = OutVT.isFloatingPoint();
3384 
3385  unsigned InElementSize = InVT.getScalarSizeInBits();
3386  unsigned OutElementSize = OutVT.getScalarSizeInBits();
3387 
3388  // Determine the split output VT. If its legal we can just split dirctly.
3389  EVT LoOutVT, HiOutVT;
3390  std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
3391  assert(LoOutVT == HiOutVT && "Unequal split?");
3392 
3393  // If the input elements are only 1/2 the width of the result elements,
3394  // just use the normal splitting. Our trick only work if there's room
3395  // to split more than once.
3396  if (isTypeLegal(LoOutVT) ||
3397  InElementSize <= OutElementSize * 2)
3398  return SplitVecOp_UnaryOp(N);
3399  SDLoc DL(N);
3400 
3401  // Don't touch if this will be scalarized.
3402  EVT FinalVT = InVT;
3403  while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
3404  FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
3405 
3406  if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector)
3407  return SplitVecOp_UnaryOp(N);
3408 
3409  // Get the split input vector.
3410  SDValue InLoVec, InHiVec;
3411  GetSplitVector(InVec, InLoVec, InHiVec);
3412 
3413  // Truncate them to 1/2 the element size.
3414  //
3415  // This assumes the number of elements is a power of two; any vector that
3416  // isn't should be widened, not split.
3417  EVT HalfElementVT = IsFloat ?
3418  EVT::getFloatingPointVT(InElementSize/2) :
3419  EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
3420  EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
3421  NumElements.divideCoefficientBy(2));
3422 
3423  SDValue HalfLo;
3424  SDValue HalfHi;
3425  SDValue Chain;
3426  if (N->isStrictFPOpcode()) {
3427  HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
3428  {N->getOperand(0), InLoVec});
3429  HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
3430  {N->getOperand(0), InHiVec});
3431  // Legalize the chain result - switch anything that used the old chain to
3432  // use the new one.
3433  Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1),
3434  HalfHi.getValue(1));
3435  } else {
3436  HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
3437  HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
3438  }
3439 
3440  // Concatenate them to get the full intermediate truncation result.
3441  EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
3442  SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
3443  HalfHi);
3444  // Now finish up by truncating all the way down to the original result
3445  // type. This should normally be something that ends up being legal directly,
3446  // but in theory if a target has very wide vectors and an annoyingly
3447  // restricted set of legal types, this split can chain to build things up.
3448 
3449  if (N->isStrictFPOpcode()) {
3450  SDValue Res = DAG.getNode(
3451  ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other},
3452  {Chain, InterVec,
3453  DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))});
3454  // Relink the chain
3455  ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
3456  return Res;
3457  }
3458 
3459  return IsFloat
3460  ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
3461  DAG.getTargetConstant(
3462  0, DL, TLI.getPointerTy(DAG.getDataLayout())))
3463  : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
3464 }
3465 
3466 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
3467  assert(N->getValueType(0).isVector() &&
3468  N->getOperand(0).getValueType().isVector() &&
3469  "Operand types must be vectors");
3470  // The result has a legal vector type, but the input needs splitting.
3471  SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
3472  SDLoc DL(N);
3473  GetSplitVector(N->getOperand(0), Lo0, Hi0);
3474  GetSplitVector(N->getOperand(1), Lo1, Hi1);
3475  auto PartEltCnt = Lo0.getValueType().getVectorElementCount();
3476 
3477  LLVMContext &Context = *DAG.getContext();
3478  EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt);
3479  EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2);
3480 
3481  LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
3482  HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
3483  SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
3484 
3485  EVT OpVT = N->getOperand(0).getValueType();
3486  ISD::NodeType ExtendCode =
3488  return DAG.getNode(ExtendCode, DL, N->getValueType(0), Con);
3489 }
3490 
3491 
3492 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
3493  // The result has a legal vector type, but the input needs splitting.
3494  EVT ResVT = N->getValueType(0);
3495  SDValue Lo, Hi;
3496  SDLoc DL(N);
3497  GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3498  EVT InVT = Lo.getValueType();
3499 
3500  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
3501  InVT.getVectorElementCount());
3502 
3503  if (N->isStrictFPOpcode()) {
3504  Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
3505  { N->getOperand(0), Lo, N->getOperand(2) });
3506  Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
3507  { N->getOperand(0), Hi, N->getOperand(2) });
3508  // Legalize the chain result - switch anything that used the old chain to
3509  // use the new one.
3510  SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
3511  Lo.getValue(1), Hi.getValue(1));
3512  ReplaceValueWith(SDValue(N, 1), NewChain);
3513  } else {
3514  Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
3515  Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
3516  }
3517 
3518  return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
3519 }
3520 
3521 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
3522  // The result (and the first input) has a legal vector type, but the second
3523  // input needs splitting.
3524  return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
3525 }
3526 
3527 SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
3528  EVT ResVT = N->getValueType(0);
3529  SDValue Lo, Hi;
3530  SDLoc dl(N);
3531  GetSplitVector(N->getOperand(0), Lo, Hi);
3532  EVT InVT = Lo.getValueType();
3533 
3534  EVT NewResVT =
3535  EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
3536  InVT.getVectorElementCount());
3537 
3538  Lo = DAG.getNode(N->getOpcode(), dl, NewResVT, Lo, N->getOperand(1));
3539  Hi = DAG.getNode(N->getOpcode(), dl, NewResVT, Hi, N->getOperand(1));
3540 
3541  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
3542 }
3543 
3544 //===----------------------------------------------------------------------===//
3545 // Result Vector Widening
3546 //===----------------------------------------------------------------------===//
3547 
3548 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
3549  LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
3550  dbgs() << "\n");
3551 
3552  // See if the target wants to custom widen this node.
3553  if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
3554  return;
3555 
3556  SDValue Res = SDValue();
3557  switch (N->getOpcode()) {
3558  default:
3559 #ifndef NDEBUG
3560  dbgs() << "WidenVectorResult #" << ResNo << ": ";
3561  N->dump(&DAG);
3562  dbgs() << "\n";
3563 #endif
3564  llvm_unreachable("Do not know how to widen the result of this operator!");
3565 
3566  case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
3567  case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
3568  case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
3569  case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
3570  case ISD::INSERT_SUBVECTOR:
3571  Res = WidenVecRes_INSERT_SUBVECTOR(N);
3572  break;
3573  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
3574  case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
3575  case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
3576  case ISD::SPLAT_VECTOR:
3577  case ISD::SCALAR_TO_VECTOR:
3578  Res = WidenVecRes_ScalarOp(N);
3579  break;
3580  case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
3581  case ISD::VSELECT:
3582  case ISD::SELECT:
3583  case ISD::VP_SELECT:
3584  case ISD::VP_MERGE:
3585  Res = WidenVecRes_Select(N);
3586  break;
3587  case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
3588  case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
3589  case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
3590  case ISD::VECTOR_SHUFFLE:
3591  Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
3592  break;
3593  case ISD::VP_LOAD:
3594  Res = WidenVecRes_VP_LOAD(cast<VPLoadSDNode>(N));
3595  break;
3596  case ISD::MLOAD:
3597  Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
3598  break;
3599  case ISD::MGATHER:
3600  Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
3601  break;
3602  case ISD::VP_GATHER:
3603  Res = WidenVecRes_VP_GATHER(cast<VPGatherSDNode>(N));
3604  break;
3605 
3606  case ISD::ADD: case ISD::VP_ADD:
3607  case ISD::AND: case ISD::VP_AND:
3608  case ISD::MUL: case ISD::VP_MUL:
3609  case ISD::MULHS:
3610  case ISD::MULHU:
3611  case ISD::OR: case ISD::VP_OR:
3612  case ISD::SUB: case ISD::VP_SUB:
3613  case ISD::XOR: case ISD::VP_XOR:
3614  case ISD::SHL: case ISD::VP_SHL:
3615  case ISD::SRA: case ISD::VP_ASHR:
3616  case ISD::SRL: case ISD::VP_LSHR:
3617  case ISD::FMINNUM:
3618  case ISD::FMAXNUM:
3619  case ISD::FMINIMUM:
3620  case ISD::FMAXIMUM:
3621  case ISD::SMIN:
3622  case ISD::SMAX:
3623  case ISD::UMIN:
3624  case ISD::UMAX:
3625  case ISD::UADDSAT:
3626  case ISD::SADDSAT:
3627  case ISD::USUBSAT:
3628  case ISD::SSUBSAT:
3629  case ISD::SSHLSAT:
3630  case ISD::USHLSAT:
3631  case ISD::ROTL:
3632  case ISD::ROTR:
3633  case ISD::AVGFLOORS:
3634  case ISD::AVGFLOORU:
3635  case ISD::AVGCEILS:
3636  case ISD::AVGCEILU:
3637  // Vector-predicated binary op widening. Note that -- unlike the
3638  // unpredicated versions -- we don't have to worry about trapping on
3639  // operations like UDIV, FADD, etc., as we pass on the original vector
3640  // length parameter. This means the widened elements containing garbage
3641  // aren't active.
3642  case ISD::VP_SDIV:
3643  case ISD::VP_UDIV:
3644  case ISD::VP_SREM:
3645  case ISD::VP_UREM:
3646  case ISD::VP_FADD:
3647  case ISD::VP_FSUB:
3648  case ISD::VP_FMUL:
3649  case ISD::VP_FDIV:
3650  case ISD::VP_FREM:
3651  Res = WidenVecRes_Binary(N);
3652  break;
3653 
3654  case ISD::FADD:
3655  case ISD::FMUL:
3656  case ISD::FPOW:
3657  case ISD::FSUB:
3658  case ISD::FDIV:
3659  case ISD::FREM:
3660  case ISD::SDIV:
3661  case ISD::UDIV:
3662  case ISD::SREM:
3663  case ISD::UREM:
3664  Res = WidenVecRes_BinaryCanTrap(N);
3665  break;
3666 
3667  case ISD::SMULFIX:
3668  case ISD::SMULFIXSAT:
3669  case ISD::UMULFIX:
3670  case ISD::UMULFIXSAT:
3671  // These are binary operations, but with an extra operand that shouldn't
3672  // be widened (the scale).
3673  Res = WidenVecRes_BinaryWithExtraScalarOp(N);
3674  break;
3675 
3676 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
3677  case ISD::STRICT_##DAGN:
3678 #include "llvm/IR/ConstrainedOps.def"
3679  Res = WidenVecRes_StrictFP(N);
3680  break;
3681 
3682  case ISD::UADDO:
3683  case ISD::SADDO:
3684  case ISD::USUBO:
3685  case ISD::SSUBO:
3686  case ISD::UMULO:
3687  case ISD::SMULO:
3688  Res = WidenVecRes_OverflowOp(N, ResNo);
3689  break;
3690 
3691  case ISD::FCOPYSIGN:
3692  Res = WidenVecRes_FCOPYSIGN(N);
3693  break;
3694 
3695  case ISD::IS_FPCLASS:
3696  Res = WidenVecRes_IS_FPCLASS(N);
3697  break;
3698 
3699  case ISD::FPOWI:
3700  Res = WidenVecRes_POWI(N);
3701  break;
3702 
3706  Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
3707  break;
3708 
3709  case ISD::ANY_EXTEND:
3710  case ISD::FP_EXTEND:
3711  case ISD::FP_ROUND:
3712  case ISD::FP_TO_SINT:
3713  case ISD::FP_TO_UINT:
3714  case ISD::SIGN_EXTEND:
3715  case ISD::SINT_TO_FP:
3716  case ISD::TRUNCATE:
3717  case ISD::UINT_TO_FP:
3718  case ISD::ZERO_EXTEND:
3719  Res = WidenVecRes_Convert(N);
3720  break;
3721 
3722  case ISD::FP_TO_SINT_SAT:
3723  case ISD::FP_TO_UINT_SAT:
3724  Res = WidenVecRes_FP_TO_XINT_SAT(N);
3725  break;
3726 
3727  case ISD::FABS:
3728  case ISD::FCEIL:
3729  case ISD::FCOS:
3730  case ISD::FEXP:
3731  case ISD::FEXP2:
3732  case ISD::FFLOOR:
3733  case ISD::FLOG:
3734  case ISD::FLOG10:
3735  case ISD::FLOG2:
3736  case ISD::FNEARBYINT:
3737  case ISD::FRINT:
3738  case ISD::FROUND:
3739  case ISD::FROUNDEVEN:
3740  case ISD::FSIN:
3741  case ISD::FSQRT:
3742  case ISD::FTRUNC: {
3743  // We're going to widen this vector op to a legal type by padding with undef
3744  // elements. If the wide vector op is eventually going to be expanded to
3745  // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
3746  // libcalls on the undef elements.
3747  EVT VT = N->getValueType(0);
3748  EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3749  if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
3750  TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
3751  Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
3752  break;
3753  }
3754  }
3755  // If the target has custom/legal support for the scalar FP intrinsic ops
3756  // (they are probably not destined to become libcalls), then widen those like
3757  // any other unary ops.
3759 
3760  case ISD::ABS:
3761  case ISD::BITREVERSE:
3762  case ISD::BSWAP:
3763  case ISD::CTLZ:
3764  case ISD::CTLZ_ZERO_UNDEF:
3765  case ISD::CTPOP:
3766  case ISD::CTTZ:
3767  case ISD::CTTZ_ZERO_UNDEF:
3768  case ISD::FNEG: case ISD::VP_FNEG:
3769  case ISD::FREEZE:
3770  case ISD::ARITH_FENCE:
3771  case ISD::FCANONICALIZE:
3772  Res = WidenVecRes_Unary(N);
3773  break;
3774  case ISD::FMA: case ISD::VP_FMA:
3775  case ISD::FSHL:
3776  case ISD::FSHR:
3777  Res = WidenVecRes_Ternary(N);
3778  break;
3779  }
3780 
3781  // If Res is null, the sub-method took care of registering the result.
3782  if (Res.getNode())
3783  SetWidenedVector(SDValue(N, ResNo), Res);
3784 }
3785 
3786 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
3787  // Ternary op widening.
3788  SDLoc dl(N);
3789  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3790  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3791  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3792  SDValue InOp3 = GetWidenedVector(N->getOperand(2));
3793  if (N->getNumOperands() == 3)
3794  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
3795 
3796  assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
3797  assert(N->isVPOpcode() && "Expected VP opcode");
3798 
3799  SDValue Mask =
3800  GetWidenedMask(N->getOperand(3), WidenVT.getVectorElementCount());
3801  return DAG.getNode(N->getOpcode(), dl, WidenVT,
3802  {InOp1, InOp2, InOp3, Mask, N->getOperand(4)});
3803 }
3804 
3805 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
3806  // Binary op widening.
3807  SDLoc dl(N);
3808  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3809  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3810  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3811  if (N->getNumOperands() == 2)
3812  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2,
3813  N->getFlags());
3814 
3815  assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
3816  assert(N->isVPOpcode() && "Expected VP opcode");
3817 
3818  SDValue Mask =
3819  GetWidenedMask(N->getOperand(2), WidenVT.getVectorElementCount());
3820  return DAG.getNode(N->getOpcode(), dl, WidenVT,
3821  {InOp1, InOp2, Mask, N->getOperand(3)}, N->getFlags());
3822 }
3823 
3824 SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
3825  // Binary op widening, but with an extra operand that shouldn't be widened.
3826  SDLoc dl(N);
3827  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3828  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3829  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3830  SDValue InOp3 = N->getOperand(2);
3831  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3,
3832  N->getFlags());
3833 }
3834 
3835 // Given a vector of operations that have been broken up to widen, see
3836 // if we can collect them together into the next widest legal VT. This
3837 // implementation is trap-safe.
3839  SmallVectorImpl<SDValue> &ConcatOps,
3840  unsigned ConcatEnd, EVT VT, EVT MaxVT,
3841  EVT WidenVT) {
3842  // Check to see if we have a single operation with the widen type.
3843  if (ConcatEnd == 1) {
3844  VT = ConcatOps[0].getValueType();
3845  if (VT == WidenVT)
3846  return ConcatOps[0];
3847  }
3848 
3849  SDLoc dl(ConcatOps[0]);
3850  EVT WidenEltVT = WidenVT.getVectorElementType();
3851 
3852  // while (Some element of ConcatOps is not of type MaxVT) {
3853  // From the end of ConcatOps, collect elements of the same type and put
3854  // them into an op of the next larger supported type
3855  // }
3856  while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
3857  int Idx = ConcatEnd - 1;
3858  VT = ConcatOps[Idx--].getValueType();
3859  while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
3860  Idx--;
3861 
3862  int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
3863  EVT NextVT;
3864  do {
3865  NextSize *= 2;
3866  NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
3867  } while (!TLI.isTypeLegal(NextVT));
3868 
3869  if (!VT.isVector()) {
3870  // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
3871  SDValue VecOp = DAG.getUNDEF(NextVT);
3872  unsigned NumToInsert = ConcatEnd - Idx - 1;
3873  for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
3874  VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp,
3875  ConcatOps[OpIdx], DAG.getVectorIdxConstant(i, dl));
3876  }
3877  ConcatOps[Idx+1] = VecOp;
3878  ConcatEnd = Idx + 2;
3879  } else {
3880  // Vector type, create a CONCAT_VECTORS of type NextVT
3881  SDValue undefVec = DAG.getUNDEF(VT);
3882  unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
3883  SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
3884  unsigned RealVals = ConcatEnd - Idx - 1;
3885  unsigned SubConcatEnd = 0;
3886  unsigned SubConcatIdx = Idx + 1;
3887  while (SubConcatEnd < RealVals)
3888  SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
3889  while (SubConcatEnd < OpsToConcat)
3890  SubConcatOps[SubConcatEnd++] = undefVec;
3891  ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
3892  NextVT, SubConcatOps);
3893  ConcatEnd = SubConcatIdx + 1;
3894  }
3895  }
3896 
3897  // Check to see if we have a single operation with the widen type.
3898  if (ConcatEnd == 1) {
3899  VT = ConcatOps[0].getValueType();
3900  if (VT == WidenVT)
3901  return ConcatOps[0];
3902  }
3903 
3904  // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
3905  unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
3906  if (NumOps != ConcatEnd ) {
3907  SDValue UndefVal = DAG.getUNDEF(MaxVT);
3908  for (unsigned j = ConcatEnd; j < NumOps; ++j)
3909  ConcatOps[j] = UndefVal;
3910  }
3911  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
3912  makeArrayRef(ConcatOps.data(), NumOps));
3913 }
3914 
3915 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
3916  // Binary op widening for operations that can trap.
3917  unsigned Opcode = N->getOpcode();
3918  SDLoc dl(N);
3919  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3920  EVT WidenEltVT = WidenVT.getVectorElementType();
3921  EVT VT = WidenVT;
3922  unsigned NumElts = VT.getVectorNumElements();
3923  const SDNodeFlags Flags = N->getFlags();
3924  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
3925  NumElts = NumElts / 2;
3926  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3927  }
3928 
3929  if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
3930  // Operation doesn't trap so just widen as normal.
3931  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3932  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3933  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
3934  }
3935 
3936  // No legal vector version so unroll the vector operation and then widen.
3937  if (NumElts == 1)
3938  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
3939 
3940  // Since the operation can trap, apply operation on the original vector.
3941  EVT MaxVT = VT;
3942  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3943  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3944  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
3945 
3946  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3947  unsigned ConcatEnd = 0; // Current ConcatOps index.
3948  int Idx = 0; // Current Idx into input vectors.
3949 
3950  // NumElts := greatest legal vector size (at most WidenVT)
3951  // while (orig. vector has unhandled elements) {
3952  // take munches of size NumElts from the beginning and add to ConcatOps
3953  // NumElts := next smaller supported vector size or 1
3954  // }
3955  while (CurNumElts != 0) {
3956  while (CurNumElts >= NumElts) {
3957  SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
3958  DAG.getVectorIdxConstant(Idx, dl));
3959  SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
3960  DAG.getVectorIdxConstant(Idx, dl));
3961  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
3962  Idx += NumElts;
3963  CurNumElts -= NumElts;
3964  }
3965  do {
3966  NumElts = NumElts / 2;
3967  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3968  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
3969 
3970  if (NumElts == 1) {
3971  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
3972  SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
3973  InOp1, DAG.getVectorIdxConstant(Idx, dl));
3974  SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
3975  InOp2, DAG.getVectorIdxConstant(Idx, dl));
3976  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
3977  EOp1, EOp2, Flags);
3978  }
3979  CurNumElts = 0;
3980  }
3981  }
3982 
3983  return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3984 }
3985 
3986 SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
3987  switch (N->getOpcode()) {
3988  case ISD::STRICT_FSETCC:
3989  case ISD::STRICT_FSETCCS:
3990  return WidenVecRes_STRICT_FSETCC(N);
3991  case ISD::STRICT_FP_EXTEND:
3992  case ISD::STRICT_FP_ROUND:
3997  return WidenVecRes_Convert_StrictFP(N);
3998  default:
3999  break;
4000  }
4001 
4002  // StrictFP op widening for operations that can trap.
4003  unsigned NumOpers = N->getNumOperands();
4004  unsigned Opcode = N->getOpcode();
4005  SDLoc dl(N);
4006  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4007  EVT WidenEltVT = WidenVT.getVectorElementType();
4008  EVT VT = WidenVT;
4009  unsigned NumElts = VT.getVectorNumElements();
4010  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
4011  NumElts = NumElts / 2;
4012  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
4013  }
4014 
4015  // No legal vector version so unroll the vector operation and then widen.
4016  if (NumElts == 1)
4017  return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements());
4018 
4019  // Since the operation can trap, apply operation on the original vector.
4020  EVT MaxVT = VT;
4022  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
4023 
4024  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
4025  SmallVector<SDValue, 16> Chains;
4026  unsigned ConcatEnd = 0; // Current ConcatOps index.
4027  int Idx = 0; // Current Idx into input vectors.
4028 
4029  // The Chain is the first operand.
4030  InOps.push_back(N->getOperand(0));
4031 
4032  // Now process the remaining operands.
4033  for (unsigned i = 1; i < NumOpers; ++i) {
4034  SDValue Oper = N->getOperand(i);
4035 
4036  if (Oper.getValueType().isVector()) {
4037  assert(Oper.getValueType() == N->getValueType(0) &&
4038  "Invalid operand type to widen!");
4039  Oper = GetWidenedVector(Oper);
4040  }
4041 
4042  InOps.push_back(Oper);
4043  }
4044 
4045  // NumElts := greatest legal vector size (at most WidenVT)
4046  // while (orig. vector has unhandled elements) {
4047  // take munches of size NumElts from the beginning and add to ConcatOps
4048  // NumElts := next smaller supported vector size or 1
4049  // }
4050  while (CurNumElts != 0) {
4051  while (CurNumElts >= NumElts) {
4053 
4054  for (unsigned i = 0; i < NumOpers; ++i) {
4055  SDValue Op = InOps[i];
4056 
4057  if (Op.getValueType().isVector())
4058  Op = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Op,
4059  DAG.getVectorIdxConstant(Idx, dl));
4060 
4061  EOps.push_back(Op);
4062  }
4063 
4064  EVT OperVT[] = {VT, MVT::Other};
4065  SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
4066  ConcatOps[ConcatEnd++] = Oper;
4067  Chains.push_back(Oper.getValue(1));
4068  Idx += NumElts;
4069  CurNumElts -= NumElts;
4070  }
4071  do {
4072  NumElts = NumElts / 2;
4073  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
4074  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
4075 
4076  if (NumElts == 1) {
4077  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
4079 
4080  for (unsigned i = 0; i < NumOpers; ++i) {
4081  SDValue Op = InOps[i];
4082 
4083  if (Op.getValueType().isVector())
4084  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op,
4085  DAG.getVectorIdxConstant(Idx, dl));
4086 
4087  EOps.push_back(Op);
4088  }
4089 
4090  EVT WidenVT[] = {WidenEltVT, MVT::Other};
4091  SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
4092  ConcatOps[ConcatEnd++] = Oper;
4093  Chains.push_back(Oper.getValue(1));
4094  }
4095  CurNumElts = 0;
4096  }
4097  }
4098 
4099  // Build a factor node to remember all the Ops that have been created.
4100  SDValue NewChain;
4101  if (Chains.size() == 1)
4102  NewChain = Chains[0];
4103  else
4104  NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
4105  ReplaceValueWith(SDValue(N, 1), NewChain);
4106 
4107  return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
4108 }
4109 
4110 SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
4111  SDLoc DL(N);
4112  EVT ResVT = N->getValueType(0);
4113  EVT OvVT = N->getValueType(1);
4114  EVT WideResVT, WideOvVT;
4115  SDValue WideLHS, WideRHS;
4116 
4117  // TODO: This might result in a widen/split loop.
4118  if (ResNo == 0) {
4119  WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
4120  WideOvVT = EVT::getVectorVT(
4121  *DAG.getContext(), OvVT.getVectorElementType(),
4122  WideResVT.getVectorNumElements());
4123 
4124  WideLHS = GetWidenedVector(N->getOperand(0));
4125  WideRHS = GetWidenedVector(N->getOperand(1));
4126  } else {
4127  WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT);
4128  WideResVT = EVT::getVectorVT(
4129  *DAG.getContext(), ResVT.getVectorElementType(),
4130  WideOvVT.getVectorNumElements());
4131 
4132  SDValue Zero = DAG.getVectorIdxConstant(0, DL);
4133  WideLHS = DAG.getNode(
4134  ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
4135  N->getOperand(0), Zero);
4136  WideRHS = DAG.getNode(
4137  ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
4138  N->getOperand(1), Zero);
4139  }
4140 
4141  SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT);
4142  SDNode *WideNode = DAG.getNode(
4143  N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode();
4144 
4145  // Replace the other vector result not being explicitly widened here.
4146  unsigned OtherNo = 1 - ResNo;
4147  EVT OtherVT = N->getValueType(OtherNo);
4148  if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) {
4149  SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo));
4150  } else {
4151  SDValue Zero = DAG.getVectorIdxConstant(0, DL);
4152  SDValue OtherVal = DAG.getNode(
4153  ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero);
4154  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
4155  }
4156 
4157  return SDValue(WideNode, ResNo);
4158 }
4159 
4160 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
4161  LLVMContext &Ctx = *DAG.getContext();
4162  SDValue InOp = N->getOperand(0);
4163  SDLoc DL(N);
4164 
4165  EVT WidenVT = TLI.getTypeToTransformTo(Ctx, N->getValueType(0));
4166  ElementCount WidenEC = WidenVT.getVectorElementCount();
4167 
4168  EVT InVT = InOp.getValueType();
4169 
4170  unsigned Opcode = N->getOpcode();
4171  const SDNodeFlags Flags = N->getFlags();
4172 
4173  // Handle the case of ZERO_EXTEND where the promoted InVT element size does
4174  // not equal that of WidenVT.
4175  if (N->getOpcode() == ISD::ZERO_EXTEND &&
4176  getTypeAction(InVT) == TargetLowering::TypePromoteInteger &&
4177  TLI.getTypeToTransformTo(Ctx, InVT).getScalarSizeInBits() !=
4178  WidenVT.getScalarSizeInBits()) {
4179  InOp = ZExtPromotedInteger(InOp);
4180  InVT = InOp.getValueType();
4181  if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
4182  Opcode = ISD::TRUNCATE;
4183  }
4184 
4185  EVT InEltVT = InVT.getVectorElementType();
4186  EVT InWidenVT = EVT::getVectorVT(Ctx, InEltVT, WidenEC);
4187  ElementCount InVTEC = InVT.getVectorElementCount();
4188 
4189  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
4190  InOp = GetWidenedVector(N->getOperand(0));
4191  InVT = InOp.getValueType();
4192  InVTEC = InVT.getVectorElementCount();
4193  if (InVTEC == WidenEC) {
4194  if (N->getNumOperands() == 1)
4195  return DAG.getNode(Opcode, DL, WidenVT, InOp);
4196  return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
4197  }
4198  if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
4199  // If both input and result vector types are of same width, extend
4200  // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
4201  // accepts fewer elements in the result than in the input.
4202  if (Opcode == ISD::ANY_EXTEND)
4203  return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
4204  if (Opcode == ISD::SIGN_EXTEND)
4205  return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
4206  if (Opcode == ISD::ZERO_EXTEND)
4207  return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
4208  }
4209  }
4210 
4211  if (TLI.isTypeLegal(InWidenVT)) {
4212  // Because the result and the input are different vector types, widening
4213  // the result could create a legal type but widening the input might make
4214  // it an illegal type that might lead to repeatedly splitting the input
4215  // and then widening it. To avoid this, we widen the input only if
4216  // it results in a legal type.
4217  if (WidenEC.isKnownMultipleOf(InVTEC.getKnownMinValue())) {
4218  // Widen the input and call convert on the widened input vector.
4219  unsigned NumConcat =
4220  WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
4221  SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
4222  Ops[0] = InOp;
4223  SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
4224  if (N->getNumOperands() == 1)
4225  return DAG.getNode(Opcode, DL, WidenVT, InVec);
4226  return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
4227  }
4228 
4229  if (InVTEC.isKnownMultipleOf(WidenEC.getKnownMinValue())) {
4230  SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
4231  DAG.getVectorIdxConstant(0, DL));
4232  // Extract the input and convert the shorten input vector.
4233  if (N->getNumOperands() == 1)
4234  return DAG.getNode(Opcode, DL, WidenVT, InVal);
4235  return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
4236  }
4237  }
4238 
4239  // Otherwise unroll into some nasty scalar code and rebuild the vector.
4240  EVT EltVT = WidenVT.getVectorElementType();
4241  SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getUNDEF(EltVT));
4242  // Use the original element count so we don't do more scalar opts than
4243  // necessary.
4244  unsigned MinElts = N->getValueType(0).getVectorNumElements();
4245  for (unsigned i=0; i < MinElts; ++i) {
4246  SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
4247  DAG.getVectorIdxConstant(i, DL));
4248  if (N->getNumOperands() == 1)
4249  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
4250  else
4251  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
4252  }
4253 
4254  return DAG.getBuildVector(WidenVT, DL, Ops);
4255 }
4256 
4257 SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
4258  SDLoc dl(N);
4259  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4260  ElementCount WidenNumElts = WidenVT.getVectorElementCount();
4261 
4262  SDValue Src = N->getOperand(0);
4263  EVT SrcVT = Src.getValueType();
4264 
4265  // Also widen the input.
4266  if (getTypeAction(SrcVT) == TargetLowering::TypeWidenVector) {
4267  Src = GetWidenedVector(Src);
4268  SrcVT = Src.getValueType();
4269  }
4270 
4271  // Input and output not widened to the same size, give up.
4272  if (WidenNumElts != SrcVT.getVectorElementCount())
4273  return DAG.UnrollVectorOp(N, WidenNumElts.getKnownMinValue());
4274 
4275  return DAG.getNode(N->getOpcode(), dl, WidenVT, Src, N->getOperand(1));
4276 }
4277 
4278 SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
4279  SDValue InOp = N->getOperand(1);
4280  SDLoc DL(N);
4281  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
4282 
4283  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4284  unsigned WidenNumElts = WidenVT.getVectorNumElements();
4285 
4286  EVT InVT = InOp.getValueType();
4287  EVT InEltVT = InVT.getVectorElementType();
4288 
4289  unsigned Opcode = N->getOpcode();
4290 
4291  // FIXME: Optimizations need to be implemented here.
4292 
4293  // Otherwise unroll into some nasty scalar code and rebuild the vector.
4294  EVT EltVT = WidenVT.getVectorElementType();
4295  std::array<EVT, 2> EltVTs = {{EltVT, MVT::Other}};
4296  SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
4297  SmallVector<SDValue, 32> OpChains;
4298  // Use the original element count so we don't do more scalar opts than
4299  // necessary.
4300  unsigned MinElts = N->getValueType(0).getVectorNumElements();
4301  for (unsigned i=0; i < MinElts; ++i) {
4302  NewOps[1] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
4303  DAG.getVectorIdxConstant(i, DL));
4304  Ops[i] = DAG.getNode(Opcode, DL, EltVTs, NewOps);
4305  OpChains.push_back(Ops[i].getValue(1));
4306  }
4307  SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains);
4308  ReplaceValueWith(SDValue(N, 1), NewChain);
4309 
4310  return DAG.getBuildVector(WidenVT, DL, Ops);
4311 }
4312 
4313 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
4314  unsigned Opcode = N->getOpcode();
4315  SDValue InOp = N->getOperand(0);
4316  SDLoc DL(N);
4317 
4318  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4319  EVT WidenSVT = WidenVT.getVectorElementType();
4320  unsigned WidenNumElts = WidenVT.getVectorNumElements();
4321 
4322  EVT InVT = InOp.getValueType();
4323  EVT InSVT = InVT.getVectorElementType();
4324  unsigned InVTNumElts = InVT.getVectorNumElements();
4325 
4326  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
4327  InOp = GetWidenedVector(InOp);
4328  InVT = InOp.getValueType();
4329  if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
4330  switch (Opcode) {
4334  return DAG.getNode(Opcode, DL, WidenVT, InOp);
4335  }
4336  }
4337  }
4338 
4339  // Unroll, extend the scalars and rebuild the vector.
4341  for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
4342  SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
4343  DAG.getVectorIdxConstant(i, DL));
4344  switch (Opcode) {
4346  Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
4347  break;
4349  Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
4350  break;
4352  Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
4353  break;
4354  default:
4355  llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
4356  }
4357  Ops.push_back(Val);
4358  }
4359 
4360  while (Ops.size() != WidenNumElts)
4361  Ops.push_back(DAG.getUNDEF(WidenSVT));
4362 
4363  return DAG.getBuildVector(WidenVT, DL, Ops);
4364 }
4365 
4366 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
4367  // If this is an FCOPYSIGN with same input types, we can treat it as a
4368  // normal (can trap) binary op.
4369  if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
4370  return WidenVecRes_BinaryCanTrap(N);
4371 
4372  // If the types are different, fall back to unrolling.
4373  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4374  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
4375 }
4376 
4377 SDValue DAGTypeLegalizer::WidenVecRes_IS_FPCLASS(SDNode *N) {
4378  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4379  SDValue Arg = GetWidenedVector(N->getOperand(0));
4380  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, {Arg, N->getOperand(1)},
4381  N->getFlags());
4382 }
4383 
4384 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
4385  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4386  SDValue InOp = GetWidenedVector(N->getOperand(0));
4387  SDValue ShOp = N->getOperand(1);
4388  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
4389 }
4390 
4391 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
4392  // Unary op widening.
4393  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4394  SDValue InOp = GetWidenedVector(N->getOperand(0));
4395  if (N->getNumOperands() == 1)
4396  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
4397 
4398  assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
4399  assert(N->isVPOpcode() && "Expected VP opcode");
4400 
4401  SDValue Mask =
4402  GetWidenedMask(N->getOperand(1), WidenVT.getVectorElementCount());
4403  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT,
4404  {InOp, Mask, N->getOperand(2)});
4405 }
4406 
4407 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
4408  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4409  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
4410  cast<VTSDNode>(N->getOperand(1))->getVT()
4411  .getVectorElementType(),
4412  WidenVT.getVectorNumElements());
4413  SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
4414  return DAG.getNode(N->getOpcode(), SDLoc(N),
4415  WidenVT, WidenLHS, DAG.getValueType(ExtVT));
4416 }
4417 
4418 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
4419  SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
4420  return GetWidenedVector(WidenVec);
4421 }
4422 
4423 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
4424  SDValue InOp = N->getOperand(0);
4425  EVT InVT = InOp.getValueType();
4426  EVT VT = N->getValueType(0);
4427  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4428  SDLoc dl(N);
4429 
4430  switch (getTypeAction(InVT)) {
4432  break;
4434  report_fatal_error("Scalarization of scalable vectors is not supported.");
4436  // If the incoming type is a vector that is being promoted, then
4437  // we know that the elements are arranged differently and that we
4438  // must perform the conversion using a stack slot.
4439  if (InVT.isVector())
4440  break;
4441 
4442  // If the InOp is promoted to the same size, convert it. Otherwise,
4443  // fall out of the switch and widen the promoted input.
4444  SDValue NInOp = GetPromotedInteger(InOp);
4445  EVT NInVT = NInOp.getValueType();
4446  if (WidenVT.bitsEq(NInVT)) {
4447  // For big endian targets we need to shift the input integer or the
4448  // interesting bits will end up at the wrong place.
4449  if (DAG.getDataLayout().isBigEndian()) {
4450  unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
4451  EVT ShiftAmtTy = TLI.getShiftAmountTy(NInVT, DAG.getDataLayout());
4452  assert(ShiftAmt < WidenVT.getSizeInBits() && "Too large shift amount!");
4453  NInOp = DAG.getNode(ISD::SHL, dl, NInVT, NInOp,
4454  DAG.getConstant(ShiftAmt, dl, ShiftAmtTy));
4455  }
4456  return DAG.getNode(ISD::BITCAST, dl, WidenVT, NInOp);
4457  }
4458  InOp = NInOp;
4459  InVT = NInVT;
4460  break;
4461  }
4469  break;
4471  // If the InOp is widened to the same size, convert it. Otherwise, fall
4472  // out of the switch and widen the widened input.
4473  InOp = GetWidenedVector(InOp);
4474  InVT = InOp.getValueType();
4475  if (WidenVT.bitsEq(InVT))
4476  // The input widens to the same size. Convert to the widen value.
4477  return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
4478  break;
4479  }
4480 
4481  unsigned WidenSize = WidenVT.getSizeInBits();
4482  unsigned InSize = InVT.getSizeInBits();
4483  // x86mmx is not an acceptable vector element type, so don't try.
4484  if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
4485  // Determine new input vector type. The new input vector type will use
4486  // the same element type (if its a vector) or use the input type as a
4487  // vector. It is the same size as the type to widen to.
4488  EVT NewInVT;
4489  unsigned NewNumElts = WidenSize / InSize;
4490  if (InVT.isVector()) {
4491  EVT InEltVT = InVT.getVectorElementType();
4492  NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
4493  WidenSize / InEltVT.getSizeInBits());
4494  } else {
4495  NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
4496  }
4497 
4498  if (TLI.isTypeLegal(NewInVT)) {
4499  SDValue NewVec;
4500  if (InVT.isVector()) {
4501  // Because the result and the input are different vector types, widening
4502  // the result could create a legal type but widening the input might make
4503  // it an illegal type that might lead to repeatedly splitting the input
4504  // and then widening it. To avoid this, we widen the input only if
4505  // it results in a legal type.
4506  SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT));
4507  Ops[0] = InOp;
4508 
4509  NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
4510  } else {
4511  NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp);
4512  }
4513  return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
4514  }
4515  }
4516 
4517  return CreateStackStoreLoad(InOp, WidenVT);
4518 }
4519 
4520 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
4521  SDLoc dl(N);
4522  // Build a vector with undefined for the new nodes.
4523  EVT VT = N->getValueType(0);
4524 
4525  // Integer BUILD_VECTOR operands may be larger than the node's vector element
4526  // type. The UNDEFs need to have the same type as the existing operands.
4527  EVT EltVT = N->getOperand(0).getValueType();
4528  unsigned NumElts = VT.getVectorNumElements();
4529 
4530  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4531  unsigned WidenNumElts = WidenVT.getVectorNumElements();
4532 
4533  SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
4534  assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
4535  NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
4536 
4537  return DAG.getBuildVector(WidenVT, dl, NewOps);
4538 }
4539 
4540 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
4541  EVT InVT = N->getOperand(0).getValueType();
4542  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4543  SDLoc dl(N);
4544  unsigned NumOperands = N->getNumOperands();
4545 
4546  bool InputWidened = false; // Indicates we need to widen the input.
4547  if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
4548  unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
4549  unsigned NumInElts = InVT.getVectorMinNumElements();
4550  if (WidenNumElts % NumInElts == 0) {
4551  // Add undef vectors to widen to correct length.
4552  unsigned NumConcat = WidenNumElts / NumInElts;
4553  SDValue UndefVal = DAG.getUNDEF(InVT);
4554  SmallVector<SDValue, 16> Ops(NumConcat);
4555  for (unsigned i=0; i < NumOperands; ++i)
4556  Ops[i] = N->getOperand(i);
4557  for (unsigned i = NumOperands; i != NumConcat; ++i)
4558  Ops[i] = UndefVal;
4559  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
4560  }
4561  } else {
4562  InputWidened = true;
4563  if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
4564  // The inputs and the result are widen to the same value.
4565  unsigned i;
4566  for (i=1; i < NumOperands; ++i)
4567  if (!N->getOperand(i).isUndef())
4568  break;
4569 
4570  if (i == NumOperands)
4571  // Everything but the first operand is an UNDEF so just return the
4572  // widened first operand.
4573  return GetWidenedVector(N->getOperand(0));
4574 
4575  if (NumOperands == 2) {
4576  assert(!WidenVT.isScalableVector() &&
4577  "Cannot use vector shuffles to widen CONCAT_VECTOR result");
4578  unsigned WidenNumElts = WidenVT.getVectorNumElements();
4579  unsigned NumInElts = InVT.getVectorNumElements();
4580 
4581  // Replace concat of two operands with a shuffle.
4582  SmallVector<int, 16> MaskOps(WidenNumElts, -1);
4583  for (unsigned i = 0; i < NumInElts; ++i) {
4584  MaskOps[i] = i;
4585  MaskOps[i + NumInElts] = i + WidenNumElts;
4586  }
4587  return DAG.getVectorShuffle(WidenVT, dl,
4588  GetWidenedVector(N->getOperand(0)),
4589  GetWidenedVector(N->getOperand(1)),
4590  MaskOps);
4591  }
4592  }
4593  }
4594 
4595  assert(!WidenVT.isScalableVector() &&
4596  "Cannot use build vectors to widen CONCAT_VECTOR result");
4597  unsigned WidenNumElts = WidenVT.getVectorNumElements();
4598  unsigned NumInElts = InVT.getVectorNumElements();
4599 
4600  // Fall back to use extracts and build vector.
4601  EVT EltVT = WidenVT.getVectorElementType();
4602  SmallVector<SDValue, 16> Ops(WidenNumElts);
4603  unsigned Idx = 0;
4604  for (unsigned i=0; i < NumOperands; ++i) {
4605  SDValue InOp = N->getOperand(i);
4606  if (InputWidened)
4607  InOp = GetWidenedVector(InOp);
4608  for (unsigned j = 0; j < NumInElts; ++j)
4609  Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4610  DAG.getVectorIdxConstant(j, dl));
4611  }
4612  SDValue UndefVal = DAG.getUNDEF(EltVT);
4613  for (; Idx < WidenNumElts; ++Idx)
4614  Ops[Idx] = UndefVal;
4615  return DAG.getBuildVector(WidenVT, dl, Ops);
4616 }
4617 
4618 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) {
4619  EVT VT = N->getValueType(0);
4620  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4621  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4622  SDValue InOp2 = N->getOperand(1);
4623  SDValue Idx = N->getOperand(2);
4624  SDLoc dl(N);
4625  return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WidenVT, InOp1, InOp2, Idx);
4626 }
4627 
4628 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
4629  EVT VT = N->getValueType(0);