LLVM  14.0.0git
LegalizeVectorTypes.cpp
Go to the documentation of this file.
1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file performs vector type splitting and scalarization for LegalizeTypes.
10 // Scalarization is the act of changing a computation in an illegal one-element
11 // vector type to be a computation in its scalar element type. For example,
12 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
14 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15 // types.
16 // Splitting is the act of changing a computation in an invalid vector type to
17 // be a computation in two vectors of half the size. For example, implementing
18 // <128 x f32> operations in terms of two <64 x f32> operations.
19 //
20 //===----------------------------------------------------------------------===//
21 
22 #include "LegalizeTypes.h"
24 #include "llvm/IR/DataLayout.h"
26 #include "llvm/Support/TypeSize.h"
28 using namespace llvm;
29 
30 #define DEBUG_TYPE "legalize-types"
31 
32 //===----------------------------------------------------------------------===//
33 // Result Vector Scalarization: <1 x ty> -> ty.
34 //===----------------------------------------------------------------------===//
35 
36 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
37  LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
38  dbgs() << "\n");
39  SDValue R = SDValue();
40 
41  switch (N->getOpcode()) {
42  default:
43 #ifndef NDEBUG
44  dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
45  N->dump(&DAG);
46  dbgs() << "\n";
47 #endif
48  report_fatal_error("Do not know how to scalarize the result of this "
49  "operator!\n");
50 
51  case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
52  case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
53  case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
54  case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
55  case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
56  case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
57  case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
58  case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
59  case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
60  case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
61  case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
62  case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
63  case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
64  case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
65  case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
66  case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
70  R = ScalarizeVecRes_VecInregOp(N);
71  break;
72  case ISD::ABS:
73  case ISD::ANY_EXTEND:
74  case ISD::BITREVERSE:
75  case ISD::BSWAP:
76  case ISD::CTLZ:
78  case ISD::CTPOP:
79  case ISD::CTTZ:
81  case ISD::FABS:
82  case ISD::FCEIL:
83  case ISD::FCOS:
84  case ISD::FEXP:
85  case ISD::FEXP2:
86  case ISD::FFLOOR:
87  case ISD::FLOG:
88  case ISD::FLOG10:
89  case ISD::FLOG2:
90  case ISD::FNEARBYINT:
91  case ISD::FNEG:
92  case ISD::FREEZE:
93  case ISD::ARITH_FENCE:
94  case ISD::FP_EXTEND:
95  case ISD::FP_TO_SINT:
96  case ISD::FP_TO_UINT:
97  case ISD::FRINT:
98  case ISD::FROUND:
99  case ISD::FROUNDEVEN:
100  case ISD::FSIN:
101  case ISD::FSQRT:
102  case ISD::FTRUNC:
103  case ISD::SIGN_EXTEND:
104  case ISD::SINT_TO_FP:
105  case ISD::TRUNCATE:
106  case ISD::UINT_TO_FP:
107  case ISD::ZERO_EXTEND:
108  case ISD::FCANONICALIZE:
109  R = ScalarizeVecRes_UnaryOp(N);
110  break;
111 
112  case ISD::ADD:
113  case ISD::AND:
114  case ISD::FADD:
115  case ISD::FCOPYSIGN:
116  case ISD::FDIV:
117  case ISD::FMUL:
118  case ISD::FMINNUM:
119  case ISD::FMAXNUM:
120  case ISD::FMINNUM_IEEE:
121  case ISD::FMAXNUM_IEEE:
122  case ISD::FMINIMUM:
123  case ISD::FMAXIMUM:
124  case ISD::SMIN:
125  case ISD::SMAX:
126  case ISD::UMIN:
127  case ISD::UMAX:
128 
129  case ISD::SADDSAT:
130  case ISD::UADDSAT:
131  case ISD::SSUBSAT:
132  case ISD::USUBSAT:
133  case ISD::SSHLSAT:
134  case ISD::USHLSAT:
135 
136  case ISD::FPOW:
137  case ISD::FREM:
138  case ISD::FSUB:
139  case ISD::MUL:
140  case ISD::OR:
141  case ISD::SDIV:
142  case ISD::SREM:
143  case ISD::SUB:
144  case ISD::UDIV:
145  case ISD::UREM:
146  case ISD::XOR:
147  case ISD::SHL:
148  case ISD::SRA:
149  case ISD::SRL:
150  case ISD::ROTL:
151  case ISD::ROTR:
152  R = ScalarizeVecRes_BinOp(N);
153  break;
154  case ISD::FMA:
155  case ISD::FSHL:
156  case ISD::FSHR:
157  R = ScalarizeVecRes_TernaryOp(N);
158  break;
159 
160 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
161  case ISD::STRICT_##DAGN:
162 #include "llvm/IR/ConstrainedOps.def"
163  R = ScalarizeVecRes_StrictFPOp(N);
164  break;
165 
166  case ISD::FP_TO_UINT_SAT:
167  case ISD::FP_TO_SINT_SAT:
168  R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
169  break;
170 
171  case ISD::UADDO:
172  case ISD::SADDO:
173  case ISD::USUBO:
174  case ISD::SSUBO:
175  case ISD::UMULO:
176  case ISD::SMULO:
177  R = ScalarizeVecRes_OverflowOp(N, ResNo);
178  break;
179  case ISD::SMULFIX:
180  case ISD::SMULFIXSAT:
181  case ISD::UMULFIX:
182  case ISD::UMULFIXSAT:
183  case ISD::SDIVFIX:
184  case ISD::SDIVFIXSAT:
185  case ISD::UDIVFIX:
186  case ISD::UDIVFIXSAT:
187  R = ScalarizeVecRes_FIX(N);
188  break;
189  }
190 
191  // If R is null, the sub-method took care of registering the result.
192  if (R.getNode())
193  SetScalarizedVector(SDValue(N, ResNo), R);
194 }
195 
196 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
197  SDValue LHS = GetScalarizedVector(N->getOperand(0));
198  SDValue RHS = GetScalarizedVector(N->getOperand(1));
199  return DAG.getNode(N->getOpcode(), SDLoc(N),
200  LHS.getValueType(), LHS, RHS, N->getFlags());
201 }
202 
203 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
204  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
205  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
206  SDValue Op2 = GetScalarizedVector(N->getOperand(2));
207  return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
208  Op2, N->getFlags());
209 }
210 
211 SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
212  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
213  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
214  SDValue Op2 = N->getOperand(2);
215  return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
216  Op2, N->getFlags());
217 }
218 
219 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
220  EVT VT = N->getValueType(0).getVectorElementType();
221  unsigned NumOpers = N->getNumOperands();
222  SDValue Chain = N->getOperand(0);
223  EVT ValueVTs[] = {VT, MVT::Other};
224  SDLoc dl(N);
225 
226  SmallVector<SDValue, 4> Opers(NumOpers);
227 
228  // The Chain is the first operand.
229  Opers[0] = Chain;
230 
231  // Now process the remaining operands.
232  for (unsigned i = 1; i < NumOpers; ++i) {
233  SDValue Oper = N->getOperand(i);
234 
235  if (Oper.getValueType().isVector())
236  Oper = GetScalarizedVector(Oper);
237 
238  Opers[i] = Oper;
239  }
240 
241  SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs),
242  Opers, N->getFlags());
243 
244  // Legalize the chain result - switch anything that used the old chain to
245  // use the new one.
246  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
247  return Result;
248 }
249 
250 SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
251  unsigned ResNo) {
252  SDLoc DL(N);
253  EVT ResVT = N->getValueType(0);
254  EVT OvVT = N->getValueType(1);
255 
256  SDValue ScalarLHS, ScalarRHS;
257  if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
258  ScalarLHS = GetScalarizedVector(N->getOperand(0));
259  ScalarRHS = GetScalarizedVector(N->getOperand(1));
260  } else {
261  SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
262  DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
263  DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
264  ScalarLHS = ElemsLHS[0];
265  ScalarRHS = ElemsRHS[0];
266  }
267 
268  SDVTList ScalarVTs = DAG.getVTList(
270  SDNode *ScalarNode = DAG.getNode(
271  N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
272  ScalarNode->setFlags(N->getFlags());
273 
274  // Replace the other vector result not being explicitly scalarized here.
275  unsigned OtherNo = 1 - ResNo;
276  EVT OtherVT = N->getValueType(OtherNo);
277  if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
278  SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
279  } else {
280  SDValue OtherVal = DAG.getNode(
281  ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
282  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
283  }
284 
285  return SDValue(ScalarNode, ResNo);
286 }
287 
288 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
289  unsigned ResNo) {
290  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
291  return GetScalarizedVector(Op);
292 }
293 
294 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
295  SDValue Op = N->getOperand(0);
296  if (Op.getValueType().isVector()
297  && Op.getValueType().getVectorNumElements() == 1
298  && !isSimpleLegalType(Op.getValueType()))
299  Op = GetScalarizedVector(Op);
300  EVT NewVT = N->getValueType(0).getVectorElementType();
301  return DAG.getNode(ISD::BITCAST, SDLoc(N),
302  NewVT, Op);
303 }
304 
305 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
306  EVT EltVT = N->getValueType(0).getVectorElementType();
307  SDValue InOp = N->getOperand(0);
308  // The BUILD_VECTOR operands may be of wider element types and
309  // we may need to truncate them back to the requested return type.
310  if (EltVT.isInteger())
311  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
312  return InOp;
313 }
314 
315 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
316  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
317  N->getValueType(0).getVectorElementType(),
318  N->getOperand(0), N->getOperand(1));
319 }
320 
321 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
322  SDLoc DL(N);
323  SDValue Op = N->getOperand(0);
324  EVT OpVT = Op.getValueType();
325  // The result needs scalarizing, but it's not a given that the source does.
326  // See similar logic in ScalarizeVecRes_UnaryOp.
327  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
328  Op = GetScalarizedVector(Op);
329  } else {
330  EVT VT = OpVT.getVectorElementType();
331  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
332  DAG.getVectorIdxConstant(0, DL));
333  }
334  return DAG.getNode(ISD::FP_ROUND, DL,
335  N->getValueType(0).getVectorElementType(), Op,
336  N->getOperand(1));
337 }
338 
339 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
340  SDValue Op = GetScalarizedVector(N->getOperand(0));
341  return DAG.getNode(ISD::FPOWI, SDLoc(N),
342  Op.getValueType(), Op, N->getOperand(1));
343 }
344 
345 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
346  // The value to insert may have a wider type than the vector element type,
347  // so be sure to truncate it to the element type if necessary.
348  SDValue Op = N->getOperand(1);
349  EVT EltVT = N->getValueType(0).getVectorElementType();
350  if (Op.getValueType() != EltVT)
351  // FIXME: Can this happen for floating point types?
352  Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
353  return Op;
354 }
355 
356 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
357  assert(N->isUnindexed() && "Indexed vector load?");
358 
359  SDValue Result = DAG.getLoad(
360  ISD::UNINDEXED, N->getExtensionType(),
361  N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
362  N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
363  N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
364  N->getOriginalAlign(), N->getMemOperand()->getFlags(), N->getAAInfo());
365 
366  // Legalize the chain result - switch anything that used the old chain to
367  // use the new one.
368  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
369  return Result;
370 }
371 
372 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
373  // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
374  EVT DestVT = N->getValueType(0).getVectorElementType();
375  SDValue Op = N->getOperand(0);
376  EVT OpVT = Op.getValueType();
377  SDLoc DL(N);
378  // The result needs scalarizing, but it's not a given that the source does.
379  // This is a workaround for targets where it's impossible to scalarize the
380  // result of a conversion, because the source type is legal.
381  // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
382  // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
383  // legal and was not scalarized.
384  // See the similar logic in ScalarizeVecRes_SETCC
385  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
386  Op = GetScalarizedVector(Op);
387  } else {
388  EVT VT = OpVT.getVectorElementType();
389  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
390  DAG.getVectorIdxConstant(0, DL));
391  }
392  return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op, N->getFlags());
393 }
394 
395 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
396  EVT EltVT = N->getValueType(0).getVectorElementType();
397  EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
398  SDValue LHS = GetScalarizedVector(N->getOperand(0));
399  return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
400  LHS, DAG.getValueType(ExtVT));
401 }
402 
403 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
404  SDLoc DL(N);
405  SDValue Op = N->getOperand(0);
406 
407  EVT OpVT = Op.getValueType();
408  EVT OpEltVT = OpVT.getVectorElementType();
409  EVT EltVT = N->getValueType(0).getVectorElementType();
410 
411  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
412  Op = GetScalarizedVector(Op);
413  } else {
414  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
415  DAG.getVectorIdxConstant(0, DL));
416  }
417 
418  switch (N->getOpcode()) {
420  return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
422  return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
424  return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
425  }
426 
427  llvm_unreachable("Illegal extend_vector_inreg opcode");
428 }
429 
430 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
431  // If the operand is wider than the vector element type then it is implicitly
432  // truncated. Make that explicit here.
433  EVT EltVT = N->getValueType(0).getVectorElementType();
434  SDValue InOp = N->getOperand(0);
435  if (InOp.getValueType() != EltVT)
436  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
437  return InOp;
438 }
439 
440 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
441  SDValue Cond = N->getOperand(0);
442  EVT OpVT = Cond.getValueType();
443  SDLoc DL(N);
444  // The vselect result and true/value operands needs scalarizing, but it's
445  // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
446  // See the similar logic in ScalarizeVecRes_SETCC
447  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
448  Cond = GetScalarizedVector(Cond);
449  } else {
450  EVT VT = OpVT.getVectorElementType();
452  DAG.getVectorIdxConstant(0, DL));
453  }
454 
455  SDValue LHS = GetScalarizedVector(N->getOperand(1));
456  TargetLowering::BooleanContent ScalarBool =
457  TLI.getBooleanContents(false, false);
458  TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
459 
460  // If integer and float booleans have different contents then we can't
461  // reliably optimize in all cases. There is a full explanation for this in
462  // DAGCombiner::visitSELECT() where the same issue affects folding
463  // (select C, 0, 1) to (xor C, 1).
464  if (TLI.getBooleanContents(false, false) !=
465  TLI.getBooleanContents(false, true)) {
466  // At least try the common case where the boolean is generated by a
467  // comparison.
468  if (Cond->getOpcode() == ISD::SETCC) {
469  EVT OpVT = Cond->getOperand(0).getValueType();
470  ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
471  VecBool = TLI.getBooleanContents(OpVT);
472  } else
474  }
475 
476  EVT CondVT = Cond.getValueType();
477  if (ScalarBool != VecBool) {
478  switch (ScalarBool) {
480  break;
484  // Vector read from all ones, scalar expects a single 1 so mask.
485  Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
486  Cond, DAG.getConstant(1, SDLoc(N), CondVT));
487  break;
491  // Vector reads from a one, scalar from all ones so sign extend.
492  Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
493  Cond, DAG.getValueType(MVT::i1));
494  break;
495  }
496  }
497 
498  // Truncate the condition if needed
499  auto BoolVT = getSetCCResultType(CondVT);
500  if (BoolVT.bitsLT(CondVT))
501  Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
502 
503  return DAG.getSelect(SDLoc(N),
504  LHS.getValueType(), Cond, LHS,
505  GetScalarizedVector(N->getOperand(2)));
506 }
507 
508 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
509  SDValue LHS = GetScalarizedVector(N->getOperand(1));
510  return DAG.getSelect(SDLoc(N),
511  LHS.getValueType(), N->getOperand(0), LHS,
512  GetScalarizedVector(N->getOperand(2)));
513 }
514 
515 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
516  SDValue LHS = GetScalarizedVector(N->getOperand(2));
517  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
518  N->getOperand(0), N->getOperand(1),
519  LHS, GetScalarizedVector(N->getOperand(3)),
520  N->getOperand(4));
521 }
522 
523 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
524  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
525 }
526 
527 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
528  // Figure out if the scalar is the LHS or RHS and return it.
529  SDValue Arg = N->getOperand(2).getOperand(0);
530  if (Arg.isUndef())
531  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
532  unsigned Op = !cast<ConstantSDNode>(Arg)->isZero();
533  return GetScalarizedVector(N->getOperand(Op));
534 }
535 
536 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
537  SDValue Src = N->getOperand(0);
538  EVT SrcVT = Src.getValueType();
539  SDLoc dl(N);
540 
541  // Handle case where result is scalarized but operand is not
542  if (getTypeAction(SrcVT) == TargetLowering::TypeScalarizeVector)
543  Src = GetScalarizedVector(Src);
544  else
545  Src = DAG.getNode(
547  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
548 
549  EVT DstVT = N->getValueType(0).getVectorElementType();
550  return DAG.getNode(N->getOpcode(), dl, DstVT, Src, N->getOperand(1));
551 }
552 
553 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
554  assert(N->getValueType(0).isVector() &&
555  N->getOperand(0).getValueType().isVector() &&
556  "Operand types must be vectors");
557  SDValue LHS = N->getOperand(0);
558  SDValue RHS = N->getOperand(1);
559  EVT OpVT = LHS.getValueType();
560  EVT NVT = N->getValueType(0).getVectorElementType();
561  SDLoc DL(N);
562 
563  // The result needs scalarizing, but it's not a given that the source does.
564  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
565  LHS = GetScalarizedVector(LHS);
566  RHS = GetScalarizedVector(RHS);
567  } else {
568  EVT VT = OpVT.getVectorElementType();
569  LHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
570  DAG.getVectorIdxConstant(0, DL));
571  RHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
572  DAG.getVectorIdxConstant(0, DL));
573  }
574 
575  // Turn it into a scalar SETCC.
576  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
577  N->getOperand(2));
578  // Vectors may have a different boolean contents to scalars. Promote the
579  // value appropriately.
580  ISD::NodeType ExtendCode =
582  return DAG.getNode(ExtendCode, DL, NVT, Res);
583 }
584 
585 
586 //===----------------------------------------------------------------------===//
587 // Operand Vector Scalarization <1 x ty> -> ty.
588 //===----------------------------------------------------------------------===//
589 
590 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
591  LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
592  dbgs() << "\n");
593  SDValue Res = SDValue();
594 
595  switch (N->getOpcode()) {
596  default:
597 #ifndef NDEBUG
598  dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
599  N->dump(&DAG);
600  dbgs() << "\n";
601 #endif
602  report_fatal_error("Do not know how to scalarize this operator's "
603  "operand!\n");
604  case ISD::BITCAST:
605  Res = ScalarizeVecOp_BITCAST(N);
606  break;
607  case ISD::ANY_EXTEND:
608  case ISD::ZERO_EXTEND:
609  case ISD::SIGN_EXTEND:
610  case ISD::TRUNCATE:
611  case ISD::FP_TO_SINT:
612  case ISD::FP_TO_UINT:
613  case ISD::SINT_TO_FP:
614  case ISD::UINT_TO_FP:
615  Res = ScalarizeVecOp_UnaryOp(N);
616  break;
621  Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
622  break;
623  case ISD::CONCAT_VECTORS:
624  Res = ScalarizeVecOp_CONCAT_VECTORS(N);
625  break;
627  Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
628  break;
629  case ISD::VSELECT:
630  Res = ScalarizeVecOp_VSELECT(N);
631  break;
632  case ISD::SETCC:
633  Res = ScalarizeVecOp_VSETCC(N);
634  break;
635  case ISD::STORE:
636  Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
637  break;
639  Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
640  break;
641  case ISD::FP_ROUND:
642  Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
643  break;
645  Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
646  break;
647  case ISD::FP_EXTEND:
648  Res = ScalarizeVecOp_FP_EXTEND(N);
649  break;
650  case ISD::VECREDUCE_FADD:
651  case ISD::VECREDUCE_FMUL:
652  case ISD::VECREDUCE_ADD:
653  case ISD::VECREDUCE_MUL:
654  case ISD::VECREDUCE_AND:
655  case ISD::VECREDUCE_OR:
656  case ISD::VECREDUCE_XOR:
657  case ISD::VECREDUCE_SMAX:
658  case ISD::VECREDUCE_SMIN:
659  case ISD::VECREDUCE_UMAX:
660  case ISD::VECREDUCE_UMIN:
661  case ISD::VECREDUCE_FMAX:
662  case ISD::VECREDUCE_FMIN:
663  Res = ScalarizeVecOp_VECREDUCE(N);
664  break;
667  Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
668  break;
669  }
670 
671  // If the result is null, the sub-method took care of registering results etc.
672  if (!Res.getNode()) return false;
673 
674  // If the result is N, the sub-method updated N in place. Tell the legalizer
675  // core about this.
676  if (Res.getNode() == N)
677  return true;
678 
679  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
680  "Invalid operand expansion");
681 
682  ReplaceValueWith(SDValue(N, 0), Res);
683  return false;
684 }
685 
686 /// If the value to convert is a vector that needs to be scalarized, it must be
687 /// <1 x ty>. Convert the element instead.
688 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
689  SDValue Elt = GetScalarizedVector(N->getOperand(0));
690  return DAG.getNode(ISD::BITCAST, SDLoc(N),
691  N->getValueType(0), Elt);
692 }
693 
694 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
695 /// Do the operation on the element instead.
696 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
697  assert(N->getValueType(0).getVectorNumElements() == 1 &&
698  "Unexpected vector type!");
699  SDValue Elt = GetScalarizedVector(N->getOperand(0));
700  SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
701  N->getValueType(0).getScalarType(), Elt);
702  // Revectorize the result so the types line up with what the uses of this
703  // expression expect.
704  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
705 }
706 
707 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
708 /// Do the strict FP operation on the element instead.
709 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
710  assert(N->getValueType(0).getVectorNumElements() == 1 &&
711  "Unexpected vector type!");
712  SDValue Elt = GetScalarizedVector(N->getOperand(1));
713  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
714  { N->getValueType(0).getScalarType(), MVT::Other },
715  { N->getOperand(0), Elt });
716  // Legalize the chain result - switch anything that used the old chain to
717  // use the new one.
718  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
719  // Revectorize the result so the types line up with what the uses of this
720  // expression expect.
721  Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
722 
723  // Do our own replacement and return SDValue() to tell the caller that we
724  // handled all replacements since caller can only handle a single result.
725  ReplaceValueWith(SDValue(N, 0), Res);
726  return SDValue();
727 }
728 
729 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
730 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
731  SmallVector<SDValue, 8> Ops(N->getNumOperands());
732  for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
733  Ops[i] = GetScalarizedVector(N->getOperand(i));
734  return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
735 }
736 
737 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
738 /// so just return the element, ignoring the index.
739 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
740  EVT VT = N->getValueType(0);
741  SDValue Res = GetScalarizedVector(N->getOperand(0));
742  if (Res.getValueType() != VT)
743  Res = VT.isFloatingPoint()
744  ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
745  : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
746  return Res;
747 }
748 
749 /// If the input condition is a vector that needs to be scalarized, it must be
750 /// <1 x i1>, so just convert to a normal ISD::SELECT
751 /// (still with vector output type since that was acceptable if we got here).
752 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
753  SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
754  EVT VT = N->getValueType(0);
755 
756  return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
757  N->getOperand(2));
758 }
759 
760 /// If the operand is a vector that needs to be scalarized then the
761 /// result must be v1i1, so just convert to a scalar SETCC and wrap
762 /// with a scalar_to_vector since the res type is legal if we got here
763 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
764  assert(N->getValueType(0).isVector() &&
765  N->getOperand(0).getValueType().isVector() &&
766  "Operand types must be vectors");
767  assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
768 
769  EVT VT = N->getValueType(0);
770  SDValue LHS = GetScalarizedVector(N->getOperand(0));
771  SDValue RHS = GetScalarizedVector(N->getOperand(1));
772 
773  EVT OpVT = N->getOperand(0).getValueType();
774  EVT NVT = VT.getVectorElementType();
775  SDLoc DL(N);
776  // Turn it into a scalar SETCC.
777  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
778  N->getOperand(2));
779 
780  // Vectors may have a different boolean contents to scalars. Promote the
781  // value appropriately.
782  ISD::NodeType ExtendCode =
784 
785  Res = DAG.getNode(ExtendCode, DL, NVT, Res);
786 
787  return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
788 }
789 
790 /// If the value to store is a vector that needs to be scalarized, it must be
791 /// <1 x ty>. Just store the element.
792 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
793  assert(N->isUnindexed() && "Indexed store of one-element vector?");
794  assert(OpNo == 1 && "Do not know how to scalarize this operand!");
795  SDLoc dl(N);
796 
797  if (N->isTruncatingStore())
798  return DAG.getTruncStore(
799  N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
800  N->getBasePtr(), N->getPointerInfo(),
801  N->getMemoryVT().getVectorElementType(), N->getOriginalAlign(),
802  N->getMemOperand()->getFlags(), N->getAAInfo());
803 
804  return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
805  N->getBasePtr(), N->getPointerInfo(),
806  N->getOriginalAlign(), N->getMemOperand()->getFlags(),
807  N->getAAInfo());
808 }
809 
810 /// If the value to round is a vector that needs to be scalarized, it must be
811 /// <1 x ty>. Convert the element instead.
812 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
813  assert(OpNo == 0 && "Wrong operand for scalarization!");
814  SDValue Elt = GetScalarizedVector(N->getOperand(0));
815  SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
816  N->getValueType(0).getVectorElementType(), Elt,
817  N->getOperand(1));
818  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
819 }
820 
821 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
822  unsigned OpNo) {
823  assert(OpNo == 1 && "Wrong operand for scalarization!");
824  SDValue Elt = GetScalarizedVector(N->getOperand(1));
826  { N->getValueType(0).getVectorElementType(),
827  MVT::Other },
828  { N->getOperand(0), Elt, N->getOperand(2) });
829  // Legalize the chain result - switch anything that used the old chain to
830  // use the new one.
831  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
832 
833  Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
834 
835  // Do our own replacement and return SDValue() to tell the caller that we
836  // handled all replacements since caller can only handle a single result.
837  ReplaceValueWith(SDValue(N, 0), Res);
838  return SDValue();
839 }
840 
841 /// If the value to extend is a vector that needs to be scalarized, it must be
842 /// <1 x ty>. Convert the element instead.
843 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
844  SDValue Elt = GetScalarizedVector(N->getOperand(0));
845  SDValue Res = DAG.getNode(ISD::FP_EXTEND, SDLoc(N),
846  N->getValueType(0).getVectorElementType(), Elt);
847  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
848 }
849 
850 /// If the value to extend is a vector that needs to be scalarized, it must be
851 /// <1 x ty>. Convert the element instead.
852 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
853  SDValue Elt = GetScalarizedVector(N->getOperand(1));
854  SDValue Res =
856  {N->getValueType(0).getVectorElementType(), MVT::Other},
857  {N->getOperand(0), Elt});
858  // Legalize the chain result - switch anything that used the old chain to
859  // use the new one.
860  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
861 
862  Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
863 
864  // Do our own replacement and return SDValue() to tell the caller that we
865  // handled all replacements since caller can only handle a single result.
866  ReplaceValueWith(SDValue(N, 0), Res);
867  return SDValue();
868 }
869 
870 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
871  SDValue Res = GetScalarizedVector(N->getOperand(0));
872  // Result type may be wider than element type.
873  if (Res.getValueType() != N->getValueType(0))
874  Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res);
875  return Res;
876 }
877 
878 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
879  SDValue AccOp = N->getOperand(0);
880  SDValue VecOp = N->getOperand(1);
881 
882  unsigned BaseOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
883 
884  SDValue Op = GetScalarizedVector(VecOp);
885  return DAG.getNode(BaseOpc, SDLoc(N), N->getValueType(0),
886  AccOp, Op, N->getFlags());
887 }
888 
889 //===----------------------------------------------------------------------===//
890 // Result Vector Splitting
891 //===----------------------------------------------------------------------===//
892 
893 /// This method is called when the specified result of the specified node is
894 /// found to need vector splitting. At this point, the node may also have
895 /// invalid operands or may have other results that need legalization, we just
896 /// know that (at least) one result needs vector splitting.
897 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
898  LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
899  SDValue Lo, Hi;
900 
901  // See if the target wants to custom expand this node.
902  if (CustomLowerNode(N, N->getValueType(ResNo), true))
903  return;
904 
905  switch (N->getOpcode()) {
906  default:
907 #ifndef NDEBUG
908  dbgs() << "SplitVectorResult #" << ResNo << ": ";
909  N->dump(&DAG);
910  dbgs() << "\n";
911 #endif
912  report_fatal_error("Do not know how to split the result of this "
913  "operator!\n");
914 
915  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
916  case ISD::VSELECT:
917  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
918  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
919  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
920  case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
921  case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
922  case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
923  case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
924  case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
925  case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
926  case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
927  case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
928  case ISD::SPLAT_VECTOR:
930  SplitVecRes_ScalarOp(N, Lo, Hi);
931  break;
932  case ISD::STEP_VECTOR:
933  SplitVecRes_STEP_VECTOR(N, Lo, Hi);
934  break;
935  case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
936  case ISD::LOAD:
937  SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
938  break;
939  case ISD::MLOAD:
940  SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
941  break;
942  case ISD::MGATHER:
943  SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
944  break;
945  case ISD::SETCC:
946  SplitVecRes_SETCC(N, Lo, Hi);
947  break;
948  case ISD::VECTOR_REVERSE:
949  SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
950  break;
951  case ISD::VECTOR_SHUFFLE:
952  SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
953  break;
954  case ISD::VECTOR_SPLICE:
955  SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
956  break;
957  case ISD::VAARG:
958  SplitVecRes_VAARG(N, Lo, Hi);
959  break;
960 
964  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
965  break;
966 
967  case ISD::ABS:
968  case ISD::BITREVERSE:
969  case ISD::BSWAP:
970  case ISD::CTLZ:
971  case ISD::CTTZ:
974  case ISD::CTPOP:
975  case ISD::FABS:
976  case ISD::FCEIL:
977  case ISD::FCOS:
978  case ISD::FEXP:
979  case ISD::FEXP2:
980  case ISD::FFLOOR:
981  case ISD::FLOG:
982  case ISD::FLOG10:
983  case ISD::FLOG2:
984  case ISD::FNEARBYINT:
985  case ISD::FNEG:
986  case ISD::FREEZE:
987  case ISD::ARITH_FENCE:
988  case ISD::FP_EXTEND:
989  case ISD::FP_ROUND:
990  case ISD::FP_TO_SINT:
991  case ISD::FP_TO_UINT:
992  case ISD::FRINT:
993  case ISD::FROUND:
994  case ISD::FROUNDEVEN:
995  case ISD::FSIN:
996  case ISD::FSQRT:
997  case ISD::FTRUNC:
998  case ISD::SINT_TO_FP:
999  case ISD::TRUNCATE:
1000  case ISD::UINT_TO_FP:
1001  case ISD::FCANONICALIZE:
1002  SplitVecRes_UnaryOp(N, Lo, Hi);
1003  break;
1004 
1005  case ISD::ANY_EXTEND:
1006  case ISD::SIGN_EXTEND:
1007  case ISD::ZERO_EXTEND:
1008  SplitVecRes_ExtendOp(N, Lo, Hi);
1009  break;
1010 
1011  case ISD::ADD:
1012  case ISD::SUB:
1013  case ISD::MUL:
1014  case ISD::MULHS:
1015  case ISD::MULHU:
1016  case ISD::FADD:
1017  case ISD::FSUB:
1018  case ISD::FMUL:
1019  case ISD::FMINNUM:
1020  case ISD::FMAXNUM:
1021  case ISD::FMINIMUM:
1022  case ISD::FMAXIMUM:
1023  case ISD::SDIV:
1024  case ISD::UDIV:
1025  case ISD::FDIV:
1026  case ISD::FPOW:
1027  case ISD::AND:
1028  case ISD::OR:
1029  case ISD::XOR:
1030  case ISD::SHL:
1031  case ISD::SRA:
1032  case ISD::SRL:
1033  case ISD::UREM:
1034  case ISD::SREM:
1035  case ISD::FREM:
1036  case ISD::SMIN:
1037  case ISD::SMAX:
1038  case ISD::UMIN:
1039  case ISD::UMAX:
1040  case ISD::SADDSAT:
1041  case ISD::UADDSAT:
1042  case ISD::SSUBSAT:
1043  case ISD::USUBSAT:
1044  case ISD::SSHLSAT:
1045  case ISD::USHLSAT:
1046  case ISD::ROTL:
1047  case ISD::ROTR:
1048  SplitVecRes_BinOp(N, Lo, Hi, /*IsVP*/ false);
1049  break;
1050  case ISD::FMA:
1051  case ISD::FSHL:
1052  case ISD::FSHR:
1053  SplitVecRes_TernaryOp(N, Lo, Hi);
1054  break;
1055 
1056 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1057  case ISD::STRICT_##DAGN:
1058 #include "llvm/IR/ConstrainedOps.def"
1059  SplitVecRes_StrictFPOp(N, Lo, Hi);
1060  break;
1061 
1062  case ISD::FP_TO_UINT_SAT:
1063  case ISD::FP_TO_SINT_SAT:
1064  SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1065  break;
1066 
1067  case ISD::UADDO:
1068  case ISD::SADDO:
1069  case ISD::USUBO:
1070  case ISD::SSUBO:
1071  case ISD::UMULO:
1072  case ISD::SMULO:
1073  SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1074  break;
1075  case ISD::SMULFIX:
1076  case ISD::SMULFIXSAT:
1077  case ISD::UMULFIX:
1078  case ISD::UMULFIXSAT:
1079  case ISD::SDIVFIX:
1080  case ISD::SDIVFIXSAT:
1081  case ISD::UDIVFIX:
1082  case ISD::UDIVFIXSAT:
1083  SplitVecRes_FIX(N, Lo, Hi);
1084  break;
1085  case ISD::VP_ADD:
1086  case ISD::VP_AND:
1087  case ISD::VP_MUL:
1088  case ISD::VP_OR:
1089  case ISD::VP_SUB:
1090  case ISD::VP_XOR:
1091  case ISD::VP_SHL:
1092  case ISD::VP_LSHR:
1093  case ISD::VP_ASHR:
1094  case ISD::VP_SDIV:
1095  case ISD::VP_UDIV:
1096  case ISD::VP_SREM:
1097  case ISD::VP_UREM:
1098  case ISD::VP_FADD:
1099  case ISD::VP_FSUB:
1100  case ISD::VP_FMUL:
1101  case ISD::VP_FDIV:
1102  case ISD::VP_FREM:
1103  SplitVecRes_BinOp(N, Lo, Hi, /*IsVP*/ true);
1104  break;
1105  }
1106 
1107  // If Lo/Hi is null, the sub-method took care of registering results etc.
1108  if (Lo.getNode())
1109  SetSplitVector(SDValue(N, ResNo), Lo, Hi);
1110 }
1111 
1112 void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1113  MachinePointerInfo &MPI, SDValue &Ptr,
1114  uint64_t *ScaledOffset) {
1115  SDLoc DL(N);
1116  unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinSize() / 8;
1117 
1118  if (MemVT.isScalableVector()) {
1119  SDNodeFlags Flags;
1120  SDValue BytesIncrement = DAG.getVScale(
1121  DL, Ptr.getValueType(),
1122  APInt(Ptr.getValueSizeInBits().getFixedSize(), IncrementSize));
1123  MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1124  Flags.setNoUnsignedWrap(true);
1125  if (ScaledOffset)
1126  *ScaledOffset += IncrementSize;
1127  Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, BytesIncrement,
1128  Flags);
1129  } else {
1130  MPI = N->getPointerInfo().getWithOffset(IncrementSize);
1131  // Increment the pointer to the other half.
1132  Ptr = DAG.getObjectPtrOffset(DL, Ptr, TypeSize::Fixed(IncrementSize));
1133  }
1134 }
1135 
1136 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi,
1137  bool IsVP) {
1138  SDValue LHSLo, LHSHi;
1139  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1140  SDValue RHSLo, RHSHi;
1141  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1142  SDLoc dl(N);
1143 
1144  const SDNodeFlags Flags = N->getFlags();
1145  unsigned Opcode = N->getOpcode();
1146  if (!IsVP) {
1147  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
1148  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
1149  return;
1150  }
1151 
1152  // Split the mask.
1153  SDValue MaskLo, MaskHi;
1154  SDValue Mask = N->getOperand(2);
1155  EVT MaskVT = Mask.getValueType();
1156  if (getTypeAction(MaskVT) == TargetLowering::TypeSplitVector)
1157  GetSplitVector(Mask, MaskLo, MaskHi);
1158  else
1159  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, SDLoc(Mask));
1160 
1161  // Split the vector length parameter.
1162  // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
1163  SDValue EVL = N->getOperand(3);
1164  EVT VecVT = N->getValueType(0);
1165  EVT EVLVT = EVL.getValueType();
1167  "Expecting the mask to be an evenly-sized vector");
1168  unsigned HalfMinNumElts = VecVT.getVectorMinNumElements() / 2;
1169  SDValue HalfNumElts =
1170  VecVT.isFixedLengthVector()
1171  ? DAG.getConstant(HalfMinNumElts, dl, EVLVT)
1172  : DAG.getVScale(dl, EVLVT,
1173  APInt(EVLVT.getScalarSizeInBits(), HalfMinNumElts));
1174  SDValue EVLLo = DAG.getNode(ISD::UMIN, dl, EVLVT, EVL, HalfNumElts);
1175  SDValue EVLHi = DAG.getNode(ISD::USUBSAT, dl, EVLVT, EVL, HalfNumElts);
1176 
1177  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(),
1178  {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1179  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(),
1180  {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1181 }
1182 
1183 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1184  SDValue &Hi) {
1185  SDValue Op0Lo, Op0Hi;
1186  GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
1187  SDValue Op1Lo, Op1Hi;
1188  GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
1189  SDValue Op2Lo, Op2Hi;
1190  GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
1191  SDLoc dl(N);
1192 
1193  Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(), Op0Lo, Op1Lo,
1194  Op2Lo, N->getFlags());
1195  Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(), Op0Hi, Op1Hi,
1196  Op2Hi, N->getFlags());
1197 }
1198 
1199 void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1200  SDValue LHSLo, LHSHi;
1201  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1202  SDValue RHSLo, RHSHi;
1203  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1204  SDLoc dl(N);
1205  SDValue Op2 = N->getOperand(2);
1206 
1207  unsigned Opcode = N->getOpcode();
1208  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2,
1209  N->getFlags());
1210  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2,
1211  N->getFlags());
1212 }
1213 
1214 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1215  SDValue &Hi) {
1216  // We know the result is a vector. The input may be either a vector or a
1217  // scalar value.
1218  EVT LoVT, HiVT;
1219  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1220  SDLoc dl(N);
1221 
1222  SDValue InOp = N->getOperand(0);
1223  EVT InVT = InOp.getValueType();
1224 
1225  // Handle some special cases efficiently.
1226  switch (getTypeAction(InVT)) {
1234  break;
1237  // A scalar to vector conversion, where the scalar needs expansion.
1238  // If the vector is being split in two then we can just convert the
1239  // expanded pieces.
1240  if (LoVT == HiVT) {
1241  GetExpandedOp(InOp, Lo, Hi);
1242  if (DAG.getDataLayout().isBigEndian())
1243  std::swap(Lo, Hi);
1244  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1245  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1246  return;
1247  }
1248  break;
1250  // If the input is a vector that needs to be split, convert each split
1251  // piece of the input now.
1252  GetSplitVector(InOp, Lo, Hi);
1253  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1254  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1255  return;
1257  report_fatal_error("Scalarization of scalable vectors is not supported.");
1258  }
1259 
1260  // In the general case, convert the input to an integer and split it by hand.
1261  EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
1262  EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
1263  if (DAG.getDataLayout().isBigEndian())
1264  std::swap(LoIntVT, HiIntVT);
1265 
1266  SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
1267 
1268  if (DAG.getDataLayout().isBigEndian())
1269  std::swap(Lo, Hi);
1270  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1271  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1272 }
1273 
1274 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1275  SDValue &Hi) {
1276  EVT LoVT, HiVT;
1277  SDLoc dl(N);
1278  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1279  unsigned LoNumElts = LoVT.getVectorNumElements();
1280  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1281  Lo = DAG.getBuildVector(LoVT, dl, LoOps);
1282 
1283  SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1284  Hi = DAG.getBuildVector(HiVT, dl, HiOps);
1285 }
1286 
1287 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1288  SDValue &Hi) {
1289  assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1290  SDLoc dl(N);
1291  unsigned NumSubvectors = N->getNumOperands() / 2;
1292  if (NumSubvectors == 1) {
1293  Lo = N->getOperand(0);
1294  Hi = N->getOperand(1);
1295  return;
1296  }
1297 
1298  EVT LoVT, HiVT;
1299  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1300 
1301  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1302  Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
1303 
1304  SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1305  Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
1306 }
1307 
1308 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1309  SDValue &Hi) {
1310  SDValue Vec = N->getOperand(0);
1311  SDValue Idx = N->getOperand(1);
1312  SDLoc dl(N);
1313 
1314  EVT LoVT, HiVT;
1315  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1316 
1317  Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
1318  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1319  Hi = DAG.getNode(
1320  ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
1321  DAG.getVectorIdxConstant(IdxVal + LoVT.getVectorMinNumElements(), dl));
1322 }
1323 
1324 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1325  SDValue &Hi) {
1326  SDValue Vec = N->getOperand(0);
1327  SDValue SubVec = N->getOperand(1);
1328  SDValue Idx = N->getOperand(2);
1329  SDLoc dl(N);
1330  GetSplitVector(Vec, Lo, Hi);
1331 
1332  EVT VecVT = Vec.getValueType();
1333  EVT LoVT = Lo.getValueType();
1334  EVT SubVecVT = SubVec.getValueType();
1335  unsigned VecElems = VecVT.getVectorMinNumElements();
1336  unsigned SubElems = SubVecVT.getVectorMinNumElements();
1337  unsigned LoElems = LoVT.getVectorMinNumElements();
1338 
1339  // If we know the index is in the first half, and we know the subvector
1340  // doesn't cross the boundary between the halves, we can avoid spilling the
1341  // vector, and insert into the lower half of the split vector directly.
1342  unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1343  if (IdxVal + SubElems <= LoElems) {
1344  Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1345  return;
1346  }
1347  // Similarly if the subvector is fully in the high half, but mind that we
1348  // can't tell whether a fixed-length subvector is fully within the high half
1349  // of a scalable vector.
1350  if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1351  IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1352  Hi = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, Hi.getValueType(), Hi, SubVec,
1353  DAG.getVectorIdxConstant(IdxVal - LoElems, dl));
1354  return;
1355  }
1356 
1357  // Spill the vector to the stack.
1358  // In cases where the vector is illegal it will be broken down into parts
1359  // and stored in parts - we should use the alignment for the smallest part.
1360  Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1361  SDValue StackPtr =
1362  DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1363  auto &MF = DAG.getMachineFunction();
1364  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1365  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1366 
1367  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1368  SmallestAlign);
1369 
1370  // Store the new subvector into the specified index.
1371  SDValue SubVecPtr =
1372  TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1373  Store = DAG.getStore(Store, dl, SubVec, SubVecPtr,
1375 
1376  // Load the Lo part from the stack slot.
1377  Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, PtrInfo,
1378  SmallestAlign);
1379 
1380  // Increment the pointer to the other part.
1381  auto *Load = cast<LoadSDNode>(Lo);
1382  MachinePointerInfo MPI = Load->getPointerInfo();
1383  IncrementPointer(Load, LoVT, MPI, StackPtr);
1384 
1385  // Load the Hi part from the stack slot.
1386  Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MPI, SmallestAlign);
1387 }
1388 
1389 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1390  SDValue &Hi) {
1391  SDLoc dl(N);
1392  GetSplitVector(N->getOperand(0), Lo, Hi);
1393  Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
1394  Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
1395 }
1396 
1397 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1398  SDValue &Hi) {
1399  SDValue LHSLo, LHSHi;
1400  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1401  SDLoc DL(N);
1402 
1403  SDValue RHSLo, RHSHi;
1404  SDValue RHS = N->getOperand(1);
1405  EVT RHSVT = RHS.getValueType();
1406  if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1407  GetSplitVector(RHS, RHSLo, RHSHi);
1408  else
1409  std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1410 
1411 
1412  Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
1413  Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
1414 }
1415 
1416 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1417  SDValue &Hi) {
1418  SDValue LHSLo, LHSHi;
1419  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1420  SDLoc dl(N);
1421 
1422  EVT LoVT, HiVT;
1423  std::tie(LoVT, HiVT) =
1424  DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1425 
1426  Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1427  DAG.getValueType(LoVT));
1428  Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1429  DAG.getValueType(HiVT));
1430 }
1431 
1432 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1433  SDValue &Hi) {
1434  unsigned Opcode = N->getOpcode();
1435  SDValue N0 = N->getOperand(0);
1436 
1437  SDLoc dl(N);
1438  SDValue InLo, InHi;
1439 
1440  if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1441  GetSplitVector(N0, InLo, InHi);
1442  else
1443  std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1444 
1445  EVT InLoVT = InLo.getValueType();
1446  unsigned InNumElements = InLoVT.getVectorNumElements();
1447 
1448  EVT OutLoVT, OutHiVT;
1449  std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1450  unsigned OutNumElements = OutLoVT.getVectorNumElements();
1451  assert((2 * OutNumElements) <= InNumElements &&
1452  "Illegal extend vector in reg split");
1453 
1454  // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1455  // input vector (i.e. we only use InLo):
1456  // OutLo will extend the first OutNumElements from InLo.
1457  // OutHi will extend the next OutNumElements from InLo.
1458 
1459  // Shuffle the elements from InLo for OutHi into the bottom elements to
1460  // create a 'fake' InHi.
1461  SmallVector<int, 8> SplitHi(InNumElements, -1);
1462  for (unsigned i = 0; i != OutNumElements; ++i)
1463  SplitHi[i] = i + OutNumElements;
1464  InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1465 
1466  Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1467  Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1468 }
1469 
1470 void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1471  SDValue &Hi) {
1472  unsigned NumOps = N->getNumOperands();
1473  SDValue Chain = N->getOperand(0);
1474  EVT LoVT, HiVT;
1475  SDLoc dl(N);
1476  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1477 
1478  SmallVector<SDValue, 4> OpsLo(NumOps);
1479  SmallVector<SDValue, 4> OpsHi(NumOps);
1480 
1481  // The Chain is the first operand.
1482  OpsLo[0] = Chain;
1483  OpsHi[0] = Chain;
1484 
1485  // Now process the remaining operands.
1486  for (unsigned i = 1; i < NumOps; ++i) {
1487  SDValue Op = N->getOperand(i);
1488  SDValue OpLo = Op;
1489  SDValue OpHi = Op;
1490 
1491  EVT InVT = Op.getValueType();
1492  if (InVT.isVector()) {
1493  // If the input also splits, handle it directly for a
1494  // compile time speedup. Otherwise split it by hand.
1495  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1496  GetSplitVector(Op, OpLo, OpHi);
1497  else
1498  std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1499  }
1500 
1501  OpsLo[i] = OpLo;
1502  OpsHi[i] = OpHi;
1503  }
1504 
1505  EVT LoValueVTs[] = {LoVT, MVT::Other};
1506  EVT HiValueVTs[] = {HiVT, MVT::Other};
1507  Lo = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo,
1508  N->getFlags());
1509  Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi,
1510  N->getFlags());
1511 
1512  // Build a factor node to remember that this Op is independent of the
1513  // other one.
1514  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1515  Lo.getValue(1), Hi.getValue(1));
1516 
1517  // Legalize the chain result - switch anything that used the old chain to
1518  // use the new one.
1519  ReplaceValueWith(SDValue(N, 1), Chain);
1520 }
1521 
1522 SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1523  SDValue Chain = N->getOperand(0);
1524  EVT VT = N->getValueType(0);
1525  unsigned NE = VT.getVectorNumElements();
1526  EVT EltVT = VT.getVectorElementType();
1527  SDLoc dl(N);
1528 
1529  SmallVector<SDValue, 8> Scalars;
1530  SmallVector<SDValue, 4> Operands(N->getNumOperands());
1531 
1532  // If ResNE is 0, fully unroll the vector op.
1533  if (ResNE == 0)
1534  ResNE = NE;
1535  else if (NE > ResNE)
1536  NE = ResNE;
1537 
1538  //The results of each unrolled operation, including the chain.
1539  EVT ChainVTs[] = {EltVT, MVT::Other};
1540  SmallVector<SDValue, 8> Chains;
1541 
1542  unsigned i;
1543  for (i = 0; i != NE; ++i) {
1544  Operands[0] = Chain;
1545  for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
1546  SDValue Operand = N->getOperand(j);
1547  EVT OperandVT = Operand.getValueType();
1548  if (OperandVT.isVector()) {
1549  EVT OperandEltVT = OperandVT.getVectorElementType();
1550  Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
1551  Operand, DAG.getVectorIdxConstant(i, dl));
1552  } else {
1553  Operands[j] = Operand;
1554  }
1555  }
1556  SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1557  Scalar.getNode()->setFlags(N->getFlags());
1558 
1559  //Add in the scalar as well as its chain value to the
1560  //result vectors.
1561  Scalars.push_back(Scalar);
1562  Chains.push_back(Scalar.getValue(1));
1563  }
1564 
1565  for (; i < ResNE; ++i)
1566  Scalars.push_back(DAG.getUNDEF(EltVT));
1567 
1568  // Build a new factor node to connect the chain back together.
1569  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1570  ReplaceValueWith(SDValue(N, 1), Chain);
1571 
1572  // Create a new BUILD_VECTOR node
1573  EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE);
1574  return DAG.getBuildVector(VecVT, dl, Scalars);
1575 }
1576 
1577 void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1578  SDValue &Lo, SDValue &Hi) {
1579  SDLoc dl(N);
1580  EVT ResVT = N->getValueType(0);
1581  EVT OvVT = N->getValueType(1);
1582  EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1583  std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
1584  std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
1585 
1586  SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1587  if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
1588  GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
1589  GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
1590  } else {
1591  std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
1592  std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
1593  }
1594 
1595  unsigned Opcode = N->getOpcode();
1596  SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
1597  SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
1598  SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
1599  SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
1600  LoNode->setFlags(N->getFlags());
1601  HiNode->setFlags(N->getFlags());
1602 
1603  Lo = SDValue(LoNode, ResNo);
1604  Hi = SDValue(HiNode, ResNo);
1605 
1606  // Replace the other vector result not being explicitly split here.
1607  unsigned OtherNo = 1 - ResNo;
1608  EVT OtherVT = N->getValueType(OtherNo);
1609  if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
1610  SetSplitVector(SDValue(N, OtherNo),
1611  SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1612  } else {
1613  SDValue OtherVal = DAG.getNode(
1614  ISD::CONCAT_VECTORS, dl, OtherVT,
1615  SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1616  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
1617  }
1618 }
1619 
1620 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1621  SDValue &Hi) {
1622  SDValue Vec = N->getOperand(0);
1623  SDValue Elt = N->getOperand(1);
1624  SDValue Idx = N->getOperand(2);
1625  SDLoc dl(N);
1626  GetSplitVector(Vec, Lo, Hi);
1627 
1628  if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1629  unsigned IdxVal = CIdx->getZExtValue();
1630  unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
1631  if (IdxVal < LoNumElts) {
1632  Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1633  Lo.getValueType(), Lo, Elt, Idx);
1634  return;
1635  } else if (!Vec.getValueType().isScalableVector()) {
1636  Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1637  DAG.getVectorIdxConstant(IdxVal - LoNumElts, dl));
1638  return;
1639  }
1640  }
1641 
1642  // See if the target wants to custom expand this node.
1643  if (CustomLowerNode(N, N->getValueType(0), true))
1644  return;
1645 
1646  // Make the vector elements byte-addressable if they aren't already.
1647  EVT VecVT = Vec.getValueType();
1648  EVT EltVT = VecVT.getVectorElementType();
1649  if (VecVT.getScalarSizeInBits() < 8) {
1650  EltVT = MVT::i8;
1651  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1652  VecVT.getVectorElementCount());
1653  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1654  // Extend the element type to match if needed.
1655  if (EltVT.bitsGT(Elt.getValueType()))
1656  Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1657  }
1658 
1659  // Spill the vector to the stack.
1660  // In cases where the vector is illegal it will be broken down into parts
1661  // and stored in parts - we should use the alignment for the smallest part.
1662  Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1663  SDValue StackPtr =
1664  DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1665  auto &MF = DAG.getMachineFunction();
1666  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1667  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1668 
1669  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1670  SmallestAlign);
1671 
1672  // Store the new element. This may be larger than the vector element type,
1673  // so use a truncating store.
1674  SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1675  Store = DAG.getTruncStore(
1676  Store, dl, Elt, EltPtr, MachinePointerInfo::getUnknownStack(MF), EltVT,
1677  commonAlignment(SmallestAlign,
1678  EltVT.getFixedSizeInBits() / 8));
1679 
1680  EVT LoVT, HiVT;
1681  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1682 
1683  // Load the Lo part from the stack slot.
1684  Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo, SmallestAlign);
1685 
1686  // Increment the pointer to the other part.
1687  auto Load = cast<LoadSDNode>(Lo);
1688  MachinePointerInfo MPI = Load->getPointerInfo();
1689  IncrementPointer(Load, LoVT, MPI, StackPtr);
1690 
1691  Hi = DAG.getLoad(HiVT, dl, Store, StackPtr, MPI, SmallestAlign);
1692 
1693  // If we adjusted the original type, we need to truncate the results.
1694  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1695  if (LoVT != Lo.getValueType())
1696  Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1697  if (HiVT != Hi.getValueType())
1698  Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1699 }
1700 
1701 void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
1702  SDValue &Hi) {
1703  EVT LoVT, HiVT;
1704  SDLoc dl(N);
1705  assert(N->getValueType(0).isScalableVector() &&
1706  "Only scalable vectors are supported for STEP_VECTOR");
1707  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1708  SDValue Step = N->getOperand(0);
1709 
1710  Lo = DAG.getNode(ISD::STEP_VECTOR, dl, LoVT, Step);
1711 
1712  // Hi = Lo + (EltCnt * Step)
1713  EVT EltVT = Step.getValueType();
1714  APInt StepVal = cast<ConstantSDNode>(Step)->getAPIntValue();
1715  SDValue StartOfHi =
1716  DAG.getVScale(dl, EltVT, StepVal * LoVT.getVectorMinNumElements());
1717  StartOfHi = DAG.getSExtOrTrunc(StartOfHi, dl, HiVT.getVectorElementType());
1718  StartOfHi = DAG.getNode(ISD::SPLAT_VECTOR, dl, HiVT, StartOfHi);
1719 
1720  Hi = DAG.getNode(ISD::STEP_VECTOR, dl, HiVT, Step);
1721  Hi = DAG.getNode(ISD::ADD, dl, HiVT, Hi, StartOfHi);
1722 }
1723 
1724 void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
1725  SDValue &Hi) {
1726  EVT LoVT, HiVT;
1727  SDLoc dl(N);
1728  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1729  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, N->getOperand(0));
1730  if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
1731  Hi = DAG.getUNDEF(HiVT);
1732  } else {
1733  assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
1734  Hi = Lo;
1735  }
1736 }
1737 
1738 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1739  SDValue &Hi) {
1740  assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1741  EVT LoVT, HiVT;
1742  SDLoc dl(LD);
1743  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1744 
1745  ISD::LoadExtType ExtType = LD->getExtensionType();
1746  SDValue Ch = LD->getChain();
1747  SDValue Ptr = LD->getBasePtr();
1748  SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1749  EVT MemoryVT = LD->getMemoryVT();
1750  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1751  AAMDNodes AAInfo = LD->getAAInfo();
1752 
1753  EVT LoMemVT, HiMemVT;
1754  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1755 
1756  if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
1757  SDValue Value, NewChain;
1758  std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
1759  std::tie(Lo, Hi) = DAG.SplitVector(Value, dl);
1760  ReplaceValueWith(SDValue(LD, 1), NewChain);
1761  return;
1762  }
1763 
1764  Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1765  LD->getPointerInfo(), LoMemVT, LD->getOriginalAlign(),
1766  MMOFlags, AAInfo);
1767 
1768  MachinePointerInfo MPI;
1769  IncrementPointer(LD, LoMemVT, MPI, Ptr);
1770 
1771  Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, MPI,
1772  HiMemVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
1773 
1774  // Build a factor node to remember that this load is independent of the
1775  // other one.
1776  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1777  Hi.getValue(1));
1778 
1779  // Legalize the chain result - switch anything that used the old chain to
1780  // use the new one.
1781  ReplaceValueWith(SDValue(LD, 1), Ch);
1782 }
1783 
1784 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
1785  SDValue &Lo, SDValue &Hi) {
1786  assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
1787  EVT LoVT, HiVT;
1788  SDLoc dl(MLD);
1789  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1790 
1791  SDValue Ch = MLD->getChain();
1792  SDValue Ptr = MLD->getBasePtr();
1793  SDValue Offset = MLD->getOffset();
1794  assert(Offset.isUndef() && "Unexpected indexed masked load offset");
1795  SDValue Mask = MLD->getMask();
1796  SDValue PassThru = MLD->getPassThru();
1797  Align Alignment = MLD->getOriginalAlign();
1798  ISD::LoadExtType ExtType = MLD->getExtensionType();
1799 
1800  // Split Mask operand
1801  SDValue MaskLo, MaskHi;
1802  if (Mask.getOpcode() == ISD::SETCC) {
1803  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1804  } else {
1805  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1806  GetSplitVector(Mask, MaskLo, MaskHi);
1807  else
1808  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1809  }
1810 
1811  EVT MemoryVT = MLD->getMemoryVT();
1812  EVT LoMemVT, HiMemVT;
1813  bool HiIsEmpty = false;
1814  std::tie(LoMemVT, HiMemVT) =
1815  DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
1816 
1817  SDValue PassThruLo, PassThruHi;
1818  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1819  GetSplitVector(PassThru, PassThruLo, PassThruHi);
1820  else
1821  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1822 
1823  unsigned LoSize = MemoryLocation::getSizeOrUnknown(LoMemVT.getStoreSize());
1825  MLD->getPointerInfo(), MachineMemOperand::MOLoad, LoSize, Alignment,
1826  MLD->getAAInfo(), MLD->getRanges());
1827 
1828  Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
1829  MMO, MLD->getAddressingMode(), ExtType,
1830  MLD->isExpandingLoad());
1831 
1832  if (HiIsEmpty) {
1833  // The hi masked load has zero storage size. We therefore simply set it to
1834  // the low masked load and rely on subsequent removal from the chain.
1835  Hi = Lo;
1836  } else {
1837  // Generate hi masked load.
1838  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1839  MLD->isExpandingLoad());
1840  unsigned HiSize = MemoryLocation::getSizeOrUnknown(HiMemVT.getStoreSize());
1841 
1842  MachinePointerInfo MPI;
1843  if (LoMemVT.isScalableVector())
1845  else
1846  MPI = MLD->getPointerInfo().getWithOffset(
1847  LoMemVT.getStoreSize().getFixedSize());
1848 
1850  MPI, MachineMemOperand::MOLoad, HiSize, Alignment, MLD->getAAInfo(),
1851  MLD->getRanges());
1852 
1853  Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
1854  HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
1855  MLD->isExpandingLoad());
1856  }
1857 
1858  // Build a factor node to remember that this load is independent of the
1859  // other one.
1860  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1861  Hi.getValue(1));
1862 
1863  // Legalize the chain result - switch anything that used the old chain to
1864  // use the new one.
1865  ReplaceValueWith(SDValue(MLD, 1), Ch);
1866 
1867 }
1868 
1869 void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
1870  SDValue &Lo, SDValue &Hi) {
1871  EVT LoVT, HiVT;
1872  SDLoc dl(MGT);
1873  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1874 
1875  SDValue Ch = MGT->getChain();
1876  SDValue Ptr = MGT->getBasePtr();
1877  SDValue Mask = MGT->getMask();
1878  SDValue PassThru = MGT->getPassThru();
1879  SDValue Index = MGT->getIndex();
1880  SDValue Scale = MGT->getScale();
1881  EVT MemoryVT = MGT->getMemoryVT();
1882  Align Alignment = MGT->getOriginalAlign();
1883  ISD::LoadExtType ExtType = MGT->getExtensionType();
1884 
1885  // Split Mask operand
1886  SDValue MaskLo, MaskHi;
1887  if (Mask.getOpcode() == ISD::SETCC) {
1888  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1889  } else {
1890  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1891  GetSplitVector(Mask, MaskLo, MaskHi);
1892  else
1893  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1894  }
1895 
1896  EVT LoMemVT, HiMemVT;
1897  // Split MemoryVT
1898  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1899 
1900  SDValue PassThruLo, PassThruHi;
1901  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1902  GetSplitVector(PassThru, PassThruLo, PassThruHi);
1903  else
1904  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1905 
1906  SDValue IndexHi, IndexLo;
1907  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1908  GetSplitVector(Index, IndexLo, IndexHi);
1909  else
1910  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1911 
1914  MemoryLocation::UnknownSize, Alignment, MGT->getAAInfo(),
1915  MGT->getRanges());
1916 
1917  SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
1918  Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, OpsLo,
1919  MMO, MGT->getIndexType(), ExtType);
1920 
1921  SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
1922  Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, OpsHi,
1923  MMO, MGT->getIndexType(), ExtType);
1924 
1925  // Build a factor node to remember that this load is independent of the
1926  // other one.
1927  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1928  Hi.getValue(1));
1929 
1930  // Legalize the chain result - switch anything that used the old chain to
1931  // use the new one.
1932  ReplaceValueWith(SDValue(MGT, 1), Ch);
1933 }
1934 
1935 
1936 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1937  assert(N->getValueType(0).isVector() &&
1938  N->getOperand(0).getValueType().isVector() &&
1939  "Operand types must be vectors");
1940 
1941  EVT LoVT, HiVT;
1942  SDLoc DL(N);
1943  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1944 
1945  // If the input also splits, handle it directly. Otherwise split it by hand.
1946  SDValue LL, LH, RL, RH;
1947  if (getTypeAction(N->getOperand(0).getValueType()) ==
1949  GetSplitVector(N->getOperand(0), LL, LH);
1950  else
1951  std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
1952 
1953  if (getTypeAction(N->getOperand(1).getValueType()) ==
1955  GetSplitVector(N->getOperand(1), RL, RH);
1956  else
1957  std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
1958 
1959  Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
1960  Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
1961 }
1962 
1963 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
1964  SDValue &Hi) {
1965  // Get the dest types - they may not match the input types, e.g. int_to_fp.
1966  EVT LoVT, HiVT;
1967  SDLoc dl(N);
1968  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1969 
1970  // If the input also splits, handle it directly for a compile time speedup.
1971  // Otherwise split it by hand.
1972  unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1973  EVT InVT = N->getOperand(OpNo).getValueType();
1974  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1975  GetSplitVector(N->getOperand(OpNo), Lo, Hi);
1976  else
1977  std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, OpNo);
1978 
1979  if (N->getOpcode() == ISD::FP_ROUND) {
1980  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1),
1981  N->getFlags());
1982  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1),
1983  N->getFlags());
1984  } else {
1985  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getFlags());
1986  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getFlags());
1987  }
1988 }
1989 
1990 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
1991  SDValue &Hi) {
1992  SDLoc dl(N);
1993  EVT SrcVT = N->getOperand(0).getValueType();
1994  EVT DestVT = N->getValueType(0);
1995  EVT LoVT, HiVT;
1996  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
1997 
1998  // We can do better than a generic split operation if the extend is doing
1999  // more than just doubling the width of the elements and the following are
2000  // true:
2001  // - The number of vector elements is even,
2002  // - the source type is legal,
2003  // - the type of a split source is illegal,
2004  // - the type of an extended (by doubling element size) source is legal, and
2005  // - the type of that extended source when split is legal.
2006  //
2007  // This won't necessarily completely legalize the operation, but it will
2008  // more effectively move in the right direction and prevent falling down
2009  // to scalarization in many cases due to the input vector being split too
2010  // far.
2011  if (SrcVT.getVectorElementCount().isKnownEven() &&
2012  SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
2013  LLVMContext &Ctx = *DAG.getContext();
2014  EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
2015  EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
2016 
2017  EVT SplitLoVT, SplitHiVT;
2018  std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
2019  if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
2020  TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
2021  LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
2022  N->dump(&DAG); dbgs() << "\n");
2023  // Extend the source vector by one step.
2024  SDValue NewSrc =
2025  DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
2026  // Get the low and high halves of the new, extended one step, vector.
2027  std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
2028  // Extend those vector halves the rest of the way.
2029  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
2030  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
2031  return;
2032  }
2033  }
2034  // Fall back to the generic unary operator splitting otherwise.
2035  SplitVecRes_UnaryOp(N, Lo, Hi);
2036 }
2037 
2038 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
2039  SDValue &Lo, SDValue &Hi) {
2040  // The low and high parts of the original input give four input vectors.
2041  SDValue Inputs[4];
2042  SDLoc dl(N);
2043  GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
2044  GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
2045  EVT NewVT = Inputs[0].getValueType();
2046  unsigned NewElts = NewVT.getVectorNumElements();
2047 
2048  // If Lo or Hi uses elements from at most two of the four input vectors, then
2049  // express it as a vector shuffle of those two inputs. Otherwise extract the
2050  // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
2052  for (unsigned High = 0; High < 2; ++High) {
2053  SDValue &Output = High ? Hi : Lo;
2054 
2055  // Build a shuffle mask for the output, discovering on the fly which
2056  // input vectors to use as shuffle operands (recorded in InputUsed).
2057  // If building a suitable shuffle vector proves too hard, then bail
2058  // out with useBuildVector set.
2059  unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
2060  unsigned FirstMaskIdx = High * NewElts;
2061  bool useBuildVector = false;
2062  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
2063  // The mask element. This indexes into the input.
2064  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
2065 
2066  // The input vector this mask element indexes into.
2067  unsigned Input = (unsigned)Idx / NewElts;
2068 
2069  if (Input >= array_lengthof(Inputs)) {
2070  // The mask element does not index into any input vector.
2071  Ops.push_back(-1);
2072  continue;
2073  }
2074 
2075  // Turn the index into an offset from the start of the input vector.
2076  Idx -= Input * NewElts;
2077 
2078  // Find or create a shuffle vector operand to hold this input.
2079  unsigned OpNo;
2080  for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
2081  if (InputUsed[OpNo] == Input) {
2082  // This input vector is already an operand.
2083  break;
2084  } else if (InputUsed[OpNo] == -1U) {
2085  // Create a new operand for this input vector.
2086  InputUsed[OpNo] = Input;
2087  break;
2088  }
2089  }
2090 
2091  if (OpNo >= array_lengthof(InputUsed)) {
2092  // More than two input vectors used! Give up on trying to create a
2093  // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
2094  useBuildVector = true;
2095  break;
2096  }
2097 
2098  // Add the mask index for the new shuffle vector.
2099  Ops.push_back(Idx + OpNo * NewElts);
2100  }
2101 
2102  if (useBuildVector) {
2103  EVT EltVT = NewVT.getVectorElementType();
2105 
2106  // Extract the input elements by hand.
2107  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
2108  // The mask element. This indexes into the input.
2109  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
2110 
2111  // The input vector this mask element indexes into.
2112  unsigned Input = (unsigned)Idx / NewElts;
2113 
2114  if (Input >= array_lengthof(Inputs)) {
2115  // The mask element is "undef" or indexes off the end of the input.
2116  SVOps.push_back(DAG.getUNDEF(EltVT));
2117  continue;
2118  }
2119 
2120  // Turn the index into an offset from the start of the input vector.
2121  Idx -= Input * NewElts;
2122 
2123  // Extract the vector element by hand.
2124  SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
2125  Inputs[Input],
2126  DAG.getVectorIdxConstant(Idx, dl)));
2127  }
2128 
2129  // Construct the Lo/Hi output using a BUILD_VECTOR.
2130  Output = DAG.getBuildVector(NewVT, dl, SVOps);
2131  } else if (InputUsed[0] == -1U) {
2132  // No input vectors were used! The result is undefined.
2133  Output = DAG.getUNDEF(NewVT);
2134  } else {
2135  SDValue Op0 = Inputs[InputUsed[0]];
2136  // If only one input was used, use an undefined vector for the other.
2137  SDValue Op1 = InputUsed[1] == -1U ?
2138  DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
2139  // At least one input vector was used. Create a new shuffle vector.
2140  Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
2141  }
2142 
2143  Ops.clear();
2144  }
2145 }
2146 
2147 void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2148  EVT OVT = N->getValueType(0);
2149  EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext());
2150  SDValue Chain = N->getOperand(0);
2151  SDValue Ptr = N->getOperand(1);
2152  SDValue SV = N->getOperand(2);
2153  SDLoc dl(N);
2154 
2155  const Align Alignment =
2157 
2158  Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment.value());
2159  Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment.value());
2160  Chain = Hi.getValue(1);
2161 
2162  // Modified the chain - switch anything that used the old chain to use
2163  // the new one.
2164  ReplaceValueWith(SDValue(N, 1), Chain);
2165 }
2166 
2167 void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
2168  SDValue &Hi) {
2169  EVT DstVTLo, DstVTHi;
2170  std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(N->getValueType(0));
2171  SDLoc dl(N);
2172 
2173  SDValue SrcLo, SrcHi;
2174  EVT SrcVT = N->getOperand(0).getValueType();
2175  if (getTypeAction(SrcVT) == TargetLowering::TypeSplitVector)
2176  GetSplitVector(N->getOperand(0), SrcLo, SrcHi);
2177  else
2178  std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(N, 0);
2179 
2180  Lo = DAG.getNode(N->getOpcode(), dl, DstVTLo, SrcLo, N->getOperand(1));
2181  Hi = DAG.getNode(N->getOpcode(), dl, DstVTHi, SrcHi, N->getOperand(1));
2182 }
2183 
2184 //===----------------------------------------------------------------------===//
2185 // Operand Vector Splitting
2186 //===----------------------------------------------------------------------===//
2187 
2188 /// This method is called when the specified operand of the specified node is
2189 /// found to need vector splitting. At this point, all of the result types of
2190 /// the node are known to be legal, but other operands of the node may need
2191 /// legalization as well as the specified one.
2192 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
2193  LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
2194  SDValue Res = SDValue();
2195 
2196  // See if the target wants to custom split this node.
2197  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2198  return false;
2199 
2200  switch (N->getOpcode()) {
2201  default:
2202 #ifndef NDEBUG
2203  dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
2204  N->dump(&DAG);
2205  dbgs() << "\n";
2206 #endif
2207  report_fatal_error("Do not know how to split this operator's "
2208  "operand!\n");
2209 
2210  case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
2211  case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
2212  case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
2213  case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
2214  case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
2215  case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
2216  case ISD::TRUNCATE:
2217  Res = SplitVecOp_TruncateHelper(N);
2218  break;
2219  case ISD::STRICT_FP_ROUND:
2220  case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
2221  case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
2222  case ISD::STORE:
2223  Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
2224  break;
2225  case ISD::MSTORE:
2226  Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
2227  break;
2228  case ISD::MSCATTER:
2229  Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
2230  break;
2231  case ISD::MGATHER:
2232  Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
2233  break;
2234  case ISD::VSELECT:
2235  Res = SplitVecOp_VSELECT(N, OpNo);
2236  break;
2239  case ISD::SINT_TO_FP:
2240  case ISD::UINT_TO_FP:
2241  if (N->getValueType(0).bitsLT(
2242  N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType()))
2243  Res = SplitVecOp_TruncateHelper(N);
2244  else
2245  Res = SplitVecOp_UnaryOp(N);
2246  break;
2247  case ISD::FP_TO_SINT_SAT:
2248  case ISD::FP_TO_UINT_SAT:
2249  Res = SplitVecOp_FP_TO_XINT_SAT(N);
2250  break;
2251  case ISD::FP_TO_SINT:
2252  case ISD::FP_TO_UINT:
2255  case ISD::STRICT_FP_EXTEND:
2256  case ISD::FP_EXTEND:
2257  case ISD::SIGN_EXTEND:
2258  case ISD::ZERO_EXTEND:
2259  case ISD::ANY_EXTEND:
2260  case ISD::FTRUNC:
2261  Res = SplitVecOp_UnaryOp(N);
2262  break;
2263 
2267  Res = SplitVecOp_ExtVecInRegOp(N);
2268  break;
2269 
2270  case ISD::VECREDUCE_FADD:
2271  case ISD::VECREDUCE_FMUL:
2272  case ISD::VECREDUCE_ADD:
2273  case ISD::VECREDUCE_MUL:
2274  case ISD::VECREDUCE_AND:
2275  case ISD::VECREDUCE_OR:
2276  case ISD::VECREDUCE_XOR:
2277  case ISD::VECREDUCE_SMAX:
2278  case ISD::VECREDUCE_SMIN:
2279  case ISD::VECREDUCE_UMAX:
2280  case ISD::VECREDUCE_UMIN:
2281  case ISD::VECREDUCE_FMAX:
2282  case ISD::VECREDUCE_FMIN:
2283  Res = SplitVecOp_VECREDUCE(N, OpNo);
2284  break;
2287  Res = SplitVecOp_VECREDUCE_SEQ(N);
2288  break;
2289  }
2290 
2291  // If the result is null, the sub-method took care of registering results etc.
2292  if (!Res.getNode()) return false;
2293 
2294  // If the result is N, the sub-method updated N in place. Tell the legalizer
2295  // core about this.
2296  if (Res.getNode() == N)
2297  return true;
2298 
2299  if (N->isStrictFPOpcode())
2300  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
2301  "Invalid operand expansion");
2302  else
2303  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2304  "Invalid operand expansion");
2305 
2306  ReplaceValueWith(SDValue(N, 0), Res);
2307  return false;
2308 }
2309 
2310 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
2311  // The only possibility for an illegal operand is the mask, since result type
2312  // legalization would have handled this node already otherwise.
2313  assert(OpNo == 0 && "Illegal operand must be mask");
2314 
2315  SDValue Mask = N->getOperand(0);
2316  SDValue Src0 = N->getOperand(1);
2317  SDValue Src1 = N->getOperand(2);
2318  EVT Src0VT = Src0.getValueType();
2319  SDLoc DL(N);
2320  assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
2321 
2322  SDValue Lo, Hi;
2323  GetSplitVector(N->getOperand(0), Lo, Hi);
2324  assert(Lo.getValueType() == Hi.getValueType() &&
2325  "Lo and Hi have differing types");
2326 
2327  EVT LoOpVT, HiOpVT;
2328  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
2329  assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
2330 
2331  SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
2332  std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
2333  std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
2334  std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
2335 
2336  SDValue LoSelect =
2337  DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
2338  SDValue HiSelect =
2339  DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
2340 
2341  return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
2342 }
2343 
2344 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
2345  EVT ResVT = N->getValueType(0);
2346  SDValue Lo, Hi;
2347  SDLoc dl(N);
2348 
2349  SDValue VecOp = N->getOperand(OpNo);
2350  EVT VecVT = VecOp.getValueType();
2351  assert(VecVT.isVector() && "Can only split reduce vector operand");
2352  GetSplitVector(VecOp, Lo, Hi);
2353  EVT LoOpVT, HiOpVT;
2354  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2355 
2356  // Use the appropriate scalar instruction on the split subvectors before
2357  // reducing the now partially reduced smaller vector.
2358  unsigned CombineOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
2359  SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
2360  return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
2361 }
2362 
2363 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
2364  EVT ResVT = N->getValueType(0);
2365  SDValue Lo, Hi;
2366  SDLoc dl(N);
2367 
2368  SDValue AccOp = N->getOperand(0);
2369  SDValue VecOp = N->getOperand(1);
2370  SDNodeFlags Flags = N->getFlags();
2371 
2372  EVT VecVT = VecOp.getValueType();
2373  assert(VecVT.isVector() && "Can only split reduce vector operand");
2374  GetSplitVector(VecOp, Lo, Hi);
2375  EVT LoOpVT, HiOpVT;
2376  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2377 
2378  // Reduce low half.
2379  SDValue Partial = DAG.getNode(N->getOpcode(), dl, ResVT, AccOp, Lo, Flags);
2380 
2381  // Reduce high half, using low half result as initial value.
2382  return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, Hi, Flags);
2383 }
2384 
2385 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
2386  // The result has a legal vector type, but the input needs splitting.
2387  EVT ResVT = N->getValueType(0);
2388  SDValue Lo, Hi;
2389  SDLoc dl(N);
2390  GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
2391  EVT InVT = Lo.getValueType();
2392 
2393  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2394  InVT.getVectorElementCount());
2395 
2396  if (N->isStrictFPOpcode()) {
2397  Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2398  { N->getOperand(0), Lo });
2399  Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2400  { N->getOperand(0), Hi });
2401 
2402  // Build a factor node to remember that this operation is independent
2403  // of the other one.
2404  SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2405  Hi.getValue(1));
2406 
2407  // Legalize the chain result - switch anything that used the old chain to
2408  // use the new one.
2409  ReplaceValueWith(SDValue(N, 1), Ch);
2410  } else {
2411  Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
2412  Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
2413  }
2414 
2415  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
2416 }
2417 
2418 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
2419  // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
2420  // end up being split all the way down to individual components. Convert the
2421  // split pieces into integers and reassemble.
2422  SDValue Lo, Hi;
2423  GetSplitVector(N->getOperand(0), Lo, Hi);
2424  Lo = BitConvertToInteger(Lo);
2425  Hi = BitConvertToInteger(Hi);
2426 
2427  if (DAG.getDataLayout().isBigEndian())
2428  std::swap(Lo, Hi);
2429 
2430  return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
2431  JoinIntegers(Lo, Hi));
2432 }
2433 
2434 SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
2435  unsigned OpNo) {
2436  assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
2437  // We know that the result type is legal.
2438  EVT ResVT = N->getValueType(0);
2439 
2440  SDValue Vec = N->getOperand(0);
2441  SDValue SubVec = N->getOperand(1);
2442  SDValue Idx = N->getOperand(2);
2443  SDLoc dl(N);
2444 
2445  SDValue Lo, Hi;
2446  GetSplitVector(SubVec, Lo, Hi);
2447 
2448  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2449  uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2450 
2451  SDValue FirstInsertion =
2452  DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Lo, Idx);
2453  SDValue SecondInsertion =
2454  DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, FirstInsertion, Hi,
2455  DAG.getVectorIdxConstant(IdxVal + LoElts, dl));
2456 
2457  return SecondInsertion;
2458 }
2459 
2460 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
2461  // We know that the extracted result type is legal.
2462  EVT SubVT = N->getValueType(0);
2463 
2464  SDValue Idx = N->getOperand(1);
2465  SDLoc dl(N);
2466  SDValue Lo, Hi;
2467 
2468  if (SubVT.isScalableVector() !=
2469  N->getOperand(0).getValueType().isScalableVector())
2470  report_fatal_error("Extracting a fixed-length vector from an illegal "
2471  "scalable vector is not yet supported");
2472 
2473  GetSplitVector(N->getOperand(0), Lo, Hi);
2474 
2475  uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2476  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2477 
2478  if (IdxVal < LoElts) {
2479  assert(IdxVal + SubVT.getVectorMinNumElements() <= LoElts &&
2480  "Extracted subvector crosses vector split!");
2481  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
2482  } else {
2483  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
2484  DAG.getVectorIdxConstant(IdxVal - LoElts, dl));
2485  }
2486 }
2487 
2488 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
2489  SDValue Vec = N->getOperand(0);
2490  SDValue Idx = N->getOperand(1);
2491  EVT VecVT = Vec.getValueType();
2492 
2493  if (isa<ConstantSDNode>(Idx)) {
2494  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2495 
2496  SDValue Lo, Hi;
2497  GetSplitVector(Vec, Lo, Hi);
2498 
2499  uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2500 
2501  if (IdxVal < LoElts)
2502  return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
2503  else if (!Vec.getValueType().isScalableVector())
2504  return SDValue(DAG.UpdateNodeOperands(N, Hi,
2505  DAG.getConstant(IdxVal - LoElts, SDLoc(N),
2506  Idx.getValueType())), 0);
2507  }
2508 
2509  // See if the target wants to custom expand this node.
2510  if (CustomLowerNode(N, N->getValueType(0), true))
2511  return SDValue();
2512 
2513  // Make the vector elements byte-addressable if they aren't already.
2514  SDLoc dl(N);
2515  EVT EltVT = VecVT.getVectorElementType();
2516  if (VecVT.getScalarSizeInBits() < 8) {
2517  EltVT = MVT::i8;
2518  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
2519  VecVT.getVectorElementCount());
2520  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
2521  }
2522 
2523  // Store the vector to the stack.
2524  // In cases where the vector is illegal it will be broken down into parts
2525  // and stored in parts - we should use the alignment for the smallest part.
2526  Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
2527  SDValue StackPtr =
2528  DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
2529  auto &MF = DAG.getMachineFunction();
2530  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
2531  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
2532  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
2533  SmallestAlign);
2534 
2535  // Load back the required element.
2536  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
2537 
2538  // FIXME: This is to handle i1 vectors with elements promoted to i8.
2539  // i1 vector handling needs general improvement.
2540  if (N->getValueType(0).bitsLT(EltVT)) {
2541  SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
2543  return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
2544  }
2545 
2546  return DAG.getExtLoad(
2547  ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
2549  commonAlignment(SmallestAlign, EltVT.getFixedSizeInBits() / 8));
2550 }
2551 
2552 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
2553  SDValue Lo, Hi;
2554 
2555  // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
2556  // splitting the result has the same effect as splitting the input operand.
2557  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
2558 
2559  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
2560 }
2561 
2562 SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
2563  unsigned OpNo) {
2564  EVT LoVT, HiVT;
2565  SDLoc dl(MGT);
2566  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
2567 
2568  SDValue Ch = MGT->getChain();
2569  SDValue Ptr = MGT->getBasePtr();
2570  SDValue Index = MGT->getIndex();
2571  SDValue Scale = MGT->getScale();
2572  SDValue Mask = MGT->getMask();
2573  SDValue PassThru = MGT->getPassThru();
2574  Align Alignment = MGT->getOriginalAlign();
2575  ISD::LoadExtType ExtType = MGT->getExtensionType();
2576 
2577  SDValue MaskLo, MaskHi;
2578  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2579  // Split Mask operand
2580  GetSplitVector(Mask, MaskLo, MaskHi);
2581  else
2582  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2583 
2584  EVT MemoryVT = MGT->getMemoryVT();
2585  EVT LoMemVT, HiMemVT;
2586  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2587 
2588  SDValue PassThruLo, PassThruHi;
2589  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2590  GetSplitVector(PassThru, PassThruLo, PassThruHi);
2591  else
2592  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2593 
2594  SDValue IndexHi, IndexLo;
2595  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2596  GetSplitVector(Index, IndexLo, IndexHi);
2597  else
2598  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
2599 
2602  MemoryLocation::UnknownSize, Alignment, MGT->getAAInfo(),
2603  MGT->getRanges());
2604 
2605  SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
2606  SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl,
2607  OpsLo, MMO, MGT->getIndexType(), ExtType);
2608 
2609  SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
2610  SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl,
2611  OpsHi, MMO, MGT->getIndexType(), ExtType);
2612 
2613  // Build a factor node to remember that this load is independent of the
2614  // other one.
2615  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2616  Hi.getValue(1));
2617 
2618  // Legalize the chain result - switch anything that used the old chain to
2619  // use the new one.
2620  ReplaceValueWith(SDValue(MGT, 1), Ch);
2621 
2622  SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
2623  Hi);
2624  ReplaceValueWith(SDValue(MGT, 0), Res);
2625  return SDValue();
2626 }
2627 
2628 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
2629  unsigned OpNo) {
2630  assert(N->isUnindexed() && "Indexed masked store of vector?");
2631  SDValue Ch = N->getChain();
2632  SDValue Ptr = N->getBasePtr();
2633  SDValue Offset = N->getOffset();
2634  assert(Offset.isUndef() && "Unexpected indexed masked store offset");
2635  SDValue Mask = N->getMask();
2636  SDValue Data = N->getValue();
2637  Align Alignment = N->getOriginalAlign();
2638  SDLoc DL(N);
2639 
2640  SDValue DataLo, DataHi;
2641  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2642  // Split Data operand
2643  GetSplitVector(Data, DataLo, DataHi);
2644  else
2645  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2646 
2647  // Split Mask operand
2648  SDValue MaskLo, MaskHi;
2649  if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
2650  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2651  } else {
2652  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2653  GetSplitVector(Mask, MaskLo, MaskHi);
2654  else
2655  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2656  }
2657 
2658  EVT MemoryVT = N->getMemoryVT();
2659  EVT LoMemVT, HiMemVT;
2660  bool HiIsEmpty = false;
2661  std::tie(LoMemVT, HiMemVT) =
2662  DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
2663 
2664  SDValue Lo, Hi, Res;
2665  unsigned LoSize = MemoryLocation::getSizeOrUnknown(LoMemVT.getStoreSize());
2667  N->getPointerInfo(), MachineMemOperand::MOStore, LoSize, Alignment,
2668  N->getAAInfo(), N->getRanges());
2669 
2670  Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
2671  N->getAddressingMode(), N->isTruncatingStore(),
2672  N->isCompressingStore());
2673 
2674  if (HiIsEmpty) {
2675  // The hi masked store has zero storage size.
2676  // Only the lo masked store is needed.
2677  Res = Lo;
2678  } else {
2679 
2680  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
2681  N->isCompressingStore());
2682 
2683  MachinePointerInfo MPI;
2684  if (LoMemVT.isScalableVector()) {
2685  Alignment = commonAlignment(
2686  Alignment, LoMemVT.getSizeInBits().getKnownMinSize() / 8);
2687  MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
2688  } else
2689  MPI = N->getPointerInfo().getWithOffset(
2690  LoMemVT.getStoreSize().getFixedSize());
2691 
2692  unsigned HiSize = MemoryLocation::getSizeOrUnknown(HiMemVT.getStoreSize());
2694  MPI, MachineMemOperand::MOStore, HiSize, Alignment, N->getAAInfo(),
2695  N->getRanges());
2696 
2697  Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
2698  N->getAddressingMode(), N->isTruncatingStore(),
2699  N->isCompressingStore());
2700 
2701  // Build a factor node to remember that this store is independent of the
2702  // other one.
2703  Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2704  }
2705 
2706  return Res;
2707 }
2708 
2709 SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
2710  unsigned OpNo) {
2711  SDValue Ch = N->getChain();
2712  SDValue Ptr = N->getBasePtr();
2713  SDValue Mask = N->getMask();
2714  SDValue Index = N->getIndex();
2715  SDValue Scale = N->getScale();
2716  SDValue Data = N->getValue();
2717  EVT MemoryVT = N->getMemoryVT();
2718  Align Alignment = N->getOriginalAlign();
2719  SDLoc DL(N);
2720 
2721  // Split all operands
2722 
2723  EVT LoMemVT, HiMemVT;
2724  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2725 
2726  SDValue DataLo, DataHi;
2727  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2728  // Split Data operand
2729  GetSplitVector(Data, DataLo, DataHi);
2730  else
2731  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2732 
2733  // Split Mask operand
2734  SDValue MaskLo, MaskHi;
2735  if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
2736  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2737  } else {
2738  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2739  GetSplitVector(Mask, MaskLo, MaskHi);
2740  else
2741  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2742  }
2743 
2744  SDValue IndexHi, IndexLo;
2745  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2746  GetSplitVector(Index, IndexLo, IndexHi);
2747  else
2748  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
2749 
2750  SDValue Lo;
2752  N->getPointerInfo(), MachineMemOperand::MOStore,
2753  MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
2754 
2755  SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale};
2756  Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), LoMemVT,
2757  DL, OpsLo, MMO, N->getIndexType(),
2758  N->isTruncatingStore());
2759 
2760  // The order of the Scatter operation after split is well defined. The "Hi"
2761  // part comes after the "Lo". So these two operations should be chained one
2762  // after another.
2763  SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale};
2764  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), HiMemVT,
2765  DL, OpsHi, MMO, N->getIndexType(),
2766  N->isTruncatingStore());
2767 }
2768 
2769 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
2770  assert(N->isUnindexed() && "Indexed store of vector?");
2771  assert(OpNo == 1 && "Can only split the stored value");
2772  SDLoc DL(N);
2773 
2774  bool isTruncating = N->isTruncatingStore();
2775  SDValue Ch = N->getChain();
2776  SDValue Ptr = N->getBasePtr();
2777  EVT MemoryVT = N->getMemoryVT();
2778  Align Alignment = N->getOriginalAlign();
2779  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2780  AAMDNodes AAInfo = N->getAAInfo();
2781  SDValue Lo, Hi;
2782  GetSplitVector(N->getOperand(1), Lo, Hi);
2783 
2784  EVT LoMemVT, HiMemVT;
2785  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2786 
2787  // Scalarize if the split halves are not byte-sized.
2788  if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
2789  return TLI.scalarizeVectorStore(N, DAG);
2790 
2791  if (isTruncating)
2792  Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
2793  Alignment, MMOFlags, AAInfo);
2794  else
2795  Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2796  AAInfo);
2797 
2798  MachinePointerInfo MPI;
2799  IncrementPointer(N, LoMemVT, MPI, Ptr);
2800 
2801  if (isTruncating)
2802  Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, MPI,
2803  HiMemVT, Alignment, MMOFlags, AAInfo);
2804  else
2805  Hi = DAG.getStore(Ch, DL, Hi, Ptr, MPI, Alignment, MMOFlags, AAInfo);
2806 
2807  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2808 }
2809 
2810 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
2811  SDLoc DL(N);
2812 
2813  // The input operands all must have the same type, and we know the result
2814  // type is valid. Convert this to a buildvector which extracts all the
2815  // input elements.
2816  // TODO: If the input elements are power-two vectors, we could convert this to
2817  // a new CONCAT_VECTORS node with elements that are half-wide.
2819  EVT EltVT = N->getValueType(0).getVectorElementType();
2820  for (const SDValue &Op : N->op_values()) {
2821  for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
2822  i != e; ++i) {
2823  Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
2824  DAG.getVectorIdxConstant(i, DL)));
2825  }
2826  }
2827 
2828  return DAG.getBuildVector(N->getValueType(0), DL, Elts);
2829 }
2830 
2831 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
2832  // The result type is legal, but the input type is illegal. If splitting
2833  // ends up with the result type of each half still being legal, just
2834  // do that. If, however, that would result in an illegal result type,
2835  // we can try to get more clever with power-two vectors. Specifically,
2836  // split the input type, but also widen the result element size, then
2837  // concatenate the halves and truncate again. For example, consider a target
2838  // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
2839  // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
2840  // %inlo = v4i32 extract_subvector %in, 0
2841  // %inhi = v4i32 extract_subvector %in, 4
2842  // %lo16 = v4i16 trunc v4i32 %inlo
2843  // %hi16 = v4i16 trunc v4i32 %inhi
2844  // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
2845  // %res = v8i8 trunc v8i16 %in16
2846  //
2847  // Without this transform, the original truncate would end up being
2848  // scalarized, which is pretty much always a last resort.
2849  unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
2850  SDValue InVec = N->getOperand(OpNo);
2851  EVT InVT = InVec->getValueType(0);
2852  EVT OutVT = N->getValueType(0);
2853  ElementCount NumElements = OutVT.getVectorElementCount();
2854  bool IsFloat = OutVT.isFloatingPoint();
2855 
2856  unsigned InElementSize = InVT.getScalarSizeInBits();
2857  unsigned OutElementSize = OutVT.getScalarSizeInBits();
2858 
2859  // Determine the split output VT. If its legal we can just split dirctly.
2860  EVT LoOutVT, HiOutVT;
2861  std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
2862  assert(LoOutVT == HiOutVT && "Unequal split?");
2863 
2864  // If the input elements are only 1/2 the width of the result elements,
2865  // just use the normal splitting. Our trick only work if there's room
2866  // to split more than once.
2867  if (isTypeLegal(LoOutVT) ||
2868  InElementSize <= OutElementSize * 2)
2869  return SplitVecOp_UnaryOp(N);
2870  SDLoc DL(N);
2871 
2872  // Don't touch if this will be scalarized.
2873  EVT FinalVT = InVT;
2874  while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
2875  FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
2876 
2877  if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector)
2878  return SplitVecOp_UnaryOp(N);
2879 
2880  // Get the split input vector.
2881  SDValue InLoVec, InHiVec;
2882  GetSplitVector(InVec, InLoVec, InHiVec);
2883 
2884  // Truncate them to 1/2 the element size.
2885  //
2886  // This assumes the number of elements is a power of two; any vector that
2887  // isn't should be widened, not split.
2888  EVT HalfElementVT = IsFloat ?
2889  EVT::getFloatingPointVT(InElementSize/2) :
2890  EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
2891  EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
2892  NumElements.divideCoefficientBy(2));
2893 
2894  SDValue HalfLo;
2895  SDValue HalfHi;
2896  SDValue Chain;
2897  if (N->isStrictFPOpcode()) {
2898  HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
2899  {N->getOperand(0), InLoVec});
2900  HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
2901  {N->getOperand(0), InHiVec});
2902  // Legalize the chain result - switch anything that used the old chain to
2903  // use the new one.
2904  Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1),
2905  HalfHi.getValue(1));
2906  } else {
2907  HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
2908  HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
2909  }
2910 
2911  // Concatenate them to get the full intermediate truncation result.
2912  EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
2913  SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
2914  HalfHi);
2915  // Now finish up by truncating all the way down to the original result
2916  // type. This should normally be something that ends up being legal directly,
2917  // but in theory if a target has very wide vectors and an annoyingly
2918  // restricted set of legal types, this split can chain to build things up.
2919 
2920  if (N->isStrictFPOpcode()) {
2921  SDValue Res = DAG.getNode(
2922  ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other},
2923  {Chain, InterVec,
2924  DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))});
2925  // Relink the chain
2926  ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
2927  return Res;
2928  }
2929 
2930  return IsFloat
2931  ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
2932  DAG.getTargetConstant(
2933  0, DL, TLI.getPointerTy(DAG.getDataLayout())))
2934  : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
2935 }
2936 
2937 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
2938  assert(N->getValueType(0).isVector() &&
2939  N->getOperand(0).getValueType().isVector() &&
2940  "Operand types must be vectors");
2941  // The result has a legal vector type, but the input needs splitting.
2942  SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
2943  SDLoc DL(N);
2944  GetSplitVector(N->getOperand(0), Lo0, Hi0);
2945  GetSplitVector(N->getOperand(1), Lo1, Hi1);
2946  auto PartEltCnt = Lo0.getValueType().getVectorElementCount();
2947 
2948  LLVMContext &Context = *DAG.getContext();
2949  EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt);
2950  EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2);
2951 
2952  LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
2953  HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
2954  SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
2955 
2956  EVT OpVT = N->getOperand(0).getValueType();
2957  ISD::NodeType ExtendCode =
2959  return DAG.getNode(ExtendCode, DL, N->getValueType(0), Con);
2960 }
2961 
2962 
2963 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
2964  // The result has a legal vector type, but the input needs splitting.
2965  EVT ResVT = N->getValueType(0);
2966  SDValue Lo, Hi;
2967  SDLoc DL(N);
2968  GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
2969  EVT InVT = Lo.getValueType();
2970 
2971  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2972  InVT.getVectorElementCount());
2973 
2974  if (N->isStrictFPOpcode()) {
2975  Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
2976  { N->getOperand(0), Lo, N->getOperand(2) });
2977  Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
2978  { N->getOperand(0), Hi, N->getOperand(2) });
2979  // Legalize the chain result - switch anything that used the old chain to
2980  // use the new one.
2981  SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
2982  Lo.getValue(1), Hi.getValue(1));
2983  ReplaceValueWith(SDValue(N, 1), NewChain);
2984  } else {
2985  Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
2986  Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
2987  }
2988 
2989  return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
2990 }
2991 
2992 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
2993  // The result (and the first input) has a legal vector type, but the second
2994  // input needs splitting.
2995  return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
2996 }
2997 
2998 SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
2999  EVT ResVT = N->getValueType(0);
3000  SDValue Lo, Hi;
3001  SDLoc dl(N);
3002  GetSplitVector(N->getOperand(0), Lo, Hi);
3003  EVT InVT = Lo.getValueType();
3004 
3005  EVT NewResVT =
3007  InVT.getVectorElementCount());
3008 
3009  Lo = DAG.getNode(N->getOpcode(), dl, NewResVT, Lo, N->getOperand(1));
3010  Hi = DAG.getNode(N->getOpcode(), dl, NewResVT, Hi, N->getOperand(1));
3011 
3012  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
3013 }
3014 
3015 //===----------------------------------------------------------------------===//
3016 // Result Vector Widening
3017 //===----------------------------------------------------------------------===//
3018 
3019 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
3020  LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
3021  dbgs() << "\n");
3022 
3023  // See if the target wants to custom widen this node.
3024  if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
3025  return;
3026 
3027  SDValue Res = SDValue();
3028  switch (N->getOpcode()) {
3029  default:
3030 #ifndef NDEBUG
3031  dbgs() << "WidenVectorResult #" << ResNo << ": ";
3032  N->dump(&DAG);
3033  dbgs() << "\n";
3034 #endif
3035  llvm_unreachable("Do not know how to widen the result of this operator!");
3036 
3037  case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
3038  case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
3039  case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
3040  case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
3041  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
3042  case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
3043  case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
3044  case ISD::SPLAT_VECTOR:
3045  case ISD::SCALAR_TO_VECTOR:
3046  Res = WidenVecRes_ScalarOp(N);
3047  break;
3048  case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
3049  case ISD::VSELECT:
3050  case ISD::SELECT: Res = WidenVecRes_SELECT(N); break;
3051  case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
3052  case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
3053  case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
3054  case ISD::VECTOR_SHUFFLE:
3055  Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
3056  break;
3057  case ISD::MLOAD:
3058  Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
3059  break;
3060  case ISD::MGATHER:
3061  Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
3062  break;
3063 
3064  case ISD::ADD:
3065  case ISD::AND:
3066  case ISD::MUL:
3067  case ISD::MULHS:
3068  case ISD::MULHU:
3069  case ISD::OR:
3070  case ISD::SUB:
3071  case ISD::XOR:
3072  case ISD::SHL:
3073  case ISD::SRA:
3074  case ISD::SRL:
3075  case ISD::FMINNUM:
3076  case ISD::FMAXNUM:
3077  case ISD::FMINIMUM:
3078  case ISD::FMAXIMUM:
3079  case ISD::SMIN:
3080  case ISD::SMAX:
3081  case ISD::UMIN:
3082  case ISD::UMAX:
3083  case ISD::UADDSAT:
3084  case ISD::SADDSAT:
3085  case ISD::USUBSAT:
3086  case ISD::SSUBSAT:
3087  case ISD::SSHLSAT:
3088  case ISD::USHLSAT:
3089  case ISD::ROTL:
3090  case ISD::ROTR:
3091  Res = WidenVecRes_Binary(N, /*IsVP*/ false);
3092  break;
3093 
3094  case ISD::FADD:
3095  case ISD::FMUL:
3096  case ISD::FPOW:
3097  case ISD::FSUB:
3098  case ISD::FDIV:
3099  case ISD::FREM:
3100  case ISD::SDIV:
3101  case ISD::UDIV:
3102  case ISD::SREM:
3103  case ISD::UREM:
3104  Res = WidenVecRes_BinaryCanTrap(N);
3105  break;
3106 
3107  case ISD::SMULFIX:
3108  case ISD::SMULFIXSAT:
3109  case ISD::UMULFIX:
3110  case ISD::UMULFIXSAT:
3111  // These are binary operations, but with an extra operand that shouldn't
3112  // be widened (the scale).
3113  Res = WidenVecRes_BinaryWithExtraScalarOp(N);
3114  break;
3115 
3116 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
3117  case ISD::STRICT_##DAGN:
3118 #include "llvm/IR/ConstrainedOps.def"
3119  Res = WidenVecRes_StrictFP(N);
3120  break;
3121 
3122  case ISD::UADDO:
3123  case ISD::SADDO:
3124  case ISD::USUBO:
3125  case ISD::SSUBO:
3126  case ISD::UMULO:
3127  case ISD::SMULO:
3128  Res = WidenVecRes_OverflowOp(N, ResNo);
3129  break;
3130 
3131  case ISD::FCOPYSIGN:
3132  Res = WidenVecRes_FCOPYSIGN(N);
3133  break;
3134 
3135  case ISD::FPOWI:
3136  Res = WidenVecRes_POWI(N);
3137  break;
3138 
3142  Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
3143  break;
3144 
3145  case ISD::ANY_EXTEND:
3146  case ISD::FP_EXTEND:
3147  case ISD::FP_ROUND:
3148  case ISD::FP_TO_SINT:
3149  case ISD::FP_TO_UINT:
3150  case ISD::SIGN_EXTEND:
3151  case ISD::SINT_TO_FP:
3152  case ISD::TRUNCATE:
3153  case ISD::UINT_TO_FP:
3154  case ISD::ZERO_EXTEND:
3155  Res = WidenVecRes_Convert(N);
3156  break;
3157 
3158  case ISD::FP_TO_SINT_SAT:
3159  case ISD::FP_TO_UINT_SAT:
3160  Res = WidenVecRes_FP_TO_XINT_SAT(N);
3161  break;
3162 
3163  case ISD::FABS:
3164  case ISD::FCEIL:
3165  case ISD::FCOS:
3166  case ISD::FEXP:
3167  case ISD::FEXP2:
3168  case ISD::FFLOOR:
3169  case ISD::FLOG:
3170  case ISD::FLOG10:
3171  case ISD::FLOG2:
3172  case ISD::FNEARBYINT:
3173  case ISD::FRINT:
3174  case ISD::FROUND:
3175  case ISD::FROUNDEVEN:
3176  case ISD::FSIN:
3177  case ISD::FSQRT:
3178  case ISD::FTRUNC: {
3179  // We're going to widen this vector op to a legal type by padding with undef
3180  // elements. If the wide vector op is eventually going to be expanded to
3181  // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
3182  // libcalls on the undef elements.
3183  EVT VT = N->getValueType(0);
3184  EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3185  if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
3186  TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
3187  Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
3188  break;
3189  }
3190  }
3191  // If the target has custom/legal support for the scalar FP intrinsic ops
3192  // (they are probably not destined to become libcalls), then widen those like
3193  // any other unary ops.
3195 
3196  case ISD::ABS:
3197  case ISD::BITREVERSE:
3198  case ISD::BSWAP:
3199  case ISD::CTLZ:
3200  case ISD::CTLZ_ZERO_UNDEF:
3201  case ISD::CTPOP:
3202  case ISD::CTTZ:
3203  case ISD::CTTZ_ZERO_UNDEF:
3204  case ISD::FNEG:
3205  case ISD::FREEZE:
3206  case ISD::ARITH_FENCE:
3207  case ISD::FCANONICALIZE:
3208  Res = WidenVecRes_Unary(N);
3209  break;
3210  case ISD::FMA:
3211  case ISD::FSHL:
3212  case ISD::FSHR:
3213  Res = WidenVecRes_Ternary(N);
3214  break;
3215  case ISD::VP_ADD:
3216  case ISD::VP_AND:
3217  case ISD::VP_MUL:
3218  case ISD::VP_OR:
3219  case ISD::VP_SUB:
3220  case ISD::VP_XOR:
3221  case ISD::VP_SHL:
3222  case ISD::VP_LSHR:
3223  case ISD::VP_ASHR:
3224  case ISD::VP_SDIV:
3225  case ISD::VP_UDIV:
3226  case ISD::VP_SREM:
3227  case ISD::VP_UREM:
3228  case ISD::VP_FADD:
3229  case ISD::VP_FSUB:
3230  case ISD::VP_FMUL:
3231  case ISD::VP_FDIV:
3232  case ISD::VP_FREM:
3233  // Vector-predicated binary op widening. Note that -- unlike the
3234  // unpredicated versions -- we don't have to worry about trapping on
3235  // operations like UDIV, FADD, etc., as we pass on the original vector
3236  // length parameter. This means the widened elements containing garbage
3237  // aren't active.
3238  Res = WidenVecRes_Binary(N, /*IsVP*/ true);
3239  break;
3240  }
3241 
3242  // If Res is null, the sub-method took care of registering the result.
3243  if (Res.getNode())
3244  SetWidenedVector(SDValue(N, ResNo), Res);
3245 }
3246 
3247 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
3248  // Ternary op widening.
3249  SDLoc dl(N);
3250  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3251  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3252  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3253  SDValue InOp3 = GetWidenedVector(N->getOperand(2));
3254  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
3255 }
3256 
3257 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N, bool IsVP) {
3258  // Binary op widening.
3259  SDLoc dl(N);
3260  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3261  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3262  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3263  if (!IsVP)
3264  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2,
3265  N->getFlags());
3266  // For VP operations, we must also widen the mask. Note that the mask type
3267  // may not actually need widening, leading it be split along with the VP
3268  // operation.
3269  // FIXME: This could lead to an infinite split/widen loop. We only handle the
3270  // case where the mask needs widening to an identically-sized type as the
3271  // vector inputs.
3272  SDValue Mask = N->getOperand(2);
3273  assert(getTypeAction(Mask.getValueType()) ==
3275  "Unable to widen binary VP op");
3276  Mask = GetWidenedVector(Mask);
3277  assert(Mask.getValueType().getVectorElementCount() ==
3278  WidenVT.getVectorElementCount() &&
3279  "Unable to widen binary VP op");
3280  return DAG.getNode(N->getOpcode(), dl, WidenVT,
3281  {InOp1, InOp2, Mask, N->getOperand(3)}, N->getFlags());
3282 }
3283 
3284 SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
3285  // Binary op widening, but with an extra operand that shouldn't be widened.
3286  SDLoc dl(N);
3287  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3288  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3289  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3290  SDValue InOp3 = N->getOperand(2);
3291  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3,
3292  N->getFlags());
3293 }
3294 
3295 // Given a vector of operations that have been broken up to widen, see
3296 // if we can collect them together into the next widest legal VT. This
3297 // implementation is trap-safe.
3299  SmallVectorImpl<SDValue> &ConcatOps,
3300  unsigned ConcatEnd, EVT VT, EVT MaxVT,
3301  EVT WidenVT) {
3302  // Check to see if we have a single operation with the widen type.
3303  if (ConcatEnd == 1) {
3304  VT = ConcatOps[0].getValueType();
3305  if (VT == WidenVT)
3306  return ConcatOps[0];
3307  }
3308 
3309  SDLoc dl(ConcatOps[0]);
3310  EVT WidenEltVT = WidenVT.getVectorElementType();
3311 
3312  // while (Some element of ConcatOps is not of type MaxVT) {
3313  // From the end of ConcatOps, collect elements of the same type and put
3314  // them into an op of the next larger supported type
3315  // }
3316  while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
3317  int Idx = ConcatEnd - 1;
3318  VT = ConcatOps[Idx--].getValueType();
3319  while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
3320  Idx--;
3321 
3322  int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
3323  EVT NextVT;
3324  do {
3325  NextSize *= 2;
3326  NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
3327  } while (!TLI.isTypeLegal(NextVT));
3328 
3329  if (!VT.isVector()) {
3330  // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
3331  SDValue VecOp = DAG.getUNDEF(NextVT);
3332  unsigned NumToInsert = ConcatEnd - Idx - 1;
3333  for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
3334  VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp,
3335  ConcatOps[OpIdx], DAG.getVectorIdxConstant(i, dl));
3336  }
3337  ConcatOps[Idx+1] = VecOp;
3338  ConcatEnd = Idx + 2;
3339  } else {
3340  // Vector type, create a CONCAT_VECTORS of type NextVT
3341  SDValue undefVec = DAG.getUNDEF(VT);
3342  unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
3343  SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
3344  unsigned RealVals = ConcatEnd - Idx - 1;
3345  unsigned SubConcatEnd = 0;
3346  unsigned SubConcatIdx = Idx + 1;
3347  while (SubConcatEnd < RealVals)
3348  SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
3349  while (SubConcatEnd < OpsToConcat)
3350  SubConcatOps[SubConcatEnd++] = undefVec;
3351  ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
3352  NextVT, SubConcatOps);
3353  ConcatEnd = SubConcatIdx + 1;
3354  }
3355  }
3356 
3357  // Check to see if we have a single operation with the widen type.
3358  if (ConcatEnd == 1) {
3359  VT = ConcatOps[0].getValueType();
3360  if (VT == WidenVT)
3361  return ConcatOps[0];
3362  }
3363 
3364  // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
3365  unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
3366  if (NumOps != ConcatEnd ) {
3367  SDValue UndefVal = DAG.getUNDEF(MaxVT);
3368  for (unsigned j = ConcatEnd; j < NumOps; ++j)
3369  ConcatOps[j] = UndefVal;
3370  }
3371  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
3372  makeArrayRef(ConcatOps.data(), NumOps));
3373 }
3374 
3375 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
3376  // Binary op widening for operations that can trap.
3377  unsigned Opcode = N->getOpcode();
3378  SDLoc dl(N);
3379  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3380  EVT WidenEltVT = WidenVT.getVectorElementType();
3381  EVT VT = WidenVT;
3382  unsigned NumElts = VT.getVectorNumElements();
3383  const SDNodeFlags Flags = N->getFlags();
3384  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
3385  NumElts = NumElts / 2;
3386  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3387  }
3388 
3389  if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
3390  // Operation doesn't trap so just widen as normal.
3391  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3392  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3393  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
3394  }
3395 
3396  // No legal vector version so unroll the vector operation and then widen.
3397  if (NumElts == 1)
3398  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
3399 
3400  // Since the operation can trap, apply operation on the original vector.
3401  EVT MaxVT = VT;
3402  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3403  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3404  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
3405 
3406  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3407  unsigned ConcatEnd = 0; // Current ConcatOps index.
3408  int Idx = 0; // Current Idx into input vectors.
3409 
3410  // NumElts := greatest legal vector size (at most WidenVT)
3411  // while (orig. vector has unhandled elements) {
3412  // take munches of size NumElts from the beginning and add to ConcatOps
3413  // NumElts := next smaller supported vector size or 1
3414  // }
3415  while (CurNumElts != 0) {
3416  while (CurNumElts >= NumElts) {
3417  SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
3418  DAG.getVectorIdxConstant(Idx, dl));
3419  SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
3420  DAG.getVectorIdxConstant(Idx, dl));
3421  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
3422  Idx += NumElts;
3423  CurNumElts -= NumElts;
3424  }
3425  do {
3426  NumElts = NumElts / 2;
3427  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3428  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
3429 
3430  if (NumElts == 1) {
3431  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
3432  SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
3433  InOp1, DAG.getVectorIdxConstant(Idx, dl));
3434  SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
3435  InOp2, DAG.getVectorIdxConstant(Idx, dl));
3436  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
3437  EOp1, EOp2, Flags);
3438  }
3439  CurNumElts = 0;
3440  }
3441  }
3442 
3443  return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3444 }
3445 
3446 SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
3447  switch (N->getOpcode()) {
3448  case ISD::STRICT_FSETCC:
3449  case ISD::STRICT_FSETCCS:
3450  return WidenVecRes_STRICT_FSETCC(N);
3451  case ISD::STRICT_FP_EXTEND:
3452  case ISD::STRICT_FP_ROUND:
3457  return WidenVecRes_Convert_StrictFP(N);
3458  default:
3459  break;
3460  }
3461 
3462  // StrictFP op widening for operations that can trap.
3463  unsigned NumOpers = N->getNumOperands();
3464  unsigned Opcode = N->getOpcode();
3465  SDLoc dl(N);
3466  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3467  EVT WidenEltVT = WidenVT.getVectorElementType();
3468  EVT VT = WidenVT;
3469  unsigned NumElts = VT.getVectorNumElements();
3470  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
3471  NumElts = NumElts / 2;
3472  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3473  }
3474 
3475  // No legal vector version so unroll the vector operation and then widen.
3476  if (NumElts == 1)
3477  return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements());
3478 
3479  // Since the operation can trap, apply operation on the original vector.
3480  EVT MaxVT = VT;
3482  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
3483 
3484  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3485  SmallVector<SDValue, 16> Chains;
3486  unsigned ConcatEnd = 0; // Current ConcatOps index.
3487  int Idx = 0; // Current Idx into input vectors.
3488 
3489  // The Chain is the first operand.
3490  InOps.push_back(N->getOperand(0));
3491 
3492  // Now process the remaining operands.
3493  for (unsigned i = 1; i < NumOpers; ++i) {
3494  SDValue Oper = N->getOperand(i);
3495 
3496  if (Oper.getValueType().isVector()) {
3497  assert(Oper.getValueType() == N->getValueType(0) &&
3498  "Invalid operand type to widen!");
3499  Oper = GetWidenedVector(Oper);
3500  }
3501 
3502  InOps.push_back(Oper);
3503  }
3504 
3505  // NumElts := greatest legal vector size (at most WidenVT)
3506  // while (orig. vector has unhandled elements) {
3507  // take munches of size NumElts from the beginning and add to ConcatOps
3508  // NumElts := next smaller supported vector size or 1
3509  // }
3510  while (CurNumElts != 0) {
3511  while (CurNumElts >= NumElts) {
3513 
3514  for (unsigned i = 0; i < NumOpers; ++i) {
3515  SDValue Op = InOps[i];
3516 
3517  if (Op.getValueType().isVector())
3518  Op = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Op,
3519  DAG.getVectorIdxConstant(Idx, dl));
3520 
3521  EOps.push_back(Op);
3522  }
3523 
3524  EVT OperVT[] = {VT, MVT::Other};
3525  SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
3526  ConcatOps[ConcatEnd++] = Oper;
3527  Chains.push_back(Oper.getValue(1));
3528  Idx += NumElts;
3529  CurNumElts -= NumElts;
3530  }
3531  do {
3532  NumElts = NumElts / 2;
3533  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3534  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
3535 
3536  if (NumElts == 1) {
3537  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
3539 
3540  for (unsigned i = 0; i < NumOpers; ++i) {
3541  SDValue Op = InOps[i];
3542 
3543  if (Op.getValueType().isVector())
3544  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op,
3545  DAG.getVectorIdxConstant(Idx, dl));
3546 
3547  EOps.push_back(Op);
3548  }
3549 
3550  EVT WidenVT[] = {WidenEltVT, MVT::Other};
3551  SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
3552  ConcatOps[ConcatEnd++] = Oper;
3553  Chains.push_back(Oper.getValue(1));
3554  }
3555  CurNumElts = 0;
3556  }
3557  }
3558 
3559  // Build a factor node to remember all the Ops that have been created.
3560  SDValue NewChain;
3561  if (Chains.size() == 1)
3562  NewChain = Chains[0];
3563  else
3564  NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
3565  ReplaceValueWith(SDValue(N, 1), NewChain);
3566 
3567  return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3568 }
3569 
3570 SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
3571  SDLoc DL(N);
3572  EVT ResVT = N->getValueType(0);
3573  EVT OvVT = N->getValueType(1);
3574  EVT WideResVT, WideOvVT;
3575  SDValue WideLHS, WideRHS;
3576 
3577  // TODO: This might result in a widen/split loop.
3578  if (ResNo == 0) {
3579  WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
3580  WideOvVT = EVT::getVectorVT(
3581  *DAG.getContext(), OvVT.getVectorElementType(),
3582  WideResVT.getVectorNumElements());
3583 
3584  WideLHS = GetWidenedVector(N->getOperand(0));
3585  WideRHS = GetWidenedVector(N->getOperand(1));
3586  } else {
3587  WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT);
3588  WideResVT = EVT::getVectorVT(
3589  *DAG.getContext(), ResVT.getVectorElementType(),
3590  WideOvVT.getVectorNumElements());
3591 
3592  SDValue Zero = DAG.getVectorIdxConstant(0, DL);
3593  WideLHS = DAG.getNode(
3594  ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3595  N->getOperand(0), Zero);
3596  WideRHS = DAG.getNode(
3597  ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3598  N->getOperand(1), Zero);
3599  }
3600 
3601  SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT);
3602  SDNode *WideNode = DAG.getNode(
3603  N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode();
3604 
3605  // Replace the other vector result not being explicitly widened here.
3606  unsigned OtherNo = 1 - ResNo;
3607  EVT OtherVT = N->getValueType(OtherNo);
3608  if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) {
3609  SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo));
3610  } else {
3611  SDValue Zero = DAG.getVectorIdxConstant(0, DL);
3612  SDValue OtherVal = DAG.getNode(
3613  ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero);
3614  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
3615  }
3616 
3617  return SDValue(WideNode, ResNo);
3618 }
3619 
3620 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
3621  LLVMContext &Ctx = *DAG.getContext();
3622  SDValue InOp = N->getOperand(0);
3623  SDLoc DL(N);
3624 
3625  EVT WidenVT = TLI.getTypeToTransformTo(Ctx, N->getValueType(0));
3626  ElementCount WidenEC = WidenVT.getVectorElementCount();
3627 
3628  EVT InVT = InOp.getValueType();
3629 
3630  unsigned Opcode = N->getOpcode();
3631  const SDNodeFlags Flags = N->getFlags();
3632 
3633  // Handle the case of ZERO_EXTEND where the promoted InVT element size does
3634  // not equal that of WidenVT.
3635  if (N->getOpcode() == ISD::ZERO_EXTEND &&
3636  getTypeAction(InVT) == TargetLowering::TypePromoteInteger &&
3637  TLI.getTypeToTransformTo(Ctx, InVT).getScalarSizeInBits() !=
3638  WidenVT.getScalarSizeInBits()) {
3639  InOp = ZExtPromotedInteger(InOp);
3640  InVT = InOp.getValueType();
3641  if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
3642  Opcode = ISD::TRUNCATE;
3643  }
3644 
3645  EVT InEltVT = InVT.getVectorElementType();
3646  EVT InWidenVT = EVT::getVectorVT(Ctx, InEltVT, WidenEC);
3647  ElementCount InVTEC = InVT.getVectorElementCount();
3648 
3649  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3650  InOp = GetWidenedVector(N->getOperand(0));
3651  InVT = InOp.getValueType();
3652  InVTEC = InVT.getVectorElementCount();
3653  if (InVTEC == WidenEC) {
3654  if (N->getNumOperands() == 1)
3655  return DAG.getNode(Opcode, DL, WidenVT, InOp);
3656  return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
3657  }
3658  if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
3659  // If both input and result vector types are of same width, extend
3660  // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
3661  // accepts fewer elements in the result than in the input.
3662  if (Opcode == ISD::ANY_EXTEND)
3663  return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3664  if (Opcode == ISD::SIGN_EXTEND)
3665  return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3666  if (Opcode == ISD::ZERO_EXTEND)
3667  return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3668  }
3669  }
3670 
3671  if (TLI.isTypeLegal(InWidenVT)) {
3672  // Because the result and the input are different vector types, widening
3673  // the result could create a legal type but widening the input might make
3674  // it an illegal type that might lead to repeatedly splitting the input
3675  // and then widening it. To avoid this, we widen the input only if
3676  // it results in a legal type.
3677  if (WidenEC.isKnownMultipleOf(InVTEC.getKnownMinValue())) {
3678  // Widen the input and call convert on the widened input vector.
3679  unsigned NumConcat =
3680  WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
3681  SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
3682  Ops[0] = InOp;
3683  SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
3684  if (N->getNumOperands() == 1)
3685  return DAG.getNode(Opcode, DL, WidenVT, InVec);
3686  return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
3687  }
3688 
3689  if (InVTEC.isKnownMultipleOf(WidenEC.getKnownMinValue())) {
3690  SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
3691  DAG.getVectorIdxConstant(0, DL));
3692  // Extract the input and convert the shorten input vector.
3693  if (N->getNumOperands() == 1)
3694  return DAG.getNode(Opcode, DL, WidenVT, InVal);
3695  return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
3696  }
3697  }
3698 
3699  // Otherwise unroll into some nasty scalar code and rebuild the vector.
3700  EVT EltVT = WidenVT.getVectorElementType();
3701  SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getUNDEF(EltVT));
3702  // Use the original element count so we don't do more scalar opts than
3703  // necessary.
3704  unsigned MinElts = N->getValueType(0).getVectorNumElements();
3705  for (unsigned i=0; i < MinElts; ++i) {
3706  SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3707  DAG.getVectorIdxConstant(i, DL));
3708  if (N->getNumOperands() == 1)
3709  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
3710  else
3711  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
3712  }
3713 
3714  return DAG.getBuildVector(WidenVT, DL, Ops);
3715 }
3716 
3717 SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
3718  SDLoc dl(N);
3719  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3720  ElementCount WidenNumElts = WidenVT.getVectorElementCount();
3721 
3722  SDValue Src = N->getOperand(0);
3723  EVT SrcVT = Src.getValueType();
3724 
3725  // Also widen the input.
3726  if (getTypeAction(SrcVT) == TargetLowering::TypeWidenVector) {
3727  Src = GetWidenedVector(Src);
3728  SrcVT = Src.getValueType();
3729  }
3730 
3731  // Input and output not widened to the same size, give up.
3732  if (WidenNumElts != SrcVT.getVectorElementCount())
3733  return DAG.UnrollVectorOp(N, WidenNumElts.getKnownMinValue());
3734 
3735  return DAG.getNode(N->getOpcode(), dl, WidenVT, Src, N->getOperand(1));
3736 }
3737 
3738 SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
3739  SDValue InOp = N->getOperand(1);
3740  SDLoc DL(N);
3741  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
3742 
3743  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3744  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3745 
3746  EVT InVT = InOp.getValueType();
3747  EVT InEltVT = InVT.getVectorElementType();
3748 
3749  unsigned Opcode = N->getOpcode();
3750 
3751  // FIXME: Optimizations need to be implemented here.
3752 
3753  // Otherwise unroll into some nasty scalar code and rebuild the vector.
3754  EVT EltVT = WidenVT.getVectorElementType();
3755  std::array<EVT, 2> EltVTs = {{EltVT, MVT::Other}};
3756  SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3757  SmallVector<SDValue, 32> OpChains;
3758  // Use the original element count so we don't do more scalar opts than
3759  // necessary.
3760  unsigned MinElts = N->getValueType(0).getVectorNumElements();
3761  for (unsigned i=0; i < MinElts; ++i) {
3762  NewOps[1] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3763  DAG.getVectorIdxConstant(i, DL));
3764  Ops[i] = DAG.getNode(Opcode, DL, EltVTs, NewOps);
3765  OpChains.push_back(Ops[i].getValue(1));
3766  }
3767  SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains);
3768  ReplaceValueWith(SDValue(N, 1), NewChain);
3769 
3770  return DAG.getBuildVector(WidenVT, DL, Ops);
3771 }
3772 
3773 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
3774  unsigned Opcode = N->getOpcode();
3775  SDValue InOp = N->getOperand(0);
3776  SDLoc DL(N);
3777 
3778  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3779  EVT WidenSVT = WidenVT.getVectorElementType();
3780  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3781 
3782  EVT InVT = InOp.getValueType();
3783  EVT InSVT = InVT.getVectorElementType();
3784  unsigned InVTNumElts = InVT.getVectorNumElements();
3785 
3786  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3787  InOp = GetWidenedVector(InOp);
3788  InVT = InOp.getValueType();
3789  if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
3790  switch (Opcode) {
3794  return DAG.getNode(Opcode, DL, WidenVT, InOp);
3795  }
3796  }
3797  }
3798 
3799  // Unroll, extend the scalars and rebuild the vector.
3801  for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
3802  SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
3803  DAG.getVectorIdxConstant(i, DL));
3804  switch (Opcode) {
3806  Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
3807  break;
3809  Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
3810  break;
3812  Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
3813  break;
3814  default:
3815  llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
3816  }
3817  Ops.push_back(Val);
3818  }
3819 
3820  while (Ops.size() != WidenNumElts)
3821  Ops.push_back(DAG.getUNDEF(WidenSVT));
3822 
3823  return DAG.getBuildVector(WidenVT, DL, Ops);
3824 }
3825 
3826 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
3827  // If this is an FCOPYSIGN with same input types, we can treat it as a
3828  // normal (can trap) binary op.
3829  if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
3830  return WidenVecRes_BinaryCanTrap(N);
3831 
3832  // If the types are different, fall back to unrolling.
3833  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3834  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
3835 }
3836 
3837 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
3838  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3839  SDValue InOp = GetWidenedVector(N->getOperand(0));
3840  SDValue ShOp = N->getOperand(1);
3841  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
3842 }
3843 
3844 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
3845  // Unary op widening.
3846  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3847  SDValue InOp = GetWidenedVector(N->getOperand(0));
3848  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
3849 }
3850 
3851 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
3852  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3853  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
3854  cast<VTSDNode>(N->getOperand(1))->getVT()
3855  .getVectorElementType(),
3856  WidenVT.getVectorNumElements());
3857  SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
3858  return DAG.getNode(N->getOpcode(), SDLoc(N),
3859  WidenVT, WidenLHS, DAG.getValueType(ExtVT));
3860 }
3861 
3862 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
3863  SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
3864  return GetWidenedVector(WidenVec);
3865 }
3866 
3867 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
3868  SDValue InOp = N->getOperand(0);
3869  EVT InVT = InOp.getValueType();
3870  EVT VT = N->getValueType(0);
3871  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3872  SDLoc dl(N);
3873 
3874  switch (getTypeAction(InVT)) {
3876  break;
3878  report_fatal_error("Scalarization of scalable vectors is not supported.");
3880  // If the incoming type is a vector that is being promoted, then
3881  // we know that the elements are arranged differently and that we
3882  // must perform the conversion using a stack slot.
3883  if (InVT.isVector())
3884  break;
3885 
3886  // If the InOp is promoted to the same size, convert it. Otherwise,
3887  // fall out of the switch and widen the promoted input.
3888  SDValue NInOp = GetPromotedInteger(InOp);
3889  EVT NInVT = NInOp.getValueType();
3890  if (WidenVT.bitsEq(NInVT)) {
3891  // For big endian targets we need to shift the input integer or the
3892  // interesting bits will end up at the wrong place.
3893  if (DAG.getDataLayout().isBigEndian()) {
3894  unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
3895  EVT ShiftAmtTy = TLI.getShiftAmountTy(NInVT, DAG.getDataLayout());
3896  assert(ShiftAmt < WidenVT.getSizeInBits() && "Too large shift amount!");
3897  NInOp = DAG.getNode(ISD::SHL, dl, NInVT, NInOp,
3898  DAG.getConstant(ShiftAmt, dl, ShiftAmtTy));
3899  }
3900  return DAG.getNode(ISD::BITCAST, dl, WidenVT, NInOp);
3901  }
3902  InOp = NInOp;
3903  InVT = NInVT;
3904  break;
3905  }
3913  break;
3915  // If the InOp is widened to the same size, convert it. Otherwise, fall
3916  // out of the switch and widen the widened input.
3917  InOp = GetWidenedVector(InOp);
3918  InVT = InOp.getValueType();
3919  if (WidenVT.bitsEq(InVT))
3920  // The input widens to the same size. Convert to the widen value.
3921  return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
3922  break;
3923  }
3924 
3925  unsigned WidenSize = WidenVT.getSizeInBits();
3926  unsigned InSize = InVT.getSizeInBits();
3927  // x86mmx is not an acceptable vector element type, so don't try.
3928  if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
3929  // Determine new input vector type. The new input vector type will use
3930  // the same element type (if its a vector) or use the input type as a
3931  // vector. It is the same size as the type to widen to.
3932  EVT NewInVT;
3933  unsigned NewNumElts = WidenSize / InSize;
3934  if (InVT.isVector()) {
3935  EVT InEltVT = InVT.getVectorElementType();
3936  NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
3937  WidenSize / InEltVT.getSizeInBits());
3938  } else {
3939  NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
3940  }
3941 
3942  if (TLI.isTypeLegal(NewInVT)) {
3943  SDValue NewVec;
3944  if (InVT.isVector()) {
3945  // Because the result and the input are different vector types, widening
3946  // the result could create a legal type but widening the input might make
3947  // it an illegal type that might lead to repeatedly splitting the input
3948  // and then widening it. To avoid this, we widen the input only if
3949  // it results in a legal type.
3950  SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT));
3951  Ops[0] = InOp;
3952 
3953  NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
3954  } else {
3955  NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp);
3956  }
3957  return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
3958  }
3959  }
3960 
3961  return CreateStackStoreLoad(InOp, WidenVT);
3962 }
3963 
3964 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
3965  SDLoc dl(N);
3966  // Build a vector with undefined for the new nodes.
3967  EVT VT = N->getValueType(0);
3968 
3969  // Integer BUILD_VECTOR operands may be larger than the node's vector element
3970  // type. The UNDEFs need to have the same type as the existing operands.
3971  EVT EltVT = N->getOperand(0).getValueType();
3972  unsigned NumElts = VT.getVectorNumElements();
3973 
3974  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3975  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3976 
3977  SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
3978  assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
3979  NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
3980 
3981  return DAG.getBuildVector(WidenVT, dl, NewOps);
3982 }
3983 
3984 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
3985  EVT InVT = N->getOperand(0).getValueType();
3986  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3987  SDLoc dl(N);
3988  unsigned NumOperands = N->getNumOperands();
3989 
3990  bool InputWidened = false; // Indicates we need to widen the input.
3991  if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
3992  unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
3993  unsigned NumInElts = InVT.getVectorMinNumElements();
3994  if (WidenNumElts % NumInElts == 0) {
3995  // Add undef vectors to widen to correct length.
3996  unsigned NumConcat = WidenNumElts / NumInElts;
3997  SDValue UndefVal = DAG.getUNDEF(InVT);
3998  SmallVector<SDValue, 16> Ops(NumConcat);
3999  for (unsigned i=0; i < NumOperands; ++i)
4000  Ops[i] = N->getOperand(i);
4001  for (unsigned i = NumOperands; i != NumConcat; ++i)
4002  Ops[i] = UndefVal;
4003  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
4004  }
4005  } else {
4006  InputWidened = true;
4007  if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
4008  // The inputs and the result are widen to the same value.
4009  unsigned i;
4010  for (i=1; i < NumOperands; ++i)
4011  if (!N->getOperand(i).isUndef())
4012  break;
4013 
4014  if (i == NumOperands)
4015  // Everything but the first operand is an UNDEF so just return the
4016  // widened first operand.
4017  return GetWidenedVector(N->getOperand(0));
4018 
4019  if (NumOperands == 2) {
4020  assert(!WidenVT.isScalableVector() &&
4021  "Cannot use vector shuffles to widen CONCAT_VECTOR result");
4022  unsigned WidenNumElts = WidenVT.getVectorNumElements();
4023  unsigned NumInElts = InVT.getVectorNumElements();
4024 
4025  // Replace concat of two operands with a shuffle.
4026  SmallVector<int, 16> MaskOps(WidenNumElts, -1);
4027  for (unsigned i = 0; i < NumInElts; ++i) {
4028  MaskOps[i] = i;
4029  MaskOps[i + NumInElts] = i + WidenNumElts;
4030  }
4031  return DAG.getVectorShuffle(WidenVT, dl,
4032  GetWidenedVector(N->getOperand(0)),
4033  GetWidenedVector(N->getOperand(1)),
4034  MaskOps);
4035  }
4036  }
4037  }
4038 
4039  assert(!WidenVT.isScalableVector() &&
4040  "Cannot use build vectors to widen CONCAT_VECTOR result");
4041  unsigned WidenNumElts = WidenVT.getVectorNumElements();
4042  unsigned NumInElts = InVT.getVectorNumElements();
4043 
4044  // Fall back to use extracts and build vector.
4045  EVT EltVT = WidenVT.getVectorElementType();
4046  SmallVector<SDValue, 16> Ops(WidenNumElts);
4047  unsigned Idx = 0;
4048  for (unsigned i=0; i < NumOperands; ++i) {
4049  SDValue InOp = N->getOperand(i);
4050  if (InputWidened)
4051  InOp = GetWidenedVector(InOp);
4052  for (unsigned j = 0; j < NumInElts; ++j)
4053  Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4054  DAG.getVectorIdxConstant(j, dl));
4055  }
4056  SDValue UndefVal = DAG.getUNDEF(EltVT);
4057  for (; Idx < WidenNumElts; ++Idx)
4058  Ops[Idx] = UndefVal;
4059  return DAG.getBuildVector(WidenVT, dl, Ops);
4060 }
4061 
4062 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
4063  EVT VT = N->getValueType(0);
4064  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4065  SDValue InOp = N->getOperand(0);
4066  SDValue Idx = N->getOperand(1);
4067  SDLoc dl(N);
4068 
4069  if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
4070  InOp = GetWidenedVector(InOp);
4071 
4072  EVT InVT = InOp.getValueType();
4073 
4074  // Check if we can just return the input vector after widening.
4075  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
4076  if (IdxVal == 0 && InVT == WidenVT)
4077  return InOp;
4078 
4079  if (VT.isScalableVector())
4080  report_fatal_error("Don't know how to widen the result of "
4081  "EXTRACT_SUBVECTOR for scalable vectors");
4082 
4083  // Check if we can extract from the vector.
4084  unsigned WidenNumElts = WidenVT.getVectorNumElements();
4085  unsigned InNumElts = InVT.getVectorNumElements();
4086  if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
4087  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
4088 
4089  // We could try widening the input to the right length but for now, extract
4090  // the original elements, fill the rest with undefs and build a vector.
4091  SmallVector<SDValue, 16> Ops(WidenNumElts);
4092  EVT EltVT = VT.getVectorElementType();
4093  unsigned NumElts = VT.getVectorNumElements();
4094  unsigned i;
4095  for (i = 0; i < NumElts; ++i)
4096  Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4097  DAG.getVectorIdxConstant(IdxVal + i, dl));
4098 
4099  SDValue UndefVal = DAG.getUNDEF(EltVT);
4100  for (; i < WidenNumElts; ++i)
4101  Ops[i] = UndefVal;
4102  return DAG.getBuildVector(WidenVT, dl, Ops);
4103 }
4104 
4105 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
4106  SDValue InOp = GetWidenedVector(N->getOperand(0));
4107  return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
4108  InOp.getValueType(), InOp,
4109  N->getOperand(1), N->getOperand(2));
4110 }
4111 
4112 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
4113  LoadSDNode *LD = cast<LoadSDNode>(N);
4114  ISD::LoadExtType ExtType = LD->getExtensionType();
4115 
4116  // A vector must always be stored in memory as-is, i.e. without any padding
4117  // between the elements, since various code depend on it, e.g. in the
4118  // handling of a bitcast of a vector type to int, which may be done with a
4119  // vector store followed by an integer load. A vector that does not have
4120  // elements that are byte-sized must therefore be stored as an integer
4121  // built out of the extracted vector elements.
4122  if (!LD->getMemoryVT().isByteSized()) {
4123  SDValue Value, NewChain;
4124  std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
4125  ReplaceValueWith(SDValue(LD, 0), Value);
4126  ReplaceValueWith(SDValue(LD, 1), NewChain);
4127  return SDValue();
4128  }
4129 
4130  SDValue Result;
4131  SmallVector<SDValue, 16> LdChain; // Chain for the series of load
4132  if (ExtType != ISD::NON_EXTLOAD)
4133  Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
4134  else
4135  Result = GenWidenVectorLoads(LdChain, LD);
4136 
4137  // If we generate a single load, we can use that for the chain. Otherwise,
4138  // build a factor node to remember the multiple loads are independent and
4139  // chain to that.
4140  SDValue NewChain;
4141  if (LdChain.size() == 1)
4142  NewChain = LdChain[0];
4143  else
4144  NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
4145 
4146  // Modified the chain - switch anything that used the old chain to use
4147  // the new one.
4148  ReplaceValueWith(SDValue(N, 1), NewChain);
4149 
4150  return Result;
4151 }
4152 
4153 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
4154 
4155  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
4156  SDValue Mask = N->getMask();
4157  EVT MaskVT = Mask.getValueType();
4158  SDValue PassThru = GetWidenedVector(N->getPassThru());
4159  ISD::LoadExtType ExtType = N->getExtensionType();
4160  SDLoc dl(N);
4161 
4162  // The mask should be widened as well
4163  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4164  MaskVT.getVectorElementType(),
4165  WidenVT.getVectorNumElements());
4166  Mask = ModifyToType(Mask, WideMaskVT, true);
4167 
4168  SDValue Res = DAG.getMaskedLoad(
4169  WidenVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask,
4170  PassThru, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(),
4171  ExtType, N->isExpandingLoad());
4172  // Legalize the chain result - switch anything that used the old chain to
4173  // use the new one.
4174  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4175  return Res;
4176 }
4177 
4178 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
4179 
4180  EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4181  SDValue Mask = N->getMask();
4182  EVT MaskVT = Mask.getValueType();
4183  SDValue PassThru = GetWidenedVector(N->getPassThru());
4184  SDValue Scale = N->getScale();
4185  unsigned NumElts = WideVT.getVectorNumElements();
4186  SDLoc dl(N);
4187 
4188  // The mask should be widened as well
4189  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4190  MaskVT.getVectorElementType(),
4191  WideVT.getVectorNumElements());
4192  Mask = ModifyToType(Mask, WideMaskVT, true);
4193 
4194  // Widen the Index operand
4195  SDValue Index = N->getIndex();
4196  EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
4197  Index.getValueType().getScalarType(),
4198  NumElts);
4199  Index = ModifyToType(Index, WideIndexVT);
4200  SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
4201  Scale };
4202 
4203  // Widen the MemoryType
4204  EVT WideMemVT = EVT::getVectorVT(*DAG.getContext(),
4205  N->getMemoryVT().getScalarType(), NumElts);
4206  SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
4207  WideMemVT, dl, Ops, N->getMemOperand(),
4208  N->getIndexType(), N->getExtensionType());
4209 
4210  // Legalize the chain result - switch anything that used the old chain to
4211  // use the new one.
4212  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4213  return Res;
4214 }
4215 
4216 SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
4217  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4218  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, N->getOperand(0));
4219 }
4220 
4221 // Return true is this is a SETCC node or a strict version of it.
4222 static inline bool isSETCCOp(unsigned Opcode) {
4223  switch (Opcode) {
4224  case ISD::SETCC:
4225  case ISD::STRICT_FSETCC:
4226  case ISD::STRICT_FSETCCS:
4227  return true;
4228  }
4229  return false;
4230 }
4231 
4232 // Return true if this is a node that could have two SETCCs as operands.
4233 static inline bool isLogicalMaskOp(unsigned Opcode) {
4234  switch (Opcode) {
4235  case ISD::AND:
4236  case ISD::OR:
4237  case ISD::XOR:
4238  return true;
4239  }
4240  return false;
4241 }
4242 
4243 // If N is a SETCC or a strict variant of it, return the type
4244 // of the compare operands.
4246  unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
4247  return N->getOperand(OpNo).getValueType();
4248 }
4249 
4250 // This is used just for the assert in convertMask(). Check that this either
4251 // a SETCC or a previously handled SETCC by convertMask().
4252 #ifndef NDEBUG
4253 static inline bool isSETCCorConvertedSETCC(SDValue N) {
4254  if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
4255  N = N.getOperand(0);
4256  else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
4257  for (unsigned i = 1; i < N->getNumOperands(); ++i)
4258  if (!N->getOperand(i)->isUndef())
4259  return false;
4260  N = N.getOperand(0);
4261  }
4262 
4263  if (N.getOpcode() == ISD::TRUNCATE)
4264  N = N.getOperand(0);
4265  else if (N.getOpcode() == ISD::SIGN_EXTEND)
4266  N = N.getOperand(0);
4267 
4268  if (isLogicalMaskOp(N.getOpcode()))
4269  return isSETCCorConvertedSETCC(N.getOperand(0)) &&
4270  isSETCCorConvertedSETCC(N.getOperand(1));
4271 
4272  return (isSETCCOp(N.getOpcode()) ||
4274 }
4275 #endif
4276 
4277 // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
4278 // to ToMaskVT if needed with vector extension or truncation.
4279 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
4280  EVT ToMaskVT) {
4281  // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
4282  // FIXME: This code seems to be too restrictive, we might consider
4283  // generalizing it or dropping it.
4284  assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
4285 
4286  // Make a new Mask node, with a legal result VT.
4287  SDValue Mask;
4289  for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
4290  Ops.push_back(InMask->getOperand(i));
4291  if (InMask->isStrictFPOpcode()) {
4292  Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask),
4293  { MaskVT, MVT::Other }, Ops);
4294  ReplaceValueWith(InMask.getValue(1), Mask.getValue(1));
4295  }
4296  else
4297  Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
4298 
4299  // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
4300  // extend or truncate is needed.
4301  LLVMContext &Ctx = *DAG.getContext();
4302  unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
4303  unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
4304  if (MaskScalarBits < ToMaskScalBits) {
4305  EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
4306  MaskVT.getVectorNumElements());
4307  Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
4308  } else if (MaskScalarBits > ToMaskScalBits) {
4309  EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
4310  MaskVT.getVectorNumElements());
4311  Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
4312  }
4313 
4314  assert(Mask->getValueType(0).getScalarSizeInBits() ==
4315  ToMaskVT.getScalarSizeInBits() &&
4316  "Mask should have the right element size by now.");
4317 
4318  // Adjust Mask to the right number of elements.
4319  unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
4320  if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
4321  SDValue ZeroIdx = DAG.getVectorIdxConstant(0, SDLoc(Mask));
4322  Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
4323  ZeroIdx);
4324  } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
4325  unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
4326  EVT SubVT = Mask->getValueType(0);
4327  SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
4328  SubOps[0] = Mask;
4329  Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
4330  }
4331 
4332  assert((Mask->getValueType(0) == ToMaskVT) &&
4333  "A mask of ToMaskVT should have been produced by now.");
4334 
4335  return Mask;
4336 }
4337 
4338 // This method tries to handle some special cases for the vselect mask
4339 // and if needed adjusting the mask vector type to match that of the VSELECT.
4340 // Without it, many cases end up with scalarization of the SETCC, with many
4341 // unnecessary instructions.
4342 SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
4343  LLVMContext &Ctx = *DAG.getContext();
4344  SDValue Cond = N->getOperand(0);
4345 
4346  if (N->getOpcode() != ISD::VSELECT)
4347  return SDValue();
4348 
4349  if (!isSETCCOp(Cond->getOpcode()) && !isLogicalMaskOp(Cond->getOpcode()))
4350  return SDValue();
4351 
4352  // If this is a splitted VSELECT that was previously already handled, do
4353  // nothing.
4354  EVT CondVT = Cond->getValueType(0);
4355  if (CondVT.getScalarSizeInBits() != 1)
4356  return SDValue();
4357 
4358  EVT VSelVT = N->getValueType(0);
4359 
4360  // This method can't handle scalable vector types.
4361  // FIXME: This support could be added in the future.
4362  if (VSelVT.isScalableVector())
4363  return SDValue();
4364 
4365  // Only handle vector types which are a power of 2.
4366  if (!isPowerOf2_64(VSelVT.getSizeInBits()))
4367  return SDValue();
4368 
4369  // Don't touch if this will be scalarized.
4370  EVT FinalVT = VSelVT;
4371  while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
4372  FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
4373 
4374  if (FinalVT.getVectorNumElements() == 1)
4375  return SDValue();
4376 
4377  // If there is support for an i1 vector mask, don't touch.
4378  if (isSETCCOp(Cond.getOpcode())) {
4379  EVT SetCCOpVT = getSETCCOperandType(Cond);
4380  while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
4381  SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
4382  EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
4383  if (SetCCResVT.getScalarSizeInBits() == 1)
4384  return SDValue();
4385  } else if (CondVT.getScalarType() == MVT::i1) {
4386  // If there is support for an i1 vector mask (or only scalar i1 conditions),
4387  // don't touch.
4388  while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal)
4389  CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
4390 
4391  if (CondVT.getScalarType() == MVT::i1)
4392  return SDValue();
4393  }
4394 
4395  // Widen the vselect result type if needed.
4396  if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector)
4397  VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
4398 
4399  // The mask of the VSELECT should have integer elements.
4400  EVT ToMaskVT = VSelVT;
4401  if (!ToMaskVT.getScalarType().isInteger())
4402  ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
4403 
4404  SDValue Mask;
4405  if (isSETCCOp(Cond->getOpcode())) {
4406  EVT MaskVT = getSetCCResultType(getSETCCOperandType(Cond));
4407  Mask = convertMask(Cond, MaskVT, ToMaskVT);
4408  } else if (isLogicalMaskOp(Cond->getOpcode()) &&
4409  isSETCCOp(Cond->getOperand(0).getOpcode()) &&
4410  isSETCCOp(Cond->getOperand(1).getOpcode())) {
4411  // Cond is (AND/OR/XOR (SETCC, SETCC))
4412  SDValue SETCC0 = Cond->getOperand(0);
4413  SDValue SETCC1 = Cond->getOperand(1);
4414  EVT VT0 = getSetCCResultType(getSETCCOperandType(SETCC0));
4415  EVT VT1 = getSetCCResultType(getSETCCOperandType(SETCC1));
4416  unsigned ScalarBits0 = VT0.getScalarSizeInBits();
4417  unsigned ScalarBits1 = VT1.getScalarSizeInBits();
4418  unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
4419  EVT MaskVT;
4420  // If the two SETCCs have different VTs, either extend/truncate one of
4421  // them to the other "towards" ToMaskVT, or truncate one and extend the
4422  // other to ToMaskVT.
4423