LLVM  16.0.0git
LegalizeVectorTypes.cpp
Go to the documentation of this file.
1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file performs vector type splitting and scalarization for LegalizeTypes.
10 // Scalarization is the act of changing a computation in an illegal one-element
11 // vector type to be a computation in its scalar element type. For example,
12 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
14 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15 // types.
16 // Splitting is the act of changing a computation in an invalid vector type to
17 // be a computation in two vectors of half the size. For example, implementing
18 // <128 x f32> operations in terms of two <64 x f32> operations.
19 //
20 //===----------------------------------------------------------------------===//
21 
22 #include "LegalizeTypes.h"
26 #include "llvm/IR/DataLayout.h"
28 #include "llvm/Support/TypeSize.h"
30 #include <numeric>
31 
32 using namespace llvm;
33 
34 #define DEBUG_TYPE "legalize-types"
35 
36 //===----------------------------------------------------------------------===//
37 // Result Vector Scalarization: <1 x ty> -> ty.
38 //===----------------------------------------------------------------------===//
39 
40 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
41  LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
42  dbgs() << "\n");
43  SDValue R = SDValue();
44 
45  switch (N->getOpcode()) {
46  default:
47 #ifndef NDEBUG
48  dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
49  N->dump(&DAG);
50  dbgs() << "\n";
51 #endif
52  report_fatal_error("Do not know how to scalarize the result of this "
53  "operator!\n");
54 
55  case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
56  case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
57  case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
58  case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
59  case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
60  case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
61  case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
62  case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
63  case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
64  case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
65  case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
66  case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
67  case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
68  case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
69  case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
70  case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
71  case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
75  R = ScalarizeVecRes_VecInregOp(N);
76  break;
77  case ISD::ABS:
78  case ISD::ANY_EXTEND:
79  case ISD::BITREVERSE:
80  case ISD::BSWAP:
81  case ISD::CTLZ:
83  case ISD::CTPOP:
84  case ISD::CTTZ:
86  case ISD::FABS:
87  case ISD::FCEIL:
88  case ISD::FCOS:
89  case ISD::FEXP:
90  case ISD::FEXP2:
91  case ISD::FFLOOR:
92  case ISD::FLOG:
93  case ISD::FLOG10:
94  case ISD::FLOG2:
95  case ISD::FNEARBYINT:
96  case ISD::FNEG:
97  case ISD::FREEZE:
98  case ISD::ARITH_FENCE:
99  case ISD::FP_EXTEND:
100  case ISD::FP_TO_SINT:
101  case ISD::FP_TO_UINT:
102  case ISD::FRINT:
103  case ISD::FROUND:
104  case ISD::FROUNDEVEN:
105  case ISD::FSIN:
106  case ISD::FSQRT:
107  case ISD::FTRUNC:
108  case ISD::SIGN_EXTEND:
109  case ISD::SINT_TO_FP:
110  case ISD::TRUNCATE:
111  case ISD::UINT_TO_FP:
112  case ISD::ZERO_EXTEND:
113  case ISD::FCANONICALIZE:
114  R = ScalarizeVecRes_UnaryOp(N);
115  break;
116 
117  case ISD::ADD:
118  case ISD::AND:
119  case ISD::FADD:
120  case ISD::FCOPYSIGN:
121  case ISD::FDIV:
122  case ISD::FMUL:
123  case ISD::FMINNUM:
124  case ISD::FMAXNUM:
125  case ISD::FMINNUM_IEEE:
126  case ISD::FMAXNUM_IEEE:
127  case ISD::FMINIMUM:
128  case ISD::FMAXIMUM:
129  case ISD::SMIN:
130  case ISD::SMAX:
131  case ISD::UMIN:
132  case ISD::UMAX:
133 
134  case ISD::SADDSAT:
135  case ISD::UADDSAT:
136  case ISD::SSUBSAT:
137  case ISD::USUBSAT:
138  case ISD::SSHLSAT:
139  case ISD::USHLSAT:
140 
141  case ISD::FPOW:
142  case ISD::FREM:
143  case ISD::FSUB:
144  case ISD::MUL:
145  case ISD::OR:
146  case ISD::SDIV:
147  case ISD::SREM:
148  case ISD::SUB:
149  case ISD::UDIV:
150  case ISD::UREM:
151  case ISD::XOR:
152  case ISD::SHL:
153  case ISD::SRA:
154  case ISD::SRL:
155  case ISD::ROTL:
156  case ISD::ROTR:
157  R = ScalarizeVecRes_BinOp(N);
158  break;
159  case ISD::FMA:
160  case ISD::FSHL:
161  case ISD::FSHR:
162  R = ScalarizeVecRes_TernaryOp(N);
163  break;
164 
165 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
166  case ISD::STRICT_##DAGN:
167 #include "llvm/IR/ConstrainedOps.def"
168  R = ScalarizeVecRes_StrictFPOp(N);
169  break;
170 
171  case ISD::FP_TO_UINT_SAT:
172  case ISD::FP_TO_SINT_SAT:
173  R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
174  break;
175 
176  case ISD::UADDO:
177  case ISD::SADDO:
178  case ISD::USUBO:
179  case ISD::SSUBO:
180  case ISD::UMULO:
181  case ISD::SMULO:
182  R = ScalarizeVecRes_OverflowOp(N, ResNo);
183  break;
184  case ISD::SMULFIX:
185  case ISD::SMULFIXSAT:
186  case ISD::UMULFIX:
187  case ISD::UMULFIXSAT:
188  case ISD::SDIVFIX:
189  case ISD::SDIVFIXSAT:
190  case ISD::UDIVFIX:
191  case ISD::UDIVFIXSAT:
192  R = ScalarizeVecRes_FIX(N);
193  break;
194  }
195 
196  // If R is null, the sub-method took care of registering the result.
197  if (R.getNode())
198  SetScalarizedVector(SDValue(N, ResNo), R);
199 }
200 
201 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
202  SDValue LHS = GetScalarizedVector(N->getOperand(0));
203  SDValue RHS = GetScalarizedVector(N->getOperand(1));
204  return DAG.getNode(N->getOpcode(), SDLoc(N),
205  LHS.getValueType(), LHS, RHS, N->getFlags());
206 }
207 
208 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
209  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
210  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
211  SDValue Op2 = GetScalarizedVector(N->getOperand(2));
212  return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
213  Op2, N->getFlags());
214 }
215 
216 SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
217  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
218  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
219  SDValue Op2 = N->getOperand(2);
220  return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
221  Op2, N->getFlags());
222 }
223 
224 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
225  EVT VT = N->getValueType(0).getVectorElementType();
226  unsigned NumOpers = N->getNumOperands();
227  SDValue Chain = N->getOperand(0);
228  EVT ValueVTs[] = {VT, MVT::Other};
229  SDLoc dl(N);
230 
231  SmallVector<SDValue, 4> Opers(NumOpers);
232 
233  // The Chain is the first operand.
234  Opers[0] = Chain;
235 
236  // Now process the remaining operands.
237  for (unsigned i = 1; i < NumOpers; ++i) {
238  SDValue Oper = N->getOperand(i);
239  EVT OperVT = Oper.getValueType();
240 
241  if (OperVT.isVector()) {
242  if (getTypeAction(OperVT) == TargetLowering::TypeScalarizeVector)
243  Oper = GetScalarizedVector(Oper);
244  else
245  Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
246  OperVT.getVectorElementType(), Oper,
247  DAG.getVectorIdxConstant(0, dl));
248  }
249 
250  Opers[i] = Oper;
251  }
252 
253  SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs),
254  Opers, N->getFlags());
255 
256  // Legalize the chain result - switch anything that used the old chain to
257  // use the new one.
258  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
259  return Result;
260 }
261 
262 SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
263  unsigned ResNo) {
264  SDLoc DL(N);
265  EVT ResVT = N->getValueType(0);
266  EVT OvVT = N->getValueType(1);
267 
268  SDValue ScalarLHS, ScalarRHS;
269  if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
270  ScalarLHS = GetScalarizedVector(N->getOperand(0));
271  ScalarRHS = GetScalarizedVector(N->getOperand(1));
272  } else {
273  SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
274  DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
275  DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
276  ScalarLHS = ElemsLHS[0];
277  ScalarRHS = ElemsRHS[0];
278  }
279 
280  SDVTList ScalarVTs = DAG.getVTList(
282  SDNode *ScalarNode = DAG.getNode(
283  N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
284  ScalarNode->setFlags(N->getFlags());
285 
286  // Replace the other vector result not being explicitly scalarized here.
287  unsigned OtherNo = 1 - ResNo;
288  EVT OtherVT = N->getValueType(OtherNo);
289  if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
290  SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
291  } else {
292  SDValue OtherVal = DAG.getNode(
293  ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
294  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
295  }
296 
297  return SDValue(ScalarNode, ResNo);
298 }
299 
300 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
301  unsigned ResNo) {
302  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
303  return GetScalarizedVector(Op);
304 }
305 
306 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
307  SDValue Op = N->getOperand(0);
308  if (Op.getValueType().isVector()
309  && Op.getValueType().getVectorNumElements() == 1
310  && !isSimpleLegalType(Op.getValueType()))
311  Op = GetScalarizedVector(Op);
312  EVT NewVT = N->getValueType(0).getVectorElementType();
313  return DAG.getNode(ISD::BITCAST, SDLoc(N),
314  NewVT, Op);
315 }
316 
317 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
318  EVT EltVT = N->getValueType(0).getVectorElementType();
319  SDValue InOp = N->getOperand(0);
320  // The BUILD_VECTOR operands may be of wider element types and
321  // we may need to truncate them back to the requested return type.
322  if (EltVT.isInteger())
323  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
324  return InOp;
325 }
326 
327 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
328  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
329  N->getValueType(0).getVectorElementType(),
330  N->getOperand(0), N->getOperand(1));
331 }
332 
333 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
334  SDLoc DL(N);
335  SDValue Op = N->getOperand(0);
336  EVT OpVT = Op.getValueType();
337  // The result needs scalarizing, but it's not a given that the source does.
338  // See similar logic in ScalarizeVecRes_UnaryOp.
339  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
340  Op = GetScalarizedVector(Op);
341  } else {
342  EVT VT = OpVT.getVectorElementType();
343  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
344  DAG.getVectorIdxConstant(0, DL));
345  }
346  return DAG.getNode(ISD::FP_ROUND, DL,
347  N->getValueType(0).getVectorElementType(), Op,
348  N->getOperand(1));
349 }
350 
351 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
352  SDValue Op = GetScalarizedVector(N->getOperand(0));
353  return DAG.getNode(ISD::FPOWI, SDLoc(N),
354  Op.getValueType(), Op, N->getOperand(1));
355 }
356 
357 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
358  // The value to insert may have a wider type than the vector element type,
359  // so be sure to truncate it to the element type if necessary.
360  SDValue Op = N->getOperand(1);
361  EVT EltVT = N->getValueType(0).getVectorElementType();
362  if (Op.getValueType() != EltVT)
363  // FIXME: Can this happen for floating point types?
364  Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
365  return Op;
366 }
367 
368 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
369  assert(N->isUnindexed() && "Indexed vector load?");
370 
371  SDValue Result = DAG.getLoad(
372  ISD::UNINDEXED, N->getExtensionType(),
373  N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
374  N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
375  N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
376  N->getOriginalAlign(), N->getMemOperand()->getFlags(), N->getAAInfo());
377 
378  // Legalize the chain result - switch anything that used the old chain to
379  // use the new one.
380  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
381  return Result;
382 }
383 
384 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
385  // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
386  EVT DestVT = N->getValueType(0).getVectorElementType();
387  SDValue Op = N->getOperand(0);
388  EVT OpVT = Op.getValueType();
389  SDLoc DL(N);
390  // The result needs scalarizing, but it's not a given that the source does.
391  // This is a workaround for targets where it's impossible to scalarize the
392  // result of a conversion, because the source type is legal.
393  // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
394  // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
395  // legal and was not scalarized.
396  // See the similar logic in ScalarizeVecRes_SETCC
397  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
398  Op = GetScalarizedVector(Op);
399  } else {
400  EVT VT = OpVT.getVectorElementType();
401  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
402  DAG.getVectorIdxConstant(0, DL));
403  }
404  return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op, N->getFlags());
405 }
406 
407 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
408  EVT EltVT = N->getValueType(0).getVectorElementType();
409  EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
410  SDValue LHS = GetScalarizedVector(N->getOperand(0));
411  return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
412  LHS, DAG.getValueType(ExtVT));
413 }
414 
415 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
416  SDLoc DL(N);
417  SDValue Op = N->getOperand(0);
418 
419  EVT OpVT = Op.getValueType();
420  EVT OpEltVT = OpVT.getVectorElementType();
421  EVT EltVT = N->getValueType(0).getVectorElementType();
422 
423  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
424  Op = GetScalarizedVector(Op);
425  } else {
426  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
427  DAG.getVectorIdxConstant(0, DL));
428  }
429 
430  switch (N->getOpcode()) {
432  return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
434  return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
436  return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
437  }
438 
439  llvm_unreachable("Illegal extend_vector_inreg opcode");
440 }
441 
442 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
443  // If the operand is wider than the vector element type then it is implicitly
444  // truncated. Make that explicit here.
445  EVT EltVT = N->getValueType(0).getVectorElementType();
446  SDValue InOp = N->getOperand(0);
447  if (InOp.getValueType() != EltVT)
448  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
449  return InOp;
450 }
451 
452 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
453  SDValue Cond = N->getOperand(0);
454  EVT OpVT = Cond.getValueType();
455  SDLoc DL(N);
456  // The vselect result and true/value operands needs scalarizing, but it's
457  // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
458  // See the similar logic in ScalarizeVecRes_SETCC
459  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
460  Cond = GetScalarizedVector(Cond);
461  } else {
462  EVT VT = OpVT.getVectorElementType();
464  DAG.getVectorIdxConstant(0, DL));
465  }
466 
467  SDValue LHS = GetScalarizedVector(N->getOperand(1));
468  TargetLowering::BooleanContent ScalarBool =
469  TLI.getBooleanContents(false, false);
470  TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
471 
472  // If integer and float booleans have different contents then we can't
473  // reliably optimize in all cases. There is a full explanation for this in
474  // DAGCombiner::visitSELECT() where the same issue affects folding
475  // (select C, 0, 1) to (xor C, 1).
476  if (TLI.getBooleanContents(false, false) !=
477  TLI.getBooleanContents(false, true)) {
478  // At least try the common case where the boolean is generated by a
479  // comparison.
480  if (Cond->getOpcode() == ISD::SETCC) {
481  EVT OpVT = Cond->getOperand(0).getValueType();
482  ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
483  VecBool = TLI.getBooleanContents(OpVT);
484  } else
486  }
487 
488  EVT CondVT = Cond.getValueType();
489  if (ScalarBool != VecBool) {
490  switch (ScalarBool) {
492  break;
496  // Vector read from all ones, scalar expects a single 1 so mask.
497  Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
498  Cond, DAG.getConstant(1, SDLoc(N), CondVT));
499  break;
503  // Vector reads from a one, scalar from all ones so sign extend.
504  Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
505  Cond, DAG.getValueType(MVT::i1));
506  break;
507  }
508  }
509 
510  // Truncate the condition if needed
511  auto BoolVT = getSetCCResultType(CondVT);
512  if (BoolVT.bitsLT(CondVT))
513  Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
514 
515  return DAG.getSelect(SDLoc(N),
516  LHS.getValueType(), Cond, LHS,
517  GetScalarizedVector(N->getOperand(2)));
518 }
519 
520 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
521  SDValue LHS = GetScalarizedVector(N->getOperand(1));
522  return DAG.getSelect(SDLoc(N),
523  LHS.getValueType(), N->getOperand(0), LHS,
524  GetScalarizedVector(N->getOperand(2)));
525 }
526 
527 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
528  SDValue LHS = GetScalarizedVector(N->getOperand(2));
529  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
530  N->getOperand(0), N->getOperand(1),
531  LHS, GetScalarizedVector(N->getOperand(3)),
532  N->getOperand(4));
533 }
534 
535 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
536  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
537 }
538 
539 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
540  // Figure out if the scalar is the LHS or RHS and return it.
541  SDValue Arg = N->getOperand(2).getOperand(0);
542  if (Arg.isUndef())
543  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
544  unsigned Op = !cast<ConstantSDNode>(Arg)->isZero();
545  return GetScalarizedVector(N->getOperand(Op));
546 }
547 
548 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
549  SDValue Src = N->getOperand(0);
550  EVT SrcVT = Src.getValueType();
551  SDLoc dl(N);
552 
553  // Handle case where result is scalarized but operand is not
554  if (getTypeAction(SrcVT) == TargetLowering::TypeScalarizeVector)
555  Src = GetScalarizedVector(Src);
556  else
557  Src = DAG.getNode(
559  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
560 
561  EVT DstVT = N->getValueType(0).getVectorElementType();
562  return DAG.getNode(N->getOpcode(), dl, DstVT, Src, N->getOperand(1));
563 }
564 
565 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
566  assert(N->getValueType(0).isVector() &&
567  N->getOperand(0).getValueType().isVector() &&
568  "Operand types must be vectors");
569  SDValue LHS = N->getOperand(0);
570  SDValue RHS = N->getOperand(1);
571  EVT OpVT = LHS.getValueType();
572  EVT NVT = N->getValueType(0).getVectorElementType();
573  SDLoc DL(N);
574 
575  // The result needs scalarizing, but it's not a given that the source does.
576  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
577  LHS = GetScalarizedVector(LHS);
578  RHS = GetScalarizedVector(RHS);
579  } else {
580  EVT VT = OpVT.getVectorElementType();
582  DAG.getVectorIdxConstant(0, DL));
584  DAG.getVectorIdxConstant(0, DL));
585  }
586 
587  // Turn it into a scalar SETCC.
588  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
589  N->getOperand(2));
590  // Vectors may have a different boolean contents to scalars. Promote the
591  // value appropriately.
592  ISD::NodeType ExtendCode =
594  return DAG.getNode(ExtendCode, DL, NVT, Res);
595 }
596 
597 SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
598  SDLoc DL(N);
599  SDValue Arg = N->getOperand(0);
600  SDValue Test = N->getOperand(1);
601  EVT ArgVT = Arg.getValueType();
602  EVT ResultVT = N->getValueType(0).getVectorElementType();
603 
604  if (getTypeAction(ArgVT) == TargetLowering::TypeScalarizeVector) {
605  Arg = GetScalarizedVector(Arg);
606  } else {
607  EVT VT = ArgVT.getVectorElementType();
609  DAG.getVectorIdxConstant(0, DL));
610  }
611 
612  SDValue Res =
613  DAG.getNode(ISD::IS_FPCLASS, DL, MVT::i1, {Arg, Test}, N->getFlags());
614  // Vectors may have a different boolean contents to scalars. Promote the
615  // value appropriately.
616  ISD::NodeType ExtendCode =
618  return DAG.getNode(ExtendCode, DL, ResultVT, Res);
619 }
620 
621 //===----------------------------------------------------------------------===//
622 // Operand Vector Scalarization <1 x ty> -> ty.
623 //===----------------------------------------------------------------------===//
624 
625 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
626  LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
627  dbgs() << "\n");
628  SDValue Res = SDValue();
629 
630  switch (N->getOpcode()) {
631  default:
632 #ifndef NDEBUG
633  dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
634  N->dump(&DAG);
635  dbgs() << "\n";
636 #endif
637  report_fatal_error("Do not know how to scalarize this operator's "
638  "operand!\n");
639  case ISD::BITCAST:
640  Res = ScalarizeVecOp_BITCAST(N);
641  break;
642  case ISD::ANY_EXTEND:
643  case ISD::ZERO_EXTEND:
644  case ISD::SIGN_EXTEND:
645  case ISD::TRUNCATE:
646  case ISD::FP_TO_SINT:
647  case ISD::FP_TO_UINT:
648  case ISD::SINT_TO_FP:
649  case ISD::UINT_TO_FP:
650  Res = ScalarizeVecOp_UnaryOp(N);
651  break;
656  Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
657  break;
658  case ISD::CONCAT_VECTORS:
659  Res = ScalarizeVecOp_CONCAT_VECTORS(N);
660  break;
662  Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
663  break;
664  case ISD::VSELECT:
665  Res = ScalarizeVecOp_VSELECT(N);
666  break;
667  case ISD::SETCC:
668  Res = ScalarizeVecOp_VSETCC(N);
669  break;
670  case ISD::STORE:
671  Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
672  break;
674  Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
675  break;
676  case ISD::FP_ROUND:
677  Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
678  break;
680  Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
681  break;
682  case ISD::FP_EXTEND:
683  Res = ScalarizeVecOp_FP_EXTEND(N);
684  break;
685  case ISD::VECREDUCE_FADD:
686  case ISD::VECREDUCE_FMUL:
687  case ISD::VECREDUCE_ADD:
688  case ISD::VECREDUCE_MUL:
689  case ISD::VECREDUCE_AND:
690  case ISD::VECREDUCE_OR:
691  case ISD::VECREDUCE_XOR:
692  case ISD::VECREDUCE_SMAX:
693  case ISD::VECREDUCE_SMIN:
694  case ISD::VECREDUCE_UMAX:
695  case ISD::VECREDUCE_UMIN:
696  case ISD::VECREDUCE_FMAX:
697  case ISD::VECREDUCE_FMIN:
698  Res = ScalarizeVecOp_VECREDUCE(N);
699  break;
702  Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
703  break;
704  }
705 
706  // If the result is null, the sub-method took care of registering results etc.
707  if (!Res.getNode()) return false;
708 
709  // If the result is N, the sub-method updated N in place. Tell the legalizer
710  // core about this.
711  if (Res.getNode() == N)
712  return true;
713 
714  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
715  "Invalid operand expansion");
716 
717  ReplaceValueWith(SDValue(N, 0), Res);
718  return false;
719 }
720 
721 /// If the value to convert is a vector that needs to be scalarized, it must be
722 /// <1 x ty>. Convert the element instead.
723 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
724  SDValue Elt = GetScalarizedVector(N->getOperand(0));
725  return DAG.getNode(ISD::BITCAST, SDLoc(N),
726  N->getValueType(0), Elt);
727 }
728 
729 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
730 /// Do the operation on the element instead.
731 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
732  assert(N->getValueType(0).getVectorNumElements() == 1 &&
733  "Unexpected vector type!");
734  SDValue Elt = GetScalarizedVector(N->getOperand(0));
735  SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
736  N->getValueType(0).getScalarType(), Elt);
737  // Revectorize the result so the types line up with what the uses of this
738  // expression expect.
739  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
740 }
741 
742 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
743 /// Do the strict FP operation on the element instead.
744 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
745  assert(N->getValueType(0).getVectorNumElements() == 1 &&
746  "Unexpected vector type!");
747  SDValue Elt = GetScalarizedVector(N->getOperand(1));
748  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
749  { N->getValueType(0).getScalarType(), MVT::Other },
750  { N->getOperand(0), Elt });
751  // Legalize the chain result - switch anything that used the old chain to
752  // use the new one.
753  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
754  // Revectorize the result so the types line up with what the uses of this
755  // expression expect.
756  Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
757 
758  // Do our own replacement and return SDValue() to tell the caller that we
759  // handled all replacements since caller can only handle a single result.
760  ReplaceValueWith(SDValue(N, 0), Res);
761  return SDValue();
762 }
763 
764 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
765 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
766  SmallVector<SDValue, 8> Ops(N->getNumOperands());
767  for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
768  Ops[i] = GetScalarizedVector(N->getOperand(i));
769  return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
770 }
771 
772 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
773 /// so just return the element, ignoring the index.
774 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
775  EVT VT = N->getValueType(0);
776  SDValue Res = GetScalarizedVector(N->getOperand(0));
777  if (Res.getValueType() != VT)
778  Res = VT.isFloatingPoint()
779  ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
780  : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
781  return Res;
782 }
783 
784 /// If the input condition is a vector that needs to be scalarized, it must be
785 /// <1 x i1>, so just convert to a normal ISD::SELECT
786 /// (still with vector output type since that was acceptable if we got here).
787 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
788  SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
789  EVT VT = N->getValueType(0);
790 
791  return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
792  N->getOperand(2));
793 }
794 
795 /// If the operand is a vector that needs to be scalarized then the
796 /// result must be v1i1, so just convert to a scalar SETCC and wrap
797 /// with a scalar_to_vector since the res type is legal if we got here
798 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
799  assert(N->getValueType(0).isVector() &&
800  N->getOperand(0).getValueType().isVector() &&
801  "Operand types must be vectors");
802  assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
803 
804  EVT VT = N->getValueType(0);
805  SDValue LHS = GetScalarizedVector(N->getOperand(0));
806  SDValue RHS = GetScalarizedVector(N->getOperand(1));
807 
808  EVT OpVT = N->getOperand(0).getValueType();
809  EVT NVT = VT.getVectorElementType();
810  SDLoc DL(N);
811  // Turn it into a scalar SETCC.
812  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
813  N->getOperand(2));
814 
815  // Vectors may have a different boolean contents to scalars. Promote the
816  // value appropriately.
817  ISD::NodeType ExtendCode =
819 
820  Res = DAG.getNode(ExtendCode, DL, NVT, Res);
821 
822  return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
823 }
824 
825 /// If the value to store is a vector that needs to be scalarized, it must be
826 /// <1 x ty>. Just store the element.
827 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
828  assert(N->isUnindexed() && "Indexed store of one-element vector?");
829  assert(OpNo == 1 && "Do not know how to scalarize this operand!");
830  SDLoc dl(N);
831 
832  if (N->isTruncatingStore())
833  return DAG.getTruncStore(
834  N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
835  N->getBasePtr(), N->getPointerInfo(),
836  N->getMemoryVT().getVectorElementType(), N->getOriginalAlign(),
837  N->getMemOperand()->getFlags(), N->getAAInfo());
838 
839  return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
840  N->getBasePtr(), N->getPointerInfo(),
841  N->getOriginalAlign(), N->getMemOperand()->getFlags(),
842  N->getAAInfo());
843 }
844 
845 /// If the value to round is a vector that needs to be scalarized, it must be
846 /// <1 x ty>. Convert the element instead.
847 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
848  assert(OpNo == 0 && "Wrong operand for scalarization!");
849  SDValue Elt = GetScalarizedVector(N->getOperand(0));
850  SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
851  N->getValueType(0).getVectorElementType(), Elt,
852  N->getOperand(1));
853  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
854 }
855 
856 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
857  unsigned OpNo) {
858  assert(OpNo == 1 && "Wrong operand for scalarization!");
859  SDValue Elt = GetScalarizedVector(N->getOperand(1));
861  { N->getValueType(0).getVectorElementType(),
862  MVT::Other },
863  { N->getOperand(0), Elt, N->getOperand(2) });
864  // Legalize the chain result - switch anything that used the old chain to
865  // use the new one.
866  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
867 
868  Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
869 
870  // Do our own replacement and return SDValue() to tell the caller that we
871  // handled all replacements since caller can only handle a single result.
872  ReplaceValueWith(SDValue(N, 0), Res);
873  return SDValue();
874 }
875 
876 /// If the value to extend is a vector that needs to be scalarized, it must be
877 /// <1 x ty>. Convert the element instead.
878 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
879  SDValue Elt = GetScalarizedVector(N->getOperand(0));
880  SDValue Res = DAG.getNode(ISD::FP_EXTEND, SDLoc(N),
881  N->getValueType(0).getVectorElementType(), Elt);
882  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
883 }
884 
885 /// If the value to extend is a vector that needs to be scalarized, it must be
886 /// <1 x ty>. Convert the element instead.
887 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
888  SDValue Elt = GetScalarizedVector(N->getOperand(1));
889  SDValue Res =
891  {N->getValueType(0).getVectorElementType(), MVT::Other},
892  {N->getOperand(0), Elt});
893  // Legalize the chain result - switch anything that used the old chain to
894  // use the new one.
895  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
896 
897  Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
898 
899  // Do our own replacement and return SDValue() to tell the caller that we
900  // handled all replacements since caller can only handle a single result.
901  ReplaceValueWith(SDValue(N, 0), Res);
902  return SDValue();
903 }
904 
905 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
906  SDValue Res = GetScalarizedVector(N->getOperand(0));
907  // Result type may be wider than element type.
908  if (Res.getValueType() != N->getValueType(0))
909  Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res);
910  return Res;
911 }
912 
913 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
914  SDValue AccOp = N->getOperand(0);
915  SDValue VecOp = N->getOperand(1);
916 
917  unsigned BaseOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
918 
919  SDValue Op = GetScalarizedVector(VecOp);
920  return DAG.getNode(BaseOpc, SDLoc(N), N->getValueType(0),
921  AccOp, Op, N->getFlags());
922 }
923 
924 //===----------------------------------------------------------------------===//
925 // Result Vector Splitting
926 //===----------------------------------------------------------------------===//
927 
928 /// This method is called when the specified result of the specified node is
929 /// found to need vector splitting. At this point, the node may also have
930 /// invalid operands or may have other results that need legalization, we just
931 /// know that (at least) one result needs vector splitting.
932 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
933  LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
934  SDValue Lo, Hi;
935 
936  // See if the target wants to custom expand this node.
937  if (CustomLowerNode(N, N->getValueType(ResNo), true))
938  return;
939 
940  switch (N->getOpcode()) {
941  default:
942 #ifndef NDEBUG
943  dbgs() << "SplitVectorResult #" << ResNo << ": ";
944  N->dump(&DAG);
945  dbgs() << "\n";
946 #endif
947  report_fatal_error("Do not know how to split the result of this "
948  "operator!\n");
949 
950  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
951  case ISD::VSELECT:
952  case ISD::SELECT:
953  case ISD::VP_MERGE:
954  case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break;
955  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
956  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
957  case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
958  case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
959  case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
960  case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
961  case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
962  case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
963  case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
964  case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
965  case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
966  case ISD::SPLAT_VECTOR:
968  SplitVecRes_ScalarOp(N, Lo, Hi);
969  break;
970  case ISD::STEP_VECTOR:
971  SplitVecRes_STEP_VECTOR(N, Lo, Hi);
972  break;
973  case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
974  case ISD::LOAD:
975  SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
976  break;
977  case ISD::VP_LOAD:
978  SplitVecRes_VP_LOAD(cast<VPLoadSDNode>(N), Lo, Hi);
979  break;
980  case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
981  SplitVecRes_VP_STRIDED_LOAD(cast<VPStridedLoadSDNode>(N), Lo, Hi);
982  break;
983  case ISD::MLOAD:
984  SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
985  break;
986  case ISD::MGATHER:
987  case ISD::VP_GATHER:
988  SplitVecRes_Gather(cast<MemSDNode>(N), Lo, Hi, /*SplitSETCC*/ true);
989  break;
990  case ISD::SETCC:
991  case ISD::VP_SETCC:
992  SplitVecRes_SETCC(N, Lo, Hi);
993  break;
994  case ISD::VECTOR_REVERSE:
995  SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
996  break;
997  case ISD::VECTOR_SHUFFLE:
998  SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
999  break;
1000  case ISD::VECTOR_SPLICE:
1001  SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
1002  break;
1003  case ISD::VAARG:
1004  SplitVecRes_VAARG(N, Lo, Hi);
1005  break;
1006 
1010  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1011  break;
1012 
1013  case ISD::ABS:
1014  case ISD::BITREVERSE:
1015  case ISD::BSWAP:
1016  case ISD::CTLZ:
1017  case ISD::CTTZ:
1018  case ISD::CTLZ_ZERO_UNDEF:
1019  case ISD::CTTZ_ZERO_UNDEF:
1020  case ISD::CTPOP:
1021  case ISD::FABS: case ISD::VP_FABS:
1022  case ISD::FCEIL:
1023  case ISD::VP_FCEIL:
1024  case ISD::FCOS:
1025  case ISD::FEXP:
1026  case ISD::FEXP2:
1027  case ISD::FFLOOR:
1028  case ISD::VP_FFLOOR:
1029  case ISD::FLOG:
1030  case ISD::FLOG10:
1031  case ISD::FLOG2:
1032  case ISD::FNEARBYINT:
1033  case ISD::FNEG: case ISD::VP_FNEG:
1034  case ISD::FREEZE:
1035  case ISD::ARITH_FENCE:
1036  case ISD::FP_EXTEND:
1037  case ISD::VP_FP_EXTEND:
1038  case ISD::FP_ROUND:
1039  case ISD::VP_FP_ROUND:
1040  case ISD::FP_TO_SINT:
1041  case ISD::VP_FP_TO_SINT:
1042  case ISD::FP_TO_UINT:
1043  case ISD::VP_FP_TO_UINT:
1044  case ISD::FRINT:
1045  case ISD::FROUND:
1046  case ISD::VP_FROUND:
1047  case ISD::FROUNDEVEN:
1048  case ISD::VP_FROUNDEVEN:
1049  case ISD::FSIN:
1050  case ISD::FSQRT: case ISD::VP_SQRT:
1051  case ISD::FTRUNC:
1052  case ISD::SINT_TO_FP:
1053  case ISD::VP_SINT_TO_FP:
1054  case ISD::TRUNCATE:
1055  case ISD::VP_TRUNCATE:
1056  case ISD::UINT_TO_FP:
1057  case ISD::VP_UINT_TO_FP:
1058  case ISD::FCANONICALIZE:
1059  SplitVecRes_UnaryOp(N, Lo, Hi);
1060  break;
1061 
1062  case ISD::ANY_EXTEND:
1063  case ISD::SIGN_EXTEND:
1064  case ISD::ZERO_EXTEND:
1065  case ISD::VP_SIGN_EXTEND:
1066  case ISD::VP_ZERO_EXTEND:
1067  SplitVecRes_ExtendOp(N, Lo, Hi);
1068  break;
1069 
1070  case ISD::ADD: case ISD::VP_ADD:
1071  case ISD::SUB: case ISD::VP_SUB:
1072  case ISD::MUL: case ISD::VP_MUL:
1073  case ISD::MULHS:
1074  case ISD::MULHU:
1075  case ISD::FADD: case ISD::VP_FADD:
1076  case ISD::FSUB: case ISD::VP_FSUB:
1077  case ISD::FMUL: case ISD::VP_FMUL:
1078  case ISD::FMINNUM: case ISD::VP_FMINNUM:
1079  case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
1080  case ISD::FMINIMUM:
1081  case ISD::FMAXIMUM:
1082  case ISD::SDIV: case ISD::VP_SDIV:
1083  case ISD::UDIV: case ISD::VP_UDIV:
1084  case ISD::FDIV: case ISD::VP_FDIV:
1085  case ISD::FPOW:
1086  case ISD::AND: case ISD::VP_AND:
1087  case ISD::OR: case ISD::VP_OR:
1088  case ISD::XOR: case ISD::VP_XOR:
1089  case ISD::SHL: case ISD::VP_SHL:
1090  case ISD::SRA: case ISD::VP_ASHR:
1091  case ISD::SRL: case ISD::VP_LSHR:
1092  case ISD::UREM: case ISD::VP_UREM:
1093  case ISD::SREM: case ISD::VP_SREM:
1094  case ISD::FREM: case ISD::VP_FREM:
1095  case ISD::SMIN:
1096  case ISD::SMAX:
1097  case ISD::UMIN:
1098  case ISD::UMAX:
1099  case ISD::SADDSAT:
1100  case ISD::UADDSAT:
1101  case ISD::SSUBSAT:
1102  case ISD::USUBSAT:
1103  case ISD::SSHLSAT:
1104  case ISD::USHLSAT:
1105  case ISD::ROTL:
1106  case ISD::ROTR:
1107  SplitVecRes_BinOp(N, Lo, Hi);
1108  break;
1109  case ISD::FMA: case ISD::VP_FMA:
1110  case ISD::FSHL:
1111  case ISD::FSHR:
1112  SplitVecRes_TernaryOp(N, Lo, Hi);
1113  break;
1114 
1115 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1116  case ISD::STRICT_##DAGN:
1117 #include "llvm/IR/ConstrainedOps.def"
1118  SplitVecRes_StrictFPOp(N, Lo, Hi);
1119  break;
1120 
1121  case ISD::FP_TO_UINT_SAT:
1122  case ISD::FP_TO_SINT_SAT:
1123  SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1124  break;
1125 
1126  case ISD::UADDO:
1127  case ISD::SADDO:
1128  case ISD::USUBO:
1129  case ISD::SSUBO:
1130  case ISD::UMULO:
1131  case ISD::SMULO:
1132  SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1133  break;
1134  case ISD::SMULFIX:
1135  case ISD::SMULFIXSAT:
1136  case ISD::UMULFIX:
1137  case ISD::UMULFIXSAT:
1138  case ISD::SDIVFIX:
1139  case ISD::SDIVFIXSAT:
1140  case ISD::UDIVFIX:
1141  case ISD::UDIVFIXSAT:
1142  SplitVecRes_FIX(N, Lo, Hi);
1143  break;
1144  }
1145 
1146  // If Lo/Hi is null, the sub-method took care of registering results etc.
1147  if (Lo.getNode())
1148  SetSplitVector(SDValue(N, ResNo), Lo, Hi);
1149 }
1150 
1151 void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1153  uint64_t *ScaledOffset) {
1154  SDLoc DL(N);
1155  unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinSize() / 8;
1156 
1157  if (MemVT.isScalableVector()) {
1158  SDNodeFlags Flags;
1159  SDValue BytesIncrement = DAG.getVScale(
1160  DL, Ptr.getValueType(),
1161  APInt(Ptr.getValueSizeInBits().getFixedSize(), IncrementSize));
1162  MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1163  Flags.setNoUnsignedWrap(true);
1164  if (ScaledOffset)
1165  *ScaledOffset += IncrementSize;
1166  Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, BytesIncrement,
1167  Flags);
1168  } else {
1169  MPI = N->getPointerInfo().getWithOffset(IncrementSize);
1170  // Increment the pointer to the other half.
1171  Ptr = DAG.getObjectPtrOffset(DL, Ptr, TypeSize::Fixed(IncrementSize));
1172  }
1173 }
1174 
1175 std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1176  return SplitMask(Mask, SDLoc(Mask));
1177 }
1178 
1179 std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1180  const SDLoc &DL) {
1181  SDValue MaskLo, MaskHi;
1182  EVT MaskVT = Mask.getValueType();
1183  if (getTypeAction(MaskVT) == TargetLowering::TypeSplitVector)
1184  GetSplitVector(Mask, MaskLo, MaskHi);
1185  else
1186  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
1187  return std::make_pair(MaskLo, MaskHi);
1188 }
1189 
1190 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1191  SDValue LHSLo, LHSHi;
1192  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1193  SDValue RHSLo, RHSHi;
1194  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1195  SDLoc dl(N);
1196 
1197  const SDNodeFlags Flags = N->getFlags();
1198  unsigned Opcode = N->getOpcode();
1199  if (N->getNumOperands() == 2) {
1200  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
1201  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
1202  return;
1203  }
1204 
1205  assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1206  assert(N->isVPOpcode() && "Expected VP opcode");
1207 
1208  SDValue MaskLo, MaskHi;
1209  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2));
1210 
1211  SDValue EVLLo, EVLHi;
1212  std::tie(EVLLo, EVLHi) =
1213  DAG.SplitEVL(N->getOperand(3), N->getValueType(0), dl);
1214 
1215  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(),
1216  {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1217  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(),
1218  {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1219 }
1220 
1221 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1222  SDValue &Hi) {
1223  SDValue Op0Lo, Op0Hi;
1224  GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
1225  SDValue Op1Lo, Op1Hi;
1226  GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
1227  SDValue Op2Lo, Op2Hi;
1228  GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
1229  SDLoc dl(N);
1230 
1231  const SDNodeFlags Flags = N->getFlags();
1232  unsigned Opcode = N->getOpcode();
1233  if (N->getNumOperands() == 3) {
1234  Lo = DAG.getNode(Opcode, dl, Op0Lo.getValueType(), Op0Lo, Op1Lo, Op2Lo, Flags);
1235  Hi = DAG.getNode(Opcode, dl, Op0Hi.getValueType(), Op0Hi, Op1Hi, Op2Hi, Flags);
1236  return;
1237  }
1238 
1239  assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
1240  assert(N->isVPOpcode() && "Expected VP opcode");
1241 
1242  SDValue MaskLo, MaskHi;
1243  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
1244 
1245  SDValue EVLLo, EVLHi;
1246  std::tie(EVLLo, EVLHi) =
1247  DAG.SplitEVL(N->getOperand(4), N->getValueType(0), dl);
1248 
1249  Lo = DAG.getNode(Opcode, dl, Op0Lo.getValueType(),
1250  {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags);
1251  Hi = DAG.getNode(Opcode, dl, Op0Hi.getValueType(),
1252  {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags);
1253 }
1254 
1255 void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1256  SDValue LHSLo, LHSHi;
1257  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1258  SDValue RHSLo, RHSHi;
1259  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1260  SDLoc dl(N);
1261  SDValue Op2 = N->getOperand(2);
1262 
1263  unsigned Opcode = N->getOpcode();
1264  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2,
1265  N->getFlags());
1266  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2,
1267  N->getFlags());
1268 }
1269 
1270 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1271  SDValue &Hi) {
1272  // We know the result is a vector. The input may be either a vector or a
1273  // scalar value.
1274  EVT LoVT, HiVT;
1275  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1276  SDLoc dl(N);
1277 
1278  SDValue InOp = N->getOperand(0);
1279  EVT InVT = InOp.getValueType();
1280 
1281  // Handle some special cases efficiently.
1282  switch (getTypeAction(InVT)) {
1290  break;
1293  // A scalar to vector conversion, where the scalar needs expansion.
1294  // If the vector is being split in two then we can just convert the
1295  // expanded pieces.
1296  if (LoVT == HiVT) {
1297  GetExpandedOp(InOp, Lo, Hi);
1298  if (DAG.getDataLayout().isBigEndian())
1299  std::swap(Lo, Hi);
1300  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1301  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1302  return;
1303  }
1304  break;
1306  // If the input is a vector that needs to be split, convert each split
1307  // piece of the input now.
1308  GetSplitVector(InOp, Lo, Hi);
1309  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1310  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1311  return;
1313  report_fatal_error("Scalarization of scalable vectors is not supported.");
1314  }
1315 
1316  // In the general case, convert the input to an integer and split it by hand.
1317  EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
1318  EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
1319  if (DAG.getDataLayout().isBigEndian())
1320  std::swap(LoIntVT, HiIntVT);
1321 
1322  SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
1323 
1324  if (DAG.getDataLayout().isBigEndian())
1325  std::swap(Lo, Hi);
1326  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1327  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1328 }
1329 
1330 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1331  SDValue &Hi) {
1332  EVT LoVT, HiVT;
1333  SDLoc dl(N);
1334  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1335  unsigned LoNumElts = LoVT.getVectorNumElements();
1336  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1337  Lo = DAG.getBuildVector(LoVT, dl, LoOps);
1338 
1339  SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1340  Hi = DAG.getBuildVector(HiVT, dl, HiOps);
1341 }
1342 
1343 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1344  SDValue &Hi) {
1345  assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1346  SDLoc dl(N);
1347  unsigned NumSubvectors = N->getNumOperands() / 2;
1348  if (NumSubvectors == 1) {
1349  Lo = N->getOperand(0);
1350  Hi = N->getOperand(1);
1351  return;
1352  }
1353 
1354  EVT LoVT, HiVT;
1355  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1356 
1357  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1358  Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
1359 
1360  SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1361  Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
1362 }
1363 
1364 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1365  SDValue &Hi) {
1366  SDValue Vec = N->getOperand(0);
1367  SDValue Idx = N->getOperand(1);
1368  SDLoc dl(N);
1369 
1370  EVT LoVT, HiVT;
1371  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1372 
1373  Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
1374  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1375  Hi = DAG.getNode(
1376  ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
1377  DAG.getVectorIdxConstant(IdxVal + LoVT.getVectorMinNumElements(), dl));
1378 }
1379 
1380 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1381  SDValue &Hi) {
1382  SDValue Vec = N->getOperand(0);
1383  SDValue SubVec = N->getOperand(1);
1384  SDValue Idx = N->getOperand(2);
1385  SDLoc dl(N);
1386  GetSplitVector(Vec, Lo, Hi);
1387 
1388  EVT VecVT = Vec.getValueType();
1389  EVT LoVT = Lo.getValueType();
1390  EVT SubVecVT = SubVec.getValueType();
1391  unsigned VecElems = VecVT.getVectorMinNumElements();
1392  unsigned SubElems = SubVecVT.getVectorMinNumElements();
1393  unsigned LoElems = LoVT.getVectorMinNumElements();
1394 
1395  // If we know the index is in the first half, and we know the subvector
1396  // doesn't cross the boundary between the halves, we can avoid spilling the
1397  // vector, and insert into the lower half of the split vector directly.
1398  unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1399  if (IdxVal + SubElems <= LoElems) {
1400  Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1401  return;
1402  }
1403  // Similarly if the subvector is fully in the high half, but mind that we
1404  // can't tell whether a fixed-length subvector is fully within the high half
1405  // of a scalable vector.
1406  if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1407  IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1408  Hi = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, Hi.getValueType(), Hi, SubVec,
1409  DAG.getVectorIdxConstant(IdxVal - LoElems, dl));
1410  return;
1411  }
1412 
1413  // Spill the vector to the stack.
1414  // In cases where the vector is illegal it will be broken down into parts
1415  // and stored in parts - we should use the alignment for the smallest part.
1416  Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1417  SDValue StackPtr =
1418  DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1419  auto &MF = DAG.getMachineFunction();
1420  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1421  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1422 
1423  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1424  SmallestAlign);
1425 
1426  // Store the new subvector into the specified index.
1427  SDValue SubVecPtr =
1428  TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1429  Store = DAG.getStore(Store, dl, SubVec, SubVecPtr,
1431 
1432  // Load the Lo part from the stack slot.
1433  Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, PtrInfo,
1434  SmallestAlign);
1435 
1436  // Increment the pointer to the other part.
1437  auto *Load = cast<LoadSDNode>(Lo);
1438  MachinePointerInfo MPI = Load->getPointerInfo();
1439  IncrementPointer(Load, LoVT, MPI, StackPtr);
1440 
1441  // Load the Hi part from the stack slot.
1442  Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MPI, SmallestAlign);
1443 }
1444 
1445 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1446  SDValue &Hi) {
1447  SDLoc dl(N);
1448  GetSplitVector(N->getOperand(0), Lo, Hi);
1449  Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
1450  Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
1451 }
1452 
1453 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1454  SDValue &Hi) {
1455  SDValue LHSLo, LHSHi;
1456  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1457  SDLoc DL(N);
1458 
1459  SDValue RHSLo, RHSHi;
1460  SDValue RHS = N->getOperand(1);
1461  EVT RHSVT = RHS.getValueType();
1462  if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1463  GetSplitVector(RHS, RHSLo, RHSHi);
1464  else
1465  std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1466 
1467 
1468  Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
1469  Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
1470 }
1471 
1472 void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
1473  SDValue &Hi) {
1474  SDLoc DL(N);
1475  SDValue ArgLo, ArgHi;
1476  SDValue Test = N->getOperand(1);
1477  GetSplitVector(N->getOperand(0), ArgLo, ArgHi);
1478  EVT LoVT, HiVT;
1479  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1480 
1481  Lo = DAG.getNode(ISD::IS_FPCLASS, DL, LoVT, ArgLo, Test, N->getFlags());
1482  Hi = DAG.getNode(ISD::IS_FPCLASS, DL, HiVT, ArgHi, Test, N->getFlags());
1483 }
1484 
1485 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1486  SDValue &Hi) {
1487  SDValue LHSLo, LHSHi;
1488  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1489  SDLoc dl(N);
1490 
1491  EVT LoVT, HiVT;
1492  std::tie(LoVT, HiVT) =
1493  DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1494 
1495  Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1496  DAG.getValueType(LoVT));
1497  Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1498  DAG.getValueType(HiVT));
1499 }
1500 
1501 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1502  SDValue &Hi) {
1503  unsigned Opcode = N->getOpcode();
1504  SDValue N0 = N->getOperand(0);
1505 
1506  SDLoc dl(N);
1507  SDValue InLo, InHi;
1508 
1509  if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1510  GetSplitVector(N0, InLo, InHi);
1511  else
1512  std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1513 
1514  EVT InLoVT = InLo.getValueType();
1515  unsigned InNumElements = InLoVT.getVectorNumElements();
1516 
1517  EVT OutLoVT, OutHiVT;
1518  std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1519  unsigned OutNumElements = OutLoVT.getVectorNumElements();
1520  assert((2 * OutNumElements) <= InNumElements &&
1521  "Illegal extend vector in reg split");
1522 
1523  // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1524  // input vector (i.e. we only use InLo):
1525  // OutLo will extend the first OutNumElements from InLo.
1526  // OutHi will extend the next OutNumElements from InLo.
1527 
1528  // Shuffle the elements from InLo for OutHi into the bottom elements to
1529  // create a 'fake' InHi.
1530  SmallVector<int, 8> SplitHi(InNumElements, -1);
1531  for (unsigned i = 0; i != OutNumElements; ++i)
1532  SplitHi[i] = i + OutNumElements;
1533  InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1534 
1535  Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1536  Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1537 }
1538 
1539 void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1540  SDValue &Hi) {
1541  unsigned NumOps = N->getNumOperands();
1542  SDValue Chain = N->getOperand(0);
1543  EVT LoVT, HiVT;
1544  SDLoc dl(N);
1545  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1546 
1547  SmallVector<SDValue, 4> OpsLo(NumOps);
1548  SmallVector<SDValue, 4> OpsHi(NumOps);
1549 
1550  // The Chain is the first operand.
1551  OpsLo[0] = Chain;
1552  OpsHi[0] = Chain;
1553 
1554  // Now process the remaining operands.
1555  for (unsigned i = 1; i < NumOps; ++i) {
1556  SDValue Op = N->getOperand(i);
1557  SDValue OpLo = Op;
1558  SDValue OpHi = Op;
1559 
1560  EVT InVT = Op.getValueType();
1561  if (InVT.isVector()) {
1562  // If the input also splits, handle it directly for a
1563  // compile time speedup. Otherwise split it by hand.
1564  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1565  GetSplitVector(Op, OpLo, OpHi);
1566  else
1567  std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1568  }
1569 
1570  OpsLo[i] = OpLo;
1571  OpsHi[i] = OpHi;
1572  }
1573 
1574  EVT LoValueVTs[] = {LoVT, MVT::Other};
1575  EVT HiValueVTs[] = {HiVT, MVT::Other};
1576  Lo = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo,
1577  N->getFlags());
1578  Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi,
1579  N->getFlags());
1580 
1581  // Build a factor node to remember that this Op is independent of the
1582  // other one.
1583  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1584  Lo.getValue(1), Hi.getValue(1));
1585 
1586  // Legalize the chain result - switch anything that used the old chain to
1587  // use the new one.
1588  ReplaceValueWith(SDValue(N, 1), Chain);
1589 }
1590 
1591 SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1592  SDValue Chain = N->getOperand(0);
1593  EVT VT = N->getValueType(0);
1594  unsigned NE = VT.getVectorNumElements();
1595  EVT EltVT = VT.getVectorElementType();
1596  SDLoc dl(N);
1597 
1598  SmallVector<SDValue, 8> Scalars;
1599  SmallVector<SDValue, 4> Operands(N->getNumOperands());
1600 
1601  // If ResNE is 0, fully unroll the vector op.
1602  if (ResNE == 0)
1603  ResNE = NE;
1604  else if (NE > ResNE)
1605  NE = ResNE;
1606 
1607  //The results of each unrolled operation, including the chain.
1608  EVT ChainVTs[] = {EltVT, MVT::Other};
1609  SmallVector<SDValue, 8> Chains;
1610 
1611  unsigned i;
1612  for (i = 0; i != NE; ++i) {
1613  Operands[0] = Chain;
1614  for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
1615  SDValue Operand = N->getOperand(j);
1616  EVT OperandVT = Operand.getValueType();
1617  if (OperandVT.isVector()) {
1618  EVT OperandEltVT = OperandVT.getVectorElementType();
1619  Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
1620  Operand, DAG.getVectorIdxConstant(i, dl));
1621  } else {
1622  Operands[j] = Operand;
1623  }
1624  }
1625  SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1626  Scalar.getNode()->setFlags(N->getFlags());
1627 
1628  //Add in the scalar as well as its chain value to the
1629  //result vectors.
1630  Scalars.push_back(Scalar);
1631  Chains.push_back(Scalar.getValue(1));
1632  }
1633 
1634  for (; i < ResNE; ++i)
1635  Scalars.push_back(DAG.getUNDEF(EltVT));
1636 
1637  // Build a new factor node to connect the chain back together.
1638  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1639  ReplaceValueWith(SDValue(N, 1), Chain);
1640 
1641  // Create a new BUILD_VECTOR node
1642  EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE);
1643  return DAG.getBuildVector(VecVT, dl, Scalars);
1644 }
1645 
1646 void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1647  SDValue &Lo, SDValue &Hi) {
1648  SDLoc dl(N);
1649  EVT ResVT = N->getValueType(0);
1650  EVT OvVT = N->getValueType(1);
1651  EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1652  std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
1653  std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
1654 
1655  SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1656  if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
1657  GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
1658  GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
1659  } else {
1660  std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
1661  std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
1662  }
1663 
1664  unsigned Opcode = N->getOpcode();
1665  SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
1666  SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
1667  SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
1668  SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
1669  LoNode->setFlags(N->getFlags());
1670  HiNode->setFlags(N->getFlags());
1671 
1672  Lo = SDValue(LoNode, ResNo);
1673  Hi = SDValue(HiNode, ResNo);
1674 
1675  // Replace the other vector result not being explicitly split here.
1676  unsigned OtherNo = 1 - ResNo;
1677  EVT OtherVT = N->getValueType(OtherNo);
1678  if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
1679  SetSplitVector(SDValue(N, OtherNo),
1680  SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1681  } else {
1682  SDValue OtherVal = DAG.getNode(
1683  ISD::CONCAT_VECTORS, dl, OtherVT,
1684  SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1685  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
1686  }
1687 }
1688 
1689 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1690  SDValue &Hi) {
1691  SDValue Vec = N->getOperand(0);
1692  SDValue Elt = N->getOperand(1);
1693  SDValue Idx = N->getOperand(2);
1694  SDLoc dl(N);
1695  GetSplitVector(Vec, Lo, Hi);
1696 
1697  if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1698  unsigned IdxVal = CIdx->getZExtValue();
1699  unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
1700  if (IdxVal < LoNumElts) {
1701  Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1702  Lo.getValueType(), Lo, Elt, Idx);
1703  return;
1704  } else if (!Vec.getValueType().isScalableVector()) {
1705  Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1706  DAG.getVectorIdxConstant(IdxVal - LoNumElts, dl));
1707  return;
1708  }
1709  }
1710 
1711  // See if the target wants to custom expand this node.
1712  if (CustomLowerNode(N, N->getValueType(0), true))
1713  return;
1714 
1715  // Make the vector elements byte-addressable if they aren't already.
1716  EVT VecVT = Vec.getValueType();
1717  EVT EltVT = VecVT.getVectorElementType();
1718  if (VecVT.getScalarSizeInBits() < 8) {
1719  EltVT = MVT::i8;
1720  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1721  VecVT.getVectorElementCount());
1722  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1723  // Extend the element type to match if needed.
1724  if (EltVT.bitsGT(Elt.getValueType()))
1725  Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1726  }
1727 
1728  // Spill the vector to the stack.
1729  // In cases where the vector is illegal it will be broken down into parts
1730  // and stored in parts - we should use the alignment for the smallest part.
1731  Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1732  SDValue StackPtr =
1733  DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1734  auto &MF = DAG.getMachineFunction();
1735  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1736  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1737 
1738  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1739  SmallestAlign);
1740 
1741  // Store the new element. This may be larger than the vector element type,
1742  // so use a truncating store.
1743  SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1744  Store = DAG.getTruncStore(
1745  Store, dl, Elt, EltPtr, MachinePointerInfo::getUnknownStack(MF), EltVT,
1746  commonAlignment(SmallestAlign,
1747  EltVT.getFixedSizeInBits() / 8));
1748 
1749  EVT LoVT, HiVT;
1750  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1751 
1752  // Load the Lo part from the stack slot.
1753  Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo, SmallestAlign);
1754 
1755  // Increment the pointer to the other part.
1756  auto Load = cast<LoadSDNode>(Lo);
1757  MachinePointerInfo MPI = Load->getPointerInfo();
1758  IncrementPointer(Load, LoVT, MPI, StackPtr);
1759 
1760  Hi = DAG.getLoad(HiVT, dl, Store, StackPtr, MPI, SmallestAlign);
1761 
1762  // If we adjusted the original type, we need to truncate the results.
1763  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1764  if (LoVT != Lo.getValueType())
1765  Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1766  if (HiVT != Hi.getValueType())
1767  Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1768 }
1769 
1770 void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
1771  SDValue &Hi) {
1772  EVT LoVT, HiVT;
1773  SDLoc dl(N);
1774  assert(N->getValueType(0).isScalableVector() &&
1775  "Only scalable vectors are supported for STEP_VECTOR");
1776  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1777  SDValue Step = N->getOperand(0);
1778 
1779  Lo = DAG.getNode(ISD::STEP_VECTOR, dl, LoVT, Step);
1780 
1781  // Hi = Lo + (EltCnt * Step)
1782  EVT EltVT = Step.getValueType();
1783  APInt StepVal = cast<ConstantSDNode>(Step)->getAPIntValue();
1784  SDValue StartOfHi =
1785  DAG.getVScale(dl, EltVT, StepVal * LoVT.getVectorMinNumElements());
1786  StartOfHi = DAG.getSExtOrTrunc(StartOfHi, dl, HiVT.getVectorElementType());
1787  StartOfHi = DAG.getNode(ISD::SPLAT_VECTOR, dl, HiVT, StartOfHi);
1788 
1789  Hi = DAG.getNode(ISD::STEP_VECTOR, dl, HiVT, Step);
1790  Hi = DAG.getNode(ISD::ADD, dl, HiVT, Hi, StartOfHi);
1791 }
1792 
1793 void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
1794  SDValue &Hi) {
1795  EVT LoVT, HiVT;
1796  SDLoc dl(N);
1797  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1798  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, N->getOperand(0));
1799  if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
1800  Hi = DAG.getUNDEF(HiVT);
1801  } else {
1802  assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
1803  Hi = Lo;
1804  }
1805 }
1806 
1807 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1808  SDValue &Hi) {
1809  assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1810  EVT LoVT, HiVT;
1811  SDLoc dl(LD);
1812  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1813 
1814  ISD::LoadExtType ExtType = LD->getExtensionType();
1815  SDValue Ch = LD->getChain();
1816  SDValue Ptr = LD->getBasePtr();
1817  SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1818  EVT MemoryVT = LD->getMemoryVT();
1819  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1820  AAMDNodes AAInfo = LD->getAAInfo();
1821 
1822  EVT LoMemVT, HiMemVT;
1823  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1824 
1825  if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
1826  SDValue Value, NewChain;
1827  std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
1828  std::tie(Lo, Hi) = DAG.SplitVector(Value, dl);
1829  ReplaceValueWith(SDValue(LD, 1), NewChain);
1830  return;
1831  }
1832 
1833  Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1834  LD->getPointerInfo(), LoMemVT, LD->getOriginalAlign(),
1835  MMOFlags, AAInfo);
1836 
1837  MachinePointerInfo MPI;
1838  IncrementPointer(LD, LoMemVT, MPI, Ptr);
1839 
1840  Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, MPI,
1841  HiMemVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
1842 
1843  // Build a factor node to remember that this load is independent of the
1844  // other one.
1845  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1846  Hi.getValue(1));
1847 
1848  // Legalize the chain result - switch anything that used the old chain to
1849  // use the new one.
1850  ReplaceValueWith(SDValue(LD, 1), Ch);
1851 }
1852 
1853 void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
1854  SDValue &Hi) {
1855  assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
1856  EVT LoVT, HiVT;
1857  SDLoc dl(LD);
1858  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1859 
1860  ISD::LoadExtType ExtType = LD->getExtensionType();
1861  SDValue Ch = LD->getChain();
1862  SDValue Ptr = LD->getBasePtr();
1863  SDValue Offset = LD->getOffset();
1864  assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
1865  Align Alignment = LD->getOriginalAlign();
1866  SDValue Mask = LD->getMask();
1867  SDValue EVL = LD->getVectorLength();
1868  EVT MemoryVT = LD->getMemoryVT();
1869 
1870  EVT LoMemVT, HiMemVT;
1871  bool HiIsEmpty = false;
1872  std::tie(LoMemVT, HiMemVT) =
1873  DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
1874 
1875  // Split Mask operand
1876  SDValue MaskLo, MaskHi;
1877  if (Mask.getOpcode() == ISD::SETCC) {
1878  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1879  } else {
1880  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1881  GetSplitVector(Mask, MaskLo, MaskHi);
1882  else
1883  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1884  }
1885 
1886  // Split EVL operand
1887  SDValue EVLLo, EVLHi;
1888  std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, LD->getValueType(0), dl);
1889 
1891  LD->getPointerInfo(), MachineMemOperand::MOLoad,
1892  MemoryLocation::UnknownSize, Alignment, LD->getAAInfo(), LD->getRanges());
1893 
1894  Lo =
1895  DAG.getLoadVP(LD->getAddressingMode(), ExtType, LoVT, dl, Ch, Ptr, Offset,
1896  MaskLo, EVLLo, LoMemVT, MMO, LD->isExpandingLoad());
1897 
1898  if (HiIsEmpty) {
1899  // The hi vp_load has zero storage size. We therefore simply set it to
1900  // the low vp_load and rely on subsequent removal from the chain.
1901  Hi = Lo;
1902  } else {
1903  // Generate hi vp_load.
1904  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1905  LD->isExpandingLoad());
1906 
1907  MachinePointerInfo MPI;
1908  if (LoMemVT.isScalableVector())
1909  MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
1910  else
1911  MPI = LD->getPointerInfo().getWithOffset(
1912  LoMemVT.getStoreSize().getFixedSize());
1913 
1916  LD->getAAInfo(), LD->getRanges());
1917 
1918  Hi = DAG.getLoadVP(LD->getAddressingMode(), ExtType, HiVT, dl, Ch, Ptr,
1919  Offset, MaskHi, EVLHi, HiMemVT, MMO,
1920  LD->isExpandingLoad());
1921  }
1922 
1923  // Build a factor node to remember that this load is independent of the
1924  // other one.
1925  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1926  Hi.getValue(1));
1927 
1928  // Legalize the chain result - switch anything that used the old chain to
1929  // use the new one.
1930  ReplaceValueWith(SDValue(LD, 1), Ch);
1931 }
1932 
1933 void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
1934  SDValue &Lo, SDValue &Hi) {
1935  assert(SLD->isUnindexed() &&
1936  "Indexed VP strided load during type legalization!");
1937  assert(SLD->getOffset().isUndef() &&
1938  "Unexpected indexed variable-length load offset");
1939 
1940  SDLoc DL(SLD);
1941 
1942  EVT LoVT, HiVT;
1943  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(SLD->getValueType(0));
1944 
1945  EVT LoMemVT, HiMemVT;
1946  bool HiIsEmpty = false;
1947  std::tie(LoMemVT, HiMemVT) =
1948  DAG.GetDependentSplitDestVTs(SLD->getMemoryVT(), LoVT, &HiIsEmpty);
1949 
1950  SDValue Mask = SLD->getMask();
1951  SDValue LoMask, HiMask;
1952  if (Mask.getOpcode() == ISD::SETCC) {
1953  SplitVecRes_SETCC(Mask.getNode(), LoMask, HiMask);
1954  } else {
1955  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1956  GetSplitVector(Mask, LoMask, HiMask);
1957  else
1958  std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
1959  }
1960 
1961  SDValue LoEVL, HiEVL;
1962  std::tie(LoEVL, HiEVL) =
1963  DAG.SplitEVL(SLD->getVectorLength(), SLD->getValueType(0), DL);
1964 
1965  // Generate the low vp_strided_load
1966  Lo = DAG.getStridedLoadVP(
1967  SLD->getAddressingMode(), SLD->getExtensionType(), LoVT, DL,
1968  SLD->getChain(), SLD->getBasePtr(), SLD->getOffset(), SLD->getStride(),
1969  LoMask, LoEVL, LoMemVT, SLD->getMemOperand(), SLD->isExpandingLoad());
1970 
1971  if (HiIsEmpty) {
1972  // The high vp_strided_load has zero storage size. We therefore simply set
1973  // it to the low vp_strided_load and rely on subsequent removal from the
1974  // chain.
1975  Hi = Lo;
1976  } else {
1977  // Generate the high vp_strided_load.
1978  // To calculate the high base address, we need to sum to the low base
1979  // address stride number of bytes for each element already loaded by low,
1980  // that is: Ptr = Ptr + (LoEVL * Stride)
1981  EVT PtrVT = SLD->getBasePtr().getValueType();
1982  SDValue Increment =
1983  DAG.getNode(ISD::MUL, DL, PtrVT, LoEVL,
1984  DAG.getSExtOrTrunc(SLD->getStride(), DL, PtrVT));
1985  SDValue Ptr =
1986  DAG.getNode(ISD::ADD, DL, PtrVT, SLD->getBasePtr(), Increment);
1987 
1988  Align Alignment = SLD->getOriginalAlign();
1989  if (LoMemVT.isScalableVector())
1990  Alignment = commonAlignment(
1991  Alignment, LoMemVT.getSizeInBits().getKnownMinSize() / 8);
1992 
1996  SLD->getAAInfo(), SLD->getRanges());
1997 
1999  HiVT, DL, SLD->getChain(), Ptr, SLD->getOffset(),
2000  SLD->getStride(), HiMask, HiEVL, HiMemVT, MMO,
2001  SLD->isExpandingLoad());
2002  }
2003 
2004  // Build a factor node to remember that this load is independent of the
2005  // other one.
2006  SDValue Ch = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
2007  Hi.getValue(1));
2008 
2009  // Legalize the chain result - switch anything that used the old chain to
2010  // use the new one.
2011  ReplaceValueWith(SDValue(SLD, 1), Ch);
2012 }
2013 
2014 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
2015  SDValue &Lo, SDValue &Hi) {
2016  assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
2017  EVT LoVT, HiVT;
2018  SDLoc dl(MLD);
2019  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
2020 
2021  SDValue Ch = MLD->getChain();
2022  SDValue Ptr = MLD->getBasePtr();
2023  SDValue Offset = MLD->getOffset();
2024  assert(Offset.isUndef() && "Unexpected indexed masked load offset");
2025  SDValue Mask = MLD->getMask();
2026  SDValue PassThru = MLD->getPassThru();
2027  Align Alignment = MLD->getOriginalAlign();
2028  ISD::LoadExtType ExtType = MLD->getExtensionType();
2029 
2030  // Split Mask operand
2031  SDValue MaskLo, MaskHi;
2032  if (Mask.getOpcode() == ISD::SETCC) {
2033  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2034  } else {
2035  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2036  GetSplitVector(Mask, MaskLo, MaskHi);
2037  else
2038  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2039  }
2040 
2041  EVT MemoryVT = MLD->getMemoryVT();
2042  EVT LoMemVT, HiMemVT;
2043  bool HiIsEmpty = false;
2044  std::tie(LoMemVT, HiMemVT) =
2045  DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
2046 
2047  SDValue PassThruLo, PassThruHi;
2048  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2049  GetSplitVector(PassThru, PassThruLo, PassThruHi);
2050  else
2051  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2052 
2055  MemoryLocation::UnknownSize, Alignment, MLD->getAAInfo(),
2056  MLD->getRanges());
2057 
2058  Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
2059  MMO, MLD->getAddressingMode(), ExtType,
2060  MLD->isExpandingLoad());
2061 
2062  if (HiIsEmpty) {
2063  // The hi masked load has zero storage size. We therefore simply set it to
2064  // the low masked load and rely on subsequent removal from the chain.
2065  Hi = Lo;
2066  } else {
2067  // Generate hi masked load.
2068  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
2069  MLD->isExpandingLoad());
2070 
2071  MachinePointerInfo MPI;
2072  if (LoMemVT.isScalableVector())
2074  else
2075  MPI = MLD->getPointerInfo().getWithOffset(
2076  LoMemVT.getStoreSize().getFixedSize());
2077 
2080  MLD->getAAInfo(), MLD->getRanges());
2081 
2082  Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
2083  HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
2084  MLD->isExpandingLoad());
2085  }
2086 
2087  // Build a factor node to remember that this load is independent of the
2088  // other one.
2089  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2090  Hi.getValue(1));
2091 
2092  // Legalize the chain result - switch anything that used the old chain to
2093  // use the new one.
2094  ReplaceValueWith(SDValue(MLD, 1), Ch);
2095 
2096 }
2097 
2098 void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2099  SDValue &Hi, bool SplitSETCC) {
2100  EVT LoVT, HiVT;
2101  SDLoc dl(N);
2102  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2103 
2104  SDValue Ch = N->getChain();
2105  SDValue Ptr = N->getBasePtr();
2106  struct Operands {
2107  SDValue Mask;
2108  SDValue Index;
2109  SDValue Scale;
2110  } Ops = [&]() -> Operands {
2111  if (auto *MSC = dyn_cast<MaskedGatherSDNode>(N)) {
2112  return {MSC->getMask(), MSC->getIndex(), MSC->getScale()};
2113  }
2114  auto *VPSC = cast<VPGatherSDNode>(N);
2115  return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale()};
2116  }();
2117 
2118  EVT MemoryVT = N->getMemoryVT();
2119  Align Alignment = N->getOriginalAlign();
2120 
2121  // Split Mask operand
2122  SDValue MaskLo, MaskHi;
2123  if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2124  SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
2125  } else {
2126  std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, dl);
2127  }
2128 
2129  EVT LoMemVT, HiMemVT;
2130  // Split MemoryVT
2131  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2132 
2133  SDValue IndexHi, IndexLo;
2134  if (getTypeAction(Ops.Index.getValueType()) ==
2136  GetSplitVector(Ops.Index, IndexLo, IndexHi);
2137  else
2138  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, dl);
2139 
2141  N->getPointerInfo(), MachineMemOperand::MOLoad,
2142  MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
2143 
2144  if (auto *MGT = dyn_cast<MaskedGatherSDNode>(N)) {
2145  SDValue PassThru = MGT->getPassThru();
2146  SDValue PassThruLo, PassThruHi;
2147  if (getTypeAction(PassThru.getValueType()) ==
2149  GetSplitVector(PassThru, PassThruLo, PassThruHi);
2150  else
2151  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2152 
2153  ISD::LoadExtType ExtType = MGT->getExtensionType();
2154  ISD::MemIndexType IndexTy = MGT->getIndexType();
2155 
2156  SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2157  Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl,
2158  OpsLo, MMO, IndexTy, ExtType);
2159 
2160  SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2161  Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl,
2162  OpsHi, MMO, IndexTy, ExtType);
2163  } else {
2164  auto *VPGT = cast<VPGatherSDNode>(N);
2165  SDValue EVLLo, EVLHi;
2166  std::tie(EVLLo, EVLHi) =
2167  DAG.SplitEVL(VPGT->getVectorLength(), MemoryVT, dl);
2168 
2169  SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2170  Lo = DAG.getGatherVP(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, OpsLo,
2171  MMO, VPGT->getIndexType());
2172 
2173  SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2174  Hi = DAG.getGatherVP(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, OpsHi,
2175  MMO, VPGT->getIndexType());
2176  }
2177 
2178  // Build a factor node to remember that this load is independent of the
2179  // other one.
2180  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2181  Hi.getValue(1));
2182 
2183  // Legalize the chain result - switch anything that used the old chain to
2184  // use the new one.
2185  ReplaceValueWith(SDValue(N, 1), Ch);
2186 }
2187 
2188 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
2189  assert(N->getValueType(0).isVector() &&
2190  N->getOperand(0).getValueType().isVector() &&
2191  "Operand types must be vectors");
2192 
2193  EVT LoVT, HiVT;
2194  SDLoc DL(N);
2195  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2196 
2197  // If the input also splits, handle it directly. Otherwise split it by hand.
2198  SDValue LL, LH, RL, RH;
2199  if (getTypeAction(N->getOperand(0).getValueType()) ==
2201  GetSplitVector(N->getOperand(0), LL, LH);
2202  else
2203  std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
2204 
2205  if (getTypeAction(N->getOperand(1).getValueType()) ==
2207  GetSplitVector(N->getOperand(1), RL, RH);
2208  else
2209  std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
2210 
2211  if (N->getOpcode() == ISD::SETCC) {
2212  Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
2213  Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
2214  } else {
2215  assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
2216  SDValue MaskLo, MaskHi, EVLLo, EVLHi;
2217  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
2218  std::tie(EVLLo, EVLHi) =
2219  DAG.SplitEVL(N->getOperand(4), N->getValueType(0), DL);
2220  Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2), MaskLo,
2221  EVLLo);
2222  Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2), MaskHi,
2223  EVLHi);
2224  }
2225 }
2226 
2227 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
2228  SDValue &Hi) {
2229  // Get the dest types - they may not match the input types, e.g. int_to_fp.
2230  EVT LoVT, HiVT;
2231  SDLoc dl(N);
2232  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2233 
2234  // If the input also splits, handle it directly for a compile time speedup.
2235  // Otherwise split it by hand.
2236  EVT InVT = N->getOperand(0).getValueType();
2237  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
2238  GetSplitVector(N->getOperand(0), Lo, Hi);
2239  else
2240  std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
2241 
2242  const SDNodeFlags Flags = N->getFlags();
2243  unsigned Opcode = N->getOpcode();
2244  if (N->getNumOperands() <= 2) {
2245  if (Opcode == ISD::FP_ROUND) {
2246  Lo = DAG.getNode(Opcode, dl, LoVT, Lo, N->getOperand(1), Flags);
2247  Hi = DAG.getNode(Opcode, dl, HiVT, Hi, N->getOperand(1), Flags);
2248  } else {
2249  Lo = DAG.getNode(Opcode, dl, LoVT, Lo, Flags);
2250  Hi = DAG.getNode(Opcode, dl, HiVT, Hi, Flags);
2251  }
2252  return;
2253  }
2254 
2255  assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
2256  assert(N->isVPOpcode() && "Expected VP opcode");
2257 
2258  SDValue MaskLo, MaskHi;
2259  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
2260 
2261  SDValue EVLLo, EVLHi;
2262  std::tie(EVLLo, EVLHi) =
2263  DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
2264 
2265  Lo = DAG.getNode(Opcode, dl, LoVT, {Lo, MaskLo, EVLLo}, Flags);
2266  Hi = DAG.getNode(Opcode, dl, HiVT, {Hi, MaskHi, EVLHi}, Flags);
2267 }
2268 
2269 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
2270  SDValue &Hi) {
2271  SDLoc dl(N);
2272  EVT SrcVT = N->getOperand(0).getValueType();
2273  EVT DestVT = N->getValueType(0);
2274  EVT LoVT, HiVT;
2275  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
2276 
2277  // We can do better than a generic split operation if the extend is doing
2278  // more than just doubling the width of the elements and the following are
2279  // true:
2280  // - The number of vector elements is even,
2281  // - the source type is legal,
2282  // - the type of a split source is illegal,
2283  // - the type of an extended (by doubling element size) source is legal, and
2284  // - the type of that extended source when split is legal.
2285  //
2286  // This won't necessarily completely legalize the operation, but it will
2287  // more effectively move in the right direction and prevent falling down
2288  // to scalarization in many cases due to the input vector being split too
2289  // far.
2290  if (SrcVT.getVectorElementCount().isKnownEven() &&
2291  SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
2292  LLVMContext &Ctx = *DAG.getContext();
2293  EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
2294  EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
2295 
2296  EVT SplitLoVT, SplitHiVT;
2297  std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
2298  if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
2299  TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
2300  LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
2301  N->dump(&DAG); dbgs() << "\n");
2302  if (!N->isVPOpcode()) {
2303  // Extend the source vector by one step.
2304  SDValue NewSrc =
2305  DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
2306  // Get the low and high halves of the new, extended one step, vector.
2307  std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
2308  // Extend those vector halves the rest of the way.
2309  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
2310  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
2311  return;
2312  }
2313 
2314  // Extend the source vector by one step.
2315  SDValue NewSrc =
2316  DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0),
2317  N->getOperand(1), N->getOperand(2));
2318  // Get the low and high halves of the new, extended one step, vector.
2319  std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
2320 
2321  SDValue MaskLo, MaskHi;
2322  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
2323 
2324  SDValue EVLLo, EVLHi;
2325  std::tie(EVLLo, EVLHi) =
2326  DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
2327  // Extend those vector halves the rest of the way.
2328  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, {Lo, MaskLo, EVLLo});
2329  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, {Hi, MaskHi, EVLHi});
2330  return;
2331  }
2332  }
2333  // Fall back to the generic unary operator splitting otherwise.
2334  SplitVecRes_UnaryOp(N, Lo, Hi);
2335 }
2336 
2337 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
2338  SDValue &Lo, SDValue &Hi) {
2339  // The low and high parts of the original input give four input vectors.
2340  SDValue Inputs[4];
2341  SDLoc DL(N);
2342  GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
2343  GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
2344  EVT NewVT = Inputs[0].getValueType();
2345  unsigned NewElts = NewVT.getVectorNumElements();
2346 
2347  auto &&IsConstant = [](const SDValue &N) {
2348  APInt SplatValue;
2349  return N.getResNo() == 0 &&
2350  (ISD::isConstantSplatVector(N.getNode(), SplatValue) ||
2352  };
2353  auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
2354  SDValue &Input2,
2355  ArrayRef<int> Mask) {
2356  assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
2357  Input2->getOpcode() == ISD::BUILD_VECTOR &&
2358  "Expected build vector node.");
2359  EVT EltVT = NewVT.getVectorElementType();
2360  SmallVector<SDValue> Ops(NewElts, DAG.getUNDEF(EltVT));
2361  for (unsigned I = 0; I < NewElts; ++I) {
2362  if (Mask[I] == UndefMaskElem)
2363  continue;
2364  unsigned Idx = Mask[I];
2365  if (Idx >= NewElts)
2366  Ops[I] = Input2.getOperand(Idx - NewElts);
2367  else
2368  Ops[I] = Input1.getOperand(Idx);
2369  // Make the type of all elements the same as the element type.
2370  if (Ops[I].getValueType().bitsGT(EltVT))
2371  Ops[I] = DAG.getNode(ISD::TRUNCATE, DL, EltVT, Ops[I]);
2372  }
2373  return DAG.getBuildVector(NewVT, DL, Ops);
2374  };
2375 
2376  // If Lo or Hi uses elements from at most two of the four input vectors, then
2377  // express it as a vector shuffle of those two inputs. Otherwise extract the
2378  // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
2379  SmallVector<int> OrigMask(N->getMask());
2380  // Try to pack incoming shuffles/inputs.
2381  auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
2383  // Check if all inputs are shuffles of the same operands or non-shuffles.
2385  for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) {
2386  SDValue Input = Inputs[Idx];
2387  auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Input.getNode());
2388  if (!Shuffle ||
2389  Input.getOperand(0).getValueType() != Input.getValueType())
2390  continue;
2391  ShufflesIdxs[std::make_pair(Input.getOperand(0), Input.getOperand(1))]
2392  .push_back(Idx);
2393  ShufflesIdxs[std::make_pair(Input.getOperand(1), Input.getOperand(0))]
2394  .push_back(Idx);
2395  }
2396  for (auto &P : ShufflesIdxs) {
2397  if (P.second.size() < 2)
2398  continue;
2399  // Use shuffles operands instead of shuffles themselves.
2400  // 1. Adjust mask.
2401  for (int &Idx : Mask) {
2402  if (Idx == UndefMaskElem)
2403  continue;
2404  unsigned SrcRegIdx = Idx / NewElts;
2405  if (Inputs[SrcRegIdx].isUndef()) {
2406  Idx = UndefMaskElem;
2407  continue;
2408  }
2409  auto *Shuffle =
2410  dyn_cast<ShuffleVectorSDNode>(Inputs[SrcRegIdx].getNode());
2411  if (!Shuffle || !is_contained(P.second, SrcRegIdx))
2412  continue;
2413  int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2414  if (MaskElt == UndefMaskElem) {
2415  Idx = UndefMaskElem;
2416  continue;
2417  }
2418  Idx = MaskElt % NewElts +
2419  P.second[Shuffle->getOperand(MaskElt / NewElts) == P.first.first
2420  ? 0
2421  : 1] *
2422  NewElts;
2423  }
2424  // 2. Update inputs.
2425  Inputs[P.second[0]] = P.first.first;
2426  Inputs[P.second[1]] = P.first.second;
2427  // Clear the pair data.
2428  P.second.clear();
2429  ShufflesIdxs[std::make_pair(P.first.second, P.first.first)].clear();
2430  }
2431  // Check if any concat_vectors can be simplified.
2432  SmallBitVector UsedSubVector(2 * std::size(Inputs));
2433  for (int &Idx : Mask) {
2434  if (Idx == UndefMaskElem)
2435  continue;
2436  unsigned SrcRegIdx = Idx / NewElts;
2437  if (Inputs[SrcRegIdx].isUndef()) {
2438  Idx = UndefMaskElem;
2439  continue;
2440  }
2442  getTypeAction(Inputs[SrcRegIdx].getValueType());
2443  if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
2444  Inputs[SrcRegIdx].getNumOperands() == 2 &&
2445  !Inputs[SrcRegIdx].getOperand(1).isUndef() &&
2446  (TypeAction == TargetLowering::TypeLegal ||
2447  TypeAction == TargetLowering::TypeWidenVector))
2448  UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
2449  }
2450  if (UsedSubVector.count() > 1) {
2452  for (unsigned I = 0; I < std::size(Inputs); ++I) {
2453  if (UsedSubVector.test(2 * I) == UsedSubVector.test(2 * I + 1))
2454  continue;
2455  if (Pairs.empty() || Pairs.back().size() == 2)
2456  Pairs.emplace_back();
2457  if (UsedSubVector.test(2 * I)) {
2458  Pairs.back().emplace_back(I, 0);
2459  } else {
2460  assert(UsedSubVector.test(2 * I + 1) &&
2461  "Expected to be used one of the subvectors.");
2462  Pairs.back().emplace_back(I, 1);
2463  }
2464  }
2465  if (!Pairs.empty() && Pairs.front().size() > 1) {
2466  // Adjust mask.
2467  for (int &Idx : Mask) {
2468  if (Idx == UndefMaskElem)
2469  continue;
2470  unsigned SrcRegIdx = Idx / NewElts;
2471  auto *It = find_if(
2472  Pairs, [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
2473  return Idxs.front().first == SrcRegIdx ||
2474  Idxs.back().first == SrcRegIdx;
2475  });
2476  if (It == Pairs.end())
2477  continue;
2478  Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
2479  (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
2480  }
2481  // Adjust inputs.
2482  for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
2483  Inputs[Idxs.front().first] = DAG.getNode(
2485  Inputs[Idxs.front().first].getValueType(),
2486  Inputs[Idxs.front().first].getOperand(Idxs.front().second),
2487  Inputs[Idxs.back().first].getOperand(Idxs.back().second));
2488  }
2489  }
2490  }
2491  bool Changed;
2492  do {
2493  // Try to remove extra shuffles (except broadcasts) and shuffles with the
2494  // reused operands.
2495  Changed = false;
2496  for (unsigned I = 0; I < std::size(Inputs); ++I) {
2497  auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Inputs[I].getNode());
2498  if (!Shuffle)
2499  continue;
2500  if (Shuffle->getOperand(0).getValueType() != NewVT)
2501  continue;
2502  int Op = -1;
2503  if (!Inputs[I].hasOneUse() && Shuffle->getOperand(1).isUndef() &&
2504  !Shuffle->isSplat()) {
2505  Op = 0;
2506  } else if (!Inputs[I].hasOneUse() &&
2507  !Shuffle->getOperand(1).isUndef()) {
2508  // Find the only used operand, if possible.
2509  for (int &Idx : Mask) {
2510  if (Idx == UndefMaskElem)
2511  continue;
2512  unsigned SrcRegIdx = Idx / NewElts;
2513  if (SrcRegIdx != I)
2514  continue;
2515  int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2516  if (MaskElt == UndefMaskElem) {
2517  Idx = UndefMaskElem;
2518  continue;
2519  }
2520  int OpIdx = MaskElt / NewElts;
2521  if (Op == -1) {
2522  Op = OpIdx;
2523  continue;
2524  }
2525  if (Op != OpIdx) {
2526  Op = -1;
2527  break;
2528  }
2529  }
2530  }
2531  if (Op < 0) {
2532  // Try to check if one of the shuffle operands is used already.
2533  for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
2534  if (Shuffle->getOperand(OpIdx).isUndef())
2535  continue;
2536  auto *It = find(Inputs, Shuffle->getOperand(OpIdx));
2537  if (It == std::end(Inputs))
2538  continue;
2539  int FoundOp = std::distance(std::begin(Inputs), It);
2540  // Found that operand is used already.
2541  // 1. Fix the mask for the reused operand.
2542  for (int &Idx : Mask) {
2543  if (Idx == UndefMaskElem)
2544  continue;
2545  unsigned SrcRegIdx = Idx / NewElts;
2546  if (SrcRegIdx != I)
2547  continue;
2548  int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2549  if (MaskElt == UndefMaskElem) {
2550  Idx = UndefMaskElem;
2551  continue;
2552  }
2553  int MaskIdx = MaskElt / NewElts;
2554  if (OpIdx == MaskIdx)
2555  Idx = MaskElt % NewElts + FoundOp * NewElts;
2556  }
2557  // 2. Set Op to the unused OpIdx.
2558  Op = (OpIdx + 1) % 2;
2559  break;
2560  }
2561  }
2562  if (Op >= 0) {
2563  Changed = true;
2564  Inputs[I] = Shuffle->getOperand(Op);
2565  // Adjust mask.
2566  for (int &Idx : Mask) {
2567  if (Idx == UndefMaskElem)
2568  continue;
2569  unsigned SrcRegIdx = Idx / NewElts;
2570  if (SrcRegIdx != I)
2571  continue;
2572  int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2573  int OpIdx = MaskElt / NewElts;
2574  if (OpIdx != Op)
2575  continue;
2576  Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
2577  }
2578  }
2579  }
2580  } while (Changed);
2581  };
2582  TryPeekThroughShufflesInputs(OrigMask);
2583  // Proces unique inputs.
2584  auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
2585  NewElts](SmallVectorImpl<int> &Mask) {
2586  SetVector<SDValue> UniqueInputs;
2587  SetVector<SDValue> UniqueConstantInputs;
2588  for (const auto &I : Inputs) {
2589  if (IsConstant(I))
2590  UniqueConstantInputs.insert(I);
2591  else if (!I.isUndef())
2592  UniqueInputs.insert(I);
2593  }
2594  // Adjust mask in case of reused inputs. Also, need to insert constant
2595  // inputs at first, otherwise it affects the final outcome.
2596  if (UniqueInputs.size() != std::size(Inputs)) {
2597  auto &&UniqueVec = UniqueInputs.takeVector();
2598  auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
2599  unsigned ConstNum = UniqueConstantVec.size();
2600  for (int &Idx : Mask) {
2601  if (Idx == UndefMaskElem)
2602  continue;
2603  unsigned SrcRegIdx = Idx / NewElts;
2604  if (Inputs[SrcRegIdx].isUndef()) {
2605  Idx = UndefMaskElem;
2606  continue;
2607  }
2608  const auto It = find(UniqueConstantVec, Inputs[SrcRegIdx]);
2609  if (It != UniqueConstantVec.end()) {
2610  Idx = (Idx % NewElts) +
2611  NewElts * std::distance(UniqueConstantVec.begin(), It);
2612  assert(Idx >= 0 && "Expected defined mask idx.");
2613  continue;
2614  }
2615  const auto RegIt = find(UniqueVec, Inputs[SrcRegIdx]);
2616  assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
2617  Idx = (Idx % NewElts) +
2618  NewElts * (std::distance(UniqueVec.begin(), RegIt) + ConstNum);
2619  assert(Idx >= 0 && "Expected defined mask idx.");
2620  }
2621  copy(UniqueConstantVec, std::begin(Inputs));
2622  copy(UniqueVec, std::next(std::begin(Inputs), ConstNum));
2623  }
2624  };
2625  MakeUniqueInputs(OrigMask);
2626  SDValue OrigInputs[4];
2627  copy(Inputs, std::begin(OrigInputs));
2628  for (unsigned High = 0; High < 2; ++High) {
2629  SDValue &Output = High ? Hi : Lo;
2630 
2631  // Build a shuffle mask for the output, discovering on the fly which
2632  // input vectors to use as shuffle operands.
2633  unsigned FirstMaskIdx = High * NewElts;
2634  SmallVector<int> Mask(NewElts * std::size(Inputs), UndefMaskElem);
2635  copy(makeArrayRef(OrigMask).slice(FirstMaskIdx, NewElts), Mask.begin());
2636  assert(!Output && "Expected default initialized initial value.");
2637  TryPeekThroughShufflesInputs(Mask);
2638  MakeUniqueInputs(Mask);
2639  SDValue TmpInputs[4];
2640  copy(Inputs, std::begin(TmpInputs));
2641  // Track changes in the output registers.
2642  int UsedIdx = -1;
2643  bool SecondIteration = false;
2644  auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
2645  if (UsedIdx < 0) {
2646  UsedIdx = Idx;
2647  return false;
2648  }
2649  if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
2650  SecondIteration = true;
2651  return SecondIteration;
2652  };
2654  Mask, std::size(Inputs), std::size(Inputs),
2655  /*NumOfUsedRegs=*/1,
2656  [&Output, &DAG = DAG, NewVT]() { Output = DAG.getUNDEF(NewVT); },
2657  [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
2658  &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
2659  if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
2660  Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
2661  else
2662  Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx],
2663  DAG.getUNDEF(NewVT), Mask);
2664  Inputs[Idx] = Output;
2665  },
2666  [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
2667  &TmpInputs,
2668  &BuildVector](ArrayRef<int> Mask, unsigned Idx1, unsigned Idx2) {
2669  if (AccumulateResults(Idx1)) {
2670  if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
2671  Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
2672  Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
2673  else
2674  Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx1],
2675  Inputs[Idx2], Mask);
2676  } else {
2677  if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
2678  TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
2679  Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
2680  else
2681  Output = DAG.getVectorShuffle(NewVT, DL, TmpInputs[Idx1],
2682  TmpInputs[Idx2], Mask);
2683  }
2684  Inputs[Idx1] = Output;
2685  });
2686  copy(OrigInputs, std::begin(Inputs));
2687  }
2688 }
2689 
2690 void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2691  EVT OVT = N->getValueType(0);
2692  EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext());
2693  SDValue Chain = N->getOperand(0);
2694  SDValue Ptr = N->getOperand(1);
2695  SDValue SV = N->getOperand(2);
2696  SDLoc dl(N);
2697 
2698  const Align Alignment =
2699  DAG.getDataLayout().getABITypeAlign(NVT.getTypeForEVT(*DAG.getContext()));
2700 
2701  Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment.value());
2702  Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment.value());
2703  Chain = Hi.getValue(1);
2704 
2705  // Modified the chain - switch anything that used the old chain to use
2706  // the new one.
2707  ReplaceValueWith(SDValue(N, 1), Chain);
2708 }
2709 
2710 void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
2711  SDValue &Hi) {
2712  EVT DstVTLo, DstVTHi;
2713  std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(N->getValueType(0));
2714  SDLoc dl(N);
2715 
2716  SDValue SrcLo, SrcHi;
2717  EVT SrcVT = N->getOperand(0).getValueType();
2718  if (getTypeAction(SrcVT) == TargetLowering::TypeSplitVector)
2719  GetSplitVector(N->getOperand(0), SrcLo, SrcHi);
2720  else
2721  std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(N, 0);
2722 
2723  Lo = DAG.getNode(N->getOpcode(), dl, DstVTLo, SrcLo, N->getOperand(1));
2724  Hi = DAG.getNode(N->getOpcode(), dl, DstVTHi, SrcHi, N->getOperand(1));
2725 }
2726 
2727 void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
2728  SDValue &Hi) {
2729  SDValue InLo, InHi;
2730  GetSplitVector(N->getOperand(0), InLo, InHi);
2731  SDLoc DL(N);
2732 
2733  Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, InHi.getValueType(), InHi);
2734  Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, InLo.getValueType(), InLo);
2735 }
2736 
2737 void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
2738  SDValue &Hi) {
2739  EVT VT = N->getValueType(0);
2740  SDLoc DL(N);
2741 
2742  EVT LoVT, HiVT;
2743  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
2744 
2745  SDValue Expanded = TLI.expandVectorSplice(N, DAG);
2746  Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, Expanded,
2747  DAG.getVectorIdxConstant(0, DL));
2748  Hi =
2749  DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, Expanded,
2750  DAG.getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL));
2751 }
2752 
2753 //===----------------------------------------------------------------------===//
2754 // Operand Vector Splitting
2755 //===----------------------------------------------------------------------===//
2756 
2757 /// This method is called when the specified operand of the specified node is
2758 /// found to need vector splitting. At this point, all of the result types of
2759 /// the node are known to be legal, but other operands of the node may need
2760 /// legalization as well as the specified one.
2761 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
2762  LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
2763  SDValue Res = SDValue();
2764 
2765  // See if the target wants to custom split this node.
2766  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2767  return false;
2768 
2769  switch (N->getOpcode()) {
2770  default:
2771 #ifndef NDEBUG
2772  dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
2773  N->dump(&DAG);
2774  dbgs() << "\n";
2775 #endif
2776  report_fatal_error("Do not know how to split this operator's "
2777  "operand!\n");
2778 
2779  case ISD::VP_SETCC:
2780  case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
2781  case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
2782  case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
2783  case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
2784  case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
2785  case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
2786  case ISD::VP_TRUNCATE:
2787  case ISD::TRUNCATE:
2788  Res = SplitVecOp_TruncateHelper(N);
2789  break;
2790  case ISD::STRICT_FP_ROUND:
2791  case ISD::VP_FP_ROUND:
2792  case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
2793  case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
2794  case ISD::STORE:
2795  Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
2796  break;
2797  case ISD::VP_STORE:
2798  Res = SplitVecOp_VP_STORE(cast<VPStoreSDNode>(N), OpNo);
2799  break;
2800  case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
2801  Res = SplitVecOp_VP_STRIDED_STORE(cast<VPStridedStoreSDNode>(N), OpNo);
2802  break;
2803  case ISD::MSTORE:
2804  Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
2805  break;
2806  case ISD::MSCATTER:
2807  case ISD::VP_SCATTER:
2808  Res = SplitVecOp_Scatter(cast<MemSDNode>(N), OpNo);
2809  break;
2810  case ISD::MGATHER:
2811  case ISD::VP_GATHER:
2812  Res = SplitVecOp_Gather(cast<MemSDNode>(N), OpNo);
2813  break;
2814  case ISD::VSELECT:
2815  Res = SplitVecOp_VSELECT(N, OpNo);
2816  break;
2819  case ISD::SINT_TO_FP:
2820  case ISD::UINT_TO_FP:
2821  case ISD::VP_SINT_TO_FP:
2822  case ISD::VP_UINT_TO_FP:
2823  if (N->getValueType(0).bitsLT(
2824  N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType()))
2825  Res = SplitVecOp_TruncateHelper(N);
2826  else
2827  Res = SplitVecOp_UnaryOp(N);
2828  break;
2829  case ISD::FP_TO_SINT_SAT:
2830  case ISD::FP_TO_UINT_SAT:
2831  Res = SplitVecOp_FP_TO_XINT_SAT(N);
2832  break;
2833  case ISD::FP_TO_SINT:
2834  case ISD::FP_TO_UINT:
2835  case ISD::VP_FP_TO_SINT:
2836  case ISD::VP_FP_TO_UINT:
2839  case ISD::STRICT_FP_EXTEND:
2840  case ISD::FP_EXTEND:
2841  case ISD::SIGN_EXTEND:
2842  case ISD::ZERO_EXTEND:
2843  case ISD::ANY_EXTEND:
2844  case ISD::FTRUNC:
2845  Res = SplitVecOp_UnaryOp(N);
2846  break;
2847 
2851  Res = SplitVecOp_ExtVecInRegOp(N);
2852  break;
2853 
2854  case ISD::VECREDUCE_FADD:
2855  case ISD::VECREDUCE_FMUL:
2856  case ISD::VECREDUCE_ADD:
2857  case ISD::VECREDUCE_MUL:
2858  case ISD::VECREDUCE_AND:
2859  case ISD::VECREDUCE_OR:
2860  case ISD::VECREDUCE_XOR:
2861  case ISD::VECREDUCE_SMAX:
2862  case ISD::VECREDUCE_SMIN:
2863  case ISD::VECREDUCE_UMAX:
2864  case ISD::VECREDUCE_UMIN:
2865  case ISD::VECREDUCE_FMAX:
2866  case ISD::VECREDUCE_FMIN:
2867  Res = SplitVecOp_VECREDUCE(N, OpNo);
2868  break;
2871  Res = SplitVecOp_VECREDUCE_SEQ(N);
2872  break;
2873  case ISD::VP_REDUCE_FADD:
2874  case ISD::VP_REDUCE_SEQ_FADD:
2875  case ISD::VP_REDUCE_FMUL:
2876  case ISD::VP_REDUCE_SEQ_FMUL:
2877  case ISD::VP_REDUCE_ADD:
2878  case ISD::VP_REDUCE_MUL:
2879  case ISD::VP_REDUCE_AND:
2880  case ISD::VP_REDUCE_OR:
2881  case ISD::VP_REDUCE_XOR:
2882  case ISD::VP_REDUCE_SMAX:
2883  case ISD::VP_REDUCE_SMIN:
2884  case ISD::VP_REDUCE_UMAX:
2885  case ISD::VP_REDUCE_UMIN:
2886  case ISD::VP_REDUCE_FMAX:
2887  case ISD::VP_REDUCE_FMIN:
2888  Res = SplitVecOp_VP_REDUCE(N, OpNo);
2889  break;
2890  }
2891 
2892  // If the result is null, the sub-method took care of registering results etc.
2893  if (!Res.getNode()) return false;
2894 
2895  // If the result is N, the sub-method updated N in place. Tell the legalizer
2896  // core about this.
2897  if (Res.getNode() == N)
2898  return true;
2899 
2900  if (N->isStrictFPOpcode())
2901  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
2902  "Invalid operand expansion");
2903  else
2904  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2905  "Invalid operand expansion");
2906 
2907  ReplaceValueWith(SDValue(N, 0), Res);
2908  return false;
2909 }
2910 
2911 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
2912  // The only possibility for an illegal operand is the mask, since result type
2913  // legalization would have handled this node already otherwise.
2914  assert(OpNo == 0 && "Illegal operand must be mask");
2915 
2916  SDValue Mask = N->getOperand(0);
2917  SDValue Src0 = N->getOperand(1);
2918  SDValue Src1 = N->getOperand(2);
2919  EVT Src0VT = Src0.getValueType();
2920  SDLoc DL(N);
2921  assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
2922 
2923  SDValue Lo, Hi;
2924  GetSplitVector(N->getOperand(0), Lo, Hi);
2925  assert(Lo.getValueType() == Hi.getValueType() &&
2926  "Lo and Hi have differing types");
2927 
2928  EVT LoOpVT, HiOpVT;
2929  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
2930  assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
2931 
2932  SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
2933  std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
2934  std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
2935  std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
2936 
2937  SDValue LoSelect =
2938  DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
2939  SDValue HiSelect =
2940  DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
2941 
2942  return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
2943 }
2944 
2945 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
2946  EVT ResVT = N->getValueType(0);
2947  SDValue Lo, Hi;
2948  SDLoc dl(N);
2949 
2950  SDValue VecOp = N->getOperand(OpNo);
2951  EVT VecVT = VecOp.getValueType();
2952  assert(VecVT.isVector() && "Can only split reduce vector operand");
2953  GetSplitVector(VecOp, Lo, Hi);
2954  EVT LoOpVT, HiOpVT;
2955  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2956 
2957  // Use the appropriate scalar instruction on the split subvectors before
2958  // reducing the now partially reduced smaller vector.
2959  unsigned CombineOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
2960  SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
2961  return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
2962 }
2963 
2964 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
2965  EVT ResVT = N->getValueType(0);
2966  SDValue Lo, Hi;
2967  SDLoc dl(N);
2968 
2969  SDValue AccOp = N->getOperand(0);
2970  SDValue VecOp = N->getOperand(1);
2971  SDNodeFlags Flags = N->getFlags();
2972 
2973  EVT VecVT = VecOp.getValueType();
2974  assert(VecVT.isVector() && "Can only split reduce vector operand");
2975  GetSplitVector(VecOp, Lo, Hi);
2976  EVT LoOpVT, HiOpVT;
2977  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2978 
2979  // Reduce low half.
2980  SDValue Partial = DAG.getNode(N->getOpcode(), dl, ResVT, AccOp, Lo, Flags);
2981 
2982  // Reduce high half, using low half result as initial value.
2983  return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, Hi, Flags);
2984 }
2985 
2986 SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
2987  assert(N->isVPOpcode() && "Expected VP opcode");
2988  assert(OpNo == 1 && "Can only split reduce vector operand");
2989 
2990  unsigned Opc = N->getOpcode();
2991  EVT ResVT = N->getValueType(0);
2992  SDValue Lo, Hi;
2993  SDLoc dl(N);
2994 
2995  SDValue VecOp = N->getOperand(OpNo);
2996  EVT VecVT = VecOp.getValueType();
2997  assert(VecVT.isVector() && "Can only split reduce vector operand");
2998  GetSplitVector(VecOp, Lo, Hi);
2999 
3000  SDValue MaskLo, MaskHi;
3001  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2));
3002 
3003  SDValue EVLLo, EVLHi;
3004  std::tie(EVLLo, EVLHi) = DAG.SplitEVL(N->getOperand(3), VecVT, dl);
3005 
3006  const SDNodeFlags Flags = N->getFlags();
3007 
3008  SDValue ResLo =
3009  DAG.getNode(Opc, dl, ResVT, {N->getOperand(0), Lo, MaskLo, EVLLo}, Flags);
3010  return DAG.getNode(Opc, dl, ResVT, {ResLo, Hi, MaskHi, EVLHi}, Flags);
3011 }
3012 
3013 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
3014  // The result has a legal vector type, but the input needs splitting.
3015  EVT ResVT = N->getValueType(0);
3016  SDValue Lo, Hi;
3017  SDLoc dl(N);
3018  GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3019  EVT InVT = Lo.getValueType();
3020 
3021  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
3022  InVT.getVectorElementCount());
3023 
3024  if (N->isStrictFPOpcode()) {
3025  Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
3026  { N->getOperand(0), Lo });
3027  Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
3028  { N->getOperand(0), Hi });
3029 
3030  // Build a factor node to remember that this operation is independent
3031  // of the other one.
3032  SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3033  Hi.getValue(1));
3034 
3035  // Legalize the chain result - switch anything that used the old chain to
3036  // use the new one.
3037  ReplaceValueWith(SDValue(N, 1), Ch);
3038  } else if (N->getNumOperands() == 3) {
3039  assert(N->isVPOpcode() && "Expected VP opcode");
3040  SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3041  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
3042  std::tie(EVLLo, EVLHi) =
3043  DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
3044  Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo, MaskLo, EVLLo);
3045  Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi, MaskHi, EVLHi);
3046  } else {
3047  Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
3048  Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
3049  }
3050 
3051  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
3052 }
3053 
3054 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
3055  // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
3056  // end up being split all the way down to individual components. Convert the
3057  // split pieces into integers and reassemble.
3058  SDValue Lo, Hi;
3059  GetSplitVector(N->getOperand(0), Lo, Hi);
3060  Lo = BitConvertToInteger(Lo);
3061  Hi = BitConvertToInteger(Hi);
3062 
3063  if (DAG.getDataLayout().isBigEndian())
3064  std::swap(Lo, Hi);
3065 
3066  return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
3067  JoinIntegers(Lo, Hi));
3068 }
3069 
3070 SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
3071  unsigned OpNo) {
3072  assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
3073  // We know that the result type is legal.
3074  EVT ResVT = N->getValueType(0);
3075 
3076  SDValue Vec = N->getOperand(0);
3077  SDValue SubVec = N->getOperand(1);
3078  SDValue Idx = N->getOperand(2);
3079  SDLoc dl(N);
3080 
3081  SDValue Lo, Hi;
3082  GetSplitVector(SubVec, Lo, Hi);
3083 
3084  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3085  uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
3086 
3087  SDValue FirstInsertion =
3088  DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Lo, Idx);
3089  SDValue SecondInsertion =
3090  DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, FirstInsertion, Hi,
3091  DAG.getVectorIdxConstant(IdxVal + LoElts, dl));
3092 
3093  return SecondInsertion;
3094 }
3095 
3096 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
3097  // We know that the extracted result type is legal.
3098  EVT SubVT = N->getValueType(0);
3099  SDValue Idx = N->getOperand(1);
3100  SDLoc dl(N);
3101  SDValue Lo, Hi;
3102 
3103  GetSplitVector(N->getOperand(0), Lo, Hi);
3104 
3105  uint64_t LoEltsMin = Lo.getValueType().getVectorMinNumElements();
3106  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3107 
3108  if (IdxVal < LoEltsMin) {
3109  assert(IdxVal + SubVT.getVectorMinNumElements() <= LoEltsMin &&
3110  "Extracted subvector crosses vector split!");
3111  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
3112  } else if (SubVT.isScalableVector() ==
3113  N->getOperand(0).getValueType().isScalableVector())
3114  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
3115  DAG.getVectorIdxConstant(IdxVal - LoEltsMin, dl));
3116 
3117  // After this point the DAG node only permits extracting fixed-width
3118  // subvectors from scalable vectors.
3119  assert(SubVT.isFixedLengthVector() &&
3120  "Extracting scalable subvector from fixed-width unsupported");
3121 
3122  // If the element type is i1 and we're not promoting the result, then we may
3123  // end up loading the wrong data since the bits are packed tightly into
3124  // bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal)
3125  // type at index 4, then we will load a byte starting at index 0.
3126  if (SubVT.getScalarType() == MVT::i1)
3127  report_fatal_error("Don't know how to extract fixed-width predicate "
3128  "subvector from a scalable predicate vector");
3129 
3130  // Spill the vector to the stack. We should use the alignment for
3131  // the smallest part.
3132  SDValue Vec = N->getOperand(0);
3133  EVT VecVT = Vec.getValueType();
3134  Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
3135  SDValue StackPtr =
3136  DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
3137  auto &MF = DAG.getMachineFunction();
3138  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
3139  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
3140 
3141  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
3142  SmallestAlign);
3143 
3144  // Extract the subvector by loading the correct part.
3145  StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVT, Idx);
3146 
3147  return DAG.getLoad(
3148  SubVT, dl, Store, StackPtr,
3149  MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
3150 }
3151 
3152 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3153  SDValue Vec = N->getOperand(0);
3154  SDValue Idx = N->getOperand(1);
3155  EVT VecVT = Vec.getValueType();
3156 
3157  if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Idx)) {
3158  uint64_t IdxVal = Index->getZExtValue();
3159 
3160  SDValue Lo, Hi;
3161  GetSplitVector(Vec, Lo, Hi);
3162 
3163  uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
3164 
3165  if (IdxVal < LoElts)
3166  return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
3167  else if (!Vec.getValueType().isScalableVector())
3168  return SDValue(DAG.UpdateNodeOperands(N, Hi,
3169  DAG.getConstant(IdxVal - LoElts, SDLoc(N),
3170  Idx.getValueType())), 0);
3171  }
3172 
3173  // See if the target wants to custom expand this node.
3174  if (CustomLowerNode(N, N->getValueType(0), true))
3175  return SDValue();
3176 
3177  // Make the vector elements byte-addressable if they aren't already.
3178  SDLoc dl(N);
3179  EVT EltVT = VecVT.getVectorElementType();
3180  if (VecVT.getScalarSizeInBits() < 8) {
3181  EltVT = MVT::i8;
3182  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
3183  VecVT.getVectorElementCount());
3184  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
3185  }
3186 
3187  // Store the vector to the stack.
3188  // In cases where the vector is illegal it will be broken down into parts
3189  // and stored in parts - we should use the alignment for the smallest part.
3190  Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
3191  SDValue StackPtr =
3192  DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
3193  auto &MF = DAG.getMachineFunction();
3194  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
3195  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
3196  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
3197  SmallestAlign);
3198 
3199  // Load back the required element.
3200  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
3201 
3202  // FIXME: This is to handle i1 vectors with elements promoted to i8.
3203  // i1 vector handling needs general improvement.
3204  if (N->getValueType(0).bitsLT(EltVT)) {
3205  SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
3206  MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
3207  return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
3208  }
3209 
3210  return DAG.getExtLoad(
3211  ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
3212  MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT,
3213  commonAlignment(SmallestAlign, EltVT.getFixedSizeInBits() / 8));
3214 }
3215 
3216 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
3217  SDValue Lo, Hi;
3218 
3219  // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
3220  // splitting the result has the same effect as splitting the input operand.
3221  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
3222 
3223  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
3224 }
3225 
3226 SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
3227  (void)OpNo;
3228  SDValue Lo, Hi;
3229  SplitVecRes_Gather(N, Lo, Hi);
3230 
3231  SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, N, N->getValueType(0), Lo, Hi);
3232  ReplaceValueWith(SDValue(N, 0), Res);
3233  return SDValue();
3234 }
3235 
3236 SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
3237  assert(N->isUnindexed() && "Indexed vp_store of vector?");
3238  SDValue Ch = N->getChain();
3239  SDValue Ptr = N->getBasePtr();
3240  SDValue Offset = N->getOffset();
3241  assert(Offset.isUndef() && "Unexpected VP store offset");
3242  SDValue Mask = N->getMask();
3243  SDValue EVL = N->getVectorLength();
3244  SDValue Data = N->getValue();
3245  Align Alignment = N->getOriginalAlign();
3246  SDLoc DL(N);
3247 
3248  SDValue DataLo, DataHi;
3249  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3250  // Split Data operand
3251  GetSplitVector(Data, DataLo, DataHi);
3252  else
3253  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
3254 
3255  // Split Mask operand
3256  SDValue MaskLo, MaskHi;
3257  if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
3258  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
3259  } else {
3260  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
3261  GetSplitVector(Mask, MaskLo, MaskHi);
3262  else
3263  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
3264  }
3265 
3266  EVT MemoryVT = N->getMemoryVT();
3267  EVT LoMemVT, HiMemVT;
3268  bool HiIsEmpty = false;
3269  std::tie(LoMemVT, HiMemVT) =
3270  DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
3271 
3272  // Split EVL
3273  SDValue EVLLo, EVLHi;
3274  std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, Data.getValueType(), DL);
3275 
3276  SDValue Lo, Hi;
3277  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3278  N->getPointerInfo(), MachineMemOperand::MOStore,
3279  MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3280 
3281  Lo = DAG.getStoreVP(Ch, DL, DataLo, Ptr, Offset, MaskLo, EVLLo, LoMemVT, MMO,
3282  N->getAddressingMode(), N->isTruncatingStore(),
3283  N->isCompressingStore());
3284 
3285  // If the hi vp_store has zero storage size, only the lo vp_store is needed.
3286  if (HiIsEmpty)
3287  return Lo;
3288 
3289  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
3290  N->isCompressingStore());
3291 
3292  MachinePointerInfo MPI;
3293  if (LoMemVT.isScalableVector()) {
3294  Alignment = commonAlignment(Alignment,
3295  LoMemVT.getSizeInBits().getKnownMinSize() / 8);
3296  MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
3297  } else
3298  MPI = N->getPointerInfo().getWithOffset(
3299  LoMemVT.getStoreSize().getFixedSize());
3300 
3301  MMO = DAG.getMachineFunction().getMachineMemOperand(
3303  N->getAAInfo(), N->getRanges());
3304 
3305  Hi = DAG.getStoreVP(Ch, DL, DataHi, Ptr, Offset, MaskHi, EVLHi, HiMemVT, MMO,
3306  N->getAddressingMode(), N->isTruncatingStore(),
3307  N->isCompressingStore());
3308 
3309  // Build a factor node to remember that this store is independent of the
3310  // other one.
3311  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3312 }
3313 
3314 SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
3315  unsigned OpNo) {
3316  assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?");
3317  assert(N->getOffset().isUndef() && "Unexpected VP strided store offset");
3318 
3319  SDLoc DL(N);
3320 
3321  SDValue Data = N->getValue();
3322  SDValue LoData, HiData;
3323  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3324  GetSplitVector(Data, LoData, HiData);
3325  else
3326  std::tie(LoData, HiData) = DAG.SplitVector(Data, DL);
3327 
3328  EVT LoMemVT, HiMemVT;
3329  bool HiIsEmpty = false;
3330  std::tie(LoMemVT, HiMemVT) = DAG.GetDependentSplitDestVTs(
3331  N->getMemoryVT(), LoData.getValueType(), &HiIsEmpty);
3332 
3333  SDValue Mask = N->getMask();
3334  SDValue LoMask, HiMask;
3335  if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC)
3336  SplitVecRes_SETCC(Mask.getNode(), LoMask, HiMask);
3337  else if (getTypeAction(Mask.getValueType()) ==
3339  GetSplitVector(Mask, LoMask, HiMask);
3340  else
3341  std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
3342 
3343  SDValue LoEVL, HiEVL;
3344  std::tie(LoEVL, HiEVL) =
3345  DAG.SplitEVL(N->getVectorLength(), Data.getValueType(), DL);
3346 
3347  // Generate the low vp_strided_store
3348  SDValue Lo = DAG.getStridedStoreVP(
3349  N->getChain(), DL, LoData, N->getBasePtr(), N->getOffset(),
3350  N->getStride(), LoMask, LoEVL, LoMemVT, N->getMemOperand(),
3351  N->getAddressingMode(), N->isTruncatingStore(), N->isCompressingStore());
3352 
3353  // If the high vp_strided_store has zero storage size, only the low
3354  // vp_strided_store is needed.
3355  if (HiIsEmpty)
3356  return Lo;
3357 
3358  // Generate the high vp_strided_store.
3359  // To calculate the high base address, we need to sum to the low base
3360  // address stride number of bytes for each element already stored by low,
3361  // that is: Ptr = Ptr + (LoEVL * Stride)
3362  EVT PtrVT = N->getBasePtr().getValueType();
3363  SDValue Increment =
3364  DAG.getNode(ISD::MUL, DL, PtrVT, LoEVL,
3365  DAG.getSExtOrTrunc(N->getStride(), DL, PtrVT));
3366  SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, N->getBasePtr(), Increment);
3367 
3368  Align Alignment = N->getOriginalAlign();
3369  if (LoMemVT.isScalableVector())
3370  Alignment = commonAlignment(Alignment,
3371  LoMemVT.getSizeInBits().getKnownMinSize() / 8);
3372 
3373  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3374  MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
3376  N->getAAInfo(), N->getRanges());
3377 
3378  SDValue Hi = DAG.getStridedStoreVP(
3379  N->getChain(), DL, HiData, Ptr, N->getOffset(), N->getStride(), HiMask,
3380  HiEVL, HiMemVT, MMO, N->getAddressingMode(), N->isTruncatingStore(),
3381  N->isCompressingStore());
3382 
3383  // Build a factor node to remember that this store is independent of the
3384  // other one.
3385  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3386 }
3387 
3388 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
3389  unsigned OpNo) {
3390  assert(N->isUnindexed() && "Indexed masked store of vector?");
3391  SDValue Ch = N->getChain();
3392  SDValue Ptr = N->getBasePtr();
3393  SDValue Offset = N->getOffset();
3394  assert(Offset.isUndef() && "Unexpected indexed masked store offset");
3395  SDValue Mask = N->getMask();
3396  SDValue Data = N->getValue();
3397  Align Alignment = N->getOriginalAlign();
3398  SDLoc DL(N);
3399 
3400  SDValue DataLo, DataHi;
3401  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3402  // Split Data operand
3403  GetSplitVector(Data, DataLo, DataHi);
3404  else
3405  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
3406 
3407  // Split Mask operand
3408  SDValue MaskLo, MaskHi;
3409  if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
3410  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
3411  } else {
3412  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
3413  GetSplitVector(Mask, MaskLo, MaskHi);
3414  else
3415  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
3416  }
3417 
3418  EVT MemoryVT = N->getMemoryVT();
3419  EVT LoMemVT, HiMemVT;
3420  bool HiIsEmpty = false;
3421  std::tie(LoMemVT, HiMemVT) =
3422  DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
3423 
3424  SDValue Lo, Hi, Res;
3425  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3426  N->getPointerInfo(), MachineMemOperand::MOStore,
3427  MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3428 
3429  Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
3430  N->getAddressingMode(), N->isTruncatingStore(),
3431  N->isCompressingStore());
3432 
3433  if (HiIsEmpty) {
3434  // The hi masked store has zero storage size.
3435  // Only the lo masked store is needed.
3436  Res = Lo;
3437  } else {
3438 
3439  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
3440  N->isCompressingStore());
3441 
3442  MachinePointerInfo MPI;
3443  if (LoMemVT.isScalableVector()) {
3444  Alignment = commonAlignment(
3445  Alignment, LoMemVT.getSizeInBits().getKnownMinSize() / 8);
3446  MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
3447  } else
3448  MPI = N->getPointerInfo().getWithOffset(
3449  LoMemVT.getStoreSize().getFixedSize());
3450 
3451  MMO = DAG.getMachineFunction().getMachineMemOperand(
3453  N->getAAInfo(), N->getRanges());
3454 
3455  Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
3456  N->getAddressingMode(), N->isTruncatingStore(),
3457  N->isCompressingStore());
3458 
3459  // Build a factor node to remember that this store is independent of the
3460  // other one.
3461  Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3462  }
3463 
3464  return Res;
3465 }
3466 
3467 SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
3468  SDValue Ch = N->getChain();
3469  SDValue Ptr = N->getBasePtr();
3470  EVT MemoryVT = N->getMemoryVT();
3471  Align Alignment = N->getOriginalAlign();
3472  SDLoc DL(N);
3473  struct Operands {
3474  SDValue Mask;
3475  SDValue Index;
3476  SDValue Scale;
3477  SDValue Data;
3478  } Ops = [&]() -> Operands {
3479  if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
3480  return {MSC->getMask(), MSC->getIndex(), MSC->getScale(),
3481  MSC->getValue()};
3482  }
3483  auto *VPSC = cast<VPScatterSDNode>(N);
3484  return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale(),
3485  VPSC->getValue()};
3486  }();
3487  // Split all operands
3488 
3489  EVT LoMemVT, HiMemVT;
3490  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
3491 
3492  SDValue DataLo, DataHi;
3493  if (getTypeAction(Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
3494  // Split Data operand
3495  GetSplitVector(Ops.Data, DataLo, DataHi);
3496  else
3497  std::tie(DataLo, DataHi) = DAG.SplitVector(Ops.Data, DL);
3498 
3499  // Split Mask operand
3500  SDValue MaskLo, MaskHi;
3501  if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
3502  SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
3503  } else {
3504  std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, DL);
3505  }
3506 
3507  SDValue IndexHi, IndexLo;
3508  if (getTypeAction(Ops.Index.getValueType()) ==
3510  GetSplitVector(Ops.Index, IndexLo, IndexHi);
3511  else
3512  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, DL);
3513 
3514  SDValue Lo;
3515  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3516  N->getPointerInfo(), MachineMemOperand::MOStore,
3517  MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3518 
3519  if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
3520  SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
3521  Lo =
3522  DAG.getMaskedScatter(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO,
3523  MSC->getIndexType(), MSC->isTruncatingStore());
3524 
3525  // The order of the Scatter operation after split is well defined. The "Hi"
3526  // part comes after the "Lo". So these two operations should be chained one
3527  // after another.
3528  SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
3529  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi,
3530  MMO, MSC->getIndexType(),
3531  MSC->isTruncatingStore());
3532  }
3533  auto *VPSC = cast<VPScatterSDNode>(N);
3534  SDValue EVLLo, EVLHi;
3535  std::tie(EVLLo, EVLHi) =
3536  DAG.SplitEVL(VPSC->getVectorLength(), Ops.Data.getValueType(), DL);
3537 
3538  SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
3539  Lo = DAG.getScatterVP(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO,
3540  VPSC->getIndexType());
3541 
3542  // The order of the Scatter operation after split is well defined. The "Hi"
3543  // part comes after the "Lo". So these two operations should be chained one
3544  // after another.
3545  SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
3546  return DAG.getScatterVP(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi, MMO,
3547  VPSC->getIndexType());
3548 }
3549 
3550 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
3551  assert(N->isUnindexed() && "Indexed store of vector?");
3552  assert(OpNo == 1 && "Can only split the stored value");
3553  SDLoc DL(N);
3554 
3555  bool isTruncating = N->isTruncatingStore();
3556  SDValue Ch = N->getChain();
3557  SDValue Ptr = N->getBasePtr();
3558  EVT MemoryVT = N->getMemoryVT();
3559  Align Alignment = N->getOriginalAlign();
3560  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3561  AAMDNodes AAInfo = N->getAAInfo();
3562  SDValue Lo, Hi;
3563  GetSplitVector(N->getOperand(1), Lo, Hi);
3564 
3565  EVT LoMemVT, HiMemVT;
3566  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
3567 
3568  // Scalarize if the split halves are not byte-sized.
3569  if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
3570  return TLI.scalarizeVectorStore(N, DAG);
3571 
3572  if (isTruncating)
3573  Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
3574  Alignment, MMOFlags, AAInfo);
3575  else
3576  Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3577  AAInfo);
3578 
3579  MachinePointerInfo MPI;
3580  IncrementPointer(N, LoMemVT, MPI, Ptr);
3581 
3582  if (isTruncating)
3583  Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, MPI,
3584  HiMemVT, Alignment, MMOFlags, AAInfo);
3585  else
3586  Hi = DAG.getStore(Ch, DL, Hi, Ptr, MPI, Alignment, MMOFlags, AAInfo);
3587 
3588  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3589 }
3590 
3591 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
3592  SDLoc DL(N);
3593 
3594  // The input operands all must have the same type, and we know the result
3595  // type is valid. Convert this to a buildvector which extracts all the
3596  // input elements.
3597  // TODO: If the input elements are power-two vectors, we could convert this to
3598  // a new CONCAT_VECTORS node with elements that are half-wide.
3600  EVT EltVT = N->getValueType(0).getVectorElementType();
3601  for (const SDValue &Op : N->op_values()) {
3602  for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
3603  i != e; ++i) {
3604  Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
3605  DAG.getVectorIdxConstant(i, DL)));
3606  }
3607  }
3608 
3609  return DAG.getBuildVector(N->getValueType(0), DL, Elts);
3610 }
3611 
3612 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
3613  // The result type is legal, but the input type is illegal. If splitting
3614  // ends up with the result type of each half still being legal, just
3615  // do that. If, however, that would result in an illegal result type,
3616  // we can try to get more clever with power-two vectors. Specifically,
3617  // split the input type, but also widen the result element size, then
3618  // concatenate the halves and truncate again. For example, consider a target
3619  // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
3620  // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
3621  // %inlo = v4i32 extract_subvector %in, 0
3622  // %inhi = v4i32 extract_subvector %in, 4
3623  // %lo16 = v4i16 trunc v4i32 %inlo
3624  // %hi16 = v4i16 trunc v4i32 %inhi
3625  // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
3626  // %res = v8i8 trunc v8i16 %in16
3627  //
3628  // Without this transform, the original truncate would end up being
3629  // scalarized, which is pretty much always a last resort.
3630  unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
3631  SDValue InVec = N->getOperand(OpNo);
3632  EVT InVT = InVec->getValueType(0);
3633  EVT OutVT = N->getValueType(0);
3634  ElementCount NumElements = OutVT.getVectorElementCount();
3635  bool IsFloat = OutVT.isFloatingPoint();
3636 
3637  unsigned InElementSize = InVT.getScalarSizeInBits();
3638  unsigned OutElementSize = OutVT.getScalarSizeInBits();
3639 
3640  // Determine the split output VT. If its legal we can just split dirctly.
3641  EVT LoOutVT, HiOutVT;
3642  std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
3643  assert(LoOutVT == HiOutVT && "Unequal split?");
3644 
3645  // If the input elements are only 1/2 the width of the result elements,
3646  // just use the normal splitting. Our trick only work if there's room
3647  // to split more than once.
3648  if (isTypeLegal(LoOutVT) ||
3649  InElementSize <= OutElementSize * 2)
3650  return SplitVecOp_UnaryOp(N);
3651  SDLoc DL(N);
3652 
3653  // Don't touch if this will be scalarized.
3654  EVT FinalVT = InVT;
3655  while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
3656  FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
3657 
3658  if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector)
3659  return SplitVecOp_UnaryOp(N);
3660 
3661  // Get the split input vector.
3662  SDValue InLoVec, InHiVec;
3663  GetSplitVector(InVec, InLoVec, InHiVec);
3664 
3665  // Truncate them to 1/2 the element size.
3666  //
3667  // This assumes the number of elements is a power of two; any vector that
3668  // isn't should be widened, not split.
3669  EVT HalfElementVT = IsFloat ?
3670  EVT::getFloatingPointVT(InElementSize/2) :
3671  EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
3672  EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
3673  NumElements.divideCoefficientBy(2));
3674 
3675  SDValue HalfLo;
3676  SDValue HalfHi;
3677  SDValue Chain;
3678  if (N->isStrictFPOpcode()) {
3679  HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
3680  {N->getOperand(0), InLoVec});
3681  HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
3682  {N->getOperand(0), InHiVec});
3683  // Legalize the chain result - switch anything that used the old chain to
3684  // use the new one.
3685  Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1),
3686  HalfHi.getValue(1));
3687  } else {
3688  HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
3689  HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
3690  }
3691 
3692  // Concatenate them to get the full intermediate truncation result.
3693  EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
3694  SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
3695  HalfHi);
3696  // Now finish up by truncating all the way down to the original result
3697  // type. This should normally be something that ends up being legal directly,
3698  // but in theory if a target has very wide vectors and an annoyingly
3699  // restricted set of legal types, this split can chain to build things up.
3700 
3701  if (N->isStrictFPOpcode()) {
3702  SDValue Res = DAG.getNode(
3703  ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other},
3704  {Chain, InterVec,
3705  DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))});
3706  // Relink the chain
3707  ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
3708  return Res;
3709  }
3710 
3711  return IsFloat
3712  ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
3713  DAG.getTargetConstant(
3714  0, DL, TLI.getPointerTy(DAG.getDataLayout())))
3715  : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
3716 }
3717 
3718 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
3719  assert(N->getValueType(0).isVector() &&
3720  N->getOperand(0).getValueType().isVector() &&
3721  "Operand types must be vectors");
3722  // The result has a legal vector type, but the input needs splitting.
3723  SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
3724  SDLoc DL(N);
3725  GetSplitVector(N->getOperand(0), Lo0, Hi0);
3726  GetSplitVector(N->getOperand(1), Lo1, Hi1);
3727  auto PartEltCnt = Lo0.getValueType().getVectorElementCount();
3728 
3729  LLVMContext &Context = *DAG.getContext();
3730  EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt);
3731  EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2);
3732 
3733  if (N->getOpcode() == ISD::SETCC) {
3734  LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
3735  HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
3736  } else {
3737  assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
3738  SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3739  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
3740  std::tie(EVLLo, EVLHi) =
3741  DAG.SplitEVL(N->getOperand(4), N->getValueType(0), DL);
3742  LoRes = DAG.getNode(ISD::VP_SETCC, DL, PartResVT, Lo0, Lo1,
3743  N->getOperand(2), MaskLo, EVLLo);
3744  HiRes = DAG.getNode(ISD::VP_SETCC, DL, PartResVT, Hi0, Hi1,
3745  N->getOperand(2), MaskHi, EVLHi);
3746  }
3747  SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
3748 
3749  EVT OpVT = N->getOperand(0).getValueType();
3750  ISD::NodeType ExtendCode =
3752  return DAG.getNode(ExtendCode, DL, N->getValueType(0), Con);
3753 }
3754 
3755 
3756 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
3757  // The result has a legal vector type, but the input needs splitting.
3758  EVT ResVT = N->getValueType(0);
3759  SDValue Lo, Hi;
3760  SDLoc DL(N);
3761  GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3762  EVT InVT = Lo.getValueType();
3763 
3764  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
3765  InVT.getVectorElementCount());
3766 
3767  if (N->isStrictFPOpcode()) {
3768  Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
3769  { N->getOperand(0), Lo, N->getOperand(2) });
3770  Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
3771  { N->getOperand(0), Hi, N->getOperand(2) });
3772  // Legalize the chain result - switch anything that used the old chain to
3773  // use the new one.
3774  SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
3775  Lo.getValue(1), Hi.getValue(1));
3776  ReplaceValueWith(SDValue(N, 1), NewChain);
3777  } else if (N->getOpcode() == ISD::VP_FP_ROUND) {
3778  SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3779  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
3780  std::tie(EVLLo, EVLHi) =
3781  DAG.SplitEVL(N->getOperand(2), N->getValueType(0), DL);
3782  Lo = DAG.getNode(ISD::VP_FP_ROUND, DL, OutVT, Lo, MaskLo, EVLLo);
3783  Hi = DAG.getNode(ISD::VP_FP_ROUND, DL, OutVT, Hi, MaskHi, EVLHi);
3784  } else {
3785  Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
3786  Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
3787  }
3788 
3789  return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
3790 }
3791 
3792 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
3793  // The result (and the first input) has a legal vector type, but the second
3794  // input needs splitting.
3795 
3796  SDLoc DL(N);
3797 
3798  EVT LHSLoVT, LHSHiVT;
3799  std::tie(LHSLoVT, LHSHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
3800 
3801  if (!isTypeLegal(LHSLoVT) || !isTypeLegal(LHSHiVT))
3802  return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
3803 
3804  SDValue LHSLo, LHSHi;
3805  std::tie(LHSLo, LHSHi) =
3806  DAG.SplitVector(N->getOperand(0), DL, LHSLoVT, LHSHiVT);
3807 
3808  SDValue RHSLo, RHSHi;
3809  std::tie(RHSLo, RHSHi) = DAG.SplitVector(N->getOperand(1), DL);
3810 
3811  SDValue Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLoVT, LHSLo, RHSLo);
3812  SDValue Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHiVT, LHSHi, RHSHi);
3813 
3814  return DAG.getNode(ISD::CONCAT_VECTORS, DL, N->getValueType(0), Lo, Hi);
3815 }
3816 
3817 SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
3818  EVT ResVT = N->getValueType(0);
3819  SDValue Lo, Hi;
3820  SDLoc dl(N);
3821  GetSplitVector(N->getOperand(0), Lo, Hi);
3822  EVT InVT = Lo.getValueType();
3823 
3824  EVT NewResVT =
3825  EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
3826  InVT.getVectorElementCount());
3827 
3828  Lo = DAG.getNode(N->getOpcode(), dl, NewResVT, Lo, N->getOperand(1));
3829  Hi = DAG.getNode(N->getOpcode(), dl, NewResVT, Hi, N->getOperand(1));
3830 
3831  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
3832 }
3833 
3834 //===----------------------------------------------------------------------===//
3835 // Result Vector Widening
3836 //===----------------------------------------------------------------------===//
3837 
3838 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
3839  LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
3840  dbgs() << "\n");
3841 
3842  // See if the target wants to custom widen this node.
3843  if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
3844  return;
3845 
3846  SDValue Res = SDValue();
3847 
3848  auto unrollExpandedOp = [&]() {
3849  // We're going to widen this vector op to a legal type by padding with undef
3850  // elements. If the wide vector op is eventually going to be expanded to
3851  // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
3852  // libcalls on the undef elements.
3853  EVT VT = N->getValueType(0);
3854  EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3855  if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
3856  TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
3857  Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
3858  return true;
3859  }
3860  return false;
3861  };
3862 
3863  switch (N->getOpcode()) {
3864  default:
3865 #ifndef NDEBUG
3866  dbgs() << "WidenVectorResult #" << ResNo << ": ";
3867  N->dump(&DAG);
3868  dbgs() << "\n";
3869 #endif
3870  llvm_unreachable("Do not know how to widen the result of this operator!");
3871 
3872  case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
3873  case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
3874  case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
3875  case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
3876  case ISD::INSERT_SUBVECTOR:
3877  Res = WidenVecRes_INSERT_SUBVECTOR(N);
3878  break;
3879  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
3880  case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
3881  case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
3882  case ISD::STEP_VECTOR:
3883  case ISD::SPLAT_VECTOR:
3884  case ISD::SCALAR_TO_VECTOR:
3885  Res = WidenVecRes_ScalarOp(N);
3886  break;
3887  case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
3888  case ISD::VSELECT:
3889  case ISD::SELECT:
3890  case ISD::VP_SELECT:
3891  case ISD::VP_MERGE:
3892  Res = WidenVecRes_Select(N);
3893  break;
3894  case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
3895  case ISD::VP_SETCC:
3896  case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
3897  case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
3898  case ISD::VECTOR_SHUFFLE:
3899  Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
3900  break;
3901  case ISD::VP_LOAD:
3902  Res = WidenVecRes_VP_LOAD(cast<VPLoadSDNode>(N));
3903  break;
3904  case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
3905  Res = WidenVecRes_VP_STRIDED_LOAD(cast<VPStridedLoadSDNode>(N));
3906  break;
3907  case ISD::MLOAD:
3908  Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
3909  break;
3910  case ISD::MGATHER:
3911  Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
3912  break;
3913  case ISD::VP_GATHER:
3914  Res = WidenVecRes_VP_GATHER(cast<VPGatherSDNode>(N));
3915  break;
3916  case ISD::VECTOR_REVERSE:
3917  Res = WidenVecRes_VECTOR_REVERSE(N);
3918  break;
3919 
3920  case ISD::ADD: case ISD::VP_ADD:
3921  case ISD::AND: case ISD::VP_AND:
3922  case ISD::MUL: case ISD::VP_MUL:
3923  case ISD::MULHS:
3924  case ISD::MULHU:
3925  case ISD::OR: case ISD::VP_OR:
3926  case ISD::SUB: case ISD::VP_SUB:
3927  case ISD::XOR: case ISD::VP_XOR:
3928  case ISD::SHL: case ISD::VP_SHL:
3929  case ISD::SRA: case ISD::VP_ASHR:
3930  case ISD::SRL: case ISD::VP_LSHR:
3931  case ISD::FMINNUM: case ISD::VP_FMINNUM:
3932  case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
3933  case ISD::FMINIMUM:
3934  case ISD::FMAXIMUM:
3935  case ISD::SMIN:
3936  case ISD::SMAX:
3937  case ISD::UMIN:
3938  case ISD::UMAX:
3939  case ISD::UADDSAT:
3940  case ISD::SADDSAT:
3941  case ISD::USUBSAT:
3942  case ISD::SSUBSAT:
3943  case ISD::SSHLSAT:
3944  case ISD::USHLSAT:
3945  case ISD::ROTL:
3946  case ISD::ROTR:
3947  case ISD::AVGFLOORS:
3948  case ISD::AVGFLOORU:
3949  case ISD::AVGCEILS:
3950  case ISD::AVGCEILU:
3951  // Vector-predicated binary op widening. Note that -- unlike the
3952  // unpredicated versions -- we don't have to worry about trapping on
3953  // operations like UDIV, FADD, etc., as we pass on the original vector
3954  // length parameter. This means the widened elements containing garbage
3955  // aren't active.
3956  case ISD::VP_SDIV:
3957  case ISD::VP_UDIV:
3958  case ISD::VP_SREM:
3959  case ISD::VP_UREM:
3960  case ISD::VP_FADD:
3961  case ISD::VP_FSUB:
3962  case ISD::VP_FMUL:
3963  case ISD::VP_FDIV:
3964  case ISD::VP_FREM:
3965  Res = WidenVecRes_Binary(N);
3966  break;
3967 
3968  case ISD::FPOW:
3969  case ISD::FREM:
3970  if (unrollExpandedOp())
3971  break;
3972  // If the target has custom/legal support for the scalar FP intrinsic ops
3973  // (they are probably not destined to become libcalls), then widen those
3974  // like any other binary ops.
3975  [[fallthrough]];
3976 
3977  case ISD::FADD:
3978  case ISD::FMUL:
3979  case ISD::FSUB:
3980  case ISD::FDIV:
3981  case ISD::SDIV:
3982  case ISD::UDIV:
3983  case ISD::SREM:
3984  case ISD::UREM:
3985  Res = WidenVecRes_BinaryCanTrap(N);
3986  break;
3987 
3988  case ISD::SMULFIX:
3989  case ISD::SMULFIXSAT:
3990  case ISD::UMULFIX:
3991  case ISD::UMULFIXSAT:
3992  // These are binary operations, but with an extra operand that shouldn't
3993  // be widened (the scale).
3994  Res = WidenVecRes_BinaryWithExtraScalarOp(N);
3995  break;
3996 
3997 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
3998  case ISD::STRICT_##DAGN:
3999 #include "llvm/IR/ConstrainedOps.def"
4000  Res = WidenVecRes_StrictFP(N);
4001  break;
4002 
4003  case ISD::UADDO:
4004  case ISD::SADDO:
4005  case ISD::USUBO:
4006  case ISD::SSUBO:
4007  case ISD::UMULO:
4008  case ISD::SMULO:
4009  Res = WidenVecRes_OverflowOp(N, ResNo);
4010  break;
4011 
4012  case ISD::FCOPYSIGN:
4013  Res = WidenVecRes_FCOPYSIGN(N);
4014  break;
4015 
4016  case ISD::IS_FPCLASS:
4017  Res = WidenVecRes_IS_FPCLASS(N);
4018  break;
4019 
4020  case ISD::FPOWI:
4021  Res = WidenVecRes_POWI(N);
4022  break;
4023 
4027  Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
4028  break;
4029 
4030  case ISD::ANY_EXTEND:
4031  case ISD::FP_EXTEND:
4032  case ISD::VP_FP_EXTEND:
4033  case ISD::FP_ROUND:
4034  case ISD::VP_FP_ROUND:
4035  case ISD::FP_TO_SINT:
4036  case ISD::VP_FP_TO_SINT:
4037  case ISD::FP_TO_UINT:
4038  case ISD::VP_FP_TO_UINT:
4039  case ISD::SIGN_EXTEND:
4040  case ISD::VP_SIGN_EXTEND:
4041  case ISD::SINT_TO_FP:
4042  case ISD::VP_SINT_TO_FP:
4043  case ISD::VP_TRUNCATE:
4044  case ISD::TRUNCATE:
4045  case ISD::UINT_TO_FP:
4046  case ISD::VP_UINT_TO_FP:
4047  case ISD::ZERO_EXTEND:
4048  case ISD::VP_ZERO_EXTEND:
4049  Res = WidenVecRes_Convert(N);
4050  break;
4051 
4052  case ISD::FP_TO_SINT_SAT:
4053  case ISD::FP_TO_UINT_SAT:
4054  Res = WidenVecRes_FP_TO_XINT_SAT(N);
4055  break;
4056 
4057  case ISD::FABS:
4058  case ISD::FCEIL:
4059  case ISD::FCOS:
4060  case ISD::FEXP:
4061  case ISD::FEXP2:
4062  case ISD::FFLOOR:
4063  case ISD::FLOG:
4064  case ISD::FLOG10:
4065  case ISD::FLOG2:
4066  case ISD::FNEARBYINT:
4067  case ISD::FRINT:
4068  case ISD::FROUND:
4069  case ISD::FROUNDEVEN:
4070  case ISD::FSIN:
4071  case ISD::FSQRT:
4072  case ISD::FTRUNC:
4073  if (unrollExpandedOp())
4074  break;
4075  // If the target has custom/legal support for the scalar FP intrinsic ops
4076  // (they are probably not destined to become libcalls), then widen those
4077  // like any other unary ops.
4078  [[fallthrough]];
4079 
4080  case ISD::ABS:
4081  case ISD::BITREVERSE:
4082  case ISD::BSWAP:
4083  case ISD::CTLZ:
4084  case ISD::CTLZ_ZERO_UNDEF:
4085  case ISD::CTPOP:
4086  case ISD::CTTZ:
4087  case ISD::CTTZ_ZERO_UNDEF:
4088  case ISD::FNEG: case ISD::VP_FNEG:
4089  case ISD::VP_FABS:
4090  case ISD::VP_SQRT:
4091  case ISD::VP_FCEIL:
4092  case ISD::VP_FFLOOR:
4093  case ISD::VP_FROUND:
4094  case ISD::VP_FROUNDEVEN:
4095  case ISD::FREEZE:
4096  case ISD::ARITH_FENCE:
4097  case ISD::FCANONICALIZE:
4098  Res = WidenVecRes_Unary(N);
4099  break;
4100  case ISD::FMA: case ISD::VP_FMA:
4101  case ISD::FSHL:
4102  case ISD::FSHR:
4103  Res = WidenVecRes_Ternary(N);
4104  break;
4105  }
4106 
4107  // If Res is null, the sub-method took care of registering the result.
4108  if (Res.getNode())
4109  SetWidenedVector(SDValue(N, ResNo), Res);
4110 }
4111 
4112 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
4113  // Ternary op widening.
4114  SDLoc dl(N);
4115  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4116  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4117  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4118  SDValue InOp3 = GetWidenedVector(N->getOperand(2));
4119  if (N->getNumOperands() == 3)
4120  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
4121 
4122  assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
4123  assert(N->isVPOpcode() && "Expected VP opcode");
4124 
4125  SDValue Mask =
4126  GetWidenedMask(N->getOperand(3), WidenVT.getVectorElementCount());
4127  return DAG.getNode(N->getOpcode(), dl, WidenVT,
4128  {InOp1, InOp2, InOp3, Mask, N->getOperand(4)});
4129 }
4130 
4131 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
4132  // Binary op widening.
4133  SDLoc dl(N);
4134  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4135  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4136  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4137  if (N->getNumOperands() == 2)
4138  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2,
4139  N->getFlags());
4140 
4141  assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
4142  assert(N->isVPOpcode() && "Expected VP opcode");
4143 
4144  SDValue Mask =
4145  GetWidenedMask(N->getOperand(2), WidenVT.getVectorElementCount());
4146  return DAG.getNode(N->getOpcode(), dl, WidenVT,
4147  {InOp1, InOp2, Mask, N->getOperand(3)}, N->getFlags());
4148 }
4149 
4150 SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
4151  // Binary op widening, but with an extra operand that shouldn't be widened.
4152  SDLoc dl(N);
4153  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4154  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4155  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4156  SDValue InOp3 = N->getOperand(2);
4157  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3,
4158  N->getFlags());
4159 }
4160 
4161 // Given a vector of operations that have been broken up to widen, see
4162 // if we can collect them together into the next widest legal VT. This
4163 // implementation is trap-safe.
4165  SmallVectorImpl<SDValue> &ConcatOps,
4166  unsigned ConcatEnd, EVT VT, EVT MaxVT,
4167  EVT WidenVT) {
4168  // Check to see if we have a single operation with the widen type.
4169  if (ConcatEnd == 1) {
4170  VT = ConcatOps[0].getValueType();
4171  if (VT == WidenVT)
4172  return ConcatOps[0];
4173  }
4174 
4175  SDLoc dl(ConcatOps[0]);
4176  EVT WidenEltVT = WidenVT.getVectorElementType();
4177 
4178  // while (Some element of ConcatOps is not of type MaxVT) {
4179  // From the end of ConcatOps, collect elements of the same type and put
4180  // them into an op of the next larger supported type
4181  // }
4182  while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
4183  int Idx = ConcatEnd - 1;
4184  VT = ConcatOps[Idx--].getValueType();
4185  while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
4186  Idx--;
4187 
4188  int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
4189  EVT NextVT;
4190  do {
4191  NextSize *= 2;
4192  NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
4193  } while (!TLI.isTypeLegal(NextVT));
4194 
4195  if (!VT.isVector()) {
4196  // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
4197  SDValue VecOp = DAG.getUNDEF(NextVT);
4198  unsigned NumToInsert = ConcatEnd - Idx - 1;
4199  for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
4200  VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp,
4201  ConcatOps[OpIdx], DAG.getVectorIdxConstant(i, dl));
4202  }
4203  ConcatOps[Idx+1] = VecOp;
4204  ConcatEnd = Idx + 2;
4205  } else {
4206  // Vector type, create a CONCAT_VECTORS of type NextVT
4207  SDValue undefVec = DAG.getUNDEF(VT);
4208  unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
4209  SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
4210  unsigned RealVals = ConcatEnd - Idx - 1;
4211  unsigned SubConcatEnd = 0;
4212  unsigned SubConcatIdx = Idx + 1;
4213  while (SubConcatEnd < RealVals)
4214  SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
4215  while (SubConcatEnd < OpsToConcat)
4216  SubConcatOps[SubConcatEnd++] = undefVec;
4217  ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
4218  NextVT, SubConcatOps);
4219  ConcatEnd = SubConcatIdx + 1;
4220  }
4221  }
4222 
4223  // Check to see if we have a single operation with the widen type.
4224  if (ConcatEnd == 1) {
4225  VT = ConcatOps[0].getValueType();
4226  if (VT == WidenVT)
4227  return ConcatOps[0];
4228  }
4229 
4230  // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
4231  unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
4232  if (NumOps != ConcatEnd ) {
4233  SDValue UndefVal = DAG.getUNDEF(MaxVT);
4234  for (unsigned j = ConcatEnd; j < NumOps; ++j)
4235  ConcatOps[j] = UndefVal;
4236  }
4237  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
4238  makeArrayRef(ConcatOps.data(), NumOps));
4239 }
4240 
4241 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
4242  // Binary op widening for operations that can trap.
4243  unsigned Opcode = N->getOpcode();
4244  SDLoc dl(N);
4245  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4246  EVT WidenEltVT = WidenVT.getVectorElementType();
4247  EVT VT = WidenVT;
4248  unsigned NumElts = VT.getVectorMinNumElements();
4249  const SDNodeFlags Flags = N->getFlags();
4250  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
4251  NumElts = NumElts / 2;
4252  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
4253  }
4254 
4255  if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
4256  // Operation doesn't trap so just widen as normal.
4257  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4258  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4259  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
4260  }
4261 
4262  // FIXME: Improve support for scalable vectors.
4263  assert(!VT.isScalableVector() && "Scalable vectors not handled yet.");
4264 
4265  // No legal vector version so unroll the vector operation and then widen.
4266  if (NumElts == 1)
4267  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
4268 
4269  // Since the operation can trap, apply operation on the original vector.
4270  EVT MaxVT = VT;
4271  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4272  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4273  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
4274 
4275  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
4276  unsigned ConcatEnd = 0; // Current ConcatOps index.
4277  int Idx = 0; // Current Idx into input vectors.
4278 
4279  // NumElts := greatest legal vector size (at most WidenVT)
4280  // while (orig. vector has unhandled elements) {
4281  // take munches of size NumElts from the beginning and add to ConcatOps
4282  // NumElts := next smaller supported vector size or 1
4283  // }
4284  while (CurNumElts != 0) {
4285  while (CurNumElts >= NumElts) {
4286  SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
4287  DAG.getVectorIdxConstant(Idx, dl));
4288  SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
4289  DAG.getVectorIdxConstant(Idx, dl));
4290  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
4291  Idx += NumElts;
4292  CurNumElts -= NumElts;
4293  }
4294  do {
4295  NumElts = NumElts / 2;
4296  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
4297  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
4298 
4299  if (NumElts == 1) {
4300  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
4301  SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
4302  InOp1, DAG.getVectorIdxConstant(Idx, dl));
4303  SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
4304  InOp2, DAG.getVectorIdxConstant(Idx, dl));
4305  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
4306  EOp1, EOp2, Flags);
4307  }
4308  CurNumElts = 0;
4309  }
4310  }
4311 
4312  return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
4313 }
4314 
4315 SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
4316  switch (N->getOpcode()) {
4317  case ISD::STRICT_FSETCC:
4318  case ISD::STRICT_FSETCCS:
4319  return WidenVecRes_STRICT_FSETCC(N);
4320  case ISD::STRICT_FP_EXTEND:
4321  case ISD::STRICT_FP_ROUND:
4326  return WidenVecRes_Convert_StrictFP(N);
4327  default:
4328  break;
4329  }
4330 
4331  // StrictFP op widening for operations that can trap.
4332  unsigned NumOpers = N->getNumOperands();
4333  unsigned Opcode = N->getOpcode();
4334  SDLoc dl(N);
4335  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4336  EVT WidenEltVT = WidenVT.getVectorElementType();
4337  EVT VT = WidenVT;
4338  unsigned NumElts = VT.getVectorNumElements();
4339  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
4340  NumElts = NumElts / 2;
4341  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
4342  }
4343 
4344  // No legal vector version so unroll the vector operation and then widen.
4345  if (NumElts == 1)
4346  return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements());
4347 
4348  // Since the operation can trap, apply operation on the original vector.
4349  EVT MaxVT = VT;
4351  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
4352 
4353  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
4354  SmallVector<SDValue, 16> Chains;
4355  unsigned ConcatEnd = 0; // Current ConcatOps index.
4356  int Idx = 0; // Current Idx into input vectors.
4357 
4358  // The Chain is the first operand.
4359  InOps.push_back(N->getOperand(0));
4360 
4361  // Now process the remaining operands.
4362  for (unsigned i = 1; i < NumOpers; ++i) {
4363  SDValue Oper = N->getOperand(i);
4364 
4365  if (Oper.getValueType().isVector()) {
4366  assert(Oper.getValueType() == N->getValueType(0) &&
4367  "Invalid operand type to widen!");
4368  Oper = GetWidenedVector(Oper);
4369  }
4370 
4371  InOps.push_back(Oper);
4372  }
4373 
4374  // NumElts := greatest legal vector size (at most WidenVT)
4375  // while (orig. vector has unhandled elements) {
4376  // take munches of size NumElts from the beginning and add to ConcatOps
4377  // NumElts := next smaller supported vector size or 1
4378  // }
4379  while (CurNumElts != 0) {
4380  while (CurNumElts >= NumElts) {
4382 
4383  for (unsigned i = 0; i < NumOpers; ++i) {
4384  SDValue Op = InOps[i];
4385 
4386  if (Op.getValueType().isVector())
4387  Op = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Op,
4388  DAG.getVectorIdxConstant(Idx, dl));
4389 
4390  EOps.push_back(Op);
4391  }
4392 
4393  EVT OperVT[] = {VT, MVT::Other};
4394  SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
4395  ConcatOps[ConcatEnd++] = Oper;
4396  Chains.push_back(Oper.getValue(1));
4397  Idx += NumElts;
4398  CurNumElts -= NumElts;
4399  }
4400  do {
4401  NumElts = NumElts / 2;
4402  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
4403  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
4404 
4405  if (NumElts == 1) {
4406  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
4408 
4409  for (unsigned i = 0; i < NumOpers; ++i) {
4410  SDValue Op = InOps[i];
4411 
4412  if (Op.getValueType().isVector())
4413  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op,
4414  DAG.getVectorIdxConstant(Idx, dl));
4415 
4416  EOps.push_back(Op);
4417  }
4418 
4419  EVT WidenVT[] = {WidenEltVT, MVT::Other};
4420  SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
4421  ConcatOps[ConcatEnd++] = Oper;
4422  Chains.push_back(Oper.getValue(1));
4423  }
4424  CurNumElts = 0;
4425  }
4426  }
4427 
4428  // Build a factor node to remember all the Ops that have been created.
4429  SDValue NewChain;
4430  if (Chains.size() == 1)
4431  NewChain = Chains[0];
4432  else
4433  NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
4434  ReplaceValueWith(SDValue(N, 1), NewChain);
4435 
4436  return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
4437 }
4438 
4439 SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
4440  SDLoc DL(N);
4441  EVT ResVT = N->getValueType(0);
4442  EVT OvVT = N->getValueType(1);
4443  EVT WideResVT, WideOvVT;
4444  SDValue WideLHS, WideRHS;
4445 
4446  // TODO: This might result in a widen/split loop.
4447  if (ResNo == 0) {
4448  WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
4449  WideOvVT = EVT::getVectorVT(
4450  *DAG.getContext(), OvVT.getVectorElementType(),
4451  WideResVT.getVectorNumElements());
4452 
4453  WideLHS = GetWidenedVector(N->getOperand(0));
4454  WideRHS = GetWidenedVector(N->getOperand(1));
4455  } else {
4456  WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT);
4457  WideResVT = EVT::getVectorVT(
4458  *DAG.getContext(), ResVT.getVectorElementType(),
4459  WideOvVT.getVectorNumElements());
4460 
4461  SDValue Zero = DAG.getVectorIdxConstant(0, DL);
4462  WideLHS = DAG.getNode(
4463  ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
4464  N->getOperand(0), Zero);
4465  WideRHS = DAG.getNode(
4466  ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
4467  N->getOperand(1), Zero);
4468  }
4469 
4470  SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT);
4471  SDNode *WideNode = DAG.getNode(
4472  N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode();
4473 
4474  // Replace the other vector result not being explicitly widened here.
4475  unsigned OtherNo = 1 - ResNo;
4476  EVT OtherVT = N->getValueType(OtherNo);
4477  if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) {
4478  SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo));
4479  } else {
4480  SDValue Zero = DAG.getVectorIdxConstant(0, DL);
4481  SDValue OtherVal = DAG.getNode(
4482  ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero);
4483  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
4484  }
4485 
4486  return SDValue(WideNode, ResNo);
4487 }
4488 
4489 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
4490  LLVMContext &Ctx = *DAG.getContext();
4491  SDValue InOp = N->getOperand(0);
4492  SDLoc DL(N);
4493 
4494  EVT WidenVT = TLI.getTypeToTransformTo(Ctx, N->getValueType(0));
4495  ElementCount WidenEC = WidenVT.getVectorElementCount();
4496 
4497  EVT InVT = InOp.getValueType();
4498 
4499  unsigned Opcode = N->getOpcode();
4500  const SDNodeFlags Flags = N->getFlags();
4501 
4502  // Handle the case of ZERO_EXTEND where the promoted InVT element size does
4503  // not equal that of WidenVT.
4504  if (N->getOpcode() == ISD::ZERO_EXTEND &&
4505  getTypeAction(InVT) == TargetLowering::TypePromoteInteger &&
4506  TLI.getTypeToTransformTo(Ctx, InVT).getScalarSizeInBits() !=
4507  WidenVT.getScalarSizeInBits()) {
4508  InOp = ZExtPromotedInteger(InOp);
4509  InVT = InOp.getValueType();
4510  if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
4511  Opcode = ISD::TRUNCATE;
4512  }
4513 
4514  EVT InEltVT = InVT.getVectorElementType();
4515  EVT InWidenVT = EVT::getVectorVT(Ctx, InEltVT, WidenEC);
4516  ElementCount InVTEC = InVT.getVectorElementCount();
4517 
4518  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
4519  InOp = GetWidenedVector(N->getOperand(0));
4520  InVT = InOp.getValueType();
4521  InVTEC = InVT.getVectorElementCount();
4522  if (InVTEC == WidenEC) {
4523  if (N->getNumOperands() == 1)
4524  return DAG.getNode(Opcode, DL, WidenVT, InOp);
4525  if (N->getNumOperands() == 3) {
4526  assert(N->isVPOpcode() && "Expected VP opcode");
4527  SDValue Mask =
4528  GetWidenedMask(N->getOperand(1), WidenVT.getVectorElementCount());
4529  return DAG.getNode(Opcode, DL, WidenVT, InOp, Mask, N->getOperand(2));
4530  }
4531  return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
4532  }
4533  if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
4534  // If both input and result vector types are of same width, extend
4535  // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
4536  // accepts fewer elements in the result than in the input.
4537  if (Opcode == ISD::ANY_EXTEND)
4538  return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
4539  if (Opcode == ISD::SIGN_EXTEND)
4540  return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
4541  if (Opcode == ISD::ZERO_EXTEND)
4542  return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
4543  }
4544  }
4545 
4546  if (TLI.isTypeLegal(InWidenVT)) {
4547  // Because the result and the input are different vector types, widening
4548  // the result could create a legal type but widening the input might make
4549  // it an illegal type that might lead to repeatedly splitting the input
4550  // and then widening it. To avoid this, we widen the input only if
4551  // it results in a legal type.
4552  if (WidenEC.isKnownMultipleOf(InVTEC.getKnownMinValue())) {
4553  // Widen the input and call convert on the widened input vector.
4554  unsigned NumConcat =
4555  WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
4556  SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
4557  Ops[0] = InOp;
4558  SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
4559  if (N->getNumOperands() == 1)
4560  return DAG.getNode(Opcode, DL, WidenVT, InVec);
4561  return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
4562  }
4563 
4564  if (InVTEC.isKnownMultipleOf(WidenEC.getKnownMinValue())) {
4565  SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
4566  DAG.getVectorIdxConstant(0, DL));
4567  // Extract the input and convert the shorten input vector.
4568  if (N->getNumOperands() == 1)
4569  return DAG.getNode(Opcode, DL, WidenVT, InVal);
4570  return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
4571  }
4572  }
4573 
4574  // Otherwise unroll into some nasty scalar code and rebuild the vector.
4575  EVT EltVT = WidenVT.getVectorElementType();
4576  SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getUNDEF(EltVT));
4577  // Use the original element count so we don't do more scalar opts than
4578  // necessary.
4579  unsigned MinElts = N->getValueType(0).getVectorNumElements();
4580  for (unsigned i=0; i < MinElts; ++i) {
4581  SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
4582  DAG.getVectorIdxConstant(i, DL));
4583  if (N->getNumOperands() == 1)
4584  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
4585  else
4586  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
4587  }
4588 
4589  return DAG.getBuildVector(WidenVT, DL, Ops);
4590 }
4591 
4592 SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
4593  SDLoc dl(N);
4594  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4595  ElementCount WidenNumElts = WidenVT.getVectorElementCount();
4596 
4597  SDValue Src = N->getOperand(0);
4598  EVT SrcVT = Src.getValueType();
4599 
4600  // Also widen the input.
4601  if (getTypeAction(SrcVT) == TargetLowering::TypeWidenVector) {
4602  Src = GetWidenedVector(Src);
4603  SrcVT = Src.getValueType();
4604  }
4605 
4606  // Input and output not widened to the same size, give up.
4607  if (WidenNumElts != SrcVT.get