LLVM 19.0.0git
LegalizeVectorTypes.cpp
Go to the documentation of this file.
1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file performs vector type splitting and scalarization for LegalizeTypes.
10// Scalarization is the act of changing a computation in an illegal one-element
11// vector type to be a computation in its scalar element type. For example,
12// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13// as a base case when scalarizing vector arithmetic like <4 x f32>, which
14// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15// types.
16// Splitting is the act of changing a computation in an invalid vector type to
17// be a computation in two vectors of half the size. For example, implementing
18// <128 x f32> operations in terms of two <64 x f32> operations.
19//
20//===----------------------------------------------------------------------===//
21
22#include "LegalizeTypes.h"
27#include "llvm/IR/DataLayout.h"
31#include <numeric>
32
33using namespace llvm;
34
35#define DEBUG_TYPE "legalize-types"
36
37//===----------------------------------------------------------------------===//
38// Result Vector Scalarization: <1 x ty> -> ty.
39//===----------------------------------------------------------------------===//
40
41void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
42 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
43 N->dump(&DAG));
44 SDValue R = SDValue();
45
46 switch (N->getOpcode()) {
47 default:
48#ifndef NDEBUG
49 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
50 N->dump(&DAG);
51 dbgs() << "\n";
52#endif
53 report_fatal_error("Do not know how to scalarize the result of this "
54 "operator!\n");
55
56 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
57 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
58 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
59 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
60 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
61 case ISD::FPOWI: R = ScalarizeVecRes_ExpOp(N); break;
62 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
63 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
64 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
65 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
66 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
67 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
68 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
69 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
70 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
71 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
72 case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
76 R = ScalarizeVecRes_VecInregOp(N);
77 break;
78 case ISD::ABS:
79 case ISD::ANY_EXTEND:
80 case ISD::BITREVERSE:
81 case ISD::BSWAP:
82 case ISD::CTLZ:
84 case ISD::CTPOP:
85 case ISD::CTTZ:
87 case ISD::FABS:
88 case ISD::FCEIL:
89 case ISD::FCOS:
90 case ISD::FEXP:
91 case ISD::FEXP2:
92 case ISD::FEXP10:
93 case ISD::FFLOOR:
94 case ISD::FLOG:
95 case ISD::FLOG10:
96 case ISD::FLOG2:
97 case ISD::FNEARBYINT:
98 case ISD::FNEG:
99 case ISD::FREEZE:
100 case ISD::ARITH_FENCE:
101 case ISD::FP_EXTEND:
102 case ISD::FP_TO_SINT:
103 case ISD::FP_TO_UINT:
104 case ISD::FRINT:
105 case ISD::LRINT:
106 case ISD::LLRINT:
107 case ISD::FROUND:
108 case ISD::FROUNDEVEN:
109 case ISD::FSIN:
110 case ISD::FSQRT:
111 case ISD::FTRUNC:
112 case ISD::SIGN_EXTEND:
113 case ISD::SINT_TO_FP:
114 case ISD::TRUNCATE:
115 case ISD::UINT_TO_FP:
116 case ISD::ZERO_EXTEND:
118 R = ScalarizeVecRes_UnaryOp(N);
119 break;
121 R = ScalarizeVecRes_ADDRSPACECAST(N);
122 break;
123 case ISD::FFREXP:
124 R = ScalarizeVecRes_FFREXP(N, ResNo);
125 break;
126 case ISD::ADD:
127 case ISD::AND:
128 case ISD::FADD:
129 case ISD::FCOPYSIGN:
130 case ISD::FDIV:
131 case ISD::FMUL:
132 case ISD::FMINNUM:
133 case ISD::FMAXNUM:
136 case ISD::FMINIMUM:
137 case ISD::FMAXIMUM:
138 case ISD::FLDEXP:
139 case ISD::SMIN:
140 case ISD::SMAX:
141 case ISD::UMIN:
142 case ISD::UMAX:
143
144 case ISD::SADDSAT:
145 case ISD::UADDSAT:
146 case ISD::SSUBSAT:
147 case ISD::USUBSAT:
148 case ISD::SSHLSAT:
149 case ISD::USHLSAT:
150
151 case ISD::FPOW:
152 case ISD::FREM:
153 case ISD::FSUB:
154 case ISD::MUL:
155 case ISD::MULHS:
156 case ISD::MULHU:
157 case ISD::OR:
158 case ISD::SDIV:
159 case ISD::SREM:
160 case ISD::SUB:
161 case ISD::UDIV:
162 case ISD::UREM:
163 case ISD::XOR:
164 case ISD::SHL:
165 case ISD::SRA:
166 case ISD::SRL:
167 case ISD::ROTL:
168 case ISD::ROTR:
169 R = ScalarizeVecRes_BinOp(N);
170 break;
171 case ISD::FMA:
172 case ISD::FSHL:
173 case ISD::FSHR:
174 R = ScalarizeVecRes_TernaryOp(N);
175 break;
176
177#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
178 case ISD::STRICT_##DAGN:
179#include "llvm/IR/ConstrainedOps.def"
180 R = ScalarizeVecRes_StrictFPOp(N);
181 break;
182
185 R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
186 break;
187
188 case ISD::UADDO:
189 case ISD::SADDO:
190 case ISD::USUBO:
191 case ISD::SSUBO:
192 case ISD::UMULO:
193 case ISD::SMULO:
194 R = ScalarizeVecRes_OverflowOp(N, ResNo);
195 break;
196 case ISD::SMULFIX:
197 case ISD::SMULFIXSAT:
198 case ISD::UMULFIX:
199 case ISD::UMULFIXSAT:
200 case ISD::SDIVFIX:
201 case ISD::SDIVFIXSAT:
202 case ISD::UDIVFIX:
203 case ISD::UDIVFIXSAT:
204 R = ScalarizeVecRes_FIX(N);
205 break;
206 }
207
208 // If R is null, the sub-method took care of registering the result.
209 if (R.getNode())
210 SetScalarizedVector(SDValue(N, ResNo), R);
211}
212
213SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
214 SDValue LHS = GetScalarizedVector(N->getOperand(0));
215 SDValue RHS = GetScalarizedVector(N->getOperand(1));
216 return DAG.getNode(N->getOpcode(), SDLoc(N),
217 LHS.getValueType(), LHS, RHS, N->getFlags());
218}
219
220SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
221 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
222 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
223 SDValue Op2 = GetScalarizedVector(N->getOperand(2));
224 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
225 Op2, N->getFlags());
226}
227
228SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
229 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
230 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
231 SDValue Op2 = N->getOperand(2);
232 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
233 Op2, N->getFlags());
234}
235
236SDValue DAGTypeLegalizer::ScalarizeVecRes_FFREXP(SDNode *N, unsigned ResNo) {
237 assert(N->getValueType(0).getVectorNumElements() == 1 &&
238 "Unexpected vector type!");
239 SDValue Elt = GetScalarizedVector(N->getOperand(0));
240
241 EVT VT0 = N->getValueType(0);
242 EVT VT1 = N->getValueType(1);
243 SDLoc dl(N);
244
245 SDNode *ScalarNode =
246 DAG.getNode(N->getOpcode(), dl,
247 {VT0.getScalarType(), VT1.getScalarType()}, Elt)
248 .getNode();
249
250 // Replace the other vector result not being explicitly scalarized here.
251 unsigned OtherNo = 1 - ResNo;
252 EVT OtherVT = N->getValueType(OtherNo);
253 if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
254 SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
255 } else {
256 SDValue OtherVal = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, OtherVT,
257 SDValue(ScalarNode, OtherNo));
258 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
259 }
260
261 return SDValue(ScalarNode, ResNo);
262}
263
264SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
265 EVT VT = N->getValueType(0).getVectorElementType();
266 unsigned NumOpers = N->getNumOperands();
267 SDValue Chain = N->getOperand(0);
268 EVT ValueVTs[] = {VT, MVT::Other};
269 SDLoc dl(N);
270
271 SmallVector<SDValue, 4> Opers(NumOpers);
272
273 // The Chain is the first operand.
274 Opers[0] = Chain;
275
276 // Now process the remaining operands.
277 for (unsigned i = 1; i < NumOpers; ++i) {
278 SDValue Oper = N->getOperand(i);
279 EVT OperVT = Oper.getValueType();
280
281 if (OperVT.isVector()) {
282 if (getTypeAction(OperVT) == TargetLowering::TypeScalarizeVector)
283 Oper = GetScalarizedVector(Oper);
284 else
285 Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
286 OperVT.getVectorElementType(), Oper,
287 DAG.getVectorIdxConstant(0, dl));
288 }
289
290 Opers[i] = Oper;
291 }
292
293 SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs),
294 Opers, N->getFlags());
295
296 // Legalize the chain result - switch anything that used the old chain to
297 // use the new one.
298 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
299 return Result;
300}
301
302SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
303 unsigned ResNo) {
304 SDLoc DL(N);
305 EVT ResVT = N->getValueType(0);
306 EVT OvVT = N->getValueType(1);
307
308 SDValue ScalarLHS, ScalarRHS;
309 if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
310 ScalarLHS = GetScalarizedVector(N->getOperand(0));
311 ScalarRHS = GetScalarizedVector(N->getOperand(1));
312 } else {
313 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
314 DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
315 DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
316 ScalarLHS = ElemsLHS[0];
317 ScalarRHS = ElemsRHS[0];
318 }
319
320 SDVTList ScalarVTs = DAG.getVTList(
322 SDNode *ScalarNode = DAG.getNode(
323 N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
324 ScalarNode->setFlags(N->getFlags());
325
326 // Replace the other vector result not being explicitly scalarized here.
327 unsigned OtherNo = 1 - ResNo;
328 EVT OtherVT = N->getValueType(OtherNo);
329 if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
330 SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
331 } else {
332 SDValue OtherVal = DAG.getNode(
333 ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
334 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
335 }
336
337 return SDValue(ScalarNode, ResNo);
338}
339
340SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
341 unsigned ResNo) {
342 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
343 return GetScalarizedVector(Op);
344}
345
346SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
347 SDValue Op = N->getOperand(0);
348 if (Op.getValueType().isVector()
349 && Op.getValueType().getVectorNumElements() == 1
350 && !isSimpleLegalType(Op.getValueType()))
351 Op = GetScalarizedVector(Op);
352 EVT NewVT = N->getValueType(0).getVectorElementType();
353 return DAG.getNode(ISD::BITCAST, SDLoc(N),
354 NewVT, Op);
355}
356
357SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
358 EVT EltVT = N->getValueType(0).getVectorElementType();
359 SDValue InOp = N->getOperand(0);
360 // The BUILD_VECTOR operands may be of wider element types and
361 // we may need to truncate them back to the requested return type.
362 if (EltVT.isInteger())
363 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
364 return InOp;
365}
366
367SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
369 N->getValueType(0).getVectorElementType(),
370 N->getOperand(0), N->getOperand(1));
371}
372
373SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
374 SDLoc DL(N);
375 SDValue Op = N->getOperand(0);
376 EVT OpVT = Op.getValueType();
377 // The result needs scalarizing, but it's not a given that the source does.
378 // See similar logic in ScalarizeVecRes_UnaryOp.
379 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
380 Op = GetScalarizedVector(Op);
381 } else {
382 EVT VT = OpVT.getVectorElementType();
384 DAG.getVectorIdxConstant(0, DL));
385 }
386 return DAG.getNode(ISD::FP_ROUND, DL,
387 N->getValueType(0).getVectorElementType(), Op,
388 N->getOperand(1));
389}
390
391SDValue DAGTypeLegalizer::ScalarizeVecRes_ExpOp(SDNode *N) {
392 SDValue Op = GetScalarizedVector(N->getOperand(0));
393 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op,
394 N->getOperand(1));
395}
396
397SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
398 // The value to insert may have a wider type than the vector element type,
399 // so be sure to truncate it to the element type if necessary.
400 SDValue Op = N->getOperand(1);
401 EVT EltVT = N->getValueType(0).getVectorElementType();
402 if (Op.getValueType() != EltVT)
403 // FIXME: Can this happen for floating point types?
404 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
405 return Op;
406}
407
408SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
409 assert(N->isUnindexed() && "Indexed vector load?");
410
411 SDValue Result = DAG.getLoad(
412 ISD::UNINDEXED, N->getExtensionType(),
413 N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
414 N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
415 N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
416 N->getOriginalAlign(), N->getMemOperand()->getFlags(), N->getAAInfo());
417
418 // Legalize the chain result - switch anything that used the old chain to
419 // use the new one.
420 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
421 return Result;
422}
423
424SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
425 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
426 EVT DestVT = N->getValueType(0).getVectorElementType();
427 SDValue Op = N->getOperand(0);
428 EVT OpVT = Op.getValueType();
429 SDLoc DL(N);
430 // The result needs scalarizing, but it's not a given that the source does.
431 // This is a workaround for targets where it's impossible to scalarize the
432 // result of a conversion, because the source type is legal.
433 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
434 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
435 // legal and was not scalarized.
436 // See the similar logic in ScalarizeVecRes_SETCC
437 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
438 Op = GetScalarizedVector(Op);
439 } else {
440 EVT VT = OpVT.getVectorElementType();
442 DAG.getVectorIdxConstant(0, DL));
443 }
444 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op, N->getFlags());
445}
446
447SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
448 EVT EltVT = N->getValueType(0).getVectorElementType();
449 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
450 SDValue LHS = GetScalarizedVector(N->getOperand(0));
451 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
452 LHS, DAG.getValueType(ExtVT));
453}
454
455SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
456 SDLoc DL(N);
457 SDValue Op = N->getOperand(0);
458
459 EVT OpVT = Op.getValueType();
460 EVT OpEltVT = OpVT.getVectorElementType();
461 EVT EltVT = N->getValueType(0).getVectorElementType();
462
463 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
464 Op = GetScalarizedVector(Op);
465 } else {
466 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
467 DAG.getVectorIdxConstant(0, DL));
468 }
469
470 switch (N->getOpcode()) {
472 return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
474 return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
476 return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
477 }
478
479 llvm_unreachable("Illegal extend_vector_inreg opcode");
480}
481
482SDValue DAGTypeLegalizer::ScalarizeVecRes_ADDRSPACECAST(SDNode *N) {
483 EVT DestVT = N->getValueType(0).getVectorElementType();
484 SDValue Op = N->getOperand(0);
485 EVT OpVT = Op.getValueType();
486 SDLoc DL(N);
487 // The result needs scalarizing, but it's not a given that the source does.
488 // This is a workaround for targets where it's impossible to scalarize the
489 // result of a conversion, because the source type is legal.
490 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
491 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
492 // legal and was not scalarized.
493 // See the similar logic in ScalarizeVecRes_SETCC
494 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
495 Op = GetScalarizedVector(Op);
496 } else {
497 EVT VT = OpVT.getVectorElementType();
499 DAG.getVectorIdxConstant(0, DL));
500 }
501 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(N);
502 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
503 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
504 return DAG.getAddrSpaceCast(DL, DestVT, Op, SrcAS, DestAS);
505}
506
507SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
508 // If the operand is wider than the vector element type then it is implicitly
509 // truncated. Make that explicit here.
510 EVT EltVT = N->getValueType(0).getVectorElementType();
511 SDValue InOp = N->getOperand(0);
512 if (InOp.getValueType() != EltVT)
513 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
514 return InOp;
515}
516
517SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
518 SDValue Cond = N->getOperand(0);
519 EVT OpVT = Cond.getValueType();
520 SDLoc DL(N);
521 // The vselect result and true/value operands needs scalarizing, but it's
522 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
523 // See the similar logic in ScalarizeVecRes_SETCC
524 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
525 Cond = GetScalarizedVector(Cond);
526 } else {
527 EVT VT = OpVT.getVectorElementType();
529 DAG.getVectorIdxConstant(0, DL));
530 }
531
532 SDValue LHS = GetScalarizedVector(N->getOperand(1));
534 TLI.getBooleanContents(false, false);
535 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
536
537 // If integer and float booleans have different contents then we can't
538 // reliably optimize in all cases. There is a full explanation for this in
539 // DAGCombiner::visitSELECT() where the same issue affects folding
540 // (select C, 0, 1) to (xor C, 1).
541 if (TLI.getBooleanContents(false, false) !=
542 TLI.getBooleanContents(false, true)) {
543 // At least try the common case where the boolean is generated by a
544 // comparison.
545 if (Cond->getOpcode() == ISD::SETCC) {
546 EVT OpVT = Cond->getOperand(0).getValueType();
547 ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
548 VecBool = TLI.getBooleanContents(OpVT);
549 } else
551 }
552
553 EVT CondVT = Cond.getValueType();
554 if (ScalarBool != VecBool) {
555 switch (ScalarBool) {
557 break;
561 // Vector read from all ones, scalar expects a single 1 so mask.
562 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
563 Cond, DAG.getConstant(1, SDLoc(N), CondVT));
564 break;
568 // Vector reads from a one, scalar from all ones so sign extend.
569 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
570 Cond, DAG.getValueType(MVT::i1));
571 break;
572 }
573 }
574
575 // Truncate the condition if needed
576 auto BoolVT = getSetCCResultType(CondVT);
577 if (BoolVT.bitsLT(CondVT))
578 Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
579
580 return DAG.getSelect(SDLoc(N),
581 LHS.getValueType(), Cond, LHS,
582 GetScalarizedVector(N->getOperand(2)));
583}
584
585SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
586 SDValue LHS = GetScalarizedVector(N->getOperand(1));
587 return DAG.getSelect(SDLoc(N),
588 LHS.getValueType(), N->getOperand(0), LHS,
589 GetScalarizedVector(N->getOperand(2)));
590}
591
592SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
593 SDValue LHS = GetScalarizedVector(N->getOperand(2));
594 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
595 N->getOperand(0), N->getOperand(1),
596 LHS, GetScalarizedVector(N->getOperand(3)),
597 N->getOperand(4));
598}
599
600SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
601 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
602}
603
604SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
605 // Figure out if the scalar is the LHS or RHS and return it.
606 SDValue Arg = N->getOperand(2).getOperand(0);
607 if (Arg.isUndef())
608 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
609 unsigned Op = !cast<ConstantSDNode>(Arg)->isZero();
610 return GetScalarizedVector(N->getOperand(Op));
611}
612
613SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
614 SDValue Src = N->getOperand(0);
615 EVT SrcVT = Src.getValueType();
616 SDLoc dl(N);
617
618 // Handle case where result is scalarized but operand is not
619 if (getTypeAction(SrcVT) == TargetLowering::TypeScalarizeVector)
620 Src = GetScalarizedVector(Src);
621 else
622 Src = DAG.getNode(
624 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
625
626 EVT DstVT = N->getValueType(0).getVectorElementType();
627 return DAG.getNode(N->getOpcode(), dl, DstVT, Src, N->getOperand(1));
628}
629
630SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
631 assert(N->getValueType(0).isVector() &&
632 N->getOperand(0).getValueType().isVector() &&
633 "Operand types must be vectors");
634 SDValue LHS = N->getOperand(0);
635 SDValue RHS = N->getOperand(1);
636 EVT OpVT = LHS.getValueType();
637 EVT NVT = N->getValueType(0).getVectorElementType();
638 SDLoc DL(N);
639
640 // The result needs scalarizing, but it's not a given that the source does.
641 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
642 LHS = GetScalarizedVector(LHS);
643 RHS = GetScalarizedVector(RHS);
644 } else {
645 EVT VT = OpVT.getVectorElementType();
646 LHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
647 DAG.getVectorIdxConstant(0, DL));
648 RHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
649 DAG.getVectorIdxConstant(0, DL));
650 }
651
652 // Turn it into a scalar SETCC.
653 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
654 N->getOperand(2));
655 // Vectors may have a different boolean contents to scalars. Promote the
656 // value appropriately.
657 ISD::NodeType ExtendCode =
659 return DAG.getNode(ExtendCode, DL, NVT, Res);
660}
661
662SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
663 SDLoc DL(N);
664 SDValue Arg = N->getOperand(0);
665 SDValue Test = N->getOperand(1);
666 EVT ArgVT = Arg.getValueType();
667 EVT ResultVT = N->getValueType(0).getVectorElementType();
668
669 if (getTypeAction(ArgVT) == TargetLowering::TypeScalarizeVector) {
670 Arg = GetScalarizedVector(Arg);
671 } else {
672 EVT VT = ArgVT.getVectorElementType();
673 Arg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Arg,
674 DAG.getVectorIdxConstant(0, DL));
675 }
676
677 SDValue Res =
678 DAG.getNode(ISD::IS_FPCLASS, DL, MVT::i1, {Arg, Test}, N->getFlags());
679 // Vectors may have a different boolean contents to scalars. Promote the
680 // value appropriately.
681 ISD::NodeType ExtendCode =
683 return DAG.getNode(ExtendCode, DL, ResultVT, Res);
684}
685
686//===----------------------------------------------------------------------===//
687// Operand Vector Scalarization <1 x ty> -> ty.
688//===----------------------------------------------------------------------===//
689
690bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
691 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
692 N->dump(&DAG));
693 SDValue Res = SDValue();
694
695 switch (N->getOpcode()) {
696 default:
697#ifndef NDEBUG
698 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
699 N->dump(&DAG);
700 dbgs() << "\n";
701#endif
702 report_fatal_error("Do not know how to scalarize this operator's "
703 "operand!\n");
704 case ISD::BITCAST:
705 Res = ScalarizeVecOp_BITCAST(N);
706 break;
707 case ISD::ANY_EXTEND:
708 case ISD::ZERO_EXTEND:
709 case ISD::SIGN_EXTEND:
710 case ISD::TRUNCATE:
711 case ISD::FP_TO_SINT:
712 case ISD::FP_TO_UINT:
713 case ISD::SINT_TO_FP:
714 case ISD::UINT_TO_FP:
715 case ISD::LRINT:
716 case ISD::LLRINT:
717 Res = ScalarizeVecOp_UnaryOp(N);
718 break;
723 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
724 break;
726 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
727 break;
729 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
730 break;
731 case ISD::VSELECT:
732 Res = ScalarizeVecOp_VSELECT(N);
733 break;
734 case ISD::SETCC:
735 Res = ScalarizeVecOp_VSETCC(N);
736 break;
737 case ISD::STORE:
738 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
739 break;
741 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
742 break;
743 case ISD::FP_ROUND:
744 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
745 break;
747 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
748 break;
749 case ISD::FP_EXTEND:
750 Res = ScalarizeVecOp_FP_EXTEND(N);
751 break;
767 Res = ScalarizeVecOp_VECREDUCE(N);
768 break;
771 Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
772 break;
773 }
774
775 // If the result is null, the sub-method took care of registering results etc.
776 if (!Res.getNode()) return false;
777
778 // If the result is N, the sub-method updated N in place. Tell the legalizer
779 // core about this.
780 if (Res.getNode() == N)
781 return true;
782
783 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
784 "Invalid operand expansion");
785
786 ReplaceValueWith(SDValue(N, 0), Res);
787 return false;
788}
789
790/// If the value to convert is a vector that needs to be scalarized, it must be
791/// <1 x ty>. Convert the element instead.
792SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
793 SDValue Elt = GetScalarizedVector(N->getOperand(0));
794 return DAG.getNode(ISD::BITCAST, SDLoc(N),
795 N->getValueType(0), Elt);
796}
797
798/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
799/// Do the operation on the element instead.
800SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
801 assert(N->getValueType(0).getVectorNumElements() == 1 &&
802 "Unexpected vector type!");
803 SDValue Elt = GetScalarizedVector(N->getOperand(0));
804 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
805 N->getValueType(0).getScalarType(), Elt);
806 // Revectorize the result so the types line up with what the uses of this
807 // expression expect.
808 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
809}
810
811/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
812/// Do the strict FP operation on the element instead.
813SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
814 assert(N->getValueType(0).getVectorNumElements() == 1 &&
815 "Unexpected vector type!");
816 SDValue Elt = GetScalarizedVector(N->getOperand(1));
817 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
818 { N->getValueType(0).getScalarType(), MVT::Other },
819 { N->getOperand(0), Elt });
820 // Legalize the chain result - switch anything that used the old chain to
821 // use the new one.
822 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
823 // Revectorize the result so the types line up with what the uses of this
824 // expression expect.
825 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
826
827 // Do our own replacement and return SDValue() to tell the caller that we
828 // handled all replacements since caller can only handle a single result.
829 ReplaceValueWith(SDValue(N, 0), Res);
830 return SDValue();
831}
832
833/// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
834SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
835 SmallVector<SDValue, 8> Ops(N->getNumOperands());
836 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
837 Ops[i] = GetScalarizedVector(N->getOperand(i));
838 return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
839}
840
841/// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
842/// so just return the element, ignoring the index.
843SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
844 EVT VT = N->getValueType(0);
845 SDValue Res = GetScalarizedVector(N->getOperand(0));
846 if (Res.getValueType() != VT)
847 Res = VT.isFloatingPoint()
848 ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
849 : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
850 return Res;
851}
852
853/// If the input condition is a vector that needs to be scalarized, it must be
854/// <1 x i1>, so just convert to a normal ISD::SELECT
855/// (still with vector output type since that was acceptable if we got here).
856SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
857 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
858 EVT VT = N->getValueType(0);
859
860 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
861 N->getOperand(2));
862}
863
864/// If the operand is a vector that needs to be scalarized then the
865/// result must be v1i1, so just convert to a scalar SETCC and wrap
866/// with a scalar_to_vector since the res type is legal if we got here
867SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
868 assert(N->getValueType(0).isVector() &&
869 N->getOperand(0).getValueType().isVector() &&
870 "Operand types must be vectors");
871 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
872
873 EVT VT = N->getValueType(0);
874 SDValue LHS = GetScalarizedVector(N->getOperand(0));
875 SDValue RHS = GetScalarizedVector(N->getOperand(1));
876
877 EVT OpVT = N->getOperand(0).getValueType();
878 EVT NVT = VT.getVectorElementType();
879 SDLoc DL(N);
880 // Turn it into a scalar SETCC.
881 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
882 N->getOperand(2));
883
884 // Vectors may have a different boolean contents to scalars. Promote the
885 // value appropriately.
886 ISD::NodeType ExtendCode =
888
889 Res = DAG.getNode(ExtendCode, DL, NVT, Res);
890
891 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
892}
893
894/// If the value to store is a vector that needs to be scalarized, it must be
895/// <1 x ty>. Just store the element.
896SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
897 assert(N->isUnindexed() && "Indexed store of one-element vector?");
898 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
899 SDLoc dl(N);
900
901 if (N->isTruncatingStore())
902 return DAG.getTruncStore(
903 N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
904 N->getBasePtr(), N->getPointerInfo(),
905 N->getMemoryVT().getVectorElementType(), N->getOriginalAlign(),
906 N->getMemOperand()->getFlags(), N->getAAInfo());
907
908 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
909 N->getBasePtr(), N->getPointerInfo(),
910 N->getOriginalAlign(), N->getMemOperand()->getFlags(),
911 N->getAAInfo());
912}
913
914/// If the value to round is a vector that needs to be scalarized, it must be
915/// <1 x ty>. Convert the element instead.
916SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
917 assert(OpNo == 0 && "Wrong operand for scalarization!");
918 SDValue Elt = GetScalarizedVector(N->getOperand(0));
919 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
920 N->getValueType(0).getVectorElementType(), Elt,
921 N->getOperand(1));
922 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
923}
924
925SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
926 unsigned OpNo) {
927 assert(OpNo == 1 && "Wrong operand for scalarization!");
928 SDValue Elt = GetScalarizedVector(N->getOperand(1));
930 { N->getValueType(0).getVectorElementType(),
931 MVT::Other },
932 { N->getOperand(0), Elt, N->getOperand(2) });
933 // Legalize the chain result - switch anything that used the old chain to
934 // use the new one.
935 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
936
937 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
938
939 // Do our own replacement and return SDValue() to tell the caller that we
940 // handled all replacements since caller can only handle a single result.
941 ReplaceValueWith(SDValue(N, 0), Res);
942 return SDValue();
943}
944
945/// If the value to extend is a vector that needs to be scalarized, it must be
946/// <1 x ty>. Convert the element instead.
947SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
948 SDValue Elt = GetScalarizedVector(N->getOperand(0));
950 N->getValueType(0).getVectorElementType(), Elt);
951 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
952}
953
954/// If the value to extend is a vector that needs to be scalarized, it must be
955/// <1 x ty>. Convert the element instead.
956SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
957 SDValue Elt = GetScalarizedVector(N->getOperand(1));
958 SDValue Res =
960 {N->getValueType(0).getVectorElementType(), MVT::Other},
961 {N->getOperand(0), Elt});
962 // Legalize the chain result - switch anything that used the old chain to
963 // use the new one.
964 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
965
966 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
967
968 // Do our own replacement and return SDValue() to tell the caller that we
969 // handled all replacements since caller can only handle a single result.
970 ReplaceValueWith(SDValue(N, 0), Res);
971 return SDValue();
972}
973
974SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
975 SDValue Res = GetScalarizedVector(N->getOperand(0));
976 // Result type may be wider than element type.
977 if (Res.getValueType() != N->getValueType(0))
978 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res);
979 return Res;
980}
981
982SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
983 SDValue AccOp = N->getOperand(0);
984 SDValue VecOp = N->getOperand(1);
985
986 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
987
988 SDValue Op = GetScalarizedVector(VecOp);
989 return DAG.getNode(BaseOpc, SDLoc(N), N->getValueType(0),
990 AccOp, Op, N->getFlags());
991}
992
993//===----------------------------------------------------------------------===//
994// Result Vector Splitting
995//===----------------------------------------------------------------------===//
996
997/// This method is called when the specified result of the specified node is
998/// found to need vector splitting. At this point, the node may also have
999/// invalid operands or may have other results that need legalization, we just
1000/// know that (at least) one result needs vector splitting.
1001void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
1002 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG));
1003 SDValue Lo, Hi;
1004
1005 // See if the target wants to custom expand this node.
1006 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1007 return;
1008
1009 switch (N->getOpcode()) {
1010 default:
1011#ifndef NDEBUG
1012 dbgs() << "SplitVectorResult #" << ResNo << ": ";
1013 N->dump(&DAG);
1014 dbgs() << "\n";
1015#endif
1016 report_fatal_error("Do not know how to split the result of this "
1017 "operator!\n");
1018
1019 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1020 case ISD::AssertZext: SplitVecRes_AssertZext(N, Lo, Hi); break;
1021 case ISD::VSELECT:
1022 case ISD::SELECT:
1023 case ISD::VP_MERGE:
1024 case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break;
1025 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1026 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1027 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
1028 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
1029 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
1030 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
1031 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
1032 case ISD::FPOWI:
1033 case ISD::FLDEXP:
1034 case ISD::FCOPYSIGN: SplitVecRes_FPOp_MultiType(N, Lo, Hi); break;
1035 case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
1036 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
1037 case ISD::SPLAT_VECTOR:
1039 SplitVecRes_ScalarOp(N, Lo, Hi);
1040 break;
1041 case ISD::STEP_VECTOR:
1042 SplitVecRes_STEP_VECTOR(N, Lo, Hi);
1043 break;
1044 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
1045 case ISD::LOAD:
1046 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
1047 break;
1048 case ISD::VP_LOAD:
1049 SplitVecRes_VP_LOAD(cast<VPLoadSDNode>(N), Lo, Hi);
1050 break;
1051 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1052 SplitVecRes_VP_STRIDED_LOAD(cast<VPStridedLoadSDNode>(N), Lo, Hi);
1053 break;
1054 case ISD::MLOAD:
1055 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
1056 break;
1057 case ISD::MGATHER:
1058 case ISD::VP_GATHER:
1059 SplitVecRes_Gather(cast<MemSDNode>(N), Lo, Hi, /*SplitSETCC*/ true);
1060 break;
1061 case ISD::SETCC:
1062 case ISD::VP_SETCC:
1063 SplitVecRes_SETCC(N, Lo, Hi);
1064 break;
1066 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
1067 break;
1069 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
1070 break;
1071 case ISD::VECTOR_SPLICE:
1072 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
1073 break;
1075 SplitVecRes_VECTOR_DEINTERLEAVE(N);
1076 return;
1078 SplitVecRes_VECTOR_INTERLEAVE(N);
1079 return;
1080 case ISD::VAARG:
1081 SplitVecRes_VAARG(N, Lo, Hi);
1082 break;
1083
1087 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1088 break;
1089
1090 case ISD::ABS:
1091 case ISD::VP_ABS:
1092 case ISD::BITREVERSE:
1093 case ISD::VP_BITREVERSE:
1094 case ISD::BSWAP:
1095 case ISD::VP_BSWAP:
1096 case ISD::CTLZ:
1097 case ISD::VP_CTLZ:
1098 case ISD::CTTZ:
1099 case ISD::VP_CTTZ:
1101 case ISD::VP_CTLZ_ZERO_UNDEF:
1103 case ISD::VP_CTTZ_ZERO_UNDEF:
1104 case ISD::CTPOP:
1105 case ISD::VP_CTPOP:
1106 case ISD::FABS: case ISD::VP_FABS:
1107 case ISD::FCEIL:
1108 case ISD::VP_FCEIL:
1109 case ISD::FCOS:
1110 case ISD::FEXP:
1111 case ISD::FEXP2:
1112 case ISD::FEXP10:
1113 case ISD::FFLOOR:
1114 case ISD::VP_FFLOOR:
1115 case ISD::FLOG:
1116 case ISD::FLOG10:
1117 case ISD::FLOG2:
1118 case ISD::FNEARBYINT:
1119 case ISD::VP_FNEARBYINT:
1120 case ISD::FNEG: case ISD::VP_FNEG:
1121 case ISD::FREEZE:
1122 case ISD::ARITH_FENCE:
1123 case ISD::FP_EXTEND:
1124 case ISD::VP_FP_EXTEND:
1125 case ISD::FP_ROUND:
1126 case ISD::VP_FP_ROUND:
1127 case ISD::FP_TO_SINT:
1128 case ISD::VP_FP_TO_SINT:
1129 case ISD::FP_TO_UINT:
1130 case ISD::VP_FP_TO_UINT:
1131 case ISD::FRINT:
1132 case ISD::VP_FRINT:
1133 case ISD::LRINT:
1134 case ISD::VP_LRINT:
1135 case ISD::LLRINT:
1136 case ISD::VP_LLRINT:
1137 case ISD::FROUND:
1138 case ISD::VP_FROUND:
1139 case ISD::FROUNDEVEN:
1140 case ISD::VP_FROUNDEVEN:
1141 case ISD::FSIN:
1142 case ISD::FSQRT: case ISD::VP_SQRT:
1143 case ISD::FTRUNC:
1144 case ISD::VP_FROUNDTOZERO:
1145 case ISD::SINT_TO_FP:
1146 case ISD::VP_SINT_TO_FP:
1147 case ISD::TRUNCATE:
1148 case ISD::VP_TRUNCATE:
1149 case ISD::UINT_TO_FP:
1150 case ISD::VP_UINT_TO_FP:
1151 case ISD::FCANONICALIZE:
1152 SplitVecRes_UnaryOp(N, Lo, Hi);
1153 break;
1154 case ISD::ADDRSPACECAST:
1155 SplitVecRes_ADDRSPACECAST(N, Lo, Hi);
1156 break;
1157 case ISD::FFREXP:
1158 SplitVecRes_FFREXP(N, ResNo, Lo, Hi);
1159 break;
1160
1161 case ISD::ANY_EXTEND:
1162 case ISD::SIGN_EXTEND:
1163 case ISD::ZERO_EXTEND:
1164 case ISD::VP_SIGN_EXTEND:
1165 case ISD::VP_ZERO_EXTEND:
1166 SplitVecRes_ExtendOp(N, Lo, Hi);
1167 break;
1168
1169 case ISD::ADD: case ISD::VP_ADD:
1170 case ISD::SUB: case ISD::VP_SUB:
1171 case ISD::MUL: case ISD::VP_MUL:
1172 case ISD::MULHS:
1173 case ISD::MULHU:
1174 case ISD::FADD: case ISD::VP_FADD:
1175 case ISD::FSUB: case ISD::VP_FSUB:
1176 case ISD::FMUL: case ISD::VP_FMUL:
1177 case ISD::FMINNUM: case ISD::VP_FMINNUM:
1178 case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
1179 case ISD::FMINIMUM:
1180 case ISD::VP_FMINIMUM:
1181 case ISD::FMAXIMUM:
1182 case ISD::VP_FMAXIMUM:
1183 case ISD::SDIV: case ISD::VP_SDIV:
1184 case ISD::UDIV: case ISD::VP_UDIV:
1185 case ISD::FDIV: case ISD::VP_FDIV:
1186 case ISD::FPOW:
1187 case ISD::AND: case ISD::VP_AND:
1188 case ISD::OR: case ISD::VP_OR:
1189 case ISD::XOR: case ISD::VP_XOR:
1190 case ISD::SHL: case ISD::VP_SHL:
1191 case ISD::SRA: case ISD::VP_ASHR:
1192 case ISD::SRL: case ISD::VP_LSHR:
1193 case ISD::UREM: case ISD::VP_UREM:
1194 case ISD::SREM: case ISD::VP_SREM:
1195 case ISD::FREM: case ISD::VP_FREM:
1196 case ISD::SMIN: case ISD::VP_SMIN:
1197 case ISD::SMAX: case ISD::VP_SMAX:
1198 case ISD::UMIN: case ISD::VP_UMIN:
1199 case ISD::UMAX: case ISD::VP_UMAX:
1200 case ISD::SADDSAT: case ISD::VP_SADDSAT:
1201 case ISD::UADDSAT: case ISD::VP_UADDSAT:
1202 case ISD::SSUBSAT: case ISD::VP_SSUBSAT:
1203 case ISD::USUBSAT: case ISD::VP_USUBSAT:
1204 case ISD::SSHLSAT:
1205 case ISD::USHLSAT:
1206 case ISD::ROTL:
1207 case ISD::ROTR:
1208 case ISD::VP_FCOPYSIGN:
1209 SplitVecRes_BinOp(N, Lo, Hi);
1210 break;
1211 case ISD::FMA: case ISD::VP_FMA:
1212 case ISD::FSHL:
1213 case ISD::VP_FSHL:
1214 case ISD::FSHR:
1215 case ISD::VP_FSHR:
1216 SplitVecRes_TernaryOp(N, Lo, Hi);
1217 break;
1218
1219#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1220 case ISD::STRICT_##DAGN:
1221#include "llvm/IR/ConstrainedOps.def"
1222 SplitVecRes_StrictFPOp(N, Lo, Hi);
1223 break;
1224
1227 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1228 break;
1229
1230 case ISD::UADDO:
1231 case ISD::SADDO:
1232 case ISD::USUBO:
1233 case ISD::SSUBO:
1234 case ISD::UMULO:
1235 case ISD::SMULO:
1236 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1237 break;
1238 case ISD::SMULFIX:
1239 case ISD::SMULFIXSAT:
1240 case ISD::UMULFIX:
1241 case ISD::UMULFIXSAT:
1242 case ISD::SDIVFIX:
1243 case ISD::SDIVFIXSAT:
1244 case ISD::UDIVFIX:
1245 case ISD::UDIVFIXSAT:
1246 SplitVecRes_FIX(N, Lo, Hi);
1247 break;
1248 case ISD::EXPERIMENTAL_VP_REVERSE:
1249 SplitVecRes_VP_REVERSE(N, Lo, Hi);
1250 break;
1251 }
1252
1253 // If Lo/Hi is null, the sub-method took care of registering results etc.
1254 if (Lo.getNode())
1255 SetSplitVector(SDValue(N, ResNo), Lo, Hi);
1256}
1257
1258void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1260 uint64_t *ScaledOffset) {
1261 SDLoc DL(N);
1262 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8;
1263
1264 if (MemVT.isScalableVector()) {
1266 SDValue BytesIncrement = DAG.getVScale(
1267 DL, Ptr.getValueType(),
1268 APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
1269 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1270 Flags.setNoUnsignedWrap(true);
1271 if (ScaledOffset)
1272 *ScaledOffset += IncrementSize;
1273 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, BytesIncrement,
1274 Flags);
1275 } else {
1276 MPI = N->getPointerInfo().getWithOffset(IncrementSize);
1277 // Increment the pointer to the other half.
1278 Ptr = DAG.getObjectPtrOffset(DL, Ptr, TypeSize::getFixed(IncrementSize));
1279 }
1280}
1281
1282std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1283 return SplitMask(Mask, SDLoc(Mask));
1284}
1285
1286std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1287 const SDLoc &DL) {
1288 SDValue MaskLo, MaskHi;
1289 EVT MaskVT = Mask.getValueType();
1290 if (getTypeAction(MaskVT) == TargetLowering::TypeSplitVector)
1291 GetSplitVector(Mask, MaskLo, MaskHi);
1292 else
1293 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
1294 return std::make_pair(MaskLo, MaskHi);
1295}
1296
1297void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1298 SDValue LHSLo, LHSHi;
1299 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1300 SDValue RHSLo, RHSHi;
1301 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1302 SDLoc dl(N);
1303
1304 const SDNodeFlags Flags = N->getFlags();
1305 unsigned Opcode = N->getOpcode();
1306 if (N->getNumOperands() == 2) {
1307 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
1308 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
1309 return;
1310 }
1311
1312 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1313 assert(N->isVPOpcode() && "Expected VP opcode");
1314
1315 SDValue MaskLo, MaskHi;
1316 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2));
1317
1318 SDValue EVLLo, EVLHi;
1319 std::tie(EVLLo, EVLHi) =
1320 DAG.SplitEVL(N->getOperand(3), N->getValueType(0), dl);
1321
1322 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(),
1323 {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1324 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(),
1325 {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1326}
1327
1328void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1329 SDValue &Hi) {
1330 SDValue Op0Lo, Op0Hi;
1331 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
1332 SDValue Op1Lo, Op1Hi;
1333 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
1334 SDValue Op2Lo, Op2Hi;
1335 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
1336 SDLoc dl(N);
1337
1338 const SDNodeFlags Flags = N->getFlags();
1339 unsigned Opcode = N->getOpcode();
1340 if (N->getNumOperands() == 3) {
1341 Lo = DAG.getNode(Opcode, dl, Op0Lo.getValueType(), Op0Lo, Op1Lo, Op2Lo, Flags);
1342 Hi = DAG.getNode(Opcode, dl, Op0Hi.getValueType(), Op0Hi, Op1Hi, Op2Hi, Flags);
1343 return;
1344 }
1345
1346 assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
1347 assert(N->isVPOpcode() && "Expected VP opcode");
1348
1349 SDValue MaskLo, MaskHi;
1350 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
1351
1352 SDValue EVLLo, EVLHi;
1353 std::tie(EVLLo, EVLHi) =
1354 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), dl);
1355
1356 Lo = DAG.getNode(Opcode, dl, Op0Lo.getValueType(),
1357 {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags);
1358 Hi = DAG.getNode(Opcode, dl, Op0Hi.getValueType(),
1359 {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags);
1360}
1361
1362void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1363 SDValue LHSLo, LHSHi;
1364 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1365 SDValue RHSLo, RHSHi;
1366 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1367 SDLoc dl(N);
1368 SDValue Op2 = N->getOperand(2);
1369
1370 unsigned Opcode = N->getOpcode();
1371 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2,
1372 N->getFlags());
1373 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2,
1374 N->getFlags());
1375}
1376
1377void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1378 SDValue &Hi) {
1379 // We know the result is a vector. The input may be either a vector or a
1380 // scalar value.
1381 EVT LoVT, HiVT;
1382 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1383 SDLoc dl(N);
1384
1385 SDValue InOp = N->getOperand(0);
1386 EVT InVT = InOp.getValueType();
1387
1388 // Handle some special cases efficiently.
1389 switch (getTypeAction(InVT)) {
1397 break;
1400 // A scalar to vector conversion, where the scalar needs expansion.
1401 // If the vector is being split in two then we can just convert the
1402 // expanded pieces.
1403 if (LoVT == HiVT) {
1404 GetExpandedOp(InOp, Lo, Hi);
1405 if (DAG.getDataLayout().isBigEndian())
1406 std::swap(Lo, Hi);
1407 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1408 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1409 return;
1410 }
1411 break;
1413 // If the input is a vector that needs to be split, convert each split
1414 // piece of the input now.
1415 GetSplitVector(InOp, Lo, Hi);
1416 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1417 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1418 return;
1420 report_fatal_error("Scalarization of scalable vectors is not supported.");
1421 }
1422
1423 if (LoVT.isScalableVector()) {
1424 auto [InLo, InHi] = DAG.SplitVectorOperand(N, 0);
1425 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, InLo);
1426 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, InHi);
1427 return;
1428 }
1429
1430 // In the general case, convert the input to an integer and split it by hand.
1431 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
1432 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
1433 if (DAG.getDataLayout().isBigEndian())
1434 std::swap(LoIntVT, HiIntVT);
1435
1436 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
1437
1438 if (DAG.getDataLayout().isBigEndian())
1439 std::swap(Lo, Hi);
1440 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1441 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1442}
1443
1444void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1445 SDValue &Hi) {
1446 EVT LoVT, HiVT;
1447 SDLoc dl(N);
1448 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1449 unsigned LoNumElts = LoVT.getVectorNumElements();
1450 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1451 Lo = DAG.getBuildVector(LoVT, dl, LoOps);
1452
1453 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1454 Hi = DAG.getBuildVector(HiVT, dl, HiOps);
1455}
1456
1457void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1458 SDValue &Hi) {
1459 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1460 SDLoc dl(N);
1461 unsigned NumSubvectors = N->getNumOperands() / 2;
1462 if (NumSubvectors == 1) {
1463 Lo = N->getOperand(0);
1464 Hi = N->getOperand(1);
1465 return;
1466 }
1467
1468 EVT LoVT, HiVT;
1469 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1470
1471 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1472 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
1473
1474 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1475 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
1476}
1477
1478void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1479 SDValue &Hi) {
1480 SDValue Vec = N->getOperand(0);
1481 SDValue Idx = N->getOperand(1);
1482 SDLoc dl(N);
1483
1484 EVT LoVT, HiVT;
1485 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1486
1487 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
1488 uint64_t IdxVal = Idx->getAsZExtVal();
1489 Hi = DAG.getNode(
1490 ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
1491 DAG.getVectorIdxConstant(IdxVal + LoVT.getVectorMinNumElements(), dl));
1492}
1493
1494void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1495 SDValue &Hi) {
1496 SDValue Vec = N->getOperand(0);
1497 SDValue SubVec = N->getOperand(1);
1498 SDValue Idx = N->getOperand(2);
1499 SDLoc dl(N);
1500 GetSplitVector(Vec, Lo, Hi);
1501
1502 EVT VecVT = Vec.getValueType();
1503 EVT LoVT = Lo.getValueType();
1504 EVT SubVecVT = SubVec.getValueType();
1505 unsigned VecElems = VecVT.getVectorMinNumElements();
1506 unsigned SubElems = SubVecVT.getVectorMinNumElements();
1507 unsigned LoElems = LoVT.getVectorMinNumElements();
1508
1509 // If we know the index is in the first half, and we know the subvector
1510 // doesn't cross the boundary between the halves, we can avoid spilling the
1511 // vector, and insert into the lower half of the split vector directly.
1512 unsigned IdxVal = Idx->getAsZExtVal();
1513 if (IdxVal + SubElems <= LoElems) {
1514 Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1515 return;
1516 }
1517 // Similarly if the subvector is fully in the high half, but mind that we
1518 // can't tell whether a fixed-length subvector is fully within the high half
1519 // of a scalable vector.
1520 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1521 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1522 Hi = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, Hi.getValueType(), Hi, SubVec,
1523 DAG.getVectorIdxConstant(IdxVal - LoElems, dl));
1524 return;
1525 }
1526
1527 // Spill the vector to the stack.
1528 // In cases where the vector is illegal it will be broken down into parts
1529 // and stored in parts - we should use the alignment for the smallest part.
1530 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1532 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1533 auto &MF = DAG.getMachineFunction();
1534 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1535 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1536
1537 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1538 SmallestAlign);
1539
1540 // Store the new subvector into the specified index.
1541 SDValue SubVecPtr =
1542 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1543 Store = DAG.getStore(Store, dl, SubVec, SubVecPtr,
1545
1546 // Load the Lo part from the stack slot.
1547 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, PtrInfo,
1548 SmallestAlign);
1549
1550 // Increment the pointer to the other part.
1551 auto *Load = cast<LoadSDNode>(Lo);
1552 MachinePointerInfo MPI = Load->getPointerInfo();
1553 IncrementPointer(Load, LoVT, MPI, StackPtr);
1554
1555 // Load the Hi part from the stack slot.
1556 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MPI, SmallestAlign);
1557}
1558
1559// Handle splitting an FP where the second operand does not match the first
1560// type. The second operand may be a scalar, or a vector that has exactly as
1561// many elements as the first
1562void DAGTypeLegalizer::SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo,
1563 SDValue &Hi) {
1564 SDValue LHSLo, LHSHi;
1565 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1566 SDLoc DL(N);
1567
1568 SDValue RHSLo, RHSHi;
1569 SDValue RHS = N->getOperand(1);
1570 EVT RHSVT = RHS.getValueType();
1571 if (RHSVT.isVector()) {
1572 if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1573 GetSplitVector(RHS, RHSLo, RHSHi);
1574 else
1575 std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1576
1577 Lo = DAG.getNode(N->getOpcode(), DL, LHSLo.getValueType(), LHSLo, RHSLo);
1578 Hi = DAG.getNode(N->getOpcode(), DL, LHSHi.getValueType(), LHSHi, RHSHi);
1579 } else {
1580 Lo = DAG.getNode(N->getOpcode(), DL, LHSLo.getValueType(), LHSLo, RHS);
1581 Hi = DAG.getNode(N->getOpcode(), DL, LHSHi.getValueType(), LHSHi, RHS);
1582 }
1583}
1584
1585void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
1586 SDValue &Hi) {
1587 SDLoc DL(N);
1588 SDValue ArgLo, ArgHi;
1589 SDValue Test = N->getOperand(1);
1590 SDValue FpValue = N->getOperand(0);
1591 if (getTypeAction(FpValue.getValueType()) == TargetLowering::TypeSplitVector)
1592 GetSplitVector(FpValue, ArgLo, ArgHi);
1593 else
1594 std::tie(ArgLo, ArgHi) = DAG.SplitVector(FpValue, SDLoc(FpValue));
1595 EVT LoVT, HiVT;
1596 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1597
1598 Lo = DAG.getNode(ISD::IS_FPCLASS, DL, LoVT, ArgLo, Test, N->getFlags());
1599 Hi = DAG.getNode(ISD::IS_FPCLASS, DL, HiVT, ArgHi, Test, N->getFlags());
1600}
1601
1602void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1603 SDValue &Hi) {
1604 SDValue LHSLo, LHSHi;
1605 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1606 SDLoc dl(N);
1607
1608 EVT LoVT, HiVT;
1609 std::tie(LoVT, HiVT) =
1610 DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1611
1612 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1613 DAG.getValueType(LoVT));
1614 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1615 DAG.getValueType(HiVT));
1616}
1617
1618void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1619 SDValue &Hi) {
1620 unsigned Opcode = N->getOpcode();
1621 SDValue N0 = N->getOperand(0);
1622
1623 SDLoc dl(N);
1624 SDValue InLo, InHi;
1625
1626 if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1627 GetSplitVector(N0, InLo, InHi);
1628 else
1629 std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1630
1631 EVT InLoVT = InLo.getValueType();
1632 unsigned InNumElements = InLoVT.getVectorNumElements();
1633
1634 EVT OutLoVT, OutHiVT;
1635 std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1636 unsigned OutNumElements = OutLoVT.getVectorNumElements();
1637 assert((2 * OutNumElements) <= InNumElements &&
1638 "Illegal extend vector in reg split");
1639
1640 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1641 // input vector (i.e. we only use InLo):
1642 // OutLo will extend the first OutNumElements from InLo.
1643 // OutHi will extend the next OutNumElements from InLo.
1644
1645 // Shuffle the elements from InLo for OutHi into the bottom elements to
1646 // create a 'fake' InHi.
1647 SmallVector<int, 8> SplitHi(InNumElements, -1);
1648 for (unsigned i = 0; i != OutNumElements; ++i)
1649 SplitHi[i] = i + OutNumElements;
1650 InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1651
1652 Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1653 Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1654}
1655
1656void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1657 SDValue &Hi) {
1658 unsigned NumOps = N->getNumOperands();
1659 SDValue Chain = N->getOperand(0);
1660 EVT LoVT, HiVT;
1661 SDLoc dl(N);
1662 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1663
1664 SmallVector<SDValue, 4> OpsLo(NumOps);
1665 SmallVector<SDValue, 4> OpsHi(NumOps);
1666
1667 // The Chain is the first operand.
1668 OpsLo[0] = Chain;
1669 OpsHi[0] = Chain;
1670
1671 // Now process the remaining operands.
1672 for (unsigned i = 1; i < NumOps; ++i) {
1673 SDValue Op = N->getOperand(i);
1674 SDValue OpLo = Op;
1675 SDValue OpHi = Op;
1676
1677 EVT InVT = Op.getValueType();
1678 if (InVT.isVector()) {
1679 // If the input also splits, handle it directly for a
1680 // compile time speedup. Otherwise split it by hand.
1681 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1682 GetSplitVector(Op, OpLo, OpHi);
1683 else
1684 std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1685 }
1686
1687 OpsLo[i] = OpLo;
1688 OpsHi[i] = OpHi;
1689 }
1690
1691 EVT LoValueVTs[] = {LoVT, MVT::Other};
1692 EVT HiValueVTs[] = {HiVT, MVT::Other};
1693 Lo = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo,
1694 N->getFlags());
1695 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi,
1696 N->getFlags());
1697
1698 // Build a factor node to remember that this Op is independent of the
1699 // other one.
1700 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1701 Lo.getValue(1), Hi.getValue(1));
1702
1703 // Legalize the chain result - switch anything that used the old chain to
1704 // use the new one.
1705 ReplaceValueWith(SDValue(N, 1), Chain);
1706}
1707
1708SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1709 SDValue Chain = N->getOperand(0);
1710 EVT VT = N->getValueType(0);
1711 unsigned NE = VT.getVectorNumElements();
1712 EVT EltVT = VT.getVectorElementType();
1713 SDLoc dl(N);
1714
1716 SmallVector<SDValue, 4> Operands(N->getNumOperands());
1717
1718 // If ResNE is 0, fully unroll the vector op.
1719 if (ResNE == 0)
1720 ResNE = NE;
1721 else if (NE > ResNE)
1722 NE = ResNE;
1723
1724 //The results of each unrolled operation, including the chain.
1725 EVT ChainVTs[] = {EltVT, MVT::Other};
1727
1728 unsigned i;
1729 for (i = 0; i != NE; ++i) {
1730 Operands[0] = Chain;
1731 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
1732 SDValue Operand = N->getOperand(j);
1733 EVT OperandVT = Operand.getValueType();
1734 if (OperandVT.isVector()) {
1735 EVT OperandEltVT = OperandVT.getVectorElementType();
1736 Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
1737 Operand, DAG.getVectorIdxConstant(i, dl));
1738 } else {
1739 Operands[j] = Operand;
1740 }
1741 }
1742 SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1743 Scalar.getNode()->setFlags(N->getFlags());
1744
1745 //Add in the scalar as well as its chain value to the
1746 //result vectors.
1747 Scalars.push_back(Scalar);
1748 Chains.push_back(Scalar.getValue(1));
1749 }
1750
1751 for (; i < ResNE; ++i)
1752 Scalars.push_back(DAG.getUNDEF(EltVT));
1753
1754 // Build a new factor node to connect the chain back together.
1755 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1756 ReplaceValueWith(SDValue(N, 1), Chain);
1757
1758 // Create a new BUILD_VECTOR node
1759 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE);
1760 return DAG.getBuildVector(VecVT, dl, Scalars);
1761}
1762
1763void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1764 SDValue &Lo, SDValue &Hi) {
1765 SDLoc dl(N);
1766 EVT ResVT = N->getValueType(0);
1767 EVT OvVT = N->getValueType(1);
1768 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1769 std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
1770 std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
1771
1772 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1773 if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
1774 GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
1775 GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
1776 } else {
1777 std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
1778 std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
1779 }
1780
1781 unsigned Opcode = N->getOpcode();
1782 SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
1783 SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
1784 SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
1785 SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
1786 LoNode->setFlags(N->getFlags());
1787 HiNode->setFlags(N->getFlags());
1788
1789 Lo = SDValue(LoNode, ResNo);
1790 Hi = SDValue(HiNode, ResNo);
1791
1792 // Replace the other vector result not being explicitly split here.
1793 unsigned OtherNo = 1 - ResNo;
1794 EVT OtherVT = N->getValueType(OtherNo);
1795 if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
1796 SetSplitVector(SDValue(N, OtherNo),
1797 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1798 } else {
1799 SDValue OtherVal = DAG.getNode(
1800 ISD::CONCAT_VECTORS, dl, OtherVT,
1801 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1802 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
1803 }
1804}
1805
1806void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1807 SDValue &Hi) {
1808 SDValue Vec = N->getOperand(0);
1809 SDValue Elt = N->getOperand(1);
1810 SDValue Idx = N->getOperand(2);
1811 SDLoc dl(N);
1812 GetSplitVector(Vec, Lo, Hi);
1813
1814 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1815 unsigned IdxVal = CIdx->getZExtValue();
1816 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
1817 if (IdxVal < LoNumElts) {
1819 Lo.getValueType(), Lo, Elt, Idx);
1820 return;
1821 } else if (!Vec.getValueType().isScalableVector()) {
1822 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1823 DAG.getVectorIdxConstant(IdxVal - LoNumElts, dl));
1824 return;
1825 }
1826 }
1827
1828 // See if the target wants to custom expand this node.
1829 if (CustomLowerNode(N, N->getValueType(0), true))
1830 return;
1831
1832 // Make the vector elements byte-addressable if they aren't already.
1833 EVT VecVT = Vec.getValueType();
1834 EVT EltVT = VecVT.getVectorElementType();
1835 if (VecVT.getScalarSizeInBits() < 8) {
1836 EltVT = MVT::i8;
1837 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1838 VecVT.getVectorElementCount());
1839 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1840 // Extend the element type to match if needed.
1841 if (EltVT.bitsGT(Elt.getValueType()))
1842 Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1843 }
1844
1845 // Spill the vector to the stack.
1846 // In cases where the vector is illegal it will be broken down into parts
1847 // and stored in parts - we should use the alignment for the smallest part.
1848 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1850 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1851 auto &MF = DAG.getMachineFunction();
1852 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1853 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1854
1855 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1856 SmallestAlign);
1857
1858 // Store the new element. This may be larger than the vector element type,
1859 // so use a truncating store.
1860 SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1861 Store = DAG.getTruncStore(
1862 Store, dl, Elt, EltPtr, MachinePointerInfo::getUnknownStack(MF), EltVT,
1863 commonAlignment(SmallestAlign,
1864 EltVT.getFixedSizeInBits() / 8));
1865
1866 EVT LoVT, HiVT;
1867 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1868
1869 // Load the Lo part from the stack slot.
1870 Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo, SmallestAlign);
1871
1872 // Increment the pointer to the other part.
1873 auto Load = cast<LoadSDNode>(Lo);
1874 MachinePointerInfo MPI = Load->getPointerInfo();
1875 IncrementPointer(Load, LoVT, MPI, StackPtr);
1876
1877 Hi = DAG.getLoad(HiVT, dl, Store, StackPtr, MPI, SmallestAlign);
1878
1879 // If we adjusted the original type, we need to truncate the results.
1880 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1881 if (LoVT != Lo.getValueType())
1882 Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1883 if (HiVT != Hi.getValueType())
1884 Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1885}
1886
1887void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
1888 SDValue &Hi) {
1889 EVT LoVT, HiVT;
1890 SDLoc dl(N);
1891 assert(N->getValueType(0).isScalableVector() &&
1892 "Only scalable vectors are supported for STEP_VECTOR");
1893 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1894 SDValue Step = N->getOperand(0);
1895
1896 Lo = DAG.getNode(ISD::STEP_VECTOR, dl, LoVT, Step);
1897
1898 // Hi = Lo + (EltCnt * Step)
1899 EVT EltVT = Step.getValueType();
1900 APInt StepVal = Step->getAsAPIntVal();
1901 SDValue StartOfHi =
1902 DAG.getVScale(dl, EltVT, StepVal * LoVT.getVectorMinNumElements());
1903 StartOfHi = DAG.getSExtOrTrunc(StartOfHi, dl, HiVT.getVectorElementType());
1904 StartOfHi = DAG.getNode(ISD::SPLAT_VECTOR, dl, HiVT, StartOfHi);
1905
1906 Hi = DAG.getNode(ISD::STEP_VECTOR, dl, HiVT, Step);
1907 Hi = DAG.getNode(ISD::ADD, dl, HiVT, Hi, StartOfHi);
1908}
1909
1910void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
1911 SDValue &Hi) {
1912 EVT LoVT, HiVT;
1913 SDLoc dl(N);
1914 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1915 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, N->getOperand(0));
1916 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
1917 Hi = DAG.getUNDEF(HiVT);
1918 } else {
1919 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
1920 Hi = Lo;
1921 }
1922}
1923
1924void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1925 SDValue &Hi) {
1926 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1927 EVT LoVT, HiVT;
1928 SDLoc dl(LD);
1929 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1930
1931 ISD::LoadExtType ExtType = LD->getExtensionType();
1932 SDValue Ch = LD->getChain();
1933 SDValue Ptr = LD->getBasePtr();
1934 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1935 EVT MemoryVT = LD->getMemoryVT();
1936 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1937 AAMDNodes AAInfo = LD->getAAInfo();
1938
1939 EVT LoMemVT, HiMemVT;
1940 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1941
1942 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
1943 SDValue Value, NewChain;
1944 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
1945 std::tie(Lo, Hi) = DAG.SplitVector(Value, dl);
1946 ReplaceValueWith(SDValue(LD, 1), NewChain);
1947 return;
1948 }
1949
1950 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1951 LD->getPointerInfo(), LoMemVT, LD->getOriginalAlign(),
1952 MMOFlags, AAInfo);
1953
1955 IncrementPointer(LD, LoMemVT, MPI, Ptr);
1956
1957 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, MPI,
1958 HiMemVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
1959
1960 // Build a factor node to remember that this load is independent of the
1961 // other one.
1962 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1963 Hi.getValue(1));
1964
1965 // Legalize the chain result - switch anything that used the old chain to
1966 // use the new one.
1967 ReplaceValueWith(SDValue(LD, 1), Ch);
1968}
1969
1970void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
1971 SDValue &Hi) {
1972 assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
1973 EVT LoVT, HiVT;
1974 SDLoc dl(LD);
1975 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1976
1977 ISD::LoadExtType ExtType = LD->getExtensionType();
1978 SDValue Ch = LD->getChain();
1979 SDValue Ptr = LD->getBasePtr();
1980 SDValue Offset = LD->getOffset();
1981 assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
1982 Align Alignment = LD->getOriginalAlign();
1983 SDValue Mask = LD->getMask();
1984 SDValue EVL = LD->getVectorLength();
1985 EVT MemoryVT = LD->getMemoryVT();
1986
1987 EVT LoMemVT, HiMemVT;
1988 bool HiIsEmpty = false;
1989 std::tie(LoMemVT, HiMemVT) =
1990 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
1991
1992 // Split Mask operand
1993 SDValue MaskLo, MaskHi;
1994 if (Mask.getOpcode() == ISD::SETCC) {
1995 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1996 } else {
1997 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1998 GetSplitVector(Mask, MaskLo, MaskHi);
1999 else
2000 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2001 }
2002
2003 // Split EVL operand
2004 SDValue EVLLo, EVLHi;
2005 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, LD->getValueType(0), dl);
2006
2008 LD->getPointerInfo(), MachineMemOperand::MOLoad,
2009 LocationSize::beforeOrAfterPointer(), Alignment, LD->getAAInfo(),
2010 LD->getRanges());
2011
2012 Lo =
2013 DAG.getLoadVP(LD->getAddressingMode(), ExtType, LoVT, dl, Ch, Ptr, Offset,
2014 MaskLo, EVLLo, LoMemVT, MMO, LD->isExpandingLoad());
2015
2016 if (HiIsEmpty) {
2017 // The hi vp_load has zero storage size. We therefore simply set it to
2018 // the low vp_load and rely on subsequent removal from the chain.
2019 Hi = Lo;
2020 } else {
2021 // Generate hi vp_load.
2022 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
2023 LD->isExpandingLoad());
2024
2026 if (LoMemVT.isScalableVector())
2027 MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
2028 else
2029 MPI = LD->getPointerInfo().getWithOffset(
2030 LoMemVT.getStoreSize().getFixedValue());
2031
2034 Alignment, LD->getAAInfo(), LD->getRanges());
2035
2036 Hi = DAG.getLoadVP(LD->getAddressingMode(), ExtType, HiVT, dl, Ch, Ptr,
2037 Offset, MaskHi, EVLHi, HiMemVT, MMO,
2038 LD->isExpandingLoad());
2039 }
2040
2041 // Build a factor node to remember that this load is independent of the
2042 // other one.
2043 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2044 Hi.getValue(1));
2045
2046 // Legalize the chain result - switch anything that used the old chain to
2047 // use the new one.
2048 ReplaceValueWith(SDValue(LD, 1), Ch);
2049}
2050
2051void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
2052 SDValue &Lo, SDValue &Hi) {
2053 assert(SLD->isUnindexed() &&
2054 "Indexed VP strided load during type legalization!");
2055 assert(SLD->getOffset().isUndef() &&
2056 "Unexpected indexed variable-length load offset");
2057
2058 SDLoc DL(SLD);
2059
2060 EVT LoVT, HiVT;
2061 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(SLD->getValueType(0));
2062
2063 EVT LoMemVT, HiMemVT;
2064 bool HiIsEmpty = false;
2065 std::tie(LoMemVT, HiMemVT) =
2066 DAG.GetDependentSplitDestVTs(SLD->getMemoryVT(), LoVT, &HiIsEmpty);
2067
2068 SDValue Mask = SLD->getMask();
2069 SDValue LoMask, HiMask;
2070 if (Mask.getOpcode() == ISD::SETCC) {
2071 SplitVecRes_SETCC(Mask.getNode(), LoMask, HiMask);
2072 } else {
2073 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2074 GetSplitVector(Mask, LoMask, HiMask);
2075 else
2076 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
2077 }
2078
2079 SDValue LoEVL, HiEVL;
2080 std::tie(LoEVL, HiEVL) =
2081 DAG.SplitEVL(SLD->getVectorLength(), SLD->getValueType(0), DL);
2082
2083 // Generate the low vp_strided_load
2084 Lo = DAG.getStridedLoadVP(
2085 SLD->getAddressingMode(), SLD->getExtensionType(), LoVT, DL,
2086 SLD->getChain(), SLD->getBasePtr(), SLD->getOffset(), SLD->getStride(),
2087 LoMask, LoEVL, LoMemVT, SLD->getMemOperand(), SLD->isExpandingLoad());
2088
2089 if (HiIsEmpty) {
2090 // The high vp_strided_load has zero storage size. We therefore simply set
2091 // it to the low vp_strided_load and rely on subsequent removal from the
2092 // chain.
2093 Hi = Lo;
2094 } else {
2095 // Generate the high vp_strided_load.
2096 // To calculate the high base address, we need to sum to the low base
2097 // address stride number of bytes for each element already loaded by low,
2098 // that is: Ptr = Ptr + (LoEVL * Stride)
2099 EVT PtrVT = SLD->getBasePtr().getValueType();
2101 DAG.getNode(ISD::MUL, DL, PtrVT, LoEVL,
2102 DAG.getSExtOrTrunc(SLD->getStride(), DL, PtrVT));
2103 SDValue Ptr =
2104 DAG.getNode(ISD::ADD, DL, PtrVT, SLD->getBasePtr(), Increment);
2105
2106 Align Alignment = SLD->getOriginalAlign();
2107 if (LoMemVT.isScalableVector())
2108 Alignment = commonAlignment(
2109 Alignment, LoMemVT.getSizeInBits().getKnownMinValue() / 8);
2110
2114 Alignment, SLD->getAAInfo(), SLD->getRanges());
2115
2117 HiVT, DL, SLD->getChain(), Ptr, SLD->getOffset(),
2118 SLD->getStride(), HiMask, HiEVL, HiMemVT, MMO,
2119 SLD->isExpandingLoad());
2120 }
2121
2122 // Build a factor node to remember that this load is independent of the
2123 // other one.
2124 SDValue Ch = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
2125 Hi.getValue(1));
2126
2127 // Legalize the chain result - switch anything that used the old chain to
2128 // use the new one.
2129 ReplaceValueWith(SDValue(SLD, 1), Ch);
2130}
2131
2132void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
2133 SDValue &Lo, SDValue &Hi) {
2134 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
2135 EVT LoVT, HiVT;
2136 SDLoc dl(MLD);
2137 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
2138
2139 SDValue Ch = MLD->getChain();
2140 SDValue Ptr = MLD->getBasePtr();
2141 SDValue Offset = MLD->getOffset();
2142 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
2143 SDValue Mask = MLD->getMask();
2144 SDValue PassThru = MLD->getPassThru();
2145 Align Alignment = MLD->getOriginalAlign();
2146 ISD::LoadExtType ExtType = MLD->getExtensionType();
2147
2148 // Split Mask operand
2149 SDValue MaskLo, MaskHi;
2150 if (Mask.getOpcode() == ISD::SETCC) {
2151 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2152 } else {
2153 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2154 GetSplitVector(Mask, MaskLo, MaskHi);
2155 else
2156 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2157 }
2158
2159 EVT MemoryVT = MLD->getMemoryVT();
2160 EVT LoMemVT, HiMemVT;
2161 bool HiIsEmpty = false;
2162 std::tie(LoMemVT, HiMemVT) =
2163 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
2164
2165 SDValue PassThruLo, PassThruHi;
2166 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2167 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2168 else
2169 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2170
2173 LocationSize::beforeOrAfterPointer(), Alignment, MLD->getAAInfo(),
2174 MLD->getRanges());
2175
2176 Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
2177 MMO, MLD->getAddressingMode(), ExtType,
2178 MLD->isExpandingLoad());
2179
2180 if (HiIsEmpty) {
2181 // The hi masked load has zero storage size. We therefore simply set it to
2182 // the low masked load and rely on subsequent removal from the chain.
2183 Hi = Lo;
2184 } else {
2185 // Generate hi masked load.
2186 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
2187 MLD->isExpandingLoad());
2188
2190 if (LoMemVT.isScalableVector())
2192 else
2193 MPI = MLD->getPointerInfo().getWithOffset(
2194 LoMemVT.getStoreSize().getFixedValue());
2195
2198 Alignment, MLD->getAAInfo(), MLD->getRanges());
2199
2200 Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
2201 HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
2202 MLD->isExpandingLoad());
2203 }
2204
2205 // Build a factor node to remember that this load is independent of the
2206 // other one.
2207 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2208 Hi.getValue(1));
2209
2210 // Legalize the chain result - switch anything that used the old chain to
2211 // use the new one.
2212 ReplaceValueWith(SDValue(MLD, 1), Ch);
2213
2214}
2215
2216void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2217 SDValue &Hi, bool SplitSETCC) {
2218 EVT LoVT, HiVT;
2219 SDLoc dl(N);
2220 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2221
2222 SDValue Ch = N->getChain();
2223 SDValue Ptr = N->getBasePtr();
2224 struct Operands {
2225 SDValue Mask;
2226 SDValue Index;
2227 SDValue Scale;
2228 } Ops = [&]() -> Operands {
2229 if (auto *MSC = dyn_cast<MaskedGatherSDNode>(N)) {
2230 return {MSC->getMask(), MSC->getIndex(), MSC->getScale()};
2231 }
2232 auto *VPSC = cast<VPGatherSDNode>(N);
2233 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale()};
2234 }();
2235
2236 EVT MemoryVT = N->getMemoryVT();
2237 Align Alignment = N->getOriginalAlign();
2238
2239 // Split Mask operand
2240 SDValue MaskLo, MaskHi;
2241 if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2242 SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
2243 } else {
2244 std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, dl);
2245 }
2246
2247 EVT LoMemVT, HiMemVT;
2248 // Split MemoryVT
2249 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2250
2251 SDValue IndexHi, IndexLo;
2252 if (getTypeAction(Ops.Index.getValueType()) ==
2254 GetSplitVector(Ops.Index, IndexLo, IndexHi);
2255 else
2256 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, dl);
2257
2259 N->getPointerInfo(), MachineMemOperand::MOLoad,
2260 LocationSize::beforeOrAfterPointer(), Alignment, N->getAAInfo(),
2261 N->getRanges());
2262
2263 if (auto *MGT = dyn_cast<MaskedGatherSDNode>(N)) {
2264 SDValue PassThru = MGT->getPassThru();
2265 SDValue PassThruLo, PassThruHi;
2266 if (getTypeAction(PassThru.getValueType()) ==
2268 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2269 else
2270 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2271
2272 ISD::LoadExtType ExtType = MGT->getExtensionType();
2273 ISD::MemIndexType IndexTy = MGT->getIndexType();
2274
2275 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2276 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl,
2277 OpsLo, MMO, IndexTy, ExtType);
2278
2279 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2280 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl,
2281 OpsHi, MMO, IndexTy, ExtType);
2282 } else {
2283 auto *VPGT = cast<VPGatherSDNode>(N);
2284 SDValue EVLLo, EVLHi;
2285 std::tie(EVLLo, EVLHi) =
2286 DAG.SplitEVL(VPGT->getVectorLength(), MemoryVT, dl);
2287
2288 SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2289 Lo = DAG.getGatherVP(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, OpsLo,
2290 MMO, VPGT->getIndexType());
2291
2292 SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2293 Hi = DAG.getGatherVP(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, OpsHi,
2294 MMO, VPGT->getIndexType());
2295 }
2296
2297 // Build a factor node to remember that this load is independent of the
2298 // other one.
2299 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2300 Hi.getValue(1));
2301
2302 // Legalize the chain result - switch anything that used the old chain to
2303 // use the new one.
2304 ReplaceValueWith(SDValue(N, 1), Ch);
2305}
2306
2307void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
2308 assert(N->getValueType(0).isVector() &&
2309 N->getOperand(0).getValueType().isVector() &&
2310 "Operand types must be vectors");
2311
2312 EVT LoVT, HiVT;
2313 SDLoc DL(N);
2314 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2315
2316 // If the input also splits, handle it directly. Otherwise split it by hand.
2317 SDValue LL, LH, RL, RH;
2318 if (getTypeAction(N->getOperand(0).getValueType()) ==
2320 GetSplitVector(N->getOperand(0), LL, LH);
2321 else
2322 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
2323
2324 if (getTypeAction(N->getOperand(1).getValueType()) ==
2326 GetSplitVector(N->getOperand(1), RL, RH);
2327 else
2328 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
2329
2330 if (N->getOpcode() == ISD::SETCC) {
2331 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
2332 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
2333 } else {
2334 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
2335 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
2336 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
2337 std::tie(EVLLo, EVLHi) =
2338 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), DL);
2339 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2), MaskLo,
2340 EVLLo);
2341 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2), MaskHi,
2342 EVLHi);
2343 }
2344}
2345
2346void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
2347 SDValue &Hi) {
2348 // Get the dest types - they may not match the input types, e.g. int_to_fp.
2349 EVT LoVT, HiVT;
2350 SDLoc dl(N);
2351 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2352
2353 // If the input also splits, handle it directly for a compile time speedup.
2354 // Otherwise split it by hand.
2355 EVT InVT = N->getOperand(0).getValueType();
2356 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
2357 GetSplitVector(N->getOperand(0), Lo, Hi);
2358 else
2359 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
2360
2361 const SDNodeFlags Flags = N->getFlags();
2362 unsigned Opcode = N->getOpcode();
2363 if (N->getNumOperands() <= 2) {
2364 if (Opcode == ISD::FP_ROUND) {
2365 Lo = DAG.getNode(Opcode, dl, LoVT, Lo, N->getOperand(1), Flags);
2366 Hi = DAG.getNode(Opcode, dl, HiVT, Hi, N->getOperand(1), Flags);
2367 } else {
2368 Lo = DAG.getNode(Opcode, dl, LoVT, Lo, Flags);
2369 Hi = DAG.getNode(Opcode, dl, HiVT, Hi, Flags);
2370 }
2371 return;
2372 }
2373
2374 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
2375 assert(N->isVPOpcode() && "Expected VP opcode");
2376
2377 SDValue MaskLo, MaskHi;
2378 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
2379
2380 SDValue EVLLo, EVLHi;
2381 std::tie(EVLLo, EVLHi) =
2382 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
2383
2384 Lo = DAG.getNode(Opcode, dl, LoVT, {Lo, MaskLo, EVLLo}, Flags);
2385 Hi = DAG.getNode(Opcode, dl, HiVT, {Hi, MaskHi, EVLHi}, Flags);
2386}
2387
2388void DAGTypeLegalizer::SplitVecRes_ADDRSPACECAST(SDNode *N, SDValue &Lo,
2389 SDValue &Hi) {
2390 SDLoc dl(N);
2391 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(N->getValueType(0));
2392
2393 // If the input also splits, handle it directly for a compile time speedup.
2394 // Otherwise split it by hand.
2395 EVT InVT = N->getOperand(0).getValueType();
2396 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
2397 GetSplitVector(N->getOperand(0), Lo, Hi);
2398 else
2399 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
2400
2401 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(N);
2402 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
2403 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
2404 Lo = DAG.getAddrSpaceCast(dl, LoVT, Lo, SrcAS, DestAS);
2405 Hi = DAG.getAddrSpaceCast(dl, HiVT, Hi, SrcAS, DestAS);
2406}
2407
2408void DAGTypeLegalizer::SplitVecRes_FFREXP(SDNode *N, unsigned ResNo,
2409 SDValue &Lo, SDValue &Hi) {
2410 SDLoc dl(N);
2411 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(N->getValueType(0));
2412 auto [LoVT1, HiVT1] = DAG.GetSplitDestVTs(N->getValueType(1));
2413
2414 // If the input also splits, handle it directly for a compile time speedup.
2415 // Otherwise split it by hand.
2416 EVT InVT = N->getOperand(0).getValueType();
2417 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
2418 GetSplitVector(N->getOperand(0), Lo, Hi);
2419 else
2420 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
2421
2422 Lo = DAG.getNode(N->getOpcode(), dl, {LoVT, LoVT1}, Lo);
2423 Hi = DAG.getNode(N->getOpcode(), dl, {HiVT, HiVT1}, Hi);
2424 Lo->setFlags(N->getFlags());
2425 Hi->setFlags(N->getFlags());
2426
2427 SDNode *HiNode = Hi.getNode();
2428 SDNode *LoNode = Lo.getNode();
2429
2430 // Replace the other vector result not being explicitly split here.
2431 unsigned OtherNo = 1 - ResNo;
2432 EVT OtherVT = N->getValueType(OtherNo);
2433 if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
2434 SetSplitVector(SDValue(N, OtherNo), SDValue(LoNode, OtherNo),
2435 SDValue(HiNode, OtherNo));
2436 } else {
2437 SDValue OtherVal =
2438 DAG.getNode(ISD::CONCAT_VECTORS, dl, OtherVT, SDValue(LoNode, OtherNo),
2439 SDValue(HiNode, OtherNo));
2440 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
2441 }
2442}
2443
2444void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
2445 SDValue &Hi) {
2446 SDLoc dl(N);
2447 EVT SrcVT = N->getOperand(0).getValueType();
2448 EVT DestVT = N->getValueType(0);
2449 EVT LoVT, HiVT;
2450 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
2451
2452 // We can do better than a generic split operation if the extend is doing
2453 // more than just doubling the width of the elements and the following are
2454 // true:
2455 // - The number of vector elements is even,
2456 // - the source type is legal,
2457 // - the type of a split source is illegal,
2458 // - the type of an extended (by doubling element size) source is legal, and
2459 // - the type of that extended source when split is legal.
2460 //
2461 // This won't necessarily completely legalize the operation, but it will
2462 // more effectively move in the right direction and prevent falling down
2463 // to scalarization in many cases due to the input vector being split too
2464 // far.
2465 if (SrcVT.getVectorElementCount().isKnownEven() &&
2466 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
2467 LLVMContext &Ctx = *DAG.getContext();
2468 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
2469 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
2470
2471 EVT SplitLoVT, SplitHiVT;
2472 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
2473 if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
2474 TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
2475 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
2476 N->dump(&DAG); dbgs() << "\n");
2477 if (!N->isVPOpcode()) {
2478 // Extend the source vector by one step.
2479 SDValue NewSrc =
2480 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
2481 // Get the low and high halves of the new, extended one step, vector.
2482 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
2483 // Extend those vector halves the rest of the way.
2484 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
2485 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
2486 return;
2487 }
2488
2489 // Extend the source vector by one step.
2490 SDValue NewSrc =
2491 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0),
2492 N->getOperand(1), N->getOperand(2));
2493 // Get the low and high halves of the new, extended one step, vector.
2494 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
2495
2496 SDValue MaskLo, MaskHi;
2497 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
2498
2499 SDValue EVLLo, EVLHi;
2500 std::tie(EVLLo, EVLHi) =
2501 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
2502 // Extend those vector halves the rest of the way.
2503 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, {Lo, MaskLo, EVLLo});
2504 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, {Hi, MaskHi, EVLHi});
2505 return;
2506 }
2507 }
2508 // Fall back to the generic unary operator splitting otherwise.
2509 SplitVecRes_UnaryOp(N, Lo, Hi);
2510}
2511
2512void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
2513 SDValue &Lo, SDValue &Hi) {
2514 // The low and high parts of the original input give four input vectors.
2515 SDValue Inputs[4];
2516 SDLoc DL(N);
2517 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
2518 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
2519 EVT NewVT = Inputs[0].getValueType();
2520 unsigned NewElts = NewVT.getVectorNumElements();
2521
2522 auto &&IsConstant = [](const SDValue &N) {
2523 APInt SplatValue;
2524 return N.getResNo() == 0 &&
2525 (ISD::isConstantSplatVector(N.getNode(), SplatValue) ||
2527 };
2528 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
2529 SDValue &Input2,
2531 assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
2532 Input2->getOpcode() == ISD::BUILD_VECTOR &&
2533 "Expected build vector node.");
2534 EVT EltVT = NewVT.getVectorElementType();
2535 SmallVector<SDValue> Ops(NewElts, DAG.getUNDEF(EltVT));
2536 for (unsigned I = 0; I < NewElts; ++I) {
2537 if (Mask[I] == PoisonMaskElem)
2538 continue;
2539 unsigned Idx = Mask[I];
2540 if (Idx >= NewElts)
2541 Ops[I] = Input2.getOperand(Idx - NewElts);
2542 else
2543 Ops[I] = Input1.getOperand(Idx);
2544 // Make the type of all elements the same as the element type.
2545 if (Ops[I].getValueType().bitsGT(EltVT))
2546 Ops[I] = DAG.getNode(ISD::TRUNCATE, DL, EltVT, Ops[I]);
2547 }
2548 return DAG.getBuildVector(NewVT, DL, Ops);
2549 };
2550
2551 // If Lo or Hi uses elements from at most two of the four input vectors, then
2552 // express it as a vector shuffle of those two inputs. Otherwise extract the
2553 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
2554 SmallVector<int> OrigMask(N->getMask());
2555 // Try to pack incoming shuffles/inputs.
2556 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
2558 // Check if all inputs are shuffles of the same operands or non-shuffles.
2560 for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) {
2561 SDValue Input = Inputs[Idx];
2562 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Input.getNode());
2563 if (!Shuffle ||
2564 Input.getOperand(0).getValueType() != Input.getValueType())
2565 continue;
2566 ShufflesIdxs[std::make_pair(Input.getOperand(0), Input.getOperand(1))]
2567 .push_back(Idx);
2568 ShufflesIdxs[std::make_pair(Input.getOperand(1), Input.getOperand(0))]
2569 .push_back(Idx);
2570 }
2571 for (auto &P : ShufflesIdxs) {
2572 if (P.second.size() < 2)
2573 continue;
2574 // Use shuffles operands instead of shuffles themselves.
2575 // 1. Adjust mask.
2576 for (int &Idx : Mask) {
2577 if (Idx == PoisonMaskElem)
2578 continue;
2579 unsigned SrcRegIdx = Idx / NewElts;
2580 if (Inputs[SrcRegIdx].isUndef()) {
2582 continue;
2583 }
2584 auto *Shuffle =
2585 dyn_cast<ShuffleVectorSDNode>(Inputs[SrcRegIdx].getNode());
2586 if (!Shuffle || !is_contained(P.second, SrcRegIdx))
2587 continue;
2588 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2589 if (MaskElt == PoisonMaskElem) {
2591 continue;
2592 }
2593 Idx = MaskElt % NewElts +
2594 P.second[Shuffle->getOperand(MaskElt / NewElts) == P.first.first
2595 ? 0
2596 : 1] *
2597 NewElts;
2598 }
2599 // 2. Update inputs.
2600 Inputs[P.second[0]] = P.first.first;
2601 Inputs[P.second[1]] = P.first.second;
2602 // Clear the pair data.
2603 P.second.clear();
2604 ShufflesIdxs[std::make_pair(P.first.second, P.first.first)].clear();
2605 }
2606 // Check if any concat_vectors can be simplified.
2607 SmallBitVector UsedSubVector(2 * std::size(Inputs));
2608 for (int &Idx : Mask) {
2609 if (Idx == PoisonMaskElem)
2610 continue;
2611 unsigned SrcRegIdx = Idx / NewElts;
2612 if (Inputs[SrcRegIdx].isUndef()) {
2614 continue;
2615 }
2617 getTypeAction(Inputs[SrcRegIdx].getValueType());
2618 if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
2619 Inputs[SrcRegIdx].getNumOperands() == 2 &&
2620 !Inputs[SrcRegIdx].getOperand(1).isUndef() &&
2621 (TypeAction == TargetLowering::TypeLegal ||
2622 TypeAction == TargetLowering::TypeWidenVector))
2623 UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
2624 }
2625 if (UsedSubVector.count() > 1) {
2627 for (unsigned I = 0; I < std::size(Inputs); ++I) {
2628 if (UsedSubVector.test(2 * I) == UsedSubVector.test(2 * I + 1))
2629 continue;
2630 if (Pairs.empty() || Pairs.back().size() == 2)
2631 Pairs.emplace_back();
2632 if (UsedSubVector.test(2 * I)) {
2633 Pairs.back().emplace_back(I, 0);
2634 } else {
2635 assert(UsedSubVector.test(2 * I + 1) &&
2636 "Expected to be used one of the subvectors.");
2637 Pairs.back().emplace_back(I, 1);
2638 }
2639 }
2640 if (!Pairs.empty() && Pairs.front().size() > 1) {
2641 // Adjust mask.
2642 for (int &Idx : Mask) {
2643 if (Idx == PoisonMaskElem)
2644 continue;
2645 unsigned SrcRegIdx = Idx / NewElts;
2646 auto *It = find_if(
2647 Pairs, [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
2648 return Idxs.front().first == SrcRegIdx ||
2649 Idxs.back().first == SrcRegIdx;
2650 });
2651 if (It == Pairs.end())
2652 continue;
2653 Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
2654 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
2655 }
2656 // Adjust inputs.
2657 for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
2658 Inputs[Idxs.front().first] = DAG.getNode(
2660 Inputs[Idxs.front().first].getValueType(),
2661 Inputs[Idxs.front().first].getOperand(Idxs.front().second),
2662 Inputs[Idxs.back().first].getOperand(Idxs.back().second));
2663 }
2664 }
2665 }
2666 bool Changed;
2667 do {
2668 // Try to remove extra shuffles (except broadcasts) and shuffles with the
2669 // reused operands.
2670 Changed = false;
2671 for (unsigned I = 0; I < std::size(Inputs); ++I) {
2672 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Inputs[I].getNode());
2673 if (!Shuffle)
2674 continue;
2675 if (Shuffle->getOperand(0).getValueType() != NewVT)
2676 continue;
2677 int Op = -1;
2678 if (!Inputs[I].hasOneUse() && Shuffle->getOperand(1).isUndef() &&
2679 !Shuffle->isSplat()) {
2680 Op = 0;
2681 } else if (!Inputs[I].hasOneUse() &&
2682 !Shuffle->getOperand(1).isUndef()) {
2683 // Find the only used operand, if possible.
2684 for (int &Idx : Mask) {
2685 if (Idx == PoisonMaskElem)
2686 continue;
2687 unsigned SrcRegIdx = Idx / NewElts;
2688 if (SrcRegIdx != I)
2689 continue;
2690 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2691 if (MaskElt == PoisonMaskElem) {
2693 continue;
2694 }
2695 int OpIdx = MaskElt / NewElts;
2696 if (Op == -1) {
2697 Op = OpIdx;
2698 continue;
2699 }
2700 if (Op != OpIdx) {
2701 Op = -1;
2702 break;
2703 }
2704 }
2705 }
2706 if (Op < 0) {
2707 // Try to check if one of the shuffle operands is used already.
2708 for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
2709 if (Shuffle->getOperand(OpIdx).isUndef())
2710 continue;
2711 auto *It = find(Inputs, Shuffle->getOperand(OpIdx));
2712 if (It == std::end(Inputs))
2713 continue;
2714 int FoundOp = std::distance(std::begin(Inputs), It);
2715 // Found that operand is used already.
2716 // 1. Fix the mask for the reused operand.
2717 for (int &Idx : Mask) {
2718 if (Idx == PoisonMaskElem)
2719 continue;
2720 unsigned SrcRegIdx = Idx / NewElts;
2721 if (SrcRegIdx != I)
2722 continue;
2723 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2724 if (MaskElt == PoisonMaskElem) {
2726 continue;
2727 }
2728 int MaskIdx = MaskElt / NewElts;
2729 if (OpIdx == MaskIdx)
2730 Idx = MaskElt % NewElts + FoundOp * NewElts;
2731 }
2732 // 2. Set Op to the unused OpIdx.
2733 Op = (OpIdx + 1) % 2;
2734 break;
2735 }
2736 }
2737 if (Op >= 0) {
2738 Changed = true;
2739 Inputs[I] = Shuffle->getOperand(Op);
2740 // Adjust mask.
2741 for (int &Idx : Mask) {
2742 if (Idx == PoisonMaskElem)
2743 continue;
2744 unsigned SrcRegIdx = Idx / NewElts;
2745 if (SrcRegIdx != I)
2746 continue;
2747 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2748 int OpIdx = MaskElt / NewElts;
2749 if (OpIdx != Op)
2750 continue;
2751 Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
2752 }
2753 }
2754 }
2755 } while (Changed);
2756 };
2757 TryPeekThroughShufflesInputs(OrigMask);
2758 // Proces unique inputs.
2759 auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
2760 NewElts](SmallVectorImpl<int> &Mask) {
2761 SetVector<SDValue> UniqueInputs;
2762 SetVector<SDValue> UniqueConstantInputs;
2763 for (const auto &I : Inputs) {
2764 if (IsConstant(I))
2765 UniqueConstantInputs.insert(I);
2766 else if (!I.isUndef())
2767 UniqueInputs.insert(I);
2768 }
2769 // Adjust mask in case of reused inputs. Also, need to insert constant
2770 // inputs at first, otherwise it affects the final outcome.
2771 if (UniqueInputs.size() != std::size(Inputs)) {
2772 auto &&UniqueVec = UniqueInputs.takeVector();
2773 auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
2774 unsigned ConstNum = UniqueConstantVec.size();
2775 for (int &Idx : Mask) {
2776 if (Idx == PoisonMaskElem)
2777 continue;
2778 unsigned SrcRegIdx = Idx / NewElts;
2779 if (Inputs[SrcRegIdx].isUndef()) {
2781 continue;
2782 }
2783 const auto It = find(UniqueConstantVec, Inputs[SrcRegIdx]);
2784 if (It != UniqueConstantVec.end()) {
2785 Idx = (Idx % NewElts) +
2786 NewElts * std::distance(UniqueConstantVec.begin(), It);
2787 assert(Idx >= 0 && "Expected defined mask idx.");
2788 continue;
2789 }
2790 const auto RegIt = find(UniqueVec, Inputs[SrcRegIdx]);
2791 assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
2792 Idx = (Idx % NewElts) +
2793 NewElts * (std::distance(UniqueVec.begin(), RegIt) + ConstNum);
2794 assert(Idx >= 0 && "Expected defined mask idx.");
2795 }
2796 copy(UniqueConstantVec, std::begin(Inputs));
2797 copy(UniqueVec, std::next(std::begin(Inputs), ConstNum));
2798 }
2799 };
2800 MakeUniqueInputs(OrigMask);
2801 SDValue OrigInputs[4];
2802 copy(Inputs, std::begin(OrigInputs));
2803 for (unsigned High = 0; High < 2; ++High) {
2804 SDValue &Output = High ? Hi : Lo;
2805
2806 // Build a shuffle mask for the output, discovering on the fly which
2807 // input vectors to use as shuffle operands.
2808 unsigned FirstMaskIdx = High * NewElts;
2809 SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem);
2810 copy(ArrayRef(OrigMask).slice(FirstMaskIdx, NewElts), Mask.begin());
2811 assert(!Output && "Expected default initialized initial value.");
2812 TryPeekThroughShufflesInputs(Mask);
2813 MakeUniqueInputs(Mask);
2814 SDValue TmpInputs[4];
2815 copy(Inputs, std::begin(TmpInputs));
2816 // Track changes in the output registers.
2817 int UsedIdx = -1;
2818 bool SecondIteration = false;
2819 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
2820 if (UsedIdx < 0) {
2821 UsedIdx = Idx;
2822 return false;
2823 }
2824 if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
2825 SecondIteration = true;
2826 return SecondIteration;
2827 };
2829 Mask, std::size(Inputs), std::size(Inputs),
2830 /*NumOfUsedRegs=*/1,
2831 [&Output, &DAG = DAG, NewVT]() { Output = DAG.getUNDEF(NewVT); },
2832 [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
2833 &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
2834 if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
2835 Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
2836 else
2837 Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx],
2838 DAG.getUNDEF(NewVT), Mask);
2839 Inputs[Idx] = Output;
2840 },
2841 [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
2842 &TmpInputs,
2843 &BuildVector](ArrayRef<int> Mask, unsigned Idx1, unsigned Idx2) {
2844 if (AccumulateResults(Idx1)) {
2845 if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
2846 Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
2847 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
2848 else
2849 Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx1],
2850 Inputs[Idx2], Mask);
2851 } else {
2852 if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
2853 TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
2854 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
2855 else
2856 Output = DAG.getVectorShuffle(NewVT, DL, TmpInputs[Idx1],
2857 TmpInputs[Idx2], Mask);
2858 }
2859 Inputs[Idx1] = Output;
2860 });
2861 copy(OrigInputs, std::begin(Inputs));
2862 }
2863}
2864
2865void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2866 EVT OVT = N->getValueType(0);
2867 EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext());
2868 SDValue Chain = N->getOperand(0);
2869 SDValue Ptr = N->getOperand(1);
2870 SDValue SV = N->getOperand(2);
2871 SDLoc dl(N);
2872
2873 const Align Alignment =
2874 DAG.getDataLayout().getABITypeAlign(NVT.getTypeForEVT(*DAG.getContext()));
2875
2876 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment.value());
2877 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment.value());
2878 Chain = Hi.getValue(1);
2879
2880 // Modified the chain - switch anything that used the old chain to use
2881 // the new one.
2882 ReplaceValueWith(SDValue(N, 1), Chain);
2883}
2884
2885void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
2886 SDValue &Hi) {
2887 EVT DstVTLo, DstVTHi;
2888 std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(N->getValueType(0));
2889 SDLoc dl(N);
2890
2891 SDValue SrcLo, SrcHi;
2892 EVT SrcVT = N->getOperand(0).getValueType();
2893 if (getTypeAction(SrcVT) == TargetLowering::TypeSplitVector)
2894 GetSplitVector(N->getOperand(0), SrcLo, SrcHi);
2895 else
2896 std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(N, 0);
2897
2898 Lo = DAG.getNode(N->getOpcode(), dl, DstVTLo, SrcLo, N->getOperand(1));
2899 Hi = DAG.getNode(N->getOpcode(), dl, DstVTHi, SrcHi, N->getOperand(1));
2900}
2901
2902void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
2903 SDValue &Hi) {
2904 SDValue InLo, InHi;
2905 GetSplitVector(N->getOperand(0), InLo, InHi);
2906 SDLoc DL(N);
2907
2908 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, InHi.getValueType(), InHi);
2909 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, InLo.getValueType(), InLo);
2910}
2911
2912void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
2913 SDValue &Hi) {
2914 SDLoc DL(N);
2915
2916 SDValue Expanded = TLI.expandVectorSplice(N, DAG);
2917 std::tie(Lo, Hi) = DAG.SplitVector(Expanded, DL);
2918}
2919
2920void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo,
2921 SDValue &Hi) {
2922 EVT VT = N->getValueType(0);
2923 SDValue Val = N->getOperand(0);
2924 SDValue Mask = N->getOperand(1);
2925 SDValue EVL = N->getOperand(2);
2926 SDLoc DL(N);
2927
2928 // Fallback to VP_STRIDED_STORE to stack followed by VP_LOAD.
2929 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
2930
2931 EVT MemVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
2933 SDValue StackPtr = DAG.CreateStackTemporary(MemVT.getStoreSize(), Alignment);
2934 EVT PtrVT = StackPtr.getValueType();
2935 auto &MF = DAG.getMachineFunction();
2936 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
2937 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
2938
2939 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
2941 Alignment);
2942 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
2944 Alignment);
2945
2946 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
2947 SDValue NumElemMinus1 =
2948 DAG.getNode(ISD::SUB, DL, PtrVT, DAG.getZExtOrTrunc(EVL, DL, PtrVT),
2949 DAG.getConstant(1, DL, PtrVT));
2950 SDValue StartOffset = DAG.getNode(ISD::MUL, DL, PtrVT, NumElemMinus1,
2951 DAG.getConstant(EltWidth, DL, PtrVT));
2952 SDValue StorePtr = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, StartOffset);
2953 SDValue Stride = DAG.getConstant(-(int64_t)EltWidth, DL, PtrVT);
2954
2955 SDValue TrueMask = DAG.getBoolConstant(true, DL, Mask.getValueType(), VT);
2956 SDValue Store = DAG.getStridedStoreVP(DAG.getEntryNode(), DL, Val, StorePtr,
2957 DAG.getUNDEF(PtrVT), Stride, TrueMask,
2958 EVL, MemVT, StoreMMO, ISD::UNINDEXED);
2959
2960 SDValue Load = DAG.getLoadVP(VT, DL, Store, StackPtr, Mask, EVL, LoadMMO);
2961
2962 std::tie(Lo, Hi) = DAG.SplitVector(Load, DL);
2963}
2964
2965void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
2966
2967 SDValue Op0Lo, Op0Hi, Op1Lo, Op1Hi;
2968 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
2969 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
2970 EVT VT = Op0Lo.getValueType();
2971 SDLoc DL(N);
2973 DAG.getVTList(VT, VT), Op0Lo, Op0Hi);
2975 DAG.getVTList(VT, VT), Op1Lo, Op1Hi);
2976
2977 SetSplitVector(SDValue(N, 0), ResLo.getValue(0), ResHi.getValue(0));
2978 SetSplitVector(SDValue(N, 1), ResLo.getValue(1), ResHi.getValue(1));
2979}
2980
2981void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) {
2982 SDValue Op0Lo, Op0Hi, Op1Lo, Op1Hi;
2983 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
2984 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
2985 EVT VT = Op0Lo.getValueType();
2986 SDLoc DL(N);
2988 DAG.getVTList(VT, VT), Op0Lo, Op1Lo),
2989 DAG.getNode(ISD::VECTOR_INTERLEAVE, DL,
2990 DAG.getVTList(VT, VT), Op0Hi, Op1Hi)};
2991
2992 SetSplitVector(SDValue(N, 0), Res[0].getValue(0), Res[0].getValue(1));
2993 SetSplitVector(SDValue(N, 1), Res[1].getValue(0), Res[1].getValue(1));
2994}
2995
2996//===----------------------------------------------------------------------===//
2997// Operand Vector Splitting
2998//===----------------------------------------------------------------------===//
2999
3000/// This method is called when the specified operand of the specified node is
3001/// found to need vector splitting. At this point, all of the result types of
3002/// the node are known to be legal, but other operands of the node may need
3003/// legalization as well as the specified one.
3004bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
3005 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG));
3006 SDValue Res = SDValue();
3007
3008 // See if the target wants to custom split this node.
3009 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
3010 return false;
3011
3012 switch (N->getOpcode()) {
3013 default:
3014#ifndef NDEBUG
3015 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
3016 N->dump(&DAG);
3017 dbgs() << "\n";
3018#endif
3019 report_fatal_error("Do not know how to split this operator's "
3020 "operand!\n");
3021
3022 case ISD::VP_SETCC:
3023 case ISD::STRICT_FSETCC:
3024 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
3025 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
3026 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
3027 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
3028 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
3029 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
3030 case ISD::VP_TRUNCATE:
3031 case ISD::TRUNCATE:
3032 Res = SplitVecOp_TruncateHelper(N);
3033 break;
3035 case ISD::VP_FP_ROUND:
3036 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
3037 case ISD::FCOPYSIGN: Res = SplitVecOp_FPOpDifferentTypes(N); break;
3038 case ISD::STORE:
3039 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
3040 break;
3041 case ISD::VP_STORE:
3042 Res = SplitVecOp_VP_STORE(cast<VPStoreSDNode>(N), OpNo);
3043 break;
3044 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
3045 Res = SplitVecOp_VP_STRIDED_STORE(cast<VPStridedStoreSDNode>(N), OpNo);
3046 break;
3047 case ISD::MSTORE:
3048 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
3049 break;
3050 case ISD::MSCATTER:
3051 case ISD::VP_SCATTER:
3052 Res = SplitVecOp_Scatter(cast<MemSDNode>(N), OpNo);
3053 break;
3054 case ISD::MGATHER:
3055 case ISD::VP_GATHER:
3056 Res = SplitVecOp_Gather(cast<MemSDNode>(N), OpNo);
3057 break;
3058 case ISD::VSELECT:
3059 Res = SplitVecOp_VSELECT(N, OpNo);
3060 break;
3063 case ISD::SINT_TO_FP:
3064 case ISD::UINT_TO_FP:
3065 case ISD::VP_SINT_TO_FP:
3066 case ISD::VP_UINT_TO_FP:
3067 if (N->getValueType(0).bitsLT(
3068 N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType()))
3069 Res = SplitVecOp_TruncateHelper(N);
3070 else
3071 Res = SplitVecOp_UnaryOp(N);
3072 break;
3075 Res = SplitVecOp_FP_TO_XINT_SAT(N);
3076 break;
3077 case ISD::FP_TO_SINT:
3078 case ISD::FP_TO_UINT:
3079 case ISD::VP_FP_TO_SINT:
3080 case ISD::VP_FP_TO_UINT:
3084 case ISD::FP_EXTEND:
3085 case ISD::SIGN_EXTEND:
3086 case ISD::ZERO_EXTEND:
3087 case ISD::ANY_EXTEND:
3088 case ISD::FTRUNC:
3089 case ISD::LRINT:
3090 case ISD::LLRINT:
3091 Res = SplitVecOp_UnaryOp(N);
3092 break;
3093 case ISD::FLDEXP:
3094 Res = SplitVecOp_FPOpDifferentTypes(N);
3095 break;
3096
3100 Res = SplitVecOp_ExtVecInRegOp(N);
3101 break;
3102
3105 case ISD::VECREDUCE_ADD:
3106 case ISD::VECREDUCE_MUL:
3107 case ISD::VECREDUCE_AND:
3108 case ISD::VECREDUCE_OR:
3109 case ISD::VECREDUCE_XOR:
3118 Res = SplitVecOp_VECREDUCE(N, OpNo);
3119 break;
3122 Res = SplitVecOp_VECREDUCE_SEQ(N);
3123 break;
3124 case ISD::VP_REDUCE_FADD:
3125 case ISD::VP_REDUCE_SEQ_FADD:
3126 case ISD::VP_REDUCE_FMUL:
3127 case ISD::VP_REDUCE_SEQ_FMUL:
3128 case ISD::VP_REDUCE_ADD:
3129 case ISD::VP_REDUCE_MUL:
3130 case ISD::VP_REDUCE_AND:
3131 case ISD::VP_REDUCE_OR:
3132 case ISD::VP_REDUCE_XOR:
3133 case ISD::VP_REDUCE_SMAX:
3134 case ISD::VP_REDUCE_SMIN:
3135 case ISD::VP_REDUCE_UMAX:
3136 case ISD::VP_REDUCE_UMIN:
3137 case ISD::VP_REDUCE_FMAX:
3138 case ISD::VP_REDUCE_FMIN:
3139 case ISD::VP_REDUCE_FMAXIMUM:
3140 case ISD::VP_REDUCE_FMINIMUM:
3141 Res = SplitVecOp_VP_REDUCE(N, OpNo);
3142 break;
3143 case ISD::VP_CTTZ_ELTS:
3144 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
3145 Res = SplitVecOp_VP_CttzElements(N);
3146 break;
3147 }
3148
3149 // If the result is null, the sub-method took care of registering results etc.
3150 if (!Res.getNode()) return false;
3151
3152 // If the result is N, the sub-method updated N in place. Tell the legalizer
3153 // core about this.
3154 if (Res.getNode() == N)
3155 return true;
3156
3157 if (N->isStrictFPOpcode())
3158 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
3159 "Invalid operand expansion");
3160 else
3161 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3162 "Invalid operand expansion");
3163
3164 ReplaceValueWith(SDValue(N, 0), Res);
3165 return false;
3166}
3167
3168SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
3169 // The only possibility for an illegal operand is the mask, since result type
3170 // legalization would have handled this node already otherwise.
3171 assert(OpNo == 0 && "Illegal operand must be mask");
3172
3173 SDValue Mask = N->getOperand(0);
3174 SDValue Src0 = N->getOperand(1);
3175 SDValue Src1 = N->getOperand(2);
3176 EVT Src0VT = Src0.getValueType();
3177 SDLoc DL(N);
3178 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
3179
3180 SDValue Lo, Hi;
3181 GetSplitVector(N->getOperand(0), Lo, Hi);
3182 assert(Lo.getValueType() == Hi.getValueType() &&
3183 "Lo and Hi have differing types");
3184
3185 EVT LoOpVT, HiOpVT;
3186 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
3187 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
3188
3189 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
3190 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
3191 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
3192 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
3193
3194 SDValue LoSelect =
3195 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
3196 SDValue HiSelect =
3197 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
3198
3199 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
3200}
3201
3202SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
3203 EVT ResVT = N->getValueType(0);
3204 SDValue Lo, Hi;
3205 SDLoc dl(N);
3206
3207 SDValue VecOp = N->getOperand(OpNo);
3208 EVT VecVT = VecOp.getValueType();
3209 assert(VecVT.isVector() && "Can only split reduce vector operand");
3210 GetSplitVector(VecOp, Lo, Hi);
3211 EVT LoOpVT, HiOpVT;
3212 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
3213
3214 // Use the appropriate scalar instruction on the split subvectors before
3215 // reducing the now partially reduced smaller vector.
3216 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
3217 SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
3218 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
3219}
3220
3221SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
3222 EVT ResVT = N->getValueType(0);
3223 SDValue Lo, Hi;
3224 SDLoc dl(N);
3225
3226 SDValue AccOp = N->getOperand(0);
3227 SDValue VecOp = N->getOperand(1);
3228 SDNodeFlags Flags = N->getFlags();
3229
3230 EVT VecVT = VecOp.getValueType();
3231 assert(VecVT.isVector() && "Can only split reduce vector operand");
3232 GetSplitVector(VecOp, Lo, Hi);
3233 EVT LoOpVT, HiOpVT;
3234 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
3235
3236 // Reduce low half.
3237 SDValue Partial = DAG.getNode(N->getOpcode(), dl, ResVT, AccOp, Lo, Flags);
3238
3239 // Reduce high half, using low half result as initial value.
3240 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, Hi, Flags);
3241}
3242
3243SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
3244 assert(N->isVPOpcode() && "Expected VP opcode");
3245 assert(OpNo == 1 && "Can only split reduce vector operand");
3246
3247 unsigned Opc = N->getOpcode();
3248 EVT ResVT = N->getValueType(0);
3249 SDValue Lo, Hi;
3250 SDLoc dl(N);
3251
3252 SDValue VecOp = N->getOperand(OpNo);
3253 EVT VecVT = VecOp.getValueType();
3254 assert(VecVT.isVector() && "Can only split reduce vector operand");
3255 GetSplitVector(VecOp, Lo, Hi);
3256
3257 SDValue MaskLo, MaskHi;
3258 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2));
3259
3260 SDValue EVLLo, EVLHi;
3261 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(N->getOperand(3), VecVT, dl);
3262
3263 const SDNodeFlags Flags = N->getFlags();
3264
3265 SDValue ResLo =
3266 DAG.getNode(Opc, dl, ResVT, {N->getOperand(0), Lo, MaskLo, EVLLo}, Flags);
3267 return DAG.getNode(Opc, dl, ResVT, {ResLo, Hi, MaskHi, EVLHi}, Flags);
3268}
3269
3270SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
3271 // The result has a legal vector type, but the input needs splitting.
3272 EVT ResVT = N->getValueType(0);
3273 SDValue Lo, Hi;
3274 SDLoc dl(N);
3275 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3276 EVT InVT = Lo.getValueType();
3277
3278 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
3279 InVT.getVectorElementCount());
3280
3281 if (N->isStrictFPOpcode()) {
3282 Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
3283 { N->getOperand(0), Lo });
3284 Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
3285 { N->getOperand(0), Hi });
3286
3287 // Build a factor node to remember that this operation is independent
3288 // of the other one.
3289 SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3290 Hi.getValue(1));
3291
3292 // Legalize the chain result - switch anything that used the old chain to
3293 // use the new one.
3294 ReplaceValueWith(SDValue(N, 1), Ch);
3295 } else if (N->getNumOperands() == 3) {
3296 assert(N->isVPOpcode() && "Expected VP opcode");
3297 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3298 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
3299 std::tie(EVLLo, EVLHi) =
3300 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
3301 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo, MaskLo, EVLLo);
3302 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi, MaskHi, EVLHi);
3303 } else {
3304 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
3305 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
3306 }
3307
3308 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
3309}
3310
3311SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
3312 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
3313 // end up being split all the way down to individual components. Convert the
3314 // split pieces into integers and reassemble.
3315 EVT ResVT = N->getValueType(0);
3316 SDValue Lo, Hi;
3317 GetSplitVector(N->getOperand(0), Lo, Hi);
3318 SDLoc dl(N);
3319
3320 if (ResVT.isScalableVector()) {
3321 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(ResVT);
3322 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
3323 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
3324 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
3325 }
3326
3327 Lo = BitConvertToInteger(Lo);
3328 Hi = BitConvertToInteger(Hi);
3329
3330 if (DAG.getDataLayout().isBigEndian())
3331 std::swap(Lo, Hi);
3332
3333 return DAG.getNode(ISD::BITCAST, dl, ResVT, JoinIntegers(Lo, Hi));
3334}
3335
3336SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
3337 unsigned OpNo) {
3338 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
3339 // We know that the result type is legal.
3340 EVT ResVT = N->getValueType(0);
3341
3342 SDValue Vec = N->getOperand(0);
3343 SDValue SubVec = N->getOperand(1);
3344 SDValue Idx = N->getOperand(2);
3345 SDLoc dl(N);
3346
3347 SDValue Lo, Hi;
3348 GetSplitVector(SubVec, Lo, Hi);
3349
3350 uint64_t IdxVal = Idx->getAsZExtVal();
3351 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
3352
3353 SDValue FirstInsertion =
3354 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Lo, Idx);
3355 SDValue SecondInsertion =
3356 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, FirstInsertion, Hi,
3357 DAG.getVectorIdxConstant(IdxVal + LoElts, dl));
3358
3359 return SecondInsertion;
3360}
3361
3362SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
3363 // We know that the extracted result type is legal.
3364 EVT SubVT = N->getValueType(0);
3365 SDValue Idx = N->getOperand(1);
3366 SDLoc dl(N);
3367 SDValue Lo, Hi;
3368
3369 GetSplitVector(N->getOperand(0), Lo, Hi);
3370
3371 uint64_t LoEltsMin = Lo.getValueType().getVectorMinNumElements();
3372 uint64_t IdxVal = Idx->getAsZExtVal();
3373
3374 if (IdxVal < LoEltsMin) {
3375 assert(IdxVal + SubVT.getVectorMinNumElements() <= LoEltsMin &&
3376 "Extracted subvector crosses vector split!");
3377 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
3378 } else if (SubVT.isScalableVector() ==
3379 N->getOperand(0).getValueType().isScalableVector())
3380 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
3381 DAG.getVectorIdxConstant(IdxVal - LoEltsMin, dl));
3382
3383 // After this point the DAG node only permits extracting fixed-width
3384 // subvectors from scalable vectors.
3385 assert(SubVT.isFixedLengthVector() &&
3386 "Extracting scalable subvector from fixed-width unsupported");
3387
3388 // If the element type is i1 and we're not promoting the result, then we may
3389 // end up loading the wrong data since the bits are packed tightly into
3390 // bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal)
3391 // type at index 4, then we will load a byte starting at index 0.
3392 if (SubVT.getScalarType() == MVT::i1)
3393 report_fatal_error("Don't know how to extract fixed-width predicate "
3394 "subvector from a scalable predicate vector");
3395
3396 // Spill the vector to the stack. We should use the alignment for
3397 // the smallest part.
3398 SDValue Vec = N->getOperand(0);
3399 EVT VecVT = Vec.getValueType();
3400 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
3402 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
3403 auto &MF = DAG.getMachineFunction();
3404 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
3405 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
3406
3407 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
3408 SmallestAlign);
3409
3410 // Extract the subvector by loading the correct part.
3411 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVT, Idx);
3412
3413 return DAG.getLoad(
3414 SubVT, dl, Store, StackPtr,
3415 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
3416}
3417
3418SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3419 SDValue Vec = N->getOperand(0);
3420 SDValue Idx = N->getOperand(1);
3421 EVT VecVT = Vec.getValueType();
3422
3423 if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Idx)) {
3424 uint64_t IdxVal = Index->getZExtValue();
3425
3426 SDValue Lo, Hi;
3427 GetSplitVector(Vec, Lo, Hi);
3428
3429 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
3430
3431 if (IdxVal < LoElts)
3432 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
3433 else if (!Vec.getValueType().isScalableVector())
3434 return SDValue(DAG.UpdateNodeOperands(N, Hi,
3435 DAG.getConstant(IdxVal - LoElts, SDLoc(N),
3436 Idx.getValueType())), 0);
3437 }
3438
3439 // See if the target wants to custom expand this node.
3440 if (CustomLowerNode(N, N->getValueType(0), true))
3441 return SDValue();
3442
3443 // Make the vector elements byte-addressable if they aren't already.
3444 SDLoc dl(N);
3445 EVT EltVT = VecVT.getVectorElementType();
3446 if (VecVT.getScalarSizeInBits() < 8) {
3447 EltVT = MVT::i8;
3448 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
3449 VecVT.getVectorElementCount());
3450 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
3451 }
3452
3453 // Store the vector to the stack.
3454 // In cases where the vector is illegal it will be broken down into parts
3455 // and stored in parts - we should use the alignment for the smallest part.
3456 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
3458 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
3459 auto &MF = DAG.getMachineFunction();
3460 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
3461 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
3462 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
3463 SmallestAlign);
3464
3465 // Load back the required element.
3466 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
3467
3468 // FIXME: This is to handle i1 vectors with elements promoted to i8.
3469 // i1 vector handling needs general improvement.
3470 if (N->getValueType(0).bitsLT(EltVT)) {
3471 SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
3472 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
3473 return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
3474 }
3475
3476 return DAG.getExtLoad(
3477 ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
3478 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT,
3479 commonAlignment(SmallestAlign, EltVT.getFixedSizeInBits() / 8));
3480}
3481
3482SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
3483 SDValue Lo, Hi;
3484
3485 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
3486 // splitting the result has the same effect as splitting the input operand.
3487 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
3488
3489 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
3490}
3491
3492SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
3493 (void)OpNo;
3494 SDValue Lo, Hi;
3495 SplitVecRes_Gather(N, Lo, Hi);
3496
3497 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, N, N->getValueType(0), Lo, Hi);
3498 ReplaceValueWith(SDValue(N, 0), Res);
3499 return SDValue();
3500}
3501
3502SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
3503 assert(N->isUnindexed() && "Indexed vp_store of vector?");
3504 SDValue Ch = N->getChain();
3505 SDValue Ptr = N->getBasePtr();
3506 SDValue Offset = N->getOffset();
3507 assert(Offset.isUndef() && "Unexpected VP store offset");
3508 SDValue Mask = N->getMask();
3509 SDValue EVL = N->getVectorLength();
3510 SDValue Data = N->getValue();
3511 Align Alignment = N->getOriginalAlign();
3512 SDLoc DL(N);
3513
3514 SDValue DataLo, DataHi;
3515 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3516 // Split Data operand
3517 GetSplitVector(Data, DataLo, DataHi);
3518 else
3519 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
3520
3521 // Split Mask operand
3522 SDValue MaskLo, MaskHi;
3523 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
3524 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
3525 } else {
3526 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
3527 GetSplitVector(Mask, MaskLo, MaskHi);
3528 else
3529 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
3530 }
3531
3532 EVT MemoryVT = N->getMemoryVT();
3533 EVT LoMemVT, HiMemVT;
3534 bool HiIsEmpty = false;
3535 std::tie(LoMemVT, HiMemVT) =
3536 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
3537
3538 // Split EVL
3539 SDValue EVLLo, EVLHi;
3540 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, Data.getValueType(), DL);
3541
3542 SDValue Lo, Hi;
3543 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3544 N->getPointerInfo(), MachineMemOperand::MOStore,
3545 LocationSize::beforeOrAfterPointer(), Alignment, N->getAAInfo(),
3546 N->getRanges());
3547
3548 Lo = DAG.getStoreVP(Ch, DL, DataLo, Ptr, Offset, MaskLo, EVLLo, LoMemVT, MMO,
3549 N->getAddressingMode(), N->isTruncatingStore(),
3550 N->isCompressingStore());
3551
3552 // If the hi vp_store has zero storage size, only the lo vp_store is needed.
3553 if (HiIsEmpty)
3554 return Lo;
3555
3556 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
3557 N->isCompressingStore());
3558
3560 if (LoMemVT.isScalableVector()) {
3561 Alignment = commonAlignment(Alignment,
3562 LoMemVT.getSizeInBits().getKnownMinValue() / 8);
3563 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
3564 } else
3565 MPI = N->getPointerInfo().getWithOffset(
3566 LoMemVT.getStoreSize().getFixedValue());
3567
3568 MMO = DAG.getMachineFunction().getMachineMemOperand(
3570 Alignment, N->getAAInfo(), N->getRanges());
3571
3572 Hi = DAG.getStoreVP(Ch, DL, DataHi, Ptr, Offset, MaskHi, EVLHi, HiMemVT, MMO,
3573 N->getAddressingMode(), N->isTruncatingStore(),
3574 N->isCompressingStore());
3575
3576 // Build a factor node to remember that this store is independent of the
3577 // other one.
3578 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3579}
3580
3581SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
3582 unsigned OpNo) {
3583 assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?");
3584 assert(N->getOffset().isUndef() && "Unexpected VP strided store offset");
3585
3586 SDLoc DL(N);
3587
3588 SDValue Data = N->getValue();
3589 SDValue LoData, HiData;
3590 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3591 GetSplitVector(Data, LoData, HiData);
3592 else
3593 std::tie(LoData, HiData) = DAG.SplitVector(Data, DL);
3594
3595 EVT LoMemVT, HiMemVT;
3596 bool HiIsEmpty = false;
3597 std::tie(LoMemVT, HiMemVT) = DAG.GetDependentSplitDestVTs(
3598 N->getMemoryVT(), LoData.getValueType(), &HiIsEmpty);
3599
3600 SDValue Mask = N->getMask();
3601 SDValue LoMask, HiMask;
3602 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC)
3603 SplitVecRes_SETCC(Mask.getNode(), LoMask, HiMask);
3604 else if (getTypeAction(Mask.getValueType()) ==
3606 GetSplitVector(Mask, LoMask, HiMask);
3607 else
3608 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
3609
3610 SDValue LoEVL, HiEVL;
3611 std::tie(LoEVL, HiEVL) =
3612 DAG.SplitEVL(N->getVectorLength(), Data.getValueType(), DL);
3613
3614 // Generate the low vp_strided_store
3615 SDValue Lo = DAG.getStridedStoreVP(
3616 N->getChain(), DL, LoData, N->getBasePtr(), N->getOffset(),
3617 N->getStride(), LoMask, LoEVL, LoMemVT, N->getMemOperand(),
3618 N->getAddressingMode(), N->isTruncatingStore(), N->isCompressingStore());
3619
3620 // If the high vp_strided_store has zero storage size, only the low
3621 // vp_strided_store is needed.
3622 if (HiIsEmpty)
3623 return Lo;
3624
3625 // Generate the high vp_strided_store.
3626 // To calculate the high base address, we need to sum to the low base
3627 // address stride number of bytes for each element already stored by low,
3628 // that is: Ptr = Ptr + (LoEVL * Stride)
3629 EVT PtrVT = N->getBasePtr().getValueType();
3631 DAG.getNode(ISD::MUL, DL, PtrVT, LoEVL,
3632 DAG.getSExtOrTrunc(N->getStride(), DL, PtrVT));
3633 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, N->getBasePtr(), Increment);
3634
3635 Align Alignment = N->getOriginalAlign();
3636 if (LoMemVT.isScalableVector())
3637 Alignment = commonAlignment(Alignment,
3638 LoMemVT.getSizeInBits().getKnownMinValue() / 8);
3639
3640 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3641 MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
3643 Alignment, N->getAAInfo(), N->getRanges());
3644
3645 SDValue Hi = DAG.getStridedStoreVP(
3646 N->getChain(), DL, HiData, Ptr, N->getOffset(), N->getStride(), HiMask,
3647 HiEVL, HiMemVT, MMO, N->getAddressingMode(), N->isTruncatingStore(),
3648 N->isCompressingStore());
3649
3650 // Build a factor node to remember that this store is independent of the
3651 // other one.
3652 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3653}
3654
3655SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
3656 unsigned OpNo) {
3657 assert(N->isUnindexed() && "Indexed masked store of vector?");
3658 SDValue Ch = N->getChain();
3659 SDValue Ptr = N->getBasePtr();
3660 SDValue Offset = N->getOffset();
3661 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
3662 SDValue Mask = N->getMask();
3663 SDValue Data = N->getValue();
3664 Align Alignment = N->getOriginalAlign();
3665 SDLoc DL(N);
3666
3667 SDValue DataLo, DataHi;
3668 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3669 // Split Data operand
3670 GetSplitVector(Data, DataLo, DataHi);
3671 else
3672 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
3673
3674 // Split Mask operand
3675 SDValue MaskLo, MaskHi;
3676 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
3677 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
3678 } else {
3679 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
3680 GetSplitVector(Mask, MaskLo, MaskHi);
3681 else
3682 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
3683 }
3684
3685 EVT MemoryVT = N->getMemoryVT();
3686 EVT LoMemVT, HiMemVT;
3687 bool HiIsEmpty = false;
3688 std::tie(LoMemVT, HiMemVT) =
3689 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
3690
3691 SDValue Lo, Hi, Res;
3692 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3693 N->getPointerInfo(), MachineMemOperand::MOStore,
3694 LocationSize::beforeOrAfterPointer(), Alignment, N->getAAInfo(),
3695 N->getRanges());
3696
3697 Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
3698 N->getAddressingMode(), N->isTruncatingStore(),
3699 N->isCompressingStore());
3700
3701 if (HiIsEmpty) {
3702 // The hi masked store has zero storage size.
3703 // Only the lo masked store is needed.
3704 Res = Lo;
3705 } else {
3706
3707 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
3708 N->isCompressingStore());
3709
3711 if (LoMemVT.isScalableVector()) {
3712 Alignment = commonAlignment(
3713 Alignment, LoMemVT.getSizeInBits().getKnownMinValue() / 8);
3714 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
3715 } else
3716 MPI = N->getPointerInfo().getWithOffset(
3717 LoMemVT.getStoreSize().getFixedValue());
3718
3719 MMO = DAG.getMachineFunction().getMachineMemOperand(
3721 Alignment, N->getAAInfo(), N->getRanges());
3722
3723 Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
3724 N->getAddressingMode(), N->isTruncatingStore(),
3725 N->isCompressingStore());
3726
3727 // Build a factor node to remember that this store is independent of the
3728 // other one.
3729 Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3730 }
3731
3732 return Res;
3733}
3734
3735SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
3736 SDValue Ch = N->getChain();
3737 SDValue Ptr = N->getBasePtr();
3738 EVT MemoryVT = N->getMemoryVT();
3739 Align Alignment = N->getOriginalAlign();
3740 SDLoc DL(N);
3741 struct Operands {
3742 SDValue Mask;
3743 SDValue Index;
3744 SDValue Scale;
3745 SDValue Data;
3746 } Ops = [&]() -> Operands {
3747 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
3748 return {MSC->getMask(), MSC->getIndex(), MSC->getScale(),
3749 MSC->getValue()};
3750 }
3751 auto *VPSC = cast<VPScatterSDNode>(N);
3752 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale(),
3753 VPSC->getValue()};
3754 }();
3755 // Split all operands
3756
3757 EVT LoMemVT, HiMemVT;
3758 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
3759
3760 SDValue DataLo, DataHi;
3761 if (getTypeAction(Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
3762 // Split Data operand
3763 GetSplitVector(Ops.Data, DataLo, DataHi);
3764 else
3765 std::tie(DataLo, DataHi) = DAG.SplitVector(Ops.Data, DL);
3766
3767 // Split Mask operand
3768 SDValue MaskLo, MaskHi;
3769 if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
3770 SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
3771 } else {
3772 std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, DL);
3773 }
3774
3775 SDValue IndexHi, IndexLo;
3776 if (getTypeAction(Ops.Index.getValueType()) ==
3778 GetSplitVector(Ops.Index, IndexLo, IndexHi);
3779 else
3780 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, DL);
3781
3782 SDValue Lo;
3783 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3784 N->getPointerInfo(), MachineMemOperand::MOStore,
3785 LocationSize::beforeOrAfterPointer(), Alignment, N->getAAInfo(),
3786 N->getRanges());
3787
3788 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
3789 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
3790 Lo =
3791 DAG.getMaskedScatter(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO,
3792 MSC->getIndexType(), MSC->isTruncatingStore());
3793
3794 // The order of the Scatter operation after split is well defined. The "Hi"
3795 // part comes after the "Lo". So these two operations should be chained one
3796 // after another.
3797 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
3798 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi,
3799 MMO, MSC->getIndexType(),
3800 MSC->isTruncatingStore());
3801 }
3802 auto *VPSC = cast<VPScatterSDNode>(N);
3803 SDValue EVLLo, EVLHi;
3804 std::tie(EVLLo, EVLHi) =
3805 DAG.SplitEVL(VPSC->getVectorLength(), Ops.Data.getValueType(), DL);
3806
3807 SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
3808 Lo = DAG.getScatterVP(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO,
3809 VPSC->getIndexType());
3810
3811 // The order of the Scatter operation after split is well defined. The "Hi"
3812 // part comes after the "Lo". So these two operations should be chained one
3813 // after another.
3814 SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
3815 return DAG.getScatterVP(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi, MMO,
3816 VPSC->getIndexType());
3817}
3818
3819SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
3820 assert(N->isUnindexed() && "Indexed store of vector?");
3821 assert(OpNo == 1 && "Can only split the stored value");
3822 SDLoc DL(N);
3823
3824 bool isTruncating = N->isTruncatingStore();
3825 SDValue Ch = N->getChain();
3826 SDValue Ptr = N->getBasePtr();
3827 EVT MemoryVT = N->getMemoryVT();
3828 Align Alignment = N->getOriginalAlign();
3829 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3830 AAMDNodes AAInfo = N->getAAInfo();
3831 SDValue Lo, Hi;
3832 GetSplitVector(N->getOperand(1), Lo, Hi);
3833
3834 EVT LoMemVT, HiMemVT;
3835 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
3836
3837 // Scalarize if the split halves are not byte-sized.
3838 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
3839 return TLI.scalarizeVectorStore(N, DAG);
3840
3841 if (isTruncating)
3842 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
3843 Alignment, MMOFlags, AAInfo);
3844 else
3845 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3846 AAInfo);
3847
3849 IncrementPointer(N, LoMemVT, MPI, Ptr);
3850
3851 if (isTruncating)
3852 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, MPI,
3853 HiMemVT, Alignment, MMOFlags, AAInfo);
3854 else
3855 Hi = DAG.getStore(Ch, DL, Hi, Ptr, MPI, Alignment, MMOFlags, AAInfo);
3856
3857 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3858}
3859
3860SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
3861 SDLoc DL(N);
3862
3863 // The input operands all must have the same type, and we know the result
3864 // type is valid. Convert this to a buildvector which extracts all the
3865 // input elements.
3866 // TODO: If the input elements are power-two vectors, we could convert this to
3867 // a new CONCAT_VECTORS node with elements that are half-wide.
3869 EVT EltVT = N->getValueType(0).getVectorElementType();
3870 for (const SDValue &Op : N->op_values()) {
3871 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
3872 i != e; ++i) {
3873 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
3874 DAG.getVectorIdxConstant(i, DL)));
3875 }
3876 }
3877
3878 return DAG.getBuildVector(N->getValueType(0), DL, Elts);
3879}
3880
3881SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
3882 // The result type is legal, but the input type is illegal. If splitting
3883 // ends up with the result type of each half still being legal, just
3884 // do that. If, however, that would result in an illegal result type,
3885 // we can try to get more clever with power-two vectors. Specifically,
3886 // split the input type, but also widen the result element size, then
3887 // concatenate the halves and truncate again. For example, consider a target
3888 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
3889 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
3890 // %inlo = v4i32 extract_subvector %in, 0
3891 // %inhi = v4i32 extract_subvector %in, 4
3892 // %lo16 = v4i16 trunc v4i32 %inlo
3893 // %hi16 = v4i16 trunc v4i32 %inhi
3894 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
3895 // %res = v8i8 trunc v8i16 %in16
3896 //
3897 // Without this transform, the original truncate would end up being
3898 // scalarized, which is pretty much always a last resort.
3899 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
3900 SDValue InVec = N->getOperand(OpNo);
3901 EVT InVT = InVec->getValueType(0);
3902 EVT OutVT = N->getValueType(0);
3903 ElementCount NumElements = OutVT.getVectorElementCount();
3904 bool IsFloat = OutVT.isFloatingPoint();
3905
3906 unsigned InElementSize = InVT.getScalarSizeInBits();
3907 unsigned OutElementSize = OutVT.getScalarSizeInBits();
3908
3909 // Determine the split output VT. If its legal we can just split dirctly.
3910 EVT LoOutVT, HiOutVT;
3911 std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
3912 assert(LoOutVT == HiOutVT && "Unequal split?");
3913
3914 // If the input elements are only 1/2 the width of the result elements,
3915 // just use the normal splitting. Our trick only work if there's room
3916 // to split more than once.
3917 if (isTypeLegal(LoOutVT) ||
3918 InElementSize <= OutElementSize * 2)
3919 return SplitVecOp_UnaryOp(N);
3920 SDLoc DL(N);
3921
3922 // Don't touch if this will be scalarized.
3923 EVT FinalVT = InVT;
3924 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
3925 FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
3926
3927 if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector)
3928 return SplitVecOp_UnaryOp(N);
3929
3930 // Get the split input vector.
3931 SDValue InLoVec, InHiVec;
3932 GetSplitVector(InVec, InLoVec, InHiVec);
3933
3934 // Truncate them to 1/2 the element size.
3935 //
3936 // This assumes the number of elements is a power of two; any vector that
3937 // isn't should be widened, not split.
3938 EVT HalfElementVT = IsFloat ?
3939 EVT::getFloatingPointVT(InElementSize/2) :
3940 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
3941 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
3942 NumElements.divideCoefficientBy(2));
3943
3944 SDValue HalfLo;
3945 SDValue HalfHi;
3946 SDValue Chain;
3947 if (N->isStrictFPOpcode()) {
3948 HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
3949 {N->getOperand(0), InLoVec});
3950 HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
3951 {N->getOperand(0), InHiVec});
3952 // Legalize the chain result - switch anything that used the old chain to
3953 // use the new one.
3954 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1),
3955 HalfHi.getValue(1));
3956 } else {
3957 HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
3958 HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
3959 }
3960
3961 // Concatenate them to get the full intermediate truncation result.
3962 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
3963 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
3964 HalfHi);
3965 // Now finish up by truncating all the way down to the original result
3966 // type. This should normally be something that ends up being legal directly,
3967 // but in theory if a target has very wide vectors and an annoyingly
3968 // restricted set of legal types, this split can chain to build things up.
3969
3970 if (N->isStrictFPOpcode()) {
3971 SDValue Res = DAG.getNode(
3972 ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other},
3973 {Chain, InterVec,
3974 DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))});
3975 // Relink the chain
3976 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
3977 return Res;
3978 }
3979
3980 return IsFloat
3981 ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
3982 DAG.getTargetConstant(
3983 0, DL, TLI.getPointerTy(DAG.getDataLayout())))
3984 : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
3985}
3986
3987SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
3988 bool isStrict = N->getOpcode() == ISD::STRICT_FSETCC;
3989 assert(N->getValueType(0).isVector() &&
3990 N->getOperand(isStrict ? 1 : 0).getValueType().isVector() &&
3991 "Operand types must be vectors");
3992 // The result has a legal vector type, but the input needs splitting.
3993 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
3994 SDLoc DL(N);
3995 GetSplitVector(N->getOperand(isStrict ? 1 : 0), Lo0, Hi0);
3996 GetSplitVector(N->getOperand(isStrict ? 2 : 1), Lo1, Hi1);
3997
3998 auto PartEltCnt = Lo0.getValueType().getVectorElementCount();
3999
4000 LLVMContext &Context = *DAG.getContext();
4001 EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt);
4002 EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2);
4003
4004 if (N->getOpcode() == ISD::SETCC) {
4005 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
4006 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
4007 } else if (N->getOpcode() == ISD::STRICT_FSETCC) {
4008 LoRes = DAG.getNode(ISD::STRICT_FSETCC, DL,
4009 DAG.getVTList(PartResVT, N->getValueType(1)),
4010 N->getOperand(0), Lo0, Lo1, N->getOperand(3));
4011 HiRes = DAG.getNode(ISD::STRICT_FSETCC, DL,
4012 DAG.getVTList(PartResVT, N->getValueType(1)),
4013 N->getOperand(0), Hi0, Hi1, N->getOperand(3));
4014 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
4015 LoRes.getValue(1), HiRes.getValue(1));
4016 ReplaceValueWith(SDValue(N, 1), NewChain);
4017 } else {
4018 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
4019 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
4020 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
4021 std::tie(EVLLo, EVLHi) =
4022 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), DL);
4023 LoRes = DAG.getNode(ISD::VP_SETCC, DL, PartResVT, Lo0, Lo1,
4024 N->getOperand(2), MaskLo, EVLLo);
4025 HiRes = DAG.getNode(ISD::VP_SETCC, DL, PartResVT, Hi0, Hi1,
4026 N->getOperand(2), MaskHi, EVLHi);
4027 }
4028 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
4029
4030 EVT OpVT = N->getOperand(0).getValueType();
4031 ISD::NodeType ExtendCode =
4033 return DAG.getNode(ExtendCode, DL, N->getValueType(0), Con);
4034}
4035
4036
4037SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
4038 // The result has a legal vector type, but the input needs splitting.
4039 EVT ResVT = N->getValueType(0);
4040 SDValue Lo, Hi;
4041 SDLoc DL(N);
4042 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
4043 EVT InVT = Lo.getValueType();
4044
4045 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
4046 InVT.getVectorElementCount());
4047
4048 if (N->isStrictFPOpcode()) {
4049 Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
4050 { N->getOperand(0), Lo, N->getOperand(2) });
4051 Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
4052 { N->getOperand(0), Hi, N->getOperand(2) });
4053 // Legalize the chain result - switch anything that used the old chain to
4054 // use the new one.
4055 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
4056 Lo.getValue(1), Hi.getValue(1));
4057 ReplaceValueWith(SDValue(N, 1), NewChain);
4058 } else if (N->getOpcode() == ISD::VP_FP_ROUND) {
4059 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
4060 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
4061 std::tie(EVLLo, EVLHi) =
4062 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), DL);
4063 Lo = DAG.getNode(ISD::VP_FP_ROUND, DL, OutVT, Lo, MaskLo, EVLLo);
4064 Hi = DAG.getNode(ISD::VP_FP_ROUND, DL, OutVT, Hi, MaskHi, EVLHi);
4065 } else {
4066 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
4067 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
4068 }
4069
4070 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
4071}
4072
4073// Split a vector type in an FP binary operation where the second operand has a
4074// different type from the first.
4075//
4076// The result (and the first input) has a legal vector type, but the second
4077// input needs splitting.
4078SDValue DAGTypeLegalizer::SplitVecOp_FPOpDifferentTypes(SDNode *N) {
4079 SDLoc DL(N);
4080
4081 EVT LHSLoVT, LHSHiVT;
4082 std::tie(LHSLoVT, LHSHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
4083
4084 if (!isTypeLegal(LHSLoVT) || !isTypeLegal(LHSHiVT))
4085 return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
4086
4087 SDValue LHSLo, LHSHi;
4088 std::tie(LHSLo, LHSHi) =
4089 DAG.SplitVector(N->getOperand(0), DL, LHSLoVT, LHSHiVT);
4090
4091 SDValue RHSLo, RHSHi;
4092 std::tie(RHSLo, RHSHi) = DAG.SplitVector(N->getOperand(1), DL);
4093
4094 SDValue Lo = DAG.getNode(N->getOpcode(), DL, LHSLoVT, LHSLo, RHSLo);
4095 SDValue Hi = DAG.getNode(N->getOpcode(), DL, LHSHiVT, LHSHi, RHSHi);
4096
4097 return DAG.getNode(ISD::CONCAT_VECTORS, DL, N->getValueType(0), Lo, Hi);
4098}
4099
4100SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
4101 EVT ResVT = N->getValueType(0);
4102 SDValue Lo, Hi;
4103 SDLoc dl(N);
4104 GetSplitVector(N->getOperand(0), Lo, Hi);
4105 EVT InVT = Lo.getValueType();
4106
4107 EVT NewResVT =
4108 EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
4109 InVT.getVectorElementCount());
4110
4111 Lo = DAG.getNode(N->getOpcode(), dl, NewResVT, Lo, N->getOperand(1));
4112 Hi = DAG.getNode(N->getOpcode(), dl, NewResVT, Hi, N->getOperand(1));
4113
4114 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
4115}
4116
4117SDValue DAGTypeLegalizer::SplitVecOp_VP_CttzElements(SDNode *N) {