LLVM 17.0.0git
LegalizeVectorTypes.cpp
Go to the documentation of this file.
1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file performs vector type splitting and scalarization for LegalizeTypes.
10// Scalarization is the act of changing a computation in an illegal one-element
11// vector type to be a computation in its scalar element type. For example,
12// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13// as a base case when scalarizing vector arithmetic like <4 x f32>, which
14// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15// types.
16// Splitting is the act of changing a computation in an invalid vector type to
17// be a computation in two vectors of half the size. For example, implementing
18// <128 x f32> operations in terms of two <64 x f32> operations.
19//
20//===----------------------------------------------------------------------===//
21
22#include "LegalizeTypes.h"
26#include "llvm/IR/DataLayout.h"
30#include <numeric>
31
32using namespace llvm;
33
34#define DEBUG_TYPE "legalize-types"
35
36//===----------------------------------------------------------------------===//
37// Result Vector Scalarization: <1 x ty> -> ty.
38//===----------------------------------------------------------------------===//
39
40void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
41 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
42 dbgs() << "\n");
43 SDValue R = SDValue();
44
45 switch (N->getOpcode()) {
46 default:
47#ifndef NDEBUG
48 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
49 N->dump(&DAG);
50 dbgs() << "\n";
51#endif
52 report_fatal_error("Do not know how to scalarize the result of this "
53 "operator!\n");
54
55 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
56 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
57 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
58 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
59 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
60 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
61 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
62 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
63 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
64 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
65 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
66 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
67 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
68 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
69 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
70 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
71 case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
75 R = ScalarizeVecRes_VecInregOp(N);
76 break;
77 case ISD::ABS:
78 case ISD::ANY_EXTEND:
79 case ISD::BITREVERSE:
80 case ISD::BSWAP:
81 case ISD::CTLZ:
83 case ISD::CTPOP:
84 case ISD::CTTZ:
86 case ISD::FABS:
87 case ISD::FCEIL:
88 case ISD::FCOS:
89 case ISD::FEXP:
90 case ISD::FEXP2:
91 case ISD::FFLOOR:
92 case ISD::FLOG:
93 case ISD::FLOG10:
94 case ISD::FLOG2:
95 case ISD::FNEARBYINT:
96 case ISD::FNEG:
97 case ISD::FREEZE:
99 case ISD::FP_EXTEND:
100 case ISD::FP_TO_SINT:
101 case ISD::FP_TO_UINT:
102 case ISD::FRINT:
103 case ISD::FROUND:
104 case ISD::FROUNDEVEN:
105 case ISD::FSIN:
106 case ISD::FSQRT:
107 case ISD::FTRUNC:
108 case ISD::SIGN_EXTEND:
109 case ISD::SINT_TO_FP:
110 case ISD::TRUNCATE:
111 case ISD::UINT_TO_FP:
112 case ISD::ZERO_EXTEND:
114 R = ScalarizeVecRes_UnaryOp(N);
115 break;
116
117 case ISD::ADD:
118 case ISD::AND:
119 case ISD::FADD:
120 case ISD::FCOPYSIGN:
121 case ISD::FDIV:
122 case ISD::FMUL:
123 case ISD::FMINNUM:
124 case ISD::FMAXNUM:
127 case ISD::FMINIMUM:
128 case ISD::FMAXIMUM:
129 case ISD::SMIN:
130 case ISD::SMAX:
131 case ISD::UMIN:
132 case ISD::UMAX:
133
134 case ISD::SADDSAT:
135 case ISD::UADDSAT:
136 case ISD::SSUBSAT:
137 case ISD::USUBSAT:
138 case ISD::SSHLSAT:
139 case ISD::USHLSAT:
140
141 case ISD::FPOW:
142 case ISD::FREM:
143 case ISD::FSUB:
144 case ISD::MUL:
145 case ISD::OR:
146 case ISD::SDIV:
147 case ISD::SREM:
148 case ISD::SUB:
149 case ISD::UDIV:
150 case ISD::UREM:
151 case ISD::XOR:
152 case ISD::SHL:
153 case ISD::SRA:
154 case ISD::SRL:
155 case ISD::ROTL:
156 case ISD::ROTR:
157 R = ScalarizeVecRes_BinOp(N);
158 break;
159 case ISD::FMA:
160 case ISD::FSHL:
161 case ISD::FSHR:
162 R = ScalarizeVecRes_TernaryOp(N);
163 break;
164
165#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
166 case ISD::STRICT_##DAGN:
167#include "llvm/IR/ConstrainedOps.def"
168 R = ScalarizeVecRes_StrictFPOp(N);
169 break;
170
173 R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
174 break;
175
176 case ISD::UADDO:
177 case ISD::SADDO:
178 case ISD::USUBO:
179 case ISD::SSUBO:
180 case ISD::UMULO:
181 case ISD::SMULO:
182 R = ScalarizeVecRes_OverflowOp(N, ResNo);
183 break;
184 case ISD::SMULFIX:
185 case ISD::SMULFIXSAT:
186 case ISD::UMULFIX:
187 case ISD::UMULFIXSAT:
188 case ISD::SDIVFIX:
189 case ISD::SDIVFIXSAT:
190 case ISD::UDIVFIX:
191 case ISD::UDIVFIXSAT:
192 R = ScalarizeVecRes_FIX(N);
193 break;
194 }
195
196 // If R is null, the sub-method took care of registering the result.
197 if (R.getNode())
198 SetScalarizedVector(SDValue(N, ResNo), R);
199}
200
201SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
202 SDValue LHS = GetScalarizedVector(N->getOperand(0));
203 SDValue RHS = GetScalarizedVector(N->getOperand(1));
204 return DAG.getNode(N->getOpcode(), SDLoc(N),
205 LHS.getValueType(), LHS, RHS, N->getFlags());
206}
207
208SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
209 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
210 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
211 SDValue Op2 = GetScalarizedVector(N->getOperand(2));
212 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
213 Op2, N->getFlags());
214}
215
216SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
217 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
218 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
219 SDValue Op2 = N->getOperand(2);
220 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
221 Op2, N->getFlags());
222}
223
224SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
225 EVT VT = N->getValueType(0).getVectorElementType();
226 unsigned NumOpers = N->getNumOperands();
227 SDValue Chain = N->getOperand(0);
228 EVT ValueVTs[] = {VT, MVT::Other};
229 SDLoc dl(N);
230
231 SmallVector<SDValue, 4> Opers(NumOpers);
232
233 // The Chain is the first operand.
234 Opers[0] = Chain;
235
236 // Now process the remaining operands.
237 for (unsigned i = 1; i < NumOpers; ++i) {
238 SDValue Oper = N->getOperand(i);
239 EVT OperVT = Oper.getValueType();
240
241 if (OperVT.isVector()) {
242 if (getTypeAction(OperVT) == TargetLowering::TypeScalarizeVector)
243 Oper = GetScalarizedVector(Oper);
244 else
245 Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
246 OperVT.getVectorElementType(), Oper,
247 DAG.getVectorIdxConstant(0, dl));
248 }
249
250 Opers[i] = Oper;
251 }
252
253 SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs),
254 Opers, N->getFlags());
255
256 // Legalize the chain result - switch anything that used the old chain to
257 // use the new one.
258 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
259 return Result;
260}
261
262SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
263 unsigned ResNo) {
264 SDLoc DL(N);
265 EVT ResVT = N->getValueType(0);
266 EVT OvVT = N->getValueType(1);
267
268 SDValue ScalarLHS, ScalarRHS;
269 if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
270 ScalarLHS = GetScalarizedVector(N->getOperand(0));
271 ScalarRHS = GetScalarizedVector(N->getOperand(1));
272 } else {
273 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
274 DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
275 DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
276 ScalarLHS = ElemsLHS[0];
277 ScalarRHS = ElemsRHS[0];
278 }
279
280 SDVTList ScalarVTs = DAG.getVTList(
282 SDNode *ScalarNode = DAG.getNode(
283 N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
284 ScalarNode->setFlags(N->getFlags());
285
286 // Replace the other vector result not being explicitly scalarized here.
287 unsigned OtherNo = 1 - ResNo;
288 EVT OtherVT = N->getValueType(OtherNo);
289 if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
290 SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
291 } else {
292 SDValue OtherVal = DAG.getNode(
293 ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
294 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
295 }
296
297 return SDValue(ScalarNode, ResNo);
298}
299
300SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
301 unsigned ResNo) {
302 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
303 return GetScalarizedVector(Op);
304}
305
306SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
307 SDValue Op = N->getOperand(0);
308 if (Op.getValueType().isVector()
309 && Op.getValueType().getVectorNumElements() == 1
310 && !isSimpleLegalType(Op.getValueType()))
311 Op = GetScalarizedVector(Op);
312 EVT NewVT = N->getValueType(0).getVectorElementType();
313 return DAG.getNode(ISD::BITCAST, SDLoc(N),
314 NewVT, Op);
315}
316
317SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
318 EVT EltVT = N->getValueType(0).getVectorElementType();
319 SDValue InOp = N->getOperand(0);
320 // The BUILD_VECTOR operands may be of wider element types and
321 // we may need to truncate them back to the requested return type.
322 if (EltVT.isInteger())
323 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
324 return InOp;
325}
326
327SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
329 N->getValueType(0).getVectorElementType(),
330 N->getOperand(0), N->getOperand(1));
331}
332
333SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
334 SDLoc DL(N);
335 SDValue Op = N->getOperand(0);
336 EVT OpVT = Op.getValueType();
337 // The result needs scalarizing, but it's not a given that the source does.
338 // See similar logic in ScalarizeVecRes_UnaryOp.
339 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
340 Op = GetScalarizedVector(Op);
341 } else {
342 EVT VT = OpVT.getVectorElementType();
343 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
344 DAG.getVectorIdxConstant(0, DL));
345 }
346 return DAG.getNode(ISD::FP_ROUND, DL,
347 N->getValueType(0).getVectorElementType(), Op,
348 N->getOperand(1));
349}
350
351SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
352 SDValue Op = GetScalarizedVector(N->getOperand(0));
353 return DAG.getNode(ISD::FPOWI, SDLoc(N),
354 Op.getValueType(), Op, N->getOperand(1));
355}
356
357SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
358 // The value to insert may have a wider type than the vector element type,
359 // so be sure to truncate it to the element type if necessary.
360 SDValue Op = N->getOperand(1);
361 EVT EltVT = N->getValueType(0).getVectorElementType();
362 if (Op.getValueType() != EltVT)
363 // FIXME: Can this happen for floating point types?
364 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
365 return Op;
366}
367
368SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
369 assert(N->isUnindexed() && "Indexed vector load?");
370
371 SDValue Result = DAG.getLoad(
372 ISD::UNINDEXED, N->getExtensionType(),
373 N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
374 N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
375 N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
376 N->getOriginalAlign(), N->getMemOperand()->getFlags(), N->getAAInfo());
377
378 // Legalize the chain result - switch anything that used the old chain to
379 // use the new one.
380 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
381 return Result;
382}
383
384SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
385 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
386 EVT DestVT = N->getValueType(0).getVectorElementType();
387 SDValue Op = N->getOperand(0);
388 EVT OpVT = Op.getValueType();
389 SDLoc DL(N);
390 // The result needs scalarizing, but it's not a given that the source does.
391 // This is a workaround for targets where it's impossible to scalarize the
392 // result of a conversion, because the source type is legal.
393 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
394 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
395 // legal and was not scalarized.
396 // See the similar logic in ScalarizeVecRes_SETCC
397 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
398 Op = GetScalarizedVector(Op);
399 } else {
400 EVT VT = OpVT.getVectorElementType();
401 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
402 DAG.getVectorIdxConstant(0, DL));
403 }
404 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op, N->getFlags());
405}
406
407SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
408 EVT EltVT = N->getValueType(0).getVectorElementType();
409 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
410 SDValue LHS = GetScalarizedVector(N->getOperand(0));
411 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
412 LHS, DAG.getValueType(ExtVT));
413}
414
415SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
416 SDLoc DL(N);
417 SDValue Op = N->getOperand(0);
418
419 EVT OpVT = Op.getValueType();
420 EVT OpEltVT = OpVT.getVectorElementType();
421 EVT EltVT = N->getValueType(0).getVectorElementType();
422
423 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
424 Op = GetScalarizedVector(Op);
425 } else {
426 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
427 DAG.getVectorIdxConstant(0, DL));
428 }
429
430 switch (N->getOpcode()) {
432 return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
434 return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
436 return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
437 }
438
439 llvm_unreachable("Illegal extend_vector_inreg opcode");
440}
441
442SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
443 // If the operand is wider than the vector element type then it is implicitly
444 // truncated. Make that explicit here.
445 EVT EltVT = N->getValueType(0).getVectorElementType();
446 SDValue InOp = N->getOperand(0);
447 if (InOp.getValueType() != EltVT)
448 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
449 return InOp;
450}
451
452SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
453 SDValue Cond = N->getOperand(0);
454 EVT OpVT = Cond.getValueType();
455 SDLoc DL(N);
456 // The vselect result and true/value operands needs scalarizing, but it's
457 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
458 // See the similar logic in ScalarizeVecRes_SETCC
459 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
460 Cond = GetScalarizedVector(Cond);
461 } else {
462 EVT VT = OpVT.getVectorElementType();
464 DAG.getVectorIdxConstant(0, DL));
465 }
466
467 SDValue LHS = GetScalarizedVector(N->getOperand(1));
469 TLI.getBooleanContents(false, false);
470 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
471
472 // If integer and float booleans have different contents then we can't
473 // reliably optimize in all cases. There is a full explanation for this in
474 // DAGCombiner::visitSELECT() where the same issue affects folding
475 // (select C, 0, 1) to (xor C, 1).
476 if (TLI.getBooleanContents(false, false) !=
477 TLI.getBooleanContents(false, true)) {
478 // At least try the common case where the boolean is generated by a
479 // comparison.
480 if (Cond->getOpcode() == ISD::SETCC) {
481 EVT OpVT = Cond->getOperand(0).getValueType();
482 ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
483 VecBool = TLI.getBooleanContents(OpVT);
484 } else
486 }
487
488 EVT CondVT = Cond.getValueType();
489 if (ScalarBool != VecBool) {
490 switch (ScalarBool) {
492 break;
496 // Vector read from all ones, scalar expects a single 1 so mask.
497 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
498 Cond, DAG.getConstant(1, SDLoc(N), CondVT));
499 break;
503 // Vector reads from a one, scalar from all ones so sign extend.
504 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
506 break;
507 }
508 }
509
510 // Truncate the condition if needed
511 auto BoolVT = getSetCCResultType(CondVT);
512 if (BoolVT.bitsLT(CondVT))
513 Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
514
515 return DAG.getSelect(SDLoc(N),
516 LHS.getValueType(), Cond, LHS,
517 GetScalarizedVector(N->getOperand(2)));
518}
519
520SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
521 SDValue LHS = GetScalarizedVector(N->getOperand(1));
522 return DAG.getSelect(SDLoc(N),
523 LHS.getValueType(), N->getOperand(0), LHS,
524 GetScalarizedVector(N->getOperand(2)));
525}
526
527SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
528 SDValue LHS = GetScalarizedVector(N->getOperand(2));
529 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
530 N->getOperand(0), N->getOperand(1),
531 LHS, GetScalarizedVector(N->getOperand(3)),
532 N->getOperand(4));
533}
534
535SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
536 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
537}
538
539SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
540 // Figure out if the scalar is the LHS or RHS and return it.
541 SDValue Arg = N->getOperand(2).getOperand(0);
542 if (Arg.isUndef())
543 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
544 unsigned Op = !cast<ConstantSDNode>(Arg)->isZero();
545 return GetScalarizedVector(N->getOperand(Op));
546}
547
548SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
549 SDValue Src = N->getOperand(0);
550 EVT SrcVT = Src.getValueType();
551 SDLoc dl(N);
552
553 // Handle case where result is scalarized but operand is not
554 if (getTypeAction(SrcVT) == TargetLowering::TypeScalarizeVector)
555 Src = GetScalarizedVector(Src);
556 else
557 Src = DAG.getNode(
559 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
560
561 EVT DstVT = N->getValueType(0).getVectorElementType();
562 return DAG.getNode(N->getOpcode(), dl, DstVT, Src, N->getOperand(1));
563}
564
565SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
566 assert(N->getValueType(0).isVector() &&
567 N->getOperand(0).getValueType().isVector() &&
568 "Operand types must be vectors");
569 SDValue LHS = N->getOperand(0);
570 SDValue RHS = N->getOperand(1);
571 EVT OpVT = LHS.getValueType();
572 EVT NVT = N->getValueType(0).getVectorElementType();
573 SDLoc DL(N);
574
575 // The result needs scalarizing, but it's not a given that the source does.
576 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
577 LHS = GetScalarizedVector(LHS);
578 RHS = GetScalarizedVector(RHS);
579 } else {
580 EVT VT = OpVT.getVectorElementType();
581 LHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
582 DAG.getVectorIdxConstant(0, DL));
583 RHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
584 DAG.getVectorIdxConstant(0, DL));
585 }
586
587 // Turn it into a scalar SETCC.
588 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
589 N->getOperand(2));
590 // Vectors may have a different boolean contents to scalars. Promote the
591 // value appropriately.
592 ISD::NodeType ExtendCode =
594 return DAG.getNode(ExtendCode, DL, NVT, Res);
595}
596
597SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
598 SDLoc DL(N);
599 SDValue Arg = N->getOperand(0);
600 SDValue Test = N->getOperand(1);
601 EVT ArgVT = Arg.getValueType();
602 EVT ResultVT = N->getValueType(0).getVectorElementType();
603
604 if (getTypeAction(ArgVT) == TargetLowering::TypeScalarizeVector) {
605 Arg = GetScalarizedVector(Arg);
606 } else {
607 EVT VT = ArgVT.getVectorElementType();
609 DAG.getVectorIdxConstant(0, DL));
610 }
611
612 SDValue Res =
613 DAG.getNode(ISD::IS_FPCLASS, DL, MVT::i1, {Arg, Test}, N->getFlags());
614 // Vectors may have a different boolean contents to scalars. Promote the
615 // value appropriately.
616 ISD::NodeType ExtendCode =
618 return DAG.getNode(ExtendCode, DL, ResultVT, Res);
619}
620
621//===----------------------------------------------------------------------===//
622// Operand Vector Scalarization <1 x ty> -> ty.
623//===----------------------------------------------------------------------===//
624
625bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
626 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
627 dbgs() << "\n");
628 SDValue Res = SDValue();
629
630 switch (N->getOpcode()) {
631 default:
632#ifndef NDEBUG
633 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
634 N->dump(&DAG);
635 dbgs() << "\n";
636#endif
637 report_fatal_error("Do not know how to scalarize this operator's "
638 "operand!\n");
639 case ISD::BITCAST:
640 Res = ScalarizeVecOp_BITCAST(N);
641 break;
642 case ISD::ANY_EXTEND:
643 case ISD::ZERO_EXTEND:
644 case ISD::SIGN_EXTEND:
645 case ISD::TRUNCATE:
646 case ISD::FP_TO_SINT:
647 case ISD::FP_TO_UINT:
648 case ISD::SINT_TO_FP:
649 case ISD::UINT_TO_FP:
650 Res = ScalarizeVecOp_UnaryOp(N);
651 break;
656 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
657 break;
659 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
660 break;
662 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
663 break;
664 case ISD::VSELECT:
665 Res = ScalarizeVecOp_VSELECT(N);
666 break;
667 case ISD::SETCC:
668 Res = ScalarizeVecOp_VSETCC(N);
669 break;
670 case ISD::STORE:
671 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
672 break;
674 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
675 break;
676 case ISD::FP_ROUND:
677 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
678 break;
680 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
681 break;
682 case ISD::FP_EXTEND:
683 Res = ScalarizeVecOp_FP_EXTEND(N);
684 break;
698 Res = ScalarizeVecOp_VECREDUCE(N);
699 break;
702 Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
703 break;
704 }
705
706 // If the result is null, the sub-method took care of registering results etc.
707 if (!Res.getNode()) return false;
708
709 // If the result is N, the sub-method updated N in place. Tell the legalizer
710 // core about this.
711 if (Res.getNode() == N)
712 return true;
713
714 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
715 "Invalid operand expansion");
716
717 ReplaceValueWith(SDValue(N, 0), Res);
718 return false;
719}
720
721/// If the value to convert is a vector that needs to be scalarized, it must be
722/// <1 x ty>. Convert the element instead.
723SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
724 SDValue Elt = GetScalarizedVector(N->getOperand(0));
725 return DAG.getNode(ISD::BITCAST, SDLoc(N),
726 N->getValueType(0), Elt);
727}
728
729/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
730/// Do the operation on the element instead.
731SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
732 assert(N->getValueType(0).getVectorNumElements() == 1 &&
733 "Unexpected vector type!");
734 SDValue Elt = GetScalarizedVector(N->getOperand(0));
735 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
736 N->getValueType(0).getScalarType(), Elt);
737 // Revectorize the result so the types line up with what the uses of this
738 // expression expect.
739 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
740}
741
742/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
743/// Do the strict FP operation on the element instead.
744SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
745 assert(N->getValueType(0).getVectorNumElements() == 1 &&
746 "Unexpected vector type!");
747 SDValue Elt = GetScalarizedVector(N->getOperand(1));
748 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
749 { N->getValueType(0).getScalarType(), MVT::Other },
750 { N->getOperand(0), Elt });
751 // Legalize the chain result - switch anything that used the old chain to
752 // use the new one.
753 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
754 // Revectorize the result so the types line up with what the uses of this
755 // expression expect.
756 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
757
758 // Do our own replacement and return SDValue() to tell the caller that we
759 // handled all replacements since caller can only handle a single result.
760 ReplaceValueWith(SDValue(N, 0), Res);
761 return SDValue();
762}
763
764/// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
765SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
766 SmallVector<SDValue, 8> Ops(N->getNumOperands());
767 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
768 Ops[i] = GetScalarizedVector(N->getOperand(i));
769 return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
770}
771
772/// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
773/// so just return the element, ignoring the index.
774SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
775 EVT VT = N->getValueType(0);
776 SDValue Res = GetScalarizedVector(N->getOperand(0));
777 if (Res.getValueType() != VT)
778 Res = VT.isFloatingPoint()
779 ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
780 : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
781 return Res;
782}
783
784/// If the input condition is a vector that needs to be scalarized, it must be
785/// <1 x i1>, so just convert to a normal ISD::SELECT
786/// (still with vector output type since that was acceptable if we got here).
787SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
788 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
789 EVT VT = N->getValueType(0);
790
791 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
792 N->getOperand(2));
793}
794
795/// If the operand is a vector that needs to be scalarized then the
796/// result must be v1i1, so just convert to a scalar SETCC and wrap
797/// with a scalar_to_vector since the res type is legal if we got here
798SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
799 assert(N->getValueType(0).isVector() &&
800 N->getOperand(0).getValueType().isVector() &&
801 "Operand types must be vectors");
802 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
803
804 EVT VT = N->getValueType(0);
805 SDValue LHS = GetScalarizedVector(N->getOperand(0));
806 SDValue RHS = GetScalarizedVector(N->getOperand(1));
807
808 EVT OpVT = N->getOperand(0).getValueType();
809 EVT NVT = VT.getVectorElementType();
810 SDLoc DL(N);
811 // Turn it into a scalar SETCC.
812 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
813 N->getOperand(2));
814
815 // Vectors may have a different boolean contents to scalars. Promote the
816 // value appropriately.
817 ISD::NodeType ExtendCode =
819
820 Res = DAG.getNode(ExtendCode, DL, NVT, Res);
821
822 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
823}
824
825/// If the value to store is a vector that needs to be scalarized, it must be
826/// <1 x ty>. Just store the element.
827SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
828 assert(N->isUnindexed() && "Indexed store of one-element vector?");
829 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
830 SDLoc dl(N);
831
832 if (N->isTruncatingStore())
833 return DAG.getTruncStore(
834 N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
835 N->getBasePtr(), N->getPointerInfo(),
836 N->getMemoryVT().getVectorElementType(), N->getOriginalAlign(),
837 N->getMemOperand()->getFlags(), N->getAAInfo());
838
839 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
840 N->getBasePtr(), N->getPointerInfo(),
841 N->getOriginalAlign(), N->getMemOperand()->getFlags(),
842 N->getAAInfo());
843}
844
845/// If the value to round is a vector that needs to be scalarized, it must be
846/// <1 x ty>. Convert the element instead.
847SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
848 assert(OpNo == 0 && "Wrong operand for scalarization!");
849 SDValue Elt = GetScalarizedVector(N->getOperand(0));
850 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
851 N->getValueType(0).getVectorElementType(), Elt,
852 N->getOperand(1));
853 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
854}
855
856SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
857 unsigned OpNo) {
858 assert(OpNo == 1 && "Wrong operand for scalarization!");
859 SDValue Elt = GetScalarizedVector(N->getOperand(1));
861 { N->getValueType(0).getVectorElementType(),
862 MVT::Other },
863 { N->getOperand(0), Elt, N->getOperand(2) });
864 // Legalize the chain result - switch anything that used the old chain to
865 // use the new one.
866 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
867
868 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
869
870 // Do our own replacement and return SDValue() to tell the caller that we
871 // handled all replacements since caller can only handle a single result.
872 ReplaceValueWith(SDValue(N, 0), Res);
873 return SDValue();
874}
875
876/// If the value to extend is a vector that needs to be scalarized, it must be
877/// <1 x ty>. Convert the element instead.
878SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
879 SDValue Elt = GetScalarizedVector(N->getOperand(0));
881 N->getValueType(0).getVectorElementType(), Elt);
882 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
883}
884
885/// If the value to extend is a vector that needs to be scalarized, it must be
886/// <1 x ty>. Convert the element instead.
887SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
888 SDValue Elt = GetScalarizedVector(N->getOperand(1));
889 SDValue Res =
891 {N->getValueType(0).getVectorElementType(), MVT::Other},
892 {N->getOperand(0), Elt});
893 // Legalize the chain result - switch anything that used the old chain to
894 // use the new one.
895 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
896
897 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
898
899 // Do our own replacement and return SDValue() to tell the caller that we
900 // handled all replacements since caller can only handle a single result.
901 ReplaceValueWith(SDValue(N, 0), Res);
902 return SDValue();
903}
904
905SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
906 SDValue Res = GetScalarizedVector(N->getOperand(0));
907 // Result type may be wider than element type.
908 if (Res.getValueType() != N->getValueType(0))
909 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res);
910 return Res;
911}
912
913SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
914 SDValue AccOp = N->getOperand(0);
915 SDValue VecOp = N->getOperand(1);
916
917 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
918
919 SDValue Op = GetScalarizedVector(VecOp);
920 return DAG.getNode(BaseOpc, SDLoc(N), N->getValueType(0),
921 AccOp, Op, N->getFlags());
922}
923
924//===----------------------------------------------------------------------===//
925// Result Vector Splitting
926//===----------------------------------------------------------------------===//
927
928/// This method is called when the specified result of the specified node is
929/// found to need vector splitting. At this point, the node may also have
930/// invalid operands or may have other results that need legalization, we just
931/// know that (at least) one result needs vector splitting.
932void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
933 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
934 SDValue Lo, Hi;
935
936 // See if the target wants to custom expand this node.
937 if (CustomLowerNode(N, N->getValueType(ResNo), true))
938 return;
939
940 switch (N->getOpcode()) {
941 default:
942#ifndef NDEBUG
943 dbgs() << "SplitVectorResult #" << ResNo << ": ";
944 N->dump(&DAG);
945 dbgs() << "\n";
946#endif
947 report_fatal_error("Do not know how to split the result of this "
948 "operator!\n");
949
950 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
951 case ISD::VSELECT:
952 case ISD::SELECT:
953 case ISD::VP_MERGE:
954 case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break;
955 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
956 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
957 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
958 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
959 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
960 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
961 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
962 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
963 case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
964 case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
965 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
968 SplitVecRes_ScalarOp(N, Lo, Hi);
969 break;
970 case ISD::STEP_VECTOR:
971 SplitVecRes_STEP_VECTOR(N, Lo, Hi);
972 break;
973 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
974 case ISD::LOAD:
975 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
976 break;
977 case ISD::VP_LOAD:
978 SplitVecRes_VP_LOAD(cast<VPLoadSDNode>(N), Lo, Hi);
979 break;
980 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
981 SplitVecRes_VP_STRIDED_LOAD(cast<VPStridedLoadSDNode>(N), Lo, Hi);
982 break;
983 case ISD::MLOAD:
984 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
985 break;
986 case ISD::MGATHER:
987 case ISD::VP_GATHER:
988 SplitVecRes_Gather(cast<MemSDNode>(N), Lo, Hi, /*SplitSETCC*/ true);
989 break;
990 case ISD::SETCC:
991 case ISD::VP_SETCC:
992 SplitVecRes_SETCC(N, Lo, Hi);
993 break;
995 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
996 break;
998 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
999 break;
1000 case ISD::VECTOR_SPLICE:
1001 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
1002 break;
1003 case ISD::VAARG:
1004 SplitVecRes_VAARG(N, Lo, Hi);
1005 break;
1006
1010 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1011 break;
1012
1013 case ISD::ABS:
1014 case ISD::VP_ABS:
1015 case ISD::BITREVERSE:
1016 case ISD::VP_BITREVERSE:
1017 case ISD::BSWAP:
1018 case ISD::VP_BSWAP:
1019 case ISD::CTLZ:
1020 case ISD::VP_CTLZ:
1021 case ISD::CTTZ:
1022 case ISD::VP_CTTZ:
1024 case ISD::VP_CTLZ_ZERO_UNDEF:
1026 case ISD::VP_CTTZ_ZERO_UNDEF:
1027 case ISD::CTPOP:
1028 case ISD::VP_CTPOP:
1029 case ISD::FABS: case ISD::VP_FABS:
1030 case ISD::FCEIL:
1031 case ISD::VP_FCEIL:
1032 case ISD::FCOS:
1033 case ISD::FEXP:
1034 case ISD::FEXP2:
1035 case ISD::FFLOOR:
1036 case ISD::VP_FFLOOR:
1037 case ISD::FLOG:
1038 case ISD::FLOG10:
1039 case ISD::FLOG2:
1040 case ISD::FNEARBYINT:
1041 case ISD::VP_FNEARBYINT:
1042 case ISD::FNEG: case ISD::VP_FNEG:
1043 case ISD::FREEZE:
1044 case ISD::ARITH_FENCE:
1045 case ISD::FP_EXTEND:
1046 case ISD::VP_FP_EXTEND:
1047 case ISD::FP_ROUND:
1048 case ISD::VP_FP_ROUND:
1049 case ISD::FP_TO_SINT:
1050 case ISD::VP_FP_TO_SINT:
1051 case ISD::FP_TO_UINT:
1052 case ISD::VP_FP_TO_UINT:
1053 case ISD::FRINT:
1054 case ISD::VP_FRINT:
1055 case ISD::FROUND:
1056 case ISD::VP_FROUND:
1057 case ISD::FROUNDEVEN:
1058 case ISD::VP_FROUNDEVEN:
1059 case ISD::FSIN:
1060 case ISD::FSQRT: case ISD::VP_SQRT:
1061 case ISD::FTRUNC:
1062 case ISD::VP_FROUNDTOZERO:
1063 case ISD::SINT_TO_FP:
1064 case ISD::VP_SINT_TO_FP:
1065 case ISD::TRUNCATE:
1066 case ISD::VP_TRUNCATE:
1067 case ISD::UINT_TO_FP:
1068 case ISD::VP_UINT_TO_FP:
1069 case ISD::FCANONICALIZE:
1070 SplitVecRes_UnaryOp(N, Lo, Hi);
1071 break;
1072
1073 case ISD::ANY_EXTEND:
1074 case ISD::SIGN_EXTEND:
1075 case ISD::ZERO_EXTEND:
1076 case ISD::VP_SIGN_EXTEND:
1077 case ISD::VP_ZERO_EXTEND:
1078 SplitVecRes_ExtendOp(N, Lo, Hi);
1079 break;
1080
1081 case ISD::ADD: case ISD::VP_ADD:
1082 case ISD::SUB: case ISD::VP_SUB:
1083 case ISD::MUL: case ISD::VP_MUL:
1084 case ISD::MULHS:
1085 case ISD::MULHU:
1086 case ISD::FADD: case ISD::VP_FADD:
1087 case ISD::FSUB: case ISD::VP_FSUB:
1088 case ISD::FMUL: case ISD::VP_FMUL:
1089 case ISD::FMINNUM: case ISD::VP_FMINNUM:
1090 case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
1091 case ISD::FMINIMUM:
1092 case ISD::FMAXIMUM:
1093 case ISD::SDIV: case ISD::VP_SDIV:
1094 case ISD::UDIV: case ISD::VP_UDIV:
1095 case ISD::FDIV: case ISD::VP_FDIV:
1096 case ISD::FPOW:
1097 case ISD::AND: case ISD::VP_AND:
1098 case ISD::OR: case ISD::VP_OR:
1099 case ISD::XOR: case ISD::VP_XOR:
1100 case ISD::SHL: case ISD::VP_SHL:
1101 case ISD::SRA: case ISD::VP_ASHR:
1102 case ISD::SRL: case ISD::VP_LSHR:
1103 case ISD::UREM: case ISD::VP_UREM:
1104 case ISD::SREM: case ISD::VP_SREM:
1105 case ISD::FREM: case ISD::VP_FREM:
1106 case ISD::SMIN: case ISD::VP_SMIN:
1107 case ISD::SMAX: case ISD::VP_SMAX:
1108 case ISD::UMIN: case ISD::VP_UMIN:
1109 case ISD::UMAX: case ISD::VP_UMAX:
1110 case ISD::SADDSAT:
1111 case ISD::UADDSAT:
1112 case ISD::SSUBSAT:
1113 case ISD::USUBSAT:
1114 case ISD::SSHLSAT:
1115 case ISD::USHLSAT:
1116 case ISD::ROTL:
1117 case ISD::ROTR:
1118 case ISD::VP_FCOPYSIGN:
1119 SplitVecRes_BinOp(N, Lo, Hi);
1120 break;
1121 case ISD::FMA: case ISD::VP_FMA:
1122 case ISD::FSHL:
1123 case ISD::VP_FSHL:
1124 case ISD::FSHR:
1125 case ISD::VP_FSHR:
1126 SplitVecRes_TernaryOp(N, Lo, Hi);
1127 break;
1128
1129#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1130 case ISD::STRICT_##DAGN:
1131#include "llvm/IR/ConstrainedOps.def"
1132 SplitVecRes_StrictFPOp(N, Lo, Hi);
1133 break;
1134
1137 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1138 break;
1139
1140 case ISD::UADDO:
1141 case ISD::SADDO:
1142 case ISD::USUBO:
1143 case ISD::SSUBO:
1144 case ISD::UMULO:
1145 case ISD::SMULO:
1146 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1147 break;
1148 case ISD::SMULFIX:
1149 case ISD::SMULFIXSAT:
1150 case ISD::UMULFIX:
1151 case ISD::UMULFIXSAT:
1152 case ISD::SDIVFIX:
1153 case ISD::SDIVFIXSAT:
1154 case ISD::UDIVFIX:
1155 case ISD::UDIVFIXSAT:
1156 SplitVecRes_FIX(N, Lo, Hi);
1157 break;
1158 }
1159
1160 // If Lo/Hi is null, the sub-method took care of registering results etc.
1161 if (Lo.getNode())
1162 SetSplitVector(SDValue(N, ResNo), Lo, Hi);
1163}
1164
1165void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1167 uint64_t *ScaledOffset) {
1168 SDLoc DL(N);
1169 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8;
1170
1171 if (MemVT.isScalableVector()) {
1173 SDValue BytesIncrement = DAG.getVScale(
1174 DL, Ptr.getValueType(),
1175 APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
1176 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1177 Flags.setNoUnsignedWrap(true);
1178 if (ScaledOffset)
1179 *ScaledOffset += IncrementSize;
1180 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, BytesIncrement,
1181 Flags);
1182 } else {
1183 MPI = N->getPointerInfo().getWithOffset(IncrementSize);
1184 // Increment the pointer to the other half.
1185 Ptr = DAG.getObjectPtrOffset(DL, Ptr, TypeSize::Fixed(IncrementSize));
1186 }
1187}
1188
1189std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1190 return SplitMask(Mask, SDLoc(Mask));
1191}
1192
1193std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1194 const SDLoc &DL) {
1195 SDValue MaskLo, MaskHi;
1196 EVT MaskVT = Mask.getValueType();
1197 if (getTypeAction(MaskVT) == TargetLowering::TypeSplitVector)
1198 GetSplitVector(Mask, MaskLo, MaskHi);
1199 else
1200 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
1201 return std::make_pair(MaskLo, MaskHi);
1202}
1203
1204void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1205 SDValue LHSLo, LHSHi;
1206 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1207 SDValue RHSLo, RHSHi;
1208 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1209 SDLoc dl(N);
1210
1211 const SDNodeFlags Flags = N->getFlags();
1212 unsigned Opcode = N->getOpcode();
1213 if (N->getNumOperands() == 2) {
1214 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
1215 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
1216 return;
1217 }
1218
1219 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1220 assert(N->isVPOpcode() && "Expected VP opcode");
1221
1222 SDValue MaskLo, MaskHi;
1223 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2));
1224
1225 SDValue EVLLo, EVLHi;
1226 std::tie(EVLLo, EVLHi) =
1227 DAG.SplitEVL(N->getOperand(3), N->getValueType(0), dl);
1228
1229 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(),
1230 {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1231 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(),
1232 {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1233}
1234
1235void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1236 SDValue &Hi) {
1237 SDValue Op0Lo, Op0Hi;
1238 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
1239 SDValue Op1Lo, Op1Hi;
1240 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
1241 SDValue Op2Lo, Op2Hi;
1242 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
1243 SDLoc dl(N);
1244
1245 const SDNodeFlags Flags = N->getFlags();
1246 unsigned Opcode = N->getOpcode();
1247 if (N->getNumOperands() == 3) {
1248 Lo = DAG.getNode(Opcode, dl, Op0Lo.getValueType(), Op0Lo, Op1Lo, Op2Lo, Flags);
1249 Hi = DAG.getNode(Opcode, dl, Op0Hi.getValueType(), Op0Hi, Op1Hi, Op2Hi, Flags);
1250 return;
1251 }
1252
1253 assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
1254 assert(N->isVPOpcode() && "Expected VP opcode");
1255
1256 SDValue MaskLo, MaskHi;
1257 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
1258
1259 SDValue EVLLo, EVLHi;
1260 std::tie(EVLLo, EVLHi) =
1261 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), dl);
1262
1263 Lo = DAG.getNode(Opcode, dl, Op0Lo.getValueType(),
1264 {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags);
1265 Hi = DAG.getNode(Opcode, dl, Op0Hi.getValueType(),
1266 {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags);
1267}
1268
1269void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1270 SDValue LHSLo, LHSHi;
1271 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1272 SDValue RHSLo, RHSHi;
1273 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1274 SDLoc dl(N);
1275 SDValue Op2 = N->getOperand(2);
1276
1277 unsigned Opcode = N->getOpcode();
1278 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2,
1279 N->getFlags());
1280 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2,
1281 N->getFlags());
1282}
1283
1284void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1285 SDValue &Hi) {
1286 // We know the result is a vector. The input may be either a vector or a
1287 // scalar value.
1288 EVT LoVT, HiVT;
1289 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1290 SDLoc dl(N);
1291
1292 SDValue InOp = N->getOperand(0);
1293 EVT InVT = InOp.getValueType();
1294
1295 // Handle some special cases efficiently.
1296 switch (getTypeAction(InVT)) {
1304 break;
1307 // A scalar to vector conversion, where the scalar needs expansion.
1308 // If the vector is being split in two then we can just convert the
1309 // expanded pieces.
1310 if (LoVT == HiVT) {
1311 GetExpandedOp(InOp, Lo, Hi);
1312 if (DAG.getDataLayout().isBigEndian())
1313 std::swap(Lo, Hi);
1314 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1315 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1316 return;
1317 }
1318 break;
1320 // If the input is a vector that needs to be split, convert each split
1321 // piece of the input now.
1322 GetSplitVector(InOp, Lo, Hi);
1323 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1324 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1325 return;
1327 report_fatal_error("Scalarization of scalable vectors is not supported.");
1328 }
1329
1330 // In the general case, convert the input to an integer and split it by hand.
1331 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
1332 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
1333 if (DAG.getDataLayout().isBigEndian())
1334 std::swap(LoIntVT, HiIntVT);
1335
1336 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
1337
1338 if (DAG.getDataLayout().isBigEndian())
1339 std::swap(Lo, Hi);
1340 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1341 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1342}
1343
1344void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1345 SDValue &Hi) {
1346 EVT LoVT, HiVT;
1347 SDLoc dl(N);
1348 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1349 unsigned LoNumElts = LoVT.getVectorNumElements();
1350 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1351 Lo = DAG.getBuildVector(LoVT, dl, LoOps);
1352
1353 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1354 Hi = DAG.getBuildVector(HiVT, dl, HiOps);
1355}
1356
1357void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1358 SDValue &Hi) {
1359 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1360 SDLoc dl(N);
1361 unsigned NumSubvectors = N->getNumOperands() / 2;
1362 if (NumSubvectors == 1) {
1363 Lo = N->getOperand(0);
1364 Hi = N->getOperand(1);
1365 return;
1366 }
1367
1368 EVT LoVT, HiVT;
1369 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1370
1371 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1372 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
1373
1374 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1375 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
1376}
1377
1378void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1379 SDValue &Hi) {
1380 SDValue Vec = N->getOperand(0);
1381 SDValue Idx = N->getOperand(1);
1382 SDLoc dl(N);
1383
1384 EVT LoVT, HiVT;
1385 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1386
1387 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
1388 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1389 Hi = DAG.getNode(
1390 ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
1391 DAG.getVectorIdxConstant(IdxVal + LoVT.getVectorMinNumElements(), dl));
1392}
1393
1394void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1395 SDValue &Hi) {
1396 SDValue Vec = N->getOperand(0);
1397 SDValue SubVec = N->getOperand(1);
1398 SDValue Idx = N->getOperand(2);
1399 SDLoc dl(N);
1400 GetSplitVector(Vec, Lo, Hi);
1401
1402 EVT VecVT = Vec.getValueType();
1403 EVT LoVT = Lo.getValueType();
1404 EVT SubVecVT = SubVec.getValueType();
1405 unsigned VecElems = VecVT.getVectorMinNumElements();
1406 unsigned SubElems = SubVecVT.getVectorMinNumElements();
1407 unsigned LoElems = LoVT.getVectorMinNumElements();
1408
1409 // If we know the index is in the first half, and we know the subvector
1410 // doesn't cross the boundary between the halves, we can avoid spilling the
1411 // vector, and insert into the lower half of the split vector directly.
1412 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1413 if (IdxVal + SubElems <= LoElems) {
1414 Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1415 return;
1416 }
1417 // Similarly if the subvector is fully in the high half, but mind that we
1418 // can't tell whether a fixed-length subvector is fully within the high half
1419 // of a scalable vector.
1420 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1421 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1422 Hi = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, Hi.getValueType(), Hi, SubVec,
1423 DAG.getVectorIdxConstant(IdxVal - LoElems, dl));
1424 return;
1425 }
1426
1427 // Spill the vector to the stack.
1428 // In cases where the vector is illegal it will be broken down into parts
1429 // and stored in parts - we should use the alignment for the smallest part.
1430 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1432 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1433 auto &MF = DAG.getMachineFunction();
1434 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1435 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1436
1437 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1438 SmallestAlign);
1439
1440 // Store the new subvector into the specified index.
1441 SDValue SubVecPtr =
1442 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1443 Store = DAG.getStore(Store, dl, SubVec, SubVecPtr,
1445
1446 // Load the Lo part from the stack slot.
1447 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, PtrInfo,
1448 SmallestAlign);
1449
1450 // Increment the pointer to the other part.
1451 auto *Load = cast<LoadSDNode>(Lo);
1452 MachinePointerInfo MPI = Load->getPointerInfo();
1453 IncrementPointer(Load, LoVT, MPI, StackPtr);
1454
1455 // Load the Hi part from the stack slot.
1456 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MPI, SmallestAlign);
1457}
1458
1459void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1460 SDValue &Hi) {
1461 SDLoc dl(N);
1462 GetSplitVector(N->getOperand(0), Lo, Hi);
1463 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
1464 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
1465}
1466
1467void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1468 SDValue &Hi) {
1469 SDValue LHSLo, LHSHi;
1470 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1471 SDLoc DL(N);
1472
1473 SDValue RHSLo, RHSHi;
1474 SDValue RHS = N->getOperand(1);
1475 EVT RHSVT = RHS.getValueType();
1476 if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1477 GetSplitVector(RHS, RHSLo, RHSHi);
1478 else
1479 std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1480
1481
1482 Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
1483 Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
1484}
1485
1486void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
1487 SDValue &Hi) {
1488 SDLoc DL(N);
1489 SDValue ArgLo, ArgHi;
1490 SDValue Test = N->getOperand(1);
1491 SDValue FpValue = N->getOperand(0);
1492 if (getTypeAction(FpValue.getValueType()) == TargetLowering::TypeSplitVector)
1493 GetSplitVector(FpValue, ArgLo, ArgHi);
1494 else
1495 std::tie(ArgLo, ArgHi) = DAG.SplitVector(FpValue, SDLoc(FpValue));
1496 EVT LoVT, HiVT;
1497 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1498
1499 Lo = DAG.getNode(ISD::IS_FPCLASS, DL, LoVT, ArgLo, Test, N->getFlags());
1500 Hi = DAG.getNode(ISD::IS_FPCLASS, DL, HiVT, ArgHi, Test, N->getFlags());
1501}
1502
1503void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1504 SDValue &Hi) {
1505 SDValue LHSLo, LHSHi;
1506 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1507 SDLoc dl(N);
1508
1509 EVT LoVT, HiVT;
1510 std::tie(LoVT, HiVT) =
1511 DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1512
1513 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1514 DAG.getValueType(LoVT));
1515 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1516 DAG.getValueType(HiVT));
1517}
1518
1519void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1520 SDValue &Hi) {
1521 unsigned Opcode = N->getOpcode();
1522 SDValue N0 = N->getOperand(0);
1523
1524 SDLoc dl(N);
1525 SDValue InLo, InHi;
1526
1527 if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1528 GetSplitVector(N0, InLo, InHi);
1529 else
1530 std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1531
1532 EVT InLoVT = InLo.getValueType();
1533 unsigned InNumElements = InLoVT.getVectorNumElements();
1534
1535 EVT OutLoVT, OutHiVT;
1536 std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1537 unsigned OutNumElements = OutLoVT.getVectorNumElements();
1538 assert((2 * OutNumElements) <= InNumElements &&
1539 "Illegal extend vector in reg split");
1540
1541 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1542 // input vector (i.e. we only use InLo):
1543 // OutLo will extend the first OutNumElements from InLo.
1544 // OutHi will extend the next OutNumElements from InLo.
1545
1546 // Shuffle the elements from InLo for OutHi into the bottom elements to
1547 // create a 'fake' InHi.
1548 SmallVector<int, 8> SplitHi(InNumElements, -1);
1549 for (unsigned i = 0; i != OutNumElements; ++i)
1550 SplitHi[i] = i + OutNumElements;
1551 InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1552
1553 Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1554 Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1555}
1556
1557void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1558 SDValue &Hi) {
1559 unsigned NumOps = N->getNumOperands();
1560 SDValue Chain = N->getOperand(0);
1561 EVT LoVT, HiVT;
1562 SDLoc dl(N);
1563 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1564
1565 SmallVector<SDValue, 4> OpsLo(NumOps);
1566 SmallVector<SDValue, 4> OpsHi(NumOps);
1567
1568 // The Chain is the first operand.
1569 OpsLo[0] = Chain;
1570 OpsHi[0] = Chain;
1571
1572 // Now process the remaining operands.
1573 for (unsigned i = 1; i < NumOps; ++i) {
1574 SDValue Op = N->getOperand(i);
1575 SDValue OpLo = Op;
1576 SDValue OpHi = Op;
1577
1578 EVT InVT = Op.getValueType();
1579 if (InVT.isVector()) {
1580 // If the input also splits, handle it directly for a
1581 // compile time speedup. Otherwise split it by hand.
1582 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1583 GetSplitVector(Op, OpLo, OpHi);
1584 else
1585 std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1586 }
1587
1588 OpsLo[i] = OpLo;
1589 OpsHi[i] = OpHi;
1590 }
1591
1592 EVT LoValueVTs[] = {LoVT, MVT::Other};
1593 EVT HiValueVTs[] = {HiVT, MVT::Other};
1594 Lo = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo,
1595 N->getFlags());
1596 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi,
1597 N->getFlags());
1598
1599 // Build a factor node to remember that this Op is independent of the
1600 // other one.
1601 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1602 Lo.getValue(1), Hi.getValue(1));
1603
1604 // Legalize the chain result - switch anything that used the old chain to
1605 // use the new one.
1606 ReplaceValueWith(SDValue(N, 1), Chain);
1607}
1608
1609SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1610 SDValue Chain = N->getOperand(0);
1611 EVT VT = N->getValueType(0);
1612 unsigned NE = VT.getVectorNumElements();
1613 EVT EltVT = VT.getVectorElementType();
1614 SDLoc dl(N);
1615
1617 SmallVector<SDValue, 4> Operands(N->getNumOperands());
1618
1619 // If ResNE is 0, fully unroll the vector op.
1620 if (ResNE == 0)
1621 ResNE = NE;
1622 else if (NE > ResNE)
1623 NE = ResNE;
1624
1625 //The results of each unrolled operation, including the chain.
1626 EVT ChainVTs[] = {EltVT, MVT::Other};
1628
1629 unsigned i;
1630 for (i = 0; i != NE; ++i) {
1631 Operands[0] = Chain;
1632 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
1633 SDValue Operand = N->getOperand(j);
1634 EVT OperandVT = Operand.getValueType();
1635 if (OperandVT.isVector()) {
1636 EVT OperandEltVT = OperandVT.getVectorElementType();
1637 Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
1638 Operand, DAG.getVectorIdxConstant(i, dl));
1639 } else {
1640 Operands[j] = Operand;
1641 }
1642 }
1643 SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1644 Scalar.getNode()->setFlags(N->getFlags());
1645
1646 //Add in the scalar as well as its chain value to the
1647 //result vectors.
1648 Scalars.push_back(Scalar);
1649 Chains.push_back(Scalar.getValue(1));
1650 }
1651
1652 for (; i < ResNE; ++i)
1653 Scalars.push_back(DAG.getUNDEF(EltVT));
1654
1655 // Build a new factor node to connect the chain back together.
1656 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1657 ReplaceValueWith(SDValue(N, 1), Chain);
1658
1659 // Create a new BUILD_VECTOR node
1660 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE);
1661 return DAG.getBuildVector(VecVT, dl, Scalars);
1662}
1663
1664void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1665 SDValue &Lo, SDValue &Hi) {
1666 SDLoc dl(N);
1667 EVT ResVT = N->getValueType(0);
1668 EVT OvVT = N->getValueType(1);
1669 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1670 std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
1671 std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
1672
1673 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1674 if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
1675 GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
1676 GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
1677 } else {
1678 std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
1679 std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
1680 }
1681
1682 unsigned Opcode = N->getOpcode();
1683 SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
1684 SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
1685 SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
1686 SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
1687 LoNode->setFlags(N->getFlags());
1688 HiNode->setFlags(N->getFlags());
1689
1690 Lo = SDValue(LoNode, ResNo);
1691 Hi = SDValue(HiNode, ResNo);
1692
1693 // Replace the other vector result not being explicitly split here.
1694 unsigned OtherNo = 1 - ResNo;
1695 EVT OtherVT = N->getValueType(OtherNo);
1696 if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
1697 SetSplitVector(SDValue(N, OtherNo),
1698 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1699 } else {
1700 SDValue OtherVal = DAG.getNode(
1701 ISD::CONCAT_VECTORS, dl, OtherVT,
1702 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1703 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
1704 }
1705}
1706
1707void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1708 SDValue &Hi) {
1709 SDValue Vec = N->getOperand(0);
1710 SDValue Elt = N->getOperand(1);
1711 SDValue Idx = N->getOperand(2);
1712 SDLoc dl(N);
1713 GetSplitVector(Vec, Lo, Hi);
1714
1715 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1716 unsigned IdxVal = CIdx->getZExtValue();
1717 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
1718 if (IdxVal < LoNumElts) {
1720 Lo.getValueType(), Lo, Elt, Idx);
1721 return;
1722 } else if (!Vec.getValueType().isScalableVector()) {
1723 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1724 DAG.getVectorIdxConstant(IdxVal - LoNumElts, dl));
1725 return;
1726 }
1727 }
1728
1729 // See if the target wants to custom expand this node.
1730 if (CustomLowerNode(N, N->getValueType(0), true))
1731 return;
1732
1733 // Make the vector elements byte-addressable if they aren't already.
1734 EVT VecVT = Vec.getValueType();
1735 EVT EltVT = VecVT.getVectorElementType();
1736 if (VecVT.getScalarSizeInBits() < 8) {
1737 EltVT = MVT::i8;
1738 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1739 VecVT.getVectorElementCount());
1740 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1741 // Extend the element type to match if needed.
1742 if (EltVT.bitsGT(Elt.getValueType()))
1743 Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1744 }
1745
1746 // Spill the vector to the stack.
1747 // In cases where the vector is illegal it will be broken down into parts
1748 // and stored in parts - we should use the alignment for the smallest part.
1749 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1751 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1752 auto &MF = DAG.getMachineFunction();
1753 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1754 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1755
1756 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1757 SmallestAlign);
1758
1759 // Store the new element. This may be larger than the vector element type,
1760 // so use a truncating store.
1761 SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1762 Store = DAG.getTruncStore(
1763 Store, dl, Elt, EltPtr, MachinePointerInfo::getUnknownStack(MF), EltVT,
1764 commonAlignment(SmallestAlign,
1765 EltVT.getFixedSizeInBits() / 8));
1766
1767 EVT LoVT, HiVT;
1768 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1769
1770 // Load the Lo part from the stack slot.
1771 Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo, SmallestAlign);
1772
1773 // Increment the pointer to the other part.
1774 auto Load = cast<LoadSDNode>(Lo);
1775 MachinePointerInfo MPI = Load->getPointerInfo();
1776 IncrementPointer(Load, LoVT, MPI, StackPtr);
1777
1778 Hi = DAG.getLoad(HiVT, dl, Store, StackPtr, MPI, SmallestAlign);
1779
1780 // If we adjusted the original type, we need to truncate the results.
1781 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1782 if (LoVT != Lo.getValueType())
1783 Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1784 if (HiVT != Hi.getValueType())
1785 Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1786}
1787
1788void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
1789 SDValue &Hi) {
1790 EVT LoVT, HiVT;
1791 SDLoc dl(N);
1792 assert(N->getValueType(0).isScalableVector() &&
1793 "Only scalable vectors are supported for STEP_VECTOR");
1794 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1795 SDValue Step = N->getOperand(0);
1796
1797 Lo = DAG.getNode(ISD::STEP_VECTOR, dl, LoVT, Step);
1798
1799 // Hi = Lo + (EltCnt * Step)
1800 EVT EltVT = Step.getValueType();
1801 APInt StepVal = cast<ConstantSDNode>(Step)->getAPIntValue();
1802 SDValue StartOfHi =
1803 DAG.getVScale(dl, EltVT, StepVal * LoVT.getVectorMinNumElements());
1804 StartOfHi = DAG.getSExtOrTrunc(StartOfHi, dl, HiVT.getVectorElementType());
1805 StartOfHi = DAG.getNode(ISD::SPLAT_VECTOR, dl, HiVT, StartOfHi);
1806
1807 Hi = DAG.getNode(ISD::STEP_VECTOR, dl, HiVT, Step);
1808 Hi = DAG.getNode(ISD::ADD, dl, HiVT, Hi, StartOfHi);
1809}
1810
1811void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
1812 SDValue &Hi) {
1813 EVT LoVT, HiVT;
1814 SDLoc dl(N);
1815 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1816 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, N->getOperand(0));
1817 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
1818 Hi = DAG.getUNDEF(HiVT);
1819 } else {
1820 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
1821 Hi = Lo;
1822 }
1823}
1824
1825void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1826 SDValue &Hi) {
1827 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1828 EVT LoVT, HiVT;
1829 SDLoc dl(LD);
1830 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1831
1832 ISD::LoadExtType ExtType = LD->getExtensionType();
1833 SDValue Ch = LD->getChain();
1834 SDValue Ptr = LD->getBasePtr();
1835 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1836 EVT MemoryVT = LD->getMemoryVT();
1837 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1838 AAMDNodes AAInfo = LD->getAAInfo();
1839
1840 EVT LoMemVT, HiMemVT;
1841 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1842
1843 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
1844 SDValue Value, NewChain;
1845 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
1846 std::tie(Lo, Hi) = DAG.SplitVector(Value, dl);
1847 ReplaceValueWith(SDValue(LD, 1), NewChain);
1848 return;
1849 }
1850
1851 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1852 LD->getPointerInfo(), LoMemVT, LD->getOriginalAlign(),
1853 MMOFlags, AAInfo);
1854
1856 IncrementPointer(LD, LoMemVT, MPI, Ptr);
1857
1858 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, MPI,
1859 HiMemVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
1860
1861 // Build a factor node to remember that this load is independent of the
1862 // other one.
1863 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1864 Hi.getValue(1));
1865
1866 // Legalize the chain result - switch anything that used the old chain to
1867 // use the new one.
1868 ReplaceValueWith(SDValue(LD, 1), Ch);
1869}
1870
1871void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
1872 SDValue &Hi) {
1873 assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
1874 EVT LoVT, HiVT;
1875 SDLoc dl(LD);
1876 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1877
1878 ISD::LoadExtType ExtType = LD->getExtensionType();
1879 SDValue Ch = LD->getChain();
1880 SDValue Ptr = LD->getBasePtr();
1881 SDValue Offset = LD->getOffset();
1882 assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
1883 Align Alignment = LD->getOriginalAlign();
1884 SDValue Mask = LD->getMask();
1885 SDValue EVL = LD->getVectorLength();
1886 EVT MemoryVT = LD->getMemoryVT();
1887
1888 EVT LoMemVT, HiMemVT;
1889 bool HiIsEmpty = false;
1890 std::tie(LoMemVT, HiMemVT) =
1891 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
1892
1893 // Split Mask operand
1894 SDValue MaskLo, MaskHi;
1895 if (Mask.getOpcode() == ISD::SETCC) {
1896 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1897 } else {
1898 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1899 GetSplitVector(Mask, MaskLo, MaskHi);
1900 else
1901 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1902 }
1903
1904 // Split EVL operand
1905 SDValue EVLLo, EVLHi;
1906 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, LD->getValueType(0), dl);
1907
1909 LD->getPointerInfo(), MachineMemOperand::MOLoad,
1910 MemoryLocation::UnknownSize, Alignment, LD->getAAInfo(), LD->getRanges());
1911
1912 Lo =
1913 DAG.getLoadVP(LD->getAddressingMode(), ExtType, LoVT, dl, Ch, Ptr, Offset,
1914 MaskLo, EVLLo, LoMemVT, MMO, LD->isExpandingLoad());
1915
1916 if (HiIsEmpty) {
1917 // The hi vp_load has zero storage size. We therefore simply set it to
1918 // the low vp_load and rely on subsequent removal from the chain.
1919 Hi = Lo;
1920 } else {
1921 // Generate hi vp_load.
1922 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1923 LD->isExpandingLoad());
1924
1926 if (LoMemVT.isScalableVector())
1927 MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
1928 else
1929 MPI = LD->getPointerInfo().getWithOffset(
1930 LoMemVT.getStoreSize().getFixedValue());
1931
1934 LD->getAAInfo(), LD->getRanges());
1935
1936 Hi = DAG.getLoadVP(LD->getAddressingMode(), ExtType, HiVT, dl, Ch, Ptr,
1937 Offset, MaskHi, EVLHi, HiMemVT, MMO,
1938 LD->isExpandingLoad());
1939 }
1940
1941 // Build a factor node to remember that this load is independent of the
1942 // other one.
1943 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1944 Hi.getValue(1));
1945
1946 // Legalize the chain result - switch anything that used the old chain to
1947 // use the new one.
1948 ReplaceValueWith(SDValue(LD, 1), Ch);
1949}
1950
1951void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
1952 SDValue &Lo, SDValue &Hi) {
1953 assert(SLD->isUnindexed() &&
1954 "Indexed VP strided load during type legalization!");
1955 assert(SLD->getOffset().isUndef() &&
1956 "Unexpected indexed variable-length load offset");
1957
1958 SDLoc DL(SLD);
1959
1960 EVT LoVT, HiVT;
1961 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(SLD->getValueType(0));
1962
1963 EVT LoMemVT, HiMemVT;
1964 bool HiIsEmpty = false;
1965 std::tie(LoMemVT, HiMemVT) =
1966 DAG.GetDependentSplitDestVTs(SLD->getMemoryVT(), LoVT, &HiIsEmpty);
1967
1968 SDValue Mask = SLD->getMask();
1969 SDValue LoMask, HiMask;
1970 if (Mask.getOpcode() == ISD::SETCC) {
1971 SplitVecRes_SETCC(Mask.getNode(), LoMask, HiMask);
1972 } else {
1973 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1974 GetSplitVector(Mask, LoMask, HiMask);
1975 else
1976 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
1977 }
1978
1979 SDValue LoEVL, HiEVL;
1980 std::tie(LoEVL, HiEVL) =
1981 DAG.SplitEVL(SLD->getVectorLength(), SLD->getValueType(0), DL);
1982
1983 // Generate the low vp_strided_load
1984 Lo = DAG.getStridedLoadVP(
1985 SLD->getAddressingMode(), SLD->getExtensionType(), LoVT, DL,
1986 SLD->getChain(), SLD->getBasePtr(), SLD->getOffset(), SLD->getStride(),
1987 LoMask, LoEVL, LoMemVT, SLD->getMemOperand(), SLD->isExpandingLoad());
1988
1989 if (HiIsEmpty) {
1990 // The high vp_strided_load has zero storage size. We therefore simply set
1991 // it to the low vp_strided_load and rely on subsequent removal from the
1992 // chain.
1993 Hi = Lo;
1994 } else {
1995 // Generate the high vp_strided_load.
1996 // To calculate the high base address, we need to sum to the low base
1997 // address stride number of bytes for each element already loaded by low,
1998 // that is: Ptr = Ptr + (LoEVL * Stride)
1999 EVT PtrVT = SLD->getBasePtr().getValueType();
2001 DAG.getNode(ISD::MUL, DL, PtrVT, LoEVL,
2002 DAG.getSExtOrTrunc(SLD->getStride(), DL, PtrVT));
2003 SDValue Ptr =
2004 DAG.getNode(ISD::ADD, DL, PtrVT, SLD->getBasePtr(), Increment);
2005
2006 Align Alignment = SLD->getOriginalAlign();
2007 if (LoMemVT.isScalableVector())
2008 Alignment = commonAlignment(
2009 Alignment, LoMemVT.getSizeInBits().getKnownMinValue() / 8);
2010
2014 SLD->getAAInfo(), SLD->getRanges());
2015
2017 HiVT, DL, SLD->getChain(), Ptr, SLD->getOffset(),
2018 SLD->getStride(), HiMask, HiEVL, HiMemVT, MMO,
2019 SLD->isExpandingLoad());
2020 }
2021
2022 // Build a factor node to remember that this load is independent of the
2023 // other one.
2024 SDValue Ch = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
2025 Hi.getValue(1));
2026
2027 // Legalize the chain result - switch anything that used the old chain to
2028 // use the new one.
2029 ReplaceValueWith(SDValue(SLD, 1), Ch);
2030}
2031
2032void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
2033 SDValue &Lo, SDValue &Hi) {
2034 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
2035 EVT LoVT, HiVT;
2036 SDLoc dl(MLD);
2037 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
2038
2039 SDValue Ch = MLD->getChain();
2040 SDValue Ptr = MLD->getBasePtr();
2041 SDValue Offset = MLD->getOffset();
2042 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
2043 SDValue Mask = MLD->getMask();
2044 SDValue PassThru = MLD->getPassThru();
2045 Align Alignment = MLD->getOriginalAlign();
2046 ISD::LoadExtType ExtType = MLD->getExtensionType();
2047
2048 // Split Mask operand
2049 SDValue MaskLo, MaskHi;
2050 if (Mask.getOpcode() == ISD::SETCC) {
2051 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2052 } else {
2053 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2054 GetSplitVector(Mask, MaskLo, MaskHi);
2055 else
2056 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2057 }
2058
2059 EVT MemoryVT = MLD->getMemoryVT();
2060 EVT LoMemVT, HiMemVT;
2061 bool HiIsEmpty = false;
2062 std::tie(LoMemVT, HiMemVT) =
2063 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
2064
2065 SDValue PassThruLo, PassThruHi;
2066 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2067 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2068 else
2069 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2070
2073 MemoryLocation::UnknownSize, Alignment, MLD->getAAInfo(),
2074 MLD->getRanges());
2075
2076 Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
2077 MMO, MLD->getAddressingMode(), ExtType,
2078 MLD->isExpandingLoad());
2079
2080 if (HiIsEmpty) {
2081 // The hi masked load has zero storage size. We therefore simply set it to
2082 // the low masked load and rely on subsequent removal from the chain.
2083 Hi = Lo;
2084 } else {
2085 // Generate hi masked load.
2086 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
2087 MLD->isExpandingLoad());
2088
2090 if (LoMemVT.isScalableVector())
2092 else
2093 MPI = MLD->getPointerInfo().getWithOffset(
2094 LoMemVT.getStoreSize().getFixedValue());
2095
2098 MLD->getAAInfo(), MLD->getRanges());
2099
2100 Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
2101 HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
2102 MLD->isExpandingLoad());
2103 }
2104
2105 // Build a factor node to remember that this load is independent of the
2106 // other one.
2107 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2108 Hi.getValue(1));
2109
2110 // Legalize the chain result - switch anything that used the old chain to
2111 // use the new one.
2112 ReplaceValueWith(SDValue(MLD, 1), Ch);
2113
2114}
2115
2116void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2117 SDValue &Hi, bool SplitSETCC) {
2118 EVT LoVT, HiVT;
2119 SDLoc dl(N);
2120 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2121
2122 SDValue Ch = N->getChain();
2123 SDValue Ptr = N->getBasePtr();
2124 struct Operands {
2125 SDValue Mask;
2126 SDValue Index;
2127 SDValue Scale;
2128 } Ops = [&]() -> Operands {
2129 if (auto *MSC = dyn_cast<MaskedGatherSDNode>(N)) {
2130 return {MSC->getMask(), MSC->getIndex(), MSC->getScale()};
2131 }
2132 auto *VPSC = cast<VPGatherSDNode>(N);
2133 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale()};
2134 }();
2135
2136 EVT MemoryVT = N->getMemoryVT();
2137 Align Alignment = N->getOriginalAlign();
2138
2139 // Split Mask operand
2140 SDValue MaskLo, MaskHi;
2141 if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2142 SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
2143 } else {
2144 std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, dl);
2145 }
2146
2147 EVT LoMemVT, HiMemVT;
2148 // Split MemoryVT
2149 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2150
2151 SDValue IndexHi, IndexLo;
2152 if (getTypeAction(Ops.Index.getValueType()) ==
2154 GetSplitVector(Ops.Index, IndexLo, IndexHi);
2155 else
2156 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, dl);
2157
2159 N->getPointerInfo(), MachineMemOperand::MOLoad,
2160 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
2161
2162 if (auto *MGT = dyn_cast<MaskedGatherSDNode>(N)) {
2163 SDValue PassThru = MGT->getPassThru();
2164 SDValue PassThruLo, PassThruHi;
2165 if (getTypeAction(PassThru.getValueType()) ==
2167 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2168 else
2169 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2170
2171 ISD::LoadExtType ExtType = MGT->getExtensionType();
2172 ISD::MemIndexType IndexTy = MGT->getIndexType();
2173
2174 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2175 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl,
2176 OpsLo, MMO, IndexTy, ExtType);
2177
2178 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2179 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl,
2180 OpsHi, MMO, IndexTy, ExtType);
2181 } else {
2182 auto *VPGT = cast<VPGatherSDNode>(N);
2183 SDValue EVLLo, EVLHi;
2184 std::tie(EVLLo, EVLHi) =
2185 DAG.SplitEVL(VPGT->getVectorLength(), MemoryVT, dl);
2186
2187 SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2188 Lo = DAG.getGatherVP(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, OpsLo,
2189 MMO, VPGT->getIndexType());
2190
2191 SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2192 Hi = DAG.getGatherVP(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, OpsHi,
2193 MMO, VPGT->getIndexType());
2194 }
2195
2196 // Build a factor node to remember that this load is independent of the
2197 // other one.
2198 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2199 Hi.getValue(1));
2200
2201 // Legalize the chain result - switch anything that used the old chain to
2202 // use the new one.
2203 ReplaceValueWith(SDValue(N, 1), Ch);
2204}
2205
2206void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
2207 assert(N->getValueType(0).isVector() &&
2208 N->getOperand(0).getValueType().isVector() &&
2209 "Operand types must be vectors");
2210
2211 EVT LoVT, HiVT;
2212 SDLoc DL(N);
2213 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2214
2215 // If the input also splits, handle it directly. Otherwise split it by hand.
2216 SDValue LL, LH, RL, RH;
2217 if (getTypeAction(N->getOperand(0).getValueType()) ==
2219 GetSplitVector(N->getOperand(0), LL, LH);
2220 else
2221 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
2222
2223 if (getTypeAction(N->getOperand(1).getValueType()) ==
2225 GetSplitVector(N->getOperand(1), RL, RH);
2226 else
2227 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
2228
2229 if (N->getOpcode() == ISD::SETCC) {
2230 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
2231 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
2232 } else {
2233 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
2234 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
2235 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
2236 std::tie(EVLLo, EVLHi) =
2237 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), DL);
2238 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2), MaskLo,
2239 EVLLo);
2240 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2), MaskHi,
2241 EVLHi);
2242 }
2243}
2244
2245void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
2246 SDValue &Hi) {
2247 // Get the dest types - they may not match the input types, e.g. int_to_fp.
2248 EVT LoVT, HiVT;
2249 SDLoc dl(N);
2250 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2251
2252 // If the input also splits, handle it directly for a compile time speedup.
2253 // Otherwise split it by hand.
2254 EVT InVT = N->getOperand(0).getValueType();
2255 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
2256 GetSplitVector(N->getOperand(0), Lo, Hi);
2257 else
2258 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
2259
2260 const SDNodeFlags Flags = N->getFlags();
2261 unsigned Opcode = N->getOpcode();
2262 if (N->getNumOperands() <= 2) {
2263 if (Opcode == ISD::FP_ROUND) {
2264 Lo = DAG.getNode(Opcode, dl, LoVT, Lo, N->getOperand(1), Flags);
2265 Hi = DAG.getNode(Opcode, dl, HiVT, Hi, N->getOperand(1), Flags);
2266 } else {
2267 Lo = DAG.getNode(Opcode, dl, LoVT, Lo, Flags);
2268 Hi = DAG.getNode(Opcode, dl, HiVT, Hi, Flags);
2269 }
2270 return;
2271 }
2272
2273 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
2274 assert(N->isVPOpcode() && "Expected VP opcode");
2275
2276 SDValue MaskLo, MaskHi;
2277 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
2278
2279 SDValue EVLLo, EVLHi;
2280 std::tie(EVLLo, EVLHi) =
2281 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
2282
2283 Lo = DAG.getNode(Opcode, dl, LoVT, {Lo, MaskLo, EVLLo}, Flags);
2284 Hi = DAG.getNode(Opcode, dl, HiVT, {Hi, MaskHi, EVLHi}, Flags);
2285}
2286
2287void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
2288 SDValue &Hi) {
2289 SDLoc dl(N);
2290 EVT SrcVT = N->getOperand(0).getValueType();
2291 EVT DestVT = N->getValueType(0);
2292 EVT LoVT, HiVT;
2293 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
2294
2295 // We can do better than a generic split operation if the extend is doing
2296 // more than just doubling the width of the elements and the following are
2297 // true:
2298 // - The number of vector elements is even,
2299 // - the source type is legal,
2300 // - the type of a split source is illegal,
2301 // - the type of an extended (by doubling element size) source is legal, and
2302 // - the type of that extended source when split is legal.
2303 //
2304 // This won't necessarily completely legalize the operation, but it will
2305 // more effectively move in the right direction and prevent falling down
2306 // to scalarization in many cases due to the input vector being split too
2307 // far.
2308 if (SrcVT.getVectorElementCount().isKnownEven() &&
2309 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
2310 LLVMContext &Ctx = *DAG.getContext();
2311 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
2312 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
2313
2314 EVT SplitLoVT, SplitHiVT;
2315 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
2316 if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
2317 TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
2318 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
2319 N->dump(&DAG); dbgs() << "\n");
2320 if (!N->isVPOpcode()) {
2321 // Extend the source vector by one step.
2322 SDValue NewSrc =
2323 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
2324 // Get the low and high halves of the new, extended one step, vector.
2325 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
2326 // Extend those vector halves the rest of the way.
2327 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
2328 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
2329 return;
2330 }
2331
2332 // Extend the source vector by one step.
2333 SDValue NewSrc =
2334 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0),
2335 N->getOperand(1), N->getOperand(2));
2336 // Get the low and high halves of the new, extended one step, vector.
2337 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
2338
2339 SDValue MaskLo, MaskHi;
2340 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
2341
2342 SDValue EVLLo, EVLHi;
2343 std::tie(EVLLo, EVLHi) =
2344 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
2345 // Extend those vector halves the rest of the way.
2346 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, {Lo, MaskLo, EVLLo});
2347 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, {Hi, MaskHi, EVLHi});
2348 return;
2349 }
2350 }
2351 // Fall back to the generic unary operator splitting otherwise.
2352 SplitVecRes_UnaryOp(N, Lo, Hi);
2353}
2354
2355void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
2356 SDValue &Lo, SDValue &Hi) {
2357 // The low and high parts of the original input give four input vectors.
2358 SDValue Inputs[4];
2359 SDLoc DL(N);
2360 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
2361 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
2362 EVT NewVT = Inputs[0].getValueType();
2363 unsigned NewElts = NewVT.getVectorNumElements();
2364
2365 auto &&IsConstant = [](const SDValue &N) {
2366 APInt SplatValue;
2367 return N.getResNo() == 0 &&
2368 (ISD::isConstantSplatVector(N.getNode(), SplatValue) ||
2370 };
2371 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
2372 SDValue &Input2,
2374 assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
2375 Input2->getOpcode() == ISD::BUILD_VECTOR &&
2376 "Expected build vector node.");
2377 EVT EltVT = NewVT.getVectorElementType();
2378 SmallVector<SDValue> Ops(NewElts, DAG.getUNDEF(EltVT));
2379 for (unsigned I = 0; I < NewElts; ++I) {
2380 if (Mask[I] == UndefMaskElem)
2381 continue;
2382 unsigned Idx = Mask[I];
2383 if (Idx >= NewElts)
2384 Ops[I] = Input2.getOperand(Idx - NewElts);
2385 else
2386 Ops[I] = Input1.getOperand(Idx);
2387 // Make the type of all elements the same as the element type.
2388 if (Ops[I].getValueType().bitsGT(EltVT))
2389 Ops[I] = DAG.getNode(ISD::TRUNCATE, DL, EltVT, Ops[I]);
2390 }
2391 return DAG.getBuildVector(NewVT, DL, Ops);
2392 };
2393
2394 // If Lo or Hi uses elements from at most two of the four input vectors, then
2395 // express it as a vector shuffle of those two inputs. Otherwise extract the
2396 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
2397 SmallVector<int> OrigMask(N->getMask());
2398 // Try to pack incoming shuffles/inputs.
2399 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
2401 // Check if all inputs are shuffles of the same operands or non-shuffles.
2403 for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) {
2404 SDValue Input = Inputs[Idx];
2405 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Input.getNode());
2406 if (!Shuffle ||
2407 Input.getOperand(0).getValueType() != Input.getValueType())
2408 continue;
2409 ShufflesIdxs[std::make_pair(Input.getOperand(0), Input.getOperand(1))]
2410 .push_back(Idx);
2411 ShufflesIdxs[std::make_pair(Input.getOperand(1), Input.getOperand(0))]
2412 .push_back(Idx);
2413 }
2414 for (auto &P : ShufflesIdxs) {
2415 if (P.second.size() < 2)
2416 continue;
2417 // Use shuffles operands instead of shuffles themselves.
2418 // 1. Adjust mask.
2419 for (int &Idx : Mask) {
2420 if (Idx == UndefMaskElem)
2421 continue;
2422 unsigned SrcRegIdx = Idx / NewElts;
2423 if (Inputs[SrcRegIdx].isUndef()) {
2425 continue;
2426 }
2427 auto *Shuffle =
2428 dyn_cast<ShuffleVectorSDNode>(Inputs[SrcRegIdx].getNode());
2429 if (!Shuffle || !is_contained(P.second, SrcRegIdx))
2430 continue;
2431 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2432 if (MaskElt == UndefMaskElem) {
2434 continue;
2435 }
2436 Idx = MaskElt % NewElts +
2437 P.second[Shuffle->getOperand(MaskElt / NewElts) == P.first.first
2438 ? 0
2439 : 1] *
2440 NewElts;
2441 }
2442 // 2. Update inputs.
2443 Inputs[P.second[0]] = P.first.first;
2444 Inputs[P.second[1]] = P.first.second;
2445 // Clear the pair data.
2446 P.second.clear();
2447 ShufflesIdxs[std::make_pair(P.first.second, P.first.first)].clear();
2448 }
2449 // Check if any concat_vectors can be simplified.
2450 SmallBitVector UsedSubVector(2 * std::size(Inputs));
2451 for (int &Idx : Mask) {
2452 if (Idx == UndefMaskElem)
2453 continue;
2454 unsigned SrcRegIdx = Idx / NewElts;
2455 if (Inputs[SrcRegIdx].isUndef()) {
2457 continue;
2458 }
2460 getTypeAction(Inputs[SrcRegIdx].getValueType());
2461 if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
2462 Inputs[SrcRegIdx].getNumOperands() == 2 &&
2463 !Inputs[SrcRegIdx].getOperand(1).isUndef() &&
2464 (TypeAction == TargetLowering::TypeLegal ||
2465 TypeAction == TargetLowering::TypeWidenVector))
2466 UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
2467 }
2468 if (UsedSubVector.count() > 1) {
2470 for (unsigned I = 0; I < std::size(Inputs); ++I) {
2471 if (UsedSubVector.test(2 * I) == UsedSubVector.test(2 * I + 1))
2472 continue;
2473 if (Pairs.empty() || Pairs.back().size() == 2)
2474 Pairs.emplace_back();
2475 if (UsedSubVector.test(2 * I)) {
2476 Pairs.back().emplace_back(I, 0);
2477 } else {
2478 assert(UsedSubVector.test(2 * I + 1) &&
2479 "Expected to be used one of the subvectors.");
2480 Pairs.back().emplace_back(I, 1);
2481 }
2482 }
2483 if (!Pairs.empty() && Pairs.front().size() > 1) {
2484 // Adjust mask.
2485 for (int &Idx : Mask) {
2486 if (Idx == UndefMaskElem)
2487 continue;
2488 unsigned SrcRegIdx = Idx / NewElts;
2489 auto *It = find_if(
2490 Pairs, [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
2491 return Idxs.front().first == SrcRegIdx ||
2492 Idxs.back().first == SrcRegIdx;
2493 });
2494 if (It == Pairs.end())
2495 continue;
2496 Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
2497 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
2498 }
2499 // Adjust inputs.
2500 for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
2501 Inputs[Idxs.front().first] = DAG.getNode(
2503 Inputs[Idxs.front().first].getValueType(),
2504 Inputs[Idxs.front().first].getOperand(Idxs.front().second),
2505 Inputs[Idxs.back().first].getOperand(Idxs.back().second));
2506 }
2507 }
2508 }
2509 bool Changed;
2510 do {
2511 // Try to remove extra shuffles (except broadcasts) and shuffles with the
2512 // reused operands.
2513 Changed = false;
2514 for (unsigned I = 0; I < std::size(Inputs); ++I) {
2515 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Inputs[I].getNode());
2516 if (!Shuffle)
2517 continue;
2518 if (Shuffle->getOperand(0).getValueType() != NewVT)
2519 continue;
2520 int Op = -1;
2521 if (!Inputs[I].hasOneUse() && Shuffle->getOperand(1).isUndef() &&
2522 !Shuffle->isSplat()) {
2523 Op = 0;
2524 } else if (!Inputs[I].hasOneUse() &&
2525 !Shuffle->getOperand(1).isUndef()) {
2526 // Find the only used operand, if possible.
2527 for (int &Idx : Mask) {
2528 if (Idx == UndefMaskElem)
2529 continue;
2530 unsigned SrcRegIdx = Idx / NewElts;
2531 if (SrcRegIdx != I)
2532 continue;
2533 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2534 if (MaskElt == UndefMaskElem) {
2536 continue;
2537 }
2538 int OpIdx = MaskElt / NewElts;
2539 if (Op == -1) {
2540 Op = OpIdx;
2541 continue;
2542 }
2543 if (Op != OpIdx) {
2544 Op = -1;
2545 break;
2546 }
2547 }
2548 }
2549 if (Op < 0) {
2550 // Try to check if one of the shuffle operands is used already.
2551 for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
2552 if (Shuffle->getOperand(OpIdx).isUndef())
2553 continue;
2554 auto *It = find(Inputs, Shuffle->getOperand(OpIdx));
2555 if (It == std::end(Inputs))
2556 continue;
2557 int FoundOp = std::distance(std::begin(Inputs), It);
2558 // Found that operand is used already.
2559 // 1. Fix the mask for the reused operand.
2560 for (int &Idx : Mask) {
2561 if (Idx == UndefMaskElem)
2562 continue;
2563 unsigned SrcRegIdx = Idx / NewElts;
2564 if (SrcRegIdx != I)
2565 continue;
2566 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2567 if (MaskElt == UndefMaskElem) {
2569 continue;
2570 }
2571 int MaskIdx = MaskElt / NewElts;
2572 if (OpIdx == MaskIdx)
2573 Idx = MaskElt % NewElts + FoundOp * NewElts;
2574 }
2575 // 2. Set Op to the unused OpIdx.
2576 Op = (OpIdx + 1) % 2;
2577 break;
2578 }
2579 }
2580 if (Op >= 0) {
2581 Changed = true;
2582 Inputs[I] = Shuffle->getOperand(Op);
2583 // Adjust mask.
2584 for (int &Idx : Mask) {
2585 if (Idx == UndefMaskElem)
2586 continue;
2587 unsigned SrcRegIdx = Idx / NewElts;
2588 if (SrcRegIdx != I)
2589 continue;
2590 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2591 int OpIdx = MaskElt / NewElts;
2592 if (OpIdx != Op)
2593 continue;
2594 Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
2595 }
2596 }
2597 }
2598 } while (Changed);
2599 };
2600 TryPeekThroughShufflesInputs(OrigMask);
2601 // Proces unique inputs.
2602 auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
2603 NewElts](SmallVectorImpl<int> &Mask) {
2604 SetVector<SDValue> UniqueInputs;
2605 SetVector<SDValue> UniqueConstantInputs;
2606 for (const auto &I : Inputs) {
2607 if (IsConstant(I))
2608 UniqueConstantInputs.insert(I);
2609 else if (!I.isUndef())
2610 UniqueInputs.insert(I);
2611 }
2612 // Adjust mask in case of reused inputs. Also, need to insert constant
2613 // inputs at first, otherwise it affects the final outcome.
2614 if (UniqueInputs.size() != std::size(Inputs)) {
2615 auto &&UniqueVec = UniqueInputs.takeVector();
2616 auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
2617 unsigned ConstNum = UniqueConstantVec.size();
2618 for (int &Idx : Mask) {
2619 if (Idx == UndefMaskElem)
2620 continue;
2621 unsigned SrcRegIdx = Idx / NewElts;
2622 if (Inputs[SrcRegIdx].isUndef()) {
2624 continue;
2625 }
2626 const auto It = find(UniqueConstantVec, Inputs[SrcRegIdx]);
2627 if (It != UniqueConstantVec.end()) {
2628 Idx = (Idx % NewElts) +
2629 NewElts * std::distance(UniqueConstantVec.begin(), It);
2630 assert(Idx >= 0 && "Expected defined mask idx.");
2631 continue;
2632 }
2633 const auto RegIt = find(UniqueVec, Inputs[SrcRegIdx]);
2634 assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
2635 Idx = (Idx % NewElts) +
2636 NewElts * (std::distance(UniqueVec.begin(), RegIt) + ConstNum);
2637 assert(Idx >= 0 && "Expected defined mask idx.");
2638 }
2639 copy(UniqueConstantVec, std::begin(Inputs));
2640 copy(UniqueVec, std::next(std::begin(Inputs), ConstNum));
2641 }
2642 };
2643 MakeUniqueInputs(OrigMask);
2644 SDValue OrigInputs[4];
2645 copy(Inputs, std::begin(OrigInputs));
2646 for (unsigned High = 0; High < 2; ++High) {
2647 SDValue &Output = High ? Hi : Lo;
2648
2649 // Build a shuffle mask for the output, discovering on the fly which
2650 // input vectors to use as shuffle operands.
2651 unsigned FirstMaskIdx = High * NewElts;
2652 SmallVector<int> Mask(NewElts * std::size(Inputs), UndefMaskElem);
2653 copy(ArrayRef(OrigMask).slice(FirstMaskIdx, NewElts), Mask.begin());
2654 assert(!Output && "Expected default initialized initial value.");
2655 TryPeekThroughShufflesInputs(Mask);
2656 MakeUniqueInputs(Mask);
2657 SDValue TmpInputs[4];
2658 copy(Inputs, std::begin(TmpInputs));
2659 // Track changes in the output registers.
2660 int UsedIdx = -1;
2661 bool SecondIteration = false;
2662 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
2663 if (UsedIdx < 0) {
2664 UsedIdx = Idx;
2665 return false;
2666 }
2667 if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
2668 SecondIteration = true;
2669 return SecondIteration;
2670 };
2672 Mask, std::size(Inputs), std::size(Inputs),
2673 /*NumOfUsedRegs=*/1,
2674 [&Output, &DAG = DAG, NewVT]() { Output = DAG.getUNDEF(NewVT); },
2675 [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
2676 &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
2677 if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
2678 Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
2679 else
2680 Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx],
2681 DAG.getUNDEF(NewVT), Mask);
2682 Inputs[Idx] = Output;
2683 },
2684 [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
2685 &TmpInputs,
2686 &BuildVector](ArrayRef<int> Mask, unsigned Idx1, unsigned Idx2) {
2687 if (AccumulateResults(Idx1)) {
2688 if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
2689 Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
2690 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
2691 else
2692 Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx1],
2693 Inputs[Idx2], Mask);
2694 } else {
2695 if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
2696 TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
2697 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
2698 else
2699 Output = DAG.getVectorShuffle(NewVT, DL, TmpInputs[Idx1],
2700 TmpInputs[Idx2], Mask);
2701 }
2702 Inputs[Idx1] = Output;
2703 });
2704 copy(OrigInputs, std::begin(Inputs));
2705 }
2706}
2707
2708void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2709 EVT OVT = N->getValueType(0);
2710 EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext());
2711 SDValue Chain = N->getOperand(0);
2712 SDValue Ptr = N->getOperand(1);
2713 SDValue SV = N->getOperand(2);
2714 SDLoc dl(N);
2715
2716 const Align Alignment =
2717 DAG.getDataLayout().getABITypeAlign(NVT.getTypeForEVT(*DAG.getContext()));
2718
2719 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment.value());
2720 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment.value());
2721 Chain = Hi.getValue(1);
2722
2723 // Modified the chain - switch anything that used the old chain to use
2724 // the new one.
2725 ReplaceValueWith(SDValue(N, 1), Chain);
2726}
2727
2728void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
2729 SDValue &Hi) {
2730 EVT DstVTLo, DstVTHi;
2731 std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(N->getValueType(0));
2732 SDLoc dl(N);
2733
2734 SDValue SrcLo, SrcHi;
2735 EVT SrcVT = N->getOperand(0).getValueType();
2736 if (getTypeAction(SrcVT) == TargetLowering::TypeSplitVector)
2737 GetSplitVector(N->getOperand(0), SrcLo, SrcHi);
2738 else
2739 std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(N, 0);
2740
2741 Lo = DAG.getNode(N->getOpcode(), dl, DstVTLo, SrcLo, N->getOperand(1));
2742 Hi = DAG.getNode(N->getOpcode(), dl, DstVTHi, SrcHi, N->getOperand(1));
2743}
2744
2745void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
2746 SDValue &Hi) {
2747 SDValue InLo, InHi;
2748 GetSplitVector(N->getOperand(0), InLo, InHi);
2749 SDLoc DL(N);
2750
2751 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, InHi.getValueType(), InHi);
2752 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, InLo.getValueType(), InLo);
2753}
2754
2755void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
2756 SDValue &Hi) {
2757 EVT VT = N->getValueType(0);
2758 SDLoc DL(N);
2759
2760 EVT LoVT, HiVT;
2761 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
2762
2763 SDValue Expanded = TLI.expandVectorSplice(N, DAG);
2764 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, Expanded,
2765 DAG.getVectorIdxConstant(0, DL));
2766 Hi =
2767 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, Expanded,
2768 DAG.getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL));
2769}
2770
2771//===----------------------------------------------------------------------===//
2772// Operand Vector Splitting
2773//===----------------------------------------------------------------------===//
2774
2775/// This method is called when the specified operand of the specified node is
2776/// found to need vector splitting. At this point, all of the result types of
2777/// the node are known to be legal, but other operands of the node may need
2778/// legalization as well as the specified one.
2779bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
2780 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
2781 SDValue Res = SDValue();
2782
2783 // See if the target wants to custom split this node.
2784 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2785 return false;
2786
2787 switch (N->getOpcode()) {
2788 default:
2789#ifndef NDEBUG
2790 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
2791 N->dump(&DAG);
2792 dbgs() << "\n";
2793#endif
2794 report_fatal_error("Do not know how to split this operator's "
2795 "operand!\n");
2796
2797 case ISD::VP_SETCC:
2798 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
2799 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
2800 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
2801 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
2802 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
2803 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
2804 case ISD::VP_TRUNCATE:
2805 case ISD::TRUNCATE:
2806 Res = SplitVecOp_TruncateHelper(N);
2807 break;
2809 case ISD::VP_FP_ROUND:
2810 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
2811 case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
2812 case ISD::STORE:
2813 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
2814 break;
2815 case ISD::VP_STORE:
2816 Res = SplitVecOp_VP_STORE(cast<VPStoreSDNode>(N), OpNo);
2817 break;
2818 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
2819 Res = SplitVecOp_VP_STRIDED_STORE(cast<VPStridedStoreSDNode>(N), OpNo);
2820 break;
2821 case ISD::MSTORE:
2822 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
2823 break;
2824 case ISD::MSCATTER:
2825 case ISD::VP_SCATTER:
2826 Res = SplitVecOp_Scatter(cast<MemSDNode>(N), OpNo);
2827 break;
2828 case ISD::MGATHER:
2829 case ISD::VP_GATHER:
2830 Res = SplitVecOp_Gather(cast<MemSDNode>(N), OpNo);
2831 break;
2832 case ISD::VSELECT:
2833 Res = SplitVecOp_VSELECT(N, OpNo);
2834 break;
2837 case ISD::SINT_TO_FP:
2838 case ISD::UINT_TO_FP:
2839 case ISD::VP_SINT_TO_FP:
2840 case ISD::VP_UINT_TO_FP:
2841 if (N->getValueType(0).bitsLT(
2842 N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType()))
2843 Res = SplitVecOp_TruncateHelper(N);
2844 else
2845 Res = SplitVecOp_UnaryOp(N);
2846 break;
2849 Res = SplitVecOp_FP_TO_XINT_SAT(N);
2850 break;
2851 case ISD::FP_TO_SINT:
2852 case ISD::FP_TO_UINT:
2853 case ISD::VP_FP_TO_SINT:
2854 case ISD::VP_FP_TO_UINT:
2858 case ISD::FP_EXTEND:
2859 case ISD::SIGN_EXTEND:
2860 case ISD::ZERO_EXTEND:
2861 case ISD::ANY_EXTEND:
2862 case ISD::FTRUNC:
2863 Res = SplitVecOp_UnaryOp(N);
2864 break;
2865
2869 Res = SplitVecOp_ExtVecInRegOp(N);
2870 break;
2871
2874 case ISD::VECREDUCE_ADD:
2875 case ISD::VECREDUCE_MUL:
2876 case ISD::VECREDUCE_AND:
2877 case ISD::VECREDUCE_OR:
2878 case ISD::VECREDUCE_XOR:
2885 Res = SplitVecOp_VECREDUCE(N, OpNo);
2886 break;
2889 Res = SplitVecOp_VECREDUCE_SEQ(N);
2890 break;
2891 case ISD::VP_REDUCE_FADD:
2892 case ISD::VP_REDUCE_SEQ_FADD:
2893 case ISD::VP_REDUCE_FMUL:
2894 case ISD::VP_REDUCE_SEQ_FMUL:
2895 case ISD::VP_REDUCE_ADD:
2896 case ISD::VP_REDUCE_MUL:
2897 case ISD::VP_REDUCE_AND:
2898 case ISD::VP_REDUCE_OR:
2899 case ISD::VP_REDUCE_XOR:
2900 case ISD::VP_REDUCE_SMAX:
2901 case ISD::VP_REDUCE_SMIN:
2902 case ISD::VP_REDUCE_UMAX:
2903 case ISD::VP_REDUCE_UMIN:
2904 case ISD::VP_REDUCE_FMAX:
2905 case ISD::VP_REDUCE_FMIN:
2906 Res = SplitVecOp_VP_REDUCE(N, OpNo);
2907 break;
2908 }
2909
2910 // If the result is null, the sub-method took care of registering results etc.
2911 if (!Res.getNode()) return false;
2912
2913 // If the result is N, the sub-method updated N in place. Tell the legalizer
2914 // core about this.
2915 if (Res.getNode() == N)
2916 return true;
2917
2918 if (N->isStrictFPOpcode())
2919 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
2920 "Invalid operand expansion");
2921 else
2922 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2923 "Invalid operand expansion");
2924
2925 ReplaceValueWith(SDValue(N, 0), Res);
2926 return false;
2927}
2928
2929SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
2930 // The only possibility for an illegal operand is the mask, since result type
2931 // legalization would have handled this node already otherwise.
2932 assert(OpNo == 0 && "Illegal operand must be mask");
2933
2934 SDValue Mask = N->getOperand(0);
2935 SDValue Src0 = N->getOperand(1);
2936 SDValue Src1 = N->getOperand(2);
2937 EVT Src0VT = Src0.getValueType();
2938 SDLoc DL(N);
2939 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
2940
2941 SDValue Lo, Hi;
2942 GetSplitVector(N->getOperand(0), Lo, Hi);
2943 assert(Lo.getValueType() == Hi.getValueType() &&
2944 "Lo and Hi have differing types");
2945
2946 EVT LoOpVT, HiOpVT;
2947 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
2948 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
2949
2950 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
2951 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
2952 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
2953 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
2954
2955 SDValue LoSelect =
2956 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
2957 SDValue HiSelect =
2958 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
2959
2960 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
2961}
2962
2963SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
2964 EVT ResVT = N->getValueType(0);
2965 SDValue Lo, Hi;
2966 SDLoc dl(N);
2967
2968 SDValue VecOp = N->getOperand(OpNo);
2969 EVT VecVT = VecOp.getValueType();
2970 assert(VecVT.isVector() && "Can only split reduce vector operand");
2971 GetSplitVector(VecOp, Lo, Hi);
2972 EVT LoOpVT, HiOpVT;
2973 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2974
2975 // Use the appropriate scalar instruction on the split subvectors before
2976 // reducing the now partially reduced smaller vector.
2977 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
2978 SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
2979 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
2980}
2981
2982SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
2983 EVT ResVT = N->getValueType(0);
2984 SDValue Lo, Hi;
2985 SDLoc dl(N);
2986
2987 SDValue AccOp = N->getOperand(0);
2988 SDValue VecOp = N->getOperand(1);
2989 SDNodeFlags Flags = N->getFlags();
2990
2991 EVT VecVT = VecOp.getValueType();
2992 assert(VecVT.isVector() && "Can only split reduce vector operand");
2993 GetSplitVector(VecOp, Lo, Hi);
2994 EVT LoOpVT, HiOpVT;
2995 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2996
2997 // Reduce low half.
2998 SDValue Partial = DAG.getNode(N->getOpcode(), dl, ResVT, AccOp, Lo, Flags);
2999
3000 // Reduce high half, using low half result as initial value.
3001 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, Hi, Flags);
3002}
3003
3004SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
3005 assert(N->isVPOpcode() && "Expected VP opcode");
3006 assert(OpNo == 1 && "Can only split reduce vector operand");
3007
3008 unsigned Opc = N->getOpcode();
3009 EVT ResVT = N->getValueType(0);
3010 SDValue Lo, Hi;
3011 SDLoc dl(N);
3012
3013 SDValue VecOp = N->getOperand(OpNo);
3014 EVT VecVT = VecOp.getValueType();
3015 assert(VecVT.isVector() && "Can only split reduce vector operand");
3016 GetSplitVector(VecOp, Lo, Hi);
3017
3018 SDValue MaskLo, MaskHi;
3019 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2));
3020
3021 SDValue EVLLo, EVLHi;
3022 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(N->getOperand(3), VecVT, dl);
3023
3024 const SDNodeFlags Flags = N->getFlags();
3025
3026 SDValue ResLo =
3027 DAG.getNode(Opc, dl, ResVT, {N->getOperand(0), Lo, MaskLo, EVLLo}, Flags);
3028 return DAG.getNode(Opc, dl, ResVT, {ResLo, Hi, MaskHi, EVLHi}, Flags);
3029}
3030
3031SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
3032 // The result has a legal vector type, but the input needs splitting.
3033 EVT ResVT = N->getValueType(0);
3034 SDValue Lo, Hi;
3035 SDLoc dl(N);
3036 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3037 EVT InVT = Lo.getValueType();
3038
3039 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
3040 InVT.getVectorElementCount());
3041
3042 if (N->isStrictFPOpcode()) {
3043 Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
3044 { N->getOperand(0), Lo });
3045 Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
3046 { N->getOperand(0), Hi });
3047
3048 // Build a factor node to remember that this operation is independent
3049 // of the other one.
3050 SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3051 Hi.getValue(1));
3052
3053 // Legalize the chain result - switch anything that used the old chain to
3054 // use the new one.
3055 ReplaceValueWith(SDValue(N, 1), Ch);
3056 } else if (N->getNumOperands() == 3) {
3057 assert(N->isVPOpcode() && "Expected VP opcode");
3058 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3059 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
3060 std::tie(EVLLo, EVLHi) =
3061 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
3062 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo, MaskLo, EVLLo);
3063 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi, MaskHi, EVLHi);
3064 } else {
3065 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
3066 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
3067 }
3068
3069 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
3070}
3071
3072SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
3073 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
3074 // end up being split all the way down to individual components. Convert the
3075 // split pieces into integers and reassemble.
3076 SDValue Lo, Hi;
3077 GetSplitVector(N->getOperand(0), Lo, Hi);
3078 Lo = BitConvertToInteger(Lo);
3079 Hi = BitConvertToInteger(Hi);
3080
3081 if (DAG.getDataLayout().isBigEndian())
3082 std::swap(Lo, Hi);
3083
3084 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
3085 JoinIntegers(Lo, Hi));
3086}
3087
3088SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
3089 unsigned OpNo) {
3090 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
3091 // We know that the result type is legal.
3092 EVT ResVT = N->getValueType(0);
3093
3094 SDValue Vec = N->getOperand(0);
3095 SDValue SubVec = N->getOperand(1);
3096 SDValue Idx = N->getOperand(2);
3097 SDLoc dl(N);
3098
3099 SDValue Lo, Hi;
3100 GetSplitVector(SubVec, Lo, Hi);
3101
3102 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3103 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
3104
3105 SDValue FirstInsertion =
3106 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Lo, Idx);
3107 SDValue SecondInsertion =
3108 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, FirstInsertion, Hi,
3109 DAG.getVectorIdxConstant(IdxVal + LoElts, dl));
3110
3111 return SecondInsertion;
3112}
3113
3114SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
3115 // We know that the extracted result type is legal.
3116 EVT SubVT = N->getValueType(0);
3117 SDValue Idx = N->getOperand(1);
3118 SDLoc dl(N);
3119 SDValue Lo, Hi;
3120
3121 GetSplitVector(N->getOperand(0), Lo, Hi);
3122
3123 uint64_t LoEltsMin = Lo.getValueType().getVectorMinNumElements();
3124 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3125
3126 if (IdxVal < LoEltsMin) {
3127 assert(IdxVal + SubVT.getVectorMinNumElements() <= LoEltsMin &&
3128 "Extracted subvector crosses vector split!");
3129 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
3130 } else if (SubVT.isScalableVector() ==
3131 N->getOperand(0).getValueType().isScalableVector())
3132 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
3133 DAG.getVectorIdxConstant(IdxVal - LoEltsMin, dl));
3134
3135 // After this point the DAG node only permits extracting fixed-width
3136 // subvectors from scalable vectors.
3137 assert(SubVT.isFixedLengthVector() &&
3138 "Extracting scalable subvector from fixed-width unsupported");
3139
3140 // If the element type is i1 and we're not promoting the result, then we may
3141 // end up loading the wrong data since the bits are packed tightly into
3142 // bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal)
3143 // type at index 4, then we will load a byte starting at index 0.
3144 if (SubVT.getScalarType() == MVT::i1)
3145 report_fatal_error("Don't know how to extract fixed-width predicate "
3146 "subvector from a scalable predicate vector");
3147
3148 // Spill the vector to the stack. We should use the alignment for
3149 // the smallest part.
3150 SDValue Vec = N->getOperand(0);
3151 EVT VecVT = Vec.getValueType();
3152 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
3154 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
3155 auto &MF = DAG.getMachineFunction();
3156 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
3157 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
3158
3159 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
3160 SmallestAlign);
3161
3162 // Extract the subvector by loading the correct part.
3163 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVT, Idx);
3164
3165 return DAG.getLoad(
3166 SubVT, dl, Store, StackPtr,
3167 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
3168}
3169
3170SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3171 SDValue Vec = N->getOperand(0);
3172 SDValue Idx = N->getOperand(1);
3173 EVT VecVT = Vec.getValueType();
3174
3175 if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Idx)) {
3176 uint64_t IdxVal = Index->getZExtValue();
3177
3178 SDValue Lo, Hi;
3179 GetSplitVector(Vec, Lo, Hi);
3180
3181 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
3182
3183 if (IdxVal < LoElts)
3184 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
3185 else if (!Vec.getValueType().isScalableVector())
3186 return SDValue(DAG.UpdateNodeOperands(N, Hi,
3187 DAG.getConstant(IdxVal - LoElts, SDLoc(N),
3188 Idx.getValueType())), 0);
3189 }
3190
3191 // See if the target wants to custom expand this node.
3192 if (CustomLowerNode(N, N->getValueType(0), true))
3193 return SDValue();
3194
3195 // Make the vector elements byte-addressable if they aren't already.
3196 SDLoc dl(N);
3197 EVT EltVT = VecVT.getVectorElementType();
3198 if (VecVT.getScalarSizeInBits() < 8) {
3199 EltVT = MVT::i8;
3200 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
3201 VecVT.getVectorElementCount());
3202 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
3203 }
3204
3205 // Store the vector to the stack.
3206 // In cases where the vector is illegal it will be broken down into parts
3207 // and stored in parts - we should use the alignment for the smallest part.
3208 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
3210 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
3211 auto &MF = DAG.getMachineFunction();
3212 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
3213 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
3214 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
3215 SmallestAlign);
3216
3217 // Load back the required element.
3218 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
3219
3220 // FIXME: This is to handle i1 vectors with elements promoted to i8.
3221 // i1 vector handling needs general improvement.
3222 if (N->getValueType(0).bitsLT(EltVT)) {
3223 SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
3224 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
3225 return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
3226 }
3227
3228 return DAG.getExtLoad(
3229 ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
3230 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT,
3231 commonAlignment(SmallestAlign, EltVT.getFixedSizeInBits() / 8));
3232}
3233
3234SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
3235 SDValue Lo, Hi;
3236
3237 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
3238 // splitting the result has the same effect as splitting the input operand.
3239 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
3240
3241 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
3242}
3243
3244SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
3245 (void)OpNo;
3246 SDValue Lo, Hi;
3247 SplitVecRes_Gather(N, Lo, Hi);
3248
3249 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, N, N->getValueType(0), Lo, Hi);
3250 ReplaceValueWith(SDValue(N, 0), Res);
3251 return SDValue();
3252}
3253
3254SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
3255 assert(N->isUnindexed() && "Indexed vp_store of vector?");
3256 SDValue Ch = N->getChain();
3257 SDValue Ptr = N->getBasePtr();
3258 SDValue Offset = N->getOffset();
3259 assert(Offset.isUndef() && "Unexpected VP store offset");
3260 SDValue Mask = N->getMask();
3261 SDValue EVL = N->getVectorLength();
3262 SDValue Data = N->getValue();
3263 Align Alignment = N->getOriginalAlign();
3264 SDLoc DL(N);
3265
3266 SDValue DataLo, DataHi;
3267 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3268 // Split Data operand
3269 GetSplitVector(Data, DataLo, DataHi);
3270 else
3271 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
3272
3273 // Split Mask operand
3274 SDValue MaskLo, MaskHi;
3275 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
3276 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
3277 } else {
3278 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
3279 GetSplitVector(Mask, MaskLo, MaskHi);
3280 else
3281 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
3282 }
3283
3284 EVT MemoryVT = N->getMemoryVT();
3285 EVT LoMemVT, HiMemVT;
3286 bool HiIsEmpty = false;
3287 std::tie(LoMemVT, HiMemVT) =
3288 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
3289
3290 // Split EVL
3291 SDValue EVLLo, EVLHi;
3292 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, Data.getValueType(), DL);
3293
3294 SDValue Lo, Hi;
3295 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3296 N->getPointerInfo(), MachineMemOperand::MOStore,
3297 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3298
3299 Lo = DAG.getStoreVP(Ch, DL, DataLo, Ptr, Offset, MaskLo, EVLLo, LoMemVT, MMO,
3300 N->getAddressingMode(), N->isTruncatingStore(),
3301 N->isCompressingStore());
3302
3303 // If the hi vp_store has zero storage size, only the lo vp_store is needed.
3304 if (HiIsEmpty)
3305 return Lo;
3306
3307 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
3308 N->isCompressingStore());
3309
3311 if (LoMemVT.isScalableVector()) {
3312 Alignment = commonAlignment(Alignment,
3313 LoMemVT.getSizeInBits().getKnownMinValue() / 8);
3314 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
3315 } else
3316 MPI = N->getPointerInfo().getWithOffset(
3317 LoMemVT.getStoreSize().getFixedValue());
3318
3319 MMO = DAG.getMachineFunction().getMachineMemOperand(
3321 N->getAAInfo(), N->getRanges());
3322
3323 Hi = DAG.getStoreVP(Ch, DL, DataHi, Ptr, Offset, MaskHi, EVLHi, HiMemVT, MMO,
3324 N->getAddressingMode(), N->isTruncatingStore(),
3325 N->isCompressingStore());
3326
3327 // Build a factor node to remember that this store is independent of the
3328 // other one.
3329 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3330}
3331
3332SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
3333 unsigned OpNo) {
3334 assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?");
3335 assert(N->getOffset().isUndef() && "Unexpected VP strided store offset");
3336
3337 SDLoc DL(N);
3338
3339 SDValue Data = N->getValue();
3340 SDValue LoData, HiData;
3341 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3342 GetSplitVector(Data, LoData, HiData);
3343 else
3344 std::tie(LoData, HiData) = DAG.SplitVector(Data, DL);
3345
3346 EVT LoMemVT, HiMemVT;
3347 bool HiIsEmpty = false;
3348 std::tie(LoMemVT, HiMemVT) = DAG.GetDependentSplitDestVTs(
3349 N->getMemoryVT(), LoData.getValueType(), &HiIsEmpty);
3350
3351 SDValue Mask = N->getMask();
3352 SDValue LoMask, HiMask;
3353 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC)
3354 SplitVecRes_SETCC(Mask.getNode(), LoMask, HiMask);
3355 else if (getTypeAction(Mask.getValueType()) ==
3357 GetSplitVector(Mask, LoMask, HiMask);
3358 else
3359 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
3360
3361 SDValue LoEVL, HiEVL;
3362 std::tie(LoEVL, HiEVL) =
3363 DAG.SplitEVL(N->getVectorLength(), Data.getValueType(), DL);
3364
3365 // Generate the low vp_strided_store
3366 SDValue Lo = DAG.getStridedStoreVP(
3367 N->getChain(), DL, LoData, N->getBasePtr(), N->getOffset(),
3368 N->getStride(), LoMask, LoEVL, LoMemVT, N->getMemOperand(),
3369 N->getAddressingMode(), N->isTruncatingStore(), N->isCompressingStore());
3370
3371 // If the high vp_strided_store has zero storage size, only the low
3372 // vp_strided_store is needed.
3373 if (HiIsEmpty)
3374 return Lo;
3375
3376 // Generate the high vp_strided_store.
3377 // To calculate the high base address, we need to sum to the low base
3378 // address stride number of bytes for each element already stored by low,
3379 // that is: Ptr = Ptr + (LoEVL * Stride)
3380 EVT PtrVT = N->getBasePtr().getValueType();
3382 DAG.getNode(ISD::MUL, DL, PtrVT, LoEVL,
3383 DAG.getSExtOrTrunc(N->getStride(), DL, PtrVT));
3384 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, N->getBasePtr(), Increment);
3385
3386 Align Alignment = N->getOriginalAlign();
3387 if (LoMemVT.isScalableVector())
3388 Alignment = commonAlignment(Alignment,
3389 LoMemVT.getSizeInBits().getKnownMinValue() / 8);
3390
3391 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3392 MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
3394 N->getAAInfo(), N->getRanges());
3395
3396 SDValue Hi = DAG.getStridedStoreVP(
3397 N->getChain(), DL, HiData, Ptr, N->getOffset(), N->getStride(), HiMask,
3398 HiEVL, HiMemVT, MMO, N->getAddressingMode(), N->isTruncatingStore(),
3399 N->isCompressingStore());
3400
3401 // Build a factor node to remember that this store is independent of the
3402 // other one.
3403 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3404}
3405
3406SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
3407 unsigned OpNo) {
3408 assert(N->isUnindexed() && "Indexed masked store of vector?");
3409 SDValue Ch = N->getChain();
3410 SDValue Ptr = N->getBasePtr();
3411 SDValue Offset = N->getOffset();
3412 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
3413 SDValue Mask = N->getMask();
3414 SDValue Data = N->getValue();
3415 Align Alignment = N->getOriginalAlign();
3416 SDLoc DL(N);
3417
3418 SDValue DataLo, DataHi;
3419 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3420 // Split Data operand
3421 GetSplitVector(Data, DataLo, DataHi);
3422 else
3423 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
3424
3425 // Split Mask operand
3426 SDValue MaskLo, MaskHi;
3427 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
3428 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
3429 } else {
3430 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
3431 GetSplitVector(Mask, MaskLo, MaskHi);
3432 else
3433 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
3434 }
3435
3436 EVT MemoryVT = N->getMemoryVT();
3437 EVT LoMemVT, HiMemVT;
3438 bool HiIsEmpty = false;
3439 std::tie(LoMemVT, HiMemVT) =
3440 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
3441
3442 SDValue Lo, Hi, Res;
3443 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3444 N->getPointerInfo(), MachineMemOperand::MOStore,
3445 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3446
3447 Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
3448 N->getAddressingMode(), N->isTruncatingStore(),
3449 N->isCompressingStore());
3450
3451 if (HiIsEmpty) {
3452 // The hi masked store has zero storage size.
3453 // Only the lo masked store is needed.
3454 Res = Lo;
3455 } else {
3456
3457 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
3458 N->isCompressingStore());
3459
3461 if (LoMemVT.isScalableVector()) {
3462 Alignment = commonAlignment(
3463 Alignment, LoMemVT.getSizeInBits().getKnownMinValue() / 8);
3464 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
3465 } else
3466 MPI = N->getPointerInfo().getWithOffset(
3467 LoMemVT.getStoreSize().getFixedValue());
3468
3469 MMO = DAG.getMachineFunction().getMachineMemOperand(
3471 N->getAAInfo(), N->getRanges());
3472
3473 Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
3474 N->getAddressingMode(), N->isTruncatingStore(),
3475 N->isCompressingStore());
3476
3477 // Build a factor node to remember that this store is independent of the
3478 // other one.
3479 Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3480 }
3481
3482 return Res;
3483}
3484
3485SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
3486 SDValue Ch = N->getChain();
3487 SDValue Ptr = N->getBasePtr();
3488 EVT MemoryVT = N->getMemoryVT();
3489 Align Alignment = N->getOriginalAlign();
3490 SDLoc DL(N);
3491 struct Operands {
3492 SDValue Mask;
3493 SDValue Index;
3494 SDValue Scale;
3495 SDValue Data;
3496 } Ops = [&]() -> Operands {
3497 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
3498 return {MSC->getMask(), MSC->getIndex(), MSC->getScale(),
3499 MSC->getValue()};
3500 }
3501 auto *VPSC = cast<VPScatterSDNode>(N);
3502 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale(),
3503 VPSC->getValue()};
3504 }();
3505 // Split all operands
3506
3507 EVT LoMemVT, HiMemVT;
3508 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
3509
3510 SDValue DataLo, DataHi;
3511 if (getTypeAction(Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
3512 // Split Data operand
3513 GetSplitVector(Ops.Data, DataLo, DataHi);
3514 else
3515 std::tie(DataLo, DataHi) = DAG.SplitVector(Ops.Data, DL);
3516
3517 // Split Mask operand
3518 SDValue MaskLo, MaskHi;
3519 if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
3520 SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
3521 } else {
3522 std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, DL);
3523 }
3524
3525 SDValue IndexHi, IndexLo;
3526 if (getTypeAction(Ops.Index.getValueType()) ==
3528 GetSplitVector(Ops.Index, IndexLo, IndexHi);
3529 else
3530 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, DL);
3531
3532 SDValue Lo;
3533 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3534 N->getPointerInfo(), MachineMemOperand::MOStore,
3535 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3536
3537 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
3538 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
3539 Lo =
3540 DAG.getMaskedScatter(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO,
3541 MSC->getIndexType(), MSC->isTruncatingStore());
3542
3543 // The order of the Scatter operation after split is well defined. The "Hi"
3544 // part comes after the "Lo". So these two operations should be chained one
3545 // after another.
3546 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
3547 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi,
3548 MMO, MSC->getIndexType(),
3549 MSC->isTruncatingStore());
3550 }
3551 auto *VPSC = cast<VPScatterSDNode>(N);
3552 SDValue EVLLo, EVLHi;
3553 std::tie(EVLLo, EVLHi) =
3554 DAG.SplitEVL(VPSC->getVectorLength(), Ops.Data.getValueType(), DL);
3555
3556 SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
3557 Lo = DAG.getScatterVP(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO,
3558 VPSC->getIndexType());
3559
3560 // The order of the Scatter operation after split is well defined. The "Hi"
3561 // part comes after the "Lo". So these two operations should be chained one
3562 // after another.
3563 SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
3564 return DAG.getScatterVP(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi, MMO,
3565 VPSC->getIndexType());
3566}
3567
3568SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
3569 assert(N->isUnindexed() && "Indexed store of vector?");
3570 assert(OpNo == 1 && "Can only split the stored value");
3571 SDLoc DL(N);
3572
3573 bool isTruncating = N->isTruncatingStore();
3574 SDValue Ch = N->getChain();
3575 SDValue Ptr = N->getBasePtr();
3576 EVT MemoryVT = N->getMemoryVT();
3577 Align Alignment = N->getOriginalAlign();
3578 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3579 AAMDNodes AAInfo = N->getAAInfo();
3580 SDValue Lo, Hi;
3581 GetSplitVector(N->getOperand(1), Lo, Hi);
3582
3583 EVT LoMemVT, HiMemVT;
3584 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
3585
3586 // Scalarize if the split halves are not byte-sized.
3587 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
3588 return TLI.scalarizeVectorStore(N, DAG);
3589
3590 if (isTruncating)
3591 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
3592 Alignment, MMOFlags, AAInfo);
3593 else
3594 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3595 AAInfo);
3596
3598 IncrementPointer(N, LoMemVT, MPI, Ptr);
3599
3600 if (isTruncating)
3601 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, MPI,
3602 HiMemVT, Alignment, MMOFlags, AAInfo);
3603 else
3604 Hi = DAG.getStore(Ch, DL, Hi, Ptr, MPI, Alignment, MMOFlags, AAInfo);
3605
3606 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3607}
3608
3609SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
3610 SDLoc DL(N);
3611
3612 // The input operands all must have the same type, and we know the result
3613 // type is valid. Convert this to a buildvector which extracts all the
3614 // input elements.
3615 // TODO: If the input elements are power-two vectors, we could convert this to
3616 // a new CONCAT_VECTORS node with elements that are half-wide.
3618 EVT EltVT = N->getValueType(0).getVectorElementType();
3619 for (const SDValue &Op : N->op_values()) {
3620 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
3621 i != e; ++i) {
3622 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
3623 DAG.getVectorIdxConstant(i, DL)));
3624 }
3625 }
3626
3627 return DAG.getBuildVector(N->getValueType(0), DL, Elts);
3628}
3629
3630SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
3631 // The result type is legal, but the input type is illegal. If splitting
3632 // ends up with the result type of each half still being legal, just
3633 // do that. If, however, that would result in an illegal result type,
3634 // we can try to get more clever with power-two vectors. Specifically,
3635 // split the input type, but also widen the result element size, then
3636 // concatenate the halves and truncate again. For example, consider a target
3637 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
3638 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
3639 // %inlo = v4i32 extract_subvector %in, 0
3640 // %inhi = v4i32 extract_subvector %in, 4
3641 // %lo16 = v4i16 trunc v4i32 %inlo
3642 // %hi16 = v4i16 trunc v4i32 %inhi
3643 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
3644 // %res = v8i8 trunc v8i16 %in16
3645 //
3646 // Without this transform, the original truncate would end up being
3647 // scalarized, which is pretty much always a last resort.
3648 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
3649 SDValue InVec = N->getOperand(OpNo);
3650 EVT InVT = InVec->getValueType(0);
3651 EVT OutVT = N->getValueType(0);
3652 ElementCount NumElements = OutVT.getVectorElementCount();
3653 bool IsFloat = OutVT.isFloatingPoint();
3654
3655 unsigned InElementSize = InVT.getScalarSizeInBits();
3656 unsigned OutElementSize = OutVT.getScalarSizeInBits();
3657
3658 // Determine the split output VT. If its legal we can just split dirctly.
3659 EVT LoOutVT, HiOutVT;
3660 std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
3661 assert(LoOutVT == HiOutVT && "Unequal split?");
3662
3663 // If the input elements are only 1/2 the width of the result elements,
3664 // just use the normal splitting. Our trick only work if there's room
3665 // to split more than once.
3666 if (isTypeLegal(LoOutVT) ||
3667 InElementSize <= OutElementSize * 2)
3668 return SplitVecOp_UnaryOp(N);
3669 SDLoc DL(N);
3670
3671 // Don't touch if this will be scalarized.
3672 EVT FinalVT = InVT;
3673 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
3674 FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
3675
3676 if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector)
3677 return SplitVecOp_UnaryOp(N);
3678
3679 // Get the split input vector.
3680 SDValue InLoVec, InHiVec;
3681 GetSplitVector(InVec, InLoVec, InHiVec);
3682
3683 // Truncate them to 1/2 the element size.
3684 //
3685 // This assumes the number of elements is a power of two; any vector that
3686 // isn't should be widened, not split.
3687 EVT HalfElementVT = IsFloat ?
3688 EVT::getFloatingPointVT(InElementSize/2) :
3689 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
3690 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
3691 NumElements.divideCoefficientBy(2));
3692
3693 SDValue HalfLo;
3694 SDValue HalfHi;
3695 SDValue Chain;
3696 if (N->isStrictFPOpcode()) {
3697 HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
3698 {N->getOperand(0), InLoVec});
3699 HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
3700 {N->getOperand(0), InHiVec});
3701 // Legalize the chain result - switch anything that used the old chain to
3702 // use the new one.
3703 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1),
3704 HalfHi.getValue(1));
3705 } else {
3706 HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
3707 HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
3708 }
3709
3710 // Concatenate them to get the full intermediate truncation result.
3711 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
3712 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
3713 HalfHi);
3714 // Now finish up by truncating all the way down to the original result
3715 // type. This should normally be something that ends up being legal directly,
3716 // but in theory if a target has very wide vectors and an annoyingly
3717 // restricted set of legal types, this split can chain to build things up.
3718
3719 if (N->isStrictFPOpcode()) {
3720 SDValue Res = DAG.getNode(
3722 {Chain, InterVec,
3723 DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))});
3724 // Relink the chain
3725 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
3726 return Res;
3727 }
3728
3729 return IsFloat
3730 ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
3731 DAG.getTargetConstant(
3732 0, DL, TLI.getPointerTy(DAG.getDataLayout())))
3733 : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
3734}
3735
3736SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
3737 assert(N->getValueType(0).isVector() &&
3738 N->getOperand(0).getValueType().isVector() &&
3739 "Operand types must be vectors");
3740 // The result has a legal vector type, but the input needs splitting.
3741 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
3742 SDLoc DL(N);
3743 GetSplitVector(N->getOperand(0), Lo0, Hi0);
3744 GetSplitVector(N->getOperand(1), Lo1, Hi1);
3745 auto PartEltCnt = Lo0.getValueType().getVectorElementCount();
3746
3747 LLVMContext &Context = *DAG.getContext();
3748 EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt);
3749 EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2);
3750
3751 if (N->getOpcode() == ISD::SETCC) {
3752 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
3753 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
3754 } else {
3755 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
3756 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3757 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
3758 std::tie(EVLLo, EVLHi) =
3759 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), DL);
3760 LoRes = DAG.getNode(ISD::VP_SETCC, DL, PartResVT, Lo0, Lo1,
3761 N->getOperand(2), MaskLo, EVLLo);
3762 HiRes = DAG.getNode(ISD::VP_SETCC, DL, PartResVT, Hi0, Hi1,
3763 N->getOperand(2), MaskHi, EVLHi);
3764 }
3765 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
3766
3767 EVT OpVT = N->getOperand(0).getValueType();
3768 ISD::NodeType ExtendCode =
3770 return DAG.getNode(ExtendCode, DL, N->getValueType(0), Con);
3771}
3772
3773
3774SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
3775 // The result has a legal vector type, but the input needs splitting.
3776 EVT ResVT = N->getValueType(0);
3777 SDValue Lo, Hi;
3778 SDLoc DL(N);
3779 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3780 EVT InVT = Lo.getValueType();
3781
3782 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
3783 InVT.getVectorElementCount());
3784
3785 if (N->isStrictFPOpcode()) {
3786 Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
3787 { N->getOperand(0), Lo, N->getOperand(2) });
3788 Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
3789 { N->getOperand(0), Hi, N->getOperand(2) });
3790 // Legalize the chain result - switch anything that used the old chain to
3791 // use the new one.
3792 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
3793 Lo.getValue(1), Hi.getValue(1));
3794 ReplaceValueWith(SDValue(N, 1), NewChain);
3795 } else if (N->getOpcode() == ISD::VP_FP_ROUND) {
3796 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3797 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
3798 std::tie(EVLLo, EVLHi) =
3799 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), DL);
3800 Lo = DAG.getNode(ISD::VP_FP_ROUND, DL, OutVT, Lo, MaskLo, EVLLo);
3801 Hi = DAG.getNode(ISD::VP_FP_ROUND, DL, OutVT, Hi, MaskHi, EVLHi);
3802 } else {
3803 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
3804 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
3805 }
3806
3807 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
3808}
3809
3810SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
3811 // The result (and the first input) has a legal vector type, but the second
3812 // input needs splitting.
3813
3814 SDLoc DL(N);
3815
3816 EVT LHSLoVT, LHSHiVT;
3817 std::tie(LHSLoVT, LHSHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
3818
3819 if (!isTypeLegal(LHSLoVT) || !isTypeLegal(LHSHiVT))
3820 return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
3821
3822 SDValue LHSLo, LHSHi;
3823 std::tie(LHSLo, LHSHi) =
3824 DAG.SplitVector(N->getOperand(0), DL, LHSLoVT, LHSHiVT);
3825
3826 SDValue RHSLo, RHSHi;
3827 std::tie(RHSLo, RHSHi) = DAG.SplitVector(N->getOperand(1), DL);
3828
3829 SDValue Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLoVT, LHSLo, RHSLo);
3830 SDValue Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHiVT, LHSHi, RHSHi);
3831
3832 return DAG.getNode(ISD::CONCAT_VECTORS, DL, N->getValueType(0), Lo, Hi);
3833}
3834
3835SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
3836 EVT ResVT = N->getValueType(0);
3837 SDValue Lo, Hi;
3838 SDLoc dl(N);
3839 GetSplitVector(N->getOperand(0), Lo, Hi);
3840 EVT InVT = Lo.getValueType();
3841
3842 EVT NewResVT =
3843 EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
3844 InVT.getVectorElementCount());
3845
3846 Lo = DAG.getNode(N->getOpcode(), dl, NewResVT, Lo, N->getOperand(1));
3847 Hi = DAG.getNode(N->getOpcode(), dl, NewResVT, Hi, N->getOperand(1));
3848
3849 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
3850}
3851
3852//===----------------------------------------------------------------------===//
3853// Result Vector Widening
3854//===----------------------------------------------------------------------===//
3855
3856void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
3857 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
3858 dbgs() << "\n");
3859
3860 // See if the target wants to custom widen this node.
3861 if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
3862 return;
3863
3864 SDValue Res = SDValue();
3865
3866 auto unrollExpandedOp = [&]() {
3867 // We're going to widen this vector op to a legal type by padding with undef
3868 // elements. If the wide vector op is eventually going to be expanded to
3869 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
3870 // libcalls on the undef elements.
3871 EVT VT = N->getValueType(0);
3872 EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3873 if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
3874 TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
3875 Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
3876 return true;
3877 }
3878 return false;
3879 };
3880
3881 switch (N->getOpcode()) {
3882 default:
3883#ifndef NDEBUG
3884 dbgs() << "WidenVectorResult #" << ResNo << ": ";
3885 N->dump(&DAG);
3886 dbgs() << "\n";
3887#endif
3888 llvm_unreachable("Do not know how to widen the result of this operator!");
3889
3890 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
3891 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
3892 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
3893 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
3895 Res = WidenVecRes_INSERT_SUBVECTOR(N);
3896 break;
3897 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
3898 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
3899 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
3900 case ISD::STEP_VECTOR:
3901 case ISD::SPLAT_VECTOR:
3903 Res = WidenVecRes_ScalarOp(N);
3904 break;
3905 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
3906 case ISD::VSELECT:
3907 case ISD::SELECT:
3908 case ISD::VP_SELECT:
3909 case ISD::VP_MERGE:
3910 Res = WidenVecRes_Select(N);
3911 break;
3912 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
3913 case ISD::VP_SETCC:
3914 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
3915 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
3917 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
3918 break;
3919 case ISD::VP_LOAD:
3920 Res = WidenVecRes_VP_LOAD(cast<VPLoadSDNode>(N));
3921 break;
3922 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
3923 Res = WidenVecRes_VP_STRIDED_LOAD(cast<VPStridedLoadSDNode>(N));
3924 break;
3925 case ISD::MLOAD:
3926 Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
3927 break;
3928 case ISD::MGATHER:
3929 Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
3930 break;
3931 case ISD::VP_GATHER:
3932 Res = WidenVecRes_VP_GATHER(cast<VPGatherSDNode>(N));
3933 break;
3935 Res = WidenVecRes_VECTOR_REVERSE(N);
3936 break;
3937
3938 case ISD::ADD: case ISD::VP_ADD:
3939 case ISD::AND: case ISD::VP_AND:
3940 case ISD::MUL: case ISD::VP_MUL:
3941 case ISD::MULHS:
3942 case ISD::MULHU:
3943 case ISD::OR: case ISD::VP_OR:
3944 case ISD::SUB: case ISD::VP_SUB:
3945 case ISD::XOR: case ISD::VP_XOR:
3946 case ISD::SHL: case ISD::VP_SHL:
3947 case ISD::SRA: case ISD::VP_ASHR:
3948 case ISD::SRL: case ISD::VP_LSHR:
3949 case ISD::FMINNUM: case ISD::VP_FMINNUM:
3950 case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
3951 case ISD::FMINIMUM:
3952 case ISD::FMAXIMUM:
3953 case ISD::SMIN: case ISD::VP_SMIN:
3954 case ISD::SMAX: case ISD::VP_SMAX:
3955 case ISD::UMIN: case ISD::VP_UMIN:
3956 case ISD::UMAX: case ISD::VP_UMAX:
3957 case ISD::UADDSAT:
3958 case ISD::SADDSAT:
3959 case ISD::USUBSAT:
3960 case ISD::SSUBSAT:
3961 case ISD::SSHLSAT:
3962 case ISD::USHLSAT:
3963 case ISD::ROTL:
3964 case ISD::ROTR:
3965 case ISD::AVGFLOORS:
3966 case ISD::AVGFLOORU:
3967 case ISD::AVGCEILS:
3968 case ISD::AVGCEILU:
3969 // Vector-predicated binary op widening. Note that -- unlike the
3970 // unpredicated versions -- we don't have to worry about trapping on
3971 // operations like UDIV, FADD, etc., as we pass on the original vector
3972 // length parameter. This means the widened elements containing garbage
3973 // aren't active.
3974 case ISD::VP_SDIV:
3975 case ISD::VP_UDIV:
3976 case ISD::VP_SREM:
3977 case ISD::VP_UREM:
3978 case ISD::VP_FADD:
3979 case ISD::VP_FSUB:
3980 case ISD::VP_FMUL:
3981 case ISD::VP_FDIV:
3982 case ISD::VP_FREM:
3983 case ISD::VP_FCOPYSIGN:
3984 Res = WidenVecRes_Binary(N);
3985 break;
3986
3987 case ISD::FPOW:
3988 case ISD::FREM:
3989 if (unrollExpandedOp())
3990 break;
3991 // If the target has custom/legal support for the scalar FP intrinsic ops
3992 // (they are probably not destined to become libcalls), then widen those
3993 // like any other binary ops.
3994 [[fallthrough]];
3995
3996 case ISD::FADD:
3997 case ISD::FMUL:
3998 case ISD::FSUB:
3999 case ISD::FDIV:
4000 case ISD::SDIV:
4001 case ISD::UDIV:
4002 case ISD::SREM:
4003 case ISD::UREM:
4004 Res = WidenVecRes_BinaryCanTrap(N);
4005 break;
4006
4007 case ISD::SMULFIX:
4008 case ISD::SMULFIXSAT:
4009 case ISD::UMULFIX:
4010 case ISD::UMULFIXSAT:
4011 // These are binary operations, but with an extra operand that shouldn't
4012 // be widened (the scale).
4013 Res = WidenVecRes_BinaryWithExtraScalarOp(N);
4014 break;
4015
4016#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
4017 case ISD::STRICT_##DAGN:
4018#include "llvm/IR/ConstrainedOps.def"
4019 Res = WidenVecRes_StrictFP(N);
4020 break;
4021
4022 case ISD::UADDO:
4023 case ISD::SADDO:
4024 case ISD::USUBO:
4025 case ISD::SSUBO:
4026 case ISD::UMULO:
4027 case ISD::SMULO:
4028 Res = WidenVecRes_OverflowOp(N, ResNo);
4029 break;
4030
4031 case ISD::FCOPYSIGN:
4032 Res = WidenVecRes_FCOPYSIGN(N);
4033 break;
4034
4035 case ISD::IS_FPCLASS:
4036 Res = WidenVecRes_IS_FPCLASS(N);
4037 break;
4038
4039 case ISD::FPOWI:
4040 Res = WidenVecRes_POWI(N);
4041 break;
4042
4046 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
4047 break;
4048
4049 case ISD::ANY_EXTEND:
4050 case ISD::FP_EXTEND:
4051 case ISD::VP_FP_EXTEND:
4052 case ISD::FP_ROUND:
4053 case ISD::VP_FP_ROUND:
4054 case ISD::FP_TO_SINT:
4055 case ISD::VP_FP_TO_SINT:
4056 case ISD::FP_TO_UINT:
4057 case ISD::VP_FP_TO_UINT:
4058 case ISD::SIGN_EXTEND:
4059 case ISD::VP_SIGN_EXTEND:
4060 case ISD::SINT_TO_FP:
4061 case ISD::VP_SINT_TO_FP:
4062 case ISD::VP_TRUNCATE:
4063 case ISD::TRUNCATE:
4064 case ISD::UINT_TO_FP:
4065 case ISD::VP_UINT_TO_FP:
4066 case ISD::ZERO_EXTEND:
4067 case ISD::VP_ZERO_EXTEND:
4068 Res = WidenVecRes_Convert(N);
4069 break;
4070
4073 Res = WidenVecRes_FP_TO_XINT_SAT(N);
4074 break;
4075
4076 case ISD::FABS:
4077 case ISD::FCEIL:
4078 case ISD::FCOS:
4079 case ISD::FEXP:
4080 case ISD::FEXP2:
4081 case ISD::FFLOOR:
4082 case ISD::FLOG:
4083 case ISD::FLOG10:
4084 case ISD::FLOG2:
4085 case ISD::FNEARBYINT:
4086 case ISD::FRINT:
4087 case ISD::FROUND:
4088 case ISD::FROUNDEVEN:
4089 case ISD::FSIN:
4090 case ISD::FSQRT:
4091 case ISD::FTRUNC:
4092 if (unrollExpandedOp())
4093 break;
4094 // If the target has custom/legal support for the scalar FP intrinsic ops
4095 // (they are probably not destined to become libcalls), then widen those
4096 // like any other unary ops.
4097 [[fallthrough]];
4098
4099 case ISD::ABS:
4100 case ISD::VP_ABS:
4101 case ISD::BITREVERSE:
4102 case ISD::VP_BITREVERSE:
4103 case ISD::BSWAP:
4104 case ISD::VP_BSWAP:
4105 case ISD::CTLZ:
4106 case ISD::VP_CTLZ:
4108 case ISD::VP_CTLZ_ZERO_UNDEF:
4109 case ISD::CTPOP:
4110 case ISD::VP_CTPOP:
4111 case ISD::CTTZ:
4112 case ISD::VP_CTTZ:
4114 case ISD::VP_CTTZ_ZERO_UNDEF:
4115 case ISD::FNEG: case ISD::VP_FNEG:
4116 case ISD::VP_FABS:
4117 case ISD::VP_SQRT:
4118 case ISD::VP_FCEIL:
4119 case ISD::VP_FFLOOR:
4120 case ISD::VP_FRINT:
4121 case ISD::VP_FNEARBYINT:
4122 case ISD::VP_FROUND:
4123 case ISD::VP_FROUNDEVEN:
4124 case ISD::VP_FROUNDTOZERO:
4125 case ISD::FREEZE:
4126 case ISD::ARITH_FENCE:
4127 case