LLVM 18.0.0git
LegalizeVectorOps.cpp
Go to the documentation of this file.
1//===- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::LegalizeVectors method.
10//
11// The vector legalizer looks for vector operations which might need to be
12// scalarized and legalizes them. This is a separate step from Legalize because
13// scalarizing can introduce illegal types. For example, suppose we have an
14// ISD::SDIV of type v2i64 on x86-32. The type is legal (for example, addition
15// on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the
16// operation, which introduces nodes with the illegal type i64 which must be
17// expanded. Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC;
18// the operation must be unrolled, which introduces nodes with the illegal
19// type i8 which must be promoted.
20//
21// This does not legalize vector manipulations like ISD::BUILD_VECTOR,
22// or operations that happen to take a vector which are custom-lowered;
23// the legalization for such operations never produces nodes
24// with illegal types, so it's okay to put off legalizing them until
25// SelectionDAG::Legalize runs.
26//
27//===----------------------------------------------------------------------===//
28
29#include "llvm/ADT/DenseMap.h"
37#include "llvm/IR/DataLayout.h"
40#include "llvm/Support/Debug.h"
42#include <cassert>
43#include <cstdint>
44#include <iterator>
45#include <utility>
46
47using namespace llvm;
48
49#define DEBUG_TYPE "legalizevectorops"
50
51namespace {
52
53class VectorLegalizer {
54 SelectionDAG& DAG;
55 const TargetLowering &TLI;
56 bool Changed = false; // Keep track of whether anything changed
57
58 /// For nodes that are of legal width, and that have more than one use, this
59 /// map indicates what regularized operand to use. This allows us to avoid
60 /// legalizing the same thing more than once.
62
63 /// Adds a node to the translation cache.
64 void AddLegalizedOperand(SDValue From, SDValue To) {
65 LegalizedNodes.insert(std::make_pair(From, To));
66 // If someone requests legalization of the new node, return itself.
67 if (From != To)
68 LegalizedNodes.insert(std::make_pair(To, To));
69 }
70
71 /// Legalizes the given node.
72 SDValue LegalizeOp(SDValue Op);
73
74 /// Assuming the node is legal, "legalize" the results.
75 SDValue TranslateLegalizeResults(SDValue Op, SDNode *Result);
76
77 /// Make sure Results are legal and update the translation cache.
78 SDValue RecursivelyLegalizeResults(SDValue Op,
80
81 /// Wrapper to interface LowerOperation with a vector of Results.
82 /// Returns false if the target wants to use default expansion. Otherwise
83 /// returns true. If return is true and the Results are empty, then the
84 /// target wants to keep the input node as is.
85 bool LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results);
86
87 /// Implements unrolling a VSETCC.
88 SDValue UnrollVSETCC(SDNode *Node);
89
90 /// Implement expand-based legalization of vector operations.
91 ///
92 /// This is just a high-level routine to dispatch to specific code paths for
93 /// operations to legalize them.
95
96 /// Implements expansion for FP_TO_UINT; falls back to UnrollVectorOp if
97 /// FP_TO_SINT isn't legal.
98 void ExpandFP_TO_UINT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
99
100 /// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if
101 /// SINT_TO_FLOAT and SHR on vectors isn't legal.
102 void ExpandUINT_TO_FLOAT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
103
104 /// Implement expansion for SIGN_EXTEND_INREG using SRL and SRA.
105 SDValue ExpandSEXTINREG(SDNode *Node);
106
107 /// Implement expansion for ANY_EXTEND_VECTOR_INREG.
108 ///
109 /// Shuffles the low lanes of the operand into place and bitcasts to the proper
110 /// type. The contents of the bits in the extended part of each element are
111 /// undef.
112 SDValue ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node);
113
114 /// Implement expansion for SIGN_EXTEND_VECTOR_INREG.
115 ///
116 /// Shuffles the low lanes of the operand into place, bitcasts to the proper
117 /// type, then shifts left and arithmetic shifts right to introduce a sign
118 /// extension.
119 SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node);
120
121 /// Implement expansion for ZERO_EXTEND_VECTOR_INREG.
122 ///
123 /// Shuffles the low lanes of the operand into place and blends zeros into
124 /// the remaining lanes, finally bitcasting to the proper type.
125 SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node);
126
127 /// Expand bswap of vectors into a shuffle if legal.
128 SDValue ExpandBSWAP(SDNode *Node);
129
130 /// Implement vselect in terms of XOR, AND, OR when blend is not
131 /// supported by the target.
132 SDValue ExpandVSELECT(SDNode *Node);
133 SDValue ExpandVP_SELECT(SDNode *Node);
134 SDValue ExpandVP_MERGE(SDNode *Node);
135 SDValue ExpandVP_REM(SDNode *Node);
136 SDValue ExpandSELECT(SDNode *Node);
137 std::pair<SDValue, SDValue> ExpandLoad(SDNode *N);
138 SDValue ExpandStore(SDNode *N);
139 SDValue ExpandFNEG(SDNode *Node);
140 void ExpandFSUB(SDNode *Node, SmallVectorImpl<SDValue> &Results);
141 void ExpandSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
142 void ExpandBITREVERSE(SDNode *Node, SmallVectorImpl<SDValue> &Results);
143 void ExpandUADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
144 void ExpandSADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
145 void ExpandMULO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
146 void ExpandFixedPointDiv(SDNode *Node, SmallVectorImpl<SDValue> &Results);
147 void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
148 void ExpandREM(SDNode *Node, SmallVectorImpl<SDValue> &Results);
149
150 void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
151
152 /// Implements vector promotion.
153 ///
154 /// This is essentially just bitcasting the operands to a different type and
155 /// bitcasting the result back to the original type.
157
158 /// Implements [SU]INT_TO_FP vector promotion.
159 ///
160 /// This is a [zs]ext of the input operand to a larger integer type.
161 void PromoteINT_TO_FP(SDNode *Node, SmallVectorImpl<SDValue> &Results);
162
163 /// Implements FP_TO_[SU]INT vector promotion of the result type.
164 ///
165 /// It is promoted to a larger integer type. The result is then
166 /// truncated back to the original type.
167 void PromoteFP_TO_INT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
168
169 /// Implements vector reduce operation promotion.
170 ///
171 /// All vector operands are promoted to a vector type with larger element
172 /// type, and the start value is promoted to a larger scalar type. Then the
173 /// result is truncated back to the original scalar type.
174 void PromoteReduction(SDNode *Node, SmallVectorImpl<SDValue> &Results);
175
176public:
177 VectorLegalizer(SelectionDAG& dag) :
178 DAG(dag), TLI(dag.getTargetLoweringInfo()) {}
179
180 /// Begin legalizer the vector operations in the DAG.
181 bool Run();
182};
183
184} // end anonymous namespace
185
186bool VectorLegalizer::Run() {
187 // Before we start legalizing vector nodes, check if there are any vectors.
188 bool HasVectors = false;
189 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
190 E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I) {
191 // Check if the values of the nodes contain vectors. We don't need to check
192 // the operands because we are going to check their values at some point.
193 HasVectors = llvm::any_of(I->values(), [](EVT T) { return T.isVector(); });
194
195 // If we found a vector node we can start the legalization.
196 if (HasVectors)
197 break;
198 }
199
200 // If this basic block has no vectors then no need to legalize vectors.
201 if (!HasVectors)
202 return false;
203
204 // The legalize process is inherently a bottom-up recursive process (users
205 // legalize their uses before themselves). Given infinite stack space, we
206 // could just start legalizing on the root and traverse the whole graph. In
207 // practice however, this causes us to run out of stack space on large basic
208 // blocks. To avoid this problem, compute an ordering of the nodes where each
209 // node is only legalized after all of its operands are legalized.
210 DAG.AssignTopologicalOrder();
211 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
212 E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I)
213 LegalizeOp(SDValue(&*I, 0));
214
215 // Finally, it's possible the root changed. Get the new root.
216 SDValue OldRoot = DAG.getRoot();
217 assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
218 DAG.setRoot(LegalizedNodes[OldRoot]);
219
220 LegalizedNodes.clear();
221
222 // Remove dead nodes now.
223 DAG.RemoveDeadNodes();
224
225 return Changed;
226}
227
228SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDNode *Result) {
229 assert(Op->getNumValues() == Result->getNumValues() &&
230 "Unexpected number of results");
231 // Generic legalization: just pass the operand through.
232 for (unsigned i = 0, e = Op->getNumValues(); i != e; ++i)
233 AddLegalizedOperand(Op.getValue(i), SDValue(Result, i));
234 return SDValue(Result, Op.getResNo());
235}
236
238VectorLegalizer::RecursivelyLegalizeResults(SDValue Op,
240 assert(Results.size() == Op->getNumValues() &&
241 "Unexpected number of results");
242 // Make sure that the generated code is itself legal.
243 for (unsigned i = 0, e = Results.size(); i != e; ++i) {
244 Results[i] = LegalizeOp(Results[i]);
245 AddLegalizedOperand(Op.getValue(i), Results[i]);
246 }
247
248 return Results[Op.getResNo()];
249}
250
251SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
252 // Note that LegalizeOp may be reentered even from single-use nodes, which
253 // means that we always must cache transformed nodes.
254 DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op);
255 if (I != LegalizedNodes.end()) return I->second;
256
257 // Legalize the operands
259 for (const SDValue &Oper : Op->op_values())
260 Ops.push_back(LegalizeOp(Oper));
261
262 SDNode *Node = DAG.UpdateNodeOperands(Op.getNode(), Ops);
263
264 bool HasVectorValueOrOp =
265 llvm::any_of(Node->values(), [](EVT T) { return T.isVector(); }) ||
266 llvm::any_of(Node->op_values(),
267 [](SDValue O) { return O.getValueType().isVector(); });
268 if (!HasVectorValueOrOp)
269 return TranslateLegalizeResults(Op, Node);
270
271 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
272 EVT ValVT;
273 switch (Op.getOpcode()) {
274 default:
275 return TranslateLegalizeResults(Op, Node);
276 case ISD::LOAD: {
277 LoadSDNode *LD = cast<LoadSDNode>(Node);
278 ISD::LoadExtType ExtType = LD->getExtensionType();
279 EVT LoadedVT = LD->getMemoryVT();
280 if (LoadedVT.isVector() && ExtType != ISD::NON_EXTLOAD)
281 Action = TLI.getLoadExtAction(ExtType, LD->getValueType(0), LoadedVT);
282 break;
283 }
284 case ISD::STORE: {
285 StoreSDNode *ST = cast<StoreSDNode>(Node);
286 EVT StVT = ST->getMemoryVT();
287 MVT ValVT = ST->getValue().getSimpleValueType();
288 if (StVT.isVector() && ST->isTruncatingStore())
289 Action = TLI.getTruncStoreAction(ValVT, StVT);
290 break;
291 }
293 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
294 // This operation lies about being legal: when it claims to be legal,
295 // it should actually be expanded.
296 if (Action == TargetLowering::Legal)
297 Action = TargetLowering::Expand;
298 break;
299#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
300 case ISD::STRICT_##DAGN:
301#include "llvm/IR/ConstrainedOps.def"
302 ValVT = Node->getValueType(0);
303 if (Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
304 Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
305 ValVT = Node->getOperand(1).getValueType();
306 if (Op.getOpcode() == ISD::STRICT_FSETCC ||
307 Op.getOpcode() == ISD::STRICT_FSETCCS) {
308 MVT OpVT = Node->getOperand(1).getSimpleValueType();
309 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(3))->get();
310 Action = TLI.getCondCodeAction(CCCode, OpVT);
311 if (Action == TargetLowering::Legal)
312 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
313 } else {
314 Action = TLI.getOperationAction(Node->getOpcode(), ValVT);
315 }
316 // If we're asked to expand a strict vector floating-point operation,
317 // by default we're going to simply unroll it. That is usually the
318 // best approach, except in the case where the resulting strict (scalar)
319 // operations would themselves use the fallback mutation to non-strict.
320 // In that specific case, just do the fallback on the vector op.
321 if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
322 TLI.getStrictFPOperationAction(Node->getOpcode(), ValVT) ==
323 TargetLowering::Legal) {
324 EVT EltVT = ValVT.getVectorElementType();
325 if (TLI.getOperationAction(Node->getOpcode(), EltVT)
326 == TargetLowering::Expand &&
327 TLI.getStrictFPOperationAction(Node->getOpcode(), EltVT)
328 == TargetLowering::Legal)
329 Action = TargetLowering::Legal;
330 }
331 break;
332 case ISD::ADD:
333 case ISD::SUB:
334 case ISD::MUL:
335 case ISD::MULHS:
336 case ISD::MULHU:
337 case ISD::SDIV:
338 case ISD::UDIV:
339 case ISD::SREM:
340 case ISD::UREM:
341 case ISD::SDIVREM:
342 case ISD::UDIVREM:
343 case ISD::FADD:
344 case ISD::FSUB:
345 case ISD::FMUL:
346 case ISD::FDIV:
347 case ISD::FREM:
348 case ISD::AND:
349 case ISD::OR:
350 case ISD::XOR:
351 case ISD::SHL:
352 case ISD::SRA:
353 case ISD::SRL:
354 case ISD::FSHL:
355 case ISD::FSHR:
356 case ISD::ROTL:
357 case ISD::ROTR:
358 case ISD::ABS:
359 case ISD::BSWAP:
360 case ISD::BITREVERSE:
361 case ISD::CTLZ:
362 case ISD::CTTZ:
365 case ISD::CTPOP:
366 case ISD::SELECT:
367 case ISD::VSELECT:
368 case ISD::SELECT_CC:
369 case ISD::ZERO_EXTEND:
370 case ISD::ANY_EXTEND:
371 case ISD::TRUNCATE:
372 case ISD::SIGN_EXTEND:
373 case ISD::FP_TO_SINT:
374 case ISD::FP_TO_UINT:
375 case ISD::FNEG:
376 case ISD::FABS:
377 case ISD::FMINNUM:
378 case ISD::FMAXNUM:
381 case ISD::FMINIMUM:
382 case ISD::FMAXIMUM:
383 case ISD::FCOPYSIGN:
384 case ISD::FSQRT:
385 case ISD::FSIN:
386 case ISD::FCOS:
387 case ISD::FLDEXP:
388 case ISD::FPOWI:
389 case ISD::FPOW:
390 case ISD::FLOG:
391 case ISD::FLOG2:
392 case ISD::FLOG10:
393 case ISD::FEXP:
394 case ISD::FEXP2:
395 case ISD::FEXP10:
396 case ISD::FCEIL:
397 case ISD::FTRUNC:
398 case ISD::FRINT:
399 case ISD::FNEARBYINT:
400 case ISD::FROUND:
401 case ISD::FROUNDEVEN:
402 case ISD::FFLOOR:
403 case ISD::FP_ROUND:
404 case ISD::FP_EXTEND:
405 case ISD::FMA:
410 case ISD::SMIN:
411 case ISD::SMAX:
412 case ISD::UMIN:
413 case ISD::UMAX:
414 case ISD::SMUL_LOHI:
415 case ISD::UMUL_LOHI:
416 case ISD::SADDO:
417 case ISD::UADDO:
418 case ISD::SSUBO:
419 case ISD::USUBO:
420 case ISD::SMULO:
421 case ISD::UMULO:
423 case ISD::FFREXP:
424 case ISD::SADDSAT:
425 case ISD::UADDSAT:
426 case ISD::SSUBSAT:
427 case ISD::USUBSAT:
428 case ISD::SSHLSAT:
429 case ISD::USHLSAT:
432 case ISD::MGATHER:
433 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
434 break;
435 case ISD::SMULFIX:
436 case ISD::SMULFIXSAT:
437 case ISD::UMULFIX:
438 case ISD::UMULFIXSAT:
439 case ISD::SDIVFIX:
440 case ISD::SDIVFIXSAT:
441 case ISD::UDIVFIX:
442 case ISD::UDIVFIXSAT: {
443 unsigned Scale = Node->getConstantOperandVal(2);
444 Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
445 Node->getValueType(0), Scale);
446 break;
447 }
448 case ISD::SINT_TO_FP:
449 case ISD::UINT_TO_FP:
465 Action = TLI.getOperationAction(Node->getOpcode(),
466 Node->getOperand(0).getValueType());
467 break;
470 Action = TLI.getOperationAction(Node->getOpcode(),
471 Node->getOperand(1).getValueType());
472 break;
473 case ISD::SETCC: {
474 MVT OpVT = Node->getOperand(0).getSimpleValueType();
475 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
476 Action = TLI.getCondCodeAction(CCCode, OpVT);
477 if (Action == TargetLowering::Legal)
478 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
479 break;
480 }
481
482#define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...) \
483 case ISD::VPID: { \
484 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \
485 : Node->getOperand(LEGALPOS).getValueType(); \
486 if (ISD::VPID == ISD::VP_SETCC) { \
487 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); \
488 Action = TLI.getCondCodeAction(CCCode, LegalizeVT.getSimpleVT()); \
489 if (Action != TargetLowering::Legal) \
490 break; \
491 } \
492 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \
493 } break;
494#include "llvm/IR/VPIntrinsics.def"
495 }
496
497 LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG));
498
499 SmallVector<SDValue, 8> ResultVals;
500 switch (Action) {
501 default: llvm_unreachable("This action is not supported yet!");
502 case TargetLowering::Promote:
503 assert((Op.getOpcode() != ISD::LOAD && Op.getOpcode() != ISD::STORE) &&
504 "This action is not supported yet!");
505 LLVM_DEBUG(dbgs() << "Promoting\n");
506 Promote(Node, ResultVals);
507 assert(!ResultVals.empty() && "No results for promotion?");
508 break;
509 case TargetLowering::Legal:
510 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
511 break;
512 case TargetLowering::Custom:
513 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
514 if (LowerOperationWrapper(Node, ResultVals))
515 break;
516 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
517 [[fallthrough]];
518 case TargetLowering::Expand:
519 LLVM_DEBUG(dbgs() << "Expanding\n");
520 Expand(Node, ResultVals);
521 break;
522 }
523
524 if (ResultVals.empty())
525 return TranslateLegalizeResults(Op, Node);
526
527 Changed = true;
528 return RecursivelyLegalizeResults(Op, ResultVals);
529}
530
531// FIXME: This is very similar to TargetLowering::LowerOperationWrapper. Can we
532// merge them somehow?
533bool VectorLegalizer::LowerOperationWrapper(SDNode *Node,
535 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
536
537 if (!Res.getNode())
538 return false;
539
540 if (Res == SDValue(Node, 0))
541 return true;
542
543 // If the original node has one result, take the return value from
544 // LowerOperation as is. It might not be result number 0.
545 if (Node->getNumValues() == 1) {
546 Results.push_back(Res);
547 return true;
548 }
549
550 // If the original node has multiple results, then the return node should
551 // have the same number of results.
552 assert((Node->getNumValues() == Res->getNumValues()) &&
553 "Lowering returned the wrong number of results!");
554
555 // Places new result values base on N result number.
556 for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I)
557 Results.push_back(Res.getValue(I));
558
559 return true;
560}
561
562void VectorLegalizer::PromoteReduction(SDNode *Node,
564 MVT VecVT = Node->getOperand(1).getSimpleValueType();
565 MVT NewVecVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VecVT);
566 MVT ScalarVT = Node->getSimpleValueType(0);
567 MVT NewScalarVT = NewVecVT.getVectorElementType();
568
569 SDLoc DL(Node);
570 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
571
572 // promote the initial value.
573 if (Node->getOperand(0).getValueType().isFloatingPoint())
574 Operands[0] =
575 DAG.getNode(ISD::FP_EXTEND, DL, NewScalarVT, Node->getOperand(0));
576 else
577 Operands[0] =
578 DAG.getNode(ISD::ANY_EXTEND, DL, NewScalarVT, Node->getOperand(0));
579
580 for (unsigned j = 1; j != Node->getNumOperands(); ++j)
581 if (Node->getOperand(j).getValueType().isVector() &&
582 !(ISD::isVPOpcode(Node->getOpcode()) &&
583 ISD::getVPMaskIdx(Node->getOpcode()) == j)) // Skip mask operand.
584 // promote the vector operand.
585 if (Node->getOperand(j).getValueType().isFloatingPoint())
586 Operands[j] =
587 DAG.getNode(ISD::FP_EXTEND, DL, NewVecVT, Node->getOperand(j));
588 else
589 Operands[j] =
590 DAG.getNode(ISD::ANY_EXTEND, DL, NewVecVT, Node->getOperand(j));
591 else
592 Operands[j] = Node->getOperand(j); // Skip VL operand.
593
594 SDValue Res = DAG.getNode(Node->getOpcode(), DL, NewScalarVT, Operands,
595 Node->getFlags());
596
597 if (ScalarVT.isFloatingPoint())
598 Res = DAG.getNode(ISD::FP_ROUND, DL, ScalarVT, Res,
599 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
600 else
601 Res = DAG.getNode(ISD::TRUNCATE, DL, ScalarVT, Res);
602
603 Results.push_back(Res);
604}
605
606void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
607 // For a few operations there is a specific concept for promotion based on
608 // the operand's type.
609 switch (Node->getOpcode()) {
610 case ISD::SINT_TO_FP:
611 case ISD::UINT_TO_FP:
614 // "Promote" the operation by extending the operand.
615 PromoteINT_TO_FP(Node, Results);
616 return;
617 case ISD::FP_TO_UINT:
618 case ISD::FP_TO_SINT:
621 // Promote the operation by extending the operand.
622 PromoteFP_TO_INT(Node, Results);
623 return;
624 case ISD::VP_REDUCE_ADD:
625 case ISD::VP_REDUCE_MUL:
626 case ISD::VP_REDUCE_AND:
627 case ISD::VP_REDUCE_OR:
628 case ISD::VP_REDUCE_XOR:
629 case ISD::VP_REDUCE_SMAX:
630 case ISD::VP_REDUCE_SMIN:
631 case ISD::VP_REDUCE_UMAX:
632 case ISD::VP_REDUCE_UMIN:
633 case ISD::VP_REDUCE_FADD:
634 case ISD::VP_REDUCE_FMUL:
635 case ISD::VP_REDUCE_FMAX:
636 case ISD::VP_REDUCE_FMIN:
637 case ISD::VP_REDUCE_SEQ_FADD:
638 // Promote the operation by extending the operand.
639 PromoteReduction(Node, Results);
640 return;
641 case ISD::FP_ROUND:
642 case ISD::FP_EXTEND:
643 // These operations are used to do promotion so they can't be promoted
644 // themselves.
645 llvm_unreachable("Don't know how to promote this operation!");
646 }
647
648 // There are currently two cases of vector promotion:
649 // 1) Bitcasting a vector of integers to a different type to a vector of the
650 // same overall length. For example, x86 promotes ISD::AND v2i32 to v1i64.
651 // 2) Extending a vector of floats to a vector of the same number of larger
652 // floats. For example, AArch64 promotes ISD::FADD on v4f16 to v4f32.
653 assert(Node->getNumValues() == 1 &&
654 "Can't promote a vector with multiple results!");
655 MVT VT = Node->getSimpleValueType(0);
656 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
657 SDLoc dl(Node);
658 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
659
660 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
661 // Do not promote the mask operand of a VP OP.
662 bool SkipPromote = ISD::isVPOpcode(Node->getOpcode()) &&
663 ISD::getVPMaskIdx(Node->getOpcode()) == j;
664 if (Node->getOperand(j).getValueType().isVector() && !SkipPromote)
665 if (Node->getOperand(j)
666 .getValueType()
667 .getVectorElementType()
668 .isFloatingPoint() &&
670 Operands[j] = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(j));
671 else
672 Operands[j] = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(j));
673 else
674 Operands[j] = Node->getOperand(j);
675 }
676
677 SDValue Res =
678 DAG.getNode(Node->getOpcode(), dl, NVT, Operands, Node->getFlags());
679
680 if ((VT.isFloatingPoint() && NVT.isFloatingPoint()) ||
683 Res = DAG.getNode(ISD::FP_ROUND, dl, VT, Res,
684 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
685 else
686 Res = DAG.getNode(ISD::BITCAST, dl, VT, Res);
687
688 Results.push_back(Res);
689}
690
691void VectorLegalizer::PromoteINT_TO_FP(SDNode *Node,
693 // INT_TO_FP operations may require the input operand be promoted even
694 // when the type is otherwise legal.
695 bool IsStrict = Node->isStrictFPOpcode();
696 MVT VT = Node->getOperand(IsStrict ? 1 : 0).getSimpleValueType();
697 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
699 "Vectors have different number of elements!");
700
701 SDLoc dl(Node);
702 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
703
704 unsigned Opc = (Node->getOpcode() == ISD::UINT_TO_FP ||
705 Node->getOpcode() == ISD::STRICT_UINT_TO_FP)
708 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
709 if (Node->getOperand(j).getValueType().isVector())
710 Operands[j] = DAG.getNode(Opc, dl, NVT, Node->getOperand(j));
711 else
712 Operands[j] = Node->getOperand(j);
713 }
714
715 if (IsStrict) {
716 SDValue Res = DAG.getNode(Node->getOpcode(), dl,
717 {Node->getValueType(0), MVT::Other}, Operands);
718 Results.push_back(Res);
719 Results.push_back(Res.getValue(1));
720 return;
721 }
722
723 SDValue Res =
724 DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Operands);
725 Results.push_back(Res);
726}
727
728// For FP_TO_INT we promote the result type to a vector type with wider
729// elements and then truncate the result. This is different from the default
730// PromoteVector which uses bitcast to promote thus assumning that the
731// promoted vector type has the same overall size.
732void VectorLegalizer::PromoteFP_TO_INT(SDNode *Node,
734 MVT VT = Node->getSimpleValueType(0);
735 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
736 bool IsStrict = Node->isStrictFPOpcode();
738 "Vectors have different number of elements!");
739
740 unsigned NewOpc = Node->getOpcode();
741 // Change FP_TO_UINT to FP_TO_SINT if possible.
742 // TODO: Should we only do this if FP_TO_UINT itself isn't legal?
743 if (NewOpc == ISD::FP_TO_UINT &&
744 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
745 NewOpc = ISD::FP_TO_SINT;
746
747 if (NewOpc == ISD::STRICT_FP_TO_UINT &&
748 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT))
749 NewOpc = ISD::STRICT_FP_TO_SINT;
750
751 SDLoc dl(Node);
752 SDValue Promoted, Chain;
753 if (IsStrict) {
754 Promoted = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
755 {Node->getOperand(0), Node->getOperand(1)});
756 Chain = Promoted.getValue(1);
757 } else
758 Promoted = DAG.getNode(NewOpc, dl, NVT, Node->getOperand(0));
759
760 // Assert that the converted value fits in the original type. If it doesn't
761 // (eg: because the value being converted is too big), then the result of the
762 // original operation was undefined anyway, so the assert is still correct.
763 if (Node->getOpcode() == ISD::FP_TO_UINT ||
764 Node->getOpcode() == ISD::STRICT_FP_TO_UINT)
765 NewOpc = ISD::AssertZext;
766 else
767 NewOpc = ISD::AssertSext;
768
769 Promoted = DAG.getNode(NewOpc, dl, NVT, Promoted,
770 DAG.getValueType(VT.getScalarType()));
771 Promoted = DAG.getNode(ISD::TRUNCATE, dl, VT, Promoted);
772 Results.push_back(Promoted);
773 if (IsStrict)
774 Results.push_back(Chain);
775}
776
777std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(SDNode *N) {
778 LoadSDNode *LD = cast<LoadSDNode>(N);
779 return TLI.scalarizeVectorLoad(LD, DAG);
780}
781
782SDValue VectorLegalizer::ExpandStore(SDNode *N) {
783 StoreSDNode *ST = cast<StoreSDNode>(N);
784 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
785 return TF;
786}
787
788void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
789 switch (Node->getOpcode()) {
790 case ISD::LOAD: {
791 std::pair<SDValue, SDValue> Tmp = ExpandLoad(Node);
792 Results.push_back(Tmp.first);
793 Results.push_back(Tmp.second);
794 return;
795 }
796 case ISD::STORE:
797 Results.push_back(ExpandStore(Node));
798 return;
800 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
801 Results.push_back(Node->getOperand(i));
802 return;
804 Results.push_back(ExpandSEXTINREG(Node));
805 return;
807 Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(Node));
808 return;
810 Results.push_back(ExpandSIGN_EXTEND_VECTOR_INREG(Node));
811 return;
813 Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(Node));
814 return;
815 case ISD::BSWAP:
816 Results.push_back(ExpandBSWAP(Node));
817 return;
818 case ISD::VP_BSWAP:
819 Results.push_back(TLI.expandVPBSWAP(Node, DAG));
820 return;
821 case ISD::VSELECT:
822 Results.push_back(ExpandVSELECT(Node));
823 return;
824 case ISD::VP_SELECT:
825 Results.push_back(ExpandVP_SELECT(Node));
826 return;
827 case ISD::VP_SREM:
828 case ISD::VP_UREM:
829 if (SDValue Expanded = ExpandVP_REM(Node)) {
830 Results.push_back(Expanded);
831 return;
832 }
833 break;
834 case ISD::SELECT:
835 Results.push_back(ExpandSELECT(Node));
836 return;
837 case ISD::SELECT_CC: {
838 if (Node->getValueType(0).isScalableVector()) {
839 EVT CondVT = TLI.getSetCCResultType(
840 DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));
841 SDValue SetCC =
842 DAG.getNode(ISD::SETCC, SDLoc(Node), CondVT, Node->getOperand(0),
843 Node->getOperand(1), Node->getOperand(4));
844 Results.push_back(DAG.getSelect(SDLoc(Node), Node->getValueType(0), SetCC,
845 Node->getOperand(2),
846 Node->getOperand(3)));
847 return;
848 }
849 break;
850 }
851 case ISD::FP_TO_UINT:
852 ExpandFP_TO_UINT(Node, Results);
853 return;
854 case ISD::UINT_TO_FP:
855 ExpandUINT_TO_FLOAT(Node, Results);
856 return;
857 case ISD::FNEG:
858 Results.push_back(ExpandFNEG(Node));
859 return;
860 case ISD::FSUB:
861 ExpandFSUB(Node, Results);
862 return;
863 case ISD::SETCC:
864 case ISD::VP_SETCC:
865 ExpandSETCC(Node, Results);
866 return;
867 case ISD::ABS:
868 if (SDValue Expanded = TLI.expandABS(Node, DAG)) {
869 Results.push_back(Expanded);
870 return;
871 }
872 break;
873 case ISD::ABDS:
874 case ISD::ABDU:
875 if (SDValue Expanded = TLI.expandABD(Node, DAG)) {
876 Results.push_back(Expanded);
877 return;
878 }
879 break;
880 case ISD::BITREVERSE:
881 ExpandBITREVERSE(Node, Results);
882 return;
883 case ISD::VP_BITREVERSE:
884 if (SDValue Expanded = TLI.expandVPBITREVERSE(Node, DAG)) {
885 Results.push_back(Expanded);
886 return;
887 }
888 break;
889 case ISD::CTPOP:
890 if (SDValue Expanded = TLI.expandCTPOP(Node, DAG)) {
891 Results.push_back(Expanded);
892 return;
893 }
894 break;
895 case ISD::VP_CTPOP:
896 if (SDValue Expanded = TLI.expandVPCTPOP(Node, DAG)) {
897 Results.push_back(Expanded);
898 return;
899 }
900 break;
901 case ISD::CTLZ:
903 if (SDValue Expanded = TLI.expandCTLZ(Node, DAG)) {
904 Results.push_back(Expanded);
905 return;
906 }
907 break;
908 case ISD::VP_CTLZ:
909 case ISD::VP_CTLZ_ZERO_UNDEF:
910 if (SDValue Expanded = TLI.expandVPCTLZ(Node, DAG)) {
911 Results.push_back(Expanded);
912 return;
913 }
914 break;
915 case ISD::CTTZ:
917 if (SDValue Expanded = TLI.expandCTTZ(Node, DAG)) {
918 Results.push_back(Expanded);
919 return;
920 }
921 break;
922 case ISD::VP_CTTZ:
923 case ISD::VP_CTTZ_ZERO_UNDEF:
924 if (SDValue Expanded = TLI.expandVPCTTZ(Node, DAG)) {
925 Results.push_back(Expanded);
926 return;
927 }
928 break;
929 case ISD::FSHL:
930 case ISD::VP_FSHL:
931 case ISD::FSHR:
932 case ISD::VP_FSHR:
933 if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG)) {
934 Results.push_back(Expanded);
935 return;
936 }
937 break;
938 case ISD::ROTL:
939 case ISD::ROTR:
940 if (SDValue Expanded = TLI.expandROT(Node, false /*AllowVectorOps*/, DAG)) {
941 Results.push_back(Expanded);
942 return;
943 }
944 break;
945 case ISD::FMINNUM:
946 case ISD::FMAXNUM:
947 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) {
948 Results.push_back(Expanded);
949 return;
950 }
951 break;
952 case ISD::SMIN:
953 case ISD::SMAX:
954 case ISD::UMIN:
955 case ISD::UMAX:
956 if (SDValue Expanded = TLI.expandIntMINMAX(Node, DAG)) {
957 Results.push_back(Expanded);
958 return;
959 }
960 break;
961 case ISD::UADDO:
962 case ISD::USUBO:
963 ExpandUADDSUBO(Node, Results);
964 return;
965 case ISD::SADDO:
966 case ISD::SSUBO:
967 ExpandSADDSUBO(Node, Results);
968 return;
969 case ISD::UMULO:
970 case ISD::SMULO:
971 ExpandMULO(Node, Results);
972 return;
973 case ISD::USUBSAT:
974 case ISD::SSUBSAT:
975 case ISD::UADDSAT:
976 case ISD::SADDSAT:
977 if (SDValue Expanded = TLI.expandAddSubSat(Node, DAG)) {
978 Results.push_back(Expanded);
979 return;
980 }
981 break;
982 case ISD::USHLSAT:
983 case ISD::SSHLSAT:
984 if (SDValue Expanded = TLI.expandShlSat(Node, DAG)) {
985 Results.push_back(Expanded);
986 return;
987 }
988 break;
991 // Expand the fpsosisat if it is scalable to prevent it from unrolling below.
992 if (Node->getValueType(0).isScalableVector()) {
993 if (SDValue Expanded = TLI.expandFP_TO_INT_SAT(Node, DAG)) {
994 Results.push_back(Expanded);
995 return;
996 }
997 }
998 break;
999 case ISD::SMULFIX:
1000 case ISD::UMULFIX:
1001 if (SDValue Expanded = TLI.expandFixedPointMul(Node, DAG)) {
1002 Results.push_back(Expanded);
1003 return;
1004 }
1005 break;
1006 case ISD::SMULFIXSAT:
1007 case ISD::UMULFIXSAT:
1008 // FIXME: We do not expand SMULFIXSAT/UMULFIXSAT here yet, not sure exactly
1009 // why. Maybe it results in worse codegen compared to the unroll for some
1010 // targets? This should probably be investigated. And if we still prefer to
1011 // unroll an explanation could be helpful.
1012 break;
1013 case ISD::SDIVFIX:
1014 case ISD::UDIVFIX:
1015 ExpandFixedPointDiv(Node, Results);
1016 return;
1017 case ISD::SDIVFIXSAT:
1018 case ISD::UDIVFIXSAT:
1019 break;
1020#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1021 case ISD::STRICT_##DAGN:
1022#include "llvm/IR/ConstrainedOps.def"
1023 ExpandStrictFPOp(Node, Results);
1024 return;
1025 case ISD::VECREDUCE_ADD:
1026 case ISD::VECREDUCE_MUL:
1027 case ISD::VECREDUCE_AND:
1028 case ISD::VECREDUCE_OR:
1029 case ISD::VECREDUCE_XOR:
1040 Results.push_back(TLI.expandVecReduce(Node, DAG));
1041 return;
1044 Results.push_back(TLI.expandVecReduceSeq(Node, DAG));
1045 return;
1046 case ISD::SREM:
1047 case ISD::UREM:
1048 ExpandREM(Node, Results);
1049 return;
1050 case ISD::VP_MERGE:
1051 Results.push_back(ExpandVP_MERGE(Node));
1052 return;
1053 }
1054
1055 SDValue Unrolled = DAG.UnrollVectorOp(Node);
1056 for (unsigned I = 0, E = Unrolled->getNumValues(); I != E; ++I)
1057 Results.push_back(Unrolled.getValue(I));
1058}
1059
1060SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) {
1061 // Lower a select instruction where the condition is a scalar and the
1062 // operands are vectors. Lower this select to VSELECT and implement it
1063 // using XOR AND OR. The selector bit is broadcasted.
1064 EVT VT = Node->getValueType(0);
1065 SDLoc DL(Node);
1066
1067 SDValue Mask = Node->getOperand(0);
1068 SDValue Op1 = Node->getOperand(1);
1069 SDValue Op2 = Node->getOperand(2);
1070
1071 assert(VT.isVector() && !Mask.getValueType().isVector()
1072 && Op1.getValueType() == Op2.getValueType() && "Invalid type");
1073
1074 // If we can't even use the basic vector operations of
1075 // AND,OR,XOR, we will have to scalarize the op.
1076 // Notice that the operation may be 'promoted' which means that it is
1077 // 'bitcasted' to another type which is handled.
1078 // Also, we need to be able to construct a splat vector using either
1079 // BUILD_VECTOR or SPLAT_VECTOR.
1080 // FIXME: Should we also permit fixed-length SPLAT_VECTOR as a fallback to
1081 // BUILD_VECTOR?
1082 if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
1083 TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
1084 TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand ||
1085 TLI.getOperationAction(VT.isFixedLengthVector() ? ISD::BUILD_VECTOR
1087 VT) == TargetLowering::Expand)
1088 return DAG.UnrollVectorOp(Node);
1089
1090 // Generate a mask operand.
1092
1093 // What is the size of each element in the vector mask.
1094 EVT BitTy = MaskTy.getScalarType();
1095
1096 Mask = DAG.getSelect(DL, BitTy, Mask, DAG.getAllOnesConstant(DL, BitTy),
1097 DAG.getConstant(0, DL, BitTy));
1098
1099 // Broadcast the mask so that the entire vector is all one or all zero.
1100 Mask = DAG.getSplat(MaskTy, DL, Mask);
1101
1102 // Bitcast the operands to be the same type as the mask.
1103 // This is needed when we select between FP types because
1104 // the mask is a vector of integers.
1105 Op1 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op1);
1106 Op2 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op2);
1107
1108 SDValue NotMask = DAG.getNOT(DL, Mask, MaskTy);
1109
1110 Op1 = DAG.getNode(ISD::AND, DL, MaskTy, Op1, Mask);
1111 Op2 = DAG.getNode(ISD::AND, DL, MaskTy, Op2, NotMask);
1112 SDValue Val = DAG.getNode(ISD::OR, DL, MaskTy, Op1, Op2);
1113 return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val);
1114}
1115
1116SDValue VectorLegalizer::ExpandSEXTINREG(SDNode *Node) {
1117 EVT VT = Node->getValueType(0);
1118
1119 // Make sure that the SRA and SHL instructions are available.
1120 if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Expand ||
1121 TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Expand)
1122 return DAG.UnrollVectorOp(Node);
1123
1124 SDLoc DL(Node);
1125 EVT OrigTy = cast<VTSDNode>(Node->getOperand(1))->getVT();
1126
1127 unsigned BW = VT.getScalarSizeInBits();
1128 unsigned OrigBW = OrigTy.getScalarSizeInBits();
1129 SDValue ShiftSz = DAG.getConstant(BW - OrigBW, DL, VT);
1130
1131 SDValue Op = DAG.getNode(ISD::SHL, DL, VT, Node->getOperand(0), ShiftSz);
1132 return DAG.getNode(ISD::SRA, DL, VT, Op, ShiftSz);
1133}
1134
1135// Generically expand a vector anyext in register to a shuffle of the relevant
1136// lanes into the appropriate locations, with other lanes left undef.
1137SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node) {
1138 SDLoc DL(Node);
1139 EVT VT = Node->getValueType(0);
1140 int NumElements = VT.getVectorNumElements();
1141 SDValue Src = Node->getOperand(0);
1142 EVT SrcVT = Src.getValueType();
1143 int NumSrcElements = SrcVT.getVectorNumElements();
1144
1145 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1146 // into a larger vector type.
1147 if (SrcVT.bitsLE(VT)) {
1148 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1149 "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1150 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1151 SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
1152 NumSrcElements);
1153 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT),
1154 Src, DAG.getVectorIdxConstant(0, DL));
1155 }
1156
1157 // Build a base mask of undef shuffles.
1158 SmallVector<int, 16> ShuffleMask;
1159 ShuffleMask.resize(NumSrcElements, -1);
1160
1161 // Place the extended lanes into the correct locations.
1162 int ExtLaneScale = NumSrcElements / NumElements;
1163 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1164 for (int i = 0; i < NumElements; ++i)
1165 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1166
1167 return DAG.getNode(
1168 ISD::BITCAST, DL, VT,
1169 DAG.getVectorShuffle(SrcVT, DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
1170}
1171
1172SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node) {
1173 SDLoc DL(Node);
1174 EVT VT = Node->getValueType(0);
1175 SDValue Src = Node->getOperand(0);
1176 EVT SrcVT = Src.getValueType();
1177
1178 // First build an any-extend node which can be legalized above when we
1179 // recurse through it.
1180 SDValue Op = DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Src);
1181
1182 // Now we need sign extend. Do this by shifting the elements. Even if these
1183 // aren't legal operations, they have a better chance of being legalized
1184 // without full scalarization than the sign extension does.
1185 unsigned EltWidth = VT.getScalarSizeInBits();
1186 unsigned SrcEltWidth = SrcVT.getScalarSizeInBits();
1187 SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth, DL, VT);
1188 return DAG.getNode(ISD::SRA, DL, VT,
1189 DAG.getNode(ISD::SHL, DL, VT, Op, ShiftAmount),
1190 ShiftAmount);
1191}
1192
1193// Generically expand a vector zext in register to a shuffle of the relevant
1194// lanes into the appropriate locations, a blend of zero into the high bits,
1195// and a bitcast to the wider element type.
1196SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node) {
1197 SDLoc DL(Node);
1198 EVT VT = Node->getValueType(0);
1199 int NumElements = VT.getVectorNumElements();
1200 SDValue Src = Node->getOperand(0);
1201 EVT SrcVT = Src.getValueType();
1202 int NumSrcElements = SrcVT.getVectorNumElements();
1203
1204 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1205 // into a larger vector type.
1206 if (SrcVT.bitsLE(VT)) {
1207 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1208 "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1209 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1210 SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
1211 NumSrcElements);
1212 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT),
1213 Src, DAG.getVectorIdxConstant(0, DL));
1214 }
1215
1216 // Build up a zero vector to blend into this one.
1217 SDValue Zero = DAG.getConstant(0, DL, SrcVT);
1218
1219 // Shuffle the incoming lanes into the correct position, and pull all other
1220 // lanes from the zero vector.
1221 auto ShuffleMask = llvm::to_vector<16>(llvm::seq<int>(0, NumSrcElements));
1222
1223 int ExtLaneScale = NumSrcElements / NumElements;
1224 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1225 for (int i = 0; i < NumElements; ++i)
1226 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1227
1228 return DAG.getNode(ISD::BITCAST, DL, VT,
1229 DAG.getVectorShuffle(SrcVT, DL, Zero, Src, ShuffleMask));
1230}
1231
1232static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl<int> &ShuffleMask) {
1233 int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
1234 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I)
1235 for (int J = ScalarSizeInBytes - 1; J >= 0; --J)
1236 ShuffleMask.push_back((I * ScalarSizeInBytes) + J);
1237}
1238
1239SDValue VectorLegalizer::ExpandBSWAP(SDNode *Node) {
1240 EVT VT = Node->getValueType(0);
1241
1242 // Scalable vectors can't use shuffle expansion.
1243 if (VT.isScalableVector())
1244 return TLI.expandBSWAP(Node, DAG);
1245
1246 // Generate a byte wise shuffle mask for the BSWAP.
1247 SmallVector<int, 16> ShuffleMask;
1248 createBSWAPShuffleMask(VT, ShuffleMask);
1249 EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, ShuffleMask.size());
1250
1251 // Only emit a shuffle if the mask is legal.
1252 if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1253 SDLoc DL(Node);
1254 SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0));
1255 Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT), ShuffleMask);
1256 return DAG.getNode(ISD::BITCAST, DL, VT, Op);
1257 }
1258
1259 // If we have the appropriate vector bit operations, it is better to use them
1260 // than unrolling and expanding each component.
1261 if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
1262 TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
1263 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&
1264 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT))
1265 return TLI.expandBSWAP(Node, DAG);
1266
1267 // Otherwise unroll.
1268 return DAG.UnrollVectorOp(Node);
1269}
1270
1271void VectorLegalizer::ExpandBITREVERSE(SDNode *Node,
1273 EVT VT = Node->getValueType(0);
1274
1275 // We can't unroll or use shuffles for scalable vectors.
1276 if (VT.isScalableVector()) {
1277 Results.push_back(TLI.expandBITREVERSE(Node, DAG));
1278 return;
1279 }
1280
1281 // If we have the scalar operation, it's probably cheaper to unroll it.
1282 if (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, VT.getScalarType())) {
1283 SDValue Tmp = DAG.UnrollVectorOp(Node);
1284 Results.push_back(Tmp);
1285 return;
1286 }
1287
1288 // If the vector element width is a whole number of bytes, test if its legal
1289 // to BSWAP shuffle the bytes and then perform the BITREVERSE on the byte
1290 // vector. This greatly reduces the number of bit shifts necessary.
1291 unsigned ScalarSizeInBits = VT.getScalarSizeInBits();
1292 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1293 SmallVector<int, 16> BSWAPMask;
1294 createBSWAPShuffleMask(VT, BSWAPMask);
1295
1296 EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, BSWAPMask.size());
1297 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1298 (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, ByteVT) ||
1299 (TLI.isOperationLegalOrCustom(ISD::SHL, ByteVT) &&
1300 TLI.isOperationLegalOrCustom(ISD::SRL, ByteVT) &&
1301 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, ByteVT) &&
1302 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, ByteVT)))) {
1303 SDLoc DL(Node);
1304 SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0));
1305 Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT),
1306 BSWAPMask);
1307 Op = DAG.getNode(ISD::BITREVERSE, DL, ByteVT, Op);
1308 Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
1309 Results.push_back(Op);
1310 return;
1311 }
1312 }
1313
1314 // If we have the appropriate vector bit operations, it is better to use them
1315 // than unrolling and expanding each component.
1316 if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
1317 TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
1318 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&
1319 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT)) {
1320 Results.push_back(TLI.expandBITREVERSE(Node, DAG));
1321 return;
1322 }
1323
1324 // Otherwise unroll.
1325 SDValue Tmp = DAG.UnrollVectorOp(Node);
1326 Results.push_back(Tmp);
1327}
1328
1329SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) {
1330 // Implement VSELECT in terms of XOR, AND, OR
1331 // on platforms which do not support blend natively.
1332 SDLoc DL(Node);
1333
1334 SDValue Mask = Node->getOperand(0);
1335 SDValue Op1 = Node->getOperand(1);
1336 SDValue Op2 = Node->getOperand(2);
1337
1338 EVT VT = Mask.getValueType();
1339
1340 // If we can't even use the basic vector operations of
1341 // AND,OR,XOR, we will have to scalarize the op.
1342 // Notice that the operation may be 'promoted' which means that it is
1343 // 'bitcasted' to another type which is handled.
1344 if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
1345 TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
1346 TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand)
1347 return DAG.UnrollVectorOp(Node);
1348
1349 // This operation also isn't safe with AND, OR, XOR when the boolean type is
1350 // 0/1 and the select operands aren't also booleans, as we need an all-ones
1351 // vector constant to mask with.
1352 // FIXME: Sign extend 1 to all ones if that's legal on the target.
1353 auto BoolContents = TLI.getBooleanContents(Op1.getValueType());
1354 if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
1355 !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1356 Op1.getValueType().getVectorElementType() == MVT::i1))
1357 return DAG.UnrollVectorOp(Node);
1358
1359 // If the mask and the type are different sizes, unroll the vector op. This
1360 // can occur when getSetCCResultType returns something that is different in
1361 // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8.
1362 if (VT.getSizeInBits() != Op1.getValueSizeInBits())
1363 return DAG.UnrollVectorOp(Node);
1364
1365 // Bitcast the operands to be the same type as the mask.
1366 // This is needed when we select between FP types because
1367 // the mask is a vector of integers.
1368 Op1 = DAG.getNode(ISD::BITCAST, DL, VT, Op1);
1369 Op2 = DAG.getNode(ISD::BITCAST, DL, VT, Op2);
1370
1371 SDValue NotMask = DAG.getNOT(DL, Mask, VT);
1372
1373 Op1 = DAG.getNode(ISD::AND, DL, VT, Op1, Mask);
1374 Op2 = DAG.getNode(ISD::AND, DL, VT, Op2, NotMask);
1375 SDValue Val = DAG.getNode(ISD::OR, DL, VT, Op1, Op2);
1376 return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val);
1377}
1378
1379SDValue VectorLegalizer::ExpandVP_SELECT(SDNode *Node) {
1380 // Implement VP_SELECT in terms of VP_XOR, VP_AND and VP_OR on platforms which
1381 // do not support it natively.
1382 SDLoc DL(Node);
1383
1384 SDValue Mask = Node->getOperand(0);
1385 SDValue Op1 = Node->getOperand(1);
1386 SDValue Op2 = Node->getOperand(2);
1387 SDValue EVL = Node->getOperand(3);
1388
1389 EVT VT = Mask.getValueType();
1390
1391 // If we can't even use the basic vector operations of
1392 // VP_AND,VP_OR,VP_XOR, we will have to scalarize the op.
1393 if (TLI.getOperationAction(ISD::VP_AND, VT) == TargetLowering::Expand ||
1394 TLI.getOperationAction(ISD::VP_XOR, VT) == TargetLowering::Expand ||
1395 TLI.getOperationAction(ISD::VP_OR, VT) == TargetLowering::Expand)
1396 return DAG.UnrollVectorOp(Node);
1397
1398 // This operation also isn't safe when the operands aren't also booleans.
1399 if (Op1.getValueType().getVectorElementType() != MVT::i1)
1400 return DAG.UnrollVectorOp(Node);
1401
1402 SDValue Ones = DAG.getAllOnesConstant(DL, VT);
1403 SDValue NotMask = DAG.getNode(ISD::VP_XOR, DL, VT, Mask, Ones, Ones, EVL);
1404
1405 Op1 = DAG.getNode(ISD::VP_AND, DL, VT, Op1, Mask, Ones, EVL);
1406 Op2 = DAG.getNode(ISD::VP_AND, DL, VT, Op2, NotMask, Ones, EVL);
1407 return DAG.getNode(ISD::VP_OR, DL, VT, Op1, Op2, Ones, EVL);
1408}
1409
1410SDValue VectorLegalizer::ExpandVP_MERGE(SDNode *Node) {
1411 // Implement VP_MERGE in terms of VSELECT. Construct a mask where vector
1412 // indices less than the EVL/pivot are true. Combine that with the original
1413 // mask for a full-length mask. Use a full-length VSELECT to select between
1414 // the true and false values.
1415 SDLoc DL(Node);
1416
1417 SDValue Mask = Node->getOperand(0);
1418 SDValue Op1 = Node->getOperand(1);
1419 SDValue Op2 = Node->getOperand(2);
1420 SDValue EVL = Node->getOperand(3);
1421
1422 EVT MaskVT = Mask.getValueType();
1423 bool IsFixedLen = MaskVT.isFixedLengthVector();
1424
1425 EVT EVLVecVT = EVT::getVectorVT(*DAG.getContext(), EVL.getValueType(),
1426 MaskVT.getVectorElementCount());
1427
1428 // If we can't construct the EVL mask efficiently, it's better to unroll.
1429 if ((IsFixedLen &&
1430 !TLI.isOperationLegalOrCustom(ISD::BUILD_VECTOR, EVLVecVT)) ||
1431 (!IsFixedLen &&
1432 (!TLI.isOperationLegalOrCustom(ISD::STEP_VECTOR, EVLVecVT) ||
1433 !TLI.isOperationLegalOrCustom(ISD::SPLAT_VECTOR, EVLVecVT))))
1434 return DAG.UnrollVectorOp(Node);
1435
1436 // If using a SETCC would result in a different type than the mask type,
1437 // unroll.
1438 if (TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
1439 EVLVecVT) != MaskVT)
1440 return DAG.UnrollVectorOp(Node);
1441
1442 SDValue StepVec = DAG.getStepVector(DL, EVLVecVT);
1443 SDValue SplatEVL = DAG.getSplat(EVLVecVT, DL, EVL);
1444 SDValue EVLMask =
1445 DAG.getSetCC(DL, MaskVT, StepVec, SplatEVL, ISD::CondCode::SETULT);
1446
1447 SDValue FullMask = DAG.getNode(ISD::AND, DL, MaskVT, Mask, EVLMask);
1448 return DAG.getSelect(DL, Node->getValueType(0), FullMask, Op1, Op2);
1449}
1450
1451SDValue VectorLegalizer::ExpandVP_REM(SDNode *Node) {
1452 // Implement VP_SREM/UREM in terms of VP_SDIV/VP_UDIV, VP_MUL, VP_SUB.
1453 EVT VT = Node->getValueType(0);
1454
1455 unsigned DivOpc = Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV;
1456
1457 if (!TLI.isOperationLegalOrCustom(DivOpc, VT) ||
1458 !TLI.isOperationLegalOrCustom(ISD::VP_MUL, VT) ||
1459 !TLI.isOperationLegalOrCustom(ISD::VP_SUB, VT))
1460 return SDValue();
1461
1462 SDLoc DL(Node);
1463
1464 SDValue Dividend = Node->getOperand(0);
1465 SDValue Divisor = Node->getOperand(1);
1466 SDValue Mask = Node->getOperand(2);
1467 SDValue EVL = Node->getOperand(3);
1468
1469 // X % Y -> X-X/Y*Y
1470 SDValue Div = DAG.getNode(DivOpc, DL, VT, Dividend, Divisor, Mask, EVL);
1471 SDValue Mul = DAG.getNode(ISD::VP_MUL, DL, VT, Divisor, Div, Mask, EVL);
1472 return DAG.getNode(ISD::VP_SUB, DL, VT, Dividend, Mul, Mask, EVL);
1473}
1474
1475void VectorLegalizer::ExpandFP_TO_UINT(SDNode *Node,
1477 // Attempt to expand using TargetLowering.
1478 SDValue Result, Chain;
1479 if (TLI.expandFP_TO_UINT(Node, Result, Chain, DAG)) {
1480 Results.push_back(Result);
1481 if (Node->isStrictFPOpcode())
1482 Results.push_back(Chain);
1483 return;
1484 }
1485
1486 // Otherwise go ahead and unroll.
1487 if (Node->isStrictFPOpcode()) {
1488 UnrollStrictFPOp(Node, Results);
1489 return;
1490 }
1491
1492 Results.push_back(DAG.UnrollVectorOp(Node));
1493}
1494
1495void VectorLegalizer::ExpandUINT_TO_FLOAT(SDNode *Node,
1497 bool IsStrict = Node->isStrictFPOpcode();
1498 unsigned OpNo = IsStrict ? 1 : 0;
1499 SDValue Src = Node->getOperand(OpNo);
1500 EVT VT = Src.getValueType();
1501 SDLoc DL(Node);
1502
1503 // Attempt to expand using TargetLowering.
1505 SDValue Chain;
1506 if (TLI.expandUINT_TO_FP(Node, Result, Chain, DAG)) {
1507 Results.push_back(Result);
1508 if (IsStrict)
1509 Results.push_back(Chain);
1510 return;
1511 }
1512
1513 // Make sure that the SINT_TO_FP and SRL instructions are available.
1514 if (((!IsStrict && TLI.getOperationAction(ISD::SINT_TO_FP, VT) ==
1515 TargetLowering::Expand) ||
1516 (IsStrict && TLI.getOperationAction(ISD::STRICT_SINT_TO_FP, VT) ==
1517 TargetLowering::Expand)) ||
1518 TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Expand) {
1519 if (IsStrict) {
1520 UnrollStrictFPOp(Node, Results);
1521 return;
1522 }
1523
1524 Results.push_back(DAG.UnrollVectorOp(Node));
1525 return;
1526 }
1527
1528 unsigned BW = VT.getScalarSizeInBits();
1529 assert((BW == 64 || BW == 32) &&
1530 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1531
1532 SDValue HalfWord = DAG.getConstant(BW / 2, DL, VT);
1533
1534 // Constants to clear the upper part of the word.
1535 // Notice that we can also use SHL+SHR, but using a constant is slightly
1536 // faster on x86.
1537 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1538 SDValue HalfWordMask = DAG.getConstant(HWMask, DL, VT);
1539
1540 // Two to the power of half-word-size.
1541 SDValue TWOHW =
1542 DAG.getConstantFP(1ULL << (BW / 2), DL, Node->getValueType(0));
1543
1544 // Clear upper part of LO, lower HI
1545 SDValue HI = DAG.getNode(ISD::SRL, DL, VT, Src, HalfWord);
1546 SDValue LO = DAG.getNode(ISD::AND, DL, VT, Src, HalfWordMask);
1547
1548 if (IsStrict) {
1549 // Convert hi and lo to floats
1550 // Convert the hi part back to the upper values
1551 // TODO: Can any fast-math-flags be set on these nodes?
1553 {Node->getValueType(0), MVT::Other},
1554 {Node->getOperand(0), HI});
1555 fHI = DAG.getNode(ISD::STRICT_FMUL, DL, {Node->getValueType(0), MVT::Other},
1556 {fHI.getValue(1), fHI, TWOHW});
1558 {Node->getValueType(0), MVT::Other},
1559 {Node->getOperand(0), LO});
1560
1561 SDValue TF = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, fHI.getValue(1),
1562 fLO.getValue(1));
1563
1564 // Add the two halves
1565 SDValue Result =
1566 DAG.getNode(ISD::STRICT_FADD, DL, {Node->getValueType(0), MVT::Other},
1567 {TF, fHI, fLO});
1568
1569 Results.push_back(Result);
1570 Results.push_back(Result.getValue(1));
1571 return;
1572 }
1573
1574 // Convert hi and lo to floats
1575 // Convert the hi part back to the upper values
1576 // TODO: Can any fast-math-flags be set on these nodes?
1577 SDValue fHI = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), HI);
1578 fHI = DAG.getNode(ISD::FMUL, DL, Node->getValueType(0), fHI, TWOHW);
1579 SDValue fLO = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), LO);
1580
1581 // Add the two halves
1582 Results.push_back(
1583 DAG.getNode(ISD::FADD, DL, Node->getValueType(0), fHI, fLO));
1584}
1585
1586SDValue VectorLegalizer::ExpandFNEG(SDNode *Node) {
1587 if (TLI.isOperationLegalOrCustom(ISD::FSUB, Node->getValueType(0))) {
1588 SDLoc DL(Node);
1589 SDValue Zero = DAG.getConstantFP(-0.0, DL, Node->getValueType(0));
1590 // TODO: If FNEG had fast-math-flags, they'd get propagated to this FSUB.
1591 return DAG.getNode(ISD::FSUB, DL, Node->getValueType(0), Zero,
1592 Node->getOperand(0));
1593 }
1594 return DAG.UnrollVectorOp(Node);
1595}
1596
1597void VectorLegalizer::ExpandFSUB(SDNode *Node,
1599 // For floating-point values, (a-b) is the same as a+(-b). If FNEG is legal,
1600 // we can defer this to operation legalization where it will be lowered as
1601 // a+(-b).
1602 EVT VT = Node->getValueType(0);
1603 if (TLI.isOperationLegalOrCustom(ISD::FNEG, VT) &&
1604 TLI.isOperationLegalOrCustom(ISD::FADD, VT))
1605 return; // Defer to LegalizeDAG
1606
1607 SDValue Tmp = DAG.UnrollVectorOp(Node);
1608 Results.push_back(Tmp);
1609}
1610
1611void VectorLegalizer::ExpandSETCC(SDNode *Node,
1613 bool NeedInvert = false;
1614 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
1615 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
1616 Node->getOpcode() == ISD::STRICT_FSETCCS;
1617 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
1618 unsigned Offset = IsStrict ? 1 : 0;
1619
1620 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
1621 SDValue LHS = Node->getOperand(0 + Offset);
1622 SDValue RHS = Node->getOperand(1 + Offset);
1623 SDValue CC = Node->getOperand(2 + Offset);
1624
1625 MVT OpVT = LHS.getSimpleValueType();
1626 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1627
1628 if (TLI.getCondCodeAction(CCCode, OpVT) != TargetLowering::Expand) {
1629 if (IsStrict) {
1630 UnrollStrictFPOp(Node, Results);
1631 return;
1632 }
1633 Results.push_back(UnrollVSETCC(Node));
1634 return;
1635 }
1636
1637 SDValue Mask, EVL;
1638 if (IsVP) {
1639 Mask = Node->getOperand(3 + Offset);
1640 EVL = Node->getOperand(4 + Offset);
1641 }
1642
1643 SDLoc dl(Node);
1644 bool Legalized =
1645 TLI.LegalizeSetCCCondCode(DAG, Node->getValueType(0), LHS, RHS, CC, Mask,
1646 EVL, NeedInvert, dl, Chain, IsSignaling);
1647
1648 if (Legalized) {
1649 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
1650 // condition code, create a new SETCC node.
1651 if (CC.getNode()) {
1652 if (IsStrict) {
1653 LHS = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(),
1654 {Chain, LHS, RHS, CC}, Node->getFlags());
1655 Chain = LHS.getValue(1);
1656 } else if (IsVP) {
1657 LHS = DAG.getNode(ISD::VP_SETCC, dl, Node->getValueType(0),
1658 {LHS, RHS, CC, Mask, EVL}, Node->getFlags());
1659 } else {
1660 LHS = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), LHS, RHS, CC,
1661 Node->getFlags());
1662 }
1663 }
1664
1665 // If we expanded the SETCC by inverting the condition code, then wrap
1666 // the existing SETCC in a NOT to restore the intended condition.
1667 if (NeedInvert) {
1668 if (!IsVP)
1669 LHS = DAG.getLogicalNOT(dl, LHS, LHS->getValueType(0));
1670 else
1671 LHS = DAG.getVPLogicalNOT(dl, LHS, Mask, EVL, LHS->getValueType(0));
1672 }
1673 } else {
1674 assert(!IsStrict && "Don't know how to expand for strict nodes.");
1675
1676 // Otherwise, SETCC for the given comparison type must be completely
1677 // illegal; expand it into a SELECT_CC.
1678 EVT VT = Node->getValueType(0);
1679 LHS =
1680 DAG.getNode(ISD::SELECT_CC, dl, VT, LHS, RHS,
1681 DAG.getBoolConstant(true, dl, VT, LHS.getValueType()),
1682 DAG.getBoolConstant(false, dl, VT, LHS.getValueType()), CC);
1683 LHS->setFlags(Node->getFlags());
1684 }
1685
1686 Results.push_back(LHS);
1687 if (IsStrict)
1688 Results.push_back(Chain);
1689}
1690
1691void VectorLegalizer::ExpandUADDSUBO(SDNode *Node,
1693 SDValue Result, Overflow;
1694 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
1695 Results.push_back(Result);
1696 Results.push_back(Overflow);
1697}
1698
1699void VectorLegalizer::ExpandSADDSUBO(SDNode *Node,
1701 SDValue Result, Overflow;
1702 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
1703 Results.push_back(Result);
1704 Results.push_back(Overflow);
1705}
1706
1707void VectorLegalizer::ExpandMULO(SDNode *Node,
1709 SDValue Result, Overflow;
1710 if (!TLI.expandMULO(Node, Result, Overflow, DAG))
1711 std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(Node);
1712
1713 Results.push_back(Result);
1714 Results.push_back(Overflow);
1715}
1716
1717void VectorLegalizer::ExpandFixedPointDiv(SDNode *Node,
1719 SDNode *N = Node;
1720 if (SDValue Expanded = TLI.expandFixedPointDiv(N->getOpcode(), SDLoc(N),
1721 N->getOperand(0), N->getOperand(1), N->getConstantOperandVal(2), DAG))
1722 Results.push_back(Expanded);
1723}
1724
1725void VectorLegalizer::ExpandStrictFPOp(SDNode *Node,
1727 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP) {
1728 ExpandUINT_TO_FLOAT(Node, Results);
1729 return;
1730 }
1731 if (Node->getOpcode() == ISD::STRICT_FP_TO_UINT) {
1732 ExpandFP_TO_UINT(Node, Results);
1733 return;
1734 }
1735
1736 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1737 Node->getOpcode() == ISD::STRICT_FSETCCS) {
1738 ExpandSETCC(Node, Results);
1739 return;
1740 }
1741
1742 UnrollStrictFPOp(Node, Results);
1743}
1744
1745void VectorLegalizer::ExpandREM(SDNode *Node,
1747 assert((Node->getOpcode() == ISD::SREM || Node->getOpcode() == ISD::UREM) &&
1748 "Expected REM node");
1749
1751 if (!TLI.expandREM(Node, Result, DAG))
1752 Result = DAG.UnrollVectorOp(Node);
1753 Results.push_back(Result);
1754}
1755
1756void VectorLegalizer::UnrollStrictFPOp(SDNode *Node,
1758 EVT VT = Node->getValueType(0);
1759 EVT EltVT = VT.getVectorElementType();
1760 unsigned NumElems = VT.getVectorNumElements();
1761 unsigned NumOpers = Node->getNumOperands();
1762 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1763
1764 EVT TmpEltVT = EltVT;
1765 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1766 Node->getOpcode() == ISD::STRICT_FSETCCS)
1767 TmpEltVT = TLI.getSetCCResultType(DAG.getDataLayout(),
1768 *DAG.getContext(), TmpEltVT);
1769
1770 EVT ValueVTs[] = {TmpEltVT, MVT::Other};
1771 SDValue Chain = Node->getOperand(0);
1772 SDLoc dl(Node);
1773
1774 SmallVector<SDValue, 32> OpValues;
1775 SmallVector<SDValue, 32> OpChains;
1776 for (unsigned i = 0; i < NumElems; ++i) {
1778 SDValue Idx = DAG.getVectorIdxConstant(i, dl);
1779
1780 // The Chain is the first operand.
1781 Opers.push_back(Chain);
1782
1783 // Now process the remaining operands.
1784 for (unsigned j = 1; j < NumOpers; ++j) {
1785 SDValue Oper = Node->getOperand(j);
1786 EVT OperVT = Oper.getValueType();
1787
1788 if (OperVT.isVector())
1789 Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
1790 OperVT.getVectorElementType(), Oper, Idx);
1791
1792 Opers.push_back(Oper);
1793 }
1794
1795 SDValue ScalarOp = DAG.getNode(Node->getOpcode(), dl, ValueVTs, Opers);
1796 SDValue ScalarResult = ScalarOp.getValue(0);
1797 SDValue ScalarChain = ScalarOp.getValue(1);
1798
1799 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1800 Node->getOpcode() == ISD::STRICT_FSETCCS)
1801 ScalarResult = DAG.getSelect(dl, EltVT, ScalarResult,
1802 DAG.getAllOnesConstant(dl, EltVT),
1803 DAG.getConstant(0, dl, EltVT));
1804
1805 OpValues.push_back(ScalarResult);
1806 OpChains.push_back(ScalarChain);
1807 }
1808
1809 SDValue Result = DAG.getBuildVector(VT, dl, OpValues);
1810 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
1811
1812 Results.push_back(Result);
1813 Results.push_back(NewChain);
1814}
1815
1816SDValue VectorLegalizer::UnrollVSETCC(SDNode *Node) {
1817 EVT VT = Node->getValueType(0);
1818 unsigned NumElems = VT.getVectorNumElements();
1819 EVT EltVT = VT.getVectorElementType();
1820 SDValue LHS = Node->getOperand(0);
1821 SDValue RHS = Node->getOperand(1);
1822 SDValue CC = Node->getOperand(2);
1823 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
1824 SDLoc dl(Node);
1825 SmallVector<SDValue, 8> Ops(NumElems);
1826 for (unsigned i = 0; i < NumElems; ++i) {
1827 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
1828 DAG.getVectorIdxConstant(i, dl));
1829 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
1830 DAG.getVectorIdxConstant(i, dl));
1831 Ops[i] = DAG.getNode(ISD::SETCC, dl,
1832 TLI.getSetCCResultType(DAG.getDataLayout(),
1833 *DAG.getContext(), TmpEltVT),
1834 LHSElem, RHSElem, CC);
1835 Ops[i] = DAG.getSelect(dl, EltVT, Ops[i], DAG.getAllOnesConstant(dl, EltVT),
1836 DAG.getConstant(0, dl, EltVT));
1837 }
1838 return DAG.getBuildVector(VT, dl, Ops);
1839}
1840
1842 return VectorLegalizer(*this).Run();
1843}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
BlockVerifier::State From
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the DenseMap class.
static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl< int > &ShuffleMask)
#define I(x, y, z)
Definition: MD5.cpp:58
mir Rename Register Operands
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
Value * RHS
Value * LHS
BinaryOperator * Mul
DEMANGLE_DUMP_METHOD void dump() const
This class represents an Operation in the Expression.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:220
size_t size() const
Definition: Function.h:768
This class is used to represent ISD::LOAD nodes.
Machine Value Type.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:307
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:478
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void resize(size_type N)
Definition: SmallVector.h:642
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
This class is used to represent ISD::STORE nodes.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Iterator for intrusive lists based on ilist_node.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:119
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:750
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:723
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:476
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1328
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1359
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:250
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:559
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:714
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:367
@ FMAXNUM_IEEE
Definition: ISDOpcodes.h:974
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1026
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:373
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:780
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:483
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:787
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1344
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1348
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:688
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:817
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:255
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1358
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:477
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:900
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:934
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:380
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:450
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1341
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:722
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1345
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:925
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:327
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
Definition: ISDOpcodes.h:646
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:500
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:349
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:727
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1360
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:627
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:323
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1353
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:651
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimum or maximum on two values,...
Definition: ISDOpcodes.h:973
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:535
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:777
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:853
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:742
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:966
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:359
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:331
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:806
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:795
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:674
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:386
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:885
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:736
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:449
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1361
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:443
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:442
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:979
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:833
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1342
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:400
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:939
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:866
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:828
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:852
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1349
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:783
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1329
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:493
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:340
@ AssertZext
Definition: ISDOpcodes.h:62
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:515
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1503
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1483
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:440
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1734
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
#define N
Extended Value Type.
Definition: ValueTypes.h:34
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:93
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:333
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:351
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:363
bool isFixedLengthVector() const
Definition: ValueTypes.h:170
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:160
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:306
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition: ValueTypes.h:166
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:311
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:319
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:291