LLVM 19.0.0git
LegalizeVectorOps.cpp
Go to the documentation of this file.
1//===- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::LegalizeVectors method.
10//
11// The vector legalizer looks for vector operations which might need to be
12// scalarized and legalizes them. This is a separate step from Legalize because
13// scalarizing can introduce illegal types. For example, suppose we have an
14// ISD::SDIV of type v2i64 on x86-32. The type is legal (for example, addition
15// on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the
16// operation, which introduces nodes with the illegal type i64 which must be
17// expanded. Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC;
18// the operation must be unrolled, which introduces nodes with the illegal
19// type i8 which must be promoted.
20//
21// This does not legalize vector manipulations like ISD::BUILD_VECTOR,
22// or operations that happen to take a vector which are custom-lowered;
23// the legalization for such operations never produces nodes
24// with illegal types, so it's okay to put off legalizing them until
25// SelectionDAG::Legalize runs.
26//
27//===----------------------------------------------------------------------===//
28
29#include "llvm/ADT/DenseMap.h"
39#include "llvm/IR/DataLayout.h"
42#include "llvm/Support/Debug.h"
44#include <cassert>
45#include <cstdint>
46#include <iterator>
47#include <utility>
48
49using namespace llvm;
50
51#define DEBUG_TYPE "legalizevectorops"
52
53namespace {
54
55class VectorLegalizer {
56 SelectionDAG& DAG;
57 const TargetLowering &TLI;
58 bool Changed = false; // Keep track of whether anything changed
59
60 /// For nodes that are of legal width, and that have more than one use, this
61 /// map indicates what regularized operand to use. This allows us to avoid
62 /// legalizing the same thing more than once.
64
65 /// Adds a node to the translation cache.
66 void AddLegalizedOperand(SDValue From, SDValue To) {
67 LegalizedNodes.insert(std::make_pair(From, To));
68 // If someone requests legalization of the new node, return itself.
69 if (From != To)
70 LegalizedNodes.insert(std::make_pair(To, To));
71 }
72
73 /// Legalizes the given node.
74 SDValue LegalizeOp(SDValue Op);
75
76 /// Assuming the node is legal, "legalize" the results.
77 SDValue TranslateLegalizeResults(SDValue Op, SDNode *Result);
78
79 /// Make sure Results are legal and update the translation cache.
80 SDValue RecursivelyLegalizeResults(SDValue Op,
82
83 /// Wrapper to interface LowerOperation with a vector of Results.
84 /// Returns false if the target wants to use default expansion. Otherwise
85 /// returns true. If return is true and the Results are empty, then the
86 /// target wants to keep the input node as is.
87 bool LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results);
88
89 /// Implements unrolling a VSETCC.
90 SDValue UnrollVSETCC(SDNode *Node);
91
92 /// Implement expand-based legalization of vector operations.
93 ///
94 /// This is just a high-level routine to dispatch to specific code paths for
95 /// operations to legalize them.
97
98 /// Implements expansion for FP_TO_UINT; falls back to UnrollVectorOp if
99 /// FP_TO_SINT isn't legal.
100 void ExpandFP_TO_UINT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
101
102 /// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if
103 /// SINT_TO_FLOAT and SHR on vectors isn't legal.
104 void ExpandUINT_TO_FLOAT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
105
106 /// Implement expansion for SIGN_EXTEND_INREG using SRL and SRA.
107 SDValue ExpandSEXTINREG(SDNode *Node);
108
109 /// Implement expansion for ANY_EXTEND_VECTOR_INREG.
110 ///
111 /// Shuffles the low lanes of the operand into place and bitcasts to the proper
112 /// type. The contents of the bits in the extended part of each element are
113 /// undef.
114 SDValue ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node);
115
116 /// Implement expansion for SIGN_EXTEND_VECTOR_INREG.
117 ///
118 /// Shuffles the low lanes of the operand into place, bitcasts to the proper
119 /// type, then shifts left and arithmetic shifts right to introduce a sign
120 /// extension.
121 SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node);
122
123 /// Implement expansion for ZERO_EXTEND_VECTOR_INREG.
124 ///
125 /// Shuffles the low lanes of the operand into place and blends zeros into
126 /// the remaining lanes, finally bitcasting to the proper type.
127 SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node);
128
129 /// Expand bswap of vectors into a shuffle if legal.
130 SDValue ExpandBSWAP(SDNode *Node);
131
132 /// Implement vselect in terms of XOR, AND, OR when blend is not
133 /// supported by the target.
134 SDValue ExpandVSELECT(SDNode *Node);
135 SDValue ExpandVP_SELECT(SDNode *Node);
136 SDValue ExpandVP_MERGE(SDNode *Node);
137 SDValue ExpandVP_REM(SDNode *Node);
138 SDValue ExpandSELECT(SDNode *Node);
139 std::pair<SDValue, SDValue> ExpandLoad(SDNode *N);
140 SDValue ExpandStore(SDNode *N);
141 SDValue ExpandFNEG(SDNode *Node);
142 void ExpandFSUB(SDNode *Node, SmallVectorImpl<SDValue> &Results);
143 void ExpandSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
144 void ExpandBITREVERSE(SDNode *Node, SmallVectorImpl<SDValue> &Results);
145 void ExpandUADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
146 void ExpandSADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
147 void ExpandMULO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
148 void ExpandFixedPointDiv(SDNode *Node, SmallVectorImpl<SDValue> &Results);
149 void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
150 void ExpandREM(SDNode *Node, SmallVectorImpl<SDValue> &Results);
151
152 bool tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall LC,
154 bool tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall Call_F32,
155 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
156 RTLIB::Libcall Call_F128,
157 RTLIB::Libcall Call_PPCF128,
159
160 void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
161
162 /// Implements vector promotion.
163 ///
164 /// This is essentially just bitcasting the operands to a different type and
165 /// bitcasting the result back to the original type.
167
168 /// Implements [SU]INT_TO_FP vector promotion.
169 ///
170 /// This is a [zs]ext of the input operand to a larger integer type.
171 void PromoteINT_TO_FP(SDNode *Node, SmallVectorImpl<SDValue> &Results);
172
173 /// Implements FP_TO_[SU]INT vector promotion of the result type.
174 ///
175 /// It is promoted to a larger integer type. The result is then
176 /// truncated back to the original type.
177 void PromoteFP_TO_INT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
178
179 /// Implements vector setcc operation promotion.
180 ///
181 /// All vector operands are promoted to a vector type with larger element
182 /// type.
183 void PromoteSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
184
185 void PromoteSTRICT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
186
187public:
188 VectorLegalizer(SelectionDAG& dag) :
189 DAG(dag), TLI(dag.getTargetLoweringInfo()) {}
190
191 /// Begin legalizer the vector operations in the DAG.
192 bool Run();
193};
194
195} // end anonymous namespace
196
197bool VectorLegalizer::Run() {
198 // Before we start legalizing vector nodes, check if there are any vectors.
199 bool HasVectors = false;
200 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
201 E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I) {
202 // Check if the values of the nodes contain vectors. We don't need to check
203 // the operands because we are going to check their values at some point.
204 HasVectors = llvm::any_of(I->values(), [](EVT T) { return T.isVector(); });
205
206 // If we found a vector node we can start the legalization.
207 if (HasVectors)
208 break;
209 }
210
211 // If this basic block has no vectors then no need to legalize vectors.
212 if (!HasVectors)
213 return false;
214
215 // The legalize process is inherently a bottom-up recursive process (users
216 // legalize their uses before themselves). Given infinite stack space, we
217 // could just start legalizing on the root and traverse the whole graph. In
218 // practice however, this causes us to run out of stack space on large basic
219 // blocks. To avoid this problem, compute an ordering of the nodes where each
220 // node is only legalized after all of its operands are legalized.
221 DAG.AssignTopologicalOrder();
222 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
223 E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I)
224 LegalizeOp(SDValue(&*I, 0));
225
226 // Finally, it's possible the root changed. Get the new root.
227 SDValue OldRoot = DAG.getRoot();
228 assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
229 DAG.setRoot(LegalizedNodes[OldRoot]);
230
231 LegalizedNodes.clear();
232
233 // Remove dead nodes now.
234 DAG.RemoveDeadNodes();
235
236 return Changed;
237}
238
239SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDNode *Result) {
240 assert(Op->getNumValues() == Result->getNumValues() &&
241 "Unexpected number of results");
242 // Generic legalization: just pass the operand through.
243 for (unsigned i = 0, e = Op->getNumValues(); i != e; ++i)
244 AddLegalizedOperand(Op.getValue(i), SDValue(Result, i));
245 return SDValue(Result, Op.getResNo());
246}
247
249VectorLegalizer::RecursivelyLegalizeResults(SDValue Op,
251 assert(Results.size() == Op->getNumValues() &&
252 "Unexpected number of results");
253 // Make sure that the generated code is itself legal.
254 for (unsigned i = 0, e = Results.size(); i != e; ++i) {
255 Results[i] = LegalizeOp(Results[i]);
256 AddLegalizedOperand(Op.getValue(i), Results[i]);
257 }
258
259 return Results[Op.getResNo()];
260}
261
262SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
263 // Note that LegalizeOp may be reentered even from single-use nodes, which
264 // means that we always must cache transformed nodes.
265 DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op);
266 if (I != LegalizedNodes.end()) return I->second;
267
268 // Legalize the operands
270 for (const SDValue &Oper : Op->op_values())
271 Ops.push_back(LegalizeOp(Oper));
272
273 SDNode *Node = DAG.UpdateNodeOperands(Op.getNode(), Ops);
274
275 bool HasVectorValueOrOp =
276 llvm::any_of(Node->values(), [](EVT T) { return T.isVector(); }) ||
277 llvm::any_of(Node->op_values(),
278 [](SDValue O) { return O.getValueType().isVector(); });
279 if (!HasVectorValueOrOp)
280 return TranslateLegalizeResults(Op, Node);
281
282 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
283 EVT ValVT;
284 switch (Op.getOpcode()) {
285 default:
286 return TranslateLegalizeResults(Op, Node);
287 case ISD::LOAD: {
288 LoadSDNode *LD = cast<LoadSDNode>(Node);
289 ISD::LoadExtType ExtType = LD->getExtensionType();
290 EVT LoadedVT = LD->getMemoryVT();
291 if (LoadedVT.isVector() && ExtType != ISD::NON_EXTLOAD)
292 Action = TLI.getLoadExtAction(ExtType, LD->getValueType(0), LoadedVT);
293 break;
294 }
295 case ISD::STORE: {
296 StoreSDNode *ST = cast<StoreSDNode>(Node);
297 EVT StVT = ST->getMemoryVT();
298 MVT ValVT = ST->getValue().getSimpleValueType();
299 if (StVT.isVector() && ST->isTruncatingStore())
300 Action = TLI.getTruncStoreAction(ValVT, StVT);
301 break;
302 }
304 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
305 // This operation lies about being legal: when it claims to be legal,
306 // it should actually be expanded.
307 if (Action == TargetLowering::Legal)
308 Action = TargetLowering::Expand;
309 break;
310#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
311 case ISD::STRICT_##DAGN:
312#include "llvm/IR/ConstrainedOps.def"
313 ValVT = Node->getValueType(0);
314 if (Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
315 Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
316 ValVT = Node->getOperand(1).getValueType();
317 if (Op.getOpcode() == ISD::STRICT_FSETCC ||
318 Op.getOpcode() == ISD::STRICT_FSETCCS) {
319 MVT OpVT = Node->getOperand(1).getSimpleValueType();
320 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(3))->get();
321 Action = TLI.getCondCodeAction(CCCode, OpVT);
322 if (Action == TargetLowering::Legal)
323 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
324 } else {
325 Action = TLI.getOperationAction(Node->getOpcode(), ValVT);
326 }
327 // If we're asked to expand a strict vector floating-point operation,
328 // by default we're going to simply unroll it. That is usually the
329 // best approach, except in the case where the resulting strict (scalar)
330 // operations would themselves use the fallback mutation to non-strict.
331 // In that specific case, just do the fallback on the vector op.
332 if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
333 TLI.getStrictFPOperationAction(Node->getOpcode(), ValVT) ==
334 TargetLowering::Legal) {
335 EVT EltVT = ValVT.getVectorElementType();
336 if (TLI.getOperationAction(Node->getOpcode(), EltVT)
337 == TargetLowering::Expand &&
338 TLI.getStrictFPOperationAction(Node->getOpcode(), EltVT)
339 == TargetLowering::Legal)
340 Action = TargetLowering::Legal;
341 }
342 break;
343 case ISD::ADD:
344 case ISD::SUB:
345 case ISD::MUL:
346 case ISD::MULHS:
347 case ISD::MULHU:
348 case ISD::SDIV:
349 case ISD::UDIV:
350 case ISD::SREM:
351 case ISD::UREM:
352 case ISD::SDIVREM:
353 case ISD::UDIVREM:
354 case ISD::FADD:
355 case ISD::FSUB:
356 case ISD::FMUL:
357 case ISD::FDIV:
358 case ISD::FREM:
359 case ISD::AND:
360 case ISD::OR:
361 case ISD::XOR:
362 case ISD::SHL:
363 case ISD::SRA:
364 case ISD::SRL:
365 case ISD::FSHL:
366 case ISD::FSHR:
367 case ISD::ROTL:
368 case ISD::ROTR:
369 case ISD::ABS:
370 case ISD::ABDS:
371 case ISD::ABDU:
372 case ISD::AVGCEILS:
373 case ISD::AVGCEILU:
374 case ISD::AVGFLOORS:
375 case ISD::AVGFLOORU:
376 case ISD::BSWAP:
377 case ISD::BITREVERSE:
378 case ISD::CTLZ:
379 case ISD::CTTZ:
382 case ISD::CTPOP:
383 case ISD::SELECT:
384 case ISD::VSELECT:
385 case ISD::SELECT_CC:
386 case ISD::ZERO_EXTEND:
387 case ISD::ANY_EXTEND:
388 case ISD::TRUNCATE:
389 case ISD::SIGN_EXTEND:
390 case ISD::FP_TO_SINT:
391 case ISD::FP_TO_UINT:
392 case ISD::FNEG:
393 case ISD::FABS:
394 case ISD::FMINNUM:
395 case ISD::FMAXNUM:
398 case ISD::FMINIMUM:
399 case ISD::FMAXIMUM:
400 case ISD::FCOPYSIGN:
401 case ISD::FSQRT:
402 case ISD::FSIN:
403 case ISD::FCOS:
404 case ISD::FTAN:
405 case ISD::FLDEXP:
406 case ISD::FPOWI:
407 case ISD::FPOW:
408 case ISD::FLOG:
409 case ISD::FLOG2:
410 case ISD::FLOG10:
411 case ISD::FEXP:
412 case ISD::FEXP2:
413 case ISD::FEXP10:
414 case ISD::FCEIL:
415 case ISD::FTRUNC:
416 case ISD::FRINT:
417 case ISD::FNEARBYINT:
418 case ISD::FROUND:
419 case ISD::FROUNDEVEN:
420 case ISD::FFLOOR:
421 case ISD::FP_ROUND:
422 case ISD::FP_EXTEND:
424 case ISD::FMA:
429 case ISD::SMIN:
430 case ISD::SMAX:
431 case ISD::UMIN:
432 case ISD::UMAX:
433 case ISD::SMUL_LOHI:
434 case ISD::UMUL_LOHI:
435 case ISD::SADDO:
436 case ISD::UADDO:
437 case ISD::SSUBO:
438 case ISD::USUBO:
439 case ISD::SMULO:
440 case ISD::UMULO:
442 case ISD::FFREXP:
443 case ISD::SADDSAT:
444 case ISD::UADDSAT:
445 case ISD::SSUBSAT:
446 case ISD::USUBSAT:
447 case ISD::SSHLSAT:
448 case ISD::USHLSAT:
451 case ISD::MGATHER:
452 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
453 break;
454 case ISD::SMULFIX:
455 case ISD::SMULFIXSAT:
456 case ISD::UMULFIX:
457 case ISD::UMULFIXSAT:
458 case ISD::SDIVFIX:
459 case ISD::SDIVFIXSAT:
460 case ISD::UDIVFIX:
461 case ISD::UDIVFIXSAT: {
462 unsigned Scale = Node->getConstantOperandVal(2);
463 Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
464 Node->getValueType(0), Scale);
465 break;
466 }
467 case ISD::LRINT:
468 case ISD::LLRINT:
469 case ISD::SINT_TO_FP:
470 case ISD::UINT_TO_FP:
486 Action = TLI.getOperationAction(Node->getOpcode(),
487 Node->getOperand(0).getValueType());
488 break;
491 Action = TLI.getOperationAction(Node->getOpcode(),
492 Node->getOperand(1).getValueType());
493 break;
494 case ISD::SETCC: {
495 MVT OpVT = Node->getOperand(0).getSimpleValueType();
496 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
497 Action = TLI.getCondCodeAction(CCCode, OpVT);
498 if (Action == TargetLowering::Legal)
499 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
500 break;
501 }
502
503#define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...) \
504 case ISD::VPID: { \
505 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \
506 : Node->getOperand(LEGALPOS).getValueType(); \
507 if (ISD::VPID == ISD::VP_SETCC) { \
508 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); \
509 Action = TLI.getCondCodeAction(CCCode, LegalizeVT.getSimpleVT()); \
510 if (Action != TargetLowering::Legal) \
511 break; \
512 } \
513 /* Defer non-vector results to LegalizeDAG. */ \
514 if (!Node->getValueType(0).isVector() && \
515 Node->getValueType(0) != MVT::Other) { \
516 Action = TargetLowering::Legal; \
517 break; \
518 } \
519 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \
520 } break;
521#include "llvm/IR/VPIntrinsics.def"
522 }
523
524 LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG));
525
526 SmallVector<SDValue, 8> ResultVals;
527 switch (Action) {
528 default: llvm_unreachable("This action is not supported yet!");
529 case TargetLowering::Promote:
530 assert((Op.getOpcode() != ISD::LOAD && Op.getOpcode() != ISD::STORE) &&
531 "This action is not supported yet!");
532 LLVM_DEBUG(dbgs() << "Promoting\n");
533 Promote(Node, ResultVals);
534 assert(!ResultVals.empty() && "No results for promotion?");
535 break;
536 case TargetLowering::Legal:
537 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
538 break;
539 case TargetLowering::Custom:
540 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
541 if (LowerOperationWrapper(Node, ResultVals))
542 break;
543 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
544 [[fallthrough]];
545 case TargetLowering::Expand:
546 LLVM_DEBUG(dbgs() << "Expanding\n");
547 Expand(Node, ResultVals);
548 break;
549 }
550
551 if (ResultVals.empty())
552 return TranslateLegalizeResults(Op, Node);
553
554 Changed = true;
555 return RecursivelyLegalizeResults(Op, ResultVals);
556}
557
558// FIXME: This is very similar to TargetLowering::LowerOperationWrapper. Can we
559// merge them somehow?
560bool VectorLegalizer::LowerOperationWrapper(SDNode *Node,
562 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
563
564 if (!Res.getNode())
565 return false;
566
567 if (Res == SDValue(Node, 0))
568 return true;
569
570 // If the original node has one result, take the return value from
571 // LowerOperation as is. It might not be result number 0.
572 if (Node->getNumValues() == 1) {
573 Results.push_back(Res);
574 return true;
575 }
576
577 // If the original node has multiple results, then the return node should
578 // have the same number of results.
579 assert((Node->getNumValues() == Res->getNumValues()) &&
580 "Lowering returned the wrong number of results!");
581
582 // Places new result values base on N result number.
583 for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I)
584 Results.push_back(Res.getValue(I));
585
586 return true;
587}
588
589void VectorLegalizer::PromoteSETCC(SDNode *Node,
591 MVT VecVT = Node->getOperand(0).getSimpleValueType();
592 MVT NewVecVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VecVT);
593
594 unsigned ExtOp = VecVT.isFloatingPoint() ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
595
596 SDLoc DL(Node);
597 SmallVector<SDValue, 5> Operands(Node->getNumOperands());
598
599 Operands[0] = DAG.getNode(ExtOp, DL, NewVecVT, Node->getOperand(0));
600 Operands[1] = DAG.getNode(ExtOp, DL, NewVecVT, Node->getOperand(1));
601 Operands[2] = Node->getOperand(2);
602
603 if (Node->getOpcode() == ISD::VP_SETCC) {
604 Operands[3] = Node->getOperand(3); // mask
605 Operands[4] = Node->getOperand(4); // evl
606 }
607
608 SDValue Res = DAG.getNode(Node->getOpcode(), DL, Node->getSimpleValueType(0),
609 Operands, Node->getFlags());
610
611 Results.push_back(Res);
612}
613
614void VectorLegalizer::PromoteSTRICT(SDNode *Node,
616 MVT VecVT = Node->getOperand(1).getSimpleValueType();
617 MVT NewVecVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VecVT);
618
619 assert(VecVT.isFloatingPoint());
620
621 SDLoc DL(Node);
622 SmallVector<SDValue, 5> Operands(Node->getNumOperands());
624
625 for (unsigned j = 1; j != Node->getNumOperands(); ++j)
626 if (Node->getOperand(j).getValueType().isVector() &&
627 !(ISD::isVPOpcode(Node->getOpcode()) &&
628 ISD::getVPMaskIdx(Node->getOpcode()) == j)) // Skip mask operand.
629 {
630 // promote the vector operand.
631 SDValue Ext =
632 DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {NewVecVT, MVT::Other},
633 {Node->getOperand(0), Node->getOperand(j)});
634 Operands[j] = Ext.getValue(0);
635 Chains.push_back(Ext.getValue(1));
636 } else
637 Operands[j] = Node->getOperand(j); // Skip no vector operand.
638
639 SDVTList VTs = DAG.getVTList(NewVecVT, Node->getValueType(1));
640
641 Operands[0] = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
642
643 SDValue Res =
644 DAG.getNode(Node->getOpcode(), DL, VTs, Operands, Node->getFlags());
645
646 SDValue Round =
647 DAG.getNode(ISD::STRICT_FP_ROUND, DL, {VecVT, MVT::Other},
648 {Res.getValue(1), Res.getValue(0),
649 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)});
650
651 Results.push_back(Round.getValue(0));
652 Results.push_back(Round.getValue(1));
653}
654
655void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
656 // For a few operations there is a specific concept for promotion based on
657 // the operand's type.
658 switch (Node->getOpcode()) {
659 case ISD::SINT_TO_FP:
660 case ISD::UINT_TO_FP:
663 // "Promote" the operation by extending the operand.
664 PromoteINT_TO_FP(Node, Results);
665 return;
666 case ISD::FP_TO_UINT:
667 case ISD::FP_TO_SINT:
670 // Promote the operation by extending the operand.
671 PromoteFP_TO_INT(Node, Results);
672 return;
673 case ISD::VP_SETCC:
674 case ISD::SETCC:
675 // Promote the operation by extending the operand.
676 PromoteSETCC(Node, Results);
677 return;
678 case ISD::STRICT_FADD:
679 case ISD::STRICT_FSUB:
680 case ISD::STRICT_FMUL:
681 case ISD::STRICT_FDIV:
683 case ISD::STRICT_FMA:
684 PromoteSTRICT(Node, Results);
685 return;
686 case ISD::FP_ROUND:
687 case ISD::FP_EXTEND:
688 // These operations are used to do promotion so they can't be promoted
689 // themselves.
690 llvm_unreachable("Don't know how to promote this operation!");
691 }
692
693 // There are currently two cases of vector promotion:
694 // 1) Bitcasting a vector of integers to a different type to a vector of the
695 // same overall length. For example, x86 promotes ISD::AND v2i32 to v1i64.
696 // 2) Extending a vector of floats to a vector of the same number of larger
697 // floats. For example, AArch64 promotes ISD::FADD on v4f16 to v4f32.
698 assert(Node->getNumValues() == 1 &&
699 "Can't promote a vector with multiple results!");
700 MVT VT = Node->getSimpleValueType(0);
701 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
702 SDLoc dl(Node);
703 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
704
705 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
706 // Do not promote the mask operand of a VP OP.
707 bool SkipPromote = ISD::isVPOpcode(Node->getOpcode()) &&
708 ISD::getVPMaskIdx(Node->getOpcode()) == j;
709 if (Node->getOperand(j).getValueType().isVector() && !SkipPromote)
710 if (Node->getOperand(j)
711 .getValueType()
712 .getVectorElementType()
713 .isFloatingPoint() &&
715 Operands[j] = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(j));
716 else
717 Operands[j] = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(j));
718 else
719 Operands[j] = Node->getOperand(j);
720 }
721
722 SDValue Res =
723 DAG.getNode(Node->getOpcode(), dl, NVT, Operands, Node->getFlags());
724
725 if ((VT.isFloatingPoint() && NVT.isFloatingPoint()) ||
728 Res = DAG.getNode(ISD::FP_ROUND, dl, VT, Res,
729 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
730 else
731 Res = DAG.getNode(ISD::BITCAST, dl, VT, Res);
732
733 Results.push_back(Res);
734}
735
736void VectorLegalizer::PromoteINT_TO_FP(SDNode *Node,
738 // INT_TO_FP operations may require the input operand be promoted even
739 // when the type is otherwise legal.
740 bool IsStrict = Node->isStrictFPOpcode();
741 MVT VT = Node->getOperand(IsStrict ? 1 : 0).getSimpleValueType();
742 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
744 "Vectors have different number of elements!");
745
746 SDLoc dl(Node);
747 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
748
749 unsigned Opc = (Node->getOpcode() == ISD::UINT_TO_FP ||
750 Node->getOpcode() == ISD::STRICT_UINT_TO_FP)
753 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
754 if (Node->getOperand(j).getValueType().isVector())
755 Operands[j] = DAG.getNode(Opc, dl, NVT, Node->getOperand(j));
756 else
757 Operands[j] = Node->getOperand(j);
758 }
759
760 if (IsStrict) {
761 SDValue Res = DAG.getNode(Node->getOpcode(), dl,
762 {Node->getValueType(0), MVT::Other}, Operands);
763 Results.push_back(Res);
764 Results.push_back(Res.getValue(1));
765 return;
766 }
767
768 SDValue Res =
769 DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Operands);
770 Results.push_back(Res);
771}
772
773// For FP_TO_INT we promote the result type to a vector type with wider
774// elements and then truncate the result. This is different from the default
775// PromoteVector which uses bitcast to promote thus assumning that the
776// promoted vector type has the same overall size.
777void VectorLegalizer::PromoteFP_TO_INT(SDNode *Node,
779 MVT VT = Node->getSimpleValueType(0);
780 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
781 bool IsStrict = Node->isStrictFPOpcode();
783 "Vectors have different number of elements!");
784
785 unsigned NewOpc = Node->getOpcode();
786 // Change FP_TO_UINT to FP_TO_SINT if possible.
787 // TODO: Should we only do this if FP_TO_UINT itself isn't legal?
788 if (NewOpc == ISD::FP_TO_UINT &&
789 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
790 NewOpc = ISD::FP_TO_SINT;
791
792 if (NewOpc == ISD::STRICT_FP_TO_UINT &&
793 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT))
794 NewOpc = ISD::STRICT_FP_TO_SINT;
795
796 SDLoc dl(Node);
797 SDValue Promoted, Chain;
798 if (IsStrict) {
799 Promoted = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
800 {Node->getOperand(0), Node->getOperand(1)});
801 Chain = Promoted.getValue(1);
802 } else
803 Promoted = DAG.getNode(NewOpc, dl, NVT, Node->getOperand(0));
804
805 // Assert that the converted value fits in the original type. If it doesn't
806 // (eg: because the value being converted is too big), then the result of the
807 // original operation was undefined anyway, so the assert is still correct.
808 if (Node->getOpcode() == ISD::FP_TO_UINT ||
809 Node->getOpcode() == ISD::STRICT_FP_TO_UINT)
810 NewOpc = ISD::AssertZext;
811 else
812 NewOpc = ISD::AssertSext;
813
814 Promoted = DAG.getNode(NewOpc, dl, NVT, Promoted,
815 DAG.getValueType(VT.getScalarType()));
816 Promoted = DAG.getNode(ISD::TRUNCATE, dl, VT, Promoted);
817 Results.push_back(Promoted);
818 if (IsStrict)
819 Results.push_back(Chain);
820}
821
822std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(SDNode *N) {
823 LoadSDNode *LD = cast<LoadSDNode>(N);
824 return TLI.scalarizeVectorLoad(LD, DAG);
825}
826
827SDValue VectorLegalizer::ExpandStore(SDNode *N) {
828 StoreSDNode *ST = cast<StoreSDNode>(N);
829 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
830 return TF;
831}
832
833void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
834 switch (Node->getOpcode()) {
835 case ISD::LOAD: {
836 std::pair<SDValue, SDValue> Tmp = ExpandLoad(Node);
837 Results.push_back(Tmp.first);
838 Results.push_back(Tmp.second);
839 return;
840 }
841 case ISD::STORE:
842 Results.push_back(ExpandStore(Node));
843 return;
845 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
846 Results.push_back(Node->getOperand(i));
847 return;
849 Results.push_back(ExpandSEXTINREG(Node));
850 return;
852 Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(Node));
853 return;
855 Results.push_back(ExpandSIGN_EXTEND_VECTOR_INREG(Node));
856 return;
858 Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(Node));
859 return;
860 case ISD::BSWAP:
861 Results.push_back(ExpandBSWAP(Node));
862 return;
863 case ISD::VP_BSWAP:
864 Results.push_back(TLI.expandVPBSWAP(Node, DAG));
865 return;
866 case ISD::VSELECT:
867 Results.push_back(ExpandVSELECT(Node));
868 return;
869 case ISD::VP_SELECT:
870 Results.push_back(ExpandVP_SELECT(Node));
871 return;
872 case ISD::VP_SREM:
873 case ISD::VP_UREM:
874 if (SDValue Expanded = ExpandVP_REM(Node)) {
875 Results.push_back(Expanded);
876 return;
877 }
878 break;
879 case ISD::SELECT:
880 Results.push_back(ExpandSELECT(Node));
881 return;
882 case ISD::SELECT_CC: {
883 if (Node->getValueType(0).isScalableVector()) {
884 EVT CondVT = TLI.getSetCCResultType(
885 DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));
886 SDValue SetCC =
887 DAG.getNode(ISD::SETCC, SDLoc(Node), CondVT, Node->getOperand(0),
888 Node->getOperand(1), Node->getOperand(4));
889 Results.push_back(DAG.getSelect(SDLoc(Node), Node->getValueType(0), SetCC,
890 Node->getOperand(2),
891 Node->getOperand(3)));
892 return;
893 }
894 break;
895 }
896 case ISD::FP_TO_UINT:
897 ExpandFP_TO_UINT(Node, Results);
898 return;
899 case ISD::UINT_TO_FP:
900 ExpandUINT_TO_FLOAT(Node, Results);
901 return;
902 case ISD::FNEG:
903 Results.push_back(ExpandFNEG(Node));
904 return;
905 case ISD::FSUB:
906 ExpandFSUB(Node, Results);
907 return;
908 case ISD::SETCC:
909 case ISD::VP_SETCC:
910 ExpandSETCC(Node, Results);
911 return;
912 case ISD::ABS:
913 if (SDValue Expanded = TLI.expandABS(Node, DAG)) {
914 Results.push_back(Expanded);
915 return;
916 }
917 break;
918 case ISD::ABDS:
919 case ISD::ABDU:
920 if (SDValue Expanded = TLI.expandABD(Node, DAG)) {
921 Results.push_back(Expanded);
922 return;
923 }
924 break;
925 case ISD::AVGCEILS:
926 case ISD::AVGCEILU:
927 case ISD::AVGFLOORS:
928 case ISD::AVGFLOORU:
929 if (SDValue Expanded = TLI.expandAVG(Node, DAG)) {
930 Results.push_back(Expanded);
931 return;
932 }
933 break;
934 case ISD::BITREVERSE:
935 ExpandBITREVERSE(Node, Results);
936 return;
937 case ISD::VP_BITREVERSE:
938 if (SDValue Expanded = TLI.expandVPBITREVERSE(Node, DAG)) {
939 Results.push_back(Expanded);
940 return;
941 }
942 break;
943 case ISD::CTPOP:
944 if (SDValue Expanded = TLI.expandCTPOP(Node, DAG)) {
945 Results.push_back(Expanded);
946 return;
947 }
948 break;
949 case ISD::VP_CTPOP:
950 if (SDValue Expanded = TLI.expandVPCTPOP(Node, DAG)) {
951 Results.push_back(Expanded);
952 return;
953 }
954 break;
955 case ISD::CTLZ:
957 if (SDValue Expanded = TLI.expandCTLZ(Node, DAG)) {
958 Results.push_back(Expanded);
959 return;
960 }
961 break;
962 case ISD::VP_CTLZ:
963 case ISD::VP_CTLZ_ZERO_UNDEF:
964 if (SDValue Expanded = TLI.expandVPCTLZ(Node, DAG)) {
965 Results.push_back(Expanded);
966 return;
967 }
968 break;
969 case ISD::CTTZ:
971 if (SDValue Expanded = TLI.expandCTTZ(Node, DAG)) {
972 Results.push_back(Expanded);
973 return;
974 }
975 break;
976 case ISD::VP_CTTZ:
977 case ISD::VP_CTTZ_ZERO_UNDEF:
978 if (SDValue Expanded = TLI.expandVPCTTZ(Node, DAG)) {
979 Results.push_back(Expanded);
980 return;
981 }
982 break;
983 case ISD::FSHL:
984 case ISD::VP_FSHL:
985 case ISD::FSHR:
986 case ISD::VP_FSHR:
987 if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG)) {
988 Results.push_back(Expanded);
989 return;
990 }
991 break;
992 case ISD::ROTL:
993 case ISD::ROTR:
994 if (SDValue Expanded = TLI.expandROT(Node, false /*AllowVectorOps*/, DAG)) {
995 Results.push_back(Expanded);
996 return;
997 }
998 break;
999 case ISD::FMINNUM:
1000 case ISD::FMAXNUM:
1001 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) {
1002 Results.push_back(Expanded);
1003 return;
1004 }
1005 break;
1006 case ISD::FMINIMUM:
1007 case ISD::FMAXIMUM:
1008 Results.push_back(TLI.expandFMINIMUM_FMAXIMUM(Node, DAG));
1009 return;
1010 case ISD::SMIN:
1011 case ISD::SMAX:
1012 case ISD::UMIN:
1013 case ISD::UMAX:
1014 if (SDValue Expanded = TLI.expandIntMINMAX(Node, DAG)) {
1015 Results.push_back(Expanded);
1016 return;
1017 }
1018 break;
1019 case ISD::UADDO:
1020 case ISD::USUBO:
1021 ExpandUADDSUBO(Node, Results);
1022 return;
1023 case ISD::SADDO:
1024 case ISD::SSUBO:
1025 ExpandSADDSUBO(Node, Results);
1026 return;
1027 case ISD::UMULO:
1028 case ISD::SMULO:
1029 ExpandMULO(Node, Results);
1030 return;
1031 case ISD::USUBSAT:
1032 case ISD::SSUBSAT:
1033 case ISD::UADDSAT:
1034 case ISD::SADDSAT:
1035 if (SDValue Expanded = TLI.expandAddSubSat(Node, DAG)) {
1036 Results.push_back(Expanded);
1037 return;
1038 }
1039 break;
1040 case ISD::USHLSAT:
1041 case ISD::SSHLSAT:
1042 if (SDValue Expanded = TLI.expandShlSat(Node, DAG)) {
1043 Results.push_back(Expanded);
1044 return;
1045 }
1046 break;
1049 // Expand the fpsosisat if it is scalable to prevent it from unrolling below.
1050 if (Node->getValueType(0).isScalableVector()) {
1051 if (SDValue Expanded = TLI.expandFP_TO_INT_SAT(Node, DAG)) {
1052 Results.push_back(Expanded);
1053 return;
1054 }
1055 }
1056 break;
1057 case ISD::SMULFIX:
1058 case ISD::UMULFIX:
1059 if (SDValue Expanded = TLI.expandFixedPointMul(Node, DAG)) {
1060 Results.push_back(Expanded);
1061 return;
1062 }
1063 break;
1064 case ISD::SMULFIXSAT:
1065 case ISD::UMULFIXSAT:
1066 // FIXME: We do not expand SMULFIXSAT/UMULFIXSAT here yet, not sure exactly
1067 // why. Maybe it results in worse codegen compared to the unroll for some
1068 // targets? This should probably be investigated. And if we still prefer to
1069 // unroll an explanation could be helpful.
1070 break;
1071 case ISD::SDIVFIX:
1072 case ISD::UDIVFIX:
1073 ExpandFixedPointDiv(Node, Results);
1074 return;
1075 case ISD::SDIVFIXSAT:
1076 case ISD::UDIVFIXSAT:
1077 break;
1078#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1079 case ISD::STRICT_##DAGN:
1080#include "llvm/IR/ConstrainedOps.def"
1081 ExpandStrictFPOp(Node, Results);
1082 return;
1083 case ISD::VECREDUCE_ADD:
1084 case ISD::VECREDUCE_MUL:
1085 case ISD::VECREDUCE_AND:
1086 case ISD::VECREDUCE_OR:
1087 case ISD::VECREDUCE_XOR:
1098 Results.push_back(TLI.expandVecReduce(Node, DAG));
1099 return;
1102 Results.push_back(TLI.expandVecReduceSeq(Node, DAG));
1103 return;
1104 case ISD::SREM:
1105 case ISD::UREM:
1106 ExpandREM(Node, Results);
1107 return;
1108 case ISD::VP_MERGE:
1109 Results.push_back(ExpandVP_MERGE(Node));
1110 return;
1111 case ISD::FREM:
1112 if (tryExpandVecMathCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
1113 RTLIB::REM_F80, RTLIB::REM_F128,
1114 RTLIB::REM_PPCF128, Results))
1115 return;
1116
1117 break;
1118 }
1119
1120 SDValue Unrolled = DAG.UnrollVectorOp(Node);
1121 if (Node->getNumValues() == 1) {
1122 Results.push_back(Unrolled);
1123 } else {
1124 assert(Node->getNumValues() == Unrolled->getNumValues() &&
1125 "VectorLegalizer Expand returned wrong number of results!");
1126 for (unsigned I = 0, E = Unrolled->getNumValues(); I != E; ++I)
1127 Results.push_back(Unrolled.getValue(I));
1128 }
1129}
1130
1131SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) {
1132 // Lower a select instruction where the condition is a scalar and the
1133 // operands are vectors. Lower this select to VSELECT and implement it
1134 // using XOR AND OR. The selector bit is broadcasted.
1135 EVT VT = Node->getValueType(0);
1136 SDLoc DL(Node);
1137
1138 SDValue Mask = Node->getOperand(0);
1139 SDValue Op1 = Node->getOperand(1);
1140 SDValue Op2 = Node->getOperand(2);
1141
1142 assert(VT.isVector() && !Mask.getValueType().isVector()
1143 && Op1.getValueType() == Op2.getValueType() && "Invalid type");
1144
1145 // If we can't even use the basic vector operations of
1146 // AND,OR,XOR, we will have to scalarize the op.
1147 // Notice that the operation may be 'promoted' which means that it is
1148 // 'bitcasted' to another type which is handled.
1149 // Also, we need to be able to construct a splat vector using either
1150 // BUILD_VECTOR or SPLAT_VECTOR.
1151 // FIXME: Should we also permit fixed-length SPLAT_VECTOR as a fallback to
1152 // BUILD_VECTOR?
1153 if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
1154 TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
1155 TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand ||
1156 TLI.getOperationAction(VT.isFixedLengthVector() ? ISD::BUILD_VECTOR
1158 VT) == TargetLowering::Expand)
1159 return DAG.UnrollVectorOp(Node);
1160
1161 // Generate a mask operand.
1163
1164 // What is the size of each element in the vector mask.
1165 EVT BitTy = MaskTy.getScalarType();
1166
1167 Mask = DAG.getSelect(DL, BitTy, Mask, DAG.getAllOnesConstant(DL, BitTy),
1168 DAG.getConstant(0, DL, BitTy));
1169
1170 // Broadcast the mask so that the entire vector is all one or all zero.
1171 Mask = DAG.getSplat(MaskTy, DL, Mask);
1172
1173 // Bitcast the operands to be the same type as the mask.
1174 // This is needed when we select between FP types because
1175 // the mask is a vector of integers.
1176 Op1 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op1);
1177 Op2 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op2);
1178
1179 SDValue NotMask = DAG.getNOT(DL, Mask, MaskTy);
1180
1181 Op1 = DAG.getNode(ISD::AND, DL, MaskTy, Op1, Mask);
1182 Op2 = DAG.getNode(ISD::AND, DL, MaskTy, Op2, NotMask);
1183 SDValue Val = DAG.getNode(ISD::OR, DL, MaskTy, Op1, Op2);
1184 return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val);
1185}
1186
1187SDValue VectorLegalizer::ExpandSEXTINREG(SDNode *Node) {
1188 EVT VT = Node->getValueType(0);
1189
1190 // Make sure that the SRA and SHL instructions are available.
1191 if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Expand ||
1192 TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Expand)
1193 return DAG.UnrollVectorOp(Node);
1194
1195 SDLoc DL(Node);
1196 EVT OrigTy = cast<VTSDNode>(Node->getOperand(1))->getVT();
1197
1198 unsigned BW = VT.getScalarSizeInBits();
1199 unsigned OrigBW = OrigTy.getScalarSizeInBits();
1200 SDValue ShiftSz = DAG.getConstant(BW - OrigBW, DL, VT);
1201
1202 SDValue Op = DAG.getNode(ISD::SHL, DL, VT, Node->getOperand(0), ShiftSz);
1203 return DAG.getNode(ISD::SRA, DL, VT, Op, ShiftSz);
1204}
1205
1206// Generically expand a vector anyext in register to a shuffle of the relevant
1207// lanes into the appropriate locations, with other lanes left undef.
1208SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node) {
1209 SDLoc DL(Node);
1210 EVT VT = Node->getValueType(0);
1211 int NumElements = VT.getVectorNumElements();
1212 SDValue Src = Node->getOperand(0);
1213 EVT SrcVT = Src.getValueType();
1214 int NumSrcElements = SrcVT.getVectorNumElements();
1215
1216 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1217 // into a larger vector type.
1218 if (SrcVT.bitsLE(VT)) {
1219 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1220 "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1221 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1222 SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
1223 NumSrcElements);
1224 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT),
1225 Src, DAG.getVectorIdxConstant(0, DL));
1226 }
1227
1228 // Build a base mask of undef shuffles.
1229 SmallVector<int, 16> ShuffleMask;
1230 ShuffleMask.resize(NumSrcElements, -1);
1231
1232 // Place the extended lanes into the correct locations.
1233 int ExtLaneScale = NumSrcElements / NumElements;
1234 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1235 for (int i = 0; i < NumElements; ++i)
1236 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1237
1238 return DAG.getNode(
1239 ISD::BITCAST, DL, VT,
1240 DAG.getVectorShuffle(SrcVT, DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
1241}
1242
1243SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node) {
1244 SDLoc DL(Node);
1245 EVT VT = Node->getValueType(0);
1246 SDValue Src = Node->getOperand(0);
1247 EVT SrcVT = Src.getValueType();
1248
1249 // First build an any-extend node which can be legalized above when we
1250 // recurse through it.
1251 SDValue Op = DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Src);
1252
1253 // Now we need sign extend. Do this by shifting the elements. Even if these
1254 // aren't legal operations, they have a better chance of being legalized
1255 // without full scalarization than the sign extension does.
1256 unsigned EltWidth = VT.getScalarSizeInBits();
1257 unsigned SrcEltWidth = SrcVT.getScalarSizeInBits();
1258 SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth, DL, VT);
1259 return DAG.getNode(ISD::SRA, DL, VT,
1260 DAG.getNode(ISD::SHL, DL, VT, Op, ShiftAmount),
1261 ShiftAmount);
1262}
1263
1264// Generically expand a vector zext in register to a shuffle of the relevant
1265// lanes into the appropriate locations, a blend of zero into the high bits,
1266// and a bitcast to the wider element type.
1267SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node) {
1268 SDLoc DL(Node);
1269 EVT VT = Node->getValueType(0);
1270 int NumElements = VT.getVectorNumElements();
1271 SDValue Src = Node->getOperand(0);
1272 EVT SrcVT = Src.getValueType();
1273 int NumSrcElements = SrcVT.getVectorNumElements();
1274
1275 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1276 // into a larger vector type.
1277 if (SrcVT.bitsLE(VT)) {
1278 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1279 "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1280 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1281 SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
1282 NumSrcElements);
1283 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT),
1284 Src, DAG.getVectorIdxConstant(0, DL));
1285 }
1286
1287 // Build up a zero vector to blend into this one.
1288 SDValue Zero = DAG.getConstant(0, DL, SrcVT);
1289
1290 // Shuffle the incoming lanes into the correct position, and pull all other
1291 // lanes from the zero vector.
1292 auto ShuffleMask = llvm::to_vector<16>(llvm::seq<int>(0, NumSrcElements));
1293
1294 int ExtLaneScale = NumSrcElements / NumElements;
1295 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1296 for (int i = 0; i < NumElements; ++i)
1297 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1298
1299 return DAG.getNode(ISD::BITCAST, DL, VT,
1300 DAG.getVectorShuffle(SrcVT, DL, Zero, Src, ShuffleMask));
1301}
1302
1303static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl<int> &ShuffleMask) {
1304 int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
1305 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I)
1306 for (int J = ScalarSizeInBytes - 1; J >= 0; --J)
1307 ShuffleMask.push_back((I * ScalarSizeInBytes) + J);
1308}
1309
1310SDValue VectorLegalizer::ExpandBSWAP(SDNode *Node) {
1311 EVT VT = Node->getValueType(0);
1312
1313 // Scalable vectors can't use shuffle expansion.
1314 if (VT.isScalableVector())
1315 return TLI.expandBSWAP(Node, DAG);
1316
1317 // Generate a byte wise shuffle mask for the BSWAP.
1318 SmallVector<int, 16> ShuffleMask;
1319 createBSWAPShuffleMask(VT, ShuffleMask);
1320 EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, ShuffleMask.size());
1321
1322 // Only emit a shuffle if the mask is legal.
1323 if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1324 SDLoc DL(Node);
1325 SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0));
1326 Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT), ShuffleMask);
1327 return DAG.getNode(ISD::BITCAST, DL, VT, Op);
1328 }
1329
1330 // If we have the appropriate vector bit operations, it is better to use them
1331 // than unrolling and expanding each component.
1332 if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
1333 TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
1334 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&
1335 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT))
1336 return TLI.expandBSWAP(Node, DAG);
1337
1338 // Otherwise unroll.
1339 return DAG.UnrollVectorOp(Node);
1340}
1341
1342void VectorLegalizer::ExpandBITREVERSE(SDNode *Node,
1344 EVT VT = Node->getValueType(0);
1345
1346 // We can't unroll or use shuffles for scalable vectors.
1347 if (VT.isScalableVector()) {
1348 Results.push_back(TLI.expandBITREVERSE(Node, DAG));
1349 return;
1350 }
1351
1352 // If we have the scalar operation, it's probably cheaper to unroll it.
1353 if (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, VT.getScalarType())) {
1354 SDValue Tmp = DAG.UnrollVectorOp(Node);
1355 Results.push_back(Tmp);
1356 return;
1357 }
1358
1359 // If the vector element width is a whole number of bytes, test if its legal
1360 // to BSWAP shuffle the bytes and then perform the BITREVERSE on the byte
1361 // vector. This greatly reduces the number of bit shifts necessary.
1362 unsigned ScalarSizeInBits = VT.getScalarSizeInBits();
1363 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1364 SmallVector<int, 16> BSWAPMask;
1365 createBSWAPShuffleMask(VT, BSWAPMask);
1366
1367 EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, BSWAPMask.size());
1368 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1369 (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, ByteVT) ||
1370 (TLI.isOperationLegalOrCustom(ISD::SHL, ByteVT) &&
1371 TLI.isOperationLegalOrCustom(ISD::SRL, ByteVT) &&
1372 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, ByteVT) &&
1373 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, ByteVT)))) {
1374 SDLoc DL(Node);
1375 SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0));
1376 Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT),
1377 BSWAPMask);
1378 Op = DAG.getNode(ISD::BITREVERSE, DL, ByteVT, Op);
1379 Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
1380 Results.push_back(Op);
1381 return;
1382 }
1383 }
1384
1385 // If we have the appropriate vector bit operations, it is better to use them
1386 // than unrolling and expanding each component.
1387 if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
1388 TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
1389 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&
1390 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT)) {
1391 Results.push_back(TLI.expandBITREVERSE(Node, DAG));
1392 return;
1393 }
1394
1395 // Otherwise unroll.
1396 SDValue Tmp = DAG.UnrollVectorOp(Node);
1397 Results.push_back(Tmp);
1398}
1399
1400SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) {
1401 // Implement VSELECT in terms of XOR, AND, OR
1402 // on platforms which do not support blend natively.
1403 SDLoc DL(Node);
1404
1405 SDValue Mask = Node->getOperand(0);
1406 SDValue Op1 = Node->getOperand(1);
1407 SDValue Op2 = Node->getOperand(2);
1408
1409 EVT VT = Mask.getValueType();
1410
1411 // If we can't even use the basic vector operations of
1412 // AND,OR,XOR, we will have to scalarize the op.
1413 // Notice that the operation may be 'promoted' which means that it is
1414 // 'bitcasted' to another type which is handled.
1415 if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
1416 TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
1417 TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand)
1418 return DAG.UnrollVectorOp(Node);
1419
1420 // This operation also isn't safe with AND, OR, XOR when the boolean type is
1421 // 0/1 and the select operands aren't also booleans, as we need an all-ones
1422 // vector constant to mask with.
1423 // FIXME: Sign extend 1 to all ones if that's legal on the target.
1424 auto BoolContents = TLI.getBooleanContents(Op1.getValueType());
1425 if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
1426 !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1427 Op1.getValueType().getVectorElementType() == MVT::i1))
1428 return DAG.UnrollVectorOp(Node);
1429
1430 // If the mask and the type are different sizes, unroll the vector op. This
1431 // can occur when getSetCCResultType returns something that is different in
1432 // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8.
1433 if (VT.getSizeInBits() != Op1.getValueSizeInBits())
1434 return DAG.UnrollVectorOp(Node);
1435
1436 // Bitcast the operands to be the same type as the mask.
1437 // This is needed when we select between FP types because
1438 // the mask is a vector of integers.
1439 Op1 = DAG.getNode(ISD::BITCAST, DL, VT, Op1);
1440 Op2 = DAG.getNode(ISD::BITCAST, DL, VT, Op2);
1441
1442 SDValue NotMask = DAG.getNOT(DL, Mask, VT);
1443
1444 Op1 = DAG.getNode(ISD::AND, DL, VT, Op1, Mask);
1445 Op2 = DAG.getNode(ISD::AND, DL, VT, Op2, NotMask);
1446 SDValue Val = DAG.getNode(ISD::OR, DL, VT, Op1, Op2);
1447 return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val);
1448}
1449
1450SDValue VectorLegalizer::ExpandVP_SELECT(SDNode *Node) {
1451 // Implement VP_SELECT in terms of VP_XOR, VP_AND and VP_OR on platforms which
1452 // do not support it natively.
1453 SDLoc DL(Node);
1454
1455 SDValue Mask = Node->getOperand(0);
1456 SDValue Op1 = Node->getOperand(1);
1457 SDValue Op2 = Node->getOperand(2);
1458 SDValue EVL = Node->getOperand(3);
1459
1460 EVT VT = Mask.getValueType();
1461
1462 // If we can't even use the basic vector operations of
1463 // VP_AND,VP_OR,VP_XOR, we will have to scalarize the op.
1464 if (TLI.getOperationAction(ISD::VP_AND, VT) == TargetLowering::Expand ||
1465 TLI.getOperationAction(ISD::VP_XOR, VT) == TargetLowering::Expand ||
1466 TLI.getOperationAction(ISD::VP_OR, VT) == TargetLowering::Expand)
1467 return DAG.UnrollVectorOp(Node);
1468
1469 // This operation also isn't safe when the operands aren't also booleans.
1470 if (Op1.getValueType().getVectorElementType() != MVT::i1)
1471 return DAG.UnrollVectorOp(Node);
1472
1473 SDValue Ones = DAG.getAllOnesConstant(DL, VT);
1474 SDValue NotMask = DAG.getNode(ISD::VP_XOR, DL, VT, Mask, Ones, Ones, EVL);
1475
1476 Op1 = DAG.getNode(ISD::VP_AND, DL, VT, Op1, Mask, Ones, EVL);
1477 Op2 = DAG.getNode(ISD::VP_AND, DL, VT, Op2, NotMask, Ones, EVL);
1478 return DAG.getNode(ISD::VP_OR, DL, VT, Op1, Op2, Ones, EVL);
1479}
1480
1481SDValue VectorLegalizer::ExpandVP_MERGE(SDNode *Node) {
1482 // Implement VP_MERGE in terms of VSELECT. Construct a mask where vector
1483 // indices less than the EVL/pivot are true. Combine that with the original
1484 // mask for a full-length mask. Use a full-length VSELECT to select between
1485 // the true and false values.
1486 SDLoc DL(Node);
1487
1488 SDValue Mask = Node->getOperand(0);
1489 SDValue Op1 = Node->getOperand(1);
1490 SDValue Op2 = Node->getOperand(2);
1491 SDValue EVL = Node->getOperand(3);
1492
1493 EVT MaskVT = Mask.getValueType();
1494 bool IsFixedLen = MaskVT.isFixedLengthVector();
1495
1496 EVT EVLVecVT = EVT::getVectorVT(*DAG.getContext(), EVL.getValueType(),
1497 MaskVT.getVectorElementCount());
1498
1499 // If we can't construct the EVL mask efficiently, it's better to unroll.
1500 if ((IsFixedLen &&
1501 !TLI.isOperationLegalOrCustom(ISD::BUILD_VECTOR, EVLVecVT)) ||
1502 (!IsFixedLen &&
1503 (!TLI.isOperationLegalOrCustom(ISD::STEP_VECTOR, EVLVecVT) ||
1504 !TLI.isOperationLegalOrCustom(ISD::SPLAT_VECTOR, EVLVecVT))))
1505 return DAG.UnrollVectorOp(Node);
1506
1507 // If using a SETCC would result in a different type than the mask type,
1508 // unroll.
1509 if (TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
1510 EVLVecVT) != MaskVT)
1511 return DAG.UnrollVectorOp(Node);
1512
1513 SDValue StepVec = DAG.getStepVector(DL, EVLVecVT);
1514 SDValue SplatEVL = DAG.getSplat(EVLVecVT, DL, EVL);
1515 SDValue EVLMask =
1516 DAG.getSetCC(DL, MaskVT, StepVec, SplatEVL, ISD::CondCode::SETULT);
1517
1518 SDValue FullMask = DAG.getNode(ISD::AND, DL, MaskVT, Mask, EVLMask);
1519 return DAG.getSelect(DL, Node->getValueType(0), FullMask, Op1, Op2);
1520}
1521
1522SDValue VectorLegalizer::ExpandVP_REM(SDNode *Node) {
1523 // Implement VP_SREM/UREM in terms of VP_SDIV/VP_UDIV, VP_MUL, VP_SUB.
1524 EVT VT = Node->getValueType(0);
1525
1526 unsigned DivOpc = Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV;
1527
1528 if (!TLI.isOperationLegalOrCustom(DivOpc, VT) ||
1529 !TLI.isOperationLegalOrCustom(ISD::VP_MUL, VT) ||
1530 !TLI.isOperationLegalOrCustom(ISD::VP_SUB, VT))
1531 return SDValue();
1532
1533 SDLoc DL(Node);
1534
1535 SDValue Dividend = Node->getOperand(0);
1536 SDValue Divisor = Node->getOperand(1);
1537 SDValue Mask = Node->getOperand(2);
1538 SDValue EVL = Node->getOperand(3);
1539
1540 // X % Y -> X-X/Y*Y
1541 SDValue Div = DAG.getNode(DivOpc, DL, VT, Dividend, Divisor, Mask, EVL);
1542 SDValue Mul = DAG.getNode(ISD::VP_MUL, DL, VT, Divisor, Div, Mask, EVL);
1543 return DAG.getNode(ISD::VP_SUB, DL, VT, Dividend, Mul, Mask, EVL);
1544}
1545
1546void VectorLegalizer::ExpandFP_TO_UINT(SDNode *Node,
1548 // Attempt to expand using TargetLowering.
1549 SDValue Result, Chain;
1550 if (TLI.expandFP_TO_UINT(Node, Result, Chain, DAG)) {
1551 Results.push_back(Result);
1552 if (Node->isStrictFPOpcode())
1553 Results.push_back(Chain);
1554 return;
1555 }
1556
1557 // Otherwise go ahead and unroll.
1558 if (Node->isStrictFPOpcode()) {
1559 UnrollStrictFPOp(Node, Results);
1560 return;
1561 }
1562
1563 Results.push_back(DAG.UnrollVectorOp(Node));
1564}
1565
1566void VectorLegalizer::ExpandUINT_TO_FLOAT(SDNode *Node,
1568 bool IsStrict = Node->isStrictFPOpcode();
1569 unsigned OpNo = IsStrict ? 1 : 0;
1570 SDValue Src = Node->getOperand(OpNo);
1571 EVT VT = Src.getValueType();
1572 SDLoc DL(Node);
1573
1574 // Attempt to expand using TargetLowering.
1576 SDValue Chain;
1577 if (TLI.expandUINT_TO_FP(Node, Result, Chain, DAG)) {
1578 Results.push_back(Result);
1579 if (IsStrict)
1580 Results.push_back(Chain);
1581 return;
1582 }
1583
1584 // Make sure that the SINT_TO_FP and SRL instructions are available.
1585 if (((!IsStrict && TLI.getOperationAction(ISD::SINT_TO_FP, VT) ==
1586 TargetLowering::Expand) ||
1587 (IsStrict && TLI.getOperationAction(ISD::STRICT_SINT_TO_FP, VT) ==
1588 TargetLowering::Expand)) ||
1589 TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Expand) {
1590 if (IsStrict) {
1591 UnrollStrictFPOp(Node, Results);
1592 return;
1593 }
1594
1595 Results.push_back(DAG.UnrollVectorOp(Node));
1596 return;
1597 }
1598
1599 unsigned BW = VT.getScalarSizeInBits();
1600 assert((BW == 64 || BW == 32) &&
1601 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1602
1603 SDValue HalfWord = DAG.getConstant(BW / 2, DL, VT);
1604
1605 // Constants to clear the upper part of the word.
1606 // Notice that we can also use SHL+SHR, but using a constant is slightly
1607 // faster on x86.
1608 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1609 SDValue HalfWordMask = DAG.getConstant(HWMask, DL, VT);
1610
1611 // Two to the power of half-word-size.
1612 SDValue TWOHW =
1613 DAG.getConstantFP(1ULL << (BW / 2), DL, Node->getValueType(0));
1614
1615 // Clear upper part of LO, lower HI
1616 SDValue HI = DAG.getNode(ISD::SRL, DL, VT, Src, HalfWord);
1617 SDValue LO = DAG.getNode(ISD::AND, DL, VT, Src, HalfWordMask);
1618
1619 if (IsStrict) {
1620 // Convert hi and lo to floats
1621 // Convert the hi part back to the upper values
1622 // TODO: Can any fast-math-flags be set on these nodes?
1624 {Node->getValueType(0), MVT::Other},
1625 {Node->getOperand(0), HI});
1626 fHI = DAG.getNode(ISD::STRICT_FMUL, DL, {Node->getValueType(0), MVT::Other},
1627 {fHI.getValue(1), fHI, TWOHW});
1629 {Node->getValueType(0), MVT::Other},
1630 {Node->getOperand(0), LO});
1631
1632 SDValue TF = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, fHI.getValue(1),
1633 fLO.getValue(1));
1634
1635 // Add the two halves
1636 SDValue Result =
1637 DAG.getNode(ISD::STRICT_FADD, DL, {Node->getValueType(0), MVT::Other},
1638 {TF, fHI, fLO});
1639
1640 Results.push_back(Result);
1641 Results.push_back(Result.getValue(1));
1642 return;
1643 }
1644
1645 // Convert hi and lo to floats
1646 // Convert the hi part back to the upper values
1647 // TODO: Can any fast-math-flags be set on these nodes?
1648 SDValue fHI = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), HI);
1649 fHI = DAG.getNode(ISD::FMUL, DL, Node->getValueType(0), fHI, TWOHW);
1650 SDValue fLO = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), LO);
1651
1652 // Add the two halves
1653 Results.push_back(
1654 DAG.getNode(ISD::FADD, DL, Node->getValueType(0), fHI, fLO));
1655}
1656
1657SDValue VectorLegalizer::ExpandFNEG(SDNode *Node) {
1658 if (TLI.isOperationLegalOrCustom(ISD::FSUB, Node->getValueType(0))) {
1659 SDLoc DL(Node);
1660 SDValue Zero = DAG.getConstantFP(-0.0, DL, Node->getValueType(0));
1661 // TODO: If FNEG had fast-math-flags, they'd get propagated to this FSUB.
1662 return DAG.getNode(ISD::FSUB, DL, Node->getValueType(0), Zero,
1663 Node->getOperand(0));
1664 }
1665 return DAG.UnrollVectorOp(Node);
1666}
1667
1668void VectorLegalizer::ExpandFSUB(SDNode *Node,
1670 // For floating-point values, (a-b) is the same as a+(-b). If FNEG is legal,
1671 // we can defer this to operation legalization where it will be lowered as
1672 // a+(-b).
1673 EVT VT = Node->getValueType(0);
1674 if (TLI.isOperationLegalOrCustom(ISD::FNEG, VT) &&
1675 TLI.isOperationLegalOrCustom(ISD::FADD, VT))
1676 return; // Defer to LegalizeDAG
1677
1678 SDValue Tmp = DAG.UnrollVectorOp(Node);
1679 Results.push_back(Tmp);
1680}
1681
1682void VectorLegalizer::ExpandSETCC(SDNode *Node,
1684 bool NeedInvert = false;
1685 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
1686 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
1687 Node->getOpcode() == ISD::STRICT_FSETCCS;
1688 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
1689 unsigned Offset = IsStrict ? 1 : 0;
1690
1691 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
1692 SDValue LHS = Node->getOperand(0 + Offset);
1693 SDValue RHS = Node->getOperand(1 + Offset);
1694 SDValue CC = Node->getOperand(2 + Offset);
1695
1696 MVT OpVT = LHS.getSimpleValueType();
1697 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1698
1699 if (TLI.getCondCodeAction(CCCode, OpVT) != TargetLowering::Expand) {
1700 if (IsStrict) {
1701 UnrollStrictFPOp(Node, Results);
1702 return;
1703 }
1704 Results.push_back(UnrollVSETCC(Node));
1705 return;
1706 }
1707
1708 SDValue Mask, EVL;
1709 if (IsVP) {
1710 Mask = Node->getOperand(3 + Offset);
1711 EVL = Node->getOperand(4 + Offset);
1712 }
1713
1714 SDLoc dl(Node);
1715 bool Legalized =
1716 TLI.LegalizeSetCCCondCode(DAG, Node->getValueType(0), LHS, RHS, CC, Mask,
1717 EVL, NeedInvert, dl, Chain, IsSignaling);
1718
1719 if (Legalized) {
1720 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
1721 // condition code, create a new SETCC node.
1722 if (CC.getNode()) {
1723 if (IsStrict) {
1724 LHS = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(),
1725 {Chain, LHS, RHS, CC}, Node->getFlags());
1726 Chain = LHS.getValue(1);
1727 } else if (IsVP) {
1728 LHS = DAG.getNode(ISD::VP_SETCC, dl, Node->getValueType(0),
1729 {LHS, RHS, CC, Mask, EVL}, Node->getFlags());
1730 } else {
1731 LHS = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), LHS, RHS, CC,
1732 Node->getFlags());
1733 }
1734 }
1735
1736 // If we expanded the SETCC by inverting the condition code, then wrap
1737 // the existing SETCC in a NOT to restore the intended condition.
1738 if (NeedInvert) {
1739 if (!IsVP)
1740 LHS = DAG.getLogicalNOT(dl, LHS, LHS->getValueType(0));
1741 else
1742 LHS = DAG.getVPLogicalNOT(dl, LHS, Mask, EVL, LHS->getValueType(0));
1743 }
1744 } else {
1745 assert(!IsStrict && "Don't know how to expand for strict nodes.");
1746
1747 // Otherwise, SETCC for the given comparison type must be completely
1748 // illegal; expand it into a SELECT_CC.
1749 EVT VT = Node->getValueType(0);
1750 LHS =
1751 DAG.getNode(ISD::SELECT_CC, dl, VT, LHS, RHS,
1752 DAG.getBoolConstant(true, dl, VT, LHS.getValueType()),
1753 DAG.getBoolConstant(false, dl, VT, LHS.getValueType()), CC);
1754 LHS->setFlags(Node->getFlags());
1755 }
1756
1757 Results.push_back(LHS);
1758 if (IsStrict)
1759 Results.push_back(Chain);
1760}
1761
1762void VectorLegalizer::ExpandUADDSUBO(SDNode *Node,
1764 SDValue Result, Overflow;
1765 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
1766 Results.push_back(Result);
1767 Results.push_back(Overflow);
1768}
1769
1770void VectorLegalizer::ExpandSADDSUBO(SDNode *Node,
1772 SDValue Result, Overflow;
1773 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
1774 Results.push_back(Result);
1775 Results.push_back(Overflow);
1776}
1777
1778void VectorLegalizer::ExpandMULO(SDNode *Node,
1780 SDValue Result, Overflow;
1781 if (!TLI.expandMULO(Node, Result, Overflow, DAG))
1782 std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(Node);
1783
1784 Results.push_back(Result);
1785 Results.push_back(Overflow);
1786}
1787
1788void VectorLegalizer::ExpandFixedPointDiv(SDNode *Node,
1790 SDNode *N = Node;
1791 if (SDValue Expanded = TLI.expandFixedPointDiv(N->getOpcode(), SDLoc(N),
1792 N->getOperand(0), N->getOperand(1), N->getConstantOperandVal(2), DAG))
1793 Results.push_back(Expanded);
1794}
1795
1796void VectorLegalizer::ExpandStrictFPOp(SDNode *Node,
1798 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP) {
1799 ExpandUINT_TO_FLOAT(Node, Results);
1800 return;
1801 }
1802 if (Node->getOpcode() == ISD::STRICT_FP_TO_UINT) {
1803 ExpandFP_TO_UINT(Node, Results);
1804 return;
1805 }
1806
1807 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1808 Node->getOpcode() == ISD::STRICT_FSETCCS) {
1809 ExpandSETCC(Node, Results);
1810 return;
1811 }
1812
1813 UnrollStrictFPOp(Node, Results);
1814}
1815
1816void VectorLegalizer::ExpandREM(SDNode *Node,
1818 assert((Node->getOpcode() == ISD::SREM || Node->getOpcode() == ISD::UREM) &&
1819 "Expected REM node");
1820
1822 if (!TLI.expandREM(Node, Result, DAG))
1823 Result = DAG.UnrollVectorOp(Node);
1824 Results.push_back(Result);
1825}
1826
1827// Try to expand libm nodes into vector math routine calls. Callers provide the
1828// LibFunc equivalent of the passed in Node, which is used to lookup mappings
1829// within TargetLibraryInfo. The only mappings considered are those where the
1830// result and all operands are the same vector type. While predicated nodes are
1831// not supported, we will emit calls to masked routines by passing in an all
1832// true mask.
1833bool VectorLegalizer::tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall LC,
1835 // Chain must be propagated but currently strict fp operations are down
1836 // converted to their none strict counterpart.
1837 assert(!Node->isStrictFPOpcode() && "Unexpected strict fp operation!");
1838
1839 const char *LCName = TLI.getLibcallName(LC);
1840 if (!LCName)
1841 return false;
1842 LLVM_DEBUG(dbgs() << "Looking for vector variant of " << LCName << "\n");
1843
1844 EVT VT = Node->getValueType(0);
1846
1847 // Lookup a vector function equivalent to the specified libcall. Prefer
1848 // unmasked variants but we will generate a mask if need be.
1849 const TargetLibraryInfo &TLibInfo = DAG.getLibInfo();
1850 const VecDesc *VD = TLibInfo.getVectorMappingInfo(LCName, VL, false);
1851 if (!VD)
1852 VD = TLibInfo.getVectorMappingInfo(LCName, VL, /*Masked=*/true);
1853 if (!VD)
1854 return false;
1855
1856 LLVMContext *Ctx = DAG.getContext();
1857 Type *Ty = VT.getTypeForEVT(*Ctx);
1858 Type *ScalarTy = Ty->getScalarType();
1859
1860 // Construct a scalar function type based on Node's operands.
1862 for (unsigned i = 0; i < Node->getNumOperands(); ++i) {
1863 assert(Node->getOperand(i).getValueType() == VT &&
1864 "Expected matching vector types!");
1865 ArgTys.push_back(ScalarTy);
1866 }
1867 FunctionType *ScalarFTy = FunctionType::get(ScalarTy, ArgTys, false);
1868
1869 // Generate call information for the vector function.
1870 const std::string MangledName = VD->getVectorFunctionABIVariantString();
1871 auto OptVFInfo = VFABI::tryDemangleForVFABI(MangledName, ScalarFTy);
1872 if (!OptVFInfo)
1873 return false;
1874
1875 LLVM_DEBUG(dbgs() << "Found vector variant " << VD->getVectorFnName()
1876 << "\n");
1877
1878 // Sanity check just in case OptVFInfo has unexpected parameters.
1879 if (OptVFInfo->Shape.Parameters.size() !=
1880 Node->getNumOperands() + VD->isMasked())
1881 return false;
1882
1883 // Collect vector call operands.
1884
1885 SDLoc DL(Node);
1888 Entry.IsSExt = false;
1889 Entry.IsZExt = false;
1890
1891 unsigned OpNum = 0;
1892 for (auto &VFParam : OptVFInfo->Shape.Parameters) {
1893 if (VFParam.ParamKind == VFParamKind::GlobalPredicate) {
1894 EVT MaskVT = TLI.getSetCCResultType(DAG.getDataLayout(), *Ctx, VT);
1895 Entry.Node = DAG.getBoolConstant(true, DL, MaskVT, VT);
1896 Entry.Ty = MaskVT.getTypeForEVT(*Ctx);
1897 Args.push_back(Entry);
1898 continue;
1899 }
1900
1901 // Only vector operands are supported.
1902 if (VFParam.ParamKind != VFParamKind::Vector)
1903 return false;
1904
1905 Entry.Node = Node->getOperand(OpNum++);
1906 Entry.Ty = Ty;
1907 Args.push_back(Entry);
1908 }
1909
1910 // Emit a call to the vector function.
1911 SDValue Callee = DAG.getExternalSymbol(VD->getVectorFnName().data(),
1912 TLI.getPointerTy(DAG.getDataLayout()));
1914 CLI.setDebugLoc(DL)
1915 .setChain(DAG.getEntryNode())
1916 .setLibCallee(CallingConv::C, Ty, Callee, std::move(Args));
1917
1918 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
1919 Results.push_back(CallResult.first);
1920 return true;
1921}
1922
1923/// Try to expand the node to a vector libcall based on the result type.
1924bool VectorLegalizer::tryExpandVecMathCall(
1925 SDNode *Node, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
1926 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
1929 Node->getValueType(0).getVectorElementType(), Call_F32, Call_F64,
1930 Call_F80, Call_F128, Call_PPCF128);
1931
1932 if (LC == RTLIB::UNKNOWN_LIBCALL)
1933 return false;
1934
1935 return tryExpandVecMathCall(Node, LC, Results);
1936}
1937
1938void VectorLegalizer::UnrollStrictFPOp(SDNode *Node,
1940 EVT VT = Node->getValueType(0);
1941 EVT EltVT = VT.getVectorElementType();
1942 unsigned NumElems = VT.getVectorNumElements();
1943 unsigned NumOpers = Node->getNumOperands();
1944 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1945
1946 EVT TmpEltVT = EltVT;
1947 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1948 Node->getOpcode() == ISD::STRICT_FSETCCS)
1949 TmpEltVT = TLI.getSetCCResultType(DAG.getDataLayout(),
1950 *DAG.getContext(), TmpEltVT);
1951
1952 EVT ValueVTs[] = {TmpEltVT, MVT::Other};
1953 SDValue Chain = Node->getOperand(0);
1954 SDLoc dl(Node);
1955
1956 SmallVector<SDValue, 32> OpValues;
1957 SmallVector<SDValue, 32> OpChains;
1958 for (unsigned i = 0; i < NumElems; ++i) {
1960 SDValue Idx = DAG.getVectorIdxConstant(i, dl);
1961
1962 // The Chain is the first operand.
1963 Opers.push_back(Chain);
1964
1965 // Now process the remaining operands.
1966 for (unsigned j = 1; j < NumOpers; ++j) {
1967 SDValue Oper = Node->getOperand(j);
1968 EVT OperVT = Oper.getValueType();
1969
1970 if (OperVT.isVector())
1971 Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
1972 OperVT.getVectorElementType(), Oper, Idx);
1973
1974 Opers.push_back(Oper);
1975 }
1976
1977 SDValue ScalarOp = DAG.getNode(Node->getOpcode(), dl, ValueVTs, Opers);
1978 SDValue ScalarResult = ScalarOp.getValue(0);
1979 SDValue ScalarChain = ScalarOp.getValue(1);
1980
1981 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1982 Node->getOpcode() == ISD::STRICT_FSETCCS)
1983 ScalarResult = DAG.getSelect(dl, EltVT, ScalarResult,
1984 DAG.getAllOnesConstant(dl, EltVT),
1985 DAG.getConstant(0, dl, EltVT));
1986
1987 OpValues.push_back(ScalarResult);
1988 OpChains.push_back(ScalarChain);
1989 }
1990
1991 SDValue Result = DAG.getBuildVector(VT, dl, OpValues);
1992 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
1993
1994 Results.push_back(Result);
1995 Results.push_back(NewChain);
1996}
1997
1998SDValue VectorLegalizer::UnrollVSETCC(SDNode *Node) {
1999 EVT VT = Node->getValueType(0);
2000 unsigned NumElems = VT.getVectorNumElements();
2001 EVT EltVT = VT.getVectorElementType();
2002 SDValue LHS = Node->getOperand(0);
2003 SDValue RHS = Node->getOperand(1);
2004 SDValue CC = Node->getOperand(2);
2005 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
2006 SDLoc dl(Node);
2007 SmallVector<SDValue, 8> Ops(NumElems);
2008 for (unsigned i = 0; i < NumElems; ++i) {
2009 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
2010 DAG.getVectorIdxConstant(i, dl));
2011 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
2012 DAG.getVectorIdxConstant(i, dl));
2013 Ops[i] = DAG.getNode(ISD::SETCC, dl,
2014 TLI.getSetCCResultType(DAG.getDataLayout(),
2015 *DAG.getContext(), TmpEltVT),
2016 LHSElem, RHSElem, CC);
2017 Ops[i] = DAG.getSelect(dl, EltVT, Ops[i], DAG.getAllOnesConstant(dl, EltVT),
2018 DAG.getConstant(0, dl, EltVT));
2019 }
2020 return DAG.getBuildVector(VT, dl, Ops);
2021}
2022
2024 return VectorLegalizer(*this).Run();
2025}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
BlockVerifier::State From
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the DenseMap class.
static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl< int > &ShuffleMask)
#define I(x, y, z)
Definition: MD5.cpp:58
mir Rename Register Operands
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
Value * RHS
Value * LHS
BinaryOperator * Mul
DEMANGLE_DUMP_METHOD void dump() const
This class represents an Operation in the Expression.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:220
size_t size() const
Definition: Function.h:811
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
This class is used to represent ISD::LOAD nodes.
Machine Value Type.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:307
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:227
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:492
ilist< SDNode >::iterator allnodes_iterator
Definition: SelectionDAG.h:548
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void resize(size_type N)
Definition: SmallVector.h:651
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
This class is used to represent ISD::STORE nodes.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:131
Provides information about what library functions are available for the current target.
const VecDesc * getVectorMappingInfo(StringRef F, const ElementCount &VF, bool Masked) const
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:348
Provides info so a possible vectorization of a function can be computed.
bool isMasked() const
std::string getVectorFunctionABIVariantString() const
Returns a vector function ABI variant string on the form: ZGV<isa><mask><vlen><vparams><scalarname>(<...
StringRef getVectorFnName() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:752
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:237
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:725
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:478
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1348
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1379
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:251
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:561
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:716
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:368
@ FMAXNUM_IEEE
Definition: ISDOpcodes.h:988
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:240
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1040
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:374
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:785
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:485
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:792
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1364
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:391
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1368
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:690
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:822
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:256
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1378
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:479
@ FPTRUNC_ROUND
Definition: ISDOpcodes.h:482
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:905
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:942
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:381
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:412
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:776
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
Definition: ISDOpcodes.h:664
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:452
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1361
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:724
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1365
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:932
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:328
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
Definition: ISDOpcodes.h:648
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:502
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:350
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:729
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1380
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:629
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:324
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1373
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:653
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:707
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
Definition: ISDOpcodes.h:987
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:537
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:782
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:858
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:744
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:974
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:360
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:332
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:811
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:800
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:676
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:387
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:890
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:738
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:451
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1381
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:445
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:467
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:444
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:993
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:838
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:472
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:682
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
Definition: ISDOpcodes.h:659
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1362
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:401
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:947
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:871
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:833
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:857
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1369
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:788
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1349
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:495
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:341
@ AssertZext
Definition: ISDOpcodes.h:62
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:517
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1542
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1522
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
std::optional< VFInfo > tryDemangleForVFABI(StringRef MangledName, const FunctionType *FTy)
Function to construct a VFInfo out of a mangled names in the following format:
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:456
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1729
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
#define N
Extended Value Type.
Definition: ValueTypes.h:34
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:93
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:340
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:370
bool isFixedLengthVector() const
Definition: ValueTypes.h:177
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:167
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:313
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:203
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition: ValueTypes.h:173
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:318
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:326
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:298
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.