LLVM  15.0.0git
VVPISelLowering.cpp
Go to the documentation of this file.
1 //===-- VVPISelLowering.cpp - VE DAG Lowering Implementation --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the lowering and legalization of vector instructions to
10 // VVP_*layer SDNodes.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "VECustomDAG.h"
15 #include "VEISelLowering.h"
16 
17 using namespace llvm;
18 
19 #define DEBUG_TYPE "ve-lower"
20 
22  SelectionDAG &DAG) const {
23  VECustomDAG CDAG(DAG, Op);
24  SDValue AVL =
25  CDAG.getConstant(Op.getValueType().getVectorNumElements(), MVT::i32);
26  SDValue A = Op->getOperand(0);
27  SDValue B = Op->getOperand(1);
28  SDValue LoA = CDAG.getUnpack(MVT::v256i1, A, PackElem::Lo, AVL);
29  SDValue HiA = CDAG.getUnpack(MVT::v256i1, A, PackElem::Hi, AVL);
30  SDValue LoB = CDAG.getUnpack(MVT::v256i1, B, PackElem::Lo, AVL);
31  SDValue HiB = CDAG.getUnpack(MVT::v256i1, B, PackElem::Hi, AVL);
32  unsigned Opc = Op.getOpcode();
33  auto LoRes = CDAG.getNode(Opc, MVT::v256i1, {LoA, LoB});
34  auto HiRes = CDAG.getNode(Opc, MVT::v256i1, {HiA, HiB});
35  return CDAG.getPack(MVT::v512i1, LoRes, HiRes, AVL);
36 }
37 
39  // Can we represent this as a VVP node.
40  const unsigned Opcode = Op->getOpcode();
41  auto VVPOpcodeOpt = getVVPOpcode(Opcode);
42  if (!VVPOpcodeOpt)
43  return SDValue();
44  unsigned VVPOpcode = VVPOpcodeOpt.getValue();
45  const bool FromVP = ISD::isVPOpcode(Opcode);
46 
47  // The representative and legalized vector type of this operation.
48  VECustomDAG CDAG(DAG, Op);
49  // Dispatch to complex lowering functions.
50  switch (VVPOpcode) {
51  case VEISD::VVP_LOAD:
52  case VEISD::VVP_STORE:
53  return lowerVVP_LOAD_STORE(Op, CDAG);
54  case VEISD::VVP_GATHER:
55  case VEISD::VVP_SCATTER:
56  return lowerVVP_GATHER_SCATTER(Op, CDAG);
57  }
58 
59  EVT OpVecVT = *getIdiomaticVectorType(Op.getNode());
60  EVT LegalVecVT = getTypeToTransformTo(*DAG.getContext(), OpVecVT);
61  auto Packing = getTypePacking(LegalVecVT.getSimpleVT());
62 
63  SDValue AVL;
64  SDValue Mask;
65 
66  if (FromVP) {
67  // All upstream VP SDNodes always have a mask and avl.
68  auto MaskIdx = ISD::getVPMaskIdx(Opcode);
69  auto AVLIdx = ISD::getVPExplicitVectorLengthIdx(Opcode);
70  if (MaskIdx)
71  Mask = Op->getOperand(*MaskIdx);
72  if (AVLIdx)
73  AVL = Op->getOperand(*AVLIdx);
74  }
75 
76  // Materialize default mask and avl.
77  if (!AVL)
78  AVL = CDAG.getConstant(OpVecVT.getVectorNumElements(), MVT::i32);
79  if (!Mask)
80  Mask = CDAG.getConstantMask(Packing, true);
81 
82  assert(LegalVecVT.isSimple());
83  if (isVVPUnaryOp(VVPOpcode))
84  return CDAG.getNode(VVPOpcode, LegalVecVT, {Op->getOperand(0), Mask, AVL});
85  if (isVVPBinaryOp(VVPOpcode))
86  return CDAG.getNode(VVPOpcode, LegalVecVT,
87  {Op->getOperand(0), Op->getOperand(1), Mask, AVL});
88  if (isVVPReductionOp(VVPOpcode)) {
89  auto SrcHasStart = hasReductionStartParam(Op->getOpcode());
90  SDValue StartV = SrcHasStart ? Op->getOperand(0) : SDValue();
91  SDValue VectorV = Op->getOperand(SrcHasStart ? 1 : 0);
92  return CDAG.getLegalReductionOpVVP(VVPOpcode, Op.getValueType(), StartV,
93  VectorV, Mask, AVL, Op->getFlags());
94  }
95 
96  switch (VVPOpcode) {
97  default:
98  llvm_unreachable("lowerToVVP called for unexpected SDNode.");
99  case VEISD::VVP_FFMA: {
100  // VE has a swizzled operand order in FMA (compared to LLVM IR and
101  // SDNodes).
102  auto X = Op->getOperand(2);
103  auto Y = Op->getOperand(0);
104  auto Z = Op->getOperand(1);
105  return CDAG.getNode(VVPOpcode, LegalVecVT, {X, Y, Z, Mask, AVL});
106  }
107  case VEISD::VVP_SELECT: {
108  auto Mask = Op->getOperand(0);
109  auto OnTrue = Op->getOperand(1);
110  auto OnFalse = Op->getOperand(2);
111  return CDAG.getNode(VVPOpcode, LegalVecVT, {OnTrue, OnFalse, Mask, AVL});
112  }
113  case VEISD::VVP_SETCC: {
114  EVT LegalResVT = getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
115  auto LHS = Op->getOperand(0);
116  auto RHS = Op->getOperand(1);
117  auto Pred = Op->getOperand(2);
118  return CDAG.getNode(VVPOpcode, LegalResVT, {LHS, RHS, Pred, Mask, AVL});
119  }
120  }
121 }
122 
124  VECustomDAG &CDAG) const {
125  auto VVPOpc = *getVVPOpcode(Op->getOpcode());
126  const bool IsLoad = (VVPOpc == VEISD::VVP_LOAD);
127 
128  // Shares.
129  SDValue BasePtr = getMemoryPtr(Op);
131  SDValue Chain = getNodeChain(Op);
132  SDValue AVL = getNodeAVL(Op);
133  // Store specific.
135  // Load specific.
136  SDValue PassThru = getNodePassthru(Op);
137 
138  SDValue StrideV = getLoadStoreStride(Op, CDAG);
139 
140  auto DataVT = *getIdiomaticVectorType(Op.getNode());
141  auto Packing = getTypePacking(DataVT);
142 
143  // TODO: Infer lower AVL from mask.
144  if (!AVL)
145  AVL = CDAG.getConstant(DataVT.getVectorNumElements(), MVT::i32);
146 
147  // Default to the all-true mask.
148  if (!Mask)
149  Mask = CDAG.getConstantMask(Packing, true);
150 
151  if (IsLoad) {
152  MVT LegalDataVT = getLegalVectorType(
153  Packing, DataVT.getVectorElementType().getSimpleVT());
154 
155  auto NewLoadV = CDAG.getNode(VEISD::VVP_LOAD, {LegalDataVT, MVT::Other},
156  {Chain, BasePtr, StrideV, Mask, AVL});
157 
158  if (!PassThru || PassThru->isUndef())
159  return NewLoadV;
160 
161  // Convert passthru to an explicit select node.
162  SDValue DataV = CDAG.getNode(VEISD::VVP_SELECT, DataVT,
163  {NewLoadV, PassThru, Mask, AVL});
164  SDValue NewLoadChainV = SDValue(NewLoadV.getNode(), 1);
165 
166  // Merge them back into one node.
167  return CDAG.getMergeValues({DataV, NewLoadChainV});
168  }
169 
170  // VVP_STORE
171  assert(VVPOpc == VEISD::VVP_STORE);
172  return CDAG.getNode(VEISD::VVP_STORE, Op.getNode()->getVTList(),
173  {Chain, Data, BasePtr, StrideV, Mask, AVL});
174 }
175 
177  VECustomDAG &CDAG) const {
178  auto VVPOC = *getVVPOpcode(Op.getOpcode());
179  assert((VVPOC == VEISD::VVP_LOAD) || (VVPOC == VEISD::VVP_STORE));
180 
181  MVT DataVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
182  assert(getTypePacking(DataVT) == Packing::Dense &&
183  "Can only split packed load/store");
184  MVT SplitDataVT = splitVectorType(DataVT);
185 
187  "Should have been folded in lowering to VVP layer");
188 
189  // Analyze the operation
190  SDValue PackedMask = getNodeMask(Op);
191  SDValue PackedAVL = getAnnotatedNodeAVL(Op).first;
192  SDValue PackPtr = getMemoryPtr(Op);
193  SDValue PackData = getStoredValue(Op);
194  SDValue PackStride = getLoadStoreStride(Op, CDAG);
195 
196  unsigned ChainResIdx = PackData ? 0 : 1;
197 
198  SDValue PartOps[2];
199 
200  SDValue UpperPartAVL; // we will use this for packing things back together
201  for (PackElem Part : {PackElem::Hi, PackElem::Lo}) {
202  // VP ops already have an explicit mask and AVL. When expanding from non-VP
203  // attach those additional inputs here.
204  auto SplitTM = CDAG.getTargetSplitMask(PackedMask, PackedAVL, Part);
205 
206  // Keep track of the (higher) lvl.
207  if (Part == PackElem::Hi)
208  UpperPartAVL = SplitTM.AVL;
209 
210  // Attach non-predicating value operands
212 
213  // Chain
214  OpVec.push_back(getNodeChain(Op));
215 
216  // Data
217  if (PackData) {
218  SDValue PartData =
219  CDAG.getUnpack(SplitDataVT, PackData, Part, SplitTM.AVL);
220  OpVec.push_back(PartData);
221  }
222 
223  // Ptr & Stride
224  // Push (ptr + ElemBytes * <Part>, 2 * ElemBytes)
225  // Stride info
226  // EVT DataVT = LegalizeVectorType(getMemoryDataVT(Op), Op, DAG, Mode);
227  OpVec.push_back(CDAG.getSplitPtrOffset(PackPtr, PackStride, Part));
228  OpVec.push_back(CDAG.getSplitPtrStride(PackStride));
229 
230  // Add predicating args and generate part node
231  OpVec.push_back(SplitTM.Mask);
232  OpVec.push_back(SplitTM.AVL);
233 
234  if (PackData) {
235  // Store
236  PartOps[(int)Part] = CDAG.getNode(VVPOC, MVT::Other, OpVec);
237  } else {
238  // Load
239  PartOps[(int)Part] =
240  CDAG.getNode(VVPOC, {SplitDataVT, MVT::Other}, OpVec);
241  }
242  }
243 
244  // Merge the chains
245  SDValue LowChain = SDValue(PartOps[(int)PackElem::Lo].getNode(), ChainResIdx);
246  SDValue HiChain = SDValue(PartOps[(int)PackElem::Hi].getNode(), ChainResIdx);
247  SDValue FusedChains =
248  CDAG.getNode(ISD::TokenFactor, MVT::Other, {LowChain, HiChain});
249 
250  // Chain only [store]
251  if (PackData)
252  return FusedChains;
253 
254  // Re-pack into full packed vector result
255  MVT PackedVT =
256  getLegalVectorType(Packing::Dense, DataVT.getVectorElementType());
257  SDValue PackedVals = CDAG.getPack(PackedVT, PartOps[(int)PackElem::Lo],
258  PartOps[(int)PackElem::Hi], UpperPartAVL);
259 
260  return CDAG.getMergeValues({PackedVals, FusedChains});
261 }
262 
263 SDValue VETargetLowering::lowerVVP_GATHER_SCATTER(SDValue Op,
264  VECustomDAG &CDAG) const {
265  EVT DataVT = *getIdiomaticVectorType(Op.getNode());
266  auto Packing = getTypePacking(DataVT);
267  MVT LegalDataVT =
269 
270  SDValue AVL = getAnnotatedNodeAVL(Op).first;
272  SDValue BasePtr = getMemoryPtr(Op);
274  SDValue Chain = getNodeChain(Op);
276  SDValue PassThru = getNodePassthru(Op);
277  SDValue StoredValue = getStoredValue(Op);
278  if (PassThru && PassThru->isUndef())
279  PassThru = SDValue();
280 
281  bool IsScatter = (bool)StoredValue;
282 
283  // TODO: Infer lower AVL from mask.
284  if (!AVL)
285  AVL = CDAG.getConstant(DataVT.getVectorNumElements(), MVT::i32);
286 
287  // Default to the all-true mask.
288  if (!Mask)
289  Mask = CDAG.getConstantMask(Packing, true);
290 
291  SDValue AddressVec =
292  CDAG.getGatherScatterAddress(BasePtr, Scale, Index, Mask, AVL);
293  if (IsScatter)
294  return CDAG.getNode(VEISD::VVP_SCATTER, MVT::Other,
295  {Chain, StoredValue, AddressVec, Mask, AVL});
296 
297  // Gather.
298  SDValue NewLoadV = CDAG.getNode(VEISD::VVP_GATHER, {LegalDataVT, MVT::Other},
299  {Chain, AddressVec, Mask, AVL});
300 
301  if (!PassThru)
302  return NewLoadV;
303 
304  // TODO: Use vvp_select
305  SDValue DataV = CDAG.getNode(VEISD::VVP_SELECT, LegalDataVT,
306  {NewLoadV, PassThru, Mask, AVL});
307  SDValue NewLoadChainV = SDValue(NewLoadV.getNode(), 1);
308  return CDAG.getMergeValues({DataV, NewLoadChainV});
309 }
310 
312  VECustomDAG &CDAG) const {
313  LLVM_DEBUG(dbgs() << "::legalizeInternalLoadStoreOp\n";);
314  MVT DataVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
315 
316  // TODO: Recognize packable load,store.
317  if (isPackedVectorType(DataVT))
318  return splitPackedLoadStore(Op, CDAG);
319 
320  return legalizePackedAVL(Op, CDAG);
321 }
322 
324  SelectionDAG &DAG) const {
325  LLVM_DEBUG(dbgs() << "::legalizeInternalVectorOp\n";);
326  VECustomDAG CDAG(DAG, Op);
327 
328  // Dispatch to specialized legalization functions.
329  switch (Op->getOpcode()) {
330  case VEISD::VVP_LOAD:
331  case VEISD::VVP_STORE:
332  return legalizeInternalLoadStoreOp(Op, CDAG);
333  }
334 
335  EVT IdiomVT = Op.getValueType();
336  if (isPackedVectorType(IdiomVT) &&
337  !supportsPackedMode(Op.getOpcode(), IdiomVT))
338  return splitVectorOp(Op, CDAG);
339 
340  // TODO: Implement odd/even splitting.
341  return legalizePackedAVL(Op, CDAG);
342 }
343 
345  MVT ResVT = splitVectorType(Op.getValue(0).getSimpleValueType());
346 
347  auto AVLPos = getAVLPos(Op->getOpcode());
348  auto MaskPos = getMaskPos(Op->getOpcode());
349 
350  SDValue PackedMask = getNodeMask(Op);
351  auto AVLPair = getAnnotatedNodeAVL(Op);
352  SDValue PackedAVL = AVLPair.first;
353  assert(!AVLPair.second && "Expecting non pack-legalized oepration");
354 
355  // request the parts
356  SDValue PartOps[2];
357 
358  SDValue UpperPartAVL; // we will use this for packing things back together
359  for (PackElem Part : {PackElem::Hi, PackElem::Lo}) {
360  // VP ops already have an explicit mask and AVL. When expanding from non-VP
361  // attach those additional inputs here.
362  auto SplitTM = CDAG.getTargetSplitMask(PackedMask, PackedAVL, Part);
363 
364  if (Part == PackElem::Hi)
365  UpperPartAVL = SplitTM.AVL;
366 
367  // Attach non-predicating value operands
369  for (unsigned i = 0; i < Op.getNumOperands(); ++i) {
370  if (AVLPos && ((int)i) == *AVLPos)
371  continue;
372  if (MaskPos && ((int)i) == *MaskPos)
373  continue;
374 
375  // Value operand
376  auto PackedOperand = Op.getOperand(i);
377  auto UnpackedOpVT = splitVectorType(PackedOperand.getSimpleValueType());
378  SDValue PartV =
379  CDAG.getUnpack(UnpackedOpVT, PackedOperand, Part, SplitTM.AVL);
380  OpVec.push_back(PartV);
381  }
382 
383  // Add predicating args and generate part node.
384  OpVec.push_back(SplitTM.Mask);
385  OpVec.push_back(SplitTM.AVL);
386  // Emit legal VVP nodes.
387  PartOps[(int)Part] =
388  CDAG.getNode(Op.getOpcode(), ResVT, OpVec, Op->getFlags());
389  }
390 
391  // Re-package vectors.
392  return CDAG.getPack(Op.getValueType(), PartOps[(int)PackElem::Lo],
393  PartOps[(int)PackElem::Hi], UpperPartAVL);
394 }
395 
397  VECustomDAG &CDAG) const {
398  LLVM_DEBUG(dbgs() << "::legalizePackedAVL\n";);
399  // Only required for VEC and VVP ops.
400  if (!isVVPOrVEC(Op->getOpcode()))
401  return Op;
402 
403  // Operation already has a legal AVL.
404  auto AVL = getNodeAVL(Op);
405  if (isLegalAVL(AVL))
406  return Op;
407 
408  // Half and round up EVL for 32bit element types.
409  SDValue LegalAVL = AVL;
410  MVT IdiomVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
411  if (isPackedVectorType(IdiomVT)) {
413  "TODO Shift predication from EVL into Mask");
414 
415  if (auto *ConstAVL = dyn_cast<ConstantSDNode>(AVL)) {
416  LegalAVL = CDAG.getConstant((ConstAVL->getZExtValue() + 1) / 2, MVT::i32);
417  } else {
418  auto ConstOne = CDAG.getConstant(1, MVT::i32);
419  auto PlusOne = CDAG.getNode(ISD::ADD, MVT::i32, {AVL, ConstOne});
420  LegalAVL = CDAG.getNode(ISD::SRL, MVT::i32, {PlusOne, ConstOne});
421  }
422  }
423 
424  SDValue AnnotatedLegalAVL = CDAG.annotateLegalAVL(LegalAVL);
425 
426  // Copy the operand list.
427  int NumOp = Op->getNumOperands();
428  auto AVLPos = getAVLPos(Op->getOpcode());
429  std::vector<SDValue> FixedOperands;
430  for (int i = 0; i < NumOp; ++i) {
431  if (AVLPos && (i == *AVLPos)) {
432  FixedOperands.push_back(AnnotatedLegalAVL);
433  continue;
434  }
435  FixedOperands.push_back(Op->getOperand(i));
436  }
437 
438  // Clone the operation with fixed operands.
439  auto Flags = Op->getFlags();
440  SDValue NewN =
441  CDAG.getNode(Op->getOpcode(), Op->getVTList(), FixedOperands, Flags);
442  return NewN;
443 }
i
i
Definition: README.txt:29
llvm::VETargetLowering::legalizeInternalVectorOp
SDValue legalizeInternalVectorOp(SDValue Op, SelectionDAG &DAG) const
Definition: VVPISelLowering.cpp:323
VECustomDAG.h
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:17
llvm::getNodeChain
SDValue getNodeChain(SDValue Op)
Node Properties {.
Definition: VECustomDAG.cpp:218
llvm::PackElem
PackElem
Definition: VECustomDAG.h:130
llvm::VETargetLowering::lowerVVP_LOAD_STORE
SDValue lowerVVP_LOAD_STORE(SDValue Op, VECustomDAG &) const
Definition: VVPISelLowering.cpp:123
llvm::VETargetLowering::splitVectorOp
SDValue splitVectorOp(SDValue Op, VECustomDAG &CDAG) const
Definition: VVPISelLowering.cpp:344
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:151
llvm::getNodeMask
SDValue getNodeMask(SDValue Op)
Definition: VECustomDAG.cpp:390
llvm::VETargetLowering::legalizePackedAVL
SDValue legalizePackedAVL(SDValue Op, VECustomDAG &CDAG) const
Definition: VVPISelLowering.cpp:396
llvm::SDNode::isUndef
bool isUndef() const
Return true if the type of the node type undefined.
Definition: SelectionDAGNodes.h:655
llvm::VECustomDAG::getMergeValues
SDValue getMergeValues(ArrayRef< SDValue > Values) const
} Packing
Definition: VECustomDAG.h:194
llvm::isPackedVectorType
bool isPackedVectorType(EVT SomeVT)
Definition: VECustomDAG.cpp:22
llvm::ISD::getVPExplicitVectorLengthIdx
Optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
Definition: SelectionDAG.cpp:465
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1185
llvm::splitVectorType
MVT splitVectorType(MVT VT)
Definition: VECustomDAG.cpp:28
llvm::Packing::Dense
@ Dense
llvm::getTypePacking
Packing getTypePacking(EVT VT)
Definition: VECustomDAG.cpp:39
RHS
Value * RHS
Definition: X86PartialReduction.cpp:76
VEISelLowering.h
llvm::VECustomDAG::getPack
SDValue getPack(EVT DestVT, SDValue LoVec, SDValue HiVec, SDValue AVL) const
Definition: VECustomDAG.cpp:490
llvm::getMaskPos
Optional< int > getMaskPos(unsigned Opc)
Definition: VECustomDAG.cpp:191
llvm::Data
@ Data
Definition: SIMachineScheduler.h:55
llvm::SelectionDAG::getContext
LLVMContext * getContext() const
Definition: SelectionDAG.h:462
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::EVT::isSimple
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:129
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
LHS
Value * LHS
Definition: X86PartialReduction.cpp:75
llvm::SelectionDAG
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:220
llvm::getStoredValue
SDValue getStoredValue(SDValue Op)
Definition: VECustomDAG.cpp:332
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:34
llvm::VETargetLowering::splitMaskArithmetic
SDValue splitMaskArithmetic(SDValue Op, SelectionDAG &DAG) const
Definition: VVPISelLowering.cpp:21
llvm::EVT::getVectorNumElements
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:308
int
Clang compiles this i1 i64 store i64 i64 store i64 i64 store i64 i64 store i64 align Which gets codegen d xmm0 movaps rbp movaps rbp movaps rbp movaps rbp rbp rbp rbp rbp It would be better to have movq s of instead of the movaps s LLVM produces ret int
Definition: README.txt:536
Y
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
llvm::dwarf::Index
Index
Definition: Dwarf.h:472
llvm::VECustomDAG::getGatherScatterAddress
SDValue getGatherScatterAddress(SDValue BasePtr, SDValue Scale, SDValue Index, SDValue Mask, SDValue AVL) const
Definition: VECustomDAG.cpp:536
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::getNodeAVL
SDValue getNodeAVL(SDValue Op)
} Node Properties
Definition: VECustomDAG.cpp:385
llvm::PackElem::Hi
@ Hi
llvm::TargetLoweringBase::getTypeToTransformTo
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
Definition: TargetLowering.h:981
X
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
llvm::getAnnotatedNodeAVL
std::pair< SDValue, bool > getAnnotatedNodeAVL(SDValue Op)
Definition: VECustomDAG.cpp:395
llvm::VECustomDAG::getConstantMask
SDValue getConstantMask(Packing Packing, bool AllTrue) const
Definition: VECustomDAG.cpp:409
llvm::ISD::isVPOpcode
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
Definition: SelectionDAG.cpp:417
llvm::supportsPackedMode
bool supportsPackedMode(unsigned Opcode, EVT IdiomVT)
Definition: VECustomDAG.cpp:101
llvm::PackElem::Lo
@ Lo
llvm::VECustomDAG::annotateLegalAVL
SDValue annotateLegalAVL(SDValue AVL) const
Definition: VECustomDAG.cpp:474
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::MVT::Other
@ Other
Definition: MachineValueType.h:42
llvm::VECustomDAG::getUnpack
SDValue getUnpack(EVT DestVT, SDValue Vec, PackElem Part, SDValue AVL) const
} Legalizing getNode
Definition: VECustomDAG.cpp:480
llvm::VECustomDAG::getLegalReductionOpVVP
SDValue getLegalReductionOpVVP(unsigned VVPOpcode, EVT ResVT, SDValue StartV, SDValue VectorV, SDValue Mask, SDValue AVL, SDNodeFlags Flags) const
} getNode
Definition: VECustomDAG.cpp:562
llvm::MVT
Machine Value Type.
Definition: MachineValueType.h:31
llvm::MVT::v256i1
@ v256i1
Definition: MachineValueType.h:74
llvm::VECustomDAG::getSplitPtrOffset
SDValue getSplitPtrOffset(SDValue Ptr, SDValue ByteStride, PackElem Part) const
Definition: VECustomDAG.cpp:521
llvm::ISD::getVPMaskIdx
Optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
Definition: SelectionDAG.cpp:453
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::getIdiomaticVectorType
Optional< EVT > getIdiomaticVectorType(SDNode *Op)
} AVL Functions
Definition: VECustomDAG.cpp:243
llvm::VECustomDAG
Definition: VECustomDAG.h:142
llvm::isLegalAVL
bool isLegalAVL(SDValue AVL)
Definition: VECustomDAG.cpp:214
llvm::isVVPOrVEC
bool isVVPOrVEC(unsigned Opcode)
Definition: VECustomDAG.cpp:126
llvm::maySafelyIgnoreMask
bool maySafelyIgnoreMask(SDValue Op)
Definition: VECustomDAG.cpp:85
llvm::getGatherScatterIndex
SDValue getGatherScatterIndex(SDValue Op)
Definition: VECustomDAG.cpp:316
llvm::MVT::v512i1
@ v512i1
Definition: MachineValueType.h:75
llvm::isVVPBinaryOp
bool isVVPBinaryOp(unsigned VVPOpcode)
Definition: VECustomDAG.cpp:146
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:345
llvm::isVVPUnaryOp
bool isVVPUnaryOp(unsigned VVPOpcode)
Definition: VECustomDAG.cpp:136
llvm::getNodePassthru
SDValue getNodePassthru(SDValue Op)
Definition: VECustomDAG.cpp:353
llvm::VECustomDAG::getConstant
SDValue getConstant(uint64_t Val, EVT VT, bool IsTarget=false, bool IsOpaque=false) const
Definition: VECustomDAG.cpp:404
llvm::VETargetLowering::legalizeInternalLoadStoreOp
SDValue legalizeInternalLoadStoreOp(SDValue Op, VECustomDAG &CDAG) const
Definition: VVPISelLowering.cpp:311
llvm::MVT::i32
@ i32
Definition: MachineValueType.h:48
llvm::Packing
Packing
} Node Properties
Definition: VECustomDAG.h:119
llvm::VETargetLowering::lowerVVP_GATHER_SCATTER
SDValue lowerVVP_GATHER_SCATTER(SDValue Op, VECustomDAG &) const
Definition: VVPISelLowering.cpp:263
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:137
llvm::getVVPOpcode
Optional< unsigned > getVVPOpcode(unsigned Opcode)
Definition: VECustomDAG.cpp:62
llvm::ISD::ADD
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
llvm::isVVPReductionOp
bool isVVPReductionOp(unsigned Opcode)
Definition: VECustomDAG.cpp:156
llvm::EVT::getVectorElementType
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:300
llvm::VECustomDAG::getSplitPtrStride
SDValue getSplitPtrStride(SDValue PackStride) const
Definition: VECustomDAG.cpp:530
llvm::VECustomDAG::getNode
SDValue getNode(unsigned OC, SDVTList VTL, ArrayRef< SDValue > OpV, Optional< SDNodeFlags > Flags=None) const
getNode {
Definition: VECustomDAG.h:156
llvm::VETargetLowering::splitPackedLoadStore
SDValue splitPackedLoadStore(SDValue Op, VECustomDAG &CDAG) const
Definition: VVPISelLowering.cpp:176
llvm::getGatherScatterScale
SDValue getGatherScatterScale(SDValue Op)
Definition: VECustomDAG.cpp:324
llvm::ISD::SRL
@ SRL
Definition: ISDOpcodes.h:693
llvm::getAVLPos
Optional< int > getAVLPos(unsigned Opc)
The VE backend uses a two-staged process to lower and legalize vector instructions:
Definition: VECustomDAG.cpp:166
llvm::getLegalVectorType
MVT getLegalVectorType(Packing P, MVT ElemVT)
Definition: VECustomDAG.cpp:34
llvm::getLoadStoreStride
SDValue getLoadStoreStride(SDValue Op, VECustomDAG &CDAG)
Definition: VECustomDAG.cpp:292
llvm::VETargetLowering::lowerToVVP
SDValue lowerToVVP(SDValue Op, SelectionDAG &DAG) const
} Custom Inserter
Definition: VVPISelLowering.cpp:38
llvm::hasReductionStartParam
bool hasReductionStartParam(unsigned OPC)
Definition: VECustomDAG.cpp:362
llvm::getMemoryPtr
SDValue getMemoryPtr(SDValue Op)
Definition: VECustomDAG.cpp:230
llvm::ISD::TokenFactor
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
llvm::EVT::getSimpleVT
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:288
llvm::VECustomDAG::getTargetSplitMask
VETargetMasks getTargetSplitMask(SDValue RawMask, SDValue RawAVL, PackElem Part) const
Definition: VECustomDAG.cpp:498