LLVM  11.0.0git
HexagonISelLoweringHVX.cpp
Go to the documentation of this file.
1 //===-- HexagonISelLoweringHVX.cpp --- Lowering HVX operations ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "HexagonISelLowering.h"
10 #include "HexagonRegisterInfo.h"
11 #include "HexagonSubtarget.h"
12 #include "llvm/IR/IntrinsicsHexagon.h"
14 
15 using namespace llvm;
16 
21 
22 
23 void
24 HexagonTargetLowering::initializeHVXLowering() {
25  if (Subtarget.useHVX64BOps()) {
26  addRegisterClass(MVT::v64i8, &Hexagon::HvxVRRegClass);
27  addRegisterClass(MVT::v32i16, &Hexagon::HvxVRRegClass);
28  addRegisterClass(MVT::v16i32, &Hexagon::HvxVRRegClass);
29  addRegisterClass(MVT::v128i8, &Hexagon::HvxWRRegClass);
30  addRegisterClass(MVT::v64i16, &Hexagon::HvxWRRegClass);
31  addRegisterClass(MVT::v32i32, &Hexagon::HvxWRRegClass);
32  // These "short" boolean vector types should be legal because
33  // they will appear as results of vector compares. If they were
34  // not legal, type legalization would try to make them legal
35  // and that would require using operations that do not use or
36  // produce such types. That, in turn, would imply using custom
37  // nodes, which would be unoptimizable by the DAG combiner.
38  // The idea is to rely on target-independent operations as much
39  // as possible.
40  addRegisterClass(MVT::v16i1, &Hexagon::HvxQRRegClass);
41  addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass);
42  addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass);
43  } else if (Subtarget.useHVX128BOps()) {
44  addRegisterClass(MVT::v128i8, &Hexagon::HvxVRRegClass);
45  addRegisterClass(MVT::v64i16, &Hexagon::HvxVRRegClass);
46  addRegisterClass(MVT::v32i32, &Hexagon::HvxVRRegClass);
47  addRegisterClass(MVT::v256i8, &Hexagon::HvxWRRegClass);
48  addRegisterClass(MVT::v128i16, &Hexagon::HvxWRRegClass);
49  addRegisterClass(MVT::v64i32, &Hexagon::HvxWRRegClass);
50  addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass);
51  addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass);
52  addRegisterClass(MVT::v128i1, &Hexagon::HvxQRRegClass);
53  }
54 
55  // Set up operation actions.
56 
57  bool Use64b = Subtarget.useHVX64BOps();
58  ArrayRef<MVT> LegalV = Use64b ? LegalV64 : LegalV128;
59  ArrayRef<MVT> LegalW = Use64b ? LegalW64 : LegalW128;
60  MVT ByteV = Use64b ? MVT::v64i8 : MVT::v128i8;
61  MVT ByteW = Use64b ? MVT::v128i8 : MVT::v256i8;
62 
63  auto setPromoteTo = [this] (unsigned Opc, MVT FromTy, MVT ToTy) {
64  setOperationAction(Opc, FromTy, Promote);
65  AddPromotedToType(Opc, FromTy, ToTy);
66  };
67 
68  // Handle bitcasts of vector predicates to scalars (e.g. v32i1 to i32).
69  // Note: v16i1 -> i16 is handled in type legalization instead of op
70  // legalization.
80 
81  for (MVT T : LegalV) {
84 
92  if (T != ByteV) {
96  }
97 
104  // Make concat-vectors custom to handle concats of more than 2 vectors.
113  if (T != ByteV) {
115  // HVX only has shifts of words and halfwords.
119 
120  // Promote all shuffles to operate on vectors of bytes.
121  setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteV);
122  }
123 
131  }
132 
133  for (MVT T : LegalW) {
134  // Custom-lower BUILD_VECTOR for vector pairs. The standard (target-
135  // independent) handling of it would convert it to a load, which is
136  // not always the optimal choice.
138  // Make concat-vectors custom to handle concats of more than 2 vectors.
140 
141  // Custom-lower these operations for pairs. Expand them into a concat
142  // of the corresponding operations on individual vectors.
150 
156 
167  if (T != ByteW) {
171 
172  // Promote all shuffles to operate on vectors of bytes.
173  setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteW);
174  }
175  }
176 
177  // Boolean vectors.
178 
179  for (MVT T : LegalW) {
180  // Boolean types for vector pairs will overlap with the boolean
181  // types for single vectors, e.g.
182  // v64i8 -> v64i1 (single)
183  // v64i16 -> v64i1 (pair)
184  // Set these actions first, and allow the single actions to overwrite
185  // any duplicates.
186  MVT BoolW = MVT::getVectorVT(MVT::i1, T.getVectorNumElements());
191  }
192 
193  for (MVT T : LegalV) {
194  MVT BoolV = MVT::getVectorVT(MVT::i1, T.getVectorNumElements());
204  }
205 
206  if (Use64b) {
209  } else {
212  }
213 
215 }
216 
217 SDValue
218 HexagonTargetLowering::getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops,
219  const SDLoc &dl, SelectionDAG &DAG) const {
220  SmallVector<SDValue,4> IntOps;
221  IntOps.push_back(DAG.getConstant(IntId, dl, MVT::i32));
222  for (const SDValue &Op : Ops)
223  IntOps.push_back(Op);
224  return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, ResTy, IntOps);
225 }
226 
227 MVT
228 HexagonTargetLowering::typeJoin(const TypePair &Tys) const {
229  assert(Tys.first.getVectorElementType() == Tys.second.getVectorElementType());
230 
231  MVT ElemTy = Tys.first.getVectorElementType();
232  return MVT::getVectorVT(ElemTy, Tys.first.getVectorNumElements() +
233  Tys.second.getVectorNumElements());
234 }
235 
236 HexagonTargetLowering::TypePair
237 HexagonTargetLowering::typeSplit(MVT VecTy) const {
238  assert(VecTy.isVector());
239  unsigned NumElem = VecTy.getVectorNumElements();
240  assert((NumElem % 2) == 0 && "Expecting even-sized vector type");
241  MVT HalfTy = MVT::getVectorVT(VecTy.getVectorElementType(), NumElem/2);
242  return { HalfTy, HalfTy };
243 }
244 
245 MVT
246 HexagonTargetLowering::typeExtElem(MVT VecTy, unsigned Factor) const {
247  MVT ElemTy = VecTy.getVectorElementType();
248  MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() * Factor);
249  return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements());
250 }
251 
252 MVT
253 HexagonTargetLowering::typeTruncElem(MVT VecTy, unsigned Factor) const {
254  MVT ElemTy = VecTy.getVectorElementType();
255  MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() / Factor);
256  return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements());
257 }
258 
259 SDValue
260 HexagonTargetLowering::opCastElem(SDValue Vec, MVT ElemTy,
261  SelectionDAG &DAG) const {
262  if (ty(Vec).getVectorElementType() == ElemTy)
263  return Vec;
264  MVT CastTy = tyVector(Vec.getValueType().getSimpleVT(), ElemTy);
265  return DAG.getBitcast(CastTy, Vec);
266 }
267 
268 SDValue
269 HexagonTargetLowering::opJoin(const VectorPair &Ops, const SDLoc &dl,
270  SelectionDAG &DAG) const {
271  return DAG.getNode(ISD::CONCAT_VECTORS, dl, typeJoin(ty(Ops)),
272  Ops.second, Ops.first);
273 }
274 
275 HexagonTargetLowering::VectorPair
276 HexagonTargetLowering::opSplit(SDValue Vec, const SDLoc &dl,
277  SelectionDAG &DAG) const {
278  TypePair Tys = typeSplit(ty(Vec));
279  if (Vec.getOpcode() == HexagonISD::QCAT)
280  return VectorPair(Vec.getOperand(0), Vec.getOperand(1));
281  return DAG.SplitVector(Vec, dl, Tys.first, Tys.second);
282 }
283 
284 bool
285 HexagonTargetLowering::isHvxSingleTy(MVT Ty) const {
286  return Subtarget.isHVXVectorType(Ty) &&
287  Ty.getSizeInBits() == 8 * Subtarget.getVectorLength();
288 }
289 
290 bool
291 HexagonTargetLowering::isHvxPairTy(MVT Ty) const {
292  return Subtarget.isHVXVectorType(Ty) &&
293  Ty.getSizeInBits() == 16 * Subtarget.getVectorLength();
294 }
295 
296 bool
297 HexagonTargetLowering::isHvxBoolTy(MVT Ty) const {
298  return Subtarget.isHVXVectorType(Ty, true) &&
300 }
301 
302 bool HexagonTargetLowering::allowsHvxMemoryAccess(
303  MVT VecTy, MachineMemOperand::Flags Flags, bool *Fast) const {
304  // Bool vectors are excluded by default, but make it explicit to
305  // emphasize that bool vectors cannot be loaded or stored.
306  // Also, disallow double vector stores (to prevent unnecessary
307  // store widening in DAG combiner).
308  if (VecTy.getSizeInBits() > 8*Subtarget.getVectorLength())
309  return false;
310  if (!Subtarget.isHVXVectorType(VecTy, /*IncludeBool=*/false))
311  return false;
312  if (Fast)
313  *Fast = true;
314  return true;
315 }
316 
317 bool HexagonTargetLowering::allowsHvxMisalignedMemoryAccesses(
318  MVT VecTy, MachineMemOperand::Flags Flags, bool *Fast) const {
319  if (!Subtarget.isHVXVectorType(VecTy))
320  return false;
321  // XXX Should this be false? vmemu are a bit slower than vmem.
322  if (Fast)
323  *Fast = true;
324  return true;
325 }
326 
327 SDValue
328 HexagonTargetLowering::convertToByteIndex(SDValue ElemIdx, MVT ElemTy,
329  SelectionDAG &DAG) const {
330  if (ElemIdx.getValueType().getSimpleVT() != MVT::i32)
331  ElemIdx = DAG.getBitcast(MVT::i32, ElemIdx);
332 
333  unsigned ElemWidth = ElemTy.getSizeInBits();
334  if (ElemWidth == 8)
335  return ElemIdx;
336 
337  unsigned L = Log2_32(ElemWidth/8);
338  const SDLoc &dl(ElemIdx);
339  return DAG.getNode(ISD::SHL, dl, MVT::i32,
340  {ElemIdx, DAG.getConstant(L, dl, MVT::i32)});
341 }
342 
343 SDValue
344 HexagonTargetLowering::getIndexInWord32(SDValue Idx, MVT ElemTy,
345  SelectionDAG &DAG) const {
346  unsigned ElemWidth = ElemTy.getSizeInBits();
347  assert(ElemWidth >= 8 && ElemWidth <= 32);
348  if (ElemWidth == 32)
349  return Idx;
350 
351  if (ty(Idx) != MVT::i32)
352  Idx = DAG.getBitcast(MVT::i32, Idx);
353  const SDLoc &dl(Idx);
354  SDValue Mask = DAG.getConstant(32/ElemWidth - 1, dl, MVT::i32);
355  SDValue SubIdx = DAG.getNode(ISD::AND, dl, MVT::i32, {Idx, Mask});
356  return SubIdx;
357 }
358 
359 SDValue
360 HexagonTargetLowering::getByteShuffle(const SDLoc &dl, SDValue Op0,
362  SelectionDAG &DAG) const {
363  MVT OpTy = ty(Op0);
364  assert(OpTy == ty(Op1));
365 
366  MVT ElemTy = OpTy.getVectorElementType();
367  if (ElemTy == MVT::i8)
368  return DAG.getVectorShuffle(OpTy, dl, Op0, Op1, Mask);
369  assert(ElemTy.getSizeInBits() >= 8);
370 
371  MVT ResTy = tyVector(OpTy, MVT::i8);
372  unsigned ElemSize = ElemTy.getSizeInBits() / 8;
373 
374  SmallVector<int,128> ByteMask;
375  for (int M : Mask) {
376  if (M < 0) {
377  for (unsigned I = 0; I != ElemSize; ++I)
378  ByteMask.push_back(-1);
379  } else {
380  int NewM = M*ElemSize;
381  for (unsigned I = 0; I != ElemSize; ++I)
382  ByteMask.push_back(NewM+I);
383  }
384  }
385  assert(ResTy.getVectorNumElements() == ByteMask.size());
386  return DAG.getVectorShuffle(ResTy, dl, opCastElem(Op0, MVT::i8, DAG),
387  opCastElem(Op1, MVT::i8, DAG), ByteMask);
388 }
389 
390 SDValue
391 HexagonTargetLowering::buildHvxVectorReg(ArrayRef<SDValue> Values,
392  const SDLoc &dl, MVT VecTy,
393  SelectionDAG &DAG) const {
394  unsigned VecLen = Values.size();
396  MVT ElemTy = VecTy.getVectorElementType();
397  unsigned ElemWidth = ElemTy.getSizeInBits();
398  unsigned HwLen = Subtarget.getVectorLength();
399 
400  unsigned ElemSize = ElemWidth / 8;
401  assert(ElemSize*VecLen == HwLen);
403 
404  if (VecTy.getVectorElementType() != MVT::i32) {
405  assert((ElemSize == 1 || ElemSize == 2) && "Invalid element size");
406  unsigned OpsPerWord = (ElemSize == 1) ? 4 : 2;
407  MVT PartVT = MVT::getVectorVT(VecTy.getVectorElementType(), OpsPerWord);
408  for (unsigned i = 0; i != VecLen; i += OpsPerWord) {
409  SDValue W = buildVector32(Values.slice(i, OpsPerWord), dl, PartVT, DAG);
410  Words.push_back(DAG.getBitcast(MVT::i32, W));
411  }
412  } else {
413  Words.assign(Values.begin(), Values.end());
414  }
415 
416  unsigned NumWords = Words.size();
417  bool IsSplat = true, IsUndef = true;
418  SDValue SplatV;
419  for (unsigned i = 0; i != NumWords && IsSplat; ++i) {
420  if (isUndef(Words[i]))
421  continue;
422  IsUndef = false;
423  if (!SplatV.getNode())
424  SplatV = Words[i];
425  else if (SplatV != Words[i])
426  IsSplat = false;
427  }
428  if (IsUndef)
429  return DAG.getUNDEF(VecTy);
430  if (IsSplat) {
431  assert(SplatV.getNode());
432  auto *IdxN = dyn_cast<ConstantSDNode>(SplatV.getNode());
433  if (IdxN && IdxN->isNullValue())
434  return getZero(dl, VecTy, DAG);
435  return DAG.getNode(HexagonISD::VSPLATW, dl, VecTy, SplatV);
436  }
437 
438  // Delay recognizing constant vectors until here, so that we can generate
439  // a vsplat.
440  SmallVector<ConstantInt*, 128> Consts(VecLen);
441  bool AllConst = getBuildVectorConstInts(Values, VecTy, DAG, Consts);
442  if (AllConst) {
443  ArrayRef<Constant*> Tmp((Constant**)Consts.begin(),
444  (Constant**)Consts.end());
445  Constant *CV = ConstantVector::get(Tmp);
446  Align Alignment(HwLen);
447  SDValue CP =
448  LowerConstantPool(DAG.getConstantPool(CV, VecTy, Alignment), DAG);
449  return DAG.getLoad(VecTy, dl, DAG.getEntryNode(), CP,
450  MachinePointerInfo::getConstantPool(MF), Alignment);
451  }
452 
453  // A special case is a situation where the vector is built entirely from
454  // elements extracted from another vector. This could be done via a shuffle
455  // more efficiently, but typically, the size of the source vector will not
456  // match the size of the vector being built (which precludes the use of a
457  // shuffle directly).
458  // This only handles a single source vector, and the vector being built
459  // should be of a sub-vector type of the source vector type.
460  auto IsBuildFromExtracts = [this,&Values] (SDValue &SrcVec,
461  SmallVectorImpl<int> &SrcIdx) {
462  SDValue Vec;
463  for (SDValue V : Values) {
464  if (isUndef(V)) {
465  SrcIdx.push_back(-1);
466  continue;
467  }
468  if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
469  return false;
470  // All extracts should come from the same vector.
471  SDValue T = V.getOperand(0);
472  if (Vec.getNode() != nullptr && T.getNode() != Vec.getNode())
473  return false;
474  Vec = T;
475  ConstantSDNode *C = dyn_cast<ConstantSDNode>(V.getOperand(1));
476  if (C == nullptr)
477  return false;
478  int I = C->getSExtValue();
479  assert(I >= 0 && "Negative element index");
480  SrcIdx.push_back(I);
481  }
482  SrcVec = Vec;
483  return true;
484  };
485 
486  SmallVector<int,128> ExtIdx;
487  SDValue ExtVec;
488  if (IsBuildFromExtracts(ExtVec, ExtIdx)) {
489  MVT ExtTy = ty(ExtVec);
490  unsigned ExtLen = ExtTy.getVectorNumElements();
491  if (ExtLen == VecLen || ExtLen == 2*VecLen) {
492  // Construct a new shuffle mask that will produce a vector with the same
493  // number of elements as the input vector, and such that the vector we
494  // want will be the initial subvector of it.
496  BitVector Used(ExtLen);
497 
498  for (int M : ExtIdx) {
499  Mask.push_back(M);
500  if (M >= 0)
501  Used.set(M);
502  }
503  // Fill the rest of the mask with the unused elements of ExtVec in hopes
504  // that it will result in a permutation of ExtVec's elements. It's still
505  // fine if it doesn't (e.g. if undefs are present, or elements are
506  // repeated), but permutations can always be done efficiently via vdelta
507  // and vrdelta.
508  for (unsigned I = 0; I != ExtLen; ++I) {
509  if (Mask.size() == ExtLen)
510  break;
511  if (!Used.test(I))
512  Mask.push_back(I);
513  }
514 
515  SDValue S = DAG.getVectorShuffle(ExtTy, dl, ExtVec,
516  DAG.getUNDEF(ExtTy), Mask);
517  if (ExtLen == VecLen)
518  return S;
519  return DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, VecTy, S);
520  }
521  }
522 
523  // Construct two halves in parallel, then or them together.
524  assert(4*Words.size() == Subtarget.getVectorLength());
525  SDValue HalfV0 = getInstr(Hexagon::V6_vd0, dl, VecTy, {}, DAG);
526  SDValue HalfV1 = getInstr(Hexagon::V6_vd0, dl, VecTy, {}, DAG);
527  SDValue S = DAG.getConstant(4, dl, MVT::i32);
528  for (unsigned i = 0; i != NumWords/2; ++i) {
529  SDValue N = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy,
530  {HalfV0, Words[i]});
531  SDValue M = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy,
532  {HalfV1, Words[i+NumWords/2]});
533  HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {N, S});
534  HalfV1 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {M, S});
535  }
536 
537  HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy,
538  {HalfV0, DAG.getConstant(HwLen/2, dl, MVT::i32)});
539  SDValue DstV = DAG.getNode(ISD::OR, dl, VecTy, {HalfV0, HalfV1});
540  return DstV;
541 }
542 
543 SDValue
544 HexagonTargetLowering::createHvxPrefixPred(SDValue PredV, const SDLoc &dl,
545  unsigned BitBytes, bool ZeroFill, SelectionDAG &DAG) const {
546  MVT PredTy = ty(PredV);
547  unsigned HwLen = Subtarget.getVectorLength();
548  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
549 
550  if (Subtarget.isHVXVectorType(PredTy, true)) {
551  // Move the vector predicate SubV to a vector register, and scale it
552  // down to match the representation (bytes per type element) that VecV
553  // uses. The scaling down will pick every 2nd or 4th (every Scale-th
554  // in general) element and put them at the front of the resulting
555  // vector. This subvector will then be inserted into the Q2V of VecV.
556  // To avoid having an operation that generates an illegal type (short
557  // vector), generate a full size vector.
558  //
559  SDValue T = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, PredV);
560  SmallVector<int,128> Mask(HwLen);
561  // Scale = BitBytes(PredV) / Given BitBytes.
562  unsigned Scale = HwLen / (PredTy.getVectorNumElements() * BitBytes);
563  unsigned BlockLen = PredTy.getVectorNumElements() * BitBytes;
564 
565  for (unsigned i = 0; i != HwLen; ++i) {
566  unsigned Num = i % Scale;
567  unsigned Off = i / Scale;
568  Mask[BlockLen*Num + Off] = i;
569  }
570  SDValue S = DAG.getVectorShuffle(ByteTy, dl, T, DAG.getUNDEF(ByteTy), Mask);
571  if (!ZeroFill)
572  return S;
573  // Fill the bytes beyond BlockLen with 0s.
574  MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen);
575  SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
576  {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG);
577  SDValue M = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, Q);
578  return DAG.getNode(ISD::AND, dl, ByteTy, S, M);
579  }
580 
581  // Make sure that this is a valid scalar predicate.
582  assert(PredTy == MVT::v2i1 || PredTy == MVT::v4i1 || PredTy == MVT::v8i1);
583 
584  unsigned Bytes = 8 / PredTy.getVectorNumElements();
585  SmallVector<SDValue,4> Words[2];
586  unsigned IdxW = 0;
587 
588  auto Lo32 = [&DAG, &dl] (SDValue P) {
589  return DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, P);
590  };
591  auto Hi32 = [&DAG, &dl] (SDValue P) {
592  return DAG.getTargetExtractSubreg(Hexagon::isub_hi, dl, MVT::i32, P);
593  };
594 
595  SDValue W0 = isUndef(PredV)
596  ? DAG.getUNDEF(MVT::i64)
597  : DAG.getNode(HexagonISD::P2D, dl, MVT::i64, PredV);
598  Words[IdxW].push_back(Hi32(W0));
599  Words[IdxW].push_back(Lo32(W0));
600 
601  while (Bytes < BitBytes) {
602  IdxW ^= 1;
603  Words[IdxW].clear();
604 
605  if (Bytes < 4) {
606  for (const SDValue &W : Words[IdxW ^ 1]) {
607  SDValue T = expandPredicate(W, dl, DAG);
608  Words[IdxW].push_back(Hi32(T));
609  Words[IdxW].push_back(Lo32(T));
610  }
611  } else {
612  for (const SDValue &W : Words[IdxW ^ 1]) {
613  Words[IdxW].push_back(W);
614  Words[IdxW].push_back(W);
615  }
616  }
617  Bytes *= 2;
618  }
619 
620  assert(Bytes == BitBytes);
621 
622  SDValue Vec = ZeroFill ? getZero(dl, ByteTy, DAG) : DAG.getUNDEF(ByteTy);
623  SDValue S4 = DAG.getConstant(HwLen-4, dl, MVT::i32);
624  for (const SDValue &W : Words[IdxW]) {
625  Vec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Vec, S4);
626  Vec = DAG.getNode(HexagonISD::VINSERTW0, dl, ByteTy, Vec, W);
627  }
628 
629  return Vec;
630 }
631 
632 SDValue
633 HexagonTargetLowering::buildHvxVectorPred(ArrayRef<SDValue> Values,
634  const SDLoc &dl, MVT VecTy,
635  SelectionDAG &DAG) const {
636  // Construct a vector V of bytes, such that a comparison V >u 0 would
637  // produce the required vector predicate.
638  unsigned VecLen = Values.size();
639  unsigned HwLen = Subtarget.getVectorLength();
640  assert(VecLen <= HwLen || VecLen == 8*HwLen);
642  bool AllT = true, AllF = true;
643 
644  auto IsTrue = [] (SDValue V) {
645  if (const auto *N = dyn_cast<ConstantSDNode>(V.getNode()))
646  return !N->isNullValue();
647  return false;
648  };
649  auto IsFalse = [] (SDValue V) {
650  if (const auto *N = dyn_cast<ConstantSDNode>(V.getNode()))
651  return N->isNullValue();
652  return false;
653  };
654 
655  if (VecLen <= HwLen) {
656  // In the hardware, each bit of a vector predicate corresponds to a byte
657  // of a vector register. Calculate how many bytes does a bit of VecTy
658  // correspond to.
659  assert(HwLen % VecLen == 0);
660  unsigned BitBytes = HwLen / VecLen;
661  for (SDValue V : Values) {
662  AllT &= IsTrue(V);
663  AllF &= IsFalse(V);
664 
665  SDValue Ext = !V.isUndef() ? DAG.getZExtOrTrunc(V, dl, MVT::i8)
666  : DAG.getUNDEF(MVT::i8);
667  for (unsigned B = 0; B != BitBytes; ++B)
668  Bytes.push_back(Ext);
669  }
670  } else {
671  // There are as many i1 values, as there are bits in a vector register.
672  // Divide the values into groups of 8 and check that each group consists
673  // of the same value (ignoring undefs).
674  for (unsigned I = 0; I != VecLen; I += 8) {
675  unsigned B = 0;
676  // Find the first non-undef value in this group.
677  for (; B != 8; ++B) {
678  if (!Values[I+B].isUndef())
679  break;
680  }
681  SDValue F = Values[I+B];
682  AllT &= IsTrue(F);
683  AllF &= IsFalse(F);
684 
685  SDValue Ext = (B < 8) ? DAG.getZExtOrTrunc(F, dl, MVT::i8)
686  : DAG.getUNDEF(MVT::i8);
687  Bytes.push_back(Ext);
688  // Verify that the rest of values in the group are the same as the
689  // first.
690  for (; B != 8; ++B)
691  assert(Values[I+B].isUndef() || Values[I+B] == F);
692  }
693  }
694 
695  if (AllT)
696  return DAG.getNode(HexagonISD::QTRUE, dl, VecTy);
697  if (AllF)
698  return DAG.getNode(HexagonISD::QFALSE, dl, VecTy);
699 
700  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
701  SDValue ByteVec = buildHvxVectorReg(Bytes, dl, ByteTy, DAG);
702  return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec);
703 }
704 
705 SDValue
706 HexagonTargetLowering::extractHvxElementReg(SDValue VecV, SDValue IdxV,
707  const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
708  MVT ElemTy = ty(VecV).getVectorElementType();
709 
710  unsigned ElemWidth = ElemTy.getSizeInBits();
711  assert(ElemWidth >= 8 && ElemWidth <= 32);
712  (void)ElemWidth;
713 
714  SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
715  SDValue ExWord = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32,
716  {VecV, ByteIdx});
717  if (ElemTy == MVT::i32)
718  return ExWord;
719 
720  // Have an extracted word, need to extract the smaller element out of it.
721  // 1. Extract the bits of (the original) IdxV that correspond to the index
722  // of the desired element in the 32-bit word.
723  SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
724  // 2. Extract the element from the word.
725  SDValue ExVec = DAG.getBitcast(tyVector(ty(ExWord), ElemTy), ExWord);
726  return extractVector(ExVec, SubIdx, dl, ElemTy, MVT::i32, DAG);
727 }
728 
729 SDValue
730 HexagonTargetLowering::extractHvxElementPred(SDValue VecV, SDValue IdxV,
731  const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
732  // Implement other return types if necessary.
733  assert(ResTy == MVT::i1);
734 
735  unsigned HwLen = Subtarget.getVectorLength();
736  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
737  SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
738 
739  unsigned Scale = HwLen / ty(VecV).getVectorNumElements();
740  SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32);
741  IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV);
742 
743  SDValue ExtB = extractHvxElementReg(ByteVec, IdxV, dl, MVT::i32, DAG);
744  SDValue Zero = DAG.getTargetConstant(0, dl, MVT::i32);
745  return getInstr(Hexagon::C2_cmpgtui, dl, MVT::i1, {ExtB, Zero}, DAG);
746 }
747 
748 SDValue
749 HexagonTargetLowering::insertHvxElementReg(SDValue VecV, SDValue IdxV,
750  SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
751  MVT ElemTy = ty(VecV).getVectorElementType();
752 
753  unsigned ElemWidth = ElemTy.getSizeInBits();
754  assert(ElemWidth >= 8 && ElemWidth <= 32);
755  (void)ElemWidth;
756 
757  auto InsertWord = [&DAG,&dl,this] (SDValue VecV, SDValue ValV,
758  SDValue ByteIdxV) {
759  MVT VecTy = ty(VecV);
760  unsigned HwLen = Subtarget.getVectorLength();
761  SDValue MaskV = DAG.getNode(ISD::AND, dl, MVT::i32,
762  {ByteIdxV, DAG.getConstant(-4, dl, MVT::i32)});
763  SDValue RotV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {VecV, MaskV});
764  SDValue InsV = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy, {RotV, ValV});
765  SDValue SubV = DAG.getNode(ISD::SUB, dl, MVT::i32,
766  {DAG.getConstant(HwLen, dl, MVT::i32), MaskV});
767  SDValue TorV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {InsV, SubV});
768  return TorV;
769  };
770 
771  SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
772  if (ElemTy == MVT::i32)
773  return InsertWord(VecV, ValV, ByteIdx);
774 
775  // If this is not inserting a 32-bit word, convert it into such a thing.
776  // 1. Extract the existing word from the target vector.
777  SDValue WordIdx = DAG.getNode(ISD::SRL, dl, MVT::i32,
778  {ByteIdx, DAG.getConstant(2, dl, MVT::i32)});
779  SDValue Ext = extractHvxElementReg(opCastElem(VecV, MVT::i32, DAG), WordIdx,
780  dl, MVT::i32, DAG);
781 
782  // 2. Treating the extracted word as a 32-bit vector, insert the given
783  // value into it.
784  SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
785  MVT SubVecTy = tyVector(ty(Ext), ElemTy);
786  SDValue Ins = insertVector(DAG.getBitcast(SubVecTy, Ext),
787  ValV, SubIdx, dl, ElemTy, DAG);
788 
789  // 3. Insert the 32-bit word back into the original vector.
790  return InsertWord(VecV, Ins, ByteIdx);
791 }
792 
793 SDValue
794 HexagonTargetLowering::insertHvxElementPred(SDValue VecV, SDValue IdxV,
795  SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
796  unsigned HwLen = Subtarget.getVectorLength();
797  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
798  SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
799 
800  unsigned Scale = HwLen / ty(VecV).getVectorNumElements();
801  SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32);
802  IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV);
803  ValV = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, ValV);
804 
805  SDValue InsV = insertHvxElementReg(ByteVec, IdxV, ValV, dl, DAG);
806  return DAG.getNode(HexagonISD::V2Q, dl, ty(VecV), InsV);
807 }
808 
809 SDValue
810 HexagonTargetLowering::extractHvxSubvectorReg(SDValue VecV, SDValue IdxV,
811  const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
812  MVT VecTy = ty(VecV);
813  unsigned HwLen = Subtarget.getVectorLength();
814  unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
815  MVT ElemTy = VecTy.getVectorElementType();
816  unsigned ElemWidth = ElemTy.getSizeInBits();
817 
818  // If the source vector is a vector pair, get the single vector containing
819  // the subvector of interest. The subvector will never overlap two single
820  // vectors.
821  if (isHvxPairTy(VecTy)) {
822  unsigned SubIdx;
823  if (Idx * ElemWidth >= 8*HwLen) {
824  SubIdx = Hexagon::vsub_hi;
825  Idx -= VecTy.getVectorNumElements() / 2;
826  } else {
827  SubIdx = Hexagon::vsub_lo;
828  }
829  VecTy = typeSplit(VecTy).first;
830  VecV = DAG.getTargetExtractSubreg(SubIdx, dl, VecTy, VecV);
831  if (VecTy == ResTy)
832  return VecV;
833  }
834 
835  // The only meaningful subvectors of a single HVX vector are those that
836  // fit in a scalar register.
837  assert(ResTy.getSizeInBits() == 32 || ResTy.getSizeInBits() == 64);
838 
839  MVT WordTy = tyVector(VecTy, MVT::i32);
840  SDValue WordVec = DAG.getBitcast(WordTy, VecV);
841  unsigned WordIdx = (Idx*ElemWidth) / 32;
842 
843  SDValue W0Idx = DAG.getConstant(WordIdx, dl, MVT::i32);
844  SDValue W0 = extractHvxElementReg(WordVec, W0Idx, dl, MVT::i32, DAG);
845  if (ResTy.getSizeInBits() == 32)
846  return DAG.getBitcast(ResTy, W0);
847 
848  SDValue W1Idx = DAG.getConstant(WordIdx+1, dl, MVT::i32);
849  SDValue W1 = extractHvxElementReg(WordVec, W1Idx, dl, MVT::i32, DAG);
850  SDValue WW = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, {W1, W0});
851  return DAG.getBitcast(ResTy, WW);
852 }
853 
854 SDValue
855 HexagonTargetLowering::extractHvxSubvectorPred(SDValue VecV, SDValue IdxV,
856  const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
857  MVT VecTy = ty(VecV);
858  unsigned HwLen = Subtarget.getVectorLength();
859  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
860  SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
861  // IdxV is required to be a constant.
862  unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
863 
864  unsigned ResLen = ResTy.getVectorNumElements();
865  unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
866  unsigned Offset = Idx * BitBytes;
867  SDValue Undef = DAG.getUNDEF(ByteTy);
869 
870  if (Subtarget.isHVXVectorType(ResTy, true)) {
871  // Converting between two vector predicates. Since the result is shorter
872  // than the source, it will correspond to a vector predicate with the
873  // relevant bits replicated. The replication count is the ratio of the
874  // source and target vector lengths.
875  unsigned Rep = VecTy.getVectorNumElements() / ResLen;
876  assert(isPowerOf2_32(Rep) && HwLen % Rep == 0);
877  for (unsigned i = 0; i != HwLen/Rep; ++i) {
878  for (unsigned j = 0; j != Rep; ++j)
879  Mask.push_back(i + Offset);
880  }
881  SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask);
882  return DAG.getNode(HexagonISD::V2Q, dl, ResTy, ShuffV);
883  }
884 
885  // Converting between a vector predicate and a scalar predicate. In the
886  // vector predicate, a group of BitBytes bits will correspond to a single
887  // i1 element of the source vector type. Those bits will all have the same
888  // value. The same will be true for ByteVec, where each byte corresponds
889  // to a bit in the vector predicate.
890  // The algorithm is to traverse the ByteVec, going over the i1 values from
891  // the source vector, and generate the corresponding representation in an
892  // 8-byte vector. To avoid repeated extracts from ByteVec, shuffle the
893  // elements so that the interesting 8 bytes will be in the low end of the
894  // vector.
895  unsigned Rep = 8 / ResLen;
896  // Make sure the output fill the entire vector register, so repeat the
897  // 8-byte groups as many times as necessary.
898  for (unsigned r = 0; r != HwLen/ResLen; ++r) {
899  // This will generate the indexes of the 8 interesting bytes.
900  for (unsigned i = 0; i != ResLen; ++i) {
901  for (unsigned j = 0; j != Rep; ++j)
902  Mask.push_back(Offset + i*BitBytes);
903  }
904  }
905 
906  SDValue Zero = getZero(dl, MVT::i32, DAG);
907  SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask);
908  // Combine the two low words from ShuffV into a v8i8, and byte-compare
909  // them against 0.
910  SDValue W0 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32, {ShuffV, Zero});
912  {ShuffV, DAG.getConstant(4, dl, MVT::i32)});
913  SDValue Vec64 = DAG.getNode(HexagonISD::COMBINE, dl, MVT::v8i8, {W1, W0});
914  return getInstr(Hexagon::A4_vcmpbgtui, dl, ResTy,
915  {Vec64, DAG.getTargetConstant(0, dl, MVT::i32)}, DAG);
916 }
917 
918 SDValue
919 HexagonTargetLowering::insertHvxSubvectorReg(SDValue VecV, SDValue SubV,
920  SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
921  MVT VecTy = ty(VecV);
922  MVT SubTy = ty(SubV);
923  unsigned HwLen = Subtarget.getVectorLength();
924  MVT ElemTy = VecTy.getVectorElementType();
925  unsigned ElemWidth = ElemTy.getSizeInBits();
926 
927  bool IsPair = isHvxPairTy(VecTy);
928  MVT SingleTy = MVT::getVectorVT(ElemTy, (8*HwLen)/ElemWidth);
929  // The two single vectors that VecV consists of, if it's a pair.
930  SDValue V0, V1;
931  SDValue SingleV = VecV;
932  SDValue PickHi;
933 
934  if (IsPair) {
935  V0 = DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, SingleTy, VecV);
936  V1 = DAG.getTargetExtractSubreg(Hexagon::vsub_hi, dl, SingleTy, VecV);
937 
938  SDValue HalfV = DAG.getConstant(SingleTy.getVectorNumElements(),
939  dl, MVT::i32);
940  PickHi = DAG.getSetCC(dl, MVT::i1, IdxV, HalfV, ISD::SETUGT);
941  if (isHvxSingleTy(SubTy)) {
942  if (const auto *CN = dyn_cast<const ConstantSDNode>(IdxV.getNode())) {
943  unsigned Idx = CN->getZExtValue();
944  assert(Idx == 0 || Idx == VecTy.getVectorNumElements()/2);
945  unsigned SubIdx = (Idx == 0) ? Hexagon::vsub_lo : Hexagon::vsub_hi;
946  return DAG.getTargetInsertSubreg(SubIdx, dl, VecTy, VecV, SubV);
947  }
948  // If IdxV is not a constant, generate the two variants: with the
949  // SubV as the high and as the low subregister, and select the right
950  // pair based on the IdxV.
951  SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SubV, V1});
952  SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SubV});
953  return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo);
954  }
955  // The subvector being inserted must be entirely contained in one of
956  // the vectors V0 or V1. Set SingleV to the correct one, and update
957  // IdxV to be the index relative to the beginning of that vector.
958  SDValue S = DAG.getNode(ISD::SUB, dl, MVT::i32, IdxV, HalfV);
959  IdxV = DAG.getNode(ISD::SELECT, dl, MVT::i32, PickHi, S, IdxV);
960  SingleV = DAG.getNode(ISD::SELECT, dl, SingleTy, PickHi, V1, V0);
961  }
962 
963  // The only meaningful subvectors of a single HVX vector are those that
964  // fit in a scalar register.
965  assert(SubTy.getSizeInBits() == 32 || SubTy.getSizeInBits() == 64);
966  // Convert IdxV to be index in bytes.
967  auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode());
968  if (!IdxN || !IdxN->isNullValue()) {
969  IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
970  DAG.getConstant(ElemWidth/8, dl, MVT::i32));
971  SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, IdxV);
972  }
973  // When inserting a single word, the rotation back to the original position
974  // would be by HwLen-Idx, but if two words are inserted, it will need to be
975  // by (HwLen-4)-Idx.
976  unsigned RolBase = HwLen;
977  if (VecTy.getSizeInBits() == 32) {
978  SDValue V = DAG.getBitcast(MVT::i32, SubV);
979  SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, V);
980  } else {
981  SDValue V = DAG.getBitcast(MVT::i64, SubV);
982  SDValue R0 = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, V);
983  SDValue R1 = DAG.getTargetExtractSubreg(Hexagon::isub_hi, dl, MVT::i32, V);
984  SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R0);
985  SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV,
986  DAG.getConstant(4, dl, MVT::i32));
987  SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R1);
988  RolBase = HwLen-4;
989  }
990  // If the vector wasn't ror'ed, don't ror it back.
991  if (RolBase != 4 || !IdxN || !IdxN->isNullValue()) {
992  SDValue RolV = DAG.getNode(ISD::SUB, dl, MVT::i32,
993  DAG.getConstant(RolBase, dl, MVT::i32), IdxV);
994  SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, RolV);
995  }
996 
997  if (IsPair) {
998  SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SingleV, V1});
999  SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SingleV});
1000  return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo);
1001  }
1002  return SingleV;
1003 }
1004 
1005 SDValue
1006 HexagonTargetLowering::insertHvxSubvectorPred(SDValue VecV, SDValue SubV,
1007  SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
1008  MVT VecTy = ty(VecV);
1009  MVT SubTy = ty(SubV);
1010  assert(Subtarget.isHVXVectorType(VecTy, true));
1011  // VecV is an HVX vector predicate. SubV may be either an HVX vector
1012  // predicate as well, or it can be a scalar predicate.
1013 
1014  unsigned VecLen = VecTy.getVectorNumElements();
1015  unsigned HwLen = Subtarget.getVectorLength();
1016  assert(HwLen % VecLen == 0 && "Unexpected vector type");
1017 
1018  unsigned Scale = VecLen / SubTy.getVectorNumElements();
1019  unsigned BitBytes = HwLen / VecLen;
1020  unsigned BlockLen = HwLen / Scale;
1021 
1022  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
1023  SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
1024  SDValue ByteSub = createHvxPrefixPred(SubV, dl, BitBytes, false, DAG);
1025  SDValue ByteIdx;
1026 
1027  auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode());
1028  if (!IdxN || !IdxN->isNullValue()) {
1029  ByteIdx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
1030  DAG.getConstant(BitBytes, dl, MVT::i32));
1031  ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteIdx);
1032  }
1033 
1034  // ByteVec is the target vector VecV rotated in such a way that the
1035  // subvector should be inserted at index 0. Generate a predicate mask
1036  // and use vmux to do the insertion.
1037  MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen);
1038  SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
1039  {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG);
1040  ByteVec = getInstr(Hexagon::V6_vmux, dl, ByteTy, {Q, ByteSub, ByteVec}, DAG);
1041  // Rotate ByteVec back, and convert to a vector predicate.
1042  if (!IdxN || !IdxN->isNullValue()) {
1043  SDValue HwLenV = DAG.getConstant(HwLen, dl, MVT::i32);
1044  SDValue ByteXdi = DAG.getNode(ISD::SUB, dl, MVT::i32, HwLenV, ByteIdx);
1045  ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteXdi);
1046  }
1047  return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec);
1048 }
1049 
1050 SDValue
1051 HexagonTargetLowering::extendHvxVectorPred(SDValue VecV, const SDLoc &dl,
1052  MVT ResTy, bool ZeroExt, SelectionDAG &DAG) const {
1053  // Sign- and any-extending of a vector predicate to a vector register is
1054  // equivalent to Q2V. For zero-extensions, generate a vmux between 0 and
1055  // a vector of 1s (where the 1s are of type matching the vector type).
1056  assert(Subtarget.isHVXVectorType(ResTy));
1057  if (!ZeroExt)
1058  return DAG.getNode(HexagonISD::Q2V, dl, ResTy, VecV);
1059 
1060  assert(ty(VecV).getVectorNumElements() == ResTy.getVectorNumElements());
1061  SDValue True = DAG.getNode(HexagonISD::VSPLAT, dl, ResTy,
1062  DAG.getConstant(1, dl, MVT::i32));
1063  SDValue False = getZero(dl, ResTy, DAG);
1064  return DAG.getSelect(dl, ResTy, VecV, True, False);
1065 }
1066 
1067 SDValue
1068 HexagonTargetLowering::compressHvxPred(SDValue VecQ, const SDLoc &dl,
1069  MVT ResTy, SelectionDAG &DAG) const {
1070  // Given a predicate register VecQ, transfer bits VecQ[0..HwLen-1]
1071  // (i.e. the entire predicate register) to bits [0..HwLen-1] of a
1072  // vector register. The remaining bits of the vector register are
1073  // unspecified.
1074 
1075  MachineFunction &MF = DAG.getMachineFunction();
1076  unsigned HwLen = Subtarget.getVectorLength();
1077  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
1078  MVT PredTy = ty(VecQ);
1079  unsigned PredLen = PredTy.getVectorNumElements();
1080  assert(HwLen % PredLen == 0);
1081  MVT VecTy = MVT::getVectorVT(MVT::getIntegerVT(8*HwLen/PredLen), PredLen);
1082 
1083  Type *Int8Ty = Type::getInt8Ty(*DAG.getContext());
1085  // Create an array of bytes (hex): 01,02,04,08,10,20,40,80, 01,02,04,08,...
1086  // These are bytes with the LSB rotated left with respect to their index.
1087  for (unsigned i = 0; i != HwLen/8; ++i) {
1088  for (unsigned j = 0; j != 8; ++j)
1089  Tmp.push_back(ConstantInt::get(Int8Ty, 1ull << j));
1090  }
1091  Constant *CV = ConstantVector::get(Tmp);
1092  Align Alignment(HwLen);
1093  SDValue CP =
1094  LowerConstantPool(DAG.getConstantPool(CV, ByteTy, Alignment), DAG);
1095  SDValue Bytes =
1096  DAG.getLoad(ByteTy, dl, DAG.getEntryNode(), CP,
1097  MachinePointerInfo::getConstantPool(MF), Alignment);
1098 
1099  // Select the bytes that correspond to true bits in the vector predicate.
1100  SDValue Sel = DAG.getSelect(dl, VecTy, VecQ, DAG.getBitcast(VecTy, Bytes),
1101  getZero(dl, VecTy, DAG));
1102  // Calculate the OR of all bytes in each group of 8. That will compress
1103  // all the individual bits into a single byte.
1104  // First, OR groups of 4, via vrmpy with 0x01010101.
1105  SDValue All1 =
1106  DAG.getSplatBuildVector(MVT::v4i8, dl, DAG.getConstant(1, dl, MVT::i32));
1107  SDValue Vrmpy = getInstr(Hexagon::V6_vrmpyub, dl, ByteTy, {Sel, All1}, DAG);
1108  // Then rotate the accumulated vector by 4 bytes, and do the final OR.
1109  SDValue Rot = getInstr(Hexagon::V6_valignbi, dl, ByteTy,
1110  {Vrmpy, Vrmpy, DAG.getTargetConstant(4, dl, MVT::i32)}, DAG);
1111  SDValue Vor = DAG.getNode(ISD::OR, dl, ByteTy, {Vrmpy, Rot});
1112 
1113  // Pick every 8th byte and coalesce them at the beginning of the output.
1114  // For symmetry, coalesce every 1+8th byte after that, then every 2+8th
1115  // byte and so on.
1117  for (unsigned i = 0; i != HwLen; ++i)
1118  Mask.push_back((8*i) % HwLen + i/(HwLen/8));
1119  SDValue Collect =
1120  DAG.getVectorShuffle(ByteTy, dl, Vor, DAG.getUNDEF(ByteTy), Mask);
1121  return DAG.getBitcast(ResTy, Collect);
1122 }
1123 
1124 SDValue
1125 HexagonTargetLowering::LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG)
1126  const {
1127  const SDLoc &dl(Op);
1128  MVT VecTy = ty(Op);
1129 
1130  unsigned Size = Op.getNumOperands();
1132  for (unsigned i = 0; i != Size; ++i)
1133  Ops.push_back(Op.getOperand(i));
1134 
1135  if (VecTy.getVectorElementType() == MVT::i1)
1136  return buildHvxVectorPred(Ops, dl, VecTy, DAG);
1137 
1138  if (VecTy.getSizeInBits() == 16*Subtarget.getVectorLength()) {
1139  ArrayRef<SDValue> A(Ops);
1140  MVT SingleTy = typeSplit(VecTy).first;
1141  SDValue V0 = buildHvxVectorReg(A.take_front(Size/2), dl, SingleTy, DAG);
1142  SDValue V1 = buildHvxVectorReg(A.drop_front(Size/2), dl, SingleTy, DAG);
1143  return DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, V0, V1);
1144  }
1145 
1146  return buildHvxVectorReg(Ops, dl, VecTy, DAG);
1147 }
1148 
1149 SDValue
1150 HexagonTargetLowering::LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG)
1151  const {
1152  // Vector concatenation of two integer (non-bool) vectors does not need
1153  // special lowering. Custom-lower concats of bool vectors and expand
1154  // concats of more than 2 vectors.
1155  MVT VecTy = ty(Op);
1156  const SDLoc &dl(Op);
1157  unsigned NumOp = Op.getNumOperands();
1158  if (VecTy.getVectorElementType() != MVT::i1) {
1159  if (NumOp == 2)
1160  return Op;
1161  // Expand the other cases into a build-vector.
1162  SmallVector<SDValue,8> Elems;
1163  for (SDValue V : Op.getNode()->ops())
1164  DAG.ExtractVectorElements(V, Elems);
1165  // A vector of i16 will be broken up into a build_vector of i16's.
1166  // This is a problem, since at the time of operation legalization,
1167  // all operations are expected to be type-legalized, and i16 is not
1168  // a legal type. If any of the extracted elements is not of a valid
1169  // type, sign-extend it to a valid one.
1170  for (unsigned i = 0, e = Elems.size(); i != e; ++i) {
1171  SDValue V = Elems[i];
1172  MVT Ty = ty(V);
1173  if (!isTypeLegal(Ty)) {
1174  EVT NTy = getTypeToTransformTo(*DAG.getContext(), Ty);
1175  if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
1176  Elems[i] = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NTy,
1177  DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NTy,
1178  V.getOperand(0), V.getOperand(1)),
1179  DAG.getValueType(Ty));
1180  continue;
1181  }
1182  // A few less complicated cases.
1183  if (V.getOpcode() == ISD::Constant)
1184  Elems[i] = DAG.getSExtOrTrunc(V, dl, NTy);
1185  else if (V.isUndef())
1186  Elems[i] = DAG.getUNDEF(NTy);
1187  else
1188  llvm_unreachable("Unexpected vector element");
1189  }
1190  }
1191  return DAG.getBuildVector(VecTy, dl, Elems);
1192  }
1193 
1194  assert(VecTy.getVectorElementType() == MVT::i1);
1195  unsigned HwLen = Subtarget.getVectorLength();
1196  assert(isPowerOf2_32(NumOp) && HwLen % NumOp == 0);
1197 
1198  SDValue Op0 = Op.getOperand(0);
1199 
1200  // If the operands are HVX types (i.e. not scalar predicates), then
1201  // defer the concatenation, and create QCAT instead.
1202  if (Subtarget.isHVXVectorType(ty(Op0), true)) {
1203  if (NumOp == 2)
1204  return DAG.getNode(HexagonISD::QCAT, dl, VecTy, Op0, Op.getOperand(1));
1205 
1206  ArrayRef<SDUse> U(Op.getNode()->ops());
1207  SmallVector<SDValue,4> SV(U.begin(), U.end());
1208  ArrayRef<SDValue> Ops(SV);
1209 
1210  MVT HalfTy = typeSplit(VecTy).first;
1211  SDValue V0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy,
1212  Ops.take_front(NumOp/2));
1213  SDValue V1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy,
1214  Ops.take_back(NumOp/2));
1215  return DAG.getNode(HexagonISD::QCAT, dl, VecTy, V0, V1);
1216  }
1217 
1218  // Count how many bytes (in a vector register) each bit in VecTy
1219  // corresponds to.
1220  unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
1221 
1222  SmallVector<SDValue,8> Prefixes;
1223  for (SDValue V : Op.getNode()->op_values()) {
1224  SDValue P = createHvxPrefixPred(V, dl, BitBytes, true, DAG);
1225  Prefixes.push_back(P);
1226  }
1227 
1228  unsigned InpLen = ty(Op.getOperand(0)).getVectorNumElements();
1229  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
1230  SDValue S = DAG.getConstant(InpLen*BitBytes, dl, MVT::i32);
1231  SDValue Res = getZero(dl, ByteTy, DAG);
1232  for (unsigned i = 0, e = Prefixes.size(); i != e; ++i) {
1233  Res = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Res, S);
1234  Res = DAG.getNode(ISD::OR, dl, ByteTy, Res, Prefixes[e-i-1]);
1235  }
1236  return DAG.getNode(HexagonISD::V2Q, dl, VecTy, Res);
1237 }
1238 
1239 SDValue
1240 HexagonTargetLowering::LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG)
1241  const {
1242  // Change the type of the extracted element to i32.
1243  SDValue VecV = Op.getOperand(0);
1244  MVT ElemTy = ty(VecV).getVectorElementType();
1245  const SDLoc &dl(Op);
1246  SDValue IdxV = Op.getOperand(1);
1247  if (ElemTy == MVT::i1)
1248  return extractHvxElementPred(VecV, IdxV, dl, ty(Op), DAG);
1249 
1250  return extractHvxElementReg(VecV, IdxV, dl, ty(Op), DAG);
1251 }
1252 
1253 SDValue
1254 HexagonTargetLowering::LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG)
1255  const {
1256  const SDLoc &dl(Op);
1257  SDValue VecV = Op.getOperand(0);
1258  SDValue ValV = Op.getOperand(1);
1259  SDValue IdxV = Op.getOperand(2);
1260  MVT ElemTy = ty(VecV).getVectorElementType();
1261  if (ElemTy == MVT::i1)
1262  return insertHvxElementPred(VecV, IdxV, ValV, dl, DAG);
1263 
1264  return insertHvxElementReg(VecV, IdxV, ValV, dl, DAG);
1265 }
1266 
1267 SDValue
1268 HexagonTargetLowering::LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG)
1269  const {
1270  SDValue SrcV = Op.getOperand(0);
1271  MVT SrcTy = ty(SrcV);
1272  MVT DstTy = ty(Op);
1273  SDValue IdxV = Op.getOperand(1);
1274  unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
1275  assert(Idx % DstTy.getVectorNumElements() == 0);
1276  (void)Idx;
1277  const SDLoc &dl(Op);
1278 
1279  MVT ElemTy = SrcTy.getVectorElementType();
1280  if (ElemTy == MVT::i1)
1281  return extractHvxSubvectorPred(SrcV, IdxV, dl, DstTy, DAG);
1282 
1283  return extractHvxSubvectorReg(SrcV, IdxV, dl, DstTy, DAG);
1284 }
1285 
1286 SDValue
1287 HexagonTargetLowering::LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG)
1288  const {
1289  // Idx does not need to be a constant.
1290  SDValue VecV = Op.getOperand(0);
1291  SDValue ValV = Op.getOperand(1);
1292  SDValue IdxV = Op.getOperand(2);
1293 
1294  const SDLoc &dl(Op);
1295  MVT VecTy = ty(VecV);
1296  MVT ElemTy = VecTy.getVectorElementType();
1297  if (ElemTy == MVT::i1)
1298  return insertHvxSubvectorPred(VecV, ValV, IdxV, dl, DAG);
1299 
1300  return insertHvxSubvectorReg(VecV, ValV, IdxV, dl, DAG);
1301 }
1302 
1303 SDValue
1304 HexagonTargetLowering::LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const {
1305  // Lower any-extends of boolean vectors to sign-extends, since they
1306  // translate directly to Q2V. Zero-extending could also be done equally
1307  // fast, but Q2V is used/recognized in more places.
1308  // For all other vectors, use zero-extend.
1309  MVT ResTy = ty(Op);
1310  SDValue InpV = Op.getOperand(0);
1311  MVT ElemTy = ty(InpV).getVectorElementType();
1312  if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1313  return LowerHvxSignExt(Op, DAG);
1314  return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Op), ResTy, InpV);
1315 }
1316 
1317 SDValue
1318 HexagonTargetLowering::LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const {
1319  MVT ResTy = ty(Op);
1320  SDValue InpV = Op.getOperand(0);
1321  MVT ElemTy = ty(InpV).getVectorElementType();
1322  if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1323  return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), false, DAG);
1324  return Op;
1325 }
1326 
1327 SDValue
1328 HexagonTargetLowering::LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const {
1329  MVT ResTy = ty(Op);
1330  SDValue InpV = Op.getOperand(0);
1331  MVT ElemTy = ty(InpV).getVectorElementType();
1332  if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1333  return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), true, DAG);
1334  return Op;
1335 }
1336 
1337 SDValue
1338 HexagonTargetLowering::LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const {
1339  // Lower vector CTTZ into a computation using CTLZ (Hacker's Delight):
1340  // cttz(x) = bitwidth(x) - ctlz(~x & (x-1))
1341  const SDLoc &dl(Op);
1342  MVT ResTy = ty(Op);
1343  SDValue InpV = Op.getOperand(0);
1344  assert(ResTy == ty(InpV));
1345 
1346  // Calculate the vectors of 1 and bitwidth(x).
1347  MVT ElemTy = ty(InpV).getVectorElementType();
1348  unsigned ElemWidth = ElemTy.getSizeInBits();
1349  // Using uint64_t because a shift by 32 can happen.
1350  uint64_t Splat1 = 0, SplatW = 0;
1351  assert(isPowerOf2_32(ElemWidth) && ElemWidth <= 32);
1352  for (unsigned i = 0; i != 32/ElemWidth; ++i) {
1353  Splat1 = (Splat1 << ElemWidth) | 1;
1354  SplatW = (SplatW << ElemWidth) | ElemWidth;
1355  }
1356  SDValue Vec1 = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1357  DAG.getConstant(uint32_t(Splat1), dl, MVT::i32));
1358  SDValue VecW = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1359  DAG.getConstant(uint32_t(SplatW), dl, MVT::i32));
1360  SDValue VecN1 = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1361  DAG.getConstant(-1, dl, MVT::i32));
1362  // Do not use DAG.getNOT, because that would create BUILD_VECTOR with
1363  // a BITCAST. Here we can skip the BITCAST (so we don't have to handle
1364  // it separately in custom combine or selection).
1365  SDValue A = DAG.getNode(ISD::AND, dl, ResTy,
1366  {DAG.getNode(ISD::XOR, dl, ResTy, {InpV, VecN1}),
1367  DAG.getNode(ISD::SUB, dl, ResTy, {InpV, Vec1})});
1368  return DAG.getNode(ISD::SUB, dl, ResTy,
1369  {VecW, DAG.getNode(ISD::CTLZ, dl, ResTy, A)});
1370 }
1371 
1372 SDValue
1373 HexagonTargetLowering::LowerHvxMul(SDValue Op, SelectionDAG &DAG) const {
1374  MVT ResTy = ty(Op);
1375  assert(ResTy.isVector() && isHvxSingleTy(ResTy));
1376  const SDLoc &dl(Op);
1377  SmallVector<int,256> ShuffMask;
1378 
1379  MVT ElemTy = ResTy.getVectorElementType();
1380  unsigned VecLen = ResTy.getVectorNumElements();
1381  SDValue Vs = Op.getOperand(0);
1382  SDValue Vt = Op.getOperand(1);
1383 
1384  switch (ElemTy.SimpleTy) {
1385  case MVT::i8: {
1386  // For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...),
1387  // V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo,
1388  // where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...).
1389  MVT ExtTy = typeExtElem(ResTy, 2);
1390  unsigned MpyOpc = ElemTy == MVT::i8 ? Hexagon::V6_vmpybv
1391  : Hexagon::V6_vmpyhv;
1392  SDValue M = getInstr(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG);
1393 
1394  // Discard high halves of the resulting values, collect the low halves.
1395  for (unsigned I = 0; I < VecLen; I += 2) {
1396  ShuffMask.push_back(I); // Pick even element.
1397  ShuffMask.push_back(I+VecLen); // Pick odd element.
1398  }
1399  VectorPair P = opSplit(opCastElem(M, ElemTy, DAG), dl, DAG);
1400  SDValue BS = getByteShuffle(dl, P.first, P.second, ShuffMask, DAG);
1401  return DAG.getBitcast(ResTy, BS);
1402  }
1403  case MVT::i16:
1404  // For i16 there is V6_vmpyih, which acts exactly like the MUL opcode.
1405  // (There is also V6_vmpyhv, which behaves in an analogous way to
1406  // V6_vmpybv.)
1407  return getInstr(Hexagon::V6_vmpyih, dl, ResTy, {Vs, Vt}, DAG);
1408  case MVT::i32: {
1409  // Use the following sequence for signed word multiply:
1410  // T0 = V6_vmpyiowh Vs, Vt
1411  // T1 = V6_vaslw T0, 16
1412  // T2 = V6_vmpyiewuh_acc T1, Vs, Vt
1413  SDValue S16 = DAG.getConstant(16, dl, MVT::i32);
1414  SDValue T0 = getInstr(Hexagon::V6_vmpyiowh, dl, ResTy, {Vs, Vt}, DAG);
1415  SDValue T1 = getInstr(Hexagon::V6_vaslw, dl, ResTy, {T0, S16}, DAG);
1416  SDValue T2 = getInstr(Hexagon::V6_vmpyiewuh_acc, dl, ResTy,
1417  {T1, Vs, Vt}, DAG);
1418  return T2;
1419  }
1420  default:
1421  break;
1422  }
1423  return SDValue();
1424 }
1425 
1426 SDValue
1427 HexagonTargetLowering::LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const {
1428  MVT ResTy = ty(Op);
1429  assert(ResTy.isVector());
1430  const SDLoc &dl(Op);
1431  SmallVector<int,256> ShuffMask;
1432 
1433  MVT ElemTy = ResTy.getVectorElementType();
1434  unsigned VecLen = ResTy.getVectorNumElements();
1435  SDValue Vs = Op.getOperand(0);
1436  SDValue Vt = Op.getOperand(1);
1437  bool IsSigned = Op.getOpcode() == ISD::MULHS;
1438 
1439  if (ElemTy == MVT::i8 || ElemTy == MVT::i16) {
1440  // For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...),
1441  // V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo,
1442  // where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...).
1443  // For i16, use V6_vmpyhv, which behaves in an analogous way to
1444  // V6_vmpybv: results Lo and Hi are products of even/odd elements
1445  // respectively.
1446  MVT ExtTy = typeExtElem(ResTy, 2);
1447  unsigned MpyOpc = ElemTy == MVT::i8
1448  ? (IsSigned ? Hexagon::V6_vmpybv : Hexagon::V6_vmpyubv)
1449  : (IsSigned ? Hexagon::V6_vmpyhv : Hexagon::V6_vmpyuhv);
1450  SDValue M = getInstr(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG);
1451 
1452  // Discard low halves of the resulting values, collect the high halves.
1453  for (unsigned I = 0; I < VecLen; I += 2) {
1454  ShuffMask.push_back(I+1); // Pick even element.
1455  ShuffMask.push_back(I+VecLen+1); // Pick odd element.
1456  }
1457  VectorPair P = opSplit(opCastElem(M, ElemTy, DAG), dl, DAG);
1458  SDValue BS = getByteShuffle(dl, P.first, P.second, ShuffMask, DAG);
1459  return DAG.getBitcast(ResTy, BS);
1460  }
1461 
1462  assert(ElemTy == MVT::i32);
1463  SDValue S16 = DAG.getConstant(16, dl, MVT::i32);
1464 
1465  if (IsSigned) {
1466  // mulhs(Vs,Vt) =
1467  // = [(Hi(Vs)*2^16 + Lo(Vs)) *s (Hi(Vt)*2^16 + Lo(Vt))] >> 32
1468  // = [Hi(Vs)*2^16 *s Hi(Vt)*2^16 + Hi(Vs) *su Lo(Vt)*2^16
1469  // + Lo(Vs) *us (Hi(Vt)*2^16 + Lo(Vt))] >> 32
1470  // = [Hi(Vs) *s Hi(Vt)*2^32 + Hi(Vs) *su Lo(Vt)*2^16
1471  // + Lo(Vs) *us Vt] >> 32
1472  // The low half of Lo(Vs)*Lo(Vt) will be discarded (it's not added to
1473  // anything, so it cannot produce any carry over to higher bits),
1474  // so everything in [] can be shifted by 16 without loss of precision.
1475  // = [Hi(Vs) *s Hi(Vt)*2^16 + Hi(Vs)*su Lo(Vt) + Lo(Vs)*Vt >> 16] >> 16
1476  // = [Hi(Vs) *s Hi(Vt)*2^16 + Hi(Vs)*su Lo(Vt) + V6_vmpyewuh(Vs,Vt)] >> 16
1477  // Denote Hi(Vs) = Vs':
1478  // = [Vs'*s Hi(Vt)*2^16 + Vs' *su Lo(Vt) + V6_vmpyewuh(Vt,Vs)] >> 16
1479  // = Vs'*s Hi(Vt) + (V6_vmpyiewuh(Vs',Vt) + V6_vmpyewuh(Vt,Vs)) >> 16
1480  SDValue T0 = getInstr(Hexagon::V6_vmpyewuh, dl, ResTy, {Vt, Vs}, DAG);
1481  // Get Vs':
1482  SDValue S0 = getInstr(Hexagon::V6_vasrw, dl, ResTy, {Vs, S16}, DAG);
1483  SDValue T1 = getInstr(Hexagon::V6_vmpyiewuh_acc, dl, ResTy,
1484  {T0, S0, Vt}, DAG);
1485  // Shift by 16:
1486  SDValue S2 = getInstr(Hexagon::V6_vasrw, dl, ResTy, {T1, S16}, DAG);
1487  // Get Vs'*Hi(Vt):
1488  SDValue T2 = getInstr(Hexagon::V6_vmpyiowh, dl, ResTy, {S0, Vt}, DAG);
1489  // Add:
1490  SDValue T3 = DAG.getNode(ISD::ADD, dl, ResTy, {S2, T2});
1491  return T3;
1492  }
1493 
1494  // Unsigned mulhw. (Would expansion using signed mulhw be better?)
1495 
1496  auto LoVec = [&DAG,ResTy,dl] (SDValue Pair) {
1497  return DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, ResTy, Pair);
1498  };
1499  auto HiVec = [&DAG,ResTy,dl] (SDValue Pair) {
1500  return DAG.getTargetExtractSubreg(Hexagon::vsub_hi, dl, ResTy, Pair);
1501  };
1502 
1503  MVT PairTy = typeJoin({ResTy, ResTy});
1504  SDValue P = getInstr(Hexagon::V6_lvsplatw, dl, ResTy,
1505  {DAG.getConstant(0x02020202, dl, MVT::i32)}, DAG);
1506  // Multiply-unsigned halfwords:
1507  // LoVec = Vs.uh[2i] * Vt.uh[2i],
1508  // HiVec = Vs.uh[2i+1] * Vt.uh[2i+1]
1509  SDValue T0 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {Vs, Vt}, DAG);
1510  // The low halves in the LoVec of the pair can be discarded. They are
1511  // not added to anything (in the full-precision product), so they cannot
1512  // produce a carry into the higher bits.
1513  SDValue T1 = getInstr(Hexagon::V6_vlsrw, dl, ResTy, {LoVec(T0), S16}, DAG);
1514  // Swap low and high halves in Vt, and do the halfword multiplication
1515  // to get products Vs.uh[2i] * Vt.uh[2i+1] and Vs.uh[2i+1] * Vt.uh[2i].
1516  SDValue D0 = getInstr(Hexagon::V6_vdelta, dl, ResTy, {Vt, P}, DAG);
1517  SDValue T2 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {Vs, D0}, DAG);
1518  // T2 has mixed products of halfwords: Lo(Vt)*Hi(Vs) and Hi(Vt)*Lo(Vs).
1519  // These products are words, but cannot be added directly because the
1520  // sums could overflow. Add these products, by halfwords, where each sum
1521  // of a pair of halfwords gives a word.
1522  SDValue T3 = getInstr(Hexagon::V6_vadduhw, dl, PairTy,
1523  {LoVec(T2), HiVec(T2)}, DAG);
1524  // Add the high halfwords from the products of the low halfwords.
1525  SDValue T4 = DAG.getNode(ISD::ADD, dl, ResTy, {T1, LoVec(T3)});
1526  SDValue T5 = getInstr(Hexagon::V6_vlsrw, dl, ResTy, {T4, S16}, DAG);
1527  SDValue T6 = DAG.getNode(ISD::ADD, dl, ResTy, {HiVec(T0), HiVec(T3)});
1528  SDValue T7 = DAG.getNode(ISD::ADD, dl, ResTy, {T5, T6});
1529  return T7;
1530 }
1531 
1532 SDValue
1533 HexagonTargetLowering::LowerHvxBitcast(SDValue Op, SelectionDAG &DAG) const {
1534  SDValue ValQ = Op.getOperand(0);
1535  MVT ResTy = ty(Op);
1536  MVT VecTy = ty(ValQ);
1537  const SDLoc &dl(Op);
1538 
1539  if (isHvxBoolTy(VecTy) && ResTy.isScalarInteger()) {
1540  unsigned HwLen = Subtarget.getVectorLength();
1541  MVT WordTy = MVT::getVectorVT(MVT::i32, HwLen/4);
1542  SDValue VQ = compressHvxPred(ValQ, dl, WordTy, DAG);
1543  unsigned BitWidth = ResTy.getSizeInBits();
1544 
1545  if (BitWidth < 64) {
1546  SDValue W0 = extractHvxElementReg(VQ, DAG.getConstant(0, dl, MVT::i32),
1547  dl, MVT::i32, DAG);
1548  if (BitWidth == 32)
1549  return W0;
1550  assert(BitWidth < 32u);
1551  return DAG.getZExtOrTrunc(W0, dl, ResTy);
1552  }
1553 
1554  // The result is >= 64 bits. The only options are 64 or 128.
1555  assert(BitWidth == 64 || BitWidth == 128);
1556  SmallVector<SDValue,4> Words;
1557  for (unsigned i = 0; i != BitWidth/32; ++i) {
1558  SDValue W = extractHvxElementReg(
1559  VQ, DAG.getConstant(i, dl, MVT::i32), dl, MVT::i32, DAG);
1560  Words.push_back(W);
1561  }
1562  SmallVector<SDValue,2> Combines;
1563  assert(Words.size() % 2 == 0);
1564  for (unsigned i = 0, e = Words.size(); i < e; i += 2) {
1565  SDValue C = DAG.getNode(
1566  HexagonISD::COMBINE, dl, MVT::i64, {Words[i+1], Words[i]});
1567  Combines.push_back(C);
1568  }
1569 
1570  if (BitWidth == 64)
1571  return Combines[0];
1572 
1573  return DAG.getNode(ISD::BUILD_PAIR, dl, ResTy, Combines);
1574  }
1575 
1576  return Op;
1577 }
1578 
1579 SDValue
1580 HexagonTargetLowering::LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const {
1581  // Sign- and zero-extends are legal.
1583  return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(Op), ty(Op),
1584  Op.getOperand(0));
1585 }
1586 
1587 SDValue
1588 HexagonTargetLowering::LowerHvxShift(SDValue Op, SelectionDAG &DAG) const {
1589  if (SDValue S = getVectorShiftByInt(Op, DAG))
1590  return S;
1591  return Op;
1592 }
1593 
1594 SDValue
1595 HexagonTargetLowering::LowerHvxIntrinsic(SDValue Op, SelectionDAG &DAG) const {
1596  const SDLoc &dl(Op);
1597  MVT ResTy = ty(Op);
1598 
1599  unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1600  bool Use64b = Subtarget.useHVX64BOps();
1601  unsigned IntPredCast = Use64b ? Intrinsic::hexagon_V6_pred_typecast
1602  : Intrinsic::hexagon_V6_pred_typecast_128B;
1603  if (IntNo == IntPredCast) {
1604  SDValue Vs = Op.getOperand(1);
1605  MVT OpTy = ty(Vs);
1606  if (isHvxBoolTy(ResTy) && isHvxBoolTy(OpTy)) {
1607  if (ResTy == OpTy)
1608  return Vs;
1609  return DAG.getNode(HexagonISD::TYPECAST, dl, ResTy, Vs);
1610  }
1611  }
1612 
1613  return Op;
1614 }
1615 
1616 SDValue
1617 HexagonTargetLowering::SplitHvxPairOp(SDValue Op, SelectionDAG &DAG) const {
1618  assert(!Op.isMachineOpcode());
1619  SmallVector<SDValue,2> OpsL, OpsH;
1620  const SDLoc &dl(Op);
1621 
1622  auto SplitVTNode = [&DAG,this] (const VTSDNode *N) {
1623  MVT Ty = typeSplit(N->getVT().getSimpleVT()).first;
1624  SDValue TV = DAG.getValueType(Ty);
1625  return std::make_pair(TV, TV);
1626  };
1627 
1628  for (SDValue A : Op.getNode()->ops()) {
1629  VectorPair P = Subtarget.isHVXVectorType(ty(A), true)
1630  ? opSplit(A, dl, DAG)
1631  : std::make_pair(A, A);
1632  // Special case for type operand.
1633  if (Op.getOpcode() == ISD::SIGN_EXTEND_INREG) {
1634  if (const auto *N = dyn_cast<const VTSDNode>(A.getNode()))
1635  P = SplitVTNode(N);
1636  }
1637  OpsL.push_back(P.first);
1638  OpsH.push_back(P.second);
1639  }
1640 
1641  MVT ResTy = ty(Op);
1642  MVT HalfTy = typeSplit(ResTy).first;
1643  SDValue L = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsL);
1644  SDValue H = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsH);
1645  SDValue S = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy, L, H);
1646  return S;
1647 }
1648 
1649 SDValue
1650 HexagonTargetLowering::SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const {
1651  LSBaseSDNode *BN = cast<LSBaseSDNode>(Op.getNode());
1652  assert(BN->isUnindexed());
1653  MVT MemTy = BN->getMemoryVT().getSimpleVT();
1654  if (!isHvxPairTy(MemTy))
1655  return Op;
1656 
1657  const SDLoc &dl(Op);
1658  unsigned HwLen = Subtarget.getVectorLength();
1659  MVT SingleTy = typeSplit(MemTy).first;
1660  SDValue Chain = BN->getChain();
1661  SDValue Base0 = BN->getBasePtr();
1662  SDValue Base1 = DAG.getMemBasePlusOffset(Base0, HwLen, dl);
1663 
1664  MachineMemOperand *MOp0 = nullptr, *MOp1 = nullptr;
1665  if (MachineMemOperand *MMO = BN->getMemOperand()) {
1666  MachineFunction &MF = DAG.getMachineFunction();
1667  MOp0 = MF.getMachineMemOperand(MMO, 0, HwLen);
1668  MOp1 = MF.getMachineMemOperand(MMO, HwLen, HwLen);
1669  }
1670 
1671  unsigned MemOpc = BN->getOpcode();
1672  SDValue NewOp;
1673 
1674  if (MemOpc == ISD::LOAD) {
1675  SDValue Load0 = DAG.getLoad(SingleTy, dl, Chain, Base0, MOp0);
1676  SDValue Load1 = DAG.getLoad(SingleTy, dl, Chain, Base1, MOp1);
1677  NewOp = DAG.getMergeValues(
1678  { DAG.getNode(ISD::CONCAT_VECTORS, dl, MemTy, Load0, Load1),
1680  Load0.getValue(1), Load1.getValue(1)) }, dl);
1681  } else {
1682  assert(MemOpc == ISD::STORE);
1683  VectorPair Vals = opSplit(cast<StoreSDNode>(Op)->getValue(), dl, DAG);
1684  SDValue Store0 = DAG.getStore(Chain, dl, Vals.first, Base0, MOp0);
1685  SDValue Store1 = DAG.getStore(Chain, dl, Vals.second, Base1, MOp1);
1686  NewOp = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store0, Store1);
1687  }
1688 
1689  return NewOp;
1690 }
1691 
1692 SDValue
1693 HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const {
1694  unsigned Opc = Op.getOpcode();
1695  bool IsPairOp = isHvxPairTy(ty(Op)) ||
1696  llvm::any_of(Op.getNode()->ops(), [this] (SDValue V) {
1697  return isHvxPairTy(ty(V));
1698  });
1699 
1700  if (IsPairOp) {
1701  switch (Opc) {
1702  default:
1703  break;
1704  case ISD::LOAD:
1705  case ISD::STORE:
1706  return SplitHvxMemOp(Op, DAG);
1707  case ISD::CTPOP:
1708  case ISD::CTLZ:
1709  case ISD::CTTZ:
1710  case ISD::MUL:
1711  case ISD::MULHS:
1712  case ISD::MULHU:
1713  case ISD::AND:
1714  case ISD::OR:
1715  case ISD::XOR:
1716  case ISD::SRA:
1717  case ISD::SHL:
1718  case ISD::SRL:
1719  case ISD::SETCC:
1720  case ISD::VSELECT:
1721  case ISD::SIGN_EXTEND:
1722  case ISD::ZERO_EXTEND:
1724  return SplitHvxPairOp(Op, DAG);
1725  }
1726  }
1727 
1728  switch (Opc) {
1729  default:
1730  break;
1731  case ISD::BUILD_VECTOR: return LowerHvxBuildVector(Op, DAG);
1732  case ISD::CONCAT_VECTORS: return LowerHvxConcatVectors(Op, DAG);
1733  case ISD::INSERT_SUBVECTOR: return LowerHvxInsertSubvector(Op, DAG);
1734  case ISD::INSERT_VECTOR_ELT: return LowerHvxInsertElement(Op, DAG);
1735  case ISD::EXTRACT_SUBVECTOR: return LowerHvxExtractSubvector(Op, DAG);
1736  case ISD::EXTRACT_VECTOR_ELT: return LowerHvxExtractElement(Op, DAG);
1737  case ISD::BITCAST: return LowerHvxBitcast(Op, DAG);
1738  case ISD::ANY_EXTEND: return LowerHvxAnyExt(Op, DAG);
1739  case ISD::SIGN_EXTEND: return LowerHvxSignExt(Op, DAG);
1740  case ISD::ZERO_EXTEND: return LowerHvxZeroExt(Op, DAG);
1741  case ISD::CTTZ: return LowerHvxCttz(Op, DAG);
1742  case ISD::SRA:
1743  case ISD::SHL:
1744  case ISD::SRL: return LowerHvxShift(Op, DAG);
1745  case ISD::MUL: return LowerHvxMul(Op, DAG);
1746  case ISD::MULHS:
1747  case ISD::MULHU: return LowerHvxMulh(Op, DAG);
1748  case ISD::ANY_EXTEND_VECTOR_INREG: return LowerHvxExtend(Op, DAG);
1749  case ISD::SETCC:
1750  case ISD::INTRINSIC_VOID: return Op;
1751  case ISD::INTRINSIC_WO_CHAIN: return LowerHvxIntrinsic(Op, DAG);
1752  // Unaligned loads will be handled by the default lowering.
1753  case ISD::LOAD: return SDValue();
1754  }
1755 #ifndef NDEBUG
1756  Op.dumpr(&DAG);
1757 #endif
1758  llvm_unreachable("Unhandled HVX operation");
1759 }
1760 
1761 void
1762 HexagonTargetLowering::LowerHvxOperationWrapper(SDNode *N,
1764 }
1765 
1766 void
1767 HexagonTargetLowering::ReplaceHvxNodeResults(SDNode *N,
1768  SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
1769  unsigned Opc = N->getOpcode();
1770  switch (Opc) {
1771  case ISD::BITCAST:
1772  if (isHvxBoolTy(ty(N->getOperand(0)))) {
1773  SDValue Op(N, 0);
1774  SDValue C = LowerHvxBitcast(Op, DAG);
1775  Results.push_back(C);
1776  }
1777  break;
1778  default:
1779  break;
1780  }
1781 }
1782 
1783 SDValue
1784 HexagonTargetLowering::PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI)
1785  const {
1786  const SDLoc &dl(N);
1787  SDValue Op(N, 0);
1788 
1789  unsigned Opc = Op.getOpcode();
1790  if (Opc == ISD::VSELECT) {
1791  // (vselect (xor x, qtrue), v0, v1) -> (vselect x, v1, v0)
1792  SDValue Cond = Op.getOperand(0);
1793  if (Cond->getOpcode() == ISD::XOR) {
1794  SDValue C0 = Cond.getOperand(0), C1 = Cond.getOperand(1);
1795  if (C1->getOpcode() == HexagonISD::QTRUE) {
1796  SDValue VSel = DCI.DAG.getNode(ISD::VSELECT, dl, ty(Op), C0,
1797  Op.getOperand(2), Op.getOperand(1));
1798  return VSel;
1799  }
1800  }
1801  }
1802  return SDValue();
1803 }
1804 
1805 bool
1806 HexagonTargetLowering::isHvxOperation(SDValue Op) const {
1807  // If the type of the result, or any operand type are HVX vector types,
1808  // this is an HVX operation.
1809  return Subtarget.isHVXVectorType(ty(Op), true) ||
1810  llvm::any_of(Op.getNode()->ops(),
1811  [this] (SDValue V) {
1812  return Subtarget.isHVXVectorType(ty(V), true);
1813  });
1814 }
1815 
1816 bool
1817 HexagonTargetLowering::isHvxOperation(SDNode *N) const {
1818  // If the type of any result, or any operand type are HVX vector types,
1819  // this is an HVX operation.
1820  auto IsHvxTy = [this] (EVT Ty) {
1821  return Ty.isSimple() && Subtarget.isHVXVectorType(Ty.getSimpleVT(), true);
1822  };
1823  auto IsHvxOp = [this] (SDValue Op) {
1824  return Subtarget.isHVXVectorType(ty(Op), true);
1825  };
1826  return llvm::any_of(N->values(), IsHvxTy) || llvm::any_of(N->ops(), IsHvxOp);
1827 }
uint64_t CallInst * C
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:744
static MVT getIntegerVT(unsigned BitWidth)
BitVector & set()
Definition: BitVector.h:398
EVT getValueType() const
Return the ValueType of the referenced return value.
LLVM_NODISCARD std::enable_if_t< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type > dyn_cast(const Y &Val)
Definition: Casting.h:334
bool isUndef() const
ArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
Definition: ArrayRef.h:219
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:511
This class represents lattice values for constants.
Definition: AllocatorList.h:23
static MVT getVectorVT(MVT VT, unsigned NumElements)
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:520
iterator_range< value_iterator > values() const
iterator begin() const
Definition: ArrayRef.h:144
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:698
bool isVector() const
Return true if this is a vector value type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:260
static const MVT LegalW128[]
bool test(unsigned Idx) const
Definition: BitVector.h:502
unsigned getVectorNumElements() const
const SDValue & getChain() const
Function Alias Analysis Results
F(f)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:687
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
static const MVT LegalV64[]
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1...
Definition: ISDOpcodes.h:498
std::pair< MCSymbol *, MachineModuleInfoImpl::StubValueTy > PairTy
A description of a memory reference used in the backend.
Shift and rotation operations.
Definition: ISDOpcodes.h:576
Base class for LoadSDNode and StoreSDNode.
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:213
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:43
SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const
void setCondCodeAction(ISD::CondCode CC, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn&#39;t supported on the target and indicate what to d...
SimpleValueType SimpleTy
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:497
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:80
This class is used to represent EVT&#39;s, which are used to parameterize some operations.
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
void assign(size_type NumElts, const T &Elt)
Definition: SmallVector.h:458
int64_t getSExtValue() const
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:421
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:606
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:223
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC&#39;s if you just have an ISD::CondCode instead of an SD...
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:926
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:618
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:676
ArrayRef< SDUse > ops() const
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements. ...
Definition: SelectionDAG.h:790
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:168
MVT getVectorElementType() const
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified, possibly variable, elements.
Definition: ISDOpcodes.h:456
SmallVector< MachineOperand, 4 > Cond
#define P(N)
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:183
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:492
Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type...
void setTargetDAGCombine(ISD::NodeType NT)
Targets should invoke this method for each target independent node that they want to provide a custom...
bool isMachineOpcode() const
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:156
This is an important base class in LLVM.
Definition: Constant.h:41
iterator_range< value_op_iterator > op_values() const
const SDValue & getOperand(unsigned Num) const
static const MVT LegalW64[]
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:465
#define H(x, y, z)
Definition: MD5.cpp:58
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static const MVT LegalV128[]
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1498
constexpr double e
Definition: MathExtras.h:58
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:147
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
Extended Value Type.
Definition: ValueTypes.h:35
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type...
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:52
bool isHVXVectorType(MVT VecTy, bool IncludeBool=false) const
SDValue getMemBasePlusOffset(SDValue Base, int64_t Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:476
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provides VTs and return the low/high part...
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:223
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:883
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:773
iterator end() const
Definition: ArrayRef.h:145
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:585
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:786
Represents one node in the SelectionDAG.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:597
void setIndexedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
EVT getMemoryVT() const
Return the type of the in-memory value.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:597
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:647
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:650
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array...
Definition: ArrayRef.h:186
Flags
Flags values. These may be or&#39;d together.
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
void dumpr() const
ArrayRef< T > take_back(size_t N=1) const
Return a copy of *this with only the last N elements.
Definition: ArrayRef.h:226
unsigned getVectorLength() const
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:551
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:665
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
Definition: ArrayRef.h:195
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:817
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
#define I(x, y, z)
Definition: MD5.cpp:59
#define N
uint32_t Size
Definition: Profile.cpp:46
unsigned getOpcode() const
SDValue getValue(unsigned R) const
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:484
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=None, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDValue getValueType(EVT)
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:620
unsigned getNumOperands() const
Conversion operators.
Definition: ISDOpcodes.h:644
const SDValue & getOperand(unsigned i) const
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
static IntegerType * getInt8Ty(LLVMContext &C)
Definition: Type.cpp:184
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:42
const SDValue & getBasePtr() const
LLVMContext * getContext() const
Definition: SelectionDAG.h:431
static Constant * get(ArrayRef< Constant *> V)
Definition: Constants.cpp:1254
#define T1
void setIndexedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:540