LLVM  16.0.0git
RISCVISelDAGToDAG.cpp
Go to the documentation of this file.
1 //===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISCV ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines an instruction selector for the RISCV target.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "RISCVISelDAGToDAG.h"
14 #include "MCTargetDesc/RISCVMCTargetDesc.h"
15 #include "MCTargetDesc/RISCVMatInt.h"
16 #include "RISCVISelLowering.h"
17 #include "RISCVMachineFunctionInfo.h"
18 #include "llvm/CodeGen/MachineFrameInfo.h"
19 #include "llvm/IR/IntrinsicsRISCV.h"
20 #include "llvm/Support/Alignment.h"
21 #include "llvm/Support/Debug.h"
22 #include "llvm/Support/KnownBits.h"
23 #include "llvm/Support/MathExtras.h"
24 #include "llvm/Support/raw_ostream.h"
25 #include <optional>
26 
27 using namespace llvm;
28 
29 #define DEBUG_TYPE "riscv-isel"
30 
31 namespace llvm::RISCV {
32 #define GET_RISCVVSSEGTable_IMPL
33 #define GET_RISCVVLSEGTable_IMPL
34 #define GET_RISCVVLXSEGTable_IMPL
35 #define GET_RISCVVSXSEGTable_IMPL
36 #define GET_RISCVVLETable_IMPL
37 #define GET_RISCVVSETable_IMPL
38 #define GET_RISCVVLXTable_IMPL
39 #define GET_RISCVVSXTable_IMPL
40 #define GET_RISCVMaskedPseudosTable_IMPL
41 #include "RISCVGenSearchableTables.inc"
42 } // namespace llvm::RISCV
43 
44 static unsigned getLastNonGlueOrChainOpIdx(const SDNode *Node) {
45  assert(Node->getNumOperands() > 0 && "Node with no operands");
46  unsigned LastOpIdx = Node->getNumOperands() - 1;
47  if (Node->getOperand(LastOpIdx).getValueType() == MVT::Glue)
48  --LastOpIdx;
49  if (Node->getOperand(LastOpIdx).getValueType() == MVT::Other)
50  --LastOpIdx;
51  return LastOpIdx;
52 }
53 
54 static unsigned getVecPolicyOpIdx(const SDNode *Node, const MCInstrDesc &MCID) {
55  assert(RISCVII::hasVecPolicyOp(MCID.TSFlags));
56  (void)MCID;
57  return getLastNonGlueOrChainOpIdx(Node);
58 }
59 
60 void RISCVDAGToDAGISel::PreprocessISelDAG() {
61  SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
62 
63  bool MadeChange = false;
64  while (Position != CurDAG->allnodes_begin()) {
65  SDNode *N = &*--Position;
66  if (N->use_empty())
67  continue;
68 
69  SDValue Result;
70  switch (N->getOpcode()) {
71  case ISD::SPLAT_VECTOR: {
72  // Convert integer SPLAT_VECTOR to VMV_V_X_VL and floating-point
73  // SPLAT_VECTOR to VFMV_V_F_VL to reduce isel burden.
74  MVT VT = N->getSimpleValueType(0);
75  unsigned Opc =
76  VT.isInteger() ? RISCVISD::VMV_V_X_VL : RISCVISD::VFMV_V_F_VL;
77  SDLoc DL(N);
78  SDValue VL = CurDAG->getRegister(RISCV::X0, Subtarget->getXLenVT());
79  Result = CurDAG->getNode(Opc, DL, VT, CurDAG->getUNDEF(VT),
80  N->getOperand(0), VL);
81  break;
82  }
83  case RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL: {
84  // Lower SPLAT_VECTOR_SPLIT_I64 to two scalar stores and a stride 0 vector
85  // load. Done after lowering and combining so that we have a chance to
86  // optimize this to VMV_V_X_VL when the upper bits aren't needed.
87  assert(N->getNumOperands() == 4 && "Unexpected number of operands");
88  MVT VT = N->getSimpleValueType(0);
89  SDValue Passthru = N->getOperand(0);
90  SDValue Lo = N->getOperand(1);
91  SDValue Hi = N->getOperand(2);
92  SDValue VL = N->getOperand(3);
93  assert(VT.getVectorElementType() == MVT::i64 && VT.isScalableVector() &&
94  Lo.getValueType() == MVT::i32 && Hi.getValueType() == MVT::i32 &&
95  "Unexpected VTs!");
96  MachineFunction &MF = CurDAG->getMachineFunction();
97  RISCVMachineFunctionInfo *FuncInfo =
98  MF.getInfo<RISCVMachineFunctionInfo>();
99  SDLoc DL(N);
100 
101  // We use the same frame index we use for moving two i32s into 64-bit FPR.
102  // This is an analogous operation.
103  int FI = FuncInfo->getMoveF64FrameIndex(MF);
104  MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
105  const TargetLowering &TLI = CurDAG->getTargetLoweringInfo();
106  SDValue StackSlot =
107  CurDAG->getFrameIndex(FI, TLI.getPointerTy(CurDAG->getDataLayout()));
108 
109  SDValue Chain = CurDAG->getEntryNode();
110  Lo = CurDAG->getStore(Chain, DL, Lo, StackSlot, MPI, Align(8));
111 
112  SDValue OffsetSlot =
113  CurDAG->getMemBasePlusOffset(StackSlot, TypeSize::Fixed(4), DL);
114  Hi = CurDAG->getStore(Chain, DL, Hi, OffsetSlot, MPI.getWithOffset(4),
115  Align(8));
116 
117  Chain = CurDAG->getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
118 
119  SDVTList VTs = CurDAG->getVTList({VT, MVT::Other});
120  SDValue IntID =
121  CurDAG->getTargetConstant(Intrinsic::riscv_vlse, DL, MVT::i64);
122  SDValue Ops[] = {Chain,
123  IntID,
124  Passthru,
125  StackSlot,
126  CurDAG->getRegister(RISCV::X0, MVT::i64),
127  VL};
128 
129  Result = CurDAG->getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
130  MVT::i64, MPI, Align(8),
131  MachineMemOperand::MOLoad);
132  break;
133  }
134  }
135 
136  if (Result) {
137  LLVM_DEBUG(dbgs() << "RISCV DAG preprocessing replacing:\nOld: ");
138  LLVM_DEBUG(N->dump(CurDAG));
139  LLVM_DEBUG(dbgs() << "\nNew: ");
140  LLVM_DEBUG(Result->dump(CurDAG));
141  LLVM_DEBUG(dbgs() << "\n");
142 
143  CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
144  MadeChange = true;
145  }
146  }
147 
148  if (MadeChange)
149  CurDAG->RemoveDeadNodes();
150 }
151 
152 void RISCVDAGToDAGISel::PostprocessISelDAG() {
153  HandleSDNode Dummy(CurDAG->getRoot());
154  SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
155 
156  bool MadeChange = false;
157  while (Position != CurDAG->allnodes_begin()) {
158  SDNode *N = &*--Position;
159  // Skip dead nodes and any non-machine opcodes.
160  if (N->use_empty() || !N->isMachineOpcode())
161  continue;
162 
163  MadeChange |= doPeepholeSExtW(N);
164  MadeChange |= doPeepholeMaskedRVV(N);
165  }
166 
167  CurDAG->setRoot(Dummy.getValue());
168 
169  MadeChange |= doPeepholeMergeVVMFold();
170 
171  if (MadeChange)
172  CurDAG->RemoveDeadNodes();
173 }
174 
175 static SDNode *selectImmSeq(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
176  RISCVMatInt::InstSeq &Seq) {
177  SDNode *Result = nullptr;
178  SDValue SrcReg = CurDAG->getRegister(RISCV::X0, VT);
179  for (RISCVMatInt::Inst &Inst : Seq) {
180  SDValue SDImm = CurDAG->getTargetConstant(Inst.Imm, DL, VT);
181  switch (Inst.getOpndKind()) {
182  case RISCVMatInt::Imm:
183  Result = CurDAG->getMachineNode(Inst.Opc, DL, VT, SDImm);
184  break;
185  case RISCVMatInt::RegX0:
186  Result = CurDAG->getMachineNode(Inst.Opc, DL, VT, SrcReg,
187  CurDAG->getRegister(RISCV::X0, VT));
188  break;
189  case RISCVMatInt::RegReg:
190  Result = CurDAG->getMachineNode(Inst.Opc, DL, VT, SrcReg, SrcReg);
191  break;
192  case RISCVMatInt::RegImm:
193  Result = CurDAG->getMachineNode(Inst.Opc, DL, VT, SrcReg, SDImm);
194  break;
195  }
196 
197  // Only the first instruction has X0 as its source.
198  SrcReg = SDValue(Result, 0);
199  }
200 
201  return Result;
202 }
203 
204 static SDNode *selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
205  int64_t Imm, const RISCVSubtarget &Subtarget) {
206  RISCVMatInt::InstSeq Seq =
207  RISCVMatInt::generateInstSeq(Imm, Subtarget.getFeatureBits());
208 
209  return selectImmSeq(CurDAG, DL, VT, Seq);
210 }
211 
212 static SDValue createTuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
213  unsigned NF, RISCVII::VLMUL LMUL) {
214  static const unsigned M1TupleRegClassIDs[] = {
215  RISCV::VRN2M1RegClassID, RISCV::VRN3M1RegClassID, RISCV::VRN4M1RegClassID,
216  RISCV::VRN5M1RegClassID, RISCV::VRN6M1RegClassID, RISCV::VRN7M1RegClassID,
217  RISCV::VRN8M1RegClassID};
218  static const unsigned M2TupleRegClassIDs[] = {RISCV::VRN2M2RegClassID,
219  RISCV::VRN3M2RegClassID,
220  RISCV::VRN4M2RegClassID};
221 
222  assert(Regs.size() >= 2 && Regs.size() <= 8);
223 
224  unsigned RegClassID;
225  unsigned SubReg0;
226  switch (LMUL) {
227  default:
228  llvm_unreachable("Invalid LMUL.");
229  case RISCVII::VLMUL::LMUL_F8:
230  case RISCVII::VLMUL::LMUL_F4:
231  case RISCVII::VLMUL::LMUL_F2:
232  case RISCVII::VLMUL::LMUL_1:
233  static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
234  "Unexpected subreg numbering");
235  SubReg0 = RISCV::sub_vrm1_0;
236  RegClassID = M1TupleRegClassIDs[NF - 2];
237  break;
238  case RISCVII::VLMUL::LMUL_2:
239  static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
240  "Unexpected subreg numbering");
241  SubReg0 = RISCV::sub_vrm2_0;
242  RegClassID = M2TupleRegClassIDs[NF - 2];
243  break;
244  case RISCVII::VLMUL::LMUL_4:
245  static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
246  "Unexpected subreg numbering");
247  SubReg0 = RISCV::sub_vrm4_0;
248  RegClassID = RISCV::VRN2M4RegClassID;
249  break;
250  }
251 
252  SDLoc DL(Regs[0]);
253  SmallVector<SDValue, 8> Ops;
254 
255  Ops.push_back(CurDAG.getTargetConstant(RegClassID, DL, MVT::i32));
256 
257  for (unsigned I = 0; I < Regs.size(); ++I) {
258  Ops.push_back(Regs[I]);
259  Ops.push_back(CurDAG.getTargetConstant(SubReg0 + I, DL, MVT::i32));
260  }
261  SDNode *N =
262  CurDAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops);
263  return SDValue(N, 0);
264 }
265 
266 void RISCVDAGToDAGISel::addVectorLoadStoreOperands(
267  SDNode *Node, unsigned Log2SEW, const SDLoc &DL, unsigned CurOp,
268  bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl<SDValue> &Operands,
269  bool IsLoad, MVT *IndexVT) {
270  SDValue Chain = Node->getOperand(0);
271  SDValue Glue;
272 
273  Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
274 
275  if (IsStridedOrIndexed) {
276  Operands.push_back(Node->getOperand(CurOp++)); // Index.
277  if (IndexVT)
278  *IndexVT = Operands.back()->getSimpleValueType(0);
279  }
280 
281  if (IsMasked) {
282  // Mask needs to be copied to V0.
283  SDValue Mask = Node->getOperand(CurOp++);
284  Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
285  Glue = Chain.getValue(1);
286  Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
287  }
288  SDValue VL;
289  selectVLOp(Node->getOperand(CurOp++), VL);
290  Operands.push_back(VL);
291 
292  MVT XLenVT = Subtarget->getXLenVT();
293  SDValue SEWOp = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT);
294  Operands.push_back(SEWOp);
295 
296  // Masked load has the tail policy argument.
297  if (IsMasked && IsLoad) {
298  // Policy must be a constant.
299  uint64_t Policy = Node->getConstantOperandVal(CurOp++);
300  SDValue PolicyOp = CurDAG->getTargetConstant(Policy, DL, XLenVT);
301  Operands.push_back(PolicyOp);
302  }
303 
304  Operands.push_back(Chain); // Chain.
305  if (Glue)
306  Operands.push_back(Glue);
307 }
308 
309 static bool isAllUndef(ArrayRef<SDValue> Values) {
310  return llvm::all_of(Values, [](SDValue V) { return V->isUndef(); });
311 }
312 
313 void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, bool IsMasked,
314  bool IsStrided) {
315  SDLoc DL(Node);
316  unsigned NF = Node->getNumValues() - 1;
317  MVT VT = Node->getSimpleValueType(0);
318  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
319  RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
320 
321  unsigned CurOp = 2;
322  SmallVector<SDValue, 8> Operands;
323 
324  SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
325  Node->op_begin() + CurOp + NF);
326  bool IsTU = IsMasked || !isAllUndef(Regs);
327  if (IsTU) {
328  SDValue Merge = createTuple(*CurDAG, Regs, NF, LMUL);
329  Operands.push_back(Merge);
330  }
331  CurOp += NF;
332 
333  addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
334  Operands, /*IsLoad=*/true);
335 
336  const RISCV::VLSEGPseudo *P =
337  RISCV::getVLSEGPseudo(NF, IsMasked, IsTU, IsStrided, /*FF*/ false, Log2SEW,
338  static_cast<unsigned>(LMUL));
339  MachineSDNode *Load =
340  CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
341 
342  if (auto *MemOp = dyn_cast<MemSDNode>(Node))
343  CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
344 
345  SDValue SuperReg = SDValue(Load, 0);
346  for (unsigned I = 0; I < NF; ++I) {
347  unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
348  ReplaceUses(SDValue(Node, I),
349  CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
350  }
351 
352  ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
353  CurDAG->RemoveDeadNode(Node);
354 }
355 
356 void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, bool IsMasked) {
357  SDLoc DL(Node);
358  unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain.
359  MVT VT = Node->getSimpleValueType(0);
360  MVT XLenVT = Subtarget->getXLenVT();
361  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
362  RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
363 
364  unsigned CurOp = 2;
365  SmallVector<SDValue, 7> Operands;
366 
367  SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
368  Node->op_begin() + CurOp + NF);
369  bool IsTU = IsMasked || !isAllUndef(Regs);
370  if (IsTU) {
371  SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
372  Operands.push_back(MaskedOff);
373  }
374  CurOp += NF;
375 
376  addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
377  /*IsStridedOrIndexed*/ false, Operands,
378  /*IsLoad=*/true);
379 
380  const RISCV::VLSEGPseudo *P =
381  RISCV::getVLSEGPseudo(NF, IsMasked, IsTU, /*Strided*/ false, /*FF*/ true,
382  Log2SEW, static_cast<unsigned>(LMUL));
383  MachineSDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped,
384  XLenVT, MVT::Other, Operands);
385 
386  if (auto *MemOp = dyn_cast<MemSDNode>(Node))
387  CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
388 
389  SDValue SuperReg = SDValue(Load, 0);
390  for (unsigned I = 0; I < NF; ++I) {
391  unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
392  ReplaceUses(SDValue(Node, I),
393  CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
394  }
395 
396  ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); // VL
397  ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 2)); // Chain
398  CurDAG->RemoveDeadNode(Node);
399 }
400 
401 void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, bool IsMasked,
402  bool IsOrdered) {
403  SDLoc DL(Node);
404  unsigned NF = Node->getNumValues() - 1;
405  MVT VT = Node->getSimpleValueType(0);
406  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
407  RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
408 
409  unsigned CurOp = 2;
410  SmallVector<SDValue, 8> Operands;
411 
412  SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
413  Node->op_begin() + CurOp + NF);
414  bool IsTU = IsMasked || !isAllUndef(Regs);
415  if (IsTU) {
416  SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
417  Operands.push_back(MaskedOff);
418  }
419  CurOp += NF;
420 
421  MVT IndexVT;
422  addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
423  /*IsStridedOrIndexed*/ true, Operands,
424  /*IsLoad=*/true, &IndexVT);
425 
426  assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
427  "Element count mismatch");
428 
429  RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
430  unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
431  if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
432  report_fatal_error("The V extension does not support EEW=64 for index "
433  "values when XLEN=32");
434  }
435  const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo(
436  NF, IsMasked, IsTU, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
437  static_cast<unsigned>(IndexLMUL));
438  MachineSDNode *Load =
439  CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
440 
441  if (auto *MemOp = dyn_cast<MemSDNode>(Node))
442  CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
443 
444  SDValue SuperReg = SDValue(Load, 0);
445  for (unsigned I = 0; I < NF; ++I) {
446  unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
447  ReplaceUses(SDValue(Node, I),
448  CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
449  }
450 
451  ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
452  CurDAG->RemoveDeadNode(Node);
453 }
454 
455 void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, bool IsMasked,
456  bool IsStrided) {
457  SDLoc DL(Node);
458  unsigned NF = Node->getNumOperands() - 4;
459  if (IsStrided)
460  NF--;
461  if (IsMasked)
462  NF--;
463  MVT VT = Node->getOperand(2)->getSimpleValueType(0);
464  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
465  RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
466  SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
467  SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
468 
469  SmallVector<SDValue, 8> Operands;
470  Operands.push_back(StoreVal);
471  unsigned CurOp = 2 + NF;
472 
473  addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
474  Operands);
475 
476  const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo(
477  NF, IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL));
478  MachineSDNode *Store =
479  CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
480 
481  if (auto *MemOp = dyn_cast<MemSDNode>(Node))
482  CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
483 
484  ReplaceNode(Node, Store);
485 }
486 
487 void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, bool IsMasked,
488  bool IsOrdered) {
489  SDLoc DL(Node);
490  unsigned NF = Node->getNumOperands() - 5;
491  if (IsMasked)
492  --NF;
493  MVT VT = Node->getOperand(2)->getSimpleValueType(0);
494  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
495  RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
496  SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
497  SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
498 
499  SmallVector<SDValue, 8> Operands;
500  Operands.push_back(StoreVal);
501  unsigned CurOp = 2 + NF;
502 
503  MVT IndexVT;
504  addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
505  /*IsStridedOrIndexed*/ true, Operands,
506  /*IsLoad=*/false, &IndexVT);
507 
508  assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
509  "Element count mismatch");
510 
511  RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
512  unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
513  if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
514  report_fatal_error("The V extension does not support EEW=64 for index "
515  "values when XLEN=32");
516  }
517  const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo(
518  NF, IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
519  static_cast<unsigned>(IndexLMUL));
520  MachineSDNode *Store =
521  CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
522 
523  if (auto *MemOp = dyn_cast<MemSDNode>(Node))
524  CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
525 
526  ReplaceNode(Node, Store);
527 }
528 
529 void RISCVDAGToDAGISel::selectVSETVLI(SDNode *Node) {
530  if (!Subtarget->hasVInstructions())
531  return;
532 
533  assert((Node->getOpcode() == ISD::INTRINSIC_W_CHAIN ||
534  Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN) &&
535  "Unexpected opcode");
536 
537  SDLoc DL(Node);
538  MVT XLenVT = Subtarget->getXLenVT();
539 
540  bool HasChain = Node->getOpcode() == ISD::INTRINSIC_W_CHAIN;
541  unsigned IntNoOffset = HasChain ? 1 : 0;
542  unsigned IntNo = Node->getConstantOperandVal(IntNoOffset);
543 
544  assert((IntNo == Intrinsic::riscv_vsetvli ||
545  IntNo == Intrinsic::riscv_vsetvlimax ||
546  IntNo == Intrinsic::riscv_vsetvli_opt ||
547  IntNo == Intrinsic::riscv_vsetvlimax_opt) &&
548  "Unexpected vsetvli intrinsic");
549 
550  bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax ||
551  IntNo == Intrinsic::riscv_vsetvlimax_opt;
552  unsigned Offset = IntNoOffset + (VLMax ? 1 : 2);
553 
554  assert(Node->getNumOperands() == Offset + 2 &&
555  "Unexpected number of operands");
556 
557  unsigned SEW =
558  RISCVVType::decodeVSEW(Node->getConstantOperandVal(Offset) & 0x7);
559  RISCVII::VLMUL VLMul = static_cast<RISCVII::VLMUL>(
560  Node->getConstantOperandVal(Offset + 1) & 0x7);
561 
562  unsigned VTypeI = RISCVVType::encodeVTYPE(VLMul, SEW, /*TailAgnostic*/ true,
563  /*MaskAgnostic*/ false);
564  SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT);
565 
566  SmallVector<EVT, 2> VTs = {XLenVT};
567  if (HasChain)
568  VTs.push_back(MVT::Other);
569 
570  SDValue VLOperand;
571  unsigned Opcode = RISCV::PseudoVSETVLI;
572  if (VLMax) {
573  VLOperand = CurDAG->getRegister(RISCV::X0, XLenVT);
574  Opcode = RISCV::PseudoVSETVLIX0;
575  } else {
576  VLOperand = Node->getOperand(IntNoOffset + 1);
577 
578  if (auto *C = dyn_cast<ConstantSDNode>(VLOperand)) {
579  uint64_t AVL = C->getZExtValue();
580  if (isUInt<5>(AVL)) {
581  SDValue VLImm = CurDAG->getTargetConstant(AVL, DL, XLenVT);
582  SmallVector<SDValue, 3> Ops = {VLImm, VTypeIOp};
583  if (HasChain)
584  Ops.push_back(Node->getOperand(0));
585  ReplaceNode(
586  Node, CurDAG->getMachineNode(RISCV::PseudoVSETIVLI, DL, VTs, Ops));
587  return;
588  }
589  }
590  }
591 
592  SmallVector<SDValue, 3> Ops = {VLOperand, VTypeIOp};
593  if (HasChain)
594  Ops.push_back(Node->getOperand(0));
595 
596  ReplaceNode(Node, CurDAG->getMachineNode(Opcode, DL, VTs, Ops));
597 }
598 
599 bool RISCVDAGToDAGISel::tryShrinkShlLogicImm(SDNode *Node) {
600  MVT VT = Node->getSimpleValueType(0);
601  unsigned Opcode = Node->getOpcode();
602  assert((Opcode == ISD::AND || Opcode == ISD::OR || Opcode == ISD::XOR) &&
603  "Unexpected opcode");
604  SDLoc DL(Node);
605 
606  // For operations of the form (x << C1) op C2, check if we can use
607  // ANDI/ORI/XORI by transforming it into (x op (C2>>C1)) << C1.
608  SDValue N0 = Node->getOperand(0);
609  SDValue N1 = Node->getOperand(1);
610 
611  ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(N1);
612  if (!Cst)
613  return false;
614 
615  int64_t Val = Cst->getSExtValue();
616 
617  // Check if immediate can already use ANDI/ORI/XORI.
618  if (isInt<12>(Val))
619  return false;
620 
621  SDValue Shift = N0;
622 
623  // If Val is simm32 and we have a sext_inreg from i32, then the binop
624  // produces at least 33 sign bits. We can peek through the sext_inreg and use
625  // a SLLIW at the end.
626  bool SignExt = false;
627  if (isInt<32>(Val) && N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
628  N0.hasOneUse() && cast<VTSDNode>(N0.getOperand(1))->getVT() == MVT::i32) {
629  SignExt = true;
630  Shift = N0.getOperand(0);
631  }
632 
633  if (Shift.getOpcode() != ISD::SHL || !Shift.hasOneUse())
634  return false;
635 
636  ConstantSDNode *ShlCst = dyn_cast<ConstantSDNode>(Shift.getOperand(1));
637  if (!ShlCst)
638  return false;
639 
640  uint64_t ShAmt = ShlCst->getZExtValue();
641 
642  // Make sure that we don't change the operation by removing bits.
643  // This only matters for OR and XOR, AND is unaffected.
644  uint64_t RemovedBitsMask = maskTrailingOnes<uint64_t>(ShAmt);
645  if (Opcode != ISD::AND && (Val & RemovedBitsMask) != 0)
646  return false;
647 
648  int64_t ShiftedVal = Val >> ShAmt;
649  if (!isInt<12>(ShiftedVal))
650  return false;
651 
652  // If we peeked through a sext_inreg, make sure the shift is valid for SLLIW.
653  if (SignExt && ShAmt >= 32)
654  return false;
655 
656  // Ok, we can reorder to get a smaller immediate.
657  unsigned BinOpc;
658  switch (Opcode) {
659  default: llvm_unreachable("Unexpected opcode");
660  case ISD::AND: BinOpc = RISCV::ANDI; break;
661  case ISD::OR: BinOpc = RISCV::ORI; break;
662  case ISD::XOR: BinOpc = RISCV::XORI; break;
663  }
664 
665  unsigned ShOpc = SignExt ? RISCV::SLLIW : RISCV::SLLI;
666 
667  SDNode *BinOp =
668  CurDAG->getMachineNode(BinOpc, DL, VT, Shift.getOperand(0),
669  CurDAG->getTargetConstant(ShiftedVal, DL, VT));
670  SDNode *SLLI =
671  CurDAG->getMachineNode(ShOpc, DL, VT, SDValue(BinOp, 0),
672  CurDAG->getTargetConstant(ShAmt, DL, VT));
673  ReplaceNode(Node, SLLI);
674  return true;
675 }
676 
677 void RISCVDAGToDAGISel::Select(SDNode *Node) {
678  // If we have a custom node, we have already selected.
679  if (Node->isMachineOpcode()) {
680  LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n");
681  Node->setNodeId(-1);
682  return;
683  }
684 
685  // Instruction Selection not handled by the auto-generated tablegen selection
686  // should be handled here.
687  unsigned Opcode = Node->getOpcode();
688  MVT XLenVT = Subtarget->getXLenVT();
689  SDLoc DL(Node);
690  MVT VT = Node->getSimpleValueType(0);
691 
692  switch (Opcode) {
693  case ISD::Constant: {
694  auto *ConstNode = cast<ConstantSDNode>(Node);
695  if (VT == XLenVT && ConstNode->isZero()) {
696  SDValue New =
697  CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, XLenVT);
698  ReplaceNode(Node, New.getNode());
699  return;
700  }
701  int64_t Imm = ConstNode->getSExtValue();
702  // If the upper XLen-16 bits are not used, try to convert this to a simm12
703  // by sign extending bit 15.
704  if (isUInt<16>(Imm) && isInt<12>(SignExtend64<16>(Imm)) &&
705  hasAllHUsers(Node))
706  Imm = SignExtend64<16>(Imm);
707  // If the upper 32-bits are not used try to convert this into a simm32 by
708  // sign extending bit 32.
709  if (!isInt<32>(Imm) && isUInt<32>(Imm) && hasAllWUsers(Node))
710  Imm = SignExtend64<32>(Imm);
711 
712  ReplaceNode(Node, selectImm(CurDAG, DL, VT, Imm, *Subtarget));
713  return;
714  }
715  case ISD::SHL: {
716  auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
717  if (!N1C)
718  break;
719  SDValue N0 = Node->getOperand(0);
720  if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() ||
721  !isa<ConstantSDNode>(N0.getOperand(1)))
722  break;
723  unsigned ShAmt = N1C->getZExtValue();
724  uint64_t Mask = N0.getConstantOperandVal(1);
725 
726  // Optimize (shl (and X, C2), C) -> (slli (srliw X, C3), C3+C) where C2 has
727  // 32 leading zeros and C3 trailing zeros.
728  if (ShAmt <= 32 && isShiftedMask_64(Mask)) {
729  unsigned XLen = Subtarget->getXLen();
730  unsigned LeadingZeros = XLen - (64 - countLeadingZeros(Mask));
731  unsigned TrailingZeros = countTrailingZeros(Mask);
732  if (TrailingZeros > 0 && LeadingZeros == 32) {
733  SDNode *SRLIW = CurDAG->getMachineNode(
734  RISCV::SRLIW, DL, VT, N0->getOperand(0),
735  CurDAG->getTargetConstant(TrailingZeros, DL, VT));
736  SDNode *SLLI = CurDAG->getMachineNode(
737  RISCV::SLLI, DL, VT, SDValue(SRLIW, 0),
738  CurDAG->getTargetConstant(TrailingZeros + ShAmt, DL, VT));
739  ReplaceNode(Node, SLLI);
740  return;
741  }
742  }
743  break;
744  }
745  case ISD::SRL: {
746  auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
747  if (!N1C)
748  break;
749  SDValue N0 = Node->getOperand(0);
750  if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(N0.getOperand(1)))
751  break;
752  unsigned ShAmt = N1C->getZExtValue();
753  uint64_t Mask = N0.getConstantOperandVal(1);
754 
755  // Optimize (srl (and X, C2), C) -> (slli (srliw X, C3), C3-C) where C2 has
756  // 32 leading zeros and C3 trailing zeros.
757  if (isShiftedMask_64(Mask) && N0.hasOneUse()) {
758  unsigned XLen = Subtarget->getXLen();
759  unsigned LeadingZeros = XLen - (64 - countLeadingZeros(Mask));
760  unsigned TrailingZeros = countTrailingZeros(Mask);
761  if (LeadingZeros == 32 && TrailingZeros > ShAmt) {
762  SDNode *SRLIW = CurDAG->getMachineNode(
763  RISCV::SRLIW, DL, VT, N0->getOperand(0),
764  CurDAG->getTargetConstant(TrailingZeros, DL, VT));
765  SDNode *SLLI = CurDAG->getMachineNode(
766  RISCV::SLLI, DL, VT, SDValue(SRLIW, 0),
767  CurDAG->getTargetConstant(TrailingZeros - ShAmt, DL, VT));
768  ReplaceNode(Node, SLLI);
769  return;
770  }
771  }
772 
773  // Optimize (srl (and X, C2), C) ->
774  // (srli (slli X, (XLen-C3), (XLen-C3) + C)
775  // Where C2 is a mask with C3 trailing ones.
776  // Taking into account that the C2 may have had lower bits unset by
777  // SimplifyDemandedBits. This avoids materializing the C2 immediate.
778  // This pattern occurs when type legalizing right shifts for types with
779  // less than XLen bits.
780  Mask |= maskTrailingOnes<uint64_t>(ShAmt);
781  if (!isMask_64(Mask))
782  break;
783  unsigned TrailingOnes = countTrailingOnes(Mask);
784  if (ShAmt >= TrailingOnes)
785  break;
786  // If the mask has 32 trailing ones, use SRLIW.
787  if (TrailingOnes == 32) {
788  SDNode *SRLIW =
789  CurDAG->getMachineNode(RISCV::SRLIW, DL, VT, N0->getOperand(0),
790  CurDAG->getTargetConstant(ShAmt, DL, VT));
791  ReplaceNode(Node, SRLIW);
792  return;
793  }
794 
795  // Only do the remaining transforms if the shift has one use.
796  if (!N0.hasOneUse())
797  break;
798 
799  // If C2 is (1 << ShAmt) use bexti if possible.
800  if (Subtarget->hasStdExtZbs() && ShAmt + 1 == TrailingOnes) {
801  SDNode *BEXTI =
802  CurDAG->getMachineNode(RISCV::BEXTI, DL, VT, N0->getOperand(0),
803  CurDAG->getTargetConstant(ShAmt, DL, VT));
804  ReplaceNode(Node, BEXTI);
805  return;
806  }
807  unsigned LShAmt = Subtarget->getXLen() - TrailingOnes;
808  SDNode *SLLI =
809  CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0),
810  CurDAG->getTargetConstant(LShAmt, DL, VT));
811  SDNode *SRLI = CurDAG->getMachineNode(
812  RISCV::SRLI, DL, VT, SDValue(SLLI, 0),
813  CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT));
814  ReplaceNode(Node, SRLI);
815  return;
816  }
817  case ISD::SRA: {
818  // Optimize (sra (sext_inreg X, i16), C) ->
819  // (srai (slli X, (XLen-16), (XLen-16) + C)
820  // And (sra (sext_inreg X, i8), C) ->
821  // (srai (slli X, (XLen-8), (XLen-8) + C)
822  // This can occur when Zbb is enabled, which makes sext_inreg i16/i8 legal.
823  // This transform matches the code we get without Zbb. The shifts are more
824  // compressible, and this can help expose CSE opportunities in the sdiv by
825  // constant optimization.
826  auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
827  if (!N1C)
828  break;
829  SDValue N0 = Node->getOperand(0);
830  if (N0.getOpcode() != ISD::SIGN_EXTEND_INREG || !N0.hasOneUse())
831  break;
832  unsigned ShAmt = N1C->getZExtValue();
833  unsigned ExtSize =
834  cast<VTSDNode>(N0.getOperand(1))->getVT().getSizeInBits();
835  // ExtSize of 32 should use sraiw via tablegen pattern.
836  if (ExtSize >= 32 || ShAmt >= ExtSize)
837  break;
838  unsigned LShAmt = Subtarget->getXLen() - ExtSize;
839  SDNode *SLLI =
840  CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0),
841  CurDAG->getTargetConstant(LShAmt, DL, VT));
842  SDNode *SRAI = CurDAG->getMachineNode(
843  RISCV::SRAI, DL, VT, SDValue(SLLI, 0),
844  CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT));
845  ReplaceNode(Node, SRAI);
846  return;
847  }
848  case ISD::OR:
849  case ISD::XOR:
850  if (tryShrinkShlLogicImm(Node))
851  return;
852 
853  break;
854  case ISD::AND: {
855  auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
856  if (!N1C)
857  break;
858 
859  SDValue N0 = Node->getOperand(0);
860 
861  bool LeftShift = N0.getOpcode() == ISD::SHL;
862  if (LeftShift || N0.getOpcode() == ISD::SRL) {
863  auto *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
864  if (!C)
865  break;
866  unsigned C2 = C->getZExtValue();
867  unsigned XLen = Subtarget->getXLen();
868  assert((C2 > 0 && C2 < XLen) && "Unexpected shift amount!");
869 
870  uint64_t C1 = N1C->getZExtValue();
871 
872  // Keep track of whether this is a c.andi. If we can't use c.andi, the
873  // shift pair might offer more compression opportunities.
874  // TODO: We could check for C extension here, but we don't have many lit
875  // tests with the C extension enabled so not checking gets better
876  // coverage.
877  // TODO: What if ANDI faster than shift?
878  bool IsCANDI = isInt<6>(N1C->getSExtValue());
879 
880  // Clear irrelevant bits in the mask.
881  if (LeftShift)
882  C1 &= maskTrailingZeros<uint64_t>(C2);
883  else
884  C1 &= maskTrailingOnes<uint64_t>(XLen - C2);
885 
886  // Some transforms should only be done if the shift has a single use or
887  // the AND would become (srli (slli X, 32), 32)
888  bool OneUseOrZExtW = N0.hasOneUse() || C1 == UINT64_C(0xFFFFFFFF);
889 
890  SDValue X = N0.getOperand(0);
891 
892  // Turn (and (srl x, c2) c1) -> (srli (slli x, c3-c2), c3) if c1 is a mask
893  // with c3 leading zeros.
894  if (!LeftShift && isMask_64(C1)) {
895  unsigned Leading = XLen - (64 - countLeadingZeros(C1));
896  if (C2 < Leading) {
897  // If the number of leading zeros is C2+32 this can be SRLIW.
898  if (C2 + 32 == Leading) {
899  SDNode *SRLIW = CurDAG->getMachineNode(
900  RISCV::SRLIW, DL, VT, X, CurDAG->getTargetConstant(C2, DL, VT));
901  ReplaceNode(Node, SRLIW);
902  return;
903  }
904 
905  // (and (srl (sexti32 Y), c2), c1) -> (srliw (sraiw Y, 31), c3 - 32)
906  // if c1 is a mask with c3 leading zeros and c2 >= 32 and c3-c2==1.
907  //
908  // This pattern occurs when (i32 (srl (sra 31), c3 - 32)) is type
909  // legalized and goes through DAG combine.
910  if (C2 >= 32 && (Leading - C2) == 1 && N0.hasOneUse() &&
911  X.getOpcode() == ISD::SIGN_EXTEND_INREG &&
912  cast<VTSDNode>(X.getOperand(1))->getVT() == MVT::i32) {
913  SDNode *SRAIW =
914  CurDAG->getMachineNode(RISCV::SRAIW, DL, VT, X.getOperand(0),
915  CurDAG->getTargetConstant(31, DL, VT));
916  SDNode *SRLIW = CurDAG->getMachineNode(
917  RISCV::SRLIW, DL, VT, SDValue(SRAIW, 0),
918  CurDAG->getTargetConstant(Leading - 32, DL, VT));
919  ReplaceNode(Node, SRLIW);
920  return;
921  }
922 
923  // (srli (slli x, c3-c2), c3).
924  // Skip if we could use (zext.w (sraiw X, C2)).
925  bool Skip = Subtarget->hasStdExtZba() && Leading == 32 &&
926  X.getOpcode() == ISD::SIGN_EXTEND_INREG &&
927  cast<VTSDNode>(X.getOperand(1))->getVT() == MVT::i32;
928  // Also Skip if we can use bexti.
929  Skip |= Subtarget->hasStdExtZbs() && Leading == XLen - 1;
930  if (OneUseOrZExtW && !Skip) {
931  SDNode *SLLI = CurDAG->getMachineNode(
932  RISCV::SLLI, DL, VT, X,
933  CurDAG->getTargetConstant(Leading - C2, DL, VT));
934  SDNode *SRLI = CurDAG->getMachineNode(
935  RISCV::SRLI, DL, VT, SDValue(SLLI, 0),
936  CurDAG->getTargetConstant(Leading, DL, VT));
937  ReplaceNode(Node, SRLI);
938  return;
939  }
940  }
941  }
942 
943  // Turn (and (shl x, c2), c1) -> (srli (slli c2+c3), c3) if c1 is a mask
944  // shifted by c2 bits with c3 leading zeros.
945  if (LeftShift && isShiftedMask_64(C1)) {
946  unsigned Leading = XLen - (64 - countLeadingZeros(C1));
947 
948  if (C2 + Leading < XLen &&
949  C1 == (maskTrailingOnes<uint64_t>(XLen - (C2 + Leading)) << C2)) {
950  // Use slli.uw when possible.
951  if ((XLen - (C2 + Leading)) == 32 && Subtarget->hasStdExtZba()) {
952  SDNode *SLLI_UW =
953  CurDAG->getMachineNode(RISCV::SLLI_UW, DL, VT, X,
954  CurDAG->getTargetConstant(C2, DL, VT));
955  ReplaceNode(Node, SLLI_UW);
956  return;
957  }
958 
959  // (srli (slli c2+c3), c3)
960  if (OneUseOrZExtW && !IsCANDI) {
961  SDNode *SLLI = CurDAG->getMachineNode(
962  RISCV::SLLI, DL, VT, X,
963  CurDAG->getTargetConstant(C2 + Leading, DL, VT));
964  SDNode *SRLI = CurDAG->getMachineNode(
965  RISCV::SRLI, DL, VT, SDValue(SLLI, 0),
966  CurDAG->getTargetConstant(Leading, DL, VT));
967  ReplaceNode(Node, SRLI);
968  return;
969  }
970  }
971  }
972 
973  // Turn (and (shr x, c2), c1) -> (slli (srli x, c2+c3), c3) if c1 is a
974  // shifted mask with c2 leading zeros and c3 trailing zeros.
975  if (!LeftShift && isShiftedMask_64(C1)) {
976  unsigned Leading = XLen - (64 - countLeadingZeros(C1));
977  unsigned Trailing = countTrailingZeros(C1);
978  if (Leading == C2 && C2 + Trailing < XLen && OneUseOrZExtW &&
979  !IsCANDI) {
980  unsigned SrliOpc = RISCV::SRLI;
981  // If the input is zexti32 we should use SRLIW.
982  if (X.getOpcode() == ISD::AND &&
983  isa<ConstantSDNode>(X.getOperand(1)) &&
984  X.getConstantOperandVal(1) == UINT64_C(0xFFFFFFFF)) {
985  SrliOpc = RISCV::SRLIW;
986  X = X.getOperand(0);
987  }
988  SDNode *SRLI = CurDAG->getMachineNode(
989  SrliOpc, DL, VT, X,
990  CurDAG->getTargetConstant(C2 + Trailing, DL, VT));
991  SDNode *SLLI = CurDAG->getMachineNode(
992  RISCV::SLLI, DL, VT, SDValue(SRLI, 0),
993  CurDAG->getTargetConstant(Trailing, DL, VT));
994  ReplaceNode(Node, SLLI);
995  return;
996  }
997  // If the leading zero count is C2+32, we can use SRLIW instead of SRLI.
998  if (Leading > 32 && (Leading - 32) == C2 && C2 + Trailing < 32 &&
999  OneUseOrZExtW && !IsCANDI) {
1000  SDNode *SRLIW = CurDAG->getMachineNode(
1001  RISCV::SRLIW, DL, VT, X,
1002  CurDAG->getTargetConstant(C2 + Trailing, DL, VT));
1003  SDNode *SLLI = CurDAG->getMachineNode(
1004  RISCV::SLLI, DL, VT, SDValue(SRLIW, 0),
1005  CurDAG->getTargetConstant(Trailing, DL, VT));
1006  ReplaceNode(Node, SLLI);
1007  return;
1008  }
1009  }
1010 
1011  // Turn (and (shl x, c2), c1) -> (slli (srli x, c3-c2), c3) if c1 is a
1012  // shifted mask with no leading zeros and c3 trailing zeros.
1013  if (LeftShift && isShiftedMask_64(C1)) {
1014  unsigned Leading = XLen - (64 - countLeadingZeros(C1));
1015  unsigned Trailing = countTrailingZeros(C1);
1016  if (Leading == 0 && C2 < Trailing && OneUseOrZExtW && !IsCANDI) {
1017  SDNode *SRLI = CurDAG->getMachineNode(
1018  RISCV::SRLI, DL, VT, X,
1019  CurDAG->getTargetConstant(Trailing - C2, DL, VT));
1020  SDNode *SLLI = CurDAG->getMachineNode(
1021  RISCV::SLLI, DL, VT, SDValue(SRLI, 0),
1022  CurDAG->getTargetConstant(Trailing, DL, VT));
1023  ReplaceNode(Node, SLLI);
1024  return;
1025  }
1026  // If we have (32-C2) leading zeros, we can use SRLIW instead of SRLI.
1027  if (C2 < Trailing && Leading + C2 == 32 && OneUseOrZExtW && !IsCANDI) {
1028  SDNode *SRLIW = CurDAG->getMachineNode(
1029  RISCV::SRLIW, DL, VT, X,
1030  CurDAG->getTargetConstant(Trailing - C2, DL, VT));
1031  SDNode *SLLI = CurDAG->getMachineNode(
1032  RISCV::SLLI, DL, VT, SDValue(SRLIW, 0),
1033  CurDAG->getTargetConstant(Trailing, DL, VT));
1034  ReplaceNode(Node, SLLI);
1035  return;
1036  }
1037  }
1038  }
1039 
1040  if (tryShrinkShlLogicImm(Node))
1041  return;
1042 
1043  break;
1044  }
1045  case ISD::MUL: {
1046  // Special case for calculating (mul (and X, C2), C1) where the full product
1047  // fits in XLen bits. We can shift X left by the number of leading zeros in
1048  // C2 and shift C1 left by XLen-lzcnt(C2). This will ensure the final
1049  // product has XLen trailing zeros, putting it in the output of MULHU. This
1050  // can avoid materializing a constant in a register for C2.
1051 
1052  // RHS should be a constant.
1053  auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1054  if (!N1C || !N1C->hasOneUse())
1055  break;
1056 
1057  // LHS should be an AND with constant.
1058  SDValue N0 = Node->getOperand(0);
1059  if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(N0.getOperand(1)))
1060  break;
1061 
1062  uint64_t C2 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue();
1063 
1064  // Constant should be a mask.
1065  if (!isMask_64(C2))
1066  break;
1067 
1068  // If this can be an ANDI, ZEXT.H or ZEXT.W, don't do this if the ANDI/ZEXT
1069  // has multiple users or the constant is a simm12. This prevents inserting
1070  // a shift and still have uses of the AND/ZEXT. Shifting a simm12 will
1071  // likely make it more costly to materialize. Otherwise, using a SLLI
1072  // might allow it to be compressed.
1073  bool IsANDIOrZExt =
1074  isInt<12>(C2) ||
1075  (C2 == UINT64_C(0xFFFF) && Subtarget->hasStdExtZbb()) ||
1076  (C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasStdExtZba());
1077  if (IsANDIOrZExt && (isInt<12>(N1C->getSExtValue()) || !N0.hasOneUse()))
1078  break;
1079 
1080  // We need to shift left the AND input and C1 by a total of XLen bits.
1081 
1082  // How far left do we need to shift the AND input?
1083  unsigned XLen = Subtarget->getXLen();
1084  unsigned LeadingZeros = XLen - (64 - countLeadingZeros(C2));
1085 
1086  // The constant gets shifted by the remaining amount unless that would
1087  // shift bits out.
1088  uint64_t C1 = N1C->getZExtValue();
1089  unsigned ConstantShift = XLen - LeadingZeros;
1090  if (ConstantShift > (XLen - (64 - countLeadingZeros(C1))))
1091  break;
1092 
1093  uint64_t ShiftedC1 = C1 << ConstantShift;
1094  // If this RV32, we need to sign extend the constant.
1095  if (XLen == 32)
1096  ShiftedC1 = SignExtend64<32>(ShiftedC1);
1097 
1098  // Create (mulhu (slli X, lzcnt(C2)), C1 << (XLen - lzcnt(C2))).
1099  SDNode *Imm = selectImm(CurDAG, DL, VT, ShiftedC1, *Subtarget);
1100  SDNode *SLLI =
1101  CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0.getOperand(0),
1102  CurDAG->getTargetConstant(LeadingZeros, DL, VT));
1103  SDNode *MULHU = CurDAG->getMachineNode(RISCV::MULHU, DL, VT,
1104  SDValue(SLLI, 0), SDValue(Imm, 0));
1105  ReplaceNode(Node, MULHU);
1106  return;
1107  }
1108  case ISD::INTRINSIC_WO_CHAIN: {
1109  unsigned IntNo = Node->getConstantOperandVal(0);
1110  switch (IntNo) {
1111  // By default we do not custom select any intrinsic.
1112  default:
1113  break;
1114  case Intrinsic::riscv_vmsgeu:
1115  case Intrinsic::riscv_vmsge: {
1116  SDValue Src1 = Node->getOperand(1);
1117  SDValue Src2 = Node->getOperand(2);
1118  bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu;
1119  bool IsCmpUnsignedZero = false;
1120  // Only custom select scalar second operand.
1121  if (Src2.getValueType() != XLenVT)
1122  break;
1123  // Small constants are handled with patterns.
1124  if (auto *C = dyn_cast<ConstantSDNode>(Src2)) {
1125  int64_t CVal = C->getSExtValue();
1126  if (CVal >= -15 && CVal <= 16) {
1127  if (!IsUnsigned || CVal != 0)
1128  break;
1129  IsCmpUnsignedZero = true;
1130  }
1131  }
1132  MVT Src1VT = Src1.getSimpleValueType();
1133  unsigned VMSLTOpcode, VMNANDOpcode, VMSetOpcode;
1134  switch (RISCVTargetLowering::getLMUL(Src1VT)) {
1135  default:
1136  llvm_unreachable("Unexpected LMUL!");
1137 #define CASE_VMSLT_VMNAND_VMSET_OPCODES(lmulenum, suffix, suffix_b) \
1138  case RISCVII::VLMUL::lmulenum: \
1139  VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \
1140  : RISCV::PseudoVMSLT_VX_##suffix; \
1141  VMNANDOpcode = RISCV::PseudoVMNAND_MM_##suffix; \
1142  VMSetOpcode = RISCV::PseudoVMSET_M_##suffix_b; \
1143  break;
1144  CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F8, MF8, B1)
1145  CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F4, MF4, B2)
1146  CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F2, MF2, B4)
1147  CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_1, M1, B8)
1148  CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_2, M2, B16)
1149  CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_4, M4, B32)
1150  CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_8, M8, B64)
1151 #undef CASE_VMSLT_VMNAND_VMSET_OPCODES
1152  }
1153  SDValue SEW = CurDAG->getTargetConstant(
1154  Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT);
1155  SDValue VL;
1156  selectVLOp(Node->getOperand(3), VL);
1157 
1158  // If vmsgeu with 0 immediate, expand it to vmset.
1159  if (IsCmpUnsignedZero) {
1160  ReplaceNode(Node, CurDAG->getMachineNode(VMSetOpcode, DL, VT, VL, SEW));
1161  return;
1162  }
1163 
1164  // Expand to
1165  // vmslt{u}.vx vd, va, x; vmnand.mm vd, vd, vd
1166  SDValue Cmp = SDValue(
1167  CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}),
1168  0);
1169  ReplaceNode(Node, CurDAG->getMachineNode(VMNANDOpcode, DL, VT,
1170  {Cmp, Cmp, VL, SEW}));
1171  return;
1172  }
1173  case Intrinsic::riscv_vmsgeu_mask:
1174  case Intrinsic::riscv_vmsge_mask: {
1175  SDValue Src1 = Node->getOperand(2);
1176  SDValue Src2 = Node->getOperand(3);
1177  bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu_mask;
1178  bool IsCmpUnsignedZero = false;
1179  // Only custom select scalar second operand.
1180  if (Src2.getValueType() != XLenVT)
1181  break;
1182  // Small constants are handled with patterns.
1183  if (auto *C = dyn_cast<ConstantSDNode>(Src2)) {
1184  int64_t CVal = C->getSExtValue();
1185  if (CVal >= -15 && CVal <= 16) {
1186  if (!IsUnsigned || CVal != 0)
1187  break;
1188  IsCmpUnsignedZero = true;
1189  }
1190  }
1191  MVT Src1VT = Src1.getSimpleValueType();
1192  unsigned VMSLTOpcode, VMSLTMaskOpcode, VMXOROpcode, VMANDNOpcode,
1193  VMOROpcode;
1194  switch (RISCVTargetLowering::getLMUL(Src1VT)) {
1195  default:
1196  llvm_unreachable("Unexpected LMUL!");
1197 #define CASE_VMSLT_OPCODES(lmulenum, suffix, suffix_b) \
1198  case RISCVII::VLMUL::lmulenum: \
1199  VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \
1200  : RISCV::PseudoVMSLT_VX_##suffix; \
1201  VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix##_MASK \
1202  : RISCV::PseudoVMSLT_VX_##suffix##_MASK; \
1203  break;
1204  CASE_VMSLT_OPCODES(LMUL_F8, MF8, B1)
1205  CASE_VMSLT_OPCODES(LMUL_F4, MF4, B2)
1206  CASE_VMSLT_OPCODES(LMUL_F2, MF2, B4)
1207  CASE_VMSLT_OPCODES(LMUL_1, M1, B8)
1208  CASE_VMSLT_OPCODES(LMUL_2, M2, B16)
1209  CASE_VMSLT_OPCODES(LMUL_4, M4, B32)
1210  CASE_VMSLT_OPCODES(LMUL_8, M8, B64)
1211 #undef CASE_VMSLT_OPCODES
1212  }
1213  // Mask operations use the LMUL from the mask type.
1214  switch (RISCVTargetLowering::getLMUL(VT)) {
1215  default:
1216  llvm_unreachable("Unexpected LMUL!");
1217 #define CASE_VMXOR_VMANDN_VMOR_OPCODES(lmulenum, suffix) \
1218  case RISCVII::VLMUL::lmulenum: \
1219  VMXOROpcode = RISCV::PseudoVMXOR_MM_##suffix; \
1220  VMANDNOpcode = RISCV::PseudoVMANDN_MM_##suffix; \
1221  VMOROpcode = RISCV::PseudoVMOR_MM_##suffix; \
1222  break;
1223  CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F8, MF8)
1224  CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F4, MF4)
1225  CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F2, MF2)
1226  CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_1, M1)
1227  CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_2, M2)
1228  CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_4, M4)
1229  CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_8, M8)
1230 #undef CASE_VMXOR_VMANDN_VMOR_OPCODES
1231  }
1232  SDValue SEW = CurDAG->getTargetConstant(
1233  Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT);
1234  SDValue MaskSEW = CurDAG->getTargetConstant(0, DL, XLenVT);
1235  SDValue VL;
1236  selectVLOp(Node->getOperand(5), VL);
1237  SDValue MaskedOff = Node->getOperand(1);
1238  SDValue Mask = Node->getOperand(4);
1239 
1240  // If vmsgeu_mask with 0 immediate, expand it to vmor mask, maskedoff.
1241  if (IsCmpUnsignedZero) {
1242  // We don't need vmor if the MaskedOff and the Mask are the same
1243  // value.
1244  if (Mask == MaskedOff) {
1245  ReplaceUses(Node, Mask.getNode());
1246  return;
1247  }
1248  ReplaceNode(Node,
1249  CurDAG->getMachineNode(VMOROpcode, DL, VT,
1250  {Mask, MaskedOff, VL, MaskSEW}));
1251  return;
1252  }
1253 
1254  // If the MaskedOff value and the Mask are the same value use
1255  // vmslt{u}.vx vt, va, x; vmandn.mm vd, vd, vt
1256  // This avoids needing to copy v0 to vd before starting the next sequence.
1257  if (Mask == MaskedOff) {
1258  SDValue Cmp = SDValue(
1259  CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}),
1260  0);
1261  ReplaceNode(Node, CurDAG->getMachineNode(VMANDNOpcode, DL, VT,
1262  {Mask, Cmp, VL, MaskSEW}));
1263  return;
1264  }
1265 
1266  // Mask needs to be copied to V0.
1267  SDValue Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL,
1268  RISCV::V0, Mask, SDValue());
1269  SDValue Glue = Chain.getValue(1);
1270  SDValue V0 = CurDAG->getRegister(RISCV::V0, VT);
1271 
1272  // Otherwise use
1273  // vmslt{u}.vx vd, va, x, v0.t; vmxor.mm vd, vd, v0
1274  // The result is mask undisturbed.
1275  // We use the same instructions to emulate mask agnostic behavior, because
1276  // the agnostic result can be either undisturbed or all 1.
1277  SDValue Cmp = SDValue(
1278  CurDAG->getMachineNode(VMSLTMaskOpcode, DL, VT,
1279  {MaskedOff, Src1, Src2, V0, VL, SEW, Glue}),
1280  0);
1281  // vmxor.mm vd, vd, v0 is used to update active value.
1282  ReplaceNode(Node, CurDAG->getMachineNode(VMXOROpcode, DL, VT,
1283  {Cmp, Mask, VL, MaskSEW}));
1284  return;
1285  }
1286  case Intrinsic::riscv_vsetvli_opt:
1287  case Intrinsic::riscv_vsetvlimax_opt:
1288  return selectVSETVLI(Node);
1289  }
1290  break;
1291  }
1292  case ISD::INTRINSIC_W_CHAIN: {
1293  unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
1294  switch (IntNo) {
1295  // By default we do not custom select any intrinsic.
1296  default:
1297  break;
1298  case Intrinsic::riscv_vsetvli:
1299  case Intrinsic::riscv_vsetvlimax:
1300  return selectVSETVLI(Node);
1301  case Intrinsic::riscv_vlseg2:
1302  case Intrinsic::riscv_vlseg3:
1303  case Intrinsic::riscv_vlseg4:
1304  case Intrinsic::riscv_vlseg5:
1305  case Intrinsic::riscv_vlseg6:
1306  case Intrinsic::riscv_vlseg7:
1307  case Intrinsic::riscv_vlseg8: {
1308  selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
1309  return;
1310  }
1311  case Intrinsic::riscv_vlseg2_mask:
1312  case Intrinsic::riscv_vlseg3_mask:
1313  case Intrinsic::riscv_vlseg4_mask:
1314  case Intrinsic::riscv_vlseg5_mask:
1315  case Intrinsic::riscv_vlseg6_mask:
1316  case Intrinsic::riscv_vlseg7_mask:
1317  case Intrinsic::riscv_vlseg8_mask: {
1318  selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
1319  return;
1320  }
1321  case Intrinsic::riscv_vlsseg2:
1322  case Intrinsic::riscv_vlsseg3:
1323  case Intrinsic::riscv_vlsseg4:
1324  case Intrinsic::riscv_vlsseg5:
1325  case Intrinsic::riscv_vlsseg6:
1326  case Intrinsic::riscv_vlsseg7:
1327  case Intrinsic::riscv_vlsseg8: {
1328  selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
1329  return;
1330  }
1331  case Intrinsic::riscv_vlsseg2_mask:
1332  case Intrinsic::riscv_vlsseg3_mask:
1333  case Intrinsic::riscv_vlsseg4_mask:
1334  case Intrinsic::riscv_vlsseg5_mask:
1335  case Intrinsic::riscv_vlsseg6_mask:
1336  case Intrinsic::riscv_vlsseg7_mask:
1337  case Intrinsic::riscv_vlsseg8_mask: {
1338  selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
1339  return;
1340  }
1341  case Intrinsic::riscv_vloxseg2:
1342  case Intrinsic::riscv_vloxseg3:
1343  case Intrinsic::riscv_vloxseg4:
1344  case Intrinsic::riscv_vloxseg5:
1345  case Intrinsic::riscv_vloxseg6:
1346  case Intrinsic::riscv_vloxseg7:
1347  case Intrinsic::riscv_vloxseg8:
1348  selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
1349  return;
1350  case Intrinsic::riscv_vluxseg2:
1351  case Intrinsic::riscv_vluxseg3:
1352  case Intrinsic::riscv_vluxseg4:
1353  case Intrinsic::riscv_vluxseg5:
1354  case Intrinsic::riscv_vluxseg6:
1355  case Intrinsic::riscv_vluxseg7:
1356  case Intrinsic::riscv_vluxseg8:
1357  selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
1358  return;
1359  case Intrinsic::riscv_vloxseg2_mask:
1360  case Intrinsic::riscv_vloxseg3_mask:
1361  case Intrinsic::riscv_vloxseg4_mask:
1362  case Intrinsic::riscv_vloxseg5_mask:
1363  case Intrinsic::riscv_vloxseg6_mask:
1364  case Intrinsic::riscv_vloxseg7_mask:
1365  case Intrinsic::riscv_vloxseg8_mask:
1366  selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
1367  return;
1368  case Intrinsic::riscv_vluxseg2_mask:
1369  case Intrinsic::riscv_vluxseg3_mask:
1370  case Intrinsic::riscv_vluxseg4_mask:
1371  case Intrinsic::riscv_vluxseg5_mask:
1372  case Intrinsic::riscv_vluxseg6_mask:
1373  case Intrinsic::riscv_vluxseg7_mask:
1374  case Intrinsic::riscv_vluxseg8_mask:
1375  selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
1376  return;
1377  case Intrinsic::riscv_vlseg8ff:
1378  case Intrinsic::riscv_vlseg7ff:
1379  case Intrinsic::riscv_vlseg6ff:
1380  case Intrinsic::riscv_vlseg5ff:
1381  case Intrinsic::riscv_vlseg4ff:
1382  case Intrinsic::riscv_vlseg3ff:
1383  case Intrinsic::riscv_vlseg2ff: {
1384  selectVLSEGFF(Node, /*IsMasked*/ false);
1385  return;
1386  }
1387  case Intrinsic::riscv_vlseg8ff_mask:
1388  case Intrinsic::riscv_vlseg7ff_mask:
1389  case Intrinsic::riscv_vlseg6ff_mask:
1390  case Intrinsic::riscv_vlseg5ff_mask:
1391  case Intrinsic::riscv_vlseg4ff_mask:
1392  case Intrinsic::riscv_vlseg3ff_mask:
1393  case Intrinsic::riscv_vlseg2ff_mask: {
1394  selectVLSEGFF(Node, /*IsMasked*/ true);
1395  return;
1396  }
1397  case Intrinsic::riscv_vloxei:
1398  case Intrinsic::riscv_vloxei_mask:
1399  case Intrinsic::riscv_vluxei:
1400  case Intrinsic::riscv_vluxei_mask: {
1401  bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask ||
1402  IntNo == Intrinsic::riscv_vluxei_mask;
1403  bool IsOrdered = IntNo == Intrinsic::riscv_vloxei ||
1404  IntNo == Intrinsic::riscv_vloxei_mask;
1405 
1406  MVT VT = Node->getSimpleValueType(0);
1407  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1408 
1409  unsigned CurOp = 2;
1410  // Masked intrinsic only have TU version pseduo instructions.
1411  bool IsTU = IsMasked || !Node->getOperand(CurOp).isUndef();
1412  SmallVector<SDValue, 8> Operands;
1413  if (IsTU)
1414  Operands.push_back(Node->getOperand(CurOp++));
1415  else
1416  // Skip the undef passthru operand for nomask TA version pseudo
1417  CurOp++;
1418 
1419  MVT IndexVT;
1420  addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
1421  /*IsStridedOrIndexed*/ true, Operands,
1422  /*IsLoad=*/true, &IndexVT);
1423 
1424  assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
1425  "Element count mismatch");
1426 
1427  RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
1428  RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
1429  unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
1430  if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
1431  report_fatal_error("The V extension does not support EEW=64 for index "
1432  "values when XLEN=32");
1433  }
1434  const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
1435  IsMasked, IsTU, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
1436  static_cast<unsigned>(IndexLMUL));
1437  MachineSDNode *Load =
1438  CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1439 
1440  if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1441  CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
1442 
1443  ReplaceNode(Node, Load);
1444  return;
1445  }
1446  case Intrinsic::riscv_vlm:
1447  case Intrinsic::riscv_vle:
1448  case Intrinsic::riscv_vle_mask:
1449  case Intrinsic::riscv_vlse:
1450  case Intrinsic::riscv_vlse_mask: {
1451  bool IsMasked = IntNo == Intrinsic::riscv_vle_mask ||
1452  IntNo == Intrinsic::riscv_vlse_mask;
1453  bool IsStrided =
1454  IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask;
1455 
1456  MVT VT = Node->getSimpleValueType(0);
1457  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1458 
1459  unsigned CurOp = 2;
1460  // The riscv_vlm intrinsic are always tail agnostic and no passthru operand.
1461  bool HasPassthruOperand = IntNo != Intrinsic::riscv_vlm;
1462  // Masked intrinsic only have TU version pseduo instructions.
1463  bool IsTU = HasPassthruOperand &&
1464  (IsMasked || !Node->getOperand(CurOp).isUndef());
1465  SmallVector<SDValue, 8> Operands;
1466  if (IsTU)
1467  Operands.push_back(Node->getOperand(CurOp++));
1468  else if (HasPassthruOperand)
1469  // Skip the undef passthru operand for nomask TA version pseudo
1470  CurOp++;
1471 
1472  addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
1473  Operands, /*IsLoad=*/true);
1474 
1475  RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
1476  const RISCV::VLEPseudo *P =
1477  RISCV::getVLEPseudo(IsMasked, IsTU, IsStrided, /*FF*/ false, Log2SEW,
1478  static_cast<unsigned>(LMUL));
1479  MachineSDNode *Load =
1480  CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1481 
1482  if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1483  CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
1484 
1485  ReplaceNode(Node, Load);
1486  return;
1487  }
1488  case Intrinsic::riscv_vleff:
1489  case Intrinsic::riscv_vleff_mask: {
1490  bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask;
1491 
1492  MVT VT = Node->getSimpleValueType(0);
1493  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1494 
1495  unsigned CurOp = 2;
1496  // Masked intrinsic only have TU version pseduo instructions.
1497  bool IsTU = IsMasked || !Node->getOperand(CurOp).isUndef();
1498  SmallVector<SDValue, 7> Operands;
1499  if (IsTU)
1500  Operands.push_back(Node->getOperand(CurOp++));
1501  else
1502  // Skip the undef passthru operand for nomask TA version pseudo
1503  CurOp++;
1504 
1505  addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
1506  /*IsStridedOrIndexed*/ false, Operands,
1507  /*IsLoad=*/true);
1508 
1509  RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
1510  const RISCV::VLEPseudo *P =
1511  RISCV::getVLEPseudo(IsMasked, IsTU, /*Strided*/ false, /*FF*/ true,
1512  Log2SEW, static_cast<unsigned>(LMUL));
1513  MachineSDNode *Load = CurDAG->getMachineNode(
1514  P->Pseudo, DL, Node->getVTList(), Operands);
1515  if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1516  CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
1517 
1518  ReplaceNode(Node, Load);
1519  return;
1520  }
1521  }
1522  break;
1523  }
1524  case ISD::INTRINSIC_VOID: {
1525  unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
1526  switch (IntNo) {
1527  case Intrinsic::riscv_vsseg2:
1528  case Intrinsic::riscv_vsseg3:
1529  case Intrinsic::riscv_vsseg4:
1530  case Intrinsic::riscv_vsseg5:
1531  case Intrinsic::riscv_vsseg6:
1532  case Intrinsic::riscv_vsseg7:
1533  case Intrinsic::riscv_vsseg8: {
1534  selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
1535  return;
1536  }
1537  case Intrinsic::riscv_vsseg2_mask:
1538  case Intrinsic::riscv_vsseg3_mask:
1539  case Intrinsic::riscv_vsseg4_mask:
1540  case Intrinsic::riscv_vsseg5_mask:
1541  case Intrinsic::riscv_vsseg6_mask:
1542  case Intrinsic::riscv_vsseg7_mask:
1543  case Intrinsic::riscv_vsseg8_mask: {
1544  selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
1545  return;
1546  }
1547  case Intrinsic::riscv_vssseg2:
1548  case Intrinsic::riscv_vssseg3:
1549  case Intrinsic::riscv_vssseg4:
1550  case Intrinsic::riscv_vssseg5:
1551  case Intrinsic::riscv_vssseg6:
1552  case Intrinsic::riscv_vssseg7:
1553  case Intrinsic::riscv_vssseg8: {
1554  selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
1555  return;
1556  }
1557  case Intrinsic::riscv_vssseg2_mask:
1558  case Intrinsic::riscv_vssseg3_mask:
1559  case Intrinsic::riscv_vssseg4_mask:
1560  case Intrinsic::riscv_vssseg5_mask:
1561  case Intrinsic::riscv_vssseg6_mask:
1562  case Intrinsic::riscv_vssseg7_mask:
1563  case Intrinsic::riscv_vssseg8_mask: {
1564  selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
1565  return;
1566  }
1567  case Intrinsic::riscv_vsoxseg2:
1568  case Intrinsic::riscv_vsoxseg3:
1569  case Intrinsic::riscv_vsoxseg4:
1570  case Intrinsic::riscv_vsoxseg5:
1571  case Intrinsic::riscv_vsoxseg6:
1572  case Intrinsic::riscv_vsoxseg7:
1573  case Intrinsic::riscv_vsoxseg8:
1574  selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
1575  return;
1576  case Intrinsic::riscv_vsuxseg2:
1577  case Intrinsic::riscv_vsuxseg3:
1578  case Intrinsic::riscv_vsuxseg4:
1579  case Intrinsic::riscv_vsuxseg5:
1580  case Intrinsic::riscv_vsuxseg6:
1581  case Intrinsic::riscv_vsuxseg7:
1582  case Intrinsic::riscv_vsuxseg8:
1583  selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
1584  return;
1585  case Intrinsic::riscv_vsoxseg2_mask:
1586  case Intrinsic::riscv_vsoxseg3_mask:
1587  case Intrinsic::riscv_vsoxseg4_mask:
1588  case Intrinsic::riscv_vsoxseg5_mask:
1589  case Intrinsic::riscv_vsoxseg6_mask:
1590  case Intrinsic::riscv_vsoxseg7_mask:
1591  case Intrinsic::riscv_vsoxseg8_mask:
1592  selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
1593  return;
1594  case Intrinsic::riscv_vsuxseg2_mask:
1595  case Intrinsic::riscv_vsuxseg3_mask:
1596  case Intrinsic::riscv_vsuxseg4_mask:
1597  case Intrinsic::riscv_vsuxseg5_mask:
1598  case Intrinsic::riscv_vsuxseg6_mask:
1599  case Intrinsic::riscv_vsuxseg7_mask:
1600  case Intrinsic::riscv_vsuxseg8_mask:
1601  selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
1602  return;
1603  case Intrinsic::riscv_vsoxei:
1604  case Intrinsic::riscv_vsoxei_mask:
1605  case Intrinsic::riscv_vsuxei:
1606  case Intrinsic::riscv_vsuxei_mask: {
1607  bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask ||
1608  IntNo == Intrinsic::riscv_vsuxei_mask;
1609  bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei ||
1610  IntNo == Intrinsic::riscv_vsoxei_mask;
1611 
1612  MVT VT = Node->getOperand(2)->getSimpleValueType(0);
1613  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1614 
1615  unsigned CurOp = 2;
1616  SmallVector<SDValue, 8> Operands;
1617  Operands.push_back(Node->getOperand(CurOp++)); // Store value.
1618 
1619  MVT IndexVT;
1620  addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
1621  /*IsStridedOrIndexed*/ true, Operands,
1622  /*IsLoad=*/false, &IndexVT);
1623 
1624  assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
1625  "Element count mismatch");
1626 
1627  RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
1628  RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
1629  unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
1630  if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
1631  report_fatal_error("The V extension does not support EEW=64 for index "
1632  "values when XLEN=32");
1633  }
1634  const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
1635  IsMasked, /*TU*/ false, IsOrdered, IndexLog2EEW,
1636  static_cast<unsigned>(LMUL), static_cast<unsigned>(IndexLMUL));
1637  MachineSDNode *Store =
1638  CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1639 
1640  if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1641  CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
1642 
1643  ReplaceNode(Node, Store);
1644  return;
1645  }
1646  case Intrinsic::riscv_vsm:
1647  case Intrinsic::riscv_vse:
1648  case Intrinsic::riscv_vse_mask:
1649  case Intrinsic::riscv_vsse:
1650  case Intrinsic::riscv_vsse_mask: {
1651  bool IsMasked = IntNo == Intrinsic::riscv_vse_mask ||
1652  IntNo == Intrinsic::riscv_vsse_mask;
1653  bool IsStrided =
1654  IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask;
1655 
1656  MVT VT = Node->getOperand(2)->getSimpleValueType(0);
1657  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1658 
1659  unsigned CurOp = 2;
1660  SmallVector<SDValue, 8> Operands;
1661  Operands.push_back(Node->getOperand(CurOp++)); // Store value.
1662 
1663  addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
1664  Operands);
1665 
1666  RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
1667  const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
1668  IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL));
1669  MachineSDNode *Store =
1670  CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1671  if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1672  CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
1673 
1674  ReplaceNode(Node, Store);
1675  return;
1676  }
1677  }
1678  break;
1679  }
1680  case ISD::BITCAST: {
1681  MVT SrcVT = Node->getOperand(0).getSimpleValueType();
1682  // Just drop bitcasts between vectors if both are fixed or both are
1683  // scalable.
1684  if ((VT.isScalableVector() && SrcVT.isScalableVector()) ||
1685  (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) {
1686  ReplaceUses(SDValue(Node, 0), Node->getOperand(0));
1687  CurDAG->RemoveDeadNode(Node);
1688  return;
1689  }
1690  break;
1691  }
1692  case ISD::INSERT_SUBVECTOR: {
1693  SDValue V = Node->getOperand(0);
1694  SDValue SubV = Node->getOperand(1);
1695  SDLoc DL(SubV);
1696  auto Idx = Node->getConstantOperandVal(2);
1697  MVT SubVecVT = SubV.getSimpleValueType();
1698 
1699  const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
1700  MVT SubVecContainerVT = SubVecVT;
1701  // Establish the correct scalable-vector types for any fixed-length type.
1702  if (SubVecVT.isFixedLengthVector())
1703  SubVecContainerVT = TLI.getContainerForFixedLengthVector(SubVecVT);
1704  if (VT.isFixedLengthVector())
1705  VT = TLI.getContainerForFixedLengthVector(VT);
1706 
1707  const auto *TRI = Subtarget->getRegisterInfo();
1708  unsigned SubRegIdx;
1709  std::tie(SubRegIdx, Idx) =
1710  RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1711  VT, SubVecContainerVT, Idx, TRI);
1712 
1713  // If the Idx hasn't been completely eliminated then this is a subvector
1714  // insert which doesn't naturally align to a vector register. These must
1715  // be handled using instructions to manipulate the vector registers.
1716  if (Idx != 0)
1717  break;
1718 
1719  RISCVII::VLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecContainerVT);
1720  bool IsSubVecPartReg = SubVecLMUL == RISCVII::VLMUL::LMUL_F2 ||
1721  SubVecLMUL == RISCVII::VLMUL::LMUL_F4 ||
1722  SubVecLMUL == RISCVII::VLMUL::LMUL_F8;
1723  (void)IsSubVecPartReg; // Silence unused variable warning without asserts.
1724  assert((!IsSubVecPartReg || V.isUndef()) &&
1725  "Expecting lowering to have created legal INSERT_SUBVECTORs when "
1726  "the subvector is smaller than a full-sized register");
1727 
1728  // If we haven't set a SubRegIdx, then we must be going between
1729  // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
1730  if (SubRegIdx == RISCV::NoSubRegister) {
1731  unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT);
1732  assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
1733  InRegClassID &&
1734  "Unexpected subvector extraction");
1735  SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
1736  SDNode *NewNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1737  DL, VT, SubV, RC);
1738  ReplaceNode(Node, NewNode);
1739  return;
1740  }
1741 
1742  SDValue Insert = CurDAG->getTargetInsertSubreg(SubRegIdx, DL, VT, V, SubV);
1743  ReplaceNode(Node, Insert.getNode());
1744  return;
1745  }
1746  case ISD::EXTRACT_SUBVECTOR: {
1747  SDValue V = Node->getOperand(0);
1748  auto Idx = Node->getConstantOperandVal(1);
1749  MVT InVT = V.getSimpleValueType();
1750  SDLoc DL(V);
1751 
1752  const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
1753  MVT SubVecContainerVT = VT;
1754  // Establish the correct scalable-vector types for any fixed-length type.
1755  if (VT.isFixedLengthVector())
1756  SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT);
1757  if (InVT.isFixedLengthVector())
1758  InVT = TLI.getContainerForFixedLengthVector(InVT);
1759 
1760  const auto *TRI = Subtarget->getRegisterInfo();
1761  unsigned SubRegIdx;
1762  std::tie(SubRegIdx, Idx) =
1763  RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1764  InVT, SubVecContainerVT, Idx, TRI);
1765 
1766  // If the Idx hasn't been completely eliminated then this is a subvector
1767  // extract which doesn't naturally align to a vector register. These must
1768  // be handled using instructions to manipulate the vector registers.
1769  if (Idx != 0)
1770  break;
1771 
1772  // If we haven't set a SubRegIdx, then we must be going between
1773  // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy.
1774  if (SubRegIdx == RISCV::NoSubRegister) {
1775  unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(InVT);
1776  assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
1777  InRegClassID &&
1778  "Unexpected subvector extraction");
1779  SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
1780  SDNode *NewNode =
1781  CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, V, RC);
1782  ReplaceNode(Node, NewNode);
1783  return;
1784  }
1785 
1786  SDValue Extract = CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, V);
1787  ReplaceNode(Node, Extract.getNode());
1788  return;
1789  }
1790  case RISCVISD::VMV_S_X_VL:
1791  case RISCVISD::VFMV_S_F_VL:
1792  case RISCVISD::VMV_V_X_VL:
1793  case RISCVISD::VFMV_V_F_VL: {
1794  // Only if we have optimized zero-stride vector load.
1795  if (!Subtarget->hasOptimizedZeroStrideLoad())
1796  break;
1797 
1798  // Try to match splat of a scalar load to a strided load with stride of x0.
1799  bool IsScalarMove = Node->getOpcode() == RISCVISD::VMV_S_X_VL ||
1800  Node->getOpcode() == RISCVISD::VFMV_S_F_VL;
1801  if (!Node->getOperand(0).isUndef())
1802  break;
1803  SDValue Src = Node->getOperand(1);
1804  auto *Ld = dyn_cast<LoadSDNode>(Src);
1805  if (!Ld)
1806  break;
1807  EVT MemVT = Ld->getMemoryVT();
1808  // The memory VT should be the same size as the element type.
1809  if (MemVT.getStoreSize() != VT.getVectorElementType().getStoreSize())
1810  break;
1811  if (!IsProfitableToFold(Src, Node, Node) ||
1812  !IsLegalToFold(Src, Node, Node, TM.getOptLevel()))
1813  break;
1814 
1815  SDValue VL;
1816  if (IsScalarMove) {
1817  // We could deal with more VL if we update the VSETVLI insert pass to
1818  // avoid introducing more VSETVLI.
1819  if (!isOneConstant(Node->getOperand(2)))
1820  break;
1821  selectVLOp(Node->getOperand(2), VL);
1822  } else
1823  selectVLOp(Node->getOperand(2), VL);
1824 
1825  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1826  SDValue SEW = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT);
1827 
1828  SDValue Operands[] = {Ld->getBasePtr(),
1829  CurDAG->getRegister(RISCV::X0, XLenVT), VL, SEW,
1830  Ld->getChain()};
1831 
1832  RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
1833  const RISCV::VLEPseudo *P = RISCV::getVLEPseudo(
1834  /*IsMasked*/ false, /*IsTU*/ false, /*IsStrided*/ true, /*FF*/ false,
1835  Log2SEW, static_cast<unsigned>(LMUL));
1836  MachineSDNode *Load =
1837  CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1838 
1839  CurDAG->setNodeMemRefs(Load, {Ld->getMemOperand()});
1840 
1841  ReplaceNode(Node, Load);
1842  return;
1843  }
1844  }
1845 
1846  // Select the default instruction.
1847  SelectCode(Node);
1848 }
1849 
1850 bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand(
1851  const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
1852  switch (ConstraintID) {
1853  case InlineAsm::Constraint_m:
1854  // We just support simple memory operands that have a single address
1855  // operand and need no special handling.
1856  OutOps.push_back(Op);
1857  return false;
1858  case InlineAsm::Constraint_A:
1859  OutOps.push_back(Op);
1860  return false;
1861  default:
1862  break;
1863  }
1864 
1865  return true;
1866 }
1867 
1868 bool RISCVDAGToDAGISel::SelectAddrFrameIndex(SDValue Addr, SDValue &Base,
1869  SDValue &Offset) {
1870  if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
1871  Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
1872  Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), Subtarget->getXLenVT());
1873  return true;
1874  }
1875 
1876  return false;
1877 }
1878 
1879 // Select a frame index and an optional immediate offset from an ADD or OR.
1880 bool RISCVDAGToDAGISel::SelectFrameAddrRegImm(SDValue Addr, SDValue &Base,
1881  SDValue &Offset) {
1882  if (SelectAddrFrameIndex(Addr, Base, Offset))
1883  return true;
1884 
1885  if (!CurDAG->isBaseWithConstantOffset(Addr))
1886  return false;
1887 
1888  if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr.getOperand(0))) {
1889  int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
1890  if (isInt<12>(CVal)) {
1891  Base = CurDAG->getTargetFrameIndex(FIN->getIndex(),
1892  Subtarget->getXLenVT());
1893  Offset = CurDAG->getTargetConstant(CVal, SDLoc(Addr),
1894  Subtarget->getXLenVT());
1895  return true;
1896  }
1897  }
1898 
1899  return false;
1900 }
1901 
1902 // Fold constant addresses.
1903 static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL,
1904  const MVT VT, const RISCVSubtarget *Subtarget,
1905  SDValue Addr, SDValue &Base, SDValue &Offset) {
1906  if (!isa<ConstantSDNode>(Addr))
1907  return false;
1908 
1909  int64_t CVal = cast<ConstantSDNode>(Addr)->getSExtValue();
1910 
1911  // If the constant is a simm12, we can fold the whole constant and use X0 as
1912  // the base. If the constant can be materialized with LUI+simm12, use LUI as
1913  // the base. We can't use generateInstSeq because it favors LUI+ADDIW.
1914  int64_t Lo12 = SignExtend64<12>(CVal);
1915  int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12;
1916  if (!Subtarget->is64Bit() || isInt<32>(Hi)) {
1917  if (Hi) {
1918  int64_t Hi20 = (Hi >> 12) & 0xfffff;
1919  Base = SDValue(
1920  CurDAG->getMachineNode(RISCV::LUI, DL, VT,
1921  CurDAG->getTargetConstant(Hi20, DL, VT)),
1922  0);
1923  } else {
1924  Base = CurDAG->getRegister(RISCV::X0, VT);
1925  }
1926  Offset = CurDAG->getTargetConstant(Lo12, DL, VT);
1927  return true;
1928  }
1929 
1930  // Ask how constant materialization would handle this constant.
1931  RISCVMatInt::InstSeq Seq =
1932  RISCVMatInt::generateInstSeq(CVal, Subtarget->getFeatureBits());
1933 
1934  // If the last instruction would be an ADDI, we can fold its immediate and
1935  // emit the rest of the sequence as the base.
1936  if (Seq.back().Opc != RISCV::ADDI)
1937  return false;
1938  Lo12 = Seq.back().Imm;
1939 
1940  // Drop the last instruction.
1941  Seq.pop_back();
1942  assert(!Seq.empty() && "Expected more instructions in sequence");
1943 
1944  Base = SDValue(selectImmSeq(CurDAG, DL, VT, Seq), 0);
1945  Offset = CurDAG->getTargetConstant(Lo12, DL, VT);
1946  return true;
1947 }
1948 
1949 // Is this ADD instruction only used as the base pointer of scalar loads and
1950 // stores?
1951 static bool isWorthFoldingAdd(SDValue Add) {
1952  for (auto *Use : Add->uses()) {
1953  if (Use->getOpcode() != ISD::LOAD && Use->getOpcode() != ISD::STORE &&
1954  Use->getOpcode() != ISD::ATOMIC_LOAD &&
1955  Use->getOpcode() != ISD::ATOMIC_STORE)
1956  return false;
1957  EVT VT = cast<MemSDNode>(Use)->getMemoryVT();
1958  if (!VT.isScalarInteger() && VT != MVT::f16 && VT != MVT::f32 &&
1959  VT != MVT::f64)
1960  return false;
1961  // Don't allow stores of the value. It must be used as the address.
1962  if (Use->getOpcode() == ISD::STORE &&
1963  cast<StoreSDNode>(Use)->getValue() == Add)
1964  return false;
1965  if (Use->getOpcode() == ISD::ATOMIC_STORE &&
1966  cast<AtomicSDNode>(Use)->getVal() == Add)
1967  return false;
1968  }
1969 
1970  return true;
1971 }
1972 
1973 bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base,
1974  SDValue &Offset) {
1975  if (SelectAddrFrameIndex(Addr, Base, Offset))
1976  return true;
1977 
1978  SDLoc DL(Addr);
1979  MVT VT = Addr.getSimpleValueType();
1980 
1981  if (Addr.getOpcode() == RISCVISD::ADD_LO) {
1982  Base = Addr.getOperand(0);
1983  Offset = Addr.getOperand(1);
1984  return true;
1985  }
1986 
1987  if (CurDAG->isBaseWithConstantOffset(Addr)) {
1988  int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
1989  if (isInt<12>(CVal)) {
1990  Base = Addr.getOperand(0);
1991  if (Base.getOpcode() == RISCVISD::ADD_LO) {
1992  SDValue LoOperand = Base.getOperand(1);
1993  if (auto *GA = dyn_cast<GlobalAddressSDNode>(LoOperand)) {
1994  // If the Lo in (ADD_LO hi, lo) is a global variable's address
1995  // (its low part, really), then we can rely on the alignment of that
1996  // variable to provide a margin of safety before low part can overflow
1997  // the 12 bits of the load/store offset. Check if CVal falls within
1998  // that margin; if so (low part + CVal) can't overflow.
1999  const DataLayout &DL = CurDAG->getDataLayout();
2000  Align Alignment = commonAlignment(
2001  GA->getGlobal()->getPointerAlignment(DL), GA->getOffset());
2002  if (CVal == 0 || Alignment > CVal) {
2003  int64_t CombinedOffset = CVal + GA->getOffset();
2004  Base = Base.getOperand(0);
2005  Offset = CurDAG->getTargetGlobalAddress(
2006  GA->getGlobal(), SDLoc(LoOperand), LoOperand.getValueType(),
2007  CombinedOffset, GA->getTargetFlags());
2008  return true;
2009  }
2010  }
2011  }
2012 
2013  if (auto *FIN = dyn_cast<FrameIndexSDNode>(Base))
2014  Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), VT);
2015  Offset = CurDAG->getTargetConstant(CVal, DL, VT);
2016  return true;
2017  }
2018  }
2019 
2020  // Handle ADD with large immediates.
2021  if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Addr.getOperand(1))) {
2022  int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
2023  assert(!isInt<12>(CVal) && "simm12 not already handled?");
2024 
2025  // Handle immediates in the range [-4096,-2049] or [2048, 4094]. We can use
2026  // an ADDI for part of the offset and fold the rest into the load/store.
2027  // This mirrors the AddiPair PatFrag in RISCVInstrInfo.td.
2028  if (isInt<12>(CVal / 2) && isInt<12>(CVal - CVal / 2)) {
2029  int64_t Adj = CVal < 0 ? -2048 : 2047;
2030  Base = SDValue(
2031  CurDAG->getMachineNode(RISCV::ADDI, DL, VT, Addr.getOperand(0),
2032  CurDAG->getTargetConstant(Adj, DL, VT)),
2033  0);
2034  Offset = CurDAG->getTargetConstant(CVal - Adj, DL, VT);
2035  return true;
2036  }
2037 
2038  // For larger immediates, we might be able to save one instruction from
2039  // constant materialization by folding the Lo12 bits of the immediate into
2040  // the address. We should only do this if the ADD is only used by loads and
2041  // stores that can fold the lo12 bits. Otherwise, the ADD will get iseled
2042  // separately with the full materialized immediate creating extra
2043  // instructions.
2044  if (isWorthFoldingAdd(Addr) &&
2045  selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr.getOperand(1), Base,
2046  Offset)) {
2047  // Insert an ADD instruction with the materialized Hi52 bits.
2048  Base = SDValue(
2049  CurDAG->getMachineNode(RISCV::ADD, DL, VT, Addr.getOperand(0), Base),
2050  0);
2051  return true;
2052  }
2053  }
2054 
2055  if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr, Base, Offset))
2056  return true;
2057 
2058  Base = Addr;
2059  Offset = CurDAG->getTargetConstant(0, DL, VT);
2060  return true;
2061 }
2062 
2063 bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth,
2064  SDValue &ShAmt) {
2065  ShAmt = N;
2066 
2067  // Shift instructions on RISCV only read the lower 5 or 6 bits of the shift
2068  // amount. If there is an AND on the shift amount, we can bypass it if it
2069  // doesn't affect any of those bits.
2070  if (ShAmt.getOpcode() == ISD::AND && isa<ConstantSDNode>(ShAmt.getOperand(1))) {
2071  const APInt &AndMask = ShAmt.getConstantOperandAPInt(1);
2072 
2073  // Since the max shift amount is a power of 2 we can subtract 1 to make a
2074  // mask that covers the bits needed to represent all shift amounts.
2075  assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
2076  APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
2077 
2078  if (ShMask.isSubsetOf(AndMask)) {
2079  ShAmt = ShAmt.getOperand(0);
2080  } else {
2081  // SimplifyDemandedBits may have optimized the mask so try restoring any
2082  // bits that are known zero.
2083  KnownBits Known = CurDAG->computeKnownBits(ShAmt.getOperand(0));
2084  if (!ShMask.isSubsetOf(AndMask | Known.Zero))
2085  return true;
2086  ShAmt = ShAmt.getOperand(0);
2087  }
2088  }
2089 
2090  if (ShAmt.getOpcode() == ISD::SUB &&
2091  isa<ConstantSDNode>(ShAmt.getOperand(0))) {
2092  uint64_t Imm = ShAmt.getConstantOperandVal(0);
2093  // If we are shifting by N-X where N == 0 mod Size, then just shift by -X to
2094  // generate a NEG instead of a SUB of a constant.
2095  if (Imm != 0 && Imm % ShiftWidth == 0) {
2096  SDLoc DL(ShAmt);
2097  EVT VT = ShAmt.getValueType();
2098  SDValue Zero = CurDAG->getRegister(RISCV::X0, VT);
2099  unsigned NegOpc = VT == MVT::i64 ? RISCV::SUBW : RISCV::SUB;
2100  MachineSDNode *Neg = CurDAG->getMachineNode(NegOpc, DL, VT, Zero,
2101  ShAmt.getOperand(1));
2102  ShAmt = SDValue(Neg, 0);
2103  return true;
2104  }
2105  }
2106 
2107  return true;
2108 }
2109 
2110 bool RISCVDAGToDAGISel::selectSExti32(SDValue N, SDValue &Val) {
2111  if (N.getOpcode() == ISD::SIGN_EXTEND_INREG &&
2112  cast<VTSDNode>(N.getOperand(1))->getVT() == MVT::i32) {
2113  Val = N.getOperand(0);
2114  return true;
2115  }
2116  MVT VT = N.getSimpleValueType();
2117  if (CurDAG->ComputeNumSignBits(N) > (VT.getSizeInBits() - 32)) {
2118  Val = N;
2119  return true;
2120  }
2121 
2122  return false;
2123 }
2124 
2125 bool RISCVDAGToDAGISel::selectZExtBits(SDValue N, unsigned Bits, SDValue &Val) {
2126  if (N.getOpcode() == ISD::AND) {
2127  auto *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
2128  if (C && C->getZExtValue() == maskTrailingOnes<uint64_t>(Bits)) {
2129  Val = N.getOperand(0);
2130  return true;
2131  }
2132  }
2133  MVT VT = N.getSimpleValueType();
2134  APInt Mask = APInt::getBitsSetFrom(VT.getSizeInBits(), Bits);
2135  if (CurDAG->MaskedValueIsZero(N, Mask)) {
2136  Val = N;
2137  return true;
2138  }
2139 
2140  return false;
2141 }
2142 
2143 /// Look for various patterns that can be done with a SHL that can be folded
2144 /// into a SHXADD. \p ShAmt contains 1, 2, or 3 and is set based on which
2145 /// SHXADD we are trying to match.
2146 bool RISCVDAGToDAGISel::selectSHXADDOp(SDValue N, unsigned ShAmt,
2147  SDValue &Val) {
2148  if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1))) {
2149  SDValue N0 = N.getOperand(0);
2150 
2151  bool LeftShift = N0.getOpcode() == ISD::SHL;
2152  if ((LeftShift || N0.getOpcode() == ISD::SRL) &&
2153  isa<ConstantSDNode>(N0.getOperand(1))) {
2154  uint64_t Mask = N.getConstantOperandVal(1);
2155  unsigned C2 = N0.getConstantOperandVal(1);
2156 
2157  unsigned XLen = Subtarget->getXLen();
2158  if (LeftShift)
2159  Mask &= maskTrailingZeros<uint64_t>(C2);
2160  else
2161  Mask &= maskTrailingOnes<uint64_t>(XLen - C2);
2162 
2163  // Look for (and (shl y, c2), c1) where c1 is a shifted mask with no
2164  // leading zeros and c3 trailing zeros. We can use an SRLI by c2+c3
2165  // followed by a SHXADD with c3 for the X amount.
2166  if (isShiftedMask_64(Mask)) {
2167  unsigned Leading = XLen - (64 - countLeadingZeros(Mask));
2168  unsigned Trailing = countTrailingZeros(Mask);
2169  if (LeftShift && Leading == 0 && C2 < Trailing && Trailing == ShAmt) {
2170  SDLoc DL(N);
2171  EVT VT = N.getValueType();
2172  Val = SDValue(CurDAG->getMachineNode(
2173  RISCV::SRLI, DL, VT, N0.getOperand(0),
2174  CurDAG->getTargetConstant(Trailing - C2, DL, VT)),
2175  0);
2176  return true;
2177  }
2178  // Look for (and (shr y, c2), c1) where c1 is a shifted mask with c2
2179  // leading zeros and c3 trailing zeros. We can use an SRLI by C3
2180  // followed by a SHXADD using c3 for the X amount.
2181  if (!LeftShift && Leading == C2 && Trailing == ShAmt) {
2182  SDLoc DL(N);
2183  EVT VT = N.getValueType();
2184  Val = SDValue(
2185  CurDAG->getMachineNode(
2186  RISCV::SRLI, DL, VT, N0.getOperand(0),
2187  CurDAG->getTargetConstant(Leading + Trailing, DL, VT)),
2188  0);
2189  return true;
2190  }
2191  }
2192  }
2193  }
2194 
2195  bool LeftShift = N.getOpcode() == ISD::SHL;
2196  if ((LeftShift || N.getOpcode() == ISD::SRL) &&
2197  isa<ConstantSDNode>(N.getOperand(1))) {
2198  SDValue N0 = N.getOperand(0);
2199  if (N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
2200  isa<ConstantSDNode>(N0.getOperand(1))) {
2201  uint64_t Mask = N0.getConstantOperandVal(1);
2202  if (isShiftedMask_64(Mask)) {
2203  unsigned C1 = N.getConstantOperandVal(1);
2204  unsigned XLen = Subtarget->getXLen();
2205  unsigned Leading = XLen - (64 - countLeadingZeros(Mask));
2206  unsigned Trailing = countTrailingZeros(Mask);
2207  // Look for (shl (and X, Mask), C1) where Mask has 32 leading zeros and
2208  // C3 trailing zeros. If C1+C3==ShAmt we can use SRLIW+SHXADD.
2209  if (LeftShift && Leading == 32 && Trailing > 0 &&
2210  (Trailing + C1) == ShAmt) {
2211  SDLoc DL(N);
2212  EVT VT = N.getValueType();
2213  Val = SDValue(CurDAG->getMachineNode(
2214  RISCV::SRLIW, DL, VT, N0.getOperand(0),
2215  CurDAG->getTargetConstant(Trailing, DL, VT)),
2216  0);
2217  return true;
2218  }
2219  // Look for (srl (and X, Mask), C1) where Mask has 32 leading zeros and
2220  // C3 trailing zeros. If C3-C1==ShAmt we can use SRLIW+SHXADD.
2221  if (!LeftShift && Leading == 32 && Trailing > C1 &&
2222  (Trailing - C1) == ShAmt) {
2223  SDLoc DL(N);
2224  EVT VT = N.getValueType();
2225  Val = SDValue(CurDAG->getMachineNode(
2226  RISCV::SRLIW, DL, VT, N0.getOperand(0),
2227  CurDAG->getTargetConstant(Trailing, DL, VT)),
2228  0);
2229  return true;
2230  }
2231  }
2232  }
2233  }
2234 
2235  return false;
2236 }
2237 
2238 /// Look for various patterns that can be done with a SHL that can be folded
2239 /// into a SHXADD_UW. \p ShAmt contains 1, 2, or 3 and is set based on which
2240 /// SHXADD_UW we are trying to match.
2241 bool RISCVDAGToDAGISel::selectSHXADD_UWOp(SDValue N, unsigned ShAmt,
2242  SDValue &Val) {
2243  if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1)) &&
2244  N.hasOneUse()) {
2245  SDValue N0 = N.getOperand(0);
2246  if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N0.getOperand(1)) &&
2247  N0.hasOneUse()) {
2248  uint64_t Mask = N.getConstantOperandVal(1);
2249  unsigned C2 = N0.getConstantOperandVal(1);
2250 
2251  Mask &= maskTrailingZeros<uint64_t>(C2);
2252 
2253  // Look for (and (shl y, c2), c1) where c1 is a shifted mask with
2254  // 32-ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
2255  // c2-ShAmt followed by SHXADD_UW with ShAmt for the X amount.
2256  if (isShiftedMask_64(Mask)) {
2257  unsigned Leading = countLeadingZeros(Mask);
2258  unsigned Trailing = countTrailingZeros(Mask);
2259  if (Leading == 32 - ShAmt && Trailing == C2 && Trailing > ShAmt) {
2260  SDLoc DL(N);
2261  EVT VT = N.getValueType();
2262  Val = SDValue(CurDAG->getMachineNode(
2263  RISCV::SLLI, DL, VT, N0.getOperand(0),
2264  CurDAG->getTargetConstant(C2 - ShAmt, DL, VT)),
2265  0);
2266  return true;
2267  }
2268  }
2269  }
2270  }
2271 
2272  return false;
2273 }
2274 
2275 // Return true if all users of this SDNode* only consume the lower \p Bits.
2276 // This can be used to form W instructions for add/sub/mul/shl even when the
2277 // root isn't a sext_inreg. This can allow the ADDW/SUBW/MULW/SLLIW to CSE if
2278 // SimplifyDemandedBits has made it so some users see a sext_inreg and some
2279 // don't. The sext_inreg+add/sub/mul/shl will get selected, but still leave
2280 // the add/sub/mul/shl to become non-W instructions. By checking the users we
2281 // may be able to use a W instruction and CSE with the other instruction if
2282 // this has happened. We could try to detect that the CSE opportunity exists
2283 // before doing this, but that would be more complicated.
2284 // TODO: Does this need to look through AND/OR/XOR to their users to find more
2285 // opportunities.
2286 bool RISCVDAGToDAGISel::hasAllNBitUsers(SDNode *Node, unsigned Bits) const {
2287  assert((Node->getOpcode() == ISD::ADD || Node->getOpcode() == ISD::SUB ||
2288  Node->getOpcode() == ISD::MUL || Node->getOpcode() == ISD::SHL ||
2289  Node->getOpcode() == ISD::SRL || Node->getOpcode() == ISD::AND ||
2290  Node->getOpcode() == ISD::OR || Node->getOpcode() == ISD::XOR ||
2291  Node->getOpcode() == ISD::SIGN_EXTEND_INREG ||
2292  isa<ConstantSDNode>(Node)) &&
2293  "Unexpected opcode");
2294 
2295  for (auto UI = Node->use_begin(), UE = Node->use_end(); UI != UE; ++UI) {
2296  SDNode *User = *UI;
2297  // Users of this node should have already been instruction selected
2298  if (!User->isMachineOpcode())
2299  return false;
2300 
2301  // TODO: Add more opcodes?
2302  switch (User->getMachineOpcode()) {
2303  default:
2304  return false;
2305  case RISCV::ADDW:
2306  case RISCV::ADDIW:
2307  case RISCV::SUBW:
2308  case RISCV::MULW:
2309  case RISCV::SLLW:
2310  case RISCV::SLLIW:
2311  case RISCV::SRAW:
2312  case RISCV::SRAIW:
2313  case RISCV::SRLW:
2314  case RISCV::SRLIW:
2315  case RISCV::DIVW:
2316  case RISCV::DIVUW:
2317  case RISCV::REMW:
2318  case RISCV::REMUW:
2319  case RISCV::ROLW:
2320  case RISCV::RORW:
2321  case RISCV::RORIW:
2322  case RISCV::CLZW:
2323  case RISCV::CTZW:
2324  case RISCV::CPOPW:
2325  case RISCV::SLLI_UW:
2326  case RISCV::FMV_W_X:
2327  case RISCV::FCVT_H_W:
2328  case RISCV::FCVT_H_WU:
2329  case RISCV::FCVT_S_W:
2330  case RISCV::FCVT_S_WU:
2331  case RISCV::FCVT_D_W:
2332  case RISCV::FCVT_D_WU:
2333  if (Bits < 32)
2334  return false;
2335  break;
2336  case RISCV::SLL:
2337  case RISCV::SRA:
2338  case RISCV::SRL:
2339  case RISCV::ROL:
2340  case RISCV::ROR:
2341  case RISCV::BSET:
2342  case RISCV::BCLR:
2343  case RISCV::BINV:
2344  // Shift amount operands only use log2(Xlen) bits.
2345  if (UI.getOperandNo() != 1 || Bits < Log2_32(Subtarget->getXLen()))
2346  return false;
2347  break;
2348  case RISCV::SLLI:
2349  // SLLI only uses the lower (XLen - ShAmt) bits.
2350  if (Bits < Subtarget->getXLen() - User->getConstantOperandVal(1))
2351  return false;
2352  break;
2353  case RISCV::ANDI:
2354  if (Bits < (64 - countLeadingZeros(User->getConstantOperandVal(1))))
2355  return false;
2356  break;
2357  case RISCV::ORI: {
2358  uint64_t Imm = cast<ConstantSDNode>(User->getOperand(1))->getSExtValue();
2359  if (Bits < (64 - countLeadingOnes(Imm)))
2360  return false;
2361  break;
2362  }
2363  case RISCV::SEXT_B:
2364  case RISCV::PACKH:
2365  if (Bits < 8)
2366  return false;
2367  break;
2368  case RISCV::SEXT_H:
2369  case RISCV::FMV_H_X:
2370  case RISCV::ZEXT_H_RV32:
2371  case RISCV::ZEXT_H_RV64:
2372  case RISCV::PACKW:
2373  if (Bits < 16)
2374  return false;
2375  break;
2376  case RISCV::PACK:
2377  if (Bits < (Subtarget->getXLen() / 2))
2378  return false;
2379  break;
2380  case RISCV::ADD_UW:
2381  case RISCV::SH1ADD_UW:
2382  case RISCV::SH2ADD_UW:
2383  case RISCV::SH3ADD_UW:
2384  // The first operand to add.uw/shXadd.uw is implicitly zero extended from
2385  // 32 bits.
2386  if (UI.getOperandNo() != 0 || Bits < 32)
2387  return false;
2388  break;
2389  case RISCV::SB:
2390  if (UI.getOperandNo() != 0 || Bits < 8)
2391  return false;
2392  break;
2393  case RISCV::SH:
2394  if (UI.getOperandNo() != 0 || Bits < 16)
2395  return false;
2396  break;
2397  case RISCV::SW:
2398  if (UI.getOperandNo() != 0 || Bits < 32)
2399  return false;
2400  break;
2401  }
2402  }
2403 
2404  return true;
2405 }
2406 
2407 // Select VL as a 5 bit immediate or a value that will become a register. This
2408 // allows us to choose betwen VSETIVLI or VSETVLI later.
2409 bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) {
2410  auto *C = dyn_cast<ConstantSDNode>(N);
2411  if (C && isUInt<5>(C->getZExtValue())) {
2412  VL = CurDAG->getTargetConstant(C->getZExtValue(), SDLoc(N),
2413  N->getValueType(0));
2414  } else if (C && C->isAllOnesValue()) {
2415  // Treat all ones as VLMax.
2416  VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, SDLoc(N),
2417  N->getValueType(0));
2418  } else if (isa<RegisterSDNode>(N) &&
2419  cast<RegisterSDNode>(N)->getReg() == RISCV::X0) {
2420  // All our VL operands use an operand that allows GPRNoX0 or an immediate
2421  // as the register class. Convert X0 to a special immediate to pass the
2422  // MachineVerifier. This is recognized specially by the vsetvli insertion
2423  // pass.
2424  VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, SDLoc(N),
2425  N->getValueType(0));
2426  } else {
2427  VL = N;
2428  }
2429 
2430  return true;
2431 }
2432 
2433 bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) {
2434  if (N.getOpcode() != RISCVISD::VMV_V_X_VL || !N.getOperand(0).isUndef())
2435  return false;
2436  assert(N.getNumOperands() == 3 && "Unexpected number of operands");
2437  SplatVal = N.getOperand(1);
2438  return true;
2439 }
2440 
2441 using ValidateFn = bool (*)(int64_t);
2442 
2443 static bool selectVSplatSimmHelper(SDValue N, SDValue &SplatVal,
2444  SelectionDAG &DAG,
2445  const RISCVSubtarget &Subtarget,
2446  ValidateFn ValidateImm) {
2447  if (N.getOpcode() != RISCVISD::VMV_V_X_VL || !N.getOperand(0).isUndef() ||
2448  !isa<ConstantSDNode>(N.getOperand(1)))
2449  return false;
2450  assert(N.getNumOperands() == 3 && "Unexpected number of operands");
2451 
2452  int64_t SplatImm =
2453  cast<ConstantSDNode>(N.getOperand(1))->getSExtValue();
2454 
2455  // The semantics of RISCVISD::VMV_V_X_VL is that when the operand
2456  // type is wider than the resulting vector element type: an implicit
2457  // truncation first takes place. Therefore, perform a manual
2458  // truncation/sign-extension in order to ignore any truncated bits and catch
2459  // any zero-extended immediate.
2460  // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first
2461  // sign-extending to (XLenVT -1).
2462  MVT XLenVT = Subtarget.getXLenVT();
2463  assert(XLenVT == N.getOperand(1).getSimpleValueType() &&
2464  "Unexpected splat operand type");
2465  MVT EltVT = N.getSimpleValueType().getVectorElementType();
2466  if (EltVT.bitsLT(XLenVT))
2467  SplatImm = SignExtend64(SplatImm, EltVT.getSizeInBits());
2468 
2469  if (!ValidateImm(SplatImm))
2470  return false;
2471 
2472  SplatVal = DAG.getTargetConstant(SplatImm, SDLoc(N), XLenVT);
2473  return true;
2474 }
2475 
2476 bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) {
2477  return selectVSplatSimmHelper(N, SplatVal, *CurDAG, *Subtarget,
2478  [](int64_t Imm) { return isInt<5>(Imm); });
2479 }
2480 
2481 bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1(SDValue N, SDValue &SplatVal) {
2482  return selectVSplatSimmHelper(
2483  N, SplatVal, *CurDAG, *Subtarget,
2484  [](int64_t Imm) { return (isInt<5>(Imm) && Imm != -16) || Imm == 16; });
2485 }
2486 
2487 bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1NonZero(SDValue N,
2488  SDValue &SplatVal) {
2489  return selectVSplatSimmHelper(
2490  N, SplatVal, *CurDAG, *Subtarget, [](int64_t Imm) {
2491  return Imm != 0 && ((isInt<5>(Imm) && Imm != -16) || Imm == 16);
2492  });
2493 }
2494 
2495 bool RISCVDAGToDAGISel::selectVSplatUimm5(SDValue N, SDValue &SplatVal) {
2496  if (N.getOpcode() != RISCVISD::VMV_V_X_VL || !N.getOperand(0).isUndef() ||
2497  !isa<ConstantSDNode>(N.getOperand(1)))
2498  return false;
2499 
2500  int64_t SplatImm =
2501  cast<ConstantSDNode>(N.getOperand(1))->getSExtValue();
2502 
2503  if (!isUInt<5>(SplatImm))
2504  return false;
2505 
2506  SplatVal =
2507  CurDAG->getTargetConstant(SplatImm, SDLoc(N), Subtarget->getXLenVT());
2508 
2509  return true;
2510 }
2511 
2512 bool RISCVDAGToDAGISel::selectRVVSimm5(SDValue N, unsigned Width,
2513  SDValue &Imm) {
2514  if (auto *C = dyn_cast<ConstantSDNode>(N)) {
2515  int64_t ImmVal = SignExtend64(C->getSExtValue(), Width);
2516 
2517  if (!isInt<5>(ImmVal))
2518  return false;
2519 
2520  Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT());
2521  return true;
2522  }
2523 
2524  return false;
2525 }
2526 
2527 // Try to remove sext.w if the input is a W instruction or can be made into
2528 // a W instruction cheaply.
2529 bool RISCVDAGToDAGISel::doPeepholeSExtW(SDNode *N) {
2530  // Look for the sext.w pattern, addiw rd, rs1, 0.
2531  if (N->getMachineOpcode() != RISCV::ADDIW ||
2532  !isNullConstant(N->getOperand(1)))
2533  return false;
2534 
2535  SDValue N0 = N->getOperand(0);
2536  if (!N0.isMachineOpcode())
2537  return false;
2538 
2539  switch (N0.getMachineOpcode()) {
2540  default:
2541  break;
2542  case RISCV::ADD:
2543  case RISCV::ADDI:
2544  case RISCV::SUB:
2545  case RISCV::MUL:
2546  case RISCV::SLLI: {
2547  // Convert sext.w+add/sub/mul to their W instructions. This will create
2548  // a new independent instruction. This improves latency.
2549  unsigned Opc;
2550  switch (N0.getMachineOpcode()) {
2551  default:
2552  llvm_unreachable("Unexpected opcode!");
2553  case RISCV::ADD: Opc = RISCV::ADDW; break;
2554  case RISCV::ADDI: Opc = RISCV::ADDIW; break;
2555  case RISCV::SUB: Opc = RISCV::SUBW; break;
2556  case RISCV::MUL: Opc = RISCV::MULW; break;
2557  case RISCV::SLLI: Opc = RISCV::SLLIW; break;
2558  }
2559 
2560  SDValue N00 = N0.getOperand(0);
2561  SDValue N01 = N0.getOperand(1);
2562 
2563  // Shift amount needs to be uimm5.
2564  if (N0.getMachineOpcode() == RISCV::SLLI &&
2565  !isUInt<5>(cast<ConstantSDNode>(N01)->getSExtValue()))
2566  break;
2567 
2568  SDNode *Result =
2569  CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0),
2570  N00, N01);
2571  ReplaceUses(N, Result);
2572  return true;
2573  }
2574  case RISCV::ADDW:
2575  case RISCV::ADDIW:
2576  case RISCV::SUBW:
2577  case RISCV::MULW:
2578  case RISCV::SLLIW:
2579  case RISCV::PACKW:
2580  // Result is already sign extended just remove the sext.w.
2581  // NOTE: We only handle the nodes that are selected with hasAllWUsers.
2582  ReplaceUses(N, N0.getNode());
2583  return true;
2584  }
2585 
2586  return false;
2587 }
2588 
2589 // Return true if we can make sure mask of N is all-ones mask.
2590 static bool usesAllOnesMask(SDNode *N, unsigned MaskOpIdx) {
2591  // Check that we're using V0 as a mask register.
2592  if (!isa<RegisterSDNode>(N->getOperand(MaskOpIdx)) ||
2593  cast<RegisterSDNode>(N->getOperand(MaskOpIdx))->getReg() != RISCV::V0)
2594  return false;
2595 
2596  // The glued user defines V0.
2597  const auto *Glued = N->getGluedNode();
2598 
2599  if (!Glued || Glued->getOpcode() != ISD::CopyToReg)
2600  return false;
2601 
2602  // Check that we're defining V0 as a mask register.
2603  if (!isa<RegisterSDNode>(Glued->getOperand(1)) ||
2604  cast<RegisterSDNode>(Glued->getOperand(1))->getReg() != RISCV::V0)
2605  return false;
2606 
2607  // Check the instruction defining V0; it needs to be a VMSET pseudo.
2608  SDValue MaskSetter = Glued->getOperand(2);
2609 
2610  const auto IsVMSet = [](unsigned Opc) {
2611  return Opc == RISCV::PseudoVMSET_M_B1 || Opc == RISCV::PseudoVMSET_M_B16 ||
2612  Opc == RISCV::PseudoVMSET_M_B2 || Opc == RISCV::PseudoVMSET_M_B32 ||
2613  Opc == RISCV::PseudoVMSET_M_B4 || Opc == RISCV::PseudoVMSET_M_B64 ||
2614  Opc == RISCV::PseudoVMSET_M_B8;
2615  };
2616 
2617  // TODO: Check that the VMSET is the expected bitwidth? The pseudo has
2618  // undefined behaviour if it's the wrong bitwidth, so we could choose to
2619  // assume that it's all-ones? Same applies to its VL.
2620  return MaskSetter->isMachineOpcode() &&
2621  IsVMSet(MaskSetter.getMachineOpcode());
2622 }
2623 
2624 // Optimize masked RVV pseudo instructions with a known all-ones mask to their
2625 // corresponding "unmasked" pseudo versions. The mask we're interested in will
2626 // take the form of a V0 physical register operand, with a glued
2627 // register-setting instruction.
2628 bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(SDNode *N) {
2629  const RISCV::RISCVMaskedPseudoInfo *I =
2630  RISCV::getMaskedPseudoInfo(N->getMachineOpcode());
2631  if (!I)
2632  return false;
2633 
2634  unsigned MaskOpIdx = I->MaskOpIdx;
2635 
2636  if (!usesAllOnesMask(N, MaskOpIdx))
2637  return false;
2638 
2639  // Retrieve the tail policy operand index, if any.
2640  std::optional<unsigned> TailPolicyOpIdx;
2641  const RISCVInstrInfo &TII = *Subtarget->getInstrInfo();
2642  const MCInstrDesc &MaskedMCID = TII.get(N->getMachineOpcode());
2643 
2644  bool IsTA = true;
2645  if (RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags)) {
2646  TailPolicyOpIdx = getVecPolicyOpIdx(N, MaskedMCID);
2647  if (!(N->getConstantOperandVal(*TailPolicyOpIdx) &
2648  RISCVII::TAIL_AGNOSTIC)) {
2649  // Keep the true-masked instruction when there is no unmasked TU
2650  // instruction
2651  if (I->UnmaskedTUPseudo == I->MaskedPseudo && !N->getOperand(0).isUndef())
2652  return false;
2653  // We can't use TA if the tie-operand is not IMPLICIT_DEF
2654  if (!N->getOperand(0).isUndef())
2655  IsTA = false;
2656  }
2657  }
2658 
2659  unsigned Opc = IsTA ? I->UnmaskedPseudo : I->UnmaskedTUPseudo;
2660 
2661  // Check that we're dropping the mask operand and any policy operand
2662  // when we transform to this unmasked pseudo. Additionally, if this insturtion
2663  // is tail agnostic, the unmasked instruction should not have a merge op.
2664  uint64_t TSFlags = TII.get(Opc).TSFlags;
2665  assert((IsTA != RISCVII::hasMergeOp(TSFlags)) &&
2666  RISCVII::hasDummyMaskOp(TSFlags) &&
2667  !RISCVII::hasVecPolicyOp(TSFlags) &&
2668  "Unexpected pseudo to transform to");
2669  (void)TSFlags;
2670 
2671  SmallVector<SDValue, 8> Ops;
2672  // Skip the merge operand at index 0 if IsTA
2673  for (unsigned I = IsTA, E = N->getNumOperands(); I != E; I++) {
2674  // Skip the mask, the policy, and the Glue.
2675  SDValue Op = N->getOperand(I);
2676  if (I == MaskOpIdx || I == TailPolicyOpIdx ||
2677  Op.getValueType() == MVT::Glue)
2678  continue;
2679  Ops.push_back(Op);
2680  }
2681 
2682  // Transitively apply any node glued to our new node.
2683  const auto *Glued = N->getGluedNode();
2684  if (auto *TGlued = Glued->getGluedNode())
2685  Ops.push_back(SDValue(TGlued, TGlued->getNumValues() - 1));
2686 
2687  SDNode *Result = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
2688  Result->setFlags(N->getFlags());
2689  ReplaceUses(N, Result);
2690 
2691  return true;
2692 }
2693 
2694 // Try to fold VMERGE_VVM with unmasked intrinsic to masked intrinsic. The
2695 // peephole only deals with VMERGE_VVM which is TU and has false operand same as
2696 // its true operand now. E.g. (VMERGE_VVM_M1_TU False, False, (VADD_M1 ...),
2697 // ...) -> (VADD_VV_M1_MASK)
2698 bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N, bool IsTA) {
2699  unsigned Offset = IsTA ? 0 : 1;
2700  uint64_t Policy = IsTA ? RISCVII::TAIL_AGNOSTIC : /*TUMU*/ 0;
2701 
2702  SDValue False = N->getOperand(0 + Offset);
2703  SDValue True = N->getOperand(1 + Offset);
2704  SDValue Mask = N->getOperand(2 + Offset);
2705  SDValue VL = N->getOperand(3 + Offset);
2706 
2707  assert(True.getResNo() == 0 &&
2708  "Expect True is the first output of an instruction.");
2709 
2710  // Need N is the exactly one using True.
2711  if (!True.hasOneUse())
2712  return false;
2713 
2714  if (!True.isMachineOpcode())
2715  return false;
2716 
2717  unsigned TrueOpc = True.getMachineOpcode();
2718 
2719  // Skip if True has merge operand.
2720  // TODO: Deal with True having same merge operand with N.
2721  if (RISCVII::hasMergeOp(TII->get(TrueOpc).TSFlags))
2722  return false;
2723 
2724  // Skip if True has side effect.
2725  // TODO: Support velff and vlsegff.
2726  if (TII->get(TrueOpc).hasUnmodeledSideEffects())
2727  return false;
2728 
2729  // Only deal with True when True is unmasked intrinsic now.
2730  const RISCV::RISCVMaskedPseudoInfo *Info =
2731  RISCV::lookupMaskedIntrinsicByUnmaskedTA(TrueOpc);
2732 
2733  if (!Info)
2734  return false;
2735 
2736  // The last operand of unmasked intrinsic should be sew or chain.
2737  bool HasChainOp =
2738  True.getOperand(True.getNumOperands() - 1).getValueType() == MVT::Other;
2739 
2740  if (HasChainOp) {
2741  // Avoid creating cycles in the DAG. We must ensure that none of the other
2742  // operands depend on True through it's Chain.
2743  SmallVector<const SDNode *, 4> LoopWorklist;
2744  SmallPtrSet<const SDNode *, 16> Visited;
2745  LoopWorklist.push_back(False.getNode());
2746  LoopWorklist.push_back(Mask.getNode());
2747  LoopWorklist.push_back(VL.getNode());
2748  if (SDNode *Glued = N->getGluedNode())
2749  LoopWorklist.push_back(Glued);
2750  if (SDNode::hasPredecessorHelper(True.getNode(), Visited, LoopWorklist))
2751  return false;
2752  }
2753 
2754  // Need True has same VL with N.
2755  unsigned TrueVLIndex = True.getNumOperands() - HasChainOp - 2;
2756  SDValue TrueVL = True.getOperand(TrueVLIndex);
2757 
2758  auto IsNoFPExcept = [this](SDValue N) {
2759  return !this->mayRaiseFPException(N.getNode()) ||
2760  N->getFlags().hasNoFPExcept();
2761  };
2762 
2763  // Allow the peephole for non-exception True with VLMAX vector length, since
2764  // all the values after VL of N are dependent on Merge. VLMAX should be
2765  // lowered to (XLenVT -1).
2766  if (TrueVL != VL && !(IsNoFPExcept(True) && isAllOnesConstant(TrueVL)))
2767  return false;
2768 
2769  SDLoc DL(N);
2770  unsigned MaskedOpc = Info->MaskedPseudo;
2771  assert(RISCVII::hasVecPolicyOp(TII->get(MaskedOpc).TSFlags) &&
2772  "Expected instructions with mask have policy operand.");
2773  assert(RISCVII::hasMergeOp(TII->get(MaskedOpc).TSFlags) &&
2774  "Expected instructions with mask have merge operand.");
2775 
2776  SmallVector<SDValue, 8> Ops;
2777  Ops.push_back(False);
2778  Ops.append(True->op_begin(), True->op_begin() + TrueVLIndex);
2779  Ops.append({Mask, VL, /* SEW */ True.getOperand(TrueVLIndex + 1)});
2780  Ops.push_back(CurDAG->getTargetConstant(Policy, DL, Subtarget->getXLenVT()));
2781 
2782  // Result node should have chain operand of True.
2783  if (HasChainOp)
2784  Ops.push_back(True.getOperand(True.getNumOperands() - 1));
2785 
2786  // Result node should take over glued node of N.
2787  if (N->getGluedNode())
2788  Ops.push_back(N->getOperand(N->getNumOperands() - 1));
2789 
2790  SDNode *Result =
2791  CurDAG->getMachineNode(MaskedOpc, DL, True->getVTList(), Ops);
2792  Result->setFlags(True->getFlags());
2793 
2794  // Replace vmerge.vvm node by Result.
2795  ReplaceUses(SDValue(N, 0), SDValue(Result, 0));
2796 
2797  // Replace another value of True. E.g. chain and VL.
2798  for (unsigned Idx = 1; Idx < True->getNumValues(); ++Idx)
2799  ReplaceUses(True.getValue(Idx), SDValue(Result, Idx));
2800 
2801  // Try to transform Result to unmasked intrinsic.
2802  doPeepholeMaskedRVV(Result);
2803  return true;
2804 }
2805 
2806 // Transform (VMERGE_VVM_<LMUL>_TU false, false, true, allones, vl, sew) to
2807 // (VADD_VI_<LMUL>_TU false, true, 0, vl, sew). It may decrease uses of VMSET.
2808 bool RISCVDAGToDAGISel::performVMergeToVAdd(SDNode *N) {
2809  unsigned NewOpc;
2810  switch (N->getMachineOpcode()) {
2811  default:
2812  llvm_unreachable("Expected VMERGE_VVM_<LMUL>_TU instruction.");
2813  case RISCV::PseudoVMERGE_VVM_MF8_TU:
2814  NewOpc = RISCV::PseudoVADD_VI_MF8_TU;
2815  break;
2816  case RISCV::PseudoVMERGE_VVM_MF4_TU:
2817  NewOpc = RISCV::PseudoVADD_VI_MF4_TU;
2818  break;
2819  case RISCV::PseudoVMERGE_VVM_MF2_TU:
2820  NewOpc = RISCV::PseudoVADD_VI_MF2_TU;
2821  break;
2822  case RISCV::PseudoVMERGE_VVM_M1_TU:
2823  NewOpc = RISCV::PseudoVADD_VI_M1_TU;
2824  break;
2825  case RISCV::PseudoVMERGE_VVM_M2_TU:
2826  NewOpc = RISCV::PseudoVADD_VI_M2_TU;
2827  break;
2828  case RISCV::PseudoVMERGE_VVM_M4_TU:
2829  NewOpc = RISCV::PseudoVADD_VI_M4_TU;
2830  break;
2831  case RISCV::PseudoVMERGE_VVM_M8_TU:
2832  NewOpc = RISCV::PseudoVADD_VI_M8_TU;
2833  break;
2834  }
2835 
2836  if (!usesAllOnesMask(N, /* MaskOpIdx */ 3))
2837  return false;
2838 
2839  SDLoc DL(N);
2840  EVT VT = N->getValueType(0);
2841  SDValue Ops[] = {N->getOperand(1), N->getOperand(2),
2842  CurDAG->getTargetConstant(0, DL, Subtarget->getXLenVT()),
2843  N->getOperand(4), N->getOperand(5)};
2844  SDNode *Result = CurDAG->getMachineNode(NewOpc, DL, VT, Ops);
2845  ReplaceUses(N, Result);
2846  return true;
2847 }
2848 
2849 bool RISCVDAGToDAGISel::doPeepholeMergeVVMFold() {
2850  bool MadeChange = false;
2851  SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
2852 
2853  while (Position != CurDAG->allnodes_begin()) {
2854  SDNode *N = &*--Position;
2855  if (N->use_empty() || !N->isMachineOpcode())
2856  continue;
2857 
2858  auto IsVMergeTU = [](unsigned Opcode) {
2859  return Opcode == RISCV::PseudoVMERGE_VVM_MF8_TU ||
2860  Opcode == RISCV::PseudoVMERGE_VVM_MF4_TU ||
2861  Opcode == RISCV::PseudoVMERGE_VVM_MF2_TU ||
2862  Opcode == RISCV::PseudoVMERGE_VVM_M1_TU ||
2863  Opcode == RISCV::PseudoVMERGE_VVM_M2_TU ||
2864  Opcode == RISCV::PseudoVMERGE_VVM_M4_TU ||
2865  Opcode == RISCV::PseudoVMERGE_VVM_M8_TU;
2866  };
2867 
2868  auto IsVMergeTA = [](unsigned Opcode) {
2869  return Opcode == RISCV::PseudoVMERGE_VVM_MF8 ||
2870  Opcode == RISCV::PseudoVMERGE_VVM_MF4 ||
2871  Opcode == RISCV::PseudoVMERGE_VVM_MF2 ||
2872  Opcode == RISCV::PseudoVMERGE_VVM_M1 ||
2873  Opcode == RISCV::PseudoVMERGE_VVM_M2 ||
2874  Opcode == RISCV::PseudoVMERGE_VVM_M4 ||
2875  Opcode == RISCV::PseudoVMERGE_VVM_M8;
2876  };
2877 
2878  unsigned Opc = N->getMachineOpcode();
2879  // The following optimizations require that the merge operand of N is same
2880  // as the false operand of N.
2881  if ((IsVMergeTU(Opc) && N->getOperand(0) == N->getOperand(1)) ||
2882  IsVMergeTA(Opc))
2883  MadeChange |= performCombineVMergeAndVOps(N, IsVMergeTA(Opc));
2884  if (IsVMergeTU(Opc) && N->getOperand(0) == N->getOperand(1))
2885  MadeChange |= performVMergeToVAdd(N);
2886  }
2887  return MadeChange;
2888 }
2889 
2890 // This pass converts a legalized DAG into a RISCV-specific DAG, ready
2891 // for instruction scheduling.
2892 FunctionPass *llvm::createRISCVISelDag(RISCVTargetMachine &TM,
2893  CodeGenOpt::Level OptLevel) {
2894  return new RISCVDAGToDAGISel(TM, OptLevel);
2895 }
llvm::ISD::SUB
@ SUB
Definition: ISDOpcodes.h:240
llvm::RISCVII::LMUL_1
@ LMUL_1
Definition: RISCVBaseInfo.h:109
llvm::TargetMachine::getOptLevel
CodeGenOpt::Level getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
Definition: TargetMachine.cpp:182
llvm::RISCVISD::VFMV_S_F_VL
@ VFMV_S_F_VL
Definition: RISCVISelLowering.h:145
llvm::RISCVMatInt::Inst
Definition: RISCVMatInt.h:28
llvm::ConstantSDNode
Definition: SelectionDAGNodes.h:1582
llvm::MVT::getVectorElementType
MVT getVectorElementType() const
Definition: MachineValueType.h:542
llvm::MVT::getStoreSize
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: MachineValueType.h:1148
llvm::ISD::INTRINSIC_VOID
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:199
llvm::RISCVDAGToDAGISel::selectVLXSEG
void selectVLXSEG(SDNode *Node, bool IsMasked, bool IsOrdered)
Definition: RISCVISelDAGToDAG.cpp:401
MathExtras.h
Merge
R600 Clause Merge
Definition: R600ClauseMergePass.cpp:70
llvm::SelectionDAGISel::TLI
const TargetLowering * TLI
Definition: SelectionDAGISel.h:56
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::RISCVISD::SLLW
@ SLLW
Definition: RISCVISelLowering.h:64
llvm::SelectionDAGISel::TM
TargetMachine & TM
Definition: SelectionDAGISel.h:43
llvm::EVT::isScalarInteger
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition: ValueTypes.h:149
llvm::RISCV::VLSEGPseudo
Definition: RISCVISelDAGToDAG.h:143
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1106
llvm::MVT::isFixedLengthVector
bool isFixedLengthVector() const
Definition: MachineValueType.h:398
llvm::RISCVDAGToDAGISel::selectVSplatSimm5Plus1
bool selectVSplatSimm5Plus1(SDValue N, SDValue &SplatVal)
Definition: RISCVISelDAGToDAG.cpp:2481
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::ISD::OR
@ OR
Definition: ISDOpcodes.h:667
llvm::MVT::isInteger
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: MachineValueType.h:370
llvm::RISCVDAGToDAGISel::PreprocessISelDAG
void PreprocessISelDAG() override
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
Definition: RISCVISelDAGToDAG.cpp:60
llvm::ISD::BITCAST
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:886
Insert
Vector Rotate Left Mask Mask Insert
Definition: README_P9.txt:112
llvm::RISCVSubtarget::getTargetLowering
const RISCVTargetLowering * getTargetLowering() const override
Definition: RISCVSubtarget.h:144
llvm::SelectionDAG::getCopyToReg
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:767
llvm::RISCV::VLXSEGPseudo
Definition: RISCVISelDAGToDAG.h:154
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:159
llvm::isOneConstant
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Definition: SelectionDAG.cpp:10802
llvm::SelectionDAG::allnodes_end
allnodes_const_iterator allnodes_end() const
Definition: SelectionDAG.h:526
P
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Definition: README-SSE.txt:411
llvm::RISCVISD::FMV_H_X
@ FMV_H_X
Definition: RISCVISelLowering.h:96
llvm::SDNode::isUndef
bool isUndef() const
Return true if the type of the node type undefined.
Definition: SelectionDAGNodes.h:667
llvm::KnownBits::Zero
APInt Zero
Definition: KnownBits.h:24
C1
instcombine should handle this C2 when C1
Definition: README.txt:263
llvm::RISCVSubtarget::hasOptimizedZeroStrideLoad
bool hasOptimizedZeroStrideLoad() const
Definition: RISCVSubtarget.h:210
llvm::RISCVISD::DIVUW
@ DIVUW
Definition: RISCVISelLowering.h:71
llvm::MVT::bitsLT
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
Definition: MachineValueType.h:1210
llvm::SelectionDAG::getFrameIndex
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
Definition: SelectionDAG.cpp:1739
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1199
llvm::RISCVDAGToDAGISel::selectVSETVLI
void selectVSETVLI(SDNode *Node)
Definition: RISCVISelDAGToDAG.cpp:529
llvm::SelectionDAG::getVTList
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
Definition: SelectionDAG.cpp:9378
llvm::MachineSDNode
An SDNode that represents everything that will be needed to construct a MachineInstr.
Definition: SelectionDAGNodes.h:2897
llvm::RISCVSubtarget::hasVInstructions
bool hasVInstructions() const
Definition: RISCVSubtarget.h:243
llvm::RISCVDAGToDAGISel::SelectFrameAddrRegImm
bool SelectFrameAddrRegImm(SDValue Addr, SDValue &Base, SDValue &Offset)
Definition: RISCVISelDAGToDAG.cpp:1880
llvm::SelectionDAG::allnodes_begin
allnodes_const_iterator allnodes_begin() const
Definition: SelectionDAG.h:525
llvm::SelectionDAG::getRoot
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:545
llvm::RISCVDAGToDAGISel::selectSHXADD_UWOp
bool selectSHXADD_UWOp(SDValue N, unsigned ShAmt, SDValue &Val)
Look for various patterns that can be done with a SHL that can be folded into a SHXADD_UW.
Definition: RISCVISelDAGToDAG.cpp:2241
llvm::HandleSDNode
This class is used to form a handle around another node that is persistent and is updated across invo...
Definition: SelectionDAGNodes.h:1233
llvm::RISCVMatInt::generateInstSeq
InstSeq generateInstSeq(int64_t Val, const FeatureBitset &ActiveFeatures)
Definition: RISCVMatInt.cpp:176
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:463
llvm::RISCVTargetMachine
Definition: RISCVTargetMachine.h:23
llvm::RISCVDAGToDAGISel::selectVSplat
bool selectVSplat(SDValue N, SDValue &SplatVal)
Definition: RISCVISelDAGToDAG.cpp:2433
llvm::RISCVII::LMUL_8
@ LMUL_8
Definition: RISCVBaseInfo.h:112
llvm::MVT::Glue
@ Glue
Definition: MachineValueType.h:282
llvm::MemOp
Definition: TargetLowering.h:111
llvm::RISCVDAGToDAGISel
Definition: RISCVISelDAGToDAG.h:23
llvm::SelectionDAG::getMemBasePlusOffset
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
Definition: SelectionDAG.cpp:6813
llvm::RISCVDAGToDAGISel::selectZExtBits
bool selectZExtBits(SDValue N, unsigned Bits, SDValue &Val)
Definition: RISCVISelDAGToDAG.cpp:2125
llvm::RISCVSubtarget::hasStdExtZbs
bool hasStdExtZbs() const
Definition: RISCVSubtarget.h:170
Shift
bool Shift
Definition: README.txt:468
llvm::APInt::getBitWidth
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1431
llvm::tgtok::Bits
@ Bits
Definition: TGLexer.h:50
llvm::SelectionDAG::getStore
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
Definition: SelectionDAG.cpp:8038
llvm::SelectionDAG::isBaseWithConstantOffset
bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
Definition: SelectionDAG.cpp:4709
llvm::RISCV::VLX_VSXPseudo
Definition: RISCVISelDAGToDAG.h:202
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:450
llvm::SPII::Store
@ Store
Definition: SparcInstrInfo.h:33
llvm::SelectionDAG::RemoveDeadNodes
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
Definition: SelectionDAG.cpp:943
llvm::RISCVTargetLowering::getRegClassIDForVecVT
static unsigned getRegClassIDForVecVT(MVT VT)
Definition: RISCVISelLowering.cpp:1617
llvm::RISCV::VLMaxSentinel
static constexpr int64_t VLMaxSentinel
Definition: RISCVInstrInfo.h:232
Log2SEW
unsigned Log2SEW
Definition: RISCVInsertVSETVLI.cpp:641
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:458
llvm::countLeadingOnes
unsigned countLeadingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the most significant bit to the first zero bit.
Definition: MathExtras.h:476
llvm::RISCVDAGToDAGISel::selectVSSEG
void selectVSSEG(SDNode *Node, bool IsMasked, bool IsStrided)
Definition: RISCVISelDAGToDAG.cpp:455
llvm::RISCVII::hasVecPolicyOp
static bool hasVecPolicyOp(uint64_t TSFlags)
Definition: RISCVBaseInfo.h:159
RISCVMatInt.h
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1628
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::commonAlignment
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:213
llvm::RISCVDAGToDAGISel::selectVSplatSimm5Plus1NonZero
bool selectVSplatSimm5Plus1NonZero(SDValue N, SDValue &SplatVal)
Definition: RISCVISelDAGToDAG.cpp:2487
llvm::SDNode::getVTList
SDVTList getVTList() const
Definition: SelectionDAGNodes.h:949
llvm::MCInstrDesc::hasUnmodeledSideEffects
bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by other flags.
Definition: MCInstrDesc.h:460
KnownBits.h
llvm::MVT::isScalableVector
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
Definition: MachineValueType.h:393
llvm::SelectionDAG::getRegister
SDValue getRegister(unsigned Reg, EVT VT)
Definition: SelectionDAG.cpp:2121
llvm::InlineAsm::Constraint_A
@ Constraint_A
Definition: InlineAsm.h:262
llvm::RISCVDAGToDAGISel::SelectAddrFrameIndex
bool SelectAddrFrameIndex(SDValue Addr, SDValue &Base, SDValue &Offset)
Definition: RISCVISelDAGToDAG.cpp:1868
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
usesAllOnesMask
static bool usesAllOnesMask(SDNode *N, unsigned MaskOpIdx)
Definition: RISCVISelDAGToDAG.cpp:2590
llvm::RISCVSubtarget::is64Bit
bool is64Bit() const
Definition: RISCVSubtarget.h:200
llvm::RISCV::VSSEGPseudo
Definition: RISCVISelDAGToDAG.h:165
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
llvm::RISCVII::LMUL_4
@ LMUL_4
Definition: RISCVBaseInfo.h:111
llvm::RISCV::RISCVMaskedPseudoInfo
Definition: RISCVISelDAGToDAG.h:212
llvm::all_of
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1709
llvm::EVT::getStoreSize
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:362
llvm::MCInstrDesc::TSFlags
uint64_t TSFlags
Definition: MCInstrDesc.h:205
llvm::RISCVDAGToDAGISel::selectShiftMask
bool selectShiftMask(SDValue N, unsigned ShiftWidth, SDValue &ShAmt)
Definition: RISCVISelDAGToDAG.cpp:2063
llvm::SelectionDAG::getTargetFrameIndex
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:720
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1141
llvm::SelectionDAG
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:220
llvm::ISD::Constant
@ Constant
Definition: ISDOpcodes.h:76
getReg
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
Definition: MipsDisassembler.cpp:517
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::MachineFunction::getInfo
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Definition: MachineFunction.h:755
llvm::User
Definition: User.h:44
llvm::SelectionDAG::getUNDEF
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:1023
llvm::ISD::CopyToReg
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition: ISDOpcodes.h:203
llvm::ISD::SIGN_EXTEND_INREG
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:781
createTuple
static SDValue createTuple(SelectionDAG &CurDAG, ArrayRef< SDValue > Regs, unsigned NF, RISCVII::VLMUL LMUL)
Definition: RISCVISelDAGToDAG.cpp:212
llvm::SelectionDAG::getTargetLoweringInfo
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:475
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:34
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::MVT::f64
@ f64
Definition: MachineValueType.h:58
llvm::isShiftedMask_64
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
Definition: MathExtras.h:452
llvm::TargetLowering
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Definition: TargetLowering.h:3490
llvm::MVT::getScalarSizeInBits
uint64_t getScalarSizeInBits() const
Definition: MachineValueType.h:1138
llvm::SelectionDAG::MaskedValueIsZero
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
Definition: SelectionDAG.cpp:2532
llvm::RISCVTargetLowering::getSubregIndexByMVT
static unsigned getSubregIndexByMVT(MVT VT, unsigned Index)
Definition: RISCVISelLowering.cpp:1594
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
llvm::ISD::SRA
@ SRA
Definition: ISDOpcodes.h:692
llvm::RISCVSubtarget::getXLenVT
MVT getXLenVT() const
Definition: RISCVSubtarget.h:211
RISCVISelDAGToDAG.h
llvm::SelectionDAGISel::ReplaceNode
void ReplaceNode(SDNode *F, SDNode *T)
Replace all uses of F with T, then remove F from the DAG.
Definition: SelectionDAGISel.h:231
llvm::TypeSize::Fixed
static TypeSize Fixed(ScalarTy MinVal)
Definition: TypeSize.h:441
llvm::Log2_32
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:547
llvm::MCInstrDesc
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:197
llvm::SelectionDAG::setRoot
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:554
llvm::RISCVVType::decodeVSEW
static unsigned decodeVSEW(unsigned VSEW)
Definition: RISCVBaseInfo.h:437
RISCVMCTargetDesc.h
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
llvm::ISD::ATOMIC_STORE
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1163
llvm::RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand
bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, std::vector< SDValue > &OutOps) override
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
Definition: RISCVISelDAGToDAG.cpp:1850
llvm::SelectionDAG::getMemIntrinsicNode
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, uint64_t Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
Definition: SelectionDAG.cpp:7763
llvm::RISCVSubtarget::hasStdExtZbb
bool hasStdExtZbb() const
Definition: RISCVSubtarget.h:168
Info
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
llvm::ISD::AND
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:666
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:687
llvm::RISCVSubtarget::getInstrInfo
const RISCVInstrInfo * getInstrInfo() const override
Definition: RISCVSubtarget.h:140
CASE_VMSLT_OPCODES
#define CASE_VMSLT_OPCODES(lmulenum, suffix, suffix_b)
Align
uint64_t Align
Definition: ELFObjHandler.cpp:82
llvm::ISD::SPLAT_VECTOR
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:613
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::SDValue::getConstantOperandVal
uint64_t getConstantOperandVal(unsigned i) const
Definition: SelectionDAGNodes.h:1153
isWorthFoldingAdd
static bool isWorthFoldingAdd(SDValue Add)
Definition: RISCVISelDAGToDAG.cpp:1951
llvm::AArch64_AM::ROR
@ ROR
Definition: AArch64AddressingModes.h:38
llvm::RISCVISD::DIVW
@ DIVW
Definition: RISCVISelLowering.h:70
X
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
llvm::SelectionDAG::getTargetGlobalAddress
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:715
llvm::RISCVISD::CLZW
@ CLZW
Definition: RISCVISelLowering.h:79
llvm::RISCVDAGToDAGISel::SelectAddrRegImm
bool SelectAddrRegImm(SDValue Addr, SDValue &Base, SDValue &Offset)
Definition: RISCVISelDAGToDAG.cpp:1973
Operands
mir Rename Register Operands
Definition: MIRNamerPass.cpp:74
llvm::APInt::isSubsetOf
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1227
llvm::InlineAsm::Constraint_m
@ Constraint_m
Definition: InlineAsm.h:259
TSFlags
uint64_t TSFlags
Definition: RISCVInsertVSETVLI.cpp:595
llvm::createRISCVISelDag
FunctionPass * createRISCVISelDag(RISCVTargetMachine &TM, CodeGenOpt::Level OptLevel)
Definition: RISCVISelDAGToDAG.cpp:2892
llvm::RISCVISD::VMV_S_X_VL
@ VMV_S_X_VL
Definition: RISCVISelLowering.h:143
llvm::SDValue::getNumOperands
unsigned getNumOperands() const
Definition: SelectionDAGNodes.h:1145
llvm::SelectionDAG::RemoveDeadNode
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
Definition: SelectionDAG.cpp:997
llvm::RISCV::VSEPseudo
Definition: RISCVISelDAGToDAG.h:194
llvm::RISCVDAGToDAGISel::selectVLOp
bool selectVLOp(SDValue N, SDValue &VL)
Definition: RISCVISelDAGToDAG.cpp:2409
llvm::RISCVDAGToDAGISel::selectVSXSEG
void selectVSXSEG(SDNode *Node, bool IsMasked, bool IsOrdered)
Definition: RISCVISelDAGToDAG.cpp:487
llvm::SelectionDAGISel::IsProfitableToFold
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
Definition: SelectionDAGISel.cpp:2072
llvm::AMDGPU::Hwreg::Offset
Offset
Definition: SIDefines.h:416
selectVSplatSimmHelper
static bool selectVSplatSimmHelper(SDValue N, SDValue &SplatVal, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, ValidateFn ValidateImm)
Definition: RISCVISelDAGToDAG.cpp:2443
uint64_t
llvm::RISCVDAGToDAGISel::selectVSplatUimm5
bool selectVSplatUimm5(SDValue N, SDValue &SplatVal)
Definition: RISCVISelDAGToDAG.cpp:2495
llvm::SelectionDAGISel::TII
const TargetInstrInfo * TII
Definition: SelectionDAGISel.h:55
llvm::ISD::LOAD
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:966
Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:79
llvm::SelectionDAGISel::FuncInfo
std::unique_ptr< FunctionLoweringInfo > FuncInfo
Definition: SelectionDAGISel.h:45
llvm::MachinePointerInfo
This class contains a discriminated union of information about pointers in memory operands,...
Definition: MachineMemOperand.h:39
llvm::SelectionDAG::getCopyFromReg
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:793
llvm::SelectionDAGISel::IsLegalToFold
static bool IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, CodeGenOpt::Level OptLevel, bool IgnoreChains=false)
IsLegalToFold - Returns true if the specific operand node N of U can be folded during instruction sel...
Definition: SelectionDAGISel.cpp:2080
llvm::SelectionDAGISel::mayRaiseFPException
bool mayRaiseFPException(SDNode *Node) const
Return whether the node may raise an FP exception.
Definition: SelectionDAGISel.cpp:3727
llvm::SDNode::getOperand
const SDValue & getOperand(unsigned Num) const
Definition: SelectionDAGNodes.h:921
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:9072
llvm::countTrailingOnes
unsigned countTrailingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the least significant bit to the first zero bit.
Definition: MathExtras.h:491
llvm::RISCVISD::ROLW
@ ROLW
Definition: RISCVISelLowering.h:75
llvm::RISCVMachineFunctionInfo
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
Definition: RISCVMachineFunctionInfo.h:47
llvm::RISCVSubtarget
Definition: RISCVSubtarget.h:35
llvm::AVRISD::ROL
@ ROL
Bit rotate left.
Definition: AVRISelLowering.h:51
llvm::SDValue::getValue
SDValue getValue(unsigned R) const
Definition: SelectionDAGNodes.h:179
llvm::RISCVDAGToDAGISel::selectVLSEG
void selectVLSEG(SDNode *Node, bool IsMasked, bool IsStrided)
Definition: RISCVISelDAGToDAG.cpp:313
llvm::RISCVISD::ADD_LO
@ ADD_LO
Definition: RISCVISelLowering.h:47
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::MVT::Other
@ Other
Definition: MachineValueType.h:42
llvm::MVT::getSizeInBits
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
Definition: MachineValueType.h:919
llvm::ConstantSDNode::getZExtValue
uint64_t getZExtValue() const
Definition: SelectionDAGNodes.h:1597
llvm::SDValue::getResNo
unsigned getResNo() const
get the index which selects a specific result in the SDNode
Definition: SelectionDAGNodes.h:156
llvm::RISCVMatInt::RegX0
@ RegX0
Definition: RISCVMatInt.h:25
llvm::SelectionDAGISel::CurDAG
SelectionDAG * CurDAG
Definition: SelectionDAGISel.h:49
llvm::RISCVDAGToDAGISel::hasAllWUsers
bool hasAllWUsers(SDNode *Node) const
Definition: RISCVISelDAGToDAG.h:80
llvm::SelectionDAG::getMachineNode
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
Definition: SelectionDAG.cpp:9816
llvm::MVT
Machine Value Type.
Definition: MachineValueType.h:31
llvm::RISCVISD::SRAW
@ SRAW
Definition: RISCVISelLowering.h:65
llvm::RISCVDAGToDAGISel::selectVSplatSimm5
bool selectVSplatSimm5(SDValue N, SDValue &SplatVal)
Definition: RISCVISelDAGToDAG.cpp:2476
llvm::SelectionDAG::setNodeMemRefs
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
Definition: SelectionDAG.cpp:9584
llvm::RISCVSubtarget::hasStdExtZba
bool hasStdExtZba() const
Definition: RISCVSubtarget.h:167
llvm::MachinePointerInfo::getWithOffset
MachinePointerInfo getWithOffset(int64_t O) const
Definition: MachineMemOperand.h:79
llvm::SDNode::isMachineOpcode
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
Definition: SelectionDAGNodes.h:699
llvm::RISCVII::hasMergeOp
static bool hasMergeOp(uint64_t TSFlags)
Definition: RISCVBaseInfo.h:147
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:75
llvm::MachineFunction
Definition: MachineFunction.h:257
llvm::RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs
static std::pair< unsigned, unsigned > decomposeSubvectorInsertExtractToSubRegs(MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx, const RISCVRegisterInfo *TRI)
Definition: RISCVISelLowering.cpp:1629
llvm::RISCVISD::REMUW
@ REMUW
Definition: RISCVISelLowering.h:72
llvm::NVPTXISD::Dummy
@ Dummy
Definition: NVPTXISelLowering.h:60
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::RISCVInstrInfo
Definition: RISCVInstrInfo.h:44
llvm::MVT::i64
@ i64
Definition: MachineValueType.h:49
llvm::countTrailingZeros
unsigned countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: MathExtras.h:152
llvm::SelectionDAG::getTargetInsertSubreg
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
Definition: SelectionDAG.cpp:9944
llvm::RISCVISD::VMV_V_X_VL
@ VMV_V_X_VL
Definition: RISCVISelLowering.h:134
llvm::RISCVSubtarget::getRegisterInfo
const RISCVRegisterInfo * getRegisterInfo() const override
Definition: RISCVSubtarget.h:141
llvm::SDValue::getMachineOpcode
unsigned getMachineOpcode() const
Definition: SelectionDAGNodes.h:1173
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::RISCVII::LMUL_2
@ LMUL_2
Definition: RISCVBaseInfo.h:110
llvm::SelectionDAG::ReplaceAllUsesOfValueWith
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
Definition: SelectionDAG.cpp:10411
llvm::SDValue::getOperand
const SDValue & getOperand(unsigned i) const
Definition: SelectionDAGNodes.h:1149
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::ConstantSDNode::getSExtValue
int64_t getSExtValue() const
Definition: SelectionDAGNodes.h:1598
llvm::SDValue::hasOneUse
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
Definition: SelectionDAGNodes.h:1185
llvm::SDValue::getSimpleValueType
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:190
llvm::CodeGenOpt::Level
Level
Definition: CodeGen.h:52
llvm::SDVTList
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Definition: SelectionDAGNodes.h:79
llvm::SignExtend64
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
Definition: MathExtras.h:718
llvm::MachineMemOperand::MOLoad
@ MOLoad
The memory access reads data.
Definition: MachineMemOperand.h:134
llvm::ISD::INTRINSIC_WO_CHAIN
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:184
llvm::ISD::XOR
@ XOR
Definition: ISDOpcodes.h:668
llvm::MVT::getVectorElementCount
ElementCount getVectorElementCount() const
Definition: MachineValueType.h:901
llvm::RISCVISD::RORW
@ RORW
Definition: RISCVISelLowering.h:76
llvm::ISD::INSERT_SUBVECTOR
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:558
llvm::SelectionDAGISel::MF
MachineFunction * MF
Definition: SelectionDAGISel.h:47
llvm::RISCV
Definition: TargetParser.h:158
llvm::isAllOnesConstant
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
Definition: SelectionDAG.cpp:10797
CASE_VMSLT_VMNAND_VMSET_OPCODES
#define CASE_VMSLT_VMNAND_VMSET_OPCODES(lmulenum, suffix, suffix_b)
Alignment.h
selectImm
static SDNode * selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, int64_t Imm, const RISCVSubtarget &Subtarget)
Definition: RISCVISelDAGToDAG.cpp:204
selectImmSeq
static SDNode * selectImmSeq(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, RISCVMatInt::InstSeq &Seq)
Definition: RISCVISelDAGToDAG.cpp:175
llvm::SelectionDAG::computeKnownBits
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
Definition: SelectionDAG.cpp:2910
llvm::KnownBits
Definition: KnownBits.h:23
llvm::RISCVISD::SRLW
@ SRLW
Definition: RISCVISelLowering.h:66
llvm::ISD::EXTRACT_SUBVECTOR
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:572
llvm::isNullConstant
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
Definition: SelectionDAG.cpp:10787
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:348
llvm::RISCV::VSXSEGPseudo
Definition: RISCVISelDAGToDAG.h:174
RISCVISelLowering.h
llvm::RISCVDAGToDAGISel::PostprocessISelDAG
void PostprocessISelDAG() override
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
Definition: RISCVISelDAGToDAG.cpp:152
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
MachineFrameInfo.h
llvm::SelectionDAG::getEntryNode
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:548
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
llvm::RISCVVType::encodeVTYPE
unsigned encodeVTYPE(RISCVII::VLMUL VLMUL, unsigned SEW, bool TailAgnostic, bool MaskAgnostic)
Definition: RISCVBaseInfo.cpp:133
llvm::SelectionDAG::getDataLayout
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:469
llvm::RISCVDAGToDAGISel::selectVLSEGFF
void selectVLSEGFF(SDNode *Node, bool IsMasked)
Definition: RISCVISelDAGToDAG.cpp:356
llvm::SelectionDAG::getTargetExtractSubreg
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
Definition: SelectionDAG.cpp:9934
llvm::SelectionDAGISel::ReplaceUses
void ReplaceUses(SDValue F, SDValue T)
ReplaceUses - replace all uses of the old node F with the use of the new node T.
Definition: SelectionDAGISel.h:210
selectConstantAddr
static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, const RISCVSubtarget *Subtarget, SDValue Addr, SDValue &Base, SDValue &Offset)
Definition: RISCVISelDAGToDAG.cpp:1903
llvm::MVT::i32
@ i32
Definition: MachineValueType.h:48
llvm::SystemZISD::PACK
@ PACK
Definition: SystemZISelLowering.h:204
llvm::RISCVSubtarget::getXLen
unsigned getXLen() const
Definition: RISCVSubtarget.h:212
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:145
llvm::SDNode::getNumValues
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Definition: SelectionDAGNodes.h:983
llvm::RISCVTargetLowering
Definition: RISCVISelLowering.h:334
llvm::RISCVMatInt::RegReg
@ RegReg
Definition: RISCVMatInt.h:24
llvm::XCoreISD::LMUL
@ LMUL
Definition: XCoreISelLowering.h:59
llvm::countLeadingZeros
unsigned countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the most significant bit to the least stopping at the first 1.
Definition: MathExtras.h:220
llvm::ISD::STORE
@ STORE
Definition: ISDOpcodes.h:967
llvm::RISCVTargetLowering::getLMUL
static RISCVII::VLMUL getLMUL(MVT VT)
Definition: RISCVISelLowering.cpp:1550
llvm::AMDGPU::Hwreg::Width
Width
Definition: SIDefines.h:439
llvm::ISD::ADD
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
llvm::RISCVISD::VFMV_V_F_VL
@ VFMV_V_F_VL
Definition: RISCVISelLowering.h:138
llvm::SDValue::isUndef
bool isUndef() const
Definition: SelectionDAGNodes.h:1177
VLMul
RISCVII::VLMUL VLMul
Definition: RISCVInsertVSETVLI.cpp:639
llvm::RISCVMatInt::Imm
@ Imm
Definition: RISCVMatInt.h:23
llvm::RISCVDAGToDAGISel::tryShrinkShlLogicImm
bool tryShrinkShlLogicImm(SDNode *Node)
Definition: RISCVISelDAGToDAG.cpp:599
llvm::SDNode::op_begin
op_iterator op_begin() const
Definition: SelectionDAGNodes.h:928
SEW
unsigned SEW
Definition: RISCVInsertVSETVLI.cpp:643
llvm::RISCVII::LMUL_F8
@ LMUL_F8
Definition: RISCVBaseInfo.h:114
llvm::ISD::SHL
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:691
llvm::MachinePointerInfo::getFixedStack
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Definition: MachineOperand.cpp:1019
llvm::RISCVMatInt::RegImm
@ RegImm
Definition: RISCVMatInt.h:22
llvm::ISD::MUL
@ MUL
Definition: ISDOpcodes.h:241
llvm::SDValue::getConstantOperandAPInt
const APInt & getConstantOperandAPInt(unsigned i) const
Definition: SelectionDAGNodes.h:1157
llvm::MVT::f16
@ f16
Definition: MachineValueType.h:56
llvm::SDNode::hasPredecessorHelper
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
Definition: SelectionDAGNodes.h:849
N
#define N
llvm::ISD::SRL
@ SRL
Definition: ISDOpcodes.h:693
RISCVMachineFunctionInfo.h
llvm::RISCVDAGToDAGISel::selectRVVSimm5
bool selectRVVSimm5(SDValue N, unsigned Width, SDValue &Imm)
Definition: RISCVISelDAGToDAG.cpp:2512
llvm::ArrayRef::size
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:164
llvm::RISCVII::LMUL_F4
@ LMUL_F4
Definition: RISCVBaseInfo.h:115
llvm::RISCVDAGToDAGISel::Select
void Select(SDNode *Node) override
Main hook for targets to transform nodes into machine nodes.
Definition: RISCVISelDAGToDAG.cpp:677
llvm::RISCVII::VLMUL
VLMUL
Definition: RISCVBaseInfo.h:108
llvm::MVT::Untyped
@ Untyped
Definition: MachineValueType.h:286
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
llvm::ISD::MULHU
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:637
llvm::SDValue::getOpcode
unsigned getOpcode() const
Definition: SelectionDAGNodes.h:1137
llvm::SelectionDAG::getTargetConstant
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:669
TM
const char LLVMTargetMachineRef TM
Definition: PassBuilderBindings.cpp:47
getLastNonGlueOrChainOpIdx
static unsigned getLastNonGlueOrChainOpIdx(const SDNode *Node)
Definition: RISCVISelDAGToDAG.cpp:44
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:308
llvm::ISD::INTRINSIC_W_CHAIN
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:192
CASE_VMXOR_VMANDN_VMOR_OPCODES
#define CASE_VMXOR_VMANDN_VMOR_OPCODES(lmulenum, suffix)
llvm::SelectionDAG::getMachineFunction
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:466
llvm::SelectionDAG::ComputeNumSignBits
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
Definition: SelectionDAG.cpp:3935
llvm::RISCVII::hasDummyMaskOp
static bool hasDummyMaskOp(uint64_t TSFlags)
Definition: RISCVBaseInfo.h:139
llvm::MCInstrInfo::get
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
Definition: MCInstrInfo.h:63
llvm::RISCVII::TAIL_AGNOSTIC
@ TAIL_AGNOSTIC
Definition: RISCVBaseInfo.h:120
llvm::isMask_64
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
Definition: MathExtras.h:440
llvm::RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL
@ SPLAT_VECTOR_SPLIT_I64_VL
Definition: RISCVISelLowering.h:149
llvm::APInt::getBitsSetFrom
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition: APInt.h:269
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::M1
unsigned M1(unsigned Val)
Definition: VE.h:466
raw_ostream.h
llvm::SDValue::isMachineOpcode
bool isMachineOpcode() const
Definition: SelectionDAGNodes.h:1169
llvm::RISCVDAGToDAGISel::selectSHXADDOp
bool selectSHXADDOp(SDValue N, unsigned ShAmt, SDValue &Val)
Look for various patterns that can be done with a SHL that can be folded into a SHXADD.
Definition: RISCVISelDAGToDAG.cpp:2146
llvm::SDNode::getFlags
SDNodeFlags getFlags() const
Definition: SelectionDAGNodes.h:972
llvm::RISCV::VLEPseudo
Definition: RISCVISelDAGToDAG.h:184
llvm::MVT::f32
@ f32
Definition: MachineValueType.h:57
llvm::RISCVDAGToDAGISel::hasAllHUsers
bool hasAllHUsers(SDNode *Node) const
Definition: RISCVISelDAGToDAG.h:79
ValidateFn
bool(*)(int64_t) ValidateFn
Definition: RISCVISelDAGToDAG.cpp:2441
llvm::RISCVISD::CTZW
@ CTZW
Definition: RISCVISelLowering.h:80
llvm::RISCVDAGToDAGISel::hasAllNBitUsers
bool hasAllNBitUsers(SDNode *Node, unsigned Bits) const
Definition: RISCVISelDAGToDAG.cpp:2286
Debug.h
llvm::ISD::ATOMIC_LOAD
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1159
llvm::RISCVDAGToDAGISel::selectSExti32
bool selectSExti32(SDValue N, SDValue &Val)
Definition: RISCVISelDAGToDAG.cpp:2110
llvm::TargetLoweringBase::getPointerTy
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
Definition: TargetLowering.h:356
getVecPolicyOpIdx
static unsigned getVecPolicyOpIdx(const SDNode *Node, const MCInstrDesc &MCID)
Definition: RISCVISelDAGToDAG.cpp:54
llvm::ISD::TokenFactor
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
llvm::RISCVII::LMUL_F2
@ LMUL_F2
Definition: RISCVBaseInfo.h:116
llvm::sampleprof::Base
@ Base
Definition: Discriminator.h:58
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
llvm::RISCVDAGToDAGISel::addVectorLoadStoreOperands
void addVectorLoadStoreOperands(SDNode *Node, unsigned SEWImm, const SDLoc &DL, unsigned CurOp, bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl< SDValue > &Operands, bool IsLoad=false, MVT *IndexVT=nullptr)
Definition: RISCVISelDAGToDAG.cpp:266
isAllUndef
static bool isAllUndef(ArrayRef< SDValue > Values)
Definition: RISCVISelDAGToDAG.cpp:309
Generated on Tue Nov 29 2022 22:38:14 for LLVM by   doxygen 1.8.17