LLVM 18.0.0git
RISCVISelDAGToDAG.cpp
Go to the documentation of this file.
1//===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISC-V -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the RISC-V target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVISelDAGToDAG.h"
17#include "RISCVISelLowering.h"
20#include "llvm/IR/IntrinsicsRISCV.h"
22#include "llvm/Support/Debug.h"
25#include <optional>
26
27using namespace llvm;
28
29#define DEBUG_TYPE "riscv-isel"
30#define PASS_NAME "RISC-V DAG->DAG Pattern Instruction Selection"
31
32namespace llvm::RISCV {
33#define GET_RISCVVSSEGTable_IMPL
34#define GET_RISCVVLSEGTable_IMPL
35#define GET_RISCVVLXSEGTable_IMPL
36#define GET_RISCVVSXSEGTable_IMPL
37#define GET_RISCVVLETable_IMPL
38#define GET_RISCVVSETable_IMPL
39#define GET_RISCVVLXTable_IMPL
40#define GET_RISCVVSXTable_IMPL
41#define GET_RISCVMaskedPseudosTable_IMPL
42#include "RISCVGenSearchableTables.inc"
43} // namespace llvm::RISCV
44
47
48 bool MadeChange = false;
49 while (Position != CurDAG->allnodes_begin()) {
50 SDNode *N = &*--Position;
51 if (N->use_empty())
52 continue;
53
54 SDValue Result;
55 switch (N->getOpcode()) {
56 case ISD::SPLAT_VECTOR: {
57 // Convert integer SPLAT_VECTOR to VMV_V_X_VL and floating-point
58 // SPLAT_VECTOR to VFMV_V_F_VL to reduce isel burden.
59 MVT VT = N->getSimpleValueType(0);
60 unsigned Opc =
62 SDLoc DL(N);
63 SDValue VL = CurDAG->getRegister(RISCV::X0, Subtarget->getXLenVT());
64 Result = CurDAG->getNode(Opc, DL, VT, CurDAG->getUNDEF(VT),
65 N->getOperand(0), VL);
66 break;
67 }
69 // Lower SPLAT_VECTOR_SPLIT_I64 to two scalar stores and a stride 0 vector
70 // load. Done after lowering and combining so that we have a chance to
71 // optimize this to VMV_V_X_VL when the upper bits aren't needed.
72 assert(N->getNumOperands() == 4 && "Unexpected number of operands");
73 MVT VT = N->getSimpleValueType(0);
74 SDValue Passthru = N->getOperand(0);
75 SDValue Lo = N->getOperand(1);
76 SDValue Hi = N->getOperand(2);
77 SDValue VL = N->getOperand(3);
78 assert(VT.getVectorElementType() == MVT::i64 && VT.isScalableVector() &&
79 Lo.getValueType() == MVT::i32 && Hi.getValueType() == MVT::i32 &&
80 "Unexpected VTs!");
82 SDLoc DL(N);
83
84 // Create temporary stack for each expanding node.
85 SDValue StackSlot =
87 int FI = cast<FrameIndexSDNode>(StackSlot.getNode())->getIndex();
89
90 SDValue Chain = CurDAG->getEntryNode();
91 Lo = CurDAG->getStore(Chain, DL, Lo, StackSlot, MPI, Align(8));
92
93 SDValue OffsetSlot =
95 Hi = CurDAG->getStore(Chain, DL, Hi, OffsetSlot, MPI.getWithOffset(4),
96 Align(8));
97
98 Chain = CurDAG->getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
99
100 SDVTList VTs = CurDAG->getVTList({VT, MVT::Other});
101 SDValue IntID =
102 CurDAG->getTargetConstant(Intrinsic::riscv_vlse, DL, MVT::i64);
103 SDValue Ops[] = {Chain,
104 IntID,
105 Passthru,
106 StackSlot,
107 CurDAG->getRegister(RISCV::X0, MVT::i64),
108 VL};
109
111 MVT::i64, MPI, Align(8),
113 break;
114 }
115 }
116
117 if (Result) {
118 LLVM_DEBUG(dbgs() << "RISC-V DAG preprocessing replacing:\nOld: ");
119 LLVM_DEBUG(N->dump(CurDAG));
120 LLVM_DEBUG(dbgs() << "\nNew: ");
121 LLVM_DEBUG(Result->dump(CurDAG));
122 LLVM_DEBUG(dbgs() << "\n");
123
125 MadeChange = true;
126 }
127 }
128
129 if (MadeChange)
131}
132
134 HandleSDNode Dummy(CurDAG->getRoot());
136
137 bool MadeChange = false;
138 while (Position != CurDAG->allnodes_begin()) {
139 SDNode *N = &*--Position;
140 // Skip dead nodes and any non-machine opcodes.
141 if (N->use_empty() || !N->isMachineOpcode())
142 continue;
143
144 MadeChange |= doPeepholeSExtW(N);
145 MadeChange |= doPeepholeMaskedRVV(cast<MachineSDNode>(N));
146 }
147
148 CurDAG->setRoot(Dummy.getValue());
149
150 MadeChange |= doPeepholeMergeVVMFold();
151
152 // After we're done with everything else, convert IMPLICIT_DEF
153 // passthru operands to NoRegister. This is required to workaround
154 // an optimization deficiency in MachineCSE. This really should
155 // be merged back into each of the patterns (i.e. there's no good
156 // reason not to go directly to NoReg), but is being done this way
157 // to allow easy backporting.
158 MadeChange |= doPeepholeNoRegPassThru();
159
160 if (MadeChange)
162}
163
164static SDValue selectImmSeq(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
166 SDValue SrcReg = CurDAG->getRegister(RISCV::X0, VT);
167 for (const RISCVMatInt::Inst &Inst : Seq) {
168 SDValue SDImm = CurDAG->getTargetConstant(Inst.getImm(), DL, VT);
169 SDNode *Result = nullptr;
170 switch (Inst.getOpndKind()) {
171 case RISCVMatInt::Imm:
172 Result = CurDAG->getMachineNode(Inst.getOpcode(), DL, VT, SDImm);
173 break;
175 Result = CurDAG->getMachineNode(Inst.getOpcode(), DL, VT, SrcReg,
176 CurDAG->getRegister(RISCV::X0, VT));
177 break;
179 Result = CurDAG->getMachineNode(Inst.getOpcode(), DL, VT, SrcReg, SrcReg);
180 break;
182 Result = CurDAG->getMachineNode(Inst.getOpcode(), DL, VT, SrcReg, SDImm);
183 break;
184 }
185
186 // Only the first instruction has X0 as its source.
187 SrcReg = SDValue(Result, 0);
188 }
189
190 return SrcReg;
191}
192
193static SDValue selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
194 int64_t Imm, const RISCVSubtarget &Subtarget) {
196 RISCVMatInt::generateInstSeq(Imm, Subtarget.getFeatureBits());
197
198 // See if we can create this constant as (ADD (SLLI X, 32), X) where X is at
199 // worst an LUI+ADDIW. This will require an extra register, but avoids a
200 // constant pool.
201 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
202 // low and high 32 bits are the same and bit 31 and 63 are set.
203 if (Seq.size() > 3) {
204 int64_t LoVal = SignExtend64<32>(Imm);
205 int64_t HiVal = SignExtend64<32>(((uint64_t)Imm - (uint64_t)LoVal) >> 32);
206 if (LoVal == HiVal ||
207 (Subtarget.hasStdExtZba() && Lo_32(Imm) == Hi_32(Imm))) {
209 RISCVMatInt::generateInstSeq(LoVal, Subtarget.getFeatureBits());
210 if ((SeqLo.size() + 2) < Seq.size()) {
211 SDValue Lo = selectImmSeq(CurDAG, DL, VT, SeqLo);
212
213 SDValue SLLI = SDValue(
214 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, Lo,
215 CurDAG->getTargetConstant(32, DL, VT)),
216 0);
217 // Prefer ADD when possible.
218 unsigned AddOpc = (LoVal == HiVal) ? RISCV::ADD : RISCV::ADD_UW;
219 return SDValue(CurDAG->getMachineNode(AddOpc, DL, VT, Lo, SLLI), 0);
220 }
221 }
222 }
223
224 // Otherwise, use the original sequence.
225 return selectImmSeq(CurDAG, DL, VT, Seq);
226}
227
229 unsigned NF, RISCVII::VLMUL LMUL) {
230 static const unsigned M1TupleRegClassIDs[] = {
231 RISCV::VRN2M1RegClassID, RISCV::VRN3M1RegClassID, RISCV::VRN4M1RegClassID,
232 RISCV::VRN5M1RegClassID, RISCV::VRN6M1RegClassID, RISCV::VRN7M1RegClassID,
233 RISCV::VRN8M1RegClassID};
234 static const unsigned M2TupleRegClassIDs[] = {RISCV::VRN2M2RegClassID,
235 RISCV::VRN3M2RegClassID,
236 RISCV::VRN4M2RegClassID};
237
238 assert(Regs.size() >= 2 && Regs.size() <= 8);
239
240 unsigned RegClassID;
241 unsigned SubReg0;
242 switch (LMUL) {
243 default:
244 llvm_unreachable("Invalid LMUL.");
249 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
250 "Unexpected subreg numbering");
251 SubReg0 = RISCV::sub_vrm1_0;
252 RegClassID = M1TupleRegClassIDs[NF - 2];
253 break;
255 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
256 "Unexpected subreg numbering");
257 SubReg0 = RISCV::sub_vrm2_0;
258 RegClassID = M2TupleRegClassIDs[NF - 2];
259 break;
261 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
262 "Unexpected subreg numbering");
263 SubReg0 = RISCV::sub_vrm4_0;
264 RegClassID = RISCV::VRN2M4RegClassID;
265 break;
266 }
267
268 SDLoc DL(Regs[0]);
270
271 Ops.push_back(CurDAG.getTargetConstant(RegClassID, DL, MVT::i32));
272
273 for (unsigned I = 0; I < Regs.size(); ++I) {
274 Ops.push_back(Regs[I]);
275 Ops.push_back(CurDAG.getTargetConstant(SubReg0 + I, DL, MVT::i32));
276 }
277 SDNode *N =
278 CurDAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops);
279 return SDValue(N, 0);
280}
281
283 SDNode *Node, unsigned Log2SEW, const SDLoc &DL, unsigned CurOp,
284 bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl<SDValue> &Operands,
285 bool IsLoad, MVT *IndexVT) {
286 SDValue Chain = Node->getOperand(0);
287 SDValue Glue;
288
289 Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
290
291 if (IsStridedOrIndexed) {
292 Operands.push_back(Node->getOperand(CurOp++)); // Index.
293 if (IndexVT)
294 *IndexVT = Operands.back()->getSimpleValueType(0);
295 }
296
297 if (IsMasked) {
298 // Mask needs to be copied to V0.
299 SDValue Mask = Node->getOperand(CurOp++);
300 Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
301 Glue = Chain.getValue(1);
302 Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
303 }
304 SDValue VL;
305 selectVLOp(Node->getOperand(CurOp++), VL);
306 Operands.push_back(VL);
307
308 MVT XLenVT = Subtarget->getXLenVT();
309 SDValue SEWOp = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT);
310 Operands.push_back(SEWOp);
311
312 // At the IR layer, all the masked load intrinsics have policy operands,
313 // none of the others do. All have passthru operands. For our pseudos,
314 // all loads have policy operands.
315 if (IsLoad) {
317 if (IsMasked)
318 Policy = Node->getConstantOperandVal(CurOp++);
319 SDValue PolicyOp = CurDAG->getTargetConstant(Policy, DL, XLenVT);
320 Operands.push_back(PolicyOp);
321 }
322
323 Operands.push_back(Chain); // Chain.
324 if (Glue)
325 Operands.push_back(Glue);
326}
327
328void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, bool IsMasked,
329 bool IsStrided) {
330 SDLoc DL(Node);
331 unsigned NF = Node->getNumValues() - 1;
332 MVT VT = Node->getSimpleValueType(0);
333 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
335
336 unsigned CurOp = 2;
338
339 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
340 Node->op_begin() + CurOp + NF);
341 SDValue Merge = createTuple(*CurDAG, Regs, NF, LMUL);
342 Operands.push_back(Merge);
343 CurOp += NF;
344
345 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
346 Operands, /*IsLoad=*/true);
347
348 const RISCV::VLSEGPseudo *P =
349 RISCV::getVLSEGPseudo(NF, IsMasked, IsStrided, /*FF*/ false, Log2SEW,
350 static_cast<unsigned>(LMUL));
351 MachineSDNode *Load =
352 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
353
354 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
355 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
356
357 SDValue SuperReg = SDValue(Load, 0);
358 for (unsigned I = 0; I < NF; ++I) {
359 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
360 ReplaceUses(SDValue(Node, I),
361 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
362 }
363
364 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
365 CurDAG->RemoveDeadNode(Node);
366}
367
368void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, bool IsMasked) {
369 SDLoc DL(Node);
370 unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain.
371 MVT VT = Node->getSimpleValueType(0);
372 MVT XLenVT = Subtarget->getXLenVT();
373 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
375
376 unsigned CurOp = 2;
378
379 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
380 Node->op_begin() + CurOp + NF);
381 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
382 Operands.push_back(MaskedOff);
383 CurOp += NF;
384
385 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
386 /*IsStridedOrIndexed*/ false, Operands,
387 /*IsLoad=*/true);
388
389 const RISCV::VLSEGPseudo *P =
390 RISCV::getVLSEGPseudo(NF, IsMasked, /*Strided*/ false, /*FF*/ true,
391 Log2SEW, static_cast<unsigned>(LMUL));
392 MachineSDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped,
393 XLenVT, MVT::Other, Operands);
394
395 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
396 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
397
398 SDValue SuperReg = SDValue(Load, 0);
399 for (unsigned I = 0; I < NF; ++I) {
400 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
401 ReplaceUses(SDValue(Node, I),
402 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
403 }
404
405 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); // VL
406 ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 2)); // Chain
407 CurDAG->RemoveDeadNode(Node);
408}
409
410void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, bool IsMasked,
411 bool IsOrdered) {
412 SDLoc DL(Node);
413 unsigned NF = Node->getNumValues() - 1;
414 MVT VT = Node->getSimpleValueType(0);
415 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
417
418 unsigned CurOp = 2;
420
421 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
422 Node->op_begin() + CurOp + NF);
423 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
424 Operands.push_back(MaskedOff);
425 CurOp += NF;
426
427 MVT IndexVT;
428 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
429 /*IsStridedOrIndexed*/ true, Operands,
430 /*IsLoad=*/true, &IndexVT);
431
433 "Element count mismatch");
434
435 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
436 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
437 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
438 report_fatal_error("The V extension does not support EEW=64 for index "
439 "values when XLEN=32");
440 }
441 const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo(
442 NF, IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
443 static_cast<unsigned>(IndexLMUL));
444 MachineSDNode *Load =
445 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
446
447 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
448 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
449
450 SDValue SuperReg = SDValue(Load, 0);
451 for (unsigned I = 0; I < NF; ++I) {
452 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
453 ReplaceUses(SDValue(Node, I),
454 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
455 }
456
457 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
458 CurDAG->RemoveDeadNode(Node);
459}
460
461void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, bool IsMasked,
462 bool IsStrided) {
463 SDLoc DL(Node);
464 unsigned NF = Node->getNumOperands() - 4;
465 if (IsStrided)
466 NF--;
467 if (IsMasked)
468 NF--;
469 MVT VT = Node->getOperand(2)->getSimpleValueType(0);
470 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
472 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
473 SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
474
476 Operands.push_back(StoreVal);
477 unsigned CurOp = 2 + NF;
478
479 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
480 Operands);
481
482 const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo(
483 NF, IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL));
484 MachineSDNode *Store =
485 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
486
487 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
488 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
489
490 ReplaceNode(Node, Store);
491}
492
493void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, bool IsMasked,
494 bool IsOrdered) {
495 SDLoc DL(Node);
496 unsigned NF = Node->getNumOperands() - 5;
497 if (IsMasked)
498 --NF;
499 MVT VT = Node->getOperand(2)->getSimpleValueType(0);
500 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
502 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
503 SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
504
506 Operands.push_back(StoreVal);
507 unsigned CurOp = 2 + NF;
508
509 MVT IndexVT;
510 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
511 /*IsStridedOrIndexed*/ true, Operands,
512 /*IsLoad=*/false, &IndexVT);
513
515 "Element count mismatch");
516
517 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
518 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
519 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
520 report_fatal_error("The V extension does not support EEW=64 for index "
521 "values when XLEN=32");
522 }
523 const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo(
524 NF, IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
525 static_cast<unsigned>(IndexLMUL));
526 MachineSDNode *Store =
527 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
528
529 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
530 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
531
532 ReplaceNode(Node, Store);
533}
534
536 if (!Subtarget->hasVInstructions())
537 return;
538
539 assert(Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN && "Unexpected opcode");
540
541 SDLoc DL(Node);
542 MVT XLenVT = Subtarget->getXLenVT();
543
544 unsigned IntNo = Node->getConstantOperandVal(0);
545
546 assert((IntNo == Intrinsic::riscv_vsetvli ||
547 IntNo == Intrinsic::riscv_vsetvlimax) &&
548 "Unexpected vsetvli intrinsic");
549
550 bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax;
551 unsigned Offset = (VLMax ? 1 : 2);
552
553 assert(Node->getNumOperands() == Offset + 2 &&
554 "Unexpected number of operands");
555
556 unsigned SEW =
557 RISCVVType::decodeVSEW(Node->getConstantOperandVal(Offset) & 0x7);
558 RISCVII::VLMUL VLMul = static_cast<RISCVII::VLMUL>(
559 Node->getConstantOperandVal(Offset + 1) & 0x7);
560
561 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMul, SEW, /*TailAgnostic*/ true,
562 /*MaskAgnostic*/ true);
563 SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT);
564
565 SDValue VLOperand;
566 unsigned Opcode = RISCV::PseudoVSETVLI;
567 if (auto *C = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
568 const unsigned VLEN = Subtarget->getRealMinVLen();
569 if (VLEN == Subtarget->getRealMaxVLen())
570 if (VLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul) == C->getZExtValue())
571 VLMax = true;
572 }
573 if (VLMax || isAllOnesConstant(Node->getOperand(1))) {
574 VLOperand = CurDAG->getRegister(RISCV::X0, XLenVT);
575 Opcode = RISCV::PseudoVSETVLIX0;
576 } else {
577 VLOperand = Node->getOperand(1);
578
579 if (auto *C = dyn_cast<ConstantSDNode>(VLOperand)) {
580 uint64_t AVL = C->getZExtValue();
581 if (isUInt<5>(AVL)) {
582 SDValue VLImm = CurDAG->getTargetConstant(AVL, DL, XLenVT);
583 ReplaceNode(Node, CurDAG->getMachineNode(RISCV::PseudoVSETIVLI, DL,
584 XLenVT, VLImm, VTypeIOp));
585 return;
586 }
587 }
588 }
589
590 ReplaceNode(Node,
591 CurDAG->getMachineNode(Opcode, DL, XLenVT, VLOperand, VTypeIOp));
592}
593
595 MVT VT = Node->getSimpleValueType(0);
596 unsigned Opcode = Node->getOpcode();
597 assert((Opcode == ISD::AND || Opcode == ISD::OR || Opcode == ISD::XOR) &&
598 "Unexpected opcode");
599 SDLoc DL(Node);
600
601 // For operations of the form (x << C1) op C2, check if we can use
602 // ANDI/ORI/XORI by transforming it into (x op (C2>>C1)) << C1.
603 SDValue N0 = Node->getOperand(0);
604 SDValue N1 = Node->getOperand(1);
605
606 ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(N1);
607 if (!Cst)
608 return false;
609
610 int64_t Val = Cst->getSExtValue();
611
612 // Check if immediate can already use ANDI/ORI/XORI.
613 if (isInt<12>(Val))
614 return false;
615
616 SDValue Shift = N0;
617
618 // If Val is simm32 and we have a sext_inreg from i32, then the binop
619 // produces at least 33 sign bits. We can peek through the sext_inreg and use
620 // a SLLIW at the end.
621 bool SignExt = false;
622 if (isInt<32>(Val) && N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
623 N0.hasOneUse() && cast<VTSDNode>(N0.getOperand(1))->getVT() == MVT::i32) {
624 SignExt = true;
625 Shift = N0.getOperand(0);
626 }
627
628 if (Shift.getOpcode() != ISD::SHL || !Shift.hasOneUse())
629 return false;
630
631 ConstantSDNode *ShlCst = dyn_cast<ConstantSDNode>(Shift.getOperand(1));
632 if (!ShlCst)
633 return false;
634
635 uint64_t ShAmt = ShlCst->getZExtValue();
636
637 // Make sure that we don't change the operation by removing bits.
638 // This only matters for OR and XOR, AND is unaffected.
639 uint64_t RemovedBitsMask = maskTrailingOnes<uint64_t>(ShAmt);
640 if (Opcode != ISD::AND && (Val & RemovedBitsMask) != 0)
641 return false;
642
643 int64_t ShiftedVal = Val >> ShAmt;
644 if (!isInt<12>(ShiftedVal))
645 return false;
646
647 // If we peeked through a sext_inreg, make sure the shift is valid for SLLIW.
648 if (SignExt && ShAmt >= 32)
649 return false;
650
651 // Ok, we can reorder to get a smaller immediate.
652 unsigned BinOpc;
653 switch (Opcode) {
654 default: llvm_unreachable("Unexpected opcode");
655 case ISD::AND: BinOpc = RISCV::ANDI; break;
656 case ISD::OR: BinOpc = RISCV::ORI; break;
657 case ISD::XOR: BinOpc = RISCV::XORI; break;
658 }
659
660 unsigned ShOpc = SignExt ? RISCV::SLLIW : RISCV::SLLI;
661
662 SDNode *BinOp =
663 CurDAG->getMachineNode(BinOpc, DL, VT, Shift.getOperand(0),
664 CurDAG->getTargetConstant(ShiftedVal, DL, VT));
665 SDNode *SLLI =
666 CurDAG->getMachineNode(ShOpc, DL, VT, SDValue(BinOp, 0),
667 CurDAG->getTargetConstant(ShAmt, DL, VT));
668 ReplaceNode(Node, SLLI);
669 return true;
670}
671
673 // Only supported with XTHeadBb at the moment.
674 if (!Subtarget->hasVendorXTHeadBb())
675 return false;
676
677 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
678 if (!N1C)
679 return false;
680
681 SDValue N0 = Node->getOperand(0);
682 if (!N0.hasOneUse())
683 return false;
684
685 auto BitfieldExtract = [&](SDValue N0, unsigned Msb, unsigned Lsb, SDLoc DL,
686 MVT VT) {
687 return CurDAG->getMachineNode(RISCV::TH_EXT, DL, VT, N0.getOperand(0),
688 CurDAG->getTargetConstant(Msb, DL, VT),
689 CurDAG->getTargetConstant(Lsb, DL, VT));
690 };
691
692 SDLoc DL(Node);
693 MVT VT = Node->getSimpleValueType(0);
694 const unsigned RightShAmt = N1C->getZExtValue();
695
696 // Transform (sra (shl X, C1) C2) with C1 < C2
697 // -> (TH.EXT X, msb, lsb)
698 if (N0.getOpcode() == ISD::SHL) {
699 auto *N01C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
700 if (!N01C)
701 return false;
702
703 const unsigned LeftShAmt = N01C->getZExtValue();
704 // Make sure that this is a bitfield extraction (i.e., the shift-right
705 // amount can not be less than the left-shift).
706 if (LeftShAmt > RightShAmt)
707 return false;
708
709 const unsigned MsbPlusOne = VT.getSizeInBits() - LeftShAmt;
710 const unsigned Msb = MsbPlusOne - 1;
711 const unsigned Lsb = RightShAmt - LeftShAmt;
712
713 SDNode *TH_EXT = BitfieldExtract(N0, Msb, Lsb, DL, VT);
714 ReplaceNode(Node, TH_EXT);
715 return true;
716 }
717
718 // Transform (sra (sext_inreg X, _), C) ->
719 // (TH.EXT X, msb, lsb)
720 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) {
721 unsigned ExtSize =
722 cast<VTSDNode>(N0.getOperand(1))->getVT().getSizeInBits();
723
724 // ExtSize of 32 should use sraiw via tablegen pattern.
725 if (ExtSize == 32)
726 return false;
727
728 const unsigned Msb = ExtSize - 1;
729 const unsigned Lsb = RightShAmt;
730
731 SDNode *TH_EXT = BitfieldExtract(N0, Msb, Lsb, DL, VT);
732 ReplaceNode(Node, TH_EXT);
733 return true;
734 }
735
736 return false;
737}
738
740 // Target does not support indexed loads.
741 if (!Subtarget->hasVendorXTHeadMemIdx())
742 return false;
743
744 LoadSDNode *Ld = cast<LoadSDNode>(Node);
746 if (AM == ISD::UNINDEXED)
747 return false;
748
749 const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Ld->getOffset());
750 if (!C)
751 return false;
752
753 EVT LoadVT = Ld->getMemoryVT();
754 bool IsPre = (AM == ISD::PRE_INC || AM == ISD::PRE_DEC);
755 bool IsPost = (AM == ISD::POST_INC || AM == ISD::POST_DEC);
756 int64_t Offset = C->getSExtValue();
757
758 // Convert decrements to increments by a negative quantity.
759 if (AM == ISD::PRE_DEC || AM == ISD::POST_DEC)
760 Offset = -Offset;
761
762 // The constants that can be encoded in the THeadMemIdx instructions
763 // are of the form (sign_extend(imm5) << imm2).
764 int64_t Shift;
765 for (Shift = 0; Shift < 4; Shift++)
766 if (isInt<5>(Offset >> Shift) && ((Offset % (1LL << Shift)) == 0))
767 break;
768
769 // Constant cannot be encoded.
770 if (Shift == 4)
771 return false;
772
773 bool IsZExt = (Ld->getExtensionType() == ISD::ZEXTLOAD);
774 unsigned Opcode;
775 if (LoadVT == MVT::i8 && IsPre)
776 Opcode = IsZExt ? RISCV::TH_LBUIB : RISCV::TH_LBIB;
777 else if (LoadVT == MVT::i8 && IsPost)
778 Opcode = IsZExt ? RISCV::TH_LBUIA : RISCV::TH_LBIA;
779 else if (LoadVT == MVT::i16 && IsPre)
780 Opcode = IsZExt ? RISCV::TH_LHUIB : RISCV::TH_LHIB;
781 else if (LoadVT == MVT::i16 && IsPost)
782 Opcode = IsZExt ? RISCV::TH_LHUIA : RISCV::TH_LHIA;
783 else if (LoadVT == MVT::i32 && IsPre)
784 Opcode = IsZExt ? RISCV::TH_LWUIB : RISCV::TH_LWIB;
785 else if (LoadVT == MVT::i32 && IsPost)
786 Opcode = IsZExt ? RISCV::TH_LWUIA : RISCV::TH_LWIA;
787 else if (LoadVT == MVT::i64 && IsPre)
788 Opcode = RISCV::TH_LDIB;
789 else if (LoadVT == MVT::i64 && IsPost)
790 Opcode = RISCV::TH_LDIA;
791 else
792 return false;
793
794 EVT Ty = Ld->getOffset().getValueType();
795 SDValue Ops[] = {Ld->getBasePtr(),
796 CurDAG->getTargetConstant(Offset >> Shift, SDLoc(Node), Ty),
797 CurDAG->getTargetConstant(Shift, SDLoc(Node), Ty),
798 Ld->getChain()};
799 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(Node), Ld->getValueType(0),
800 Ld->getValueType(1), MVT::Other, Ops);
801
802 MachineMemOperand *MemOp = cast<MemSDNode>(Node)->getMemOperand();
803 CurDAG->setNodeMemRefs(cast<MachineSDNode>(New), {MemOp});
804
805 ReplaceNode(Node, New);
806
807 return true;
808}
809
811 // If we have a custom node, we have already selected.
812 if (Node->isMachineOpcode()) {
813 LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n");
814 Node->setNodeId(-1);
815 return;
816 }
817
818 // Instruction Selection not handled by the auto-generated tablegen selection
819 // should be handled here.
820 unsigned Opcode = Node->getOpcode();
821 MVT XLenVT = Subtarget->getXLenVT();
822 SDLoc DL(Node);
823 MVT VT = Node->getSimpleValueType(0);
824
825 bool HasBitTest = Subtarget->hasStdExtZbs() || Subtarget->hasVendorXTHeadBs();
826
827 switch (Opcode) {
828 case ISD::Constant: {
829 assert(VT == Subtarget->getXLenVT() && "Unexpected VT");
830 auto *ConstNode = cast<ConstantSDNode>(Node);
831 if (ConstNode->isZero()) {
832 SDValue New =
833 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, VT);
834 ReplaceNode(Node, New.getNode());
835 return;
836 }
837 int64_t Imm = ConstNode->getSExtValue();
838 // If the upper XLen-16 bits are not used, try to convert this to a simm12
839 // by sign extending bit 15.
840 if (isUInt<16>(Imm) && isInt<12>(SignExtend64<16>(Imm)) &&
841 hasAllHUsers(Node))
842 Imm = SignExtend64<16>(Imm);
843 // If the upper 32-bits are not used try to convert this into a simm32 by
844 // sign extending bit 32.
845 if (!isInt<32>(Imm) && isUInt<32>(Imm) && hasAllWUsers(Node))
846 Imm = SignExtend64<32>(Imm);
847
848 ReplaceNode(Node, selectImm(CurDAG, DL, VT, Imm, *Subtarget).getNode());
849 return;
850 }
851 case ISD::ConstantFP: {
852 const APFloat &APF = cast<ConstantFPSDNode>(Node)->getValueAPF();
853 int FPImm = static_cast<const RISCVTargetLowering *>(TLI)->getLegalZfaFPImm(
854 APF, VT);
855 if (FPImm >= 0) {
856 unsigned Opc;
857 switch (VT.SimpleTy) {
858 default:
859 llvm_unreachable("Unexpected size");
860 case MVT::f16:
861 Opc = RISCV::FLI_H;
862 break;
863 case MVT::f32:
864 Opc = RISCV::FLI_S;
865 break;
866 case MVT::f64:
867 Opc = RISCV::FLI_D;
868 break;
869 }
870
872 Opc, DL, VT, CurDAG->getTargetConstant(FPImm, DL, XLenVT));
873 ReplaceNode(Node, Res);
874 return;
875 }
876
877 bool NegZeroF64 = APF.isNegZero() && VT == MVT::f64;
878 SDValue Imm;
879 // For +0.0 or f64 -0.0 we need to start from X0. For all others, we will
880 // create an integer immediate.
881 if (APF.isPosZero() || NegZeroF64)
882 Imm = CurDAG->getRegister(RISCV::X0, XLenVT);
883 else
884 Imm = selectImm(CurDAG, DL, XLenVT, APF.bitcastToAPInt().getSExtValue(),
885 *Subtarget);
886
887 unsigned Opc;
888 switch (VT.SimpleTy) {
889 default:
890 llvm_unreachable("Unexpected size");
891 case MVT::bf16:
892 assert(Subtarget->hasStdExtZfbfmin());
893 Opc = RISCV::FMV_H_X;
894 break;
895 case MVT::f16:
896 Opc =
897 Subtarget->hasStdExtZhinxOrZhinxmin() ? RISCV::COPY : RISCV::FMV_H_X;
898 break;
899 case MVT::f32:
900 Opc = Subtarget->hasStdExtZfinx() ? RISCV::COPY : RISCV::FMV_W_X;
901 break;
902 case MVT::f64:
903 // For RV32, we can't move from a GPR, we need to convert instead. This
904 // should only happen for +0.0 and -0.0.
905 assert((Subtarget->is64Bit() || APF.isZero()) && "Unexpected constant");
906 bool HasZdinx = Subtarget->hasStdExtZdinx();
907 if (Subtarget->is64Bit())
908 Opc = HasZdinx ? RISCV::COPY : RISCV::FMV_D_X;
909 else
910 Opc = HasZdinx ? RISCV::FCVT_D_W_IN32X : RISCV::FCVT_D_W;
911 break;
912 }
913
914 SDNode *Res;
915 if (Opc == RISCV::FCVT_D_W_IN32X || Opc == RISCV::FCVT_D_W)
916 Res = CurDAG->getMachineNode(
917 Opc, DL, VT, Imm,
919 else
920 Res = CurDAG->getMachineNode(Opc, DL, VT, Imm);
921
922 // For f64 -0.0, we need to insert a fneg.d idiom.
923 if (NegZeroF64)
924 Res = CurDAG->getMachineNode(RISCV::FSGNJN_D, DL, VT, SDValue(Res, 0),
925 SDValue(Res, 0));
926
927 ReplaceNode(Node, Res);
928 return;
929 }
930 case RISCVISD::SplitF64: {
931 if (!Subtarget->hasStdExtZfa())
932 break;
933 assert(Subtarget->hasStdExtD() && !Subtarget->is64Bit() &&
934 "Unexpected subtarget");
935
936 // With Zfa, lower to fmv.x.w and fmvh.x.d.
937 if (!SDValue(Node, 0).use_empty()) {
938 SDNode *Lo = CurDAG->getMachineNode(RISCV::FMV_X_W_FPR64, DL, VT,
939 Node->getOperand(0));
940 ReplaceUses(SDValue(Node, 0), SDValue(Lo, 0));
941 }
942 if (!SDValue(Node, 1).use_empty()) {
943 SDNode *Hi = CurDAG->getMachineNode(RISCV::FMVH_X_D, DL, VT,
944 Node->getOperand(0));
945 ReplaceUses(SDValue(Node, 1), SDValue(Hi, 0));
946 }
947
948 CurDAG->RemoveDeadNode(Node);
949 return;
950 }
951 case ISD::SHL: {
952 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
953 if (!N1C)
954 break;
955 SDValue N0 = Node->getOperand(0);
956 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() ||
957 !isa<ConstantSDNode>(N0.getOperand(1)))
958 break;
959 unsigned ShAmt = N1C->getZExtValue();
960 uint64_t Mask = N0.getConstantOperandVal(1);
961
962 // Optimize (shl (and X, C2), C) -> (slli (srliw X, C3), C3+C) where C2 has
963 // 32 leading zeros and C3 trailing zeros.
964 if (ShAmt <= 32 && isShiftedMask_64(Mask)) {
965 unsigned XLen = Subtarget->getXLen();
966 unsigned LeadingZeros = XLen - llvm::bit_width(Mask);
967 unsigned TrailingZeros = llvm::countr_zero(Mask);
968 if (TrailingZeros > 0 && LeadingZeros == 32) {
969 SDNode *SRLIW = CurDAG->getMachineNode(
970 RISCV::SRLIW, DL, VT, N0->getOperand(0),
971 CurDAG->getTargetConstant(TrailingZeros, DL, VT));
973 RISCV::SLLI, DL, VT, SDValue(SRLIW, 0),
974 CurDAG->getTargetConstant(TrailingZeros + ShAmt, DL, VT));
975 ReplaceNode(Node, SLLI);
976 return;
977 }
978 }
979 break;
980 }
981 case ISD::SRL: {
982 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
983 if (!N1C)
984 break;
985 SDValue N0 = Node->getOperand(0);
986 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(N0.getOperand(1)))
987 break;
988 unsigned ShAmt = N1C->getZExtValue();
989 uint64_t Mask = N0.getConstantOperandVal(1);
990
991 // Optimize (srl (and X, C2), C) -> (slli (srliw X, C3), C3-C) where C2 has
992 // 32 leading zeros and C3 trailing zeros.
993 if (isShiftedMask_64(Mask) && N0.hasOneUse()) {
994 unsigned XLen = Subtarget->getXLen();
995 unsigned LeadingZeros = XLen - llvm::bit_width(Mask);
996 unsigned TrailingZeros = llvm::countr_zero(Mask);
997 if (LeadingZeros == 32 && TrailingZeros > ShAmt) {
998 SDNode *SRLIW = CurDAG->getMachineNode(
999 RISCV::SRLIW, DL, VT, N0->getOperand(0),
1000 CurDAG->getTargetConstant(TrailingZeros, DL, VT));
1001 SDNode *SLLI = CurDAG->getMachineNode(
1002 RISCV::SLLI, DL, VT, SDValue(SRLIW, 0),
1003 CurDAG->getTargetConstant(TrailingZeros - ShAmt, DL, VT));
1004 ReplaceNode(Node, SLLI);
1005 return;
1006 }
1007 }
1008
1009 // Optimize (srl (and X, C2), C) ->
1010 // (srli (slli X, (XLen-C3), (XLen-C3) + C)
1011 // Where C2 is a mask with C3 trailing ones.
1012 // Taking into account that the C2 may have had lower bits unset by
1013 // SimplifyDemandedBits. This avoids materializing the C2 immediate.
1014 // This pattern occurs when type legalizing right shifts for types with
1015 // less than XLen bits.
1016 Mask |= maskTrailingOnes<uint64_t>(ShAmt);
1017 if (!isMask_64(Mask))
1018 break;
1019 unsigned TrailingOnes = llvm::countr_one(Mask);
1020 if (ShAmt >= TrailingOnes)
1021 break;
1022 // If the mask has 32 trailing ones, use SRLIW.
1023 if (TrailingOnes == 32) {
1024 SDNode *SRLIW =
1025 CurDAG->getMachineNode(RISCV::SRLIW, DL, VT, N0->getOperand(0),
1026 CurDAG->getTargetConstant(ShAmt, DL, VT));
1027 ReplaceNode(Node, SRLIW);
1028 return;
1029 }
1030
1031 // Only do the remaining transforms if the AND has one use.
1032 if (!N0.hasOneUse())
1033 break;
1034
1035 // If C2 is (1 << ShAmt) use bexti or th.tst if possible.
1036 if (HasBitTest && ShAmt + 1 == TrailingOnes) {
1037 SDNode *BEXTI = CurDAG->getMachineNode(
1038 Subtarget->hasStdExtZbs() ? RISCV::BEXTI : RISCV::TH_TST, DL, VT,
1039 N0->getOperand(0), CurDAG->getTargetConstant(ShAmt, DL, VT));
1040 ReplaceNode(Node, BEXTI);
1041 return;
1042 }
1043
1044 unsigned LShAmt = Subtarget->getXLen() - TrailingOnes;
1045 SDNode *SLLI =
1046 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0),
1047 CurDAG->getTargetConstant(LShAmt, DL, VT));
1048 SDNode *SRLI = CurDAG->getMachineNode(
1049 RISCV::SRLI, DL, VT, SDValue(SLLI, 0),
1050 CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT));
1051 ReplaceNode(Node, SRLI);
1052 return;
1053 }
1054 case ISD::SRA: {
1055 if (trySignedBitfieldExtract(Node))
1056 return;
1057
1058 // Optimize (sra (sext_inreg X, i16), C) ->
1059 // (srai (slli X, (XLen-16), (XLen-16) + C)
1060 // And (sra (sext_inreg X, i8), C) ->
1061 // (srai (slli X, (XLen-8), (XLen-8) + C)
1062 // This can occur when Zbb is enabled, which makes sext_inreg i16/i8 legal.
1063 // This transform matches the code we get without Zbb. The shifts are more
1064 // compressible, and this can help expose CSE opportunities in the sdiv by
1065 // constant optimization.
1066 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1067 if (!N1C)
1068 break;
1069 SDValue N0 = Node->getOperand(0);
1070 if (N0.getOpcode() != ISD::SIGN_EXTEND_INREG || !N0.hasOneUse())
1071 break;
1072 unsigned ShAmt = N1C->getZExtValue();
1073 unsigned ExtSize =
1074 cast<VTSDNode>(N0.getOperand(1))->getVT().getSizeInBits();
1075 // ExtSize of 32 should use sraiw via tablegen pattern.
1076 if (ExtSize >= 32 || ShAmt >= ExtSize)
1077 break;
1078 unsigned LShAmt = Subtarget->getXLen() - ExtSize;
1079 SDNode *SLLI =
1080 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0),
1081 CurDAG->getTargetConstant(LShAmt, DL, VT));
1082 SDNode *SRAI = CurDAG->getMachineNode(
1083 RISCV::SRAI, DL, VT, SDValue(SLLI, 0),
1084 CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT));
1085 ReplaceNode(Node, SRAI);
1086 return;
1087 }
1088 case ISD::OR:
1089 case ISD::XOR:
1090 if (tryShrinkShlLogicImm(Node))
1091 return;
1092
1093 break;
1094 case ISD::AND: {
1095 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1096 if (!N1C)
1097 break;
1098 uint64_t C1 = N1C->getZExtValue();
1099 const bool isC1Mask = isMask_64(C1);
1100 const bool isC1ANDI = isInt<12>(C1);
1101
1102 SDValue N0 = Node->getOperand(0);
1103
1104 auto tryUnsignedBitfieldExtract = [&](SDNode *Node, SDLoc DL, MVT VT,
1105 SDValue X, unsigned Msb,
1106 unsigned Lsb) {
1107 if (!Subtarget->hasVendorXTHeadBb())
1108 return false;
1109
1110 SDNode *TH_EXTU = CurDAG->getMachineNode(
1111 RISCV::TH_EXTU, DL, VT, X, CurDAG->getTargetConstant(Msb, DL, VT),
1112 CurDAG->getTargetConstant(Lsb, DL, VT));
1113 ReplaceNode(Node, TH_EXTU);
1114 return true;
1115 };
1116
1117 bool LeftShift = N0.getOpcode() == ISD::SHL;
1118 if (LeftShift || N0.getOpcode() == ISD::SRL) {
1119 auto *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
1120 if (!C)
1121 break;
1122 unsigned C2 = C->getZExtValue();
1123 unsigned XLen = Subtarget->getXLen();
1124 assert((C2 > 0 && C2 < XLen) && "Unexpected shift amount!");
1125
1126 // Keep track of whether this is a c.andi. If we can't use c.andi, the
1127 // shift pair might offer more compression opportunities.
1128 // TODO: We could check for C extension here, but we don't have many lit
1129 // tests with the C extension enabled so not checking gets better
1130 // coverage.
1131 // TODO: What if ANDI faster than shift?
1132 bool IsCANDI = isInt<6>(N1C->getSExtValue());
1133
1134 // Clear irrelevant bits in the mask.
1135 if (LeftShift)
1136 C1 &= maskTrailingZeros<uint64_t>(C2);
1137 else
1138 C1 &= maskTrailingOnes<uint64_t>(XLen - C2);
1139
1140 // Some transforms should only be done if the shift has a single use or
1141 // the AND would become (srli (slli X, 32), 32)
1142 bool OneUseOrZExtW = N0.hasOneUse() || C1 == UINT64_C(0xFFFFFFFF);
1143
1144 SDValue X = N0.getOperand(0);
1145
1146 // Turn (and (srl x, c2) c1) -> (srli (slli x, c3-c2), c3) if c1 is a mask
1147 // with c3 leading zeros.
1148 if (!LeftShift && isC1Mask) {
1149 unsigned Leading = XLen - llvm::bit_width(C1);
1150 if (C2 < Leading) {
1151 // If the number of leading zeros is C2+32 this can be SRLIW.
1152 if (C2 + 32 == Leading) {
1153 SDNode *SRLIW = CurDAG->getMachineNode(
1154 RISCV::SRLIW, DL, VT, X, CurDAG->getTargetConstant(C2, DL, VT));
1155 ReplaceNode(Node, SRLIW);
1156 return;
1157 }
1158
1159 // (and (srl (sexti32 Y), c2), c1) -> (srliw (sraiw Y, 31), c3 - 32)
1160 // if c1 is a mask with c3 leading zeros and c2 >= 32 and c3-c2==1.
1161 //
1162 // This pattern occurs when (i32 (srl (sra 31), c3 - 32)) is type
1163 // legalized and goes through DAG combine.
1164 if (C2 >= 32 && (Leading - C2) == 1 && N0.hasOneUse() &&
1165 X.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1166 cast<VTSDNode>(X.getOperand(1))->getVT() == MVT::i32) {
1167 SDNode *SRAIW =
1168 CurDAG->getMachineNode(RISCV::SRAIW, DL, VT, X.getOperand(0),
1169 CurDAG->getTargetConstant(31, DL, VT));
1170 SDNode *SRLIW = CurDAG->getMachineNode(
1171 RISCV::SRLIW, DL, VT, SDValue(SRAIW, 0),
1172 CurDAG->getTargetConstant(Leading - 32, DL, VT));
1173 ReplaceNode(Node, SRLIW);
1174 return;
1175 }
1176
1177 // Try to use an unsigned bitfield extract (e.g., th.extu) if
1178 // available.
1179 // Transform (and (srl x, C2), C1)
1180 // -> (<bfextract> x, msb, lsb)
1181 //
1182 // Make sure to keep this below the SRLIW cases, as we always want to
1183 // prefer the more common instruction.
1184 const unsigned Msb = llvm::bit_width(C1) + C2 - 1;
1185 const unsigned Lsb = C2;
1186 if (tryUnsignedBitfieldExtract(Node, DL, VT, X, Msb, Lsb))
1187 return;
1188
1189 // (srli (slli x, c3-c2), c3).
1190 // Skip if we could use (zext.w (sraiw X, C2)).
1191 bool Skip = Subtarget->hasStdExtZba() && Leading == 32 &&
1192 X.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1193 cast<VTSDNode>(X.getOperand(1))->getVT() == MVT::i32;
1194 // Also Skip if we can use bexti or th.tst.
1195 Skip |= HasBitTest && Leading == XLen - 1;
1196 if (OneUseOrZExtW && !Skip) {
1197 SDNode *SLLI = CurDAG->getMachineNode(
1198 RISCV::SLLI, DL, VT, X,
1199 CurDAG->getTargetConstant(Leading - C2, DL, VT));
1200 SDNode *SRLI = CurDAG->getMachineNode(
1201 RISCV::SRLI, DL, VT, SDValue(SLLI, 0),
1202 CurDAG->getTargetConstant(Leading, DL, VT));
1203 ReplaceNode(Node, SRLI);
1204 return;
1205 }
1206 }
1207 }
1208
1209 // Turn (and (shl x, c2), c1) -> (srli (slli c2+c3), c3) if c1 is a mask
1210 // shifted by c2 bits with c3 leading zeros.
1211 if (LeftShift && isShiftedMask_64(C1)) {
1212 unsigned Leading = XLen - llvm::bit_width(C1);
1213
1214 if (C2 + Leading < XLen &&
1215 C1 == (maskTrailingOnes<uint64_t>(XLen - (C2 + Leading)) << C2)) {
1216 // Use slli.uw when possible.
1217 if ((XLen - (C2 + Leading)) == 32 && Subtarget->hasStdExtZba()) {
1218 SDNode *SLLI_UW =
1219 CurDAG->getMachineNode(RISCV::SLLI_UW, DL, VT, X,
1220 CurDAG->getTargetConstant(C2, DL, VT));
1221 ReplaceNode(Node, SLLI_UW);
1222 return;
1223 }
1224
1225 // (srli (slli c2+c3), c3)
1226 if (OneUseOrZExtW && !IsCANDI) {
1227 SDNode *SLLI = CurDAG->getMachineNode(
1228 RISCV::SLLI, DL, VT, X,
1229 CurDAG->getTargetConstant(C2 + Leading, DL, VT));
1230 SDNode *SRLI = CurDAG->getMachineNode(
1231 RISCV::SRLI, DL, VT, SDValue(SLLI, 0),
1232 CurDAG->getTargetConstant(Leading, DL, VT));
1233 ReplaceNode(Node, SRLI);
1234 return;
1235 }
1236 }
1237 }
1238
1239 // Turn (and (shr x, c2), c1) -> (slli (srli x, c2+c3), c3) if c1 is a
1240 // shifted mask with c2 leading zeros and c3 trailing zeros.
1241 if (!LeftShift && isShiftedMask_64(C1)) {
1242 unsigned Leading = XLen - llvm::bit_width(C1);
1243 unsigned Trailing = llvm::countr_zero(C1);
1244 if (Leading == C2 && C2 + Trailing < XLen && OneUseOrZExtW &&
1245 !IsCANDI) {
1246 unsigned SrliOpc = RISCV::SRLI;
1247 // If the input is zexti32 we should use SRLIW.
1248 if (X.getOpcode() == ISD::AND &&
1249 isa<ConstantSDNode>(X.getOperand(1)) &&
1250 X.getConstantOperandVal(1) == UINT64_C(0xFFFFFFFF)) {
1251 SrliOpc = RISCV::SRLIW;
1252 X = X.getOperand(0);
1253 }
1254 SDNode *SRLI = CurDAG->getMachineNode(
1255 SrliOpc, DL, VT, X,
1256 CurDAG->getTargetConstant(C2 + Trailing, DL, VT));
1257 SDNode *SLLI = CurDAG->getMachineNode(
1258 RISCV::SLLI, DL, VT, SDValue(SRLI, 0),
1259 CurDAG->getTargetConstant(Trailing, DL, VT));
1260 ReplaceNode(Node, SLLI);
1261 return;
1262 }
1263 // If the leading zero count is C2+32, we can use SRLIW instead of SRLI.
1264 if (Leading > 32 && (Leading - 32) == C2 && C2 + Trailing < 32 &&
1265 OneUseOrZExtW && !IsCANDI) {
1266 SDNode *SRLIW = CurDAG->getMachineNode(
1267 RISCV::SRLIW, DL, VT, X,
1268 CurDAG->getTargetConstant(C2 + Trailing, DL, VT));
1269 SDNode *SLLI = CurDAG->getMachineNode(
1270 RISCV::SLLI, DL, VT, SDValue(SRLIW, 0),
1271 CurDAG->getTargetConstant(Trailing, DL, VT));
1272 ReplaceNode(Node, SLLI);
1273 return;
1274 }
1275 }
1276
1277 // Turn (and (shl x, c2), c1) -> (slli (srli x, c3-c2), c3) if c1 is a
1278 // shifted mask with no leading zeros and c3 trailing zeros.
1279 if (LeftShift && isShiftedMask_64(C1)) {
1280 unsigned Leading = XLen - llvm::bit_width(C1);
1281 unsigned Trailing = llvm::countr_zero(C1);
1282 if (Leading == 0 && C2 < Trailing && OneUseOrZExtW && !IsCANDI) {
1283 SDNode *SRLI = CurDAG->getMachineNode(
1284 RISCV::SRLI, DL, VT, X,
1285 CurDAG->getTargetConstant(Trailing - C2, DL, VT));
1286 SDNode *SLLI = CurDAG->getMachineNode(
1287 RISCV::SLLI, DL, VT, SDValue(SRLI, 0),
1288 CurDAG->getTargetConstant(Trailing, DL, VT));
1289 ReplaceNode(Node, SLLI);
1290 return;
1291 }
1292 // If we have (32-C2) leading zeros, we can use SRLIW instead of SRLI.
1293 if (C2 < Trailing && Leading + C2 == 32 && OneUseOrZExtW && !IsCANDI) {
1294 SDNode *SRLIW = CurDAG->getMachineNode(
1295 RISCV::SRLIW, DL, VT, X,
1296 CurDAG->getTargetConstant(Trailing - C2, DL, VT));
1297 SDNode *SLLI = CurDAG->getMachineNode(
1298 RISCV::SLLI, DL, VT, SDValue(SRLIW, 0),
1299 CurDAG->getTargetConstant(Trailing, DL, VT));
1300 ReplaceNode(Node, SLLI);
1301 return;
1302 }
1303 }
1304 }
1305
1306 // If C1 masks off the upper bits only (but can't be formed as an
1307 // ANDI), use an unsigned bitfield extract (e.g., th.extu), if
1308 // available.
1309 // Transform (and x, C1)
1310 // -> (<bfextract> x, msb, lsb)
1311 if (isC1Mask && !isC1ANDI) {
1312 const unsigned Msb = llvm::bit_width(C1) - 1;
1313 if (tryUnsignedBitfieldExtract(Node, DL, VT, N0, Msb, 0))
1314 return;
1315 }
1316
1317 if (tryShrinkShlLogicImm(Node))
1318 return;
1319
1320 break;
1321 }
1322 case ISD::MUL: {
1323 // Special case for calculating (mul (and X, C2), C1) where the full product
1324 // fits in XLen bits. We can shift X left by the number of leading zeros in
1325 // C2 and shift C1 left by XLen-lzcnt(C2). This will ensure the final
1326 // product has XLen trailing zeros, putting it in the output of MULHU. This
1327 // can avoid materializing a constant in a register for C2.
1328
1329 // RHS should be a constant.
1330 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1331 if (!N1C || !N1C->hasOneUse())
1332 break;
1333
1334 // LHS should be an AND with constant.
1335 SDValue N0 = Node->getOperand(0);
1336 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(N0.getOperand(1)))
1337 break;
1338
1339 uint64_t C2 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue();
1340
1341 // Constant should be a mask.
1342 if (!isMask_64(C2))
1343 break;
1344
1345 // If this can be an ANDI or ZEXT.H, don't do this if the ANDI/ZEXT has
1346 // multiple users or the constant is a simm12. This prevents inserting a
1347 // shift and still have uses of the AND/ZEXT. Shifting a simm12 will likely
1348 // make it more costly to materialize. Otherwise, using a SLLI might allow
1349 // it to be compressed.
1350 bool IsANDIOrZExt =
1351 isInt<12>(C2) ||
1352 (C2 == UINT64_C(0xFFFF) && Subtarget->hasStdExtZbb());
1353 // With XTHeadBb, we can use TH.EXTU.
1354 IsANDIOrZExt |= C2 == UINT64_C(0xFFFF) && Subtarget->hasVendorXTHeadBb();
1355 if (IsANDIOrZExt && (isInt<12>(N1C->getSExtValue()) || !N0.hasOneUse()))
1356 break;
1357 // If this can be a ZEXT.w, don't do this if the ZEXT has multiple users or
1358 // the constant is a simm32.
1359 bool IsZExtW = C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasStdExtZba();
1360 // With XTHeadBb, we can use TH.EXTU.
1361 IsZExtW |= C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasVendorXTHeadBb();
1362 if (IsZExtW && (isInt<32>(N1C->getSExtValue()) || !N0.hasOneUse()))
1363 break;
1364
1365 // We need to shift left the AND input and C1 by a total of XLen bits.
1366
1367 // How far left do we need to shift the AND input?
1368 unsigned XLen = Subtarget->getXLen();
1369 unsigned LeadingZeros = XLen - llvm::bit_width(C2);
1370
1371 // The constant gets shifted by the remaining amount unless that would
1372 // shift bits out.
1373 uint64_t C1 = N1C->getZExtValue();
1374 unsigned ConstantShift = XLen - LeadingZeros;
1375 if (ConstantShift > (XLen - llvm::bit_width(C1)))
1376 break;
1377
1378 uint64_t ShiftedC1 = C1 << ConstantShift;
1379 // If this RV32, we need to sign extend the constant.
1380 if (XLen == 32)
1381 ShiftedC1 = SignExtend64<32>(ShiftedC1);
1382
1383 // Create (mulhu (slli X, lzcnt(C2)), C1 << (XLen - lzcnt(C2))).
1384 SDNode *Imm = selectImm(CurDAG, DL, VT, ShiftedC1, *Subtarget).getNode();
1385 SDNode *SLLI =
1386 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0.getOperand(0),
1387 CurDAG->getTargetConstant(LeadingZeros, DL, VT));
1388 SDNode *MULHU = CurDAG->getMachineNode(RISCV::MULHU, DL, VT,
1389 SDValue(SLLI, 0), SDValue(Imm, 0));
1390 ReplaceNode(Node, MULHU);
1391 return;
1392 }
1393 case ISD::LOAD: {
1394 if (tryIndexedLoad(Node))
1395 return;
1396 break;
1397 }
1399 unsigned IntNo = Node->getConstantOperandVal(0);
1400 switch (IntNo) {
1401 // By default we do not custom select any intrinsic.
1402 default:
1403 break;
1404 case Intrinsic::riscv_vmsgeu:
1405 case Intrinsic::riscv_vmsge: {
1406 SDValue Src1 = Node->getOperand(1);
1407 SDValue Src2 = Node->getOperand(2);
1408 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu;
1409 bool IsCmpUnsignedZero = false;
1410 // Only custom select scalar second operand.
1411 if (Src2.getValueType() != XLenVT)
1412 break;
1413 // Small constants are handled with patterns.
1414 if (auto *C = dyn_cast<ConstantSDNode>(Src2)) {
1415 int64_t CVal = C->getSExtValue();
1416 if (CVal >= -15 && CVal <= 16) {
1417 if (!IsUnsigned || CVal != 0)
1418 break;
1419 IsCmpUnsignedZero = true;
1420 }
1421 }
1422 MVT Src1VT = Src1.getSimpleValueType();
1423 unsigned VMSLTOpcode, VMNANDOpcode, VMSetOpcode;
1424 switch (RISCVTargetLowering::getLMUL(Src1VT)) {
1425 default:
1426 llvm_unreachable("Unexpected LMUL!");
1427#define CASE_VMSLT_VMNAND_VMSET_OPCODES(lmulenum, suffix, suffix_b) \
1428 case RISCVII::VLMUL::lmulenum: \
1429 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \
1430 : RISCV::PseudoVMSLT_VX_##suffix; \
1431 VMNANDOpcode = RISCV::PseudoVMNAND_MM_##suffix; \
1432 VMSetOpcode = RISCV::PseudoVMSET_M_##suffix_b; \
1433 break;
1434 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F8, MF8, B1)
1435 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F4, MF4, B2)
1436 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F2, MF2, B4)
1438 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_2, M2, B16)
1439 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_4, M4, B32)
1440 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_8, M8, B64)
1441#undef CASE_VMSLT_VMNAND_VMSET_OPCODES
1442 }
1444 Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT);
1445 SDValue VL;
1446 selectVLOp(Node->getOperand(3), VL);
1447
1448 // If vmsgeu with 0 immediate, expand it to vmset.
1449 if (IsCmpUnsignedZero) {
1450 ReplaceNode(Node, CurDAG->getMachineNode(VMSetOpcode, DL, VT, VL, SEW));
1451 return;
1452 }
1453
1454 // Expand to
1455 // vmslt{u}.vx vd, va, x; vmnand.mm vd, vd, vd
1456 SDValue Cmp = SDValue(
1457 CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}),
1458 0);
1459 ReplaceNode(Node, CurDAG->getMachineNode(VMNANDOpcode, DL, VT,
1460 {Cmp, Cmp, VL, SEW}));
1461 return;
1462 }
1463 case Intrinsic::riscv_vmsgeu_mask:
1464 case Intrinsic::riscv_vmsge_mask: {
1465 SDValue Src1 = Node->getOperand(2);
1466 SDValue Src2 = Node->getOperand(3);
1467 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu_mask;
1468 bool IsCmpUnsignedZero = false;
1469 // Only custom select scalar second operand.
1470 if (Src2.getValueType() != XLenVT)
1471 break;
1472 // Small constants are handled with patterns.
1473 if (auto *C = dyn_cast<ConstantSDNode>(Src2)) {
1474 int64_t CVal = C->getSExtValue();
1475 if (CVal >= -15 && CVal <= 16) {
1476 if (!IsUnsigned || CVal != 0)
1477 break;
1478 IsCmpUnsignedZero = true;
1479 }
1480 }
1481 MVT Src1VT = Src1.getSimpleValueType();
1482 unsigned VMSLTOpcode, VMSLTMaskOpcode, VMXOROpcode, VMANDNOpcode,
1483 VMOROpcode;
1484 switch (RISCVTargetLowering::getLMUL(Src1VT)) {
1485 default:
1486 llvm_unreachable("Unexpected LMUL!");
1487#define CASE_VMSLT_OPCODES(lmulenum, suffix, suffix_b) \
1488 case RISCVII::VLMUL::lmulenum: \
1489 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \
1490 : RISCV::PseudoVMSLT_VX_##suffix; \
1491 VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix##_MASK \
1492 : RISCV::PseudoVMSLT_VX_##suffix##_MASK; \
1493 break;
1494 CASE_VMSLT_OPCODES(LMUL_F8, MF8, B1)
1495 CASE_VMSLT_OPCODES(LMUL_F4, MF4, B2)
1496 CASE_VMSLT_OPCODES(LMUL_F2, MF2, B4)
1497 CASE_VMSLT_OPCODES(LMUL_1, M1, B8)
1498 CASE_VMSLT_OPCODES(LMUL_2, M2, B16)
1499 CASE_VMSLT_OPCODES(LMUL_4, M4, B32)
1500 CASE_VMSLT_OPCODES(LMUL_8, M8, B64)
1501#undef CASE_VMSLT_OPCODES
1502 }
1503 // Mask operations use the LMUL from the mask type.
1504 switch (RISCVTargetLowering::getLMUL(VT)) {
1505 default:
1506 llvm_unreachable("Unexpected LMUL!");
1507#define CASE_VMXOR_VMANDN_VMOR_OPCODES(lmulenum, suffix) \
1508 case RISCVII::VLMUL::lmulenum: \
1509 VMXOROpcode = RISCV::PseudoVMXOR_MM_##suffix; \
1510 VMANDNOpcode = RISCV::PseudoVMANDN_MM_##suffix; \
1511 VMOROpcode = RISCV::PseudoVMOR_MM_##suffix; \
1512 break;
1513 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F8, MF8)
1514 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F4, MF4)
1515 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F2, MF2)
1520#undef CASE_VMXOR_VMANDN_VMOR_OPCODES
1521 }
1523 Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT);
1524 SDValue MaskSEW = CurDAG->getTargetConstant(0, DL, XLenVT);
1525 SDValue VL;
1526 selectVLOp(Node->getOperand(5), VL);
1527 SDValue MaskedOff = Node->getOperand(1);
1528 SDValue Mask = Node->getOperand(4);
1529
1530 // If vmsgeu_mask with 0 immediate, expand it to vmor mask, maskedoff.
1531 if (IsCmpUnsignedZero) {
1532 // We don't need vmor if the MaskedOff and the Mask are the same
1533 // value.
1534 if (Mask == MaskedOff) {
1535 ReplaceUses(Node, Mask.getNode());
1536 return;
1537 }
1538 ReplaceNode(Node,
1539 CurDAG->getMachineNode(VMOROpcode, DL, VT,
1540 {Mask, MaskedOff, VL, MaskSEW}));
1541 return;
1542 }
1543
1544 // If the MaskedOff value and the Mask are the same value use
1545 // vmslt{u}.vx vt, va, x; vmandn.mm vd, vd, vt
1546 // This avoids needing to copy v0 to vd before starting the next sequence.
1547 if (Mask == MaskedOff) {
1548 SDValue Cmp = SDValue(
1549 CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}),
1550 0);
1551 ReplaceNode(Node, CurDAG->getMachineNode(VMANDNOpcode, DL, VT,
1552 {Mask, Cmp, VL, MaskSEW}));
1553 return;
1554 }
1555
1556 // Mask needs to be copied to V0.
1558 RISCV::V0, Mask, SDValue());
1559 SDValue Glue = Chain.getValue(1);
1560 SDValue V0 = CurDAG->getRegister(RISCV::V0, VT);
1561
1562 // Otherwise use
1563 // vmslt{u}.vx vd, va, x, v0.t; vmxor.mm vd, vd, v0
1564 // The result is mask undisturbed.
1565 // We use the same instructions to emulate mask agnostic behavior, because
1566 // the agnostic result can be either undisturbed or all 1.
1567 SDValue Cmp = SDValue(
1568 CurDAG->getMachineNode(VMSLTMaskOpcode, DL, VT,
1569 {MaskedOff, Src1, Src2, V0, VL, SEW, Glue}),
1570 0);
1571 // vmxor.mm vd, vd, v0 is used to update active value.
1572 ReplaceNode(Node, CurDAG->getMachineNode(VMXOROpcode, DL, VT,
1573 {Cmp, Mask, VL, MaskSEW}));
1574 return;
1575 }
1576 case Intrinsic::riscv_vsetvli:
1577 case Intrinsic::riscv_vsetvlimax:
1578 return selectVSETVLI(Node);
1579 }
1580 break;
1581 }
1583 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
1584 switch (IntNo) {
1585 // By default we do not custom select any intrinsic.
1586 default:
1587 break;
1588 case Intrinsic::riscv_vlseg2:
1589 case Intrinsic::riscv_vlseg3:
1590 case Intrinsic::riscv_vlseg4:
1591 case Intrinsic::riscv_vlseg5:
1592 case Intrinsic::riscv_vlseg6:
1593 case Intrinsic::riscv_vlseg7:
1594 case Intrinsic::riscv_vlseg8: {
1595 selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
1596 return;
1597 }
1598 case Intrinsic::riscv_vlseg2_mask:
1599 case Intrinsic::riscv_vlseg3_mask:
1600 case Intrinsic::riscv_vlseg4_mask:
1601 case Intrinsic::riscv_vlseg5_mask:
1602 case Intrinsic::riscv_vlseg6_mask:
1603 case Intrinsic::riscv_vlseg7_mask:
1604 case Intrinsic::riscv_vlseg8_mask: {
1605 selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
1606 return;
1607 }
1608 case Intrinsic::riscv_vlsseg2:
1609 case Intrinsic::riscv_vlsseg3:
1610 case Intrinsic::riscv_vlsseg4:
1611 case Intrinsic::riscv_vlsseg5:
1612 case Intrinsic::riscv_vlsseg6:
1613 case Intrinsic::riscv_vlsseg7:
1614 case Intrinsic::riscv_vlsseg8: {
1615 selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
1616 return;
1617 }
1618 case Intrinsic::riscv_vlsseg2_mask:
1619 case Intrinsic::riscv_vlsseg3_mask:
1620 case Intrinsic::riscv_vlsseg4_mask:
1621 case Intrinsic::riscv_vlsseg5_mask:
1622 case Intrinsic::riscv_vlsseg6_mask:
1623 case Intrinsic::riscv_vlsseg7_mask:
1624 case Intrinsic::riscv_vlsseg8_mask: {
1625 selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
1626 return;
1627 }
1628 case Intrinsic::riscv_vloxseg2:
1629 case Intrinsic::riscv_vloxseg3:
1630 case Intrinsic::riscv_vloxseg4:
1631 case Intrinsic::riscv_vloxseg5:
1632 case Intrinsic::riscv_vloxseg6:
1633 case Intrinsic::riscv_vloxseg7:
1634 case Intrinsic::riscv_vloxseg8:
1635 selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
1636 return;
1637 case Intrinsic::riscv_vluxseg2:
1638 case Intrinsic::riscv_vluxseg3:
1639 case Intrinsic::riscv_vluxseg4:
1640 case Intrinsic::riscv_vluxseg5:
1641 case Intrinsic::riscv_vluxseg6:
1642 case Intrinsic::riscv_vluxseg7:
1643 case Intrinsic::riscv_vluxseg8:
1644 selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
1645 return;
1646 case Intrinsic::riscv_vloxseg2_mask:
1647 case Intrinsic::riscv_vloxseg3_mask:
1648 case Intrinsic::riscv_vloxseg4_mask:
1649 case Intrinsic::riscv_vloxseg5_mask:
1650 case Intrinsic::riscv_vloxseg6_mask:
1651 case Intrinsic::riscv_vloxseg7_mask:
1652 case Intrinsic::riscv_vloxseg8_mask:
1653 selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
1654 return;
1655 case Intrinsic::riscv_vluxseg2_mask:
1656 case Intrinsic::riscv_vluxseg3_mask:
1657 case Intrinsic::riscv_vluxseg4_mask:
1658 case Intrinsic::riscv_vluxseg5_mask:
1659 case Intrinsic::riscv_vluxseg6_mask:
1660 case Intrinsic::riscv_vluxseg7_mask:
1661 case Intrinsic::riscv_vluxseg8_mask:
1662 selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
1663 return;
1664 case Intrinsic::riscv_vlseg8ff:
1665 case Intrinsic::riscv_vlseg7ff:
1666 case Intrinsic::riscv_vlseg6ff:
1667 case Intrinsic::riscv_vlseg5ff:
1668 case Intrinsic::riscv_vlseg4ff:
1669 case Intrinsic::riscv_vlseg3ff:
1670 case Intrinsic::riscv_vlseg2ff: {
1671 selectVLSEGFF(Node, /*IsMasked*/ false);
1672 return;
1673 }
1674 case Intrinsic::riscv_vlseg8ff_mask:
1675 case Intrinsic::riscv_vlseg7ff_mask:
1676 case Intrinsic::riscv_vlseg6ff_mask:
1677 case Intrinsic::riscv_vlseg5ff_mask:
1678 case Intrinsic::riscv_vlseg4ff_mask:
1679 case Intrinsic::riscv_vlseg3ff_mask:
1680 case Intrinsic::riscv_vlseg2ff_mask: {
1681 selectVLSEGFF(Node, /*IsMasked*/ true);
1682 return;
1683 }
1684 case Intrinsic::riscv_vloxei:
1685 case Intrinsic::riscv_vloxei_mask:
1686 case Intrinsic::riscv_vluxei:
1687 case Intrinsic::riscv_vluxei_mask: {
1688 bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask ||
1689 IntNo == Intrinsic::riscv_vluxei_mask;
1690 bool IsOrdered = IntNo == Intrinsic::riscv_vloxei ||
1691 IntNo == Intrinsic::riscv_vloxei_mask;
1692
1693 MVT VT = Node->getSimpleValueType(0);
1694 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1695
1696 unsigned CurOp = 2;
1698 Operands.push_back(Node->getOperand(CurOp++));
1699
1700 MVT IndexVT;
1701 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
1702 /*IsStridedOrIndexed*/ true, Operands,
1703 /*IsLoad=*/true, &IndexVT);
1704
1706 "Element count mismatch");
1707
1709 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
1710 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
1711 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
1712 report_fatal_error("The V extension does not support EEW=64 for index "
1713 "values when XLEN=32");
1714 }
1715 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
1716 IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
1717 static_cast<unsigned>(IndexLMUL));
1718 MachineSDNode *Load =
1719 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1720
1721 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1722 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
1723
1724 ReplaceNode(Node, Load);
1725 return;
1726 }
1727 case Intrinsic::riscv_vlm:
1728 case Intrinsic::riscv_vle:
1729 case Intrinsic::riscv_vle_mask:
1730 case Intrinsic::riscv_vlse:
1731 case Intrinsic::riscv_vlse_mask: {
1732 bool IsMasked = IntNo == Intrinsic::riscv_vle_mask ||
1733 IntNo == Intrinsic::riscv_vlse_mask;
1734 bool IsStrided =
1735 IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask;
1736
1737 MVT VT = Node->getSimpleValueType(0);
1738 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1739
1740 // The riscv_vlm intrinsic are always tail agnostic and no passthru
1741 // operand at the IR level. In pseudos, they have both policy and
1742 // passthru operand. The passthru operand is needed to track the
1743 // "tail undefined" state, and the policy is there just for
1744 // for consistency - it will always be "don't care" for the
1745 // unmasked form.
1746 bool HasPassthruOperand = IntNo != Intrinsic::riscv_vlm;
1747 unsigned CurOp = 2;
1749 if (HasPassthruOperand)
1750 Operands.push_back(Node->getOperand(CurOp++));
1751 else {
1752 // We eagerly lower to implicit_def (instead of undef), as we
1753 // otherwise fail to select nodes such as: nxv1i1 = undef
1754 SDNode *Passthru =
1755 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);
1756 Operands.push_back(SDValue(Passthru, 0));
1757 }
1758 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
1759 Operands, /*IsLoad=*/true);
1760
1762 const RISCV::VLEPseudo *P =
1763 RISCV::getVLEPseudo(IsMasked, IsStrided, /*FF*/ false, Log2SEW,
1764 static_cast<unsigned>(LMUL));
1765 MachineSDNode *Load =
1766 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1767
1768 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1769 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
1770
1771 ReplaceNode(Node, Load);
1772 return;
1773 }
1774 case Intrinsic::riscv_vleff:
1775 case Intrinsic::riscv_vleff_mask: {
1776 bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask;
1777
1778 MVT VT = Node->getSimpleValueType(0);
1779 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1780
1781 unsigned CurOp = 2;
1783 Operands.push_back(Node->getOperand(CurOp++));
1784 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
1785 /*IsStridedOrIndexed*/ false, Operands,
1786 /*IsLoad=*/true);
1787
1789 const RISCV::VLEPseudo *P =
1790 RISCV::getVLEPseudo(IsMasked, /*Strided*/ false, /*FF*/ true,
1791 Log2SEW, static_cast<unsigned>(LMUL));
1793 P->Pseudo, DL, Node->getVTList(), Operands);
1794 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1795 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
1796
1797 ReplaceNode(Node, Load);
1798 return;
1799 }
1800 }
1801 break;
1802 }
1803 case ISD::INTRINSIC_VOID: {
1804 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
1805 switch (IntNo) {
1806 case Intrinsic::riscv_vsseg2:
1807 case Intrinsic::riscv_vsseg3:
1808 case Intrinsic::riscv_vsseg4:
1809 case Intrinsic::riscv_vsseg5:
1810 case Intrinsic::riscv_vsseg6:
1811 case Intrinsic::riscv_vsseg7:
1812 case Intrinsic::riscv_vsseg8: {
1813 selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
1814 return;
1815 }
1816 case Intrinsic::riscv_vsseg2_mask:
1817 case Intrinsic::riscv_vsseg3_mask:
1818 case Intrinsic::riscv_vsseg4_mask:
1819 case Intrinsic::riscv_vsseg5_mask:
1820 case Intrinsic::riscv_vsseg6_mask:
1821 case Intrinsic::riscv_vsseg7_mask:
1822 case Intrinsic::riscv_vsseg8_mask: {
1823 selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
1824 return;
1825 }
1826 case Intrinsic::riscv_vssseg2:
1827 case Intrinsic::riscv_vssseg3:
1828 case Intrinsic::riscv_vssseg4:
1829 case Intrinsic::riscv_vssseg5:
1830 case Intrinsic::riscv_vssseg6:
1831 case Intrinsic::riscv_vssseg7:
1832 case Intrinsic::riscv_vssseg8: {
1833 selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
1834 return;
1835 }
1836 case Intrinsic::riscv_vssseg2_mask:
1837 case Intrinsic::riscv_vssseg3_mask:
1838 case Intrinsic::riscv_vssseg4_mask:
1839 case Intrinsic::riscv_vssseg5_mask:
1840 case Intrinsic::riscv_vssseg6_mask:
1841 case Intrinsic::riscv_vssseg7_mask:
1842 case Intrinsic::riscv_vssseg8_mask: {
1843 selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
1844 return;
1845 }
1846 case Intrinsic::riscv_vsoxseg2:
1847 case Intrinsic::riscv_vsoxseg3:
1848 case Intrinsic::riscv_vsoxseg4:
1849 case Intrinsic::riscv_vsoxseg5:
1850 case Intrinsic::riscv_vsoxseg6:
1851 case Intrinsic::riscv_vsoxseg7:
1852 case Intrinsic::riscv_vsoxseg8:
1853 selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
1854 return;
1855 case Intrinsic::riscv_vsuxseg2:
1856 case Intrinsic::riscv_vsuxseg3:
1857 case Intrinsic::riscv_vsuxseg4:
1858 case Intrinsic::riscv_vsuxseg5:
1859 case Intrinsic::riscv_vsuxseg6:
1860 case Intrinsic::riscv_vsuxseg7:
1861 case Intrinsic::riscv_vsuxseg8:
1862 selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
1863 return;
1864 case Intrinsic::riscv_vsoxseg2_mask:
1865 case Intrinsic::riscv_vsoxseg3_mask:
1866 case Intrinsic::riscv_vsoxseg4_mask:
1867 case Intrinsic::riscv_vsoxseg5_mask:
1868 case Intrinsic::riscv_vsoxseg6_mask:
1869 case Intrinsic::riscv_vsoxseg7_mask:
1870 case Intrinsic::riscv_vsoxseg8_mask:
1871 selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
1872 return;
1873 case Intrinsic::riscv_vsuxseg2_mask:
1874 case Intrinsic::riscv_vsuxseg3_mask:
1875 case Intrinsic::riscv_vsuxseg4_mask:
1876 case Intrinsic::riscv_vsuxseg5_mask:
1877 case Intrinsic::riscv_vsuxseg6_mask:
1878 case Intrinsic::riscv_vsuxseg7_mask:
1879 case Intrinsic::riscv_vsuxseg8_mask:
1880 selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
1881 return;
1882 case Intrinsic::riscv_vsoxei:
1883 case Intrinsic::riscv_vsoxei_mask:
1884 case Intrinsic::riscv_vsuxei:
1885 case Intrinsic::riscv_vsuxei_mask: {
1886 bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask ||
1887 IntNo == Intrinsic::riscv_vsuxei_mask;
1888 bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei ||
1889 IntNo == Intrinsic::riscv_vsoxei_mask;
1890
1891 MVT VT = Node->getOperand(2)->getSimpleValueType(0);
1892 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1893
1894 unsigned CurOp = 2;
1896 Operands.push_back(Node->getOperand(CurOp++)); // Store value.
1897
1898 MVT IndexVT;
1899 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
1900 /*IsStridedOrIndexed*/ true, Operands,
1901 /*IsLoad=*/false, &IndexVT);
1902
1904 "Element count mismatch");
1905
1907 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
1908 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
1909 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
1910 report_fatal_error("The V extension does not support EEW=64 for index "
1911 "values when XLEN=32");
1912 }
1913 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
1914 IsMasked, IsOrdered, IndexLog2EEW,
1915 static_cast<unsigned>(LMUL), static_cast<unsigned>(IndexLMUL));
1916 MachineSDNode *Store =
1917 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1918
1919 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1920 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
1921
1922 ReplaceNode(Node, Store);
1923 return;
1924 }
1925 case Intrinsic::riscv_vsm:
1926 case Intrinsic::riscv_vse:
1927 case Intrinsic::riscv_vse_mask:
1928 case Intrinsic::riscv_vsse:
1929 case Intrinsic::riscv_vsse_mask: {
1930 bool IsMasked = IntNo == Intrinsic::riscv_vse_mask ||
1931 IntNo == Intrinsic::riscv_vsse_mask;
1932 bool IsStrided =
1933 IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask;
1934
1935 MVT VT = Node->getOperand(2)->getSimpleValueType(0);
1936 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1937
1938 unsigned CurOp = 2;
1940 Operands.push_back(Node->getOperand(CurOp++)); // Store value.
1941
1942 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
1943 Operands);
1944
1946 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
1947 IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL));
1948 MachineSDNode *Store =
1949 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1950 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1951 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
1952
1953 ReplaceNode(Node, Store);
1954 return;
1955 }
1956 }
1957 break;
1958 }
1959 case ISD::BITCAST: {
1960 MVT SrcVT = Node->getOperand(0).getSimpleValueType();
1961 // Just drop bitcasts between vectors if both are fixed or both are
1962 // scalable.
1963 if ((VT.isScalableVector() && SrcVT.isScalableVector()) ||
1964 (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) {
1965 ReplaceUses(SDValue(Node, 0), Node->getOperand(0));
1966 CurDAG->RemoveDeadNode(Node);
1967 return;
1968 }
1969 break;
1970 }
1971 case ISD::INSERT_SUBVECTOR: {
1972 SDValue V = Node->getOperand(0);
1973 SDValue SubV = Node->getOperand(1);
1974 SDLoc DL(SubV);
1975 auto Idx = Node->getConstantOperandVal(2);
1976 MVT SubVecVT = SubV.getSimpleValueType();
1977
1978 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
1979 MVT SubVecContainerVT = SubVecVT;
1980 // Establish the correct scalable-vector types for any fixed-length type.
1981 if (SubVecVT.isFixedLengthVector())
1982 SubVecContainerVT = TLI.getContainerForFixedLengthVector(SubVecVT);
1983 if (VT.isFixedLengthVector())
1984 VT = TLI.getContainerForFixedLengthVector(VT);
1985
1986 const auto *TRI = Subtarget->getRegisterInfo();
1987 unsigned SubRegIdx;
1988 std::tie(SubRegIdx, Idx) =
1990 VT, SubVecContainerVT, Idx, TRI);
1991
1992 // If the Idx hasn't been completely eliminated then this is a subvector
1993 // insert which doesn't naturally align to a vector register. These must
1994 // be handled using instructions to manipulate the vector registers.
1995 if (Idx != 0)
1996 break;
1997
1998 RISCVII::VLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecContainerVT);
1999 bool IsSubVecPartReg = SubVecLMUL == RISCVII::VLMUL::LMUL_F2 ||
2000 SubVecLMUL == RISCVII::VLMUL::LMUL_F4 ||
2001 SubVecLMUL == RISCVII::VLMUL::LMUL_F8;
2002 (void)IsSubVecPartReg; // Silence unused variable warning without asserts.
2003 assert((!IsSubVecPartReg || V.isUndef()) &&
2004 "Expecting lowering to have created legal INSERT_SUBVECTORs when "
2005 "the subvector is smaller than a full-sized register");
2006
2007 // If we haven't set a SubRegIdx, then we must be going between
2008 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
2009 if (SubRegIdx == RISCV::NoSubRegister) {
2010 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT);
2012 InRegClassID &&
2013 "Unexpected subvector extraction");
2014 SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
2015 SDNode *NewNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
2016 DL, VT, SubV, RC);
2017 ReplaceNode(Node, NewNode);
2018 return;
2019 }
2020
2021 SDValue Insert = CurDAG->getTargetInsertSubreg(SubRegIdx, DL, VT, V, SubV);
2022 ReplaceNode(Node, Insert.getNode());
2023 return;
2024 }
2026 SDValue V = Node->getOperand(0);
2027 auto Idx = Node->getConstantOperandVal(1);
2028 MVT InVT = V.getSimpleValueType();
2029 SDLoc DL(V);
2030
2031 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
2032 MVT SubVecContainerVT = VT;
2033 // Establish the correct scalable-vector types for any fixed-length type.
2034 if (VT.isFixedLengthVector())
2035 SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT);
2036 if (InVT.isFixedLengthVector())
2037 InVT = TLI.getContainerForFixedLengthVector(InVT);
2038
2039 const auto *TRI = Subtarget->getRegisterInfo();
2040 unsigned SubRegIdx;
2041 std::tie(SubRegIdx, Idx) =
2043 InVT, SubVecContainerVT, Idx, TRI);
2044
2045 // If the Idx hasn't been completely eliminated then this is a subvector
2046 // extract which doesn't naturally align to a vector register. These must
2047 // be handled using instructions to manipulate the vector registers.
2048 if (Idx != 0)
2049 break;
2050
2051 // If we haven't set a SubRegIdx, then we must be going between
2052 // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy.
2053 if (SubRegIdx == RISCV::NoSubRegister) {
2054 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(InVT);
2056 InRegClassID &&
2057 "Unexpected subvector extraction");
2058 SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
2059 SDNode *NewNode =
2060 CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, V, RC);
2061 ReplaceNode(Node, NewNode);
2062 return;
2063 }
2064
2065 SDValue Extract = CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, V);
2066 ReplaceNode(Node, Extract.getNode());
2067 return;
2068 }
2072 case RISCVISD::VFMV_V_F_VL: {
2073 // Try to match splat of a scalar load to a strided load with stride of x0.
2074 bool IsScalarMove = Node->getOpcode() == RISCVISD::VMV_S_X_VL ||
2075 Node->getOpcode() == RISCVISD::VFMV_S_F_VL;
2076 if (!Node->getOperand(0).isUndef())
2077 break;
2078 SDValue Src = Node->getOperand(1);
2079 auto *Ld = dyn_cast<LoadSDNode>(Src);
2080 // Can't fold load update node because the second
2081 // output is used so that load update node can't be removed.
2082 if (!Ld || Ld->isIndexed())
2083 break;
2084 EVT MemVT = Ld->getMemoryVT();
2085 // The memory VT should be the same size as the element type.
2086 if (MemVT.getStoreSize() != VT.getVectorElementType().getStoreSize())
2087 break;
2088 if (!IsProfitableToFold(Src, Node, Node) ||
2089 !IsLegalToFold(Src, Node, Node, TM.getOptLevel()))
2090 break;
2091
2092 SDValue VL;
2093 if (IsScalarMove) {
2094 // We could deal with more VL if we update the VSETVLI insert pass to
2095 // avoid introducing more VSETVLI.
2096 if (!isOneConstant(Node->getOperand(2)))
2097 break;
2098 selectVLOp(Node->getOperand(2), VL);
2099 } else
2100 selectVLOp(Node->getOperand(2), VL);
2101
2102 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
2104
2105 // If VL=1, then we don't need to do a strided load and can just do a
2106 // regular load.
2107 bool IsStrided = !isOneConstant(VL);
2108
2109 // Only do a strided load if we have optimized zero-stride vector load.
2110 if (IsStrided && !Subtarget->hasOptimizedZeroStrideLoad())
2111 break;
2112
2114 SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT), 0),
2115 Ld->getBasePtr()};
2116 if (IsStrided)
2117 Operands.push_back(CurDAG->getRegister(RISCV::X0, XLenVT));
2119 SDValue PolicyOp = CurDAG->getTargetConstant(Policy, DL, XLenVT);
2120 Operands.append({VL, SEW, PolicyOp, Ld->getChain()});
2121
2123 const RISCV::VLEPseudo *P = RISCV::getVLEPseudo(
2124 /*IsMasked*/ false, IsStrided, /*FF*/ false,
2125 Log2SEW, static_cast<unsigned>(LMUL));
2126 MachineSDNode *Load =
2127 CurDAG->getMachineNode(P->Pseudo, DL, {VT, MVT::Other}, Operands);
2128 // Update the chain.
2129 ReplaceUses(Src.getValue(1), SDValue(Load, 1));
2130 // Record the mem-refs
2131 CurDAG->setNodeMemRefs(Load, {Ld->getMemOperand()});
2132 // Replace the splat with the vlse.
2133 ReplaceNode(Node, Load);
2134 return;
2135 }
2136 case ISD::PREFETCH:
2137 unsigned Locality = Node->getConstantOperandVal(3);
2138 if (Locality > 2)
2139 break;
2140
2141 if (auto *LoadStoreMem = dyn_cast<MemSDNode>(Node)) {
2142 MachineMemOperand *MMO = LoadStoreMem->getMemOperand();
2144
2145 int NontemporalLevel = 0;
2146 switch (Locality) {
2147 case 0:
2148 NontemporalLevel = 3; // NTL.ALL
2149 break;
2150 case 1:
2151 NontemporalLevel = 1; // NTL.PALL
2152 break;
2153 case 2:
2154 NontemporalLevel = 0; // NTL.P1
2155 break;
2156 default:
2157 llvm_unreachable("unexpected locality value.");
2158 }
2159
2160 if (NontemporalLevel & 0b1)
2162 if (NontemporalLevel & 0b10)
2164 }
2165 break;
2166 }
2167
2168 // Select the default instruction.
2169 SelectCode(Node);
2170}
2171
2173 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
2174 std::vector<SDValue> &OutOps) {
2175 // Always produce a register and immediate operand, as expected by
2176 // RISCVAsmPrinter::PrintAsmMemoryOperand.
2177 switch (ConstraintID) {
2180 SDValue Op0, Op1;
2181 bool Found = SelectAddrRegImm(Op, Op0, Op1);
2182 assert(Found && "SelectAddrRegImm should always succeed");
2183 (void)Found;
2184 OutOps.push_back(Op0);
2185 OutOps.push_back(Op1);
2186 return false;
2187 }
2189 OutOps.push_back(Op);
2190 OutOps.push_back(
2191 CurDAG->getTargetConstant(0, SDLoc(Op), Subtarget->getXLenVT()));
2192 return false;
2193 default:
2194 report_fatal_error("Unexpected asm memory constraint " +
2195 InlineAsm::getMemConstraintName(ConstraintID));
2196 }
2197
2198 return true;
2199}
2200
2202 SDValue &Offset) {
2203 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
2204 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
2205 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), Subtarget->getXLenVT());
2206 return true;
2207 }
2208
2209 return false;
2210}
2211
2212// Select a frame index and an optional immediate offset from an ADD or OR.
2214 SDValue &Offset) {
2216 return true;
2217
2219 return false;
2220
2221 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr.getOperand(0))) {
2222 int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
2223 if (isInt<12>(CVal)) {
2224 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(),
2225 Subtarget->getXLenVT());
2227 Subtarget->getXLenVT());
2228 return true;
2229 }
2230 }
2231
2232 return false;
2233}
2234
2235// Fold constant addresses.
2236static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL,
2237 const MVT VT, const RISCVSubtarget *Subtarget,
2239 if (!isa<ConstantSDNode>(Addr))
2240 return false;
2241
2242 int64_t CVal = cast<ConstantSDNode>(Addr)->getSExtValue();
2243
2244 // If the constant is a simm12, we can fold the whole constant and use X0 as
2245 // the base. If the constant can be materialized with LUI+simm12, use LUI as
2246 // the base. We can't use generateInstSeq because it favors LUI+ADDIW.
2247 int64_t Lo12 = SignExtend64<12>(CVal);
2248 int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12;
2249 if (!Subtarget->is64Bit() || isInt<32>(Hi)) {
2250 if (Hi) {
2251 int64_t Hi20 = (Hi >> 12) & 0xfffff;
2252 Base = SDValue(
2253 CurDAG->getMachineNode(RISCV::LUI, DL, VT,
2254 CurDAG->getTargetConstant(Hi20, DL, VT)),
2255 0);
2256 } else {
2257 Base = CurDAG->getRegister(RISCV::X0, VT);
2258 }
2259 Offset = CurDAG->getTargetConstant(Lo12, DL, VT);
2260 return true;
2261 }
2262
2263 // Ask how constant materialization would handle this constant.
2265 RISCVMatInt::generateInstSeq(CVal, Subtarget->getFeatureBits());
2266
2267 // If the last instruction would be an ADDI, we can fold its immediate and
2268 // emit the rest of the sequence as the base.
2269 if (Seq.back().getOpcode() != RISCV::ADDI)
2270 return false;
2271 Lo12 = Seq.back().getImm();
2272
2273 // Drop the last instruction.
2274 Seq.pop_back();
2275 assert(!Seq.empty() && "Expected more instructions in sequence");
2276
2277 Base = selectImmSeq(CurDAG, DL, VT, Seq);
2278 Offset = CurDAG->getTargetConstant(Lo12, DL, VT);
2279 return true;
2280}
2281
2282// Is this ADD instruction only used as the base pointer of scalar loads and
2283// stores?
2285 for (auto *Use : Add->uses()) {
2286 if (Use->getOpcode() != ISD::LOAD && Use->getOpcode() != ISD::STORE &&
2287 Use->getOpcode() != ISD::ATOMIC_LOAD &&
2288 Use->getOpcode() != ISD::ATOMIC_STORE)
2289 return false;
2290 EVT VT = cast<MemSDNode>(Use)->getMemoryVT();
2291 if (!VT.isScalarInteger() && VT != MVT::f16 && VT != MVT::f32 &&
2292 VT != MVT::f64)
2293 return false;
2294 // Don't allow stores of the value. It must be used as the address.
2295 if (Use->getOpcode() == ISD::STORE &&
2296 cast<StoreSDNode>(Use)->getValue() == Add)
2297 return false;
2298 if (Use->getOpcode() == ISD::ATOMIC_STORE &&
2299 cast<AtomicSDNode>(Use)->getVal() == Add)
2300 return false;
2301 }
2302
2303 return true;
2304}
2305
2307 unsigned MaxShiftAmount,
2309 SDValue &Scale) {
2310 EVT VT = Addr.getSimpleValueType();
2311 auto UnwrapShl = [this, VT, MaxShiftAmount](SDValue N, SDValue &Index,
2312 SDValue &Shift) {
2313 uint64_t ShiftAmt = 0;
2314 Index = N;
2315
2316 if (N.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N.getOperand(1))) {
2317 // Only match shifts by a value in range [0, MaxShiftAmount].
2318 if (N.getConstantOperandVal(1) <= MaxShiftAmount) {
2319 Index = N.getOperand(0);
2320 ShiftAmt = N.getConstantOperandVal(1);
2321 }
2322 }
2323
2324 Shift = CurDAG->getTargetConstant(ShiftAmt, SDLoc(N), VT);
2325 return ShiftAmt != 0;
2326 };
2327
2328 if (Addr.getOpcode() == ISD::ADD) {
2329 if (auto *C1 = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
2330 SDValue AddrB = Addr.getOperand(0);
2331 if (AddrB.getOpcode() == ISD::ADD &&
2332 UnwrapShl(AddrB.getOperand(0), Index, Scale) &&
2333 !isa<ConstantSDNode>(AddrB.getOperand(1)) &&
2334 isInt<12>(C1->getSExtValue())) {
2335 // (add (add (shl A C2) B) C1) -> (add (add B C1) (shl A C2))
2336 SDValue C1Val =
2337 CurDAG->getTargetConstant(C1->getZExtValue(), SDLoc(Addr), VT);
2338 Base = SDValue(CurDAG->getMachineNode(RISCV::ADDI, SDLoc(Addr), VT,
2339 AddrB.getOperand(1), C1Val),
2340 0);
2341 return true;
2342 }
2343 } else if (UnwrapShl(Addr.getOperand(0), Index, Scale)) {
2344 Base = Addr.getOperand(1);
2345 return true;
2346 } else {
2347 UnwrapShl(Addr.getOperand(1), Index, Scale);
2348 Base = Addr.getOperand(0);
2349 return true;
2350 }
2351 } else if (UnwrapShl(Addr, Index, Scale)) {
2352 EVT VT = Addr.getValueType();
2353 Base = CurDAG->getRegister(RISCV::X0, VT);
2354 return true;
2355 }
2356
2357 return false;
2358}
2359
2361 SDValue &Offset, bool IsINX) {
2363 return true;
2364
2365 SDLoc DL(Addr);
2366 MVT VT = Addr.getSimpleValueType();
2367
2368 if (Addr.getOpcode() == RISCVISD::ADD_LO) {
2369 Base = Addr.getOperand(0);
2370 Offset = Addr.getOperand(1);
2371 return true;
2372 }
2373
2374 int64_t RV32ZdinxRange = IsINX ? 4 : 0;
2376 int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
2377 if (isInt<12>(CVal) && isInt<12>(CVal + RV32ZdinxRange)) {
2378 Base = Addr.getOperand(0);
2379 if (Base.getOpcode() == RISCVISD::ADD_LO) {
2380 SDValue LoOperand = Base.getOperand(1);
2381 if (auto *GA = dyn_cast<GlobalAddressSDNode>(LoOperand)) {
2382 // If the Lo in (ADD_LO hi, lo) is a global variable's address
2383 // (its low part, really), then we can rely on the alignment of that
2384 // variable to provide a margin of safety before low part can overflow
2385 // the 12 bits of the load/store offset. Check if CVal falls within
2386 // that margin; if so (low part + CVal) can't overflow.
2387 const DataLayout &DL = CurDAG->getDataLayout();
2388 Align Alignment = commonAlignment(
2389 GA->getGlobal()->getPointerAlignment(DL), GA->getOffset());
2390 if (CVal == 0 || Alignment > CVal) {
2391 int64_t CombinedOffset = CVal + GA->getOffset();
2392 Base = Base.getOperand(0);
2394 GA->getGlobal(), SDLoc(LoOperand), LoOperand.getValueType(),
2395 CombinedOffset, GA->getTargetFlags());
2396 return true;
2397 }
2398 }
2399 }
2400
2401 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Base))
2402 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), VT);
2403 Offset = CurDAG->getTargetConstant(CVal, DL, VT);
2404 return true;
2405 }
2406 }
2407
2408 // Handle ADD with large immediates.
2409 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Addr.getOperand(1))) {
2410 int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
2411 assert(!(isInt<12>(CVal) && isInt<12>(CVal + RV32ZdinxRange)) &&
2412 "simm12 not already handled?");
2413
2414 // Handle immediates in the range [-4096,-2049] or [2048, 4094]. We can use
2415 // an ADDI for part of the offset and fold the rest into the load/store.
2416 // This mirrors the AddiPair PatFrag in RISCVInstrInfo.td.
2417 if (isInt<12>(CVal / 2) && isInt<12>(CVal - CVal / 2)) {
2418 int64_t Adj = CVal < 0 ? -2048 : 2047;
2419 Base = SDValue(
2420 CurDAG->getMachineNode(RISCV::ADDI, DL, VT, Addr.getOperand(0),
2421 CurDAG->getTargetConstant(Adj, DL, VT)),
2422 0);
2423 Offset = CurDAG->getTargetConstant(CVal - Adj, DL, VT);
2424 return true;
2425 }
2426
2427 // For larger immediates, we might be able to save one instruction from
2428 // constant materialization by folding the Lo12 bits of the immediate into
2429 // the address. We should only do this if the ADD is only used by loads and
2430 // stores that can fold the lo12 bits. Otherwise, the ADD will get iseled
2431 // separately with the full materialized immediate creating extra
2432 // instructions.
2433 if (isWorthFoldingAdd(Addr) &&
2434 selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr.getOperand(1), Base,
2435 Offset)) {
2436 // Insert an ADD instruction with the materialized Hi52 bits.
2437 Base = SDValue(
2438 CurDAG->getMachineNode(RISCV::ADD, DL, VT, Addr.getOperand(0), Base),
2439 0);
2440 return true;
2441 }
2442 }
2443
2444 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr, Base, Offset))
2445 return true;
2446
2447 Base = Addr;
2448 Offset = CurDAG->getTargetConstant(0, DL, VT);
2449 return true;
2450}
2451
2453 SDValue &ShAmt) {
2454 ShAmt = N;
2455
2456 // Shift instructions on RISC-V only read the lower 5 or 6 bits of the shift
2457 // amount. If there is an AND on the shift amount, we can bypass it if it
2458 // doesn't affect any of those bits.
2459 if (ShAmt.getOpcode() == ISD::AND && isa<ConstantSDNode>(ShAmt.getOperand(1))) {
2460 const APInt &AndMask = ShAmt.getConstantOperandAPInt(1);
2461
2462 // Since the max shift amount is a power of 2 we can subtract 1 to make a
2463 // mask that covers the bits needed to represent all shift amounts.
2464 assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
2465 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
2466
2467 if (ShMask.isSubsetOf(AndMask)) {
2468 ShAmt = ShAmt.getOperand(0);
2469 } else {
2470 // SimplifyDemandedBits may have optimized the mask so try restoring any
2471 // bits that are known zero.
2472 KnownBits Known = CurDAG->computeKnownBits(ShAmt.getOperand(0));
2473 if (!ShMask.isSubsetOf(AndMask | Known.Zero))
2474 return true;
2475 ShAmt = ShAmt.getOperand(0);
2476 }
2477 }
2478
2479 if (ShAmt.getOpcode() == ISD::ADD &&
2480 isa<ConstantSDNode>(ShAmt.getOperand(1))) {
2481 uint64_t Imm = ShAmt.getConstantOperandVal(1);
2482 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
2483 // to avoid the ADD.
2484 if (Imm != 0 && Imm % ShiftWidth == 0) {
2485 ShAmt = ShAmt.getOperand(0);
2486 return true;
2487 }
2488 } else if (ShAmt.getOpcode() == ISD::SUB &&
2489 isa<ConstantSDNode>(ShAmt.getOperand(0))) {
2490 uint64_t Imm = ShAmt.getConstantOperandVal(0);
2491 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X to
2492 // generate a NEG instead of a SUB of a constant.
2493 if (Imm != 0 && Imm % ShiftWidth == 0) {
2494 SDLoc DL(ShAmt);
2495 EVT VT = ShAmt.getValueType();
2496 SDValue Zero = CurDAG->getRegister(RISCV::X0, VT);
2497 unsigned NegOpc = VT == MVT::i64 ? RISCV::SUBW : RISCV::SUB;
2498 MachineSDNode *Neg = CurDAG->getMachineNode(NegOpc, DL, VT, Zero,
2499 ShAmt.getOperand(1));
2500 ShAmt = SDValue(Neg, 0);
2501 return true;
2502 }
2503 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
2504 // to generate a NOT instead of a SUB of a constant.
2505 if (Imm % ShiftWidth == ShiftWidth - 1) {
2506 SDLoc DL(ShAmt);
2507 EVT VT = ShAmt.getValueType();
2508 MachineSDNode *Not =
2509 CurDAG->getMachineNode(RISCV::XORI, DL, VT, ShAmt.getOperand(1),
2510 CurDAG->getTargetConstant(-1, DL, VT));
2511 ShAmt = SDValue(Not, 0);
2512 return true;
2513 }
2514 }
2515
2516 return true;
2517}
2518
2519/// RISC-V doesn't have general instructions for integer setne/seteq, but we can
2520/// check for equality with 0. This function emits instructions that convert the
2521/// seteq/setne into something that can be compared with 0.
2522/// \p ExpectedCCVal indicates the condition code to attempt to match (e.g.
2523/// ISD::SETNE).
2525 SDValue &Val) {
2526 assert(ISD::isIntEqualitySetCC(ExpectedCCVal) &&
2527 "Unexpected condition code!");
2528
2529 // We're looking for a setcc.
2530 if (N->getOpcode() != ISD::SETCC)
2531 return false;
2532
2533 // Must be an equality comparison.
2534 ISD::CondCode CCVal = cast<CondCodeSDNode>(N->getOperand(2))->get();
2535 if (CCVal != ExpectedCCVal)
2536 return false;
2537
2538 SDValue LHS = N->getOperand(0);
2539 SDValue RHS = N->getOperand(1);
2540
2541 if (!LHS.getValueType().isScalarInteger())
2542 return false;
2543
2544 // If the RHS side is 0, we don't need any extra instructions, return the LHS.
2545 if (isNullConstant(RHS)) {
2546 Val = LHS;
2547 return true;
2548 }
2549
2550 SDLoc DL(N);
2551
2552 if (auto *C = dyn_cast<ConstantSDNode>(RHS)) {
2553 int64_t CVal = C->getSExtValue();
2554 // If the RHS is -2048, we can use xori to produce 0 if the LHS is -2048 and
2555 // non-zero otherwise.
2556 if (CVal == -2048) {
2557 Val =
2559 RISCV::XORI, DL, N->getValueType(0), LHS,
2560 CurDAG->getTargetConstant(CVal, DL, N->getValueType(0))),
2561 0);
2562 return true;
2563 }
2564 // If the RHS is [-2047,2048], we can use addi with -RHS to produce 0 if the
2565 // LHS is equal to the RHS and non-zero otherwise.
2566 if (isInt<12>(CVal) || CVal == 2048) {
2567 Val =
2569 RISCV::ADDI, DL, N->getValueType(0), LHS,
2570 CurDAG->getTargetConstant(-CVal, DL, N->getValueType(0))),
2571 0);
2572 return true;
2573 }
2574 }
2575
2576 // If nothing else we can XOR the LHS and RHS to produce zero if they are
2577 // equal and a non-zero value if they aren't.
2578 Val = SDValue(
2579 CurDAG->getMachineNode(RISCV::XOR, DL, N->getValueType(0), LHS, RHS), 0);
2580 return true;
2581}
2582
2584 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG &&
2585 cast<VTSDNode>(N.getOperand(1))->getVT().getSizeInBits() == Bits) {
2586 Val = N.getOperand(0);
2587 return true;
2588 }
2589
2590 auto UnwrapShlSra = [](SDValue N, unsigned ShiftAmt) {
2591 if (N.getOpcode() != ISD::SRA || !isa<ConstantSDNode>(N.getOperand(1)))
2592 return N;
2593
2594 SDValue N0 = N.getOperand(0);
2595 if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N0.getOperand(1)) &&
2596 N.getConstantOperandVal(1) == ShiftAmt &&
2597 N0.getConstantOperandVal(1) == ShiftAmt)
2598 return N0.getOperand(0);
2599
2600 return N;
2601 };
2602
2603 MVT VT = N.getSimpleValueType();
2604 if (CurDAG->ComputeNumSignBits(N) > (VT.getSizeInBits() - Bits)) {
2605 Val = UnwrapShlSra(N, VT.getSizeInBits() - Bits);
2606 return true;
2607 }
2608
2609 return false;
2610}
2611
2613 if (N.getOpcode() == ISD::AND) {
2614 auto *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
2615 if (C && C->getZExtValue() == maskTrailingOnes<uint64_t>(Bits)) {
2616 Val = N.getOperand(0);
2617 return true;
2618 }
2619 }
2620 MVT VT = N.getSimpleValueType();
2621 APInt Mask = APInt::getBitsSetFrom(VT.getSizeInBits(), Bits);
2622 if (CurDAG->MaskedValueIsZero(N, Mask)) {
2623 Val = N;
2624 return true;
2625 }
2626
2627 return false;
2628}
2629
2630/// Look for various patterns that can be done with a SHL that can be folded
2631/// into a SHXADD. \p ShAmt contains 1, 2, or 3 and is set based on which
2632/// SHXADD we are trying to match.
2634 SDValue &Val) {
2635 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1))) {
2636 SDValue N0 = N.getOperand(0);
2637
2638 bool LeftShift = N0.getOpcode() == ISD::SHL;
2639 if ((LeftShift || N0.getOpcode() == ISD::SRL) &&
2640 isa<ConstantSDNode>(N0.getOperand(1))) {
2641 uint64_t Mask = N.getConstantOperandVal(1);
2642 unsigned C2 = N0.getConstantOperandVal(1);
2643
2644 unsigned XLen = Subtarget->getXLen();
2645 if (LeftShift)
2646 Mask &= maskTrailingZeros<uint64_t>(C2);
2647 else
2648 Mask &= maskTrailingOnes<uint64_t>(XLen - C2);
2649
2650 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with no
2651 // leading zeros and c3 trailing zeros. We can use an SRLI by c2+c3
2652 // followed by a SHXADD with c3 for the X amount.
2653 if (isShiftedMask_64(Mask)) {
2654 unsigned Leading = XLen - llvm::bit_width(Mask);
2655 unsigned Trailing = llvm::countr_zero(Mask);
2656 if (LeftShift && Leading == 0 && C2 < Trailing && Trailing == ShAmt) {
2657 SDLoc DL(N);
2658 EVT VT = N.getValueType();
2660 RISCV::SRLI, DL, VT, N0.getOperand(0),
2661 CurDAG->getTargetConstant(Trailing - C2, DL, VT)),
2662 0);
2663 return true;
2664 }
2665 // Look for (and (shr y, c2), c1) where c1 is a shifted mask with c2
2666 // leading zeros and c3 trailing zeros. We can use an SRLI by C3
2667 // followed by a SHXADD using c3 for the X amount.
2668 if (!LeftShift && Leading == C2 && Trailing == ShAmt) {
2669 SDLoc DL(N);
2670 EVT VT = N.getValueType();
2671 Val = SDValue(
2673 RISCV::SRLI, DL, VT, N0.getOperand(0),
2674 CurDAG->getTargetConstant(Leading + Trailing, DL, VT)),
2675 0);
2676 return true;
2677 }
2678 }
2679 }
2680 }
2681
2682 bool LeftShift = N.getOpcode() == ISD::SHL;
2683 if ((LeftShift || N.getOpcode() == ISD::SRL) &&
2684 isa<ConstantSDNode>(N.getOperand(1))) {
2685 SDValue N0 = N.getOperand(0);
2686 if (N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
2687 isa<ConstantSDNode>(N0.getOperand(1))) {
2688 uint64_t Mask = N0.getConstantOperandVal(1);
2689 if (isShiftedMask_64(Mask)) {
2690 unsigned C1 = N.getConstantOperandVal(1);
2691 unsigned XLen = Subtarget->getXLen();
2692 unsigned Leading = XLen - llvm::bit_width(Mask);
2693 unsigned Trailing = llvm::countr_zero(Mask);
2694 // Look for (shl (and X, Mask), C1) where Mask has 32 leading zeros and
2695 // C3 trailing zeros. If C1+C3==ShAmt we can use SRLIW+SHXADD.
2696 if (LeftShift && Leading == 32 && Trailing > 0 &&
2697 (Trailing + C1) == ShAmt) {
2698 SDLoc DL(N);
2699 EVT VT = N.getValueType();
2701 RISCV::SRLIW, DL, VT, N0.getOperand(0),
2702 CurDAG->getTargetConstant(Trailing, DL, VT)),
2703 0);
2704 return true;
2705 }
2706 // Look for (srl (and X, Mask), C1) where Mask has 32 leading zeros and
2707 // C3 trailing zeros. If C3-C1==ShAmt we can use SRLIW+SHXADD.
2708 if (!LeftShift && Leading == 32 && Trailing > C1 &&
2709 (Trailing - C1) == ShAmt) {
2710 SDLoc DL(N);
2711 EVT VT = N.getValueType();
2713 RISCV::SRLIW, DL, VT, N0.getOperand(0),
2714 CurDAG->getTargetConstant(Trailing, DL, VT)),
2715 0);
2716 return true;
2717 }
2718 }
2719 }
2720 }
2721
2722 return false;
2723}
2724
2725/// Look for various patterns that can be done with a SHL that can be folded
2726/// into a SHXADD_UW. \p ShAmt contains 1, 2, or 3 and is set based on which
2727/// SHXADD_UW we are trying to match.
2729 SDValue &Val) {
2730 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1)) &&
2731 N.hasOneUse()) {
2732 SDValue N0 = N.getOperand(0);
2733 if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N0.getOperand(1)) &&
2734 N0.hasOneUse()) {
2735 uint64_t Mask = N.getConstantOperandVal(1);
2736 unsigned C2 = N0.getConstantOperandVal(1);
2737
2738 Mask &= maskTrailingZeros<uint64_t>(C2);
2739
2740 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with
2741 // 32-ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
2742 // c2-ShAmt followed by SHXADD_UW with ShAmt for the X amount.
2743 if (isShiftedMask_64(Mask)) {
2744 unsigned Leading = llvm::countl_zero(Mask);
2745 unsigned Trailing = llvm::countr_zero(Mask);
2746 if (Leading == 32 - ShAmt && Trailing == C2 && Trailing > ShAmt) {
2747 SDLoc DL(N);
2748 EVT VT = N.getValueType();
2750 RISCV::SLLI, DL, VT, N0.getOperand(0),
2751 CurDAG->getTargetConstant(C2 - ShAmt, DL, VT)),
2752 0);
2753 return true;
2754 }
2755 }
2756 }
2757 }
2758
2759 return false;
2760}
2761
2762static bool vectorPseudoHasAllNBitUsers(SDNode *User, unsigned UserOpNo,
2763 unsigned Bits,
2764 const TargetInstrInfo *TII) {
2765 const RISCVVPseudosTable::PseudoInfo *PseudoInfo =
2766 RISCVVPseudosTable::getPseudoInfo(User->getMachineOpcode());
2767
2768 if (!PseudoInfo)
2769 return false;
2770
2771 const MCInstrDesc &MCID = TII->get(User->getMachineOpcode());
2772 const uint64_t TSFlags = MCID.TSFlags;
2774 return false;
2776
2777 bool HasGlueOp = User->getGluedNode() != nullptr;
2778 unsigned ChainOpIdx = User->getNumOperands() - HasGlueOp - 1;
2779 bool HasChainOp = User->getOperand(ChainOpIdx).getValueType() == MVT::Other;
2780 bool HasVecPolicyOp = RISCVII::hasVecPolicyOp(TSFlags);
2781 unsigned VLIdx =
2782 User->getNumOperands() - HasVecPolicyOp - HasChainOp - HasGlueOp - 2;
2783 const unsigned Log2SEW = User->getConstantOperandVal(VLIdx + 1);
2784
2785 if (UserOpNo == VLIdx)
2786 return false;
2787
2788 // TODO: Handle Zvbb instructions
2789 switch (PseudoInfo->BaseInstr) {
2790 default:
2791 return false;
2792
2793 // 11.6. Vector Single-Width Shift Instructions
2794 case RISCV::VSLL_VX:
2795 case RISCV::VSRL_VX:
2796 case RISCV::VSRA_VX:
2797 // 12.4. Vector Single-Width Scaling Shift Instructions
2798 case RISCV::VSSRL_VX:
2799 case RISCV::VSSRA_VX:
2800 // Only the low lg2(SEW) bits of the shift-amount value are used.
2801 if (Bits < Log2SEW)
2802 return false;
2803 break;
2804
2805 // 11.7 Vector Narrowing Integer Right Shift Instructions
2806 case RISCV::VNSRL_WX:
2807 case RISCV::VNSRA_WX:
2808 // 12.5. Vector Narrowing Fixed-Point Clip Instructions
2809 case RISCV::VNCLIPU_WX:
2810 case RISCV::VNCLIP_WX:
2811 // Only the low lg2(2*SEW) bits of the shift-amount value are used.
2812 if (Bits < Log2SEW + 1)
2813 return false;
2814 break;
2815
2816 // 11.1. Vector Single-Width Integer Add and Subtract
2817 case RISCV::VADD_VX:
2818 case RISCV::VSUB_VX:
2819 case RISCV::VRSUB_VX:
2820 // 11.2. Vector Widening Integer Add/Subtract
2821 case RISCV::VWADDU_VX:
2822 case RISCV::VWSUBU_VX:
2823 case RISCV::VWADD_VX:
2824 case RISCV::VWSUB_VX:
2825 case RISCV::VWADDU_WX:
2826 case RISCV::VWSUBU_WX:
2827 case RISCV::VWADD_WX:
2828 case RISCV::VWSUB_WX:
2829 // 11.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
2830 case RISCV::VADC_VXM:
2831 case RISCV::VADC_VIM:
2832 case RISCV::VMADC_VXM:
2833 case RISCV::VMADC_VIM:
2834 case RISCV::VMADC_VX:
2835 case RISCV::VSBC_VXM:
2836 case RISCV::VMSBC_VXM:
2837 case RISCV::VMSBC_VX:
2838 // 11.5 Vector Bitwise Logical Instructions
2839 case RISCV::VAND_VX:
2840 case RISCV::VOR_VX:
2841 case RISCV::VXOR_VX:
2842 // 11.8. Vector Integer Compare Instructions
2843 case RISCV::VMSEQ_VX:
2844 case RISCV::VMSNE_VX:
2845 case RISCV::VMSLTU_VX:
2846 case RISCV::VMSLT_VX:
2847 case RISCV::VMSLEU_VX:
2848 case RISCV::VMSLE_VX:
2849 case RISCV::VMSGTU_VX:
2850 case RISCV::VMSGT_VX:
2851 // 11.9. Vector Integer Min/Max Instructions
2852 case RISCV::VMINU_VX:
2853 case RISCV::VMIN_VX:
2854 case RISCV::VMAXU_VX:
2855 case RISCV::VMAX_VX:
2856 // 11.10. Vector Single-Width Integer Multiply Instructions
2857 case RISCV::VMUL_VX:
2858 case RISCV::VMULH_VX:
2859 case RISCV::VMULHU_VX:
2860 case RISCV::VMULHSU_VX:
2861 // 11.11. Vector Integer Divide Instructions
2862 case RISCV::VDIVU_VX:
2863 case RISCV::VDIV_VX:
2864 case RISCV::VREMU_VX:
2865 case RISCV::VREM_VX:
2866 // 11.12. Vector Widening Integer Multiply Instructions
2867 case RISCV::VWMUL_VX:
2868 case RISCV::VWMULU_VX:
2869 case RISCV::VWMULSU_VX:
2870 // 11.13. Vector Single-Width Integer Multiply-Add Instructions
2871 case RISCV::VMACC_VX:
2872 case RISCV::VNMSAC_VX:
2873 case RISCV::VMADD_VX:
2874 case RISCV::VNMSUB_VX:
2875 // 11.14. Vector Widening Integer Multiply-Add Instructions
2876 case RISCV::VWMACCU_VX:
2877 case RISCV::VWMACC_VX:
2878 case RISCV::VWMACCSU_VX:
2879 case RISCV::VWMACCUS_VX:
2880 // 11.15. Vector Integer Merge Instructions
2881 case RISCV::VMERGE_VXM:
2882 // 11.16. Vector Integer Move Instructions
2883 case RISCV::VMV_V_X:
2884 // 12.1. Vector Single-Width Saturating Add and Subtract
2885 case RISCV::VSADDU_VX:
2886 case RISCV::VSADD_VX:
2887 case RISCV::VSSUBU_VX:
2888 case RISCV::VSSUB_VX:
2889 // 12.2. Vector Single-Width Averaging Add and Subtract
2890 case RISCV::VAADDU_VX:
2891 case RISCV::VAADD_VX:
2892 case RISCV::VASUBU_VX:
2893 case RISCV::VASUB_VX:
2894 // 12.3. Vector Single-Width Fractional Multiply with Rounding and Saturation
2895 case RISCV::VSMUL_VX:
2896 // 16.1. Integer Scalar Move Instructions
2897 case RISCV::VMV_S_X:
2898 if (Bits < (1U << Log2SEW))
2899 return false;
2900 }
2901 return true;
2902}
2903
2904// Return true if all users of this SDNode* only consume the lower \p Bits.
2905// This can be used to form W instructions for add/sub/mul/shl even when the
2906// root isn't a sext_inreg. This can allow the ADDW/SUBW/MULW/SLLIW to CSE if
2907// SimplifyDemandedBits has made it so some users see a sext_inreg and some
2908// don't. The sext_inreg+add/sub/mul/shl will get selected, but still leave
2909// the add/sub/mul/shl to become non-W instructions. By checking the users we
2910// may be able to use a W instruction and CSE with the other instruction if
2911// this has happened. We could try to detect that the CSE opportunity exists
2912// before doing this, but that would be more complicated.
2914 const unsigned Depth) const {
2915 assert((Node->getOpcode() == ISD::ADD || Node->getOpcode() == ISD::SUB ||
2916 Node->getOpcode() == ISD::MUL || Node->getOpcode() == ISD::SHL ||
2917 Node->getOpcode() == ISD::SRL || Node->getOpcode() == ISD::AND ||
2918 Node->getOpcode() == ISD::OR || Node->getOpcode() == ISD::XOR ||
2919 Node->getOpcode() == ISD::SIGN_EXTEND_INREG ||
2920 isa<ConstantSDNode>(Node) || Depth != 0) &&
2921 "Unexpected opcode");
2922
2924 return false;
2925
2926 for (auto UI = Node->use_begin(), UE = Node->use_end(); UI != UE; ++UI) {
2927 SDNode *User = *UI;
2928 // Users of this node should have already been instruction selected
2929 if (!User->isMachineOpcode())
2930 return false;
2931
2932 // TODO: Add more opcodes?
2933 switch (User->getMachineOpcode()) {
2934 default:
2935 if (vectorPseudoHasAllNBitUsers(User, UI.getOperandNo(), Bits, TII))
2936 break;
2937 return false;
2938 case RISCV::ADDW:
2939 case RISCV::ADDIW:
2940 case RISCV::SUBW:
2941 case RISCV::MULW:
2942 case RISCV::SLLW:
2943 case RISCV::SLLIW:
2944 case RISCV::SRAW:
2945 case RISCV::SRAIW:
2946 case RISCV::SRLW:
2947 case RISCV::SRLIW:
2948 case RISCV::DIVW:
2949 case RISCV::DIVUW:
2950 case RISCV::REMW:
2951 case RISCV::REMUW:
2952 case RISCV::ROLW:
2953 case RISCV::RORW:
2954 case RISCV::RORIW:
2955 case RISCV::CLZW:
2956 case RISCV::CTZW:
2957 case RISCV::CPOPW:
2958 case RISCV::SLLI_UW:
2959 case RISCV::FMV_W_X:
2960 case RISCV::FCVT_H_W:
2961 case RISCV::FCVT_H_WU:
2962 case RISCV::FCVT_S_W:
2963 case RISCV::FCVT_S_WU:
2964 case RISCV::FCVT_D_W:
2965 case RISCV::FCVT_D_WU:
2966 case RISCV::TH_REVW:
2967 case RISCV::TH_SRRIW:
2968 if (Bits < 32)
2969 return false;
2970 break;
2971 case RISCV::SLL:
2972 case RISCV::SRA:
2973 case RISCV::SRL:
2974 case RISCV::ROL:
2975 case RISCV::ROR:
2976 case RISCV::BSET:
2977 case RISCV::BCLR:
2978 case RISCV::BINV:
2979 // Shift amount operands only use log2(Xlen) bits.
2980 if (UI.getOperandNo() != 1 || Bits < Log2_32(Subtarget->getXLen()))
2981 return false;
2982 break;
2983 case RISCV::SLLI:
2984 // SLLI only uses the lower (XLen - ShAmt) bits.
2985 if (Bits < Subtarget->getXLen() - User->getConstantOperandVal(1))
2986 return false;
2987 break;
2988 case RISCV::ANDI:
2989 if (Bits >= (unsigned)llvm::bit_width(User->getConstantOperandVal(1)))
2990 break;
2991 goto RecCheck;
2992 case RISCV::ORI: {
2993 uint64_t Imm = cast<ConstantSDNode>(User->getOperand(1))->getSExtValue();
2994 if (Bits >= (unsigned)llvm::bit_width<uint64_t>(~Imm))
2995 break;
2996 [[fallthrough]];
2997 }
2998 case RISCV::AND:
2999 case RISCV::OR:
3000 case RISCV::XOR:
3001 case RISCV::XORI:
3002 case RISCV::ANDN:
3003 case RISCV::ORN:
3004 case RISCV::XNOR:
3005 case RISCV::SH1ADD:
3006 case RISCV::SH2ADD:
3007 case RISCV::SH3ADD:
3008 RecCheck:
3009 if (hasAllNBitUsers(User, Bits, Depth + 1))
3010 break;
3011 return false;
3012 case RISCV::SRLI: {
3013 unsigned ShAmt = User->getConstantOperandVal(1);
3014 // If we are shifting right by less than Bits, and users don't demand any
3015 // bits that were shifted into [Bits-1:0], then we can consider this as an
3016 // N-Bit user.
3017 if (Bits > ShAmt && hasAllNBitUsers(User, Bits - ShAmt, Depth + 1))
3018 break;
3019 return false;
3020 }
3021 case RISCV::SEXT_B:
3022 case RISCV::PACKH:
3023 if (Bits < 8)
3024 return false;
3025 break;
3026 case RISCV::SEXT_H:
3027 case RISCV::FMV_H_X:
3028 case RISCV::ZEXT_H_RV32:
3029 case RISCV::ZEXT_H_RV64:
3030 case RISCV::PACKW:
3031 if (Bits < 16)
3032 return false;
3033 break;
3034 case RISCV::PACK:
3035 if (Bits < (Subtarget->getXLen() / 2))
3036 return false;
3037 break;
3038 case RISCV::ADD_UW:
3039 case RISCV::SH1ADD_UW:
3040 case RISCV::SH2ADD_UW:
3041 case RISCV::SH3ADD_UW:
3042 // The first operand to add.uw/shXadd.uw is implicitly zero extended from
3043 // 32 bits.
3044 if (UI.getOperandNo() != 0 || Bits < 32)
3045 return false;
3046 break;
3047 case RISCV::SB:
3048 if (UI.getOperandNo() != 0 || Bits < 8)
3049 return false;
3050 break;
3051 case RISCV::SH:
3052 if (UI.getOperandNo() != 0 || Bits < 16)
3053 return false;
3054 break;
3055 case RISCV::SW:
3056 if (UI.getOperandNo() != 0 || Bits < 32)
3057 return false;
3058 break;
3059 }
3060 }
3061
3062 return true;
3063}
3064
3065// Select a constant that can be represented as (sign_extend(imm5) << imm2).
3067 SDValue &Shl2) {
3068 if (auto *C = dyn_cast<ConstantSDNode>(N)) {
3069 int64_t Offset = C->getSExtValue();
3070 int64_t Shift;
3071 for (Shift = 0; Shift < 4; Shift++)
3072 if (isInt<5>(Offset >> Shift) && ((Offset % (1LL << Shift)) == 0))
3073 break;
3074
3075 // Constant cannot be encoded.
3076 if (Shift == 4)
3077 return false;
3078
3079 EVT Ty = N->getValueType(0);
3080 Simm5 = CurDAG->getTargetConstant(Offset >> Shift, SDLoc(N), Ty);
3081 Shl2 = CurDAG->getTargetConstant(Shift, SDLoc(N), Ty);
3082 return true;
3083 }
3084
3085 return false;
3086}
3087
3088// Select VL as a 5 bit immediate or a value that will become a register. This
3089// allows us to choose betwen VSETIVLI or VSETVLI later.
3091 auto *C = dyn_cast<ConstantSDNode>(N);
3092 if (C && isUInt<5>(C->getZExtValue())) {
3093 VL = CurDAG->getTargetConstant(C->getZExtValue(), SDLoc(N),
3094 N->getValueType(0));
3095 } else if (C && C->isAllOnes()) {
3096 // Treat all ones as VLMax.
3098 N->getValueType(0));
3099 } else if (isa<RegisterSDNode>(N) &&
3100 cast<RegisterSDNode>(N)->getReg() == RISCV::X0) {
3101 // All our VL operands use an operand that allows GPRNoX0 or an immediate
3102 // as the register class. Convert X0 to a special immediate to pass the
3103 // MachineVerifier. This is recognized specially by the vsetvli insertion
3104 // pass.
3106 N->getValueType(0));
3107 } else {
3108 VL = N;
3109 }
3110
3111 return true;
3112}
3113
3115 if (N.getOpcode() == ISD::INSERT_SUBVECTOR) {
3116 if (!N.getOperand(0).isUndef())
3117 return SDValue();
3118 N = N.getOperand(1);
3119 }
3120 SDValue Splat = N;
3121 if ((Splat.getOpcode() != RISCVISD::VMV_V_X_VL &&
3122 Splat.getOpcode() != RISCVISD::VMV_S_X_VL) ||
3123 !Splat.getOperand(0).isUndef())
3124 return SDValue();
3125 assert(Splat.getNumOperands() == 3 && "Unexpected number of operands");
3126 return Splat;
3127}
3128
3131 if (!Splat)
3132 return false;
3133
3134 SplatVal = Splat.getOperand(1);
3135 return true;
3136}
3137
3139 SelectionDAG &DAG,
3140 const RISCVSubtarget &Subtarget,
3141 std::function<bool(int64_t)> ValidateImm) {
3143 if (!Splat || !isa<ConstantSDNode>(Splat.getOperand(1)))
3144 return false;
3145
3146 const unsigned SplatEltSize = Splat.getScalarValueSizeInBits();
3147 assert(Subtarget.getXLenVT() == Splat.getOperand(1).getSimpleValueType() &&
3148 "Unexpected splat operand type");
3149
3150 // The semantics of RISCVISD::VMV_V_X_VL is that when the operand
3151 // type is wider than the resulting vector element type: an implicit
3152 // truncation first takes place. Therefore, perform a manual
3153 // truncation/sign-extension in order to ignore any truncated bits and catch
3154 // any zero-extended immediate.
3155 // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first
3156 // sign-extending to (XLenVT -1).
3157 APInt SplatConst = Splat.getConstantOperandAPInt(1).sextOrTrunc(SplatEltSize);
3158
3159 int64_t SplatImm = SplatConst.getSExtValue();
3160
3161 if (!ValidateImm(SplatImm))
3162 return false;
3163
3164 SplatVal = DAG.getTargetConstant(SplatImm, SDLoc(N), Subtarget.getXLenVT());
3165 return true;
3166}
3167
3169 return selectVSplatImmHelper(N, SplatVal, *CurDAG, *Subtarget,
3170 [](int64_t Imm) { return isInt<5>(Imm); });
3171}
3172
3174 return selectVSplatImmHelper(
3175 N, SplatVal, *CurDAG, *Subtarget,
3176 [](int64_t Imm) { return (isInt<5>(Imm) && Imm != -16) || Imm == 16; });
3177}
3178
3180 SDValue &SplatVal) {
3181 return selectVSplatImmHelper(
3182 N, SplatVal, *CurDAG, *Subtarget, [](int64_t Imm) {
3183 return Imm != 0 && ((isInt<5>(Imm) && Imm != -16) || Imm == 16);
3184 });
3185}
3186
3188 SDValue &SplatVal) {
3189 return selectVSplatImmHelper(
3190 N, SplatVal, *CurDAG, *Subtarget,
3191 [Bits](int64_t Imm) { return isUIntN(Bits, Imm); });
3192}
3193
3195 // Truncates are custom lowered during legalization.
3196 auto IsTrunc = [this](SDValue N) {
3197 if (N->getOpcode() != RISCVISD::TRUNCATE_VECTOR_VL)
3198 return false;
3199 SDValue VL;
3200 selectVLOp(N->getOperand(2), VL);
3201 // Any vmset_vl is ok, since any bits past VL are undefined and we can
3202 // assume they are set.
3203 return N->getOperand(1).getOpcode() == RISCVISD::VMSET_VL &&
3204 isa<ConstantSDNode>(VL) &&
3205 cast<ConstantSDNode>(VL)->getSExtValue() == RISCV::VLMaxSentinel;
3206 };
3207
3208 // We can have multiple nested truncates, so unravel them all if needed.
3209 while (N->getOpcode() == ISD::SIGN_EXTEND ||
3210 N->getOpcode() == ISD::ZERO_EXTEND || IsTrunc(N)) {
3211 if (!N.hasOneUse() ||
3212 N.getValueType().getSizeInBits().getKnownMinValue() < 8)
3213 return false;
3214 N = N->getOperand(0);
3215 }
3216
3217 return selectVSplat(N, SplatVal);
3218}
3219
3221 ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N.getNode());
3222 if (!CFP)
3223 return false;
3224 const APFloat &APF = CFP->getValueAPF();
3225 // td can handle +0.0 already.
3226 if (APF.isPosZero())
3227 return false;
3228
3229 MVT VT = CFP->getSimpleValueType(0);
3230
3231 if (static_cast<const RISCVTargetLowering *>(TLI)->getLegalZfaFPImm(APF,
3232 VT) >= 0)
3233 return false;
3234
3235 MVT XLenVT = Subtarget->getXLenVT();
3236 if (VT == MVT::f64 && !Subtarget->is64Bit()) {
3237 assert(APF.isNegZero() && "Unexpected constant.");
3238 return false;
3239 }
3240 SDLoc DL(N);
3241 Imm = selectImm(CurDAG, DL, XLenVT, APF.bitcastToAPInt().getSExtValue(),
3242 *Subtarget);
3243 return true;
3244}
3245
3247 SDValue &Imm) {
3248 if (auto *C = dyn_cast<ConstantSDNode>(N)) {
3249 int64_t ImmVal = SignExtend64(C->getSExtValue(), Width);
3250
3251 if (!isInt<5>(ImmVal))
3252 return false;
3253
3254 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT());
3255 return true;
3256 }
3257
3258 return false;
3259}
3260
3261// Try to remove sext.w if the input is a W instruction or can be made into
3262// a W instruction cheaply.
3263bool RISCVDAGToDAGISel::doPeepholeSExtW(SDNode *N) {
3264 // Look for the sext.w pattern, addiw rd, rs1, 0.
3265 if (N->getMachineOpcode() != RISCV::ADDIW ||
3266 !isNullConstant(N->getOperand(1)))
3267 return false;
3268
3269 SDValue N0 = N->getOperand(0);
3270 if (!N0.isMachineOpcode())
3271 return false;
3272
3273 switch (N0.getMachineOpcode()) {
3274 default:
3275 break;
3276 case RISCV::ADD:
3277 case RISCV::ADDI:
3278 case RISCV::SUB:
3279 case RISCV::MUL:
3280 case RISCV::SLLI: {
3281 // Convert sext.w+add/sub/mul to their W instructions. This will create
3282 // a new independent instruction. This improves latency.
3283 unsigned Opc;
3284 switch (N0.getMachineOpcode()) {
3285 default:
3286 llvm_unreachable("Unexpected opcode!");
3287 case RISCV::ADD: Opc = RISCV::ADDW; break;
3288 case RISCV::ADDI: Opc = RISCV::ADDIW; break;
3289 case RISCV::SUB: Opc = RISCV::SUBW; break;
3290 case RISCV::MUL: Opc = RISCV::MULW; break;
3291 case RISCV::SLLI: Opc = RISCV::SLLIW; break;
3292 }
3293
3294 SDValue N00 = N0.getOperand(0);
3295 SDValue N01 = N0.getOperand(1);
3296
3297 // Shift amount needs to be uimm5.
3298 if (N0.getMachineOpcode() == RISCV::SLLI &&
3299 !isUInt<5>(cast<ConstantSDNode>(N01)->getSExtValue()))
3300 break;
3301
3302 SDNode *Result =
3303 CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0),
3304 N00, N01);
3305 ReplaceUses(N, Result);
3306 return true;
3307 }
3308 case RISCV::ADDW:
3309 case RISCV::ADDIW:
3310 case RISCV::SUBW:
3311 case RISCV::MULW:
3312 case RISCV::SLLIW:
3313 case RISCV::PACKW:
3314 case RISCV::TH_MULAW:
3315 case RISCV::TH_MULAH:
3316 case RISCV::TH_MULSW:
3317 case RISCV::TH_MULSH:
3318 // Result is already sign extended just remove the sext.w.
3319 // NOTE: We only handle the nodes that are selected with hasAllWUsers.
3320 ReplaceUses(N, N0.getNode());
3321 return true;
3322 }
3323
3324 return false;
3325}
3326
3327static bool usesAllOnesMask(SDValue MaskOp, SDValue GlueOp) {
3328 // Check that we're using V0 as a mask register.
3329 if (!isa<RegisterSDNode>(MaskOp) ||
3330 cast<RegisterSDNode>(MaskOp)->getReg() != RISCV::V0)
3331 return false;
3332
3333 // The glued user defines V0.
3334 const auto *Glued = GlueOp.getNode();
3335
3336 if (!Glued || Glued->getOpcode() != ISD::CopyToReg)
3337 return false;
3338
3339 // Check that we're defining V0 as a mask register.
3340 if (!isa<RegisterSDNode>(Glued->getOperand(1)) ||
3341 cast<RegisterSDNode>(Glued->getOperand(1))->getReg() != RISCV::V0)
3342 return false;
3343
3344 // Check the instruction defining V0; it needs to be a VMSET pseudo.
3345 SDValue MaskSetter = Glued->getOperand(2);
3346
3347 // Sometimes the VMSET is wrapped in a COPY_TO_REGCLASS, e.g. if the mask came
3348 // from an extract_subvector or insert_subvector.
3349 if (MaskSetter->isMachineOpcode() &&
3350 MaskSetter->getMachineOpcode() == RISCV::COPY_TO_REGCLASS)
3351 MaskSetter = MaskSetter->getOperand(0);
3352
3353 const auto IsVMSet = [](unsigned Opc) {
3354 return Opc == RISCV::PseudoVMSET_M_B1 || Opc == RISCV::PseudoVMSET_M_B16 ||
3355 Opc == RISCV::PseudoVMSET_M_B2 || Opc == RISCV::PseudoVMSET_M_B32 ||
3356 Opc == RISCV::PseudoVMSET_M_B4 || Opc == RISCV::PseudoVMSET_M_B64 ||
3357 Opc == RISCV::PseudoVMSET_M_B8;
3358 };
3359
3360 // TODO: Check that the VMSET is the expected bitwidth? The pseudo has
3361 // undefined behaviour if it's the wrong bitwidth, so we could choose to
3362 // assume that it's all-ones? Same applies to its VL.
3363 return MaskSetter->isMachineOpcode() &&
3364 IsVMSet(MaskSetter.getMachineOpcode());
3365}
3366
3367// Return true if we can make sure mask of N is all-ones mask.
3368static bool usesAllOnesMask(SDNode *N, unsigned MaskOpIdx) {
3369 return usesAllOnesMask(N->getOperand(MaskOpIdx),
3370 N->getOperand(N->getNumOperands() - 1));
3371}
3372
3373static bool isImplicitDef(SDValue V) {
3374 return V.isMachineOpcode() &&
3375 V.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF;
3376}
3377
3378// Optimize masked RVV pseudo instructions with a known all-ones mask to their
3379// corresponding "unmasked" pseudo versions. The mask we're interested in will
3380// take the form of a V0 physical register operand, with a glued
3381// register-setting instruction.
3382bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(MachineSDNode *N) {
3384 RISCV::getMaskedPseudoInfo(N->getMachineOpcode());
3385 if (!I)
3386 return false;
3387
3388 unsigned MaskOpIdx = I->MaskOpIdx;
3389 if (!usesAllOnesMask(N, MaskOpIdx))
3390 return false;
3391
3392 // There are two classes of pseudos in the table - compares and
3393 // everything else. See the comment on RISCVMaskedPseudo for details.
3394 const unsigned Opc = I->UnmaskedPseudo;
3395 const MCInstrDesc &MCID = TII->get(Opc);
3396 const bool UseTUPseudo = RISCVII::hasVecPolicyOp(MCID.TSFlags);
3397#ifndef NDEBUG
3398 const MCInstrDesc &MaskedMCID = TII->get(N->getMachineOpcode());
3401 "Masked and unmasked pseudos are inconsistent");
3402 const bool HasTiedDest = RISCVII::isFirstDefTiedToFirstUse(MCID);
3403 assert(UseTUPseudo == HasTiedDest && "Unexpected pseudo structure");
3404#endif
3405
3407 // Skip the merge operand at index 0 if !UseTUPseudo.
3408 for (unsigned I = !UseTUPseudo, E = N->getNumOperands(); I != E; I++) {
3409 // Skip the mask, and the Glue.
3410 SDValue Op = N->getOperand(I);
3411 if (I == MaskOpIdx || Op.getValueType() == MVT::Glue)
3412 continue;
3413 Ops.push_back(Op);
3414 }
3415
3416 // Transitively apply any node glued to our new node.
3417 const auto *Glued = N->getGluedNode();
3418 if (auto *TGlued = Glued->getGluedNode())
3419 Ops.push_back(SDValue(TGlued, TGlued->getNumValues() - 1));
3420
3422 CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
3423
3424 if (!N->memoperands_empty())
3425 CurDAG->setNodeMemRefs(Result, N->memoperands());
3426
3427 Result->setFlags(N->getFlags());
3428 ReplaceUses(N, Result);
3429
3430 return true;
3431}
3432
3433static bool IsVMerge(SDNode *N) {
3434 unsigned Opc = N->getMachineOpcode();
3435 return Opc == RISCV::PseudoVMERGE_VVM_MF8 ||
3436 Opc == RISCV::PseudoVMERGE_VVM_MF4 ||
3437 Opc == RISCV::PseudoVMERGE_VVM_MF2 ||
3438 Opc == RISCV::PseudoVMERGE_VVM_M1 ||
3439 Opc == RISCV::PseudoVMERGE_VVM_M2 ||
3440 Opc == RISCV::PseudoVMERGE_VVM_M4 || Opc == RISCV::PseudoVMERGE_VVM_M8;
3441}
3442
3443static bool IsVMv(SDNode *N) {
3444 unsigned Opc = N->getMachineOpcode();
3445 return Opc == RISCV::PseudoVMV_V_V_MF8 || Opc == RISCV::PseudoVMV_V_V_MF4 ||
3446 Opc == RISCV::PseudoVMV_V_V_MF2 || Opc == RISCV::PseudoVMV_V_V_M1 ||
3447 Opc == RISCV::PseudoVMV_V_V_M2 || Opc == RISCV::PseudoVMV_V_V_M4 ||
3448 Opc == RISCV::PseudoVMV_V_V_M8;
3449}
3450
3451static unsigned GetVMSetForLMul(RISCVII::VLMUL LMUL) {
3452 switch (LMUL) {
3453 case RISCVII::LMUL_F8:
3454 return RISCV::PseudoVMSET_M_B1;
3455 case RISCVII::LMUL_F4:
3456 return RISCV::PseudoVMSET_M_B2;
3457 case RISCVII::LMUL_F2:
3458 return RISCV::PseudoVMSET_M_B4;
3459 case RISCVII::LMUL_1:
3460 return RISCV::PseudoVMSET_M_B8;
3461 case RISCVII::LMUL_2:
3462 return RISCV::PseudoVMSET_M_B16;
3463 case RISCVII::LMUL_4:
3464 return RISCV::PseudoVMSET_M_B32;
3465 case RISCVII::LMUL_8:
3466 return RISCV::PseudoVMSET_M_B64;
3468 llvm_unreachable("Unexpected LMUL");
3469 }
3470 llvm_unreachable("Unknown VLMUL enum");
3471}
3472
3473// Try to fold away VMERGE_VVM instructions. We handle these cases:
3474// -Masked TU VMERGE_VVM combined with an unmasked TA instruction instruction
3475// folds to a masked TU instruction. VMERGE_VVM must have have merge operand
3476// same as false operand.
3477// -Masked TA VMERGE_VVM combined with an unmasked TA instruction fold to a
3478// masked TA instruction.
3479// -Unmasked TU VMERGE_VVM combined with a masked MU TA instruction folds to
3480// masked TU instruction. Both instructions must have the same merge operand.
3481// VMERGE_VVM must have have merge operand same as false operand.
3482// Note: The VMERGE_VVM forms above (TA, and TU) refer to the policy implied,
3483// not the pseudo name. That is, a TA VMERGE_VVM can be either the _TU pseudo
3484// form with an IMPLICIT_DEF passthrough operand or the unsuffixed (TA) pseudo
3485// form.
3486bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) {
3487 SDValue Merge, False, True, VL, Mask, Glue;
3488 // A vmv.v.v is equivalent to a vmerge with an all-ones mask.
3489 if (IsVMv(N)) {
3490 Merge = N->getOperand(0);
3491 False = N->getOperand(0);
3492 True = N->getOperand(1);
3493 VL = N->getOperand(2);
3494 // A vmv.v.v won't have a Mask or Glue, instead we'll construct an all-ones
3495 // mask later below.
3496 } else {
3497 assert(IsVMerge(N));
3498 Merge = N->getOperand(0);
3499 False = N->getOperand(1);
3500 True = N->getOperand(2);
3501 Mask = N->getOperand(3);
3502 VL = N->getOperand(4);
3503 // We always have a glue node for the mask at v0.
3504 Glue = N->getOperand(N->getNumOperands() - 1);
3505 }
3506 assert(!Mask || cast<RegisterSDNode>(Mask)->getReg() == RISCV::V0);
3507 assert(!Glue || Glue.getValueType() == MVT::Glue);
3508
3509 // We require that either merge and false are the same, or that merge
3510 // is undefined.
3511 if (Merge != False && !isImplicitDef(Merge))
3512 return false;
3513
3514 assert(True.getResNo() == 0 &&
3515 "Expect True is the first output of an instruction.");
3516
3517 // Need N is the exactly one using True.
3518 if (!True.hasOneUse())
3519 return false;
3520
3521 if (!True.isMachineOpcode())
3522 return false;
3523
3524 unsigned TrueOpc = True.getMachineOpcode();
3525 const MCInstrDesc &TrueMCID = TII->get(TrueOpc);
3526 uint64_t TrueTSFlags = TrueMCID.TSFlags;
3527 bool HasTiedDest = RISCVII::isFirstDefTiedToFirstUse(TrueMCID);
3528
3529 bool IsMasked = false;
3531 RISCV::lookupMaskedIntrinsicByUnmasked(TrueOpc);
3532 if (!Info && HasTiedDest) {
3533 Info = RISCV::getMaskedPseudoInfo(TrueOpc);
3534 IsMasked = true;
3535 }
3536
3537 if (!Info)
3538 return false;
3539
3540 if (HasTiedDest && !isImplicitDef(True->getOperand(0))) {
3541 // The vmerge instruction must be TU.
3542 // FIXME: This could be relaxed, but we need to handle the policy for the
3543 // resulting op correctly.
3544 if (isImplicitDef(Merge))
3545 return false;
3546 SDValue MergeOpTrue = True->getOperand(0);
3547 // Both the vmerge instruction and the True instruction must have the same
3548 // merge operand.
3549 if (False != MergeOpTrue)
3550 return false;
3551 }
3552
3553 if (IsMasked) {
3554 assert(HasTiedDest && "Expected tied dest");
3555 // The vmerge instruction must be TU.
3556 if (isImplicitDef(Merge))
3557 return false;
3558 // The vmerge instruction must have an all 1s mask since we're going to keep
3559 // the mask from the True instruction.
3560 // FIXME: Support mask agnostic True instruction which would have an
3561 // undef merge operand.
3562 if (Mask && !usesAllOnesMask(Mask, Glue))
3563 return false;
3564 }
3565
3566 // Skip if True has side effect.
3567 // TODO: Support vleff and vlsegff.
3568 if (TII->get(TrueOpc).hasUnmodeledSideEffects())
3569 return false;
3570
3571 // The last operand of a masked instruction may be glued.
3572 bool HasGlueOp = True->getGluedNode() != nullptr;
3573
3574 // The chain operand may exist either before the glued operands or in the last
3575 // position.
3576 unsigned TrueChainOpIdx = True.getNumOperands() - HasGlueOp - 1;
3577 bool HasChainOp =
3578 True.getOperand(TrueChainOpIdx).getValueType() == MVT::Other;
3579
3580 if (HasChainOp) {
3581 // Avoid creating cycles in the DAG. We must ensure that none of the other
3582 // operands depend on True through it's Chain.
3583 SmallVector<const SDNode *, 4> LoopWorklist;
3585 LoopWorklist.push_back(False.getNode());
3586 if (Mask)
3587 LoopWorklist.push_back(Mask.getNode());
3588 LoopWorklist.push_back(VL.getNode());
3589 if (Glue)
3590 LoopWorklist.push_back(Glue.getNode());
3591 if (SDNode::hasPredecessorHelper(True.getNode(), Visited, LoopWorklist))
3592 return false;
3593 }
3594
3595 // The vector policy operand may be present for masked intrinsics
3596 bool HasVecPolicyOp = RISCVII::hasVecPolicyOp(TrueTSFlags);
3597 unsigned TrueVLIndex =
3598 True.getNumOperands() - HasVecPolicyOp - HasChainOp - HasGlueOp - 2;
3599 SDValue TrueVL = True.getOperand(TrueVLIndex);
3600 SDValue SEW = True.getOperand(TrueVLIndex + 1);
3601
3602 auto GetMinVL = [](SDValue LHS, SDValue RHS) {
3603 if (LHS == RHS)
3604 return LHS;
3605 if (isAllOnesConstant(LHS))
3606 return RHS;
3607 if (isAllOnesConstant(RHS))
3608 return LHS;
3609 auto *CLHS = dyn_cast<ConstantSDNode>(LHS);
3610 auto *CRHS = dyn_cast<ConstantSDNode>(RHS);
3611 if (!CLHS || !CRHS)
3612 return SDValue();
3613 return CLHS->getZExtValue() <= CRHS->getZExtValue() ? LHS : RHS;
3614 };
3615
3616 // Because N and True must have the same merge operand (or True's operand is
3617 // implicit_def), the "effective" body is the minimum of their VLs.
3618 SDValue OrigVL = VL;
3619 VL = GetMinVL(TrueVL, VL);
3620 if (!VL)
3621 return false;
3622
3623 // If we end up changing the VL or mask of True, then we need to make sure it
3624 // doesn't raise any observable fp exceptions, since changing the active
3625 // elements will affect how fflags is set.
3626 if (TrueVL != VL || !IsMasked)
3627 if (mayRaiseFPException(True.getNode()) &&
3628 !True->getFlags().hasNoFPExcept())
3629 return false;
3630
3631 SDLoc DL(N);
3632
3633 // From the preconditions we checked above, we know the mask and thus glue
3634 // for the result node will be taken from True.
3635 if (IsMasked) {
3636 Mask = True->getOperand(Info->MaskOpIdx);
3637 Glue = True->getOperand(True->getNumOperands() - 1);
3638 assert(Glue.getValueType() == MVT::Glue);
3639 }
3640 // If we end up using the vmerge mask the vmerge is actually a vmv.v.v, create
3641 // an all-ones mask to use.
3642 else if (IsVMv(N)) {
3643 unsigned TSFlags = TII->get(N->getMachineOpcode()).TSFlags;
3644 unsigned VMSetOpc = GetVMSetForLMul(RISCVII::getLMul(TSFlags));
3645 ElementCount EC = N->getValueType(0).getVectorElementCount();
3646 MVT MaskVT = MVT::getVectorVT(MVT::i1, EC);
3647
3648 SDValue AllOnesMask =
3649 SDValue(CurDAG->getMachineNode(VMSetOpc, DL, MaskVT, VL, SEW), 0);
3651 RISCV::V0, AllOnesMask, SDValue());
3652 Mask = CurDAG->getRegister(RISCV::V0, MaskVT);
3653 Glue = MaskCopy.getValue(1);
3654 }
3655
3656 unsigned MaskedOpc = Info->MaskedPseudo;
3657#ifndef NDEBUG
3658 const MCInstrDesc &MaskedMCID = TII->get(MaskedOpc);
3660 "Expected instructions with mask have policy operand.");
3661 assert(MaskedMCID.getOperandConstraint(MaskedMCID.getNumDefs(),
3662 MCOI::TIED_TO) == 0 &&
3663 "Expected instructions with mask have a tied dest.");
3664#endif
3665
3666 // Use a tumu policy, relaxing it to tail agnostic provided that the merge
3667 // operand is undefined.
3668 //
3669 // However, if the VL became smaller than what the vmerge had originally, then
3670 // elements past VL that were previously in the vmerge's body will have moved
3671 // to the tail. In that case we always need to use tail undisturbed to
3672 // preserve them.
3673 bool MergeVLShrunk = VL != OrigVL;
3674 uint64_t Policy = (isImplicitDef(Merge) && !MergeVLShrunk)
3676 : /*TUMU*/ 0;
3677 SDValue PolicyOp =
3678 CurDAG->getTargetConstant(Policy, DL, Subtarget->getXLenVT());
3679
3680
3682 Ops.push_back(False);
3683
3684 const bool HasRoundingMode = RISCVII::hasRoundModeOp(TrueTSFlags);
3685 const unsigned NormalOpsEnd = TrueVLIndex - IsMasked - HasRoundingMode;
3686 assert(!IsMasked || NormalOpsEnd == Info->MaskOpIdx);
3687 Ops.append(True->op_begin() + HasTiedDest, True->op_begin() + NormalOpsEnd);
3688
3689 Ops.push_back(Mask);
3690
3691 // For unmasked "VOp" with rounding mode operand, that is interfaces like
3692 // (..., rm, vl) or (..., rm, vl, policy).
3693 // Its masked version is (..., vm, rm, vl, policy).
3694 // Check the rounding mode pseudo nodes under RISCVInstrInfoVPseudos.td
3695 if (HasRoundingMode)
3696 Ops.push_back(True->getOperand(TrueVLIndex - 1));
3697
3698 Ops.append({VL, SEW, PolicyOp});
3699
3700 // Result node should have chain operand of True.
3701 if (HasChainOp)
3702 Ops.push_back(True.getOperand(TrueChainOpIdx));
3703
3704 // Add the glue for the CopyToReg of mask->v0.
3705 Ops.push_back(Glue);
3706
3708 CurDAG->getMachineNode(MaskedOpc, DL, True->getVTList(), Ops);
3709 Result->setFlags(True->getFlags());
3710
3711 if (!cast<MachineSDNode>(True)->memoperands_empty())
3712 CurDAG->setNodeMemRefs(Result, cast<MachineSDNode>(True)->memoperands());
3713
3714 // Replace vmerge.vvm node by Result.
3715 ReplaceUses(SDValue(N, 0), SDValue(Result, 0));
3716
3717 // Replace another value of True. E.g. chain and VL.
3718 for (unsigned Idx = 1; Idx < True->getNumValues(); ++Idx)
3719 ReplaceUses(True.getValue(Idx), SDValue(Result, Idx));
3720
3721 // Try to transform Result to unmasked intrinsic.
3722 doPeepholeMaskedRVV(Result);
3723 return true;
3724}
3725
3726// Transform (VMERGE_VVM_<LMUL> false, false, true, allones, vl, sew) to
3727// (VMV_V_V_<LMUL> false, true, vl, sew). It may decrease uses of VMSET.
3728bool RISCVDAGToDAGISel::performVMergeToVMv(SDNode *N) {
3729#define CASE_VMERGE_TO_VMV(lmul) \
3730 case RISCV::PseudoVMERGE_VVM_##lmul: \
3731 NewOpc = RISCV::PseudoVMV_V_V_##lmul; \
3732 break;
3733 unsigned NewOpc;
3734 switch (N->getMachineOpcode()) {
3735 default:
3736 llvm_unreachable("Expected VMERGE_VVM_<LMUL> instruction.");
3744 }
3745
3746 if (!usesAllOnesMask(N, /* MaskOpIdx */ 3))
3747 return false;
3748
3749 SDLoc DL(N);
3750 SDValue PolicyOp =
3751 CurDAG->getTargetConstant(/*TUMU*/ 0, DL, Subtarget->getXLenVT());
3753 NewOpc, DL, N->getValueType(0),
3754 {N->getOperand(1), N->getOperand(2), N->getOperand(4), N->getOperand(5),
3755 PolicyOp});
3756 ReplaceUses(N, Result);
3757 return true;
3758}
3759
3760bool RISCVDAGToDAGISel::doPeepholeMergeVVMFold() {
3761 bool MadeChange = false;
3763
3764 while (Position != CurDAG->allnodes_begin()) {
3765 SDNode *N = &*--Position;
3766 if (N->use_empty() || !N->isMachineOpcode())
3767 continue;
3768
3769 if (IsVMerge(N) || IsVMv(N))
3770 MadeChange |= performCombineVMergeAndVOps(N);
3771 if (IsVMerge(N) && N->getOperand(0) == N->getOperand(1))
3772 MadeChange |= performVMergeToVMv(N);
3773 }
3774 return MadeChange;
3775}
3776
3777/// If our passthru is an implicit_def, use noreg instead. This side
3778/// steps issues with MachineCSE not being able to CSE expressions with
3779/// IMPLICIT_DEF operands while preserving the semantic intent. See
3780/// pr64282 for context. Note that this transform is the last one
3781/// performed at ISEL DAG to DAG.
3782bool RISCVDAGToDAGISel::doPeepholeNoRegPassThru() {
3783 bool MadeChange = false;
3785
3786 while (Position != CurDAG->allnodes_begin()) {
3787 SDNode *N = &*--Position;
3788 if (N->use_empty() || !N->isMachineOpcode())
3789 continue;
3790
3791 const unsigned Opc = N->getMachineOpcode();
3792 if (!RISCVVPseudosTable::getPseudoInfo(Opc) ||
3794 !isImplicitDef(N->getOperand(0)))
3795 continue;
3796
3798 Ops.push_back(CurDAG->getRegister(RISCV::NoRegister, N->getValueType(0)));
3799 for (unsigned I = 1, E = N->getNumOperands(); I != E; I++) {
3800 SDValue Op = N->getOperand(I);
3801 Ops.push_back(Op);
3802 }
3803
3805 CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
3806 Result->setFlags(N->getFlags());
3807 CurDAG->setNodeMemRefs(Result, cast<MachineSDNode>(N)->memoperands());
3808 ReplaceUses(N, Result);
3809 MadeChange = true;
3810 }
3811 return MadeChange;
3812}
3813
3814
3815// This pass converts a legalized DAG into a RISCV-specific DAG, ready
3816// for instruction scheduling.
3818 CodeGenOptLevel OptLevel) {
3819 return new RISCVDAGToDAGISel(TM, OptLevel);
3820}
3821
3822char RISCVDAGToDAGISel::ID = 0;
3823
static Register createTuple(ArrayRef< Register > Regs, const unsigned RegClassIDs[], const unsigned SubRegs[], MachineIRBuilder &MIB)
Create a REG_SEQUENCE instruction using the registers in Regs.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Addr
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
#define DEBUG_TYPE
const HexagonInstrInfo * TII
#define I(x, y, z)
Definition: MD5.cpp:58
mir Rename Register Operands
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
#define P(N)
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
R600 Clause Merge
static SDValue selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, int64_t Imm, const RISCVSubtarget &Subtarget)
#define CASE_VMSLT_OPCODES(lmulenum, suffix, suffix_b)
static bool isWorthFoldingAdd(SDValue Add)
static SDValue selectImmSeq(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, RISCVMatInt::InstSeq &Seq)
static bool isImplicitDef(SDValue V)
static unsigned GetVMSetForLMul(RISCVII::VLMUL LMUL)
#define CASE_VMXOR_VMANDN_VMOR_OPCODES(lmulenum, suffix)
static bool usesAllOnesMask(SDValue MaskOp, SDValue GlueOp)
#define CASE_VMERGE_TO_VMV(lmul)
static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, const RISCVSubtarget *Subtarget, SDValue Addr, SDValue &Base, SDValue &Offset)
static bool vectorPseudoHasAllNBitUsers(SDNode *User, unsigned UserOpNo, unsigned Bits, const TargetInstrInfo *TII)
static bool IsVMv(SDNode *N)
#define CASE_VMSLT_VMNAND_VMSET_OPCODES(lmulenum, suffix, suffix_b)
static SDValue findVSplat(SDValue N)
static bool selectVSplatImmHelper(SDValue N, SDValue &SplatVal, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, std::function< bool(int64_t)> ValidateImm)
static bool IsVMerge(SDNode *N)
unsigned Log2SEW
RISCVII::VLMUL VLMul
unsigned SEW
uint64_t TSFlags
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
#define PASS_NAME
Value * RHS
Value * LHS
bool isZero() const
Definition: APFloat.h:1289
APInt bitcastToAPInt() const
Definition: APFloat.h:1208
bool isPosZero() const
Definition: APFloat.h:1304
bool isNegZero() const
Definition: APFloat.h:1305
Class for arbitrary precision integers.
Definition: APInt.h:76
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1433
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1229
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition: APInt.h:264
int64_t getSExtValue() const
Get sign extended value.
Definition: APInt.h:1507
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
const APFloat & getValueAPF() const
uint64_t getZExtValue() const
int64_t getSExtValue() const
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
This class is used to form a handle around another node that is persistent and is updated across invo...
static StringRef getMemConstraintName(ConstraintCode C)
Definition: InlineAsm.h:442
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc,...
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:198
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
Definition: MCInstrDesc.h:248
int getOperandConstraint(unsigned OpNum, MCOI::OperandConstraint Constraint) const
Returns the value of the specified operand constraint if it is present.
Definition: MCInstrDesc.h:219
bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by other flags.
Definition: MCInstrDesc.h:463
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
Definition: MCInstrInfo.h:63
Machine Value Type.
SimpleValueType SimpleTy
uint64_t getScalarSizeInBits() const
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool isFixedLengthVector() const
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
void setFlags(Flags f)
Bitwise OR the current flags with the given flags.
An SDNode that represents everything that will be needed to construct a MachineInstr.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
bool selectSETCC(SDValue N, ISD::CondCode ExpectedCCVal, SDValue &Val)
RISC-V doesn't have general instructions for integer setne/seteq, but we can check for equality with ...
bool selectSExtBits(SDValue N, unsigned Bits, SDValue &Val)
bool selectZExtBits(SDValue N, unsigned Bits, SDValue &Val)
bool selectSHXADD_UWOp(SDValue N, unsigned ShAmt, SDValue &Val)
Look for various patterns that can be done with a SHL that can be folded into a SHXADD_UW.
bool hasAllNBitUsers(SDNode *Node, unsigned Bits, const unsigned Depth=0) const
void selectVSSEG(SDNode *Node, bool IsMasked, bool IsStrided)
bool SelectFrameAddrRegImm(SDValue Addr, SDValue &Base, SDValue &Offset)
void selectVLSEGFF(SDNode *Node, bool IsMasked)
bool selectFPImm(SDValue N, SDValue &Imm)
bool selectSimm5Shl2(SDValue N, SDValue &Simm5, SDValue &Shl2)
bool selectLow8BitsVSplat(SDValue N, SDValue &SplatVal)
bool hasAllHUsers(SDNode *Node) const
bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps) override
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
bool selectVSplatSimm5(SDValue N, SDValue &SplatVal)
bool selectRVVSimm5(SDValue N, unsigned Width, SDValue &Imm)
bool SelectAddrFrameIndex(SDValue Addr, SDValue &Base, SDValue &Offset)
bool hasAllWUsers(SDNode *Node) const
void PreprocessISelDAG() override
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
void Select(SDNode *Node) override
Main hook for targets to transform nodes into machine nodes.
bool selectVSplat(SDValue N, SDValue &SplatVal)
void addVectorLoadStoreOperands(SDNode *Node, unsigned SEWImm, const SDLoc &DL, unsigned CurOp, bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl< SDValue > &Operands, bool IsLoad=false, MVT *IndexVT=nullptr)
void PostprocessISelDAG() override
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
void selectVLXSEG(SDNode *Node, bool IsMasked, bool IsOrdered)
bool tryShrinkShlLogicImm(SDNode *Node)
void selectVSETVLI(SDNode *Node)
bool selectVLOp(SDValue N, SDValue &VL)
bool trySignedBitfieldExtract(SDNode *Node)
void selectVSXSEG(SDNode *Node, bool IsMasked, bool IsOrdered)
bool selectVSplatSimm5Plus1(SDValue N, SDValue &SplatVal)
bool selectVSplatSimm5Plus1NonZero(SDValue N, SDValue &SplatVal)
bool SelectAddrRegImm(SDValue Addr, SDValue &Base, SDValue &Offset, bool IsINX=false)
void selectVLSEG(SDNode *Node, bool IsMasked, bool IsStrided)
bool selectShiftMask(SDValue N, unsigned ShiftWidth, SDValue &ShAmt)
bool selectSHXADDOp(SDValue N, unsigned ShAmt, SDValue &Val)
Look for various patterns that can be done with a SHL that can be folded into a SHXADD.
bool tryIndexedLoad(SDNode *Node)
bool SelectAddrRegRegScale(SDValue Addr, unsigned MaxShiftAmount, SDValue &Base, SDValue &Index, SDValue &Scale)
bool selectVSplatUimm(SDValue N, unsigned Bits, SDValue &SplatVal)
unsigned getRealMinVLen() const
unsigned getXLen() const
bool hasVInstructions() const
unsigned getRealMaxVLen() const
bool hasStdExtZhinxOrZhinxmin() const
const RISCVRegisterInfo * getRegisterInfo() const override
const RISCVTargetLowering * getTargetLowering() const override
static std::pair< unsigned, unsigned > decomposeSubvectorInsertExtractToSubRegs(MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx, const RISCVRegisterInfo *TRI)
static unsigned getSubregIndexByMVT(MVT VT, unsigned Index)
static unsigned getRegClassIDForVecVT(MVT VT)
static RISCVII::VLMUL getLMUL(MVT VT)
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
SDVTList getVTList() const
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
op_iterator op_begin() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isMachineOpcode() const
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getMachineOpcode() const
unsigned getOpcode() const
unsigned getNumOperands() const
const TargetLowering * TLI
MachineFunction * MF
const TargetInstrInfo * TII
void ReplaceUses(SDValue F, SDValue T)
ReplaceUses - replace all uses of the old node F with the use of the new node T.
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
static bool IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, CodeGenOptLevel OptLevel, bool IgnoreChains=false)
IsLegalToFold - Returns true if the specific operand node N of U can be folded during instruction sel...
bool mayRaiseFPException(SDNode *Node) const
Return whether the node may raise an FP exception.
void ReplaceNode(SDNode *F, SDNode *T)
Replace all uses of F with T, then remove F from the DAG.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:720
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:551
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
static constexpr unsigned MaxRecursionDepth
Definition: SelectionDAG.h:448
allnodes_const_iterator allnodes_begin() const
Definition: SelectionDAG.h:531
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
allnodes_const_iterator allnodes_end() const
Definition: SelectionDAG.h:532
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:725
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getRegister(unsigned Reg, EVT VT)
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:771
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, uint64_t Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:674
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:797
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:560
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:451
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:687
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
TargetInstrInfo - Interface to description of machine instruction set.
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
static constexpr TypeSize Fixed(ScalarTy ExactSize)
Definition: TypeSize.h:331
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
Value * getOperand(unsigned i) const
Definition: User.h:169
unsigned getNumOperands() const
Definition: User.h:191
Iterator for intrusive lists based on ilist_node.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:119
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:750
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:559
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1230
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1026
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:199
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:900
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:1210
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1226
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:627
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:573
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition: ISDOpcodes.h:203
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:777
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:795
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:184
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:192
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
Definition: ISDOpcodes.h:1452
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1503
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Definition: ISDOpcodes.h:1548
static bool hasRoundModeOp(uint64_t TSFlags)
static VLMUL getLMul(uint64_t TSFlags)
static bool hasVLOp(uint64_t TSFlags)
static bool hasVecPolicyOp(uint64_t TSFlags)
static bool hasSEWOp(uint64_t TSFlags)
static bool isFirstDefTiedToFirstUse(const MCInstrDesc &Desc)
InstSeq generateInstSeq(int64_t Val, const FeatureBitset &ActiveFeatures)
static unsigned decodeVSEW(unsigned VSEW)
unsigned getSEWLMULRatio(unsigned SEW, RISCVII::VLMUL VLMul)
unsigned encodeVTYPE(RISCVII::VLMUL VLMUL, unsigned SEW, bool TailAgnostic, bool MaskAgnostic)
static constexpr int64_t VLMaxSentinel
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:440
static const MachineMemOperand::Flags MONontemporalBit1
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:228
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition: bit.h:271
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
Definition: bit.h:281
static const MachineMemOperand::Flags MONontemporalBit0
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:179
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
Definition: MathExtras.h:258
unsigned M1(unsigned Val)
Definition: VE.h:376
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:313
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
Definition: bit.h:245
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:264
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
Definition: MathExtras.h:136
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
Definition: MathExtras.h:246
CodeGenOptLevel
Code generation optimization level.
Definition: CodeGen.h:54
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
Definition: MathExtras.h:141
@ Add
Sum of integers.
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
FunctionPass * createRISCVISelDag(RISCVTargetMachine &TM, CodeGenOptLevel OptLevel)
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:212
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
Definition: MathExtras.h:449
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:373
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition: ValueTypes.h:149
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
bool hasNoFPExcept() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.