LLVM 22.0.0git
RISCVInstrInfo.cpp
Go to the documentation of this file.
1//===-- RISCVInstrInfo.cpp - RISC-V Instruction Information -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the RISC-V implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVInstrInfo.h"
16#include "RISCV.h"
18#include "RISCVSubtarget.h"
19#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/Statistic.h"
33#include "llvm/IR/Module.h"
37
38using namespace llvm;
39
40#define GEN_CHECK_COMPRESS_INSTR
41#include "RISCVGenCompressInstEmitter.inc"
42
43#define GET_INSTRINFO_CTOR_DTOR
44#define GET_INSTRINFO_NAMED_OPS
45#include "RISCVGenInstrInfo.inc"
46
47#define DEBUG_TYPE "riscv-instr-info"
48STATISTIC(NumVRegSpilled,
49 "Number of registers within vector register groups spilled");
50STATISTIC(NumVRegReloaded,
51 "Number of registers within vector register groups reloaded");
52
54 "riscv-prefer-whole-register-move", cl::init(false), cl::Hidden,
55 cl::desc("Prefer whole register move for vector registers."));
56
58 "riscv-force-machine-combiner-strategy", cl::Hidden,
59 cl::desc("Force machine combiner to use a specific strategy for machine "
60 "trace metrics evaluation."),
63 "Local strategy."),
65 "MinInstrCount strategy.")));
66
68
69using namespace RISCV;
70
71#define GET_RISCVVPseudosTable_IMPL
72#include "RISCVGenSearchableTables.inc"
73
74} // namespace llvm::RISCVVPseudosTable
75
76namespace llvm::RISCV {
77
78#define GET_RISCVMaskedPseudosTable_IMPL
79#include "RISCVGenSearchableTables.inc"
80
81} // end namespace llvm::RISCV
82
84 : RISCVGenInstrInfo(STI, RISCV::ADJCALLSTACKDOWN, RISCV::ADJCALLSTACKUP),
85 STI(STI) {}
86
87#define GET_INSTRINFO_HELPERS
88#include "RISCVGenInstrInfo.inc"
89
91 if (STI.hasStdExtZca())
92 return MCInstBuilder(RISCV::C_NOP);
93 return MCInstBuilder(RISCV::ADDI)
94 .addReg(RISCV::X0)
95 .addReg(RISCV::X0)
96 .addImm(0);
97}
98
100 int &FrameIndex) const {
101 TypeSize Dummy = TypeSize::getZero();
102 return isLoadFromStackSlot(MI, FrameIndex, Dummy);
103}
104
105static std::optional<unsigned> getLMULForRVVWholeLoadStore(unsigned Opcode) {
106 switch (Opcode) {
107 default:
108 return std::nullopt;
109 case RISCV::VS1R_V:
110 case RISCV::VL1RE8_V:
111 case RISCV::VL1RE16_V:
112 case RISCV::VL1RE32_V:
113 case RISCV::VL1RE64_V:
114 return 1;
115 case RISCV::VS2R_V:
116 case RISCV::VL2RE8_V:
117 case RISCV::VL2RE16_V:
118 case RISCV::VL2RE32_V:
119 case RISCV::VL2RE64_V:
120 return 2;
121 case RISCV::VS4R_V:
122 case RISCV::VL4RE8_V:
123 case RISCV::VL4RE16_V:
124 case RISCV::VL4RE32_V:
125 case RISCV::VL4RE64_V:
126 return 4;
127 case RISCV::VS8R_V:
128 case RISCV::VL8RE8_V:
129 case RISCV::VL8RE16_V:
130 case RISCV::VL8RE32_V:
131 case RISCV::VL8RE64_V:
132 return 8;
133 }
134}
135
137 int &FrameIndex,
138 TypeSize &MemBytes) const {
139 switch (MI.getOpcode()) {
140 default:
141 return 0;
142 case RISCV::LB:
143 case RISCV::LBU:
144 MemBytes = TypeSize::getFixed(1);
145 break;
146 case RISCV::LH:
147 case RISCV::LH_INX:
148 case RISCV::LHU:
149 case RISCV::FLH:
150 MemBytes = TypeSize::getFixed(2);
151 break;
152 case RISCV::LW:
153 case RISCV::LW_INX:
154 case RISCV::FLW:
155 case RISCV::LWU:
156 MemBytes = TypeSize::getFixed(4);
157 break;
158 case RISCV::LD:
159 case RISCV::LD_RV32:
160 case RISCV::FLD:
161 MemBytes = TypeSize::getFixed(8);
162 break;
163 case RISCV::VL1RE8_V:
164 case RISCV::VL2RE8_V:
165 case RISCV::VL4RE8_V:
166 case RISCV::VL8RE8_V:
167 if (!MI.getOperand(1).isFI())
168 return Register();
169 FrameIndex = MI.getOperand(1).getIndex();
170 unsigned LMUL = *getLMULForRVVWholeLoadStore(MI.getOpcode());
172 return MI.getOperand(0).getReg();
173 }
174
175 if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
176 MI.getOperand(2).getImm() == 0) {
177 FrameIndex = MI.getOperand(1).getIndex();
178 return MI.getOperand(0).getReg();
179 }
180
181 return 0;
182}
183
185 int &FrameIndex) const {
186 TypeSize Dummy = TypeSize::getZero();
187 return isStoreToStackSlot(MI, FrameIndex, Dummy);
188}
189
191 int &FrameIndex,
192 TypeSize &MemBytes) const {
193 switch (MI.getOpcode()) {
194 default:
195 return 0;
196 case RISCV::SB:
197 MemBytes = TypeSize::getFixed(1);
198 break;
199 case RISCV::SH:
200 case RISCV::SH_INX:
201 case RISCV::FSH:
202 MemBytes = TypeSize::getFixed(2);
203 break;
204 case RISCV::SW:
205 case RISCV::SW_INX:
206 case RISCV::FSW:
207 MemBytes = TypeSize::getFixed(4);
208 break;
209 case RISCV::SD:
210 case RISCV::SD_RV32:
211 case RISCV::FSD:
212 MemBytes = TypeSize::getFixed(8);
213 break;
214 case RISCV::VS1R_V:
215 case RISCV::VS2R_V:
216 case RISCV::VS4R_V:
217 case RISCV::VS8R_V:
218 if (!MI.getOperand(1).isFI())
219 return Register();
220 FrameIndex = MI.getOperand(1).getIndex();
221 unsigned LMUL = *getLMULForRVVWholeLoadStore(MI.getOpcode());
223 return MI.getOperand(0).getReg();
224 }
225
226 if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
227 MI.getOperand(2).getImm() == 0) {
228 FrameIndex = MI.getOperand(1).getIndex();
229 return MI.getOperand(0).getReg();
230 }
231
232 return 0;
233}
234
236 const MachineInstr &MI) const {
237 switch (RISCV::getRVVMCOpcode(MI.getOpcode())) {
238 case RISCV::VMV_V_X:
239 case RISCV::VFMV_V_F:
240 case RISCV::VMV_V_I:
241 case RISCV::VMV_S_X:
242 case RISCV::VFMV_S_F:
243 case RISCV::VID_V:
244 return MI.getOperand(1).isUndef();
245 default:
247 }
248}
249
250static bool forwardCopyWillClobberTuple(unsigned DstReg, unsigned SrcReg,
251 unsigned NumRegs) {
252 return DstReg > SrcReg && (DstReg - SrcReg) < NumRegs;
253}
254
256 const MachineBasicBlock &MBB,
259 RISCVVType::VLMUL LMul) {
261 return false;
262
263 assert(MBBI->getOpcode() == TargetOpcode::COPY &&
264 "Unexpected COPY instruction.");
265 Register SrcReg = MBBI->getOperand(1).getReg();
267
268 bool FoundDef = false;
269 bool FirstVSetVLI = false;
270 unsigned FirstSEW = 0;
271 while (MBBI != MBB.begin()) {
272 --MBBI;
273 if (MBBI->isMetaInstruction())
274 continue;
275
276 if (RISCVInstrInfo::isVectorConfigInstr(*MBBI)) {
277 // There is a vsetvli between COPY and source define instruction.
278 // vy = def_vop ... (producing instruction)
279 // ...
280 // vsetvli
281 // ...
282 // vx = COPY vy
283 if (!FoundDef) {
284 if (!FirstVSetVLI) {
285 FirstVSetVLI = true;
286 unsigned FirstVType = MBBI->getOperand(2).getImm();
287 RISCVVType::VLMUL FirstLMul = RISCVVType::getVLMUL(FirstVType);
288 FirstSEW = RISCVVType::getSEW(FirstVType);
289 // The first encountered vsetvli must have the same lmul as the
290 // register class of COPY.
291 if (FirstLMul != LMul)
292 return false;
293 }
294 // Only permit `vsetvli x0, x0, vtype` between COPY and the source
295 // define instruction.
296 if (!RISCVInstrInfo::isVLPreservingConfig(*MBBI))
297 return false;
298 continue;
299 }
300
301 // MBBI is the first vsetvli before the producing instruction.
302 unsigned VType = MBBI->getOperand(2).getImm();
303 // If there is a vsetvli between COPY and the producing instruction.
304 if (FirstVSetVLI) {
305 // If SEW is different, return false.
306 if (RISCVVType::getSEW(VType) != FirstSEW)
307 return false;
308 }
309
310 // If the vsetvli is tail undisturbed, keep the whole register move.
311 if (!RISCVVType::isTailAgnostic(VType))
312 return false;
313
314 // The checking is conservative. We only have register classes for
315 // LMUL = 1/2/4/8. We should be able to convert vmv1r.v to vmv.v.v
316 // for fractional LMUL operations. However, we could not use the vsetvli
317 // lmul for widening operations. The result of widening operation is
318 // 2 x LMUL.
319 return LMul == RISCVVType::getVLMUL(VType);
320 } else if (MBBI->isInlineAsm() || MBBI->isCall()) {
321 return false;
322 } else if (MBBI->getNumDefs()) {
323 // Check all the instructions which will change VL.
324 // For example, vleff has implicit def VL.
325 if (MBBI->modifiesRegister(RISCV::VL, /*TRI=*/nullptr))
326 return false;
327
328 // Only converting whole register copies to vmv.v.v when the defining
329 // value appears in the explicit operands.
330 for (const MachineOperand &MO : MBBI->explicit_operands()) {
331 if (!MO.isReg() || !MO.isDef())
332 continue;
333 if (!FoundDef && TRI->regsOverlap(MO.getReg(), SrcReg)) {
334 // We only permit the source of COPY has the same LMUL as the defined
335 // operand.
336 // There are cases we need to keep the whole register copy if the LMUL
337 // is different.
338 // For example,
339 // $x0 = PseudoVSETIVLI 4, 73 // vsetivli zero, 4, e16,m2,ta,m
340 // $v28m4 = PseudoVWADD_VV_M2 $v26m2, $v8m2
341 // # The COPY may be created by vlmul_trunc intrinsic.
342 // $v26m2 = COPY renamable $v28m2, implicit killed $v28m4
343 //
344 // After widening, the valid value will be 4 x e32 elements. If we
345 // convert the COPY to vmv.v.v, it will only copy 4 x e16 elements.
346 // FIXME: The COPY of subregister of Zvlsseg register will not be able
347 // to convert to vmv.v.[v|i] under the constraint.
348 if (MO.getReg() != SrcReg)
349 return false;
350
351 // In widening reduction instructions with LMUL_1 input vector case,
352 // only checking the LMUL is insufficient due to reduction result is
353 // always LMUL_1.
354 // For example,
355 // $x11 = PseudoVSETIVLI 1, 64 // vsetivli a1, 1, e8, m1, ta, mu
356 // $v8m1 = PseudoVWREDSUM_VS_M1 $v26, $v27
357 // $v26 = COPY killed renamable $v8
358 // After widening, The valid value will be 1 x e16 elements. If we
359 // convert the COPY to vmv.v.v, it will only copy 1 x e8 elements.
360 uint64_t TSFlags = MBBI->getDesc().TSFlags;
362 return false;
363
364 // If the producing instruction does not depend on vsetvli, do not
365 // convert COPY to vmv.v.v. For example, VL1R_V or PseudoVRELOAD.
366 if (!RISCVII::hasSEWOp(TSFlags) || !RISCVII::hasVLOp(TSFlags))
367 return false;
368
369 // Found the definition.
370 FoundDef = true;
371 DefMBBI = MBBI;
372 break;
373 }
374 }
375 }
376 }
377
378 return false;
379}
380
383 const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc,
384 const TargetRegisterClass *RegClass) const {
385 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
387 unsigned NF = RISCVRI::getNF(RegClass->TSFlags);
388
389 uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
390 uint16_t DstEncoding = TRI->getEncodingValue(DstReg);
391 auto [LMulVal, Fractional] = RISCVVType::decodeVLMUL(LMul);
392 assert(!Fractional && "It is impossible be fractional lmul here.");
393 unsigned NumRegs = NF * LMulVal;
394 bool ReversedCopy =
395 forwardCopyWillClobberTuple(DstEncoding, SrcEncoding, NumRegs);
396 if (ReversedCopy) {
397 // If the src and dest overlap when copying a tuple, we need to copy the
398 // registers in reverse.
399 SrcEncoding += NumRegs - 1;
400 DstEncoding += NumRegs - 1;
401 }
402
403 unsigned I = 0;
404 auto GetCopyInfo = [&](uint16_t SrcEncoding, uint16_t DstEncoding)
405 -> std::tuple<RISCVVType::VLMUL, const TargetRegisterClass &, unsigned,
406 unsigned, unsigned> {
407 if (ReversedCopy) {
408 // For reversed copying, if there are enough aligned registers(8/4/2), we
409 // can do a larger copy(LMUL8/4/2).
410 // Besides, we have already known that DstEncoding is larger than
411 // SrcEncoding in forwardCopyWillClobberTuple, so the difference between
412 // DstEncoding and SrcEncoding should be >= LMUL value we try to use to
413 // avoid clobbering.
414 uint16_t Diff = DstEncoding - SrcEncoding;
415 if (I + 8 <= NumRegs && Diff >= 8 && SrcEncoding % 8 == 7 &&
416 DstEncoding % 8 == 7)
417 return {RISCVVType::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
418 RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
419 if (I + 4 <= NumRegs && Diff >= 4 && SrcEncoding % 4 == 3 &&
420 DstEncoding % 4 == 3)
421 return {RISCVVType::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
422 RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
423 if (I + 2 <= NumRegs && Diff >= 2 && SrcEncoding % 2 == 1 &&
424 DstEncoding % 2 == 1)
425 return {RISCVVType::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
426 RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
427 // Or we should do LMUL1 copying.
428 return {RISCVVType::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
429 RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
430 }
431
432 // For forward copying, if source register encoding and destination register
433 // encoding are aligned to 8/4/2, we can do a LMUL8/4/2 copying.
434 if (I + 8 <= NumRegs && SrcEncoding % 8 == 0 && DstEncoding % 8 == 0)
435 return {RISCVVType::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
436 RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
437 if (I + 4 <= NumRegs && SrcEncoding % 4 == 0 && DstEncoding % 4 == 0)
438 return {RISCVVType::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
439 RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
440 if (I + 2 <= NumRegs && SrcEncoding % 2 == 0 && DstEncoding % 2 == 0)
441 return {RISCVVType::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
442 RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
443 // Or we should do LMUL1 copying.
444 return {RISCVVType::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
445 RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
446 };
447
448 while (I != NumRegs) {
449 // For non-segment copying, we only do this once as the registers are always
450 // aligned.
451 // For segment copying, we may do this several times. If the registers are
452 // aligned to larger LMUL, we can eliminate some copyings.
453 auto [LMulCopied, RegClass, Opc, VVOpc, VIOpc] =
454 GetCopyInfo(SrcEncoding, DstEncoding);
455 auto [NumCopied, _] = RISCVVType::decodeVLMUL(LMulCopied);
456
458 if (LMul == LMulCopied &&
459 isConvertibleToVMV_V_V(STI, MBB, MBBI, DefMBBI, LMul)) {
460 Opc = VVOpc;
461 if (DefMBBI->getOpcode() == VIOpc)
462 Opc = VIOpc;
463 }
464
465 // Emit actual copying.
466 // For reversed copying, the encoding should be decreased.
467 MCRegister ActualSrcReg = TRI->findVRegWithEncoding(
468 RegClass, ReversedCopy ? (SrcEncoding - NumCopied + 1) : SrcEncoding);
469 MCRegister ActualDstReg = TRI->findVRegWithEncoding(
470 RegClass, ReversedCopy ? (DstEncoding - NumCopied + 1) : DstEncoding);
471
472 auto MIB = BuildMI(MBB, MBBI, DL, get(Opc), ActualDstReg);
473 bool UseVMV_V_I = RISCV::getRVVMCOpcode(Opc) == RISCV::VMV_V_I;
474 bool UseVMV = UseVMV_V_I || RISCV::getRVVMCOpcode(Opc) == RISCV::VMV_V_V;
475 if (UseVMV)
476 MIB.addReg(ActualDstReg, RegState::Undef);
477 if (UseVMV_V_I)
478 MIB = MIB.add(DefMBBI->getOperand(2));
479 else
480 MIB = MIB.addReg(ActualSrcReg, getKillRegState(KillSrc));
481 if (UseVMV) {
482 const MCInstrDesc &Desc = DefMBBI->getDesc();
483 MIB.add(DefMBBI->getOperand(RISCVII::getVLOpNum(Desc))); // AVL
484 unsigned Log2SEW =
485 DefMBBI->getOperand(RISCVII::getSEWOpNum(Desc)).getImm();
486 MIB.addImm(Log2SEW ? Log2SEW : 3); // SEW
487 MIB.addImm(0); // tu, mu
488 MIB.addReg(RISCV::VL, RegState::Implicit);
489 MIB.addReg(RISCV::VTYPE, RegState::Implicit);
490 }
491 // Add an implicit read of the original source to silence the verifier
492 // in the cases where some of the smaller VRs we're copying from might be
493 // undef, caused by the fact that the original, larger source VR might not
494 // be fully initialized at the time this COPY happens.
495 MIB.addReg(SrcReg, RegState::Implicit);
496
497 // If we are copying reversely, we should decrease the encoding.
498 SrcEncoding += (ReversedCopy ? -NumCopied : NumCopied);
499 DstEncoding += (ReversedCopy ? -NumCopied : NumCopied);
500 I += NumCopied;
501 }
502}
503
506 const DebugLoc &DL, Register DstReg,
507 Register SrcReg, bool KillSrc,
508 bool RenamableDest, bool RenamableSrc) const {
509 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
510 unsigned KillFlag = getKillRegState(KillSrc);
511
512 if (RISCV::GPRRegClass.contains(DstReg, SrcReg)) {
513 BuildMI(MBB, MBBI, DL, get(RISCV::ADDI), DstReg)
514 .addReg(SrcReg, KillFlag | getRenamableRegState(RenamableSrc))
515 .addImm(0);
516 return;
517 }
518
519 if (RISCV::GPRF16RegClass.contains(DstReg, SrcReg)) {
520 BuildMI(MBB, MBBI, DL, get(RISCV::PseudoMV_FPR16INX), DstReg)
521 .addReg(SrcReg, KillFlag | getRenamableRegState(RenamableSrc));
522 return;
523 }
524
525 if (RISCV::GPRF32RegClass.contains(DstReg, SrcReg)) {
526 BuildMI(MBB, MBBI, DL, get(RISCV::PseudoMV_FPR32INX), DstReg)
527 .addReg(SrcReg, KillFlag | getRenamableRegState(RenamableSrc));
528 return;
529 }
530
531 if (RISCV::GPRPairRegClass.contains(DstReg, SrcReg)) {
532 MCRegister EvenReg = TRI->getSubReg(SrcReg, RISCV::sub_gpr_even);
533 MCRegister OddReg = TRI->getSubReg(SrcReg, RISCV::sub_gpr_odd);
534 // We need to correct the odd register of X0_Pair.
535 if (OddReg == RISCV::DUMMY_REG_PAIR_WITH_X0)
536 OddReg = RISCV::X0;
537 assert(DstReg != RISCV::X0_Pair && "Cannot write to X0_Pair");
538
539 // Emit an ADDI for both parts of GPRPair.
540 BuildMI(MBB, MBBI, DL, get(RISCV::ADDI),
541 TRI->getSubReg(DstReg, RISCV::sub_gpr_even))
542 .addReg(EvenReg, KillFlag)
543 .addImm(0);
544 BuildMI(MBB, MBBI, DL, get(RISCV::ADDI),
545 TRI->getSubReg(DstReg, RISCV::sub_gpr_odd))
546 .addReg(OddReg, KillFlag)
547 .addImm(0);
548 return;
549 }
550
551 // Handle copy from csr
552 if (RISCV::VCSRRegClass.contains(SrcReg) &&
553 RISCV::GPRRegClass.contains(DstReg)) {
554 BuildMI(MBB, MBBI, DL, get(RISCV::CSRRS), DstReg)
555 .addImm(RISCVSysReg::lookupSysRegByName(TRI->getName(SrcReg))->Encoding)
556 .addReg(RISCV::X0);
557 return;
558 }
559
560 if (RISCV::FPR16RegClass.contains(DstReg, SrcReg)) {
561 unsigned Opc;
562 if (STI.hasStdExtZfh()) {
563 Opc = RISCV::FSGNJ_H;
564 } else {
565 assert(STI.hasStdExtF() &&
566 (STI.hasStdExtZfhmin() || STI.hasStdExtZfbfmin()) &&
567 "Unexpected extensions");
568 // Zfhmin/Zfbfmin doesn't have FSGNJ_H, replace FSGNJ_H with FSGNJ_S.
569 DstReg = TRI->getMatchingSuperReg(DstReg, RISCV::sub_16,
570 &RISCV::FPR32RegClass);
571 SrcReg = TRI->getMatchingSuperReg(SrcReg, RISCV::sub_16,
572 &RISCV::FPR32RegClass);
573 Opc = RISCV::FSGNJ_S;
574 }
575 BuildMI(MBB, MBBI, DL, get(Opc), DstReg)
576 .addReg(SrcReg, KillFlag)
577 .addReg(SrcReg, KillFlag);
578 return;
579 }
580
581 if (RISCV::FPR32RegClass.contains(DstReg, SrcReg)) {
582 BuildMI(MBB, MBBI, DL, get(RISCV::FSGNJ_S), DstReg)
583 .addReg(SrcReg, KillFlag)
584 .addReg(SrcReg, KillFlag);
585 return;
586 }
587
588 if (RISCV::FPR64RegClass.contains(DstReg, SrcReg)) {
589 BuildMI(MBB, MBBI, DL, get(RISCV::FSGNJ_D), DstReg)
590 .addReg(SrcReg, KillFlag)
591 .addReg(SrcReg, KillFlag);
592 return;
593 }
594
595 if (RISCV::FPR32RegClass.contains(DstReg) &&
596 RISCV::GPRRegClass.contains(SrcReg)) {
597 BuildMI(MBB, MBBI, DL, get(RISCV::FMV_W_X), DstReg)
598 .addReg(SrcReg, KillFlag);
599 return;
600 }
601
602 if (RISCV::GPRRegClass.contains(DstReg) &&
603 RISCV::FPR32RegClass.contains(SrcReg)) {
604 BuildMI(MBB, MBBI, DL, get(RISCV::FMV_X_W), DstReg)
605 .addReg(SrcReg, KillFlag);
606 return;
607 }
608
609 if (RISCV::FPR64RegClass.contains(DstReg) &&
610 RISCV::GPRRegClass.contains(SrcReg)) {
611 assert(STI.getXLen() == 64 && "Unexpected GPR size");
612 BuildMI(MBB, MBBI, DL, get(RISCV::FMV_D_X), DstReg)
613 .addReg(SrcReg, KillFlag);
614 return;
615 }
616
617 if (RISCV::GPRRegClass.contains(DstReg) &&
618 RISCV::FPR64RegClass.contains(SrcReg)) {
619 assert(STI.getXLen() == 64 && "Unexpected GPR size");
620 BuildMI(MBB, MBBI, DL, get(RISCV::FMV_X_D), DstReg)
621 .addReg(SrcReg, KillFlag);
622 return;
623 }
624
625 // VR->VR copies.
626 const TargetRegisterClass *RegClass =
627 TRI->getCommonMinimalPhysRegClass(SrcReg, DstReg);
628 if (RISCVRegisterInfo::isRVVRegClass(RegClass)) {
629 copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RegClass);
630 return;
631 }
632
633 llvm_unreachable("Impossible reg-to-reg copy");
634}
635
638 Register SrcReg, bool IsKill, int FI,
639 const TargetRegisterClass *RC,
640 const TargetRegisterInfo *TRI,
641 Register VReg,
642 MachineInstr::MIFlag Flags) const {
643 MachineFunction *MF = MBB.getParent();
644 MachineFrameInfo &MFI = MF->getFrameInfo();
645
646 unsigned Opcode;
647 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
648 Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
649 RISCV::SW : RISCV::SD;
650 } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
651 Opcode = RISCV::SH_INX;
652 } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {
653 Opcode = RISCV::SW_INX;
654 } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
655 Opcode = RISCV::PseudoRV32ZdinxSD;
656 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
657 Opcode = RISCV::FSH;
658 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
659 Opcode = RISCV::FSW;
660 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
661 Opcode = RISCV::FSD;
662 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
663 Opcode = RISCV::VS1R_V;
664 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
665 Opcode = RISCV::VS2R_V;
666 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
667 Opcode = RISCV::VS4R_V;
668 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
669 Opcode = RISCV::VS8R_V;
670 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
671 Opcode = RISCV::PseudoVSPILL2_M1;
672 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
673 Opcode = RISCV::PseudoVSPILL2_M2;
674 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
675 Opcode = RISCV::PseudoVSPILL2_M4;
676 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
677 Opcode = RISCV::PseudoVSPILL3_M1;
678 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
679 Opcode = RISCV::PseudoVSPILL3_M2;
680 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
681 Opcode = RISCV::PseudoVSPILL4_M1;
682 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
683 Opcode = RISCV::PseudoVSPILL4_M2;
684 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
685 Opcode = RISCV::PseudoVSPILL5_M1;
686 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
687 Opcode = RISCV::PseudoVSPILL6_M1;
688 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
689 Opcode = RISCV::PseudoVSPILL7_M1;
690 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
691 Opcode = RISCV::PseudoVSPILL8_M1;
692 else
693 llvm_unreachable("Can't store this register to stack slot");
694
699
701 BuildMI(MBB, I, DebugLoc(), get(Opcode))
702 .addReg(SrcReg, getKillRegState(IsKill))
703 .addFrameIndex(FI)
704 .addMemOperand(MMO)
705 .setMIFlag(Flags);
706 NumVRegSpilled += TRI->getRegSizeInBits(*RC) / RISCV::RVVBitsPerBlock;
707 } else {
710 MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
711
712 BuildMI(MBB, I, DebugLoc(), get(Opcode))
713 .addReg(SrcReg, getKillRegState(IsKill))
714 .addFrameIndex(FI)
715 .addImm(0)
716 .addMemOperand(MMO)
717 .setMIFlag(Flags);
718 }
719}
720
723 int FI, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI,
724 Register VReg, MachineInstr::MIFlag Flags) const {
725 MachineFunction *MF = MBB.getParent();
726 MachineFrameInfo &MFI = MF->getFrameInfo();
727 DebugLoc DL =
728 Flags & MachineInstr::FrameDestroy ? MBB.findDebugLoc(I) : DebugLoc();
729
730 unsigned Opcode;
731 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
732 Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
733 RISCV::LW : RISCV::LD;
734 } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
735 Opcode = RISCV::LH_INX;
736 } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {
737 Opcode = RISCV::LW_INX;
738 } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
739 Opcode = RISCV::PseudoRV32ZdinxLD;
740 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
741 Opcode = RISCV::FLH;
742 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
743 Opcode = RISCV::FLW;
744 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
745 Opcode = RISCV::FLD;
746 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
747 Opcode = RISCV::VL1RE8_V;
748 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
749 Opcode = RISCV::VL2RE8_V;
750 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
751 Opcode = RISCV::VL4RE8_V;
752 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
753 Opcode = RISCV::VL8RE8_V;
754 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
755 Opcode = RISCV::PseudoVRELOAD2_M1;
756 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
757 Opcode = RISCV::PseudoVRELOAD2_M2;
758 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
759 Opcode = RISCV::PseudoVRELOAD2_M4;
760 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
761 Opcode = RISCV::PseudoVRELOAD3_M1;
762 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
763 Opcode = RISCV::PseudoVRELOAD3_M2;
764 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
765 Opcode = RISCV::PseudoVRELOAD4_M1;
766 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
767 Opcode = RISCV::PseudoVRELOAD4_M2;
768 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
769 Opcode = RISCV::PseudoVRELOAD5_M1;
770 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
771 Opcode = RISCV::PseudoVRELOAD6_M1;
772 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
773 Opcode = RISCV::PseudoVRELOAD7_M1;
774 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
775 Opcode = RISCV::PseudoVRELOAD8_M1;
776 else
777 llvm_unreachable("Can't load this register from stack slot");
778
783
785 BuildMI(MBB, I, DL, get(Opcode), DstReg)
786 .addFrameIndex(FI)
787 .addMemOperand(MMO)
788 .setMIFlag(Flags);
789 NumVRegReloaded += TRI->getRegSizeInBits(*RC) / RISCV::RVVBitsPerBlock;
790 } else {
793 MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
794
795 BuildMI(MBB, I, DL, get(Opcode), DstReg)
796 .addFrameIndex(FI)
797 .addImm(0)
798 .addMemOperand(MMO)
799 .setMIFlag(Flags);
800 }
801}
802std::optional<unsigned> getFoldedOpcode(MachineFunction &MF, MachineInstr &MI,
804 const RISCVSubtarget &ST) {
805
806 // The below optimizations narrow the load so they are only valid for little
807 // endian.
808 // TODO: Support big endian by adding an offset into the frame object?
809 if (MF.getDataLayout().isBigEndian())
810 return std::nullopt;
811
812 // Fold load from stack followed by sext.b/sext.h/sext.w/zext.b/zext.h/zext.w.
813 if (Ops.size() != 1 || Ops[0] != 1)
814 return std::nullopt;
815
816 switch (MI.getOpcode()) {
817 default:
818 if (RISCVInstrInfo::isSEXT_W(MI))
819 return RISCV::LW;
820 if (RISCVInstrInfo::isZEXT_W(MI))
821 return RISCV::LWU;
822 if (RISCVInstrInfo::isZEXT_B(MI))
823 return RISCV::LBU;
824 break;
825 case RISCV::SEXT_H:
826 return RISCV::LH;
827 case RISCV::SEXT_B:
828 return RISCV::LB;
829 case RISCV::ZEXT_H_RV32:
830 case RISCV::ZEXT_H_RV64:
831 return RISCV::LHU;
832 }
833
834 switch (RISCV::getRVVMCOpcode(MI.getOpcode())) {
835 default:
836 return std::nullopt;
837 case RISCV::VMV_X_S: {
838 unsigned Log2SEW =
839 MI.getOperand(RISCVII::getSEWOpNum(MI.getDesc())).getImm();
840 if (ST.getXLen() < (1U << Log2SEW))
841 return std::nullopt;
842 switch (Log2SEW) {
843 case 3:
844 return RISCV::LB;
845 case 4:
846 return RISCV::LH;
847 case 5:
848 return RISCV::LW;
849 case 6:
850 return RISCV::LD;
851 default:
852 llvm_unreachable("Unexpected SEW");
853 }
854 }
855 case RISCV::VFMV_F_S: {
856 unsigned Log2SEW =
857 MI.getOperand(RISCVII::getSEWOpNum(MI.getDesc())).getImm();
858 switch (Log2SEW) {
859 case 4:
860 return RISCV::FLH;
861 case 5:
862 return RISCV::FLW;
863 case 6:
864 return RISCV::FLD;
865 default:
866 llvm_unreachable("Unexpected SEW");
867 }
868 }
869 }
870}
871
872// This is the version used during inline spilling
875 MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS,
876 VirtRegMap *VRM) const {
877
878 std::optional<unsigned> LoadOpc = getFoldedOpcode(MF, MI, Ops, STI);
879 if (!LoadOpc)
880 return nullptr;
881 Register DstReg = MI.getOperand(0).getReg();
882 return BuildMI(*MI.getParent(), InsertPt, MI.getDebugLoc(), get(*LoadOpc),
883 DstReg)
884 .addFrameIndex(FrameIndex)
885 .addImm(0);
886}
887
890 const DebugLoc &DL, Register DstReg, uint64_t Val,
891 MachineInstr::MIFlag Flag, bool DstRenamable,
892 bool DstIsDead) const {
893 Register SrcReg = RISCV::X0;
894
895 // For RV32, allow a sign or unsigned 32 bit value.
896 if (!STI.is64Bit() && !isInt<32>(Val)) {
897 // If have a uimm32 it will still fit in a register so we can allow it.
898 if (!isUInt<32>(Val))
899 report_fatal_error("Should only materialize 32-bit constants for RV32");
900
901 // Sign extend for generateInstSeq.
902 Val = SignExtend64<32>(Val);
903 }
904
906 assert(!Seq.empty());
907
908 bool SrcRenamable = false;
909 unsigned Num = 0;
910
911 for (const RISCVMatInt::Inst &Inst : Seq) {
912 bool LastItem = ++Num == Seq.size();
913 unsigned DstRegState = getDeadRegState(DstIsDead && LastItem) |
914 getRenamableRegState(DstRenamable);
915 unsigned SrcRegState = getKillRegState(SrcReg != RISCV::X0) |
916 getRenamableRegState(SrcRenamable);
917 switch (Inst.getOpndKind()) {
918 case RISCVMatInt::Imm:
919 BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()))
920 .addReg(DstReg, RegState::Define | DstRegState)
921 .addImm(Inst.getImm())
922 .setMIFlag(Flag);
923 break;
925 BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()))
926 .addReg(DstReg, RegState::Define | DstRegState)
927 .addReg(SrcReg, SrcRegState)
928 .addReg(RISCV::X0)
929 .setMIFlag(Flag);
930 break;
932 BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()))
933 .addReg(DstReg, RegState::Define | DstRegState)
934 .addReg(SrcReg, SrcRegState)
935 .addReg(SrcReg, SrcRegState)
936 .setMIFlag(Flag);
937 break;
939 BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()))
940 .addReg(DstReg, RegState::Define | DstRegState)
941 .addReg(SrcReg, SrcRegState)
942 .addImm(Inst.getImm())
943 .setMIFlag(Flag);
944 break;
945 }
946
947 // Only the first instruction has X0 as its source.
948 SrcReg = DstReg;
949 SrcRenamable = DstRenamable;
950 }
951}
952
954 switch (Opc) {
955 default:
957 case RISCV::BEQ:
958 case RISCV::BEQI:
959 case RISCV::CV_BEQIMM:
960 case RISCV::QC_BEQI:
961 case RISCV::QC_E_BEQI:
962 case RISCV::NDS_BBC:
963 case RISCV::NDS_BEQC:
964 return RISCVCC::COND_EQ;
965 case RISCV::BNE:
966 case RISCV::BNEI:
967 case RISCV::QC_BNEI:
968 case RISCV::QC_E_BNEI:
969 case RISCV::CV_BNEIMM:
970 case RISCV::NDS_BBS:
971 case RISCV::NDS_BNEC:
972 return RISCVCC::COND_NE;
973 case RISCV::BLT:
974 case RISCV::QC_BLTI:
975 case RISCV::QC_E_BLTI:
976 return RISCVCC::COND_LT;
977 case RISCV::BGE:
978 case RISCV::QC_BGEI:
979 case RISCV::QC_E_BGEI:
980 return RISCVCC::COND_GE;
981 case RISCV::BLTU:
982 case RISCV::QC_BLTUI:
983 case RISCV::QC_E_BLTUI:
984 return RISCVCC::COND_LTU;
985 case RISCV::BGEU:
986 case RISCV::QC_BGEUI:
987 case RISCV::QC_E_BGEUI:
988 return RISCVCC::COND_GEU;
989 }
990}
991
993 int64_t C1) {
994 switch (CC) {
995 default:
996 llvm_unreachable("Unexpected CC");
997 case RISCVCC::COND_EQ:
998 return C0 == C1;
999 case RISCVCC::COND_NE:
1000 return C0 != C1;
1001 case RISCVCC::COND_LT:
1002 return C0 < C1;
1003 case RISCVCC::COND_GE:
1004 return C0 >= C1;
1005 case RISCVCC::COND_LTU:
1006 return (uint64_t)C0 < (uint64_t)C1;
1007 case RISCVCC::COND_GEU:
1008 return (uint64_t)C0 >= (uint64_t)C1;
1009 }
1010}
1011
1012// The contents of values added to Cond are not examined outside of
1013// RISCVInstrInfo, giving us flexibility in what to push to it. For RISCV, we
1014// push BranchOpcode, Reg1, Reg2.
1017 // Block ends with fall-through condbranch.
1018 assert(LastInst.getDesc().isConditionalBranch() &&
1019 "Unknown conditional branch");
1020 Target = LastInst.getOperand(2).getMBB();
1021 Cond.push_back(MachineOperand::CreateImm(LastInst.getOpcode()));
1022 Cond.push_back(LastInst.getOperand(0));
1023 Cond.push_back(LastInst.getOperand(1));
1024}
1025
1026static unsigned getInverseXqcicmOpcode(unsigned Opcode) {
1027 switch (Opcode) {
1028 default:
1029 llvm_unreachable("Unexpected Opcode");
1030 case RISCV::QC_MVEQ:
1031 return RISCV::QC_MVNE;
1032 case RISCV::QC_MVNE:
1033 return RISCV::QC_MVEQ;
1034 case RISCV::QC_MVLT:
1035 return RISCV::QC_MVGE;
1036 case RISCV::QC_MVGE:
1037 return RISCV::QC_MVLT;
1038 case RISCV::QC_MVLTU:
1039 return RISCV::QC_MVGEU;
1040 case RISCV::QC_MVGEU:
1041 return RISCV::QC_MVLTU;
1042 case RISCV::QC_MVEQI:
1043 return RISCV::QC_MVNEI;
1044 case RISCV::QC_MVNEI:
1045 return RISCV::QC_MVEQI;
1046 case RISCV::QC_MVLTI:
1047 return RISCV::QC_MVGEI;
1048 case RISCV::QC_MVGEI:
1049 return RISCV::QC_MVLTI;
1050 case RISCV::QC_MVLTUI:
1051 return RISCV::QC_MVGEUI;
1052 case RISCV::QC_MVGEUI:
1053 return RISCV::QC_MVLTUI;
1054 }
1055}
1056
1057unsigned RISCVCC::getBrCond(RISCVCC::CondCode CC, unsigned SelectOpc) {
1058 switch (SelectOpc) {
1059 default:
1060 switch (CC) {
1061 default:
1062 llvm_unreachable("Unexpected condition code!");
1063 case RISCVCC::COND_EQ:
1064 return RISCV::BEQ;
1065 case RISCVCC::COND_NE:
1066 return RISCV::BNE;
1067 case RISCVCC::COND_LT:
1068 return RISCV::BLT;
1069 case RISCVCC::COND_GE:
1070 return RISCV::BGE;
1071 case RISCVCC::COND_LTU:
1072 return RISCV::BLTU;
1073 case RISCVCC::COND_GEU:
1074 return RISCV::BGEU;
1075 }
1076 break;
1077 case RISCV::Select_GPR_Using_CC_Imm5_Zibi:
1078 switch (CC) {
1079 default:
1080 llvm_unreachable("Unexpected condition code!");
1081 case RISCVCC::COND_EQ:
1082 return RISCV::BEQI;
1083 case RISCVCC::COND_NE:
1084 return RISCV::BNEI;
1085 }
1086 break;
1087 case RISCV::Select_GPR_Using_CC_SImm5_CV:
1088 switch (CC) {
1089 default:
1090 llvm_unreachable("Unexpected condition code!");
1091 case RISCVCC::COND_EQ:
1092 return RISCV::CV_BEQIMM;
1093 case RISCVCC::COND_NE:
1094 return RISCV::CV_BNEIMM;
1095 }
1096 break;
1097 case RISCV::Select_GPRNoX0_Using_CC_SImm5NonZero_QC:
1098 switch (CC) {
1099 default:
1100 llvm_unreachable("Unexpected condition code!");
1101 case RISCVCC::COND_EQ:
1102 return RISCV::QC_BEQI;
1103 case RISCVCC::COND_NE:
1104 return RISCV::QC_BNEI;
1105 case RISCVCC::COND_LT:
1106 return RISCV::QC_BLTI;
1107 case RISCVCC::COND_GE:
1108 return RISCV::QC_BGEI;
1109 }
1110 break;
1111 case RISCV::Select_GPRNoX0_Using_CC_UImm5NonZero_QC:
1112 switch (CC) {
1113 default:
1114 llvm_unreachable("Unexpected condition code!");
1115 case RISCVCC::COND_LTU:
1116 return RISCV::QC_BLTUI;
1117 case RISCVCC::COND_GEU:
1118 return RISCV::QC_BGEUI;
1119 }
1120 break;
1121 case RISCV::Select_GPRNoX0_Using_CC_SImm16NonZero_QC:
1122 switch (CC) {
1123 default:
1124 llvm_unreachable("Unexpected condition code!");
1125 case RISCVCC::COND_EQ:
1126 return RISCV::QC_E_BEQI;
1127 case RISCVCC::COND_NE:
1128 return RISCV::QC_E_BNEI;
1129 case RISCVCC::COND_LT:
1130 return RISCV::QC_E_BLTI;
1131 case RISCVCC::COND_GE:
1132 return RISCV::QC_E_BGEI;
1133 }
1134 break;
1135 case RISCV::Select_GPRNoX0_Using_CC_UImm16NonZero_QC:
1136 switch (CC) {
1137 default:
1138 llvm_unreachable("Unexpected condition code!");
1139 case RISCVCC::COND_LTU:
1140 return RISCV::QC_E_BLTUI;
1141 case RISCVCC::COND_GEU:
1142 return RISCV::QC_E_BGEUI;
1143 }
1144 break;
1145 case RISCV::Select_GPR_Using_CC_UImmLog2XLen_NDS:
1146 switch (CC) {
1147 default:
1148 llvm_unreachable("Unexpected condition code!");
1149 case RISCVCC::COND_EQ:
1150 return RISCV::NDS_BBC;
1151 case RISCVCC::COND_NE:
1152 return RISCV::NDS_BBS;
1153 }
1154 break;
1155 case RISCV::Select_GPR_Using_CC_UImm7_NDS:
1156 switch (CC) {
1157 default:
1158 llvm_unreachable("Unexpected condition code!");
1159 case RISCVCC::COND_EQ:
1160 return RISCV::NDS_BEQC;
1161 case RISCVCC::COND_NE:
1162 return RISCV::NDS_BNEC;
1163 }
1164 break;
1165 }
1166}
1167
1169 switch (CC) {
1170 default:
1171 llvm_unreachable("Unrecognized conditional branch");
1172 case RISCVCC::COND_EQ:
1173 return RISCVCC::COND_NE;
1174 case RISCVCC::COND_NE:
1175 return RISCVCC::COND_EQ;
1176 case RISCVCC::COND_LT:
1177 return RISCVCC::COND_GE;
1178 case RISCVCC::COND_GE:
1179 return RISCVCC::COND_LT;
1180 case RISCVCC::COND_LTU:
1181 return RISCVCC::COND_GEU;
1182 case RISCVCC::COND_GEU:
1183 return RISCVCC::COND_LTU;
1184 }
1185}
1186
1189 MachineBasicBlock *&FBB,
1191 bool AllowModify) const {
1192 TBB = FBB = nullptr;
1193 Cond.clear();
1194
1195 // If the block has no terminators, it just falls into the block after it.
1196 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1197 if (I == MBB.end() || !isUnpredicatedTerminator(*I))
1198 return false;
1199
1200 // Count the number of terminators and find the first unconditional or
1201 // indirect branch.
1202 MachineBasicBlock::iterator FirstUncondOrIndirectBr = MBB.end();
1203 int NumTerminators = 0;
1204 for (auto J = I.getReverse(); J != MBB.rend() && isUnpredicatedTerminator(*J);
1205 J++) {
1206 NumTerminators++;
1207 if (J->getDesc().isUnconditionalBranch() ||
1208 J->getDesc().isIndirectBranch()) {
1209 FirstUncondOrIndirectBr = J.getReverse();
1210 }
1211 }
1212
1213 // If AllowModify is true, we can erase any terminators after
1214 // FirstUncondOrIndirectBR.
1215 if (AllowModify && FirstUncondOrIndirectBr != MBB.end()) {
1216 while (std::next(FirstUncondOrIndirectBr) != MBB.end()) {
1217 std::next(FirstUncondOrIndirectBr)->eraseFromParent();
1218 NumTerminators--;
1219 }
1220 I = FirstUncondOrIndirectBr;
1221 }
1222
1223 // We can't handle blocks that end in an indirect branch.
1224 if (I->getDesc().isIndirectBranch())
1225 return true;
1226
1227 // We can't handle Generic branch opcodes from Global ISel.
1228 if (I->isPreISelOpcode())
1229 return true;
1230
1231 // We can't handle blocks with more than 2 terminators.
1232 if (NumTerminators > 2)
1233 return true;
1234
1235 // Handle a single unconditional branch.
1236 if (NumTerminators == 1 && I->getDesc().isUnconditionalBranch()) {
1238 return false;
1239 }
1240
1241 // Handle a single conditional branch.
1242 if (NumTerminators == 1 && I->getDesc().isConditionalBranch()) {
1244 return false;
1245 }
1246
1247 // Handle a conditional branch followed by an unconditional branch.
1248 if (NumTerminators == 2 && std::prev(I)->getDesc().isConditionalBranch() &&
1249 I->getDesc().isUnconditionalBranch()) {
1250 parseCondBranch(*std::prev(I), TBB, Cond);
1251 FBB = getBranchDestBlock(*I);
1252 return false;
1253 }
1254
1255 // Otherwise, we can't handle this.
1256 return true;
1257}
1258
1260 int *BytesRemoved) const {
1261 if (BytesRemoved)
1262 *BytesRemoved = 0;
1263 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1264 if (I == MBB.end())
1265 return 0;
1266
1267 if (!I->getDesc().isUnconditionalBranch() &&
1268 !I->getDesc().isConditionalBranch())
1269 return 0;
1270
1271 // Remove the branch.
1272 if (BytesRemoved)
1273 *BytesRemoved += getInstSizeInBytes(*I);
1274 I->eraseFromParent();
1275
1276 I = MBB.end();
1277
1278 if (I == MBB.begin())
1279 return 1;
1280 --I;
1281 if (!I->getDesc().isConditionalBranch())
1282 return 1;
1283
1284 // Remove the branch.
1285 if (BytesRemoved)
1286 *BytesRemoved += getInstSizeInBytes(*I);
1287 I->eraseFromParent();
1288 return 2;
1289}
1290
1291// Inserts a branch into the end of the specific MachineBasicBlock, returning
1292// the number of instructions inserted.
1295 ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
1296 if (BytesAdded)
1297 *BytesAdded = 0;
1298
1299 // Shouldn't be a fall through.
1300 assert(TBB && "insertBranch must not be told to insert a fallthrough");
1301 assert((Cond.size() == 3 || Cond.size() == 0) &&
1302 "RISC-V branch conditions have two components!");
1303
1304 // Unconditional branch.
1305 if (Cond.empty()) {
1306 MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(TBB);
1307 if (BytesAdded)
1308 *BytesAdded += getInstSizeInBytes(MI);
1309 return 1;
1310 }
1311
1312 // Either a one or two-way conditional branch.
1313 MachineInstr &CondMI = *BuildMI(&MBB, DL, get(Cond[0].getImm()))
1314 .add(Cond[1])
1315 .add(Cond[2])
1316 .addMBB(TBB);
1317 if (BytesAdded)
1318 *BytesAdded += getInstSizeInBytes(CondMI);
1319
1320 // One-way conditional branch.
1321 if (!FBB)
1322 return 1;
1323
1324 // Two-way conditional branch.
1325 MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(FBB);
1326 if (BytesAdded)
1327 *BytesAdded += getInstSizeInBytes(MI);
1328 return 2;
1329}
1330
1332 MachineBasicBlock &DestBB,
1333 MachineBasicBlock &RestoreBB,
1334 const DebugLoc &DL, int64_t BrOffset,
1335 RegScavenger *RS) const {
1336 assert(RS && "RegScavenger required for long branching");
1337 assert(MBB.empty() &&
1338 "new block should be inserted for expanding unconditional branch");
1339 assert(MBB.pred_size() == 1);
1340 assert(RestoreBB.empty() &&
1341 "restore block should be inserted for restoring clobbered registers");
1342
1343 MachineFunction *MF = MBB.getParent();
1347
1348 if (!isInt<32>(BrOffset))
1350 "Branch offsets outside of the signed 32-bit range not supported");
1351
1352 // FIXME: A virtual register must be used initially, as the register
1353 // scavenger won't work with empty blocks (SIInstrInfo::insertIndirectBranch
1354 // uses the same workaround).
1355 Register ScratchReg = MRI.createVirtualRegister(&RISCV::GPRJALRRegClass);
1356 auto II = MBB.end();
1357 // We may also update the jump target to RestoreBB later.
1358 MachineInstr &MI = *BuildMI(MBB, II, DL, get(RISCV::PseudoJump))
1359 .addReg(ScratchReg, RegState::Define | RegState::Dead)
1360 .addMBB(&DestBB, RISCVII::MO_CALL);
1361
1363 Register TmpGPR =
1364 RS->scavengeRegisterBackwards(RISCV::GPRRegClass, MI.getIterator(),
1365 /*RestoreAfter=*/false, /*SpAdj=*/0,
1366 /*AllowSpill=*/false);
1367 if (TmpGPR != RISCV::NoRegister)
1368 RS->setRegUsed(TmpGPR);
1369 else {
1370 // The case when there is no scavenged register needs special handling.
1371
1372 // Pick s11(or s1 for rve) because it doesn't make a difference.
1373 TmpGPR = STI.hasStdExtE() ? RISCV::X9 : RISCV::X27;
1374
1375 int FrameIndex = RVFI->getBranchRelaxationScratchFrameIndex();
1376 if (FrameIndex == -1)
1377 report_fatal_error("underestimated function size");
1378
1379 storeRegToStackSlot(MBB, MI, TmpGPR, /*IsKill=*/true, FrameIndex,
1380 &RISCV::GPRRegClass, TRI, Register());
1381 TRI->eliminateFrameIndex(std::prev(MI.getIterator()),
1382 /*SpAdj=*/0, /*FIOperandNum=*/1);
1383
1384 MI.getOperand(1).setMBB(&RestoreBB);
1385
1386 loadRegFromStackSlot(RestoreBB, RestoreBB.end(), TmpGPR, FrameIndex,
1387 &RISCV::GPRRegClass, TRI, Register());
1388 TRI->eliminateFrameIndex(RestoreBB.back(),
1389 /*SpAdj=*/0, /*FIOperandNum=*/1);
1390 }
1391
1392 MRI.replaceRegWith(ScratchReg, TmpGPR);
1393 MRI.clearVirtRegs();
1394}
1395
1398 assert((Cond.size() == 3) && "Invalid branch condition!");
1399 switch (Cond[0].getImm()) {
1400 default:
1401 llvm_unreachable("Unknown conditional branch!");
1402 case RISCV::BEQ:
1403 Cond[0].setImm(RISCV::BNE);
1404 break;
1405 case RISCV::BEQI:
1406 Cond[0].setImm(RISCV::BNEI);
1407 break;
1408 case RISCV::BNE:
1409 Cond[0].setImm(RISCV::BEQ);
1410 break;
1411 case RISCV::BNEI:
1412 Cond[0].setImm(RISCV::BEQI);
1413 break;
1414 case RISCV::BLT:
1415 Cond[0].setImm(RISCV::BGE);
1416 break;
1417 case RISCV::BGE:
1418 Cond[0].setImm(RISCV::BLT);
1419 break;
1420 case RISCV::BLTU:
1421 Cond[0].setImm(RISCV::BGEU);
1422 break;
1423 case RISCV::BGEU:
1424 Cond[0].setImm(RISCV::BLTU);
1425 break;
1426 case RISCV::CV_BEQIMM:
1427 Cond[0].setImm(RISCV::CV_BNEIMM);
1428 break;
1429 case RISCV::CV_BNEIMM:
1430 Cond[0].setImm(RISCV::CV_BEQIMM);
1431 break;
1432 case RISCV::QC_BEQI:
1433 Cond[0].setImm(RISCV::QC_BNEI);
1434 break;
1435 case RISCV::QC_BNEI:
1436 Cond[0].setImm(RISCV::QC_BEQI);
1437 break;
1438 case RISCV::QC_BGEI:
1439 Cond[0].setImm(RISCV::QC_BLTI);
1440 break;
1441 case RISCV::QC_BLTI:
1442 Cond[0].setImm(RISCV::QC_BGEI);
1443 break;
1444 case RISCV::QC_BGEUI:
1445 Cond[0].setImm(RISCV::QC_BLTUI);
1446 break;
1447 case RISCV::QC_BLTUI:
1448 Cond[0].setImm(RISCV::QC_BGEUI);
1449 break;
1450 case RISCV::QC_E_BEQI:
1451 Cond[0].setImm(RISCV::QC_E_BNEI);
1452 break;
1453 case RISCV::QC_E_BNEI:
1454 Cond[0].setImm(RISCV::QC_E_BEQI);
1455 break;
1456 case RISCV::QC_E_BGEI:
1457 Cond[0].setImm(RISCV::QC_E_BLTI);
1458 break;
1459 case RISCV::QC_E_BLTI:
1460 Cond[0].setImm(RISCV::QC_E_BGEI);
1461 break;
1462 case RISCV::QC_E_BGEUI:
1463 Cond[0].setImm(RISCV::QC_E_BLTUI);
1464 break;
1465 case RISCV::QC_E_BLTUI:
1466 Cond[0].setImm(RISCV::QC_E_BGEUI);
1467 break;
1468 case RISCV::NDS_BBC:
1469 Cond[0].setImm(RISCV::NDS_BBS);
1470 break;
1471 case RISCV::NDS_BBS:
1472 Cond[0].setImm(RISCV::NDS_BBC);
1473 break;
1474 case RISCV::NDS_BEQC:
1475 Cond[0].setImm(RISCV::NDS_BNEC);
1476 break;
1477 case RISCV::NDS_BNEC:
1478 Cond[0].setImm(RISCV::NDS_BEQC);
1479 break;
1480 }
1481
1482 return false;
1483}
1484
1485// Return true if the instruction is a load immediate instruction (i.e.
1486// ADDI x0, imm).
1487static bool isLoadImm(const MachineInstr *MI, int64_t &Imm) {
1488 if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(1).isReg() &&
1489 MI->getOperand(1).getReg() == RISCV::X0) {
1490 Imm = MI->getOperand(2).getImm();
1491 return true;
1492 }
1493 return false;
1494}
1495
1497 const MachineOperand &Op, int64_t &Imm) {
1498 // Either a load from immediate instruction or X0.
1499 if (!Op.isReg())
1500 return false;
1501
1502 Register Reg = Op.getReg();
1503 if (Reg == RISCV::X0) {
1504 Imm = 0;
1505 return true;
1506 }
1507 return Reg.isVirtual() && isLoadImm(MRI.getVRegDef(Reg), Imm);
1508}
1509
1511 bool IsSigned = false;
1512 bool IsEquality = false;
1513 switch (MI.getOpcode()) {
1514 default:
1515 return false;
1516 case RISCV::BEQ:
1517 case RISCV::BNE:
1518 IsEquality = true;
1519 break;
1520 case RISCV::BGE:
1521 case RISCV::BLT:
1522 IsSigned = true;
1523 break;
1524 case RISCV::BGEU:
1525 case RISCV::BLTU:
1526 break;
1527 }
1528
1529 MachineBasicBlock *MBB = MI.getParent();
1530 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
1531
1532 const MachineOperand &LHS = MI.getOperand(0);
1533 const MachineOperand &RHS = MI.getOperand(1);
1534 MachineBasicBlock *TBB = MI.getOperand(2).getMBB();
1535
1536 RISCVCC::CondCode CC = getCondFromBranchOpc(MI.getOpcode());
1538
1539 // Canonicalize conditional branches which can be constant folded into
1540 // beqz or bnez. We can't modify the CFG here.
1541 int64_t C0, C1;
1542 if (isFromLoadImm(MRI, LHS, C0) && isFromLoadImm(MRI, RHS, C1)) {
1543 unsigned NewOpc = evaluateCondBranch(CC, C0, C1) ? RISCV::BEQ : RISCV::BNE;
1544 // Build the new branch and remove the old one.
1545 BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
1546 .addReg(RISCV::X0)
1547 .addReg(RISCV::X0)
1548 .addMBB(TBB);
1549 MI.eraseFromParent();
1550 return true;
1551 }
1552
1553 if (IsEquality)
1554 return false;
1555
1556 // For two constants C0 and C1 from
1557 // ```
1558 // li Y, C0
1559 // li Z, C1
1560 // ```
1561 // 1. if C1 = C0 + 1
1562 // we can turn:
1563 // (a) blt Y, X -> bge X, Z
1564 // (b) bge Y, X -> blt X, Z
1565 //
1566 // 2. if C1 = C0 - 1
1567 // we can turn:
1568 // (a) blt X, Y -> bge Z, X
1569 // (b) bge X, Y -> blt Z, X
1570 //
1571 // To make sure this optimization is really beneficial, we only
1572 // optimize for cases where Y had only one use (i.e. only used by the branch).
1573 // Try to find the register for constant Z; return
1574 // invalid register otherwise.
1575 auto searchConst = [&](int64_t C1) -> Register {
1577 auto DefC1 = std::find_if(++II, E, [&](const MachineInstr &I) -> bool {
1578 int64_t Imm;
1579 return isLoadImm(&I, Imm) && Imm == C1 &&
1580 I.getOperand(0).getReg().isVirtual();
1581 });
1582 if (DefC1 != E)
1583 return DefC1->getOperand(0).getReg();
1584
1585 return Register();
1586 };
1587
1588 unsigned NewOpc = RISCVCC::getBrCond(getInverseBranchCondition(CC));
1589
1590 // Might be case 1.
1591 // Don't change 0 to 1 since we can use x0.
1592 // For unsigned cases changing -1U to 0 would be incorrect.
1593 // The incorrect case for signed would be INT_MAX, but isFromLoadImm can't
1594 // return that.
1595 if (isFromLoadImm(MRI, LHS, C0) && C0 != 0 && LHS.getReg().isVirtual() &&
1596 MRI.hasOneUse(LHS.getReg()) && (IsSigned || C0 != -1)) {
1597 assert(isInt<12>(C0) && "Unexpected immediate");
1598 if (Register RegZ = searchConst(C0 + 1)) {
1599 BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
1600 .add(RHS)
1601 .addReg(RegZ)
1602 .addMBB(TBB);
1603 // We might extend the live range of Z, clear its kill flag to
1604 // account for this.
1605 MRI.clearKillFlags(RegZ);
1606 MI.eraseFromParent();
1607 return true;
1608 }
1609 }
1610
1611 // Might be case 2.
1612 // For signed cases we don't want to change 0 since we can use x0.
1613 // For unsigned cases changing 0 to -1U would be incorrect.
1614 // The incorrect case for signed would be INT_MIN, but isFromLoadImm can't
1615 // return that.
1616 if (isFromLoadImm(MRI, RHS, C0) && C0 != 0 && RHS.getReg().isVirtual() &&
1617 MRI.hasOneUse(RHS.getReg())) {
1618 assert(isInt<12>(C0) && "Unexpected immediate");
1619 if (Register RegZ = searchConst(C0 - 1)) {
1620 BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
1621 .addReg(RegZ)
1622 .add(LHS)
1623 .addMBB(TBB);
1624 // We might extend the live range of Z, clear its kill flag to
1625 // account for this.
1626 MRI.clearKillFlags(RegZ);
1627 MI.eraseFromParent();
1628 return true;
1629 }
1630 }
1631
1632 return false;
1633}
1634
1637 assert(MI.getDesc().isBranch() && "Unexpected opcode!");
1638 // The branch target is always the last operand.
1639 int NumOp = MI.getNumExplicitOperands();
1640 return MI.getOperand(NumOp - 1).getMBB();
1641}
1642
1644 int64_t BrOffset) const {
1645 unsigned XLen = STI.getXLen();
1646 // Ideally we could determine the supported branch offset from the
1647 // RISCVII::FormMask, but this can't be used for Pseudo instructions like
1648 // PseudoBR.
1649 switch (BranchOp) {
1650 default:
1651 llvm_unreachable("Unexpected opcode!");
1652 case RISCV::NDS_BBC:
1653 case RISCV::NDS_BBS:
1654 case RISCV::NDS_BEQC:
1655 case RISCV::NDS_BNEC:
1656 return isInt<11>(BrOffset);
1657 case RISCV::BEQ:
1658 case RISCV::BNE:
1659 case RISCV::BLT:
1660 case RISCV::BGE:
1661 case RISCV::BLTU:
1662 case RISCV::BGEU:
1663 case RISCV::BEQI:
1664 case RISCV::BNEI:
1665 case RISCV::CV_BEQIMM:
1666 case RISCV::CV_BNEIMM:
1667 case RISCV::QC_BEQI:
1668 case RISCV::QC_BNEI:
1669 case RISCV::QC_BGEI:
1670 case RISCV::QC_BLTI:
1671 case RISCV::QC_BLTUI:
1672 case RISCV::QC_BGEUI:
1673 case RISCV::QC_E_BEQI:
1674 case RISCV::QC_E_BNEI:
1675 case RISCV::QC_E_BGEI:
1676 case RISCV::QC_E_BLTI:
1677 case RISCV::QC_E_BLTUI:
1678 case RISCV::QC_E_BGEUI:
1679 return isInt<13>(BrOffset);
1680 case RISCV::JAL:
1681 case RISCV::PseudoBR:
1682 return isInt<21>(BrOffset);
1683 case RISCV::PseudoJump:
1684 return isInt<32>(SignExtend64(BrOffset + 0x800, XLen));
1685 }
1686}
1687
1688// If the operation has a predicated pseudo instruction, return the pseudo
1689// instruction opcode. Otherwise, return RISCV::INSTRUCTION_LIST_END.
1690// TODO: Support more operations.
1691unsigned getPredicatedOpcode(unsigned Opcode) {
1692 switch (Opcode) {
1693 case RISCV::ADD: return RISCV::PseudoCCADD; break;
1694 case RISCV::SUB: return RISCV::PseudoCCSUB; break;
1695 case RISCV::SLL: return RISCV::PseudoCCSLL; break;
1696 case RISCV::SRL: return RISCV::PseudoCCSRL; break;
1697 case RISCV::SRA: return RISCV::PseudoCCSRA; break;
1698 case RISCV::AND: return RISCV::PseudoCCAND; break;
1699 case RISCV::OR: return RISCV::PseudoCCOR; break;
1700 case RISCV::XOR: return RISCV::PseudoCCXOR; break;
1701
1702 case RISCV::ADDI: return RISCV::PseudoCCADDI; break;
1703 case RISCV::SLLI: return RISCV::PseudoCCSLLI; break;
1704 case RISCV::SRLI: return RISCV::PseudoCCSRLI; break;
1705 case RISCV::SRAI: return RISCV::PseudoCCSRAI; break;
1706 case RISCV::ANDI: return RISCV::PseudoCCANDI; break;
1707 case RISCV::ORI: return RISCV::PseudoCCORI; break;
1708 case RISCV::XORI: return RISCV::PseudoCCXORI; break;
1709
1710 case RISCV::ADDW: return RISCV::PseudoCCADDW; break;
1711 case RISCV::SUBW: return RISCV::PseudoCCSUBW; break;
1712 case RISCV::SLLW: return RISCV::PseudoCCSLLW; break;
1713 case RISCV::SRLW: return RISCV::PseudoCCSRLW; break;
1714 case RISCV::SRAW: return RISCV::PseudoCCSRAW; break;
1715
1716 case RISCV::ADDIW: return RISCV::PseudoCCADDIW; break;
1717 case RISCV::SLLIW: return RISCV::PseudoCCSLLIW; break;
1718 case RISCV::SRLIW: return RISCV::PseudoCCSRLIW; break;
1719 case RISCV::SRAIW: return RISCV::PseudoCCSRAIW; break;
1720
1721 case RISCV::ANDN: return RISCV::PseudoCCANDN; break;
1722 case RISCV::ORN: return RISCV::PseudoCCORN; break;
1723 case RISCV::XNOR: return RISCV::PseudoCCXNOR; break;
1724
1725 case RISCV::NDS_BFOS: return RISCV::PseudoCCNDS_BFOS; break;
1726 case RISCV::NDS_BFOZ: return RISCV::PseudoCCNDS_BFOZ; break;
1727 }
1728
1729 return RISCV::INSTRUCTION_LIST_END;
1730}
1731
1732/// Identify instructions that can be folded into a CCMOV instruction, and
1733/// return the defining instruction.
1735 const MachineRegisterInfo &MRI,
1736 const TargetInstrInfo *TII) {
1737 if (!Reg.isVirtual())
1738 return nullptr;
1739 if (!MRI.hasOneNonDBGUse(Reg))
1740 return nullptr;
1741 MachineInstr *MI = MRI.getVRegDef(Reg);
1742 if (!MI)
1743 return nullptr;
1744 // Check if MI can be predicated and folded into the CCMOV.
1745 if (getPredicatedOpcode(MI->getOpcode()) == RISCV::INSTRUCTION_LIST_END)
1746 return nullptr;
1747 // Don't predicate li idiom.
1748 if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(1).isReg() &&
1749 MI->getOperand(1).getReg() == RISCV::X0)
1750 return nullptr;
1751 // Check if MI has any other defs or physreg uses.
1752 for (const MachineOperand &MO : llvm::drop_begin(MI->operands())) {
1753 // Reject frame index operands, PEI can't handle the predicated pseudos.
1754 if (MO.isFI() || MO.isCPI() || MO.isJTI())
1755 return nullptr;
1756 if (!MO.isReg())
1757 continue;
1758 // MI can't have any tied operands, that would conflict with predication.
1759 if (MO.isTied())
1760 return nullptr;
1761 if (MO.isDef())
1762 return nullptr;
1763 // Allow constant physregs.
1764 if (MO.getReg().isPhysical() && !MRI.isConstantPhysReg(MO.getReg()))
1765 return nullptr;
1766 }
1767 bool DontMoveAcrossStores = true;
1768 if (!MI->isSafeToMove(DontMoveAcrossStores))
1769 return nullptr;
1770 return MI;
1771}
1772
1775 unsigned &TrueOp, unsigned &FalseOp,
1776 bool &Optimizable) const {
1777 assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
1778 "Unknown select instruction");
1779 // CCMOV operands:
1780 // 0: Def.
1781 // 1: LHS of compare.
1782 // 2: RHS of compare.
1783 // 3: Condition code.
1784 // 4: False use.
1785 // 5: True use.
1786 TrueOp = 5;
1787 FalseOp = 4;
1788 Cond.push_back(MI.getOperand(1));
1789 Cond.push_back(MI.getOperand(2));
1790 Cond.push_back(MI.getOperand(3));
1791 // We can only fold when we support short forward branch opt.
1792 Optimizable = STI.hasShortForwardBranchOpt();
1793 return false;
1794}
1795
1799 bool PreferFalse) const {
1800 assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
1801 "Unknown select instruction");
1802 if (!STI.hasShortForwardBranchOpt())
1803 return nullptr;
1804
1805 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
1807 canFoldAsPredicatedOp(MI.getOperand(5).getReg(), MRI, this);
1808 bool Invert = !DefMI;
1809 if (!DefMI)
1810 DefMI = canFoldAsPredicatedOp(MI.getOperand(4).getReg(), MRI, this);
1811 if (!DefMI)
1812 return nullptr;
1813
1814 // Find new register class to use.
1815 MachineOperand FalseReg = MI.getOperand(Invert ? 5 : 4);
1816 Register DestReg = MI.getOperand(0).getReg();
1817 const TargetRegisterClass *PreviousClass = MRI.getRegClass(FalseReg.getReg());
1818 if (!MRI.constrainRegClass(DestReg, PreviousClass))
1819 return nullptr;
1820
1821 unsigned PredOpc = getPredicatedOpcode(DefMI->getOpcode());
1822 assert(PredOpc != RISCV::INSTRUCTION_LIST_END && "Unexpected opcode!");
1823
1824 // Create a new predicated version of DefMI.
1825 MachineInstrBuilder NewMI =
1826 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(PredOpc), DestReg);
1827
1828 // Copy the condition portion.
1829 NewMI.add(MI.getOperand(1));
1830 NewMI.add(MI.getOperand(2));
1831
1832 // Add condition code, inverting if necessary.
1833 auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());
1834 if (Invert)
1836 NewMI.addImm(CC);
1837
1838 // Copy the false register.
1839 NewMI.add(FalseReg);
1840
1841 // Copy all the DefMI operands.
1842 const MCInstrDesc &DefDesc = DefMI->getDesc();
1843 for (unsigned i = 1, e = DefDesc.getNumOperands(); i != e; ++i)
1844 NewMI.add(DefMI->getOperand(i));
1845
1846 // Update SeenMIs set: register newly created MI and erase removed DefMI.
1847 SeenMIs.insert(NewMI);
1848 SeenMIs.erase(DefMI);
1849
1850 // If MI is inside a loop, and DefMI is outside the loop, then kill flags on
1851 // DefMI would be invalid when transferred inside the loop. Checking for a
1852 // loop is expensive, but at least remove kill flags if they are in different
1853 // BBs.
1854 if (DefMI->getParent() != MI.getParent())
1855 NewMI->clearKillInfo();
1856
1857 // The caller will erase MI, but not DefMI.
1858 DefMI->eraseFromParent();
1859 return NewMI;
1860}
1861
1863 if (MI.isMetaInstruction())
1864 return 0;
1865
1866 unsigned Opcode = MI.getOpcode();
1867
1868 if (Opcode == TargetOpcode::INLINEASM ||
1869 Opcode == TargetOpcode::INLINEASM_BR) {
1870 const MachineFunction &MF = *MI.getParent()->getParent();
1871 return getInlineAsmLength(MI.getOperand(0).getSymbolName(),
1872 *MF.getTarget().getMCAsmInfo());
1873 }
1874
1875 if (!MI.memoperands_empty()) {
1876 MachineMemOperand *MMO = *(MI.memoperands_begin());
1877 if (STI.hasStdExtZihintntl() && MMO->isNonTemporal()) {
1878 if (STI.hasStdExtZca()) {
1879 if (isCompressibleInst(MI, STI))
1880 return 4; // c.ntl.all + c.load/c.store
1881 return 6; // c.ntl.all + load/store
1882 }
1883 return 8; // ntl.all + load/store
1884 }
1885 }
1886
1887 if (Opcode == TargetOpcode::BUNDLE)
1888 return getInstBundleLength(MI);
1889
1890 if (MI.getParent() && MI.getParent()->getParent()) {
1891 if (isCompressibleInst(MI, STI))
1892 return 2;
1893 }
1894
1895 switch (Opcode) {
1896 case RISCV::PseudoMV_FPR16INX:
1897 case RISCV::PseudoMV_FPR32INX:
1898 // MV is always compressible to either c.mv or c.li rd, 0.
1899 return STI.hasStdExtZca() ? 2 : 4;
1900 case TargetOpcode::STACKMAP:
1901 // The upper bound for a stackmap intrinsic is the full length of its shadow
1903 case TargetOpcode::PATCHPOINT:
1904 // The size of the patchpoint intrinsic is the number of bytes requested
1906 case TargetOpcode::STATEPOINT: {
1907 // The size of the statepoint intrinsic is the number of bytes requested
1908 unsigned NumBytes = StatepointOpers(&MI).getNumPatchBytes();
1909 // No patch bytes means at most a PseudoCall is emitted
1910 return std::max(NumBytes, 8U);
1911 }
1912 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1913 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
1914 case TargetOpcode::PATCHABLE_TAIL_CALL: {
1915 const MachineFunction &MF = *MI.getParent()->getParent();
1916 const Function &F = MF.getFunction();
1917 if (Opcode == TargetOpcode::PATCHABLE_FUNCTION_ENTER &&
1918 F.hasFnAttribute("patchable-function-entry")) {
1919 unsigned Num;
1920 if (F.getFnAttribute("patchable-function-entry")
1921 .getValueAsString()
1922 .getAsInteger(10, Num))
1923 return get(Opcode).getSize();
1924
1925 // Number of C.NOP or NOP
1926 return (STI.hasStdExtZca() ? 2 : 4) * Num;
1927 }
1928 // XRay uses C.JAL + 21 or 33 C.NOP for each sled in RV32 and RV64,
1929 // respectively.
1930 return STI.is64Bit() ? 68 : 44;
1931 }
1932 default:
1933 return get(Opcode).getSize();
1934 }
1935}
1936
1937unsigned RISCVInstrInfo::getInstBundleLength(const MachineInstr &MI) const {
1938 unsigned Size = 0;
1940 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
1941 while (++I != E && I->isInsideBundle()) {
1942 assert(!I->isBundle() && "No nested bundle!");
1944 }
1945 return Size;
1946}
1947
1949 const unsigned Opcode = MI.getOpcode();
1950 switch (Opcode) {
1951 default:
1952 break;
1953 case RISCV::FSGNJ_D:
1954 case RISCV::FSGNJ_S:
1955 case RISCV::FSGNJ_H:
1956 case RISCV::FSGNJ_D_INX:
1957 case RISCV::FSGNJ_D_IN32X:
1958 case RISCV::FSGNJ_S_INX:
1959 case RISCV::FSGNJ_H_INX:
1960 // The canonical floating-point move is fsgnj rd, rs, rs.
1961 return MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
1962 MI.getOperand(1).getReg() == MI.getOperand(2).getReg();
1963 case RISCV::ADDI:
1964 case RISCV::ORI:
1965 case RISCV::XORI:
1966 return (MI.getOperand(1).isReg() &&
1967 MI.getOperand(1).getReg() == RISCV::X0) ||
1968 (MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0);
1969 }
1970 return MI.isAsCheapAsAMove();
1971}
1972
1973std::optional<DestSourcePair>
1975 if (MI.isMoveReg())
1976 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
1977 switch (MI.getOpcode()) {
1978 default:
1979 break;
1980 case RISCV::ADD:
1981 case RISCV::OR:
1982 case RISCV::XOR:
1983 if (MI.getOperand(1).isReg() && MI.getOperand(1).getReg() == RISCV::X0 &&
1984 MI.getOperand(2).isReg())
1985 return DestSourcePair{MI.getOperand(0), MI.getOperand(2)};
1986 if (MI.getOperand(2).isReg() && MI.getOperand(2).getReg() == RISCV::X0 &&
1987 MI.getOperand(1).isReg())
1988 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
1989 break;
1990 case RISCV::ADDI:
1991 // Operand 1 can be a frameindex but callers expect registers
1992 if (MI.getOperand(1).isReg() && MI.getOperand(2).isImm() &&
1993 MI.getOperand(2).getImm() == 0)
1994 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
1995 break;
1996 case RISCV::SUB:
1997 if (MI.getOperand(2).isReg() && MI.getOperand(2).getReg() == RISCV::X0 &&
1998 MI.getOperand(1).isReg())
1999 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
2000 break;
2001 case RISCV::SH1ADD:
2002 case RISCV::SH1ADD_UW:
2003 case RISCV::SH2ADD:
2004 case RISCV::SH2ADD_UW:
2005 case RISCV::SH3ADD:
2006 case RISCV::SH3ADD_UW:
2007 if (MI.getOperand(1).isReg() && MI.getOperand(1).getReg() == RISCV::X0 &&
2008 MI.getOperand(2).isReg())
2009 return DestSourcePair{MI.getOperand(0), MI.getOperand(2)};
2010 break;
2011 case RISCV::FSGNJ_D:
2012 case RISCV::FSGNJ_S:
2013 case RISCV::FSGNJ_H:
2014 case RISCV::FSGNJ_D_INX:
2015 case RISCV::FSGNJ_D_IN32X:
2016 case RISCV::FSGNJ_S_INX:
2017 case RISCV::FSGNJ_H_INX:
2018 // The canonical floating-point move is fsgnj rd, rs, rs.
2019 if (MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
2020 MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
2021 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
2022 break;
2023 }
2024 return std::nullopt;
2025}
2026
2028 if (ForceMachineCombinerStrategy.getNumOccurrences() == 0) {
2029 // The option is unused. Choose Local strategy only for in-order cores. When
2030 // scheduling model is unspecified, use MinInstrCount strategy as more
2031 // generic one.
2032 const auto &SchedModel = STI.getSchedModel();
2033 return (!SchedModel.hasInstrSchedModel() || SchedModel.isOutOfOrder())
2036 }
2037 // The strategy was forced by the option.
2039}
2040
2042 MachineInstr &Root, unsigned &Pattern,
2043 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
2044 int16_t FrmOpIdx =
2045 RISCV::getNamedOperandIdx(Root.getOpcode(), RISCV::OpName::frm);
2046 if (FrmOpIdx < 0) {
2047 assert(all_of(InsInstrs,
2048 [](MachineInstr *MI) {
2049 return RISCV::getNamedOperandIdx(MI->getOpcode(),
2050 RISCV::OpName::frm) < 0;
2051 }) &&
2052 "New instructions require FRM whereas the old one does not have it");
2053 return;
2054 }
2055
2056 const MachineOperand &FRM = Root.getOperand(FrmOpIdx);
2057 MachineFunction &MF = *Root.getMF();
2058
2059 for (auto *NewMI : InsInstrs) {
2060 // We'd already added the FRM operand.
2061 if (static_cast<unsigned>(RISCV::getNamedOperandIdx(
2062 NewMI->getOpcode(), RISCV::OpName::frm)) != NewMI->getNumOperands())
2063 continue;
2064 MachineInstrBuilder MIB(MF, NewMI);
2065 MIB.add(FRM);
2066 if (FRM.getImm() == RISCVFPRndMode::DYN)
2067 MIB.addUse(RISCV::FRM, RegState::Implicit);
2068 }
2069}
2070
2071static bool isFADD(unsigned Opc) {
2072 switch (Opc) {
2073 default:
2074 return false;
2075 case RISCV::FADD_H:
2076 case RISCV::FADD_S:
2077 case RISCV::FADD_D:
2078 return true;
2079 }
2080}
2081
2082static bool isFSUB(unsigned Opc) {
2083 switch (Opc) {
2084 default:
2085 return false;
2086 case RISCV::FSUB_H:
2087 case RISCV::FSUB_S:
2088 case RISCV::FSUB_D:
2089 return true;
2090 }
2091}
2092
2093static bool isFMUL(unsigned Opc) {
2094 switch (Opc) {
2095 default:
2096 return false;
2097 case RISCV::FMUL_H:
2098 case RISCV::FMUL_S:
2099 case RISCV::FMUL_D:
2100 return true;
2101 }
2102}
2103
2104bool RISCVInstrInfo::isVectorAssociativeAndCommutative(const MachineInstr &Inst,
2105 bool Invert) const {
2106#define OPCODE_LMUL_CASE(OPC) \
2107 case RISCV::OPC##_M1: \
2108 case RISCV::OPC##_M2: \
2109 case RISCV::OPC##_M4: \
2110 case RISCV::OPC##_M8: \
2111 case RISCV::OPC##_MF2: \
2112 case RISCV::OPC##_MF4: \
2113 case RISCV::OPC##_MF8
2114
2115#define OPCODE_LMUL_MASK_CASE(OPC) \
2116 case RISCV::OPC##_M1_MASK: \
2117 case RISCV::OPC##_M2_MASK: \
2118 case RISCV::OPC##_M4_MASK: \
2119 case RISCV::OPC##_M8_MASK: \
2120 case RISCV::OPC##_MF2_MASK: \
2121 case RISCV::OPC##_MF4_MASK: \
2122 case RISCV::OPC##_MF8_MASK
2123
2124 unsigned Opcode = Inst.getOpcode();
2125 if (Invert) {
2126 if (auto InvOpcode = getInverseOpcode(Opcode))
2127 Opcode = *InvOpcode;
2128 else
2129 return false;
2130 }
2131
2132 // clang-format off
2133 switch (Opcode) {
2134 default:
2135 return false;
2136 OPCODE_LMUL_CASE(PseudoVADD_VV):
2137 OPCODE_LMUL_MASK_CASE(PseudoVADD_VV):
2138 OPCODE_LMUL_CASE(PseudoVMUL_VV):
2139 OPCODE_LMUL_MASK_CASE(PseudoVMUL_VV):
2140 return true;
2141 }
2142 // clang-format on
2143
2144#undef OPCODE_LMUL_MASK_CASE
2145#undef OPCODE_LMUL_CASE
2146}
2147
2148bool RISCVInstrInfo::areRVVInstsReassociable(const MachineInstr &Root,
2149 const MachineInstr &Prev) const {
2150 if (!areOpcodesEqualOrInverse(Root.getOpcode(), Prev.getOpcode()))
2151 return false;
2152
2153 assert(Root.getMF() == Prev.getMF());
2154 const MachineRegisterInfo *MRI = &Root.getMF()->getRegInfo();
2155 const TargetRegisterInfo *TRI = MRI->getTargetRegisterInfo();
2156
2157 // Make sure vtype operands are also the same.
2158 const MCInstrDesc &Desc = get(Root.getOpcode());
2159 const uint64_t TSFlags = Desc.TSFlags;
2160
2161 auto checkImmOperand = [&](unsigned OpIdx) {
2162 return Root.getOperand(OpIdx).getImm() == Prev.getOperand(OpIdx).getImm();
2163 };
2164
2165 auto checkRegOperand = [&](unsigned OpIdx) {
2166 return Root.getOperand(OpIdx).getReg() == Prev.getOperand(OpIdx).getReg();
2167 };
2168
2169 // PassThru
2170 // TODO: Potentially we can loosen the condition to consider Root to be
2171 // associable with Prev if Root has NoReg as passthru. In which case we
2172 // also need to loosen the condition on vector policies between these.
2173 if (!checkRegOperand(1))
2174 return false;
2175
2176 // SEW
2177 if (RISCVII::hasSEWOp(TSFlags) &&
2178 !checkImmOperand(RISCVII::getSEWOpNum(Desc)))
2179 return false;
2180
2181 // Mask
2182 if (RISCVII::usesMaskPolicy(TSFlags)) {
2183 const MachineBasicBlock *MBB = Root.getParent();
2186 Register MI1VReg;
2187
2188 bool SeenMI2 = false;
2189 for (auto End = MBB->rend(), It = It1; It != End; ++It) {
2190 if (It == It2) {
2191 SeenMI2 = true;
2192 if (!MI1VReg.isValid())
2193 // There is no V0 def between Root and Prev; they're sharing the
2194 // same V0.
2195 break;
2196 }
2197
2198 if (It->modifiesRegister(RISCV::V0, TRI)) {
2199 Register SrcReg = It->getOperand(1).getReg();
2200 // If it's not VReg it'll be more difficult to track its defs, so
2201 // bailing out here just to be safe.
2202 if (!SrcReg.isVirtual())
2203 return false;
2204
2205 if (!MI1VReg.isValid()) {
2206 // This is the V0 def for Root.
2207 MI1VReg = SrcReg;
2208 continue;
2209 }
2210
2211 // Some random mask updates.
2212 if (!SeenMI2)
2213 continue;
2214
2215 // This is the V0 def for Prev; check if it's the same as that of
2216 // Root.
2217 if (MI1VReg != SrcReg)
2218 return false;
2219 else
2220 break;
2221 }
2222 }
2223
2224 // If we haven't encountered Prev, it's likely that this function was
2225 // called in a wrong way (e.g. Root is before Prev).
2226 assert(SeenMI2 && "Prev is expected to appear before Root");
2227 }
2228
2229 // Tail / Mask policies
2230 if (RISCVII::hasVecPolicyOp(TSFlags) &&
2231 !checkImmOperand(RISCVII::getVecPolicyOpNum(Desc)))
2232 return false;
2233
2234 // VL
2235 if (RISCVII::hasVLOp(TSFlags)) {
2236 unsigned OpIdx = RISCVII::getVLOpNum(Desc);
2237 const MachineOperand &Op1 = Root.getOperand(OpIdx);
2238 const MachineOperand &Op2 = Prev.getOperand(OpIdx);
2239 if (Op1.getType() != Op2.getType())
2240 return false;
2241 switch (Op1.getType()) {
2243 if (Op1.getReg() != Op2.getReg())
2244 return false;
2245 break;
2247 if (Op1.getImm() != Op2.getImm())
2248 return false;
2249 break;
2250 default:
2251 llvm_unreachable("Unrecognized VL operand type");
2252 }
2253 }
2254
2255 // Rounding modes
2256 if (RISCVII::hasRoundModeOp(TSFlags) &&
2257 !checkImmOperand(RISCVII::getVLOpNum(Desc) - 1))
2258 return false;
2259
2260 return true;
2261}
2262
2263// Most of our RVV pseudos have passthru operand, so the real operands
2264// start from index = 2.
2265bool RISCVInstrInfo::hasReassociableVectorSibling(const MachineInstr &Inst,
2266 bool &Commuted) const {
2267 const MachineBasicBlock *MBB = Inst.getParent();
2268 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
2270 "Expect the present of passthrough operand.");
2271 MachineInstr *MI1 = MRI.getUniqueVRegDef(Inst.getOperand(2).getReg());
2272 MachineInstr *MI2 = MRI.getUniqueVRegDef(Inst.getOperand(3).getReg());
2273
2274 // If only one operand has the same or inverse opcode and it's the second
2275 // source operand, the operands must be commuted.
2276 Commuted = !areRVVInstsReassociable(Inst, *MI1) &&
2277 areRVVInstsReassociable(Inst, *MI2);
2278 if (Commuted)
2279 std::swap(MI1, MI2);
2280
2281 return areRVVInstsReassociable(Inst, *MI1) &&
2282 (isVectorAssociativeAndCommutative(*MI1) ||
2283 isVectorAssociativeAndCommutative(*MI1, /* Invert */ true)) &&
2285 MRI.hasOneNonDBGUse(MI1->getOperand(0).getReg());
2286}
2287
2289 const MachineInstr &Inst, const MachineBasicBlock *MBB) const {
2290 if (!isVectorAssociativeAndCommutative(Inst) &&
2291 !isVectorAssociativeAndCommutative(Inst, /*Invert=*/true))
2293
2294 const MachineOperand &Op1 = Inst.getOperand(2);
2295 const MachineOperand &Op2 = Inst.getOperand(3);
2296 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
2297
2298 // We need virtual register definitions for the operands that we will
2299 // reassociate.
2300 MachineInstr *MI1 = nullptr;
2301 MachineInstr *MI2 = nullptr;
2302 if (Op1.isReg() && Op1.getReg().isVirtual())
2303 MI1 = MRI.getUniqueVRegDef(Op1.getReg());
2304 if (Op2.isReg() && Op2.getReg().isVirtual())
2305 MI2 = MRI.getUniqueVRegDef(Op2.getReg());
2306
2307 // And at least one operand must be defined in MBB.
2308 return MI1 && MI2 && (MI1->getParent() == MBB || MI2->getParent() == MBB);
2309}
2310
2312 const MachineInstr &Root, unsigned Pattern,
2313 std::array<unsigned, 5> &OperandIndices) const {
2315 if (RISCV::getRVVMCOpcode(Root.getOpcode())) {
2316 // Skip the passthrough operand, so increment all indices by one.
2317 for (unsigned I = 0; I < 5; ++I)
2318 ++OperandIndices[I];
2319 }
2320}
2321
2323 bool &Commuted) const {
2324 if (isVectorAssociativeAndCommutative(Inst) ||
2325 isVectorAssociativeAndCommutative(Inst, /*Invert=*/true))
2326 return hasReassociableVectorSibling(Inst, Commuted);
2327
2328 if (!TargetInstrInfo::hasReassociableSibling(Inst, Commuted))
2329 return false;
2330
2331 const MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();
2332 unsigned OperandIdx = Commuted ? 2 : 1;
2333 const MachineInstr &Sibling =
2334 *MRI.getVRegDef(Inst.getOperand(OperandIdx).getReg());
2335
2336 int16_t InstFrmOpIdx =
2337 RISCV::getNamedOperandIdx(Inst.getOpcode(), RISCV::OpName::frm);
2338 int16_t SiblingFrmOpIdx =
2339 RISCV::getNamedOperandIdx(Sibling.getOpcode(), RISCV::OpName::frm);
2340
2341 return (InstFrmOpIdx < 0 && SiblingFrmOpIdx < 0) ||
2342 RISCV::hasEqualFRM(Inst, Sibling);
2343}
2344
2346 bool Invert) const {
2347 if (isVectorAssociativeAndCommutative(Inst, Invert))
2348 return true;
2349
2350 unsigned Opc = Inst.getOpcode();
2351 if (Invert) {
2352 auto InverseOpcode = getInverseOpcode(Opc);
2353 if (!InverseOpcode)
2354 return false;
2355 Opc = *InverseOpcode;
2356 }
2357
2358 if (isFADD(Opc) || isFMUL(Opc))
2361
2362 switch (Opc) {
2363 default:
2364 return false;
2365 case RISCV::ADD:
2366 case RISCV::ADDW:
2367 case RISCV::AND:
2368 case RISCV::OR:
2369 case RISCV::XOR:
2370 // From RISC-V ISA spec, if both the high and low bits of the same product
2371 // are required, then the recommended code sequence is:
2372 //
2373 // MULH[[S]U] rdh, rs1, rs2
2374 // MUL rdl, rs1, rs2
2375 // (source register specifiers must be in same order and rdh cannot be the
2376 // same as rs1 or rs2)
2377 //
2378 // Microarchitectures can then fuse these into a single multiply operation
2379 // instead of performing two separate multiplies.
2380 // MachineCombiner may reassociate MUL operands and lose the fusion
2381 // opportunity.
2382 case RISCV::MUL:
2383 case RISCV::MULW:
2384 case RISCV::MIN:
2385 case RISCV::MINU:
2386 case RISCV::MAX:
2387 case RISCV::MAXU:
2388 case RISCV::FMIN_H:
2389 case RISCV::FMIN_S:
2390 case RISCV::FMIN_D:
2391 case RISCV::FMAX_H:
2392 case RISCV::FMAX_S:
2393 case RISCV::FMAX_D:
2394 return true;
2395 }
2396
2397 return false;
2398}
2399
2400std::optional<unsigned>
2401RISCVInstrInfo::getInverseOpcode(unsigned Opcode) const {
2402#define RVV_OPC_LMUL_CASE(OPC, INV) \
2403 case RISCV::OPC##_M1: \
2404 return RISCV::INV##_M1; \
2405 case RISCV::OPC##_M2: \
2406 return RISCV::INV##_M2; \
2407 case RISCV::OPC##_M4: \
2408 return RISCV::INV##_M4; \
2409 case RISCV::OPC##_M8: \
2410 return RISCV::INV##_M8; \
2411 case RISCV::OPC##_MF2: \
2412 return RISCV::INV##_MF2; \
2413 case RISCV::OPC##_MF4: \
2414 return RISCV::INV##_MF4; \
2415 case RISCV::OPC##_MF8: \
2416 return RISCV::INV##_MF8
2417
2418#define RVV_OPC_LMUL_MASK_CASE(OPC, INV) \
2419 case RISCV::OPC##_M1_MASK: \
2420 return RISCV::INV##_M1_MASK; \
2421 case RISCV::OPC##_M2_MASK: \
2422 return RISCV::INV##_M2_MASK; \
2423 case RISCV::OPC##_M4_MASK: \
2424 return RISCV::INV##_M4_MASK; \
2425 case RISCV::OPC##_M8_MASK: \
2426 return RISCV::INV##_M8_MASK; \
2427 case RISCV::OPC##_MF2_MASK: \
2428 return RISCV::INV##_MF2_MASK; \
2429 case RISCV::OPC##_MF4_MASK: \
2430 return RISCV::INV##_MF4_MASK; \
2431 case RISCV::OPC##_MF8_MASK: \
2432 return RISCV::INV##_MF8_MASK
2433
2434 switch (Opcode) {
2435 default:
2436 return std::nullopt;
2437 case RISCV::FADD_H:
2438 return RISCV::FSUB_H;
2439 case RISCV::FADD_S:
2440 return RISCV::FSUB_S;
2441 case RISCV::FADD_D:
2442 return RISCV::FSUB_D;
2443 case RISCV::FSUB_H:
2444 return RISCV::FADD_H;
2445 case RISCV::FSUB_S:
2446 return RISCV::FADD_S;
2447 case RISCV::FSUB_D:
2448 return RISCV::FADD_D;
2449 case RISCV::ADD:
2450 return RISCV::SUB;
2451 case RISCV::SUB:
2452 return RISCV::ADD;
2453 case RISCV::ADDW:
2454 return RISCV::SUBW;
2455 case RISCV::SUBW:
2456 return RISCV::ADDW;
2457 // clang-format off
2458 RVV_OPC_LMUL_CASE(PseudoVADD_VV, PseudoVSUB_VV);
2459 RVV_OPC_LMUL_MASK_CASE(PseudoVADD_VV, PseudoVSUB_VV);
2460 RVV_OPC_LMUL_CASE(PseudoVSUB_VV, PseudoVADD_VV);
2461 RVV_OPC_LMUL_MASK_CASE(PseudoVSUB_VV, PseudoVADD_VV);
2462 // clang-format on
2463 }
2464
2465#undef RVV_OPC_LMUL_MASK_CASE
2466#undef RVV_OPC_LMUL_CASE
2467}
2468
2470 const MachineOperand &MO,
2471 bool DoRegPressureReduce) {
2472 if (!MO.isReg() || !MO.getReg().isVirtual())
2473 return false;
2474 const MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
2475 MachineInstr *MI = MRI.getVRegDef(MO.getReg());
2476 if (!MI || !isFMUL(MI->getOpcode()))
2477 return false;
2478
2481 return false;
2482
2483 // Try combining even if fmul has more than one use as it eliminates
2484 // dependency between fadd(fsub) and fmul. However, it can extend liveranges
2485 // for fmul operands, so reject the transformation in register pressure
2486 // reduction mode.
2487 if (DoRegPressureReduce && !MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
2488 return false;
2489
2490 // Do not combine instructions from different basic blocks.
2491 if (Root.getParent() != MI->getParent())
2492 return false;
2493 return RISCV::hasEqualFRM(Root, *MI);
2494}
2495
2497 SmallVectorImpl<unsigned> &Patterns,
2498 bool DoRegPressureReduce) {
2499 unsigned Opc = Root.getOpcode();
2500 bool IsFAdd = isFADD(Opc);
2501 if (!IsFAdd && !isFSUB(Opc))
2502 return false;
2503 bool Added = false;
2504 if (canCombineFPFusedMultiply(Root, Root.getOperand(1),
2505 DoRegPressureReduce)) {
2508 Added = true;
2509 }
2510 if (canCombineFPFusedMultiply(Root, Root.getOperand(2),
2511 DoRegPressureReduce)) {
2514 Added = true;
2515 }
2516 return Added;
2517}
2518
2519static bool getFPPatterns(MachineInstr &Root,
2520 SmallVectorImpl<unsigned> &Patterns,
2521 bool DoRegPressureReduce) {
2522 return getFPFusedMultiplyPatterns(Root, Patterns, DoRegPressureReduce);
2523}
2524
2525/// Utility routine that checks if \param MO is defined by an
2526/// \param CombineOpc instruction in the basic block \param MBB
2528 const MachineOperand &MO,
2529 unsigned CombineOpc) {
2530 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2531 const MachineInstr *MI = nullptr;
2532
2533 if (MO.isReg() && MO.getReg().isVirtual())
2534 MI = MRI.getUniqueVRegDef(MO.getReg());
2535 // And it needs to be in the trace (otherwise, it won't have a depth).
2536 if (!MI || MI->getParent() != &MBB || MI->getOpcode() != CombineOpc)
2537 return nullptr;
2538 // Must only used by the user we combine with.
2539 if (!MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
2540 return nullptr;
2541
2542 return MI;
2543}
2544
2545/// Utility routine that checks if \param MO is defined by a SLLI in \param
2546/// MBB that can be combined by splitting across 2 SHXADD instructions. The
2547/// first SHXADD shift amount is given by \param OuterShiftAmt.
2549 const MachineOperand &MO,
2550 unsigned OuterShiftAmt) {
2551 const MachineInstr *ShiftMI = canCombine(MBB, MO, RISCV::SLLI);
2552 if (!ShiftMI)
2553 return false;
2554
2555 unsigned InnerShiftAmt = ShiftMI->getOperand(2).getImm();
2556 if (InnerShiftAmt < OuterShiftAmt || (InnerShiftAmt - OuterShiftAmt) > 3)
2557 return false;
2558
2559 return true;
2560}
2561
2562// Returns the shift amount from a SHXADD instruction. Returns 0 if the
2563// instruction is not a SHXADD.
2564static unsigned getSHXADDShiftAmount(unsigned Opc) {
2565 switch (Opc) {
2566 default:
2567 return 0;
2568 case RISCV::SH1ADD:
2569 return 1;
2570 case RISCV::SH2ADD:
2571 return 2;
2572 case RISCV::SH3ADD:
2573 return 3;
2574 }
2575}
2576
2577// Returns the shift amount from a SHXADD.UW instruction. Returns 0 if the
2578// instruction is not a SHXADD.UW.
2579static unsigned getSHXADDUWShiftAmount(unsigned Opc) {
2580 switch (Opc) {
2581 default:
2582 return 0;
2583 case RISCV::SH1ADD_UW:
2584 return 1;
2585 case RISCV::SH2ADD_UW:
2586 return 2;
2587 case RISCV::SH3ADD_UW:
2588 return 3;
2589 }
2590}
2591
2592// Look for opportunities to combine (sh3add Z, (add X, (slli Y, 5))) into
2593// (sh3add (sh2add Y, Z), X).
2594static bool getSHXADDPatterns(const MachineInstr &Root,
2595 SmallVectorImpl<unsigned> &Patterns) {
2596 unsigned ShiftAmt = getSHXADDShiftAmount(Root.getOpcode());
2597 if (!ShiftAmt)
2598 return false;
2599
2600 const MachineBasicBlock &MBB = *Root.getParent();
2601
2602 const MachineInstr *AddMI = canCombine(MBB, Root.getOperand(2), RISCV::ADD);
2603 if (!AddMI)
2604 return false;
2605
2606 bool Found = false;
2607 if (canCombineShiftIntoShXAdd(MBB, AddMI->getOperand(1), ShiftAmt)) {
2609 Found = true;
2610 }
2611 if (canCombineShiftIntoShXAdd(MBB, AddMI->getOperand(2), ShiftAmt)) {
2613 Found = true;
2614 }
2615
2616 return Found;
2617}
2618
2630
2632 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
2633 bool DoRegPressureReduce) const {
2634
2635 if (getFPPatterns(Root, Patterns, DoRegPressureReduce))
2636 return true;
2637
2638 if (getSHXADDPatterns(Root, Patterns))
2639 return true;
2640
2641 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
2642 DoRegPressureReduce);
2643}
2644
2645static unsigned getFPFusedMultiplyOpcode(unsigned RootOpc, unsigned Pattern) {
2646 switch (RootOpc) {
2647 default:
2648 llvm_unreachable("Unexpected opcode");
2649 case RISCV::FADD_H:
2650 return RISCV::FMADD_H;
2651 case RISCV::FADD_S:
2652 return RISCV::FMADD_S;
2653 case RISCV::FADD_D:
2654 return RISCV::FMADD_D;
2655 case RISCV::FSUB_H:
2656 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_H
2657 : RISCV::FNMSUB_H;
2658 case RISCV::FSUB_S:
2659 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_S
2660 : RISCV::FNMSUB_S;
2661 case RISCV::FSUB_D:
2662 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_D
2663 : RISCV::FNMSUB_D;
2664 }
2665}
2666
2667static unsigned getAddendOperandIdx(unsigned Pattern) {
2668 switch (Pattern) {
2669 default:
2670 llvm_unreachable("Unexpected pattern");
2673 return 2;
2676 return 1;
2677 }
2678}
2679
2681 unsigned Pattern,
2684 MachineFunction *MF = Root.getMF();
2687
2688 MachineOperand &Mul1 = Prev.getOperand(1);
2689 MachineOperand &Mul2 = Prev.getOperand(2);
2690 MachineOperand &Dst = Root.getOperand(0);
2692
2693 Register DstReg = Dst.getReg();
2694 unsigned FusedOpc = getFPFusedMultiplyOpcode(Root.getOpcode(), Pattern);
2695 uint32_t IntersectedFlags = Root.getFlags() & Prev.getFlags();
2696 DebugLoc MergedLoc =
2698
2699 bool Mul1IsKill = Mul1.isKill();
2700 bool Mul2IsKill = Mul2.isKill();
2701 bool AddendIsKill = Addend.isKill();
2702
2703 // We need to clear kill flags since we may be extending the live range past
2704 // a kill. If the mul had kill flags, we can preserve those since we know
2705 // where the previous range stopped.
2706 MRI.clearKillFlags(Mul1.getReg());
2707 MRI.clearKillFlags(Mul2.getReg());
2708
2710 BuildMI(*MF, MergedLoc, TII->get(FusedOpc), DstReg)
2711 .addReg(Mul1.getReg(), getKillRegState(Mul1IsKill))
2712 .addReg(Mul2.getReg(), getKillRegState(Mul2IsKill))
2713 .addReg(Addend.getReg(), getKillRegState(AddendIsKill))
2714 .setMIFlags(IntersectedFlags);
2715
2716 InsInstrs.push_back(MIB);
2717 if (MRI.hasOneNonDBGUse(Prev.getOperand(0).getReg()))
2718 DelInstrs.push_back(&Prev);
2719 DelInstrs.push_back(&Root);
2720}
2721
2722// Combine patterns like (sh3add Z, (add X, (slli Y, 5))) to
2723// (sh3add (sh2add Y, Z), X) if the shift amount can be split across two
2724// shXadd instructions. The outer shXadd keeps its original opcode.
2725static void
2726genShXAddAddShift(MachineInstr &Root, unsigned AddOpIdx,
2729 DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
2730 MachineFunction *MF = Root.getMF();
2733
2734 unsigned OuterShiftAmt = getSHXADDShiftAmount(Root.getOpcode());
2735 assert(OuterShiftAmt != 0 && "Unexpected opcode");
2736
2737 MachineInstr *AddMI = MRI.getUniqueVRegDef(Root.getOperand(2).getReg());
2738 MachineInstr *ShiftMI =
2739 MRI.getUniqueVRegDef(AddMI->getOperand(AddOpIdx).getReg());
2740
2741 unsigned InnerShiftAmt = ShiftMI->getOperand(2).getImm();
2742 assert(InnerShiftAmt >= OuterShiftAmt && "Unexpected shift amount");
2743
2744 unsigned InnerOpc;
2745 switch (InnerShiftAmt - OuterShiftAmt) {
2746 default:
2747 llvm_unreachable("Unexpected shift amount");
2748 case 0:
2749 InnerOpc = RISCV::ADD;
2750 break;
2751 case 1:
2752 InnerOpc = RISCV::SH1ADD;
2753 break;
2754 case 2:
2755 InnerOpc = RISCV::SH2ADD;
2756 break;
2757 case 3:
2758 InnerOpc = RISCV::SH3ADD;
2759 break;
2760 }
2761
2762 const MachineOperand &X = AddMI->getOperand(3 - AddOpIdx);
2763 const MachineOperand &Y = ShiftMI->getOperand(1);
2764 const MachineOperand &Z = Root.getOperand(1);
2765
2766 Register NewVR = MRI.createVirtualRegister(&RISCV::GPRRegClass);
2767
2768 auto MIB1 = BuildMI(*MF, MIMetadata(Root), TII->get(InnerOpc), NewVR)
2769 .addReg(Y.getReg(), getKillRegState(Y.isKill()))
2770 .addReg(Z.getReg(), getKillRegState(Z.isKill()));
2771 auto MIB2 = BuildMI(*MF, MIMetadata(Root), TII->get(Root.getOpcode()),
2772 Root.getOperand(0).getReg())
2773 .addReg(NewVR, RegState::Kill)
2774 .addReg(X.getReg(), getKillRegState(X.isKill()));
2775
2776 InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
2777 InsInstrs.push_back(MIB1);
2778 InsInstrs.push_back(MIB2);
2779 DelInstrs.push_back(ShiftMI);
2780 DelInstrs.push_back(AddMI);
2781 DelInstrs.push_back(&Root);
2782}
2783
2785 MachineInstr &Root, unsigned Pattern,
2788 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
2790 switch (Pattern) {
2791 default:
2793 DelInstrs, InstrIdxForVirtReg);
2794 return;
2797 MachineInstr &Prev = *MRI.getVRegDef(Root.getOperand(1).getReg());
2798 combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
2799 return;
2800 }
2803 MachineInstr &Prev = *MRI.getVRegDef(Root.getOperand(2).getReg());
2804 combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
2805 return;
2806 }
2808 genShXAddAddShift(Root, 1, InsInstrs, DelInstrs, InstrIdxForVirtReg);
2809 return;
2811 genShXAddAddShift(Root, 2, InsInstrs, DelInstrs, InstrIdxForVirtReg);
2812 return;
2813 }
2814}
2815
2817 StringRef &ErrInfo) const {
2818 MCInstrDesc const &Desc = MI.getDesc();
2819
2820 for (const auto &[Index, Operand] : enumerate(Desc.operands())) {
2821 unsigned OpType = Operand.OperandType;
2822 if (OpType >= RISCVOp::OPERAND_FIRST_RISCV_IMM &&
2824 const MachineOperand &MO = MI.getOperand(Index);
2825 if (MO.isReg()) {
2826 ErrInfo = "Expected a non-register operand.";
2827 return false;
2828 }
2829 if (MO.isImm()) {
2830 int64_t Imm = MO.getImm();
2831 bool Ok;
2832 switch (OpType) {
2833 default:
2834 llvm_unreachable("Unexpected operand type");
2835
2836 // clang-format off
2837#define CASE_OPERAND_UIMM(NUM) \
2838 case RISCVOp::OPERAND_UIMM##NUM: \
2839 Ok = isUInt<NUM>(Imm); \
2840 break;
2841#define CASE_OPERAND_SIMM(NUM) \
2842 case RISCVOp::OPERAND_SIMM##NUM: \
2843 Ok = isInt<NUM>(Imm); \
2844 break;
2861 // clang-format on
2863 Ok = isShiftedUInt<1, 1>(Imm);
2864 break;
2866 Ok = isShiftedUInt<4, 1>(Imm);
2867 break;
2869 Ok = isUInt<5>(Imm) && (Imm != 0);
2870 break;
2872 Ok = isUInt<5>(Imm) && (Imm > 3);
2873 break;
2875 Ok = (isUInt<5>(Imm) && (Imm != 0)) || (Imm == 32);
2876 break;
2878 Ok = isShiftedUInt<5, 1>(Imm);
2879 break;
2881 Ok = isShiftedUInt<5, 2>(Imm);
2882 break;
2884 Ok = isShiftedUInt<4, 3>(Imm);
2885 break;
2887 Ok = isShiftedUInt<6, 2>(Imm);
2888 break;
2890 Ok = isShiftedUInt<5, 3>(Imm);
2891 break;
2893 Ok = isUInt<8>(Imm) && Imm >= 32;
2894 break;
2896 Ok = isShiftedUInt<6, 3>(Imm);
2897 break;
2899 Ok = isShiftedInt<6, 4>(Imm) && (Imm != 0);
2900 break;
2902 Ok = isShiftedUInt<8, 2>(Imm) && (Imm != 0);
2903 break;
2905 Ok = isUInt<16>(Imm) && (Imm != 0);
2906 break;
2908 Ok = Imm == 3;
2909 break;
2911 Ok = Imm == 4;
2912 break;
2914 Ok = (isUInt<5>(Imm) && Imm != 0) || Imm == -1;
2915 break;
2916 // clang-format off
2922 // clang-format on
2924 Ok = (isInt<5>(Imm) && Imm != -16) || Imm == 16;
2925 break;
2927 Ok = isInt<5>(Imm) && (Imm != 0);
2928 break;
2930 Ok = Imm != 0 && isInt<6>(Imm);
2931 break;
2933 Ok = isUInt<10>(Imm);
2934 break;
2936 Ok = isUInt<11>(Imm);
2937 break;
2939 Ok = isShiftedInt<7, 5>(Imm);
2940 break;
2942 Ok = isInt<16>(Imm) && (Imm != 0);
2943 break;
2945 Ok = isInt<20>(Imm);
2946 break;
2948 Ok = isInt<32>(Imm);
2949 break;
2951 Ok = STI.is64Bit() ? isUInt<6>(Imm) : isUInt<5>(Imm);
2952 break;
2954 Ok = STI.is64Bit() ? isUInt<6>(Imm) : isUInt<5>(Imm);
2955 Ok = Ok && Imm != 0;
2956 break;
2958 Ok = (isUInt<5>(Imm) && Imm != 0) ||
2959 (Imm >= 0xfffe0 && Imm <= 0xfffff);
2960 break;
2962 Ok = Imm >= 0 && Imm <= 10;
2963 break;
2965 Ok = Imm >= 0 && Imm <= 7;
2966 break;
2968 Ok = Imm >= 1 && Imm <= 10;
2969 break;
2971 Ok = Imm >= 2 && Imm <= 14;
2972 break;
2974 Ok = Imm >= RISCVZC::RA && Imm <= RISCVZC::RA_S0_S11;
2975 break;
2977 Ok = Imm >= RISCVZC::RA_S0 && Imm <= RISCVZC::RA_S0_S11;
2978 break;
2980 Ok = Imm >= 0 && Imm <= 48 && Imm % 16 == 0;
2981 break;
2984 break;
2986 Ok = Imm == RISCVFPRndMode::RTZ;
2987 break;
2989 Ok = Imm >= 0 && Imm < RISCVCC::COND_INVALID;
2990 break;
2992 Ok = (Imm &
2994 break;
2996 Ok = (isUInt<5>(Imm) && RISCVVType::isValidSEW(1 << Imm));
2997 break;
2999 Ok = Imm == 0;
3000 break;
3003 if (RISCVII::usesVXRM(Desc.TSFlags))
3004 Ok = isUInt<2>(Imm);
3005 else
3007 break;
3008 }
3009 if (!Ok) {
3010 ErrInfo = "Invalid immediate";
3011 return false;
3012 }
3013 }
3014 }
3015 }
3016
3017 const uint64_t TSFlags = Desc.TSFlags;
3018 if (RISCVII::hasVLOp(TSFlags)) {
3019 const MachineOperand &Op = MI.getOperand(RISCVII::getVLOpNum(Desc));
3020 if (!Op.isImm() && !Op.isReg()) {
3021 ErrInfo = "Invalid operand type for VL operand";
3022 return false;
3023 }
3024 if (Op.isReg() && Op.getReg() != RISCV::NoRegister) {
3025 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3026 auto *RC = MRI.getRegClass(Op.getReg());
3027 if (!RISCV::GPRRegClass.hasSubClassEq(RC)) {
3028 ErrInfo = "Invalid register class for VL operand";
3029 return false;
3030 }
3031 }
3032 if (!RISCVII::hasSEWOp(TSFlags)) {
3033 ErrInfo = "VL operand w/o SEW operand?";
3034 return false;
3035 }
3036 }
3037 if (RISCVII::hasSEWOp(TSFlags)) {
3038 unsigned OpIdx = RISCVII::getSEWOpNum(Desc);
3039 if (!MI.getOperand(OpIdx).isImm()) {
3040 ErrInfo = "SEW value expected to be an immediate";
3041 return false;
3042 }
3043 uint64_t Log2SEW = MI.getOperand(OpIdx).getImm();
3044 if (Log2SEW > 31) {
3045 ErrInfo = "Unexpected SEW value";
3046 return false;
3047 }
3048 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
3049 if (!RISCVVType::isValidSEW(SEW)) {
3050 ErrInfo = "Unexpected SEW value";
3051 return false;
3052 }
3053 }
3054 if (RISCVII::hasVecPolicyOp(TSFlags)) {
3056 if (!MI.getOperand(OpIdx).isImm()) {
3057 ErrInfo = "Policy operand expected to be an immediate";
3058 return false;
3059 }
3060 uint64_t Policy = MI.getOperand(OpIdx).getImm();
3062 ErrInfo = "Invalid Policy Value";
3063 return false;
3064 }
3065 if (!RISCVII::hasVLOp(TSFlags)) {
3066 ErrInfo = "policy operand w/o VL operand?";
3067 return false;
3068 }
3069
3070 // VecPolicy operands can only exist on instructions with passthru/merge
3071 // arguments. Note that not all arguments with passthru have vec policy
3072 // operands- some instructions have implicit policies.
3073 unsigned UseOpIdx;
3074 if (!MI.isRegTiedToUseOperand(0, &UseOpIdx)) {
3075 ErrInfo = "policy operand w/o tied operand?";
3076 return false;
3077 }
3078 }
3079
3080 if (int Idx = RISCVII::getFRMOpNum(Desc);
3081 Idx >= 0 && MI.getOperand(Idx).getImm() == RISCVFPRndMode::DYN &&
3082 !MI.readsRegister(RISCV::FRM, /*TRI=*/nullptr)) {
3083 ErrInfo = "dynamic rounding mode should read FRM";
3084 return false;
3085 }
3086
3087 return true;
3088}
3089
3091 const MachineInstr &AddrI,
3092 ExtAddrMode &AM) const {
3093 switch (MemI.getOpcode()) {
3094 default:
3095 return false;
3096 case RISCV::LB:
3097 case RISCV::LBU:
3098 case RISCV::LH:
3099 case RISCV::LH_INX:
3100 case RISCV::LHU:
3101 case RISCV::LW:
3102 case RISCV::LW_INX:
3103 case RISCV::LWU:
3104 case RISCV::LD:
3105 case RISCV::LD_RV32:
3106 case RISCV::FLH:
3107 case RISCV::FLW:
3108 case RISCV::FLD:
3109 case RISCV::SB:
3110 case RISCV::SH:
3111 case RISCV::SH_INX:
3112 case RISCV::SW:
3113 case RISCV::SW_INX:
3114 case RISCV::SD:
3115 case RISCV::SD_RV32:
3116 case RISCV::FSH:
3117 case RISCV::FSW:
3118 case RISCV::FSD:
3119 break;
3120 }
3121
3122 if (MemI.getOperand(0).getReg() == Reg)
3123 return false;
3124
3125 if (AddrI.getOpcode() != RISCV::ADDI || !AddrI.getOperand(1).isReg() ||
3126 !AddrI.getOperand(2).isImm())
3127 return false;
3128
3129 int64_t OldOffset = MemI.getOperand(2).getImm();
3130 int64_t Disp = AddrI.getOperand(2).getImm();
3131 int64_t NewOffset = OldOffset + Disp;
3132 if (!STI.is64Bit())
3133 NewOffset = SignExtend64<32>(NewOffset);
3134
3135 if (!isInt<12>(NewOffset))
3136 return false;
3137
3138 AM.BaseReg = AddrI.getOperand(1).getReg();
3139 AM.ScaledReg = 0;
3140 AM.Scale = 0;
3141 AM.Displacement = NewOffset;
3143 return true;
3144}
3145
3147 const ExtAddrMode &AM) const {
3148
3149 const DebugLoc &DL = MemI.getDebugLoc();
3150 MachineBasicBlock &MBB = *MemI.getParent();
3151
3152 assert(AM.ScaledReg == 0 && AM.Scale == 0 &&
3153 "Addressing mode not supported for folding");
3154
3155 return BuildMI(MBB, MemI, DL, get(MemI.getOpcode()))
3156 .addReg(MemI.getOperand(0).getReg(),
3157 MemI.mayLoad() ? RegState::Define : 0)
3158 .addReg(AM.BaseReg)
3159 .addImm(AM.Displacement)
3160 .setMemRefs(MemI.memoperands())
3161 .setMIFlags(MemI.getFlags());
3162}
3163
3164// TODO: At the moment, MIPS introduced paring of instructions operating with
3165// word or double word. This should be extended with more instructions when more
3166// vendors support load/store pairing.
3168 switch (Opc) {
3169 default:
3170 return false;
3171 case RISCV::SW:
3172 case RISCV::SD:
3173 case RISCV::LD:
3174 case RISCV::LW:
3175 return true;
3176 }
3177}
3178
3180 const TargetRegisterInfo *TRI) {
3181 // If this is a volatile load/store, don't mess with it.
3182 if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3)
3183 return false;
3184
3185 if (LdSt.getOperand(1).isFI())
3186 return true;
3187
3188 assert(LdSt.getOperand(1).isReg() && "Expected a reg operand.");
3189 // Can't cluster if the instruction modifies the base register
3190 // or it is update form. e.g. ld x5,8(x5)
3191 if (LdSt.modifiesRegister(LdSt.getOperand(1).getReg(), TRI))
3192 return false;
3193
3194 if (!LdSt.getOperand(2).isImm())
3195 return false;
3196
3197 return true;
3198}
3199
3202 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
3203 const TargetRegisterInfo *TRI) const {
3204 if (!LdSt.mayLoadOrStore())
3205 return false;
3206
3207 // Conservatively, only handle scalar loads/stores for now.
3208 switch (LdSt.getOpcode()) {
3209 case RISCV::LB:
3210 case RISCV::LBU:
3211 case RISCV::SB:
3212 case RISCV::LH:
3213 case RISCV::LH_INX:
3214 case RISCV::LHU:
3215 case RISCV::FLH:
3216 case RISCV::SH:
3217 case RISCV::SH_INX:
3218 case RISCV::FSH:
3219 case RISCV::LW:
3220 case RISCV::LW_INX:
3221 case RISCV::LWU:
3222 case RISCV::FLW:
3223 case RISCV::SW:
3224 case RISCV::SW_INX:
3225 case RISCV::FSW:
3226 case RISCV::LD:
3227 case RISCV::LD_RV32:
3228 case RISCV::FLD:
3229 case RISCV::SD:
3230 case RISCV::SD_RV32:
3231 case RISCV::FSD:
3232 break;
3233 default:
3234 return false;
3235 }
3236 const MachineOperand *BaseOp;
3237 OffsetIsScalable = false;
3238 if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
3239 return false;
3240 BaseOps.push_back(BaseOp);
3241 return true;
3242}
3243
3244// TODO: This was copied from SIInstrInfo. Could it be lifted to a common
3245// helper?
3248 const MachineInstr &MI2,
3250 // Only examine the first "base" operand of each instruction, on the
3251 // assumption that it represents the real base address of the memory access.
3252 // Other operands are typically offsets or indices from this base address.
3253 if (BaseOps1.front()->isIdenticalTo(*BaseOps2.front()))
3254 return true;
3255
3256 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
3257 return false;
3258
3259 auto MO1 = *MI1.memoperands_begin();
3260 auto MO2 = *MI2.memoperands_begin();
3261 if (MO1->getAddrSpace() != MO2->getAddrSpace())
3262 return false;
3263
3264 auto Base1 = MO1->getValue();
3265 auto Base2 = MO2->getValue();
3266 if (!Base1 || !Base2)
3267 return false;
3268 Base1 = getUnderlyingObject(Base1);
3269 Base2 = getUnderlyingObject(Base2);
3270
3271 if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
3272 return false;
3273
3274 return Base1 == Base2;
3275}
3276
3278 ArrayRef<const MachineOperand *> BaseOps1, int64_t Offset1,
3279 bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
3280 int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize,
3281 unsigned NumBytes) const {
3282 // If the mem ops (to be clustered) do not have the same base ptr, then they
3283 // should not be clustered
3284 if (!BaseOps1.empty() && !BaseOps2.empty()) {
3285 const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();
3286 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
3287 if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOps1, SecondLdSt, BaseOps2))
3288 return false;
3289 } else if (!BaseOps1.empty() || !BaseOps2.empty()) {
3290 // If only one base op is empty, they do not have the same base ptr
3291 return false;
3292 }
3293
3294 unsigned CacheLineSize =
3295 BaseOps1.front()->getParent()->getMF()->getSubtarget().getCacheLineSize();
3296 // Assume a cache line size of 64 bytes if no size is set in RISCVSubtarget.
3298 // Cluster if the memory operations are on the same or a neighbouring cache
3299 // line, but limit the maximum ClusterSize to avoid creating too much
3300 // additional register pressure.
3301 return ClusterSize <= 4 && std::abs(Offset1 - Offset2) < CacheLineSize;
3302}
3303
3304// Set BaseReg (the base register operand), Offset (the byte offset being
3305// accessed) and the access Width of the passed instruction that reads/writes
3306// memory. Returns false if the instruction does not read/write memory or the
3307// BaseReg/Offset/Width can't be determined. Is not guaranteed to always
3308// recognise base operands and offsets in all cases.
3309// TODO: Add an IsScalable bool ref argument (like the equivalent AArch64
3310// function) and set it as appropriate.
3312 const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
3313 LocationSize &Width, const TargetRegisterInfo *TRI) const {
3314 if (!LdSt.mayLoadOrStore())
3315 return false;
3316
3317 // Here we assume the standard RISC-V ISA, which uses a base+offset
3318 // addressing mode. You'll need to relax these conditions to support custom
3319 // load/store instructions.
3320 if (LdSt.getNumExplicitOperands() != 3)
3321 return false;
3322 if ((!LdSt.getOperand(1).isReg() && !LdSt.getOperand(1).isFI()) ||
3323 !LdSt.getOperand(2).isImm())
3324 return false;
3325
3326 if (!LdSt.hasOneMemOperand())
3327 return false;
3328
3329 Width = (*LdSt.memoperands_begin())->getSize();
3330 BaseReg = &LdSt.getOperand(1);
3331 Offset = LdSt.getOperand(2).getImm();
3332 return true;
3333}
3334
3336 const MachineInstr &MIa, const MachineInstr &MIb) const {
3337 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
3338 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
3339
3342 return false;
3343
3344 // Retrieve the base register, offset from the base register and width. Width
3345 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If
3346 // base registers are identical, and the offset of a lower memory access +
3347 // the width doesn't overlap the offset of a higher memory access,
3348 // then the memory accesses are different.
3349 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
3350 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
3351 int64_t OffsetA = 0, OffsetB = 0;
3353 WidthB = LocationSize::precise(0);
3354 if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
3355 getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
3356 if (BaseOpA->isIdenticalTo(*BaseOpB)) {
3357 int LowOffset = std::min(OffsetA, OffsetB);
3358 int HighOffset = std::max(OffsetA, OffsetB);
3359 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
3360 if (LowWidth.hasValue() &&
3361 LowOffset + (int)LowWidth.getValue() <= HighOffset)
3362 return true;
3363 }
3364 }
3365 return false;
3366}
3367
3368std::pair<unsigned, unsigned>
3370 const unsigned Mask = RISCVII::MO_DIRECT_FLAG_MASK;
3371 return std::make_pair(TF & Mask, TF & ~Mask);
3372}
3373
3376 using namespace RISCVII;
3377 static const std::pair<unsigned, const char *> TargetFlags[] = {
3378 {MO_CALL, "riscv-call"},
3379 {MO_LO, "riscv-lo"},
3380 {MO_HI, "riscv-hi"},
3381 {MO_PCREL_LO, "riscv-pcrel-lo"},
3382 {MO_PCREL_HI, "riscv-pcrel-hi"},
3383 {MO_GOT_HI, "riscv-got-hi"},
3384 {MO_TPREL_LO, "riscv-tprel-lo"},
3385 {MO_TPREL_HI, "riscv-tprel-hi"},
3386 {MO_TPREL_ADD, "riscv-tprel-add"},
3387 {MO_TLS_GOT_HI, "riscv-tls-got-hi"},
3388 {MO_TLS_GD_HI, "riscv-tls-gd-hi"},
3389 {MO_TLSDESC_HI, "riscv-tlsdesc-hi"},
3390 {MO_TLSDESC_LOAD_LO, "riscv-tlsdesc-load-lo"},
3391 {MO_TLSDESC_ADD_LO, "riscv-tlsdesc-add-lo"},
3392 {MO_TLSDESC_CALL, "riscv-tlsdesc-call"}};
3393 return ArrayRef(TargetFlags);
3394}
3396 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
3397 const Function &F = MF.getFunction();
3398
3399 // Can F be deduplicated by the linker? If it can, don't outline from it.
3400 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
3401 return false;
3402
3403 // Don't outline from functions with section markings; the program could
3404 // expect that all the code is in the named section.
3405 if (F.hasSection())
3406 return false;
3407
3408 // It's safe to outline from MF.
3409 return true;
3410}
3411
3413 unsigned &Flags) const {
3414 // More accurate safety checking is done in getOutliningCandidateInfo.
3416}
3417
3418// Enum values indicating how an outlined call should be constructed.
3423
3428
3430 const MachineFunction *MF = MBB.getParent();
3431 const Function &F = MF->getFunction();
3432 return F.getFnAttribute("fentry-call").getValueAsBool() ||
3433 F.hasFnAttribute("patchable-function-entry");
3434}
3435
3437 MCRegister RegNo) {
3438 return MI.readsRegister(RegNo, TRI) ||
3439 MI.getDesc().hasImplicitUseOfPhysReg(RegNo);
3440}
3441
3443 const TargetRegisterInfo *TRI, MCRegister RegNo) {
3444 return MI.modifiesRegister(RegNo, TRI) ||
3445 MI.getDesc().hasImplicitDefOfPhysReg(RegNo);
3446}
3447
3449 if (!MBB.back().isReturn())
3450 return true;
3452 return true;
3453
3454 // If the candidate reads the pre-set register
3455 // that can be used for expanding PseudoTAIL instruction,
3456 // then we cannot insert tail call.
3457 const TargetSubtargetInfo &STI = MBB.getParent()->getSubtarget();
3458 MCRegister TailExpandUseRegNo =
3460 for (const MachineInstr &MI : MBB) {
3461 if (isMIReadsReg(MI, STI.getRegisterInfo(), TailExpandUseRegNo))
3462 return true;
3463 if (isMIModifiesReg(MI, STI.getRegisterInfo(), TailExpandUseRegNo))
3464 break;
3465 }
3466 return false;
3467}
3468
3470 // If last instruction is return then we can rely on
3471 // the verification already performed in the getOutliningTypeImpl.
3472 if (C.back().isReturn()) {
3473 assert(!cannotInsertTailCall(*C.getMBB()) &&
3474 "The candidate who uses return instruction must be outlined "
3475 "using tail call");
3476 return false;
3477 }
3478
3479 // Filter out candidates where the X5 register (t0) can't be used to setup
3480 // the function call.
3481 const TargetRegisterInfo *TRI = C.getMF()->getSubtarget().getRegisterInfo();
3482 if (llvm::any_of(C, [TRI](const MachineInstr &MI) {
3483 return isMIModifiesReg(MI, TRI, RISCV::X5);
3484 }))
3485 return true;
3486
3487 return !C.isAvailableAcrossAndOutOfSeq(RISCV::X5, *TRI);
3488}
3489
3490std::optional<std::unique_ptr<outliner::OutlinedFunction>>
3492 const MachineModuleInfo &MMI,
3493 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
3494 unsigned MinRepeats) const {
3495
3496 // Analyze each candidate and erase the ones that are not viable.
3497 llvm::erase_if(RepeatedSequenceLocs, analyzeCandidate);
3498
3499 // If the sequence doesn't have enough candidates left, then we're done.
3500 if (RepeatedSequenceLocs.size() < MinRepeats)
3501 return std::nullopt;
3502
3503 // Each RepeatedSequenceLoc is identical.
3504 outliner::Candidate &Candidate = RepeatedSequenceLocs[0];
3505 unsigned InstrSizeCExt =
3506 Candidate.getMF()->getSubtarget<RISCVSubtarget>().hasStdExtZca() ? 2 : 4;
3507 unsigned CallOverhead = 0, FrameOverhead = 0;
3508
3510 if (Candidate.back().isReturn()) {
3512 // tail call = auipc + jalr in the worst case without linker relaxation.
3513 // FIXME: This code suggests the JALR can be compressed - how?
3514 CallOverhead = 4 + InstrSizeCExt;
3515 // Using tail call we move ret instruction from caller to callee.
3516 FrameOverhead = 0;
3517 } else {
3518 // call t0, function = 8 bytes.
3519 CallOverhead = 8;
3520 // jr t0 = 4 bytes, 2 bytes if compressed instructions are enabled.
3521 FrameOverhead = InstrSizeCExt;
3522 }
3523
3524 for (auto &C : RepeatedSequenceLocs)
3525 C.setCallInfo(MOCI, CallOverhead);
3526
3527 unsigned SequenceSize = 0;
3528 for (auto &MI : Candidate)
3529 SequenceSize += getInstSizeInBytes(MI);
3530
3531 return std::make_unique<outliner::OutlinedFunction>(
3532 RepeatedSequenceLocs, SequenceSize, FrameOverhead, MOCI);
3533}
3534
3538 unsigned Flags) const {
3539 MachineInstr &MI = *MBBI;
3540 MachineBasicBlock *MBB = MI.getParent();
3541 const TargetRegisterInfo *TRI =
3542 MBB->getParent()->getSubtarget().getRegisterInfo();
3543 const auto &F = MI.getMF()->getFunction();
3544
3545 // We can manually strip out CFI instructions later.
3546 if (MI.isCFIInstruction())
3547 // If current function has exception handling code, we can't outline &
3548 // strip these CFI instructions since it may break .eh_frame section
3549 // needed in unwinding.
3550 return F.needsUnwindTableEntry() ? outliner::InstrType::Illegal
3552
3553 if (cannotInsertTailCall(*MBB) &&
3554 (MI.isReturn() || isMIModifiesReg(MI, TRI, RISCV::X5)))
3556
3557 // Make sure the operands don't reference something unsafe.
3558 for (const auto &MO : MI.operands()) {
3559
3560 // pcrel-hi and pcrel-lo can't put in separate sections, filter that out
3561 // if any possible.
3562 if (MO.getTargetFlags() == RISCVII::MO_PCREL_LO &&
3563 (MI.getMF()->getTarget().getFunctionSections() || F.hasComdat() ||
3564 F.hasSection() || F.getSectionPrefix()))
3566 }
3567
3568 if (isLPAD(MI))
3570
3572}
3573
3576 const outliner::OutlinedFunction &OF) const {
3577
3578 // Strip out any CFI instructions
3579 bool Changed = true;
3580 while (Changed) {
3581 Changed = false;
3582 auto I = MBB.begin();
3583 auto E = MBB.end();
3584 for (; I != E; ++I) {
3585 if (I->isCFIInstruction()) {
3586 I->removeFromParent();
3587 Changed = true;
3588 break;
3589 }
3590 }
3591 }
3592
3593 if (OF.FrameConstructionID == MachineOutlinerTailCall)
3594 return;
3595
3596 MBB.addLiveIn(RISCV::X5);
3597
3598 // Add in a return instruction to the end of the outlined frame.
3599 MBB.insert(MBB.end(), BuildMI(MF, DebugLoc(), get(RISCV::JALR))
3600 .addReg(RISCV::X0, RegState::Define)
3601 .addReg(RISCV::X5)
3602 .addImm(0));
3603}
3604
3608
3609 if (C.CallConstructionID == MachineOutlinerTailCall) {
3610 It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(RISCV::PseudoTAIL))
3611 .addGlobalAddress(M.getNamedValue(MF.getName()),
3612 /*Offset=*/0, RISCVII::MO_CALL));
3613 return It;
3614 }
3615
3616 // Add in a call instruction to the outlined function at the given location.
3617 It = MBB.insert(It,
3618 BuildMI(MF, DebugLoc(), get(RISCV::PseudoCALLReg), RISCV::X5)
3619 .addGlobalAddress(M.getNamedValue(MF.getName()), 0,
3621 return It;
3622}
3623
3624std::optional<RegImmPair> RISCVInstrInfo::isAddImmediate(const MachineInstr &MI,
3625 Register Reg) const {
3626 // TODO: Handle cases where Reg is a super- or sub-register of the
3627 // destination register.
3628 const MachineOperand &Op0 = MI.getOperand(0);
3629 if (!Op0.isReg() || Reg != Op0.getReg())
3630 return std::nullopt;
3631
3632 // Don't consider ADDIW as a candidate because the caller may not be aware
3633 // of its sign extension behaviour.
3634 if (MI.getOpcode() == RISCV::ADDI && MI.getOperand(1).isReg() &&
3635 MI.getOperand(2).isImm())
3636 return RegImmPair{MI.getOperand(1).getReg(), MI.getOperand(2).getImm()};
3637
3638 return std::nullopt;
3639}
3640
3641// MIR printer helper function to annotate Operands with a comment.
3643 const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,
3644 const TargetRegisterInfo *TRI) const {
3645 // Print a generic comment for this operand if there is one.
3646 std::string GenericComment =
3648 if (!GenericComment.empty())
3649 return GenericComment;
3650
3651 // If not, we must have an immediate operand.
3652 if (!Op.isImm())
3653 return std::string();
3654
3655 const MCInstrDesc &Desc = MI.getDesc();
3656 if (OpIdx >= Desc.getNumOperands())
3657 return std::string();
3658
3659 std::string Comment;
3660 raw_string_ostream OS(Comment);
3661
3662 const MCOperandInfo &OpInfo = Desc.operands()[OpIdx];
3663
3664 // Print the full VType operand of vsetvli/vsetivli instructions, and the SEW
3665 // operand of vector codegen pseudos.
3666 switch (OpInfo.OperandType) {
3669 unsigned Imm = Op.getImm();
3670 RISCVVType::printVType(Imm, OS);
3671 break;
3672 }
3675 unsigned Log2SEW = Op.getImm();
3676 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
3677 assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW");
3678 OS << "e" << SEW;
3679 break;
3680 }
3682 unsigned Policy = Op.getImm();
3684 "Invalid Policy Value");
3685 OS << (Policy & RISCVVType::TAIL_AGNOSTIC ? "ta" : "tu") << ", "
3686 << (Policy & RISCVVType::MASK_AGNOSTIC ? "ma" : "mu");
3687 break;
3688 }
3689
3690 return Comment;
3691}
3692
3693// clang-format off
3694#define CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL) \
3695 RISCV::Pseudo##OP##_##LMUL
3696
3697#define CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL) \
3698 RISCV::Pseudo##OP##_##LMUL##_MASK
3699
3700#define CASE_RVV_OPCODE_LMUL(OP, LMUL) \
3701 CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL): \
3702 case CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL)
3703
3704#define CASE_RVV_OPCODE_UNMASK_WIDEN(OP) \
3705 CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF8): \
3706 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF4): \
3707 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF2): \
3708 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M1): \
3709 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M2): \
3710 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M4)
3711
3712#define CASE_RVV_OPCODE_UNMASK(OP) \
3713 CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
3714 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M8)
3715
3716#define CASE_RVV_OPCODE_MASK_WIDEN(OP) \
3717 CASE_RVV_OPCODE_MASK_LMUL(OP, MF8): \
3718 case CASE_RVV_OPCODE_MASK_LMUL(OP, MF4): \
3719 case CASE_RVV_OPCODE_MASK_LMUL(OP, MF2): \
3720 case CASE_RVV_OPCODE_MASK_LMUL(OP, M1): \
3721 case CASE_RVV_OPCODE_MASK_LMUL(OP, M2): \
3722 case CASE_RVV_OPCODE_MASK_LMUL(OP, M4)
3723
3724#define CASE_RVV_OPCODE_MASK(OP) \
3725 CASE_RVV_OPCODE_MASK_WIDEN(OP): \
3726 case CASE_RVV_OPCODE_MASK_LMUL(OP, M8)
3727
3728#define CASE_RVV_OPCODE_WIDEN(OP) \
3729 CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
3730 case CASE_RVV_OPCODE_MASK_WIDEN(OP)
3731
3732#define CASE_RVV_OPCODE(OP) \
3733 CASE_RVV_OPCODE_UNMASK(OP): \
3734 case CASE_RVV_OPCODE_MASK(OP)
3735// clang-format on
3736
3737// clang-format off
3738#define CASE_VMA_OPCODE_COMMON(OP, TYPE, LMUL) \
3739 RISCV::PseudoV##OP##_##TYPE##_##LMUL
3740
3741#define CASE_VMA_OPCODE_LMULS(OP, TYPE) \
3742 CASE_VMA_OPCODE_COMMON(OP, TYPE, MF8): \
3743 case CASE_VMA_OPCODE_COMMON(OP, TYPE, MF4): \
3744 case CASE_VMA_OPCODE_COMMON(OP, TYPE, MF2): \
3745 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M1): \
3746 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M2): \
3747 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M4): \
3748 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M8)
3749
3750// VFMA instructions are SEW specific.
3751#define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL, SEW) \
3752 RISCV::PseudoV##OP##_##TYPE##_##LMUL##_##SEW
3753
3754#define CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW) \
3755 CASE_VFMA_OPCODE_COMMON(OP, TYPE, M1, SEW): \
3756 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M2, SEW): \
3757 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M4, SEW): \
3758 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M8, SEW)
3759
3760#define CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW) \
3761 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF2, SEW): \
3762 case CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW)
3763
3764#define CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE, SEW) \
3765 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF4, SEW): \
3766 case CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW)
3767
3768#define CASE_VFMA_OPCODE_VV(OP) \
3769 CASE_VFMA_OPCODE_LMULS_MF4(OP, VV, E16): \
3770 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VV, E32): \
3771 case CASE_VFMA_OPCODE_LMULS_M1(OP, VV, E64)
3772
3773#define CASE_VFMA_SPLATS(OP) \
3774 CASE_VFMA_OPCODE_LMULS_MF4(OP, VFPR16, E16): \
3775 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VFPR32, E32): \
3776 case CASE_VFMA_OPCODE_LMULS_M1(OP, VFPR64, E64)
3777// clang-format on
3778
3780 unsigned &SrcOpIdx1,
3781 unsigned &SrcOpIdx2) const {
3782 const MCInstrDesc &Desc = MI.getDesc();
3783 if (!Desc.isCommutable())
3784 return false;
3785
3786 switch (MI.getOpcode()) {
3787 case RISCV::TH_MVEQZ:
3788 case RISCV::TH_MVNEZ:
3789 // We can't commute operands if operand 2 (i.e., rs1 in
3790 // mveqz/mvnez rd,rs1,rs2) is the zero-register (as it is
3791 // not valid as the in/out-operand 1).
3792 if (MI.getOperand(2).getReg() == RISCV::X0)
3793 return false;
3794 // Operands 1 and 2 are commutable, if we switch the opcode.
3795 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1, 2);
3796 case RISCV::QC_MVEQ:
3797 case RISCV::QC_MVNE:
3798 case RISCV::QC_MVLT:
3799 case RISCV::QC_MVGE:
3800 case RISCV::QC_MVLTU:
3801 case RISCV::QC_MVGEU:
3802 case RISCV::QC_MVEQI:
3803 case RISCV::QC_MVNEI:
3804 case RISCV::QC_MVLTI:
3805 case RISCV::QC_MVGEI:
3806 case RISCV::QC_MVLTUI:
3807 case RISCV::QC_MVGEUI:
3808 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1, 4);
3809 case RISCV::TH_MULA:
3810 case RISCV::TH_MULAW:
3811 case RISCV::TH_MULAH:
3812 case RISCV::TH_MULS:
3813 case RISCV::TH_MULSW:
3814 case RISCV::TH_MULSH:
3815 // Operands 2 and 3 are commutable.
3816 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
3817 case RISCV::PseudoCCMOVGPRNoX0:
3818 case RISCV::PseudoCCMOVGPR:
3819 // Operands 4 and 5 are commutable.
3820 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 4, 5);
3821 case CASE_RVV_OPCODE(VADD_VV):
3822 case CASE_RVV_OPCODE(VAND_VV):
3823 case CASE_RVV_OPCODE(VOR_VV):
3824 case CASE_RVV_OPCODE(VXOR_VV):
3825 case CASE_RVV_OPCODE_MASK(VMSEQ_VV):
3826 case CASE_RVV_OPCODE_MASK(VMSNE_VV):
3827 case CASE_RVV_OPCODE(VMIN_VV):
3828 case CASE_RVV_OPCODE(VMINU_VV):
3829 case CASE_RVV_OPCODE(VMAX_VV):
3830 case CASE_RVV_OPCODE(VMAXU_VV):
3831 case CASE_RVV_OPCODE(VMUL_VV):
3832 case CASE_RVV_OPCODE(VMULH_VV):
3833 case CASE_RVV_OPCODE(VMULHU_VV):
3834 case CASE_RVV_OPCODE_WIDEN(VWADD_VV):
3835 case CASE_RVV_OPCODE_WIDEN(VWADDU_VV):
3836 case CASE_RVV_OPCODE_WIDEN(VWMUL_VV):
3837 case CASE_RVV_OPCODE_WIDEN(VWMULU_VV):
3838 case CASE_RVV_OPCODE_WIDEN(VWMACC_VV):
3839 case CASE_RVV_OPCODE_WIDEN(VWMACCU_VV):
3840 case CASE_RVV_OPCODE_UNMASK(VADC_VVM):
3841 case CASE_RVV_OPCODE(VSADD_VV):
3842 case CASE_RVV_OPCODE(VSADDU_VV):
3843 case CASE_RVV_OPCODE(VAADD_VV):
3844 case CASE_RVV_OPCODE(VAADDU_VV):
3845 case CASE_RVV_OPCODE(VSMUL_VV):
3846 // Operands 2 and 3 are commutable.
3847 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
3848 case CASE_VFMA_SPLATS(FMADD):
3849 case CASE_VFMA_SPLATS(FMSUB):
3850 case CASE_VFMA_SPLATS(FMACC):
3851 case CASE_VFMA_SPLATS(FMSAC):
3854 case CASE_VFMA_SPLATS(FNMACC):
3855 case CASE_VFMA_SPLATS(FNMSAC):
3856 case CASE_VFMA_OPCODE_VV(FMACC):
3857 case CASE_VFMA_OPCODE_VV(FMSAC):
3858 case CASE_VFMA_OPCODE_VV(FNMACC):
3859 case CASE_VFMA_OPCODE_VV(FNMSAC):
3860 case CASE_VMA_OPCODE_LMULS(MADD, VX):
3861 case CASE_VMA_OPCODE_LMULS(NMSUB, VX):
3862 case CASE_VMA_OPCODE_LMULS(MACC, VX):
3863 case CASE_VMA_OPCODE_LMULS(NMSAC, VX):
3864 case CASE_VMA_OPCODE_LMULS(MACC, VV):
3865 case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
3866 // If the tail policy is undisturbed we can't commute.
3867 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
3868 if ((MI.getOperand(RISCVII::getVecPolicyOpNum(MI.getDesc())).getImm() &
3869 1) == 0)
3870 return false;
3871
3872 // For these instructions we can only swap operand 1 and operand 3 by
3873 // changing the opcode.
3874 unsigned CommutableOpIdx1 = 1;
3875 unsigned CommutableOpIdx2 = 3;
3876 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
3877 CommutableOpIdx2))
3878 return false;
3879 return true;
3880 }
3881 case CASE_VFMA_OPCODE_VV(FMADD):
3885 case CASE_VMA_OPCODE_LMULS(MADD, VV):
3886 case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
3887 // If the tail policy is undisturbed we can't commute.
3888 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
3889 if ((MI.getOperand(RISCVII::getVecPolicyOpNum(MI.getDesc())).getImm() &
3890 1) == 0)
3891 return false;
3892
3893 // For these instructions we have more freedom. We can commute with the
3894 // other multiplicand or with the addend/subtrahend/minuend.
3895
3896 // Any fixed operand must be from source 1, 2 or 3.
3897 if (SrcOpIdx1 != CommuteAnyOperandIndex && SrcOpIdx1 > 3)
3898 return false;
3899 if (SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx2 > 3)
3900 return false;
3901
3902 // It both ops are fixed one must be the tied source.
3903 if (SrcOpIdx1 != CommuteAnyOperandIndex &&
3904 SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx1 != 1 && SrcOpIdx2 != 1)
3905 return false;
3906
3907 // Look for two different register operands assumed to be commutable
3908 // regardless of the FMA opcode. The FMA opcode is adjusted later if
3909 // needed.
3910 if (SrcOpIdx1 == CommuteAnyOperandIndex ||
3911 SrcOpIdx2 == CommuteAnyOperandIndex) {
3912 // At least one of operands to be commuted is not specified and
3913 // this method is free to choose appropriate commutable operands.
3914 unsigned CommutableOpIdx1 = SrcOpIdx1;
3915 if (SrcOpIdx1 == SrcOpIdx2) {
3916 // Both of operands are not fixed. Set one of commutable
3917 // operands to the tied source.
3918 CommutableOpIdx1 = 1;
3919 } else if (SrcOpIdx1 == CommuteAnyOperandIndex) {
3920 // Only one of the operands is not fixed.
3921 CommutableOpIdx1 = SrcOpIdx2;
3922 }
3923
3924 // CommutableOpIdx1 is well defined now. Let's choose another commutable
3925 // operand and assign its index to CommutableOpIdx2.
3926 unsigned CommutableOpIdx2;
3927 if (CommutableOpIdx1 != 1) {
3928 // If we haven't already used the tied source, we must use it now.
3929 CommutableOpIdx2 = 1;
3930 } else {
3931 Register Op1Reg = MI.getOperand(CommutableOpIdx1).getReg();
3932
3933 // The commuted operands should have different registers.
3934 // Otherwise, the commute transformation does not change anything and
3935 // is useless. We use this as a hint to make our decision.
3936 if (Op1Reg != MI.getOperand(2).getReg())
3937 CommutableOpIdx2 = 2;
3938 else
3939 CommutableOpIdx2 = 3;
3940 }
3941
3942 // Assign the found pair of commutable indices to SrcOpIdx1 and
3943 // SrcOpIdx2 to return those values.
3944 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
3945 CommutableOpIdx2))
3946 return false;
3947 }
3948
3949 return true;
3950 }
3951 }
3952
3953 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
3954}
3955
3956// clang-format off
3957#define CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL) \
3958 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL: \
3959 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL; \
3960 break;
3961
3962#define CASE_VMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE) \
3963 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8) \
3964 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4) \
3965 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2) \
3966 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1) \
3967 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2) \
3968 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4) \
3969 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8)
3970
3971// VFMA depends on SEW.
3972#define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL, SEW) \
3973 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL##_##SEW: \
3974 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL##_##SEW; \
3975 break;
3976
3977#define CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW) \
3978 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1, SEW) \
3979 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2, SEW) \
3980 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4, SEW) \
3981 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8, SEW)
3982
3983#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW) \
3984 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2, SEW) \
3985 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW)
3986
3987#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW) \
3988 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4, SEW) \
3989 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW)
3990
3991#define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP) \
3992 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VV, E16) \
3993 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E32) \
3994 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VV, E64)
3995
3996#define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP) \
3997 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16, E16) \
3998 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VFPR32, E32) \
3999 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VFPR64, E64)
4000// clang-format on
4001
4003 bool NewMI,
4004 unsigned OpIdx1,
4005 unsigned OpIdx2) const {
4006 auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & {
4007 if (NewMI)
4008 return *MI.getParent()->getParent()->CloneMachineInstr(&MI);
4009 return MI;
4010 };
4011
4012 switch (MI.getOpcode()) {
4013 case RISCV::TH_MVEQZ:
4014 case RISCV::TH_MVNEZ: {
4015 auto &WorkingMI = cloneIfNew(MI);
4016 WorkingMI.setDesc(get(MI.getOpcode() == RISCV::TH_MVEQZ ? RISCV::TH_MVNEZ
4017 : RISCV::TH_MVEQZ));
4018 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, false, OpIdx1,
4019 OpIdx2);
4020 }
4021 case RISCV::QC_MVEQ:
4022 case RISCV::QC_MVNE:
4023 case RISCV::QC_MVLT:
4024 case RISCV::QC_MVGE:
4025 case RISCV::QC_MVLTU:
4026 case RISCV::QC_MVGEU:
4027 case RISCV::QC_MVEQI:
4028 case RISCV::QC_MVNEI:
4029 case RISCV::QC_MVLTI:
4030 case RISCV::QC_MVGEI:
4031 case RISCV::QC_MVLTUI:
4032 case RISCV::QC_MVGEUI: {
4033 auto &WorkingMI = cloneIfNew(MI);
4034 WorkingMI.setDesc(get(getInverseXqcicmOpcode(MI.getOpcode())));
4035 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, false, OpIdx1,
4036 OpIdx2);
4037 }
4038 case RISCV::PseudoCCMOVGPRNoX0:
4039 case RISCV::PseudoCCMOVGPR: {
4040 // CCMOV can be commuted by inverting the condition.
4041 auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());
4043 auto &WorkingMI = cloneIfNew(MI);
4044 WorkingMI.getOperand(3).setImm(CC);
4045 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI*/ false,
4046 OpIdx1, OpIdx2);
4047 }
4048 case CASE_VFMA_SPLATS(FMACC):
4049 case CASE_VFMA_SPLATS(FMADD):
4050 case CASE_VFMA_SPLATS(FMSAC):
4051 case CASE_VFMA_SPLATS(FMSUB):
4052 case CASE_VFMA_SPLATS(FNMACC):
4054 case CASE_VFMA_SPLATS(FNMSAC):
4056 case CASE_VFMA_OPCODE_VV(FMACC):
4057 case CASE_VFMA_OPCODE_VV(FMSAC):
4058 case CASE_VFMA_OPCODE_VV(FNMACC):
4059 case CASE_VFMA_OPCODE_VV(FNMSAC):
4060 case CASE_VMA_OPCODE_LMULS(MADD, VX):
4061 case CASE_VMA_OPCODE_LMULS(NMSUB, VX):
4062 case CASE_VMA_OPCODE_LMULS(MACC, VX):
4063 case CASE_VMA_OPCODE_LMULS(NMSAC, VX):
4064 case CASE_VMA_OPCODE_LMULS(MACC, VV):
4065 case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
4066 // It only make sense to toggle these between clobbering the
4067 // addend/subtrahend/minuend one of the multiplicands.
4068 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
4069 assert((OpIdx1 == 3 || OpIdx2 == 3) && "Unexpected opcode index");
4070 unsigned Opc;
4071 switch (MI.getOpcode()) {
4072 default:
4073 llvm_unreachable("Unexpected opcode");
4074 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMACC, FMADD)
4075 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMADD, FMACC)
4082 CASE_VFMA_CHANGE_OPCODE_VV(FMACC, FMADD)
4086 CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VX)
4087 CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VX)
4088 CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VX)
4089 CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VX)
4090 CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VV)
4091 CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VV)
4092 }
4093
4094 auto &WorkingMI = cloneIfNew(MI);
4095 WorkingMI.setDesc(get(Opc));
4096 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
4097 OpIdx1, OpIdx2);
4098 }
4099 case CASE_VFMA_OPCODE_VV(FMADD):
4103 case CASE_VMA_OPCODE_LMULS(MADD, VV):
4104 case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
4105 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
4106 // If one of the operands, is the addend we need to change opcode.
4107 // Otherwise we're just swapping 2 of the multiplicands.
4108 if (OpIdx1 == 3 || OpIdx2 == 3) {
4109 unsigned Opc;
4110 switch (MI.getOpcode()) {
4111 default:
4112 llvm_unreachable("Unexpected opcode");
4113 CASE_VFMA_CHANGE_OPCODE_VV(FMADD, FMACC)
4117 CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VV)
4118 CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VV)
4119 }
4120
4121 auto &WorkingMI = cloneIfNew(MI);
4122 WorkingMI.setDesc(get(Opc));
4123 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
4124 OpIdx1, OpIdx2);
4125 }
4126 // Let the default code handle it.
4127 break;
4128 }
4129 }
4130
4131 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
4132}
4133
4134#undef CASE_VMA_CHANGE_OPCODE_COMMON
4135#undef CASE_VMA_CHANGE_OPCODE_LMULS
4136#undef CASE_VFMA_CHANGE_OPCODE_COMMON
4137#undef CASE_VFMA_CHANGE_OPCODE_LMULS_M1
4138#undef CASE_VFMA_CHANGE_OPCODE_LMULS_MF2
4139#undef CASE_VFMA_CHANGE_OPCODE_LMULS_MF4
4140#undef CASE_VFMA_CHANGE_OPCODE_VV
4141#undef CASE_VFMA_CHANGE_OPCODE_SPLATS
4142
4143#undef CASE_RVV_OPCODE_UNMASK_LMUL
4144#undef CASE_RVV_OPCODE_MASK_LMUL
4145#undef CASE_RVV_OPCODE_LMUL
4146#undef CASE_RVV_OPCODE_UNMASK_WIDEN
4147#undef CASE_RVV_OPCODE_UNMASK
4148#undef CASE_RVV_OPCODE_MASK_WIDEN
4149#undef CASE_RVV_OPCODE_MASK
4150#undef CASE_RVV_OPCODE_WIDEN
4151#undef CASE_RVV_OPCODE
4152
4153#undef CASE_VMA_OPCODE_COMMON
4154#undef CASE_VMA_OPCODE_LMULS
4155#undef CASE_VFMA_OPCODE_COMMON
4156#undef CASE_VFMA_OPCODE_LMULS_M1
4157#undef CASE_VFMA_OPCODE_LMULS_MF2
4158#undef CASE_VFMA_OPCODE_LMULS_MF4
4159#undef CASE_VFMA_OPCODE_VV
4160#undef CASE_VFMA_SPLATS
4161
4163 switch (MI.getOpcode()) {
4164 default:
4165 break;
4166 case RISCV::ADD:
4167 case RISCV::OR:
4168 case RISCV::XOR:
4169 // Normalize (so we hit the next if clause).
4170 // add/[x]or rd, zero, rs => add/[x]or rd, rs, zero
4171 if (MI.getOperand(1).getReg() == RISCV::X0)
4172 commuteInstruction(MI);
4173 // add/[x]or rd, rs, zero => addi rd, rs, 0
4174 if (MI.getOperand(2).getReg() == RISCV::X0) {
4175 MI.getOperand(2).ChangeToImmediate(0);
4176 MI.setDesc(get(RISCV::ADDI));
4177 return true;
4178 }
4179 // xor rd, rs, rs => addi rd, zero, 0
4180 if (MI.getOpcode() == RISCV::XOR &&
4181 MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) {
4182 MI.getOperand(1).setReg(RISCV::X0);
4183 MI.getOperand(2).ChangeToImmediate(0);
4184 MI.setDesc(get(RISCV::ADDI));
4185 return true;
4186 }
4187 break;
4188 case RISCV::ORI:
4189 case RISCV::XORI:
4190 // [x]ori rd, zero, N => addi rd, zero, N
4191 if (MI.getOperand(1).getReg() == RISCV::X0) {
4192 MI.setDesc(get(RISCV::ADDI));
4193 return true;
4194 }
4195 break;
4196 case RISCV::SUB:
4197 // sub rd, rs, zero => addi rd, rs, 0
4198 if (MI.getOperand(2).getReg() == RISCV::X0) {
4199 MI.getOperand(2).ChangeToImmediate(0);
4200 MI.setDesc(get(RISCV::ADDI));
4201 return true;
4202 }
4203 break;
4204 case RISCV::SUBW:
4205 // subw rd, rs, zero => addiw rd, rs, 0
4206 if (MI.getOperand(2).getReg() == RISCV::X0) {
4207 MI.getOperand(2).ChangeToImmediate(0);
4208 MI.setDesc(get(RISCV::ADDIW));
4209 return true;
4210 }
4211 break;
4212 case RISCV::ADDW:
4213 // Normalize (so we hit the next if clause).
4214 // addw rd, zero, rs => addw rd, rs, zero
4215 if (MI.getOperand(1).getReg() == RISCV::X0)
4216 commuteInstruction(MI);
4217 // addw rd, rs, zero => addiw rd, rs, 0
4218 if (MI.getOperand(2).getReg() == RISCV::X0) {
4219 MI.getOperand(2).ChangeToImmediate(0);
4220 MI.setDesc(get(RISCV::ADDIW));
4221 return true;
4222 }
4223 break;
4224 case RISCV::SH1ADD:
4225 case RISCV::SH1ADD_UW:
4226 case RISCV::SH2ADD:
4227 case RISCV::SH2ADD_UW:
4228 case RISCV::SH3ADD:
4229 case RISCV::SH3ADD_UW:
4230 // shNadd[.uw] rd, zero, rs => addi rd, rs, 0
4231 if (MI.getOperand(1).getReg() == RISCV::X0) {
4232 MI.removeOperand(1);
4233 MI.addOperand(MachineOperand::CreateImm(0));
4234 MI.setDesc(get(RISCV::ADDI));
4235 return true;
4236 }
4237 // shNadd[.uw] rd, rs, zero => slli[.uw] rd, rs, N
4238 if (MI.getOperand(2).getReg() == RISCV::X0) {
4239 MI.removeOperand(2);
4240 unsigned Opc = MI.getOpcode();
4241 if (Opc == RISCV::SH1ADD_UW || Opc == RISCV::SH2ADD_UW ||
4242 Opc == RISCV::SH3ADD_UW) {
4244 MI.setDesc(get(RISCV::SLLI_UW));
4245 return true;
4246 }
4248 MI.setDesc(get(RISCV::SLLI));
4249 return true;
4250 }
4251 break;
4252 case RISCV::AND:
4253 case RISCV::MUL:
4254 case RISCV::MULH:
4255 case RISCV::MULHSU:
4256 case RISCV::MULHU:
4257 case RISCV::MULW:
4258 // and rd, zero, rs => addi rd, zero, 0
4259 // mul* rd, zero, rs => addi rd, zero, 0
4260 // and rd, rs, zero => addi rd, zero, 0
4261 // mul* rd, rs, zero => addi rd, zero, 0
4262 if (MI.getOperand(1).getReg() == RISCV::X0 ||
4263 MI.getOperand(2).getReg() == RISCV::X0) {
4264 MI.getOperand(1).setReg(RISCV::X0);
4265 MI.getOperand(2).ChangeToImmediate(0);
4266 MI.setDesc(get(RISCV::ADDI));
4267 return true;
4268 }
4269 break;
4270 case RISCV::ANDI:
4271 // andi rd, zero, C => addi rd, zero, 0
4272 if (MI.getOperand(1).getReg() == RISCV::X0) {
4273 MI.getOperand(2).setImm(0);
4274 MI.setDesc(get(RISCV::ADDI));
4275 return true;
4276 }
4277 break;
4278 case RISCV::SLL:
4279 case RISCV::SRL:
4280 case RISCV::SRA:
4281 // shift rd, zero, rs => addi rd, zero, 0
4282 if (MI.getOperand(1).getReg() == RISCV::X0) {
4283 MI.getOperand(2).ChangeToImmediate(0);
4284 MI.setDesc(get(RISCV::ADDI));
4285 return true;
4286 }
4287 // shift rd, rs, zero => addi rd, rs, 0
4288 if (MI.getOperand(2).getReg() == RISCV::X0) {
4289 MI.getOperand(2).ChangeToImmediate(0);
4290 MI.setDesc(get(RISCV::ADDI));
4291 return true;
4292 }
4293 break;
4294 case RISCV::SLLW:
4295 case RISCV::SRLW:
4296 case RISCV::SRAW:
4297 // shiftw rd, zero, rs => addi rd, zero, 0
4298 if (MI.getOperand(1).getReg() == RISCV::X0) {
4299 MI.getOperand(2).ChangeToImmediate(0);
4300 MI.setDesc(get(RISCV::ADDI));
4301 return true;
4302 }
4303 break;
4304 case RISCV::SLLI:
4305 case RISCV::SRLI:
4306 case RISCV::SRAI:
4307 case RISCV::SLLIW:
4308 case RISCV::SRLIW:
4309 case RISCV::SRAIW:
4310 case RISCV::SLLI_UW:
4311 // shiftimm rd, zero, N => addi rd, zero, 0
4312 if (MI.getOperand(1).getReg() == RISCV::X0) {
4313 MI.getOperand(2).setImm(0);
4314 MI.setDesc(get(RISCV::ADDI));
4315 return true;
4316 }
4317 break;
4318 case RISCV::SLTU:
4319 case RISCV::ADD_UW:
4320 // sltu rd, zero, zero => addi rd, zero, 0
4321 // add.uw rd, zero, zero => addi rd, zero, 0
4322 if (MI.getOperand(1).getReg() == RISCV::X0 &&
4323 MI.getOperand(2).getReg() == RISCV::X0) {
4324 MI.getOperand(2).ChangeToImmediate(0);
4325 MI.setDesc(get(RISCV::ADDI));
4326 return true;
4327 }
4328 // add.uw rd, zero, rs => addi rd, rs, 0
4329 if (MI.getOpcode() == RISCV::ADD_UW &&
4330 MI.getOperand(1).getReg() == RISCV::X0) {
4331 MI.removeOperand(1);
4332 MI.addOperand(MachineOperand::CreateImm(0));
4333 MI.setDesc(get(RISCV::ADDI));
4334 }
4335 break;
4336 case RISCV::SLTIU:
4337 // sltiu rd, zero, NZC => addi rd, zero, 1
4338 // sltiu rd, zero, 0 => addi rd, zero, 0
4339 if (MI.getOperand(1).getReg() == RISCV::X0) {
4340 MI.getOperand(2).setImm(MI.getOperand(2).getImm() != 0);
4341 MI.setDesc(get(RISCV::ADDI));
4342 return true;
4343 }
4344 break;
4345 case RISCV::SEXT_H:
4346 case RISCV::SEXT_B:
4347 case RISCV::ZEXT_H_RV32:
4348 case RISCV::ZEXT_H_RV64:
4349 // sext.[hb] rd, zero => addi rd, zero, 0
4350 // zext.h rd, zero => addi rd, zero, 0
4351 if (MI.getOperand(1).getReg() == RISCV::X0) {
4352 MI.addOperand(MachineOperand::CreateImm(0));
4353 MI.setDesc(get(RISCV::ADDI));
4354 return true;
4355 }
4356 break;
4357 case RISCV::MIN:
4358 case RISCV::MINU:
4359 case RISCV::MAX:
4360 case RISCV::MAXU:
4361 // min|max rd, rs, rs => addi rd, rs, 0
4362 if (MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) {
4363 MI.getOperand(2).ChangeToImmediate(0);
4364 MI.setDesc(get(RISCV::ADDI));
4365 return true;
4366 }
4367 break;
4368 case RISCV::BEQ:
4369 case RISCV::BNE:
4370 // b{eq,ne} zero, rs, imm => b{eq,ne} rs, zero, imm
4371 if (MI.getOperand(0).getReg() == RISCV::X0) {
4372 MachineOperand MO0 = MI.getOperand(0);
4373 MI.removeOperand(0);
4374 MI.insert(MI.operands_begin() + 1, {MO0});
4375 }
4376 break;
4377 case RISCV::BLTU:
4378 // bltu zero, rs, imm => bne rs, zero, imm
4379 if (MI.getOperand(0).getReg() == RISCV::X0) {
4380 MachineOperand MO0 = MI.getOperand(0);
4381 MI.removeOperand(0);
4382 MI.insert(MI.operands_begin() + 1, {MO0});
4383 MI.setDesc(get(RISCV::BNE));
4384 }
4385 break;
4386 case RISCV::BGEU:
4387 // bgeu zero, rs, imm => beq rs, zero, imm
4388 if (MI.getOperand(0).getReg() == RISCV::X0) {
4389 MachineOperand MO0 = MI.getOperand(0);
4390 MI.removeOperand(0);
4391 MI.insert(MI.operands_begin() + 1, {MO0});
4392 MI.setDesc(get(RISCV::BEQ));
4393 }
4394 break;
4395 }
4396 return false;
4397}
4398
4399// clang-format off
4400#define CASE_WIDEOP_OPCODE_COMMON(OP, LMUL) \
4401 RISCV::PseudoV##OP##_##LMUL##_TIED
4402
4403#define CASE_WIDEOP_OPCODE_LMULS(OP) \
4404 CASE_WIDEOP_OPCODE_COMMON(OP, MF8): \
4405 case CASE_WIDEOP_OPCODE_COMMON(OP, MF4): \
4406 case CASE_WIDEOP_OPCODE_COMMON(OP, MF2): \
4407 case CASE_WIDEOP_OPCODE_COMMON(OP, M1): \
4408 case CASE_WIDEOP_OPCODE_COMMON(OP, M2): \
4409 case CASE_WIDEOP_OPCODE_COMMON(OP, M4)
4410
4411#define CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL) \
4412 case RISCV::PseudoV##OP##_##LMUL##_TIED: \
4413 NewOpc = RISCV::PseudoV##OP##_##LMUL; \
4414 break;
4415
4416#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
4417 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF8) \
4418 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4) \
4419 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2) \
4420 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1) \
4421 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2) \
4422 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4)
4423
4424// FP Widening Ops may by SEW aware. Create SEW aware cases for these cases.
4425#define CASE_FP_WIDEOP_OPCODE_COMMON(OP, LMUL, SEW) \
4426 RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED
4427
4428#define CASE_FP_WIDEOP_OPCODE_LMULS(OP) \
4429 CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16): \
4430 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16): \
4431 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E32): \
4432 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E16): \
4433 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E32): \
4434 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E16): \
4435 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E32): \
4436 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E16): \
4437 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E32) \
4438
4439#define CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL, SEW) \
4440 case RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED: \
4441 NewOpc = RISCV::PseudoV##OP##_##LMUL##_##SEW; \
4442 break;
4443
4444#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
4445 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16) \
4446 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16) \
4447 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E32) \
4448 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E16) \
4449 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E32) \
4450 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E16) \
4451 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E32) \
4452 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16) \
4453 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E32) \
4454// clang-format on
4455
4457 LiveVariables *LV,
4458 LiveIntervals *LIS) const {
4460 switch (MI.getOpcode()) {
4461 default:
4462 return nullptr;
4463 case CASE_FP_WIDEOP_OPCODE_LMULS(FWADD_WV):
4464 case CASE_FP_WIDEOP_OPCODE_LMULS(FWSUB_WV): {
4465 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
4466 MI.getNumExplicitOperands() == 7 &&
4467 "Expect 7 explicit operands rd, rs2, rs1, rm, vl, sew, policy");
4468 // If the tail policy is undisturbed we can't convert.
4469 if ((MI.getOperand(RISCVII::getVecPolicyOpNum(MI.getDesc())).getImm() &
4470 1) == 0)
4471 return nullptr;
4472 // clang-format off
4473 unsigned NewOpc;
4474 switch (MI.getOpcode()) {
4475 default:
4476 llvm_unreachable("Unexpected opcode");
4479 }
4480 // clang-format on
4481
4482 MachineBasicBlock &MBB = *MI.getParent();
4483 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
4484 .add(MI.getOperand(0))
4485 .addReg(MI.getOperand(0).getReg(), RegState::Undef)
4486 .add(MI.getOperand(1))
4487 .add(MI.getOperand(2))
4488 .add(MI.getOperand(3))
4489 .add(MI.getOperand(4))
4490 .add(MI.getOperand(5))
4491 .add(MI.getOperand(6));
4492 break;
4493 }
4494 case CASE_WIDEOP_OPCODE_LMULS(WADD_WV):
4495 case CASE_WIDEOP_OPCODE_LMULS(WADDU_WV):
4496 case CASE_WIDEOP_OPCODE_LMULS(WSUB_WV):
4497 case CASE_WIDEOP_OPCODE_LMULS(WSUBU_WV): {
4498 // If the tail policy is undisturbed we can't convert.
4499 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
4500 MI.getNumExplicitOperands() == 6);
4501 if ((MI.getOperand(RISCVII::getVecPolicyOpNum(MI.getDesc())).getImm() &
4502 1) == 0)
4503 return nullptr;
4504
4505 // clang-format off
4506 unsigned NewOpc;
4507 switch (MI.getOpcode()) {
4508 default:
4509 llvm_unreachable("Unexpected opcode");
4514 }
4515 // clang-format on
4516
4517 MachineBasicBlock &MBB = *MI.getParent();
4518 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
4519 .add(MI.getOperand(0))
4520 .addReg(MI.getOperand(0).getReg(), RegState::Undef)
4521 .add(MI.getOperand(1))
4522 .add(MI.getOperand(2))
4523 .add(MI.getOperand(3))
4524 .add(MI.getOperand(4))
4525 .add(MI.getOperand(5));
4526 break;
4527 }
4528 }
4529 MIB.copyImplicitOps(MI);
4530
4531 if (LV) {
4532 unsigned NumOps = MI.getNumOperands();
4533 for (unsigned I = 1; I < NumOps; ++I) {
4534 MachineOperand &Op = MI.getOperand(I);
4535 if (Op.isReg() && Op.isKill())
4536 LV->replaceKillInstruction(Op.getReg(), MI, *MIB);
4537 }
4538 }
4539
4540 if (LIS) {
4541 SlotIndex Idx = LIS->ReplaceMachineInstrInMaps(MI, *MIB);
4542
4543 if (MI.getOperand(0).isEarlyClobber()) {
4544 // Use operand 1 was tied to early-clobber def operand 0, so its live
4545 // interval could have ended at an early-clobber slot. Now they are not
4546 // tied we need to update it to the normal register slot.
4547 LiveInterval &LI = LIS->getInterval(MI.getOperand(1).getReg());
4549 if (S->end == Idx.getRegSlot(true))
4550 S->end = Idx.getRegSlot();
4551 }
4552 }
4553
4554 return MIB;
4555}
4556
4557#undef CASE_WIDEOP_OPCODE_COMMON
4558#undef CASE_WIDEOP_OPCODE_LMULS
4559#undef CASE_WIDEOP_CHANGE_OPCODE_COMMON
4560#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS
4561#undef CASE_FP_WIDEOP_OPCODE_COMMON
4562#undef CASE_FP_WIDEOP_OPCODE_LMULS
4563#undef CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON
4564#undef CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS
4565
4568 Register DestReg, uint32_t Amount,
4569 MachineInstr::MIFlag Flag) const {
4571 if (llvm::has_single_bit<uint32_t>(Amount)) {
4572 uint32_t ShiftAmount = Log2_32(Amount);
4573 if (ShiftAmount == 0)
4574 return;
4575 BuildMI(MBB, II, DL, get(RISCV::SLLI), DestReg)
4576 .addReg(DestReg, RegState::Kill)
4577 .addImm(ShiftAmount)
4578 .setMIFlag(Flag);
4579 } else if (STI.hasShlAdd(3) &&
4580 ((Amount % 3 == 0 && isPowerOf2_64(Amount / 3)) ||
4581 (Amount % 5 == 0 && isPowerOf2_64(Amount / 5)) ||
4582 (Amount % 9 == 0 && isPowerOf2_64(Amount / 9)))) {
4583 // We can use Zba SHXADD+SLLI instructions for multiply in some cases.
4584 unsigned Opc;
4585 uint32_t ShiftAmount;
4586 if (Amount % 9 == 0) {
4587 Opc = RISCV::SH3ADD;
4588 ShiftAmount = Log2_64(Amount / 9);
4589 } else if (Amount % 5 == 0) {
4590 Opc = RISCV::SH2ADD;
4591 ShiftAmount = Log2_64(Amount / 5);
4592 } else if (Amount % 3 == 0) {
4593 Opc = RISCV::SH1ADD;
4594 ShiftAmount = Log2_64(Amount / 3);
4595 } else {
4596 llvm_unreachable("implied by if-clause");
4597 }
4598 if (ShiftAmount)
4599 BuildMI(MBB, II, DL, get(RISCV::SLLI), DestReg)
4600 .addReg(DestReg, RegState::Kill)
4601 .addImm(ShiftAmount)
4602 .setMIFlag(Flag);
4603 BuildMI(MBB, II, DL, get(Opc), DestReg)
4604 .addReg(DestReg, RegState::Kill)
4605 .addReg(DestReg)
4606 .setMIFlag(Flag);
4607 } else if (llvm::has_single_bit<uint32_t>(Amount - 1)) {
4608 Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass);
4609 uint32_t ShiftAmount = Log2_32(Amount - 1);
4610 BuildMI(MBB, II, DL, get(RISCV::SLLI), ScaledRegister)
4611 .addReg(DestReg)
4612 .addImm(ShiftAmount)
4613 .setMIFlag(Flag);
4614 BuildMI(MBB, II, DL, get(RISCV::ADD), DestReg)
4615 .addReg(ScaledRegister, RegState::Kill)
4616 .addReg(DestReg, RegState::Kill)
4617 .setMIFlag(Flag);
4618 } else if (llvm::has_single_bit<uint32_t>(Amount + 1)) {
4619 Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass);
4620 uint32_t ShiftAmount = Log2_32(Amount + 1);
4621 BuildMI(MBB, II, DL, get(RISCV::SLLI), ScaledRegister)
4622 .addReg(DestReg)
4623 .addImm(ShiftAmount)
4624 .setMIFlag(Flag);
4625 BuildMI(MBB, II, DL, get(RISCV::SUB), DestReg)
4626 .addReg(ScaledRegister, RegState::Kill)
4627 .addReg(DestReg, RegState::Kill)
4628 .setMIFlag(Flag);
4629 } else if (STI.hasStdExtZmmul()) {
4630 Register N = MRI.createVirtualRegister(&RISCV::GPRRegClass);
4631 movImm(MBB, II, DL, N, Amount, Flag);
4632 BuildMI(MBB, II, DL, get(RISCV::MUL), DestReg)
4633 .addReg(DestReg, RegState::Kill)
4635 .setMIFlag(Flag);
4636 } else {
4637 Register Acc;
4638 uint32_t PrevShiftAmount = 0;
4639 for (uint32_t ShiftAmount = 0; Amount >> ShiftAmount; ShiftAmount++) {
4640 if (Amount & (1U << ShiftAmount)) {
4641 if (ShiftAmount)
4642 BuildMI(MBB, II, DL, get(RISCV::SLLI), DestReg)
4643 .addReg(DestReg, RegState::Kill)
4644 .addImm(ShiftAmount - PrevShiftAmount)
4645 .setMIFlag(Flag);
4646 if (Amount >> (ShiftAmount + 1)) {
4647 // If we don't have an accmulator yet, create it and copy DestReg.
4648 if (!Acc) {
4649 Acc = MRI.createVirtualRegister(&RISCV::GPRRegClass);
4650 BuildMI(MBB, II, DL, get(TargetOpcode::COPY), Acc)
4651 .addReg(DestReg)
4652 .setMIFlag(Flag);
4653 } else {
4654 BuildMI(MBB, II, DL, get(RISCV::ADD), Acc)
4655 .addReg(Acc, RegState::Kill)
4656 .addReg(DestReg)
4657 .setMIFlag(Flag);
4658 }
4659 }
4660 PrevShiftAmount = ShiftAmount;
4661 }
4662 }
4663 assert(Acc && "Expected valid accumulator");
4664 BuildMI(MBB, II, DL, get(RISCV::ADD), DestReg)
4665 .addReg(DestReg, RegState::Kill)
4666 .addReg(Acc, RegState::Kill)
4667 .setMIFlag(Flag);
4668 }
4669}
4670
4673 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
4674 {{MONontemporalBit0, "riscv-nontemporal-domain-bit-0"},
4675 {MONontemporalBit1, "riscv-nontemporal-domain-bit-1"}};
4676 return ArrayRef(TargetFlags);
4677}
4678
4680 return OptLevel >= CodeGenOptLevel::Aggressive
4681 ? STI.getTailDupAggressiveThreshold()
4682 : 2;
4683}
4684
4686 // RVV lacks any support for immediate addressing for stack addresses, so be
4687 // conservative.
4688 unsigned Opcode = MI.getOpcode();
4689 if (!RISCVVPseudosTable::getPseudoInfo(Opcode) &&
4691 return false;
4692 return true;
4693}
4694
4695std::optional<std::pair<unsigned, unsigned>>
4697 switch (Opcode) {
4698 default:
4699 return std::nullopt;
4700 case RISCV::PseudoVSPILL2_M1:
4701 case RISCV::PseudoVRELOAD2_M1:
4702 return std::make_pair(2u, 1u);
4703 case RISCV::PseudoVSPILL2_M2:
4704 case RISCV::PseudoVRELOAD2_M2:
4705 return std::make_pair(2u, 2u);
4706 case RISCV::PseudoVSPILL2_M4:
4707 case RISCV::PseudoVRELOAD2_M4:
4708 return std::make_pair(2u, 4u);
4709 case RISCV::PseudoVSPILL3_M1:
4710 case RISCV::PseudoVRELOAD3_M1:
4711 return std::make_pair(3u, 1u);
4712 case RISCV::PseudoVSPILL3_M2:
4713 case RISCV::PseudoVRELOAD3_M2:
4714 return std::make_pair(3u, 2u);
4715 case RISCV::PseudoVSPILL4_M1:
4716 case RISCV::PseudoVRELOAD4_M1:
4717 return std::make_pair(4u, 1u);
4718 case RISCV::PseudoVSPILL4_M2:
4719 case RISCV::PseudoVRELOAD4_M2:
4720 return std::make_pair(4u, 2u);
4721 case RISCV::PseudoVSPILL5_M1:
4722 case RISCV::PseudoVRELOAD5_M1:
4723 return std::make_pair(5u, 1u);
4724 case RISCV::PseudoVSPILL6_M1:
4725 case RISCV::PseudoVRELOAD6_M1:
4726 return std::make_pair(6u, 1u);
4727 case RISCV::PseudoVSPILL7_M1:
4728 case RISCV::PseudoVRELOAD7_M1:
4729 return std::make_pair(7u, 1u);
4730 case RISCV::PseudoVSPILL8_M1:
4731 case RISCV::PseudoVRELOAD8_M1:
4732 return std::make_pair(8u, 1u);
4733 }
4734}
4735
4736bool RISCV::hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2) {
4737 int16_t MI1FrmOpIdx =
4738 RISCV::getNamedOperandIdx(MI1.getOpcode(), RISCV::OpName::frm);
4739 int16_t MI2FrmOpIdx =
4740 RISCV::getNamedOperandIdx(MI2.getOpcode(), RISCV::OpName::frm);
4741 if (MI1FrmOpIdx < 0 || MI2FrmOpIdx < 0)
4742 return false;
4743 MachineOperand FrmOp1 = MI1.getOperand(MI1FrmOpIdx);
4744 MachineOperand FrmOp2 = MI2.getOperand(MI2FrmOpIdx);
4745 return FrmOp1.getImm() == FrmOp2.getImm();
4746}
4747
4748std::optional<unsigned>
4749RISCV::getVectorLowDemandedScalarBits(unsigned Opcode, unsigned Log2SEW) {
4750 switch (Opcode) {
4751 default:
4752 return std::nullopt;
4753
4754 // 11.6. Vector Single-Width Shift Instructions
4755 case RISCV::VSLL_VX:
4756 case RISCV::VSRL_VX:
4757 case RISCV::VSRA_VX:
4758 // 12.4. Vector Single-Width Scaling Shift Instructions
4759 case RISCV::VSSRL_VX:
4760 case RISCV::VSSRA_VX:
4761 // Zvbb
4762 case RISCV::VROL_VX:
4763 case RISCV::VROR_VX:
4764 // Only the low lg2(SEW) bits of the shift-amount value are used.
4765 return Log2SEW;
4766
4767 // 11.7 Vector Narrowing Integer Right Shift Instructions
4768 case RISCV::VNSRL_WX:
4769 case RISCV::VNSRA_WX:
4770 // 12.5. Vector Narrowing Fixed-Point Clip Instructions
4771 case RISCV::VNCLIPU_WX:
4772 case RISCV::VNCLIP_WX:
4773 // Zvbb
4774 case RISCV::VWSLL_VX:
4775 // Only the low lg2(2*SEW) bits of the shift-amount value are used.
4776 return Log2SEW + 1;
4777
4778 // 11.1. Vector Single-Width Integer Add and Subtract
4779 case RISCV::VADD_VX:
4780 case RISCV::VSUB_VX:
4781 case RISCV::VRSUB_VX:
4782 // 11.2. Vector Widening Integer Add/Subtract
4783 case RISCV::VWADDU_VX:
4784 case RISCV::VWSUBU_VX:
4785 case RISCV::VWADD_VX:
4786 case RISCV::VWSUB_VX:
4787 case RISCV::VWADDU_WX:
4788 case RISCV::VWSUBU_WX:
4789 case RISCV::VWADD_WX:
4790 case RISCV::VWSUB_WX:
4791 // 11.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
4792 case RISCV::VADC_VXM:
4793 case RISCV::VADC_VIM:
4794 case RISCV::VMADC_VXM:
4795 case RISCV::VMADC_VIM:
4796 case RISCV::VMADC_VX:
4797 case RISCV::VSBC_VXM:
4798 case RISCV::VMSBC_VXM:
4799 case RISCV::VMSBC_VX:
4800 // 11.5 Vector Bitwise Logical Instructions
4801 case RISCV::VAND_VX:
4802 case RISCV::VOR_VX:
4803 case RISCV::VXOR_VX:
4804 // 11.8. Vector Integer Compare Instructions
4805 case RISCV::VMSEQ_VX:
4806 case RISCV::VMSNE_VX:
4807 case RISCV::VMSLTU_VX:
4808 case RISCV::VMSLT_VX:
4809 case RISCV::VMSLEU_VX:
4810 case RISCV::VMSLE_VX:
4811 case RISCV::VMSGTU_VX:
4812 case RISCV::VMSGT_VX:
4813 // 11.9. Vector Integer Min/Max Instructions
4814 case RISCV::VMINU_VX:
4815 case RISCV::VMIN_VX:
4816 case RISCV::VMAXU_VX:
4817 case RISCV::VMAX_VX:
4818 // 11.10. Vector Single-Width Integer Multiply Instructions
4819 case RISCV::VMUL_VX:
4820 case RISCV::VMULH_VX:
4821 case RISCV::VMULHU_VX:
4822 case RISCV::VMULHSU_VX:
4823 // 11.11. Vector Integer Divide Instructions
4824 case RISCV::VDIVU_VX:
4825 case RISCV::VDIV_VX:
4826 case RISCV::VREMU_VX:
4827 case RISCV::VREM_VX:
4828 // 11.12. Vector Widening Integer Multiply Instructions
4829 case RISCV::VWMUL_VX:
4830 case RISCV::VWMULU_VX:
4831 case RISCV::VWMULSU_VX:
4832 // 11.13. Vector Single-Width Integer Multiply-Add Instructions
4833 case RISCV::VMACC_VX:
4834 case RISCV::VNMSAC_VX:
4835 case RISCV::VMADD_VX:
4836 case RISCV::VNMSUB_VX:
4837 // 11.14. Vector Widening Integer Multiply-Add Instructions
4838 case RISCV::VWMACCU_VX:
4839 case RISCV::VWMACC_VX:
4840 case RISCV::VWMACCSU_VX:
4841 case RISCV::VWMACCUS_VX:
4842 // 11.15. Vector Integer Merge Instructions
4843 case RISCV::VMERGE_VXM:
4844 // 11.16. Vector Integer Move Instructions
4845 case RISCV::VMV_V_X:
4846 // 12.1. Vector Single-Width Saturating Add and Subtract
4847 case RISCV::VSADDU_VX:
4848 case RISCV::VSADD_VX:
4849 case RISCV::VSSUBU_VX:
4850 case RISCV::VSSUB_VX:
4851 // 12.2. Vector Single-Width Averaging Add and Subtract
4852 case RISCV::VAADDU_VX:
4853 case RISCV::VAADD_VX:
4854 case RISCV::VASUBU_VX:
4855 case RISCV::VASUB_VX:
4856 // 12.3. Vector Single-Width Fractional Multiply with Rounding and Saturation
4857 case RISCV::VSMUL_VX:
4858 // 16.1. Integer Scalar Move Instructions
4859 case RISCV::VMV_S_X:
4860 // Zvbb
4861 case RISCV::VANDN_VX:
4862 return 1U << Log2SEW;
4863 }
4864}
4865
4866unsigned RISCV::getRVVMCOpcode(unsigned RVVPseudoOpcode) {
4868 RISCVVPseudosTable::getPseudoInfo(RVVPseudoOpcode);
4869 if (!RVV)
4870 return 0;
4871 return RVV->BaseInstr;
4872}
4873
4874unsigned RISCV::getDestLog2EEW(const MCInstrDesc &Desc, unsigned Log2SEW) {
4875 unsigned DestEEW =
4877 // EEW = 1
4878 if (DestEEW == 0)
4879 return 0;
4880 // EEW = SEW * n
4881 unsigned Scaled = Log2SEW + (DestEEW - 1);
4882 assert(Scaled >= 3 && Scaled <= 6);
4883 return Scaled;
4884}
4885
4886static std::optional<int64_t> getEffectiveImm(const MachineOperand &MO) {
4887 assert(MO.isImm() || MO.getReg().isVirtual());
4888 if (MO.isImm())
4889 return MO.getImm();
4890 const MachineInstr *Def =
4891 MO.getParent()->getMF()->getRegInfo().getVRegDef(MO.getReg());
4892 int64_t Imm;
4893 if (isLoadImm(Def, Imm))
4894 return Imm;
4895 return std::nullopt;
4896}
4897
4898/// Given two VL operands, do we know that LHS <= RHS? Must be used in SSA form.
4900 assert((LHS.isImm() || LHS.getParent()->getMF()->getRegInfo().isSSA()) &&
4901 (RHS.isImm() || RHS.getParent()->getMF()->getRegInfo().isSSA()));
4902 if (LHS.isReg() && RHS.isReg() && LHS.getReg().isVirtual() &&
4903 LHS.getReg() == RHS.getReg())
4904 return true;
4905 if (RHS.isImm() && RHS.getImm() == RISCV::VLMaxSentinel)
4906 return true;
4907 if (LHS.isImm() && LHS.getImm() == 0)
4908 return true;
4909 if (LHS.isImm() && LHS.getImm() == RISCV::VLMaxSentinel)
4910 return false;
4911 std::optional<int64_t> LHSImm = getEffectiveImm(LHS),
4912 RHSImm = getEffectiveImm(RHS);
4913 if (!LHSImm || !RHSImm)
4914 return false;
4915 return LHSImm <= RHSImm;
4916}
4917
4918namespace {
4919class RISCVPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
4920 const MachineInstr *LHS;
4921 const MachineInstr *RHS;
4923
4924public:
4925 RISCVPipelinerLoopInfo(const MachineInstr *LHS, const MachineInstr *RHS,
4927 : LHS(LHS), RHS(RHS), Cond(Cond.begin(), Cond.end()) {}
4928
4929 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
4930 // Make the instructions for loop control be placed in stage 0.
4931 // The predecessors of LHS/RHS are considered by the caller.
4932 if (LHS && MI == LHS)
4933 return true;
4934 if (RHS && MI == RHS)
4935 return true;
4936 return false;
4937 }
4938
4939 std::optional<bool> createTripCountGreaterCondition(
4940 int TC, MachineBasicBlock &MBB,
4941 SmallVectorImpl<MachineOperand> &CondParam) override {
4942 // A branch instruction will be inserted as "if (Cond) goto epilogue".
4943 // Cond is normalized for such use.
4944 // The predecessors of the branch are assumed to have already been inserted.
4945 CondParam = Cond;
4946 return {};
4947 }
4948
4949 void setPreheader(MachineBasicBlock *NewPreheader) override {}
4950
4951 void adjustTripCount(int TripCountAdjust) override {}
4952};
4953} // namespace
4954
4955std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
4957 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
4959 if (analyzeBranch(*LoopBB, TBB, FBB, Cond, /*AllowModify=*/false))
4960 return nullptr;
4961
4962 // Infinite loops are not supported
4963 if (TBB == LoopBB && FBB == LoopBB)
4964 return nullptr;
4965
4966 // Must be conditional branch
4967 if (FBB == nullptr)
4968 return nullptr;
4969
4970 assert((TBB == LoopBB || FBB == LoopBB) &&
4971 "The Loop must be a single-basic-block loop");
4972
4973 // Normalization for createTripCountGreaterCondition()
4974 if (TBB == LoopBB)
4976
4977 const MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo();
4978 auto FindRegDef = [&MRI](MachineOperand &Op) -> const MachineInstr * {
4979 if (!Op.isReg())
4980 return nullptr;
4981 Register Reg = Op.getReg();
4982 if (!Reg.isVirtual())
4983 return nullptr;
4984 return MRI.getVRegDef(Reg);
4985 };
4986
4987 const MachineInstr *LHS = FindRegDef(Cond[1]);
4988 const MachineInstr *RHS = FindRegDef(Cond[2]);
4989 if (LHS && LHS->isPHI())
4990 return nullptr;
4991 if (RHS && RHS->isPHI())
4992 return nullptr;
4993
4994 return std::make_unique<RISCVPipelinerLoopInfo>(LHS, RHS, Cond);
4995}
4996
4997// FIXME: We should remove this if we have a default generic scheduling model.
4999 unsigned RVVMCOpcode = RISCV::getRVVMCOpcode(Opc);
5000 Opc = RVVMCOpcode ? RVVMCOpcode : Opc;
5001 switch (Opc) {
5002 default:
5003 return false;
5004 // Integer div/rem.
5005 case RISCV::DIV:
5006 case RISCV::DIVW:
5007 case RISCV::DIVU:
5008 case RISCV::DIVUW:
5009 case RISCV::REM:
5010 case RISCV::REMW:
5011 case RISCV::REMU:
5012 case RISCV::REMUW:
5013 // Floating-point div/sqrt.
5014 case RISCV::FDIV_H:
5015 case RISCV::FDIV_S:
5016 case RISCV::FDIV_D:
5017 case RISCV::FDIV_H_INX:
5018 case RISCV::FDIV_S_INX:
5019 case RISCV::FDIV_D_INX:
5020 case RISCV::FDIV_D_IN32X:
5021 case RISCV::FSQRT_H:
5022 case RISCV::FSQRT_S:
5023 case RISCV::FSQRT_D:
5024 case RISCV::FSQRT_H_INX:
5025 case RISCV::FSQRT_S_INX:
5026 case RISCV::FSQRT_D_INX:
5027 case RISCV::FSQRT_D_IN32X:
5028 // Vector integer div/rem
5029 case RISCV::VDIV_VV:
5030 case RISCV::VDIV_VX:
5031 case RISCV::VDIVU_VV:
5032 case RISCV::VDIVU_VX:
5033 case RISCV::VREM_VV:
5034 case RISCV::VREM_VX:
5035 case RISCV::VREMU_VV:
5036 case RISCV::VREMU_VX:
5037 // Vector floating-point div/sqrt.
5038 case RISCV::VFDIV_VV:
5039 case RISCV::VFDIV_VF:
5040 case RISCV::VFRDIV_VF:
5041 case RISCV::VFSQRT_V:
5042 case RISCV::VFRSQRT7_V:
5043 return true;
5044 }
5045}
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder MachineInstrBuilder & DefMI
static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg, unsigned NumRegs)
static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target, SmallVectorImpl< MachineOperand > &Cond)
@ MachineOutlinerTailCall
Emit a save, restore, call, and return.
@ MachineOutlinerDefault
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
SmallVector< int16_t, MAX_SRC_OPERANDS_NUM > OperandIndices
@ Scaled
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
basic Basic Alias true
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
const HexagonInstrInfo * TII
#define _
IRTranslator LLVM IR MI
Module.h This file contains the declarations for the Module class.
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define F(x, y, z)
Definition MD5.cpp:55
#define I(x, y, z)
Definition MD5.cpp:58
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
This file provides utility analysis objects describing memory locations.
MachineInstr unsigned OpIdx
uint64_t IntrinsicInst * II
static bool cannotInsertTailCall(const MachineBasicBlock &MBB)
#define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP)
#define CASE_FP_WIDEOP_OPCODE_LMULS(OP)
#define CASE_OPERAND_SIMM(NUM)
static std::optional< unsigned > getLMULForRVVWholeLoadStore(unsigned Opcode)
#define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP)
static bool analyzeCandidate(outliner::Candidate &C)
static unsigned getFPFusedMultiplyOpcode(unsigned RootOpc, unsigned Pattern)
std::optional< unsigned > getFoldedOpcode(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, const RISCVSubtarget &ST)
#define RVV_OPC_LMUL_CASE(OPC, INV)
#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(OP)
static void combineFPFusedMultiply(MachineInstr &Root, MachineInstr &Prev, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs)
static unsigned getAddendOperandIdx(unsigned Pattern)
#define CASE_RVV_OPCODE_UNMASK(OP)
#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP)
static cl::opt< bool > PreferWholeRegisterMove("riscv-prefer-whole-register-move", cl::init(false), cl::Hidden, cl::desc("Prefer whole register move for vector registers."))
#define CASE_VFMA_SPLATS(OP)
unsigned getPredicatedOpcode(unsigned Opcode)
#define CASE_WIDEOP_OPCODE_LMULS(OP)
static bool isMIReadsReg(const MachineInstr &MI, const TargetRegisterInfo *TRI, MCRegister RegNo)
#define OPCODE_LMUL_MASK_CASE(OPC)
static bool isFSUB(unsigned Opc)
#define CASE_VMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE)
#define CASE_RVV_OPCODE(OP)
static std::optional< int64_t > getEffectiveImm(const MachineOperand &MO)
#define CASE_VFMA_OPCODE_VV(OP)
MachineOutlinerConstructionID
#define CASE_RVV_OPCODE_WIDEN(OP)
static unsigned getSHXADDUWShiftAmount(unsigned Opc)
#define CASE_VMA_OPCODE_LMULS(OP, TYPE)
static bool isConvertibleToVMV_V_V(const RISCVSubtarget &STI, const MachineBasicBlock &MBB, MachineBasicBlock::const_iterator MBBI, MachineBasicBlock::const_iterator &DefMBBI, RISCVVType::VLMUL LMul)
static bool isFMUL(unsigned Opc)
static unsigned getInverseXqcicmOpcode(unsigned Opcode)
static bool getFPPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce)
#define OPCODE_LMUL_CASE(OPC)
#define CASE_OPERAND_UIMM(NUM)
static bool canCombineShiftIntoShXAdd(const MachineBasicBlock &MBB, const MachineOperand &MO, unsigned OuterShiftAmt)
Utility routine that checks if.
static bool isCandidatePatchable(const MachineBasicBlock &MBB)
static bool isFADD(unsigned Opc)
static void genShXAddAddShift(MachineInstr &Root, unsigned AddOpIdx, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg)
static bool isLoadImm(const MachineInstr *MI, int64_t &Imm)
static bool isMIModifiesReg(const MachineInstr &MI, const TargetRegisterInfo *TRI, MCRegister RegNo)
static MachineInstr * canFoldAsPredicatedOp(Register Reg, const MachineRegisterInfo &MRI, const TargetInstrInfo *TII)
Identify instructions that can be folded into a CCMOV instruction, and return the defining instructio...
static bool canCombineFPFusedMultiply(const MachineInstr &Root, const MachineOperand &MO, bool DoRegPressureReduce)
static bool getSHXADDPatterns(const MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static bool getFPFusedMultiplyPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce)
static cl::opt< MachineTraceStrategy > ForceMachineCombinerStrategy("riscv-force-machine-combiner-strategy", cl::Hidden, cl::desc("Force machine combiner to use a specific strategy for machine " "trace metrics evaluation."), cl::init(MachineTraceStrategy::TS_NumStrategies), cl::values(clEnumValN(MachineTraceStrategy::TS_Local, "local", "Local strategy."), clEnumValN(MachineTraceStrategy::TS_MinInstrCount, "min-instr", "MinInstrCount strategy.")))
static unsigned getSHXADDShiftAmount(unsigned Opc)
#define CASE_RVV_OPCODE_MASK(OP)
#define RVV_OPC_LMUL_MASK_CASE(OPC, INV)
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
This file declares the machine register scavenger class.
static bool memOpsHaveSameBasePtr(const MachineInstr &MI1, ArrayRef< const MachineOperand * > BaseOps1, const MachineInstr &MI2, ArrayRef< const MachineOperand * > BaseOps2)
This file contains some templates that are useful if you are working with the STL at all.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:480
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO, unsigned CombineOpc=0)
static cl::opt< unsigned > CacheLineSize("cache-line-size", cl::init(0), cl::Hidden, cl::desc("Use this to override the target cache line size when " "specified by the user."))
Value * RHS
Value * LHS
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:41
const T & front() const
front - Get the first element.
Definition ArrayRef.h:150
bool empty() const
empty - Check if the array is empty.
Definition ArrayRef.h:142
static LLVM_ABI DILocation * getMergedLocation(DILocation *LocA, DILocation *LocB)
Attempts to merge LocA and LocB into a single location; see DebugLoc::getMergedLocation for more deta...
bool isBigEndian() const
Definition DataLayout.h:208
A debug info location.
Definition DebugLoc.h:124
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition DenseMap.h:222
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition Function.h:703
LiveInterval - This class represents the liveness of a register, or stack slot.
LiveInterval & getInterval(Register Reg)
SlotIndex ReplaceMachineInstrInMaps(MachineInstr &MI, MachineInstr &NewMI)
const Segment * getSegmentContaining(SlotIndex Idx) const
Return the segment that contains the specified index, or null if there is none.
LLVM_ABI void replaceKillInstruction(Register Reg, MachineInstr &OldMI, MachineInstr &NewMI)
replaceKillInstruction - Update register kill info by replacing a kill instruction with a new one.
bool hasValue() const
static LocationSize precise(uint64_t Value)
TypeSize getValue() const
MCInstBuilder & addReg(MCRegister Reg)
Add a new register operand.
MCInstBuilder & addImm(int64_t Val)
Add a new integer immediate operand.
Instances of this class represent a single low-level machine instruction.
Definition MCInst.h:188
Describe properties that are true of each instruction in the target description file.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
bool isConditionalBranch() const
Return true if this is a branch which may fall through to the next instruction or may transfer contro...
This holds information about one operand of a machine instruction, indicating the register class for ...
Definition MCInstrDesc.h:87
Wrapper class representing physical registers. Should be passed by value.
Definition MCRegister.h:33
const FeatureBitset & getFeatureBits() const
Set of metadata that should be preserved when using BuildMI().
MachineInstrBundleIterator< const MachineInstr > const_iterator
MachineInstrBundleIterator< MachineInstr, true > reverse_iterator
Instructions::const_iterator const_instr_iterator
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
MachineInstrBundleIterator< const MachineInstr, true > const_reverse_iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
void setStackID(int ObjectIdx, uint8_t ID)
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & copyImplicitOps(const MachineInstr &OtherMI) const
Copy all the implicit operands from OtherMI onto this one.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
reverse_iterator getReverse() const
Get a reverse iterator to the same node.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool isReturn(QueryType Type=AnyInBundle) const
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
LLVM_ABI unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
bool modifiesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr modifies (fully define or partially define) the specified register.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
LLVM_ABI bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by mayLoad / mayStore,...
bool hasOneMemOperand() const
Return true if this instruction has exactly one MachineMemOperand.
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
LLVM_ABI bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
const MachineOperand & getOperand(unsigned i) const
uint32_t getFlags() const
Return the MI flags bitvector.
LLVM_ABI void clearKillInfo()
Clears kill flags on all operands.
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
This class contains meta information specific to a module.
MachineOperand class - Representation of each machine instruction operand.
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreateImm(int64_t Val)
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
LLVM_ABI bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
@ MO_Immediate
Immediate operand.
@ MO_Register
Register operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:67
MI-level patchpoint operands.
Definition StackMaps.h:77
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given patchpoint should emit.
Definition StackMaps.h:105
MachineInstr * convertToThreeAddress(MachineInstr &MI, LiveVariables *LV, LiveIntervals *LIS) const override
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
std::optional< std::unique_ptr< outliner::OutlinedFunction > > getOutliningCandidateInfo(const MachineModuleInfo &MMI, std::vector< outliner::Candidate > &RepeatedSequenceLocs, unsigned MinRepeats) const override
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg) const override
void movImm(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, Register DstReg, uint64_t Val, MachineInstr::MIFlag Flag=MachineInstr::NoFlags, bool DstRenamable=false, bool DstIsDead=false) const
MachineInstr * emitLdStWithAddr(MachineInstr &MemI, const ExtAddrMode &AM) const override
void mulImm(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator II, const DebugLoc &DL, Register DestReg, uint32_t Amt, MachineInstr::MIFlag Flag) const
Generate code to multiply the value in DestReg by Amt - handles all the common optimizations for this...
static bool isPairableLdStInstOpc(unsigned Opc)
Return true if pairing the given load or store may be paired with another.
RISCVInstrInfo(const RISCVSubtarget &STI)
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool IsKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
bool isFunctionSafeToOutlineFrom(MachineFunction &MF, bool OutlineFromLinkOnceODRs) const override
std::unique_ptr< TargetInstrInfo::PipelinerLoopInfo > analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &dl, int *BytesAdded=nullptr) const override
bool hasReassociableSibling(const MachineInstr &Inst, bool &Commuted) const override
static bool isLdStSafeToPair(const MachineInstr &LdSt, const TargetRegisterInfo *TRI)
void copyPhysRegVector(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc, const TargetRegisterClass *RegClass) const
bool isReMaterializableImpl(const MachineInstr &MI) const override
MachineInstr * optimizeSelect(MachineInstr &MI, SmallPtrSetImpl< MachineInstr * > &SeenMIs, bool) const override
bool canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg, const MachineInstr &AddrI, ExtAddrMode &AM) const override
void insertIndirectBranch(MachineBasicBlock &MBB, MachineBasicBlock &NewDestBB, MachineBasicBlock &RestoreBB, const DebugLoc &DL, int64_t BrOffset, RegScavenger *RS) const override
bool isAsCheapAsAMove(const MachineInstr &MI) const override
bool verifyInstruction(const MachineInstr &MI, StringRef &ErrInfo) const override
bool getMemOperandWithOffsetWidth(const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset, LocationSize &Width, const TargetRegisterInfo *TRI) const
unsigned getTailDuplicateSize(CodeGenOptLevel OptLevel) const override
void getReassociateOperandIndices(const MachineInstr &Root, unsigned Pattern, std::array< unsigned, 5 > &OperandIndices) const override
const RISCVSubtarget & STI
Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const override
std::optional< unsigned > getInverseOpcode(unsigned Opcode) const override
bool simplifyInstruction(MachineInstr &MI) const override
ArrayRef< std::pair< unsigned, const char * > > getSerializableDirectMachineOperandTargetFlags() const override
outliner::InstrType getOutliningTypeImpl(const MachineModuleInfo &MMI, MachineBasicBlock::iterator &MBBI, unsigned Flags) const override
MachineTraceStrategy getMachineCombinerTraceStrategy() const override
unsigned getInstSizeInBytes(const MachineInstr &MI) const override
std::optional< RegImmPair > isAddImmediate(const MachineInstr &MI, Register Reg) const override
bool reverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
ArrayRef< std::pair< MachineMemOperand::Flags, const char * > > getSerializableMachineMemOperandTargetFlags() const override
MCInst getNop() const override
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
bool isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, unsigned &Flags) const override
bool getMemOperandsWithOffsetWidth(const MachineInstr &MI, SmallVectorImpl< const MachineOperand * > &BaseOps, int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const override
void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF, const outliner::OutlinedFunction &OF) const override
void finalizeInsInstrs(MachineInstr &Root, unsigned &Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs) const override
std::pair< unsigned, unsigned > decomposeMachineOperandsTargetFlags(unsigned TF) const override
MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx1, unsigned OpIdx2) const override
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DstReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
bool hasReassociableOperands(const MachineInstr &Inst, const MachineBasicBlock *MBB) const override
MachineBasicBlock * getBranchDestBlock(const MachineInstr &MI) const override
std::string createMIROperandComment(const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx, const TargetRegisterInfo *TRI) const override
bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, Register DstReg, Register SrcReg, bool KillSrc, bool RenamableDest=false, bool RenamableSrc=false) const override
bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const override
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const override
MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, outliner::Candidate &C) const override
bool isBranchOffsetInRange(unsigned BranchOpc, int64_t BrOffset) const override
static RISCVCC::CondCode getCondFromBranchOpc(unsigned Opc)
bool isAssociativeAndCommutative(const MachineInstr &Inst, bool Invert) const override
CombinerObjective getCombinerObjective(unsigned Pattern) const override
bool isHighLatencyDef(int Opc) const override
static bool evaluateCondBranch(RISCVCC::CondCode CC, int64_t C0, int64_t C1)
Return the result of the evaluation of C0 CC C1, where CC is a RISCVCC::CondCode.
bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const override
bool optimizeCondBranch(MachineInstr &MI) const override
std::optional< DestSourcePair > isCopyInstrImpl(const MachineInstr &MI) const override
bool analyzeSelect(const MachineInstr &MI, SmallVectorImpl< MachineOperand > &Cond, unsigned &TrueOp, unsigned &FalseOp, bool &Optimizable) const override
static bool isFromLoadImm(const MachineRegisterInfo &MRI, const MachineOperand &Op, int64_t &Imm)
Return true if the operand is a load immediate instruction and sets Imm to the immediate value.
bool shouldClusterMemOps(ArrayRef< const MachineOperand * > BaseOps1, int64_t Offset1, bool OffsetIsScalable1, ArrayRef< const MachineOperand * > BaseOps2, int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize, unsigned NumBytes) const override
bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const override
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
const RISCVRegisterInfo * getRegisterInfo() const override
void enterBasicBlockEnd(MachineBasicBlock &MBB)
Start tracking liveness from the end of basic block MBB.
void setRegUsed(Register Reg, LaneBitmask LaneMask=LaneBitmask::getAll())
Tell the scavenger a register is used.
Register scavengeRegisterBackwards(const TargetRegisterClass &RC, MachineBasicBlock::iterator To, bool RestoreAfter, int SPAdj, bool AllowSpill=true)
Make a register of the specific register class available from the current position backwards to the p...
Wrapper class representing virtual and physical registers.
Definition Register.h:19
constexpr bool isValid() const
Definition Register.h:107
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition Register.h:74
SlotIndex - An opaque wrapper around machine indexes.
Definition SlotIndexes.h:66
SlotIndex getRegSlot(bool EC=false) const
Returns the register use/def slot in the current instruction for a normal or early-clobber def.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
Remove pointer from the set.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
MI-level stackmap operands.
Definition StackMaps.h:36
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given stackmap should emit.
Definition StackMaps.h:51
MI-level Statepoint operands.
Definition StackMaps.h:159
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given statepoint should emit.
Definition StackMaps.h:208
StringRef - Represent a constant reference to a string, i.e.
Definition StringRef.h:55
Object returned by analyzeLoopForPipelining.
TargetInstrInfo - Interface to description of machine instruction set.
virtual bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const
Returns true iff the routine could find two commutable operands in the given machine instruction.
virtual bool hasReassociableOperands(const MachineInstr &Inst, const MachineBasicBlock *MBB) const
Return true when \P Inst has reassociable operands in the same \P MBB.
virtual void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< Register, unsigned > &InstIdxForVirtReg) const
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
virtual bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
virtual bool isReMaterializableImpl(const MachineInstr &MI) const
For instructions with opcodes for which the M_REMATERIALIZABLE flag is set, this hook lets the target...
virtual bool isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, unsigned &Flags) const
Optional target hook that returns true if MBB is safe to outline from, and returns any target-specifi...
virtual void getReassociateOperandIndices(const MachineInstr &Root, unsigned Pattern, std::array< unsigned, 5 > &OperandIndices) const
The returned array encodes the operand index for each parameter because the operands may be commuted;...
virtual CombinerObjective getCombinerObjective(unsigned Pattern) const
Return the objective of a combiner pattern.
virtual MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx1, unsigned OpIdx2) const
This method commutes the operands of the given machine instruction MI.
virtual bool hasReassociableSibling(const MachineInstr &Inst, bool &Commuted) const
Return true when \P Inst has reassociable sibling.
virtual std::string createMIROperandComment(const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx, const TargetRegisterInfo *TRI) const
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
const uint8_t TSFlags
Configurable target specific flags.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
Target - Wrapper for Target specific information.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:343
static constexpr TypeSize getZero()
Definition TypeSize.h:349
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
Definition TypeSize.h:346
A raw_ostream that writes to an std::string.
Changed
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
CondCode getInverseBranchCondition(CondCode)
unsigned getBrCond(CondCode CC, unsigned SelectOpc=0)
static bool isValidRoundingMode(unsigned Mode)
static unsigned getVecPolicyOpNum(const MCInstrDesc &Desc)
static bool usesMaskPolicy(uint64_t TSFlags)
static bool hasRoundModeOp(uint64_t TSFlags)
static unsigned getVLOpNum(const MCInstrDesc &Desc)
static bool hasVLOp(uint64_t TSFlags)
static MCRegister getTailExpandUseRegNo(const FeatureBitset &FeatureBits)
static int getFRMOpNum(const MCInstrDesc &Desc)
static bool hasVecPolicyOp(uint64_t TSFlags)
static bool usesVXRM(uint64_t TSFlags)
static bool isRVVWideningReduction(uint64_t TSFlags)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
static bool hasSEWOp(uint64_t TSFlags)
static bool isFirstDefTiedToFirstUse(const MCInstrDesc &Desc)
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
SmallVector< Inst, 8 > InstSeq
Definition RISCVMatInt.h:43
@ OPERAND_SIMM10_LSB0000_NONZERO
static unsigned getNF(uint8_t TSFlags)
static RISCVVType::VLMUL getLMul(uint8_t TSFlags)
static bool isTailAgnostic(unsigned VType)
LLVM_ABI std::pair< unsigned, bool > decodeVLMUL(VLMUL VLMul)
static bool isValidSEW(unsigned SEW)
LLVM_ABI void printVType(unsigned VType, raw_ostream &OS)
static unsigned getSEW(unsigned VType)
static VLMUL getVLMUL(unsigned VType)
bool hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2)
bool isVLKnownLE(const MachineOperand &LHS, const MachineOperand &RHS)
Given two VL operands, do we know that LHS <= RHS?
unsigned getRVVMCOpcode(unsigned RVVPseudoOpcode)
unsigned getDestLog2EEW(const MCInstrDesc &Desc, unsigned Log2SEW)
std::optional< unsigned > getVectorLowDemandedScalarBits(unsigned Opcode, unsigned Log2SEW)
std::optional< std::pair< unsigned, unsigned > > isRVVSpillForZvlsseg(unsigned Opcode)
static constexpr unsigned RVVBitsPerBlock
bool isRVVSpill(const MachineInstr &MI)
static constexpr unsigned RVVBytesPerBlock
static constexpr int64_t VLMaxSentinel
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Dead
Unused definition.
@ Define
Register definition.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
InstrType
Represents how an instruction should be mapped by the outliner.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:318
@ Offset
Definition DWP.cpp:477
@ SHXADD_ADD_SLLI_OP2
@ SHXADD_ADD_SLLI_OP1
MachineTraceStrategy
Strategies for selecting traces.
@ TS_MinInstrCount
Select the trace through a block that has the fewest instructions.
@ TS_Local
Select the trace that contains only the current basic block.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1705
static const MachineMemOperand::Flags MONontemporalBit1
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
Definition MathExtras.h:174
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2452
static const MachineMemOperand::Flags MONontemporalBit0
unsigned getDeadRegState(bool B)
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:293
Op::Description Desc
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:348
constexpr bool has_single_bit(T Value) noexcept
Definition bit.h:147
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1712
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:342
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:167
CombinerObjective
The combiner's goal may differ based on which pattern it is attempting to optimize.
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
Definition MathExtras.h:198
CodeGenOptLevel
Code generation optimization level.
Definition CodeGen.h:82
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:548
unsigned getKillRegState(bool B)
unsigned getRenamableRegState(bool B)
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
constexpr bool isShiftedInt(int64_t x)
Checks if a signed integer is an N bit number shifted left by S.
Definition MathExtras.h:191
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2100
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
Definition MathExtras.h:583
LLVM_ABI const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=MaxLookupSearchDepth)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
constexpr bool isShiftedUInt(uint64_t x)
Checks if a unsigned integer is an N bit number shifted left by S.
Definition MathExtras.h:207
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:872
#define N
Used to describe addressing mode similar to ExtAddrMode in CodeGenPrepare.
This represents a simple continuous liveness interval for a value.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
static bool isRVVRegClass(const TargetRegisterClass *RC)
Used to describe a register and immediate addition.
An individual sequence of instructions to be replaced with a call to an outlined function.
MachineFunction * getMF() const
The information necessary to create an outlined function for some class of candidate.