LLVM 19.0.0git
RISCVInstrInfo.cpp
Go to the documentation of this file.
1//===-- RISCVInstrInfo.cpp - RISC-V Instruction Information -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the RISC-V implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVInstrInfo.h"
15#include "RISCV.h"
17#include "RISCVSubtarget.h"
18#include "RISCVTargetMachine.h"
19#include "llvm/ADT/STLExtras.h"
36
37using namespace llvm;
38
39#define GEN_CHECK_COMPRESS_INSTR
40#include "RISCVGenCompressInstEmitter.inc"
41
42#define GET_INSTRINFO_CTOR_DTOR
43#define GET_INSTRINFO_NAMED_OPS
44#include "RISCVGenInstrInfo.inc"
45
47 "riscv-prefer-whole-register-move", cl::init(false), cl::Hidden,
48 cl::desc("Prefer whole register move for vector registers."));
49
51 "riscv-force-machine-combiner-strategy", cl::Hidden,
52 cl::desc("Force machine combiner to use a specific strategy for machine "
53 "trace metrics evaluation."),
54 cl::init(MachineTraceStrategy::TS_NumStrategies),
55 cl::values(clEnumValN(MachineTraceStrategy::TS_Local, "local",
56 "Local strategy."),
57 clEnumValN(MachineTraceStrategy::TS_MinInstrCount, "min-instr",
58 "MinInstrCount strategy.")));
59
61
62using namespace RISCV;
63
64#define GET_RISCVVPseudosTable_IMPL
65#include "RISCVGenSearchableTables.inc"
66
67} // namespace llvm::RISCVVPseudosTable
68
69namespace llvm::RISCV {
70
71#define GET_RISCVMaskedPseudosTable_IMPL
72#include "RISCVGenSearchableTables.inc"
73
74} // end namespace llvm::RISCV
75
77 : RISCVGenInstrInfo(RISCV::ADJCALLSTACKDOWN, RISCV::ADJCALLSTACKUP),
78 STI(STI) {}
79
81 if (STI.hasStdExtCOrZca())
82 return MCInstBuilder(RISCV::C_NOP);
83 return MCInstBuilder(RISCV::ADDI)
84 .addReg(RISCV::X0)
85 .addReg(RISCV::X0)
86 .addImm(0);
87}
88
90 int &FrameIndex) const {
91 unsigned Dummy;
92 return isLoadFromStackSlot(MI, FrameIndex, Dummy);
93}
94
96 int &FrameIndex,
97 unsigned &MemBytes) const {
98 switch (MI.getOpcode()) {
99 default:
100 return 0;
101 case RISCV::LB:
102 case RISCV::LBU:
103 MemBytes = 1;
104 break;
105 case RISCV::LH:
106 case RISCV::LHU:
107 case RISCV::FLH:
108 MemBytes = 2;
109 break;
110 case RISCV::LW:
111 case RISCV::FLW:
112 case RISCV::LWU:
113 MemBytes = 4;
114 break;
115 case RISCV::LD:
116 case RISCV::FLD:
117 MemBytes = 8;
118 break;
119 }
120
121 if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
122 MI.getOperand(2).getImm() == 0) {
123 FrameIndex = MI.getOperand(1).getIndex();
124 return MI.getOperand(0).getReg();
125 }
126
127 return 0;
128}
129
131 int &FrameIndex) const {
132 unsigned Dummy;
133 return isStoreToStackSlot(MI, FrameIndex, Dummy);
134}
135
137 int &FrameIndex,
138 unsigned &MemBytes) const {
139 switch (MI.getOpcode()) {
140 default:
141 return 0;
142 case RISCV::SB:
143 MemBytes = 1;
144 break;
145 case RISCV::SH:
146 case RISCV::FSH:
147 MemBytes = 2;
148 break;
149 case RISCV::SW:
150 case RISCV::FSW:
151 MemBytes = 4;
152 break;
153 case RISCV::SD:
154 case RISCV::FSD:
155 MemBytes = 8;
156 break;
157 }
158
159 if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
160 MI.getOperand(2).getImm() == 0) {
161 FrameIndex = MI.getOperand(1).getIndex();
162 return MI.getOperand(0).getReg();
163 }
164
165 return 0;
166}
167
168static bool forwardCopyWillClobberTuple(unsigned DstReg, unsigned SrcReg,
169 unsigned NumRegs) {
170 return DstReg > SrcReg && (DstReg - SrcReg) < NumRegs;
171}
172
174 const MachineBasicBlock &MBB,
177 RISCVII::VLMUL LMul) {
179 return false;
180
181 assert(MBBI->getOpcode() == TargetOpcode::COPY &&
182 "Unexpected COPY instruction.");
183 Register SrcReg = MBBI->getOperand(1).getReg();
185
186 bool FoundDef = false;
187 bool FirstVSetVLI = false;
188 unsigned FirstSEW = 0;
189 while (MBBI != MBB.begin()) {
190 --MBBI;
191 if (MBBI->isMetaInstruction())
192 continue;
193
194 if (MBBI->getOpcode() == RISCV::PseudoVSETVLI ||
195 MBBI->getOpcode() == RISCV::PseudoVSETVLIX0 ||
196 MBBI->getOpcode() == RISCV::PseudoVSETIVLI) {
197 // There is a vsetvli between COPY and source define instruction.
198 // vy = def_vop ... (producing instruction)
199 // ...
200 // vsetvli
201 // ...
202 // vx = COPY vy
203 if (!FoundDef) {
204 if (!FirstVSetVLI) {
205 FirstVSetVLI = true;
206 unsigned FirstVType = MBBI->getOperand(2).getImm();
207 RISCVII::VLMUL FirstLMul = RISCVVType::getVLMUL(FirstVType);
208 FirstSEW = RISCVVType::getSEW(FirstVType);
209 // The first encountered vsetvli must have the same lmul as the
210 // register class of COPY.
211 if (FirstLMul != LMul)
212 return false;
213 }
214 // Only permit `vsetvli x0, x0, vtype` between COPY and the source
215 // define instruction.
216 if (MBBI->getOperand(0).getReg() != RISCV::X0)
217 return false;
218 if (MBBI->getOperand(1).isImm())
219 return false;
220 if (MBBI->getOperand(1).getReg() != RISCV::X0)
221 return false;
222 continue;
223 }
224
225 // MBBI is the first vsetvli before the producing instruction.
226 unsigned VType = MBBI->getOperand(2).getImm();
227 // If there is a vsetvli between COPY and the producing instruction.
228 if (FirstVSetVLI) {
229 // If SEW is different, return false.
230 if (RISCVVType::getSEW(VType) != FirstSEW)
231 return false;
232 }
233
234 // If the vsetvli is tail undisturbed, keep the whole register move.
235 if (!RISCVVType::isTailAgnostic(VType))
236 return false;
237
238 // The checking is conservative. We only have register classes for
239 // LMUL = 1/2/4/8. We should be able to convert vmv1r.v to vmv.v.v
240 // for fractional LMUL operations. However, we could not use the vsetvli
241 // lmul for widening operations. The result of widening operation is
242 // 2 x LMUL.
243 return LMul == RISCVVType::getVLMUL(VType);
244 } else if (MBBI->isInlineAsm() || MBBI->isCall()) {
245 return false;
246 } else if (MBBI->getNumDefs()) {
247 // Check all the instructions which will change VL.
248 // For example, vleff has implicit def VL.
249 if (MBBI->modifiesRegister(RISCV::VL, /*TRI=*/nullptr))
250 return false;
251
252 // Only converting whole register copies to vmv.v.v when the defining
253 // value appears in the explicit operands.
254 for (const MachineOperand &MO : MBBI->explicit_operands()) {
255 if (!MO.isReg() || !MO.isDef())
256 continue;
257 if (!FoundDef && TRI->regsOverlap(MO.getReg(), SrcReg)) {
258 // We only permit the source of COPY has the same LMUL as the defined
259 // operand.
260 // There are cases we need to keep the whole register copy if the LMUL
261 // is different.
262 // For example,
263 // $x0 = PseudoVSETIVLI 4, 73 // vsetivli zero, 4, e16,m2,ta,m
264 // $v28m4 = PseudoVWADD_VV_M2 $v26m2, $v8m2
265 // # The COPY may be created by vlmul_trunc intrinsic.
266 // $v26m2 = COPY renamable $v28m2, implicit killed $v28m4
267 //
268 // After widening, the valid value will be 4 x e32 elements. If we
269 // convert the COPY to vmv.v.v, it will only copy 4 x e16 elements.
270 // FIXME: The COPY of subregister of Zvlsseg register will not be able
271 // to convert to vmv.v.[v|i] under the constraint.
272 if (MO.getReg() != SrcReg)
273 return false;
274
275 // In widening reduction instructions with LMUL_1 input vector case,
276 // only checking the LMUL is insufficient due to reduction result is
277 // always LMUL_1.
278 // For example,
279 // $x11 = PseudoVSETIVLI 1, 64 // vsetivli a1, 1, e8, m1, ta, mu
280 // $v8m1 = PseudoVWREDSUM_VS_M1 $v26, $v27
281 // $v26 = COPY killed renamable $v8
282 // After widening, The valid value will be 1 x e16 elements. If we
283 // convert the COPY to vmv.v.v, it will only copy 1 x e8 elements.
284 uint64_t TSFlags = MBBI->getDesc().TSFlags;
286 return false;
287
288 // If the producing instruction does not depend on vsetvli, do not
289 // convert COPY to vmv.v.v. For example, VL1R_V or PseudoVRELOAD.
290 if (!RISCVII::hasSEWOp(TSFlags) || !RISCVII::hasVLOp(TSFlags))
291 return false;
292
293 // Found the definition.
294 FoundDef = true;
295 DefMBBI = MBBI;
296 break;
297 }
298 }
299 }
300 }
301
302 return false;
303}
304
307 const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc,
308 const TargetRegisterClass *RegClass) const {
310 RISCVII::VLMUL LMul = RISCVRI::getLMul(RegClass->TSFlags);
311 unsigned NF = RISCVRI::getNF(RegClass->TSFlags);
312
313 uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
314 uint16_t DstEncoding = TRI->getEncodingValue(DstReg);
315 auto [LMulVal, Fractional] = RISCVVType::decodeVLMUL(LMul);
316 assert(!Fractional && "It is impossible be fractional lmul here.");
317 unsigned NumRegs = NF * LMulVal;
318 bool ReversedCopy =
319 forwardCopyWillClobberTuple(DstEncoding, SrcEncoding, NumRegs);
320 if (ReversedCopy) {
321 // If the src and dest overlap when copying a tuple, we need to copy the
322 // registers in reverse.
323 SrcEncoding += NumRegs - 1;
324 DstEncoding += NumRegs - 1;
325 }
326
327 unsigned I = 0;
328 auto GetCopyInfo = [&](uint16_t SrcEncoding, uint16_t DstEncoding)
329 -> std::tuple<RISCVII::VLMUL, const TargetRegisterClass &, unsigned,
330 unsigned, unsigned> {
331 if (ReversedCopy) {
332 // For reversed copying, if there are enough aligned registers(8/4/2), we
333 // can do a larger copy(LMUL8/4/2).
334 // Besides, we have already known that DstEncoding is larger than
335 // SrcEncoding in forwardCopyWillClobberTuple, so the difference between
336 // DstEncoding and SrcEncoding should be >= LMUL value we try to use to
337 // avoid clobbering.
338 uint16_t Diff = DstEncoding - SrcEncoding;
339 if (I + 8 <= NumRegs && Diff >= 8 && SrcEncoding % 8 == 7 &&
340 DstEncoding % 8 == 7)
341 return {RISCVII::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
342 RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
343 if (I + 4 <= NumRegs && Diff >= 4 && SrcEncoding % 4 == 3 &&
344 DstEncoding % 4 == 3)
345 return {RISCVII::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
346 RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
347 if (I + 2 <= NumRegs && Diff >= 2 && SrcEncoding % 2 == 1 &&
348 DstEncoding % 2 == 1)
349 return {RISCVII::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
350 RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
351 // Or we should do LMUL1 copying.
352 return {RISCVII::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
353 RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
354 }
355
356 // For forward copying, if source register encoding and destination register
357 // encoding are aligned to 8/4/2, we can do a LMUL8/4/2 copying.
358 if (I + 8 <= NumRegs && SrcEncoding % 8 == 0 && DstEncoding % 8 == 0)
359 return {RISCVII::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
360 RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
361 if (I + 4 <= NumRegs && SrcEncoding % 4 == 0 && DstEncoding % 4 == 0)
362 return {RISCVII::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
363 RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
364 if (I + 2 <= NumRegs && SrcEncoding % 2 == 0 && DstEncoding % 2 == 0)
365 return {RISCVII::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
366 RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
367 // Or we should do LMUL1 copying.
368 return {RISCVII::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
369 RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
370 };
371 auto FindRegWithEncoding = [TRI](const TargetRegisterClass &RegClass,
372 uint16_t Encoding) {
373 MCRegister Reg = RISCV::V0 + Encoding;
374 if (&RegClass == &RISCV::VRRegClass)
375 return Reg;
376 return TRI->getMatchingSuperReg(Reg, RISCV::sub_vrm1_0, &RegClass);
377 };
378 while (I != NumRegs) {
379 // For non-segment copying, we only do this once as the registers are always
380 // aligned.
381 // For segment copying, we may do this several times. If the registers are
382 // aligned to larger LMUL, we can eliminate some copyings.
383 auto [LMulCopied, RegClass, Opc, VVOpc, VIOpc] =
384 GetCopyInfo(SrcEncoding, DstEncoding);
385 auto [NumCopied, _] = RISCVVType::decodeVLMUL(LMulCopied);
386
388 if (LMul == LMulCopied &&
389 isConvertibleToVMV_V_V(STI, MBB, MBBI, DefMBBI, LMul)) {
390 Opc = VVOpc;
391 if (DefMBBI->getOpcode() == VIOpc)
392 Opc = VIOpc;
393 }
394
395 // Emit actual copying.
396 // For reversed copying, the encoding should be decreased.
397 MCRegister ActualSrcReg = FindRegWithEncoding(
398 RegClass, ReversedCopy ? (SrcEncoding - NumCopied + 1) : SrcEncoding);
399 MCRegister ActualDstReg = FindRegWithEncoding(
400 RegClass, ReversedCopy ? (DstEncoding - NumCopied + 1) : DstEncoding);
401
402 auto MIB = BuildMI(MBB, MBBI, DL, get(Opc), ActualDstReg);
403 bool UseVMV_V_I = RISCV::getRVVMCOpcode(Opc) == RISCV::VMV_V_I;
404 bool UseVMV = UseVMV_V_I || RISCV::getRVVMCOpcode(Opc) == RISCV::VMV_V_V;
405 if (UseVMV)
406 MIB.addReg(ActualDstReg, RegState::Undef);
407 if (UseVMV_V_I)
408 MIB = MIB.add(DefMBBI->getOperand(2));
409 else
410 MIB = MIB.addReg(ActualSrcReg, getKillRegState(KillSrc));
411 if (UseVMV) {
412 const MCInstrDesc &Desc = DefMBBI->getDesc();
413 MIB.add(DefMBBI->getOperand(RISCVII::getVLOpNum(Desc))); // AVL
414 MIB.add(DefMBBI->getOperand(RISCVII::getSEWOpNum(Desc))); // SEW
415 MIB.addImm(0); // tu, mu
416 MIB.addReg(RISCV::VL, RegState::Implicit);
417 MIB.addReg(RISCV::VTYPE, RegState::Implicit);
418 }
419
420 // If we are copying reversely, we should decrease the encoding.
421 SrcEncoding += (ReversedCopy ? -NumCopied : NumCopied);
422 DstEncoding += (ReversedCopy ? -NumCopied : NumCopied);
423 I += NumCopied;
424 }
425}
426
429 const DebugLoc &DL, MCRegister DstReg,
430 MCRegister SrcReg, bool KillSrc) const {
432
433 if (RISCV::GPRRegClass.contains(DstReg, SrcReg)) {
434 BuildMI(MBB, MBBI, DL, get(RISCV::ADDI), DstReg)
435 .addReg(SrcReg, getKillRegState(KillSrc))
436 .addImm(0);
437 return;
438 }
439
440 if (RISCV::GPRPairRegClass.contains(DstReg, SrcReg)) {
441 // Emit an ADDI for both parts of GPRPair.
442 BuildMI(MBB, MBBI, DL, get(RISCV::ADDI),
443 TRI->getSubReg(DstReg, RISCV::sub_gpr_even))
444 .addReg(TRI->getSubReg(SrcReg, RISCV::sub_gpr_even),
445 getKillRegState(KillSrc))
446 .addImm(0);
447 BuildMI(MBB, MBBI, DL, get(RISCV::ADDI),
448 TRI->getSubReg(DstReg, RISCV::sub_gpr_odd))
449 .addReg(TRI->getSubReg(SrcReg, RISCV::sub_gpr_odd),
450 getKillRegState(KillSrc))
451 .addImm(0);
452 return;
453 }
454
455 // Handle copy from csr
456 if (RISCV::VCSRRegClass.contains(SrcReg) &&
457 RISCV::GPRRegClass.contains(DstReg)) {
458 BuildMI(MBB, MBBI, DL, get(RISCV::CSRRS), DstReg)
459 .addImm(RISCVSysReg::lookupSysRegByName(TRI->getName(SrcReg))->Encoding)
460 .addReg(RISCV::X0);
461 return;
462 }
463
464 if (RISCV::FPR16RegClass.contains(DstReg, SrcReg)) {
465 unsigned Opc;
466 if (STI.hasStdExtZfh()) {
467 Opc = RISCV::FSGNJ_H;
468 } else {
469 assert(STI.hasStdExtF() &&
470 (STI.hasStdExtZfhmin() || STI.hasStdExtZfbfmin()) &&
471 "Unexpected extensions");
472 // Zfhmin/Zfbfmin doesn't have FSGNJ_H, replace FSGNJ_H with FSGNJ_S.
473 DstReg = TRI->getMatchingSuperReg(DstReg, RISCV::sub_16,
474 &RISCV::FPR32RegClass);
475 SrcReg = TRI->getMatchingSuperReg(SrcReg, RISCV::sub_16,
476 &RISCV::FPR32RegClass);
477 Opc = RISCV::FSGNJ_S;
478 }
479 BuildMI(MBB, MBBI, DL, get(Opc), DstReg)
480 .addReg(SrcReg, getKillRegState(KillSrc))
481 .addReg(SrcReg, getKillRegState(KillSrc));
482 return;
483 }
484
485 if (RISCV::FPR32RegClass.contains(DstReg, SrcReg)) {
486 BuildMI(MBB, MBBI, DL, get(RISCV::FSGNJ_S), DstReg)
487 .addReg(SrcReg, getKillRegState(KillSrc))
488 .addReg(SrcReg, getKillRegState(KillSrc));
489 return;
490 }
491
492 if (RISCV::FPR64RegClass.contains(DstReg, SrcReg)) {
493 BuildMI(MBB, MBBI, DL, get(RISCV::FSGNJ_D), DstReg)
494 .addReg(SrcReg, getKillRegState(KillSrc))
495 .addReg(SrcReg, getKillRegState(KillSrc));
496 return;
497 }
498
499 if (RISCV::FPR32RegClass.contains(DstReg) &&
500 RISCV::GPRRegClass.contains(SrcReg)) {
501 BuildMI(MBB, MBBI, DL, get(RISCV::FMV_W_X), DstReg)
502 .addReg(SrcReg, getKillRegState(KillSrc));
503 return;
504 }
505
506 if (RISCV::GPRRegClass.contains(DstReg) &&
507 RISCV::FPR32RegClass.contains(SrcReg)) {
508 BuildMI(MBB, MBBI, DL, get(RISCV::FMV_X_W), DstReg)
509 .addReg(SrcReg, getKillRegState(KillSrc));
510 return;
511 }
512
513 if (RISCV::FPR64RegClass.contains(DstReg) &&
514 RISCV::GPRRegClass.contains(SrcReg)) {
515 assert(STI.getXLen() == 64 && "Unexpected GPR size");
516 BuildMI(MBB, MBBI, DL, get(RISCV::FMV_D_X), DstReg)
517 .addReg(SrcReg, getKillRegState(KillSrc));
518 return;
519 }
520
521 if (RISCV::GPRRegClass.contains(DstReg) &&
522 RISCV::FPR64RegClass.contains(SrcReg)) {
523 assert(STI.getXLen() == 64 && "Unexpected GPR size");
524 BuildMI(MBB, MBBI, DL, get(RISCV::FMV_X_D), DstReg)
525 .addReg(SrcReg, getKillRegState(KillSrc));
526 return;
527 }
528
529 // VR->VR copies.
530 static const TargetRegisterClass *RVVRegClasses[] = {
531 &RISCV::VRRegClass, &RISCV::VRM2RegClass, &RISCV::VRM4RegClass,
532 &RISCV::VRM8RegClass, &RISCV::VRN2M1RegClass, &RISCV::VRN2M2RegClass,
533 &RISCV::VRN2M4RegClass, &RISCV::VRN3M1RegClass, &RISCV::VRN3M2RegClass,
534 &RISCV::VRN4M1RegClass, &RISCV::VRN4M2RegClass, &RISCV::VRN5M1RegClass,
535 &RISCV::VRN6M1RegClass, &RISCV::VRN7M1RegClass, &RISCV::VRN8M1RegClass};
536 for (const auto &RegClass : RVVRegClasses) {
537 if (RegClass->contains(DstReg, SrcReg)) {
538 copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RegClass);
539 return;
540 }
541 }
542
543 llvm_unreachable("Impossible reg-to-reg copy");
544}
545
548 Register SrcReg, bool IsKill, int FI,
549 const TargetRegisterClass *RC,
550 const TargetRegisterInfo *TRI,
551 Register VReg) const {
553 MachineFrameInfo &MFI = MF->getFrameInfo();
554
555 unsigned Opcode;
556 bool IsScalableVector = true;
557 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
558 Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
559 RISCV::SW : RISCV::SD;
560 IsScalableVector = false;
561 } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
562 Opcode = RISCV::PseudoRV32ZdinxSD;
563 IsScalableVector = false;
564 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
565 Opcode = RISCV::FSH;
566 IsScalableVector = false;
567 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
568 Opcode = RISCV::FSW;
569 IsScalableVector = false;
570 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
571 Opcode = RISCV::FSD;
572 IsScalableVector = false;
573 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
574 Opcode = RISCV::VS1R_V;
575 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
576 Opcode = RISCV::VS2R_V;
577 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
578 Opcode = RISCV::VS4R_V;
579 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
580 Opcode = RISCV::VS8R_V;
581 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
582 Opcode = RISCV::PseudoVSPILL2_M1;
583 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
584 Opcode = RISCV::PseudoVSPILL2_M2;
585 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
586 Opcode = RISCV::PseudoVSPILL2_M4;
587 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
588 Opcode = RISCV::PseudoVSPILL3_M1;
589 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
590 Opcode = RISCV::PseudoVSPILL3_M2;
591 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
592 Opcode = RISCV::PseudoVSPILL4_M1;
593 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
594 Opcode = RISCV::PseudoVSPILL4_M2;
595 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
596 Opcode = RISCV::PseudoVSPILL5_M1;
597 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
598 Opcode = RISCV::PseudoVSPILL6_M1;
599 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
600 Opcode = RISCV::PseudoVSPILL7_M1;
601 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
602 Opcode = RISCV::PseudoVSPILL8_M1;
603 else
604 llvm_unreachable("Can't store this register to stack slot");
605
606 if (IsScalableVector) {
610
612 BuildMI(MBB, I, DebugLoc(), get(Opcode))
613 .addReg(SrcReg, getKillRegState(IsKill))
614 .addFrameIndex(FI)
615 .addMemOperand(MMO);
616 } else {
619 MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
620
621 BuildMI(MBB, I, DebugLoc(), get(Opcode))
622 .addReg(SrcReg, getKillRegState(IsKill))
623 .addFrameIndex(FI)
624 .addImm(0)
625 .addMemOperand(MMO);
626 }
627}
628
631 Register DstReg, int FI,
632 const TargetRegisterClass *RC,
633 const TargetRegisterInfo *TRI,
634 Register VReg) const {
636 MachineFrameInfo &MFI = MF->getFrameInfo();
637
638 unsigned Opcode;
639 bool IsScalableVector = true;
640 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
641 Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
642 RISCV::LW : RISCV::LD;
643 IsScalableVector = false;
644 } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
645 Opcode = RISCV::PseudoRV32ZdinxLD;
646 IsScalableVector = false;
647 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
648 Opcode = RISCV::FLH;
649 IsScalableVector = false;
650 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
651 Opcode = RISCV::FLW;
652 IsScalableVector = false;
653 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
654 Opcode = RISCV::FLD;
655 IsScalableVector = false;
656 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
657 Opcode = RISCV::VL1RE8_V;
658 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
659 Opcode = RISCV::VL2RE8_V;
660 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
661 Opcode = RISCV::VL4RE8_V;
662 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
663 Opcode = RISCV::VL8RE8_V;
664 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
665 Opcode = RISCV::PseudoVRELOAD2_M1;
666 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
667 Opcode = RISCV::PseudoVRELOAD2_M2;
668 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
669 Opcode = RISCV::PseudoVRELOAD2_M4;
670 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
671 Opcode = RISCV::PseudoVRELOAD3_M1;
672 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
673 Opcode = RISCV::PseudoVRELOAD3_M2;
674 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
675 Opcode = RISCV::PseudoVRELOAD4_M1;
676 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
677 Opcode = RISCV::PseudoVRELOAD4_M2;
678 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
679 Opcode = RISCV::PseudoVRELOAD5_M1;
680 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
681 Opcode = RISCV::PseudoVRELOAD6_M1;
682 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
683 Opcode = RISCV::PseudoVRELOAD7_M1;
684 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
685 Opcode = RISCV::PseudoVRELOAD8_M1;
686 else
687 llvm_unreachable("Can't load this register from stack slot");
688
689 if (IsScalableVector) {
693
695 BuildMI(MBB, I, DebugLoc(), get(Opcode), DstReg)
696 .addFrameIndex(FI)
697 .addMemOperand(MMO);
698 } else {
701 MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
702
703 BuildMI(MBB, I, DebugLoc(), get(Opcode), DstReg)
704 .addFrameIndex(FI)
705 .addImm(0)
706 .addMemOperand(MMO);
707 }
708}
709
712 MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS,
713 VirtRegMap *VRM) const {
714 const MachineFrameInfo &MFI = MF.getFrameInfo();
715
716 // The below optimizations narrow the load so they are only valid for little
717 // endian.
718 // TODO: Support big endian by adding an offset into the frame object?
719 if (MF.getDataLayout().isBigEndian())
720 return nullptr;
721
722 // Fold load from stack followed by sext.b/sext.h/sext.w/zext.b/zext.h/zext.w.
723 if (Ops.size() != 1 || Ops[0] != 1)
724 return nullptr;
725
726 unsigned LoadOpc;
727 switch (MI.getOpcode()) {
728 default:
729 if (RISCV::isSEXT_W(MI)) {
730 LoadOpc = RISCV::LW;
731 break;
732 }
733 if (RISCV::isZEXT_W(MI)) {
734 LoadOpc = RISCV::LWU;
735 break;
736 }
737 if (RISCV::isZEXT_B(MI)) {
738 LoadOpc = RISCV::LBU;
739 break;
740 }
741 return nullptr;
742 case RISCV::SEXT_H:
743 LoadOpc = RISCV::LH;
744 break;
745 case RISCV::SEXT_B:
746 LoadOpc = RISCV::LB;
747 break;
748 case RISCV::ZEXT_H_RV32:
749 case RISCV::ZEXT_H_RV64:
750 LoadOpc = RISCV::LHU;
751 break;
752 }
753
755 MachinePointerInfo::getFixedStack(MF, FrameIndex),
757 MFI.getObjectAlign(FrameIndex));
758
759 Register DstReg = MI.getOperand(0).getReg();
760 return BuildMI(*MI.getParent(), InsertPt, MI.getDebugLoc(), get(LoadOpc),
761 DstReg)
762 .addFrameIndex(FrameIndex)
763 .addImm(0)
764 .addMemOperand(MMO);
765}
766
769 const DebugLoc &DL, Register DstReg, uint64_t Val,
770 MachineInstr::MIFlag Flag, bool DstRenamable,
771 bool DstIsDead) const {
772 Register SrcReg = RISCV::X0;
773
774 // For RV32, allow a sign or unsigned 32 bit value.
775 if (!STI.is64Bit() && !isInt<32>(Val)) {
776 // If have a uimm32 it will still fit in a register so we can allow it.
777 if (!isUInt<32>(Val))
778 report_fatal_error("Should only materialize 32-bit constants for RV32");
779
780 // Sign extend for generateInstSeq.
781 Val = SignExtend64<32>(Val);
782 }
783
785 assert(!Seq.empty());
786
787 bool SrcRenamable = false;
788 unsigned Num = 0;
789
790 for (const RISCVMatInt::Inst &Inst : Seq) {
791 bool LastItem = ++Num == Seq.size();
792 unsigned DstRegState = getDeadRegState(DstIsDead && LastItem) |
793 getRenamableRegState(DstRenamable);
794 unsigned SrcRegState = getKillRegState(SrcReg != RISCV::X0) |
795 getRenamableRegState(SrcRenamable);
796 switch (Inst.getOpndKind()) {
797 case RISCVMatInt::Imm:
798 BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()))
799 .addReg(DstReg, RegState::Define | DstRegState)
800 .addImm(Inst.getImm())
801 .setMIFlag(Flag);
802 break;
804 BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()))
805 .addReg(DstReg, RegState::Define | DstRegState)
806 .addReg(SrcReg, SrcRegState)
807 .addReg(RISCV::X0)
808 .setMIFlag(Flag);
809 break;
811 BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()))
812 .addReg(DstReg, RegState::Define | DstRegState)
813 .addReg(SrcReg, SrcRegState)
814 .addReg(SrcReg, SrcRegState)
815 .setMIFlag(Flag);
816 break;
818 BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()))
819 .addReg(DstReg, RegState::Define | DstRegState)
820 .addReg(SrcReg, SrcRegState)
821 .addImm(Inst.getImm())
822 .setMIFlag(Flag);
823 break;
824 }
825
826 // Only the first instruction has X0 as its source.
827 SrcReg = DstReg;
828 SrcRenamable = DstRenamable;
829 }
830}
831
833 switch (Opc) {
834 default:
836 case RISCV::CV_BEQIMM:
837 return RISCVCC::COND_EQ;
838 case RISCV::CV_BNEIMM:
839 return RISCVCC::COND_NE;
840 case RISCV::BEQ:
841 return RISCVCC::COND_EQ;
842 case RISCV::BNE:
843 return RISCVCC::COND_NE;
844 case RISCV::BLT:
845 return RISCVCC::COND_LT;
846 case RISCV::BGE:
847 return RISCVCC::COND_GE;
848 case RISCV::BLTU:
849 return RISCVCC::COND_LTU;
850 case RISCV::BGEU:
851 return RISCVCC::COND_GEU;
852 }
853}
854
855// The contents of values added to Cond are not examined outside of
856// RISCVInstrInfo, giving us flexibility in what to push to it. For RISCV, we
857// push BranchOpcode, Reg1, Reg2.
860 // Block ends with fall-through condbranch.
861 assert(LastInst.getDesc().isConditionalBranch() &&
862 "Unknown conditional branch");
863 Target = LastInst.getOperand(2).getMBB();
864 unsigned CC = getCondFromBranchOpc(LastInst.getOpcode());
866 Cond.push_back(LastInst.getOperand(0));
867 Cond.push_back(LastInst.getOperand(1));
868}
869
871 switch (CC) {
872 default:
873 llvm_unreachable("Unknown condition code!");
874 case RISCVCC::COND_EQ:
875 return Imm ? RISCV::CV_BEQIMM : RISCV::BEQ;
876 case RISCVCC::COND_NE:
877 return Imm ? RISCV::CV_BNEIMM : RISCV::BNE;
878 case RISCVCC::COND_LT:
879 return RISCV::BLT;
880 case RISCVCC::COND_GE:
881 return RISCV::BGE;
883 return RISCV::BLTU;
885 return RISCV::BGEU;
886 }
887}
888
890 bool Imm) const {
891 return get(RISCVCC::getBrCond(CC, Imm));
892}
893
895 switch (CC) {
896 default:
897 llvm_unreachable("Unrecognized conditional branch");
898 case RISCVCC::COND_EQ:
899 return RISCVCC::COND_NE;
900 case RISCVCC::COND_NE:
901 return RISCVCC::COND_EQ;
902 case RISCVCC::COND_LT:
903 return RISCVCC::COND_GE;
904 case RISCVCC::COND_GE:
905 return RISCVCC::COND_LT;
907 return RISCVCC::COND_GEU;
909 return RISCVCC::COND_LTU;
910 }
911}
912
915 MachineBasicBlock *&FBB,
917 bool AllowModify) const {
918 TBB = FBB = nullptr;
919 Cond.clear();
920
921 // If the block has no terminators, it just falls into the block after it.
923 if (I == MBB.end() || !isUnpredicatedTerminator(*I))
924 return false;
925
926 // Count the number of terminators and find the first unconditional or
927 // indirect branch.
928 MachineBasicBlock::iterator FirstUncondOrIndirectBr = MBB.end();
929 int NumTerminators = 0;
930 for (auto J = I.getReverse(); J != MBB.rend() && isUnpredicatedTerminator(*J);
931 J++) {
932 NumTerminators++;
933 if (J->getDesc().isUnconditionalBranch() ||
934 J->getDesc().isIndirectBranch()) {
935 FirstUncondOrIndirectBr = J.getReverse();
936 }
937 }
938
939 // If AllowModify is true, we can erase any terminators after
940 // FirstUncondOrIndirectBR.
941 if (AllowModify && FirstUncondOrIndirectBr != MBB.end()) {
942 while (std::next(FirstUncondOrIndirectBr) != MBB.end()) {
943 std::next(FirstUncondOrIndirectBr)->eraseFromParent();
944 NumTerminators--;
945 }
946 I = FirstUncondOrIndirectBr;
947 }
948
949 // We can't handle blocks that end in an indirect branch.
950 if (I->getDesc().isIndirectBranch())
951 return true;
952
953 // We can't handle Generic branch opcodes from Global ISel.
954 if (I->isPreISelOpcode())
955 return true;
956
957 // We can't handle blocks with more than 2 terminators.
958 if (NumTerminators > 2)
959 return true;
960
961 // Handle a single unconditional branch.
962 if (NumTerminators == 1 && I->getDesc().isUnconditionalBranch()) {
964 return false;
965 }
966
967 // Handle a single conditional branch.
968 if (NumTerminators == 1 && I->getDesc().isConditionalBranch()) {
970 return false;
971 }
972
973 // Handle a conditional branch followed by an unconditional branch.
974 if (NumTerminators == 2 && std::prev(I)->getDesc().isConditionalBranch() &&
975 I->getDesc().isUnconditionalBranch()) {
976 parseCondBranch(*std::prev(I), TBB, Cond);
977 FBB = getBranchDestBlock(*I);
978 return false;
979 }
980
981 // Otherwise, we can't handle this.
982 return true;
983}
984
986 int *BytesRemoved) const {
987 if (BytesRemoved)
988 *BytesRemoved = 0;
990 if (I == MBB.end())
991 return 0;
992
993 if (!I->getDesc().isUnconditionalBranch() &&
994 !I->getDesc().isConditionalBranch())
995 return 0;
996
997 // Remove the branch.
998 if (BytesRemoved)
999 *BytesRemoved += getInstSizeInBytes(*I);
1000 I->eraseFromParent();
1001
1002 I = MBB.end();
1003
1004 if (I == MBB.begin())
1005 return 1;
1006 --I;
1007 if (!I->getDesc().isConditionalBranch())
1008 return 1;
1009
1010 // Remove the branch.
1011 if (BytesRemoved)
1012 *BytesRemoved += getInstSizeInBytes(*I);
1013 I->eraseFromParent();
1014 return 2;
1015}
1016
1017// Inserts a branch into the end of the specific MachineBasicBlock, returning
1018// the number of instructions inserted.
1021 ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
1022 if (BytesAdded)
1023 *BytesAdded = 0;
1024
1025 // Shouldn't be a fall through.
1026 assert(TBB && "insertBranch must not be told to insert a fallthrough");
1027 assert((Cond.size() == 3 || Cond.size() == 0) &&
1028 "RISC-V branch conditions have two components!");
1029
1030 // Unconditional branch.
1031 if (Cond.empty()) {
1032 MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(TBB);
1033 if (BytesAdded)
1034 *BytesAdded += getInstSizeInBytes(MI);
1035 return 1;
1036 }
1037
1038 // Either a one or two-way conditional branch.
1039 auto CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm());
1040 MachineInstr &CondMI = *BuildMI(&MBB, DL, getBrCond(CC, Cond[2].isImm()))
1041 .add(Cond[1])
1042 .add(Cond[2])
1043 .addMBB(TBB);
1044 if (BytesAdded)
1045 *BytesAdded += getInstSizeInBytes(CondMI);
1046
1047 // One-way conditional branch.
1048 if (!FBB)
1049 return 1;
1050
1051 // Two-way conditional branch.
1052 MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(FBB);
1053 if (BytesAdded)
1054 *BytesAdded += getInstSizeInBytes(MI);
1055 return 2;
1056}
1057
1059 MachineBasicBlock &DestBB,
1060 MachineBasicBlock &RestoreBB,
1061 const DebugLoc &DL, int64_t BrOffset,
1062 RegScavenger *RS) const {
1063 assert(RS && "RegScavenger required for long branching");
1064 assert(MBB.empty() &&
1065 "new block should be inserted for expanding unconditional branch");
1066 assert(MBB.pred_size() == 1);
1067 assert(RestoreBB.empty() &&
1068 "restore block should be inserted for restoring clobbered registers");
1069
1074
1075 if (!isInt<32>(BrOffset))
1077 "Branch offsets outside of the signed 32-bit range not supported");
1078
1079 // FIXME: A virtual register must be used initially, as the register
1080 // scavenger won't work with empty blocks (SIInstrInfo::insertIndirectBranch
1081 // uses the same workaround).
1082 Register ScratchReg = MRI.createVirtualRegister(&RISCV::GPRJALRRegClass);
1083 auto II = MBB.end();
1084 // We may also update the jump target to RestoreBB later.
1085 MachineInstr &MI = *BuildMI(MBB, II, DL, get(RISCV::PseudoJump))
1086 .addReg(ScratchReg, RegState::Define | RegState::Dead)
1087 .addMBB(&DestBB, RISCVII::MO_CALL);
1088
1090 Register TmpGPR =
1091 RS->scavengeRegisterBackwards(RISCV::GPRRegClass, MI.getIterator(),
1092 /*RestoreAfter=*/false, /*SpAdj=*/0,
1093 /*AllowSpill=*/false);
1094 if (TmpGPR != RISCV::NoRegister)
1095 RS->setRegUsed(TmpGPR);
1096 else {
1097 // The case when there is no scavenged register needs special handling.
1098
1099 // Pick s11 because it doesn't make a difference.
1100 TmpGPR = RISCV::X27;
1101
1102 int FrameIndex = RVFI->getBranchRelaxationScratchFrameIndex();
1103 if (FrameIndex == -1)
1104 report_fatal_error("underestimated function size");
1105
1106 storeRegToStackSlot(MBB, MI, TmpGPR, /*IsKill=*/true, FrameIndex,
1107 &RISCV::GPRRegClass, TRI, Register());
1108 TRI->eliminateFrameIndex(std::prev(MI.getIterator()),
1109 /*SpAdj=*/0, /*FIOperandNum=*/1);
1110
1111 MI.getOperand(1).setMBB(&RestoreBB);
1112
1113 loadRegFromStackSlot(RestoreBB, RestoreBB.end(), TmpGPR, FrameIndex,
1114 &RISCV::GPRRegClass, TRI, Register());
1115 TRI->eliminateFrameIndex(RestoreBB.back(),
1116 /*SpAdj=*/0, /*FIOperandNum=*/1);
1117 }
1118
1119 MRI.replaceRegWith(ScratchReg, TmpGPR);
1120 MRI.clearVirtRegs();
1121}
1122
1125 assert((Cond.size() == 3) && "Invalid branch condition!");
1126 auto CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm());
1128 return false;
1129}
1130
1132 MachineBasicBlock *MBB = MI.getParent();
1134
1135 MachineBasicBlock *TBB, *FBB;
1137 if (analyzeBranch(*MBB, TBB, FBB, Cond, /*AllowModify=*/false))
1138 return false;
1139
1140 RISCVCC::CondCode CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm());
1142
1144 return false;
1145
1146 // For two constants C0 and C1 from
1147 // ```
1148 // li Y, C0
1149 // li Z, C1
1150 // ```
1151 // 1. if C1 = C0 + 1
1152 // we can turn:
1153 // (a) blt Y, X -> bge X, Z
1154 // (b) bge Y, X -> blt X, Z
1155 //
1156 // 2. if C1 = C0 - 1
1157 // we can turn:
1158 // (a) blt X, Y -> bge Z, X
1159 // (b) bge X, Y -> blt Z, X
1160 //
1161 // To make sure this optimization is really beneficial, we only
1162 // optimize for cases where Y had only one use (i.e. only used by the branch).
1163
1164 // Right now we only care about LI (i.e. ADDI x0, imm)
1165 auto isLoadImm = [](const MachineInstr *MI, int64_t &Imm) -> bool {
1166 if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(1).isReg() &&
1167 MI->getOperand(1).getReg() == RISCV::X0) {
1168 Imm = MI->getOperand(2).getImm();
1169 return true;
1170 }
1171 return false;
1172 };
1173 // Either a load from immediate instruction or X0.
1174 auto isFromLoadImm = [&](const MachineOperand &Op, int64_t &Imm) -> bool {
1175 if (!Op.isReg())
1176 return false;
1177 Register Reg = Op.getReg();
1178 return Reg.isVirtual() && isLoadImm(MRI.getVRegDef(Reg), Imm);
1179 };
1180
1181 MachineOperand &LHS = MI.getOperand(0);
1182 MachineOperand &RHS = MI.getOperand(1);
1183 // Try to find the register for constant Z; return
1184 // invalid register otherwise.
1185 auto searchConst = [&](int64_t C1) -> Register {
1187 auto DefC1 = std::find_if(++II, E, [&](const MachineInstr &I) -> bool {
1188 int64_t Imm;
1189 return isLoadImm(&I, Imm) && Imm == C1 &&
1190 I.getOperand(0).getReg().isVirtual();
1191 });
1192 if (DefC1 != E)
1193 return DefC1->getOperand(0).getReg();
1194
1195 return Register();
1196 };
1197
1198 bool Modify = false;
1199 int64_t C0;
1200 if (isFromLoadImm(LHS, C0) && MRI.hasOneUse(LHS.getReg())) {
1201 // Might be case 1.
1202 // Signed integer overflow is UB. (UINT64_MAX is bigger so we don't need
1203 // to worry about unsigned overflow here)
1204 if (C0 < INT64_MAX)
1205 if (Register RegZ = searchConst(C0 + 1)) {
1207 Cond[1] = MachineOperand::CreateReg(RHS.getReg(), /*isDef=*/false);
1208 Cond[2] = MachineOperand::CreateReg(RegZ, /*isDef=*/false);
1209 // We might extend the live range of Z, clear its kill flag to
1210 // account for this.
1211 MRI.clearKillFlags(RegZ);
1212 Modify = true;
1213 }
1214 } else if (isFromLoadImm(RHS, C0) && MRI.hasOneUse(RHS.getReg())) {
1215 // Might be case 2.
1216 // For unsigned cases, we don't want C1 to wrap back to UINT64_MAX
1217 // when C0 is zero.
1218 if ((CC == RISCVCC::COND_GE || CC == RISCVCC::COND_LT) || C0)
1219 if (Register RegZ = searchConst(C0 - 1)) {
1221 Cond[1] = MachineOperand::CreateReg(RegZ, /*isDef=*/false);
1222 Cond[2] = MachineOperand::CreateReg(LHS.getReg(), /*isDef=*/false);
1223 // We might extend the live range of Z, clear its kill flag to
1224 // account for this.
1225 MRI.clearKillFlags(RegZ);
1226 Modify = true;
1227 }
1228 }
1229
1230 if (!Modify)
1231 return false;
1232
1233 // Build the new branch and remove the old one.
1234 BuildMI(*MBB, MI, MI.getDebugLoc(),
1235 getBrCond(static_cast<RISCVCC::CondCode>(Cond[0].getImm())))
1236 .add(Cond[1])
1237 .add(Cond[2])
1238 .addMBB(TBB);
1239 MI.eraseFromParent();
1240
1241 return true;
1242}
1243
1246 assert(MI.getDesc().isBranch() && "Unexpected opcode!");
1247 // The branch target is always the last operand.
1248 int NumOp = MI.getNumExplicitOperands();
1249 return MI.getOperand(NumOp - 1).getMBB();
1250}
1251
1253 int64_t BrOffset) const {
1254 unsigned XLen = STI.getXLen();
1255 // Ideally we could determine the supported branch offset from the
1256 // RISCVII::FormMask, but this can't be used for Pseudo instructions like
1257 // PseudoBR.
1258 switch (BranchOp) {
1259 default:
1260 llvm_unreachable("Unexpected opcode!");
1261 case RISCV::BEQ:
1262 case RISCV::BNE:
1263 case RISCV::BLT:
1264 case RISCV::BGE:
1265 case RISCV::BLTU:
1266 case RISCV::BGEU:
1267 case RISCV::CV_BEQIMM:
1268 case RISCV::CV_BNEIMM:
1269 return isIntN(13, BrOffset);
1270 case RISCV::JAL:
1271 case RISCV::PseudoBR:
1272 return isIntN(21, BrOffset);
1273 case RISCV::PseudoJump:
1274 return isIntN(32, SignExtend64(BrOffset + 0x800, XLen));
1275 }
1276}
1277
1278// If the operation has a predicated pseudo instruction, return the pseudo
1279// instruction opcode. Otherwise, return RISCV::INSTRUCTION_LIST_END.
1280// TODO: Support more operations.
1281unsigned getPredicatedOpcode(unsigned Opcode) {
1282 switch (Opcode) {
1283 case RISCV::ADD: return RISCV::PseudoCCADD; break;
1284 case RISCV::SUB: return RISCV::PseudoCCSUB; break;
1285 case RISCV::SLL: return RISCV::PseudoCCSLL; break;
1286 case RISCV::SRL: return RISCV::PseudoCCSRL; break;
1287 case RISCV::SRA: return RISCV::PseudoCCSRA; break;
1288 case RISCV::AND: return RISCV::PseudoCCAND; break;
1289 case RISCV::OR: return RISCV::PseudoCCOR; break;
1290 case RISCV::XOR: return RISCV::PseudoCCXOR; break;
1291
1292 case RISCV::ADDI: return RISCV::PseudoCCADDI; break;
1293 case RISCV::SLLI: return RISCV::PseudoCCSLLI; break;
1294 case RISCV::SRLI: return RISCV::PseudoCCSRLI; break;
1295 case RISCV::SRAI: return RISCV::PseudoCCSRAI; break;
1296 case RISCV::ANDI: return RISCV::PseudoCCANDI; break;
1297 case RISCV::ORI: return RISCV::PseudoCCORI; break;
1298 case RISCV::XORI: return RISCV::PseudoCCXORI; break;
1299
1300 case RISCV::ADDW: return RISCV::PseudoCCADDW; break;
1301 case RISCV::SUBW: return RISCV::PseudoCCSUBW; break;
1302 case RISCV::SLLW: return RISCV::PseudoCCSLLW; break;
1303 case RISCV::SRLW: return RISCV::PseudoCCSRLW; break;
1304 case RISCV::SRAW: return RISCV::PseudoCCSRAW; break;
1305
1306 case RISCV::ADDIW: return RISCV::PseudoCCADDIW; break;
1307 case RISCV::SLLIW: return RISCV::PseudoCCSLLIW; break;
1308 case RISCV::SRLIW: return RISCV::PseudoCCSRLIW; break;
1309 case RISCV::SRAIW: return RISCV::PseudoCCSRAIW; break;
1310
1311 case RISCV::ANDN: return RISCV::PseudoCCANDN; break;
1312 case RISCV::ORN: return RISCV::PseudoCCORN; break;
1313 case RISCV::XNOR: return RISCV::PseudoCCXNOR; break;
1314 }
1315
1316 return RISCV::INSTRUCTION_LIST_END;
1317}
1318
1319/// Identify instructions that can be folded into a CCMOV instruction, and
1320/// return the defining instruction.
1322 const MachineRegisterInfo &MRI,
1323 const TargetInstrInfo *TII) {
1324 if (!Reg.isVirtual())
1325 return nullptr;
1326 if (!MRI.hasOneNonDBGUse(Reg))
1327 return nullptr;
1328 MachineInstr *MI = MRI.getVRegDef(Reg);
1329 if (!MI)
1330 return nullptr;
1331 // Check if MI can be predicated and folded into the CCMOV.
1332 if (getPredicatedOpcode(MI->getOpcode()) == RISCV::INSTRUCTION_LIST_END)
1333 return nullptr;
1334 // Don't predicate li idiom.
1335 if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(1).isReg() &&
1336 MI->getOperand(1).getReg() == RISCV::X0)
1337 return nullptr;
1338 // Check if MI has any other defs or physreg uses.
1339 for (const MachineOperand &MO : llvm::drop_begin(MI->operands())) {
1340 // Reject frame index operands, PEI can't handle the predicated pseudos.
1341 if (MO.isFI() || MO.isCPI() || MO.isJTI())
1342 return nullptr;
1343 if (!MO.isReg())
1344 continue;
1345 // MI can't have any tied operands, that would conflict with predication.
1346 if (MO.isTied())
1347 return nullptr;
1348 if (MO.isDef())
1349 return nullptr;
1350 // Allow constant physregs.
1351 if (MO.getReg().isPhysical() && !MRI.isConstantPhysReg(MO.getReg()))
1352 return nullptr;
1353 }
1354 bool DontMoveAcrossStores = true;
1355 if (!MI->isSafeToMove(/* AliasAnalysis = */ nullptr, DontMoveAcrossStores))
1356 return nullptr;
1357 return MI;
1358}
1359
1362 unsigned &TrueOp, unsigned &FalseOp,
1363 bool &Optimizable) const {
1364 assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
1365 "Unknown select instruction");
1366 // CCMOV operands:
1367 // 0: Def.
1368 // 1: LHS of compare.
1369 // 2: RHS of compare.
1370 // 3: Condition code.
1371 // 4: False use.
1372 // 5: True use.
1373 TrueOp = 5;
1374 FalseOp = 4;
1375 Cond.push_back(MI.getOperand(1));
1376 Cond.push_back(MI.getOperand(2));
1377 Cond.push_back(MI.getOperand(3));
1378 // We can only fold when we support short forward branch opt.
1379 Optimizable = STI.hasShortForwardBranchOpt();
1380 return false;
1381}
1382
1386 bool PreferFalse) const {
1387 assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
1388 "Unknown select instruction");
1389 if (!STI.hasShortForwardBranchOpt())
1390 return nullptr;
1391
1392 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
1394 canFoldAsPredicatedOp(MI.getOperand(5).getReg(), MRI, this);
1395 bool Invert = !DefMI;
1396 if (!DefMI)
1397 DefMI = canFoldAsPredicatedOp(MI.getOperand(4).getReg(), MRI, this);
1398 if (!DefMI)
1399 return nullptr;
1400
1401 // Find new register class to use.
1402 MachineOperand FalseReg = MI.getOperand(Invert ? 5 : 4);
1403 Register DestReg = MI.getOperand(0).getReg();
1404 const TargetRegisterClass *PreviousClass = MRI.getRegClass(FalseReg.getReg());
1405 if (!MRI.constrainRegClass(DestReg, PreviousClass))
1406 return nullptr;
1407
1408 unsigned PredOpc = getPredicatedOpcode(DefMI->getOpcode());
1409 assert(PredOpc != RISCV::INSTRUCTION_LIST_END && "Unexpected opcode!");
1410
1411 // Create a new predicated version of DefMI.
1412 MachineInstrBuilder NewMI =
1413 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(PredOpc), DestReg);
1414
1415 // Copy the condition portion.
1416 NewMI.add(MI.getOperand(1));
1417 NewMI.add(MI.getOperand(2));
1418
1419 // Add condition code, inverting if necessary.
1420 auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());
1421 if (Invert)
1423 NewMI.addImm(CC);
1424
1425 // Copy the false register.
1426 NewMI.add(FalseReg);
1427
1428 // Copy all the DefMI operands.
1429 const MCInstrDesc &DefDesc = DefMI->getDesc();
1430 for (unsigned i = 1, e = DefDesc.getNumOperands(); i != e; ++i)
1431 NewMI.add(DefMI->getOperand(i));
1432
1433 // Update SeenMIs set: register newly created MI and erase removed DefMI.
1434 SeenMIs.insert(NewMI);
1435 SeenMIs.erase(DefMI);
1436
1437 // If MI is inside a loop, and DefMI is outside the loop, then kill flags on
1438 // DefMI would be invalid when tranferred inside the loop. Checking for a
1439 // loop is expensive, but at least remove kill flags if they are in different
1440 // BBs.
1441 if (DefMI->getParent() != MI.getParent())
1442 NewMI->clearKillInfo();
1443
1444 // The caller will erase MI, but not DefMI.
1446 return NewMI;
1447}
1448
1450 if (MI.isMetaInstruction())
1451 return 0;
1452
1453 unsigned Opcode = MI.getOpcode();
1454
1455 if (Opcode == TargetOpcode::INLINEASM ||
1456 Opcode == TargetOpcode::INLINEASM_BR) {
1457 const MachineFunction &MF = *MI.getParent()->getParent();
1458 const auto &TM = static_cast<const RISCVTargetMachine &>(MF.getTarget());
1459 return getInlineAsmLength(MI.getOperand(0).getSymbolName(),
1460 *TM.getMCAsmInfo());
1461 }
1462
1463 if (!MI.memoperands_empty()) {
1464 MachineMemOperand *MMO = *(MI.memoperands_begin());
1465 const MachineFunction &MF = *MI.getParent()->getParent();
1466 const auto &ST = MF.getSubtarget<RISCVSubtarget>();
1467 if (ST.hasStdExtZihintntl() && MMO->isNonTemporal()) {
1468 if (ST.hasStdExtCOrZca() && ST.enableRVCHintInstrs()) {
1469 if (isCompressibleInst(MI, STI))
1470 return 4; // c.ntl.all + c.load/c.store
1471 return 6; // c.ntl.all + load/store
1472 }
1473 return 8; // ntl.all + load/store
1474 }
1475 }
1476
1477 if (Opcode == TargetOpcode::BUNDLE)
1478 return getInstBundleLength(MI);
1479
1480 if (MI.getParent() && MI.getParent()->getParent()) {
1481 if (isCompressibleInst(MI, STI))
1482 return 2;
1483 }
1484
1485 switch (Opcode) {
1486 case TargetOpcode::STACKMAP:
1487 // The upper bound for a stackmap intrinsic is the full length of its shadow
1489 case TargetOpcode::PATCHPOINT:
1490 // The size of the patchpoint intrinsic is the number of bytes requested
1492 case TargetOpcode::STATEPOINT: {
1493 // The size of the statepoint intrinsic is the number of bytes requested
1494 unsigned NumBytes = StatepointOpers(&MI).getNumPatchBytes();
1495 // No patch bytes means at most a PseudoCall is emitted
1496 return std::max(NumBytes, 8U);
1497 }
1498 default:
1499 return get(Opcode).getSize();
1500 }
1501}
1502
1503unsigned RISCVInstrInfo::getInstBundleLength(const MachineInstr &MI) const {
1504 unsigned Size = 0;
1506 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
1507 while (++I != E && I->isInsideBundle()) {
1508 assert(!I->isBundle() && "No nested bundle!");
1510 }
1511 return Size;
1512}
1513
1515 const unsigned Opcode = MI.getOpcode();
1516 switch (Opcode) {
1517 default:
1518 break;
1519 case RISCV::FSGNJ_D:
1520 case RISCV::FSGNJ_S:
1521 case RISCV::FSGNJ_H:
1522 case RISCV::FSGNJ_D_INX:
1523 case RISCV::FSGNJ_D_IN32X:
1524 case RISCV::FSGNJ_S_INX:
1525 case RISCV::FSGNJ_H_INX:
1526 // The canonical floating-point move is fsgnj rd, rs, rs.
1527 return MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
1528 MI.getOperand(1).getReg() == MI.getOperand(2).getReg();
1529 case RISCV::ADDI:
1530 case RISCV::ORI:
1531 case RISCV::XORI:
1532 return (MI.getOperand(1).isReg() &&
1533 MI.getOperand(1).getReg() == RISCV::X0) ||
1534 (MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0);
1535 }
1536 return MI.isAsCheapAsAMove();
1537}
1538
1539std::optional<DestSourcePair>
1541 if (MI.isMoveReg())
1542 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
1543 switch (MI.getOpcode()) {
1544 default:
1545 break;
1546 case RISCV::ADDI:
1547 // Operand 1 can be a frameindex but callers expect registers
1548 if (MI.getOperand(1).isReg() && MI.getOperand(2).isImm() &&
1549 MI.getOperand(2).getImm() == 0)
1550 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
1551 break;
1552 case RISCV::FSGNJ_D:
1553 case RISCV::FSGNJ_S:
1554 case RISCV::FSGNJ_H:
1555 case RISCV::FSGNJ_D_INX:
1556 case RISCV::FSGNJ_D_IN32X:
1557 case RISCV::FSGNJ_S_INX:
1558 case RISCV::FSGNJ_H_INX:
1559 // The canonical floating-point move is fsgnj rd, rs, rs.
1560 if (MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
1561 MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
1562 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
1563 break;
1564 }
1565 return std::nullopt;
1566}
1567
1569 if (ForceMachineCombinerStrategy.getNumOccurrences() == 0) {
1570 // The option is unused. Choose Local strategy only for in-order cores. When
1571 // scheduling model is unspecified, use MinInstrCount strategy as more
1572 // generic one.
1573 const auto &SchedModel = STI.getSchedModel();
1574 return (!SchedModel.hasInstrSchedModel() || SchedModel.isOutOfOrder())
1577 }
1578 // The strategy was forced by the option.
1580}
1581
1583 MachineInstr &Root, unsigned &Pattern,
1584 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
1585 int16_t FrmOpIdx =
1586 RISCV::getNamedOperandIdx(Root.getOpcode(), RISCV::OpName::frm);
1587 if (FrmOpIdx < 0) {
1588 assert(all_of(InsInstrs,
1589 [](MachineInstr *MI) {
1590 return RISCV::getNamedOperandIdx(MI->getOpcode(),
1591 RISCV::OpName::frm) < 0;
1592 }) &&
1593 "New instructions require FRM whereas the old one does not have it");
1594 return;
1595 }
1596
1597 const MachineOperand &FRM = Root.getOperand(FrmOpIdx);
1598 MachineFunction &MF = *Root.getMF();
1599
1600 for (auto *NewMI : InsInstrs) {
1601 // We'd already added the FRM operand.
1602 if (static_cast<unsigned>(RISCV::getNamedOperandIdx(
1603 NewMI->getOpcode(), RISCV::OpName::frm)) != NewMI->getNumOperands())
1604 continue;
1605 MachineInstrBuilder MIB(MF, NewMI);
1606 MIB.add(FRM);
1607 if (FRM.getImm() == RISCVFPRndMode::DYN)
1608 MIB.addUse(RISCV::FRM, RegState::Implicit);
1609 }
1610}
1611
1612static bool isFADD(unsigned Opc) {
1613 switch (Opc) {
1614 default:
1615 return false;
1616 case RISCV::FADD_H:
1617 case RISCV::FADD_S:
1618 case RISCV::FADD_D:
1619 return true;
1620 }
1621}
1622
1623static bool isFSUB(unsigned Opc) {
1624 switch (Opc) {
1625 default:
1626 return false;
1627 case RISCV::FSUB_H:
1628 case RISCV::FSUB_S:
1629 case RISCV::FSUB_D:
1630 return true;
1631 }
1632}
1633
1634static bool isFMUL(unsigned Opc) {
1635 switch (Opc) {
1636 default:
1637 return false;
1638 case RISCV::FMUL_H:
1639 case RISCV::FMUL_S:
1640 case RISCV::FMUL_D:
1641 return true;
1642 }
1643}
1644
1645bool RISCVInstrInfo::isVectorAssociativeAndCommutative(const MachineInstr &Inst,
1646 bool Invert) const {
1647#define OPCODE_LMUL_CASE(OPC) \
1648 case RISCV::OPC##_M1: \
1649 case RISCV::OPC##_M2: \
1650 case RISCV::OPC##_M4: \
1651 case RISCV::OPC##_M8: \
1652 case RISCV::OPC##_MF2: \
1653 case RISCV::OPC##_MF4: \
1654 case RISCV::OPC##_MF8
1655
1656#define OPCODE_LMUL_MASK_CASE(OPC) \
1657 case RISCV::OPC##_M1_MASK: \
1658 case RISCV::OPC##_M2_MASK: \
1659 case RISCV::OPC##_M4_MASK: \
1660 case RISCV::OPC##_M8_MASK: \
1661 case RISCV::OPC##_MF2_MASK: \
1662 case RISCV::OPC##_MF4_MASK: \
1663 case RISCV::OPC##_MF8_MASK
1664
1665 unsigned Opcode = Inst.getOpcode();
1666 if (Invert) {
1667 if (auto InvOpcode = getInverseOpcode(Opcode))
1668 Opcode = *InvOpcode;
1669 else
1670 return false;
1671 }
1672
1673 // clang-format off
1674 switch (Opcode) {
1675 default:
1676 return false;
1677 OPCODE_LMUL_CASE(PseudoVADD_VV):
1678 OPCODE_LMUL_MASK_CASE(PseudoVADD_VV):
1679 OPCODE_LMUL_CASE(PseudoVMUL_VV):
1680 OPCODE_LMUL_MASK_CASE(PseudoVMUL_VV):
1681 return true;
1682 }
1683 // clang-format on
1684
1685#undef OPCODE_LMUL_MASK_CASE
1686#undef OPCODE_LMUL_CASE
1687}
1688
1689bool RISCVInstrInfo::areRVVInstsReassociable(const MachineInstr &Root,
1690 const MachineInstr &Prev) const {
1691 if (!areOpcodesEqualOrInverse(Root.getOpcode(), Prev.getOpcode()))
1692 return false;
1693
1694 assert(Root.getMF() == Prev.getMF());
1695 const MachineRegisterInfo *MRI = &Root.getMF()->getRegInfo();
1696 const TargetRegisterInfo *TRI = MRI->getTargetRegisterInfo();
1697
1698 // Make sure vtype operands are also the same.
1699 const MCInstrDesc &Desc = get(Root.getOpcode());
1700 const uint64_t TSFlags = Desc.TSFlags;
1701
1702 auto checkImmOperand = [&](unsigned OpIdx) {
1703 return Root.getOperand(OpIdx).getImm() == Prev.getOperand(OpIdx).getImm();
1704 };
1705
1706 auto checkRegOperand = [&](unsigned OpIdx) {
1707 return Root.getOperand(OpIdx).getReg() == Prev.getOperand(OpIdx).getReg();
1708 };
1709
1710 // PassThru
1711 // TODO: Potentially we can loosen the condition to consider Root to be
1712 // associable with Prev if Root has NoReg as passthru. In which case we
1713 // also need to loosen the condition on vector policies between these.
1714 if (!checkRegOperand(1))
1715 return false;
1716
1717 // SEW
1718 if (RISCVII::hasSEWOp(TSFlags) &&
1719 !checkImmOperand(RISCVII::getSEWOpNum(Desc)))
1720 return false;
1721
1722 // Mask
1723 if (RISCVII::usesMaskPolicy(TSFlags)) {
1724 const MachineBasicBlock *MBB = Root.getParent();
1727 Register MI1VReg;
1728
1729 bool SeenMI2 = false;
1730 for (auto End = MBB->rend(), It = It1; It != End; ++It) {
1731 if (It == It2) {
1732 SeenMI2 = true;
1733 if (!MI1VReg.isValid())
1734 // There is no V0 def between Root and Prev; they're sharing the
1735 // same V0.
1736 break;
1737 }
1738
1739 if (It->modifiesRegister(RISCV::V0, TRI)) {
1740 Register SrcReg = It->getOperand(1).getReg();
1741 // If it's not VReg it'll be more difficult to track its defs, so
1742 // bailing out here just to be safe.
1743 if (!SrcReg.isVirtual())
1744 return false;
1745
1746 if (!MI1VReg.isValid()) {
1747 // This is the V0 def for Root.
1748 MI1VReg = SrcReg;
1749 continue;
1750 }
1751
1752 // Some random mask updates.
1753 if (!SeenMI2)
1754 continue;
1755
1756 // This is the V0 def for Prev; check if it's the same as that of
1757 // Root.
1758 if (MI1VReg != SrcReg)
1759 return false;
1760 else
1761 break;
1762 }
1763 }
1764
1765 // If we haven't encountered Prev, it's likely that this function was
1766 // called in a wrong way (e.g. Root is before Prev).
1767 assert(SeenMI2 && "Prev is expected to appear before Root");
1768 }
1769
1770 // Tail / Mask policies
1771 if (RISCVII::hasVecPolicyOp(TSFlags) &&
1772 !checkImmOperand(RISCVII::getVecPolicyOpNum(Desc)))
1773 return false;
1774
1775 // VL
1776 if (RISCVII::hasVLOp(TSFlags)) {
1777 unsigned OpIdx = RISCVII::getVLOpNum(Desc);
1778 const MachineOperand &Op1 = Root.getOperand(OpIdx);
1779 const MachineOperand &Op2 = Prev.getOperand(OpIdx);
1780 if (Op1.getType() != Op2.getType())
1781 return false;
1782 switch (Op1.getType()) {
1784 if (Op1.getReg() != Op2.getReg())
1785 return false;
1786 break;
1788 if (Op1.getImm() != Op2.getImm())
1789 return false;
1790 break;
1791 default:
1792 llvm_unreachable("Unrecognized VL operand type");
1793 }
1794 }
1795
1796 // Rounding modes
1797 if (RISCVII::hasRoundModeOp(TSFlags) &&
1798 !checkImmOperand(RISCVII::getVLOpNum(Desc) - 1))
1799 return false;
1800
1801 return true;
1802}
1803
1804// Most of our RVV pseudos have passthru operand, so the real operands
1805// start from index = 2.
1806bool RISCVInstrInfo::hasReassociableVectorSibling(const MachineInstr &Inst,
1807 bool &Commuted) const {
1808 const MachineBasicBlock *MBB = Inst.getParent();
1811 "Expect the present of passthrough operand.");
1812 MachineInstr *MI1 = MRI.getUniqueVRegDef(Inst.getOperand(2).getReg());
1813 MachineInstr *MI2 = MRI.getUniqueVRegDef(Inst.getOperand(3).getReg());
1814
1815 // If only one operand has the same or inverse opcode and it's the second
1816 // source operand, the operands must be commuted.
1817 Commuted = !areRVVInstsReassociable(Inst, *MI1) &&
1818 areRVVInstsReassociable(Inst, *MI2);
1819 if (Commuted)
1820 std::swap(MI1, MI2);
1821
1822 return areRVVInstsReassociable(Inst, *MI1) &&
1823 (isVectorAssociativeAndCommutative(*MI1) ||
1824 isVectorAssociativeAndCommutative(*MI1, /* Invert */ true)) &&
1826 MRI.hasOneNonDBGUse(MI1->getOperand(0).getReg());
1827}
1828
1830 const MachineInstr &Inst, const MachineBasicBlock *MBB) const {
1831 if (!isVectorAssociativeAndCommutative(Inst) &&
1832 !isVectorAssociativeAndCommutative(Inst, /*Invert=*/true))
1834
1835 const MachineOperand &Op1 = Inst.getOperand(2);
1836 const MachineOperand &Op2 = Inst.getOperand(3);
1838
1839 // We need virtual register definitions for the operands that we will
1840 // reassociate.
1841 MachineInstr *MI1 = nullptr;
1842 MachineInstr *MI2 = nullptr;
1843 if (Op1.isReg() && Op1.getReg().isVirtual())
1844 MI1 = MRI.getUniqueVRegDef(Op1.getReg());
1845 if (Op2.isReg() && Op2.getReg().isVirtual())
1846 MI2 = MRI.getUniqueVRegDef(Op2.getReg());
1847
1848 // And at least one operand must be defined in MBB.
1849 return MI1 && MI2 && (MI1->getParent() == MBB || MI2->getParent() == MBB);
1850}
1851
1853 const MachineInstr &Root, unsigned Pattern,
1854 std::array<unsigned, 5> &OperandIndices) const {
1856 if (RISCV::getRVVMCOpcode(Root.getOpcode())) {
1857 // Skip the passthrough operand, so increment all indices by one.
1858 for (unsigned I = 0; I < 5; ++I)
1859 ++OperandIndices[I];
1860 }
1861}
1862
1864 bool &Commuted) const {
1865 if (isVectorAssociativeAndCommutative(Inst) ||
1866 isVectorAssociativeAndCommutative(Inst, /*Invert=*/true))
1867 return hasReassociableVectorSibling(Inst, Commuted);
1868
1869 if (!TargetInstrInfo::hasReassociableSibling(Inst, Commuted))
1870 return false;
1871
1872 const MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();
1873 unsigned OperandIdx = Commuted ? 2 : 1;
1874 const MachineInstr &Sibling =
1875 *MRI.getVRegDef(Inst.getOperand(OperandIdx).getReg());
1876
1877 int16_t InstFrmOpIdx =
1878 RISCV::getNamedOperandIdx(Inst.getOpcode(), RISCV::OpName::frm);
1879 int16_t SiblingFrmOpIdx =
1880 RISCV::getNamedOperandIdx(Sibling.getOpcode(), RISCV::OpName::frm);
1881
1882 return (InstFrmOpIdx < 0 && SiblingFrmOpIdx < 0) ||
1883 RISCV::hasEqualFRM(Inst, Sibling);
1884}
1885
1887 bool Invert) const {
1888 if (isVectorAssociativeAndCommutative(Inst, Invert))
1889 return true;
1890
1891 unsigned Opc = Inst.getOpcode();
1892 if (Invert) {
1893 auto InverseOpcode = getInverseOpcode(Opc);
1894 if (!InverseOpcode)
1895 return false;
1896 Opc = *InverseOpcode;
1897 }
1898
1899 if (isFADD(Opc) || isFMUL(Opc))
1902
1903 switch (Opc) {
1904 default:
1905 return false;
1906 case RISCV::ADD:
1907 case RISCV::ADDW:
1908 case RISCV::AND:
1909 case RISCV::OR:
1910 case RISCV::XOR:
1911 // From RISC-V ISA spec, if both the high and low bits of the same product
1912 // are required, then the recommended code sequence is:
1913 //
1914 // MULH[[S]U] rdh, rs1, rs2
1915 // MUL rdl, rs1, rs2
1916 // (source register specifiers must be in same order and rdh cannot be the
1917 // same as rs1 or rs2)
1918 //
1919 // Microarchitectures can then fuse these into a single multiply operation
1920 // instead of performing two separate multiplies.
1921 // MachineCombiner may reassociate MUL operands and lose the fusion
1922 // opportunity.
1923 case RISCV::MUL:
1924 case RISCV::MULW:
1925 case RISCV::MIN:
1926 case RISCV::MINU:
1927 case RISCV::MAX:
1928 case RISCV::MAXU:
1929 case RISCV::FMIN_H:
1930 case RISCV::FMIN_S:
1931 case RISCV::FMIN_D:
1932 case RISCV::FMAX_H:
1933 case RISCV::FMAX_S:
1934 case RISCV::FMAX_D:
1935 return true;
1936 }
1937
1938 return false;
1939}
1940
1941std::optional<unsigned>
1942RISCVInstrInfo::getInverseOpcode(unsigned Opcode) const {
1943#define RVV_OPC_LMUL_CASE(OPC, INV) \
1944 case RISCV::OPC##_M1: \
1945 return RISCV::INV##_M1; \
1946 case RISCV::OPC##_M2: \
1947 return RISCV::INV##_M2; \
1948 case RISCV::OPC##_M4: \
1949 return RISCV::INV##_M4; \
1950 case RISCV::OPC##_M8: \
1951 return RISCV::INV##_M8; \
1952 case RISCV::OPC##_MF2: \
1953 return RISCV::INV##_MF2; \
1954 case RISCV::OPC##_MF4: \
1955 return RISCV::INV##_MF4; \
1956 case RISCV::OPC##_MF8: \
1957 return RISCV::INV##_MF8
1958
1959#define RVV_OPC_LMUL_MASK_CASE(OPC, INV) \
1960 case RISCV::OPC##_M1_MASK: \
1961 return RISCV::INV##_M1_MASK; \
1962 case RISCV::OPC##_M2_MASK: \
1963 return RISCV::INV##_M2_MASK; \
1964 case RISCV::OPC##_M4_MASK: \
1965 return RISCV::INV##_M4_MASK; \
1966 case RISCV::OPC##_M8_MASK: \
1967 return RISCV::INV##_M8_MASK; \
1968 case RISCV::OPC##_MF2_MASK: \
1969 return RISCV::INV##_MF2_MASK; \
1970 case RISCV::OPC##_MF4_MASK: \
1971 return RISCV::INV##_MF4_MASK; \
1972 case RISCV::OPC##_MF8_MASK: \
1973 return RISCV::INV##_MF8_MASK
1974
1975 switch (Opcode) {
1976 default:
1977 return std::nullopt;
1978 case RISCV::FADD_H:
1979 return RISCV::FSUB_H;
1980 case RISCV::FADD_S:
1981 return RISCV::FSUB_S;
1982 case RISCV::FADD_D:
1983 return RISCV::FSUB_D;
1984 case RISCV::FSUB_H:
1985 return RISCV::FADD_H;
1986 case RISCV::FSUB_S:
1987 return RISCV::FADD_S;
1988 case RISCV::FSUB_D:
1989 return RISCV::FADD_D;
1990 case RISCV::ADD:
1991 return RISCV::SUB;
1992 case RISCV::SUB:
1993 return RISCV::ADD;
1994 case RISCV::ADDW:
1995 return RISCV::SUBW;
1996 case RISCV::SUBW:
1997 return RISCV::ADDW;
1998 // clang-format off
1999 RVV_OPC_LMUL_CASE(PseudoVADD_VV, PseudoVSUB_VV);
2000 RVV_OPC_LMUL_MASK_CASE(PseudoVADD_VV, PseudoVSUB_VV);
2001 RVV_OPC_LMUL_CASE(PseudoVSUB_VV, PseudoVADD_VV);
2002 RVV_OPC_LMUL_MASK_CASE(PseudoVSUB_VV, PseudoVADD_VV);
2003 // clang-format on
2004 }
2005
2006#undef RVV_OPC_LMUL_MASK_CASE
2007#undef RVV_OPC_LMUL_CASE
2008}
2009
2011 const MachineOperand &MO,
2012 bool DoRegPressureReduce) {
2013 if (!MO.isReg() || !MO.getReg().isVirtual())
2014 return false;
2015 const MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
2016 MachineInstr *MI = MRI.getVRegDef(MO.getReg());
2017 if (!MI || !isFMUL(MI->getOpcode()))
2018 return false;
2019
2022 return false;
2023
2024 // Try combining even if fmul has more than one use as it eliminates
2025 // dependency between fadd(fsub) and fmul. However, it can extend liveranges
2026 // for fmul operands, so reject the transformation in register pressure
2027 // reduction mode.
2028 if (DoRegPressureReduce && !MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
2029 return false;
2030
2031 // Do not combine instructions from different basic blocks.
2032 if (Root.getParent() != MI->getParent())
2033 return false;
2034 return RISCV::hasEqualFRM(Root, *MI);
2035}
2036
2038 SmallVectorImpl<unsigned> &Patterns,
2039 bool DoRegPressureReduce) {
2040 unsigned Opc = Root.getOpcode();
2041 bool IsFAdd = isFADD(Opc);
2042 if (!IsFAdd && !isFSUB(Opc))
2043 return false;
2044 bool Added = false;
2045 if (canCombineFPFusedMultiply(Root, Root.getOperand(1),
2046 DoRegPressureReduce)) {
2049 Added = true;
2050 }
2051 if (canCombineFPFusedMultiply(Root, Root.getOperand(2),
2052 DoRegPressureReduce)) {
2055 Added = true;
2056 }
2057 return Added;
2058}
2059
2060static bool getFPPatterns(MachineInstr &Root,
2061 SmallVectorImpl<unsigned> &Patterns,
2062 bool DoRegPressureReduce) {
2063 return getFPFusedMultiplyPatterns(Root, Patterns, DoRegPressureReduce);
2064}
2065
2066/// Utility routine that checks if \param MO is defined by an
2067/// \param CombineOpc instruction in the basic block \param MBB
2069 const MachineOperand &MO,
2070 unsigned CombineOpc) {
2072 const MachineInstr *MI = nullptr;
2073
2074 if (MO.isReg() && MO.getReg().isVirtual())
2075 MI = MRI.getUniqueVRegDef(MO.getReg());
2076 // And it needs to be in the trace (otherwise, it won't have a depth).
2077 if (!MI || MI->getParent() != &MBB || MI->getOpcode() != CombineOpc)
2078 return nullptr;
2079 // Must only used by the user we combine with.
2080 if (!MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
2081 return nullptr;
2082
2083 return MI;
2084}
2085
2086/// Utility routine that checks if \param MO is defined by a SLLI in \param
2087/// MBB that can be combined by splitting across 2 SHXADD instructions. The
2088/// first SHXADD shift amount is given by \param OuterShiftAmt.
2090 const MachineOperand &MO,
2091 unsigned OuterShiftAmt) {
2092 const MachineInstr *ShiftMI = canCombine(MBB, MO, RISCV::SLLI);
2093 if (!ShiftMI)
2094 return false;
2095
2096 unsigned InnerShiftAmt = ShiftMI->getOperand(2).getImm();
2097 if (InnerShiftAmt < OuterShiftAmt || (InnerShiftAmt - OuterShiftAmt) > 3)
2098 return false;
2099
2100 return true;
2101}
2102
2103// Returns the shift amount from a SHXADD instruction. Returns 0 if the
2104// instruction is not a SHXADD.
2105static unsigned getSHXADDShiftAmount(unsigned Opc) {
2106 switch (Opc) {
2107 default:
2108 return 0;
2109 case RISCV::SH1ADD:
2110 return 1;
2111 case RISCV::SH2ADD:
2112 return 2;
2113 case RISCV::SH3ADD:
2114 return 3;
2115 }
2116}
2117
2118// Look for opportunities to combine (sh3add Z, (add X, (slli Y, 5))) into
2119// (sh3add (sh2add Y, Z), X).
2120static bool getSHXADDPatterns(const MachineInstr &Root,
2121 SmallVectorImpl<unsigned> &Patterns) {
2122 unsigned ShiftAmt = getSHXADDShiftAmount(Root.getOpcode());
2123 if (!ShiftAmt)
2124 return false;
2125
2126 const MachineBasicBlock &MBB = *Root.getParent();
2127
2128 const MachineInstr *AddMI = canCombine(MBB, Root.getOperand(2), RISCV::ADD);
2129 if (!AddMI)
2130 return false;
2131
2132 bool Found = false;
2133 if (canCombineShiftIntoShXAdd(MBB, AddMI->getOperand(1), ShiftAmt)) {
2135 Found = true;
2136 }
2137 if (canCombineShiftIntoShXAdd(MBB, AddMI->getOperand(2), ShiftAmt)) {
2139 Found = true;
2140 }
2141
2142 return Found;
2143}
2144
2146 switch (Pattern) {
2152 default:
2154 }
2155}
2156
2158 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
2159 bool DoRegPressureReduce) const {
2160
2161 if (getFPPatterns(Root, Patterns, DoRegPressureReduce))
2162 return true;
2163
2164 if (getSHXADDPatterns(Root, Patterns))
2165 return true;
2166
2167 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
2168 DoRegPressureReduce);
2169}
2170
2171static unsigned getFPFusedMultiplyOpcode(unsigned RootOpc, unsigned Pattern) {
2172 switch (RootOpc) {
2173 default:
2174 llvm_unreachable("Unexpected opcode");
2175 case RISCV::FADD_H:
2176 return RISCV::FMADD_H;
2177 case RISCV::FADD_S:
2178 return RISCV::FMADD_S;
2179 case RISCV::FADD_D:
2180 return RISCV::FMADD_D;
2181 case RISCV::FSUB_H:
2182 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_H
2183 : RISCV::FNMSUB_H;
2184 case RISCV::FSUB_S:
2185 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_S
2186 : RISCV::FNMSUB_S;
2187 case RISCV::FSUB_D:
2188 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_D
2189 : RISCV::FNMSUB_D;
2190 }
2191}
2192
2193static unsigned getAddendOperandIdx(unsigned Pattern) {
2194 switch (Pattern) {
2195 default:
2196 llvm_unreachable("Unexpected pattern");
2199 return 2;
2202 return 1;
2203 }
2204}
2205
2207 unsigned Pattern,
2210 MachineFunction *MF = Root.getMF();
2213
2214 MachineOperand &Mul1 = Prev.getOperand(1);
2215 MachineOperand &Mul2 = Prev.getOperand(2);
2216 MachineOperand &Dst = Root.getOperand(0);
2218
2219 Register DstReg = Dst.getReg();
2220 unsigned FusedOpc = getFPFusedMultiplyOpcode(Root.getOpcode(), Pattern);
2221 uint32_t IntersectedFlags = Root.getFlags() & Prev.getFlags();
2222 DebugLoc MergedLoc =
2224
2225 bool Mul1IsKill = Mul1.isKill();
2226 bool Mul2IsKill = Mul2.isKill();
2227 bool AddendIsKill = Addend.isKill();
2228
2229 // We need to clear kill flags since we may be extending the live range past
2230 // a kill. If the mul had kill flags, we can preserve those since we know
2231 // where the previous range stopped.
2232 MRI.clearKillFlags(Mul1.getReg());
2233 MRI.clearKillFlags(Mul2.getReg());
2234
2236 BuildMI(*MF, MergedLoc, TII->get(FusedOpc), DstReg)
2237 .addReg(Mul1.getReg(), getKillRegState(Mul1IsKill))
2238 .addReg(Mul2.getReg(), getKillRegState(Mul2IsKill))
2239 .addReg(Addend.getReg(), getKillRegState(AddendIsKill))
2240 .setMIFlags(IntersectedFlags);
2241
2242 InsInstrs.push_back(MIB);
2243 if (MRI.hasOneNonDBGUse(Prev.getOperand(0).getReg()))
2244 DelInstrs.push_back(&Prev);
2245 DelInstrs.push_back(&Root);
2246}
2247
2248// Combine patterns like (sh3add Z, (add X, (slli Y, 5))) to
2249// (sh3add (sh2add Y, Z), X) if the shift amount can be split across two
2250// shXadd instructions. The outer shXadd keeps its original opcode.
2251static void
2252genShXAddAddShift(MachineInstr &Root, unsigned AddOpIdx,
2255 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) {
2256 MachineFunction *MF = Root.getMF();
2259
2260 unsigned OuterShiftAmt = getSHXADDShiftAmount(Root.getOpcode());
2261 assert(OuterShiftAmt != 0 && "Unexpected opcode");
2262
2263 MachineInstr *AddMI = MRI.getUniqueVRegDef(Root.getOperand(2).getReg());
2264 MachineInstr *ShiftMI =
2265 MRI.getUniqueVRegDef(AddMI->getOperand(AddOpIdx).getReg());
2266
2267 unsigned InnerShiftAmt = ShiftMI->getOperand(2).getImm();
2268 assert(InnerShiftAmt >= OuterShiftAmt && "Unexpected shift amount");
2269
2270 unsigned InnerOpc;
2271 switch (InnerShiftAmt - OuterShiftAmt) {
2272 default:
2273 llvm_unreachable("Unexpected shift amount");
2274 case 0:
2275 InnerOpc = RISCV::ADD;
2276 break;
2277 case 1:
2278 InnerOpc = RISCV::SH1ADD;
2279 break;
2280 case 2:
2281 InnerOpc = RISCV::SH2ADD;
2282 break;
2283 case 3:
2284 InnerOpc = RISCV::SH3ADD;
2285 break;
2286 }
2287
2288 const MachineOperand &X = AddMI->getOperand(3 - AddOpIdx);
2289 const MachineOperand &Y = ShiftMI->getOperand(1);
2290 const MachineOperand &Z = Root.getOperand(1);
2291
2292 Register NewVR = MRI.createVirtualRegister(&RISCV::GPRRegClass);
2293
2294 auto MIB1 = BuildMI(*MF, MIMetadata(Root), TII->get(InnerOpc), NewVR)
2295 .addReg(Y.getReg(), getKillRegState(Y.isKill()))
2296 .addReg(Z.getReg(), getKillRegState(Z.isKill()));
2297 auto MIB2 = BuildMI(*MF, MIMetadata(Root), TII->get(Root.getOpcode()),
2298 Root.getOperand(0).getReg())
2299 .addReg(NewVR, RegState::Kill)
2300 .addReg(X.getReg(), getKillRegState(X.isKill()));
2301
2302 InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
2303 InsInstrs.push_back(MIB1);
2304 InsInstrs.push_back(MIB2);
2305 DelInstrs.push_back(ShiftMI);
2306 DelInstrs.push_back(AddMI);
2307 DelInstrs.push_back(&Root);
2308}
2309
2311 MachineInstr &Root, unsigned Pattern,
2314 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
2316 switch (Pattern) {
2317 default:
2319 DelInstrs, InstrIdxForVirtReg);
2320 return;
2323 MachineInstr &Prev = *MRI.getVRegDef(Root.getOperand(1).getReg());
2324 combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
2325 return;
2326 }
2329 MachineInstr &Prev = *MRI.getVRegDef(Root.getOperand(2).getReg());
2330 combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
2331 return;
2332 }
2334 genShXAddAddShift(Root, 1, InsInstrs, DelInstrs, InstrIdxForVirtReg);
2335 return;
2337 genShXAddAddShift(Root, 2, InsInstrs, DelInstrs, InstrIdxForVirtReg);
2338 return;
2339 }
2340}
2341
2343 StringRef &ErrInfo) const {
2344 MCInstrDesc const &Desc = MI.getDesc();
2345
2346 for (const auto &[Index, Operand] : enumerate(Desc.operands())) {
2347 unsigned OpType = Operand.OperandType;
2348 if (OpType >= RISCVOp::OPERAND_FIRST_RISCV_IMM &&
2350 const MachineOperand &MO = MI.getOperand(Index);
2351 if (MO.isImm()) {
2352 int64_t Imm = MO.getImm();
2353 bool Ok;
2354 switch (OpType) {
2355 default:
2356 llvm_unreachable("Unexpected operand type");
2357
2358 // clang-format off
2359#define CASE_OPERAND_UIMM(NUM) \
2360 case RISCVOp::OPERAND_UIMM##NUM: \
2361 Ok = isUInt<NUM>(Imm); \
2362 break;
2373 // clang-format on
2375 Ok = isShiftedUInt<1, 1>(Imm);
2376 break;
2378 Ok = isShiftedUInt<5, 2>(Imm);
2379 break;
2381 Ok = isShiftedUInt<6, 2>(Imm);
2382 break;
2384 Ok = isShiftedUInt<5, 3>(Imm);
2385 break;
2387 Ok = isUInt<8>(Imm) && Imm >= 32;
2388 break;
2390 Ok = isShiftedUInt<6, 3>(Imm);
2391 break;
2393 Ok = isShiftedInt<6, 4>(Imm) && (Imm != 0);
2394 break;
2396 Ok = isShiftedUInt<8, 2>(Imm) && (Imm != 0);
2397 break;
2399 Ok = Imm == 0;
2400 break;
2402 Ok = isInt<5>(Imm);
2403 break;
2405 Ok = (isInt<5>(Imm) && Imm != -16) || Imm == 16;
2406 break;
2408 Ok = isInt<6>(Imm);
2409 break;
2411 Ok = Imm != 0 && isInt<6>(Imm);
2412 break;
2414 Ok = isUInt<10>(Imm);
2415 break;
2417 Ok = isUInt<11>(Imm);
2418 break;
2420 Ok = isInt<12>(Imm);
2421 break;
2423 Ok = isShiftedInt<7, 5>(Imm);
2424 break;
2426 Ok = STI.is64Bit() ? isUInt<6>(Imm) : isUInt<5>(Imm);
2427 break;
2429 Ok = STI.is64Bit() ? isUInt<6>(Imm) : isUInt<5>(Imm);
2430 Ok = Ok && Imm != 0;
2431 break;
2433 Ok = (isUInt<5>(Imm) && Imm != 0) ||
2434 (Imm >= 0xfffe0 && Imm <= 0xfffff);
2435 break;
2437 Ok = Imm >= 0 && Imm <= 10;
2438 break;
2440 Ok = Imm >= 0 && Imm <= 7;
2441 break;
2443 Ok = Imm >= 1 && Imm <= 10;
2444 break;
2446 Ok = Imm >= 2 && Imm <= 14;
2447 break;
2449 Ok = (Imm & 0xf) == 0;
2450 break;
2451 }
2452 if (!Ok) {
2453 ErrInfo = "Invalid immediate";
2454 return false;
2455 }
2456 }
2457 }
2458 }
2459
2460 const uint64_t TSFlags = Desc.TSFlags;
2461 if (RISCVII::hasVLOp(TSFlags)) {
2462 const MachineOperand &Op = MI.getOperand(RISCVII::getVLOpNum(Desc));
2463 if (!Op.isImm() && !Op.isReg()) {
2464 ErrInfo = "Invalid operand type for VL operand";
2465 return false;
2466 }
2467 if (Op.isReg() && Op.getReg() != RISCV::NoRegister) {
2468 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
2469 auto *RC = MRI.getRegClass(Op.getReg());
2470 if (!RISCV::GPRRegClass.hasSubClassEq(RC)) {
2471 ErrInfo = "Invalid register class for VL operand";
2472 return false;
2473 }
2474 }
2475 if (!RISCVII::hasSEWOp(TSFlags)) {
2476 ErrInfo = "VL operand w/o SEW operand?";
2477 return false;
2478 }
2479 }
2480 if (RISCVII::hasSEWOp(TSFlags)) {
2481 unsigned OpIdx = RISCVII::getSEWOpNum(Desc);
2482 if (!MI.getOperand(OpIdx).isImm()) {
2483 ErrInfo = "SEW value expected to be an immediate";
2484 return false;
2485 }
2486 uint64_t Log2SEW = MI.getOperand(OpIdx).getImm();
2487 if (Log2SEW > 31) {
2488 ErrInfo = "Unexpected SEW value";
2489 return false;
2490 }
2491 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
2492 if (!RISCVVType::isValidSEW(SEW)) {
2493 ErrInfo = "Unexpected SEW value";
2494 return false;
2495 }
2496 }
2497 if (RISCVII::hasVecPolicyOp(TSFlags)) {
2498 unsigned OpIdx = RISCVII::getVecPolicyOpNum(Desc);
2499 if (!MI.getOperand(OpIdx).isImm()) {
2500 ErrInfo = "Policy operand expected to be an immediate";
2501 return false;
2502 }
2503 uint64_t Policy = MI.getOperand(OpIdx).getImm();
2505 ErrInfo = "Invalid Policy Value";
2506 return false;
2507 }
2508 if (!RISCVII::hasVLOp(TSFlags)) {
2509 ErrInfo = "policy operand w/o VL operand?";
2510 return false;
2511 }
2512
2513 // VecPolicy operands can only exist on instructions with passthru/merge
2514 // arguments. Note that not all arguments with passthru have vec policy
2515 // operands- some instructions have implicit policies.
2516 unsigned UseOpIdx;
2517 if (!MI.isRegTiedToUseOperand(0, &UseOpIdx)) {
2518 ErrInfo = "policy operand w/o tied operand?";
2519 return false;
2520 }
2521 }
2522
2523 return true;
2524}
2525
2527 const MachineInstr &AddrI,
2528 ExtAddrMode &AM) const {
2529 switch (MemI.getOpcode()) {
2530 default:
2531 return false;
2532 case RISCV::LB:
2533 case RISCV::LBU:
2534 case RISCV::LH:
2535 case RISCV::LHU:
2536 case RISCV::LW:
2537 case RISCV::LWU:
2538 case RISCV::LD:
2539 case RISCV::FLH:
2540 case RISCV::FLW:
2541 case RISCV::FLD:
2542 case RISCV::SB:
2543 case RISCV::SH:
2544 case RISCV::SW:
2545 case RISCV::SD:
2546 case RISCV::FSH:
2547 case RISCV::FSW:
2548 case RISCV::FSD:
2549 break;
2550 }
2551
2552 if (MemI.getOperand(0).getReg() == Reg)
2553 return false;
2554
2555 if (AddrI.getOpcode() != RISCV::ADDI || !AddrI.getOperand(1).isReg() ||
2556 !AddrI.getOperand(2).isImm())
2557 return false;
2558
2559 int64_t OldOffset = MemI.getOperand(2).getImm();
2560 int64_t Disp = AddrI.getOperand(2).getImm();
2561 int64_t NewOffset = OldOffset + Disp;
2562 if (!STI.is64Bit())
2563 NewOffset = SignExtend64<32>(NewOffset);
2564
2565 if (!isInt<12>(NewOffset))
2566 return false;
2567
2568 AM.BaseReg = AddrI.getOperand(1).getReg();
2569 AM.ScaledReg = 0;
2570 AM.Scale = 0;
2571 AM.Displacement = NewOffset;
2573 return true;
2574}
2575
2577 const ExtAddrMode &AM) const {
2578
2579 const DebugLoc &DL = MemI.getDebugLoc();
2580 MachineBasicBlock &MBB = *MemI.getParent();
2581
2582 assert(AM.ScaledReg == 0 && AM.Scale == 0 &&
2583 "Addressing mode not supported for folding");
2584
2585 return BuildMI(MBB, MemI, DL, get(MemI.getOpcode()))
2586 .addReg(MemI.getOperand(0).getReg(),
2587 MemI.mayLoad() ? RegState::Define : 0)
2588 .addReg(AM.BaseReg)
2589 .addImm(AM.Displacement)
2590 .setMemRefs(MemI.memoperands())
2591 .setMIFlags(MemI.getFlags());
2592}
2593
2596 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
2597 const TargetRegisterInfo *TRI) const {
2598 if (!LdSt.mayLoadOrStore())
2599 return false;
2600
2601 // Conservatively, only handle scalar loads/stores for now.
2602 switch (LdSt.getOpcode()) {
2603 case RISCV::LB:
2604 case RISCV::LBU:
2605 case RISCV::SB:
2606 case RISCV::LH:
2607 case RISCV::LHU:
2608 case RISCV::FLH:
2609 case RISCV::SH:
2610 case RISCV::FSH:
2611 case RISCV::LW:
2612 case RISCV::LWU:
2613 case RISCV::FLW:
2614 case RISCV::SW:
2615 case RISCV::FSW:
2616 case RISCV::LD:
2617 case RISCV::FLD:
2618 case RISCV::SD:
2619 case RISCV::FSD:
2620 break;
2621 default:
2622 return false;
2623 }
2624 const MachineOperand *BaseOp;
2625 OffsetIsScalable = false;
2626 if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
2627 return false;
2628 BaseOps.push_back(BaseOp);
2629 return true;
2630}
2631
2632// TODO: This was copied from SIInstrInfo. Could it be lifted to a common
2633// helper?
2636 const MachineInstr &MI2,
2638 // Only examine the first "base" operand of each instruction, on the
2639 // assumption that it represents the real base address of the memory access.
2640 // Other operands are typically offsets or indices from this base address.
2641 if (BaseOps1.front()->isIdenticalTo(*BaseOps2.front()))
2642 return true;
2643
2644 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
2645 return false;
2646
2647 auto MO1 = *MI1.memoperands_begin();
2648 auto MO2 = *MI2.memoperands_begin();
2649 if (MO1->getAddrSpace() != MO2->getAddrSpace())
2650 return false;
2651
2652 auto Base1 = MO1->getValue();
2653 auto Base2 = MO2->getValue();
2654 if (!Base1 || !Base2)
2655 return false;
2656 Base1 = getUnderlyingObject(Base1);
2657 Base2 = getUnderlyingObject(Base2);
2658
2659 if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
2660 return false;
2661
2662 return Base1 == Base2;
2663}
2664
2666 ArrayRef<const MachineOperand *> BaseOps1, int64_t Offset1,
2667 bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
2668 int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize,
2669 unsigned NumBytes) const {
2670 // If the mem ops (to be clustered) do not have the same base ptr, then they
2671 // should not be clustered
2672 if (!BaseOps1.empty() && !BaseOps2.empty()) {
2673 const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();
2674 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
2675 if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOps1, SecondLdSt, BaseOps2))
2676 return false;
2677 } else if (!BaseOps1.empty() || !BaseOps2.empty()) {
2678 // If only one base op is empty, they do not have the same base ptr
2679 return false;
2680 }
2681
2682 unsigned CacheLineSize =
2683 BaseOps1.front()->getParent()->getMF()->getSubtarget().getCacheLineSize();
2684 // Assume a cache line size of 64 bytes if no size is set in RISCVSubtarget.
2686 // Cluster if the memory operations are on the same or a neighbouring cache
2687 // line, but limit the maximum ClusterSize to avoid creating too much
2688 // additional register pressure.
2689 return ClusterSize <= 4 && std::abs(Offset1 - Offset2) < CacheLineSize;
2690}
2691
2692// Set BaseReg (the base register operand), Offset (the byte offset being
2693// accessed) and the access Width of the passed instruction that reads/writes
2694// memory. Returns false if the instruction does not read/write memory or the
2695// BaseReg/Offset/Width can't be determined. Is not guaranteed to always
2696// recognise base operands and offsets in all cases.
2697// TODO: Add an IsScalable bool ref argument (like the equivalent AArch64
2698// function) and set it as appropriate.
2700 const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
2701 LocationSize &Width, const TargetRegisterInfo *TRI) const {
2702 if (!LdSt.mayLoadOrStore())
2703 return false;
2704
2705 // Here we assume the standard RISC-V ISA, which uses a base+offset
2706 // addressing mode. You'll need to relax these conditions to support custom
2707 // load/store instructions.
2708 if (LdSt.getNumExplicitOperands() != 3)
2709 return false;
2710 if ((!LdSt.getOperand(1).isReg() && !LdSt.getOperand(1).isFI()) ||
2711 !LdSt.getOperand(2).isImm())
2712 return false;
2713
2714 if (!LdSt.hasOneMemOperand())
2715 return false;
2716
2717 Width = (*LdSt.memoperands_begin())->getSize();
2718 BaseReg = &LdSt.getOperand(1);
2719 Offset = LdSt.getOperand(2).getImm();
2720 return true;
2721}
2722
2724 const MachineInstr &MIa, const MachineInstr &MIb) const {
2725 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
2726 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
2727
2730 return false;
2731
2732 // Retrieve the base register, offset from the base register and width. Width
2733 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If
2734 // base registers are identical, and the offset of a lower memory access +
2735 // the width doesn't overlap the offset of a higher memory access,
2736 // then the memory accesses are different.
2738 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
2739 int64_t OffsetA = 0, OffsetB = 0;
2740 LocationSize WidthA = 0, WidthB = 0;
2741 if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
2742 getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
2743 if (BaseOpA->isIdenticalTo(*BaseOpB)) {
2744 int LowOffset = std::min(OffsetA, OffsetB);
2745 int HighOffset = std::max(OffsetA, OffsetB);
2746 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
2747 if (LowWidth.hasValue() &&
2748 LowOffset + (int)LowWidth.getValue() <= HighOffset)
2749 return true;
2750 }
2751 }
2752 return false;
2753}
2754
2755std::pair<unsigned, unsigned>
2757 const unsigned Mask = RISCVII::MO_DIRECT_FLAG_MASK;
2758 return std::make_pair(TF & Mask, TF & ~Mask);
2759}
2760
2763 using namespace RISCVII;
2764 static const std::pair<unsigned, const char *> TargetFlags[] = {
2765 {MO_CALL, "riscv-call"},
2766 {MO_LO, "riscv-lo"},
2767 {MO_HI, "riscv-hi"},
2768 {MO_PCREL_LO, "riscv-pcrel-lo"},
2769 {MO_PCREL_HI, "riscv-pcrel-hi"},
2770 {MO_GOT_HI, "riscv-got-hi"},
2771 {MO_TPREL_LO, "riscv-tprel-lo"},
2772 {MO_TPREL_HI, "riscv-tprel-hi"},
2773 {MO_TPREL_ADD, "riscv-tprel-add"},
2774 {MO_TLS_GOT_HI, "riscv-tls-got-hi"},
2775 {MO_TLS_GD_HI, "riscv-tls-gd-hi"},
2776 {MO_TLSDESC_HI, "riscv-tlsdesc-hi"},
2777 {MO_TLSDESC_LOAD_LO, "riscv-tlsdesc-load-lo"},
2778 {MO_TLSDESC_ADD_LO, "riscv-tlsdesc-add-lo"},
2779 {MO_TLSDESC_CALL, "riscv-tlsdesc-call"}};
2780 return ArrayRef(TargetFlags);
2781}
2783 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
2784 const Function &F = MF.getFunction();
2785
2786 // Can F be deduplicated by the linker? If it can, don't outline from it.
2787 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
2788 return false;
2789
2790 // Don't outline from functions with section markings; the program could
2791 // expect that all the code is in the named section.
2792 if (F.hasSection())
2793 return false;
2794
2795 // It's safe to outline from MF.
2796 return true;
2797}
2798
2800 unsigned &Flags) const {
2801 // More accurate safety checking is done in getOutliningCandidateInfo.
2803}
2804
2805// Enum values indicating how an outlined call should be constructed.
2809
2811 MachineFunction &MF) const {
2812 return MF.getFunction().hasMinSize();
2813}
2814
2815std::optional<outliner::OutlinedFunction>
2817 std::vector<outliner::Candidate> &RepeatedSequenceLocs) const {
2818
2819 // First we need to filter out candidates where the X5 register (IE t0) can't
2820 // be used to setup the function call.
2821 auto CannotInsertCall = [](outliner::Candidate &C) {
2822 const TargetRegisterInfo *TRI = C.getMF()->getSubtarget().getRegisterInfo();
2823 return !C.isAvailableAcrossAndOutOfSeq(RISCV::X5, *TRI);
2824 };
2825
2826 llvm::erase_if(RepeatedSequenceLocs, CannotInsertCall);
2827
2828 // If the sequence doesn't have enough candidates left, then we're done.
2829 if (RepeatedSequenceLocs.size() < 2)
2830 return std::nullopt;
2831
2832 unsigned SequenceSize = 0;
2833
2834 for (auto &MI : RepeatedSequenceLocs[0])
2835 SequenceSize += getInstSizeInBytes(MI);
2836
2837 // call t0, function = 8 bytes.
2838 unsigned CallOverhead = 8;
2839 for (auto &C : RepeatedSequenceLocs)
2840 C.setCallInfo(MachineOutlinerDefault, CallOverhead);
2841
2842 // jr t0 = 4 bytes, 2 bytes if compressed instructions are enabled.
2843 unsigned FrameOverhead = 4;
2844 if (RepeatedSequenceLocs[0]
2845 .getMF()
2846 ->getSubtarget<RISCVSubtarget>()
2847 .hasStdExtCOrZca())
2848 FrameOverhead = 2;
2849
2850 return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize,
2851 FrameOverhead, MachineOutlinerDefault);
2852}
2853
2856 unsigned Flags) const {
2857 MachineInstr &MI = *MBBI;
2858 MachineBasicBlock *MBB = MI.getParent();
2859 const TargetRegisterInfo *TRI =
2861 const auto &F = MI.getMF()->getFunction();
2862
2863 // We can manually strip out CFI instructions later.
2864 if (MI.isCFIInstruction())
2865 // If current function has exception handling code, we can't outline &
2866 // strip these CFI instructions since it may break .eh_frame section
2867 // needed in unwinding.
2868 return F.needsUnwindTableEntry() ? outliner::InstrType::Illegal
2870
2871 // We need support for tail calls to outlined functions before return
2872 // statements can be allowed.
2873 if (MI.isReturn())
2875
2876 // Don't allow modifying the X5 register which we use for return addresses for
2877 // these outlined functions.
2878 if (MI.modifiesRegister(RISCV::X5, TRI) ||
2879 MI.getDesc().hasImplicitDefOfPhysReg(RISCV::X5))
2881
2882 // Make sure the operands don't reference something unsafe.
2883 for (const auto &MO : MI.operands()) {
2884
2885 // pcrel-hi and pcrel-lo can't put in separate sections, filter that out
2886 // if any possible.
2887 if (MO.getTargetFlags() == RISCVII::MO_PCREL_LO &&
2888 (MI.getMF()->getTarget().getFunctionSections() || F.hasComdat() ||
2889 F.hasSection()))
2891 }
2892
2894}
2895
2898 const outliner::OutlinedFunction &OF) const {
2899
2900 // Strip out any CFI instructions
2901 bool Changed = true;
2902 while (Changed) {
2903 Changed = false;
2904 auto I = MBB.begin();
2905 auto E = MBB.end();
2906 for (; I != E; ++I) {
2907 if (I->isCFIInstruction()) {
2908 I->removeFromParent();
2909 Changed = true;
2910 break;
2911 }
2912 }
2913 }
2914
2915 MBB.addLiveIn(RISCV::X5);
2916
2917 // Add in a return instruction to the end of the outlined frame.
2918 MBB.insert(MBB.end(), BuildMI(MF, DebugLoc(), get(RISCV::JALR))
2919 .addReg(RISCV::X0, RegState::Define)
2920 .addReg(RISCV::X5)
2921 .addImm(0));
2922}
2923
2927
2928 // Add in a call instruction to the outlined function at the given location.
2929 It = MBB.insert(It,
2930 BuildMI(MF, DebugLoc(), get(RISCV::PseudoCALLReg), RISCV::X5)
2931 .addGlobalAddress(M.getNamedValue(MF.getName()), 0,
2933 return It;
2934}
2935
2936std::optional<RegImmPair> RISCVInstrInfo::isAddImmediate(const MachineInstr &MI,
2937 Register Reg) const {
2938 // TODO: Handle cases where Reg is a super- or sub-register of the
2939 // destination register.
2940 const MachineOperand &Op0 = MI.getOperand(0);
2941 if (!Op0.isReg() || Reg != Op0.getReg())
2942 return std::nullopt;
2943
2944 // Don't consider ADDIW as a candidate because the caller may not be aware
2945 // of its sign extension behaviour.
2946 if (MI.getOpcode() == RISCV::ADDI && MI.getOperand(1).isReg() &&
2947 MI.getOperand(2).isImm())
2948 return RegImmPair{MI.getOperand(1).getReg(), MI.getOperand(2).getImm()};
2949
2950 return std::nullopt;
2951}
2952
2953// MIR printer helper function to annotate Operands with a comment.
2955 const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,
2956 const TargetRegisterInfo *TRI) const {
2957 // Print a generic comment for this operand if there is one.
2958 std::string GenericComment =
2960 if (!GenericComment.empty())
2961 return GenericComment;
2962
2963 // If not, we must have an immediate operand.
2964 if (!Op.isImm())
2965 return std::string();
2966
2967 std::string Comment;
2968 raw_string_ostream OS(Comment);
2969
2970 uint64_t TSFlags = MI.getDesc().TSFlags;
2971
2972 // Print the full VType operand of vsetvli/vsetivli instructions, and the SEW
2973 // operand of vector codegen pseudos.
2974 if ((MI.getOpcode() == RISCV::VSETVLI || MI.getOpcode() == RISCV::VSETIVLI ||
2975 MI.getOpcode() == RISCV::PseudoVSETVLI ||
2976 MI.getOpcode() == RISCV::PseudoVSETIVLI ||
2977 MI.getOpcode() == RISCV::PseudoVSETVLIX0) &&
2978 OpIdx == 2) {
2979 unsigned Imm = MI.getOperand(OpIdx).getImm();
2981 } else if (RISCVII::hasSEWOp(TSFlags) &&
2982 OpIdx == RISCVII::getSEWOpNum(MI.getDesc())) {
2983 unsigned Log2SEW = MI.getOperand(OpIdx).getImm();
2984 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
2985 assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW");
2986 OS << "e" << SEW;
2987 } else if (RISCVII::hasVecPolicyOp(TSFlags) &&
2988 OpIdx == RISCVII::getVecPolicyOpNum(MI.getDesc())) {
2989 unsigned Policy = MI.getOperand(OpIdx).getImm();
2991 "Invalid Policy Value");
2992 OS << (Policy & RISCVII::TAIL_AGNOSTIC ? "ta" : "tu") << ", "
2993 << (Policy & RISCVII::MASK_AGNOSTIC ? "ma" : "mu");
2994 }
2995
2996 OS.flush();
2997 return Comment;
2998}
2999
3000// clang-format off
3001#define CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL) \
3002 RISCV::Pseudo##OP##_##LMUL
3003
3004#define CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL) \
3005 RISCV::Pseudo##OP##_##LMUL##_MASK
3006
3007#define CASE_RVV_OPCODE_LMUL(OP, LMUL) \
3008 CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL): \
3009 case CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL)
3010
3011#define CASE_RVV_OPCODE_UNMASK_WIDEN(OP) \
3012 CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF8): \
3013 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF4): \
3014 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF2): \
3015 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M1): \
3016 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M2): \
3017 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M4)
3018
3019#define CASE_RVV_OPCODE_UNMASK(OP) \
3020 CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
3021 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M8)
3022
3023#define CASE_RVV_OPCODE_MASK_WIDEN(OP) \
3024 CASE_RVV_OPCODE_MASK_LMUL(OP, MF8): \
3025 case CASE_RVV_OPCODE_MASK_LMUL(OP, MF4): \
3026 case CASE_RVV_OPCODE_MASK_LMUL(OP, MF2): \
3027 case CASE_RVV_OPCODE_MASK_LMUL(OP, M1): \
3028 case CASE_RVV_OPCODE_MASK_LMUL(OP, M2): \
3029 case CASE_RVV_OPCODE_MASK_LMUL(OP, M4)
3030
3031#define CASE_RVV_OPCODE_MASK(OP) \
3032 CASE_RVV_OPCODE_MASK_WIDEN(OP): \
3033 case CASE_RVV_OPCODE_MASK_LMUL(OP, M8)
3034
3035#define CASE_RVV_OPCODE_WIDEN(OP) \
3036 CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
3037 case CASE_RVV_OPCODE_MASK_WIDEN(OP)
3038
3039#define CASE_RVV_OPCODE(OP) \
3040 CASE_RVV_OPCODE_UNMASK(OP): \
3041 case CASE_RVV_OPCODE_MASK(OP)
3042// clang-format on
3043
3044// clang-format off
3045#define CASE_VMA_OPCODE_COMMON(OP, TYPE, LMUL) \
3046 RISCV::PseudoV##OP##_##TYPE##_##LMUL
3047
3048#define CASE_VMA_OPCODE_LMULS_M1(OP, TYPE) \
3049 CASE_VMA_OPCODE_COMMON(OP, TYPE, M1): \
3050 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M2): \
3051 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M4): \
3052 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M8)
3053
3054#define CASE_VMA_OPCODE_LMULS_MF2(OP, TYPE) \
3055 CASE_VMA_OPCODE_COMMON(OP, TYPE, MF2): \
3056 case CASE_VMA_OPCODE_LMULS_M1(OP, TYPE)
3057
3058#define CASE_VMA_OPCODE_LMULS_MF4(OP, TYPE) \
3059 CASE_VMA_OPCODE_COMMON(OP, TYPE, MF4): \
3060 case CASE_VMA_OPCODE_LMULS_MF2(OP, TYPE)
3061
3062#define CASE_VMA_OPCODE_LMULS(OP, TYPE) \
3063 CASE_VMA_OPCODE_COMMON(OP, TYPE, MF8): \
3064 case CASE_VMA_OPCODE_LMULS_MF4(OP, TYPE)
3065
3066// VFMA instructions are SEW specific.
3067#define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL, SEW) \
3068 RISCV::PseudoV##OP##_##TYPE##_##LMUL##_##SEW
3069
3070#define CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW) \
3071 CASE_VFMA_OPCODE_COMMON(OP, TYPE, M1, SEW): \
3072 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M2, SEW): \
3073 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M4, SEW): \
3074 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M8, SEW)
3075
3076#define CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW) \
3077 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF2, SEW): \
3078 case CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW)
3079
3080#define CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE, SEW) \
3081 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF4, SEW): \
3082 case CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW)
3083
3084#define CASE_VFMA_OPCODE_VV(OP) \
3085 CASE_VFMA_OPCODE_LMULS_MF4(OP, VV, E16): \
3086 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VV, E32): \
3087 case CASE_VFMA_OPCODE_LMULS_M1(OP, VV, E64)
3088
3089#define CASE_VFMA_SPLATS(OP) \
3090 CASE_VFMA_OPCODE_LMULS_MF4(OP, VFPR16, E16): \
3091 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VFPR32, E32): \
3092 case CASE_VFMA_OPCODE_LMULS_M1(OP, VFPR64, E64)
3093// clang-format on
3094
3096 unsigned &SrcOpIdx1,
3097 unsigned &SrcOpIdx2) const {
3098 const MCInstrDesc &Desc = MI.getDesc();
3099 if (!Desc.isCommutable())
3100 return false;
3101
3102 switch (MI.getOpcode()) {
3103 case RISCV::TH_MVEQZ:
3104 case RISCV::TH_MVNEZ:
3105 // We can't commute operands if operand 2 (i.e., rs1 in
3106 // mveqz/mvnez rd,rs1,rs2) is the zero-register (as it is
3107 // not valid as the in/out-operand 1).
3108 if (MI.getOperand(2).getReg() == RISCV::X0)
3109 return false;
3110 // Operands 1 and 2 are commutable, if we switch the opcode.
3111 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1, 2);
3112 case RISCV::TH_MULA:
3113 case RISCV::TH_MULAW:
3114 case RISCV::TH_MULAH:
3115 case RISCV::TH_MULS:
3116 case RISCV::TH_MULSW:
3117 case RISCV::TH_MULSH:
3118 // Operands 2 and 3 are commutable.
3119 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
3120 case RISCV::PseudoCCMOVGPRNoX0:
3121 case RISCV::PseudoCCMOVGPR:
3122 // Operands 4 and 5 are commutable.
3123 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 4, 5);
3124 case CASE_RVV_OPCODE(VADD_VV):
3125 case CASE_RVV_OPCODE(VAND_VV):
3126 case CASE_RVV_OPCODE(VOR_VV):
3127 case CASE_RVV_OPCODE(VXOR_VV):
3128 case CASE_RVV_OPCODE_MASK(VMSEQ_VV):
3129 case CASE_RVV_OPCODE_MASK(VMSNE_VV):
3130 case CASE_RVV_OPCODE(VMIN_VV):
3131 case CASE_RVV_OPCODE(VMINU_VV):
3132 case CASE_RVV_OPCODE(VMAX_VV):
3133 case CASE_RVV_OPCODE(VMAXU_VV):
3134 case CASE_RVV_OPCODE(VMUL_VV):
3135 case CASE_RVV_OPCODE(VMULH_VV):
3136 case CASE_RVV_OPCODE(VMULHU_VV):
3137 case CASE_RVV_OPCODE_WIDEN(VWADD_VV):
3138 case CASE_RVV_OPCODE_WIDEN(VWADDU_VV):
3139 case CASE_RVV_OPCODE_WIDEN(VWMUL_VV):
3140 case CASE_RVV_OPCODE_WIDEN(VWMULU_VV):
3141 case CASE_RVV_OPCODE_WIDEN(VWMACC_VV):
3142 case CASE_RVV_OPCODE_WIDEN(VWMACCU_VV):
3143 case CASE_RVV_OPCODE_UNMASK(VADC_VVM):
3144 case CASE_RVV_OPCODE(VSADD_VV):
3145 case CASE_RVV_OPCODE(VSADDU_VV):
3146 case CASE_RVV_OPCODE(VAADD_VV):
3147 case CASE_RVV_OPCODE(VAADDU_VV):
3148 case CASE_RVV_OPCODE(VSMUL_VV):
3149 // Operands 2 and 3 are commutable.
3150 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
3151 case CASE_VFMA_SPLATS(FMADD):
3152 case CASE_VFMA_SPLATS(FMSUB):
3153 case CASE_VFMA_SPLATS(FMACC):
3154 case CASE_VFMA_SPLATS(FMSAC):
3157 case CASE_VFMA_SPLATS(FNMACC):
3158 case CASE_VFMA_SPLATS(FNMSAC):
3159 case CASE_VFMA_OPCODE_VV(FMACC):
3160 case CASE_VFMA_OPCODE_VV(FMSAC):
3161 case CASE_VFMA_OPCODE_VV(FNMACC):
3162 case CASE_VFMA_OPCODE_VV(FNMSAC):
3163 case CASE_VMA_OPCODE_LMULS(MADD, VX):
3164 case CASE_VMA_OPCODE_LMULS(NMSUB, VX):
3165 case CASE_VMA_OPCODE_LMULS(MACC, VX):
3166 case CASE_VMA_OPCODE_LMULS(NMSAC, VX):
3167 case CASE_VMA_OPCODE_LMULS(MACC, VV):
3168 case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
3169 // If the tail policy is undisturbed we can't commute.
3170 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
3171 if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0)
3172 return false;
3173
3174 // For these instructions we can only swap operand 1 and operand 3 by
3175 // changing the opcode.
3176 unsigned CommutableOpIdx1 = 1;
3177 unsigned CommutableOpIdx2 = 3;
3178 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
3179 CommutableOpIdx2))
3180 return false;
3181 return true;
3182 }
3183 case CASE_VFMA_OPCODE_VV(FMADD):
3187 case CASE_VMA_OPCODE_LMULS(MADD, VV):
3188 case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
3189 // If the tail policy is undisturbed we can't commute.
3190 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
3191 if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0)
3192 return false;
3193
3194 // For these instructions we have more freedom. We can commute with the
3195 // other multiplicand or with the addend/subtrahend/minuend.
3196
3197 // Any fixed operand must be from source 1, 2 or 3.
3198 if (SrcOpIdx1 != CommuteAnyOperandIndex && SrcOpIdx1 > 3)
3199 return false;
3200 if (SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx2 > 3)
3201 return false;
3202
3203 // It both ops are fixed one must be the tied source.
3204 if (SrcOpIdx1 != CommuteAnyOperandIndex &&
3205 SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx1 != 1 && SrcOpIdx2 != 1)
3206 return false;
3207
3208 // Look for two different register operands assumed to be commutable
3209 // regardless of the FMA opcode. The FMA opcode is adjusted later if
3210 // needed.
3211 if (SrcOpIdx1 == CommuteAnyOperandIndex ||
3212 SrcOpIdx2 == CommuteAnyOperandIndex) {
3213 // At least one of operands to be commuted is not specified and
3214 // this method is free to choose appropriate commutable operands.
3215 unsigned CommutableOpIdx1 = SrcOpIdx1;
3216 if (SrcOpIdx1 == SrcOpIdx2) {
3217 // Both of operands are not fixed. Set one of commutable
3218 // operands to the tied source.
3219 CommutableOpIdx1 = 1;
3220 } else if (SrcOpIdx1 == CommuteAnyOperandIndex) {
3221 // Only one of the operands is not fixed.
3222 CommutableOpIdx1 = SrcOpIdx2;
3223 }
3224
3225 // CommutableOpIdx1 is well defined now. Let's choose another commutable
3226 // operand and assign its index to CommutableOpIdx2.
3227 unsigned CommutableOpIdx2;
3228 if (CommutableOpIdx1 != 1) {
3229 // If we haven't already used the tied source, we must use it now.
3230 CommutableOpIdx2 = 1;
3231 } else {
3232 Register Op1Reg = MI.getOperand(CommutableOpIdx1).getReg();
3233
3234 // The commuted operands should have different registers.
3235 // Otherwise, the commute transformation does not change anything and
3236 // is useless. We use this as a hint to make our decision.
3237 if (Op1Reg != MI.getOperand(2).getReg())
3238 CommutableOpIdx2 = 2;
3239 else
3240 CommutableOpIdx2 = 3;
3241 }
3242
3243 // Assign the found pair of commutable indices to SrcOpIdx1 and
3244 // SrcOpIdx2 to return those values.
3245 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
3246 CommutableOpIdx2))
3247 return false;
3248 }
3249
3250 return true;
3251 }
3252 }
3253
3254 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
3255}
3256
3257// clang-format off
3258#define CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL) \
3259 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL: \
3260 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL; \
3261 break;
3262
3263#define CASE_VMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE) \
3264 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1) \
3265 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2) \
3266 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4) \
3267 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8)
3268
3269#define CASE_VMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE) \
3270 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2) \
3271 CASE_VMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE)
3272
3273#define CASE_VMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE) \
3274 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4) \
3275 CASE_VMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE)
3276
3277#define CASE_VMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE) \
3278 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8) \
3279 CASE_VMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE)
3280
3281#define CASE_VMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP) \
3282 CASE_VMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16) \
3283 CASE_VMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VFPR32) \
3284 CASE_VMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VFPR64)
3285
3286// VFMA depends on SEW.
3287#define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL, SEW) \
3288 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL##_##SEW: \
3289 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL##_##SEW; \
3290 break;
3291
3292#define CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW) \
3293 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1, SEW) \
3294 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2, SEW) \
3295 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4, SEW) \
3296 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8, SEW)
3297
3298#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW) \
3299 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2, SEW) \
3300 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW)
3301
3302#define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP) \
3303 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VV, E16) \
3304 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E32) \
3305 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VV, E64)
3306
3307#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW) \
3308 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4, SEW) \
3309 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW)
3310
3311#define CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE, SEW) \
3312 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8, SEW) \
3313 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW)
3314
3315#define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP) \
3316 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16, E16) \
3317 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VFPR32, E32) \
3318 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VFPR64, E64)
3319
3321 bool NewMI,
3322 unsigned OpIdx1,
3323 unsigned OpIdx2) const {
3324 auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & {
3325 if (NewMI)
3326 return *MI.getParent()->getParent()->CloneMachineInstr(&MI);
3327 return MI;
3328 };
3329
3330 switch (MI.getOpcode()) {
3331 case RISCV::TH_MVEQZ:
3332 case RISCV::TH_MVNEZ: {
3333 auto &WorkingMI = cloneIfNew(MI);
3334 WorkingMI.setDesc(get(MI.getOpcode() == RISCV::TH_MVEQZ ? RISCV::TH_MVNEZ
3335 : RISCV::TH_MVEQZ));
3336 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, false, OpIdx1,
3337 OpIdx2);
3338 }
3339 case RISCV::PseudoCCMOVGPRNoX0:
3340 case RISCV::PseudoCCMOVGPR: {
3341 // CCMOV can be commuted by inverting the condition.
3342 auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());
3344 auto &WorkingMI = cloneIfNew(MI);
3345 WorkingMI.getOperand(3).setImm(CC);
3346 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI*/ false,
3347 OpIdx1, OpIdx2);
3348 }
3349 case CASE_VFMA_SPLATS(FMACC):
3350 case CASE_VFMA_SPLATS(FMADD):
3351 case CASE_VFMA_SPLATS(FMSAC):
3352 case CASE_VFMA_SPLATS(FMSUB):
3353 case CASE_VFMA_SPLATS(FNMACC):
3355 case CASE_VFMA_SPLATS(FNMSAC):
3357 case CASE_VFMA_OPCODE_VV(FMACC):
3358 case CASE_VFMA_OPCODE_VV(FMSAC):
3359 case CASE_VFMA_OPCODE_VV(FNMACC):
3360 case CASE_VFMA_OPCODE_VV(FNMSAC):
3361 case CASE_VMA_OPCODE_LMULS(MADD, VX):
3362 case CASE_VMA_OPCODE_LMULS(NMSUB, VX):
3363 case CASE_VMA_OPCODE_LMULS(MACC, VX):
3364 case CASE_VMA_OPCODE_LMULS(NMSAC, VX):
3365 case CASE_VMA_OPCODE_LMULS(MACC, VV):
3366 case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
3367 // It only make sense to toggle these between clobbering the
3368 // addend/subtrahend/minuend one of the multiplicands.
3369 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
3370 assert((OpIdx1 == 3 || OpIdx2 == 3) && "Unexpected opcode index");
3371 unsigned Opc;
3372 switch (MI.getOpcode()) {
3373 default:
3374 llvm_unreachable("Unexpected opcode");
3375 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMACC, FMADD)
3376 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMADD, FMACC)
3383 CASE_VFMA_CHANGE_OPCODE_VV(FMACC, FMADD)
3387 CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VX)
3388 CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VX)
3389 CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VX)
3390 CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VX)
3391 CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VV)
3392 CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VV)
3393 }
3394
3395 auto &WorkingMI = cloneIfNew(MI);
3396 WorkingMI.setDesc(get(Opc));
3397 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
3398 OpIdx1, OpIdx2);
3399 }
3400 case CASE_VFMA_OPCODE_VV(FMADD):
3404 case CASE_VMA_OPCODE_LMULS(MADD, VV):
3405 case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
3406 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
3407 // If one of the operands, is the addend we need to change opcode.
3408 // Otherwise we're just swapping 2 of the multiplicands.
3409 if (OpIdx1 == 3 || OpIdx2 == 3) {
3410 unsigned Opc;
3411 switch (MI.getOpcode()) {
3412 default:
3413 llvm_unreachable("Unexpected opcode");
3414 CASE_VFMA_CHANGE_OPCODE_VV(FMADD, FMACC)
3418 CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VV)
3419 CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VV)
3420 }
3421
3422 auto &WorkingMI = cloneIfNew(MI);
3423 WorkingMI.setDesc(get(Opc));
3424 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
3425 OpIdx1, OpIdx2);
3426 }
3427 // Let the default code handle it.
3428 break;
3429 }
3430 }
3431
3432 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
3433}
3434
3435#undef CASE_RVV_OPCODE_UNMASK_LMUL
3436#undef CASE_RVV_OPCODE_MASK_LMUL
3437#undef CASE_RVV_OPCODE_LMUL
3438#undef CASE_RVV_OPCODE_UNMASK_WIDEN
3439#undef CASE_RVV_OPCODE_UNMASK
3440#undef CASE_RVV_OPCODE_MASK_WIDEN
3441#undef CASE_RVV_OPCODE_MASK
3442#undef CASE_RVV_OPCODE_WIDEN
3443#undef CASE_RVV_OPCODE
3444
3445#undef CASE_VMA_OPCODE_COMMON
3446#undef CASE_VMA_OPCODE_LMULS_M1
3447#undef CASE_VMA_OPCODE_LMULS_MF2
3448#undef CASE_VMA_OPCODE_LMULS_MF4
3449#undef CASE_VMA_OPCODE_LMULS
3450#undef CASE_VFMA_OPCODE_COMMON
3451#undef CASE_VFMA_OPCODE_LMULS_M1
3452#undef CASE_VFMA_OPCODE_LMULS_MF2
3453#undef CASE_VFMA_OPCODE_LMULS_MF4
3454#undef CASE_VFMA_OPCODE_VV
3455#undef CASE_VFMA_SPLATS
3456
3457// clang-format off
3458#define CASE_WIDEOP_OPCODE_COMMON(OP, LMUL) \
3459 RISCV::PseudoV##OP##_##LMUL##_TIED
3460
3461#define CASE_WIDEOP_OPCODE_LMULS_MF4(OP) \
3462 CASE_WIDEOP_OPCODE_COMMON(OP, MF4): \
3463 case CASE_WIDEOP_OPCODE_COMMON(OP, MF2): \
3464 case CASE_WIDEOP_OPCODE_COMMON(OP, M1): \
3465 case CASE_WIDEOP_OPCODE_COMMON(OP, M2): \
3466 case CASE_WIDEOP_OPCODE_COMMON(OP, M4)
3467
3468#define CASE_WIDEOP_OPCODE_LMULS(OP) \
3469 CASE_WIDEOP_OPCODE_COMMON(OP, MF8): \
3470 case CASE_WIDEOP_OPCODE_LMULS_MF4(OP)
3471
3472#define CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL) \
3473 case RISCV::PseudoV##OP##_##LMUL##_TIED: \
3474 NewOpc = RISCV::PseudoV##OP##_##LMUL; \
3475 break;
3476
3477#define CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP) \
3478 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4) \
3479 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2) \
3480 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1) \
3481 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2) \
3482 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4)
3483
3484#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
3485 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF8) \
3486 CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)
3487
3488// FP Widening Ops may by SEW aware. Create SEW aware cases for these cases.
3489#define CASE_FP_WIDEOP_OPCODE_COMMON(OP, LMUL, SEW) \
3490 RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED
3491
3492#define CASE_FP_WIDEOP_OPCODE_LMULS_MF4(OP) \
3493 CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16): \
3494 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16): \
3495 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E32): \
3496 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E16): \
3497 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E32): \
3498 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E16): \
3499 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E32): \
3500 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E16): \
3501 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E32) \
3502
3503#define CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL, SEW) \
3504 case RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED: \
3505 NewOpc = RISCV::PseudoV##OP##_##LMUL##_##SEW; \
3506 break;
3507
3508#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP) \
3509 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16) \
3510 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16) \
3511 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E32) \
3512 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E16) \
3513 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E32) \
3514 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E16) \
3515 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E32) \
3516 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16) \
3517 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E32) \
3518
3519#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
3520 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)
3521// clang-format on
3522
3524 LiveVariables *LV,
3525 LiveIntervals *LIS) const {
3527 switch (MI.getOpcode()) {
3528 default:
3529 return nullptr;
3530 case CASE_FP_WIDEOP_OPCODE_LMULS_MF4(FWADD_WV):
3531 case CASE_FP_WIDEOP_OPCODE_LMULS_MF4(FWSUB_WV): {
3532 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
3533 MI.getNumExplicitOperands() == 7 &&
3534 "Expect 7 explicit operands rd, rs2, rs1, rm, vl, sew, policy");
3535 // If the tail policy is undisturbed we can't convert.
3536 if ((MI.getOperand(RISCVII::getVecPolicyOpNum(MI.getDesc())).getImm() &
3537 1) == 0)
3538 return nullptr;
3539 // clang-format off
3540 unsigned NewOpc;
3541 switch (MI.getOpcode()) {
3542 default:
3543 llvm_unreachable("Unexpected opcode");
3546 }
3547 // clang-format on
3548
3549 MachineBasicBlock &MBB = *MI.getParent();
3550 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
3551 .add(MI.getOperand(0))
3552 .addReg(MI.getOperand(0).getReg(), RegState::Undef)
3553 .add(MI.getOperand(1))
3554 .add(MI.getOperand(2))
3555 .add(MI.getOperand(3))
3556 .add(MI.getOperand(4))
3557 .add(MI.getOperand(5))
3558 .add(MI.getOperand(6));
3559 break;
3560 }
3561 case CASE_WIDEOP_OPCODE_LMULS(WADD_WV):
3562 case CASE_WIDEOP_OPCODE_LMULS(WADDU_WV):
3563 case CASE_WIDEOP_OPCODE_LMULS(WSUB_WV):
3564 case CASE_WIDEOP_OPCODE_LMULS(WSUBU_WV): {
3565 // If the tail policy is undisturbed we can't convert.
3566 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
3567 MI.getNumExplicitOperands() == 6);
3568 if ((MI.getOperand(5).getImm() & 1) == 0)
3569 return nullptr;
3570
3571 // clang-format off
3572 unsigned NewOpc;
3573 switch (MI.getOpcode()) {
3574 default:
3575 llvm_unreachable("Unexpected opcode");
3580 }
3581 // clang-format on
3582
3583 MachineBasicBlock &MBB = *MI.getParent();
3584 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
3585 .add(MI.getOperand(0))
3586 .addReg(MI.getOperand(0).getReg(), RegState::Undef)
3587 .add(MI.getOperand(1))
3588 .add(MI.getOperand(2))
3589 .add(MI.getOperand(3))
3590 .add(MI.getOperand(4))
3591 .add(MI.getOperand(5));
3592 break;
3593 }
3594 }
3595 MIB.copyImplicitOps(MI);
3596
3597 if (LV) {
3598 unsigned NumOps = MI.getNumOperands();
3599 for (unsigned I = 1; I < NumOps; ++I) {
3600 MachineOperand &Op = MI.getOperand(I);
3601 if (Op.isReg() && Op.isKill())
3602 LV->replaceKillInstruction(Op.getReg(), MI, *MIB);
3603 }
3604 }
3605
3606 if (LIS) {
3607 SlotIndex Idx = LIS->ReplaceMachineInstrInMaps(MI, *MIB);
3608
3609 if (MI.getOperand(0).isEarlyClobber()) {
3610 // Use operand 1 was tied to early-clobber def operand 0, so its live
3611 // interval could have ended at an early-clobber slot. Now they are not
3612 // tied we need to update it to the normal register slot.
3613 LiveInterval &LI = LIS->getInterval(MI.getOperand(1).getReg());
3615 if (S->end == Idx.getRegSlot(true))
3616 S->end = Idx.getRegSlot();
3617 }
3618 }
3619
3620 return MIB;
3621}
3622
3623#undef CASE_WIDEOP_OPCODE_COMMON
3624#undef CASE_WIDEOP_OPCODE_LMULS_MF4
3625#undef CASE_WIDEOP_OPCODE_LMULS
3626#undef CASE_WIDEOP_CHANGE_OPCODE_COMMON
3627#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4
3628#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS
3629#undef CASE_FP_WIDEOP_OPCODE_COMMON
3630#undef CASE_FP_WIDEOP_OPCODE_LMULS_MF4
3631#undef CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON
3632#undef CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4
3633#undef CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS
3634
3637 Register DestReg, uint32_t Amount,
3638 MachineInstr::MIFlag Flag) const {
3640 if (llvm::has_single_bit<uint32_t>(Amount)) {
3641 uint32_t ShiftAmount = Log2_32(Amount);
3642 if (ShiftAmount == 0)
3643 return;
3644 BuildMI(MBB, II, DL, get(RISCV::SLLI), DestReg)
3645 .addReg(DestReg, RegState::Kill)
3646 .addImm(ShiftAmount)
3647 .setMIFlag(Flag);
3648 } else if (STI.hasStdExtZba() &&
3649 ((Amount % 3 == 0 && isPowerOf2_64(Amount / 3)) ||
3650 (Amount % 5 == 0 && isPowerOf2_64(Amount / 5)) ||
3651 (Amount % 9 == 0 && isPowerOf2_64(Amount / 9)))) {
3652 // We can use Zba SHXADD+SLLI instructions for multiply in some cases.
3653 unsigned Opc;
3654 uint32_t ShiftAmount;
3655 if (Amount % 9 == 0) {
3656 Opc = RISCV::SH3ADD;
3657 ShiftAmount = Log2_64(Amount / 9);
3658 } else if (Amount % 5 == 0) {
3659 Opc = RISCV::SH2ADD;
3660 ShiftAmount = Log2_64(Amount / 5);
3661 } else if (Amount % 3 == 0) {
3662 Opc = RISCV::SH1ADD;
3663 ShiftAmount = Log2_64(Amount / 3);
3664 } else {
3665 llvm_unreachable("implied by if-clause");
3666 }
3667 if (ShiftAmount)
3668 BuildMI(MBB, II, DL, get(RISCV::SLLI), DestReg)
3669 .addReg(DestReg, RegState::Kill)
3670 .addImm(ShiftAmount)
3671 .setMIFlag(Flag);
3672 BuildMI(MBB, II, DL, get(Opc), DestReg)
3673 .addReg(DestReg, RegState::Kill)
3674 .addReg(DestReg)
3675 .setMIFlag(Flag);
3676 } else if (llvm::has_single_bit<uint32_t>(Amount - 1)) {
3677 Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass);
3678 uint32_t ShiftAmount = Log2_32(Amount - 1);
3679 BuildMI(MBB, II, DL, get(RISCV::SLLI), ScaledRegister)
3680 .addReg(DestReg)
3681 .addImm(ShiftAmount)
3682 .setMIFlag(Flag);
3683 BuildMI(MBB, II, DL, get(RISCV::ADD), DestReg)
3684 .addReg(ScaledRegister, RegState::Kill)
3685 .addReg(DestReg, RegState::Kill)
3686 .setMIFlag(Flag);
3687 } else if (llvm::has_single_bit<uint32_t>(Amount + 1)) {
3688 Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass);
3689 uint32_t ShiftAmount = Log2_32(Amount + 1);
3690 BuildMI(MBB, II, DL, get(RISCV::SLLI), ScaledRegister)
3691 .addReg(DestReg)
3692 .addImm(ShiftAmount)
3693 .setMIFlag(Flag);
3694 BuildMI(MBB, II, DL, get(RISCV::SUB), DestReg)
3695 .addReg(ScaledRegister, RegState::Kill)
3696 .addReg(DestReg, RegState::Kill)
3697 .setMIFlag(Flag);
3698 } else if (STI.hasStdExtM() || STI.hasStdExtZmmul()) {
3699 Register N = MRI.createVirtualRegister(&RISCV::GPRRegClass);
3700 movImm(MBB, II, DL, N, Amount, Flag);
3701 BuildMI(MBB, II, DL, get(RISCV::MUL), DestReg)
3702 .addReg(DestReg, RegState::Kill)
3704 .setMIFlag(Flag);
3705 } else {
3706 Register Acc;
3707 uint32_t PrevShiftAmount = 0;
3708 for (uint32_t ShiftAmount = 0; Amount >> ShiftAmount; ShiftAmount++) {
3709 if (Amount & (1U << ShiftAmount)) {
3710 if (ShiftAmount)
3711 BuildMI(MBB, II, DL, get(RISCV::SLLI), DestReg)
3712 .addReg(DestReg, RegState::Kill)
3713 .addImm(ShiftAmount - PrevShiftAmount)
3714 .setMIFlag(Flag);
3715 if (Amount >> (ShiftAmount + 1)) {
3716 // If we don't have an accmulator yet, create it and copy DestReg.
3717 if (!Acc) {
3718 Acc = MRI.createVirtualRegister(&RISCV::GPRRegClass);
3719 BuildMI(MBB, II, DL, get(TargetOpcode::COPY), Acc)
3720 .addReg(DestReg)
3721 .setMIFlag(Flag);
3722 } else {
3723 BuildMI(MBB, II, DL, get(RISCV::ADD), Acc)
3724 .addReg(Acc, RegState::Kill)
3725 .addReg(DestReg)
3726 .setMIFlag(Flag);
3727 }
3728 }
3729 PrevShiftAmount = ShiftAmount;
3730 }
3731 }
3732 assert(Acc && "Expected valid accumulator");
3733 BuildMI(MBB, II, DL, get(RISCV::ADD), DestReg)
3734 .addReg(DestReg, RegState::Kill)
3735 .addReg(Acc, RegState::Kill)
3736 .setMIFlag(Flag);
3737 }
3738}
3739
3742 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
3743 {{MONontemporalBit0, "riscv-nontemporal-domain-bit-0"},
3744 {MONontemporalBit1, "riscv-nontemporal-domain-bit-1"}};
3745 return ArrayRef(TargetFlags);
3746}
3747
3748// Returns true if this is the sext.w pattern, addiw rd, rs1, 0.
3750 return MI.getOpcode() == RISCV::ADDIW && MI.getOperand(1).isReg() &&
3751 MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0;
3752}
3753
3754// Returns true if this is the zext.w pattern, adduw rd, rs1, x0.
3756 return MI.getOpcode() == RISCV::ADD_UW && MI.getOperand(1).isReg() &&
3757 MI.getOperand(2).isReg() && MI.getOperand(2).getReg() == RISCV::X0;
3758}
3759
3760// Returns true if this is the zext.b pattern, andi rd, rs1, 255.
3762 return MI.getOpcode() == RISCV::ANDI && MI.getOperand(1).isReg() &&
3763 MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 255;
3764}
3765
3766static bool isRVVWholeLoadStore(unsigned Opcode) {
3767 switch (Opcode) {
3768 default:
3769 return false;
3770 case RISCV::VS1R_V:
3771 case RISCV::VS2R_V:
3772 case RISCV::VS4R_V:
3773 case RISCV::VS8R_V:
3774 case RISCV::VL1RE8_V:
3775 case RISCV::VL2RE8_V:
3776 case RISCV::VL4RE8_V:
3777 case RISCV::VL8RE8_V:
3778 case RISCV::VL1RE16_V:
3779 case RISCV::VL2RE16_V:
3780 case RISCV::VL4RE16_V:
3781 case RISCV::VL8RE16_V:
3782 case RISCV::VL1RE32_V:
3783 case RISCV::VL2RE32_V:
3784 case RISCV::VL4RE32_V:
3785 case RISCV::VL8RE32_V:
3786 case RISCV::VL1RE64_V:
3787 case RISCV::VL2RE64_V:
3788 case RISCV::VL4RE64_V:
3789 case RISCV::VL8RE64_V:
3790 return true;
3791 }
3792}
3793
3795 // RVV lacks any support for immediate addressing for stack addresses, so be
3796 // conservative.
3797 unsigned Opcode = MI.getOpcode();
3798 if (!RISCVVPseudosTable::getPseudoInfo(Opcode) &&
3799 !isRVVWholeLoadStore(Opcode) && !isRVVSpillForZvlsseg(Opcode))
3800 return false;
3801 return true;
3802}
3803
3804std::optional<std::pair<unsigned, unsigned>>
3806 switch (Opcode) {
3807 default:
3808 return std::nullopt;
3809 case RISCV::PseudoVSPILL2_M1:
3810 case RISCV::PseudoVRELOAD2_M1:
3811 return std::make_pair(2u, 1u);
3812 case RISCV::PseudoVSPILL2_M2:
3813 case RISCV::PseudoVRELOAD2_M2:
3814 return std::make_pair(2u, 2u);
3815 case RISCV::PseudoVSPILL2_M4:
3816 case RISCV::PseudoVRELOAD2_M4:
3817 return std::make_pair(2u, 4u);
3818 case RISCV::PseudoVSPILL3_M1:
3819 case RISCV::PseudoVRELOAD3_M1:
3820 return std::make_pair(3u, 1u);
3821 case RISCV::PseudoVSPILL3_M2:
3822 case RISCV::PseudoVRELOAD3_M2:
3823 return std::make_pair(3u, 2u);
3824 case RISCV::PseudoVSPILL4_M1:
3825 case RISCV::PseudoVRELOAD4_M1:
3826 return std::make_pair(4u, 1u);
3827 case RISCV::PseudoVSPILL4_M2:
3828 case RISCV::PseudoVRELOAD4_M2:
3829 return std::make_pair(4u, 2u);
3830 case RISCV::PseudoVSPILL5_M1:
3831 case RISCV::PseudoVRELOAD5_M1:
3832 return std::make_pair(5u, 1u);
3833 case RISCV::PseudoVSPILL6_M1:
3834 case RISCV::PseudoVRELOAD6_M1:
3835 return std::make_pair(6u, 1u);
3836 case RISCV::PseudoVSPILL7_M1:
3837 case RISCV::PseudoVRELOAD7_M1:
3838 return std::make_pair(7u, 1u);
3839 case RISCV::PseudoVSPILL8_M1:
3840 case RISCV::PseudoVRELOAD8_M1:
3841 return std::make_pair(8u, 1u);
3842 }
3843}
3844
3846 return MI.getNumExplicitDefs() == 2 &&
3847 MI.modifiesRegister(RISCV::VL, /*TRI=*/nullptr) && !MI.isInlineAsm();
3848}
3849
3850bool RISCV::hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2) {
3851 int16_t MI1FrmOpIdx =
3852 RISCV::getNamedOperandIdx(MI1.getOpcode(), RISCV::OpName::frm);
3853 int16_t MI2FrmOpIdx =
3854 RISCV::getNamedOperandIdx(MI2.getOpcode(), RISCV::OpName::frm);
3855 if (MI1FrmOpIdx < 0 || MI2FrmOpIdx < 0)
3856 return false;
3857 MachineOperand FrmOp1 = MI1.getOperand(MI1FrmOpIdx);
3858 MachineOperand FrmOp2 = MI2.getOperand(MI2FrmOpIdx);
3859 return FrmOp1.getImm() == FrmOp2.getImm();
3860}
3861
3862std::optional<unsigned>
3864 // TODO: Handle Zvbb instructions
3865 switch (Opcode) {
3866 default:
3867 return std::nullopt;
3868
3869 // 11.6. Vector Single-Width Shift Instructions
3870 case RISCV::VSLL_VX:
3871 case RISCV::VSRL_VX:
3872 case RISCV::VSRA_VX:
3873 // 12.4. Vector Single-Width Scaling Shift Instructions
3874 case RISCV::VSSRL_VX:
3875 case RISCV::VSSRA_VX:
3876 // Only the low lg2(SEW) bits of the shift-amount value are used.
3877 return Log2SEW;
3878
3879 // 11.7 Vector Narrowing Integer Right Shift Instructions
3880 case RISCV::VNSRL_WX:
3881 case RISCV::VNSRA_WX:
3882 // 12.5. Vector Narrowing Fixed-Point Clip Instructions
3883 case RISCV::VNCLIPU_WX:
3884 case RISCV::VNCLIP_WX:
3885 // Only the low lg2(2*SEW) bits of the shift-amount value are used.
3886 return Log2SEW + 1;
3887
3888 // 11.1. Vector Single-Width Integer Add and Subtract
3889 case RISCV::VADD_VX:
3890 case RISCV::VSUB_VX:
3891 case RISCV::VRSUB_VX:
3892 // 11.2. Vector Widening Integer Add/Subtract
3893 case RISCV::VWADDU_VX:
3894 case RISCV::VWSUBU_VX:
3895 case RISCV::VWADD_VX:
3896 case RISCV::VWSUB_VX:
3897 case RISCV::VWADDU_WX:
3898 case RISCV::VWSUBU_WX:
3899 case RISCV::VWADD_WX:
3900 case RISCV::VWSUB_WX:
3901 // 11.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
3902 case RISCV::VADC_VXM:
3903 case RISCV::VADC_VIM:
3904 case RISCV::VMADC_VXM:
3905 case RISCV::VMADC_VIM:
3906 case RISCV::VMADC_VX:
3907 case RISCV::VSBC_VXM:
3908 case RISCV::VMSBC_VXM:
3909 case RISCV::VMSBC_VX:
3910 // 11.5 Vector Bitwise Logical Instructions
3911 case RISCV::VAND_VX:
3912 case RISCV::VOR_VX:
3913 case RISCV::VXOR_VX:
3914 // 11.8. Vector Integer Compare Instructions
3915 case RISCV::VMSEQ_VX:
3916 case RISCV::VMSNE_VX:
3917 case RISCV::VMSLTU_VX:
3918 case RISCV::VMSLT_VX:
3919 case RISCV::VMSLEU_VX:
3920 case RISCV::VMSLE_VX:
3921 case RISCV::VMSGTU_VX:
3922 case RISCV::VMSGT_VX:
3923 // 11.9. Vector Integer Min/Max Instructions
3924 case RISCV::VMINU_VX:
3925 case RISCV::VMIN_VX:
3926 case RISCV::VMAXU_VX:
3927 case RISCV::VMAX_VX:
3928 // 11.10. Vector Single-Width Integer Multiply Instructions
3929 case RISCV::VMUL_VX:
3930 case RISCV::VMULH_VX:
3931 case RISCV::VMULHU_VX:
3932 case RISCV::VMULHSU_VX:
3933 // 11.11. Vector Integer Divide Instructions
3934 case RISCV::VDIVU_VX:
3935 case RISCV::VDIV_VX:
3936 case RISCV::VREMU_VX:
3937 case RISCV::VREM_VX:
3938 // 11.12. Vector Widening Integer Multiply Instructions
3939 case RISCV::VWMUL_VX:
3940 case RISCV::VWMULU_VX:
3941 case RISCV::VWMULSU_VX:
3942 // 11.13. Vector Single-Width Integer Multiply-Add Instructions
3943 case RISCV::VMACC_VX:
3944 case RISCV::VNMSAC_VX:
3945 case RISCV::VMADD_VX:
3946 case RISCV::VNMSUB_VX:
3947 // 11.14. Vector Widening Integer Multiply-Add Instructions
3948 case RISCV::VWMACCU_VX:
3949 case RISCV::VWMACC_VX:
3950 case RISCV::VWMACCSU_VX:
3951 case RISCV::VWMACCUS_VX:
3952 // 11.15. Vector Integer Merge Instructions
3953 case RISCV::VMERGE_VXM:
3954 // 11.16. Vector Integer Move Instructions
3955 case RISCV::VMV_V_X:
3956 // 12.1. Vector Single-Width Saturating Add and Subtract
3957 case RISCV::VSADDU_VX:
3958 case RISCV::VSADD_VX:
3959 case RISCV::VSSUBU_VX:
3960 case RISCV::VSSUB_VX:
3961 // 12.2. Vector Single-Width Averaging Add and Subtract
3962 case RISCV::VAADDU_VX:
3963 case RISCV::VAADD_VX:
3964 case RISCV::VASUBU_VX:
3965 case RISCV::VASUB_VX:
3966 // 12.3. Vector Single-Width Fractional Multiply with Rounding and Saturation
3967 case RISCV::VSMUL_VX:
3968 // 16.1. Integer Scalar Move Instructions
3969 case RISCV::VMV_S_X:
3970 return 1U << Log2SEW;
3971 }
3972}
3973
3974unsigned RISCV::getRVVMCOpcode(unsigned RVVPseudoOpcode) {
3976 RISCVVPseudosTable::getPseudoInfo(RVVPseudoOpcode);
3977 if (!RVV)
3978 return 0;
3979 return RVV->BaseInstr;
3980}
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder MachineInstrBuilder & DefMI
static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg, unsigned NumRegs)
static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO, unsigned CombineOpc, unsigned ZeroReg=0, bool CheckZeroReg=false)
static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target, SmallVectorImpl< MachineOperand > &Cond)
@ MachineOutlinerDefault
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static ARCCC::CondCode getOppositeBranchCondition(ARCCC::CondCode CC)
Return the inverse of passed condition, i.e. turning COND_E to COND_NE.
basic Basic Alias true
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Definition: CommandLine.h:693
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
uint64_t Size
bool End
Definition: ELF_riscv.cpp:480
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
#define _
IRTranslator LLVM IR MI
static M68k::CondCode getCondFromBranchOpc(unsigned BrOpc)
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const char LLVMTargetMachineRef TM
INITIALIZE_PASS(RISCVInsertVSETVLI, DEBUG_TYPE, RISCV_INSERT_VSETVLI_NAME, false, false) char RISCVCoalesceVSETVLI const LiveIntervals * LIS
#define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP)
static bool isRVVWholeLoadStore(unsigned Opcode)
#define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP)
static unsigned getFPFusedMultiplyOpcode(unsigned RootOpc, unsigned Pattern)
#define RVV_OPC_LMUL_CASE(OPC, INV)
static void combineFPFusedMultiply(MachineInstr &Root, MachineInstr &Prev, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs)
static unsigned getAddendOperandIdx(unsigned Pattern)
#define CASE_RVV_OPCODE_UNMASK(OP)
#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP)
static cl::opt< bool > PreferWholeRegisterMove("riscv-prefer-whole-register-move", cl::init(false), cl::Hidden, cl::desc("Prefer whole register move for vector registers."))
#define CASE_VFMA_SPLATS(OP)
unsigned getPredicatedOpcode(unsigned Opcode)
static void genShXAddAddShift(MachineInstr &Root, unsigned AddOpIdx, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg)
#define CASE_WIDEOP_OPCODE_LMULS(OP)
#define OPCODE_LMUL_MASK_CASE(OPC)
static bool isFSUB(unsigned Opc)
#define CASE_VMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE)
#define CASE_RVV_OPCODE(OP)
#define CASE_VFMA_OPCODE_VV(OP)
MachineOutlinerConstructionID
#define CASE_RVV_OPCODE_WIDEN(OP)
#define CASE_VMA_OPCODE_LMULS(OP, TYPE)
static bool isFMUL(unsigned Opc)
static bool getFPPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce)
#define OPCODE_LMUL_CASE(OPC)
#define CASE_OPERAND_UIMM(NUM)
static bool canCombineShiftIntoShXAdd(const MachineBasicBlock &MBB, const MachineOperand &MO, unsigned OuterShiftAmt)
Utility routine that checks if.
static bool isFADD(unsigned Opc)
#define CASE_FP_WIDEOP_OPCODE_LMULS_MF4(OP)
static bool isConvertibleToVMV_V_V(const RISCVSubtarget &STI, const MachineBasicBlock &MBB, MachineBasicBlock::const_iterator MBBI, MachineBasicBlock::const_iterator &DefMBBI, RISCVII::VLMUL LMul)
static MachineInstr * canFoldAsPredicatedOp(Register Reg, const MachineRegisterInfo &MRI, const TargetInstrInfo *TII)
Identify instructions that can be folded into a CCMOV instruction, and return the defining instructio...
static bool canCombineFPFusedMultiply(const MachineInstr &Root, const MachineOperand &MO, bool DoRegPressureReduce)
static bool getSHXADDPatterns(const MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static bool getFPFusedMultiplyPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce)
static cl::opt< MachineTraceStrategy > ForceMachineCombinerStrategy("riscv-force-machine-combiner-strategy", cl::Hidden, cl::desc("Force machine combiner to use a specific strategy for machine " "trace metrics evaluation."), cl::init(MachineTraceStrategy::TS_NumStrategies), cl::values(clEnumValN(MachineTraceStrategy::TS_Local, "local", "Local strategy."), clEnumValN(MachineTraceStrategy::TS_MinInstrCount, "min-instr", "MinInstrCount strategy.")))
static unsigned getSHXADDShiftAmount(unsigned Opc)
#define CASE_RVV_OPCODE_MASK(OP)
#define RVV_OPC_LMUL_MASK_CASE(OPC, INV)
#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
This file declares the machine register scavenger class.
static bool memOpsHaveSameBasePtr(const MachineInstr &MI1, ArrayRef< const MachineOperand * > BaseOps1, const MachineInstr &MI2, ArrayRef< const MachineOperand * > BaseOps2)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI)
This file contains some templates that are useful if you are working with the STL at all.
raw_pwrite_stream & OS
This file defines the SmallVector class.
static cl::opt< unsigned > CacheLineSize("cache-line-size", cl::init(0), cl::Hidden, cl::desc("Use this to override the target cache line size when " "specified by the user."))
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition: Value.cpp:469
Value * RHS
Value * LHS
static unsigned getSize(unsigned Kind)
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
const T & front() const
front - Get the first element.
Definition: ArrayRef.h:168
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
static DILocation * getMergedLocation(DILocation *LocA, DILocation *LocB)
When two instructions are combined into a single instruction we also need to combine the original loc...
This class represents an Operation in the Expression.
bool isBigEndian() const
Definition: DataLayout.h:239
A debug info location.
Definition: DebugLoc.h:33
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:220
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:682
LiveInterval - This class represents the liveness of a register, or stack slot.
Definition: LiveInterval.h:687
const Segment * getSegmentContaining(SlotIndex Idx) const
Return the segment that contains the specified index, or null if there is none.
Definition: LiveInterval.h:408
void replaceKillInstruction(Register Reg, MachineInstr &OldMI, MachineInstr &NewMI)
replaceKillInstruction - Update register kill info by replacing a kill instruction with a new one.
bool hasValue() const
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
TypeSize getValue() const
MCInstBuilder & addReg(unsigned Reg)
Add a new register operand.
Definition: MCInstBuilder.h:37
MCInstBuilder & addImm(int64_t Val)
Add a new integer immediate operand.
Definition: MCInstBuilder.h:43
Instances of this class represent a single low-level machine instruction.
Definition: MCInst.h:184
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:198
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
Definition: MCInstrDesc.h:237
bool isConditionalBranch() const
Return true if this is a branch which may fall through to the next instruction or may transfer contro...
Definition: MCInstrDesc.h:317
Wrapper class representing physical registers. Should be passed by value.
Definition: MCRegister.h:33
Set of metadata that should be preserved when using BuildMI().
unsigned pred_size() const
reverse_iterator rend()
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
iterator getLastNonDebugInstr(bool SkipPseudoOp=true)
Returns an iterator to the last non-debug instruction in the basic block, or end().
Instructions::const_iterator const_instr_iterator
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
void setStackID(int ObjectIdx, uint8_t ID)
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & copyImplicitOps(const MachineInstr &OtherMI) const
Copy all the implicit operands from OtherMI onto this one.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
reverse_iterator getReverse() const
Get a reverse iterator to the same node.
Representation of each machine instruction.
Definition: MachineInstr.h:69
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:558
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:341
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
Definition: MachineInstr.h:391
unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
Definition: MachineInstr.h:555
bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by mayLoad / mayStore,...
bool hasOneMemOperand() const
Return true if this instruction has exactly one MachineMemOperand.
Definition: MachineInstr.h:804
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
Definition: MachineInstr.h:789
bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
Definition: MachineInstr.h:771
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:487
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:568
uint32_t getFlags() const
Return the MI flags bitvector.
Definition: MachineInstr.h:386
void clearKillInfo()
Clears kill flags on all operands.
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
MachineOperand class - Representation of each machine instruction operand.
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
static MachineOperand CreateImm(int64_t Val)
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
@ MO_Immediate
Immediate operand.
@ MO_Register
Register operand.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
MI-level patchpoint operands.
Definition: StackMaps.h:76
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given patchpoint should emit.
Definition: StackMaps.h:104
MachineInstr * convertToThreeAddress(MachineInstr &MI, LiveVariables *LV, LiveIntervals *LIS) const override
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
std::optional< outliner::OutlinedFunction > getOutliningCandidateInfo(std::vector< outliner::Candidate > &RepeatedSequenceLocs) const override
void movImm(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, Register DstReg, uint64_t Val, MachineInstr::MIFlag Flag=MachineInstr::NoFlags, bool DstRenamable=false, bool DstIsDead=false) const
MachineInstr * emitLdStWithAddr(MachineInstr &MemI, const ExtAddrMode &AM) const override
void mulImm(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator II, const DebugLoc &DL, Register DestReg, uint32_t Amt, MachineInstr::MIFlag Flag) const
Generate code to multiply the value in DestReg by Amt - handles all the common optimizations for this...
bool isFunctionSafeToOutlineFrom(MachineFunction &MF, bool OutlineFromLinkOnceODRs) const override
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &dl, int *BytesAdded=nullptr) const override
bool hasReassociableSibling(const MachineInstr &Inst, bool &Commuted) const override
RISCVInstrInfo(RISCVSubtarget &STI)
void copyPhysRegVector(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc, const TargetRegisterClass *RegClass) const
void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg) const override
const MCInstrDesc & getBrCond(RISCVCC::CondCode CC, bool Imm=false) const
MachineInstr * optimizeSelect(MachineInstr &MI, SmallPtrSetImpl< MachineInstr * > &SeenMIs, bool) const override
bool canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg, const MachineInstr &AddrI, ExtAddrMode &AM) const override
void insertIndirectBranch(MachineBasicBlock &MBB, MachineBasicBlock &NewDestBB, MachineBasicBlock &RestoreBB, const DebugLoc &DL, int64_t BrOffset, RegScavenger *RS) const override
bool isAsCheapAsAMove(const MachineInstr &MI) const override
bool verifyInstruction(const MachineInstr &MI, StringRef &ErrInfo) const override
bool getMemOperandWithOffsetWidth(const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset, LocationSize &Width, const TargetRegisterInfo *TRI) const
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc) const override
void getReassociateOperandIndices(const MachineInstr &Root, unsigned Pattern, std::array< unsigned, 5 > &OperandIndices) const override
const RISCVSubtarget & STI
Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const override
std::optional< unsigned > getInverseOpcode(unsigned Opcode) const override
ArrayRef< std::pair< unsigned, const char * > > getSerializableDirectMachineOperandTargetFlags() const override
MachineTraceStrategy getMachineCombinerTraceStrategy() const override
unsigned getInstSizeInBytes(const MachineInstr &MI) const override
virtual outliner::InstrType getOutliningTypeImpl(MachineBasicBlock::iterator &MBBI, unsigned Flags) const override
std::optional< RegImmPair > isAddImmediate(const MachineInstr &MI, Register Reg) const override
bool reverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
ArrayRef< std::pair< MachineMemOperand::Flags, const char * > > getSerializableMachineMemOperandTargetFlags() const override
MCInst getNop() const override
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
bool isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, unsigned &Flags) const override
bool getMemOperandsWithOffsetWidth(const MachineInstr &MI, SmallVectorImpl< const MachineOperand * > &BaseOps, int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const override
void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF, const outliner::OutlinedFunction &OF) const override
void finalizeInsInstrs(MachineInstr &Root, unsigned &Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs) const override
std::pair< unsigned, unsigned > decomposeMachineOperandsTargetFlags(unsigned TF) const override
MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx1, unsigned OpIdx2) const override
bool hasReassociableOperands(const MachineInstr &Inst, const MachineBasicBlock *MBB) const override
MachineBasicBlock * getBranchDestBlock(const MachineInstr &MI) const override
std::string createMIROperandComment(const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx, const TargetRegisterInfo *TRI) const override
bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override
bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const override
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const override
MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, outliner::Candidate &C) const override
bool isBranchOffsetInRange(unsigned BranchOpc, int64_t BrOffset) const override
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DstReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
bool isAssociativeAndCommutative(const MachineInstr &Inst, bool Invert) const override
CombinerObjective getCombinerObjective(unsigned Pattern) const override
bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const override
bool optimizeCondBranch(MachineInstr &MI) const override
std::optional< DestSourcePair > isCopyInstrImpl(const MachineInstr &MI) const override
bool analyzeSelect(const MachineInstr &MI, SmallVectorImpl< MachineOperand > &Cond, unsigned &TrueOp, unsigned &FalseOp, bool &Optimizable) const override
bool shouldClusterMemOps(ArrayRef< const MachineOperand * > BaseOps1, int64_t Offset1, bool OffsetIsScalable1, ArrayRef< const MachineOperand * > BaseOps2, int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize, unsigned NumBytes) const override
bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const override
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool IsKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
bool hasStdExtCOrZca() const
unsigned getXLen() const
const RISCVRegisterInfo * getRegisterInfo() const override
void enterBasicBlockEnd(MachineBasicBlock &MBB)
Start tracking liveness from the end of basic block MBB.
void setRegUsed(Register Reg, LaneBitmask LaneMask=LaneBitmask::getAll())
Tell the scavenger a register is used.
Register scavengeRegisterBackwards(const TargetRegisterClass &RC, MachineBasicBlock::iterator To, bool RestoreAfter, int SPAdj, bool AllowSpill=true)
Make a register of the specific register class available from the current position backwards to the p...
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
constexpr bool isValid() const
Definition: Register.h:116
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:91
SlotIndex - An opaque wrapper around machine indexes.
Definition: SlotIndexes.h:68
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:321
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
Definition: SmallPtrSet.h:356
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
MI-level stackmap operands.
Definition: StackMaps.h:35
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given stackmap should emit.
Definition: StackMaps.h:50
MI-level Statepoint operands.
Definition: StackMaps.h:158
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given statepoint should emit.
Definition: StackMaps.h:207
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
TargetInstrInfo - Interface to description of machine instruction set.
virtual bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const
Returns true iff the routine could find two commutable operands in the given machine instruction.
virtual bool hasReassociableOperands(const MachineInstr &Inst, const MachineBasicBlock *MBB) const
Return true when \P Inst has reassociable operands in the same \P MBB.
virtual void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstIdxForVirtReg) const
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
virtual bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
virtual bool isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, unsigned &Flags) const
Optional target hook that returns true if MBB is safe to outline from, and returns any target-specifi...
virtual void getReassociateOperandIndices(const MachineInstr &Root, unsigned Pattern, std::array< unsigned, 5 > &OperandIndices) const
The returned array encodes the operand index for each parameter because the operands may be commuted;...
virtual CombinerObjective getCombinerObjective(unsigned Pattern) const
Return the objective of a combiner pattern.
virtual MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx1, unsigned OpIdx2) const
This method commutes the operands of the given machine instruction MI.
virtual bool hasReassociableSibling(const MachineInstr &Inst, bool &Commuted) const
Return true when \P Inst has reassociable sibling.
virtual std::string createMIROperandComment(const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx, const TargetRegisterInfo *TRI) const
const uint8_t TSFlags
Configurable target specific flags.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:660
#define INT64_MAX
Definition: DataTypes.h:71
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
CondCode getOppositeBranchCondition(CondCode)
unsigned getBrCond(CondCode CC, bool Imm=false)
static unsigned getVecPolicyOpNum(const MCInstrDesc &Desc)
static bool usesMaskPolicy(uint64_t TSFlags)
static bool hasRoundModeOp(uint64_t TSFlags)
static unsigned getVLOpNum(const MCInstrDesc &Desc)
static bool hasVLOp(uint64_t TSFlags)
static bool hasVecPolicyOp(uint64_t TSFlags)
static bool isRVVWideningReduction(uint64_t TSFlags)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
static bool hasSEWOp(uint64_t TSFlags)
static bool isFirstDefTiedToFirstUse(const MCInstrDesc &Desc)
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
@ OPERAND_UIMMLOG2XLEN_NONZERO
@ OPERAND_UIMM10_LSB00_NONZERO
@ OPERAND_SIMM10_LSB0000_NONZERO
static RISCVII::VLMUL getLMul(uint64_t TSFlags)
static unsigned getNF(uint64_t TSFlags)
static bool isTailAgnostic(unsigned VType)
static RISCVII::VLMUL getVLMUL(unsigned VType)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
static bool isValidSEW(unsigned SEW)
void printVType(unsigned VType, raw_ostream &OS)
static unsigned getSEW(unsigned VType)
bool hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2)
std::optional< unsigned > getVectorLowDemandedScalarBits(uint16_t Opcode, unsigned Log2SEW)
int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIndex)
unsigned getRVVMCOpcode(unsigned RVVPseudoOpcode)
bool isSEXT_W(const MachineInstr &MI)
bool isFaultFirstLoad(const MachineInstr &MI)
std::optional< std::pair< unsigned, unsigned > > isRVVSpillForZvlsseg(uns