LLVM 18.0.0git
RISCVInsertVSETVLI.cpp
Go to the documentation of this file.
1//===- RISCVInsertVSETVLI.cpp - Insert VSETVLI instructions ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a function pass that inserts VSETVLI instructions where
10// needed and expands the vl outputs of VLEFF/VLSEGFF to PseudoReadVL
11// instructions.
12//
13// This pass consists of 3 phases:
14//
15// Phase 1 collects how each basic block affects VL/VTYPE.
16//
17// Phase 2 uses the information from phase 1 to do a data flow analysis to
18// propagate the VL/VTYPE changes through the function. This gives us the
19// VL/VTYPE at the start of each basic block.
20//
21// Phase 3 inserts VSETVLI instructions in each basic block. Information from
22// phase 2 is used to prevent inserting a VSETVLI before the first vector
23// instruction in the block if possible.
24//
25//===----------------------------------------------------------------------===//
26
27#include "RISCV.h"
28#include "RISCVSubtarget.h"
31#include <queue>
32using namespace llvm;
33
34#define DEBUG_TYPE "riscv-insert-vsetvli"
35#define RISCV_INSERT_VSETVLI_NAME "RISC-V Insert VSETVLI pass"
36
38 "riscv-disable-insert-vsetvl-phi-opt", cl::init(false), cl::Hidden,
39 cl::desc("Disable looking through phis when inserting vsetvlis."));
40
42 "riscv-insert-vsetvl-strict-asserts", cl::init(true), cl::Hidden,
43 cl::desc("Enable strict assertion checking for the dataflow algorithm"));
44
45namespace {
46
47static unsigned getVLOpNum(const MachineInstr &MI) {
48 return RISCVII::getVLOpNum(MI.getDesc());
49}
50
51static unsigned getSEWOpNum(const MachineInstr &MI) {
52 return RISCVII::getSEWOpNum(MI.getDesc());
53}
54
55static bool isVectorConfigInstr(const MachineInstr &MI) {
56 return MI.getOpcode() == RISCV::PseudoVSETVLI ||
57 MI.getOpcode() == RISCV::PseudoVSETVLIX0 ||
58 MI.getOpcode() == RISCV::PseudoVSETIVLI;
59}
60
61/// Return true if this is 'vsetvli x0, x0, vtype' which preserves
62/// VL and only sets VTYPE.
63static bool isVLPreservingConfig(const MachineInstr &MI) {
64 if (MI.getOpcode() != RISCV::PseudoVSETVLIX0)
65 return false;
66 assert(RISCV::X0 == MI.getOperand(1).getReg());
67 return RISCV::X0 == MI.getOperand(0).getReg();
68}
69
70static uint16_t getRVVMCOpcode(uint16_t RVVPseudoOpcode) {
72 RISCVVPseudosTable::getPseudoInfo(RVVPseudoOpcode);
73 if (!RVV)
74 return 0;
75 return RVV->BaseInstr;
76}
77
78static bool isFloatScalarMoveOrScalarSplatInstr(const MachineInstr &MI) {
79 switch (getRVVMCOpcode(MI.getOpcode())) {
80 default:
81 return false;
82 case RISCV::VFMV_S_F:
83 case RISCV::VFMV_V_F:
84 return true;
85 }
86}
87
88static bool isScalarExtractInstr(const MachineInstr &MI) {
89 switch (getRVVMCOpcode(MI.getOpcode())) {
90 default:
91 return false;
92 case RISCV::VMV_X_S:
93 case RISCV::VFMV_F_S:
94 return true;
95 }
96}
97
98static bool isScalarInsertInstr(const MachineInstr &MI) {
99 switch (getRVVMCOpcode(MI.getOpcode())) {
100 default:
101 return false;
102 case RISCV::VMV_S_X:
103 case RISCV::VFMV_S_F:
104 return true;
105 }
106}
107
108static bool isScalarSplatInstr(const MachineInstr &MI) {
109 switch (getRVVMCOpcode(MI.getOpcode())) {
110 default:
111 return false;
112 case RISCV::VMV_V_I:
113 case RISCV::VMV_V_X:
114 case RISCV::VFMV_V_F:
115 return true;
116 }
117}
118
119static bool isVSlideInstr(const MachineInstr &MI) {
120 switch (getRVVMCOpcode(MI.getOpcode())) {
121 default:
122 return false;
123 case RISCV::VSLIDEDOWN_VX:
124 case RISCV::VSLIDEDOWN_VI:
125 case RISCV::VSLIDEUP_VX:
126 case RISCV::VSLIDEUP_VI:
127 return true;
128 }
129}
130
131/// Get the EEW for a load or store instruction. Return std::nullopt if MI is
132/// not a load or store which ignores SEW.
133static std::optional<unsigned> getEEWForLoadStore(const MachineInstr &MI) {
134 switch (getRVVMCOpcode(MI.getOpcode())) {
135 default:
136 return std::nullopt;
137 case RISCV::VLE8_V:
138 case RISCV::VLSE8_V:
139 case RISCV::VSE8_V:
140 case RISCV::VSSE8_V:
141 return 8;
142 case RISCV::VLE16_V:
143 case RISCV::VLSE16_V:
144 case RISCV::VSE16_V:
145 case RISCV::VSSE16_V:
146 return 16;
147 case RISCV::VLE32_V:
148 case RISCV::VLSE32_V:
149 case RISCV::VSE32_V:
150 case RISCV::VSSE32_V:
151 return 32;
152 case RISCV::VLE64_V:
153 case RISCV::VLSE64_V:
154 case RISCV::VSE64_V:
155 case RISCV::VSSE64_V:
156 return 64;
157 }
158}
159
160/// Return true if this is an operation on mask registers. Note that
161/// this includes both arithmetic/logical ops and load/store (vlm/vsm).
162static bool isMaskRegOp(const MachineInstr &MI) {
163 if (!RISCVII::hasSEWOp(MI.getDesc().TSFlags))
164 return false;
165 const unsigned Log2SEW = MI.getOperand(getSEWOpNum(MI)).getImm();
166 // A Log2SEW of 0 is an operation on mask registers only.
167 return Log2SEW == 0;
168}
169
170/// Return true if the inactive elements in the result are entirely undefined.
171/// Note that this is different from "agnostic" as defined by the vector
172/// specification. Agnostic requires each lane to either be undisturbed, or
173/// take the value -1; no other value is allowed.
174static bool hasUndefinedMergeOp(const MachineInstr &MI,
175 const MachineRegisterInfo &MRI) {
176
177 unsigned UseOpIdx;
178 if (!MI.isRegTiedToUseOperand(0, &UseOpIdx))
179 // If there is no passthrough operand, then the pass through
180 // lanes are undefined.
181 return true;
182
183 // If the tied operand is NoReg, an IMPLICIT_DEF, or a REG_SEQEUENCE whose
184 // operands are solely IMPLICIT_DEFS, then the pass through lanes are
185 // undefined.
186 const MachineOperand &UseMO = MI.getOperand(UseOpIdx);
187 if (UseMO.getReg() == RISCV::NoRegister)
188 return true;
189
190 if (MachineInstr *UseMI = MRI.getVRegDef(UseMO.getReg())) {
191 if (UseMI->isImplicitDef())
192 return true;
193
194 if (UseMI->isRegSequence()) {
195 for (unsigned i = 1, e = UseMI->getNumOperands(); i < e; i += 2) {
196 MachineInstr *SourceMI = MRI.getVRegDef(UseMI->getOperand(i).getReg());
197 if (!SourceMI || !SourceMI->isImplicitDef())
198 return false;
199 }
200 return true;
201 }
202 }
203 return false;
204}
205
206/// Which subfields of VL or VTYPE have values we need to preserve?
207struct DemandedFields {
208 // Some unknown property of VL is used. If demanded, must preserve entire
209 // value.
210 bool VLAny = false;
211 // Only zero vs non-zero is used. If demanded, can change non-zero values.
212 bool VLZeroness = false;
213 // What properties of SEW we need to preserve.
214 enum : uint8_t {
215 SEWEqual = 3, // The exact value of SEW needs to be preserved.
216 SEWGreaterThanOrEqual = 2, // SEW can be changed as long as it's greater
217 // than or equal to the original value.
218 SEWGreaterThanOrEqualAndLessThan64 =
219 1, // SEW can be changed as long as it's greater
220 // than or equal to the original value, but must be less
221 // than 64.
222 SEWNone = 0 // We don't need to preserve SEW at all.
223 } SEW = SEWNone;
224 bool LMUL = false;
225 bool SEWLMULRatio = false;
226 bool TailPolicy = false;
227 bool MaskPolicy = false;
228
229 // Return true if any part of VTYPE was used
230 bool usedVTYPE() const {
231 return SEW || LMUL || SEWLMULRatio || TailPolicy || MaskPolicy;
232 }
233
234 // Return true if any property of VL was used
235 bool usedVL() {
236 return VLAny || VLZeroness;
237 }
238
239 // Mark all VTYPE subfields and properties as demanded
240 void demandVTYPE() {
241 SEW = SEWEqual;
242 LMUL = true;
243 SEWLMULRatio = true;
244 TailPolicy = true;
245 MaskPolicy = true;
246 }
247
248 // Mark all VL properties as demanded
249 void demandVL() {
250 VLAny = true;
251 VLZeroness = true;
252 }
253
254#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
255 /// Support for debugging, callable in GDB: V->dump()
256 LLVM_DUMP_METHOD void dump() const {
257 print(dbgs());
258 dbgs() << "\n";
259 }
260
261 /// Implement operator<<.
262 void print(raw_ostream &OS) const {
263 OS << "{";
264 OS << "VLAny=" << VLAny << ", ";
265 OS << "VLZeroness=" << VLZeroness << ", ";
266 OS << "SEW=";
267 switch (SEW) {
268 case SEWEqual:
269 OS << "SEWEqual";
270 break;
271 case SEWGreaterThanOrEqual:
272 OS << "SEWGreaterThanOrEqual";
273 break;
274 case SEWGreaterThanOrEqualAndLessThan64:
275 OS << "SEWGreaterThanOrEqualAndLessThan64";
276 break;
277 case SEWNone:
278 OS << "SEWNone";
279 break;
280 };
281 OS << ", ";
282 OS << "LMUL=" << LMUL << ", ";
283 OS << "SEWLMULRatio=" << SEWLMULRatio << ", ";
284 OS << "TailPolicy=" << TailPolicy << ", ";
285 OS << "MaskPolicy=" << MaskPolicy;
286 OS << "}";
287 }
288#endif
289};
290
291#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
293inline raw_ostream &operator<<(raw_ostream &OS, const DemandedFields &DF) {
294 DF.print(OS);
295 return OS;
296}
297#endif
298
299/// Return true if moving from CurVType to NewVType is
300/// indistinguishable from the perspective of an instruction (or set
301/// of instructions) which use only the Used subfields and properties.
302static bool areCompatibleVTYPEs(uint64_t CurVType, uint64_t NewVType,
303 const DemandedFields &Used) {
304 if (Used.SEW == DemandedFields::SEWEqual &&
305 RISCVVType::getSEW(CurVType) != RISCVVType::getSEW(NewVType))
306 return false;
307
308 if (Used.SEW == DemandedFields::SEWGreaterThanOrEqual &&
309 RISCVVType::getSEW(NewVType) < RISCVVType::getSEW(CurVType))
310 return false;
311
312 if (Used.SEW == DemandedFields::SEWGreaterThanOrEqualAndLessThan64 &&
313 (RISCVVType::getSEW(NewVType) < RISCVVType::getSEW(CurVType) ||
314 RISCVVType::getSEW(NewVType) >= 64))
315 return false;
316
317 if (Used.LMUL &&
318 RISCVVType::getVLMUL(CurVType) != RISCVVType::getVLMUL(NewVType))
319 return false;
320
321 if (Used.SEWLMULRatio) {
322 auto Ratio1 = RISCVVType::getSEWLMULRatio(RISCVVType::getSEW(CurVType),
323 RISCVVType::getVLMUL(CurVType));
324 auto Ratio2 = RISCVVType::getSEWLMULRatio(RISCVVType::getSEW(NewVType),
325 RISCVVType::getVLMUL(NewVType));
326 if (Ratio1 != Ratio2)
327 return false;
328 }
329
330 if (Used.TailPolicy && RISCVVType::isTailAgnostic(CurVType) !=
332 return false;
333 if (Used.MaskPolicy && RISCVVType::isMaskAgnostic(CurVType) !=
335 return false;
336 return true;
337}
338
339/// Return the fields and properties demanded by the provided instruction.
340DemandedFields getDemanded(const MachineInstr &MI,
341 const MachineRegisterInfo *MRI) {
342 // Warning: This function has to work on both the lowered (i.e. post
343 // emitVSETVLIs) and pre-lowering forms. The main implication of this is
344 // that it can't use the value of a SEW, VL, or Policy operand as they might
345 // be stale after lowering.
346 bool HasVInstructionsF64 =
347 MI.getMF()->getSubtarget<RISCVSubtarget>().hasVInstructionsF64();
348
349 // Most instructions don't use any of these subfeilds.
350 DemandedFields Res;
351 // Start conservative if registers are used
352 if (MI.isCall() || MI.isInlineAsm() || MI.readsRegister(RISCV::VL))
353 Res.demandVL();
354 if (MI.isCall() || MI.isInlineAsm() || MI.readsRegister(RISCV::VTYPE))
355 Res.demandVTYPE();
356 // Start conservative on the unlowered form too
357 uint64_t TSFlags = MI.getDesc().TSFlags;
359 Res.demandVTYPE();
361 Res.demandVL();
362
363 // Behavior is independent of mask policy.
365 Res.MaskPolicy = false;
366 }
367
368 // Loads and stores with implicit EEW do not demand SEW or LMUL directly.
369 // They instead demand the ratio of the two which is used in computing
370 // EMUL, but which allows us the flexibility to change SEW and LMUL
371 // provided we don't change the ratio.
372 // Note: We assume that the instructions initial SEW is the EEW encoded
373 // in the opcode. This is asserted when constructing the VSETVLIInfo.
374 if (getEEWForLoadStore(MI)) {
375 Res.SEW = DemandedFields::SEWNone;
376 Res.LMUL = false;
377 }
378
379 // Store instructions don't use the policy fields.
380 if (RISCVII::hasSEWOp(TSFlags) && MI.getNumExplicitDefs() == 0) {
381 Res.TailPolicy = false;
382 Res.MaskPolicy = false;
383 }
384
385 // If this is a mask reg operation, it only cares about VLMAX.
386 // TODO: Possible extensions to this logic
387 // * Probably ok if available VLMax is larger than demanded
388 // * The policy bits can probably be ignored..
389 if (isMaskRegOp(MI)) {
390 Res.SEW = DemandedFields::SEWNone;
391 Res.LMUL = false;
392 }
393
394 // For vmv.s.x and vfmv.s.f, there are only two behaviors, VL = 0 and VL > 0.
395 if (isScalarInsertInstr(MI)) {
396 Res.LMUL = false;
397 Res.SEWLMULRatio = false;
398 Res.VLAny = false;
399 // For vmv.s.x and vfmv.s.f, if the merge operand is *undefined*, we don't
400 // need to preserve any other bits and are thus compatible with any larger,
401 // etype and can disregard policy bits. Warning: It's tempting to try doing
402 // this for any tail agnostic operation, but we can't as TA requires
403 // tail lanes to either be the original value or -1. We are writing
404 // unknown bits to the lanes here.
405 if (hasUndefinedMergeOp(MI, *MRI)) {
406 if (isFloatScalarMoveOrScalarSplatInstr(MI) && !HasVInstructionsF64)
407 Res.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
408 else
409 Res.SEW = DemandedFields::SEWGreaterThanOrEqual;
410 Res.TailPolicy = false;
411 }
412 }
413
414 // vmv.x.s, and vmv.f.s are unconditional and ignore everything except SEW.
415 if (isScalarExtractInstr(MI)) {
417 Res.LMUL = false;
418 Res.SEWLMULRatio = false;
419 Res.TailPolicy = false;
420 Res.MaskPolicy = false;
421 }
422
423 return Res;
424}
425
426/// Defines the abstract state with which the forward dataflow models the
427/// values of the VL and VTYPE registers after insertion.
428class VSETVLIInfo {
429 union {
430 Register AVLReg;
431 unsigned AVLImm;
432 };
433
434 enum : uint8_t {
436 AVLIsReg,
437 AVLIsImm,
438 Unknown,
439 } State = Uninitialized;
440
441 // Fields from VTYPE.
443 uint8_t SEW = 0;
444 uint8_t TailAgnostic : 1;
445 uint8_t MaskAgnostic : 1;
446 uint8_t SEWLMULRatioOnly : 1;
447
448public:
449 VSETVLIInfo()
450 : AVLImm(0), TailAgnostic(false), MaskAgnostic(false),
451 SEWLMULRatioOnly(false) {}
452
453 static VSETVLIInfo getUnknown() {
454 VSETVLIInfo Info;
455 Info.setUnknown();
456 return Info;
457 }
458
459 bool isValid() const { return State != Uninitialized; }
460 void setUnknown() { State = Unknown; }
461 bool isUnknown() const { return State == Unknown; }
462
463 void setAVLReg(Register Reg) {
464 AVLReg = Reg;
465 State = AVLIsReg;
466 }
467
468 void setAVLImm(unsigned Imm) {
469 AVLImm = Imm;
470 State = AVLIsImm;
471 }
472
473 bool hasAVLImm() const { return State == AVLIsImm; }
474 bool hasAVLReg() const { return State == AVLIsReg; }
475 Register getAVLReg() const {
476 assert(hasAVLReg());
477 return AVLReg;
478 }
479 unsigned getAVLImm() const {
480 assert(hasAVLImm());
481 return AVLImm;
482 }
483
484 unsigned getSEW() const { return SEW; }
485 RISCVII::VLMUL getVLMUL() const { return VLMul; }
486
487 bool hasNonZeroAVL(const MachineRegisterInfo &MRI) const {
488 if (hasAVLImm())
489 return getAVLImm() > 0;
490 if (hasAVLReg()) {
491 if (getAVLReg() == RISCV::X0)
492 return true;
493 if (MachineInstr *MI = MRI.getVRegDef(getAVLReg());
494 MI && MI->getOpcode() == RISCV::ADDI &&
495 MI->getOperand(1).isReg() && MI->getOperand(2).isImm() &&
496 MI->getOperand(1).getReg() == RISCV::X0 &&
497 MI->getOperand(2).getImm() != 0)
498 return true;
499 return false;
500 }
501 return false;
502 }
503
504 bool hasEquallyZeroAVL(const VSETVLIInfo &Other,
505 const MachineRegisterInfo &MRI) const {
506 if (hasSameAVL(Other))
507 return true;
508 return (hasNonZeroAVL(MRI) && Other.hasNonZeroAVL(MRI));
509 }
510
511 bool hasSameAVL(const VSETVLIInfo &Other) const {
512 if (hasAVLReg() && Other.hasAVLReg())
513 return getAVLReg() == Other.getAVLReg();
514
515 if (hasAVLImm() && Other.hasAVLImm())
516 return getAVLImm() == Other.getAVLImm();
517
518 return false;
519 }
520
521 void setVTYPE(unsigned VType) {
522 assert(isValid() && !isUnknown() &&
523 "Can't set VTYPE for uninitialized or unknown");
525 SEW = RISCVVType::getSEW(VType);
528 }
529 void setVTYPE(RISCVII::VLMUL L, unsigned S, bool TA, bool MA) {
530 assert(isValid() && !isUnknown() &&
531 "Can't set VTYPE for uninitialized or unknown");
532 VLMul = L;
533 SEW = S;
535 MaskAgnostic = MA;
536 }
537
538 unsigned encodeVTYPE() const {
539 assert(isValid() && !isUnknown() && !SEWLMULRatioOnly &&
540 "Can't encode VTYPE for uninitialized or unknown");
541 return RISCVVType::encodeVTYPE(VLMul, SEW, TailAgnostic, MaskAgnostic);
542 }
543
544 bool hasSEWLMULRatioOnly() const { return SEWLMULRatioOnly; }
545
546 bool hasSameVTYPE(const VSETVLIInfo &Other) const {
547 assert(isValid() && Other.isValid() &&
548 "Can't compare invalid VSETVLIInfos");
549 assert(!isUnknown() && !Other.isUnknown() &&
550 "Can't compare VTYPE in unknown state");
551 assert(!SEWLMULRatioOnly && !Other.SEWLMULRatioOnly &&
552 "Can't compare when only LMUL/SEW ratio is valid.");
553 return std::tie(VLMul, SEW, TailAgnostic, MaskAgnostic) ==
554 std::tie(Other.VLMul, Other.SEW, Other.TailAgnostic,
555 Other.MaskAgnostic);
556 }
557
558 unsigned getSEWLMULRatio() const {
559 assert(isValid() && !isUnknown() &&
560 "Can't use VTYPE for uninitialized or unknown");
561 return RISCVVType::getSEWLMULRatio(SEW, VLMul);
562 }
563
564 // Check if the VTYPE for these two VSETVLIInfos produce the same VLMAX.
565 // Note that having the same VLMAX ensures that both share the same
566 // function from AVL to VL; that is, they must produce the same VL value
567 // for any given AVL value.
568 bool hasSameVLMAX(const VSETVLIInfo &Other) const {
569 assert(isValid() && Other.isValid() &&
570 "Can't compare invalid VSETVLIInfos");
571 assert(!isUnknown() && !Other.isUnknown() &&
572 "Can't compare VTYPE in unknown state");
573 return getSEWLMULRatio() == Other.getSEWLMULRatio();
574 }
575
576 bool hasCompatibleVTYPE(const DemandedFields &Used,
577 const VSETVLIInfo &Require) const {
578 return areCompatibleVTYPEs(Require.encodeVTYPE(), encodeVTYPE(), Used);
579 }
580
581 // Determine whether the vector instructions requirements represented by
582 // Require are compatible with the previous vsetvli instruction represented
583 // by this. MI is the instruction whose requirements we're considering.
584 bool isCompatible(const DemandedFields &Used, const VSETVLIInfo &Require,
585 const MachineRegisterInfo &MRI) const {
586 assert(isValid() && Require.isValid() &&
587 "Can't compare invalid VSETVLIInfos");
588 assert(!Require.SEWLMULRatioOnly &&
589 "Expected a valid VTYPE for instruction!");
590 // Nothing is compatible with Unknown.
591 if (isUnknown() || Require.isUnknown())
592 return false;
593
594 // If only our VLMAX ratio is valid, then this isn't compatible.
595 if (SEWLMULRatioOnly)
596 return false;
597
598 if (Used.VLAny && !hasSameAVL(Require))
599 return false;
600
601 if (Used.VLZeroness && !hasEquallyZeroAVL(Require, MRI))
602 return false;
603
604 return hasCompatibleVTYPE(Used, Require);
605 }
606
607 bool operator==(const VSETVLIInfo &Other) const {
608 // Uninitialized is only equal to another Uninitialized.
609 if (!isValid())
610 return !Other.isValid();
611 if (!Other.isValid())
612 return !isValid();
613
614 // Unknown is only equal to another Unknown.
615 if (isUnknown())
616 return Other.isUnknown();
617 if (Other.isUnknown())
618 return isUnknown();
619
620 if (!hasSameAVL(Other))
621 return false;
622
623 // If the SEWLMULRatioOnly bits are different, then they aren't equal.
624 if (SEWLMULRatioOnly != Other.SEWLMULRatioOnly)
625 return false;
626
627 // If only the VLMAX is valid, check that it is the same.
628 if (SEWLMULRatioOnly)
629 return hasSameVLMAX(Other);
630
631 // If the full VTYPE is valid, check that it is the same.
632 return hasSameVTYPE(Other);
633 }
634
635 bool operator!=(const VSETVLIInfo &Other) const {
636 return !(*this == Other);
637 }
638
639 // Calculate the VSETVLIInfo visible to a block assuming this and Other are
640 // both predecessors.
641 VSETVLIInfo intersect(const VSETVLIInfo &Other) const {
642 // If the new value isn't valid, ignore it.
643 if (!Other.isValid())
644 return *this;
645
646 // If this value isn't valid, this must be the first predecessor, use it.
647 if (!isValid())
648 return Other;
649
650 // If either is unknown, the result is unknown.
651 if (isUnknown() || Other.isUnknown())
652 return VSETVLIInfo::getUnknown();
653
654 // If we have an exact, match return this.
655 if (*this == Other)
656 return *this;
657
658 // Not an exact match, but maybe the AVL and VLMAX are the same. If so,
659 // return an SEW/LMUL ratio only value.
660 if (hasSameAVL(Other) && hasSameVLMAX(Other)) {
661 VSETVLIInfo MergeInfo = *this;
662 MergeInfo.SEWLMULRatioOnly = true;
663 return MergeInfo;
664 }
665
666 // Otherwise the result is unknown.
667 return VSETVLIInfo::getUnknown();
668 }
669
670#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
671 /// Support for debugging, callable in GDB: V->dump()
672 LLVM_DUMP_METHOD void dump() const {
673 print(dbgs());
674 dbgs() << "\n";
675 }
676
677 /// Implement operator<<.
678 /// @{
679 void print(raw_ostream &OS) const {
680 OS << "{";
681 if (!isValid())
682 OS << "Uninitialized";
683 if (isUnknown())
684 OS << "unknown";
685 if (hasAVLReg())
686 OS << "AVLReg=" << (unsigned)AVLReg;
687 if (hasAVLImm())
688 OS << "AVLImm=" << (unsigned)AVLImm;
689 OS << ", "
690 << "VLMul=" << (unsigned)VLMul << ", "
691 << "SEW=" << (unsigned)SEW << ", "
692 << "TailAgnostic=" << (bool)TailAgnostic << ", "
693 << "MaskAgnostic=" << (bool)MaskAgnostic << ", "
694 << "SEWLMULRatioOnly=" << (bool)SEWLMULRatioOnly << "}";
695 }
696#endif
697};
698
699#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
701inline raw_ostream &operator<<(raw_ostream &OS, const VSETVLIInfo &V) {
702 V.print(OS);
703 return OS;
704}
705#endif
706
707struct BlockData {
708 // The VSETVLIInfo that represents the VL/VTYPE settings on exit from this
709 // block. Calculated in Phase 2.
710 VSETVLIInfo Exit;
711
712 // The VSETVLIInfo that represents the VL/VTYPE settings from all predecessor
713 // blocks. Calculated in Phase 2, and used by Phase 3.
714 VSETVLIInfo Pred;
715
716 // Keeps track of whether the block is already in the queue.
717 bool InQueue = false;
718
719 BlockData() = default;
720};
721
722class RISCVInsertVSETVLI : public MachineFunctionPass {
723 const TargetInstrInfo *TII;
725
726 std::vector<BlockData> BlockInfo;
727 std::queue<const MachineBasicBlock *> WorkList;
728
729public:
730 static char ID;
731
732 RISCVInsertVSETVLI() : MachineFunctionPass(ID) {
734 }
735 bool runOnMachineFunction(MachineFunction &MF) override;
736
737 void getAnalysisUsage(AnalysisUsage &AU) const override {
738 AU.setPreservesCFG();
740 }
741
742 StringRef getPassName() const override { return RISCV_INSERT_VSETVLI_NAME; }
743
744private:
745 bool needVSETVLI(const MachineInstr &MI, const VSETVLIInfo &Require,
746 const VSETVLIInfo &CurInfo) const;
747 bool needVSETVLIPHI(const VSETVLIInfo &Require,
748 const MachineBasicBlock &MBB) const;
749 void insertVSETVLI(MachineBasicBlock &MBB, MachineInstr &MI,
750 const VSETVLIInfo &Info, const VSETVLIInfo &PrevInfo);
751 void insertVSETVLI(MachineBasicBlock &MBB,
753 const VSETVLIInfo &Info, const VSETVLIInfo &PrevInfo);
754
755 void transferBefore(VSETVLIInfo &Info, const MachineInstr &MI) const;
756 void transferAfter(VSETVLIInfo &Info, const MachineInstr &MI) const;
757 bool computeVLVTYPEChanges(const MachineBasicBlock &MBB,
758 VSETVLIInfo &Info) const;
759 void computeIncomingVLVTYPE(const MachineBasicBlock &MBB);
760 void emitVSETVLIs(MachineBasicBlock &MBB);
761 void doLocalPostpass(MachineBasicBlock &MBB);
762 void doPRE(MachineBasicBlock &MBB);
763 void insertReadVL(MachineBasicBlock &MBB);
764};
765
766} // end anonymous namespace
767
768char RISCVInsertVSETVLI::ID = 0;
769
771 false, false)
772
773static VSETVLIInfo computeInfoForInstr(const MachineInstr &MI, uint64_t TSFlags,
775 VSETVLIInfo InstrInfo;
776
777 bool TailAgnostic = true;
778 bool MaskAgnostic = true;
779 if (!hasUndefinedMergeOp(MI, *MRI)) {
780 // Start with undisturbed.
781 TailAgnostic = false;
782 MaskAgnostic = false;
783
784 // If there is a policy operand, use it.
786 const MachineOperand &Op = MI.getOperand(MI.getNumExplicitOperands() - 1);
787 uint64_t Policy = Op.getImm();
789 "Invalid Policy Value");
792 }
793
794 // Some pseudo instructions force a tail agnostic policy despite having a
795 // tied def.
797 TailAgnostic = true;
798
800 MaskAgnostic = true;
801 }
802
804
805 unsigned Log2SEW = MI.getOperand(getSEWOpNum(MI)).getImm();
806 // A Log2SEW of 0 is an operation on mask registers only.
807 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
808 assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW");
809
811 const MachineOperand &VLOp = MI.getOperand(getVLOpNum(MI));
812 if (VLOp.isImm()) {
813 int64_t Imm = VLOp.getImm();
814 // Conver the VLMax sentintel to X0 register.
815 if (Imm == RISCV::VLMaxSentinel)
816 InstrInfo.setAVLReg(RISCV::X0);
817 else
818 InstrInfo.setAVLImm(Imm);
819 } else {
820 InstrInfo.setAVLReg(VLOp.getReg());
821 }
822 } else {
823 assert(isScalarExtractInstr(MI));
824 InstrInfo.setAVLReg(RISCV::NoRegister);
825 }
826#ifndef NDEBUG
827 if (std::optional<unsigned> EEW = getEEWForLoadStore(MI)) {
828 assert(SEW == EEW && "Initial SEW doesn't match expected EEW");
829 }
830#endif
832
833 return InstrInfo;
834}
835
836void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB, MachineInstr &MI,
837 const VSETVLIInfo &Info,
838 const VSETVLIInfo &PrevInfo) {
839 DebugLoc DL = MI.getDebugLoc();
840 insertVSETVLI(MBB, MachineBasicBlock::iterator(&MI), DL, Info, PrevInfo);
841}
842
843// Return a VSETVLIInfo representing the changes made by this VSETVLI or
844// VSETIVLI instruction.
845static VSETVLIInfo getInfoForVSETVLI(const MachineInstr &MI) {
846 VSETVLIInfo NewInfo;
847 if (MI.getOpcode() == RISCV::PseudoVSETIVLI) {
848 NewInfo.setAVLImm(MI.getOperand(1).getImm());
849 } else {
850 assert(MI.getOpcode() == RISCV::PseudoVSETVLI ||
851 MI.getOpcode() == RISCV::PseudoVSETVLIX0);
852 Register AVLReg = MI.getOperand(1).getReg();
853 assert((AVLReg != RISCV::X0 || MI.getOperand(0).getReg() != RISCV::X0) &&
854 "Can't handle X0, X0 vsetvli yet");
855 NewInfo.setAVLReg(AVLReg);
856 }
857 NewInfo.setVTYPE(MI.getOperand(2).getImm());
858
859 return NewInfo;
860}
861
862void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB,
864 const VSETVLIInfo &Info, const VSETVLIInfo &PrevInfo) {
865
866 if (PrevInfo.isValid() && !PrevInfo.isUnknown()) {
867 // Use X0, X0 form if the AVL is the same and the SEW+LMUL gives the same
868 // VLMAX.
869 if (Info.hasSameAVL(PrevInfo) && Info.hasSameVLMAX(PrevInfo)) {
870 BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))
872 .addReg(RISCV::X0, RegState::Kill)
873 .addImm(Info.encodeVTYPE())
874 .addReg(RISCV::VL, RegState::Implicit);
875 return;
876 }
877
878 // If our AVL is a virtual register, it might be defined by a VSET(I)VLI. If
879 // it has the same VLMAX we want and the last VL/VTYPE we observed is the
880 // same, we can use the X0, X0 form.
881 if (Info.hasSameVLMAX(PrevInfo) && Info.hasAVLReg() &&
882 Info.getAVLReg().isVirtual()) {
883 if (MachineInstr *DefMI = MRI->getVRegDef(Info.getAVLReg())) {
884 if (isVectorConfigInstr(*DefMI)) {
885 VSETVLIInfo DefInfo = getInfoForVSETVLI(*DefMI);
886 if (DefInfo.hasSameAVL(PrevInfo) && DefInfo.hasSameVLMAX(PrevInfo)) {
887 BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))
889 .addReg(RISCV::X0, RegState::Kill)
890 .addImm(Info.encodeVTYPE())
891 .addReg(RISCV::VL, RegState::Implicit);
892 return;
893 }
894 }
895 }
896 }
897 }
898
899 if (Info.hasAVLImm()) {
900 BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETIVLI))
902 .addImm(Info.getAVLImm())
903 .addImm(Info.encodeVTYPE());
904 return;
905 }
906
907 Register AVLReg = Info.getAVLReg();
908 if (AVLReg == RISCV::NoRegister) {
909 // We can only use x0, x0 if there's no chance of the vtype change causing
910 // the previous vl to become invalid.
911 if (PrevInfo.isValid() && !PrevInfo.isUnknown() &&
912 Info.hasSameVLMAX(PrevInfo)) {
913 BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))
915 .addReg(RISCV::X0, RegState::Kill)
916 .addImm(Info.encodeVTYPE())
917 .addReg(RISCV::VL, RegState::Implicit);
918 return;
919 }
920 // Otherwise use an AVL of 1 to avoid depending on previous vl.
921 BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETIVLI))
923 .addImm(1)
924 .addImm(Info.encodeVTYPE());
925 return;
926 }
927
928 if (AVLReg.isVirtual())
929 MRI->constrainRegClass(AVLReg, &RISCV::GPRNoX0RegClass);
930
931 // Use X0 as the DestReg unless AVLReg is X0. We also need to change the
932 // opcode if the AVLReg is X0 as they have different register classes for
933 // the AVL operand.
934 Register DestReg = RISCV::X0;
935 unsigned Opcode = RISCV::PseudoVSETVLI;
936 if (AVLReg == RISCV::X0) {
937 DestReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
938 Opcode = RISCV::PseudoVSETVLIX0;
939 }
940 BuildMI(MBB, InsertPt, DL, TII->get(Opcode))
942 .addReg(AVLReg)
943 .addImm(Info.encodeVTYPE());
944}
945
947 auto [LMul, Fractional] = RISCVVType::decodeVLMUL(LMUL);
948 return Fractional || LMul == 1;
949}
950
951/// Return true if a VSETVLI is required to transition from CurInfo to Require
952/// before MI.
953bool RISCVInsertVSETVLI::needVSETVLI(const MachineInstr &MI,
954 const VSETVLIInfo &Require,
955 const VSETVLIInfo &CurInfo) const {
956 assert(Require == computeInfoForInstr(MI, MI.getDesc().TSFlags, MRI));
957
958 if (!CurInfo.isValid() || CurInfo.isUnknown() || CurInfo.hasSEWLMULRatioOnly())
959 return true;
960
961 DemandedFields Used = getDemanded(MI, MRI);
962 bool HasVInstructionsF64 =
963 MI.getMF()->getSubtarget<RISCVSubtarget>().hasVInstructionsF64();
964
965 // A slidedown/slideup with an *undefined* merge op can freely clobber
966 // elements not copied from the source vector (e.g. masked off, tail, or
967 // slideup's prefix). Notes:
968 // * We can't modify SEW here since the slide amount is in units of SEW.
969 // * VL=1 is special only because we have existing support for zero vs
970 // non-zero VL. We could generalize this if we had a VL > C predicate.
971 // * The LMUL1 restriction is for machines whose latency may depend on VL.
972 // * As above, this is only legal for tail "undefined" not "agnostic".
973 if (isVSlideInstr(MI) && Require.hasAVLImm() && Require.getAVLImm() == 1 &&
974 isLMUL1OrSmaller(CurInfo.getVLMUL()) && hasUndefinedMergeOp(MI, *MRI)) {
975 Used.VLAny = false;
976 Used.VLZeroness = true;
977 Used.LMUL = false;
978 Used.TailPolicy = false;
979 }
980
981 // A tail undefined vmv.v.i/x or vfmv.v.f with VL=1 can be treated in the same
982 // semantically as vmv.s.x. This is particularly useful since we don't have an
983 // immediate form of vmv.s.x, and thus frequently use vmv.v.i in it's place.
984 // Since a splat is non-constant time in LMUL, we do need to be careful to not
985 // increase the number of active vector registers (unlike for vmv.s.x.)
986 if (isScalarSplatInstr(MI) && Require.hasAVLImm() && Require.getAVLImm() == 1 &&
987 isLMUL1OrSmaller(CurInfo.getVLMUL()) && hasUndefinedMergeOp(MI, *MRI)) {
988 Used.LMUL = false;
989 Used.SEWLMULRatio = false;
990 Used.VLAny = false;
991 if (isFloatScalarMoveOrScalarSplatInstr(MI) && !HasVInstructionsF64)
992 Used.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
993 else
994 Used.SEW = DemandedFields::SEWGreaterThanOrEqual;
995 Used.TailPolicy = false;
996 }
997
998 if (CurInfo.isCompatible(Used, Require, *MRI))
999 return false;
1000
1001 // We didn't find a compatible value. If our AVL is a virtual register,
1002 // it might be defined by a VSET(I)VLI. If it has the same VLMAX we need
1003 // and the last VL/VTYPE we observed is the same, we don't need a
1004 // VSETVLI here.
1005 if (Require.hasAVLReg() && Require.getAVLReg().isVirtual() &&
1006 CurInfo.hasCompatibleVTYPE(Used, Require)) {
1007 if (MachineInstr *DefMI = MRI->getVRegDef(Require.getAVLReg())) {
1008 if (isVectorConfigInstr(*DefMI)) {
1009 VSETVLIInfo DefInfo = getInfoForVSETVLI(*DefMI);
1010 if (DefInfo.hasSameAVL(CurInfo) && DefInfo.hasSameVLMAX(CurInfo))
1011 return false;
1012 }
1013 }
1014 }
1015
1016 return true;
1017}
1018
1019// Given an incoming state reaching MI, modifies that state so that it is minimally
1020// compatible with MI. The resulting state is guaranteed to be semantically legal
1021// for MI, but may not be the state requested by MI.
1022void RISCVInsertVSETVLI::transferBefore(VSETVLIInfo &Info,
1023 const MachineInstr &MI) const {
1024 uint64_t TSFlags = MI.getDesc().TSFlags;
1026 return;
1027
1028 const VSETVLIInfo NewInfo = computeInfoForInstr(MI, TSFlags, MRI);
1029 if (Info.isValid() && !needVSETVLI(MI, NewInfo, Info))
1030 return;
1031
1032 const VSETVLIInfo PrevInfo = Info;
1033 Info = NewInfo;
1034
1036 return;
1037
1038 // For vmv.s.x and vfmv.s.f, there are only two behaviors, VL = 0 and
1039 // VL > 0. We can discard the user requested AVL and just use the last
1040 // one if we can prove it equally zero. This removes a vsetvli entirely
1041 // if the types match or allows use of cheaper avl preserving variant
1042 // if VLMAX doesn't change. If VLMAX might change, we couldn't use
1043 // the 'vsetvli x0, x0, vtype" variant, so we avoid the transform to
1044 // prevent extending live range of an avl register operand.
1045 // TODO: We can probably relax this for immediates.
1046 if (isScalarInsertInstr(MI) && PrevInfo.isValid() &&
1047 PrevInfo.hasEquallyZeroAVL(Info, *MRI) &&
1048 Info.hasSameVLMAX(PrevInfo)) {
1049 if (PrevInfo.hasAVLImm())
1050 Info.setAVLImm(PrevInfo.getAVLImm());
1051 else
1052 Info.setAVLReg(PrevInfo.getAVLReg());
1053 return;
1054 }
1055
1056 // If AVL is defined by a vsetvli with the same VLMAX, we can
1057 // replace the AVL operand with the AVL of the defining vsetvli.
1058 // We avoid general register AVLs to avoid extending live ranges
1059 // without being sure we can kill the original source reg entirely.
1060 if (!Info.hasAVLReg() || !Info.getAVLReg().isVirtual())
1061 return;
1062 MachineInstr *DefMI = MRI->getVRegDef(Info.getAVLReg());
1063 if (!DefMI || !isVectorConfigInstr(*DefMI))
1064 return;
1065
1066 VSETVLIInfo DefInfo = getInfoForVSETVLI(*DefMI);
1067 if (DefInfo.hasSameVLMAX(Info) &&
1068 (DefInfo.hasAVLImm() || DefInfo.getAVLReg() == RISCV::X0)) {
1069 if (DefInfo.hasAVLImm())
1070 Info.setAVLImm(DefInfo.getAVLImm());
1071 else
1072 Info.setAVLReg(DefInfo.getAVLReg());
1073 return;
1074 }
1075}
1076
1077// Given a state with which we evaluated MI (see transferBefore above for why
1078// this might be different that the state MI requested), modify the state to
1079// reflect the changes MI might make.
1080void RISCVInsertVSETVLI::transferAfter(VSETVLIInfo &Info,
1081 const MachineInstr &MI) const {
1082 if (isVectorConfigInstr(MI)) {
1084 return;
1085 }
1086
1088 // Update AVL to vl-output of the fault first load.
1089 Info.setAVLReg(MI.getOperand(1).getReg());
1090 return;
1091 }
1092
1093 // If this is something that updates VL/VTYPE that we don't know about, set
1094 // the state to unknown.
1095 if (MI.isCall() || MI.isInlineAsm() || MI.modifiesRegister(RISCV::VL) ||
1096 MI.modifiesRegister(RISCV::VTYPE))
1097 Info = VSETVLIInfo::getUnknown();
1098}
1099
1100bool RISCVInsertVSETVLI::computeVLVTYPEChanges(const MachineBasicBlock &MBB,
1101 VSETVLIInfo &Info) const {
1102 bool HadVectorOp = false;
1103
1104 Info = BlockInfo[MBB.getNumber()].Pred;
1105 for (const MachineInstr &MI : MBB) {
1106 transferBefore(Info, MI);
1107
1108 if (isVectorConfigInstr(MI) || RISCVII::hasSEWOp(MI.getDesc().TSFlags))
1109 HadVectorOp = true;
1110
1111 transferAfter(Info, MI);
1112 }
1113
1114 return HadVectorOp;
1115}
1116
1117void RISCVInsertVSETVLI::computeIncomingVLVTYPE(const MachineBasicBlock &MBB) {
1118
1119 BlockData &BBInfo = BlockInfo[MBB.getNumber()];
1120
1121 BBInfo.InQueue = false;
1122
1123 // Start with the previous entry so that we keep the most conservative state
1124 // we have ever found.
1125 VSETVLIInfo InInfo = BBInfo.Pred;
1126 if (MBB.pred_empty()) {
1127 // There are no predecessors, so use the default starting status.
1128 InInfo.setUnknown();
1129 } else {
1131 InInfo = InInfo.intersect(BlockInfo[P->getNumber()].Exit);
1132 }
1133
1134 // If we don't have any valid predecessor value, wait until we do.
1135 if (!InInfo.isValid())
1136 return;
1137
1138 // If no change, no need to rerun block
1139 if (InInfo == BBInfo.Pred)
1140 return;
1141
1142 BBInfo.Pred = InInfo;
1143 LLVM_DEBUG(dbgs() << "Entry state of " << printMBBReference(MBB)
1144 << " changed to " << BBInfo.Pred << "\n");
1145
1146 // Note: It's tempting to cache the state changes here, but due to the
1147 // compatibility checks performed a blocks output state can change based on
1148 // the input state. To cache, we'd have to add logic for finding
1149 // never-compatible state changes.
1150 VSETVLIInfo TmpStatus;
1151 computeVLVTYPEChanges(MBB, TmpStatus);
1152
1153 // If the new exit value matches the old exit value, we don't need to revisit
1154 // any blocks.
1155 if (BBInfo.Exit == TmpStatus)
1156 return;
1157
1158 BBInfo.Exit = TmpStatus;
1159 LLVM_DEBUG(dbgs() << "Exit state of " << printMBBReference(MBB)
1160 << " changed to " << BBInfo.Exit << "\n");
1161
1162 // Add the successors to the work list so we can propagate the changed exit
1163 // status.
1164 for (MachineBasicBlock *S : MBB.successors())
1165 if (!BlockInfo[S->getNumber()].InQueue) {
1166 BlockInfo[S->getNumber()].InQueue = true;
1167 WorkList.push(S);
1168 }
1169}
1170
1171// If we weren't able to prove a vsetvli was directly unneeded, it might still
1172// be unneeded if the AVL is a phi node where all incoming values are VL
1173// outputs from the last VSETVLI in their respective basic blocks.
1174bool RISCVInsertVSETVLI::needVSETVLIPHI(const VSETVLIInfo &Require,
1175 const MachineBasicBlock &MBB) const {
1177 return true;
1178
1179 if (!Require.hasAVLReg())
1180 return true;
1181
1182 Register AVLReg = Require.getAVLReg();
1183 if (!AVLReg.isVirtual())
1184 return true;
1185
1186 // We need the AVL to be produce by a PHI node in this basic block.
1187 MachineInstr *PHI = MRI->getVRegDef(AVLReg);
1188 if (!PHI || PHI->getOpcode() != RISCV::PHI || PHI->getParent() != &MBB)
1189 return true;
1190
1191 for (unsigned PHIOp = 1, NumOps = PHI->getNumOperands(); PHIOp != NumOps;
1192 PHIOp += 2) {
1193 Register InReg = PHI->getOperand(PHIOp).getReg();
1194 MachineBasicBlock *PBB = PHI->getOperand(PHIOp + 1).getMBB();
1195 const BlockData &PBBInfo = BlockInfo[PBB->getNumber()];
1196 // If the exit from the predecessor has the VTYPE we are looking for
1197 // we might be able to avoid a VSETVLI.
1198 if (PBBInfo.Exit.isUnknown() || !PBBInfo.Exit.hasSameVTYPE(Require))
1199 return true;
1200
1201 // We need the PHI input to the be the output of a VSET(I)VLI.
1202 MachineInstr *DefMI = MRI->getVRegDef(InReg);
1203 if (!DefMI || !isVectorConfigInstr(*DefMI))
1204 return true;
1205
1206 // We found a VSET(I)VLI make sure it matches the output of the
1207 // predecessor block.
1208 VSETVLIInfo DefInfo = getInfoForVSETVLI(*DefMI);
1209 if (!DefInfo.hasSameAVL(PBBInfo.Exit) ||
1210 !DefInfo.hasSameVTYPE(PBBInfo.Exit))
1211 return true;
1212 }
1213
1214 // If all the incoming values to the PHI checked out, we don't need
1215 // to insert a VSETVLI.
1216 return false;
1217}
1218
1219void RISCVInsertVSETVLI::emitVSETVLIs(MachineBasicBlock &MBB) {
1220 VSETVLIInfo CurInfo = BlockInfo[MBB.getNumber()].Pred;
1221 // Track whether the prefix of the block we've scanned is transparent
1222 // (meaning has not yet changed the abstract state).
1223 bool PrefixTransparent = true;
1224 for (MachineInstr &MI : MBB) {
1225 const VSETVLIInfo PrevInfo = CurInfo;
1226 transferBefore(CurInfo, MI);
1227
1228 // If this is an explicit VSETVLI or VSETIVLI, update our state.
1229 if (isVectorConfigInstr(MI)) {
1230 // Conservatively, mark the VL and VTYPE as live.
1231 assert(MI.getOperand(3).getReg() == RISCV::VL &&
1232 MI.getOperand(4).getReg() == RISCV::VTYPE &&
1233 "Unexpected operands where VL and VTYPE should be");
1234 MI.getOperand(3).setIsDead(false);
1235 MI.getOperand(4).setIsDead(false);
1236 PrefixTransparent = false;
1237 }
1238
1239 uint64_t TSFlags = MI.getDesc().TSFlags;
1241 if (PrevInfo != CurInfo) {
1242 // If this is the first implicit state change, and the state change
1243 // requested can be proven to produce the same register contents, we
1244 // can skip emitting the actual state change and continue as if we
1245 // had since we know the GPR result of the implicit state change
1246 // wouldn't be used and VL/VTYPE registers are correct. Note that
1247 // we *do* need to model the state as if it changed as while the
1248 // register contents are unchanged, the abstract model can change.
1249 if (!PrefixTransparent || needVSETVLIPHI(CurInfo, MBB))
1250 insertVSETVLI(MBB, MI, CurInfo, PrevInfo);
1251 PrefixTransparent = false;
1252 }
1253
1255 MachineOperand &VLOp = MI.getOperand(getVLOpNum(MI));
1256 if (VLOp.isReg()) {
1257 // Erase the AVL operand from the instruction.
1258 VLOp.setReg(RISCV::NoRegister);
1259 VLOp.setIsKill(false);
1260 }
1261 MI.addOperand(MachineOperand::CreateReg(RISCV::VL, /*isDef*/ false,
1262 /*isImp*/ true));
1263 }
1264 MI.addOperand(MachineOperand::CreateReg(RISCV::VTYPE, /*isDef*/ false,
1265 /*isImp*/ true));
1266 }
1267
1268 if (MI.isCall() || MI.isInlineAsm() || MI.modifiesRegister(RISCV::VL) ||
1269 MI.modifiesRegister(RISCV::VTYPE))
1270 PrefixTransparent = false;
1271
1272 transferAfter(CurInfo, MI);
1273 }
1274
1275 // If we reach the end of the block and our current info doesn't match the
1276 // expected info, insert a vsetvli to correct.
1277 if (!UseStrictAsserts) {
1278 const VSETVLIInfo &ExitInfo = BlockInfo[MBB.getNumber()].Exit;
1279 if (CurInfo.isValid() && ExitInfo.isValid() && !ExitInfo.isUnknown() &&
1280 CurInfo != ExitInfo) {
1281 // Note there's an implicit assumption here that terminators never use
1282 // or modify VL or VTYPE. Also, fallthrough will return end().
1283 auto InsertPt = MBB.getFirstInstrTerminator();
1284 insertVSETVLI(MBB, InsertPt, MBB.findDebugLoc(InsertPt), ExitInfo,
1285 CurInfo);
1286 CurInfo = ExitInfo;
1287 }
1288 }
1289
1290 if (UseStrictAsserts && CurInfo.isValid()) {
1291 const auto &Info = BlockInfo[MBB.getNumber()];
1292 if (CurInfo != Info.Exit) {
1293 LLVM_DEBUG(dbgs() << "in block " << printMBBReference(MBB) << "\n");
1294 LLVM_DEBUG(dbgs() << " begin state: " << Info.Pred << "\n");
1295 LLVM_DEBUG(dbgs() << " expected end state: " << Info.Exit << "\n");
1296 LLVM_DEBUG(dbgs() << " actual end state: " << CurInfo << "\n");
1297 }
1298 assert(CurInfo == Info.Exit &&
1299 "InsertVSETVLI dataflow invariant violated");
1300 }
1301}
1302
1303/// Return true if the VL value configured must be equal to the requested one.
1304static bool hasFixedResult(const VSETVLIInfo &Info, const RISCVSubtarget &ST) {
1305 if (!Info.hasAVLImm())
1306 // VLMAX is always the same value.
1307 // TODO: Could extend to other registers by looking at the associated vreg
1308 // def placement.
1309 return RISCV::X0 == Info.getAVLReg();
1310
1311 unsigned AVL = Info.getAVLImm();
1312 unsigned SEW = Info.getSEW();
1313 unsigned AVLInBits = AVL * SEW;
1314
1315 unsigned LMul;
1316 bool Fractional;
1317 std::tie(LMul, Fractional) = RISCVVType::decodeVLMUL(Info.getVLMUL());
1318
1319 if (Fractional)
1320 return ST.getRealMinVLen() / LMul >= AVLInBits;
1321 return ST.getRealMinVLen() * LMul >= AVLInBits;
1322}
1323
1324/// Perform simple partial redundancy elimination of the VSETVLI instructions
1325/// we're about to insert by looking for cases where we can PRE from the
1326/// beginning of one block to the end of one of its predecessors. Specifically,
1327/// this is geared to catch the common case of a fixed length vsetvl in a single
1328/// block loop when it could execute once in the preheader instead.
1329void RISCVInsertVSETVLI::doPRE(MachineBasicBlock &MBB) {
1330 const MachineFunction &MF = *MBB.getParent();
1332
1333 if (!BlockInfo[MBB.getNumber()].Pred.isUnknown())
1334 return;
1335
1336 MachineBasicBlock *UnavailablePred = nullptr;
1337 VSETVLIInfo AvailableInfo;
1338 for (MachineBasicBlock *P : MBB.predecessors()) {
1339 const VSETVLIInfo &PredInfo = BlockInfo[P->getNumber()].Exit;
1340 if (PredInfo.isUnknown()) {
1341 if (UnavailablePred)
1342 return;
1343 UnavailablePred = P;
1344 } else if (!AvailableInfo.isValid()) {
1345 AvailableInfo = PredInfo;
1346 } else if (AvailableInfo != PredInfo) {
1347 return;
1348 }
1349 }
1350
1351 // Unreachable, single pred, or full redundancy. Note that FRE is handled by
1352 // phase 3.
1353 if (!UnavailablePred || !AvailableInfo.isValid())
1354 return;
1355
1356 // Critical edge - TODO: consider splitting?
1357 if (UnavailablePred->succ_size() != 1)
1358 return;
1359
1360 // If VL can be less than AVL, then we can't reduce the frequency of exec.
1361 if (!hasFixedResult(AvailableInfo, ST))
1362 return;
1363
1364 // Model the effect of changing the input state of the block MBB to
1365 // AvailableInfo. We're looking for two issues here; one legality,
1366 // one profitability.
1367 // 1) If the block doesn't use some of the fields from VL or VTYPE, we
1368 // may hit the end of the block with a different end state. We can
1369 // not make this change without reflowing later blocks as well.
1370 // 2) If we don't actually remove a transition, inserting a vsetvli
1371 // into the predecessor block would be correct, but unprofitable.
1372 VSETVLIInfo OldInfo = BlockInfo[MBB.getNumber()].Pred;
1373 VSETVLIInfo CurInfo = AvailableInfo;
1374 int TransitionsRemoved = 0;
1375 for (const MachineInstr &MI : MBB) {
1376 const VSETVLIInfo LastInfo = CurInfo;
1377 const VSETVLIInfo LastOldInfo = OldInfo;
1378 transferBefore(CurInfo, MI);
1379 transferBefore(OldInfo, MI);
1380 if (CurInfo == LastInfo)
1381 TransitionsRemoved++;
1382 if (LastOldInfo == OldInfo)
1383 TransitionsRemoved--;
1384 transferAfter(CurInfo, MI);
1385 transferAfter(OldInfo, MI);
1386 if (CurInfo == OldInfo)
1387 // Convergence. All transitions after this must match by construction.
1388 break;
1389 }
1390 if (CurInfo != OldInfo || TransitionsRemoved <= 0)
1391 // Issues 1 and 2 above
1392 return;
1393
1394 // Finally, update both data flow state and insert the actual vsetvli.
1395 // Doing both keeps the code in sync with the dataflow results, which
1396 // is critical for correctness of phase 3.
1397 auto OldExit = BlockInfo[UnavailablePred->getNumber()].Exit;
1398 LLVM_DEBUG(dbgs() << "PRE VSETVLI from " << MBB.getName() << " to "
1399 << UnavailablePred->getName() << " with state "
1400 << AvailableInfo << "\n");
1401 BlockInfo[UnavailablePred->getNumber()].Exit = AvailableInfo;
1402 BlockInfo[MBB.getNumber()].Pred = AvailableInfo;
1403
1404 // Note there's an implicit assumption here that terminators never use
1405 // or modify VL or VTYPE. Also, fallthrough will return end().
1406 auto InsertPt = UnavailablePred->getFirstInstrTerminator();
1407 insertVSETVLI(*UnavailablePred, InsertPt,
1408 UnavailablePred->findDebugLoc(InsertPt),
1409 AvailableInfo, OldExit);
1410}
1411
1412static void doUnion(DemandedFields &A, DemandedFields B) {
1413 A.VLAny |= B.VLAny;
1414 A.VLZeroness |= B.VLZeroness;
1415 A.SEW = std::max(A.SEW, B.SEW);
1416 A.LMUL |= B.LMUL;
1417 A.SEWLMULRatio |= B.SEWLMULRatio;
1418 A.TailPolicy |= B.TailPolicy;
1419 A.MaskPolicy |= B.MaskPolicy;
1420}
1421
1422static bool isNonZeroAVL(const MachineOperand &MO) {
1423 if (MO.isReg())
1424 return RISCV::X0 == MO.getReg();
1425 assert(MO.isImm());
1426 return 0 != MO.getImm();
1427}
1428
1429// Return true if we can mutate PrevMI to match MI without changing any the
1430// fields which would be observed.
1431static bool canMutatePriorConfig(const MachineInstr &PrevMI,
1432 const MachineInstr &MI,
1433 const DemandedFields &Used) {
1434 // If the VL values aren't equal, return false if either a) the former is
1435 // demanded, or b) we can't rewrite the former to be the later for
1436 // implementation reasons.
1437 if (!isVLPreservingConfig(MI)) {
1438 if (Used.VLAny)
1439 return false;
1440
1441 // We don't bother to handle the equally zero case here as it's largely
1442 // uninteresting.
1443 if (Used.VLZeroness) {
1444 if (isVLPreservingConfig(PrevMI))
1445 return false;
1446 if (!isNonZeroAVL(MI.getOperand(1)) ||
1447 !isNonZeroAVL(PrevMI.getOperand(1)))
1448 return false;
1449 }
1450
1451 // TODO: Track whether the register is defined between
1452 // PrevMI and MI.
1453 if (MI.getOperand(1).isReg() &&
1454 RISCV::X0 != MI.getOperand(1).getReg())
1455 return false;
1456 }
1457
1458 if (!PrevMI.getOperand(2).isImm() || !MI.getOperand(2).isImm())
1459 return false;
1460
1461 auto PriorVType = PrevMI.getOperand(2).getImm();
1462 auto VType = MI.getOperand(2).getImm();
1463 return areCompatibleVTYPEs(PriorVType, VType, Used);
1464}
1465
1466void RISCVInsertVSETVLI::doLocalPostpass(MachineBasicBlock &MBB) {
1467 MachineInstr *NextMI = nullptr;
1468 // We can have arbitrary code in successors, so VL and VTYPE
1469 // must be considered demanded.
1470 DemandedFields Used;
1471 Used.demandVL();
1472 Used.demandVTYPE();
1474 for (MachineInstr &MI : make_range(MBB.rbegin(), MBB.rend())) {
1475
1476 if (!isVectorConfigInstr(MI)) {
1477 doUnion(Used, getDemanded(MI, MRI));
1478 continue;
1479 }
1480
1481 Register VRegDef = MI.getOperand(0).getReg();
1482 if (VRegDef != RISCV::X0 &&
1483 !(VRegDef.isVirtual() && MRI->use_nodbg_empty(VRegDef)))
1484 Used.demandVL();
1485
1486 if (NextMI) {
1487 if (!Used.usedVL() && !Used.usedVTYPE()) {
1488 ToDelete.push_back(&MI);
1489 // Leave NextMI unchanged
1490 continue;
1491 } else if (canMutatePriorConfig(MI, *NextMI, Used)) {
1492 if (!isVLPreservingConfig(*NextMI)) {
1493 MI.getOperand(0).setReg(NextMI->getOperand(0).getReg());
1494 MI.getOperand(0).setIsDead(false);
1495 if (NextMI->getOperand(1).isImm())
1496 MI.getOperand(1).ChangeToImmediate(NextMI->getOperand(1).getImm());
1497 else
1498 MI.getOperand(1).ChangeToRegister(NextMI->getOperand(1).getReg(), false);
1499 MI.setDesc(NextMI->getDesc());
1500 }
1501 MI.getOperand(2).setImm(NextMI->getOperand(2).getImm());
1502 ToDelete.push_back(NextMI);
1503 // fallthrough
1504 }
1505 }
1506 NextMI = &MI;
1507 Used = getDemanded(MI, MRI);
1508 }
1509
1510 for (auto *MI : ToDelete)
1511 MI->eraseFromParent();
1512}
1513
1514void RISCVInsertVSETVLI::insertReadVL(MachineBasicBlock &MBB) {
1515 for (auto I = MBB.begin(), E = MBB.end(); I != E;) {
1516 MachineInstr &MI = *I++;
1518 Register VLOutput = MI.getOperand(1).getReg();
1519 if (!MRI->use_nodbg_empty(VLOutput))
1520 BuildMI(MBB, I, MI.getDebugLoc(), TII->get(RISCV::PseudoReadVL),
1521 VLOutput);
1522 // We don't use the vl output of the VLEFF/VLSEGFF anymore.
1523 MI.getOperand(1).setReg(RISCV::X0);
1524 }
1525 }
1526}
1527
1528bool RISCVInsertVSETVLI::runOnMachineFunction(MachineFunction &MF) {
1529 // Skip if the vector extension is not enabled.
1531 if (!ST.hasVInstructions())
1532 return false;
1533
1534 LLVM_DEBUG(dbgs() << "Entering InsertVSETVLI for " << MF.getName() << "\n");
1535
1536 TII = ST.getInstrInfo();
1537 MRI = &MF.getRegInfo();
1538
1539 assert(BlockInfo.empty() && "Expect empty block infos");
1540 BlockInfo.resize(MF.getNumBlockIDs());
1541
1542 bool HaveVectorOp = false;
1543
1544 // Phase 1 - determine how VL/VTYPE are affected by the each block.
1545 for (const MachineBasicBlock &MBB : MF) {
1546 VSETVLIInfo TmpStatus;
1547 HaveVectorOp |= computeVLVTYPEChanges(MBB, TmpStatus);
1548 // Initial exit state is whatever change we found in the block.
1549 BlockData &BBInfo = BlockInfo[MBB.getNumber()];
1550 BBInfo.Exit = TmpStatus;
1551 LLVM_DEBUG(dbgs() << "Initial exit state of " << printMBBReference(MBB)
1552 << " is " << BBInfo.Exit << "\n");
1553
1554 }
1555
1556 // If we didn't find any instructions that need VSETVLI, we're done.
1557 if (!HaveVectorOp) {
1558 BlockInfo.clear();
1559 return false;
1560 }
1561
1562 // Phase 2 - determine the exit VL/VTYPE from each block. We add all
1563 // blocks to the list here, but will also add any that need to be revisited
1564 // during Phase 2 processing.
1565 for (const MachineBasicBlock &MBB : MF) {
1566 WorkList.push(&MBB);
1567 BlockInfo[MBB.getNumber()].InQueue = true;
1568 }
1569 while (!WorkList.empty()) {
1570 const MachineBasicBlock &MBB = *WorkList.front();
1571 WorkList.pop();
1572 computeIncomingVLVTYPE(MBB);
1573 }
1574
1575 // Perform partial redundancy elimination of vsetvli transitions.
1576 for (MachineBasicBlock &MBB : MF)
1577 doPRE(MBB);
1578
1579 // Phase 3 - add any vsetvli instructions needed in the block. Use the
1580 // Phase 2 information to avoid adding vsetvlis before the first vector
1581 // instruction in the block if the VL/VTYPE is satisfied by its
1582 // predecessors.
1583 for (MachineBasicBlock &MBB : MF)
1584 emitVSETVLIs(MBB);
1585
1586 // Now that all vsetvlis are explicit, go through and do block local
1587 // DSE and peephole based demanded fields based transforms. Note that
1588 // this *must* be done outside the main dataflow so long as we allow
1589 // any cross block analysis within the dataflow. We can't have both
1590 // demanded fields based mutation and non-local analysis in the
1591 // dataflow at the same time without introducing inconsistencies.
1592 for (MachineBasicBlock &MBB : MF)
1593 doLocalPostpass(MBB);
1594
1595 // Insert PseudoReadVL after VLEFF/VLSEGFF and replace it with the vl output
1596 // of VLEFF/VLSEGFF.
1597 for (MachineBasicBlock &MBB : MF)
1598 insertReadVL(MBB);
1599
1600 BlockInfo.clear();
1601 return HaveVectorOp;
1602}
1603
1604/// Returns an instance of the Insert VSETVLI pass.
1606 return new RISCVInsertVSETVLI();
1607}
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
aarch64 promote const
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Rewrite undef for PHI
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
#define LLVM_ATTRIBUTE_USED
Definition: Compiler.h:151
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Definition: Compiler.h:510
#define LLVM_DEBUG(X)
Definition: Debug.h:101
static RegisterPass< DebugifyFunctionPass > DF("debugify-function", "Attach debug info to a function")
std::optional< std::vector< StOtherPiece > > Other
Definition: ELFYAML.cpp:1272
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
static ValueLatticeElement intersect(const ValueLatticeElement &A, const ValueLatticeElement &B)
Combine two sets of facts about the same value into a single set of facts.
#define I(x, y, z)
Definition: MD5.cpp:58
#define P(N)
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
static VSETVLIInfo getInfoForVSETVLI(const MachineInstr &MI)
static bool isNonZeroAVL(const MachineOperand &MO)
#define RISCV_INSERT_VSETVLI_NAME
uint64_t const MachineRegisterInfo * MRI
return InstrInfo
bool TailAgnostic
bool MaskAgnostic
unsigned Log2SEW
static bool canMutatePriorConfig(const MachineInstr &PrevMI, const MachineInstr &MI, const DemandedFields &Used)
RISCVII::VLMUL VLMul
assert(RISCVVType::isValidSEW(SEW) &&"Unexpected SEW")
InstrInfo setVTYPE(VLMul, SEW, TailAgnostic, MaskAgnostic)
static bool isLMUL1OrSmaller(RISCVII::VLMUL LMUL)
InstrInfo setAVLReg(RISCV::NoRegister)
static void doUnion(DemandedFields &A, DemandedFields B)
unsigned SEW
static cl::opt< bool > UseStrictAsserts("riscv-insert-vsetvl-strict-asserts", cl::init(true), cl::Hidden, cl::desc("Enable strict assertion checking for the dataflow algorithm"))
static cl::opt< bool > DisableInsertVSETVLPHIOpt("riscv-disable-insert-vsetvl-phi-opt", cl::init(false), cl::Hidden, cl::desc("Disable looking through phis when inserting vsetvlis."))
uint64_t TSFlags
#define DEBUG_TYPE
static bool hasFixedResult(const VSETVLIInfo &Info, const RISCVSubtarget &ST)
Return true if the VL value configured must be equal to the requested one.
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
raw_pwrite_stream & OS
BlockData()=default
Represent the analysis usage information of a pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:269
This class represents an Operation in the Expression.
A debug info location.
Definition: DebugLoc.h:33
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
reverse_iterator rend()
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
unsigned succ_size() const
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator_range< succ_iterator > successors()
instr_iterator getFirstInstrTerminator()
Same getFirstTerminator but it ignores bundles and return an instr_iterator instead.
reverse_iterator rbegin()
iterator_range< pred_iterator > predecessors()
StringRef getName() const
Return the name of the corresponding LLVM basic block, or an empty string.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
unsigned getNumBlockIDs() const
getNumBlockIDs - Return the number of MBB ID's allocated.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
Definition: MachineInstr.h:68
bool isImplicitDef() const
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:546
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
Definition: MachineInstr.h:540
bool isRegSequence() const
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:553
MachineOperand class - Representation of each machine instruction operand.
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Definition: Pass.cpp:81
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:91
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
TargetInstrInfo - Interface to description of machine instruction set.
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
static bool usesMaskPolicy(uint64_t TSFlags)
static unsigned getVLOpNum(const MCInstrDesc &Desc)
static bool doesForceTailAgnostic(uint64_t TSFlags)
static VLMUL getLMul(uint64_t TSFlags)
static bool hasVLOp(uint64_t TSFlags)
static bool hasVecPolicyOp(uint64_t TSFlags)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
static bool hasSEWOp(uint64_t TSFlags)
static bool isTailAgnostic(unsigned VType)
static RISCVII::VLMUL getVLMUL(unsigned VType)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
unsigned getSEWLMULRatio(unsigned SEW, RISCVII::VLMUL VLMul)
static bool isMaskAgnostic(unsigned VType)
static bool isValidSEW(unsigned SEW)
unsigned encodeVTYPE(RISCVII::VLMUL VLMUL, unsigned SEW, bool TailAgnostic, bool MaskAgnostic)
static unsigned getSEW(unsigned VType)
bool isFaultFirstLoad(const MachineInstr &MI)
static constexpr int64_t VLMaxSentinel
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Dead
Unused definition.
@ Define
Register definition.
@ Kill
The last use of a register.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
@ Uninitialized
Definition: Threading.h:61
bool operator!=(uint64_t V1, const APInt &V2)
Definition: APInt.h:2036
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void initializeRISCVInsertVSETVLIPass(PassRegistry &)
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
FunctionPass * createRISCVInsertVSETVLIPass()
Returns an instance of the Insert VSETVLI pass.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
Definition: APFixedPoint.h:292
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Status intersect(const Status &S) const