37#define DEBUG_TYPE "riscv-insert-vsetvli"
38#define RISCV_INSERT_VSETVLI_NAME "RISC-V Insert VSETVLI pass"
39#define RISCV_COALESCE_VSETVLI_NAME "RISC-V Coalesce VSETVLI pass"
41STATISTIC(NumInsertedVSETVL,
"Number of VSETVL inst inserted");
42STATISTIC(NumCoalescedVSETVL,
"Number of VSETVL inst coalesced");
46 cl::desc(
"Disable looking through phis when inserting vsetvlis."));
59 return MI.getOpcode() == RISCV::PseudoVSETVLI ||
60 MI.getOpcode() == RISCV::PseudoVSETVLIX0 ||
61 MI.getOpcode() == RISCV::PseudoVSETIVLI;
67 if (
MI.getOpcode() != RISCV::PseudoVSETVLIX0)
69 assert(RISCV::X0 ==
MI.getOperand(1).getReg());
70 return RISCV::X0 ==
MI.getOperand(0).getReg();
73static bool isFloatScalarMoveOrScalarSplatInstr(
const MachineInstr &
MI) {
109 case RISCV::VFMV_V_F:
118 case RISCV::VSLIDEDOWN_VX:
119 case RISCV::VSLIDEDOWN_VI:
120 case RISCV::VSLIDEUP_VX:
121 case RISCV::VSLIDEUP_VI:
128static std::optional<unsigned> getEEWForLoadStore(
const MachineInstr &
MI) {
138 case RISCV::VLSE16_V:
140 case RISCV::VSSE16_V:
143 case RISCV::VLSE32_V:
145 case RISCV::VSSE32_V:
148 case RISCV::VLSE64_V:
150 case RISCV::VSSE64_V:
156 return MI.getOpcode() == RISCV::ADDI &&
157 MI.getOperand(1).isReg() &&
MI.getOperand(2).isImm() &&
158 MI.getOperand(1).getReg() == RISCV::X0 &&
159 MI.getOperand(2).getImm() != 0;
167 const unsigned Log2SEW =
MI.getOperand(getSEWOpNum(
MI)).getImm();
180 if (!
MI.isRegTiedToUseOperand(0, &UseOpIdx))
189 if (UseMO.
getReg() == RISCV::NoRegister)
214struct DemandedFields {
219 bool VLZeroness =
false;
223 SEWGreaterThanOrEqual = 2,
225 SEWGreaterThanOrEqualAndLessThan64 =
232 bool SEWLMULRatio =
false;
233 bool TailPolicy =
false;
234 bool MaskPolicy =
false;
237 bool usedVTYPE()
const {
238 return SEW ||
LMUL || SEWLMULRatio || TailPolicy || MaskPolicy;
243 return VLAny || VLZeroness;
262 void doUnion(
const DemandedFields &
B) {
264 VLZeroness |=
B.VLZeroness;
265 SEW = std::max(SEW,
B.SEW);
267 SEWLMULRatio |=
B.SEWLMULRatio;
268 TailPolicy |=
B.TailPolicy;
269 MaskPolicy |=
B.MaskPolicy;
272#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
282 OS <<
"VLAny=" << VLAny <<
", ";
283 OS <<
"VLZeroness=" << VLZeroness <<
", ";
289 case SEWGreaterThanOrEqual:
290 OS <<
"SEWGreaterThanOrEqual";
292 case SEWGreaterThanOrEqualAndLessThan64:
293 OS <<
"SEWGreaterThanOrEqualAndLessThan64";
300 OS <<
"LMUL=" <<
LMUL <<
", ";
301 OS <<
"SEWLMULRatio=" << SEWLMULRatio <<
", ";
302 OS <<
"TailPolicy=" << TailPolicy <<
", ";
303 OS <<
"MaskPolicy=" << MaskPolicy;
309#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
321 const DemandedFields &Used) {
323 case DemandedFields::SEWNone:
325 case DemandedFields::SEWEqual:
329 case DemandedFields::SEWGreaterThanOrEqual:
333 case DemandedFields::SEWGreaterThanOrEqualAndLessThan64:
344 if (
Used.SEWLMULRatio) {
349 if (Ratio1 != Ratio2)
374 if (
MI.isCall() ||
MI.isInlineAsm() ||
375 MI.readsRegister(RISCV::VL,
nullptr))
377 if (
MI.isCall() ||
MI.isInlineAsm() ||
378 MI.readsRegister(RISCV::VTYPE,
nullptr))
389 Res.MaskPolicy =
false;
398 if (getEEWForLoadStore(
MI)) {
399 Res.SEW = DemandedFields::SEWNone;
405 Res.TailPolicy =
false;
406 Res.MaskPolicy =
false;
413 if (isMaskRegOp(
MI)) {
414 Res.SEW = DemandedFields::SEWNone;
419 if (isScalarInsertInstr(
MI)) {
421 Res.SEWLMULRatio =
false;
429 if (hasUndefinedMergeOp(
MI, *
MRI)) {
430 if (isFloatScalarMoveOrScalarSplatInstr(
MI) && !
ST->hasVInstructionsF64())
431 Res.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
433 Res.SEW = DemandedFields::SEWGreaterThanOrEqual;
434 Res.TailPolicy =
false;
439 if (isScalarExtractInstr(
MI)) {
442 Res.SEWLMULRatio =
false;
443 Res.TailPolicy =
false;
444 Res.MaskPolicy =
false;
474 uint8_t TailAgnostic : 1;
475 uint8_t MaskAgnostic : 1;
476 uint8_t SEWLMULRatioOnly : 1;
480 : AVLImm(0), TailAgnostic(
false), MaskAgnostic(
false),
481 SEWLMULRatioOnly(
false) {}
483 static VSETVLIInfo getUnknown() {
490 void setUnknown() { State =
Unknown; }
491 bool isUnknown()
const {
return State ==
Unknown; }
495 AVLRegDef.DefMI =
DefMI;
496 AVLRegDef.DefReg =
AVLReg;
500 void setAVLImm(
unsigned Imm) {
505 void setAVLVLMAX() { State = AVLIsVLMAX; }
507 void setAVLIgnored() { State = AVLIsIgnored; }
509 bool hasAVLImm()
const {
return State == AVLIsImm; }
510 bool hasAVLReg()
const {
return State == AVLIsReg; }
511 bool hasAVLVLMAX()
const {
return State == AVLIsVLMAX; }
512 bool hasAVLIgnored()
const {
return State == AVLIsIgnored; }
514 assert(hasAVLReg() && AVLRegDef.DefReg.isVirtual());
515 return AVLRegDef.DefReg;
517 unsigned getAVLImm()
const {
522 assert(hasAVLReg() && AVLRegDef.DefMI);
523 return *AVLRegDef.DefMI;
526 void setAVL(VSETVLIInfo Info) {
528 if (
Info.isUnknown())
530 else if (
Info.hasAVLReg())
532 else if (
Info.hasAVLVLMAX())
534 else if (
Info.hasAVLIgnored())
538 setAVLImm(
Info.getAVLImm());
542 unsigned getSEW()
const {
return SEW; }
544 bool getTailAgnostic()
const {
return TailAgnostic; }
545 bool getMaskAgnostic()
const {
return MaskAgnostic; }
547 bool hasNonZeroAVL()
const {
549 return getAVLImm() > 0;
551 return isNonZeroLoadImmediate(getAVLDefMI());
559 bool hasEquallyZeroAVL(
const VSETVLIInfo &
Other)
const {
560 if (hasSameAVL(
Other))
562 return (hasNonZeroAVL() &&
Other.hasNonZeroAVL());
565 bool hasSameAVL(
const VSETVLIInfo &
Other)
const {
566 if (hasAVLReg() &&
Other.hasAVLReg())
567 return AVLRegDef.DefMI ==
Other.AVLRegDef.DefMI &&
568 AVLRegDef.DefReg ==
Other.AVLRegDef.DefReg;
570 if (hasAVLImm() &&
Other.hasAVLImm())
571 return getAVLImm() ==
Other.getAVLImm();
574 return Other.hasAVLVLMAX() && hasSameVLMAX(
Other);
577 return Other.hasAVLIgnored();
584 "Can't set VTYPE for uninitialized or unknown");
592 "Can't set VTYPE for uninitialized or unknown");
603 "Can't encode VTYPE for uninitialized or unknown");
607 bool hasSEWLMULRatioOnly()
const {
return SEWLMULRatioOnly; }
609 bool hasSameVTYPE(
const VSETVLIInfo &
Other)
const {
611 "Can't compare invalid VSETVLIInfos");
613 "Can't compare VTYPE in unknown state");
614 assert(!SEWLMULRatioOnly && !
Other.SEWLMULRatioOnly &&
615 "Can't compare when only LMUL/SEW ratio is valid.");
616 return std::tie(VLMul, SEW, TailAgnostic, MaskAgnostic) ==
623 "Can't use VTYPE for uninitialized or unknown");
631 bool hasSameVLMAX(
const VSETVLIInfo &
Other)
const {
633 "Can't compare invalid VSETVLIInfos");
635 "Can't compare VTYPE in unknown state");
639 bool hasCompatibleVTYPE(
const DemandedFields &Used,
640 const VSETVLIInfo &Require)
const {
641 return areCompatibleVTYPEs(Require.encodeVTYPE(),
encodeVTYPE(), Used);
647 bool isCompatible(
const DemandedFields &Used,
const VSETVLIInfo &Require,
650 "Can't compare invalid VSETVLIInfos");
651 assert(!Require.SEWLMULRatioOnly &&
652 "Expected a valid VTYPE for instruction!");
654 if (isUnknown() || Require.isUnknown())
658 if (SEWLMULRatioOnly)
661 if (
Used.VLAny && !(hasSameAVL(Require) && hasSameVLMAX(Require)))
664 if (
Used.VLZeroness && !hasEquallyZeroAVL(Require))
667 return hasCompatibleVTYPE(Used, Require);
673 return !
Other.isValid();
674 if (!
Other.isValid())
679 return Other.isUnknown();
680 if (
Other.isUnknown())
683 if (!hasSameAVL(
Other))
687 if (SEWLMULRatioOnly !=
Other.SEWLMULRatioOnly)
691 if (SEWLMULRatioOnly)
692 return hasSameVLMAX(
Other);
695 return hasSameVTYPE(
Other);
699 return !(*
this ==
Other);
706 if (!
Other.isValid())
714 if (isUnknown() ||
Other.isUnknown())
715 return VSETVLIInfo::getUnknown();
723 if (hasSameAVL(
Other) && hasSameVLMAX(
Other)) {
724 VSETVLIInfo MergeInfo = *
this;
725 MergeInfo.SEWLMULRatioOnly =
true;
730 return VSETVLIInfo::getUnknown();
733#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
745 OS <<
"Uninitialized";
757 <<
"VLMul=" << (
unsigned)VLMul <<
", "
758 <<
"SEW=" << (
unsigned)SEW <<
", "
759 <<
"TailAgnostic=" << (
bool)TailAgnostic <<
", "
760 <<
"MaskAgnostic=" << (
bool)MaskAgnostic <<
", "
761 <<
"SEWLMULRatioOnly=" << (
bool)SEWLMULRatioOnly <<
"}";
766#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
784 bool InQueue =
false;
794 std::vector<BlockData> BlockInfo;
795 std::queue<const MachineBasicBlock *> WorkList;
811 bool needVSETVLI(
const MachineInstr &
MI,
const VSETVLIInfo &Require,
812 const VSETVLIInfo &CurInfo)
const;
813 bool needVSETVLIPHI(
const VSETVLIInfo &Require,
816 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo);
819 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo);
821 void transferBefore(VSETVLIInfo &Info,
const MachineInstr &
MI)
const;
822 void transferAfter(VSETVLIInfo &Info,
const MachineInstr &
MI)
const;
824 VSETVLIInfo &Info)
const;
863char RISCVInsertVSETVLI::ID = 0;
868char RISCVCoalesceVSETVLI::
ID = 0;
878 if (
MI.getOpcode() == RISCV::PseudoVSETIVLI) {
879 NewInfo.setAVLImm(
MI.getOperand(1).getImm());
881 assert(
MI.getOpcode() == RISCV::PseudoVSETVLI ||
882 MI.getOpcode() == RISCV::PseudoVSETVLIX0);
885 "Can't handle X0, X0 vsetvli yet");
909 VSETVLIInfo InstrInfo;
911 bool TailAgnostic =
true;
912 bool MaskAgnostic =
true;
913 if (!hasUndefinedMergeOp(
MI, *
MRI)) {
915 TailAgnostic =
false;
916 MaskAgnostic =
false;
923 "Invalid Policy Value");
939 unsigned Log2SEW =
MI.getOperand(getSEWOpNum(
MI)).getImm();
941 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
947 int64_t Imm = VLOp.
getImm();
952 const unsigned VLMAX =
computeVLMAX(ST.getRealMaxVLen(), SEW, VLMul);
953 if (ST.getRealMinVLen() == ST.getRealMaxVLen() && VLMAX <= 31)
954 InstrInfo.setAVLImm(VLMAX);
956 InstrInfo.setAVLVLMAX();
959 InstrInfo.setAVLImm(Imm);
968 InstrInfo.setAVLIgnored();
971 if (std::optional<unsigned> EEW = getEEWForLoadStore(
MI)) {
972 assert(SEW == EEW &&
"Initial SEW doesn't match expected EEW");
975 InstrInfo.setVTYPE(VLMul, SEW, TailAgnostic, MaskAgnostic);
981 if (InstrInfo.hasAVLReg()) {
983 if (isVectorConfigInstr(
DefMI)) {
984 VSETVLIInfo DefInstrInfo = getInfoForVSETVLI(
DefMI, *
MRI);
985 if (DefInstrInfo.hasSameVLMAX(InstrInfo) &&
986 (DefInstrInfo.hasAVLImm() || DefInstrInfo.hasAVLVLMAX()))
987 InstrInfo.setAVL(DefInstrInfo);
995 const VSETVLIInfo &Info,
996 const VSETVLIInfo &PrevInfo) {
1003 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo) {
1005 ++NumInsertedVSETVL;
1006 if (PrevInfo.isValid() && !PrevInfo.isUnknown()) {
1009 if (
Info.hasSameAVL(PrevInfo) &&
Info.hasSameVLMAX(PrevInfo)) {
1021 if (
Info.hasSameVLMAX(PrevInfo) &&
Info.hasAVLReg()) {
1023 if (isVectorConfigInstr(
DefMI)) {
1024 VSETVLIInfo DefInfo = getInfoForVSETVLI(
DefMI, *
MRI);
1025 if (DefInfo.hasSameAVL(PrevInfo) && DefInfo.hasSameVLMAX(PrevInfo)) {
1037 if (
Info.hasAVLImm()) {
1045 if (
Info.hasAVLIgnored()) {
1048 if (PrevInfo.isValid() && !PrevInfo.isUnknown() &&
1049 Info.hasSameVLMAX(PrevInfo)) {
1065 if (
Info.hasAVLVLMAX()) {
1066 Register DestReg =
MRI->createVirtualRegister(&RISCV::GPRRegClass);
1075 MRI->constrainRegClass(
AVLReg, &RISCV::GPRNoX0RegClass);
1084 return Fractional || LMul == 1;
1090 const VSETVLIInfo &Require,
1091 const VSETVLIInfo &CurInfo)
const {
1094 if (!CurInfo.isValid() || CurInfo.isUnknown() || CurInfo.hasSEWLMULRatioOnly())
1097 DemandedFields
Used = getDemanded(
MI,
MRI, ST);
1107 if (isVSlideInstr(
MI) && Require.hasAVLImm() && Require.getAVLImm() == 1 &&
1110 Used.VLZeroness =
true;
1112 Used.TailPolicy =
false;
1120 if (isScalarSplatInstr(
MI) && Require.hasAVLImm() && Require.getAVLImm() == 1 &&
1123 Used.SEWLMULRatio =
false;
1125 if (isFloatScalarMoveOrScalarSplatInstr(
MI) && !
ST->hasVInstructionsF64())
1126 Used.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
1128 Used.SEW = DemandedFields::SEWGreaterThanOrEqual;
1129 Used.TailPolicy =
false;
1132 if (CurInfo.isCompatible(Used, Require, *
MRI))
1139 if (Require.hasAVLReg() && CurInfo.hasCompatibleVTYPE(Used, Require)) {
1141 if (isVectorConfigInstr(
DefMI)) {
1142 VSETVLIInfo DefInfo = getInfoForVSETVLI(
DefMI, *
MRI);
1143 if (DefInfo.hasSameAVL(CurInfo) && DefInfo.hasSameVLMAX(CurInfo))
1155 DemandedFields &Demanded) {
1158 if (!Demanded.LMUL && !Demanded.SEWLMULRatio && PrevInfo.isValid() &&
1159 !PrevInfo.isUnknown()) {
1161 PrevInfo.getSEW(), PrevInfo.getVLMUL(),
Info.getSEW()))
1162 Info.setVLMul(*NewVLMul);
1163 Demanded.LMUL =
true;
1172void RISCVInsertVSETVLI::transferBefore(VSETVLIInfo &Info,
1183 const VSETVLIInfo PrevInfo =
Info;
1184 if (!
Info.isValid() ||
Info.isUnknown())
1187 DemandedFields Demanded = getDemanded(
MI,
MRI, ST);
1197 bool EquallyZero = IncomingInfo.hasEquallyZeroAVL(PrevInfo) &&
1198 IncomingInfo.hasSameVLMAX(PrevInfo);
1199 if (Demanded.VLAny || (Demanded.VLZeroness && !EquallyZero))
1200 Info.setAVL(IncomingInfo);
1203 ((Demanded.LMUL || Demanded.SEWLMULRatio) ? IncomingInfo : Info)
1205 ((Demanded.SEW || Demanded.SEWLMULRatio) ? IncomingInfo : Info).getSEW(),
1208 (Demanded.TailPolicy ? IncomingInfo : Info).getTailAgnostic() ||
1209 IncomingInfo.getTailAgnostic(),
1210 (Demanded.MaskPolicy ? IncomingInfo : Info).getMaskAgnostic() ||
1211 IncomingInfo.getMaskAgnostic());
1215 if (
Info.hasSEWLMULRatioOnly()) {
1216 VSETVLIInfo RatiolessInfo = IncomingInfo;
1217 RatiolessInfo.setAVL(Info);
1218 Info = RatiolessInfo;
1225void RISCVInsertVSETVLI::transferAfter(VSETVLIInfo &Info,
1227 if (isVectorConfigInstr(
MI)) {
1234 Info.setAVLRegDef(
MRI->getVRegDef(
MI.getOperand(1).getReg()),
1235 MI.getOperand(1).getReg());
1241 if (
MI.isCall() ||
MI.isInlineAsm() ||
1242 MI.modifiesRegister(RISCV::VL,
nullptr) ||
1243 MI.modifiesRegister(RISCV::VTYPE,
nullptr))
1244 Info = VSETVLIInfo::getUnknown();
1248 VSETVLIInfo &Info)
const {
1249 bool HadVectorOp =
false;
1253 transferBefore(Info,
MI);
1258 transferAfter(Info,
MI);
1268 BBInfo.InQueue =
false;
1272 VSETVLIInfo InInfo = BBInfo.
Pred;
1275 InInfo.setUnknown();
1278 InInfo = InInfo.
intersect(BlockInfo[
P->getNumber()].Exit);
1282 if (!InInfo.isValid())
1286 if (InInfo == BBInfo.
Pred)
1289 BBInfo.
Pred = InInfo;
1291 <<
" changed to " << BBInfo.
Pred <<
"\n");
1297 VSETVLIInfo TmpStatus;
1298 computeVLVTYPEChanges(
MBB, TmpStatus);
1302 if (BBInfo.
Exit == TmpStatus)
1305 BBInfo.
Exit = TmpStatus;
1307 <<
" changed to " << BBInfo.
Exit <<
"\n");
1312 if (!BlockInfo[S->getNumber()].InQueue) {
1313 BlockInfo[S->getNumber()].InQueue =
true;
1321bool RISCVInsertVSETVLI::needVSETVLIPHI(
const VSETVLIInfo &Require,
1326 if (!Require.hasAVLReg())
1331 if (
PHI->getOpcode() != RISCV::PHI ||
PHI->getParent() != &
MBB)
1334 for (
unsigned PHIOp = 1, NumOps =
PHI->getNumOperands(); PHIOp != NumOps;
1338 const VSETVLIInfo &PBBExit = BlockInfo[PBB->
getNumber()].Exit;
1347 VSETVLIInfo DefInfo = getInfoForVSETVLI(*
DefMI, *
MRI);
1348 if (DefInfo != PBBExit)
1354 if (PBBExit.isUnknown() || !PBBExit.hasSameVTYPE(Require))
1367 bool PrefixTransparent =
true;
1369 const VSETVLIInfo PrevInfo = CurInfo;
1370 transferBefore(CurInfo,
MI);
1373 if (isVectorConfigInstr(
MI)) {
1375 assert(
MI.getOperand(3).getReg() == RISCV::VL &&
1376 MI.getOperand(4).getReg() == RISCV::VTYPE &&
1377 "Unexpected operands where VL and VTYPE should be");
1378 MI.getOperand(3).setIsDead(
false);
1379 MI.getOperand(4).setIsDead(
false);
1380 PrefixTransparent =
false;
1385 if (PrevInfo != CurInfo) {
1393 if (!PrefixTransparent || needVSETVLIPHI(CurInfo,
MBB))
1394 insertVSETVLI(
MBB,
MI, CurInfo, PrevInfo);
1395 PrefixTransparent =
false;
1405 VLOp.
setReg(RISCV::NoRegister);
1412 if (VLOpDef &&
TII->isAddImmediate(*VLOpDef, Reg) &&
1413 MRI->use_nodbg_empty(Reg))
1423 if (
MI.isCall() ||
MI.isInlineAsm() ||
1424 MI.modifiesRegister(RISCV::VL,
nullptr) ||
1425 MI.modifiesRegister(RISCV::VTYPE,
nullptr))
1426 PrefixTransparent =
false;
1428 transferAfter(CurInfo,
MI);
1432 if (CurInfo !=
Info.Exit) {
1438 assert(CurInfo ==
Info.Exit &&
"InsertVSETVLI dataflow invariant violated");
1451 VSETVLIInfo AvailableInfo;
1453 const VSETVLIInfo &PredInfo = BlockInfo[
P->getNumber()].Exit;
1454 if (PredInfo.isUnknown()) {
1455 if (UnavailablePred)
1457 UnavailablePred =
P;
1458 }
else if (!AvailableInfo.isValid()) {
1459 AvailableInfo = PredInfo;
1460 }
else if (AvailableInfo != PredInfo) {
1467 if (!UnavailablePred || !AvailableInfo.isValid())
1472 if (AvailableInfo.hasSEWLMULRatioOnly())
1482 if (AvailableInfo.hasAVLReg()) {
1483 const MachineInstr *AVLDefMI = &AvailableInfo.getAVLDefMI();
1486 if (AVLDefMI->
getParent() != UnavailablePred)
1488 for (
auto &TermMI : UnavailablePred->
terminators())
1489 if (&TermMI == AVLDefMI)
1495 if (AvailableInfo.hasAVLIgnored())
1507 VSETVLIInfo CurInfo = AvailableInfo;
1508 int TransitionsRemoved = 0;
1510 const VSETVLIInfo LastInfo = CurInfo;
1511 const VSETVLIInfo LastOldInfo = OldInfo;
1512 transferBefore(CurInfo,
MI);
1513 transferBefore(OldInfo,
MI);
1514 if (CurInfo == LastInfo)
1515 TransitionsRemoved++;
1516 if (LastOldInfo == OldInfo)
1517 TransitionsRemoved--;
1518 transferAfter(CurInfo,
MI);
1519 transferAfter(OldInfo,
MI);
1520 if (CurInfo == OldInfo)
1524 if (CurInfo != OldInfo || TransitionsRemoved <= 0)
1531 auto OldExit = BlockInfo[UnavailablePred->
getNumber()].Exit;
1533 << UnavailablePred->
getName() <<
" with state "
1534 << AvailableInfo <<
"\n");
1535 BlockInfo[UnavailablePred->
getNumber()].Exit = AvailableInfo;
1541 insertVSETVLI(*UnavailablePred, InsertPt,
1543 AvailableInfo, OldExit);
1550 const DemandedFields &Used,
1555 if (!isVLPreservingConfig(
MI)) {
1559 if (Used.VLZeroness) {
1560 if (isVLPreservingConfig(PrevMI))
1562 if (!getInfoForVSETVLI(PrevMI,
MRI)
1563 .hasEquallyZeroAVL(getInfoForVSETVLI(
MI,
MRI)))
1567 auto &AVL =
MI.getOperand(1);
1572 if (AVL.isReg() && AVL.getReg() != RISCV::X0 &&
1573 (!
MRI.hasOneDef(AVL.getReg()) || !PrevAVL.isReg() ||
1574 PrevAVL.getReg() != AVL.getReg()))
1580 auto VType =
MI.getOperand(2).getImm();
1581 return areCompatibleVTYPEs(PriorVType, VType, Used);
1588 DemandedFields
Used;
1594 if (!isVectorConfigInstr(
MI)) {
1596 if (
MI.isCall() ||
MI.isInlineAsm() ||
1597 MI.modifiesRegister(RISCV::VL,
nullptr) ||
1598 MI.modifiesRegister(RISCV::VTYPE,
nullptr))
1605 if (RegDef != RISCV::X0 && !
MRI->use_nodbg_empty(RegDef))
1609 if (!
Used.usedVL() && !
Used.usedVTYPE()) {
1616 if (!isVLPreservingConfig(*NextMI)) {
1619 MI.getOperand(0).setReg(DefReg);
1620 MI.getOperand(0).setIsDead(
false);
1630 DefVNI->
def = MISlot;
1636 LIS->shrinkToUses(&DefLI);
1640 if (
MI.getOperand(1).isReg())
1641 OldVLReg =
MI.getOperand(1).getReg();
1653 LIS->shrinkToUses(&LIS->getInterval(OldVLReg));
1656 if (VLOpDef &&
TII->isAddImmediate(*VLOpDef, OldVLReg) &&
1657 MRI->use_nodbg_empty(OldVLReg)) {
1659 LIS->removeInterval(OldVLReg);
1673 NumCoalescedVSETVL += ToDelete.
size();
1674 for (
auto *
MI : ToDelete) {
1675 LIS->RemoveMachineInstrFromMaps(*
MI);
1676 MI->eraseFromParent();
1679 return !ToDelete.empty();
1686 Register VLOutput =
MI.getOperand(1).getReg();
1687 if (!
MRI->use_nodbg_empty(VLOutput))
1691 MI.getOperand(1).setReg(RISCV::X0);
1699 if (!
ST->hasVInstructions())
1704 TII =
ST->getInstrInfo();
1707 assert(BlockInfo.empty() &&
"Expect empty block infos");
1710 bool HaveVectorOp =
false;
1714 VSETVLIInfo TmpStatus;
1715 HaveVectorOp |= computeVLVTYPEChanges(
MBB, TmpStatus);
1718 BBInfo.
Exit = TmpStatus;
1720 <<
" is " << BBInfo.
Exit <<
"\n");
1725 if (!HaveVectorOp) {
1734 WorkList.push(&
MBB);
1737 while (!WorkList.empty()) {
1740 computeIncomingVLVTYPE(
MBB);
1760 return HaveVectorOp;
1765 return new RISCVInsertVSETVLI();
1777 if (!ST->hasVInstructions())
1779 TII = ST->getInstrInfo();
1781 LIS = &getAnalysis<LiveIntervals>();
1783 bool Changed =
false;
1785 Changed |= coalesceVSETVLIs(
MBB);
1791 return new RISCVCoalesceVSETVLI();
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Analysis containing CSE Info
#define LLVM_ATTRIBUTE_USED
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
static RegisterPass< DebugifyFunctionPass > DF("debugify-function", "Attach debug info to a function")
std::optional< std::vector< StOtherPiece > > Other
const HexagonInstrInfo * TII
static ValueLatticeElement intersect(const ValueLatticeElement &A, const ValueLatticeElement &B)
Combine two sets of facts about the same value into a single set of facts.
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
NewInfo setVTYPE(MI.getOperand(2).getImm())
#define RISCV_COALESCE_VSETVLI_NAME
assert((AVLReg !=RISCV::X0||MI.getOperand(0).getReg() !=RISCV::X0) &&"Can't handle X0, X0 vsetvli yet")
static unsigned computeVLMAX(unsigned VLEN, unsigned SEW, RISCVII::VLMUL VLMul)
static VSETVLIInfo computeInfoForInstr(const MachineInstr &MI, uint64_t TSFlags, const RISCVSubtarget &ST, const MachineRegisterInfo *MRI)
#define RISCV_INSERT_VSETVLI_NAME
else NewInfo setAVLRegDef(MRI.getVRegDef(AVLReg), AVLReg)
static VSETVLIInfo adjustIncoming(VSETVLIInfo PrevInfo, VSETVLIInfo NewInfo, DemandedFields &Demanded)
static bool isLMUL1OrSmaller(RISCVII::VLMUL LMUL)
static bool canMutatePriorConfig(const MachineInstr &PrevMI, const MachineInstr &MI, const DemandedFields &Used, const MachineRegisterInfo &MRI)
static cl::opt< bool > DisableInsertVSETVLPHIOpt("riscv-disable-insert-vsetvl-phi-opt", cl::init(false), cl::Hidden, cl::desc("Disable looking through phis when inserting vsetvlis."))
INITIALIZE_PASS(RISCVInsertVSETVLI, DEBUG_TYPE, RISCV_INSERT_VSETVLI_NAME, false, false) char RISCVCoalesceVSETVLI const MachineRegisterInfo & MRI
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
This class represents an Operation in the Expression.
FunctionPass class - This class is used to implement most global optimizations.
LiveInterval - This class represents the liveness of a register, or stack slot.
void setWeight(float Value)
iterator addSegment(Segment S)
Add the specified Segment to this range, merging segments as appropriate.
SlotIndex beginIndex() const
beginIndex - Return the lowest numbered slot covered.
VNInfo * getVNInfoAt(SlotIndex Idx) const
getVNInfoAt - Return the VNInfo that is live at Idx, or NULL.
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
unsigned succ_size() const
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
iterator_range< iterator > terminators()
iterator_range< succ_iterator > successors()
instr_iterator getFirstInstrTerminator()
Same getFirstTerminator but it ignores bundles and return an instr_iterator instead.
reverse_iterator rbegin()
iterator_range< pred_iterator > predecessors()
StringRef getName() const
Return the name of the corresponding LLVM basic block, or an empty string.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
unsigned getNumBlockIDs() const
getNumBlockIDs - Return the number of MBB ID's allocated.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
bool isImplicitDef() const
const MachineBasicBlock * getParent() const
unsigned getNumOperands() const
Retuns the total number of operands.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
bool isRegSequence() const
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
SlotIndex - An opaque wrapper around machine indexes.
SlotIndex getRegSlot(bool EC=false) const
Returns the register use/def slot in the current instruction for a normal or early-clobber def.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
TargetInstrInfo - Interface to description of machine instruction set.
VNInfo - Value Number Information.
SlotIndex def
The index of the defining instruction.
This class implements an extremely fast bulk output stream that can only output to a stream.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
static bool usesMaskPolicy(uint64_t TSFlags)
static unsigned getVLOpNum(const MCInstrDesc &Desc)
static bool doesForceTailAgnostic(uint64_t TSFlags)
static VLMUL getLMul(uint64_t TSFlags)
static bool hasVLOp(uint64_t TSFlags)
static bool hasVecPolicyOp(uint64_t TSFlags)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
static bool hasSEWOp(uint64_t TSFlags)
static bool isTailAgnostic(unsigned VType)
static RISCVII::VLMUL getVLMUL(unsigned VType)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
unsigned getSEWLMULRatio(unsigned SEW, RISCVII::VLMUL VLMul)
static bool isMaskAgnostic(unsigned VType)
static bool isValidSEW(unsigned SEW)
unsigned encodeVTYPE(RISCVII::VLMUL VLMUL, unsigned SEW, bool TailAgnostic, bool MaskAgnostic)
static unsigned getSEW(unsigned VType)
std::optional< RISCVII::VLMUL > getSameRatioLMUL(unsigned SEW, RISCVII::VLMUL VLMUL, unsigned EEW)
unsigned getRVVMCOpcode(unsigned RVVPseudoOpcode)
bool isFaultFirstLoad(const MachineInstr &MI)
static constexpr int64_t VLMaxSentinel
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool operator!=(uint64_t V1, const APInt &V2)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
FunctionPass * createRISCVInsertVSETVLIPass()
Returns an instance of the Insert VSETVLI pass.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
FunctionPass * createRISCVCoalesceVSETVLIPass()
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Status intersect(const Status &S) const
This represents a simple continuous liveness interval for a value.