37#define DEBUG_TYPE "riscv-insert-vsetvli"
38#define RISCV_INSERT_VSETVLI_NAME "RISC-V Insert VSETVLI pass"
39#define RISCV_COALESCE_VSETVLI_NAME "RISC-V Coalesce VSETVLI pass"
41STATISTIC(NumInsertedVSETVL,
"Number of VSETVL inst inserted");
42STATISTIC(NumCoalescedVSETVL,
"Number of VSETVL inst coalesced");
46 cl::desc(
"Disable looking through phis when inserting vsetvlis."));
55 auto &LI =
LIS->getInterval(Reg);
57 return LI.getVNInfoBefore(SI);
69 return MI.getOpcode() == RISCV::PseudoVSETVLI ||
70 MI.getOpcode() == RISCV::PseudoVSETVLIX0 ||
71 MI.getOpcode() == RISCV::PseudoVSETIVLI;
77 if (
MI.getOpcode() != RISCV::PseudoVSETVLIX0)
79 assert(RISCV::X0 ==
MI.getOperand(1).getReg());
80 return RISCV::X0 ==
MI.getOperand(0).getReg();
83static bool isFloatScalarMoveOrScalarSplatInstr(
const MachineInstr &
MI) {
108 case RISCV::VFMV_S_F:
119 case RISCV::VFMV_V_F:
128 case RISCV::VSLIDEDOWN_VX:
129 case RISCV::VSLIDEDOWN_VI:
130 case RISCV::VSLIDEUP_VX:
131 case RISCV::VSLIDEUP_VI:
138static std::optional<unsigned> getEEWForLoadStore(
const MachineInstr &
MI) {
148 case RISCV::VLSE16_V:
150 case RISCV::VSSE16_V:
153 case RISCV::VLSE32_V:
155 case RISCV::VSSE32_V:
158 case RISCV::VLSE64_V:
160 case RISCV::VSSE64_V:
166 return MI.getOpcode() == RISCV::ADDI &&
167 MI.getOperand(1).isReg() &&
MI.getOperand(2).isImm() &&
168 MI.getOperand(1).getReg() == RISCV::X0 &&
169 MI.getOperand(2).getImm() != 0;
177 const unsigned Log2SEW =
MI.getOperand(getSEWOpNum(
MI)).getImm();
189 if (!
MI.isRegTiedToUseOperand(0, &UseOpIdx))
197 return UseMO.
getReg() == RISCV::NoRegister || UseMO.
isUndef();
201struct DemandedFields {
206 bool VLZeroness =
false;
210 SEWGreaterThanOrEqual = 2,
212 SEWGreaterThanOrEqualAndLessThan64 =
219 bool SEWLMULRatio =
false;
220 bool TailPolicy =
false;
221 bool MaskPolicy =
false;
224 bool usedVTYPE()
const {
225 return SEW ||
LMUL || SEWLMULRatio || TailPolicy || MaskPolicy;
230 return VLAny || VLZeroness;
249 void doUnion(
const DemandedFields &
B) {
251 VLZeroness |=
B.VLZeroness;
252 SEW = std::max(SEW,
B.SEW);
254 SEWLMULRatio |=
B.SEWLMULRatio;
255 TailPolicy |=
B.TailPolicy;
256 MaskPolicy |=
B.MaskPolicy;
259#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
269 OS <<
"VLAny=" << VLAny <<
", ";
270 OS <<
"VLZeroness=" << VLZeroness <<
", ";
276 case SEWGreaterThanOrEqual:
277 OS <<
"SEWGreaterThanOrEqual";
279 case SEWGreaterThanOrEqualAndLessThan64:
280 OS <<
"SEWGreaterThanOrEqualAndLessThan64";
287 OS <<
"LMUL=" <<
LMUL <<
", ";
288 OS <<
"SEWLMULRatio=" << SEWLMULRatio <<
", ";
289 OS <<
"TailPolicy=" << TailPolicy <<
", ";
290 OS <<
"MaskPolicy=" << MaskPolicy;
296#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
308 const DemandedFields &Used) {
310 case DemandedFields::SEWNone:
312 case DemandedFields::SEWEqual:
316 case DemandedFields::SEWGreaterThanOrEqual:
320 case DemandedFields::SEWGreaterThanOrEqualAndLessThan64:
331 if (
Used.SEWLMULRatio) {
336 if (Ratio1 != Ratio2)
359 if (
MI.isCall() ||
MI.isInlineAsm() ||
360 MI.readsRegister(RISCV::VL,
nullptr))
362 if (
MI.isCall() ||
MI.isInlineAsm() ||
363 MI.readsRegister(RISCV::VTYPE,
nullptr))
374 Res.MaskPolicy =
false;
383 if (getEEWForLoadStore(
MI)) {
384 Res.SEW = DemandedFields::SEWNone;
390 Res.TailPolicy =
false;
391 Res.MaskPolicy =
false;
398 if (isMaskRegOp(
MI)) {
399 Res.SEW = DemandedFields::SEWNone;
404 if (isScalarInsertInstr(
MI)) {
406 Res.SEWLMULRatio =
false;
414 if (hasUndefinedMergeOp(
MI)) {
415 if (isFloatScalarMoveOrScalarSplatInstr(
MI) && !
ST->hasVInstructionsF64())
416 Res.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
418 Res.SEW = DemandedFields::SEWGreaterThanOrEqual;
419 Res.TailPolicy =
false;
424 if (isScalarExtractInstr(
MI)) {
427 Res.SEWLMULRatio =
false;
428 Res.TailPolicy =
false;
429 Res.MaskPolicy =
false;
460 uint8_t TailAgnostic : 1;
461 uint8_t MaskAgnostic : 1;
462 uint8_t SEWLMULRatioOnly : 1;
466 : AVLImm(0), TailAgnostic(
false), MaskAgnostic(
false),
467 SEWLMULRatioOnly(
false) {}
469 static VSETVLIInfo getUnknown() {
476 void setUnknown() { State =
Unknown; }
477 bool isUnknown()
const {
return State ==
Unknown; }
482 AVLRegDef.DefReg =
AVLReg;
486 void setAVLImm(
unsigned Imm) {
491 void setAVLVLMAX() { State = AVLIsVLMAX; }
495 bool hasAVLImm()
const {
return State == AVLIsImm; }
496 bool hasAVLReg()
const {
return State == AVLIsReg; }
497 bool hasAVLVLMAX()
const {
return State == AVLIsVLMAX; }
498 bool hasAVLIgnored()
const {
return State == AVLIsIgnored; }
500 assert(hasAVLReg() && AVLRegDef.DefReg.isVirtual());
501 return AVLRegDef.DefReg;
503 unsigned getAVLImm()
const {
507 const VNInfo *getAVLVNInfo()
const {
509 return AVLRegDef.ValNo;
516 auto *
MI =
LIS->getInstructionFromIndex(getAVLVNInfo()->def);
517 assert(!(getAVLVNInfo()->isPHIDef() &&
MI));
521 void setAVL(VSETVLIInfo Info) {
523 if (
Info.isUnknown())
525 else if (
Info.hasAVLReg())
526 setAVLRegDef(
Info.getAVLVNInfo(),
Info.getAVLReg());
527 else if (
Info.hasAVLVLMAX())
529 else if (
Info.hasAVLIgnored())
533 setAVLImm(
Info.getAVLImm());
537 unsigned getSEW()
const {
return SEW; }
539 bool getTailAgnostic()
const {
return TailAgnostic; }
540 bool getMaskAgnostic()
const {
return MaskAgnostic; }
544 return getAVLImm() > 0;
547 return isNonZeroLoadImmediate(*
DefMI);
556 bool hasEquallyZeroAVL(
const VSETVLIInfo &
Other,
558 if (hasSameAVL(
Other))
560 return (hasNonZeroAVL(
LIS) &&
Other.hasNonZeroAVL(
LIS));
563 bool hasSameAVL(
const VSETVLIInfo &
Other)
const {
564 if (hasAVLReg() &&
Other.hasAVLReg())
565 return getAVLVNInfo()->id ==
Other.getAVLVNInfo()->id &&
566 getAVLReg() ==
Other.getAVLReg();
568 if (hasAVLImm() &&
Other.hasAVLImm())
569 return getAVLImm() ==
Other.getAVLImm();
572 return Other.hasAVLVLMAX() && hasSameVLMAX(
Other);
575 return Other.hasAVLIgnored();
582 "Can't set VTYPE for uninitialized or unknown");
590 "Can't set VTYPE for uninitialized or unknown");
601 "Can't encode VTYPE for uninitialized or unknown");
605 bool hasSEWLMULRatioOnly()
const {
return SEWLMULRatioOnly; }
607 bool hasSameVTYPE(
const VSETVLIInfo &
Other)
const {
609 "Can't compare invalid VSETVLIInfos");
611 "Can't compare VTYPE in unknown state");
612 assert(!SEWLMULRatioOnly && !
Other.SEWLMULRatioOnly &&
613 "Can't compare when only LMUL/SEW ratio is valid.");
614 return std::tie(VLMul, SEW, TailAgnostic, MaskAgnostic) ==
621 "Can't use VTYPE for uninitialized or unknown");
629 bool hasSameVLMAX(
const VSETVLIInfo &
Other)
const {
631 "Can't compare invalid VSETVLIInfos");
633 "Can't compare VTYPE in unknown state");
637 bool hasCompatibleVTYPE(
const DemandedFields &Used,
638 const VSETVLIInfo &Require)
const {
639 return areCompatibleVTYPEs(Require.encodeVTYPE(),
encodeVTYPE(), Used);
645 bool isCompatible(
const DemandedFields &Used,
const VSETVLIInfo &Require,
648 "Can't compare invalid VSETVLIInfos");
649 assert(!Require.SEWLMULRatioOnly &&
650 "Expected a valid VTYPE for instruction!");
652 if (isUnknown() || Require.isUnknown())
656 if (SEWLMULRatioOnly)
659 if (
Used.VLAny && !(hasSameAVL(Require) && hasSameVLMAX(Require)))
662 if (
Used.VLZeroness && !hasEquallyZeroAVL(Require,
LIS))
665 return hasCompatibleVTYPE(Used, Require);
671 return !
Other.isValid();
672 if (!
Other.isValid())
677 return Other.isUnknown();
678 if (
Other.isUnknown())
681 if (!hasSameAVL(
Other))
685 if (SEWLMULRatioOnly !=
Other.SEWLMULRatioOnly)
689 if (SEWLMULRatioOnly)
690 return hasSameVLMAX(
Other);
693 return hasSameVTYPE(
Other);
697 return !(*
this ==
Other);
704 if (!
Other.isValid())
712 if (isUnknown() ||
Other.isUnknown())
713 return VSETVLIInfo::getUnknown();
721 if (hasSameAVL(
Other) && hasSameVLMAX(
Other)) {
722 VSETVLIInfo MergeInfo = *
this;
723 MergeInfo.SEWLMULRatioOnly =
true;
728 return VSETVLIInfo::getUnknown();
731#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
743 OS <<
"Uninitialized";
755 <<
"VLMul=" << (
unsigned)VLMul <<
", "
756 <<
"SEW=" << (
unsigned)SEW <<
", "
757 <<
"TailAgnostic=" << (
bool)TailAgnostic <<
", "
758 <<
"MaskAgnostic=" << (
bool)MaskAgnostic <<
", "
759 <<
"SEWLMULRatioOnly=" << (
bool)SEWLMULRatioOnly <<
"}";
764#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
782 bool InQueue =
false;
793 std::vector<BlockData> BlockInfo;
794 std::queue<const MachineBasicBlock *> WorkList;
818 bool needVSETVLI(
const MachineInstr &
MI,
const VSETVLIInfo &Require,
819 const VSETVLIInfo &CurInfo)
const;
820 bool needVSETVLIPHI(
const VSETVLIInfo &Require,
823 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo);
826 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo);
828 void transferBefore(VSETVLIInfo &Info,
const MachineInstr &
MI)
const;
829 void transferAfter(VSETVLIInfo &Info,
const MachineInstr &
MI)
const;
831 VSETVLIInfo &Info)
const;
870char RISCVInsertVSETVLI::ID = 0;
876char RISCVCoalesceVSETVLI::
ID = 0;
886 if (
MI.getOpcode() == RISCV::PseudoVSETIVLI) {
887 NewInfo.setAVLImm(
MI.getOperand(1).getImm());
889 assert(
MI.getOpcode() == RISCV::PseudoVSETVLI ||
890 MI.getOpcode() == RISCV::PseudoVSETVLIX0);
893 "Can't handle X0, X0 vsetvli yet");
921 VSETVLIInfo InstrInfo;
923 bool TailAgnostic =
true;
924 bool MaskAgnostic =
true;
925 if (!hasUndefinedMergeOp(
MI)) {
927 TailAgnostic =
false;
928 MaskAgnostic =
false;
935 "Invalid Policy Value");
951 unsigned Log2SEW =
MI.getOperand(getSEWOpNum(
MI)).getImm();
953 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
959 int64_t Imm = VLOp.
getImm();
964 const unsigned VLMAX =
computeVLMAX(ST.getRealMaxVLen(), SEW, VLMul);
965 if (ST.getRealMinVLen() == ST.getRealMaxVLen() && VLMAX <= 31)
966 InstrInfo.setAVLImm(VLMAX);
968 InstrInfo.setAVLVLMAX();
971 InstrInfo.setAVLImm(Imm);
973 InstrInfo.setAVLRegDef(VNI, VLOp.
getReg());
976 InstrInfo.setAVLIgnored();
983 InstrInfo.setAVLIgnored();
986 if (std::optional<unsigned> EEW = getEEWForLoadStore(
MI)) {
987 assert(SEW == EEW &&
"Initial SEW doesn't match expected EEW");
990 InstrInfo.setVTYPE(VLMul, SEW, TailAgnostic, MaskAgnostic);
996 if (InstrInfo.hasAVLReg()) {
999 VSETVLIInfo DefInstrInfo = getInfoForVSETVLI(*
DefMI,
LIS);
1000 if (DefInstrInfo.hasSameVLMAX(InstrInfo) &&
1001 (DefInstrInfo.hasAVLImm() || DefInstrInfo.hasAVLVLMAX()))
1002 InstrInfo.setAVL(DefInstrInfo);
1010 const VSETVLIInfo &Info,
1011 const VSETVLIInfo &PrevInfo) {
1018 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo) {
1020 ++NumInsertedVSETVL;
1021 if (PrevInfo.isValid() && !PrevInfo.isUnknown()) {
1024 if (
Info.hasSameAVL(PrevInfo) &&
Info.hasSameVLMAX(PrevInfo)) {
1030 LIS->InsertMachineInstrInMaps(*
MI);
1037 if (
Info.hasSameVLMAX(PrevInfo) &&
Info.hasAVLReg()) {
1040 VSETVLIInfo DefInfo = getInfoForVSETVLI(*
DefMI,
LIS);
1041 if (DefInfo.hasSameAVL(PrevInfo) && DefInfo.hasSameVLMAX(PrevInfo)) {
1047 LIS->InsertMachineInstrInMaps(*
MI);
1054 if (
Info.hasAVLImm()) {
1059 LIS->InsertMachineInstrInMaps(*
MI);
1063 if (
Info.hasAVLIgnored()) {
1066 if (PrevInfo.isValid() && !PrevInfo.isUnknown() &&
1067 Info.hasSameVLMAX(PrevInfo)) {
1073 LIS->InsertMachineInstrInMaps(*
MI);
1081 LIS->InsertMachineInstrInMaps(*
MI);
1085 if (
Info.hasAVLVLMAX()) {
1086 Register DestReg =
MRI->createVirtualRegister(&RISCV::GPRRegClass);
1091 LIS->InsertMachineInstrInMaps(*
MI);
1092 LIS->createAndComputeVirtRegInterval(DestReg);
1097 MRI->constrainRegClass(
AVLReg, &RISCV::GPRNoX0RegClass);
1102 LIS->InsertMachineInstrInMaps(*
MI);
1107 LIS->getMBBStartIdx(&
MBB),
LIS->getInstructionIndex(*MI).getRegSlot());
1112 return Fractional || LMul == 1;
1118 const VSETVLIInfo &Require,
1119 const VSETVLIInfo &CurInfo)
const {
1122 if (!CurInfo.isValid() || CurInfo.isUnknown() || CurInfo.hasSEWLMULRatioOnly())
1125 DemandedFields
Used = getDemanded(
MI, ST);
1135 if (isVSlideInstr(
MI) && Require.hasAVLImm() && Require.getAVLImm() == 1 &&
1138 Used.VLZeroness =
true;
1140 Used.TailPolicy =
false;
1148 if (isScalarSplatInstr(
MI) && Require.hasAVLImm() &&
1150 hasUndefinedMergeOp(
MI)) {
1152 Used.SEWLMULRatio =
false;
1154 if (isFloatScalarMoveOrScalarSplatInstr(
MI) && !
ST->hasVInstructionsF64())
1155 Used.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
1157 Used.SEW = DemandedFields::SEWGreaterThanOrEqual;
1158 Used.TailPolicy =
false;
1161 if (CurInfo.isCompatible(Used, Require,
LIS))
1168 if (Require.hasAVLReg() && CurInfo.hasCompatibleVTYPE(Used, Require)) {
1171 VSETVLIInfo DefInfo = getInfoForVSETVLI(*
DefMI,
LIS);
1172 if (DefInfo.hasSameAVL(CurInfo) && DefInfo.hasSameVLMAX(CurInfo))
1184 DemandedFields &Demanded) {
1187 if (!Demanded.LMUL && !Demanded.SEWLMULRatio && PrevInfo.isValid() &&
1188 !PrevInfo.isUnknown()) {
1190 PrevInfo.getSEW(), PrevInfo.getVLMUL(),
Info.getSEW()))
1191 Info.setVLMul(*NewVLMul);
1192 Demanded.LMUL =
true;
1201void RISCVInsertVSETVLI::transferBefore(VSETVLIInfo &Info,
1212 const VSETVLIInfo PrevInfo =
Info;
1213 if (!
Info.isValid() ||
Info.isUnknown())
1216 DemandedFields Demanded = getDemanded(
MI, ST);
1226 bool EquallyZero = IncomingInfo.hasEquallyZeroAVL(PrevInfo,
LIS) &&
1227 IncomingInfo.hasSameVLMAX(PrevInfo);
1228 if (Demanded.VLAny || (Demanded.VLZeroness && !EquallyZero))
1229 Info.setAVL(IncomingInfo);
1232 ((Demanded.LMUL || Demanded.SEWLMULRatio) ? IncomingInfo : Info)
1234 ((Demanded.SEW || Demanded.SEWLMULRatio) ? IncomingInfo : Info).getSEW(),
1237 (Demanded.TailPolicy ? IncomingInfo : Info).getTailAgnostic() ||
1238 IncomingInfo.getTailAgnostic(),
1239 (Demanded.MaskPolicy ? IncomingInfo : Info).getMaskAgnostic() ||
1240 IncomingInfo.getMaskAgnostic());
1244 if (
Info.hasSEWLMULRatioOnly()) {
1245 VSETVLIInfo RatiolessInfo = IncomingInfo;
1246 RatiolessInfo.setAVL(Info);
1247 Info = RatiolessInfo;
1254void RISCVInsertVSETVLI::transferAfter(VSETVLIInfo &Info,
1256 if (isVectorConfigInstr(
MI)) {
1263 assert(
MI.getOperand(1).getReg().isVirtual());
1264 auto &LI =
LIS->getInterval(
MI.getOperand(1).getReg());
1265 SlotIndex SI =
LIS->getSlotIndexes()->getInstructionIndex(
MI).getRegSlot();
1266 VNInfo *VNI = LI.getVNInfoAt(SI);
1267 Info.setAVLRegDef(VNI,
MI.getOperand(1).getReg());
1273 if (
MI.isCall() ||
MI.isInlineAsm() ||
1274 MI.modifiesRegister(RISCV::VL,
nullptr) ||
1275 MI.modifiesRegister(RISCV::VTYPE,
nullptr))
1276 Info = VSETVLIInfo::getUnknown();
1280 VSETVLIInfo &Info)
const {
1281 bool HadVectorOp =
false;
1285 transferBefore(Info,
MI);
1290 transferAfter(Info,
MI);
1300 BBInfo.InQueue =
false;
1304 VSETVLIInfo InInfo = BBInfo.
Pred;
1307 InInfo.setUnknown();
1310 InInfo = InInfo.
intersect(BlockInfo[
P->getNumber()].Exit);
1314 if (!InInfo.isValid())
1318 if (InInfo == BBInfo.
Pred)
1321 BBInfo.
Pred = InInfo;
1323 <<
" changed to " << BBInfo.
Pred <<
"\n");
1329 VSETVLIInfo TmpStatus;
1330 computeVLVTYPEChanges(
MBB, TmpStatus);
1334 if (BBInfo.
Exit == TmpStatus)
1337 BBInfo.
Exit = TmpStatus;
1339 <<
" changed to " << BBInfo.
Exit <<
"\n");
1344 if (!BlockInfo[S->getNumber()].InQueue) {
1345 BlockInfo[S->getNumber()].InQueue =
true;
1353bool RISCVInsertVSETVLI::needVSETVLIPHI(
const VSETVLIInfo &Require,
1358 if (!Require.hasAVLReg())
1362 const VNInfo *Valno = Require.getAVLVNInfo();
1366 const LiveRange &LR =
LIS->getInterval(Require.getAVLReg());
1369 const VSETVLIInfo &PBBExit = BlockInfo[PBB->getNumber()].Exit;
1381 VSETVLIInfo DefInfo = getInfoForVSETVLI(*
DefMI,
LIS);
1382 if (DefInfo != PBBExit)
1388 if (PBBExit.isUnknown() || !PBBExit.hasSameVTYPE(Require))
1401 bool PrefixTransparent =
true;
1403 const VSETVLIInfo PrevInfo = CurInfo;
1404 transferBefore(CurInfo,
MI);
1407 if (isVectorConfigInstr(
MI)) {
1409 assert(
MI.getOperand(3).getReg() == RISCV::VL &&
1410 MI.getOperand(4).getReg() == RISCV::VTYPE &&
1411 "Unexpected operands where VL and VTYPE should be");
1412 MI.getOperand(3).setIsDead(
false);
1413 MI.getOperand(4).setIsDead(
false);
1414 PrefixTransparent =
false;
1419 if (PrevInfo != CurInfo) {
1427 if (!PrefixTransparent || needVSETVLIPHI(CurInfo,
MBB))
1428 insertVSETVLI(
MBB,
MI, CurInfo, PrevInfo);
1429 PrefixTransparent =
false;
1439 VLOp.
setReg(RISCV::NoRegister);
1442 LIS->shrinkToUses(&LI, &DeadMIs);
1446 LIS->splitSeparateComponents(LI, SplitLIs);
1453 if (!
TII->isAddImmediate(*DeadMI, Reg))
1455 LIS->RemoveMachineInstrFromMaps(*DeadMI);
1456 DeadMI->eraseFromParent();
1466 if (
MI.isCall() ||
MI.isInlineAsm() ||
1467 MI.modifiesRegister(RISCV::VL,
nullptr) ||
1468 MI.modifiesRegister(RISCV::VTYPE,
nullptr))
1469 PrefixTransparent =
false;
1471 transferAfter(CurInfo,
MI);
1475 if (CurInfo !=
Info.Exit) {
1481 assert(CurInfo ==
Info.Exit &&
"InsertVSETVLI dataflow invariant violated");
1494 VSETVLIInfo AvailableInfo;
1496 const VSETVLIInfo &PredInfo = BlockInfo[
P->getNumber()].Exit;
1497 if (PredInfo.isUnknown()) {
1498 if (UnavailablePred)
1500 UnavailablePred =
P;
1501 }
else if (!AvailableInfo.isValid()) {
1502 AvailableInfo = PredInfo;
1503 }
else if (AvailableInfo != PredInfo) {
1510 if (!UnavailablePred || !AvailableInfo.isValid())
1515 if (AvailableInfo.hasSEWLMULRatioOnly())
1525 if (AvailableInfo.hasAVLReg()) {
1529 if (
LIS->getMBBFromIndex(SI) != UnavailablePred)
1538 if (AvailableInfo.hasAVLIgnored())
1550 VSETVLIInfo CurInfo = AvailableInfo;
1551 int TransitionsRemoved = 0;
1553 const VSETVLIInfo LastInfo = CurInfo;
1554 const VSETVLIInfo LastOldInfo = OldInfo;
1555 transferBefore(CurInfo,
MI);
1556 transferBefore(OldInfo,
MI);
1557 if (CurInfo == LastInfo)
1558 TransitionsRemoved++;
1559 if (LastOldInfo == OldInfo)
1560 TransitionsRemoved--;
1561 transferAfter(CurInfo,
MI);
1562 transferAfter(OldInfo,
MI);
1563 if (CurInfo == OldInfo)
1567 if (CurInfo != OldInfo || TransitionsRemoved <= 0)
1574 auto OldExit = BlockInfo[UnavailablePred->
getNumber()].Exit;
1576 << UnavailablePred->
getName() <<
" with state "
1577 << AvailableInfo <<
"\n");
1578 BlockInfo[UnavailablePred->
getNumber()].Exit = AvailableInfo;
1584 insertVSETVLI(*UnavailablePred, InsertPt,
1586 AvailableInfo, OldExit);
1593 const DemandedFields &Used,
1599 if (!isVLPreservingConfig(
MI)) {
1603 if (Used.VLZeroness) {
1604 if (isVLPreservingConfig(PrevMI))
1606 if (!getInfoForVSETVLI(PrevMI,
LIS)
1607 .hasEquallyZeroAVL(getInfoForVSETVLI(
MI,
LIS),
LIS))
1611 auto &AVL =
MI.getOperand(1);
1616 if (AVL.isReg() && AVL.getReg() != RISCV::X0 &&
1617 (!
MRI.hasOneDef(AVL.getReg()) || !PrevAVL.isReg() ||
1618 PrevAVL.getReg() != AVL.getReg()))
1624 auto VType =
MI.getOperand(2).getImm();
1625 return areCompatibleVTYPEs(PriorVType, VType, Used);
1632 DemandedFields
Used;
1638 if (!isVectorConfigInstr(
MI)) {
1639 Used.doUnion(getDemanded(
MI, ST));
1640 if (
MI.isCall() ||
MI.isInlineAsm() ||
1641 MI.modifiesRegister(RISCV::VL,
nullptr) ||
1642 MI.modifiesRegister(RISCV::VTYPE,
nullptr))
1647 if (!
MI.getOperand(0).isDead())
1651 if (!
Used.usedVL() && !
Used.usedVTYPE()) {
1658 if (!isVLPreservingConfig(*NextMI)) {
1661 MI.getOperand(0).setReg(DefReg);
1662 MI.getOperand(0).setIsDead(
false);
1672 DefVNI->
def = MISlot;
1678 LIS->shrinkToUses(&DefLI);
1682 if (
MI.getOperand(1).isReg())
1683 OldVLReg =
MI.getOperand(1).getReg();
1695 LIS->shrinkToUses(&
LIS->getInterval(OldVLReg));
1698 if (VLOpDef &&
TII->isAddImmediate(*VLOpDef, OldVLReg) &&
1699 MRI->use_nodbg_empty(OldVLReg)) {
1701 LIS->removeInterval(OldVLReg);
1712 Used = getDemanded(
MI, ST);
1715 NumCoalescedVSETVL += ToDelete.
size();
1716 for (
auto *
MI : ToDelete) {
1717 LIS->RemoveMachineInstrFromMaps(*
MI);
1718 MI->eraseFromParent();
1721 return !ToDelete.empty();
1728 Register VLOutput =
MI.getOperand(1).getReg();
1730 if (!
MI.getOperand(1).isDead()) {
1732 TII->get(RISCV::PseudoReadVL), VLOutput);
1735 LIS->InsertMachineInstrInMaps(*ReadVLMI).getRegSlot();
1739 DefVNI->
def = NewDefSI;
1742 MI.getOperand(1).setReg(RISCV::X0);
1750 if (!
ST->hasVInstructions())
1755 TII =
ST->getInstrInfo();
1757 LIS = &getAnalysis<LiveIntervals>();
1759 assert(BlockInfo.empty() &&
"Expect empty block infos");
1762 bool HaveVectorOp =
false;
1766 VSETVLIInfo TmpStatus;
1767 HaveVectorOp |= computeVLVTYPEChanges(
MBB, TmpStatus);
1770 BBInfo.
Exit = TmpStatus;
1772 <<
" is " << BBInfo.
Exit <<
"\n");
1777 if (!HaveVectorOp) {
1786 WorkList.push(&
MBB);
1789 while (!WorkList.empty()) {
1792 computeIncomingVLVTYPE(
MBB);
1812 return HaveVectorOp;
1817 return new RISCVInsertVSETVLI();
1829 if (!ST->hasVInstructions())
1831 TII = ST->getInstrInfo();
1833 LIS = &getAnalysis<LiveIntervals>();
1835 bool Changed =
false;
1837 Changed |= coalesceVSETVLIs(
MBB);
1843 return new RISCVCoalesceVSETVLI();
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder MachineInstrBuilder & DefMI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Analysis containing CSE Info
#define LLVM_ATTRIBUTE_USED
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
static RegisterPass< DebugifyFunctionPass > DF("debugify-function", "Attach debug info to a function")
std::optional< std::vector< StOtherPiece > > Other
const HexagonInstrInfo * TII
static ValueLatticeElement intersect(const ValueLatticeElement &A, const ValueLatticeElement &B)
Combine two sets of facts about the same value into a single set of facts.
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
NewInfo setVTYPE(MI.getOperand(2).getImm())
static VSETVLIInfo computeInfoForInstr(const MachineInstr &MI, uint64_t TSFlags, const RISCVSubtarget &ST, const LiveIntervals *LIS)
#define RISCV_COALESCE_VSETVLI_NAME
assert((AVLReg !=RISCV::X0||MI.getOperand(0).getReg() !=RISCV::X0) &&"Can't handle X0, X0 vsetvli yet")
static unsigned computeVLMAX(unsigned VLEN, unsigned SEW, RISCVII::VLMUL VLMul)
#define RISCV_INSERT_VSETVLI_NAME
static VSETVLIInfo adjustIncoming(VSETVLIInfo PrevInfo, VSETVLIInfo NewInfo, DemandedFields &Demanded)
static bool isLMUL1OrSmaller(RISCVII::VLMUL LMUL)
static cl::opt< bool > DisableInsertVSETVLPHIOpt("riscv-disable-insert-vsetvl-phi-opt", cl::init(false), cl::Hidden, cl::desc("Disable looking through phis when inserting vsetvlis."))
static bool canMutatePriorConfig(const MachineInstr &PrevMI, const MachineInstr &MI, const DemandedFields &Used, const MachineRegisterInfo &MRI, const LiveIntervals *LIS)
INITIALIZE_PASS(RISCVInsertVSETVLI, DEBUG_TYPE, RISCV_INSERT_VSETVLI_NAME, false, false) char RISCVCoalesceVSETVLI const LiveIntervals * LIS
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
This class represents an Operation in the Expression.
FunctionPass class - This class is used to implement most global optimizations.
LiveInterval - This class represents the liveness of a register, or stack slot.
void setWeight(float Value)
This class represents the liveness of a register, stack slot, etc.
iterator addSegment(Segment S)
Add the specified Segment to this range, merging segments as appropriate.
VNInfo * getVNInfoBefore(SlotIndex Idx) const
getVNInfoBefore - Return the VNInfo that is live up to but not necessarilly including Idx,...
SlotIndex beginIndex() const
beginIndex - Return the lowest numbered slot covered.
void removeSegment(SlotIndex Start, SlotIndex End, bool RemoveDeadValNo=false)
Remove the specified interval from this live range.
VNInfo * getVNInfoAt(SlotIndex Idx) const
getVNInfoAt - Return the VNInfo that is live at Idx, or NULL.
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
unsigned succ_size() const
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
iterator_range< iterator > terminators()
iterator_range< succ_iterator > successors()
instr_iterator getFirstInstrTerminator()
Same getFirstTerminator but it ignores bundles and return an instr_iterator instead.
reverse_iterator rbegin()
iterator_range< pred_iterator > predecessors()
StringRef getName() const
Return the name of the corresponding LLVM basic block, or an empty string.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
unsigned getNumBlockIDs() const
getNumBlockIDs - Return the number of MBB ID's allocated.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
SlotIndex - An opaque wrapper around machine indexes.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
TargetInstrInfo - Interface to description of machine instruction set.
VNInfo - Value Number Information.
SlotIndex def
The index of the defining instruction.
bool isPHIDef() const
Returns true if this value is defined by a PHI instruction (or was, PHI instructions may have been el...
LLVM Value Representation.
This class implements an extremely fast bulk output stream that can only output to a stream.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
static bool usesMaskPolicy(uint64_t TSFlags)
static unsigned getVLOpNum(const MCInstrDesc &Desc)
static bool doesForceTailAgnostic(uint64_t TSFlags)
static VLMUL getLMul(uint64_t TSFlags)
static bool hasVLOp(uint64_t TSFlags)
static bool hasVecPolicyOp(uint64_t TSFlags)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
static bool hasSEWOp(uint64_t TSFlags)
static bool isTailAgnostic(unsigned VType)
static RISCVII::VLMUL getVLMUL(unsigned VType)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
unsigned getSEWLMULRatio(unsigned SEW, RISCVII::VLMUL VLMul)
static bool isMaskAgnostic(unsigned VType)
static bool isValidSEW(unsigned SEW)
unsigned encodeVTYPE(RISCVII::VLMUL VLMUL, unsigned SEW, bool TailAgnostic, bool MaskAgnostic)
static unsigned getSEW(unsigned VType)
std::optional< RISCVII::VLMUL > getSameRatioLMUL(unsigned SEW, RISCVII::VLMUL VLMUL, unsigned EEW)
unsigned getRVVMCOpcode(unsigned RVVPseudoOpcode)
bool isFaultFirstLoad(const MachineInstr &MI)
static constexpr int64_t VLMaxSentinel
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool operator!=(uint64_t V1, const APInt &V2)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
FunctionPass * createRISCVInsertVSETVLIPass()
Returns an instance of the Insert VSETVLI pass.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
char & RISCVInsertVSETVLIID
FunctionPass * createRISCVCoalesceVSETVLIPass()
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Status intersect(const Status &S) const
This represents a simple continuous liveness interval for a value.