87#define DEBUG_TYPE "si-wqm"
96 StateStrict = StateStrictWWM | StateStrictWQM,
103 explicit PrintState(
int State) : State(State) {}
109 static const std::pair<char, const char *> Mapping[] = {
110 std::pair(StateWQM,
"WQM"), std::pair(StateStrictWWM,
"StrictWWM"),
111 std::pair(StateStrictWQM,
"StrictWQM"), std::pair(StateExact,
"Exact")};
112 char State = PS.State;
113 for (
auto M : Mapping) {
114 if (State & M.first) {
137 char InitialState = 0;
138 bool NeedsLowering =
false;
164 unsigned AndSaveExecOpc;
165 unsigned AndSaveExecTermOpc;
184 std::vector<WorkItem> &Worklist);
186 unsigned SubReg,
char Flag, std::vector<WorkItem> &Worklist);
188 std::vector<WorkItem> &Worklist);
190 std::vector<WorkItem> &Worklist);
191 char scanInstructions(
MachineFunction &MF, std::vector<WorkItem> &Worklist);
192 void propagateInstruction(
MachineInstr &
MI, std::vector<WorkItem> &Worklist);
207 Register SaveOrig,
char StrictStateNeeded);
210 char NonStrictState,
char CurrentStrictState);
223 void lowerLiveMaskQueries();
224 void lowerCopyInstrs();
225 void lowerKillInstrs(
bool IsWQM);
248 MachineFunctionProperties::Property::IsSSA);
254char SIWholeQuadMode::ID = 0;
267 return new SIWholeQuadMode;
272 for (
const auto &BII :
Blocks) {
275 <<
" InNeeds = " << PrintState(BII.second.InNeeds)
276 <<
", Needs = " << PrintState(BII.second.Needs)
277 <<
", OutNeeds = " << PrintState(BII.second.OutNeeds) <<
"\n\n";
280 auto III = Instructions.find(&
MI);
281 if (III == Instructions.end())
284 dbgs() <<
" " <<
MI <<
" Needs = " << PrintState(III->second.Needs)
285 <<
", OutNeeds = " << PrintState(III->second.OutNeeds) <<
'\n';
292 std::vector<WorkItem> &Worklist) {
295 assert(!(Flag & StateExact) && Flag != 0);
301 Flag &= ~II.Disabled;
305 if ((II.Needs & Flag) == Flag)
310 Worklist.push_back(&
MI);
316 std::vector<WorkItem> &Worklist) {
328 : (
Reg.isVirtual() ?
MRI->getMaxLaneMaskForVReg(Reg)
340 :
Phi(
Phi), PredIdx(PredIdx), DefinedLanes(DefinedLanes) {}
342 using VisitKey = std::pair<const VNInfo *, LaneBitmask>;
346 unsigned NextPredIdx = 0;
348 const VNInfo *NextValue =
nullptr;
349 const VisitKey
Key(
Value, DefinedLanes);
351 if (Visited.
insert(Key).second) {
356 if (
Value->isPHIDef()) {
359 assert(
MBB &&
"Phi-def has no defining MBB");
362 unsigned Idx = NextPredIdx;
365 for (; PI != PE && !NextValue; ++PI, ++
Idx) {
367 if (!Visited.
count(VisitKey(VN, DefinedLanes)))
377 assert(
MI &&
"Def has no defining instruction");
379 if (
Reg.isVirtual()) {
383 if (
Op.getReg() != Reg)
389 :
TRI->getSubRegIndexLaneMask(
Op.getSubReg());
393 HasDef |= Overlap.
any();
396 DefinedLanes |= OpLanes;
400 if ((DefinedLanes & UseLanes) != UseLanes) {
404 if (!Visited.
count(VisitKey(VN, DefinedLanes)))
411 markInstruction(*
MI, Flag, Worklist);
414 markInstruction(*
MI, Flag, Worklist);
418 if (!NextValue && !PhiStack.
empty()) {
420 PhiEntry &Entry = PhiStack.
back();
421 NextValue = Entry.Phi;
422 NextPredIdx = Entry.PredIdx;
423 DefinedLanes = Entry.DefinedLanes;
433 std::vector<WorkItem> &Worklist) {
440 case AMDGPU::EXEC_LO:
448 if (
Reg.isVirtual()) {
450 markDefs(
MI, LR, Reg,
Op.getSubReg(), Flag, Worklist);
461 markDefs(
MI, LR, Unit, AMDGPU::NoSubRegister, Flag, Worklist);
467void SIWholeQuadMode::markInstructionUses(
const MachineInstr &
MI,
char Flag,
468 std::vector<WorkItem> &Worklist) {
469 LLVM_DEBUG(
dbgs() <<
"markInstructionUses " << PrintState(Flag) <<
": "
473 markOperand(
MI,
Use, Flag, Worklist);
479 std::vector<WorkItem> &Worklist) {
480 char GlobalFlags = 0;
484 bool HasImplicitDerivatives =
497 unsigned Opcode =
MI.getOpcode();
500 if (
TII->isWQM(Opcode)) {
502 if (!
ST->hasExtendedImageInsts())
507 if (!HasImplicitDerivatives)
512 markInstructionUses(
MI, StateWQM, Worklist);
513 GlobalFlags |= StateWQM;
515 }
else if (Opcode == AMDGPU::WQM) {
519 LowerToCopyInstrs.push_back(&
MI);
520 }
else if (Opcode == AMDGPU::SOFT_WQM) {
521 LowerToCopyInstrs.push_back(&
MI);
524 }
else if (Opcode == AMDGPU::STRICT_WWM) {
528 markInstructionUses(
MI, StateStrictWWM, Worklist);
529 GlobalFlags |= StateStrictWWM;
530 LowerToMovInstrs.push_back(&
MI);
532 }
else if (Opcode == AMDGPU::STRICT_WQM ||
533 TII->isDualSourceBlendEXP(
MI)) {
537 markInstructionUses(
MI, StateStrictWQM, Worklist);
538 GlobalFlags |= StateStrictWQM;
540 if (Opcode == AMDGPU::STRICT_WQM) {
541 LowerToMovInstrs.push_back(&
MI);
546 BBI.Needs |= StateExact;
547 if (!(BBI.InNeeds & StateExact)) {
548 BBI.InNeeds |= StateExact;
549 Worklist.push_back(
MBB);
551 GlobalFlags |= StateExact;
552 III.Disabled = StateWQM | StateStrict;
555 }
else if (Opcode == AMDGPU::LDS_PARAM_LOAD ||
556 Opcode == AMDGPU::DS_PARAM_LOAD ||
557 Opcode == AMDGPU::LDS_DIRECT_LOAD ||
558 Opcode == AMDGPU::DS_DIRECT_LOAD) {
562 II.Needs |= StateStrictWQM;
563 GlobalFlags |= StateStrictWQM;
565 }
else if (Opcode == AMDGPU::V_SET_INACTIVE_B32 ||
566 Opcode == AMDGPU::V_SET_INACTIVE_B64) {
567 III.Disabled = StateStrict;
569 if (Inactive.
isReg()) {
571 LowerToCopyInstrs.push_back(&
MI);
573 markOperand(
MI, Inactive, StateStrictWWM, Worklist);
578 }
else if (
TII->isDisableWQM(
MI)) {
579 BBI.Needs |= StateExact;
580 if (!(BBI.InNeeds & StateExact)) {
581 BBI.InNeeds |= StateExact;
582 Worklist.push_back(
MBB);
584 GlobalFlags |= StateExact;
585 III.Disabled = StateWQM | StateStrict;
588 if (Opcode == AMDGPU::SI_PS_LIVE || Opcode == AMDGPU::SI_LIVE_MASK) {
589 LiveMaskQueries.push_back(&
MI);
590 }
else if (Opcode == AMDGPU::SI_KILL_I1_TERMINATOR ||
591 Opcode == AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR ||
592 Opcode == AMDGPU::SI_DEMOTE_I1) {
593 KillInstrs.push_back(&
MI);
594 BBI.NeedsLowering =
true;
595 }
else if (WQMOutputs) {
606 if (!
Reg.isVirtual() &&
607 TRI->hasVectorRegisters(
TRI->getPhysRegBaseClass(Reg))) {
618 markInstruction(
MI, Flags, Worklist);
619 GlobalFlags |=
Flags;
627 if (GlobalFlags & StateWQM) {
629 markInstruction(*
MI, StateWQM, Worklist);
631 markInstruction(*
MI, StateWQM, Worklist);
638 std::vector<WorkItem>& Worklist) {
645 if ((II.OutNeeds & StateWQM) && !(II.Disabled & StateWQM) &&
646 (
MI.isTerminator() || (
TII->usesVM_CNT(
MI) &&
MI.mayStore()))) {
652 if (II.Needs & StateWQM) {
653 BI.Needs |= StateWQM;
654 if (!(BI.InNeeds & StateWQM)) {
655 BI.InNeeds |= StateWQM;
656 Worklist.push_back(
MBB);
662 char InNeeds = (II.Needs & ~StateStrict) | II.OutNeeds;
663 if (!PrevMI->isPHI()) {
665 if ((PrevII.OutNeeds | InNeeds) != PrevII.OutNeeds) {
666 PrevII.OutNeeds |= InNeeds;
667 Worklist.push_back(PrevMI);
673 assert(!(II.Needs & StateExact));
676 markInstructionUses(
MI, II.Needs, Worklist);
680 if (II.Needs & StateStrictWWM)
681 BI.Needs |= StateStrictWWM;
682 if (II.Needs & StateStrictWQM)
683 BI.Needs |= StateStrictWQM;
687 std::vector<WorkItem>& Worklist) {
694 if ((LastII.OutNeeds | BI.OutNeeds) != LastII.OutNeeds) {
695 LastII.OutNeeds |= BI.OutNeeds;
696 Worklist.push_back(LastMI);
702 BlockInfo &PredBI =
Blocks[Pred];
703 if ((PredBI.OutNeeds | BI.InNeeds) == PredBI.OutNeeds)
706 PredBI.OutNeeds |= BI.InNeeds;
707 PredBI.InNeeds |= BI.InNeeds;
708 Worklist.push_back(Pred);
713 BlockInfo &SuccBI =
Blocks[Succ];
714 if ((SuccBI.InNeeds | BI.OutNeeds) == SuccBI.InNeeds)
717 SuccBI.InNeeds |= BI.OutNeeds;
718 Worklist.push_back(Succ);
723 std::vector<WorkItem> Worklist;
724 char GlobalFlags = scanInstructions(MF, Worklist);
726 while (!Worklist.empty()) {
731 propagateInstruction(*WI.MI, Worklist);
733 propagateBlock(*WI.MBB, Worklist);
742 Register SaveReg =
MRI->createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
751 LIS->InsertMachineInstrInMaps(*Save);
752 LIS->InsertMachineInstrInMaps(*Restore);
753 LIS->createAndComputeVirtRegInterval(SaveReg);
764 BB->
splitAt(*TermMI,
true, LIS);
768 unsigned NewOpcode = 0;
770 case AMDGPU::S_AND_B32:
771 NewOpcode = AMDGPU::S_AND_B32_term;
773 case AMDGPU::S_AND_B64:
774 NewOpcode = AMDGPU::S_AND_B64_term;
776 case AMDGPU::S_MOV_B32:
777 NewOpcode = AMDGPU::S_MOV_B32_term;
779 case AMDGPU::S_MOV_B64:
780 NewOpcode = AMDGPU::S_MOV_B64_term;
793 DTUpdates.
push_back({DomTreeT::Insert, SplitBB, Succ});
794 DTUpdates.
push_back({DomTreeT::Delete, BB, Succ});
796 DTUpdates.
push_back({DomTreeT::Insert, BB, SplitBB});
798 MDT->getBase().applyUpdates(DTUpdates);
800 PDT->getBase().applyUpdates(DTUpdates);
806 LIS->InsertMachineInstrInMaps(*
MI);
826 switch (
MI.getOperand(2).getImm()) {
828 Opcode = AMDGPU::V_CMP_LG_F32_e64;
831 Opcode = AMDGPU::V_CMP_GE_F32_e64;
834 Opcode = AMDGPU::V_CMP_GT_F32_e64;
837 Opcode = AMDGPU::V_CMP_LE_F32_e64;
840 Opcode = AMDGPU::V_CMP_LT_F32_e64;
843 Opcode = AMDGPU::V_CMP_EQ_F32_e64;
846 Opcode = AMDGPU::V_CMP_O_F32_e64;
849 Opcode = AMDGPU::V_CMP_U_F32_e64;
853 Opcode = AMDGPU::V_CMP_NEQ_F32_e64;
857 Opcode = AMDGPU::V_CMP_NLT_F32_e64;
861 Opcode = AMDGPU::V_CMP_NLE_F32_e64;
865 Opcode = AMDGPU::V_CMP_NGT_F32_e64;
869 Opcode = AMDGPU::V_CMP_NGE_F32_e64;
873 Opcode = AMDGPU::V_CMP_NLG_F32_e64;
885 Register VCC =
ST->isWave32() ? AMDGPU::VCC_LO : AMDGPU::VCC;
918 LIS->ReplaceMachineInstrInMaps(
MI, *VcmpMI);
921 LIS->InsertMachineInstrInMaps(*MaskUpdateMI);
922 LIS->InsertMachineInstrInMaps(*ExecMaskMI);
923 LIS->InsertMachineInstrInMaps(*EarlyTermMI);
924 LIS->InsertMachineInstrInMaps(*NewTerm);
934 const bool IsDemote = IsWQM && (
MI.getOpcode() == AMDGPU::SI_DEMOTE_I1);
936 int64_t KillVal =
MI.getOperand(1).getImm();
943 if (
Op.getImm() == KillVal) {
951 if (
MI.getOpcode() == AMDGPU::SI_DEMOTE_I1) {
952 LIS->RemoveMachineInstrFromMaps(
MI);
957 LIS->ReplaceMachineInstrInMaps(
MI, *NewTerm);
966 TmpReg =
MRI->createVirtualRegister(
TRI->getBoolRC());
967 ComputeKilledMaskMI =
992 LiveMaskWQM =
MRI->createVirtualRegister(
TRI->getBoolRC());
1001 unsigned MovOpc =
ST->isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
1003 }
else if (!IsWQM) {
1008 unsigned Opcode = KillVal ? AndN2Opc : AndOpc;
1015 LIS->RemoveMachineInstrFromMaps(
MI);
1020 if (ComputeKilledMaskMI)
1021 LIS->InsertMachineInstrInMaps(*ComputeKilledMaskMI);
1022 LIS->InsertMachineInstrInMaps(*MaskUpdateMI);
1023 LIS->InsertMachineInstrInMaps(*EarlyTermMI);
1025 LIS->InsertMachineInstrInMaps(*WQMMaskMI);
1026 LIS->InsertMachineInstrInMaps(*NewTerm);
1029 LIS->removeInterval(CndReg);
1030 LIS->createAndComputeVirtRegInterval(CndReg);
1033 LIS->createAndComputeVirtRegInterval(TmpReg);
1035 LIS->createAndComputeVirtRegInterval(LiveMaskWQM);
1046 assert(Entry->getOpcode() == AMDGPU::ENTER_STRICT_WQM);
1047 assert(
Exit->getOpcode() == AMDGPU::EXIT_STRICT_WQM);
1049 Register SaveOrig = Entry->getOperand(0).getReg();
1056 LIS->ReplaceMachineInstrInMaps(*Exit, *NewExit);
1057 Exit->eraseFromParent();
1059 LIS->ReplaceMachineInstrInMaps(*Entry, *NewEntry);
1060 Entry->eraseFromParent();
1062 LIS->removeInterval(SaveOrig);
1073 const BlockInfo &BI = BII->second;
1074 if (!BI.NeedsLowering)
1080 char State = BI.InitialState;
1085 char PreviousState = State;
1087 if (StateTransition.count(&
MI))
1088 State = StateTransition[&
MI];
1091 switch (
MI.getOpcode()) {
1092 case AMDGPU::SI_DEMOTE_I1:
1093 case AMDGPU::SI_KILL_I1_TERMINATOR:
1094 SplitPoint = lowerKillI1(
MBB,
MI, State == StateWQM);
1096 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
1097 SplitPoint = lowerKillF32(
MBB,
MI);
1099 case AMDGPU::ENTER_STRICT_WQM:
1100 StrictEntry = PreviousState == StateWQM ? &
MI :
nullptr;
1102 case AMDGPU::EXIT_STRICT_WQM:
1103 if (State == StateWQM && StrictEntry) {
1105 lowerPseudoStrictMode(
MBB, StrictEntry, &
MI);
1107 StrictEntry =
nullptr;
1109 case AMDGPU::ENTER_STRICT_WWM:
1110 case AMDGPU::EXIT_STRICT_WWM:
1111 StrictEntry =
nullptr;
1121 if (!SplitPoints.
empty()) {
1142 : LIS->getMBBEndIdx(&
MBB);
1144 Last != MBBE ? LIS->getInstructionIndex(*
Last) : LIS->getMBBEndIdx(&
MBB);
1155 if (Next < FirstIdx)
1160 assert(EndMI &&
"Segment does not end on valid instruction");
1164 SlotIndex Next = LIS->getInstructionIndex(*NextI);
1184 bool IsExecDef =
false;
1187 MO.getReg() == AMDGPU::EXEC_LO || MO.getReg() == AMDGPU::EXEC;
1205 if (!IsTerminator) {
1207 if (FirstTerm !=
MBB.
end()) {
1208 SlotIndex FirstTermIdx = LIS->getInstructionIndex(*FirstTerm);
1210 IsTerminator = BeforeIdx > FirstTermIdx;
1217 unsigned Opcode = IsTerminator ? AndSaveExecTermOpc : AndSaveExecOpc;
1221 unsigned Opcode = IsTerminator ? AndTermOpc : AndOpc;
1227 LIS->InsertMachineInstrInMaps(*
MI);
1228 StateTransition[
MI] = StateExact;
1243 LIS->InsertMachineInstrInMaps(*
MI);
1244 StateTransition[
MI] = StateWQM;
1249 Register SaveOrig,
char StrictStateNeeded) {
1252 assert(StrictStateNeeded == StateStrictWWM ||
1253 StrictStateNeeded == StateStrictWQM);
1255 if (StrictStateNeeded == StateStrictWWM) {
1264 LIS->InsertMachineInstrInMaps(*
MI);
1265 StateTransition[
MI] = StrictStateNeeded;
1270 BII->second.NeedsLowering =
true;
1275 Register SavedOrig,
char NonStrictState,
1276 char CurrentStrictState) {
1280 assert(CurrentStrictState == StateStrictWWM ||
1281 CurrentStrictState == StateStrictWQM);
1283 if (CurrentStrictState == StateStrictWWM) {
1292 LIS->InsertMachineInstrInMaps(*
MI);
1293 StateTransition[
MI] = NonStrictState;
1301 BlockInfo &BI = BII->second;
1305 if (!IsEntry && BI.Needs == StateWQM && BI.OutNeeds != StateExact) {
1306 BI.InitialState = StateWQM;
1315 bool WQMFromExec = IsEntry;
1316 char State = (IsEntry || !(BI.InNeeds & StateWQM)) ? StateExact : StateWQM;
1317 char NonStrictState = 0;
1323 if (II != IE && II->getOpcode() == AMDGPU::COPY &&
1324 II->getOperand(1).getReg() ==
TRI->getExec())
1339 BI.InitialState = State;
1343 char Needs = StateExact | StateWQM;
1349 if (FirstStrict == IE)
1357 if (
MI.isTerminator() ||
TII->mayReadEXEC(*
MRI,
MI)) {
1360 if (III->second.Needs & StateStrictWWM)
1361 Needs = StateStrictWWM;
1362 else if (III->second.Needs & StateStrictWQM)
1363 Needs = StateStrictWQM;
1364 else if (III->second.Needs & StateWQM)
1367 Needs &= ~III->second.Disabled;
1368 OutNeeds = III->second.OutNeeds;
1373 Needs = StateExact | StateWQM | StateStrict;
1377 if (
MI.isBranch() && OutNeeds == StateExact)
1383 if (BI.OutNeeds & StateWQM)
1385 else if (BI.OutNeeds == StateExact)
1388 Needs = StateWQM | StateExact;
1392 if (!(Needs & State)) {
1394 if (State == StateStrictWWM || Needs == StateStrictWWM ||
1395 State == StateStrictWQM || Needs == StateStrictWQM) {
1397 First = FirstStrict;
1404 bool SaveSCC =
false;
1407 case StateStrictWWM:
1408 case StateStrictWQM:
1412 SaveSCC = (Needs & StateStrict) || ((Needs & StateWQM) && WQMFromExec);
1416 SaveSCC = !(Needs & StateWQM);
1423 prepareInsertion(
MBB,
First, II, Needs == StateWQM, SaveSCC);
1425 if (State & StateStrict) {
1426 assert(State == StateStrictWWM || State == StateStrictWQM);
1427 assert(SavedNonStrictReg);
1428 fromStrictMode(
MBB,
Before, SavedNonStrictReg, NonStrictState, State);
1430 LIS->createAndComputeVirtRegInterval(SavedNonStrictReg);
1431 SavedNonStrictReg = 0;
1432 State = NonStrictState;
1435 if (Needs & StateStrict) {
1436 NonStrictState = State;
1437 assert(Needs == StateStrictWWM || Needs == StateStrictWQM);
1438 assert(!SavedNonStrictReg);
1439 SavedNonStrictReg =
MRI->createVirtualRegister(BoolRC);
1441 toStrictMode(
MBB,
Before, SavedNonStrictReg, Needs);
1445 if (State == StateWQM && (Needs & StateExact) && !(Needs & StateWQM)) {
1446 if (!WQMFromExec && (OutNeeds & StateWQM)) {
1448 SavedWQMReg =
MRI->createVirtualRegister(BoolRC);
1453 }
else if (State == StateExact && (Needs & StateWQM) &&
1454 !(Needs & StateExact)) {
1455 assert(WQMFromExec == (SavedWQMReg == 0));
1460 LIS->createAndComputeVirtRegInterval(SavedWQMReg);
1473 if (Needs != (StateExact | StateWQM | StateStrict)) {
1474 if (Needs != (StateExact | StateWQM))
1485 assert(!SavedNonStrictReg);
1488void SIWholeQuadMode::lowerLiveMaskQueries() {
1497 LIS->ReplaceMachineInstrInMaps(*
MI, *Copy);
1498 MI->eraseFromParent();
1502void SIWholeQuadMode::lowerCopyInstrs() {
1504 assert(
MI->getNumExplicitOperands() == 2);
1509 TRI->getRegClassForOperandReg(*
MRI,
MI->getOperand(0));
1510 if (
TRI->isVGPRClass(regClass)) {
1511 const unsigned MovOp =
TII->getMovOpcode(regClass);
1512 MI->setDesc(
TII->get(MovOp));
1517 return MO.isUse() && MO.getReg() == AMDGPU::EXEC;
1523 if (
MI->getOperand(0).isEarlyClobber()) {
1524 LIS->removeInterval(Reg);
1525 MI->getOperand(0).setIsEarlyClobber(
false);
1526 LIS->createAndComputeVirtRegInterval(Reg);
1528 int Index =
MI->findRegisterUseOperandIdx(AMDGPU::EXEC);
1529 while (
Index >= 0) {
1531 Index =
MI->findRegisterUseOperandIdx(AMDGPU::EXEC);
1533 MI->setDesc(
TII->get(AMDGPU::COPY));
1538 if (
MI->getOpcode() == AMDGPU::V_SET_INACTIVE_B32 ||
1539 MI->getOpcode() == AMDGPU::V_SET_INACTIVE_B64) {
1540 assert(
MI->getNumExplicitOperands() == 3);
1544 assert(
MI->getOperand(2).isUndef());
1545 MI->removeOperand(2);
1546 MI->untieRegOperand(1);
1548 assert(
MI->getNumExplicitOperands() == 2);
1551 unsigned CopyOp =
MI->getOperand(1).isReg()
1553 :
TII->getMovOpcode(
TRI->getRegClassForOperandReg(
1554 *
MRI,
MI->getOperand(0)));
1555 MI->setDesc(
TII->get(CopyOp));
1559void SIWholeQuadMode::lowerKillInstrs(
bool IsWQM) {
1563 switch (
MI->getOpcode()) {
1564 case AMDGPU::SI_DEMOTE_I1:
1565 case AMDGPU::SI_KILL_I1_TERMINATOR:
1566 SplitPoint = lowerKillI1(*
MBB, *
MI, IsWQM);
1568 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
1569 SplitPoint = lowerKillF32(*
MBB, *
MI);
1581 <<
" ------------- \n");
1586 LiveMaskQueries.clear();
1587 LowerToCopyInstrs.clear();
1588 LowerToMovInstrs.clear();
1590 StateTransition.clear();
1594 TII =
ST->getInstrInfo();
1595 TRI = &
TII->getRegisterInfo();
1597 LIS = &getAnalysis<LiveIntervals>();
1598 MDT = getAnalysisIfAvailable<MachineDominatorTree>();
1599 PDT = getAnalysisIfAvailable<MachinePostDominatorTree>();
1601 if (
ST->isWave32()) {
1602 AndOpc = AMDGPU::S_AND_B32;
1603 AndTermOpc = AMDGPU::S_AND_B32_term;
1604 AndN2Opc = AMDGPU::S_ANDN2_B32;
1605 XorOpc = AMDGPU::S_XOR_B32;
1606 AndSaveExecOpc = AMDGPU::S_AND_SAVEEXEC_B32;
1607 AndSaveExecTermOpc = AMDGPU::S_AND_SAVEEXEC_B32_term;
1608 WQMOpc = AMDGPU::S_WQM_B32;
1609 Exec = AMDGPU::EXEC_LO;
1611 AndOpc = AMDGPU::S_AND_B64;
1612 AndTermOpc = AMDGPU::S_AND_B64_term;
1613 AndN2Opc = AMDGPU::S_ANDN2_B64;
1614 XorOpc = AMDGPU::S_XOR_B64;
1615 AndSaveExecOpc = AMDGPU::S_AND_SAVEEXEC_B64;
1616 AndSaveExecTermOpc = AMDGPU::S_AND_SAVEEXEC_B64_term;
1617 WQMOpc = AMDGPU::S_WQM_B64;
1618 Exec = AMDGPU::EXEC;
1622 const bool NeedsLiveMask = !(KillInstrs.empty() && LiveMaskQueries.empty());
1627 if (!(GlobalFlags & (StateWQM | StateStrict)) && LowerToCopyInstrs.empty() &&
1628 LowerToMovInstrs.empty() && KillInstrs.empty()) {
1629 lowerLiveMaskQueries();
1630 return !LiveMaskQueries.empty();
1637 if (NeedsLiveMask || (GlobalFlags & StateWQM)) {
1638 LiveMaskReg =
MRI->createVirtualRegister(
TRI->getBoolRC());
1642 LIS->InsertMachineInstrInMaps(*
MI);
1647 lowerLiveMaskQueries();
1651 if (GlobalFlags == StateWQM) {
1654 LIS->InsertMachineInstrInMaps(*
MI);
1655 lowerKillInstrs(
true);
1658 processBlock(*BII.first, BII.first == &Entry);
1661 lowerBlock(*BII.first);
1665 if (LiveMaskReg != Exec)
1666 LIS->createAndComputeVirtRegInterval(LiveMaskReg);
1671 LIS->removeAllRegUnitsForPhysReg(AMDGPU::SCC);
1674 if (!KillInstrs.empty())
1675 LIS->removeAllRegUnitsForPhysReg(AMDGPU::EXEC);
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Provides AMDGPU specific target descriptions.
static void analyzeFunction(Function &Fn, const DataLayout &Layout, FunctionVarLocsBuilder *FnVarLocs)
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
DenseMap< Block *, BlockRelaxAux > Blocks
AMD GCN specific subclass of TargetSubtarget.
const HexagonInstrInfo * TII
unsigned const TargetRegisterInfo * TRI
This file implements a map that provides insertion order iteration.
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
static void splitBlock(MachineBasicBlock &MBB, MachineInstr &MI, MachineDominatorTree *MDT)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
This class represents an Operation in the Expression.
Core dominator tree base class.
FunctionPass class - This class is used to implement most global optimizations.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Result of a LiveRange query.
VNInfo * valueIn() const
Return the value that is live-in to the instruction.
This class represents the liveness of a register, stack slot, etc.
const Segment * getSegmentContaining(SlotIndex Idx) const
Return the segment that contains the specified index, or null if there is none.
LiveQueryResult Query(SlotIndex Idx) const
Query Liveness at Idx.
VNInfo * getVNInfoBefore(SlotIndex Idx) const
getVNInfoBefore - Return the VNInfo that is live up to but not necessarilly including Idx,...
static MCRegister from(unsigned Val)
Check the provided unsigned value is a valid MCRegister.
succ_iterator succ_begin()
MachineInstr * remove(MachineInstr *I)
Remove the unbundled instruction from the instruction list without deleting it.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
unsigned succ_size() const
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
pred_iterator pred_begin()
MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
iterator_range< succ_iterator > successors()
reverse_iterator rbegin()
iterator_range< pred_iterator > predecessors()
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
virtual MachineFunctionProperties getClearedProperties() const
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & set(Property P)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
void dump() const
dump - Print the current MachineFunction to cerr, useful for debugger use.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineBasicBlock & front() const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Register getReg() const
getReg - Returns the register number.
MachinePostDominatorTree - an analysis pass wrapper for DominatorTree used to compute the post-domina...
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This class implements a map that also provides access to all stored values in a deterministic order.
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
SlotIndex - An opaque wrapper around machine indexes.
SlotIndex getBaseIndex() const
Returns the base index for associated with this index.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
A Use represents the edge between a Value definition and its users.
VNInfo - Value Number Information.
LLVM Value Representation.
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
LLVM_READONLY int getVOPe32(uint16_t Opcode)
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
@ Define
Register definition.
Reg
All possible values of the reg field in the ModR/M byte.
NodeAddr< PhiNode * > Phi
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
FunctionPass * createSIWholeQuadModePass()
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
static constexpr LaneBitmask getAll()
constexpr bool any() const
static constexpr LaneBitmask getNone()
This represents a simple continuous liveness interval for a value.