63#define PASS_KEY "x86-slh"
64#define DEBUG_TYPE PASS_KEY
66STATISTIC(NumCondBranchesTraced,
"Number of conditional branches traced");
67STATISTIC(NumBranchesUntraced,
"Number of branches unable to trace");
69 "Number of address mode used registers hardaned");
71 "Number of post-load register values hardened");
73 "Number of calls or jumps requiring extra hardening");
74STATISTIC(NumInstsInserted,
"Number of instructions inserted");
75STATISTIC(NumLFENCEsInserted,
"Number of lfence instructions inserted");
78 "x86-speculative-load-hardening",
85 "Use LFENCE along each conditional edge to harden against speculative "
86 "loads rather than conditional movs and poisoned pointers."),
91 cl::desc(
"Harden the value loaded *after* it is loaded by "
92 "flushing the loaded bits to 1. This is hard to do "
93 "in general but can be done easily for GPRs."),
98 cl::desc(
"Use a full speculation fence to harden both call and ret edges "
99 "rather than a lighter weight mitigation."),
104 cl::desc(
"Harden interprocedurally by passing our state in and out of "
105 "functions in the high bits of the stack pointer."),
110 cl::desc(
"Sanitize loads from memory. When disable, no "
111 "significant security is provided."),
116 cl::desc(
"Harden indirect calls and jumps against using speculatively "
117 "stored attacker controlled addresses. This is designed to "
118 "mitigate Spectre v1.2 style attacks."),
128 return "X86 speculative load hardening";
139 struct BlockCondInfo {
151 unsigned InitialReg = 0;
152 unsigned PoisonReg = 0;
166 std::optional<PredState> PS;
191 const DebugLoc &Loc,
unsigned PredStateReg);
203 bool canHardenRegister(
Register Reg);
210 void hardenIndirectCallOrJumpInstr(
217char X86SpeculativeLoadHardeningPass::ID = 0;
219void X86SpeculativeLoadHardeningPass::getAnalysisUsage(
228 assert(!Succ.
isEHPad() &&
"Shouldn't get edges to EH pads!");
242 "Didn't start with the right target!");
252 "Without an unconditional branch, the old layout successor should "
253 "be an actual successor!");
257 UncondBr = &*BrBuilder;
268 "Cannot have a branchless successor and an unconditional branch!");
270 "A non-branch successor must have been a layout successor before "
271 "and now is a layout successor of the new block.");
277 if (SuccCount == 1) {
290 for (
int OpIdx = 1, NumOps =
MI.getNumOperands(); OpIdx < NumOps;
294 assert(OpMBB.
isMBB() &&
"Block operand to a PHI is not a block!");
299 if (SuccCount == 1) {
305 MI.addOperand(MF, OpV);
312 for (
auto &LI : Succ.
liveins())
332 for (
auto &
MI :
MBB) {
339 for (
int OpIdx = 1, NumOps =
MI.getNumOperands(); OpIdx < NumOps;
341 if (!Preds.
insert(
MI.getOperand(OpIdx + 1).getMBB()).second)
355 while (!DupIndices.
empty()) {
359 MI.removeOperand(OpIdx + 1);
360 MI.removeOperand(OpIdx);
378 if (
MI.getOpcode() == X86::LFENCE)
386 if (
MI.getOpcode() == X86::MFENCE)
398bool X86SpeculativeLoadHardeningPass::runOnMachineFunction(
411 TII = Subtarget->getInstrInfo();
412 TRI = Subtarget->getRegisterInfo();
415 PS.emplace(MF, &X86::GR64_NOSPRegClass);
423 hardenEdgesWithLFENCE(MF);
431 auto EntryInsertPt = Entry.SkipPHIsLabelsAndDebug(Entry.begin());
441 if (!HasVulnerableLoad && Infos.
empty())
446 const int PoisonVal = -1;
447 PS->PoisonReg =
MRI->createVirtualRegister(PS->RC);
448 BuildMI(Entry, EntryInsertPt, Loc,
TII->get(X86::MOV64ri32), PS->PoisonReg)
461 BuildMI(Entry, EntryInsertPt, Loc,
TII->get(X86::LFENCE));
463 ++NumLFENCEsInserted;
476 PS->InitialReg = extractPredStateFromSP(Entry, EntryInsertPt, Loc);
480 PS->InitialReg =
MRI->createVirtualRegister(PS->RC);
481 Register PredStateSubReg =
MRI->createVirtualRegister(&X86::GR32RegClass);
482 auto ZeroI =
BuildMI(Entry, EntryInsertPt, Loc,
TII->get(X86::MOV32r0),
486 ZeroI->findRegisterDefOperand(X86::EFLAGS);
488 "Must have an implicit def of EFLAGS!");
490 BuildMI(Entry, EntryInsertPt, Loc,
TII->get(X86::SUBREG_TO_REG),
504 PS->SSA.Initialize(PS->InitialReg);
505 PS->SSA.AddAvailableValue(&Entry, PS->InitialReg);
508 auto CMovs = tracePredStateThroughCFG(MF, Infos);
524 PS->SSA.AddAvailableValue(
533 unfoldCallAndJumpLoads(MF);
536 auto IndirectBrCMovs = tracePredStateThroughIndirectBranches(MF);
537 CMovs.append(IndirectBrCMovs.begin(), IndirectBrCMovs.end());
543 tracePredStateThroughBlocksAndHarden(MF);
549 if (!
Op.isReg() ||
Op.getReg() != PS->InitialReg)
552 PS->SSA.RewriteUse(
Op);
555 LLVM_DEBUG(
dbgs() <<
"Final speculative load hardened function:\n"; MF.dump();
556 dbgs() <<
"\n"; MF.verify(
this));
566void X86SpeculativeLoadHardeningPass::hardenEdgesWithLFENCE(
579 if (TermIt ==
MBB.
end() || !TermIt->isBranch())
586 if (!SuccMBB->isEHPad())
594 ++NumLFENCEsInserted;
599X86SpeculativeLoadHardeningPass::collectBlockCondInfo(
MachineFunction &MF) {
629 BlockCondInfo
Info = {&
MBB, {},
nullptr};
635 if (!
MI.isTerminator())
639 if (!
MI.isBranch()) {
640 Info.CondBrs.clear();
646 if (
MI.getOpcode() == X86::JMP_1) {
647 Info.CondBrs.clear();
662 Info.CondBrs.clear();
668 Info.CondBrs.push_back(&
MI);
670 if (
Info.CondBrs.empty()) {
671 ++NumBranchesUntraced;
672 LLVM_DEBUG(
dbgs() <<
"WARNING: unable to secure successors of block:\n";
691X86SpeculativeLoadHardeningPass::tracePredStateThroughCFG(
699 for (
const BlockCondInfo &Info : Infos) {
706 ++NumCondBranchesTraced;
711 UncondBr ? (UncondBr->
getOpcode() == X86::JMP_1
719 ++SuccCounts[UncondSucc];
720 for (
auto *CondBr : CondBrs)
721 ++SuccCounts[CondBr->getOperand(0).getMBB()];
725 auto BuildCheckingBlockForSuccAndConds =
732 (SuccCount == 1 && Succ.pred_size() == 1)
736 bool LiveEFLAGS = Succ.
isLiveIn(X86::EFLAGS);
738 CheckingMBB.addLiveIn(X86::EFLAGS);
741 auto InsertPt = CheckingMBB.begin();
742 assert((InsertPt == CheckingMBB.end() || !InsertPt->isPHI()) &&
743 "Should never have a PHI in the initial checking block as it "
744 "always has a single predecessor!");
748 unsigned CurStateReg = PS->InitialReg;
751 int PredStateSizeInBytes =
TRI->getRegSizeInBits(*PS->RC) / 8;
754 Register UpdatedStateReg =
MRI->createVirtualRegister(PS->RC);
758 TII->get(CMovOp), UpdatedStateReg)
764 if (!LiveEFLAGS &&
Cond == Conds.back())
773 if (CurStateReg == PS->InitialReg)
777 CurStateReg = UpdatedStateReg;
782 PS->SSA.AddAvailableValue(&CheckingMBB, CurStateReg);
785 std::vector<X86::CondCode> UncondCodeSeq;
786 for (
auto *CondBr : CondBrs) {
788 int &SuccCount = SuccCounts[&Succ];
792 UncondCodeSeq.push_back(
Cond);
794 BuildCheckingBlockForSuccAndConds(
MBB, Succ, SuccCount, CondBr, UncondBr,
817 assert(SuccCounts[UncondSucc] == 1 &&
818 "We should never have more than one edge to the unconditional "
819 "successor at this point because every other edge must have been "
824 UncondCodeSeq.erase(std::unique(UncondCodeSeq.begin(), UncondCodeSeq.end()),
825 UncondCodeSeq.end());
828 BuildCheckingBlockForSuccAndConds(
MBB, *UncondSucc, 1,
829 UncondBr, UncondBr, UncondCodeSeq);
844 unsigned UnfoldedOpc =
TII.getOpcodeAfterMemoryUnfold(
845 Opcode,
true,
false, &
Index);
847 return TII.getRegClass(MCID,
Index, &
TII.getRegisterInfo(), MF);
850void X86SpeculativeLoadHardeningPass::unfoldCallAndJumpLoads(
857 if (!
MI.isCall() && !
MI.isBranch())
863 switch (
MI.getOpcode()) {
866 dbgs() <<
"ERROR: Found an unexpected loading branch or call "
872 case X86::FARCALL16m:
873 case X86::FARCALL32m:
874 case X86::FARCALL64m:
883 case X86::CALL16m_NT:
885 case X86::CALL32m_NT:
887 case X86::CALL64m_NT:
894 case X86::TAILJMPm64:
895 case X86::TAILJMPm64_REX:
897 case X86::TCRETURNmi64:
898 case X86::TCRETURNmi: {
905 <<
"ERROR: Unable to unfold load from instruction:\n";
914 TII->unfoldMemoryOperand(MF,
MI, Reg,
true,
918 "Computed unfolded register class but failed to unfold");
920 for (
auto *NewMI : NewMIs)
924 if (
MI.isCandidateForCallSiteEntry())
925 MF.eraseCallSiteInfo(&
MI);
927 MI.eraseFromParent();
929 dbgs() <<
"Unfolded load successfully into:\n";
930 for (
auto *NewMI : NewMIs) {
960X86SpeculativeLoadHardeningPass::tracePredStateThroughIndirectBranches(
967 TargetAddrSSA.Initialize(
MRI->createVirtualRegister(&X86::GR64RegClass));
1000 case X86::FARJMP64m:
1006 case X86::JMP16m_NT:
1008 case X86::JMP32m_NT:
1010 case X86::JMP64m_NT:
1016 "Support for 16-bit indirect branches is not implemented.");
1019 "Support for 32-bit indirect branches is not implemented.");
1028 return !OtherTI.isDebugInstr() && &OtherTI != &TI;
1031 dbgs() <<
"ERROR: Found other terminators in a block with an indirect "
1032 "branch! This is not yet supported! Terminator sequence:\n";
1042 TargetAddrSSA.AddAvailableValue(&
MBB, TargetReg);
1047 IndirectTargetMBBs.
insert(Succ);
1054 if (IndirectTargetMBBs.
empty())
1062 if (!IndirectTargetMBBs.
count(&
MBB))
1069 "Unexpected EH pad as target of an indirect branch!");
1077 "Cannot check within a block that already has live-in EFLAGS!");
1084 if (IndirectTerminatedMBBs.
count(Pred))
1093 return Succ->isEHPad() || Succ == &MBB;
1096 dbgs() <<
"ERROR: Found conditional entry to target of indirect "
1102 "an indirect branch!");
1108 auto InsertPt = Pred->getFirstTerminator();
1109 Register TargetReg =
MRI->createVirtualRegister(&X86::GR64RegClass);
1111 !Subtarget->isPositionIndependent()) {
1114 TII->get(X86::MOV64ri32), TargetReg)
1134 TargetAddrSSA.AddAvailableValue(Pred, TargetReg);
1142 Register TargetReg = TargetAddrSSA.GetValueInMiddleOfBlock(&
MBB);
1150 !Subtarget->isPositionIndependent()) {
1160 Register AddrReg =
MRI->createVirtualRegister(&X86::GR64RegClass);
1180 int PredStateSizeInBytes =
TRI->getRegSizeInBits(*PS->RC) / 8;
1182 Register UpdatedStateReg =
MRI->createVirtualRegister(PS->RC);
1195 PS->SSA.AddAvailableValue(&
MBB, UpdatedStateReg);
1205 if (
const MachineOperand *DefOp =
MI.findRegisterDefOperand(X86::EFLAGS)) {
1206 return !DefOp->isDead();
1218 if (DefOp->isDead())
1226 if (
MI.killsRegister(X86::EFLAGS, &
TRI))
1262void X86SpeculativeLoadHardeningPass::tracePredStateThroughBlocksAndHarden(
1299 return Op.isReg() && LoadDepRegs.test(Op.getReg());
1303 LoadDepRegs.
set(
Def.getReg());
1308 if (
MI.getOpcode() == X86::LFENCE)
1316 if (
MI.getOpcode() == X86::MFENCE)
1321 if (MemRefBeginIdx < 0) {
1323 <<
"WARNING: unable to harden loading instruction: ";
1335 unsigned BaseReg = 0, IndexReg = 0;
1336 if (!BaseMO.
isFI() && BaseMO.
getReg() != X86::RIP &&
1337 BaseMO.
getReg() != X86::NoRegister)
1338 BaseReg = BaseMO.
getReg();
1339 if (IndexMO.
getReg() != X86::NoRegister)
1340 IndexReg = IndexMO.
getReg();
1342 if (!BaseReg && !IndexReg)
1350 if ((BaseReg && LoadDepRegs.
test(BaseReg)) ||
1351 (IndexReg && LoadDepRegs.
test(IndexReg)))
1361 MI.getOperand(0).isReg() &&
1362 canHardenRegister(
MI.getOperand(0).getReg()) &&
1363 !HardenedAddrRegs.
count(BaseReg) &&
1364 !HardenedAddrRegs.
count(IndexReg)) {
1366 HardenedAddrRegs.
insert(
MI.getOperand(0).getReg());
1374 HardenedAddrRegs.
insert(BaseReg);
1376 HardenedAddrRegs.
insert(IndexReg);
1380 LoadDepRegs.
set(
Def.getReg());
1392 "Requested to harden both the address and def of a load!");
1395 if (HardenLoadAddr.
erase(&
MI)) {
1397 assert(MemRefBeginIdx >= 0 &&
"Cannot have an invalid index here!");
1403 hardenLoadAddr(
MI, BaseMO, IndexMO, AddrRegToHardenedReg);
1409 if (HardenPostLoad.
erase(&
MI)) {
1410 assert(!
MI.isCall() &&
"Must not try to post-load harden a call!");
1418 MachineInstr *SunkMI = sinkPostLoadHardenedInst(
MI, HardenPostLoad);
1423 if (SunkMI != &
MI) {
1430 HardenPostLoad.
insert(SunkMI);
1435 unsigned HardenedReg = hardenPostLoad(
MI);
1438 AddrRegToHardenedReg[HardenedReg] = HardenedReg;
1449 hardenIndirectCallOrJumpInstr(
MI, AddrRegToHardenedReg);
1456 if (!
MI.isCall() && !
MI.isReturn())
1461 if (
MI.isReturn() && !
MI.isCall()) {
1462 hardenReturnInstr(
MI);
1469 assert(
MI.isCall() &&
"Should only reach here for calls!");
1470 tracePredStateThroughCall(
MI);
1473 HardenPostLoad.
clear();
1474 HardenLoadAddr.
clear();
1475 HardenedAddrRegs.
clear();
1476 AddrRegToHardenedReg.
clear();
1481 LoadDepRegs.
clear();
1491unsigned X86SpeculativeLoadHardeningPass::saveEFLAGS(
1509void X86SpeculativeLoadHardeningPass::restoreEFLAGS(
1520void X86SpeculativeLoadHardeningPass::mergePredStateIntoSP(
1522 const DebugLoc &Loc,
unsigned PredStateReg) {
1523 Register TmpReg =
MRI->createVirtualRegister(PS->RC);
1527 auto ShiftI =
BuildMI(
MBB, InsertPt, Loc,
TII->get(X86::SHL64ri), TmpReg)
1532 auto OrI =
BuildMI(
MBB, InsertPt, Loc,
TII->get(X86::OR64rr), X86::RSP)
1540unsigned X86SpeculativeLoadHardeningPass::extractPredStateFromSP(
1543 Register PredStateReg =
MRI->createVirtualRegister(PS->RC);
1544 Register TmpReg =
MRI->createVirtualRegister(PS->RC);
1549 BuildMI(
MBB, InsertPt, Loc,
TII->get(TargetOpcode::COPY), TmpReg)
1552 BuildMI(
MBB, InsertPt, Loc,
TII->get(X86::SAR64ri), PredStateReg)
1554 .
addImm(
TRI->getRegSizeInBits(*PS->RC) - 1);
1558 return PredStateReg;
1561void X86SpeculativeLoadHardeningPass::hardenLoadAddr(
1573 if (BaseMO.
isFI()) {
1577 dbgs() <<
" Skipping hardening base of explicit stack frame load: ";
1579 }
else if (BaseMO.
getReg() == X86::RSP) {
1584 "Explicit RSP access with dynamic index!");
1586 dbgs() <<
" Cannot harden base of explicit RSP offset in a load!");
1587 }
else if (BaseMO.
getReg() == X86::RIP ||
1588 BaseMO.
getReg() == X86::NoRegister) {
1598 dbgs() <<
" Cannot harden base of "
1599 << (BaseMO.
getReg() == X86::RIP ?
"RIP-relative" :
"no-base")
1600 <<
" address in a load!");
1603 "Only allowed to have a frame index or register base.");
1607 if (IndexMO.
getReg() != X86::NoRegister &&
1608 (HardenOpRegs.
empty() ||
1609 HardenOpRegs.
front()->getReg() != IndexMO.
getReg()))
1613 "Should have exactly one or two registers to harden!");
1615 HardenOpRegs[0]->getReg() != HardenOpRegs[1]->getReg()) &&
1616 "Should not have two of the same registers!");
1621 auto It = AddrRegToHardenedReg.
find(
Op->getReg());
1622 if (It == AddrRegToHardenedReg.
end())
1627 Op->setReg(It->second);
1631 if (HardenOpRegs.
empty())
1635 Register StateReg = PS->SSA.GetValueAtEndOfBlock(&
MBB);
1637 auto InsertPt =
MI.getIterator();
1642 unsigned FlagsReg = 0;
1643 if (EFLAGSLive && !Subtarget->hasBMI2()) {
1645 FlagsReg = saveEFLAGS(
MBB, InsertPt, Loc);
1650 auto *OpRC =
MRI->getRegClass(OpReg);
1651 Register TmpReg =
MRI->createVirtualRegister(OpRC);
1655 if (!Subtarget->hasVLX() && (OpRC->hasSuperClassEq(&X86::VR128RegClass) ||
1656 OpRC->hasSuperClassEq(&X86::VR256RegClass))) {
1657 assert(Subtarget->hasAVX2() &&
"AVX2-specific register classes!");
1658 bool Is128Bit = OpRC->hasSuperClassEq(&X86::VR128RegClass);
1663 Register VStateReg =
MRI->createVirtualRegister(&X86::VR128RegClass);
1665 BuildMI(
MBB, InsertPt, Loc,
TII->get(X86::VMOV64toPQIrr), VStateReg)
1672 Register VBStateReg =
MRI->createVirtualRegister(OpRC);
1673 auto BroadcastI =
BuildMI(
MBB, InsertPt, Loc,
1674 TII->get(Is128Bit ? X86::VPBROADCASTQrr
1675 : X86::VPBROADCASTQYrr),
1680 LLVM_DEBUG(
dbgs() <<
" Inserting broadcast: "; BroadcastI->dump();
1686 TII->get(Is128Bit ? X86::VPORrr : X86::VPORYrr), TmpReg)
1692 }
else if (OpRC->hasSuperClassEq(&X86::VR128XRegClass) ||
1693 OpRC->hasSuperClassEq(&X86::VR256XRegClass) ||
1694 OpRC->hasSuperClassEq(&X86::VR512RegClass)) {
1695 assert(Subtarget->hasAVX512() &&
"AVX512-specific register classes!");
1696 bool Is128Bit = OpRC->hasSuperClassEq(&X86::VR128XRegClass);
1697 bool Is256Bit = OpRC->hasSuperClassEq(&X86::VR256XRegClass);
1698 if (Is128Bit || Is256Bit)
1699 assert(Subtarget->hasVLX() &&
"AVX512VL-specific register classes!");
1702 Register VStateReg =
MRI->createVirtualRegister(OpRC);
1703 unsigned BroadcastOp =
Is128Bit ? X86::VPBROADCASTQrZ128rr
1704 : Is256Bit ? X86::VPBROADCASTQrZ256rr
1705 : X86::VPBROADCASTQrZrr;
1707 BuildMI(
MBB, InsertPt, Loc,
TII->get(BroadcastOp), VStateReg)
1711 LLVM_DEBUG(
dbgs() <<
" Inserting broadcast: "; BroadcastI->dump();
1715 unsigned OrOp =
Is128Bit ? X86::VPORQZ128rr
1716 : Is256Bit ? X86::VPORQZ256rr : X86::VPORQZrr;
1717 auto OrI =
BuildMI(
MBB, InsertPt, Loc,
TII->get(OrOp), TmpReg)
1725 assert(OpRC->hasSuperClassEq(&X86::GR64RegClass) &&
1726 "Not a supported register class for address hardening!");
1730 auto OrI =
BuildMI(
MBB, InsertPt, Loc,
TII->get(X86::OR64rr), TmpReg)
1740 BuildMI(
MBB, InsertPt, Loc,
TII->get(X86::SHRX64rr), TmpReg)
1752 "Should not have checked this register yet!");
1753 AddrRegToHardenedReg[
Op->getReg()] = TmpReg;
1755 ++NumAddrRegsHardened;
1760 restoreEFLAGS(
MBB, InsertPt, Loc, FlagsReg);
1763MachineInstr *X86SpeculativeLoadHardeningPass::sinkPostLoadHardenedInst(
1766 "Cannot get here with a non-invariant load!");
1768 "Cannot get here with a data invariant load "
1769 "that interferes with EFLAGS!");
1772 auto SinkCheckToSingleUse =
1789 "Data variant instruction being hardened!");
1796 assert(MemRefBeginIdx >= 0 &&
1797 "Should always have mem references here!");
1803 if ((BaseMO.
isReg() && BaseMO.
getReg() == DefReg) ||
1824 if (
UseMI.getDesc().getNumDefs() > 1)
1831 if (!canHardenRegister(UseDefReg))
1834 SingleUseMI = &
UseMI;
1839 return {SingleUseMI};
1843 while (std::optional<MachineInstr *> SingleUse = SinkCheckToSingleUse(*
MI)) {
1853bool X86SpeculativeLoadHardeningPass::canHardenRegister(
Register Reg) {
1855 if (!
Reg.isVirtual())
1858 auto *RC =
MRI->getRegClass(Reg);
1859 int RegBytes =
TRI->getRegSizeInBits(*RC) / 8;
1864 unsigned RegIdx =
Log2_32(RegBytes);
1865 assert(RegIdx < 4 &&
"Unsupported register size");
1875 &X86::GR8_NOREXRegClass, &X86::GR16_NOREXRegClass,
1876 &X86::GR32_NOREXRegClass, &X86::GR64_NOREXRegClass};
1877 if (RC == NOREXRegClasses[RegIdx])
1881 &X86::GR8RegClass, &X86::GR16RegClass, &X86::GR32RegClass,
1882 &X86::GR64RegClass};
1900unsigned X86SpeculativeLoadHardeningPass::hardenValueInRegister(
1903 assert(canHardenRegister(Reg) &&
"Cannot harden this register!");
1905 auto *RC =
MRI->getRegClass(Reg);
1906 int Bytes =
TRI->getRegSizeInBits(*RC) / 8;
1907 Register StateReg = PS->SSA.GetValueAtEndOfBlock(&
MBB);
1908 assert((Bytes == 1 || Bytes == 2 || Bytes == 4 || Bytes == 8) &&
1909 "Unknown register size");
1913 unsigned SubRegImms[] = {X86::sub_8bit, X86::sub_16bit, X86::sub_32bit};
1914 unsigned SubRegImm = SubRegImms[
Log2_32(Bytes)];
1915 Register NarrowStateReg =
MRI->createVirtualRegister(RC);
1916 BuildMI(
MBB, InsertPt, Loc,
TII->get(TargetOpcode::COPY), NarrowStateReg)
1917 .
addReg(StateReg, 0, SubRegImm);
1918 StateReg = NarrowStateReg;
1921 unsigned FlagsReg = 0;
1923 FlagsReg = saveEFLAGS(
MBB, InsertPt, Loc);
1926 unsigned OrOpCodes[] = {X86::OR8rr, X86::OR16rr, X86::OR32rr, X86::OR64rr};
1927 unsigned OrOpCode = OrOpCodes[
Log2_32(Bytes)];
1928 auto OrI =
BuildMI(
MBB, InsertPt, Loc,
TII->get(OrOpCode), NewReg)
1936 restoreEFLAGS(
MBB, InsertPt, Loc, FlagsReg);
1950unsigned X86SpeculativeLoadHardeningPass::hardenPostLoad(
MachineInstr &
MI) {
1954 auto &DefOp =
MI.getOperand(0);
1955 Register OldDefReg = DefOp.getReg();
1956 auto *DefRC =
MRI->getRegClass(OldDefReg);
1961 Register UnhardenedReg =
MRI->createVirtualRegister(DefRC);
1962 DefOp.setReg(UnhardenedReg);
1967 unsigned HardenedReg = hardenValueInRegister(
1968 UnhardenedReg,
MBB, std::next(
MI.getIterator()), Loc);
1972 MRI->replaceRegWith( OldDefReg, HardenedReg);
1974 ++NumPostLoadRegsHardened;
2001void X86SpeculativeLoadHardeningPass::hardenReturnInstr(
MachineInstr &
MI) {
2004 auto InsertPt =
MI.getIterator();
2014 mergePredStateIntoSP(
MBB, InsertPt, Loc, PS->SSA.GetValueAtEndOfBlock(&
MBB));
2047void X86SpeculativeLoadHardeningPass::tracePredStateThroughCall(
2051 auto InsertPt =
MI.getIterator();
2064 BuildMI(
MBB, std::next(InsertPt), Loc,
TII->get(X86::LFENCE));
2066 ++NumLFENCEsInserted;
2072 Register StateReg = PS->SSA.GetValueAtEndOfBlock(&
MBB);
2073 mergePredStateIntoSP(
MBB, InsertPt, Loc, StateReg);
2088 MI.setPostInstrSymbol(MF, RetSymbol);
2091 unsigned ExpectedRetAddrReg = 0;
2096 if (!Subtarget->getFrameLowering()->has128ByteRedZone(MF) ||
2116 ExpectedRetAddrReg =
MRI->createVirtualRegister(AddrRC);
2118 !Subtarget->isPositionIndependent()) {
2119 BuildMI(
MBB, InsertPt, Loc,
TII->get(X86::MOV64ri32), ExpectedRetAddrReg)
2122 BuildMI(
MBB, InsertPt, Loc,
TII->get(X86::LEA64r), ExpectedRetAddrReg)
2138 if (!ExpectedRetAddrReg) {
2139 ExpectedRetAddrReg =
MRI->createVirtualRegister(AddrRC);
2140 BuildMI(
MBB, InsertPt, Loc,
TII->get(X86::MOV64rm), ExpectedRetAddrReg)
2150 unsigned NewStateReg = extractPredStateFromSP(
MBB, InsertPt, Loc);
2156 !Subtarget->isPositionIndependent()) {
2163 Register ActualRetAddrReg =
MRI->createVirtualRegister(AddrRC);
2164 BuildMI(
MBB, InsertPt, Loc,
TII->get(X86::LEA64r), ActualRetAddrReg)
2177 int PredStateSizeInBytes =
TRI->getRegSizeInBits(*PS->RC) / 8;
2180 Register UpdatedStateReg =
MRI->createVirtualRegister(PS->RC);
2181 auto CMovI =
BuildMI(
MBB, InsertPt, Loc,
TII->get(CMovOp), UpdatedStateReg)
2189 PS->SSA.AddAvailableValue(&
MBB, UpdatedStateReg);
2207void X86SpeculativeLoadHardeningPass::hardenIndirectCallOrJumpInstr(
2210 switch (
MI.getOpcode()) {
2211 case X86::FARCALL16m:
2212 case X86::FARCALL32m:
2213 case X86::FARCALL64m:
2214 case X86::FARJMP16m:
2215 case X86::FARJMP32m:
2216 case X86::FARJMP64m:
2227 assert(!
MI.mayLoad() &&
"Found a lingering loading instruction!");
2231 if (!
MI.getOperand(0).isReg())
2236 auto &TargetOp =
MI.getOperand(0);
2237 Register OldTargetReg = TargetOp.getReg();
2242 unsigned &HardenedTargetReg = AddrRegToHardenedReg[OldTargetReg];
2251 if (!HardenedTargetReg)
2252 HardenedTargetReg = hardenValueInRegister(
2253 OldTargetReg, *
MI.getParent(),
MI.getIterator(),
MI.getDebugLoc());
2256 TargetOp.setReg(HardenedTargetReg);
2258 ++NumCallsOrJumpsHardened;
2262 "X86 speculative load hardener",
false,
false)
2267 return new X86SpeculativeLoadHardeningPass();
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
AMDGPU Mark last scratch load
Analysis containing CSE Info
This file defines the DenseMap class.
DenseMap< Block *, BlockRelaxAux > Blocks
const HexagonInstrInfo * TII
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
unsigned const TargetRegisterInfo * TRI
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file defines the SparseBitVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static MachineBasicBlock & splitEdge(MachineBasicBlock &MBB, MachineBasicBlock &Succ, int SuccCount, MachineInstr *Br, MachineInstr *&UncondBr, const X86InstrInfo &TII)
static cl::opt< bool > HardenLoads(PASS_KEY "-loads", cl::desc("Sanitize loads from memory. When disable, no " "significant security is provided."), cl::init(true), cl::Hidden)
static void canonicalizePHIOperands(MachineFunction &MF)
Removing duplicate PHI operands to leave the PHI in a canonical and predictable form.
static cl::opt< bool > HardenInterprocedurally(PASS_KEY "-ip", cl::desc("Harden interprocedurally by passing our state in and out of " "functions in the high bits of the stack pointer."), cl::init(true), cl::Hidden)
static cl::opt< bool > FenceCallAndRet(PASS_KEY "-fence-call-and-ret", cl::desc("Use a full speculation fence to harden both call and ret edges " "rather than a lighter weight mitigation."), cl::init(false), cl::Hidden)
static cl::opt< bool > EnablePostLoadHardening(PASS_KEY "-post-load", cl::desc("Harden the value loaded *after* it is loaded by " "flushing the loaded bits to 1. This is hard to do " "in general but can be done easily for GPRs."), cl::init(true), cl::Hidden)
static cl::opt< bool > HardenEdgesWithLFENCE(PASS_KEY "-lfence", cl::desc("Use LFENCE along each conditional edge to harden against speculative " "loads rather than conditional movs and poisoned pointers."), cl::init(false), cl::Hidden)
static bool isEFLAGSLive(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const TargetRegisterInfo &TRI)
static cl::opt< bool > EnableSpeculativeLoadHardening("x86-speculative-load-hardening", cl::desc("Force enable speculative load hardening"), cl::init(false), cl::Hidden)
static const TargetRegisterClass * getRegClassForUnfoldedLoad(MachineFunction &MF, const X86InstrInfo &TII, unsigned Opcode)
Compute the register class for the unfolded load.
static bool hasVulnerableLoad(MachineFunction &MF)
Helper to scan a function for loads vulnerable to misspeculation that we want to harden.
static bool isEFLAGSDefLive(const MachineInstr &MI)
X86 speculative load hardener
static cl::opt< bool > HardenIndirectCallsAndJumps(PASS_KEY "-indirect", cl::desc("Harden indirect calls and jumps against using speculatively " "stored attacker controlled addresses. This is designed to " "mitigate Spectre v1.2 style attacks."), cl::init(true), cl::Hidden)
Represent the analysis usage information of a pass.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
This class represents an Operation in the Expression.
iterator find(const_arg_type_t< KeyT > Val)
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
FunctionPass class - This class is used to implement most global optimizations.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
Insert branch code into the end of the specified MachineBasicBlock.
MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
Describe properties that are true of each instruction in the target description file.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
bool isEHPad() const
Returns true if the block is a landing pad.
void replaceSuccessor(MachineBasicBlock *Old, MachineBasicBlock *New)
Replace successor OLD with NEW and update probability info.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
iterator_range< livein_iterator > liveins() const
reverse_instr_iterator instr_rbegin()
iterator SkipPHIsAndLabels(iterator I)
Return the first instruction in MBB after I that is not a PHI or a label.
void splitSuccessor(MachineBasicBlock *Old, MachineBasicBlock *New, bool NormalizeSuccProbs=false)
Split the old successor into old plus new and updates the probability info.
iterator SkipPHIsLabelsAndDebug(iterator I, Register Reg=Register(), bool SkipPseudoOp=true)
Return the first instruction in MBB after I that is not a PHI, label or debug.
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
bool isEHFuncletEntry() const
Returns true if this is the entry block of an EH funclet.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
unsigned succ_size() const
bool isEHScopeEntry() const
Returns true if this is the entry block of an EH scope, i.e., the block that used to have a catchpad ...
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
reverse_instr_iterator instr_rend()
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator_range< iterator > terminators()
iterator_range< succ_iterator > successors()
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
iterator_range< pred_iterator > predecessors()
StringRef getName() const
Return the name of the corresponding LLVM basic block, or an empty string.
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
bool isCleanupFuncletEntry() const
Returns true if this is the entry block of a cleanup funclet.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
bool exposesReturnsTwice() const
exposesReturnsTwice - Returns true if the function calls setjmp or any other similar functions with a...
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool isTerminator(QueryType Type=AnyInBundle) const
Returns true if this instruction part of the terminator for a basic block.
bool isBranch(QueryType Type=AnyInBundle) const
Returns true if this is a conditional, unconditional, or indirect branch.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
const MachineOperand & getOperand(unsigned i) const
MachineOperand * findRegisterUseOperand(Register Reg, bool isKill=false, const TargetRegisterInfo *TRI=nullptr)
Wrapper for findRegisterUseOperandIdx, it returns a pointer to the MachineOperand rather than an inde...
bool addRegisterDead(Register Reg, const TargetRegisterInfo *RegInfo, bool AddIfNotFound=false)
We have determined MI defined a register without a use.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
void setIsDead(bool Val=true)
void setIsKill(bool Val=true)
void setMBB(MachineBasicBlock *MBB)
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
static MachineOperand CreateMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0)
bool isMBB() const
isMBB - Tests if this is a MO_MachineBasicBlock operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
MachineSSAUpdater - This class updates SSA form for a set of virtual registers defined in multiple bl...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
bool test(unsigned Idx) const
StringRef - Represent a constant reference to a string, i.e.
CodeModel::Model getCodeModel() const
Returns the code model.
bool hasSuperClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a super-class of or equal to this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
static bool isDataInvariantLoad(MachineInstr &MI)
Returns true if the instruction has no behavior (specified or otherwise) that is based on the value l...
static bool isDataInvariant(MachineInstr &MI)
Returns true if the instruction has no behavior (specified or otherwise) that is based on the value o...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Kill
The last use of a register.
Reg
All possible values of the reg field in the ModR/M byte.
CondCode getCondFromBranch(const MachineInstr &MI)
int getFirstAddrOperandIdx(const MachineInstr &MI)
Return the index of the instruction's first address operand, if it has a memory reference,...
CondCode GetOppositeBranchCondition(CondCode CC)
GetOppositeBranchCondition - Return the inverse of the specified cond, e.g.
unsigned getCMovOpcode(unsigned RegBytes, bool HasMemoryOperand=false, bool HasNDD=false)
Return a cmov opcode for the given register size in bytes, and operand type.
initializer< Ty > init(const Ty &Val)
NodeAddr< DefNode * > Def
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
FunctionPass * createX86SpeculativeLoadHardeningPass()
auto reverse(ContainerTy &&C)
void sort(IteratorTy Start, IteratorTy End)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...