63#define PASS_KEY "x86-slh"
64#define DEBUG_TYPE PASS_KEY
66STATISTIC(NumCondBranchesTraced,
"Number of conditional branches traced");
67STATISTIC(NumBranchesUntraced,
"Number of branches unable to trace");
69 "Number of address mode used registers hardaned");
71 "Number of post-load register values hardened");
73 "Number of calls or jumps requiring extra hardening");
74STATISTIC(NumInstsInserted,
"Number of instructions inserted");
75STATISTIC(NumLFENCEsInserted,
"Number of lfence instructions inserted");
78 "x86-speculative-load-hardening",
85 "Use LFENCE along each conditional edge to harden against speculative "
86 "loads rather than conditional movs and poisoned pointers."),
91 cl::desc(
"Harden the value loaded *after* it is loaded by "
92 "flushing the loaded bits to 1. This is hard to do "
93 "in general but can be done easily for GPRs."),
98 cl::desc(
"Use a full speculation fence to harden both call and ret edges "
99 "rather than a lighter weight mitigation."),
104 cl::desc(
"Harden interprocedurally by passing our state in and out of "
105 "functions in the high bits of the stack pointer."),
110 cl::desc(
"Sanitize loads from memory. When disable, no "
111 "significant security is provided."),
116 cl::desc(
"Harden indirect calls and jumps against using speculatively "
117 "stored attacker controlled addresses. This is designed to "
118 "mitigate Spectre v1.2 style attacks."),
128 return "X86 speculative load hardening";
139 struct BlockCondInfo {
151 unsigned InitialReg = 0;
152 unsigned PoisonReg = 0;
166 std::optional<PredState> PS;
191 const DebugLoc &Loc,
unsigned PredStateReg);
203 bool canHardenRegister(
Register Reg);
210 void hardenIndirectCallOrJumpInstr(
217char X86SpeculativeLoadHardeningPass::ID = 0;
219void X86SpeculativeLoadHardeningPass::getAnalysisUsage(
228 assert(!Succ.
isEHPad() &&
"Shouldn't get edges to EH pads!");
242 "Didn't start with the right target!");
252 "Without an unconditional branch, the old layout successor should "
253 "be an actual successor!");
257 UncondBr = &*BrBuilder;
268 "Cannot have a branchless successor and an unconditional branch!");
270 "A non-branch successor must have been a layout successor before "
271 "and now is a layout successor of the new block.");
277 if (SuccCount == 1) {
290 for (
int OpIdx = 1, NumOps =
MI.getNumOperands(); OpIdx < NumOps;
294 assert(OpMBB.
isMBB() &&
"Block operand to a PHI is not a block!");
299 if (SuccCount == 1) {
305 MI.addOperand(MF, OpV);
312 for (
auto &LI : Succ.
liveins())
332 for (
auto &
MI :
MBB) {
339 for (
int OpIdx = 1, NumOps =
MI.getNumOperands(); OpIdx < NumOps;
341 if (!Preds.
insert(
MI.getOperand(OpIdx + 1).getMBB()).second)
355 while (!DupIndices.
empty()) {
359 MI.removeOperand(OpIdx + 1);
360 MI.removeOperand(OpIdx);
378 if (
MI.getOpcode() == X86::LFENCE)
386 if (
MI.getOpcode() == X86::MFENCE)
398bool X86SpeculativeLoadHardeningPass::runOnMachineFunction(
411 TII = Subtarget->getInstrInfo();
412 TRI = Subtarget->getRegisterInfo();
415 PS.emplace(MF, &X86::GR64_NOSPRegClass);
423 hardenEdgesWithLFENCE(MF);
431 auto EntryInsertPt = Entry.SkipPHIsLabelsAndDebug(Entry.begin());
441 if (!HasVulnerableLoad && Infos.
empty())
446 const int PoisonVal = -1;
447 PS->PoisonReg =
MRI->createVirtualRegister(PS->RC);
448 BuildMI(Entry, EntryInsertPt, Loc,
TII->get(X86::MOV64ri32), PS->PoisonReg)
461 BuildMI(Entry, EntryInsertPt, Loc,
TII->get(X86::LFENCE));
463 ++NumLFENCEsInserted;
476 PS->InitialReg = extractPredStateFromSP(Entry, EntryInsertPt, Loc);
480 PS->InitialReg =
MRI->createVirtualRegister(PS->RC);
481 Register PredStateSubReg =
MRI->createVirtualRegister(&X86::GR32RegClass);
482 auto ZeroI =
BuildMI(Entry, EntryInsertPt, Loc,
TII->get(X86::MOV32r0),
486 ZeroI->findRegisterDefOperand(X86::EFLAGS,
nullptr);
488 "Must have an implicit def of EFLAGS!");
490 BuildMI(Entry, EntryInsertPt, Loc,
TII->get(X86::SUBREG_TO_REG),
504 PS->SSA.Initialize(PS->InitialReg);
505 PS->SSA.AddAvailableValue(&Entry, PS->InitialReg);
508 auto CMovs = tracePredStateThroughCFG(MF, Infos);
524 PS->SSA.AddAvailableValue(
533 unfoldCallAndJumpLoads(MF);
536 auto IndirectBrCMovs = tracePredStateThroughIndirectBranches(MF);
537 CMovs.append(IndirectBrCMovs.begin(), IndirectBrCMovs.end());
543 tracePredStateThroughBlocksAndHarden(MF);
549 if (!
Op.isReg() ||
Op.getReg() != PS->InitialReg)
552 PS->SSA.RewriteUse(
Op);
555 LLVM_DEBUG(
dbgs() <<
"Final speculative load hardened function:\n"; MF.dump();
556 dbgs() <<
"\n"; MF.verify(
this));
566void X86SpeculativeLoadHardeningPass::hardenEdgesWithLFENCE(
579 if (TermIt ==
MBB.
end() || !TermIt->isBranch())
586 if (!SuccMBB->isEHPad())
594 ++NumLFENCEsInserted;
599X86SpeculativeLoadHardeningPass::collectBlockCondInfo(
MachineFunction &MF) {
629 BlockCondInfo
Info = {&
MBB, {},
nullptr};
635 if (!
MI.isTerminator())
639 if (!
MI.isBranch()) {
640 Info.CondBrs.clear();
646 if (
MI.getOpcode() == X86::JMP_1) {
647 Info.CondBrs.clear();
662 Info.CondBrs.clear();
668 Info.CondBrs.push_back(&
MI);
670 if (
Info.CondBrs.empty()) {
671 ++NumBranchesUntraced;
672 LLVM_DEBUG(
dbgs() <<
"WARNING: unable to secure successors of block:\n";
691X86SpeculativeLoadHardeningPass::tracePredStateThroughCFG(
699 for (
const BlockCondInfo &Info : Infos) {
706 ++NumCondBranchesTraced;
711 UncondBr ? (UncondBr->
getOpcode() == X86::JMP_1
719 ++SuccCounts[UncondSucc];
720 for (
auto *CondBr : CondBrs)
721 ++SuccCounts[CondBr->getOperand(0).getMBB()];
725 auto BuildCheckingBlockForSuccAndConds =
732 (SuccCount == 1 && Succ.pred_size() == 1)
736 bool LiveEFLAGS = Succ.
isLiveIn(X86::EFLAGS);
738 CheckingMBB.addLiveIn(X86::EFLAGS);
741 auto InsertPt = CheckingMBB.begin();
742 assert((InsertPt == CheckingMBB.end() || !InsertPt->isPHI()) &&
743 "Should never have a PHI in the initial checking block as it "
744 "always has a single predecessor!");
748 unsigned CurStateReg = PS->InitialReg;
751 int PredStateSizeInBytes =
TRI->getRegSizeInBits(*PS->RC) / 8;
754 Register UpdatedStateReg =
MRI->createVirtualRegister(PS->RC);
758 TII->get(CMovOp), UpdatedStateReg)
764 if (!LiveEFLAGS &&
Cond == Conds.back())
774 if (CurStateReg == PS->InitialReg)
778 CurStateReg = UpdatedStateReg;
783 PS->SSA.AddAvailableValue(&CheckingMBB, CurStateReg);
786 std::vector<X86::CondCode> UncondCodeSeq;
787 for (
auto *CondBr : CondBrs) {
789 int &SuccCount = SuccCounts[&Succ];
793 UncondCodeSeq.push_back(
Cond);
795 BuildCheckingBlockForSuccAndConds(
MBB, Succ, SuccCount, CondBr, UncondBr,
818 assert(SuccCounts[UncondSucc] == 1 &&
819 "We should never have more than one edge to the unconditional "
820 "successor at this point because every other edge must have been "
825 UncondCodeSeq.erase(std::unique(UncondCodeSeq.begin(), UncondCodeSeq.end()),
826 UncondCodeSeq.end());
829 BuildCheckingBlockForSuccAndConds(
MBB, *UncondSucc, 1,
830 UncondBr, UncondBr, UncondCodeSeq);
845 unsigned UnfoldedOpc =
TII.getOpcodeAfterMemoryUnfold(
846 Opcode,
true,
false, &
Index);
848 return TII.getRegClass(MCID,
Index, &
TII.getRegisterInfo(), MF);
851void X86SpeculativeLoadHardeningPass::unfoldCallAndJumpLoads(
858 if (!
MI.isCall() && !
MI.isBranch())
864 switch (
MI.getOpcode()) {
867 dbgs() <<
"ERROR: Found an unexpected loading branch or call "
873 case X86::FARCALL16m:
874 case X86::FARCALL32m:
875 case X86::FARCALL64m:
884 case X86::CALL16m_NT:
886 case X86::CALL32m_NT:
888 case X86::CALL64m_NT:
895 case X86::TAILJMPm64:
896 case X86::TAILJMPm64_REX:
898 case X86::TCRETURNmi64:
899 case X86::TCRETURNmi: {
906 <<
"ERROR: Unable to unfold load from instruction:\n";
915 TII->unfoldMemoryOperand(MF,
MI, Reg,
true,
919 "Computed unfolded register class but failed to unfold");
921 for (
auto *NewMI : NewMIs)
925 if (
MI.isCandidateForCallSiteEntry())
926 MF.eraseCallSiteInfo(&
MI);
928 MI.eraseFromParent();
930 dbgs() <<
"Unfolded load successfully into:\n";
931 for (
auto *NewMI : NewMIs) {
961X86SpeculativeLoadHardeningPass::tracePredStateThroughIndirectBranches(
968 TargetAddrSSA.Initialize(
MRI->createVirtualRegister(&X86::GR64RegClass));
1000 case X86::FARJMP32m:
1001 case X86::FARJMP64m:
1007 case X86::JMP16m_NT:
1009 case X86::JMP32m_NT:
1011 case X86::JMP64m_NT:
1017 "Support for 16-bit indirect branches is not implemented.");
1020 "Support for 32-bit indirect branches is not implemented.");
1029 return !OtherTI.isDebugInstr() && &OtherTI != &TI;
1032 dbgs() <<
"ERROR: Found other terminators in a block with an indirect "
1033 "branch! This is not yet supported! Terminator sequence:\n";
1043 TargetAddrSSA.AddAvailableValue(&
MBB, TargetReg);
1048 IndirectTargetMBBs.
insert(Succ);
1055 if (IndirectTargetMBBs.
empty())
1063 if (!IndirectTargetMBBs.
count(&
MBB))
1070 "Unexpected EH pad as target of an indirect branch!");
1078 "Cannot check within a block that already has live-in EFLAGS!");
1085 if (IndirectTerminatedMBBs.
count(Pred))
1094 return Succ->isEHPad() || Succ == &MBB;
1097 dbgs() <<
"ERROR: Found conditional entry to target of indirect "
1103 "an indirect branch!");
1109 auto InsertPt = Pred->getFirstTerminator();
1110 Register TargetReg =
MRI->createVirtualRegister(&X86::GR64RegClass);
1112 !Subtarget->isPositionIndependent()) {
1115 TII->get(X86::MOV64ri32), TargetReg)
1135 TargetAddrSSA.AddAvailableValue(Pred, TargetReg);
1143 Register TargetReg = TargetAddrSSA.GetValueInMiddleOfBlock(&
MBB);
1151 !Subtarget->isPositionIndependent()) {
1161 Register AddrReg =
MRI->createVirtualRegister(&X86::GR64RegClass);
1181 int PredStateSizeInBytes =
TRI->getRegSizeInBits(*PS->RC) / 8;
1183 Register UpdatedStateReg =
MRI->createVirtualRegister(PS->RC);
1197 PS->SSA.AddAvailableValue(&
MBB, UpdatedStateReg);
1208 MI.findRegisterDefOperand(X86::EFLAGS,
nullptr)) {
1209 return !DefOp->isDead();
1220 MI.findRegisterDefOperand(X86::EFLAGS,
nullptr)) {
1222 if (DefOp->isDead())
1230 if (
MI.killsRegister(X86::EFLAGS, &
TRI))
1266void X86SpeculativeLoadHardeningPass::tracePredStateThroughBlocksAndHarden(
1303 return Op.isReg() && LoadDepRegs.test(Op.getReg());
1307 LoadDepRegs.
set(
Def.getReg());
1312 if (
MI.getOpcode() == X86::LFENCE)
1320 if (
MI.getOpcode() == X86::MFENCE)
1325 if (MemRefBeginIdx < 0) {
1327 <<
"WARNING: unable to harden loading instruction: ";
1339 unsigned BaseReg = 0, IndexReg = 0;
1340 if (!BaseMO.
isFI() && BaseMO.
getReg() != X86::RIP &&
1341 BaseMO.
getReg() != X86::NoRegister)
1342 BaseReg = BaseMO.
getReg();
1343 if (IndexMO.
getReg() != X86::NoRegister)
1344 IndexReg = IndexMO.
getReg();
1346 if (!BaseReg && !IndexReg)
1354 if ((BaseReg && LoadDepRegs.
test(BaseReg)) ||
1355 (IndexReg && LoadDepRegs.
test(IndexReg)))
1365 MI.getOperand(0).isReg() &&
1366 canHardenRegister(
MI.getOperand(0).getReg()) &&
1367 !HardenedAddrRegs.
count(BaseReg) &&
1368 !HardenedAddrRegs.
count(IndexReg)) {
1370 HardenedAddrRegs.
insert(
MI.getOperand(0).getReg());
1378 HardenedAddrRegs.
insert(BaseReg);
1380 HardenedAddrRegs.
insert(IndexReg);
1384 LoadDepRegs.
set(
Def.getReg());
1396 "Requested to harden both the address and def of a load!");
1399 if (HardenLoadAddr.
erase(&
MI)) {
1401 assert(MemRefBeginIdx >= 0 &&
"Cannot have an invalid index here!");
1407 hardenLoadAddr(
MI, BaseMO, IndexMO, AddrRegToHardenedReg);
1413 if (HardenPostLoad.
erase(&
MI)) {
1414 assert(!
MI.isCall() &&
"Must not try to post-load harden a call!");
1422 MachineInstr *SunkMI = sinkPostLoadHardenedInst(
MI, HardenPostLoad);
1427 if (SunkMI != &
MI) {
1434 HardenPostLoad.
insert(SunkMI);
1439 unsigned HardenedReg = hardenPostLoad(
MI);
1442 AddrRegToHardenedReg[HardenedReg] = HardenedReg;
1453 hardenIndirectCallOrJumpInstr(
MI, AddrRegToHardenedReg);
1460 if (!
MI.isCall() && !
MI.isReturn())
1465 if (
MI.isReturn() && !
MI.isCall()) {
1466 hardenReturnInstr(
MI);
1473 assert(
MI.isCall() &&
"Should only reach here for calls!");
1474 tracePredStateThroughCall(
MI);
1477 HardenPostLoad.
clear();
1478 HardenLoadAddr.
clear();
1479 HardenedAddrRegs.
clear();
1480 AddrRegToHardenedReg.
clear();
1485 LoadDepRegs.
clear();
1495unsigned X86SpeculativeLoadHardeningPass::saveEFLAGS(
1513void X86SpeculativeLoadHardeningPass::restoreEFLAGS(
1524void X86SpeculativeLoadHardeningPass::mergePredStateIntoSP(
1526 const DebugLoc &Loc,
unsigned PredStateReg) {
1527 Register TmpReg =
MRI->createVirtualRegister(PS->RC);
1531 auto ShiftI =
BuildMI(
MBB, InsertPt, Loc,
TII->get(X86::SHL64ri), TmpReg)
1536 auto OrI =
BuildMI(
MBB, InsertPt, Loc,
TII->get(X86::OR64rr), X86::RSP)
1544unsigned X86SpeculativeLoadHardeningPass::extractPredStateFromSP(
1547 Register PredStateReg =
MRI->createVirtualRegister(PS->RC);
1548 Register TmpReg =
MRI->createVirtualRegister(PS->RC);
1553 BuildMI(
MBB, InsertPt, Loc,
TII->get(TargetOpcode::COPY), TmpReg)
1556 BuildMI(
MBB, InsertPt, Loc,
TII->get(X86::SAR64ri), PredStateReg)
1558 .
addImm(
TRI->getRegSizeInBits(*PS->RC) - 1);
1562 return PredStateReg;
1565void X86SpeculativeLoadHardeningPass::hardenLoadAddr(
1577 if (BaseMO.
isFI()) {
1581 dbgs() <<
" Skipping hardening base of explicit stack frame load: ";
1583 }
else if (BaseMO.
getReg() == X86::RSP) {
1588 "Explicit RSP access with dynamic index!");
1590 dbgs() <<
" Cannot harden base of explicit RSP offset in a load!");
1591 }
else if (BaseMO.
getReg() == X86::RIP ||
1592 BaseMO.
getReg() == X86::NoRegister) {
1602 dbgs() <<
" Cannot harden base of "
1603 << (BaseMO.
getReg() == X86::RIP ?
"RIP-relative" :
"no-base")
1604 <<
" address in a load!");
1607 "Only allowed to have a frame index or register base.");
1611 if (IndexMO.
getReg() != X86::NoRegister &&
1612 (HardenOpRegs.
empty() ||
1613 HardenOpRegs.
front()->getReg() != IndexMO.
getReg()))
1617 "Should have exactly one or two registers to harden!");
1619 HardenOpRegs[0]->getReg() != HardenOpRegs[1]->getReg()) &&
1620 "Should not have two of the same registers!");
1625 auto It = AddrRegToHardenedReg.
find(
Op->getReg());
1626 if (It == AddrRegToHardenedReg.
end())
1631 Op->setReg(It->second);
1635 if (HardenOpRegs.
empty())
1639 Register StateReg = PS->SSA.GetValueAtEndOfBlock(&
MBB);
1641 auto InsertPt =
MI.getIterator();
1646 unsigned FlagsReg = 0;
1647 if (EFLAGSLive && !Subtarget->hasBMI2()) {
1649 FlagsReg = saveEFLAGS(
MBB, InsertPt, Loc);
1654 auto *OpRC =
MRI->getRegClass(OpReg);
1655 Register TmpReg =
MRI->createVirtualRegister(OpRC);
1659 if (!Subtarget->hasVLX() && (OpRC->hasSuperClassEq(&X86::VR128RegClass) ||
1660 OpRC->hasSuperClassEq(&X86::VR256RegClass))) {
1661 assert(Subtarget->hasAVX2() &&
"AVX2-specific register classes!");
1662 bool Is128Bit = OpRC->hasSuperClassEq(&X86::VR128RegClass);
1667 Register VStateReg =
MRI->createVirtualRegister(&X86::VR128RegClass);
1669 BuildMI(
MBB, InsertPt, Loc,
TII->get(X86::VMOV64toPQIrr), VStateReg)
1676 Register VBStateReg =
MRI->createVirtualRegister(OpRC);
1677 auto BroadcastI =
BuildMI(
MBB, InsertPt, Loc,
1678 TII->get(Is128Bit ? X86::VPBROADCASTQrr
1679 : X86::VPBROADCASTQYrr),
1684 LLVM_DEBUG(
dbgs() <<
" Inserting broadcast: "; BroadcastI->dump();
1690 TII->get(Is128Bit ? X86::VPORrr : X86::VPORYrr), TmpReg)
1696 }
else if (OpRC->hasSuperClassEq(&X86::VR128XRegClass) ||
1697 OpRC->hasSuperClassEq(&X86::VR256XRegClass) ||
1698 OpRC->hasSuperClassEq(&X86::VR512RegClass)) {
1699 assert(Subtarget->hasAVX512() &&
"AVX512-specific register classes!");
1700 bool Is128Bit = OpRC->hasSuperClassEq(&X86::VR128XRegClass);
1701 bool Is256Bit = OpRC->hasSuperClassEq(&X86::VR256XRegClass);
1702 if (Is128Bit || Is256Bit)
1703 assert(Subtarget->hasVLX() &&
"AVX512VL-specific register classes!");
1706 Register VStateReg =
MRI->createVirtualRegister(OpRC);
1707 unsigned BroadcastOp =
Is128Bit ? X86::VPBROADCASTQrZ128rr
1708 : Is256Bit ? X86::VPBROADCASTQrZ256rr
1709 : X86::VPBROADCASTQrZrr;
1711 BuildMI(
MBB, InsertPt, Loc,
TII->get(BroadcastOp), VStateReg)
1715 LLVM_DEBUG(
dbgs() <<
" Inserting broadcast: "; BroadcastI->dump();
1719 unsigned OrOp =
Is128Bit ? X86::VPORQZ128rr
1720 : Is256Bit ? X86::VPORQZ256rr : X86::VPORQZrr;
1721 auto OrI =
BuildMI(
MBB, InsertPt, Loc,
TII->get(OrOp), TmpReg)
1729 assert(OpRC->hasSuperClassEq(&X86::GR64RegClass) &&
1730 "Not a supported register class for address hardening!");
1734 auto OrI =
BuildMI(
MBB, InsertPt, Loc,
TII->get(X86::OR64rr), TmpReg)
1744 BuildMI(
MBB, InsertPt, Loc,
TII->get(X86::SHRX64rr), TmpReg)
1756 "Should not have checked this register yet!");
1757 AddrRegToHardenedReg[
Op->getReg()] = TmpReg;
1759 ++NumAddrRegsHardened;
1764 restoreEFLAGS(
MBB, InsertPt, Loc, FlagsReg);
1767MachineInstr *X86SpeculativeLoadHardeningPass::sinkPostLoadHardenedInst(
1770 "Cannot get here with a non-invariant load!");
1772 "Cannot get here with a data invariant load "
1773 "that interferes with EFLAGS!");
1776 auto SinkCheckToSingleUse =
1793 "Data variant instruction being hardened!");
1800 assert(MemRefBeginIdx >= 0 &&
1801 "Should always have mem references here!");
1807 if ((BaseMO.
isReg() && BaseMO.
getReg() == DefReg) ||
1828 if (
UseMI.getDesc().getNumDefs() > 1)
1835 if (!canHardenRegister(UseDefReg))
1838 SingleUseMI = &
UseMI;
1843 return {SingleUseMI};
1847 while (std::optional<MachineInstr *> SingleUse = SinkCheckToSingleUse(*
MI)) {
1857bool X86SpeculativeLoadHardeningPass::canHardenRegister(
Register Reg) {
1859 if (!
Reg.isVirtual())
1862 auto *RC =
MRI->getRegClass(Reg);
1863 int RegBytes =
TRI->getRegSizeInBits(*RC) / 8;
1868 unsigned RegIdx =
Log2_32(RegBytes);
1869 assert(RegIdx < 4 &&
"Unsupported register size");
1879 &X86::GR8_NOREXRegClass, &X86::GR16_NOREXRegClass,
1880 &X86::GR32_NOREXRegClass, &X86::GR64_NOREXRegClass};
1881 if (RC == NOREXRegClasses[RegIdx])
1885 &X86::GR8RegClass, &X86::GR16RegClass, &X86::GR32RegClass,
1886 &X86::GR64RegClass};
1904unsigned X86SpeculativeLoadHardeningPass::hardenValueInRegister(
1907 assert(canHardenRegister(Reg) &&
"Cannot harden this register!");
1909 auto *RC =
MRI->getRegClass(Reg);
1910 int Bytes =
TRI->getRegSizeInBits(*RC) / 8;
1911 Register StateReg = PS->SSA.GetValueAtEndOfBlock(&
MBB);
1912 assert((Bytes == 1 || Bytes == 2 || Bytes == 4 || Bytes == 8) &&
1913 "Unknown register size");
1917 unsigned SubRegImms[] = {X86::sub_8bit, X86::sub_16bit, X86::sub_32bit};
1918 unsigned SubRegImm = SubRegImms[
Log2_32(Bytes)];
1919 Register NarrowStateReg =
MRI->createVirtualRegister(RC);
1920 BuildMI(
MBB, InsertPt, Loc,
TII->get(TargetOpcode::COPY), NarrowStateReg)
1921 .
addReg(StateReg, 0, SubRegImm);
1922 StateReg = NarrowStateReg;
1925 unsigned FlagsReg = 0;
1927 FlagsReg = saveEFLAGS(
MBB, InsertPt, Loc);
1930 unsigned OrOpCodes[] = {X86::OR8rr, X86::OR16rr, X86::OR32rr, X86::OR64rr};
1931 unsigned OrOpCode = OrOpCodes[
Log2_32(Bytes)];
1932 auto OrI =
BuildMI(
MBB, InsertPt, Loc,
TII->get(OrOpCode), NewReg)
1940 restoreEFLAGS(
MBB, InsertPt, Loc, FlagsReg);
1954unsigned X86SpeculativeLoadHardeningPass::hardenPostLoad(
MachineInstr &
MI) {
1958 auto &DefOp =
MI.getOperand(0);
1959 Register OldDefReg = DefOp.getReg();
1960 auto *DefRC =
MRI->getRegClass(OldDefReg);
1965 Register UnhardenedReg =
MRI->createVirtualRegister(DefRC);
1966 DefOp.setReg(UnhardenedReg);
1971 unsigned HardenedReg = hardenValueInRegister(
1972 UnhardenedReg,
MBB, std::next(
MI.getIterator()), Loc);
1976 MRI->replaceRegWith( OldDefReg, HardenedReg);
1978 ++NumPostLoadRegsHardened;
2005void X86SpeculativeLoadHardeningPass::hardenReturnInstr(
MachineInstr &
MI) {
2008 auto InsertPt =
MI.getIterator();
2018 mergePredStateIntoSP(
MBB, InsertPt, Loc, PS->SSA.GetValueAtEndOfBlock(&
MBB));
2051void X86SpeculativeLoadHardeningPass::tracePredStateThroughCall(
2055 auto InsertPt =
MI.getIterator();
2068 BuildMI(
MBB, std::next(InsertPt), Loc,
TII->get(X86::LFENCE));
2070 ++NumLFENCEsInserted;
2076 Register StateReg = PS->SSA.GetValueAtEndOfBlock(&
MBB);
2077 mergePredStateIntoSP(
MBB, InsertPt, Loc, StateReg);
2092 MI.setPostInstrSymbol(MF, RetSymbol);
2095 unsigned ExpectedRetAddrReg = 0;
2100 if (!Subtarget->getFrameLowering()->has128ByteRedZone(MF) ||
2120 ExpectedRetAddrReg =
MRI->createVirtualRegister(AddrRC);
2122 !Subtarget->isPositionIndependent()) {
2123 BuildMI(
MBB, InsertPt, Loc,
TII->get(X86::MOV64ri32), ExpectedRetAddrReg)
2126 BuildMI(
MBB, InsertPt, Loc,
TII->get(X86::LEA64r), ExpectedRetAddrReg)
2142 if (!ExpectedRetAddrReg) {
2143 ExpectedRetAddrReg =
MRI->createVirtualRegister(AddrRC);
2144 BuildMI(
MBB, InsertPt, Loc,
TII->get(X86::MOV64rm), ExpectedRetAddrReg)
2154 unsigned NewStateReg = extractPredStateFromSP(
MBB, InsertPt, Loc);
2160 !Subtarget->isPositionIndependent()) {
2167 Register ActualRetAddrReg =
MRI->createVirtualRegister(AddrRC);
2168 BuildMI(
MBB, InsertPt, Loc,
TII->get(X86::LEA64r), ActualRetAddrReg)
2181 int PredStateSizeInBytes =
TRI->getRegSizeInBits(*PS->RC) / 8;
2184 Register UpdatedStateReg =
MRI->createVirtualRegister(PS->RC);
2185 auto CMovI =
BuildMI(
MBB, InsertPt, Loc,
TII->get(CMovOp), UpdatedStateReg)
2193 PS->SSA.AddAvailableValue(&
MBB, UpdatedStateReg);
2211void X86SpeculativeLoadHardeningPass::hardenIndirectCallOrJumpInstr(
2214 switch (
MI.getOpcode()) {
2215 case X86::FARCALL16m:
2216 case X86::FARCALL32m:
2217 case X86::FARCALL64m:
2218 case X86::FARJMP16m:
2219 case X86::FARJMP32m:
2220 case X86::FARJMP64m:
2231 assert(!
MI.mayLoad() &&
"Found a lingering loading instruction!");
2235 if (!
MI.getOperand(0).isReg())
2240 auto &TargetOp =
MI.getOperand(0);
2241 Register OldTargetReg = TargetOp.getReg();
2246 unsigned &HardenedTargetReg = AddrRegToHardenedReg[OldTargetReg];
2255 if (!HardenedTargetReg)
2256 HardenedTargetReg = hardenValueInRegister(
2257 OldTargetReg, *
MI.getParent(),
MI.getIterator(),
MI.getDebugLoc());
2260 TargetOp.setReg(HardenedTargetReg);
2262 ++NumCallsOrJumpsHardened;
2266 "X86 speculative load hardener",
false,
false)
2271 return new X86SpeculativeLoadHardeningPass();
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
AMDGPU Mark last scratch load
Analysis containing CSE Info
This file defines the DenseMap class.
DenseMap< Block *, BlockRelaxAux > Blocks
const HexagonInstrInfo * TII
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
unsigned const TargetRegisterInfo * TRI
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file defines the SparseBitVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static MachineBasicBlock & splitEdge(MachineBasicBlock &MBB, MachineBasicBlock &Succ, int SuccCount, MachineInstr *Br, MachineInstr *&UncondBr, const X86InstrInfo &TII)
static cl::opt< bool > HardenLoads(PASS_KEY "-loads", cl::desc("Sanitize loads from memory. When disable, no " "significant security is provided."), cl::init(true), cl::Hidden)
static void canonicalizePHIOperands(MachineFunction &MF)
Removing duplicate PHI operands to leave the PHI in a canonical and predictable form.
static cl::opt< bool > HardenInterprocedurally(PASS_KEY "-ip", cl::desc("Harden interprocedurally by passing our state in and out of " "functions in the high bits of the stack pointer."), cl::init(true), cl::Hidden)
static cl::opt< bool > FenceCallAndRet(PASS_KEY "-fence-call-and-ret", cl::desc("Use a full speculation fence to harden both call and ret edges " "rather than a lighter weight mitigation."), cl::init(false), cl::Hidden)
static cl::opt< bool > EnablePostLoadHardening(PASS_KEY "-post-load", cl::desc("Harden the value loaded *after* it is loaded by " "flushing the loaded bits to 1. This is hard to do " "in general but can be done easily for GPRs."), cl::init(true), cl::Hidden)
static cl::opt< bool > HardenEdgesWithLFENCE(PASS_KEY "-lfence", cl::desc("Use LFENCE along each conditional edge to harden against speculative " "loads rather than conditional movs and poisoned pointers."), cl::init(false), cl::Hidden)
static bool isEFLAGSLive(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const TargetRegisterInfo &TRI)
static cl::opt< bool > EnableSpeculativeLoadHardening("x86-speculative-load-hardening", cl::desc("Force enable speculative load hardening"), cl::init(false), cl::Hidden)
static const TargetRegisterClass * getRegClassForUnfoldedLoad(MachineFunction &MF, const X86InstrInfo &TII, unsigned Opcode)
Compute the register class for the unfolded load.
static bool hasVulnerableLoad(MachineFunction &MF)
Helper to scan a function for loads vulnerable to misspeculation that we want to harden.
static bool isEFLAGSDefLive(const MachineInstr &MI)
X86 speculative load hardener
static cl::opt< bool > HardenIndirectCallsAndJumps(PASS_KEY "-indirect", cl::desc("Harden indirect calls and jumps against using speculatively " "stored attacker controlled addresses. This is designed to " "mitigate Spectre v1.2 style attacks."), cl::init(true), cl::Hidden)
Represent the analysis usage information of a pass.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
This class represents an Operation in the Expression.
iterator find(const_arg_type_t< KeyT > Val)
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
FunctionPass class - This class is used to implement most global optimizations.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
Insert branch code into the end of the specified MachineBasicBlock.
MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
Describe properties that are true of each instruction in the target description file.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
bool isEHPad() const
Returns true if the block is a landing pad.
void replaceSuccessor(MachineBasicBlock *Old, MachineBasicBlock *New)
Replace successor OLD with NEW and update probability info.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
iterator_range< livein_iterator > liveins() const
reverse_instr_iterator instr_rbegin()
iterator SkipPHIsAndLabels(iterator I)
Return the first instruction in MBB after I that is not a PHI or a label.
void splitSuccessor(MachineBasicBlock *Old, MachineBasicBlock *New, bool NormalizeSuccProbs=false)
Split the old successor into old plus new and updates the probability info.
iterator SkipPHIsLabelsAndDebug(iterator I, Register Reg=Register(), bool SkipPseudoOp=true)
Return the first instruction in MBB after I that is not a PHI, label or debug.
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
bool isEHFuncletEntry() const
Returns true if this is the entry block of an EH funclet.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
unsigned succ_size() const
bool isEHScopeEntry() const
Returns true if this is the entry block of an EH scope, i.e., the block that used to have a catchpad ...
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
reverse_instr_iterator instr_rend()
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator_range< iterator > terminators()
iterator_range< succ_iterator > successors()
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
iterator_range< pred_iterator > predecessors()
StringRef getName() const
Return the name of the corresponding LLVM basic block, or an empty string.
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
bool isCleanupFuncletEntry() const
Returns true if this is the entry block of a cleanup funclet.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
bool exposesReturnsTwice() const
exposesReturnsTwice - Returns true if the function calls setjmp or any other similar functions with a...
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool isTerminator(QueryType Type=AnyInBundle) const
Returns true if this instruction part of the terminator for a basic block.
bool isBranch(QueryType Type=AnyInBundle) const
Returns true if this is a conditional, unconditional, or indirect branch.
MachineOperand * findRegisterUseOperand(Register Reg, const TargetRegisterInfo *TRI, bool isKill=false)
Wrapper for findRegisterUseOperandIdx, it returns a pointer to the MachineOperand rather than an inde...
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
const MachineOperand & getOperand(unsigned i) const
bool addRegisterDead(Register Reg, const TargetRegisterInfo *RegInfo, bool AddIfNotFound=false)
We have determined MI defined a register without a use.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
void setIsDead(bool Val=true)
void setIsKill(bool Val=true)
void setMBB(MachineBasicBlock *MBB)
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
static MachineOperand CreateMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0)
bool isMBB() const
isMBB - Tests if this is a MO_MachineBasicBlock operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
MachineSSAUpdater - This class updates SSA form for a set of virtual registers defined in multiple bl...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
bool test(unsigned Idx) const
StringRef - Represent a constant reference to a string, i.e.
CodeModel::Model getCodeModel() const
Returns the code model.
bool hasSuperClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a super-class of or equal to this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
static bool isDataInvariantLoad(MachineInstr &MI)
Returns true if the instruction has no behavior (specified or otherwise) that is based on the value l...
static bool isDataInvariant(MachineInstr &MI)
Returns true if the instruction has no behavior (specified or otherwise) that is based on the value o...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Kill
The last use of a register.
Reg
All possible values of the reg field in the ModR/M byte.
CondCode getCondFromBranch(const MachineInstr &MI)
int getFirstAddrOperandIdx(const MachineInstr &MI)
Return the index of the instruction's first address operand, if it has a memory reference,...
CondCode GetOppositeBranchCondition(CondCode CC)
GetOppositeBranchCondition - Return the inverse of the specified cond, e.g.
unsigned getCMovOpcode(unsigned RegBytes, bool HasMemoryOperand=false, bool HasNDD=false)
Return a cmov opcode for the given register size in bytes, and operand type.
initializer< Ty > init(const Ty &Val)
NodeAddr< DefNode * > Def
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
FunctionPass * createX86SpeculativeLoadHardeningPass()
auto reverse(ContainerTy &&C)
void sort(IteratorTy Start, IteratorTy End)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...