105using namespace wholeprogramdevirt;
107#define DEBUG_TYPE "wholeprogramdevirt"
109STATISTIC(NumDevirtTargets,
"Number of whole program devirtualization targets");
110STATISTIC(NumSingleImpl,
"Number of single implementation devirtualizations");
112STATISTIC(NumUniformRetVal,
"Number of uniform return value optimizations");
113STATISTIC(NumUniqueRetVal,
"Number of unique return value optimizations");
115 "Number of 1 bit virtual constant propagations");
116STATISTIC(NumVirtConstProp,
"Number of virtual constant propagations");
119 "wholeprogramdevirt-summary-action",
120 cl::desc(
"What to do with the summary when running this pass"),
122 clEnumValN(PassSummaryAction::Import,
"import",
123 "Import typeid resolutions from summary and globals"),
124 clEnumValN(PassSummaryAction::Export,
"export",
125 "Export typeid resolutions to summary and globals")),
129 "wholeprogramdevirt-read-summary",
131 "Read summary from given bitcode or YAML file before running pass"),
135 "wholeprogramdevirt-write-summary",
136 cl::desc(
"Write summary to given bitcode or YAML file after running pass. "
137 "Output file format is deduced from extension: *.bc means writing "
138 "bitcode, otherwise YAML"),
144 cl::desc(
"Maximum number of call targets per "
145 "call site to enable branch funnels"));
149 cl::desc(
"Print index-based devirtualization messages"));
157 cl::desc(
"Enable whole program visibility"));
162 "disable-whole-program-visibility",
cl::Hidden,
163 cl::desc(
"Disable whole program visibility (overrides enabling options)"));
168 cl::desc(
"Prevent function(s) from being devirtualized"),
179 cl::desc(
"Type of checking for incorrect devirtualizations"),
183 "Fallback to indirect when incorrect")));
187 std::vector<GlobPattern> Patterns;
188 template <
class T>
void init(
const T &StringList) {
189 for (
const auto &S : StringList)
191 Patterns.push_back(std::move(*Pat));
212 MinByte = std::max(MinByte,
Target.minAfterBytes());
214 MinByte = std::max(MinByte,
Target.minBeforeBytes());
237 std::vector<ArrayRef<uint8_t>> Used;
240 :
Target.TM->Bits->Before.BytesUsed;
242 : MinByte -
Target.minBeforeBytes();
252 for (
unsigned I = 0;; ++
I) {
253 uint8_t BitsUsed = 0;
254 for (
auto &&
B : Used)
257 if (BitsUsed != 0xff)
263 for (
unsigned I = 0;; ++
I) {
264 for (
auto &&
B : Used) {
266 while ((
I + Byte) <
B.size() && Byte < (
Size / 8)) {
272 return (MinByte +
I) * 8;
282 OffsetByte = -(AllocBefore / 8 + 1);
284 OffsetByte = -((AllocBefore + 7) / 8 + (
BitWidth + 7) / 8);
285 OffsetBit = AllocBefore % 8;
289 Target.setBeforeBit(AllocBefore);
299 OffsetByte = AllocAfter / 8;
301 OffsetByte = (AllocAfter + 7) / 8;
302 OffsetBit = AllocAfter % 8;
306 Target.setAfterBit(AllocAfter);
345 const VTableSlot &
RHS) {
346 return LHS.TypeID ==
RHS.TypeID &&
LHS.ByteOffset ==
RHS.ByteOffset;
365 return LHS.TypeID ==
RHS.TypeID &&
LHS.ByteOffset ==
RHS.ByteOffset;
388 if (!Summary->isLive())
390 if (
auto *FS = dyn_cast<FunctionSummary>(Summary->getBaseObject())) {
391 if (!FS->fflags().MustBeUnreachable)
406struct VirtualCallSite {
413 unsigned *NumUnsafeUses =
nullptr;
424 <<
NV(
"Optimization", OptName)
425 <<
": devirtualized a call to "
426 <<
NV(
"FunctionName", TargetName));
429 void replaceAndErase(
434 emitRemark(OptName, TargetName, OREGetter);
436 if (
auto *II = dyn_cast<InvokeInst>(&CB)) {
438 II->getUnwindDest()->removePredecessor(II->getParent());
455 std::vector<VirtualCallSite> CallSites;
460 bool AllCallSitesDevirted =
true;
467 bool SummaryHasTypeTestAssumeUsers =
false;
476 std::vector<FunctionSummary *> SummaryTypeCheckedLoadUsers;
477 std::vector<FunctionSummary *> SummaryTypeTestAssumeUsers;
479 bool isExported()
const {
480 return SummaryHasTypeTestAssumeUsers ||
481 !SummaryTypeCheckedLoadUsers.empty();
485 SummaryTypeCheckedLoadUsers.push_back(FS);
486 AllCallSitesDevirted =
false;
490 SummaryTypeTestAssumeUsers.push_back(FS);
491 SummaryHasTypeTestAssumeUsers =
true;
492 AllCallSitesDevirted =
false;
496 AllCallSitesDevirted =
true;
499 SummaryTypeCheckedLoadUsers.clear();
504struct VTableSlotInfo {
511 std::map<std::vector<uint64_t>, CallSiteInfo> ConstCSInfo;
513 void addCallSite(
Value *VTable,
CallBase &CB,
unsigned *NumUnsafeUses);
516 CallSiteInfo &findCallSiteInfo(
CallBase &CB);
519CallSiteInfo &VTableSlotInfo::findCallSiteInfo(
CallBase &CB) {
520 std::vector<uint64_t>
Args;
521 auto *CBType = dyn_cast<IntegerType>(CB.
getType());
522 if (!CBType || CBType->getBitWidth() > 64 || CB.
arg_empty())
525 auto *CI = dyn_cast<ConstantInt>(Arg);
526 if (!CI || CI->getBitWidth() > 64)
528 Args.push_back(CI->getZExtValue());
530 return ConstCSInfo[
Args];
534 unsigned *NumUnsafeUses) {
535 auto &CSI = findCallSiteInfo(CB);
536 CSI.AllCallSitesDevirted =
false;
537 CSI.CallSites.push_back({
VTable, CB, NumUnsafeUses});
580 std::map<CallInst *, unsigned> NumUnsafeUsesForTypeTest;
581 PatternList FunctionsToSkip;
588 :
M(
M), AARGetter(AARGetter), LookupDomTree(LookupDomTree),
589 ExportSummary(ExportSummary), ImportSummary(ImportSummary),
590 Int8Ty(
Type::getInt8Ty(
M.getContext())),
593 Int64Ty(
Type::getInt64Ty(
M.getContext())),
594 IntPtrTy(
M.getDataLayout().getIntPtrType(
M.getContext(), 0)),
596 RemarksEnabled(areRemarksEnabled()), OREGetter(OREGetter) {
597 assert(!(ExportSummary && ImportSummary));
601 bool areRemarksEnabled();
604 scanTypeTestUsers(
Function *TypeTestFunc,
606 void scanTypeCheckedLoadUsers(
Function *TypeCheckedLoadFunc);
608 void buildTypeIdentifierMap(
609 std::vector<VTableBits> &Bits,
613 tryFindVirtualCallTargets(std::vector<VirtualCallTarget> &TargetsForSlot,
614 const std::set<TypeMemberInfo> &TypeMemberInfos,
618 void applySingleImplDevirt(VTableSlotInfo &SlotInfo,
Constant *TheFn,
622 VTableSlotInfo &SlotInfo,
625 void applyICallBranchFunnel(VTableSlotInfo &SlotInfo,
Constant *JT,
628 VTableSlotInfo &SlotInfo,
631 bool tryEvaluateFunctionsWithArgs(
635 void applyUniformRetValOpt(CallSiteInfo &CSInfo,
StringRef FnName,
638 CallSiteInfo &CSInfo,
646 bool shouldExportConstantsAsAbsoluteSymbols();
666 void applyUniqueRetValOpt(CallSiteInfo &CSInfo,
StringRef FnName,
bool IsOne,
668 bool tryUniqueRetValOpt(
unsigned BitWidth,
670 CallSiteInfo &CSInfo,
674 void applyVirtualConstProp(CallSiteInfo &CSInfo,
StringRef FnName,
677 VTableSlotInfo &SlotInfo,
683 void importResolution(VTableSlot Slot, VTableSlotInfo &SlotInfo);
687 void removeRedundantTypeTests();
720 std::set<GlobalValue::GUID> &ExportedGUIDs;
724 std::map<ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap;
728 PatternList FunctionsToSkip;
732 std::set<GlobalValue::GUID> &ExportedGUIDs,
733 std::map<
ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap)
734 : ExportSummary(ExportSummary), ExportedGUIDs(ExportedGUIDs),
735 LocalWPDTargetsMap(LocalWPDTargetsMap) {
739 bool tryFindVirtualCallTargets(std::vector<ValueInfo> &TargetsForSlot,
745 VTableSlotInfo &SlotInfo,
747 std::set<ValueInfo> &DevirtTargets);
765 if (UseCommandLine) {
766 if (!DevirtModule::runForTesting(M, AARGetter, OREGetter, LookupDomTree))
770 if (!DevirtModule(M, AARGetter, OREGetter, LookupDomTree, ExportSummary,
790 if (
TypeID.ends_with(
".virtual"))
796 if (!
TypeID.consume_front(
"_ZTS"))
804 std::string typeInfo = (
"_ZTI" +
TypeID).str();
805 return IsVisibleToRegularObj(typeInfo);
814 for (
auto Type : Types)
815 if (
auto *
TypeID = dyn_cast<MDString>(
Type->getOperand(1).get()))
817 IsVisibleToRegularObj);
826 Module &M,
bool WholeProgramVisibilityEnabledInLTO,
828 bool ValidateAllVtablesHaveTypeInfos,
845 !(ValidateAllVtablesHaveTypeInfos &&
852 bool WholeProgramVisibilityEnabledInLTO) {
855 if (!PublicTypeTestFunc)
861 auto *CI = cast<CallInst>(U.getUser());
863 TypeTestFunc, {CI->getArgOperand(0), CI->getArgOperand(1)},
865 CI->replaceAllUsesWith(NewCI);
866 CI->eraseFromParent();
871 auto *CI = cast<CallInst>(U.getUser());
872 CI->replaceAllUsesWith(True);
873 CI->eraseFromParent();
884 for (
const auto &typeID :
Index.typeIdCompatibleVtableMap()) {
887 VisibleToRegularObjSymbols.
insert(
P.VTableVI.getGUID());
903 if (DynamicExportSymbols.
count(
P.first))
905 for (
auto &S :
P.second.SummaryList) {
906 auto *GVar = dyn_cast<GlobalVarSummary>(S.get());
914 if (VisibleToRegularObjSymbols.
count(
P.first))
923 std::map<
ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap) {
924 DevirtIndex(Summary, ExportedGUIDs, LocalWPDTargetsMap).run();
930 std::map<
ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap) {
931 for (
auto &
T : LocalWPDTargetsMap) {
934 assert(VI.getSummaryList().size() == 1 &&
935 "Devirt of local target has more than one copy");
936 auto &S = VI.getSummaryList()[0];
937 if (!isExported(S->modulePath(), VI))
941 for (
auto &SlotSummary :
T.second) {
942 auto *TIdSum = Summary.getTypeIdSummary(SlotSummary.TypeID);
944 auto WPDRes = TIdSum->WPDRes.find(SlotSummary.ByteOffset);
945 assert(WPDRes != TIdSum->WPDRes.end());
947 WPDRes->second.SingleImplName,
948 Summary.getModuleHash(S->modulePath()));
958 const auto &ModPaths = Summary->modulePaths();
963 "combined summary should contain Regular LTO module");
967bool DevirtModule::runForTesting(
971 std::unique_ptr<ModuleSummaryIndex>
Summary =
972 std::make_unique<ModuleSummaryIndex>(
false);
979 auto ReadSummaryFile =
981 if (
Expected<std::unique_ptr<ModuleSummaryIndex>> SummaryOrErr =
983 Summary = std::move(*SummaryOrErr);
988 yaml::Input
In(ReadSummaryFile->getBuffer());
995 DevirtModule(M, AARGetter, OREGetter, LookupDomTree,
1013 yaml::Output Out(
OS);
1021void DevirtModule::buildTypeIdentifierMap(
1022 std::vector<VTableBits> &Bits,
1025 Bits.reserve(
M.global_size());
1035 Bits.emplace_back();
1036 Bits.back().GV = &GV;
1037 Bits.back().ObjectSize =
1039 BitsPtr = &
Bits.back();
1047 cast<ConstantAsMetadata>(
Type->getOperand(0))->getValue())
1055bool DevirtModule::tryFindVirtualCallTargets(
1056 std::vector<VirtualCallTarget> &TargetsForSlot,
1057 const std::set<TypeMemberInfo> &TypeMemberInfos,
uint64_t ByteOffset,
1060 if (!
TM.Bits->GV->isConstant())
1065 if (
TM.Bits->GV->getVCallVisibility() ==
1077 if (FunctionsToSkip.match(Fn->
getName()))
1082 if (Fn->
getName() ==
"__cxa_pure_virtual")
1092 auto GV = dyn_cast<GlobalValue>(
C);
1094 TargetsForSlot.push_back({GV, &
TM});
1098 return !TargetsForSlot.empty();
1101bool DevirtIndex::tryFindVirtualCallTargets(
1102 std::vector<ValueInfo> &TargetsForSlot,
1116 bool LocalFound =
false;
1117 for (
const auto &S :
P.VTableVI.getSummaryList()) {
1123 auto *CurVS = cast<GlobalVarSummary>(S->getBaseObject());
1124 if (!CurVS->vTableFuncs().empty() ||
1146 for (
auto VTP :
VS->vTableFuncs()) {
1147 if (VTP.VTableOffset !=
P.AddressPointOffset + ByteOffset)
1153 TargetsForSlot.push_back(VTP.FuncVI);
1158 return !TargetsForSlot.empty();
1161void DevirtModule::applySingleImplDevirt(VTableSlotInfo &SlotInfo,
1162 Constant *TheFn,
bool &IsExported) {
1167 auto Apply = [&](CallSiteInfo &CSInfo) {
1168 for (
auto &&VCallSite : CSInfo.CallSites) {
1169 if (!OptimizedCalls.
insert(&VCallSite.CB).second)
1173 VCallSite.emitRemark(
"single-impl",
1176 auto &CB = VCallSite.CB;
1189 Builder.SetInsertPoint(ThenTerm);
1191 auto *CallTrap = Builder.CreateCall(TrapFn);
1209 NewInst.
setMetadata(LLVMContext::MD_prof,
nullptr);
1210 NewInst.
setMetadata(LLVMContext::MD_callees,
nullptr);
1232 CallsWithPtrAuthBundleRemoved.
push_back(&CB);
1237 if (VCallSite.NumUnsafeUses)
1238 --*VCallSite.NumUnsafeUses;
1240 if (CSInfo.isExported())
1242 CSInfo.markDevirt();
1244 Apply(SlotInfo.CSInfo);
1245 for (
auto &
P : SlotInfo.ConstCSInfo)
1251 if (Callee.getSummaryList().empty())
1258 bool IsExported =
false;
1259 auto &S = Callee.getSummaryList()[0];
1262 auto AddCalls = [&](CallSiteInfo &CSInfo) {
1263 for (
auto *FS : CSInfo.SummaryTypeCheckedLoadUsers) {
1264 FS->addCall({Callee, CI});
1265 IsExported |= S->modulePath() != FS->modulePath();
1267 for (
auto *FS : CSInfo.SummaryTypeTestAssumeUsers) {
1268 FS->addCall({Callee, CI});
1269 IsExported |= S->modulePath() != FS->modulePath();
1273 for (
auto &
P : SlotInfo.ConstCSInfo)
1278bool DevirtModule::trySingleImplDevirt(
1284 auto *TheFn = TargetsForSlot[0].Fn;
1285 for (
auto &&
Target : TargetsForSlot)
1291 TargetsForSlot[0].WasDevirt =
true;
1293 bool IsExported =
false;
1294 applySingleImplDevirt(SlotInfo, TheFn, IsExported);
1301 if (TheFn->hasLocalLinkage()) {
1302 std::string NewName = (TheFn->
getName() +
".llvm.merged").str();
1307 if (
Comdat *
C = TheFn->getComdat()) {
1308 if (
C->getName() == TheFn->
getName()) {
1309 Comdat *NewC =
M.getOrInsertComdat(NewName);
1312 if (GO.getComdat() ==
C)
1334 VTableSlotInfo &SlotInfo,
1336 std::set<ValueInfo> &DevirtTargets) {
1339 auto TheFn = TargetsForSlot[0];
1340 for (
auto &&
Target : TargetsForSlot)
1345 auto Size = TheFn.getSummaryList().size();
1351 if (FunctionsToSkip.match(TheFn.name()))
1356 for (
const auto &S : TheFn.getSummaryList())
1362 DevirtTargets.insert(TheFn);
1364 auto &S = TheFn.getSummaryList()[0];
1365 bool IsExported =
AddCalls(SlotInfo, TheFn);
1367 ExportedGUIDs.insert(TheFn.getGUID());
1378 TheFn.name(), ExportSummary.
getModuleHash(S->modulePath()));
1380 LocalWPDTargetsMap[TheFn].push_back(SlotSummary);
1394void DevirtModule::tryICallBranchFunnel(
1404 bool HasNonDevirt = !SlotInfo.CSInfo.AllCallSitesDevirted;
1406 for (
auto &
P : SlotInfo.ConstCSInfo)
1407 if (!
P.second.AllCallSitesDevirted) {
1408 HasNonDevirt =
true;
1418 if (isa<MDString>(
Slot.TypeID)) {
1420 M.getDataLayout().getProgramAddressSpace(),
1421 getGlobalName(Slot, {},
"branch_funnel"), &M);
1425 M.getDataLayout().getProgramAddressSpace(),
1426 "branch_funnel", &M);
1428 JT->addParamAttr(0, Attribute::Nest);
1430 std::vector<Value *> JTArgs;
1431 JTArgs.push_back(
JT->arg_begin());
1432 for (
auto &
T : TargetsForSlot) {
1433 JTArgs.push_back(getMemberAddr(
T.TM));
1434 JTArgs.push_back(
T.Fn);
1445 bool IsExported =
false;
1446 applyICallBranchFunnel(SlotInfo, JT, IsExported);
1451void DevirtModule::applyICallBranchFunnel(VTableSlotInfo &SlotInfo,
1453 auto Apply = [&](CallSiteInfo &CSInfo) {
1454 if (CSInfo.isExported())
1456 if (CSInfo.AllCallSitesDevirted)
1459 std::map<CallBase *, CallBase *> CallBases;
1460 for (
auto &&VCallSite : CSInfo.CallSites) {
1463 if (CallBases.find(&CB) != CallBases.end()) {
1480 VCallSite.emitRemark(
"branch-funnel",
1481 JT->stripPointerCasts()->getName(), OREGetter);
1485 std::vector<Type *> NewArgs;
1486 NewArgs.push_back(Int8PtrTy);
1494 std::vector<Value *>
Args;
1495 Args.push_back(VCallSite.VTable);
1499 if (isa<CallInst>(CB))
1500 NewCS = IRB.CreateCall(NewFT, IRB.CreateBitCast(JT, NewFTPtr), Args);
1502 NewCS = IRB.CreateInvoke(NewFT, IRB.CreateBitCast(JT, NewFTPtr),
1503 cast<InvokeInst>(CB).getNormalDest(),
1504 cast<InvokeInst>(CB).getUnwindDest(), Args);
1508 std::vector<AttributeSet> NewArgAttrs;
1511 M.getContext(), Attribute::Nest)}));
1512 for (
unsigned I = 0;
I + 2 <
Attrs.getNumAttrSets(); ++
I)
1513 NewArgAttrs.push_back(
Attrs.getParamAttrs(
I));
1516 Attrs.getRetAttrs(), NewArgAttrs));
1518 CallBases[&CB] = NewCS;
1521 if (VCallSite.NumUnsafeUses)
1522 --*VCallSite.NumUnsafeUses;
1529 for (
auto &[Old, New] : CallBases) {
1531 Old->eraseFromParent();
1534 Apply(SlotInfo.CSInfo);
1535 for (
auto &
P : SlotInfo.ConstCSInfo)
1539bool DevirtModule::tryEvaluateFunctionsWithArgs(
1548 auto Fn = dyn_cast<Function>(
Target.Fn);
1559 for (
unsigned I = 0;
I !=
Args.size(); ++
I) {
1564 EvalArgs.
push_back(ConstantInt::get(ArgTy, Args[
I]));
1568 if (!Eval.EvaluateFunction(Fn, RetVal, EvalArgs) ||
1569 !isa<ConstantInt>(RetVal))
1571 Target.RetVal = cast<ConstantInt>(RetVal)->getZExtValue();
1576void DevirtModule::applyUniformRetValOpt(CallSiteInfo &CSInfo,
StringRef FnName,
1578 for (
auto Call : CSInfo.CallSites) {
1582 Call.replaceAndErase(
1583 "uniform-ret-val", FnName, RemarksEnabled, OREGetter,
1584 ConstantInt::get(cast<IntegerType>(
Call.CB.getType()), TheRetVal));
1586 CSInfo.markDevirt();
1589bool DevirtModule::tryUniformRetValOpt(
1594 uint64_t TheRetVal = TargetsForSlot[0].RetVal;
1596 if (
Target.RetVal != TheRetVal)
1599 if (CSInfo.isExported()) {
1601 Res->
Info = TheRetVal;
1604 applyUniformRetValOpt(CSInfo, TargetsForSlot[0].Fn->
getName(), TheRetVal);
1606 for (
auto &&
Target : TargetsForSlot)
1611std::string DevirtModule::getGlobalName(VTableSlot Slot,
1614 std::string FullName =
"__typeid_";
1616 OS << cast<MDString>(
Slot.TypeID)->getString() <<
'_' <<
Slot.ByteOffset;
1623bool DevirtModule::shouldExportConstantsAsAbsoluteSymbols() {
1631 getGlobalName(Slot, Args,
Name),
C, &M);
1638 if (shouldExportConstantsAsAbsoluteSymbols()) {
1651 M.getOrInsertGlobal(getGlobalName(Slot, Args,
Name), Int8Arr0Ty);
1652 auto *GV = dyn_cast<GlobalVariable>(
C);
1661 if (!shouldExportConstantsAsAbsoluteSymbols())
1662 return ConstantInt::get(IntTy, Storage);
1665 auto *GV = cast<GlobalVariable>(
C->stripPointerCasts());
1670 if (GV->
hasMetadata(LLVMContext::MD_absolute_symbol))
1681 SetAbsRange(~0ull, ~0ull);
1683 SetAbsRange(0, 1ull << AbsWidth);
1687void DevirtModule::applyUniqueRetValOpt(CallSiteInfo &CSInfo,
StringRef FnName,
1690 for (
auto &&Call : CSInfo.CallSites) {
1696 B.CreateBitCast(UniqueMemberAddr,
Call.VTable->getType()));
1697 Cmp =
B.CreateZExt(Cmp,
Call.CB.getType());
1699 Call.replaceAndErase(
"unique-ret-val", FnName, RemarksEnabled, OREGetter,
1702 CSInfo.markDevirt();
1707 ConstantInt::get(Int64Ty,
M->Offset));
1710bool DevirtModule::tryUniqueRetValOpt(
1715 auto tryUniqueRetValOptFor = [&](
bool IsOne) {
1718 if (
Target.RetVal == (IsOne ? 1 : 0)) {
1721 UniqueMember =
Target.TM;
1729 Constant *UniqueMemberAddr = getMemberAddr(UniqueMember);
1730 if (CSInfo.isExported()) {
1734 exportGlobal(Slot, Args,
"unique_member", UniqueMemberAddr);
1738 applyUniqueRetValOpt(CSInfo, TargetsForSlot[0].Fn->
getName(), IsOne,
1743 for (
auto &&
Target : TargetsForSlot)
1750 if (tryUniqueRetValOptFor(
true))
1752 if (tryUniqueRetValOptFor(
false))
1758void DevirtModule::applyVirtualConstProp(CallSiteInfo &CSInfo,
StringRef FnName,
1760 for (
auto Call : CSInfo.CallSites) {
1763 auto *RetType = cast<IntegerType>(
Call.CB.getType());
1766 if (RetType->getBitWidth() == 1) {
1768 Value *BitsAndBit =
B.CreateAnd(Bits, Bit);
1769 auto IsBitSet =
B.CreateICmpNE(BitsAndBit, ConstantInt::get(Int8Ty, 0));
1770 NumVirtConstProp1Bit++;
1771 Call.replaceAndErase(
"virtual-const-prop-1-bit", FnName, RemarksEnabled,
1772 OREGetter, IsBitSet);
1776 Call.replaceAndErase(
"virtual-const-prop", FnName, RemarksEnabled,
1780 CSInfo.markDevirt();
1783bool DevirtModule::tryVirtualConstProp(
1789 auto Fn = dyn_cast<Function>(TargetsForSlot[0].Fn);
1796 unsigned BitWidth = RetType->getBitWidth();
1814 auto Fn = dyn_cast<Function>(
Target.Fn);
1820 .doesNotAccessMemory() ||
1826 for (
auto &&CSByConstantArg : SlotInfo.ConstCSInfo) {
1827 if (!tryEvaluateFunctionsWithArgs(TargetsForSlot, CSByConstantArg.first))
1832 ResByArg = &Res->
ResByArg[CSByConstantArg.first];
1834 if (tryUniformRetValOpt(TargetsForSlot, CSByConstantArg.second, ResByArg))
1837 if (tryUniqueRetValOpt(
BitWidth, TargetsForSlot, CSByConstantArg.second,
1838 ResByArg, Slot, CSByConstantArg.first))
1850 uint64_t TotalPaddingBefore = 0, TotalPaddingAfter = 0;
1851 for (
auto &&
Target : TargetsForSlot) {
1852 TotalPaddingBefore += std::max<int64_t>(
1853 (AllocBefore + 7) / 8 -
Target.allocatedBeforeBytes() - 1, 0);
1854 TotalPaddingAfter += std::max<int64_t>(
1855 (AllocAfter + 7) / 8 -
Target.allocatedAfterBytes() - 1, 0);
1860 if (std::min(TotalPaddingBefore, TotalPaddingAfter) > 128)
1867 if (TotalPaddingBefore <= TotalPaddingAfter)
1875 for (
auto &&
Target : TargetsForSlot)
1879 if (CSByConstantArg.second.isExported()) {
1881 exportConstant(Slot, CSByConstantArg.first,
"byte", OffsetByte,
1883 exportConstant(Slot, CSByConstantArg.first,
"bit", 1ULL << OffsetBit,
1889 Constant *BitConst = ConstantInt::get(Int8Ty, 1ULL << OffsetBit);
1890 applyVirtualConstProp(CSByConstantArg.second,
1891 TargetsForSlot[0].Fn->getName(), ByteConst, BitConst);
1897 if (
B.Before.Bytes.empty() &&
B.After.Bytes.empty())
1902 Align Alignment =
M.getDataLayout().getValueOrABITypeAlignment(
1903 B.GV->getAlign(),
B.GV->getValueType());
1904 B.Before.Bytes.resize(
alignTo(
B.Before.Bytes.size(), Alignment));
1907 for (
size_t I = 0,
Size =
B.Before.Bytes.size();
I !=
Size / 2; ++
I)
1914 B.GV->getInitializer(),
1919 NewGV->setSection(
B.GV->getSection());
1920 NewGV->setComdat(
B.GV->getComdat());
1921 NewGV->setAlignment(
B.GV->getAlign());
1925 NewGV->copyMetadata(
B.GV,
B.Before.Bytes.size());
1930 B.GV->getInitializer()->getType(), 0,
B.GV->getLinkage(),
"",
1932 NewInit->getType(), NewGV,
1934 ConstantInt::get(Int32Ty, 1)}),
1936 Alias->setVisibility(
B.GV->getVisibility());
1937 Alias->takeName(
B.GV);
1939 B.GV->replaceAllUsesWith(Alias);
1940 B.GV->eraseFromParent();
1943bool DevirtModule::areRemarksEnabled() {
1944 const auto &FL =
M.getFunctionList();
1949 return DI.isEnabled();
1954void DevirtModule::scanTypeTestUsers(
1963 auto *CI = dyn_cast<CallInst>(
U.getUser());
1970 auto &DT = LookupDomTree(*CI->getFunction());
1974 cast<MetadataAsValue>(CI->getArgOperand(1))->getMetadata();
1976 if (!Assumes.
empty()) {
1977 Value *
Ptr = CI->getArgOperand(0)->stripPointerCasts();
1979 CallSlots[{TypeId,
Call.Offset}].addCallSite(
Ptr,
Call.CB,
nullptr);
1982 auto RemoveTypeTestAssumes = [&]() {
1984 for (
auto *Assume : Assumes)
1985 Assume->eraseFromParent();
1988 if (CI->use_empty())
1989 CI->eraseFromParent();
2004 if (!TypeIdMap.count(TypeId))
2005 RemoveTypeTestAssumes();
2016 else if (ImportSummary && isa<MDString>(TypeId)) {
2020 RemoveTypeTestAssumes();
2029void DevirtModule::scanTypeCheckedLoadUsers(
Function *TypeCheckedLoadFunc) {
2033 auto *CI = dyn_cast<CallInst>(
U.getUser());
2039 Value *TypeIdValue = CI->getArgOperand(2);
2040 Metadata *TypeId = cast<MetadataAsValue>(TypeIdValue)->getMetadata();
2045 bool HasNonCallUses =
false;
2046 auto &DT = LookupDomTree(*CI->getFunction());
2048 HasNonCallUses, CI, DT);
2057 (LoadedPtrs.
size() == 1 && !HasNonCallUses) ? LoadedPtrs[0] : CI);
2059 Value *LoadedValue =
nullptr;
2061 Intrinsic::type_checked_load_relative) {
2063 LoadedValue = LoadB.CreateLoad(
Int32Ty,
GEP);
2064 LoadedValue = LoadB.CreateSExt(LoadedValue, IntPtrTy);
2065 GEP = LoadB.CreatePtrToInt(
GEP, IntPtrTy);
2066 LoadedValue = LoadB.CreateAdd(
GEP, LoadedValue);
2067 LoadedValue = LoadB.CreateIntToPtr(LoadedValue, Int8PtrTy);
2070 LoadedValue = LoadB.CreateLoad(Int8PtrTy,
GEP);
2075 LoadedPtr->eraseFromParent();
2079 IRBuilder<> CallB((Preds.
size() == 1 && !HasNonCallUses) ? Preds[0] : CI);
2080 CallInst *TypeTestCall = CallB.CreateCall(TypeTestFunc, {
Ptr, TypeIdValue});
2084 Pred->eraseFromParent();
2091 if (!CI->use_empty()) {
2094 Pair =
B.CreateInsertValue(Pair, LoadedValue, {0});
2095 Pair =
B.CreateInsertValue(Pair, TypeTestCall, {1});
2096 CI->replaceAllUsesWith(Pair);
2100 auto &NumUnsafeUses = NumUnsafeUsesForTypeTest[TypeTestCall];
2101 NumUnsafeUses = DevirtCalls.
size();
2109 CallSlots[{TypeId,
Call.Offset}].addCallSite(
Ptr,
Call.CB,
2113 CI->eraseFromParent();
2117void DevirtModule::importResolution(VTableSlot Slot, VTableSlotInfo &SlotInfo) {
2118 auto *TypeId = dyn_cast<MDString>(
Slot.TypeID);
2125 auto ResI = TidSummary->
WPDRes.find(
Slot.ByteOffset);
2126 if (ResI == TidSummary->
WPDRes.end())
2140 bool IsExported =
false;
2141 applySingleImplDevirt(SlotInfo, SingleImpl, IsExported);
2145 for (
auto &CSByConstantArg : SlotInfo.ConstCSInfo) {
2146 auto I = Res.
ResByArg.find(CSByConstantArg.first);
2149 auto &ResByArg =
I->second;
2156 applyUniformRetValOpt(CSByConstantArg.second,
"", ResByArg.
Info);
2160 importGlobal(Slot, CSByConstantArg.first,
"unique_member");
2161 applyUniqueRetValOpt(CSByConstantArg.second,
"", ResByArg.
Info,
2166 Constant *
Byte = importConstant(Slot, CSByConstantArg.first,
"byte",
2168 Constant *
Bit = importConstant(Slot, CSByConstantArg.first,
"bit", Int8Ty,
2170 applyVirtualConstProp(CSByConstantArg.second,
"", Byte, Bit);
2182 M.getOrInsertFunction(getGlobalName(Slot, {},
"branch_funnel"),
2185 bool IsExported =
false;
2186 applyICallBranchFunnel(SlotInfo, JT, IsExported);
2191void DevirtModule::removeRedundantTypeTests() {
2193 for (
auto &&U : NumUnsafeUsesForTypeTest) {
2194 if (
U.second == 0) {
2195 U.first->replaceAllUsesWith(True);
2196 U.first->eraseFromParent();
2202DevirtModule::lookUpFunctionValueInfo(
Function *TheFn,
2204 assert((ExportSummary !=
nullptr) &&
2205 "Caller guarantees ExportSummary is not nullptr");
2207 const auto TheFnGUID = TheFn->
getGUID();
2219 if ((!TheFnVI) && (TheFnGUID != TheFnGUIDWithExportedName)) {
2220 TheFnVI = ExportSummary->
getValueInfo(TheFnGUIDWithExportedName);
2225bool DevirtModule::mustBeUnreachableFunction(
2228 if (!
F->isDeclaration()) {
2231 return isa<UnreachableInst>(
F->getEntryBlock().getTerminator());
2234 return ExportSummary &&
2236 DevirtModule::lookUpFunctionValueInfo(
F, ExportSummary));
2239bool DevirtModule::run() {
2252 Function *TypeCheckedLoadRelativeFunc =
2259 if (!ExportSummary &&
2260 (!TypeTestFunc || TypeTestFunc->
use_empty() || !AssumeFunc ||
2262 (!TypeCheckedLoadFunc || TypeCheckedLoadFunc->
use_empty()) &&
2263 (!TypeCheckedLoadRelativeFunc ||
2264 TypeCheckedLoadRelativeFunc->
use_empty()))
2268 std::vector<VTableBits>
Bits;
2270 buildTypeIdentifierMap(Bits, TypeIdMap);
2272 if (TypeTestFunc && AssumeFunc)
2273 scanTypeTestUsers(TypeTestFunc, TypeIdMap);
2275 if (TypeCheckedLoadFunc)
2276 scanTypeCheckedLoadUsers(TypeCheckedLoadFunc);
2278 if (TypeCheckedLoadRelativeFunc)
2279 scanTypeCheckedLoadUsers(TypeCheckedLoadRelativeFunc);
2281 if (ImportSummary) {
2282 for (
auto &S : CallSlots)
2283 importResolution(S.first, S.second);
2285 removeRedundantTypeTests();
2298 if (TypeIdMap.
empty())
2302 if (ExportSummary) {
2304 for (
auto &
P : TypeIdMap) {
2305 if (
auto *TypeId = dyn_cast<MDString>(
P.first))
2310 for (
auto &
P : *ExportSummary) {
2311 for (
auto &S :
P.second.SummaryList) {
2312 auto *
FS = dyn_cast<FunctionSummary>(S.get());
2317 for (
Metadata *MD : MetadataByGUID[VF.GUID]) {
2318 CallSlots[{MD, VF.Offset}].CSInfo.addSummaryTypeTestAssumeUser(FS);
2322 for (
Metadata *MD : MetadataByGUID[VF.GUID]) {
2323 CallSlots[{MD, VF.Offset}].CSInfo.addSummaryTypeCheckedLoadUser(FS);
2327 FS->type_test_assume_const_vcalls()) {
2328 for (
Metadata *MD : MetadataByGUID[
VC.VFunc.GUID]) {
2329 CallSlots[{MD,
VC.VFunc.Offset}]
2330 .ConstCSInfo[
VC.Args]
2331 .addSummaryTypeTestAssumeUser(FS);
2335 FS->type_checked_load_const_vcalls()) {
2336 for (
Metadata *MD : MetadataByGUID[
VC.VFunc.GUID]) {
2337 CallSlots[{MD,
VC.VFunc.Offset}]
2338 .ConstCSInfo[
VC.Args]
2339 .addSummaryTypeCheckedLoadUser(FS);
2347 bool DidVirtualConstProp =
false;
2348 std::map<std::string, GlobalValue *> DevirtTargets;
2349 for (
auto &S : CallSlots) {
2353 std::vector<VirtualCallTarget> TargetsForSlot;
2355 const std::set<TypeMemberInfo> &TypeMemberInfos = TypeIdMap[S.first.TypeID];
2356 if (ExportSummary && isa<MDString>(S.first.TypeID) &&
2357 TypeMemberInfos.size())
2364 Res = &ExportSummary
2365 ->getOrInsertTypeIdSummary(
2366 cast<MDString>(S.first.TypeID)->getString())
2367 .WPDRes[S.first.ByteOffset];
2368 if (tryFindVirtualCallTargets(TargetsForSlot, TypeMemberInfos,
2369 S.first.ByteOffset, ExportSummary)) {
2371 if (!trySingleImplDevirt(ExportSummary, TargetsForSlot, S.second, Res)) {
2372 DidVirtualConstProp |=
2373 tryVirtualConstProp(TargetsForSlot, S.second, Res, S.first);
2375 tryICallBranchFunnel(TargetsForSlot, S.second, Res, S.first);
2380 for (
const auto &
T : TargetsForSlot)
2382 DevirtTargets[std::string(
T.Fn->getName())] =
T.Fn;
2389 if (ExportSummary && isa<MDString>(S.first.TypeID)) {
2392 for (
auto *FS : S.second.CSInfo.SummaryTypeCheckedLoadUsers)
2393 FS->addTypeTest(GUID);
2394 for (
auto &CCS : S.second.ConstCSInfo)
2395 for (
auto *FS : CCS.second.SummaryTypeCheckedLoadUsers)
2396 FS->addTypeTest(GUID);
2400 if (RemarksEnabled) {
2402 for (
const auto &DT : DevirtTargets) {
2404 auto F = dyn_cast<Function>(GV);
2406 auto A = dyn_cast<GlobalAlias>(GV);
2407 assert(
A && isa<Function>(
A->getAliasee()));
2408 F = dyn_cast<Function>(
A->getAliasee());
2412 using namespace ore;
2415 <<
NV(
"FunctionName", DT.first));
2419 NumDevirtTargets += DevirtTargets.size();
2421 removeRedundantTypeTests();
2425 if (DidVirtualConstProp)
2435 for (
auto *CI : CallsWithPtrAuthBundleRemoved)
2436 CI->eraseFromParent();
2441void DevirtIndex::run() {
2442 if (ExportSummary.typeIdCompatibleVtableMap().empty())
2446 for (
const auto &
P : ExportSummary.typeIdCompatibleVtableMap()) {
2454 ExportSummary.getOrInsertTypeIdSummary(
P.first);
2458 for (
auto &
P : ExportSummary) {
2459 for (
auto &S :
P.second.SummaryList) {
2460 auto *
FS = dyn_cast<FunctionSummary>(S.get());
2466 CallSlots[{
Name, VF.Offset}].CSInfo.addSummaryTypeTestAssumeUser(FS);
2471 CallSlots[{
Name, VF.Offset}].CSInfo.addSummaryTypeCheckedLoadUser(FS);
2475 FS->type_test_assume_const_vcalls()) {
2477 CallSlots[{
Name,
VC.VFunc.Offset}]
2478 .ConstCSInfo[
VC.Args]
2479 .addSummaryTypeTestAssumeUser(FS);
2483 FS->type_checked_load_const_vcalls()) {
2485 CallSlots[{
Name,
VC.VFunc.Offset}]
2486 .ConstCSInfo[
VC.Args]
2487 .addSummaryTypeCheckedLoadUser(FS);
2493 std::set<ValueInfo> DevirtTargets;
2495 for (
auto &S : CallSlots) {
2499 std::vector<ValueInfo> TargetsForSlot;
2500 auto TidSummary = ExportSummary.getTypeIdCompatibleVtableSummary(S.first.TypeID);
2505 &ExportSummary.getTypeIdSummary(S.first.TypeID)
2506 ->WPDRes[S.first.ByteOffset];
2507 if (tryFindVirtualCallTargets(TargetsForSlot, *TidSummary,
2508 S.first.ByteOffset)) {
2510 if (!trySingleImplDevirt(TargetsForSlot, S.first, S.second, Res,
2519 for (
const auto &DT : DevirtTargets)
2520 errs() <<
"Devirtualized call to " << DT <<
"\n";
2522 NumDevirtTargets += DevirtTargets.size();
static const Function * getParent(const Value *V)
This is the interface for LLVM's primary stateless and local alias analysis.
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file defines DenseMapInfo traits for DenseMap.
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
Provides passes for computing function attributes based on interprocedural analyses.
static cl::opt< std::string > ClReadSummary("lowertypetests-read-summary", cl::desc("Read summary from given YAML file before running pass"), cl::Hidden)
static cl::opt< PassSummaryAction > ClSummaryAction("lowertypetests-summary-action", cl::desc("What to do with the summary when running this pass"), cl::values(clEnumValN(PassSummaryAction::None, "none", "Do nothing"), clEnumValN(PassSummaryAction::Import, "import", "Import typeid resolutions from summary and globals"), clEnumValN(PassSummaryAction::Export, "export", "Export typeid resolutions to summary and globals")), cl::Hidden)
static cl::opt< std::string > ClWriteSummary("lowertypetests-write-summary", cl::desc("Write summary to given YAML file after running pass"), cl::Hidden)
This file implements a map that provides insertion order iteration.
static bool mustBeUnreachableFunction(const Function &F)
Module.h This file contains the declarations for the Module class.
FunctionAnalysisManager FAM
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static cl::opt< bool > DisableWholeProgramVisibility("disable-whole-program-visibility", cl::Hidden, cl::desc("Disable whole program visibility (overrides enabling options)"))
Provide a way to force disable whole program for debugging or workarounds, when enabled via the linke...
WPDCheckMode
Mechanism to add runtime checking of devirtualization decisions, optionally trapping or falling back ...
static cl::opt< PassSummaryAction > ClSummaryAction("wholeprogramdevirt-summary-action", cl::desc("What to do with the summary when running this pass"), cl::values(clEnumValN(PassSummaryAction::None, "none", "Do nothing"), clEnumValN(PassSummaryAction::Import, "import", "Import typeid resolutions from summary and globals"), clEnumValN(PassSummaryAction::Export, "export", "Export typeid resolutions to summary and globals")), cl::Hidden)
static cl::opt< bool > WholeProgramVisibility("whole-program-visibility", cl::Hidden, cl::desc("Enable whole program visibility"))
Provide a way to force enable whole program visibility in tests.
static bool typeIDVisibleToRegularObj(StringRef TypeID, function_ref< bool(StringRef)> IsVisibleToRegularObj)
static Error checkCombinedSummaryForTesting(ModuleSummaryIndex *Summary)
static cl::list< std::string > SkipFunctionNames("wholeprogramdevirt-skip", cl::desc("Prevent function(s) from being devirtualized"), cl::Hidden, cl::CommaSeparated)
Provide way to prevent certain function from being devirtualized.
static cl::opt< std::string > ClWriteSummary("wholeprogramdevirt-write-summary", cl::desc("Write summary to given bitcode or YAML file after running pass. " "Output file format is deduced from extension: *.bc means writing " "bitcode, otherwise YAML"), cl::Hidden)
static cl::opt< unsigned > ClThreshold("wholeprogramdevirt-branch-funnel-threshold", cl::Hidden, cl::init(10), cl::desc("Maximum number of call targets per " "call site to enable branch funnels"))
static cl::opt< WPDCheckMode > DevirtCheckMode("wholeprogramdevirt-check", cl::Hidden, cl::desc("Type of checking for incorrect devirtualizations"), cl::values(clEnumValN(WPDCheckMode::None, "none", "No checking"), clEnumValN(WPDCheckMode::Trap, "trap", "Trap when incorrect"), clEnumValN(WPDCheckMode::Fallback, "fallback", "Fallback to indirect when incorrect")))
static cl::opt< std::string > ClReadSummary("wholeprogramdevirt-read-summary", cl::desc("Read summary from given bitcode or YAML file before running pass"), cl::Hidden)
static bool skipUpdateDueToValidation(GlobalVariable &GV, function_ref< bool(StringRef)> IsVisibleToRegularObj)
static bool AddCalls(VTableSlotInfo &SlotInfo, const ValueInfo &Callee)
static cl::opt< bool > PrintSummaryDevirt("wholeprogramdevirt-print-index-based", cl::Hidden, cl::desc("Print index-based devirtualization messages"))
A manager for alias analyses.
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
static AttributeSet get(LLVMContext &C, const AttrBuilder &B)
StringRef getValueAsString() const
Return the attribute's value as a string.
bool isValid() const
Return true if the attribute is any kind of attribute.
LLVM Basic Block Representation.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
static BranchInst * Create(BasicBlock *IfTrue, BasicBlock::iterator InsertBefore)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void setCallingConv(CallingConv::ID CC)
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
CallingConv::ID getCallingConv() const
static CallBase * removeOperandBundle(CallBase *CB, uint32_t ID, Instruction *InsertPt=nullptr)
Create a clone of CB with operand bundle ID removed.
Value * getCalledOperand() const
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
FunctionType * getFunctionType() const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void setCalledOperand(Value *V)
AttributeList getAttributes() const
Return the parameter attributes for this call.
Function * getCaller()
Helper to get the caller (the parent function).
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr, BasicBlock::iterator InsertBefore)
void setSelectionKind(SelectionKind Val)
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
static Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList, bool InBounds=false, std::optional< ConstantRange > InRange=std::nullopt, Type *OnlyIfReducedTy=nullptr)
Getelementptr form.
static ConstantInt * getTrue(LLVMContext &Context)
static Constant * getAnon(ArrayRef< Constant * > V, bool Packed=false)
Return an anonymous struct that has the specified elements.
This is an important base class in LLVM.
const Constant * stripPointerCasts() const
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Implements a dense probed hash-table based set.
Analysis pass which computes a DominatorTree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Subclass of Error for the sole purpose of identifying the success path in the type system.
Lightweight error class with error context and mandatory checking.
This class evaluates LLVM IR, producing the Constant representing each SSA instruction.
Helper for check-and-exit error handling.
Tagged union holding either a T or a Error.
Function summary information to aid decisions and implementation of importing.
Type * getParamType(unsigned i) const
Parameter type accessors.
ArrayRef< Type * > params() const
Type * getReturnType() const
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
const BasicBlock & front() const
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Type * getReturnType() const
Returns the type of the ret val.
This class implements a glob pattern matcher similar to the one found in bash, but with some key diff...
static Expected< GlobPattern > create(StringRef Pat, std::optional< size_t > MaxSubPatterns={})
static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
bool hasMetadata() const
Return true if this value has any metadata attached to it.
void setMetadata(unsigned KindID, MDNode *Node)
Set a particular kind of metadata attachment.
VCallVisibility getVCallVisibility() const
bool eraseMetadata(unsigned KindID)
Erase all metadata attachments with the given kind.
@ VCallVisibilityLinkageUnit
MDNode * getMetadata(unsigned KindID) const
Get the current metadata attachments for the given kind, if any.
void setVCallVisibilityMetadata(VCallVisibility Visibility)
static bool isLocalLinkage(LinkageTypes Linkage)
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
static bool isAvailableExternallyLinkage(LinkageTypes Linkage)
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
@ HiddenVisibility
The GV is hidden.
static GUID getGUID(StringRef GlobalName)
Return a 64-bit global unique ID constructed from global value name (i.e.
void setVisibility(VisibilityTypes V)
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ InternalLinkage
Rename collisions when linking (static functions).
@ ExternalLinkage
Externally visible function.
Global variable summary information to aid decisions and implementation of importing.
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const BasicBlock * getParent() const
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Class to represent integer types.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight)
Return metadata containing two branch weights.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
This class implements a map that also provides access to all stored values in a deterministic order.
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFile(const Twine &Filename, bool IsText=false, bool RequiresNullTerminator=true, bool IsVolatile=false, std::optional< Align > Alignment=std::nullopt)
Open the specified file as a MemoryBuffer, returning a new MemoryBuffer if successful,...
Class to hold module path string table and global value map, and encapsulate methods for operating on...
const TypeIdSummary * getTypeIdSummary(StringRef TypeId) const
This returns either a pointer to the type id summary (if present in the summary map) or null (if not ...
ValueInfo getValueInfo(const GlobalValueSummaryMapTy::value_type &R) const
Return a ValueInfo for the index value_type (convenient when iterating index).
const ModuleHash & getModuleHash(const StringRef ModPath) const
Get the module SHA1 hash recorded for the given module path.
static constexpr const char * getRegularLTOModuleName()
bool partiallySplitLTOUnits() const
static std::string getGlobalNameForLocal(StringRef Name, ModuleHash ModHash)
Convenience method for creating a promoted global name for the given value name of a local,...
A Module instance is used to store all the information related to an LLVM module.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
static ReturnInst * Create(LLVMContext &C, Value *retVal, BasicBlock::iterator InsertBefore)
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
bool contains(StringRef Other) const
Return true if the given string is a substring of *this, and false otherwise.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
The instances of the Type class are immutable: once they are created, they are never changed.
TypeID
Definitions of all of the base types for the Type system.
static Type * getVoidTy(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
bool eraseMetadata(unsigned KindID)
Erase all metadata attachments with the given kind.
iterator_range< use_iterator > uses()
StringRef getName() const
Return a constant reference to the value's name.
std::pair< iterator, bool > insert(const ValueT &V)
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
An efficient, type-erasing, non-owning reference to a callable.
self_iterator getIterator()
A raw_ostream that writes to a file descriptor.
A raw_ostream that writes to an std::string.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
@ C
The default llvm calling convention, compatible with C.
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
bool match(Val *V, const Pattern &P)
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
DiagnosticInfoOptimizationBase::Argument NV
@ OF_TextWithCRLF
The file should be opened in text mode and use a carriage linefeed '\r '.
uint64_t findLowestOffset(ArrayRef< VirtualCallTarget > Targets, bool IsAfter, uint64_t Size)
void setAfterReturnValues(MutableArrayRef< VirtualCallTarget > Targets, uint64_t AllocAfter, unsigned BitWidth, int64_t &OffsetByte, uint64_t &OffsetBit)
void setBeforeReturnValues(MutableArrayRef< VirtualCallTarget > Targets, uint64_t AllocBefore, unsigned BitWidth, int64_t &OffsetByte, uint64_t &OffsetBit)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
MemoryEffects computeFunctionBodyMemoryAccess(Function &F, AAResults &AAR)
Returns the memory access properties of this copy of the function.
void writeIndexToFile(const ModuleSummaryIndex &Index, raw_ostream &Out, const std::map< std::string, GVSummaryMapTy > *ModuleToSummariesForIndex=nullptr)
Write the specified module summary index to the given raw output stream, where it will be written in ...
@ Export
Export information to summary.
@ Import
Import information from summary.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Error createStringError(std::error_code EC, char const *Fmt, const Ts &... Vals)
Create formatted StringError object.
bool hasWholeProgramVisibility(bool WholeProgramVisibilityEnabledInLTO)
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Expected< std::unique_ptr< ModuleSummaryIndex > > getModuleSummaryIndex(MemoryBufferRef Buffer)
Parse the specified bitcode buffer, returning the module summary index.
void updatePublicTypeTestCalls(Module &M, bool WholeProgramVisibilityEnabledInLTO)
void getVisibleToRegularObjVtableGUIDs(ModuleSummaryIndex &Index, DenseSet< GlobalValue::GUID > &VisibleToRegularObjSymbols, function_ref< bool(StringRef)> IsVisibleToRegularObj)
Based on typeID string, get all associated vtable GUIDS that are visible to regular objects.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void findDevirtualizableCallsForTypeCheckedLoad(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< Instruction * > &LoadedPtrs, SmallVectorImpl< Instruction * > &Preds, bool &HasNonCallUses, const CallInst *CI, DominatorTree &DT)
Given a call to the intrinsic @llvm.type.checked.load, find all devirtualizable call sites based on t...
CallBase & versionCallSite(CallBase &CB, Value *Callee, MDNode *BranchWeights)
Predicate and clone the given call site.
bool AreStatisticsEnabled()
Check if statistics are enabled.
void updateIndexWPDForExports(ModuleSummaryIndex &Summary, function_ref< bool(StringRef, ValueInfo)> isExported, std::map< ValueInfo, std::vector< VTableSlotSummary > > &LocalWPDTargetsMap)
Call after cross-module importing to update the recorded single impl devirt target names for any loca...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
void runWholeProgramDevirtOnIndex(ModuleSummaryIndex &Summary, std::set< GlobalValue::GUID > &ExportedGUIDs, std::map< ValueInfo, std::vector< VTableSlotSummary > > &LocalWPDTargetsMap)
Perform index-based whole program devirtualization on the Summary index.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Expected< T > errorOrToExpected(ErrorOr< T > &&EO)
Convert an ErrorOr<T> to an Expected<T>.
constexpr unsigned BitWidth
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
std::vector< TypeIdOffsetVtableInfo > TypeIdCompatibleVtableInfo
List of vtable definitions decorated by a particular type identifier, and their corresponding offsets...
void consumeError(Error Err)
Consume a Error without doing anything.
void findDevirtualizableCallsForTypeTest(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< CallInst * > &Assumes, const CallInst *CI, DominatorTree &DT)
Given a call to the intrinsic @llvm.type.test, find all devirtualizable call sites based on the call ...
void updateVCallVisibilityInModule(Module &M, bool WholeProgramVisibilityEnabledInLTO, const DenseSet< GlobalValue::GUID > &DynamicExportSymbols, bool ValidateAllVtablesHaveTypeInfos, function_ref< bool(StringRef)> IsVisibleToRegularObj)
If whole program visibility asserted, then upgrade all public vcall visibility metadata on vtable def...
std::pair< Function *, Constant * > getFunctionAtVTableOffset(GlobalVariable *GV, uint64_t Offset, Module &M)
Given a vtable and a specified offset, returns the function and the trivial pointer at the specified ...
void updateVCallVisibilityInIndex(ModuleSummaryIndex &Index, bool WholeProgramVisibilityEnabledInLTO, const DenseSet< GlobalValue::GUID > &DynamicExportSymbols, const DenseSet< GlobalValue::GUID > &VisibleToRegularObjSymbols)
If whole program visibility asserted, then upgrade all public vcall visibility metadata on vtable def...
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Class to accumulate and hold information about a callee.
static unsigned getHashValue(const VTableSlotSummary &I)
static bool isEqual(const VTableSlotSummary &LHS, const VTableSlotSummary &RHS)
static VTableSlotSummary getTombstoneKey()
static VTableSlotSummary getEmptyKey()
static VTableSlot getTombstoneKey()
static VTableSlot getEmptyKey()
static bool isEqual(const VTableSlot &LHS, const VTableSlot &RHS)
static unsigned getHashValue(const VTableSlot &I)
An information struct used to provide DenseMap with the various necessary components for a given valu...
A call site that could be devirtualized.
A specification for a virtual function call with all constant integer arguments.
An "identifier" for a virtual function.
The following data structures summarize type metadata information.
std::map< uint64_t, WholeProgramDevirtResolution > WPDRes
Mapping from byte offset to whole-program devirt resolution for that (typeid, byte offset) pair.
@ Unsat
Unsatisfiable type (i.e. no global has this type metadata)
enum llvm::TypeTestResolution::Kind TheKind
Struct that holds a reference to a particular GUID in a global value summary.
ArrayRef< std::unique_ptr< GlobalValueSummary > > getSummaryList() const
PreservedAnalyses run(Module &M, ModuleAnalysisManager &)
@ UniformRetVal
Uniform return value optimization.
@ VirtualConstProp
Virtual constant propagation.
@ UniqueRetVal
Unique return value optimization.
uint64_t Info
Additional information for the resolution:
enum llvm::WholeProgramDevirtResolution::ByArg::Kind TheKind
enum llvm::WholeProgramDevirtResolution::Kind TheKind
std::map< std::vector< uint64_t >, ByArg > ResByArg
Resolutions for calls with all constant integer arguments (excluding the first argument,...
std::string SingleImplName
@ SingleImpl
Single implementation devirtualization.
@ BranchFunnel
When retpoline mitigation is enabled, use a branch funnel that is defined in the merged module.
VirtualCallTarget(GlobalValue *Fn, const TypeMemberInfo *TM)