109using namespace wholeprogramdevirt;
111#define DEBUG_TYPE "wholeprogramdevirt"
113STATISTIC(NumDevirtTargets,
"Number of whole program devirtualization targets");
114STATISTIC(NumSingleImpl,
"Number of single implementation devirtualizations");
116STATISTIC(NumUniformRetVal,
"Number of uniform return value optimizations");
117STATISTIC(NumUniqueRetVal,
"Number of unique return value optimizations");
119 "Number of 1 bit virtual constant propagations");
120STATISTIC(NumVirtConstProp,
"Number of virtual constant propagations");
123 "wholeprogramdevirt-summary-action",
124 cl::desc(
"What to do with the summary when running this pass"),
126 clEnumValN(PassSummaryAction::Import,
"import",
127 "Import typeid resolutions from summary and globals"),
128 clEnumValN(PassSummaryAction::Export,
"export",
129 "Export typeid resolutions to summary and globals")),
133 "wholeprogramdevirt-read-summary",
135 "Read summary from given bitcode or YAML file before running pass"),
139 "wholeprogramdevirt-write-summary",
140 cl::desc(
"Write summary to given bitcode or YAML file after running pass. "
141 "Output file format is deduced from extension: *.bc means writing "
142 "bitcode, otherwise YAML"),
148 cl::desc(
"Maximum number of call targets per "
149 "call site to enable branch funnels"));
153 cl::desc(
"Print index-based devirtualization messages"));
161 cl::desc(
"Enable whole program visibility"));
166 "disable-whole-program-visibility",
cl::Hidden,
167 cl::desc(
"Disable whole program visibility (overrides enabling options)"));
172 cl::desc(
"Prevent function(s) from being devirtualized"),
183 cl::desc(
"Type of checking for incorrect devirtualizations"),
187 "Fallback to indirect when incorrect")));
191 std::vector<GlobPattern> Patterns;
192 template <
class T>
void init(
const T &StringList) {
193 for (
const auto &S : StringList)
195 Patterns.push_back(std::move(*Pat));
216 MinByte = std::max(MinByte,
Target.minAfterBytes());
218 MinByte = std::max(MinByte,
Target.minBeforeBytes());
241 std::vector<ArrayRef<uint8_t>> Used;
244 :
Target.TM->Bits->Before.BytesUsed;
246 : MinByte -
Target.minBeforeBytes();
256 for (
unsigned I = 0;; ++
I) {
257 uint8_t BitsUsed = 0;
258 for (
auto &&
B : Used)
261 if (BitsUsed != 0xff)
267 for (
unsigned I = 0;; ++
I) {
268 for (
auto &&
B : Used) {
270 while ((
I + Byte) <
B.size() && Byte < (
Size / 8)) {
276 return (MinByte +
I) * 8;
286 OffsetByte = -(AllocBefore / 8 + 1);
288 OffsetByte = -((AllocBefore + 7) / 8 + (
BitWidth + 7) / 8);
289 OffsetBit = AllocBefore % 8;
293 Target.setBeforeBit(AllocBefore);
303 OffsetByte = AllocAfter / 8;
305 OffsetByte = (AllocAfter + 7) / 8;
306 OffsetBit = AllocAfter % 8;
310 Target.setAfterBit(AllocAfter);
349 const VTableSlot &
RHS) {
350 return LHS.TypeID ==
RHS.TypeID &&
LHS.ByteOffset ==
RHS.ByteOffset;
369 return LHS.TypeID ==
RHS.TypeID &&
LHS.ByteOffset ==
RHS.ByteOffset;
396 if (
auto *FS = dyn_cast<FunctionSummary>(
Summary->getBaseObject())) {
397 if (!
FS->fflags().MustBeUnreachable)
411struct VirtualCallSite {
418 unsigned *NumUnsafeUses =
nullptr;
429 <<
NV(
"Optimization", OptName)
430 <<
": devirtualized a call to "
431 <<
NV(
"FunctionName", TargetName));
434 void replaceAndErase(
439 emitRemark(OptName, TargetName, OREGetter);
441 if (
auto *II = dyn_cast<InvokeInst>(&CB)) {
443 II->getUnwindDest()->removePredecessor(II->getParent());
460 std::vector<VirtualCallSite> CallSites;
465 bool AllCallSitesDevirted =
true;
472 bool SummaryHasTypeTestAssumeUsers =
false;
481 std::vector<FunctionSummary *> SummaryTypeCheckedLoadUsers;
482 std::vector<FunctionSummary *> SummaryTypeTestAssumeUsers;
484 bool isExported()
const {
485 return SummaryHasTypeTestAssumeUsers ||
486 !SummaryTypeCheckedLoadUsers.empty();
490 SummaryTypeCheckedLoadUsers.push_back(FS);
491 AllCallSitesDevirted =
false;
495 SummaryTypeTestAssumeUsers.push_back(FS);
496 SummaryHasTypeTestAssumeUsers =
true;
497 AllCallSitesDevirted =
false;
501 AllCallSitesDevirted =
true;
504 SummaryTypeCheckedLoadUsers.clear();
509struct VTableSlotInfo {
516 std::map<std::vector<uint64_t>, CallSiteInfo> ConstCSInfo;
518 void addCallSite(
Value *VTable,
CallBase &CB,
unsigned *NumUnsafeUses);
521 CallSiteInfo &findCallSiteInfo(
CallBase &CB);
524CallSiteInfo &VTableSlotInfo::findCallSiteInfo(
CallBase &CB) {
525 std::vector<uint64_t>
Args;
526 auto *CBType = dyn_cast<IntegerType>(CB.
getType());
527 if (!CBType || CBType->getBitWidth() > 64 || CB.
arg_empty())
530 auto *CI = dyn_cast<ConstantInt>(
Arg);
531 if (!CI || CI->getBitWidth() > 64)
533 Args.push_back(CI->getZExtValue());
535 return ConstCSInfo[
Args];
539 unsigned *NumUnsafeUses) {
540 auto &CSI = findCallSiteInfo(CB);
541 CSI.AllCallSitesDevirted =
false;
542 CSI.CallSites.push_back({
VTable, CB, NumUnsafeUses});
581 std::map<CallInst *, unsigned> NumUnsafeUsesForTypeTest;
582 PatternList FunctionsToSkip;
589 :
M(
M), AARGetter(AARGetter), LookupDomTree(LookupDomTree),
590 ExportSummary(ExportSummary), ImportSummary(ImportSummary),
591 Int8Ty(
Type::getInt8Ty(
M.getContext())),
592 Int8PtrTy(
Type::getInt8PtrTy(
M.getContext())),
594 Int64Ty(
Type::getInt64Ty(
M.getContext())),
595 IntPtrTy(
M.getDataLayout().getIntPtrType(
M.getContext(), 0)),
597 RemarksEnabled(areRemarksEnabled()), OREGetter(OREGetter) {
598 assert(!(ExportSummary && ImportSummary));
602 bool areRemarksEnabled();
605 scanTypeTestUsers(
Function *TypeTestFunc,
607 void scanTypeCheckedLoadUsers(
Function *TypeCheckedLoadFunc);
609 void buildTypeIdentifierMap(
610 std::vector<VTableBits> &Bits,
614 tryFindVirtualCallTargets(std::vector<VirtualCallTarget> &TargetsForSlot,
615 const std::set<TypeMemberInfo> &TypeMemberInfos,
619 void applySingleImplDevirt(VTableSlotInfo &SlotInfo,
Constant *TheFn,
623 VTableSlotInfo &SlotInfo,
626 void applyICallBranchFunnel(VTableSlotInfo &SlotInfo,
Constant *JT,
629 VTableSlotInfo &SlotInfo,
632 bool tryEvaluateFunctionsWithArgs(
636 void applyUniformRetValOpt(CallSiteInfo &CSInfo,
StringRef FnName,
639 CallSiteInfo &CSInfo,
647 bool shouldExportConstantsAsAbsoluteSymbols();
667 void applyUniqueRetValOpt(CallSiteInfo &CSInfo,
StringRef FnName,
bool IsOne,
669 bool tryUniqueRetValOpt(
unsigned BitWidth,
671 CallSiteInfo &CSInfo,
675 void applyVirtualConstProp(CallSiteInfo &CSInfo,
StringRef FnName,
678 VTableSlotInfo &SlotInfo,
684 void importResolution(VTableSlot Slot, VTableSlotInfo &SlotInfo);
688 void removeRedundantTypeTests();
721 std::set<GlobalValue::GUID> &ExportedGUIDs;
725 std::map<ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap;
729 PatternList FunctionsToSkip;
733 std::set<GlobalValue::GUID> &ExportedGUIDs,
734 std::map<
ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap)
735 : ExportSummary(ExportSummary), ExportedGUIDs(ExportedGUIDs),
736 LocalWPDTargetsMap(LocalWPDTargetsMap) {
740 bool tryFindVirtualCallTargets(std::vector<ValueInfo> &TargetsForSlot,
746 VTableSlotInfo &SlotInfo,
748 std::set<ValueInfo> &DevirtTargets);
766 if (UseCommandLine) {
767 if (!DevirtModule::runForTesting(M, AARGetter, OREGetter, LookupDomTree))
771 if (!DevirtModule(M, AARGetter, OREGetter, LookupDomTree, ExportSummary,
790 Module &M,
bool WholeProgramVisibilityEnabledInLTO,
798 if (GV.hasMetadata(LLVMContext::MD_type) &&
802 !DynamicExportSymbols.
count(GV.getGUID()))
808 bool WholeProgramVisibilityEnabledInLTO) {
811 if (!PublicTypeTestFunc)
817 auto *CI = cast<CallInst>(U.getUser());
819 TypeTestFunc, {CI->getArgOperand(0), CI->getArgOperand(1)},
820 std::nullopt,
"", CI);
822 CI->eraseFromParent();
827 auto *CI = cast<CallInst>(U.getUser());
828 CI->replaceAllUsesWith(True);
829 CI->eraseFromParent();
845 if (DynamicExportSymbols.
count(
P.first))
847 for (
auto &S :
P.second.SummaryList) {
848 auto *GVar = dyn_cast<GlobalVarSummary>(S.get());
859 std::map<
ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap) {
860 DevirtIndex(Summary, ExportedGUIDs, LocalWPDTargetsMap).run();
866 std::map<
ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap) {
867 for (
auto &
T : LocalWPDTargetsMap) {
870 assert(
VI.getSummaryList().size() == 1 &&
871 "Devirt of local target has more than one copy");
872 auto &S =
VI.getSummaryList()[0];
873 if (!isExported(S->modulePath(),
VI))
877 for (
auto &SlotSummary :
T.second) {
878 auto *TIdSum = Summary.getTypeIdSummary(SlotSummary.TypeID);
880 auto WPDRes = TIdSum->WPDRes.find(SlotSummary.ByteOffset);
881 assert(WPDRes != TIdSum->WPDRes.end());
883 WPDRes->second.SingleImplName,
884 Summary.getModuleHash(S->modulePath()));
896 const auto &ModPaths = Summary->modulePaths();
901 "combined summary should contain Regular LTO module");
905bool DevirtModule::runForTesting(
909 std::unique_ptr<ModuleSummaryIndex>
Summary =
910 std::make_unique<ModuleSummaryIndex>(
false);
917 auto ReadSummaryFile =
919 if (
Expected<std::unique_ptr<ModuleSummaryIndex>> SummaryOrErr =
921 Summary = std::move(*SummaryOrErr);
926 yaml::Input
In(ReadSummaryFile->getBuffer());
933 DevirtModule(M, AARGetter, OREGetter, LookupDomTree,
951 yaml::Output Out(
OS);
959void DevirtModule::buildTypeIdentifierMap(
960 std::vector<VTableBits> &Bits,
963 Bits.reserve(
M.global_size());
967 GV.getMetadata(LLVMContext::MD_type, Types);
968 if (GV.isDeclaration() ||
Types.empty())
974 Bits.back().GV = &GV;
975 Bits.back().ObjectSize =
976 M.getDataLayout().getTypeAllocSize(GV.getInitializer()->getType());
977 BitsPtr = &
Bits.back();
985 cast<ConstantAsMetadata>(
Type->getOperand(0))->getValue())
993bool DevirtModule::tryFindVirtualCallTargets(
994 std::vector<VirtualCallTarget> &TargetsForSlot,
995 const std::set<TypeMemberInfo> &TypeMemberInfos,
uint64_t ByteOffset,
998 if (!
TM.Bits->GV->isConstant())
1003 if (
TM.Bits->GV->getVCallVisibility() ==
1008 TM.Offset + ByteOffset, M,
TM.Bits->GV);
1012 auto C =
Ptr->stripPointerCasts();
1014 auto Fn = dyn_cast<Function>(
C);
1015 auto A = dyn_cast<GlobalAlias>(
C);
1017 Fn = dyn_cast<Function>(
A->getAliasee());
1022 if (FunctionsToSkip.match(Fn->getName()))
1027 if (Fn->getName() ==
"__cxa_pure_virtual")
1037 auto GV = dyn_cast<GlobalValue>(
C);
1039 TargetsForSlot.push_back({GV, &
TM});
1043 return !TargetsForSlot.empty();
1046bool DevirtIndex::tryFindVirtualCallTargets(
1061 bool LocalFound =
false;
1062 for (
const auto &S :
P.VTableVI.getSummaryList()) {
1068 auto *CurVS = cast<GlobalVarSummary>(S->getBaseObject());
1069 if (!CurVS->vTableFuncs().empty() ||
1091 for (
auto VTP :
VS->vTableFuncs()) {
1092 if (VTP.VTableOffset !=
P.AddressPointOffset + ByteOffset)
1098 TargetsForSlot.push_back(VTP.FuncVI);
1103 return !TargetsForSlot.empty();
1106void DevirtModule::applySingleImplDevirt(VTableSlotInfo &SlotInfo,
1107 Constant *TheFn,
bool &IsExported) {
1112 auto Apply = [&](CallSiteInfo &CSInfo) {
1113 for (
auto &&VCallSite : CSInfo.CallSites) {
1114 if (!OptimizedCalls.
insert(&VCallSite.CB).second)
1118 VCallSite.emitRemark(
"single-impl",
1121 auto &CB = VCallSite.CB;
1134 Builder.SetInsertPoint(ThenTerm);
1136 auto *CallTrap =
Builder.CreateCall(TrapFn);
1154 NewInst.
setMetadata(LLVMContext::MD_prof,
nullptr);
1155 NewInst.
setMetadata(LLVMContext::MD_callees,
nullptr);
1174 if (VCallSite.NumUnsafeUses)
1175 --*VCallSite.NumUnsafeUses;
1177 if (CSInfo.isExported())
1179 CSInfo.markDevirt();
1181 Apply(SlotInfo.CSInfo);
1182 for (
auto &
P : SlotInfo.ConstCSInfo)
1188 if (
Callee.getSummaryList().empty())
1195 bool IsExported =
false;
1196 auto &S =
Callee.getSummaryList()[0];
1198 auto AddCalls = [&](CallSiteInfo &CSInfo) {
1199 for (
auto *FS : CSInfo.SummaryTypeCheckedLoadUsers) {
1200 FS->addCall({
Callee, CI});
1201 IsExported |= S->modulePath() != FS->modulePath();
1203 for (
auto *FS : CSInfo.SummaryTypeTestAssumeUsers) {
1204 FS->addCall({
Callee, CI});
1205 IsExported |= S->modulePath() != FS->modulePath();
1209 for (
auto &
P : SlotInfo.ConstCSInfo)
1214bool DevirtModule::trySingleImplDevirt(
1220 auto *TheFn = TargetsForSlot[0].Fn;
1221 for (
auto &&
Target : TargetsForSlot)
1227 TargetsForSlot[0].WasDevirt =
true;
1229 bool IsExported =
false;
1230 applySingleImplDevirt(SlotInfo, TheFn, IsExported);
1237 if (TheFn->hasLocalLinkage()) {
1238 std::string NewName = (TheFn->
getName() +
".llvm.merged").str();
1243 if (
Comdat *
C = TheFn->getComdat()) {
1244 if (
C->getName() == TheFn->
getName()) {
1245 Comdat *NewC =
M.getOrInsertComdat(NewName);
1248 if (GO.getComdat() ==
C)
1270 VTableSlotInfo &SlotInfo,
1272 std::set<ValueInfo> &DevirtTargets) {
1275 auto TheFn = TargetsForSlot[0];
1276 for (
auto &&
Target : TargetsForSlot)
1281 auto Size = TheFn.getSummaryList().size();
1287 if (FunctionsToSkip.match(TheFn.name()))
1292 for (
const auto &S : TheFn.getSummaryList())
1298 DevirtTargets.insert(TheFn);
1300 auto &S = TheFn.getSummaryList()[0];
1301 bool IsExported =
AddCalls(SlotInfo, TheFn);
1303 ExportedGUIDs.insert(TheFn.getGUID());
1314 TheFn.name(), ExportSummary.
getModuleHash(S->modulePath()));
1316 LocalWPDTargetsMap[TheFn].push_back(SlotSummary);
1330void DevirtModule::tryICallBranchFunnel(
1340 bool HasNonDevirt = !SlotInfo.CSInfo.AllCallSitesDevirted;
1342 for (
auto &
P : SlotInfo.ConstCSInfo)
1343 if (!
P.second.AllCallSitesDevirted) {
1344 HasNonDevirt =
true;
1354 if (isa<MDString>(
Slot.TypeID)) {
1356 M.getDataLayout().getProgramAddressSpace(),
1357 getGlobalName(Slot, {},
"branch_funnel"), &M);
1361 M.getDataLayout().getProgramAddressSpace(),
1362 "branch_funnel", &M);
1364 JT->addParamAttr(0, Attribute::Nest);
1366 std::vector<Value *> JTArgs;
1367 JTArgs.push_back(
JT->arg_begin());
1368 for (
auto &
T : TargetsForSlot) {
1369 JTArgs.push_back(getMemberAddr(
T.TM));
1370 JTArgs.push_back(
T.Fn);
1381 bool IsExported =
false;
1382 applyICallBranchFunnel(SlotInfo, JT, IsExported);
1387void DevirtModule::applyICallBranchFunnel(VTableSlotInfo &SlotInfo,
1389 auto Apply = [&](CallSiteInfo &CSInfo) {
1390 if (CSInfo.isExported())
1392 if (CSInfo.AllCallSitesDevirted)
1394 for (
auto &&VCallSite : CSInfo.CallSites) {
1405 VCallSite.emitRemark(
"branch-funnel",
1406 JT->stripPointerCasts()->getName(), OREGetter);
1410 std::vector<Type *> NewArgs;
1411 NewArgs.push_back(Int8PtrTy);
1419 std::vector<Value *>
Args;
1420 Args.push_back(IRB.CreateBitCast(VCallSite.VTable, Int8PtrTy));
1424 if (isa<CallInst>(CB))
1425 NewCS = IRB.CreateCall(NewFT, IRB.CreateBitCast(JT, NewFTPtr), Args);
1427 NewCS = IRB.CreateInvoke(NewFT, IRB.CreateBitCast(JT, NewFTPtr),
1428 cast<InvokeInst>(CB).getNormalDest(),
1429 cast<InvokeInst>(CB).getUnwindDest(), Args);
1433 std::vector<AttributeSet> NewArgAttrs;
1436 M.getContext(), Attribute::Nest)}));
1437 for (
unsigned I = 0;
I + 2 <
Attrs.getNumAttrSets(); ++
I)
1438 NewArgAttrs.push_back(
Attrs.getParamAttrs(
I));
1441 Attrs.getRetAttrs(), NewArgAttrs));
1447 if (VCallSite.NumUnsafeUses)
1448 --*VCallSite.NumUnsafeUses;
1455 Apply(SlotInfo.CSInfo);
1456 for (
auto &
P : SlotInfo.ConstCSInfo)
1460bool DevirtModule::tryEvaluateFunctionsWithArgs(
1469 auto Fn = dyn_cast<Function>(
Target.Fn);
1473 if (Fn->arg_size() !=
Args.size() + 1)
1480 for (
unsigned I = 0;
I !=
Args.size(); ++
I) {
1482 dyn_cast<IntegerType>(Fn->getFunctionType()->getParamType(
I + 1));
1489 if (!Eval.EvaluateFunction(Fn, RetVal, EvalArgs) ||
1490 !isa<ConstantInt>(RetVal))
1492 Target.RetVal = cast<ConstantInt>(RetVal)->getZExtValue();
1497void DevirtModule::applyUniformRetValOpt(CallSiteInfo &CSInfo,
StringRef FnName,
1499 for (
auto Call : CSInfo.CallSites) {
1503 Call.replaceAndErase(
1504 "uniform-ret-val", FnName, RemarksEnabled, OREGetter,
1507 CSInfo.markDevirt();
1510bool DevirtModule::tryUniformRetValOpt(
1515 uint64_t TheRetVal = TargetsForSlot[0].RetVal;
1517 if (
Target.RetVal != TheRetVal)
1520 if (CSInfo.isExported()) {
1522 Res->
Info = TheRetVal;
1525 applyUniformRetValOpt(CSInfo, TargetsForSlot[0].Fn->getName(), TheRetVal);
1527 for (
auto &&
Target : TargetsForSlot)
1532std::string DevirtModule::getGlobalName(VTableSlot Slot,
1535 std::string FullName =
"__typeid_";
1537 OS << cast<MDString>(
Slot.TypeID)->getString() <<
'_' <<
Slot.ByteOffset;
1544bool DevirtModule::shouldExportConstantsAsAbsoluteSymbols() {
1552 getGlobalName(Slot, Args,
Name),
C, &M);
1559 if (shouldExportConstantsAsAbsoluteSymbols()) {
1572 M.getOrInsertGlobal(getGlobalName(Slot, Args,
Name), Int8Arr0Ty);
1573 auto *GV = dyn_cast<GlobalVariable>(
C);
1582 if (!shouldExportConstantsAsAbsoluteSymbols())
1586 auto *GV = cast<GlobalVariable>(
C->stripPointerCasts());
1591 if (GV->hasMetadata(LLVMContext::MD_absolute_symbol))
1597 GV->setMetadata(LLVMContext::MD_absolute_symbol,
1602 SetAbsRange(~0ull, ~0ull);
1604 SetAbsRange(0, 1ull << AbsWidth);
1608void DevirtModule::applyUniqueRetValOpt(CallSiteInfo &CSInfo,
StringRef FnName,
1611 for (
auto &&Call : CSInfo.CallSites) {
1617 B.CreateBitCast(UniqueMemberAddr,
Call.VTable->getType()));
1618 Cmp =
B.CreateZExt(Cmp,
Call.CB.getType());
1620 Call.replaceAndErase(
"unique-ret-val", FnName, RemarksEnabled, OREGetter,
1623 CSInfo.markDevirt();
1632bool DevirtModule::tryUniqueRetValOpt(
1637 auto tryUniqueRetValOptFor = [&](
bool IsOne) {
1640 if (
Target.RetVal == (IsOne ? 1 : 0)) {
1643 UniqueMember =
Target.TM;
1651 Constant *UniqueMemberAddr = getMemberAddr(UniqueMember);
1652 if (CSInfo.isExported()) {
1656 exportGlobal(Slot, Args,
"unique_member", UniqueMemberAddr);
1660 applyUniqueRetValOpt(CSInfo, TargetsForSlot[0].Fn->getName(), IsOne,
1665 for (
auto &&
Target : TargetsForSlot)
1672 if (tryUniqueRetValOptFor(
true))
1674 if (tryUniqueRetValOptFor(
false))
1680void DevirtModule::applyVirtualConstProp(CallSiteInfo &CSInfo,
StringRef FnName,
1682 for (
auto Call : CSInfo.CallSites) {
1685 auto *RetType = cast<IntegerType>(
Call.CB.getType());
1688 B.CreateGEP(Int8Ty,
B.CreateBitCast(
Call.VTable, Int8PtrTy), Byte);
1689 if (RetType->getBitWidth() == 1) {
1691 Value *BitsAndBit =
B.CreateAnd(Bits, Bit);
1693 NumVirtConstProp1Bit++;
1694 Call.replaceAndErase(
"virtual-const-prop-1-bit", FnName, RemarksEnabled,
1695 OREGetter, IsBitSet);
1697 Value *ValAddr =
B.CreateBitCast(
Addr, RetType->getPointerTo());
1698 Value *Val =
B.CreateLoad(RetType, ValAddr);
1700 Call.replaceAndErase(
"virtual-const-prop", FnName, RemarksEnabled,
1704 CSInfo.markDevirt();
1707bool DevirtModule::tryVirtualConstProp(
1713 auto Fn = dyn_cast<Function>(TargetsForSlot[0].Fn);
1717 auto RetType = dyn_cast<IntegerType>(Fn->getReturnType());
1720 unsigned BitWidth = RetType->getBitWidth();
1738 auto Fn = dyn_cast<Function>(
Target.Fn);
1742 if (Fn->isDeclaration() ||
1744 .doesNotAccessMemory() ||
1745 Fn->arg_empty() || !Fn->arg_begin()->use_empty() ||
1746 Fn->getReturnType() != RetType)
1750 for (
auto &&CSByConstantArg : SlotInfo.ConstCSInfo) {
1751 if (!tryEvaluateFunctionsWithArgs(TargetsForSlot, CSByConstantArg.first))
1756 ResByArg = &Res->
ResByArg[CSByConstantArg.first];
1758 if (tryUniformRetValOpt(TargetsForSlot, CSByConstantArg.second, ResByArg))
1761 if (tryUniqueRetValOpt(
BitWidth, TargetsForSlot, CSByConstantArg.second,
1762 ResByArg, Slot, CSByConstantArg.first))
1774 uint64_t TotalPaddingBefore = 0, TotalPaddingAfter = 0;
1775 for (
auto &&
Target : TargetsForSlot) {
1776 TotalPaddingBefore += std::max<int64_t>(
1777 (AllocBefore + 7) / 8 -
Target.allocatedBeforeBytes() - 1, 0);
1778 TotalPaddingAfter += std::max<int64_t>(
1779 (AllocAfter + 7) / 8 -
Target.allocatedAfterBytes() - 1, 0);
1784 if (std::min(TotalPaddingBefore, TotalPaddingAfter) > 128)
1791 if (TotalPaddingBefore <= TotalPaddingAfter)
1799 for (
auto &&
Target : TargetsForSlot)
1803 if (CSByConstantArg.second.isExported()) {
1805 exportConstant(Slot, CSByConstantArg.first,
"byte", OffsetByte,
1807 exportConstant(Slot, CSByConstantArg.first,
"bit", 1ULL << OffsetBit,
1814 applyVirtualConstProp(CSByConstantArg.second,
1815 TargetsForSlot[0].Fn->getName(), ByteConst, BitConst);
1821 if (
B.Before.Bytes.empty() &&
B.After.Bytes.empty())
1826 Align Alignment =
M.getDataLayout().getValueOrABITypeAlignment(
1827 B.GV->getAlign(),
B.GV->getValueType());
1828 B.Before.Bytes.resize(
alignTo(
B.Before.Bytes.size(), Alignment));
1831 for (
size_t I = 0,
Size =
B.Before.Bytes.size();
I !=
Size / 2; ++
I)
1838 B.GV->getInitializer(),
1843 NewGV->setSection(
B.GV->getSection());
1844 NewGV->setComdat(
B.GV->getComdat());
1845 NewGV->setAlignment(
B.GV->getAlign());
1849 NewGV->copyMetadata(
B.GV,
B.Before.Bytes.size());
1854 B.GV->getInitializer()->getType(), 0,
B.GV->getLinkage(),
"",
1856 NewInit->getType(), NewGV,
1858 ConstantInt::get(Int32Ty, 1)}),
1860 Alias->setVisibility(
B.GV->getVisibility());
1861 Alias->takeName(
B.GV);
1863 B.GV->replaceAllUsesWith(Alias);
1864 B.GV->eraseFromParent();
1867bool DevirtModule::areRemarksEnabled() {
1868 const auto &FL =
M.getFunctionList();
1873 return DI.isEnabled();
1878void DevirtModule::scanTypeTestUsers(
1887 auto *CI = dyn_cast<CallInst>(
U.getUser());
1894 auto &DT = LookupDomTree(*CI->getFunction());
1898 cast<MetadataAsValue>(CI->getArgOperand(1))->getMetadata();
1900 if (!Assumes.
empty()) {
1901 Value *
Ptr = CI->getArgOperand(0)->stripPointerCasts();
1903 CallSlots[{TypeId,
Call.Offset}].addCallSite(
Ptr,
Call.CB,
nullptr);
1906 auto RemoveTypeTestAssumes = [&]() {
1908 for (
auto *Assume : Assumes)
1909 Assume->eraseFromParent();
1912 if (CI->use_empty())
1913 CI->eraseFromParent();
1928 if (!TypeIdMap.count(TypeId))
1929 RemoveTypeTestAssumes();
1940 else if (ImportSummary && isa<MDString>(TypeId)) {
1944 RemoveTypeTestAssumes();
1953void DevirtModule::scanTypeCheckedLoadUsers(
Function *TypeCheckedLoadFunc) {
1957 auto *CI = dyn_cast<CallInst>(
U.getUser());
1963 Value *TypeIdValue = CI->getArgOperand(2);
1964 Metadata *TypeId = cast<MetadataAsValue>(TypeIdValue)->getMetadata();
1969 bool HasNonCallUses =
false;
1970 auto &DT = LookupDomTree(*CI->getFunction());
1972 HasNonCallUses, CI, DT);
1981 (LoadedPtrs.
size() == 1 && !HasNonCallUses) ? LoadedPtrs[0] : CI);
1984 Value *LoadedValue = LoadB.CreateLoad(Int8PtrTy, GEPPtr);
1988 LoadedPtr->eraseFromParent();
1992 IRBuilder<> CallB((Preds.
size() == 1 && !HasNonCallUses) ? Preds[0] : CI);
1993 CallInst *TypeTestCall = CallB.CreateCall(TypeTestFunc, {
Ptr, TypeIdValue});
1997 Pred->eraseFromParent();
2004 if (!CI->use_empty()) {
2007 Pair =
B.CreateInsertValue(Pair, LoadedValue, {0});
2008 Pair =
B.CreateInsertValue(Pair, TypeTestCall, {1});
2009 CI->replaceAllUsesWith(Pair);
2013 auto &NumUnsafeUses = NumUnsafeUsesForTypeTest[TypeTestCall];
2014 NumUnsafeUses = DevirtCalls.
size();
2022 CallSlots[{TypeId,
Call.Offset}].addCallSite(
Ptr,
Call.CB,
2026 CI->eraseFromParent();
2030void DevirtModule::importResolution(VTableSlot Slot, VTableSlotInfo &SlotInfo) {
2031 auto *TypeId = dyn_cast<MDString>(
Slot.TypeID);
2038 auto ResI = TidSummary->
WPDRes.find(
Slot.ByteOffset);
2039 if (ResI == TidSummary->
WPDRes.end())
2053 bool IsExported =
false;
2054 applySingleImplDevirt(SlotInfo, SingleImpl, IsExported);
2058 for (
auto &CSByConstantArg : SlotInfo.ConstCSInfo) {
2059 auto I = Res.
ResByArg.find(CSByConstantArg.first);
2062 auto &ResByArg =
I->second;
2069 applyUniformRetValOpt(CSByConstantArg.second,
"", ResByArg.
Info);
2073 importGlobal(Slot, CSByConstantArg.first,
"unique_member");
2074 applyUniqueRetValOpt(CSByConstantArg.second,
"", ResByArg.
Info,
2079 Constant *
Byte = importConstant(Slot, CSByConstantArg.first,
"byte",
2081 Constant *
Bit = importConstant(Slot, CSByConstantArg.first,
"bit", Int8Ty,
2083 applyVirtualConstProp(CSByConstantArg.second,
"", Byte, Bit);
2095 M.getOrInsertFunction(getGlobalName(Slot, {},
"branch_funnel"),
2098 bool IsExported =
false;
2099 applyICallBranchFunnel(SlotInfo, JT, IsExported);
2104void DevirtModule::removeRedundantTypeTests() {
2106 for (
auto &&U : NumUnsafeUsesForTypeTest) {
2107 if (
U.second == 0) {
2108 U.first->replaceAllUsesWith(True);
2109 U.first->eraseFromParent();
2115DevirtModule::lookUpFunctionValueInfo(
Function *TheFn,
2117 assert((ExportSummary !=
nullptr) &&
2118 "Caller guarantees ExportSummary is not nullptr");
2120 const auto TheFnGUID = TheFn->
getGUID();
2132 if ((!TheFnVI) && (TheFnGUID != TheFnGUIDWithExportedName)) {
2133 TheFnVI = ExportSummary->
getValueInfo(TheFnGUIDWithExportedName);
2138bool DevirtModule::mustBeUnreachableFunction(
2141 if (!
F->isDeclaration()) {
2144 return isa<UnreachableInst>(
F->getEntryBlock().getTerminator());
2147 return ExportSummary &&
2149 DevirtModule::lookUpFunctionValueInfo(
F, ExportSummary));
2152bool DevirtModule::run() {
2170 if (!ExportSummary &&
2171 (!TypeTestFunc || TypeTestFunc->
use_empty() || !AssumeFunc ||
2173 (!TypeCheckedLoadFunc || TypeCheckedLoadFunc->
use_empty()))
2177 std::vector<VTableBits>
Bits;
2179 buildTypeIdentifierMap(Bits, TypeIdMap);
2181 if (TypeTestFunc && AssumeFunc)
2182 scanTypeTestUsers(TypeTestFunc, TypeIdMap);
2184 if (TypeCheckedLoadFunc)
2185 scanTypeCheckedLoadUsers(TypeCheckedLoadFunc);
2187 if (ImportSummary) {
2188 for (
auto &S : CallSlots)
2189 importResolution(S.first, S.second);
2191 removeRedundantTypeTests();
2197 GV.eraseMetadata(LLVMContext::MD_vcall_visibility);
2204 if (TypeIdMap.
empty())
2208 if (ExportSummary) {
2210 for (
auto &
P : TypeIdMap) {
2211 if (
auto *TypeId = dyn_cast<MDString>(
P.first))
2216 for (
auto &
P : *ExportSummary) {
2217 for (
auto &S :
P.second.SummaryList) {
2218 auto *
FS = dyn_cast<FunctionSummary>(S.get());
2223 for (
Metadata *MD : MetadataByGUID[VF.GUID]) {
2224 CallSlots[{MD, VF.Offset}].CSInfo.addSummaryTypeTestAssumeUser(FS);
2228 for (
Metadata *MD : MetadataByGUID[VF.GUID]) {
2229 CallSlots[{MD, VF.Offset}].CSInfo.addSummaryTypeCheckedLoadUser(FS);
2233 FS->type_test_assume_const_vcalls()) {
2234 for (
Metadata *MD : MetadataByGUID[
VC.VFunc.GUID]) {
2235 CallSlots[{MD,
VC.VFunc.Offset}]
2236 .ConstCSInfo[
VC.Args]
2237 .addSummaryTypeTestAssumeUser(FS);
2241 FS->type_checked_load_const_vcalls()) {
2242 for (
Metadata *MD : MetadataByGUID[
VC.VFunc.GUID]) {
2243 CallSlots[{MD,
VC.VFunc.Offset}]
2244 .ConstCSInfo[
VC.Args]
2245 .addSummaryTypeCheckedLoadUser(FS);
2253 bool DidVirtualConstProp =
false;
2254 std::map<std::string, GlobalValue *> DevirtTargets;
2255 for (
auto &S : CallSlots) {
2259 std::vector<VirtualCallTarget> TargetsForSlot;
2261 const std::set<TypeMemberInfo> &TypeMemberInfos = TypeIdMap[S.first.TypeID];
2262 if (ExportSummary && isa<MDString>(S.first.TypeID) &&
2263 TypeMemberInfos.size())
2270 Res = &ExportSummary
2271 ->getOrInsertTypeIdSummary(
2272 cast<MDString>(S.first.TypeID)->getString())
2273 .WPDRes[S.first.ByteOffset];
2274 if (tryFindVirtualCallTargets(TargetsForSlot, TypeMemberInfos,
2275 S.first.ByteOffset, ExportSummary)) {
2277 if (!trySingleImplDevirt(ExportSummary, TargetsForSlot, S.second, Res)) {
2278 DidVirtualConstProp |=
2279 tryVirtualConstProp(TargetsForSlot, S.second, Res, S.first);
2281 tryICallBranchFunnel(TargetsForSlot, S.second, Res, S.first);
2286 for (
const auto &
T : TargetsForSlot)
2288 DevirtTargets[std::string(
T.Fn->getName())] =
T.Fn;
2295 if (ExportSummary && isa<MDString>(S.first.TypeID)) {
2298 for (
auto *FS : S.second.CSInfo.SummaryTypeCheckedLoadUsers)
2299 FS->addTypeTest(GUID);
2300 for (
auto &CCS : S.second.ConstCSInfo)
2301 for (
auto *FS : CCS.second.SummaryTypeCheckedLoadUsers)
2302 FS->addTypeTest(GUID);
2306 if (RemarksEnabled) {
2308 for (
const auto &DT : DevirtTargets) {
2310 auto F = dyn_cast<Function>(GV);
2312 auto A = dyn_cast<GlobalAlias>(GV);
2313 assert(
A && isa<Function>(
A->getAliasee()));
2314 F = dyn_cast<Function>(
A->getAliasee());
2318 using namespace ore;
2321 <<
NV(
"FunctionName", DT.first));
2325 NumDevirtTargets += DevirtTargets.size();
2327 removeRedundantTypeTests();
2331 if (DidVirtualConstProp)
2344void DevirtIndex::run() {
2345 if (ExportSummary.typeIdCompatibleVtableMap().empty())
2349 for (
const auto &
P : ExportSummary.typeIdCompatibleVtableMap()) {
2357 ExportSummary.getOrInsertTypeIdSummary(
P.first);
2361 for (
auto &
P : ExportSummary) {
2362 for (
auto &S :
P.second.SummaryList) {
2363 auto *
FS = dyn_cast<FunctionSummary>(S.get());
2369 CallSlots[{
Name, VF.Offset}].CSInfo.addSummaryTypeTestAssumeUser(FS);
2374 CallSlots[{
Name, VF.Offset}].CSInfo.addSummaryTypeCheckedLoadUser(FS);
2378 FS->type_test_assume_const_vcalls()) {
2380 CallSlots[{
Name,
VC.VFunc.Offset}]
2381 .ConstCSInfo[
VC.Args]
2382 .addSummaryTypeTestAssumeUser(FS);
2386 FS->type_checked_load_const_vcalls()) {
2388 CallSlots[{
Name,
VC.VFunc.Offset}]
2389 .ConstCSInfo[
VC.Args]
2390 .addSummaryTypeCheckedLoadUser(FS);
2396 std::set<ValueInfo> DevirtTargets;
2398 for (
auto &S : CallSlots) {
2402 std::vector<ValueInfo> TargetsForSlot;
2403 auto TidSummary = ExportSummary.getTypeIdCompatibleVtableSummary(S.first.TypeID);
2408 &ExportSummary.getTypeIdSummary(S.first.TypeID)
2409 ->WPDRes[S.first.ByteOffset];
2410 if (tryFindVirtualCallTargets(TargetsForSlot, *TidSummary,
2411 S.first.ByteOffset)) {
2413 if (!trySingleImplDevirt(TargetsForSlot, S.first, S.second, Res,
2422 for (
const auto &DT : DevirtTargets)
2423 errs() <<
"Devirtualized call to " << DT <<
"\n";
2425 NumDevirtTargets += DevirtTargets.size();
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
static const Function * getParent(const Value *V)
This is the interface for LLVM's primary stateless and local alias analysis.
SmallVector< MachineOperand, 4 > Cond
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file defines DenseMapInfo traits for DenseMap.
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
Provides passes for computing function attributes based on interprocedural analyses.
static cl::opt< std::string > ClReadSummary("lowertypetests-read-summary", cl::desc("Read summary from given YAML file before running pass"), cl::Hidden)
static cl::opt< PassSummaryAction > ClSummaryAction("lowertypetests-summary-action", cl::desc("What to do with the summary when running this pass"), cl::values(clEnumValN(PassSummaryAction::None, "none", "Do nothing"), clEnumValN(PassSummaryAction::Import, "import", "Import typeid resolutions from summary and globals"), clEnumValN(PassSummaryAction::Export, "export", "Export typeid resolutions to summary and globals")), cl::Hidden)
static cl::opt< std::string > ClWriteSummary("lowertypetests-write-summary", cl::desc("Write summary to given YAML file after running pass"), cl::Hidden)
This file implements a map that provides insertion order iteration.
static bool mustBeUnreachableFunction(const Function &F)
Module.h This file contains the declarations for the Module class.
FunctionAnalysisManager FAM
const char LLVMTargetMachineRef TM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static cl::opt< bool > DisableWholeProgramVisibility("disable-whole-program-visibility", cl::Hidden, cl::desc("Disable whole program visibility (overrides enabling options)"))
Provide a way to force disable whole program for debugging or workarounds, when enabled via the linke...
WPDCheckMode
Mechanism to add runtime checking of devirtualization decisions, optionally trapping or falling back ...
static cl::opt< PassSummaryAction > ClSummaryAction("wholeprogramdevirt-summary-action", cl::desc("What to do with the summary when running this pass"), cl::values(clEnumValN(PassSummaryAction::None, "none", "Do nothing"), clEnumValN(PassSummaryAction::Import, "import", "Import typeid resolutions from summary and globals"), clEnumValN(PassSummaryAction::Export, "export", "Export typeid resolutions to summary and globals")), cl::Hidden)
static cl::opt< bool > WholeProgramVisibility("whole-program-visibility", cl::Hidden, cl::desc("Enable whole program visibility"))
Provide a way to force enable whole program visibility in tests.
static Error checkCombinedSummaryForTesting(ModuleSummaryIndex *Summary)
static cl::list< std::string > SkipFunctionNames("wholeprogramdevirt-skip", cl::desc("Prevent function(s) from being devirtualized"), cl::Hidden, cl::CommaSeparated)
Provide way to prevent certain function from being devirtualized.
static cl::opt< std::string > ClWriteSummary("wholeprogramdevirt-write-summary", cl::desc("Write summary to given bitcode or YAML file after running pass. " "Output file format is deduced from extension: *.bc means writing " "bitcode, otherwise YAML"), cl::Hidden)
static cl::opt< unsigned > ClThreshold("wholeprogramdevirt-branch-funnel-threshold", cl::Hidden, cl::init(10), cl::desc("Maximum number of call targets per " "call site to enable branch funnels"))
static cl::opt< WPDCheckMode > DevirtCheckMode("wholeprogramdevirt-check", cl::Hidden, cl::desc("Type of checking for incorrect devirtualizations"), cl::values(clEnumValN(WPDCheckMode::None, "none", "No checking"), clEnumValN(WPDCheckMode::Trap, "trap", "Trap when incorrect"), clEnumValN(WPDCheckMode::Fallback, "fallback", "Fallback to indirect when incorrect")))
static cl::opt< std::string > ClReadSummary("wholeprogramdevirt-read-summary", cl::desc("Read summary from given bitcode or YAML file before running pass"), cl::Hidden)
static bool AddCalls(VTableSlotInfo &SlotInfo, const ValueInfo &Callee)
static cl::opt< bool > PrintSummaryDevirt("wholeprogramdevirt-print-index-based", cl::Hidden, cl::desc("Print index-based devirtualization messages"))
A manager for alias analyses.
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
static AttributeSet get(LLVMContext &C, const AttrBuilder &B)
StringRef getValueAsString() const
Return the attribute's value as a string.
bool isValid() const
Return true if the attribute is any kind of attribute.
LLVM Basic Block Representation.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void setCallingConv(CallingConv::ID CC)
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
CallingConv::ID getCallingConv() const
Value * getCalledOperand() const
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
FunctionType * getFunctionType() const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void setCalledOperand(Value *V)
AttributeList getAttributes() const
Return the parameter attributes for this call.
Function * getCaller()
Helper to get the caller (the parent function).
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
void setSelectionKind(SelectionKind Val)
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
static Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList, bool InBounds=false, std::optional< unsigned > InRangeIndex=std::nullopt, Type *OnlyIfReducedTy=nullptr)
Getelementptr form.
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static ConstantInt * getTrue(LLVMContext &Context)
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static Constant * getAnon(ArrayRef< Constant * > V, bool Packed=false)
Return an anonymous struct that has the specified elements.
This is an important base class in LLVM.
const Constant * stripPointerCasts() const
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Implements a dense probed hash-table based set.
Analysis pass which computes a DominatorTree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Subclass of Error for the sole purpose of identifying the success path in the type system.
Lightweight error class with error context and mandatory checking.
This class evaluates LLVM IR, producing the Constant representing each SSA instruction.
Helper for check-and-exit error handling.
Tagged union holding either a T or a Error.
Function summary information to aid decisions and implementation of importing.
ArrayRef< Type * > params() const
Type * getReturnType() const
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
static Expected< GlobPattern > create(StringRef Pat)
static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
@ VCallVisibilityLinkageUnit
static bool isLocalLinkage(LinkageTypes Linkage)
static bool isAvailableExternallyLinkage(LinkageTypes Linkage)
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
@ HiddenVisibility
The GV is hidden.
static GUID getGUID(StringRef GlobalName)
Return a 64-bit global unique ID constructed from global value name (i.e.
void setVisibility(VisibilityTypes V)
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ InternalLinkage
Rename collisions when linking (static functions).
@ ExternalLinkage
Externally visible function.
Global variable summary information to aid decisions and implementation of importing.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const BasicBlock * getParent() const
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Class to represent integer types.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight)
Return metadata containing two branch weights.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
This class implements a map that also provides access to all stored values in a deterministic order.
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFile(const Twine &Filename, bool IsText=false, bool RequiresNullTerminator=true, bool IsVolatile=false, std::optional< Align > Alignment=std::nullopt)
Open the specified file as a MemoryBuffer, returning a new MemoryBuffer if successful,...
Class to hold module path string table and global value map, and encapsulate methods for operating on...
const TypeIdSummary * getTypeIdSummary(StringRef TypeId) const
This returns either a pointer to the type id summary (if present in the summary map) or null (if not ...
ValueInfo getValueInfo(const GlobalValueSummaryMapTy::value_type &R) const
Return a ValueInfo for the index value_type (convenient when iterating index).
const ModuleHash & getModuleHash(const StringRef ModPath) const
Get the module SHA1 hash recorded for the given module path.
static constexpr const char * getRegularLTOModuleName()
bool partiallySplitLTOUnits() const
static std::string getGlobalNameForLocal(StringRef Name, ModuleHash ModHash)
Convenience method for creating a promoted global name for the given value name of a local,...
A Module instance is used to store all the information related to an LLVM module.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, Instruction *InsertBefore=nullptr)
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
bool contains(StringRef Other) const
Return true if the given string is a substring of *this, and false otherwise.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
The instances of the Type class are immutable: once they are created, they are never changed.
TypeID
Definitions of all of the base types for the Type system.
static Type * getVoidTy(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
bool eraseMetadata(unsigned KindID)
Erase all metadata attachments with the given kind.
iterator_range< use_iterator > uses()
StringRef getName() const
Return a constant reference to the value's name.
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to a file descriptor.
A raw_ostream that writes to an std::string.
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
@ C
The default llvm calling convention, compatible with C.
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
bool match(Val *V, const Pattern &P)
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
DiagnosticInfoOptimizationBase::Argument NV
@ OF_TextWithCRLF
The file should be opened in text mode and use a carriage linefeed '\r '.
uint64_t findLowestOffset(ArrayRef< VirtualCallTarget > Targets, bool IsAfter, uint64_t Size)
void setAfterReturnValues(MutableArrayRef< VirtualCallTarget > Targets, uint64_t AllocAfter, unsigned BitWidth, int64_t &OffsetByte, uint64_t &OffsetBit)
void setBeforeReturnValues(MutableArrayRef< VirtualCallTarget > Targets, uint64_t AllocBefore, unsigned BitWidth, int64_t &OffsetByte, uint64_t &OffsetBit)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
MemoryEffects computeFunctionBodyMemoryAccess(Function &F, AAResults &AAR)
Returns the memory access properties of this copy of the function.
void updateVCallVisibilityInModule(Module &M, bool WholeProgramVisibilityEnabledInLTO, const DenseSet< GlobalValue::GUID > &DynamicExportSymbols)
If whole program visibility asserted, then upgrade all public vcall visibility metadata on vtable def...
void writeIndexToFile(const ModuleSummaryIndex &Index, raw_ostream &Out, const std::map< std::string, GVSummaryMapTy > *ModuleToSummariesForIndex=nullptr)
Write the specified module summary index to the given raw output stream, where it will be written in ...
@ Export
Export information to summary.
@ Import
Import information from summary.
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Error createStringError(std::error_code EC, char const *Fmt, const Ts &... Vals)
Create formatted StringError object.
bool hasWholeProgramVisibility(bool WholeProgramVisibilityEnabledInLTO)
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Expected< std::unique_ptr< ModuleSummaryIndex > > getModuleSummaryIndex(MemoryBufferRef Buffer)
Parse the specified bitcode buffer, returning the module summary index.
void updatePublicTypeTestCalls(Module &M, bool WholeProgramVisibilityEnabledInLTO)
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void findDevirtualizableCallsForTypeCheckedLoad(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< Instruction * > &LoadedPtrs, SmallVectorImpl< Instruction * > &Preds, bool &HasNonCallUses, const CallInst *CI, DominatorTree &DT)
Given a call to the intrinsic @llvm.type.checked.load, find all devirtualizable call sites based on t...
CallBase & versionCallSite(CallBase &CB, Value *Callee, MDNode *BranchWeights)
Predicate and clone the given call site.
bool AreStatisticsEnabled()
Check if statistics are enabled.
void updateIndexWPDForExports(ModuleSummaryIndex &Summary, function_ref< bool(StringRef, ValueInfo)> isExported, std::map< ValueInfo, std::vector< VTableSlotSummary > > &LocalWPDTargetsMap)
Call after cross-module importing to update the recorded single impl devirt target names for any loca...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
void updateVCallVisibilityInIndex(ModuleSummaryIndex &Index, bool WholeProgramVisibilityEnabledInLTO, const DenseSet< GlobalValue::GUID > &DynamicExportSymbols)
If whole program visibility asserted, then upgrade all public vcall visibility metadata on vtable def...
void runWholeProgramDevirtOnIndex(ModuleSummaryIndex &Summary, std::set< GlobalValue::GUID > &ExportedGUIDs, std::map< ValueInfo, std::vector< VTableSlotSummary > > &LocalWPDTargetsMap)
Perform index-based whole program devirtualization on the Summary index.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Expected< T > errorOrToExpected(ErrorOr< T > &&EO)
Convert an ErrorOr<T> to an Expected<T>.
constexpr unsigned BitWidth
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
Instruction * SplitBlockAndInsertIfThen(Value *Cond, Instruction *SplitBefore, bool Unreachable, MDNode *BranchWeights, DominatorTree *DT, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
std::vector< TypeIdOffsetVtableInfo > TypeIdCompatibleVtableInfo
List of vtable definitions decorated by a particular type identifier, and their corresponding offsets...
void consumeError(Error Err)
Consume a Error without doing anything.
void findDevirtualizableCallsForTypeTest(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< CallInst * > &Assumes, const CallInst *CI, DominatorTree &DT)
Given a call to the intrinsic @llvm.type.test, find all devirtualizable call sites based on the call ...
Constant * getPointerAtOffset(Constant *I, uint64_t Offset, Module &M, Constant *TopLevelGlobal=nullptr)
Processes a Constant recursively looking into elements of arrays, structs and expressions to find a t...
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Class to accumulate and hold information about a callee.
static unsigned getHashValue(const VTableSlotSummary &I)
static bool isEqual(const VTableSlotSummary &LHS, const VTableSlotSummary &RHS)
static VTableSlotSummary getTombstoneKey()
static VTableSlotSummary getEmptyKey()
static VTableSlot getTombstoneKey()
static VTableSlot getEmptyKey()
static bool isEqual(const VTableSlot &LHS, const VTableSlot &RHS)
static unsigned getHashValue(const VTableSlot &I)
An information struct used to provide DenseMap with the various necessary components for a given valu...
A call site that could be devirtualized.
A specification for a virtual function call with all constant integer arguments.
An "identifier" for a virtual function.
The following data structures summarize type metadata information.
std::map< uint64_t, WholeProgramDevirtResolution > WPDRes
Mapping from byte offset to whole-program devirt resolution for that (typeid, byte offset) pair.
@ Unsat
Unsatisfiable type (i.e. no global has this type metadata)
enum llvm::TypeTestResolution::Kind TheKind
Struct that holds a reference to a particular GUID in a global value summary.
ArrayRef< std::unique_ptr< GlobalValueSummary > > getSummaryList() const
PreservedAnalyses run(Module &M, ModuleAnalysisManager &)
@ UniformRetVal
Uniform return value optimization.
@ VirtualConstProp
Virtual constant propagation.
@ UniqueRetVal
Unique return value optimization.
uint64_t Info
Additional information for the resolution:
enum llvm::WholeProgramDevirtResolution::ByArg::Kind TheKind
enum llvm::WholeProgramDevirtResolution::Kind TheKind
std::map< std::vector< uint64_t >, ByArg > ResByArg
Resolutions for calls with all constant integer arguments (excluding the first argument,...
std::string SingleImplName
@ SingleImpl
Single implementation devirtualization.
@ BranchFunnel
When retpoline mitigation is enabled, use a branch funnel that is defined in the merged module.
VirtualCallTarget(GlobalValue *Fn, const TypeMemberInfo *TM)