31#include "llvm/Config/llvm-config.h"
88 cl::desc(
"Number of metadatas above which we emit an index "
89 "to enable lazy-loading"));
92 cl::desc(
"The threshold (unit M) for flushing LLVM bitcode."));
96 cl::desc(
"Write relative block frequency to function summary "));
111 VST_BBENTRY_6_ABBREV,
115 CONSTANTS_INTEGER_ABBREV,
116 CONSTANTS_CE_CAST_Abbrev,
117 CONSTANTS_NULL_Abbrev,
121 FUNCTION_INST_UNOP_ABBREV,
122 FUNCTION_INST_UNOP_FLAGS_ABBREV,
123 FUNCTION_INST_BINOP_ABBREV,
124 FUNCTION_INST_BINOP_FLAGS_ABBREV,
125 FUNCTION_INST_CAST_ABBREV,
126 FUNCTION_INST_RET_VOID_ABBREV,
127 FUNCTION_INST_RET_VAL_ABBREV,
128 FUNCTION_INST_UNREACHABLE_ABBREV,
129 FUNCTION_INST_GEP_ABBREV,
134class BitcodeWriterBase {
145 : Stream(Stream), StrtabBuilder(StrtabBuilder) {}
148 void writeModuleVersion();
151void BitcodeWriterBase::writeModuleVersion() {
158class ModuleBitcodeWriterBase :
public BitcodeWriterBase {
172 std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
175 unsigned GlobalValueId;
186 bool ShouldPreserveUseListOrder,
188 : BitcodeWriterBase(Stream, StrtabBuilder),
M(
M),
198 for (
const auto &GUIDSummaryLists : *
Index)
200 for (
auto &Summary : GUIDSummaryLists.second.SummaryList)
201 if (
auto FS = dyn_cast<FunctionSummary>(
Summary.get()))
206 for (
auto &CallEdge :
FS->calls())
207 if (!CallEdge.first.haveGVs() || !CallEdge.first.getValue())
208 assignValueId(CallEdge.first.getGUID());
212 void writePerModuleGlobalValueSummary();
215 void writePerModuleFunctionSummaryRecord(
217 unsigned ValueID,
unsigned FSCallsAbbrev,
unsigned FSCallsProfileAbbrev,
218 unsigned CallsiteAbbrev,
unsigned AllocAbbrev,
const Function &
F);
221 unsigned FSModRefsAbbrev,
222 unsigned FSModVTableRefsAbbrev);
225 GUIDToValueIdMap[ValGUID] = ++GlobalValueId;
229 const auto &VMI = GUIDToValueIdMap.find(ValGUID);
232 assert(VMI != GUIDToValueIdMap.end() &&
233 "GUID does not have assigned value Id");
239 if (!
VI.haveGVs() || !
VI.getValue())
240 return getValueId(
VI.getGUID());
244 std::map<GlobalValue::GUID, unsigned> &valueIds() {
return GUIDToValueIdMap; }
248class ModuleBitcodeWriter :
public ModuleBitcodeWriterBase {
272 : ModuleBitcodeWriterBase(
M, StrtabBuilder, Stream,
273 ShouldPreserveUseListOrder,
Index),
274 Buffer(Buffer), GenerateHash(GenerateHash), ModHash(ModHash),
275 BitcodeStartBit(Stream.GetCurrentBitNo()) {}
281 uint64_t bitcodeStartBit() {
return BitcodeStartBit; }
285 void writeAttributeGroupTable();
286 void writeAttributeTable();
287 void writeTypeTable();
289 void writeValueSymbolTableForwardDecl();
290 void writeModuleInfo();
295 unsigned createDILocationAbbrev();
298 unsigned createGenericDINodeAbbrev();
367 unsigned createNamedMetadataAbbrev();
369 unsigned createMetadataStringsAbbrev();
374 std::vector<unsigned> *MDAbbrevs =
nullptr,
375 std::vector<uint64_t> *IndexPos =
nullptr);
376 void writeModuleMetadata();
377 void writeFunctionMetadata(
const Function &
F);
378 void writeFunctionMetadataAttachment(
const Function &
F);
381 void writeModuleMetadataKinds();
382 void writeOperandBundleTags();
383 void writeSyncScopeNames();
384 void writeConstants(
unsigned FirstVal,
unsigned LastVal,
bool isGlobal);
385 void writeModuleConstants();
386 bool pushValueAndType(
const Value *V,
unsigned InstID,
388 void writeOperandBundles(
const CallBase &CB,
unsigned InstID);
389 void pushValue(
const Value *V,
unsigned InstID,
391 void pushValueSigned(
const Value *V,
unsigned InstID,
393 void writeInstruction(
const Instruction &
I,
unsigned InstID,
396 void writeGlobalValueSymbolTable(
399 void writeUseListBlock(
const Function *
F);
403 void writeBlockInfo();
404 void writeModuleHash(
size_t BlockStartPos);
410 unsigned getEncodedAlign(
MaybeAlign Alignment) {
return encode(Alignment); }
414class IndexBitcodeWriter :
public BitcodeWriterBase {
420 const std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex;
424 std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
429 std::vector<unsigned> StackIdIndices;
432 unsigned GlobalValueId = 0;
444 const std::map<std::string, GVSummaryMapTy>
445 *ModuleToSummariesForIndex =
nullptr)
446 : BitcodeWriterBase(Stream, StrtabBuilder),
Index(
Index),
447 ModuleToSummariesForIndex(ModuleToSummariesForIndex) {
452 forEachSummary([&](GVInfo
I,
bool IsAliasee) {
453 GUIDToValueIdMap[
I.first] = ++GlobalValueId;
456 auto *
FS = dyn_cast<FunctionSummary>(
I.second);
462 for (
auto &CI :
FS->callsites())
463 for (
auto Idx : CI.StackIdIndices)
464 StackIdIndices.push_back(
Idx);
465 for (
auto &AI :
FS->allocs())
466 for (
auto &MIB : AI.MIBs)
467 for (
auto Idx : MIB.StackIdIndices)
468 StackIdIndices.push_back(
Idx);
471 StackIdIndices.erase(
472 std::unique(StackIdIndices.begin(), StackIdIndices.end()),
473 StackIdIndices.end());
477 using GVInfo = std::pair<GlobalValue::GUID, GlobalValueSummary *>;
482 template<
typename Functor>
483 void forEachSummary(Functor Callback) {
484 if (ModuleToSummariesForIndex) {
485 for (
auto &M : *ModuleToSummariesForIndex)
486 for (
auto &Summary :
M.second) {
487 Callback(Summary,
false);
491 if (
auto *AS = dyn_cast<AliasSummary>(
Summary.getSecond()))
492 Callback({AS->getAliaseeGUID(), &AS->getAliasee()},
true);
495 for (
auto &Summaries :
Index)
496 for (
auto &Summary : Summaries.second.SummaryList)
497 Callback({Summaries.first,
Summary.get()},
false);
505 template <
typename Functor>
void forEachModule(Functor Callback) {
506 if (ModuleToSummariesForIndex) {
507 for (
const auto &M : *ModuleToSummariesForIndex) {
508 const auto &MPI =
Index.modulePaths().find(
M.first);
509 if (MPI ==
Index.modulePaths().end()) {
513 assert(ModuleToSummariesForIndex->size() == 1);
523 std::vector<StringRef> ModulePaths;
524 for (
auto &[ModPath,
_] :
Index.modulePaths())
525 ModulePaths.push_back(ModPath);
526 llvm::sort(ModulePaths.begin(), ModulePaths.end());
527 for (
auto &ModPath : ModulePaths)
528 Callback(*
Index.modulePaths().find(ModPath));
536 void writeModStrings();
537 void writeCombinedGlobalValueSummary();
540 auto VMI = GUIDToValueIdMap.find(ValGUID);
541 if (VMI == GUIDToValueIdMap.end())
546 std::map<GlobalValue::GUID, unsigned> &valueIds() {
return GUIDToValueIdMap; }
580 case Instruction::Add:
582 case Instruction::Sub:
584 case Instruction::Mul:
587 case Instruction::FDiv:
590 case Instruction::FRem:
656 case Attribute::Alignment:
658 case Attribute::AllocAlign:
660 case Attribute::AllocSize:
662 case Attribute::AlwaysInline:
664 case Attribute::Builtin:
666 case Attribute::ByVal:
668 case Attribute::Convergent:
670 case Attribute::InAlloca:
672 case Attribute::Cold:
674 case Attribute::DisableSanitizerInstrumentation:
676 case Attribute::FnRetThunkExtern:
680 case Attribute::ElementType:
682 case Attribute::InlineHint:
684 case Attribute::InReg:
686 case Attribute::JumpTable:
688 case Attribute::MinSize:
690 case Attribute::AllocatedPointer:
692 case Attribute::AllocKind:
694 case Attribute::Memory:
696 case Attribute::NoFPClass:
698 case Attribute::Naked:
700 case Attribute::Nest:
702 case Attribute::NoAlias:
704 case Attribute::NoBuiltin:
706 case Attribute::NoCallback:
708 case Attribute::NoCapture:
710 case Attribute::NoDuplicate:
712 case Attribute::NoFree:
714 case Attribute::NoImplicitFloat:
716 case Attribute::NoInline:
718 case Attribute::NoRecurse:
720 case Attribute::NoMerge:
722 case Attribute::NonLazyBind:
724 case Attribute::NonNull:
726 case Attribute::Dereferenceable:
728 case Attribute::DereferenceableOrNull:
730 case Attribute::NoRedZone:
732 case Attribute::NoReturn:
734 case Attribute::NoSync:
736 case Attribute::NoCfCheck:
738 case Attribute::NoProfile:
740 case Attribute::SkipProfile:
742 case Attribute::NoUnwind:
744 case Attribute::NoSanitizeBounds:
746 case Attribute::NoSanitizeCoverage:
748 case Attribute::NullPointerIsValid:
750 case Attribute::OptForFuzzing:
752 case Attribute::OptimizeForSize:
754 case Attribute::OptimizeNone:
756 case Attribute::ReadNone:
758 case Attribute::ReadOnly:
760 case Attribute::Returned:
762 case Attribute::ReturnsTwice:
764 case Attribute::SExt:
766 case Attribute::Speculatable:
768 case Attribute::StackAlignment:
770 case Attribute::StackProtect:
772 case Attribute::StackProtectReq:
774 case Attribute::StackProtectStrong:
776 case Attribute::SafeStack:
778 case Attribute::ShadowCallStack:
780 case Attribute::StrictFP:
782 case Attribute::StructRet:
784 case Attribute::SanitizeAddress:
786 case Attribute::SanitizeHWAddress:
788 case Attribute::SanitizeThread:
790 case Attribute::SanitizeMemory:
792 case Attribute::SpeculativeLoadHardening:
794 case Attribute::SwiftError:
796 case Attribute::SwiftSelf:
798 case Attribute::SwiftAsync:
800 case Attribute::UWTable:
802 case Attribute::VScaleRange:
804 case Attribute::WillReturn:
806 case Attribute::WriteOnly:
808 case Attribute::ZExt:
810 case Attribute::ImmArg:
812 case Attribute::SanitizeMemTag:
814 case Attribute::Preallocated:
816 case Attribute::NoUndef:
818 case Attribute::ByRef:
820 case Attribute::MustProgress:
822 case Attribute::PresplitCoroutine:
836void ModuleBitcodeWriter::writeAttributeGroupTable() {
837 const std::vector<ValueEnumerator::IndexAndAttrSet> &AttrGrps =
839 if (AttrGrps.empty())
return;
845 unsigned AttrListIndex = Pair.first;
848 Record.push_back(AttrListIndex);
851 if (Attr.isEnumAttribute()) {
854 }
else if (Attr.isIntAttribute()) {
857 Record.push_back(Attr.getValueAsInt());
858 }
else if (Attr.isStringAttribute()) {
870 assert(Attr.isTypeAttribute());
871 Type *Ty = Attr.getValueAsType();
872 Record.push_back(Ty ? 6 : 5);
886void ModuleBitcodeWriter::writeAttributeTable() {
888 if (
Attrs.empty())
return;
894 for (
unsigned i :
AL.indexes()) {
908void ModuleBitcodeWriter::writeTypeTable() {
917 auto Abbv = std::make_shared<BitCodeAbbrev>();
920 unsigned OpaquePtrAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
923 Abbv = std::make_shared<BitCodeAbbrev>();
928 unsigned FunctionAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
931 Abbv = std::make_shared<BitCodeAbbrev>();
936 unsigned StructAnonAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
939 Abbv = std::make_shared<BitCodeAbbrev>();
943 unsigned StructNameAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
946 Abbv = std::make_shared<BitCodeAbbrev>();
951 unsigned StructNamedAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
954 Abbv = std::make_shared<BitCodeAbbrev>();
958 unsigned ArrayAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
966 for (
Type *
T : TypeList) {
970 switch (
T->getTypeID()) {
996 AbbrevToUse = OpaquePtrAbbrev;
1005 for (
unsigned i = 0, e = FT->getNumParams(); i != e; ++i)
1007 AbbrevToUse = FunctionAbbrev;
1015 for (
Type *ET :
ST->elements())
1018 if (
ST->isLiteral()) {
1020 AbbrevToUse = StructAnonAbbrev;
1022 if (
ST->isOpaque()) {
1026 AbbrevToUse = StructNamedAbbrev;
1030 if (!
ST->getName().empty())
1040 TypeVals.
push_back(AT->getNumElements());
1042 AbbrevToUse = ArrayAbbrev;
1051 TypeVals.
push_back(VT->getElementCount().getKnownMinValue());
1053 if (isa<ScalableVectorType>(VT))
1063 for (
Type *InnerTy :
TET->type_params())
1065 for (
unsigned IntParam :
TET->int_params())
1074 Stream.
EmitRecord(Code, TypeVals, AbbrevToUse);
1115 RawFlags |= Flags.ReadNone;
1116 RawFlags |= (Flags.ReadOnly << 1);
1117 RawFlags |= (Flags.NoRecurse << 2);
1118 RawFlags |= (Flags.ReturnDoesNotAlias << 3);
1119 RawFlags |= (Flags.NoInline << 4);
1120 RawFlags |= (Flags.AlwaysInline << 5);
1121 RawFlags |= (Flags.NoUnwind << 6);
1122 RawFlags |= (Flags.MayThrow << 7);
1123 RawFlags |= (Flags.HasUnknownCall << 8);
1124 RawFlags |= (Flags.MustBeUnreachable << 9);
1133 RawFlags |= Flags.NotEligibleToImport;
1134 RawFlags |= (Flags.Live << 1);
1135 RawFlags |= (Flags.DSOLocal << 2);
1136 RawFlags |= (Flags.CanAutoHide << 3);
1141 RawFlags = (RawFlags << 4) | Flags.Linkage;
1143 RawFlags |= (Flags.Visibility << 8);
1149 uint64_t RawFlags = Flags.MaybeReadOnly | (Flags.MaybeWriteOnly << 1) |
1150 (Flags.Constant << 2) | Flags.VCallVisibility << 3;
1174 case GlobalVariable::NotThreadLocal:
return 0;
1175 case GlobalVariable::GeneralDynamicTLSModel:
return 1;
1176 case GlobalVariable::LocalDynamicTLSModel:
return 2;
1177 case GlobalVariable::InitialExecTLSModel:
return 3;
1178 case GlobalVariable::LocalExecTLSModel:
return 4;
1184 switch (
C.getSelectionKind()) {
1201 case GlobalValue::UnnamedAddr::None:
return 0;
1202 case GlobalValue::UnnamedAddr::Local:
return 2;
1203 case GlobalValue::UnnamedAddr::Global:
return 1;
1208size_t ModuleBitcodeWriter::addToStrtab(
StringRef Str) {
1211 return StrtabBuilder.
add(Str);
1214void ModuleBitcodeWriter::writeComdats() {
1229void ModuleBitcodeWriter::writeValueSymbolTableForwardDecl() {
1234 auto Abbv = std::make_shared<BitCodeAbbrev>();
1240 unsigned VSTOffsetAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
1256 bool isChar6 =
true;
1257 for (
char C : Str) {
1260 if ((
unsigned char)
C & 128)
1270 "Sanitizer Metadata is too large for naive serialization.");
1273 return Meta.NoAddress | (Meta.NoHWAddress << 1) |
1274 (Meta.Memtag << 2) | (Meta.IsDynInit << 3);
1280void ModuleBitcodeWriter::writeModuleInfo() {
1282 if (!
M.getTargetTriple().empty())
1285 const std::string &
DL =
M.getDataLayoutStr();
1288 if (!
M.getModuleInlineAsm().empty())
1294 std::map<std::string, unsigned> SectionMap;
1295 std::map<std::string, unsigned> GCMap;
1297 unsigned MaxGlobalType = 0;
1298 const auto UpdateMaxAlignment = [&MaxAlignment](
const MaybeAlign A) {
1300 MaxAlignment = !MaxAlignment ? *
A : std::max(*MaxAlignment, *
A);
1303 UpdateMaxAlignment(GV.getAlign());
1304 MaxGlobalType = std::max(MaxGlobalType, VE.
getTypeID(GV.getValueType()));
1305 if (GV.hasSection()) {
1307 unsigned &Entry = SectionMap[std::string(GV.getSection())];
1311 Entry = SectionMap.size();
1316 UpdateMaxAlignment(
F.getAlign());
1317 if (
F.hasSection()) {
1319 unsigned &Entry = SectionMap[std::string(
F.getSection())];
1323 Entry = SectionMap.size();
1328 unsigned &Entry = GCMap[
F.getGC()];
1332 Entry = GCMap.size();
1338 unsigned SimpleGVarAbbrev = 0;
1339 if (!
M.global_empty()) {
1341 auto Abbv = std::make_shared<BitCodeAbbrev>();
1355 unsigned MaxEncAlignment = getEncodedAlign(MaxAlignment);
1359 if (SectionMap.empty())
1365 SimpleGVarAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
1379 auto Abbv = std::make_shared<BitCodeAbbrev>();
1382 Abbv->Add(AbbrevOpToUse);
1383 unsigned FilenameAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
1385 for (
const auto P :
M.getSourceFileName())
1395 unsigned AbbrevToUse = 0;
1401 Vals.
push_back(addToStrtab(GV.getName()));
1404 Vals.
push_back(GV.getType()->getAddressSpace() << 2 | 2 | GV.isConstant());
1408 Vals.
push_back(getEncodedAlign(GV.getAlign()));
1409 Vals.
push_back(GV.hasSection() ? SectionMap[std::string(GV.getSection())]
1411 if (GV.isThreadLocal() ||
1413 GV.getUnnamedAddr() != GlobalValue::UnnamedAddr::None ||
1414 GV.isExternallyInitialized() ||
1416 GV.hasComdat() || GV.hasAttributes() || GV.isDSOLocal() ||
1417 GV.hasPartition() || GV.hasSanitizerMetadata()) {
1421 Vals.
push_back(GV.isExternallyInitialized());
1429 Vals.
push_back(addToStrtab(GV.getPartition()));
1430 Vals.
push_back(GV.getPartition().size());
1433 GV.getSanitizerMetadata())
1436 AbbrevToUse = SimpleGVarAbbrev;
1456 Vals.
push_back(getEncodedAlign(
F.getAlign()));
1457 Vals.
push_back(
F.hasSection() ? SectionMap[std::string(
F.getSection())]
1469 F.hasPersonalityFn() ? (VE.
getValueID(
F.getPersonalityFn()) + 1) : 0);
1473 Vals.
push_back(addToStrtab(
F.getPartition()));
1476 unsigned AbbrevToUse = 0;
1489 Vals.
push_back(
A.getType()->getAddressSpace());
1497 Vals.
push_back(addToStrtab(
A.getPartition()));
1500 unsigned AbbrevToUse = 0;
1512 Vals.
push_back(
I.getType()->getAddressSpace());
1517 Vals.
push_back(addToStrtab(
I.getPartition()));
1523 writeValueSymbolTableForwardDecl();
1529 if (
const auto *OBO = dyn_cast<OverflowingBinaryOperator>(V)) {
1530 if (OBO->hasNoSignedWrap())
1532 if (OBO->hasNoUnsignedWrap())
1534 }
else if (
const auto *PEO = dyn_cast<PossiblyExactOperator>(V)) {
1537 }
else if (
const auto *FPMO = dyn_cast<FPMathOperator>(V)) {
1538 if (FPMO->hasAllowReassoc())
1540 if (FPMO->hasNoNaNs())
1542 if (FPMO->hasNoInfs())
1544 if (FPMO->hasNoSignedZeros())
1546 if (FPMO->hasAllowReciprocal())
1548 if (FPMO->hasAllowContract())
1550 if (FPMO->hasApproxFunc())
1557void ModuleBitcodeWriter::writeValueAsMetadata(
1567void ModuleBitcodeWriter::writeMDTuple(
const MDTuple *
N,
1570 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
1572 assert(!(MD && isa<LocalAsMetadata>(MD)) &&
1573 "Unexpected function-local metadata");
1582unsigned ModuleBitcodeWriter::createDILocationAbbrev() {
1585 auto Abbv = std::make_shared<BitCodeAbbrev>();
1596void ModuleBitcodeWriter::writeDILocation(
const DILocation *
N,
1600 Abbrev = createDILocationAbbrev();
1602 Record.push_back(
N->isDistinct());
1603 Record.push_back(
N->getLine());
1604 Record.push_back(
N->getColumn());
1607 Record.push_back(
N->isImplicitCode());
1613unsigned ModuleBitcodeWriter::createGenericDINodeAbbrev() {
1616 auto Abbv = std::make_shared<BitCodeAbbrev>();
1627void ModuleBitcodeWriter::writeGenericDINode(
const GenericDINode *
N,
1631 Abbrev = createGenericDINodeAbbrev();
1633 Record.push_back(
N->isDistinct());
1634 Record.push_back(
N->getTag());
1637 for (
auto &
I :
N->operands())
1644void ModuleBitcodeWriter::writeDISubrange(
const DISubrange *
N,
1658void ModuleBitcodeWriter::writeDIGenericSubrange(
1672 if ((int64_t)V >= 0)
1683 unsigned NumWords =
A.getActiveWords();
1684 const uint64_t *RawData =
A.getRawData();
1685 for (
unsigned i = 0; i < NumWords; i++)
1689void ModuleBitcodeWriter::writeDIEnumerator(
const DIEnumerator *
N,
1693 Record.push_back(IsBigInt | (
N->isUnsigned() << 1) |
N->isDistinct());
1694 Record.push_back(
N->getValue().getBitWidth());
1702void ModuleBitcodeWriter::writeDIBasicType(
const DIBasicType *
N,
1705 Record.push_back(
N->isDistinct());
1706 Record.push_back(
N->getTag());
1708 Record.push_back(
N->getSizeInBits());
1709 Record.push_back(
N->getAlignInBits());
1710 Record.push_back(
N->getEncoding());
1711 Record.push_back(
N->getFlags());
1717void ModuleBitcodeWriter::writeDIStringType(
const DIStringType *
N,
1720 Record.push_back(
N->isDistinct());
1721 Record.push_back(
N->getTag());
1726 Record.push_back(
N->getSizeInBits());
1727 Record.push_back(
N->getAlignInBits());
1728 Record.push_back(
N->getEncoding());
1734void ModuleBitcodeWriter::writeDIDerivedType(
const DIDerivedType *
N,
1737 Record.push_back(
N->isDistinct());
1738 Record.push_back(
N->getTag());
1741 Record.push_back(
N->getLine());
1744 Record.push_back(
N->getSizeInBits());
1745 Record.push_back(
N->getAlignInBits());
1746 Record.push_back(
N->getOffsetInBits());
1747 Record.push_back(
N->getFlags());
1752 if (
const auto &DWARFAddressSpace =
N->getDWARFAddressSpace())
1753 Record.push_back(*DWARFAddressSpace + 1);
1763void ModuleBitcodeWriter::writeDICompositeType(
1766 const unsigned IsNotUsedInOldTypeRef = 0x2;
1767 Record.push_back(IsNotUsedInOldTypeRef | (
unsigned)
N->isDistinct());
1768 Record.push_back(
N->getTag());
1771 Record.push_back(
N->getLine());
1774 Record.push_back(
N->getSizeInBits());
1775 Record.push_back(
N->getAlignInBits());
1776 Record.push_back(
N->getOffsetInBits());
1777 Record.push_back(
N->getFlags());
1779 Record.push_back(
N->getRuntimeLang());
1794void ModuleBitcodeWriter::writeDISubroutineType(
1797 const unsigned HasNoOldTypeRefs = 0x2;
1798 Record.push_back(HasNoOldTypeRefs | (
unsigned)
N->isDistinct());
1799 Record.push_back(
N->getFlags());
1807void ModuleBitcodeWriter::writeDIFile(
const DIFile *
N,
1810 Record.push_back(
N->isDistinct());
1813 if (
N->getRawChecksum()) {
1814 Record.push_back(
N->getRawChecksum()->Kind);
1822 auto Source =
N->getRawSource();
1830void ModuleBitcodeWriter::writeDICompileUnit(
const DICompileUnit *
N,
1833 assert(
N->isDistinct() &&
"Expected distinct compile units");
1835 Record.push_back(
N->getSourceLanguage());
1838 Record.push_back(
N->isOptimized());
1840 Record.push_back(
N->getRuntimeVersion());
1842 Record.push_back(
N->getEmissionKind());
1848 Record.push_back(
N->getDWOId());
1850 Record.push_back(
N->getSplitDebugInlining());
1851 Record.push_back(
N->getDebugInfoForProfiling());
1852 Record.push_back((
unsigned)
N->getNameTableKind());
1853 Record.push_back(
N->getRangesBaseAddress());
1861void ModuleBitcodeWriter::writeDISubprogram(
const DISubprogram *
N,
1864 const uint64_t HasUnitFlag = 1 << 1;
1865 const uint64_t HasSPFlagsFlag = 1 << 2;
1866 Record.push_back(
uint64_t(
N->isDistinct()) | HasUnitFlag | HasSPFlagsFlag);
1871 Record.push_back(
N->getLine());
1873 Record.push_back(
N->getScopeLine());
1875 Record.push_back(
N->getSPFlags());
1876 Record.push_back(
N->getVirtualIndex());
1877 Record.push_back(
N->getFlags());
1882 Record.push_back(
N->getThisAdjustment());
1894 Record.push_back(
N->isDistinct());
1897 Record.push_back(
N->getLine());
1898 Record.push_back(
N->getColumn());
1904void ModuleBitcodeWriter::writeDILexicalBlockFile(
1907 Record.push_back(
N->isDistinct());
1910 Record.push_back(
N->getDiscriminator());
1916void ModuleBitcodeWriter::writeDICommonBlock(
const DICommonBlock *
N,
1919 Record.push_back(
N->isDistinct());
1924 Record.push_back(
N->getLineNo());
1930void ModuleBitcodeWriter::writeDINamespace(
const DINamespace *
N,
1933 Record.push_back(
N->isDistinct() |
N->getExportSymbols() << 1);
1941void ModuleBitcodeWriter::writeDIMacro(
const DIMacro *
N,
1944 Record.push_back(
N->isDistinct());
1945 Record.push_back(
N->getMacinfoType());
1946 Record.push_back(
N->getLine());
1954void ModuleBitcodeWriter::writeDIMacroFile(
const DIMacroFile *
N,
1957 Record.push_back(
N->isDistinct());
1958 Record.push_back(
N->getMacinfoType());
1959 Record.push_back(
N->getLine());
1967void ModuleBitcodeWriter::writeDIArgList(
const DIArgList *
N,
1970 Record.reserve(
N->getArgs().size());
1978void ModuleBitcodeWriter::writeDIModule(
const DIModule *
N,
1981 Record.push_back(
N->isDistinct());
1982 for (
auto &
I :
N->operands())
1984 Record.push_back(
N->getLineNo());
1985 Record.push_back(
N->getIsDecl());
1991void ModuleBitcodeWriter::writeDIAssignID(
const DIAssignID *
N,
1995 Record.push_back(
N->isDistinct());
2000void ModuleBitcodeWriter::writeDITemplateTypeParameter(
2003 Record.push_back(
N->isDistinct());
2006 Record.push_back(
N->isDefault());
2012void ModuleBitcodeWriter::writeDITemplateValueParameter(
2015 Record.push_back(
N->isDistinct());
2016 Record.push_back(
N->getTag());
2019 Record.push_back(
N->isDefault());
2026void ModuleBitcodeWriter::writeDIGlobalVariable(
2035 Record.push_back(
N->getLine());
2037 Record.push_back(
N->isLocalToUnit());
2038 Record.push_back(
N->isDefinition());
2041 Record.push_back(
N->getAlignInBits());
2048void ModuleBitcodeWriter::writeDILocalVariable(
2064 const uint64_t HasAlignmentFlag = 1 << 1;
2069 Record.push_back(
N->getLine());
2071 Record.push_back(
N->getArg());
2072 Record.push_back(
N->getFlags());
2073 Record.push_back(
N->getAlignInBits());
2080void ModuleBitcodeWriter::writeDILabel(
2087 Record.push_back(
N->getLine());
2093void ModuleBitcodeWriter::writeDIExpression(
const DIExpression *
N,
2096 Record.reserve(
N->getElements().size() + 1);
2099 Record.append(
N->elements_begin(),
N->elements_end());
2105void ModuleBitcodeWriter::writeDIGlobalVariableExpression(
2108 Record.push_back(
N->isDistinct());
2119 Record.push_back(
N->isDistinct());
2122 Record.push_back(
N->getLine());
2125 Record.push_back(
N->getAttributes());
2132void ModuleBitcodeWriter::writeDIImportedEntity(
2135 Record.push_back(
N->isDistinct());
2136 Record.push_back(
N->getTag());
2139 Record.push_back(
N->getLine());
2148unsigned ModuleBitcodeWriter::createNamedMetadataAbbrev() {
2149 auto Abbv = std::make_shared<BitCodeAbbrev>();
2156void ModuleBitcodeWriter::writeNamedMetadata(
2158 if (
M.named_metadata_empty())
2161 unsigned Abbrev = createNamedMetadataAbbrev();
2165 Record.append(Str.bytes_begin(), Str.bytes_end());
2170 for (
const MDNode *
N : NMD.operands())
2177unsigned ModuleBitcodeWriter::createMetadataStringsAbbrev() {
2178 auto Abbv = std::make_shared<BitCodeAbbrev>();
2190void ModuleBitcodeWriter::writeMetadataStrings(
2192 if (Strings.empty())
2197 Record.push_back(Strings.size());
2204 W.EmitVBR(cast<MDString>(MD)->getLength(), 6);
2213 Blob.
append(cast<MDString>(MD)->getString());
2222#define HANDLE_MDNODE_LEAF(CLASS) CLASS##AbbrevID,
2223#include "llvm/IR/Metadata.def"
2227void ModuleBitcodeWriter::writeMetadataRecords(
2229 std::vector<unsigned> *MDAbbrevs, std::vector<uint64_t> *IndexPos) {
2234#define HANDLE_MDNODE_LEAF(CLASS) unsigned CLASS##Abbrev = 0;
2235#include "llvm/IR/Metadata.def"
2240 if (
const MDNode *
N = dyn_cast<MDNode>(MD)) {
2241 assert(
N->isResolved() &&
"Expected forward references to be resolved");
2243 switch (
N->getMetadataID()) {
2246#define HANDLE_MDNODE_LEAF(CLASS) \
2247 case Metadata::CLASS##Kind: \
2249 write##CLASS(cast<CLASS>(N), Record, \
2250 (*MDAbbrevs)[MetadataAbbrev::CLASS##AbbrevID]); \
2252 write##CLASS(cast<CLASS>(N), Record, CLASS##Abbrev); \
2254#include "llvm/IR/Metadata.def"
2257 writeValueAsMetadata(cast<ValueAsMetadata>(MD),
Record);
2261void ModuleBitcodeWriter::writeModuleMetadata() {
2262 if (!VE.
hasMDs() &&
M.named_metadata_empty())
2270 std::vector<unsigned> MDAbbrevs;
2273 MDAbbrevs[MetadataAbbrev::DILocationAbbrevID] = createDILocationAbbrev();
2274 MDAbbrevs[MetadataAbbrev::GenericDINodeAbbrevID] =
2275 createGenericDINodeAbbrev();
2277 auto Abbv = std::make_shared<BitCodeAbbrev>();
2281 unsigned OffsetAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
2283 Abbv = std::make_shared<BitCodeAbbrev>();
2287 unsigned IndexAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
2309 std::vector<uint64_t> IndexPos;
2324 uint64_t PreviousValue = IndexOffsetRecordBitPos;
2325 for (
auto &Elt : IndexPos) {
2326 auto EltDelta = Elt - PreviousValue;
2327 PreviousValue = Elt;
2336 writeNamedMetadata(
Record);
2338 auto AddDeclAttachedMetadata = [&](
const GlobalObject &GO) {
2341 pushGlobalMetadataAttachment(
Record, GO);
2345 if (
F.isDeclaration() &&
F.hasMetadata())
2346 AddDeclAttachedMetadata(
F);
2350 if (GV.hasMetadata())
2351 AddDeclAttachedMetadata(GV);
2356void ModuleBitcodeWriter::writeFunctionMetadata(
const Function &
F) {
2367void ModuleBitcodeWriter::pushGlobalMetadataAttachment(
2372 for (
const auto &
I : MDs) {
2378void ModuleBitcodeWriter::writeFunctionMetadataAttachment(
const Function &
F) {
2383 if (
F.hasMetadata()) {
2384 pushGlobalMetadataAttachment(
Record,
F);
2395 I.getAllMetadataOtherThanDebugLoc(MDs);
2398 if (MDs.
empty())
continue;
2402 for (
unsigned i = 0, e = MDs.
size(); i != e; ++i) {
2403 Record.push_back(MDs[i].first);
2413void ModuleBitcodeWriter::writeModuleMetadataKinds() {
2419 M.getMDKindNames(Names);
2421 if (Names.
empty())
return;
2425 for (
unsigned MDKindID = 0, e = Names.
size(); MDKindID != e; ++MDKindID) {
2426 Record.push_back(MDKindID);
2437void ModuleBitcodeWriter::writeOperandBundleTags() {
2445 M.getOperandBundleTags(Tags);
2454 for (
auto Tag : Tags) {
2464void ModuleBitcodeWriter::writeSyncScopeNames() {
2466 M.getContext().getSyncScopeNames(SSNs);
2473 for (
auto SSN : SSNs) {
2474 Record.append(SSN.begin(), SSN.end());
2482void ModuleBitcodeWriter::writeConstants(
unsigned FirstVal,
unsigned LastVal,
2484 if (FirstVal == LastVal)
return;
2488 unsigned AggregateAbbrev = 0;
2489 unsigned String8Abbrev = 0;
2490 unsigned CString7Abbrev = 0;
2491 unsigned CString6Abbrev = 0;
2495 auto Abbv = std::make_shared<BitCodeAbbrev>();
2499 AggregateAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
2502 Abbv = std::make_shared<BitCodeAbbrev>();
2506 String8Abbrev = Stream.
EmitAbbrev(std::move(Abbv));
2508 Abbv = std::make_shared<BitCodeAbbrev>();
2512 CString7Abbrev = Stream.
EmitAbbrev(std::move(Abbv));
2514 Abbv = std::make_shared<BitCodeAbbrev>();
2518 CString6Abbrev = Stream.
EmitAbbrev(std::move(Abbv));
2524 Type *LastTy =
nullptr;
2525 for (
unsigned i = FirstVal; i != LastVal; ++i) {
2526 const Value *
V = Vals[i].first;
2528 if (
V->getType() != LastTy) {
2529 LastTy =
V->getType();
2532 CONSTANTS_SETTYPE_ABBREV);
2536 if (
const InlineAsm *IA = dyn_cast<InlineAsm>(V)) {
2539 unsigned(
IA->hasSideEffects()) |
unsigned(
IA->isAlignStack()) << 1 |
2540 unsigned(
IA->getDialect() & 1) << 2 |
unsigned(
IA->canThrow()) << 3);
2543 const std::string &AsmStr =
IA->getAsmString();
2544 Record.push_back(AsmStr.size());
2545 Record.append(AsmStr.begin(), AsmStr.end());
2548 const std::string &ConstraintStr =
IA->getConstraintString();
2549 Record.push_back(ConstraintStr.size());
2550 Record.append(ConstraintStr.begin(), ConstraintStr.end());
2556 unsigned Code = -1U;
2557 unsigned AbbrevToUse = 0;
2558 if (
C->isNullValue()) {
2560 }
else if (isa<PoisonValue>(
C)) {
2562 }
else if (isa<UndefValue>(
C)) {
2565 if (
IV->getBitWidth() <= 64) {
2569 AbbrevToUse = CONSTANTS_INTEGER_ABBREV;
2574 }
else if (
const ConstantFP *CFP = dyn_cast<ConstantFP>(
C)) {
2576 Type *Ty = CFP->getType();
2579 Record.push_back(CFP->getValueAPF().bitcastToAPInt().getZExtValue());
2583 APInt api = CFP->getValueAPF().bitcastToAPInt();
2585 Record.push_back((p[1] << 48) | (p[0] >> 16));
2586 Record.push_back(p[0] & 0xffffLL);
2588 APInt api = CFP->getValueAPF().bitcastToAPInt();
2593 assert(0 &&
"Unknown FP type!");
2595 }
else if (isa<ConstantDataSequential>(
C) &&
2596 cast<ConstantDataSequential>(
C)->isString()) {
2599 unsigned NumElts = Str->getNumElements();
2601 if (Str->isCString()) {
2606 AbbrevToUse = String8Abbrev;
2610 for (
unsigned i = 0; i != NumElts; ++i) {
2611 unsigned char V = Str->getElementAsInteger(i);
2613 isCStr7 &= (
V & 128) == 0;
2619 AbbrevToUse = CString6Abbrev;
2621 AbbrevToUse = CString7Abbrev;
2623 dyn_cast<ConstantDataSequential>(
C)) {
2625 Type *EltTy = CDS->getElementType();
2626 if (isa<IntegerType>(EltTy)) {
2627 for (
unsigned i = 0, e = CDS->getNumElements(); i != e; ++i)
2628 Record.push_back(CDS->getElementAsInteger(i));
2630 for (
unsigned i = 0, e = CDS->getNumElements(); i != e; ++i)
2632 CDS->getElementAsAPFloat(i).bitcastToAPInt().getLimitedValue());
2634 }
else if (isa<ConstantAggregate>(
C)) {
2636 for (
const Value *
Op :
C->operands())
2638 AbbrevToUse = AggregateAbbrev;
2639 }
else if (
const ConstantExpr *CE = dyn_cast<ConstantExpr>(
C)) {
2640 switch (
CE->getOpcode()) {
2647 AbbrevToUse = CONSTANTS_CE_CAST_Abbrev;
2649 assert(
CE->getNumOperands() == 2 &&
"Unknown constant expr!");
2659 case Instruction::FNeg: {
2660 assert(
CE->getNumOperands() == 1 &&
"Unknown constant expr!");
2669 case Instruction::GetElementPtr: {
2671 const auto *GO = cast<GEPOperator>(
C);
2673 if (std::optional<unsigned>
Idx = GO->getInRangeIndex()) {
2675 Record.push_back((*
Idx << 1) | GO->isInBounds());
2676 }
else if (GO->isInBounds())
2678 for (
unsigned i = 0, e =
CE->getNumOperands(); i != e; ++i) {
2684 case Instruction::ExtractElement:
2691 case Instruction::InsertElement:
2698 case Instruction::ShuffleVector:
2703 if (
C->getType() ==
C->getOperand(0)->getType()) {
2713 case Instruction::ICmp:
2714 case Instruction::FCmp:
2719 Record.push_back(
CE->getPredicate());
2722 }
else if (
const BlockAddress *BA = dyn_cast<BlockAddress>(
C)) {
2727 }
else if (
const auto *Equiv = dyn_cast<DSOLocalEquivalent>(
C)) {
2731 }
else if (
const auto *
NC = dyn_cast<NoCFIValue>(
C)) {
2748void ModuleBitcodeWriter::writeModuleConstants() {
2753 for (
unsigned i = 0, e = Vals.size(); i != e; ++i) {
2754 if (!isa<GlobalValue>(Vals[i].first)) {
2755 writeConstants(i, Vals.size(),
true);
2769bool ModuleBitcodeWriter::pushValueAndType(
const Value *V,
unsigned InstID,
2774 if (
ValID >= InstID) {
2781void ModuleBitcodeWriter::writeOperandBundles(
const CallBase &CS,
2788 Record.push_back(
C.getOperandBundleTagID(Bundle.getTagName()));
2790 for (
auto &Input : Bundle.Inputs)
2791 pushValueAndType(Input, InstID,
Record);
2800void ModuleBitcodeWriter::pushValue(
const Value *V,
unsigned InstID,
2806void ModuleBitcodeWriter::pushValueSigned(
const Value *V,
unsigned InstID,
2809 int64_t diff = ((int32_t)InstID - (int32_t)
ValID);
2814void ModuleBitcodeWriter::writeInstruction(
const Instruction &
I,
2818 unsigned AbbrevToUse = 0;
2820 switch (
I.getOpcode()) {
2824 if (!pushValueAndType(
I.getOperand(0), InstID, Vals))
2825 AbbrevToUse = FUNCTION_INST_CAST_ABBREV;
2829 assert(isa<BinaryOperator>(
I) &&
"Unknown instruction!");
2831 if (!pushValueAndType(
I.getOperand(0), InstID, Vals))
2832 AbbrevToUse = FUNCTION_INST_BINOP_ABBREV;
2833 pushValue(
I.getOperand(1), InstID, Vals);
2837 if (AbbrevToUse == FUNCTION_INST_BINOP_ABBREV)
2838 AbbrevToUse = FUNCTION_INST_BINOP_FLAGS_ABBREV;
2843 case Instruction::FNeg: {
2845 if (!pushValueAndType(
I.getOperand(0), InstID, Vals))
2846 AbbrevToUse = FUNCTION_INST_UNOP_ABBREV;
2850 if (AbbrevToUse == FUNCTION_INST_UNOP_ABBREV)
2851 AbbrevToUse = FUNCTION_INST_UNOP_FLAGS_ABBREV;
2856 case Instruction::GetElementPtr: {
2858 AbbrevToUse = FUNCTION_INST_GEP_ABBREV;
2859 auto &GEPInst = cast<GetElementPtrInst>(
I);
2862 for (
unsigned i = 0, e =
I.getNumOperands(); i != e; ++i)
2863 pushValueAndType(
I.getOperand(i), InstID, Vals);
2866 case Instruction::ExtractValue: {
2868 pushValueAndType(
I.getOperand(0), InstID, Vals);
2873 case Instruction::InsertValue: {
2875 pushValueAndType(
I.getOperand(0), InstID, Vals);
2876 pushValueAndType(
I.getOperand(1), InstID, Vals);
2881 case Instruction::Select: {
2883 pushValueAndType(
I.getOperand(1), InstID, Vals);
2884 pushValue(
I.getOperand(2), InstID, Vals);
2885 pushValueAndType(
I.getOperand(0), InstID, Vals);
2891 case Instruction::ExtractElement:
2893 pushValueAndType(
I.getOperand(0), InstID, Vals);
2894 pushValueAndType(
I.getOperand(1), InstID, Vals);
2896 case Instruction::InsertElement:
2898 pushValueAndType(
I.getOperand(0), InstID, Vals);
2899 pushValue(
I.getOperand(1), InstID, Vals);
2900 pushValueAndType(
I.getOperand(2), InstID, Vals);
2902 case Instruction::ShuffleVector:
2904 pushValueAndType(
I.getOperand(0), InstID, Vals);
2905 pushValue(
I.getOperand(1), InstID, Vals);
2906 pushValue(cast<ShuffleVectorInst>(
I).getShuffleMaskForBitcode(), InstID,
2909 case Instruction::ICmp:
2910 case Instruction::FCmp: {
2913 pushValueAndType(
I.getOperand(0), InstID, Vals);
2914 pushValue(
I.getOperand(1), InstID, Vals);
2922 case Instruction::Ret:
2925 unsigned NumOperands =
I.getNumOperands();
2926 if (NumOperands == 0)
2927 AbbrevToUse = FUNCTION_INST_RET_VOID_ABBREV;
2928 else if (NumOperands == 1) {
2929 if (!pushValueAndType(
I.getOperand(0), InstID, Vals))
2930 AbbrevToUse = FUNCTION_INST_RET_VAL_ABBREV;
2932 for (
unsigned i = 0, e = NumOperands; i !=
e; ++i)
2933 pushValueAndType(
I.getOperand(i), InstID, Vals);
2937 case Instruction::Br:
2948 case Instruction::Switch:
2953 pushValue(
SI.getCondition(), InstID, Vals);
2955 for (
auto Case :
SI.cases()) {
2961 case Instruction::IndirectBr:
2965 pushValue(
I.getOperand(0), InstID, Vals);
2966 for (
unsigned i = 1, e =
I.getNumOperands(); i != e; ++i)
2970 case Instruction::Invoke: {
2976 writeOperandBundles(*II, InstID);
2985 pushValueAndType(Callee, InstID, Vals);
2988 for (
unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
2989 pushValue(
I.getOperand(i), InstID, Vals);
2992 if (FTy->isVarArg()) {
2993 for (
unsigned i = FTy->getNumParams(), e = II->
arg_size(); i != e; ++i)
2994 pushValueAndType(
I.getOperand(i), InstID, Vals);
2998 case Instruction::Resume:
3000 pushValueAndType(
I.getOperand(0), InstID, Vals);
3002 case Instruction::CleanupRet: {
3004 const auto &CRI = cast<CleanupReturnInst>(
I);
3005 pushValue(CRI.getCleanupPad(), InstID, Vals);
3006 if (CRI.hasUnwindDest())
3010 case Instruction::CatchRet: {
3012 const auto &CRI = cast<CatchReturnInst>(
I);
3013 pushValue(CRI.getCatchPad(), InstID, Vals);
3017 case Instruction::CleanupPad:
3018 case Instruction::CatchPad: {
3019 const auto &FuncletPad = cast<FuncletPadInst>(
I);
3022 pushValue(FuncletPad.getParentPad(), InstID, Vals);
3024 unsigned NumArgOperands = FuncletPad.arg_size();
3026 for (
unsigned Op = 0;
Op != NumArgOperands; ++
Op)
3027 pushValueAndType(FuncletPad.getArgOperand(
Op), InstID, Vals);
3030 case Instruction::CatchSwitch: {
3032 const auto &CatchSwitch = cast<CatchSwitchInst>(
I);
3034 pushValue(CatchSwitch.getParentPad(), InstID, Vals);
3036 unsigned NumHandlers = CatchSwitch.getNumHandlers();
3038 for (
const BasicBlock *CatchPadBB : CatchSwitch.handlers())
3041 if (CatchSwitch.hasUnwindDest())
3045 case Instruction::CallBr: {
3051 writeOperandBundles(*CBI, InstID);
3066 pushValueAndType(Callee, InstID, Vals);
3069 for (
unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3070 pushValue(
I.getOperand(i), InstID, Vals);
3073 if (FTy->isVarArg()) {
3074 for (
unsigned i = FTy->getNumParams(), e = CBI->
arg_size(); i != e; ++i)
3075 pushValueAndType(
I.getOperand(i), InstID, Vals);
3079 case Instruction::Unreachable:
3081 AbbrevToUse = FUNCTION_INST_UNREACHABLE_ABBREV;
3084 case Instruction::PHI: {
3085 const PHINode &PN = cast<PHINode>(
I);
3102 Stream.
EmitRecord(Code, Vals64, AbbrevToUse);
3107 case Instruction::LandingPad: {
3118 pushValueAndType(LP.
getClause(
I), InstID, Vals);
3123 case Instruction::Alloca: {
3131 unsigned EncodedAlign = getEncodedAlign(AI.
getAlign());
3132 Bitfield::set<APV::AlignLower>(
3133 Record, EncodedAlign & ((1 << APV::AlignLower::Bits) - 1));
3134 Bitfield::set<APV::AlignUpper>(
Record,
3135 EncodedAlign >> APV::AlignLower::Bits);
3137 Bitfield::set<APV::ExplicitType>(
Record,
true);
3142 if (AS !=
M.getDataLayout().getAllocaAddrSpace())
3147 case Instruction::Load:
3148 if (cast<LoadInst>(
I).isAtomic()) {
3150 pushValueAndType(
I.getOperand(0), InstID, Vals);
3153 if (!pushValueAndType(
I.getOperand(0), InstID, Vals))
3154 AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
3158 Vals.
push_back(cast<LoadInst>(
I).isVolatile());
3159 if (cast<LoadInst>(
I).isAtomic()) {
3161 Vals.
push_back(getEncodedSyncScopeID(cast<LoadInst>(
I).getSyncScopeID()));
3164 case Instruction::Store:
3165 if (cast<StoreInst>(
I).isAtomic())
3169 pushValueAndType(
I.getOperand(1), InstID, Vals);
3170 pushValueAndType(
I.getOperand(0), InstID, Vals);
3172 Vals.
push_back(cast<StoreInst>(
I).isVolatile());
3173 if (cast<StoreInst>(
I).isAtomic()) {
3176 getEncodedSyncScopeID(cast<StoreInst>(
I).getSyncScopeID()));
3179 case Instruction::AtomicCmpXchg:
3181 pushValueAndType(
I.getOperand(0), InstID, Vals);
3182 pushValueAndType(
I.getOperand(1), InstID, Vals);
3183 pushValue(
I.getOperand(2), InstID, Vals);
3184 Vals.
push_back(cast<AtomicCmpXchgInst>(
I).isVolatile());
3188 getEncodedSyncScopeID(cast<AtomicCmpXchgInst>(
I).getSyncScopeID()));
3191 Vals.
push_back(cast<AtomicCmpXchgInst>(
I).isWeak());
3194 case Instruction::AtomicRMW:
3196 pushValueAndType(
I.getOperand(0), InstID, Vals);
3197 pushValueAndType(
I.getOperand(1), InstID, Vals);
3200 Vals.
push_back(cast<AtomicRMWInst>(
I).isVolatile());
3203 getEncodedSyncScopeID(cast<AtomicRMWInst>(
I).getSyncScopeID()));
3206 case Instruction::Fence:
3209 Vals.
push_back(getEncodedSyncScopeID(cast<FenceInst>(
I).getSyncScopeID()));
3211 case Instruction::Call: {
3216 writeOperandBundles(CI, InstID);
3236 for (
unsigned i = 0, e = FTy->getNumParams(); i != e; ++i) {
3238 if (FTy->getParamType(i)->isLabelTy())
3245 if (FTy->isVarArg()) {
3246 for (
unsigned i = FTy->getNumParams(), e = CI.
arg_size(); i != e; ++i)
3251 case Instruction::VAArg:
3254 pushValue(
I.getOperand(0), InstID, Vals);
3257 case Instruction::Freeze:
3259 pushValueAndType(
I.getOperand(0), InstID, Vals);
3269void ModuleBitcodeWriter::writeGlobalValueSymbolTable(
3275 VSTOffset -= bitcodeStartBit();
3276 assert((VSTOffset & 31) == 0 &&
"VST block not 32-bit aligned");
3280 Stream.
BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32 + 1);
3284 auto Abbv = std::make_shared<BitCodeAbbrev>();
3288 unsigned FnEntryAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
3293 if (
F.isDeclaration())
3300 uint64_t BitcodeIndex = FunctionToBitcodeIndex[&
F] - bitcodeStartBit();
3301 assert((BitcodeIndex & 31) == 0 &&
"function block not 32-bit aligned");
3305 Record[1] = BitcodeIndex / 32 + 1;
3314void ModuleBitcodeWriter::writeFunctionLevelValueSymbolTable(
3329 unsigned AbbrevToUse = VST_ENTRY_8_ABBREV;
3335 if (isa<BasicBlock>(
Name.getValue())) {
3338 AbbrevToUse = VST_BBENTRY_6_ABBREV;
3342 AbbrevToUse = VST_ENTRY_6_ABBREV;
3344 AbbrevToUse = VST_ENTRY_7_ABBREV;
3347 for (
const auto P :
Name.getKey())
3351 Stream.
EmitRecord(Code, NameVals, AbbrevToUse);
3358void ModuleBitcodeWriter::writeUseList(
UseListOrder &&Order) {
3359 assert(Order.Shuffle.size() >= 2 &&
"Shuffle too small");
3361 if (isa<BasicBlock>(Order.V))
3371void ModuleBitcodeWriter::writeUseListBlock(
const Function *
F) {
3373 "Expected to be preserving use-list order");
3375 auto hasMore = [&]() {
3391void ModuleBitcodeWriter::writeFunction(
3410 unsigned CstStart, CstEnd;
3412 writeConstants(CstStart, CstEnd,
false);
3415 writeFunctionMetadata(
F);
3418 unsigned InstID = CstEnd;
3420 bool NeedsMetadataAttachment =
F.hasMetadata();
3428 writeInstruction(
I, InstID, Vals);
3430 if (!
I.getType()->isVoidTy())
3434 NeedsMetadataAttachment |=
I.hasMetadataOtherThanDebugLoc();
3461 while (!Worklist.empty()) {
3462 Value *
V = Worklist.pop_back_val();
3463 for (
User *U :
V->users()) {
3464 if (
auto *
I = dyn_cast<Instruction>(U)) {
3468 }
else if (isa<Constant>(U) && !isa<GlobalValue>(U) &&
3469 Visited.insert(U).second)
3470 Worklist.push_back(U);
3476 if (!BlockAddressUsers.
empty()) {
3485 if (
auto *Symtab =
F.getValueSymbolTable())
3486 writeFunctionLevelValueSymbolTable(*Symtab);
3488 if (NeedsMetadataAttachment)
3489 writeFunctionMetadataAttachment(
F);
3491 writeUseListBlock(&
F);
3497void ModuleBitcodeWriter::writeBlockInfo() {
3504 auto Abbv = std::make_shared<BitCodeAbbrev>();
3515 auto Abbv = std::make_shared<BitCodeAbbrev>();
3525 auto Abbv = std::make_shared<BitCodeAbbrev>();
3535 auto Abbv = std::make_shared<BitCodeAbbrev>();
3541 VST_BBENTRY_6_ABBREV)
3546 auto Abbv = std::make_shared<BitCodeAbbrev>();
3551 CONSTANTS_SETTYPE_ABBREV)
3556 auto Abbv = std::make_shared<BitCodeAbbrev>();
3560 CONSTANTS_INTEGER_ABBREV)
3565 auto Abbv = std::make_shared<BitCodeAbbrev>();
3573 CONSTANTS_CE_CAST_Abbrev)
3577 auto Abbv = std::make_shared<BitCodeAbbrev>();
3580 CONSTANTS_NULL_Abbrev)
3587 auto Abbv = std::make_shared<BitCodeAbbrev>();
3595 FUNCTION_INST_LOAD_ABBREV)
3599 auto Abbv = std::make_shared<BitCodeAbbrev>();
3604 FUNCTION_INST_UNOP_ABBREV)
3608 auto Abbv = std::make_shared<BitCodeAbbrev>();
3614 FUNCTION_INST_UNOP_FLAGS_ABBREV)
3618 auto Abbv = std::make_shared<BitCodeAbbrev>();
3624 FUNCTION_INST_BINOP_ABBREV)
3628 auto Abbv = std::make_shared<BitCodeAbbrev>();
3635 FUNCTION_INST_BINOP_FLAGS_ABBREV)
3639 auto Abbv = std::make_shared<BitCodeAbbrev>();
3646 FUNCTION_INST_CAST_ABBREV)
3651 auto Abbv = std::make_shared<BitCodeAbbrev>();
3654 FUNCTION_INST_RET_VOID_ABBREV)
3658 auto Abbv = std::make_shared<BitCodeAbbrev>();
3662 FUNCTION_INST_RET_VAL_ABBREV)
3666 auto Abbv = std::make_shared<BitCodeAbbrev>();
3669 FUNCTION_INST_UNREACHABLE_ABBREV)
3673 auto Abbv = std::make_shared<BitCodeAbbrev>();
3681 FUNCTION_INST_GEP_ABBREV)
3690void IndexBitcodeWriter::writeModStrings() {
3696 auto Abbv = std::make_shared<BitCodeAbbrev>();
3701 unsigned Abbrev8Bit = Stream.
EmitAbbrev(std::move(Abbv));
3704 Abbv = std::make_shared<BitCodeAbbrev>();
3709 unsigned Abbrev7Bit = Stream.
EmitAbbrev(std::move(Abbv));
3712 Abbv = std::make_shared<BitCodeAbbrev>();
3717 unsigned Abbrev6Bit = Stream.
EmitAbbrev(std::move(Abbv));
3720 Abbv = std::make_shared<BitCodeAbbrev>();
3727 unsigned AbbrevHash = Stream.
EmitAbbrev(std::move(Abbv));
3732 const auto &Hash = MPSE.
getValue();
3734 unsigned AbbrevToUse = Abbrev8Bit;
3736 AbbrevToUse = Abbrev6Bit;
3738 AbbrevToUse = Abbrev7Bit;
3740 auto ModuleId = ModuleIdMap.
size();
3741 ModuleIdMap[
Key] = ModuleId;
3750 Vals.
assign(Hash.begin(), Hash.end());
3762template <
typename Fn>
3766 if (!FS->type_tests().empty())
3771 auto WriteVFuncIdVec = [&](
uint64_t Ty,
3776 for (
auto &VF : VFs) {
3777 Record.push_back(VF.GUID);
3778 Record.push_back(VF.Offset);
3784 FS->type_test_assume_vcalls());
3786 FS->type_checked_load_vcalls());
3788 auto WriteConstVCallVec = [&](
uint64_t Ty,
3790 for (
auto &VC : VCs) {
3792 Record.push_back(VC.VFunc.GUID);
3793 Record.push_back(VC.VFunc.Offset);
3800 FS->type_test_assume_const_vcalls());
3802 FS->type_checked_load_const_vcalls());
3806 assert(Range.getLower().getNumWords() == 1);
3807 assert(Range.getUpper().getNumWords() == 1);
3812 if (!FS->paramAccesses().empty()) {
3814 for (
auto &Arg : FS->paramAccesses()) {
3815 size_t UndoSize =
Record.size();
3816 Record.push_back(Arg.ParamNo);
3817 WriteRange(Arg.Use);
3818 Record.push_back(Arg.Calls.size());
3819 for (
auto &Call : Arg.Calls) {
3820 Record.push_back(Call.ParamNo);
3821 std::optional<unsigned> ValueID = GetValueID(Call.Callee);
3828 Record.push_back(*ValueID);
3829 WriteRange(Call.Offsets);
3840 std::set<GlobalValue::GUID> &ReferencedTypeIds) {
3841 if (!FS->type_tests().empty())
3842 for (
auto &TT : FS->type_tests())
3843 ReferencedTypeIds.insert(TT);
3845 auto GetReferencedTypesFromVFuncIdVec =
3847 for (
auto &VF : VFs)
3848 ReferencedTypeIds.insert(VF.GUID);
3851 GetReferencedTypesFromVFuncIdVec(FS->type_test_assume_vcalls());
3852 GetReferencedTypesFromVFuncIdVec(FS->type_checked_load_vcalls());
3854 auto GetReferencedTypesFromConstVCallVec =
3856 for (
auto &VC : VCs)
3857 ReferencedTypeIds.insert(VC.VFunc.GUID);
3860 GetReferencedTypesFromConstVCallVec(FS->type_test_assume_const_vcalls());
3861 GetReferencedTypesFromConstVCallVec(FS->type_checked_load_const_vcalls());
3892 const std::string &Id,
3897 NameVals.
push_back(Summary.TTRes.TheKind);
3898 NameVals.
push_back(Summary.TTRes.SizeM1BitWidth);
3899 NameVals.
push_back(Summary.TTRes.AlignLog2);
3900 NameVals.
push_back(Summary.TTRes.SizeM1);
3901 NameVals.
push_back(Summary.TTRes.BitMask);
3902 NameVals.
push_back(Summary.TTRes.InlineBits);
3904 for (
auto &W : Summary.WPDRes)
3916 for (
auto &
P : Summary) {
3924 unsigned AllocAbbrev,
bool PerModule,
3925 std::function<
unsigned(
const ValueInfo &VI)> GetValueID,
3926 std::function<
unsigned(
unsigned)> GetStackIndex) {
3929 for (
auto &CI : FS->callsites()) {
3933 assert(!PerModule || (CI.Clones.size() == 1 && CI.Clones[0] == 0));
3934 Record.push_back(GetValueID(CI.Callee));
3936 Record.push_back(CI.StackIdIndices.size());
3937 Record.push_back(CI.Clones.size());
3939 for (
auto Id : CI.StackIdIndices)
3940 Record.push_back(GetStackIndex(Id));
3942 for (
auto V : CI.Clones)
3950 for (
auto &AI : FS->allocs()) {
3954 assert(!PerModule || (AI.Versions.size() == 1 && AI.Versions[0] == 0));
3956 Record.push_back(AI.MIBs.size());
3957 Record.push_back(AI.Versions.size());
3959 for (
auto &MIB : AI.MIBs) {
3960 Record.push_back((uint8_t)MIB.AllocType);
3961 Record.push_back(MIB.StackIdIndices.size());
3962 for (
auto Id : MIB.StackIdIndices)
3963 Record.push_back(GetStackIndex(Id));
3966 for (
auto V : AI.Versions)
3976void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
3978 unsigned ValueID,
unsigned FSCallsAbbrev,
unsigned FSCallsProfileAbbrev,
3979 unsigned CallsiteAbbrev,
unsigned AllocAbbrev,
const Function &
F) {
3985 Stream, FS, [&](
const ValueInfo &VI) -> std::optional<unsigned> {
3990 Stream, FS, CallsiteAbbrev, AllocAbbrev,
3992 [&](
const ValueInfo &VI) {
return getValueId(VI); },
3993 [&](
unsigned I) {
return I; });
3995 auto SpecialRefCnts =
FS->specialRefCounts();
4000 NameVals.
push_back(SpecialRefCnts.first);
4001 NameVals.
push_back(SpecialRefCnts.second);
4003 for (
auto &RI :
FS->refs())
4006 bool HasProfileData =
4008 for (
auto &ECI :
FS->calls()) {
4009 NameVals.
push_back(getValueId(ECI.first));
4011 NameVals.
push_back(
static_cast<uint8_t
>(ECI.second.Hotness));
4013 NameVals.
push_back(ECI.second.RelBlockFreq);
4016 unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev);
4029void ModuleBitcodeWriterBase::writeModuleLevelReferences(
4031 unsigned FSModRefsAbbrev,
unsigned FSModVTableRefsAbbrev) {
4032 auto VI =
Index->getValueInfo(
V.getGUID());
4033 if (!VI ||
VI.getSummaryList().empty()) {
4039 auto *
Summary =
VI.getSummaryList()[0].get();
4045 auto VTableFuncs =
VS->vTableFuncs();
4046 if (!VTableFuncs.empty())
4049 unsigned SizeBeforeRefs = NameVals.
size();
4050 for (
auto &RI :
VS->refs())
4056 if (VTableFuncs.empty())
4061 for (
auto &
P : VTableFuncs) {
4067 FSModVTableRefsAbbrev);
4074void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
4077 bool IsThinLTO =
true;
4079 mdconst::extract_or_null<ConstantInt>(
M.getModuleFlag(
"ThinLTO")))
4080 IsThinLTO = MD->getZExtValue();
4092 if (
Index->enableSplitLTOUnit())
4094 if (
Index->hasUnifiedLTO())
4104 for (
const auto &GVI : valueIds()) {
4109 if (!
Index->stackIds().empty()) {
4110 auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4115 unsigned StackIdAbbvId = Stream.
EmitAbbrev(std::move(StackIdAbbv));
4120 auto Abbv = std::make_shared<BitCodeAbbrev>();
4132 unsigned FSCallsProfileAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
4135 Abbv = std::make_shared<BitCodeAbbrev>();
4150 unsigned FSCallsAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
4153 Abbv = std::make_shared<BitCodeAbbrev>();
4159 unsigned FSModRefsAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
4162 Abbv = std::make_shared<BitCodeAbbrev>();
4170 unsigned FSModVTableRefsAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
4173 Abbv = std::make_shared<BitCodeAbbrev>();
4178 unsigned FSAliasAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
4181 Abbv = std::make_shared<BitCodeAbbrev>();
4188 unsigned TypeIdCompatibleVtableAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
4190 Abbv = std::make_shared<BitCodeAbbrev>();
4196 unsigned CallsiteAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
4198 Abbv = std::make_shared<BitCodeAbbrev>();
4203 unsigned AllocAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
4215 if (!VI ||
VI.getSummaryList().empty()) {
4221 auto *
Summary =
VI.getSummaryList()[0].get();
4222 writePerModuleFunctionSummaryRecord(NameVals, Summary, VE.
getValueID(&
F),
4223 FSCallsAbbrev, FSCallsProfileAbbrev,
4224 CallsiteAbbrev, AllocAbbrev,
F);
4230 writeModuleLevelReferences(
G, NameVals, FSModRefsAbbrev,
4231 FSModVTableRefsAbbrev);
4234 auto *Aliasee =
A.getAliaseeObject();
4237 if (!Aliasee->hasName() || isa<GlobalIFunc>(Aliasee))
4250 for (
auto &S :
Index->typeIdCompatibleVtableMap()) {
4254 TypeIdCompatibleVtableAbbrev);
4258 if (
Index->getBlockCount())
4266void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
4275 for (
const auto &GVI : valueIds()) {
4280 if (!StackIdIndices.empty()) {
4281 auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4286 unsigned StackIdAbbvId = Stream.
EmitAbbrev(std::move(StackIdAbbv));
4289 std::vector<uint64_t> StackIds;
4290 for (
auto &
I : StackIdIndices)
4291 StackIds.push_back(
Index.getStackIdAtIndex(
I));
4296 auto Abbv = std::make_shared<BitCodeAbbrev>();
4310 unsigned FSCallsAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
4313 Abbv = std::make_shared<BitCodeAbbrev>();
4327 unsigned FSCallsProfileAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
4330 Abbv = std::make_shared<BitCodeAbbrev>();
4337 unsigned FSModRefsAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
4340 Abbv = std::make_shared<BitCodeAbbrev>();
4346 unsigned FSAliasAbbrev = Stream.
EmitAbbrev(std::move(Abbv));
4348 Abbv = std::make_shared<BitCodeAbbrev>();