19 #define DEBUG_TYPE "orc"
43 return SPSMachOJITDylibDepInfo::AsArgList::serialize(
OB, DDI.
Sealed,
49 return SPSMachOJITDylibDepInfo::AsArgList::deserialize(IB, DDI.
Sealed,
69 void materialize(std::unique_ptr<MaterializationResponsibility> R)
override {
73 MOP.getExecutionSession().getExecutorProcessControl().getTargetTriple();
75 switch (
TT.getArch()) {
85 auto G = std::make_unique<jitlink::LinkGraph>(
89 auto &HeaderBlock = createHeaderBlock(*
G, HeaderSection);
92 G->addDefinedSymbol(HeaderBlock, 0, *
R->getInitializerSymbol(),
95 for (
auto &
HS : AdditionalHeaderSymbols)
96 G->addDefinedSymbol(HeaderBlock,
HS.Offset,
HS.Name,
106 struct HeaderSymbol {
111 static constexpr HeaderSymbol AdditionalHeaderSymbols[] = {
112 {
"___mh_executable_header", 0}};
118 switch (
G.getTargetTriple().getArch()) {
139 auto HeaderContent =
G.allocateString(
140 StringRef(
reinterpret_cast<const char *
>(&Hdr),
sizeof(Hdr)));
142 return G.createContentBlock(HeaderSection, HeaderContent,
ExecutorAddr(), 8,
152 for (
auto &
HS : AdditionalHeaderSymbols)
163 constexpr MachOHeaderMaterializationUnit::HeaderSymbol
164 MachOHeaderMaterializationUnit::AdditionalHeaderSymbols[];
166 StringRef EHFrameSectionName =
"__TEXT,__eh_frame";
167 StringRef ModInitFuncSectionName =
"__DATA,__mod_init_func";
168 StringRef ObjCClassListSectionName =
"__DATA,__objc_classlist";
169 StringRef ObjCImageInfoSectionName =
"__DATA,__objc_image_info";
170 StringRef ObjCSelRefsSectionName =
"__DATA,__objc_selrefs";
171 StringRef Swift5ProtoSectionName =
"__TEXT,__swift5_proto";
172 StringRef Swift5ProtosSectionName =
"__TEXT,__swift5_protos";
173 StringRef Swift5TypesSectionName =
"__TEXT,__swift5_types";
174 StringRef ThreadBSSSectionName =
"__DATA,__thread_bss";
175 StringRef ThreadDataSectionName =
"__DATA,__thread_data";
176 StringRef ThreadVarsSectionName =
"__DATA,__thread_vars";
179 ModInitFuncSectionName, ObjCSelRefsSectionName, ObjCClassListSectionName,
180 Swift5ProtosSectionName, Swift5ProtoSectionName, Swift5TypesSectionName};
189 JITDylib &PlatformJD,
const char *OrcRuntimePath,
195 if (!supportedTarget(EPC.getTargetTriple()))
196 return make_error<StringError>(
"Unsupported MachOPlatform triple: " +
197 EPC.getTargetTriple().str(),
202 RuntimeAliases = standardPlatformAliases(ES);
210 {{ES.
intern(
"___orc_rt_jit_dispatch"),
211 {EPC.getJITDispatchInfo().JITDispatchFunction.getValue(),
213 {ES.
intern(
"___orc_rt_jit_dispatch_ctx"),
214 {EPC.getJITDispatchInfo().JITDispatchContext.getValue(),
220 ObjLinkingLayer, OrcRuntimePath, EPC.getTargetTriple());
221 if (!OrcRuntimeArchiveGenerator)
222 return OrcRuntimeArchiveGenerator.takeError();
226 auto P = std::unique_ptr<MachOPlatform>(
228 std::move(*OrcRuntimeArchiveGenerator), Err));
235 if (
auto Err = JD.
define(std::make_unique<MachOHeaderMaterializationUnit>(
236 *
this, MachOHeaderStartSymbol)))
239 return ES.lookup({&JD}, MachOHeaderStartSymbol).takeError();
243 std::lock_guard<std::mutex>
Lock(PlatformMutex);
244 auto I = JITDylibToHeaderAddr.find(&JD);
245 if (
I != JITDylibToHeaderAddr.end()) {
246 assert(HeaderAddrToJITDylib.count(
I->second) &&
247 "HeaderAddrToJITDylib missing entry");
248 HeaderAddrToJITDylib.erase(
I->second);
249 JITDylibToHeaderAddr.erase(
I);
251 JITDylibToPThreadKey.erase(&JD);
262 RegisteredInitSymbols[&JD].add(InitSym,
265 dbgs() <<
"MachOPlatform: Registered init symbol " << *InitSym <<
" for MU "
276 ArrayRef<std::pair<const char *, const char *>>
AL) {
277 for (
auto &KV :
AL) {
278 auto AliasName = ES.
intern(KV.first);
279 assert(!Aliases.
count(AliasName) &&
"Duplicate symbol name in alias map");
287 addAliases(ES, Aliases, requiredCXXAliases());
288 addAliases(ES, Aliases, standardRuntimeUtilityAliases());
294 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
295 {
"___cxa_atexit",
"___orc_rt_macho_cxa_atexit"}};
302 static const std::pair<const char *, const char *>
303 StandardRuntimeUtilityAliases[] = {
304 {
"___orc_rt_run_program",
"___orc_rt_macho_run_program"},
305 {
"___orc_rt_log_error",
"___orc_rt_log_error_to_stderr"}};
308 StandardRuntimeUtilityAliases);
313 for (
auto &
Name : InitSectionNames) {
314 if (
Name.startswith(SegName) &&
Name.substr(7) == SectName)
320 bool MachOPlatform::supportedTarget(
const Triple &TT) {
321 switch (TT.getArch()) {
330 MachOPlatform::MachOPlatform(
333 std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator,
Error &Err)
334 : ES(ES), ObjLinkingLayer(ObjLinkingLayer),
335 MachOHeaderStartSymbol(ES.intern(
"___dso_handle")) {
338 ObjLinkingLayer.
addPlugin(std::make_unique<MachOPlatformPlugin>(*
this));
345 {{ES.
intern(
"___orc_rt_macho_register_ehframe_section"),
346 &orc_rt_macho_register_ehframe_section},
347 {ES.
intern(
"___orc_rt_macho_deregister_ehframe_section"),
348 &orc_rt_macho_deregister_ehframe_section}})) {
353 State = BootstrapPhase2;
356 if (
auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
364 if (
auto E2 = bootstrapMachORuntime(PlatformJD)) {
379 Error MachOPlatform::associateRuntimeSupportFunctions(
JITDylib &PlatformJD) {
382 using PushInitializersSPSSig =
384 WFs[ES.
intern(
"___orc_rt_macho_push_initializers_tag")] =
386 this, &MachOPlatform::rt_pushInitializers);
388 using LookupSymbolSPSSig =
390 WFs[ES.
intern(
"___orc_rt_macho_symbol_lookup_tag")] =
392 &MachOPlatform::rt_lookupSymbol);
397 void MachOPlatform::pushInitializersLoop(
398 PushInitializersSendResultFn SendResult,
JITDylibSP JD) {
404 while (!Worklist.empty()) {
407 auto DepJD = Worklist.back();
411 if (JDDepMap.count(DepJD))
415 auto &DM = JDDepMap[DepJD];
416 DepJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) {
418 if (KV.first == DepJD)
420 DM.push_back(KV.first);
421 Worklist.push_back(KV.first);
426 auto RISItr = RegisteredInitSymbols.find(DepJD);
427 if (RISItr != RegisteredInitSymbols.end()) {
428 NewInitSymbols[DepJD] =
std::move(RISItr->second);
429 RegisteredInitSymbols.erase(RISItr);
436 if (NewInitSymbols.
empty()) {
443 std::lock_guard<std::mutex>
Lock(PlatformMutex);
444 for (
auto &KV : JDDepMap) {
445 auto I = JITDylibToHeaderAddr.find(KV.first);
446 if (
I == JITDylibToHeaderAddr.end()) {
451 make_error<StringError>(
"JITDylib " + KV.first->getName() +
452 " has no registered header address",
456 HeaderAddrs[KV.first] =
I->second;
461 MachOJITDylibDepInfoMap DIM;
462 DIM.reserve(JDDepMap.size());
463 for (
auto &KV : JDDepMap) {
464 assert(HeaderAddrs.
count(KV.first) &&
"Missing header addr");
465 auto H = HeaderAddrs[KV.first];
466 MachOJITDylibDepInfo DepInfo;
467 for (
auto &Dep : KV.second) {
468 assert(HeaderAddrs.
count(Dep) &&
"Missing header addr");
469 DepInfo.DepHeaders.push_back(HeaderAddrs[Dep]);
471 DIM.push_back(std::make_pair(
H,
std::move(DepInfo)));
478 lookupInitSymbolsAsync(
479 [
this, SendResult =
std::move(SendResult), &JD](
Error Err)
mutable {
483 pushInitializersLoop(
std::move(SendResult), JD);
488 void MachOPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult,
492 std::lock_guard<std::mutex>
Lock(PlatformMutex);
493 auto I = HeaderAddrToJITDylib.find(JDHeaderAddr);
494 if (
I != HeaderAddrToJITDylib.end())
499 dbgs() <<
"MachOPlatform::rt_pushInitializers(" << JDHeaderAddr <<
") ";
501 dbgs() <<
"pushing initializers for " << JD->getName() <<
"\n";
503 dbgs() <<
"No JITDylib for header address.\n";
508 make_error<StringError>(
"No JITDylib with header addr " +
514 pushInitializersLoop(
std::move(SendResult), JD);
517 void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
520 dbgs() <<
"MachOPlatform::rt_lookupSymbol(\""
527 std::lock_guard<std::mutex>
Lock(PlatformMutex);
528 auto I = HeaderAddrToJITDylib.find(Handle);
529 if (
I != HeaderAddrToJITDylib.end())
535 dbgs() <<
" No JITDylib for handle "
538 SendResult(make_error<StringError>(
"No JITDylib associated with handle " +
545 class RtLookupNotifyComplete {
547 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
548 : SendResult(
std::
move(SendResult)) {}
551 assert(
Result->size() == 1 &&
"Unexpected result map count");
554 SendResult(
Result.takeError());
559 SendSymbolAddressFn SendResult;
565 LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
570 Error MachOPlatform::bootstrapMachORuntime(
JITDylib &PlatformJD) {
573 {{ES.
intern(
"___orc_rt_macho_platform_bootstrap"),
574 &orc_rt_macho_platform_bootstrap},
575 {ES.
intern(
"___orc_rt_macho_platform_shutdown"),
576 &orc_rt_macho_platform_shutdown},
577 {ES.
intern(
"___orc_rt_macho_register_jitdylib"),
578 &orc_rt_macho_register_jitdylib},
579 {ES.
intern(
"___orc_rt_macho_deregister_jitdylib"),
580 &orc_rt_macho_deregister_jitdylib},
581 {ES.
intern(
"___orc_rt_macho_register_object_platform_sections"),
582 &orc_rt_macho_register_object_platform_sections},
583 {ES.
intern(
"___orc_rt_macho_deregister_object_platform_sections"),
584 &orc_rt_macho_deregister_object_platform_sections},
585 {ES.
intern(
"___orc_rt_macho_create_pthread_key"),
586 &orc_rt_macho_create_pthread_key}}))
589 return ES.
callSPSWrapper<void()>(orc_rt_macho_platform_bootstrap);
593 if (!orc_rt_macho_create_pthread_key)
594 return make_error<StringError>(
595 "Attempting to create pthread key in target, but runtime support has "
596 "not been loaded yet",
601 orc_rt_macho_create_pthread_key, Result))
606 void MachOPlatform::MachOPlatformPlugin::modifyPassConfig(
610 auto PS = MP.State.load();
618 if (InitSymbol == MP.MachOHeaderStartSymbol) {
620 return associateJITDylibHeaderSymbol(
G, MR);
629 if (
auto Err = preserveInitSections(
G, MR))
631 return processObjCImageInfo(
G, MR);
636 if (PS == MachOPlatform::BootstrapPhase1) {
647 return fixTLVSectionsAndEdges(
G, JD);
654 return registerObjectPlatformSections(
G, JD);
659 MachOPlatform::MachOPlatformPlugin::getSyntheticSymbolDependencies(
661 std::lock_guard<std::mutex>
Lock(PluginMutex);
662 auto I = InitSymbolDeps.find(&MR);
663 if (
I != InitSymbolDeps.end()) {
664 SyntheticSymbolDependenciesMap
Result;
666 InitSymbolDeps.erase(&MR);
669 return SyntheticSymbolDependenciesMap();
672 Error MachOPlatform::MachOPlatformPlugin::associateJITDylibHeaderSymbol(
675 return Sym->getName() == *MP.MachOHeaderStartSymbol;
677 assert(
I !=
G.defined_symbols().end() &&
"Missing MachO header start symbol");
680 std::lock_guard<std::mutex>
Lock(MP.PlatformMutex);
681 auto HeaderAddr = (*I)->getAddress();
682 MP.JITDylibToHeaderAddr[&JD] = HeaderAddr;
683 MP.HeaderAddrToJITDylib[HeaderAddr] = &JD;
684 G.allocActions().push_back(
687 MP.orc_rt_macho_register_jitdylib, JD.getName(), HeaderAddr)),
689 MP.orc_rt_macho_deregister_jitdylib, HeaderAddr))});
693 Error MachOPlatform::MachOPlatformPlugin::preserveInitSections(
696 JITLinkSymbolSet InitSectionSymbols;
697 for (
auto &InitSectionName : InitSectionNames) {
699 auto *InitSection =
G.findSectionByName(InitSectionName);
706 for (
auto &Sym : InitSection->symbols()) {
707 auto &
B = Sym->getBlock();
708 if (Sym->isLive() && Sym->getOffset() == 0 &&
709 Sym->getSize() ==
B.getSize() && !AlreadyLiveBlocks.
count(&
B)) {
710 InitSectionSymbols.insert(Sym);
716 for (
auto *
B : InitSection->blocks())
717 if (!AlreadyLiveBlocks.
count(
B))
718 InitSectionSymbols.insert(
719 &
G.addAnonymousSymbol(*
B, 0,
B->getSize(),
false,
true));
722 if (!InitSectionSymbols.empty()) {
723 std::lock_guard<std::mutex>
Lock(PluginMutex);
724 InitSymbolDeps[&MR] =
std::move(InitSectionSymbols);
730 Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo(
739 auto *ObjCImageInfo =
G.findSectionByName(ObjCImageInfoSectionName);
743 auto ObjCImageInfoBlocks = ObjCImageInfo->blocks();
747 return make_error<StringError>(
"Empty " + ObjCImageInfoSectionName +
748 " section in " +
G.getName(),
752 if (std::next(ObjCImageInfoBlocks.begin()) != ObjCImageInfoBlocks.end())
753 return make_error<StringError>(
"Multiple blocks in " +
754 ObjCImageInfoSectionName +
755 " section in " +
G.getName(),
760 for (
auto &Sec :
G.sections()) {
761 if (&Sec != ObjCImageInfo)
762 for (
auto *
B : Sec.blocks())
763 for (
auto &
E :
B->edges())
764 if (
E.getTarget().isDefined() &&
765 &
E.getTarget().getBlock().getSection() == ObjCImageInfo)
766 return make_error<StringError>(ObjCImageInfoSectionName +
767 " is referenced within file " +
772 auto &ObjCImageInfoBlock = **ObjCImageInfoBlocks.begin();
773 auto *ObjCImageInfoData = ObjCImageInfoBlock.getContent().data();
779 std::lock_guard<std::mutex>
Lock(PluginMutex);
782 if (ObjCImageInfoItr != ObjCImageInfos.end()) {
785 if (ObjCImageInfoItr->second.first !=
Version)
786 return make_error<StringError>(
787 "ObjC version in " +
G.getName() +
788 " does not match first registered version",
790 if (ObjCImageInfoItr->second.second != Flags)
791 return make_error<StringError>(
"ObjC flags in " +
G.getName() +
792 " do not match first registered flags",
796 for (
auto *
S : ObjCImageInfo->symbols())
797 G.removeDefinedSymbol(*
S);
798 G.removeBlock(ObjCImageInfoBlock);
808 Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges(
812 for (
auto *Sym :
G.external_symbols())
813 if (Sym->getName() ==
"__tlv_bootstrap") {
814 Sym->setName(
"___orc_rt_macho_tlv_get_addr");
819 if (
auto *ThreadDataSec =
G.findSectionByName(ThreadVarsSectionName)) {
822 std::lock_guard<std::mutex>
Lock(MP.PlatformMutex);
823 auto I = MP.JITDylibToPThreadKey.find(&JD);
824 if (
I != MP.JITDylibToPThreadKey.end())
829 if (
auto KeyOrErr = MP.createPThreadKey())
832 return KeyOrErr.takeError();
838 for (
auto *
B : ThreadDataSec->blocks()) {
839 if (
B->getSize() != 3 *
G.getPointerSize())
840 return make_error<StringError>(
"__thread_vars block at " +
842 " has unexpected size",
845 auto NewBlockContent =
G.allocateBuffer(
B->getSize());
846 llvm::copy(
B->getContent(), NewBlockContent.data());
847 memcpy(NewBlockContent.data() +
G.getPointerSize(), &PlatformKeyBits,
849 B->setContent(NewBlockContent);
854 for (
auto *
B :
G.blocks())
855 for (
auto &
E :
B->edges())
858 E.setKind(jitlink::x86_64::
864 Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections(
868 if (
auto *EHFrameSection =
G.findSectionByName(EHFrameSectionName)) {
871 G.allocActions().push_back(
874 MP.orc_rt_macho_register_ehframe_section,
R.getRange())),
877 MP.orc_rt_macho_deregister_ehframe_section,
R.getRange()))});
883 G.findSectionByName(ThreadDataSectionName);
886 if (
auto *ThreadBSSSection =
G.findSectionByName(ThreadBSSSectionName)) {
890 if (ThreadDataSection)
891 G.mergeSections(*ThreadDataSection, *ThreadBSSSection);
893 ThreadDataSection = ThreadBSSSection;
900 if (ThreadDataSection) {
903 if (MP.State != MachOPlatform::Initialized)
904 return make_error<StringError>(
"__thread_data section encountered, but "
905 "MachOPlatform has not finished booting",
908 MachOPlatformSecs.push_back({ThreadDataSectionName,
R.getRange()});
915 ModInitFuncSectionName, ObjCClassListSectionName,
916 ObjCImageInfoSectionName, ObjCSelRefsSectionName,
917 Swift5ProtoSectionName, Swift5ProtosSectionName,
918 Swift5TypesSectionName,
921 for (
auto &SecName : PlatformSections) {
922 auto *Sec =
G.findSectionByName(SecName);
929 MachOPlatformSecs.push_back({SecName,
R.getRange()});
932 if (!MachOPlatformSecs.empty()) {
935 std::lock_guard<std::mutex>
Lock(MP.PlatformMutex);
936 auto I = MP.JITDylibToHeaderAddr.find(&JD);
937 if (
I != MP.JITDylibToHeaderAddr.end())
938 HeaderAddr =
I->second;
942 return make_error<StringError>(
"Missing header for " + JD.
getName(),
947 dbgs() <<
"MachOPlatform: Scraped " <<
G.getName() <<
" init sections:\n";
948 for (
auto &KV : MachOPlatformSecs)
949 dbgs() <<
" " << KV.first <<
": " << KV.second <<
"\n";
952 using SPSRegisterObjectPlatformSectionsArgs =
955 G.allocActions().push_back(
957 WrapperFunctionCall::Create<SPSRegisterObjectPlatformSectionsArgs>(
958 MP.orc_rt_macho_register_object_platform_sections, *HeaderAddr,
961 WrapperFunctionCall::Create<SPSRegisterObjectPlatformSectionsArgs>(
962 MP.orc_rt_macho_deregister_object_platform_sections,
963 *HeaderAddr, MachOPlatformSecs))});
969 Error MachOPlatform::MachOPlatformPlugin::registerEHSectionsPhase1(
973 auto *EHFrameSection =
G.findSectionByName(EHFrameSectionName);
987 for (
auto *Sym :
G.defined_symbols()) {
990 if (Sym->getName() ==
"___orc_rt_macho_register_ehframe_section")
991 orc_rt_macho_register_ehframe_section =
ExecutorAddr(Sym->getAddress());
992 else if (Sym->getName() ==
"___orc_rt_macho_deregister_ehframe_section")
993 orc_rt_macho_deregister_ehframe_section =
ExecutorAddr(Sym->getAddress());
995 if (orc_rt_macho_register_ehframe_section &&
996 orc_rt_macho_deregister_ehframe_section)
1001 if (!orc_rt_macho_register_ehframe_section ||
1002 !orc_rt_macho_deregister_ehframe_section)
1003 return make_error<StringError>(
"Could not find eh-frame registration "
1004 "functions during platform bootstrap",
1009 G.allocActions().push_back(
1011 orc_rt_macho_register_ehframe_section,
R.getRange())),
1013 orc_rt_macho_deregister_ehframe_section,
R.getRange()))});