19 #define DEBUG_TYPE "orc"
32 createDSOHandleSectionInterface(ENP, DSOHandleSymbol)),
37 void materialize(std::unique_ptr<MaterializationResponsibility> R)
override {
44 switch (TT.getArch()) {
55 auto G = std::make_unique<jitlink::LinkGraph>(
58 auto &DSOHandleSection =
60 auto &DSOHandleBlock =
G->createContentBlock(
63 auto &DSOHandleSymbol =
G->addDefinedSymbol(
64 DSOHandleBlock, 0, *R->getInitializerSymbol(), DSOHandleBlock.getSize(),
66 DSOHandleBlock.addEdge(EdgeKind, 0, DSOHandleSymbol, 0);
84 static const char Content[8] = {0};
92 StringRef EHFrameSectionName =
".eh_frame";
93 StringRef InitArrayFuncSectionName =
".init_array";
96 StringRef ThreadDataSectionName =
".tdata";
98 StringRef InitSectionNames[] = {InitArrayFuncSectionName};
108 JITDylib &PlatformJD,
const char *OrcRuntimePath,
114 if (!supportedTarget(EPC.getTargetTriple()))
115 return make_error<StringError>(
"Unsupported ELFNixPlatform triple: " +
116 EPC.getTargetTriple().str(),
120 if (!RuntimeAliases) {
121 auto StandardRuntimeAliases = standardPlatformAliases(ES, PlatformJD);
122 if (!StandardRuntimeAliases)
123 return StandardRuntimeAliases.takeError();
124 RuntimeAliases =
std::move(*StandardRuntimeAliases);
133 {{ES.
intern(
"__orc_rt_jit_dispatch"),
134 {EPC.getJITDispatchInfo().JITDispatchFunction.getValue(),
136 {ES.
intern(
"__orc_rt_jit_dispatch_ctx"),
137 {EPC.getJITDispatchInfo().JITDispatchContext.getValue(),
143 ObjLinkingLayer, OrcRuntimePath, EPC.getTargetTriple());
144 if (!OrcRuntimeArchiveGenerator)
145 return OrcRuntimeArchiveGenerator.takeError();
149 auto P = std::unique_ptr<ELFNixPlatform>(
151 std::move(*OrcRuntimeArchiveGenerator), Err));
159 std::make_unique<DSOHandleMaterializationUnit>(*
this, DSOHandleSymbol));
173 RegisteredInitSymbols[&JD].add(InitSym,
176 dbgs() <<
"ELFNixPlatform: Registered init symbol " << *InitSym
177 <<
" for MU " << MU.
getName() <<
"\n";
187 ArrayRef<std::pair<const char *, const char *>>
AL) {
188 for (
auto &KV :
AL) {
189 auto AliasName = ES.
intern(KV.first);
190 assert(!Aliases.
count(AliasName) &&
"Duplicate symbol name in alias map");
200 addAliases(ES, Aliases, requiredCXXAliases());
201 addAliases(ES, Aliases, standardRuntimeUtilityAliases());
207 auto RTRegisterFrame = ES.
intern(
"__orc_rt_register_eh_frame_section");
208 auto LibUnwindRegisterFrame = ES.
intern(
"__unw_add_dynamic_eh_frame_section");
209 auto RTDeregisterFrame = ES.
intern(
"__orc_rt_deregister_eh_frame_section");
210 auto LibUnwindDeregisterFrame =
211 ES.
intern(
"__unw_remove_dynamic_eh_frame_section");
214 .
add(LibUnwindRegisterFrame,
216 .
add(LibUnwindDeregisterFrame,
220 return SM.takeError();
221 }
else if (SM->size() == 2) {
223 dbgs() <<
"Using libunwind " << LibUnwindRegisterFrame
224 <<
" for unwind info registration\n";
226 Aliases[
std::move(RTRegisterFrame)] = {LibUnwindRegisterFrame,
228 Aliases[
std::move(RTDeregisterFrame)] = {LibUnwindDeregisterFrame,
234 dbgs() <<
"Using libgcc __register_frame"
235 <<
" for unwind info registration\n";
239 Aliases[
std::move(RTDeregisterFrame)] = {ES.
intern(
"__deregister_frame"),
248 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
249 {
"__cxa_atexit",
"__orc_rt_elfnix_cxa_atexit"},
250 {
"atexit",
"__orc_rt_elfnix_atexit"}};
257 static const std::pair<const char *, const char *>
258 StandardRuntimeUtilityAliases[] = {
259 {
"__orc_rt_run_program",
"__orc_rt_elfnix_run_program"},
260 {
"__orc_rt_log_error",
"__orc_rt_log_error_to_stderr"}};
263 StandardRuntimeUtilityAliases);
267 for (
auto &
Name : InitSectionNames) {
268 if (
Name.equals(SecName))
274 bool ELFNixPlatform::supportedTarget(
const Triple &TT) {
275 switch (TT.getArch()) {
283 ELFNixPlatform::ELFNixPlatform(
286 std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator,
Error &Err)
287 : ES(ES), ObjLinkingLayer(ObjLinkingLayer),
288 DSOHandleSymbol(ES.intern(
"__dso_handle")) {
291 ObjLinkingLayer.
addPlugin(std::make_unique<ELFNixPlatformPlugin>(*
this));
302 RegisteredInitSymbols[&PlatformJD].add(
306 if (
auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
314 if (
auto E2 = bootstrapELFNixRuntime(PlatformJD)) {
320 Error ELFNixPlatform::associateRuntimeSupportFunctions(
JITDylib &PlatformJD) {
323 using GetInitializersSPSSig =
325 WFs[ES.
intern(
"__orc_rt_elfnix_get_initializers_tag")] =
327 this, &ELFNixPlatform::rt_getInitializers);
329 using GetDeinitializersSPSSig =
331 WFs[ES.
intern(
"__orc_rt_elfnix_get_deinitializers_tag")] =
333 this, &ELFNixPlatform::rt_getDeinitializers);
335 using LookupSymbolSPSSig =
337 WFs[ES.
intern(
"__orc_rt_elfnix_symbol_lookup_tag")] =
339 &ELFNixPlatform::rt_lookupSymbol);
344 void ELFNixPlatform::getInitializersBuildSequencePhase(
345 SendInitializerSequenceFn SendResult,
JITDylib &JD,
346 std::vector<JITDylibSP> DFSLinkOrder) {
349 std::lock_guard<std::mutex>
Lock(PlatformMutex);
350 for (
auto &InitJD :
reverse(DFSLinkOrder)) {
352 dbgs() <<
"ELFNixPlatform: Appending inits for \"" << InitJD->getName()
353 <<
"\" to sequence\n";
355 auto ISItr = InitSeqs.find(InitJD.get());
356 if (ISItr != InitSeqs.end()) {
357 FullInitSeq.emplace_back(
std::move(ISItr->second));
358 InitSeqs.erase(ISItr);
366 void ELFNixPlatform::getInitializersLookupPhase(
367 SendInitializerSequenceFn SendResult,
JITDylib &JD) {
371 SendResult(DFSLinkOrder.takeError());
377 for (
auto &InitJD : *DFSLinkOrder) {
378 auto RISItr = RegisteredInitSymbols.find(InitJD.get());
379 if (RISItr != RegisteredInitSymbols.end()) {
380 NewInitSymbols[InitJD.get()] = std::move(RISItr->second);
381 RegisteredInitSymbols.erase(RISItr);
388 if (NewInitSymbols.
empty()) {
389 getInitializersBuildSequencePhase(
std::move(SendResult), JD,
396 [
this, SendResult =
std::move(SendResult), &JD](
Error Err)
mutable {
400 getInitializersLookupPhase(
std::move(SendResult), JD);
405 void ELFNixPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult,
408 dbgs() <<
"ELFNixPlatform::rt_getInitializers(\"" << JDName <<
"\")\n";
414 dbgs() <<
" No such JITDylib \"" << JDName <<
"\". Sending error.\n";
416 SendResult(make_error<StringError>(
"No JITDylib named " + JDName,
421 getInitializersLookupPhase(
std::move(SendResult), *JD);
424 void ELFNixPlatform::rt_getDeinitializers(
425 SendDeinitializerSequenceFn SendResult,
ExecutorAddr Handle) {
427 dbgs() <<
"ELFNixPlatform::rt_getDeinitializers(\""
434 std::lock_guard<std::mutex>
Lock(PlatformMutex);
435 auto I = HandleAddrToJITDylib.find(Handle);
436 if (
I != HandleAddrToJITDylib.end())
442 dbgs() <<
" No JITDylib for handle "
445 SendResult(make_error<StringError>(
"No JITDylib associated with handle " +
454 void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
458 dbgs() <<
"ELFNixPlatform::rt_lookupSymbol(\""
465 std::lock_guard<std::mutex>
Lock(PlatformMutex);
466 auto I = HandleAddrToJITDylib.find(Handle);
467 if (
I != HandleAddrToJITDylib.end())
473 dbgs() <<
" No JITDylib for handle "
476 SendResult(make_error<StringError>(
"No JITDylib associated with handle " +
483 class RtLookupNotifyComplete {
485 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
486 : SendResult(
std::
move(SendResult)) {}
489 assert(
Result->size() == 1 &&
"Unexpected result map count");
492 SendResult(
Result.takeError());
497 SendSymbolAddressFn SendResult;
506 Error ELFNixPlatform::bootstrapELFNixRuntime(
JITDylib &PlatformJD) {
508 std::pair<const char *, ExecutorAddr *> Symbols[] = {
509 {
"__orc_rt_elfnix_platform_bootstrap", &orc_rt_elfnix_platform_bootstrap},
510 {
"__orc_rt_elfnix_platform_shutdown", &orc_rt_elfnix_platform_shutdown},
511 {
"__orc_rt_elfnix_register_object_sections",
512 &orc_rt_elfnix_register_object_sections},
513 {
"__orc_rt_elfnix_create_pthread_key",
514 &orc_rt_elfnix_create_pthread_key}};
517 std::vector<std::pair<SymbolStringPtr, ExecutorAddr *>> AddrsToRecord;
518 for (
const auto &KV : Symbols) {
524 auto RuntimeSymbolAddrs = ES.
lookup(
526 if (!RuntimeSymbolAddrs)
527 return RuntimeSymbolAddrs.takeError();
529 for (
const auto &KV : AddrsToRecord) {
530 auto &
Name = KV.first;
531 assert(RuntimeSymbolAddrs->count(
Name) &&
"Missing runtime symbol?");
532 KV.second->setValue((*RuntimeSymbolAddrs)[
Name].getAddress());
535 auto PJDDSOHandle = ES.
lookup(
538 return PJDDSOHandle.takeError();
541 orc_rt_elfnix_platform_bootstrap, PJDDSOHandle->getAddress()))
547 RuntimeBootstrapped =
true;
548 std::vector<ELFPerObjectSectionsToRegister> DeferredPOSRs;
550 std::lock_guard<std::mutex>
Lock(PlatformMutex);
551 DeferredPOSRs =
std::move(BootstrapPOSRs);
554 for (
auto &
D : DeferredPOSRs)
555 if (
auto Err = registerPerObjectSections(
D))
561 Error ELFNixPlatform::registerInitInfo(
564 std::unique_lock<std::mutex>
Lock(PlatformMutex);
568 auto I = InitSeqs.find(&JD);
569 if (
I == InitSeqs.end()) {
576 if (
auto Err = ES.
lookup(SearchOrder, DSOHandleSymbol).takeError())
580 I = InitSeqs.find(&JD);
582 "Entry missing after header symbol lookup?");
584 InitSeq = &
I->second;
587 for (
auto *Sec : InitSections) {
597 Error ELFNixPlatform::registerPerObjectSections(
600 if (!orc_rt_elfnix_register_object_sections)
601 return make_error<StringError>(
"Attempting to register per-object "
602 "sections, but runtime support has not "
609 orc_rt_elfnix_register_object_sections, ErrResult, POSR))
615 if (!orc_rt_elfnix_create_pthread_key)
616 return make_error<StringError>(
617 "Attempting to create pthread key in target, but runtime support has "
618 "not been loaded yet",
623 orc_rt_elfnix_create_pthread_key, Result))
628 void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(
635 addDSOHandleSupportPasses(MR, Config);
643 addInitializerSupportPasses(MR, Config);
646 addEHAndTLVSupportPasses(MR, Config);
650 ELFNixPlatform::ELFNixPlatformPlugin::getSyntheticSymbolDependencies(
652 std::lock_guard<std::mutex>
Lock(PluginMutex);
653 auto I = InitSymbolDeps.find(&MR);
654 if (
I != InitSymbolDeps.end()) {
655 SyntheticSymbolDependenciesMap
Result;
657 InitSymbolDeps.erase(&MR);
660 return SyntheticSymbolDependenciesMap();
663 void ELFNixPlatform::ELFNixPlatformPlugin::addInitializerSupportPasses(
668 if (
auto Err = preserveInitSections(
G, MR))
675 return registerInitSections(
G, JD);
679 void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
684 auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
685 return Sym->getName() == *MP.DSOHandleSymbol;
687 assert(
I !=
G.defined_symbols().end() &&
"Missing DSO handle symbol");
689 std::lock_guard<std::mutex>
Lock(MP.PlatformMutex);
690 auto HandleAddr = (*I)->getAddress();
691 MP.HandleAddrToJITDylib[HandleAddr] = &JD;
692 assert(!MP.InitSeqs.count(&JD) &&
"InitSeq entry for JD already exists");
693 MP.InitSeqs.insert(std::make_pair(
700 void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
710 return fixTLVSectionsAndEdges(G, JD);
718 if (
auto *EHFrameSection =
G.findSectionByName(EHFrameSectionName)) {
719 jitlink::SectionRange R(*EHFrameSection);
721 POSR.EHFrameSection = {ExecutorAddr(R.getStart()),
722 ExecutorAddr(R.getEnd())};
728 G.findSectionByName(ThreadDataSectionName);
731 if (
auto *ThreadBSSSection =
G.findSectionByName(ThreadBSSSectionName)) {
735 if (ThreadDataSection)
736 G.mergeSections(*ThreadDataSection, *ThreadBSSSection);
738 ThreadDataSection = ThreadBSSSection;
743 if (ThreadDataSection) {
754 if (!MP.RuntimeBootstrapped) {
755 std::lock_guard<std::mutex>
Lock(MP.PlatformMutex);
756 MP.BootstrapPOSRs.push_back(POSR);
761 if (
auto Err = MP.registerPerObjectSections(POSR))
769 Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(
772 JITLinkSymbolSet InitSectionSymbols;
773 for (
auto &InitSectionName : InitSectionNames) {
775 auto *InitSection =
G.findSectionByName(InitSectionName);
782 for (
auto &Sym : InitSection->symbols()) {
783 auto &
B = Sym->getBlock();
784 if (Sym->isLive() && Sym->getOffset() == 0 &&
785 Sym->getSize() ==
B.getSize() && !AlreadyLiveBlocks.
count(&
B)) {
786 InitSectionSymbols.insert(Sym);
792 for (
auto *
B : InitSection->blocks())
793 if (!AlreadyLiveBlocks.
count(
B))
794 InitSectionSymbols.insert(
795 &
G.addAnonymousSymbol(*
B, 0,
B->getSize(),
false,
true));
798 if (!InitSectionSymbols.empty()) {
799 std::lock_guard<std::mutex>
Lock(PluginMutex);
800 InitSymbolDeps[&MR] =
std::move(InitSectionSymbols);
803 return Error::success();
806 Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
811 LLVM_DEBUG({
dbgs() <<
"ELFNixPlatform::registerInitSections\n"; });
813 for (
auto InitSectionName : InitSectionNames) {
814 if (
auto *Sec =
G.findSectionByName(InitSectionName)) {
815 InitSections.push_back(Sec);
821 dbgs() <<
"ELFNixPlatform: Scraped " <<
G.getName() <<
" init sections:\n";
822 for (
auto *Sec : InitSections) {
824 dbgs() <<
" " << Sec->getName() <<
": "
825 <<
formatv(
"[ {0:x} -- {1:x} ]",
R.getStart(),
R.getEnd()) <<
"\n";
829 return MP.registerInitInfo(JD, InitSections);
832 Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
836 for (
auto *Sym :
G.external_symbols())
837 if (Sym->getName() ==
"__tls_get_addr") {
838 Sym->setName(
"___orc_rt_elfnix_tls_get_addr");
841 auto *TLSInfoEntrySection =
G.findSectionByName(
"$__TLSINFO");
843 if (TLSInfoEntrySection) {
846 std::lock_guard<std::mutex>
Lock(MP.PlatformMutex);
847 auto I = MP.JITDylibToPThreadKey.find(&JD);
848 if (
I != MP.JITDylibToPThreadKey.end())
852 if (
auto KeyOrErr = MP.createPThreadKey())
855 return KeyOrErr.takeError();
861 for (
auto *
B : TLSInfoEntrySection->blocks()) {
864 assert(
B->getSize() == (
G.getPointerSize() * 2) &&
865 "TLS descriptor must be 2 words length");
866 auto TLSInfoEntryContent =
B->getMutableContent(
G);
867 memcpy(TLSInfoEntryContent.data(), &PlatformKeyBits,
G.getPointerSize());
871 return Error::success();