LLVM 19.0.0git
Core.cpp
Go to the documentation of this file.
1//===--- Core.cpp - Core ORC APIs (MaterializationUnit, JITDylib, etc.) ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
10
11#include "llvm/ADT/STLExtras.h"
12#include "llvm/Config/llvm-config.h"
17
18#include <condition_variable>
19#include <future>
20#include <optional>
21
22#define DEBUG_TYPE "orc"
23
24namespace llvm {
25namespace orc {
26
29char SymbolsNotFound::ID = 0;
35char LookupTask::ID = 0;
36
39
40void MaterializationUnit::anchor() {}
41
43 assert((reinterpret_cast<uintptr_t>(JD.get()) & 0x1) == 0 &&
44 "JITDylib must be two byte aligned");
45 JD->Retain();
46 JDAndFlag.store(reinterpret_cast<uintptr_t>(JD.get()));
47}
48
50 getJITDylib().getExecutionSession().destroyResourceTracker(*this);
52}
53
55 return getJITDylib().getExecutionSession().removeResourceTracker(*this);
56}
57
59 getJITDylib().getExecutionSession().transferResourceTracker(DstRT, *this);
60}
61
62void ResourceTracker::makeDefunct() {
63 uintptr_t Val = JDAndFlag.load();
64 Val |= 0x1U;
65 JDAndFlag.store(Val);
66}
67
69
71 : RT(std::move(RT)) {}
72
75}
76
78 OS << "Resource tracker " << (void *)RT.get() << " became defunct";
79}
80
82 std::shared_ptr<SymbolStringPool> SSP,
83 std::shared_ptr<SymbolDependenceMap> Symbols)
84 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
85 assert(this->SSP && "String pool cannot be null");
86 assert(!this->Symbols->empty() && "Can not fail to resolve an empty set");
87
88 // FIXME: Use a new dep-map type for FailedToMaterialize errors so that we
89 // don't have to manually retain/release.
90 for (auto &[JD, Syms] : *this->Symbols)
91 JD->Retain();
92}
93
95 for (auto &[JD, Syms] : *Symbols)
96 JD->Release();
97}
98
101}
102
104 OS << "Failed to materialize symbols: " << *Symbols;
105}
106
108 std::shared_ptr<SymbolStringPool> SSP, JITDylibSP JD,
109 SymbolNameSet FailedSymbols, SymbolDependenceMap BadDeps,
110 std::string Explanation)
111 : SSP(std::move(SSP)), JD(std::move(JD)),
112 FailedSymbols(std::move(FailedSymbols)), BadDeps(std::move(BadDeps)),
113 Explanation(std::move(Explanation)) {}
114
117}
118
120 OS << "In " << JD->getName() << ", failed to materialize " << FailedSymbols
121 << ", due to unsatisfied dependencies " << BadDeps;
122 if (!Explanation.empty())
123 OS << " (" << Explanation << ")";
124}
125
126SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP,
127 SymbolNameSet Symbols)
128 : SSP(std::move(SSP)) {
129 for (auto &Sym : Symbols)
130 this->Symbols.push_back(Sym);
131 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
132}
133
134SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP,
135 SymbolNameVector Symbols)
136 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
137 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
138}
139
140std::error_code SymbolsNotFound::convertToErrorCode() const {
142}
143
145 OS << "Symbols not found: " << Symbols;
146}
147
149 std::shared_ptr<SymbolStringPool> SSP, SymbolNameSet Symbols)
150 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
151 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
152}
153
156}
157
159 OS << "Symbols could not be removed: " << Symbols;
160}
161
164}
165
167 OS << "Missing definitions in module " << ModuleName
168 << ": " << Symbols;
169}
170
173}
174
176 OS << "Unexpected definitions in module " << ModuleName
177 << ": " << Symbols;
178}
179
181 const SymbolLookupSet &Symbols, SymbolState RequiredState,
182 SymbolsResolvedCallback NotifyComplete)
183 : NotifyComplete(std::move(NotifyComplete)), RequiredState(RequiredState) {
184 assert(RequiredState >= SymbolState::Resolved &&
185 "Cannot query for a symbols that have not reached the resolve state "
186 "yet");
187
188 OutstandingSymbolsCount = Symbols.size();
189
190 for (auto &[Name, Flags] : Symbols)
191 ResolvedSymbols[Name] = ExecutorSymbolDef();
192}
193
196 auto I = ResolvedSymbols.find(Name);
197 assert(I != ResolvedSymbols.end() &&
198 "Resolving symbol outside the requested set");
199 assert(I->second == ExecutorSymbolDef() &&
200 "Redundantly resolving symbol Name");
201
202 // If this is a materialization-side-effects-only symbol then drop it,
203 // otherwise update its map entry with its resolved address.
204 if (Sym.getFlags().hasMaterializationSideEffectsOnly())
205 ResolvedSymbols.erase(I);
206 else
207 I->second = std::move(Sym);
208 --OutstandingSymbolsCount;
209}
210
211void AsynchronousSymbolQuery::handleComplete(ExecutionSession &ES) {
212 assert(OutstandingSymbolsCount == 0 &&
213 "Symbols remain, handleComplete called prematurely");
214
215 class RunQueryCompleteTask : public Task {
216 public:
217 RunQueryCompleteTask(SymbolMap ResolvedSymbols,
218 SymbolsResolvedCallback NotifyComplete)
219 : ResolvedSymbols(std::move(ResolvedSymbols)),
220 NotifyComplete(std::move(NotifyComplete)) {}
221 void printDescription(raw_ostream &OS) override {
222 OS << "Execute query complete callback for " << ResolvedSymbols;
223 }
224 void run() override { NotifyComplete(std::move(ResolvedSymbols)); }
225
226 private:
227 SymbolMap ResolvedSymbols;
228 SymbolsResolvedCallback NotifyComplete;
229 };
230
231 auto T = std::make_unique<RunQueryCompleteTask>(std::move(ResolvedSymbols),
232 std::move(NotifyComplete));
233 NotifyComplete = SymbolsResolvedCallback();
234 ES.dispatchTask(std::move(T));
235}
236
237void AsynchronousSymbolQuery::handleFailed(Error Err) {
238 assert(QueryRegistrations.empty() && ResolvedSymbols.empty() &&
239 OutstandingSymbolsCount == 0 &&
240 "Query should already have been abandoned");
241 NotifyComplete(std::move(Err));
242 NotifyComplete = SymbolsResolvedCallback();
243}
244
245void AsynchronousSymbolQuery::addQueryDependence(JITDylib &JD,
246 SymbolStringPtr Name) {
247 bool Added = QueryRegistrations[&JD].insert(std::move(Name)).second;
248 (void)Added;
249 assert(Added && "Duplicate dependence notification?");
250}
251
252void AsynchronousSymbolQuery::removeQueryDependence(
253 JITDylib &JD, const SymbolStringPtr &Name) {
254 auto QRI = QueryRegistrations.find(&JD);
255 assert(QRI != QueryRegistrations.end() &&
256 "No dependencies registered for JD");
257 assert(QRI->second.count(Name) && "No dependency on Name in JD");
258 QRI->second.erase(Name);
259 if (QRI->second.empty())
260 QueryRegistrations.erase(QRI);
261}
262
263void AsynchronousSymbolQuery::dropSymbol(const SymbolStringPtr &Name) {
264 auto I = ResolvedSymbols.find(Name);
265 assert(I != ResolvedSymbols.end() &&
266 "Redundant removal of weakly-referenced symbol");
267 ResolvedSymbols.erase(I);
268 --OutstandingSymbolsCount;
269}
270
271void AsynchronousSymbolQuery::detach() {
272 ResolvedSymbols.clear();
273 OutstandingSymbolsCount = 0;
274 for (auto &[JD, Syms] : QueryRegistrations)
275 JD->detachQueryHelper(*this, Syms);
276 QueryRegistrations.clear();
277}
278
280 SymbolMap Symbols)
281 : MaterializationUnit(extractFlags(Symbols)), Symbols(std::move(Symbols)) {}
282
284 return "<Absolute Symbols>";
285}
286
287void AbsoluteSymbolsMaterializationUnit::materialize(
288 std::unique_ptr<MaterializationResponsibility> R) {
289 // Even though these are just absolute symbols we need to check for failure
290 // to resolve/emit: the tracker for these symbols may have been removed while
291 // the materialization was in flight (e.g. due to a failure in some action
292 // triggered by the queries attached to the resolution/emission of these
293 // symbols).
294 if (auto Err = R->notifyResolved(Symbols)) {
295 R->getExecutionSession().reportError(std::move(Err));
296 R->failMaterialization();
297 return;
298 }
299 if (auto Err = R->notifyEmitted({})) {
300 R->getExecutionSession().reportError(std::move(Err));
301 R->failMaterialization();
302 return;
303 }
304}
305
306void AbsoluteSymbolsMaterializationUnit::discard(const JITDylib &JD,
307 const SymbolStringPtr &Name) {
308 assert(Symbols.count(Name) && "Symbol is not part of this MU");
309 Symbols.erase(Name);
310}
311
312MaterializationUnit::Interface
313AbsoluteSymbolsMaterializationUnit::extractFlags(const SymbolMap &Symbols) {
315 for (const auto &[Name, Def] : Symbols)
316 Flags[Name] = Def.getFlags();
317 return MaterializationUnit::Interface(std::move(Flags), nullptr);
318}
319
321 JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags,
322 SymbolAliasMap Aliases)
323 : MaterializationUnit(extractFlags(Aliases)), SourceJD(SourceJD),
324 SourceJDLookupFlags(SourceJDLookupFlags), Aliases(std::move(Aliases)) {}
325
327 return "<Reexports>";
328}
329
330void ReExportsMaterializationUnit::materialize(
331 std::unique_ptr<MaterializationResponsibility> R) {
332
333 auto &ES = R->getTargetJITDylib().getExecutionSession();
334 JITDylib &TgtJD = R->getTargetJITDylib();
335 JITDylib &SrcJD = SourceJD ? *SourceJD : TgtJD;
336
337 // Find the set of requested aliases and aliasees. Return any unrequested
338 // aliases back to the JITDylib so as to not prematurely materialize any
339 // aliasees.
340 auto RequestedSymbols = R->getRequestedSymbols();
341 SymbolAliasMap RequestedAliases;
342
343 for (auto &Name : RequestedSymbols) {
344 auto I = Aliases.find(Name);
345 assert(I != Aliases.end() && "Symbol not found in aliases map?");
346 RequestedAliases[Name] = std::move(I->second);
347 Aliases.erase(I);
348 }
349
350 LLVM_DEBUG({
351 ES.runSessionLocked([&]() {
352 dbgs() << "materializing reexports: target = " << TgtJD.getName()
353 << ", source = " << SrcJD.getName() << " " << RequestedAliases
354 << "\n";
355 });
356 });
357
358 if (!Aliases.empty()) {
359 auto Err = SourceJD ? R->replace(reexports(*SourceJD, std::move(Aliases),
360 SourceJDLookupFlags))
361 : R->replace(symbolAliases(std::move(Aliases)));
362
363 if (Err) {
364 // FIXME: Should this be reported / treated as failure to materialize?
365 // Or should this be treated as a sanctioned bailing-out?
366 ES.reportError(std::move(Err));
367 R->failMaterialization();
368 return;
369 }
370 }
371
372 // The OnResolveInfo struct will hold the aliases and responsibility for each
373 // query in the list.
374 struct OnResolveInfo {
375 OnResolveInfo(std::unique_ptr<MaterializationResponsibility> R,
376 SymbolAliasMap Aliases)
377 : R(std::move(R)), Aliases(std::move(Aliases)) {}
378
379 std::unique_ptr<MaterializationResponsibility> R;
380 SymbolAliasMap Aliases;
381 std::vector<SymbolDependenceGroup> SDGs;
382 };
383
384 // Build a list of queries to issue. In each round we build a query for the
385 // largest set of aliases that we can resolve without encountering a chain of
386 // aliases (e.g. Foo -> Bar, Bar -> Baz). Such a chain would deadlock as the
387 // query would be waiting on a symbol that it itself had to resolve. Creating
388 // a new query for each link in such a chain eliminates the possibility of
389 // deadlock. In practice chains are likely to be rare, and this algorithm will
390 // usually result in a single query to issue.
391
392 std::vector<std::pair<SymbolLookupSet, std::shared_ptr<OnResolveInfo>>>
393 QueryInfos;
394 while (!RequestedAliases.empty()) {
395 SymbolNameSet ResponsibilitySymbols;
396 SymbolLookupSet QuerySymbols;
397 SymbolAliasMap QueryAliases;
398
399 // Collect as many aliases as we can without including a chain.
400 for (auto &KV : RequestedAliases) {
401 // Chain detected. Skip this symbol for this round.
402 if (&SrcJD == &TgtJD && (QueryAliases.count(KV.second.Aliasee) ||
403 RequestedAliases.count(KV.second.Aliasee)))
404 continue;
405
406 ResponsibilitySymbols.insert(KV.first);
407 QuerySymbols.add(KV.second.Aliasee,
408 KV.second.AliasFlags.hasMaterializationSideEffectsOnly()
411 QueryAliases[KV.first] = std::move(KV.second);
412 }
413
414 // Remove the aliases collected this round from the RequestedAliases map.
415 for (auto &KV : QueryAliases)
416 RequestedAliases.erase(KV.first);
417
418 assert(!QuerySymbols.empty() && "Alias cycle detected!");
419
420 auto NewR = R->delegate(ResponsibilitySymbols);
421 if (!NewR) {
422 ES.reportError(NewR.takeError());
423 R->failMaterialization();
424 return;
425 }
426
427 auto QueryInfo = std::make_shared<OnResolveInfo>(std::move(*NewR),
428 std::move(QueryAliases));
429 QueryInfos.push_back(
430 make_pair(std::move(QuerySymbols), std::move(QueryInfo)));
431 }
432
433 // Issue the queries.
434 while (!QueryInfos.empty()) {
435 auto QuerySymbols = std::move(QueryInfos.back().first);
436 auto QueryInfo = std::move(QueryInfos.back().second);
437
438 QueryInfos.pop_back();
439
440 auto RegisterDependencies = [QueryInfo,
441 &SrcJD](const SymbolDependenceMap &Deps) {
442 // If there were no materializing symbols, just bail out.
443 if (Deps.empty())
444 return;
445
446 // Otherwise the only deps should be on SrcJD.
447 assert(Deps.size() == 1 && Deps.count(&SrcJD) &&
448 "Unexpected dependencies for reexports");
449
450 auto &SrcJDDeps = Deps.find(&SrcJD)->second;
451
452 for (auto &[Alias, AliasInfo] : QueryInfo->Aliases)
453 if (SrcJDDeps.count(AliasInfo.Aliasee))
454 QueryInfo->SDGs.push_back({{Alias}, {{&SrcJD, {AliasInfo.Aliasee}}}});
455 };
456
457 auto OnComplete = [QueryInfo](Expected<SymbolMap> Result) {
458 auto &ES = QueryInfo->R->getTargetJITDylib().getExecutionSession();
459 if (Result) {
460 SymbolMap ResolutionMap;
461 for (auto &KV : QueryInfo->Aliases) {
462 assert((KV.second.AliasFlags.hasMaterializationSideEffectsOnly() ||
463 Result->count(KV.second.Aliasee)) &&
464 "Result map missing entry?");
465 // Don't try to resolve materialization-side-effects-only symbols.
466 if (KV.second.AliasFlags.hasMaterializationSideEffectsOnly())
467 continue;
468
469 ResolutionMap[KV.first] = {(*Result)[KV.second.Aliasee].getAddress(),
470 KV.second.AliasFlags};
471 }
472 if (auto Err = QueryInfo->R->notifyResolved(ResolutionMap)) {
473 ES.reportError(std::move(Err));
474 QueryInfo->R->failMaterialization();
475 return;
476 }
477 if (auto Err = QueryInfo->R->notifyEmitted(QueryInfo->SDGs)) {
478 ES.reportError(std::move(Err));
479 QueryInfo->R->failMaterialization();
480 return;
481 }
482 } else {
483 ES.reportError(Result.takeError());
484 QueryInfo->R->failMaterialization();
485 }
486 };
487
489 JITDylibSearchOrder({{&SrcJD, SourceJDLookupFlags}}),
490 QuerySymbols, SymbolState::Resolved, std::move(OnComplete),
491 std::move(RegisterDependencies));
492 }
493}
494
495void ReExportsMaterializationUnit::discard(const JITDylib &JD,
496 const SymbolStringPtr &Name) {
497 assert(Aliases.count(Name) &&
498 "Symbol not covered by this MaterializationUnit");
499 Aliases.erase(Name);
500}
501
502MaterializationUnit::Interface
503ReExportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) {
505 for (auto &KV : Aliases)
506 SymbolFlags[KV.first] = KV.second.AliasFlags;
507
508 return MaterializationUnit::Interface(std::move(SymbolFlags), nullptr);
509}
510
512 SymbolNameSet Symbols) {
513 SymbolLookupSet LookupSet(Symbols);
514 auto Flags = SourceJD.getExecutionSession().lookupFlags(
516 SymbolLookupSet(std::move(Symbols)));
517
518 if (!Flags)
519 return Flags.takeError();
520
522 for (auto &Name : Symbols) {
523 assert(Flags->count(Name) && "Missing entry in flags map");
524 Result[Name] = SymbolAliasMapEntry(Name, (*Flags)[Name]);
525 }
526
527 return Result;
528}
529
531public:
532 // FIXME: Reduce the number of SymbolStringPtrs here. See
533 // https://github.com/llvm/llvm-project/issues/55576.
534
540 }
541 virtual ~InProgressLookupState() = default;
542 virtual void complete(std::unique_ptr<InProgressLookupState> IPLS) = 0;
543 virtual void fail(Error Err) = 0;
544
549
551 bool NewJITDylib = true;
554
555 enum {
556 NotInGenerator, // Not currently using a generator.
557 ResumedForGenerator, // Resumed after being auto-suspended before generator.
558 InGenerator // Currently using generator.
560 std::vector<std::weak_ptr<DefinitionGenerator>> CurDefGeneratorStack;
561};
562
564public:
570 OnComplete(std::move(OnComplete)) {}
571
572 void complete(std::unique_ptr<InProgressLookupState> IPLS) override {
573 auto &ES = SearchOrder.front().first->getExecutionSession();
574 ES.OL_completeLookupFlags(std::move(IPLS), std::move(OnComplete));
575 }
576
577 void fail(Error Err) override { OnComplete(std::move(Err)); }
578
579private:
581};
582
584public:
588 std::shared_ptr<AsynchronousSymbolQuery> Q,
589 RegisterDependenciesFunction RegisterDependencies)
592 Q(std::move(Q)), RegisterDependencies(std::move(RegisterDependencies)) {
593 }
594
595 void complete(std::unique_ptr<InProgressLookupState> IPLS) override {
596 auto &ES = SearchOrder.front().first->getExecutionSession();
597 ES.OL_completeLookup(std::move(IPLS), std::move(Q),
598 std::move(RegisterDependencies));
599 }
600
601 void fail(Error Err) override {
602 Q->detach();
603 Q->handleFailed(std::move(Err));
604 }
605
606private:
607 std::shared_ptr<AsynchronousSymbolQuery> Q;
608 RegisterDependenciesFunction RegisterDependencies;
609};
610
612 JITDylibLookupFlags SourceJDLookupFlags,
613 SymbolPredicate Allow)
614 : SourceJD(SourceJD), SourceJDLookupFlags(SourceJDLookupFlags),
615 Allow(std::move(Allow)) {}
616
618 JITDylib &JD,
619 JITDylibLookupFlags JDLookupFlags,
620 const SymbolLookupSet &LookupSet) {
621 assert(&JD != &SourceJD && "Cannot re-export from the same dylib");
622
623 // Use lookupFlags to find the subset of symbols that match our lookup.
624 auto Flags = JD.getExecutionSession().lookupFlags(
625 K, {{&SourceJD, JDLookupFlags}}, LookupSet);
626 if (!Flags)
627 return Flags.takeError();
628
629 // Create an alias map.
630 orc::SymbolAliasMap AliasMap;
631 for (auto &KV : *Flags)
632 if (!Allow || Allow(KV.first))
633 AliasMap[KV.first] = SymbolAliasMapEntry(KV.first, KV.second);
634
635 if (AliasMap.empty())
636 return Error::success();
637
638 // Define the re-exports.
639 return JD.define(reexports(SourceJD, AliasMap, SourceJDLookupFlags));
640}
641
642LookupState::LookupState(std::unique_ptr<InProgressLookupState> IPLS)
643 : IPLS(std::move(IPLS)) {}
644
645void LookupState::reset(InProgressLookupState *IPLS) { this->IPLS.reset(IPLS); }
646
647LookupState::LookupState() = default;
648LookupState::LookupState(LookupState &&) = default;
649LookupState &LookupState::operator=(LookupState &&) = default;
650LookupState::~LookupState() = default;
651
653 assert(IPLS && "Cannot call continueLookup on empty LookupState");
654 auto &ES = IPLS->SearchOrder.begin()->first->getExecutionSession();
655 ES.OL_applyQueryPhase1(std::move(IPLS), std::move(Err));
656}
657
659 std::deque<LookupState> LookupsToFail;
660 {
661 std::lock_guard<std::mutex> Lock(M);
662 std::swap(PendingLookups, LookupsToFail);
663 InUse = false;
664 }
665
666 for (auto &LS : LookupsToFail)
667 LS.continueLookup(make_error<StringError>(
668 "Query waiting on DefinitionGenerator that was destroyed",
670}
671
673 LLVM_DEBUG(dbgs() << "Destroying JITDylib " << getName() << "\n");
674}
675
677 std::vector<ResourceTrackerSP> TrackersToRemove;
678 ES.runSessionLocked([&]() {
679 assert(State != Closed && "JD is defunct");
680 for (auto &KV : TrackerSymbols)
681 TrackersToRemove.push_back(KV.first);
682 TrackersToRemove.push_back(getDefaultResourceTracker());
683 });
684
685 Error Err = Error::success();
686 for (auto &RT : TrackersToRemove)
687 Err = joinErrors(std::move(Err), RT->remove());
688 return Err;
689}
690
692 return ES.runSessionLocked([this] {
693 assert(State != Closed && "JD is defunct");
694 if (!DefaultTracker)
695 DefaultTracker = new ResourceTracker(this);
696 return DefaultTracker;
697 });
698}
699
701 return ES.runSessionLocked([this] {
702 assert(State == Open && "JD is defunct");
703 ResourceTrackerSP RT = new ResourceTracker(this);
704 return RT;
705 });
706}
707
709 // DefGenerator moved into TmpDG to ensure that it's destroyed outside the
710 // session lock (since it may have to send errors to pending queries).
711 std::shared_ptr<DefinitionGenerator> TmpDG;
712
713 ES.runSessionLocked([&] {
714 assert(State == Open && "JD is defunct");
715 auto I = llvm::find_if(DefGenerators,
716 [&](const std::shared_ptr<DefinitionGenerator> &H) {
717 return H.get() == &G;
718 });
719 assert(I != DefGenerators.end() && "Generator not found");
720 TmpDG = std::move(*I);
721 DefGenerators.erase(I);
722 });
723}
724
726JITDylib::defineMaterializing(MaterializationResponsibility &FromMR,
727 SymbolFlagsMap SymbolFlags) {
728
729 return ES.runSessionLocked([&]() -> Expected<SymbolFlagsMap> {
730 if (FromMR.RT->isDefunct())
731 return make_error<ResourceTrackerDefunct>(FromMR.RT);
732
733 std::vector<NonOwningSymbolStringPtr> AddedSyms;
734 std::vector<NonOwningSymbolStringPtr> RejectedWeakDefs;
735
736 for (auto SFItr = SymbolFlags.begin(), SFEnd = SymbolFlags.end();
737 SFItr != SFEnd; ++SFItr) {
738
739 auto &Name = SFItr->first;
740 auto &Flags = SFItr->second;
741
742 auto EntryItr = Symbols.find(Name);
743
744 // If the entry already exists...
745 if (EntryItr != Symbols.end()) {
746
747 // If this is a strong definition then error out.
748 if (!Flags.isWeak()) {
749 // Remove any symbols already added.
750 for (auto &S : AddedSyms)
751 Symbols.erase(Symbols.find_as(S));
752
753 // FIXME: Return all duplicates.
754 return make_error<DuplicateDefinition>(std::string(*Name));
755 }
756
757 // Otherwise just make a note to discard this symbol after the loop.
758 RejectedWeakDefs.push_back(NonOwningSymbolStringPtr(Name));
759 continue;
760 } else
761 EntryItr =
762 Symbols.insert(std::make_pair(Name, SymbolTableEntry(Flags))).first;
763
764 AddedSyms.push_back(NonOwningSymbolStringPtr(Name));
765 EntryItr->second.setState(SymbolState::Materializing);
766 }
767
768 // Remove any rejected weak definitions from the SymbolFlags map.
769 while (!RejectedWeakDefs.empty()) {
770 SymbolFlags.erase(SymbolFlags.find_as(RejectedWeakDefs.back()));
771 RejectedWeakDefs.pop_back();
772 }
773
774 return SymbolFlags;
775 });
776}
777
778Error JITDylib::replace(MaterializationResponsibility &FromMR,
779 std::unique_ptr<MaterializationUnit> MU) {
780 assert(MU != nullptr && "Can not replace with a null MaterializationUnit");
781 std::unique_ptr<MaterializationUnit> MustRunMU;
782 std::unique_ptr<MaterializationResponsibility> MustRunMR;
783
784 auto Err =
785 ES.runSessionLocked([&, this]() -> Error {
786 if (FromMR.RT->isDefunct())
787 return make_error<ResourceTrackerDefunct>(std::move(FromMR.RT));
788
789#ifndef NDEBUG
790 for (auto &KV : MU->getSymbols()) {
791 auto SymI = Symbols.find(KV.first);
792 assert(SymI != Symbols.end() && "Replacing unknown symbol");
793 assert(SymI->second.getState() == SymbolState::Materializing &&
794 "Can not replace a symbol that ha is not materializing");
795 assert(!SymI->second.hasMaterializerAttached() &&
796 "Symbol should not have materializer attached already");
797 assert(UnmaterializedInfos.count(KV.first) == 0 &&
798 "Symbol being replaced should have no UnmaterializedInfo");
799 }
800#endif // NDEBUG
801
802 // If the tracker is defunct we need to bail out immediately.
803
804 // If any symbol has pending queries against it then we need to
805 // materialize MU immediately.
806 for (auto &KV : MU->getSymbols()) {
807 auto MII = MaterializingInfos.find(KV.first);
808 if (MII != MaterializingInfos.end()) {
809 if (MII->second.hasQueriesPending()) {
810 MustRunMR = ES.createMaterializationResponsibility(
811 *FromMR.RT, std::move(MU->SymbolFlags),
812 std::move(MU->InitSymbol));
813 MustRunMU = std::move(MU);
814 return Error::success();
815 }
816 }
817 }
818
819 // Otherwise, make MU responsible for all the symbols.
820 auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU),
821 FromMR.RT.get());
822 for (auto &KV : UMI->MU->getSymbols()) {
823 auto SymI = Symbols.find(KV.first);
824 assert(SymI->second.getState() == SymbolState::Materializing &&
825 "Can not replace a symbol that is not materializing");
826 assert(!SymI->second.hasMaterializerAttached() &&
827 "Can not replace a symbol that has a materializer attached");
828 assert(UnmaterializedInfos.count(KV.first) == 0 &&
829 "Unexpected materializer entry in map");
830 SymI->second.setAddress(SymI->second.getAddress());
831 SymI->second.setMaterializerAttached(true);
832
833 auto &UMIEntry = UnmaterializedInfos[KV.first];
834 assert((!UMIEntry || !UMIEntry->MU) &&
835 "Replacing symbol with materializer still attached");
836 UMIEntry = UMI;
837 }
838
839 return Error::success();
840 });
841
842 if (Err)
843 return Err;
844
845 if (MustRunMU) {
846 assert(MustRunMR && "MustRunMU set implies MustRunMR set");
847 ES.dispatchTask(std::make_unique<MaterializationTask>(
848 std::move(MustRunMU), std::move(MustRunMR)));
849 } else {
850 assert(!MustRunMR && "MustRunMU unset implies MustRunMR unset");
851 }
852
853 return Error::success();
854}
855
856Expected<std::unique_ptr<MaterializationResponsibility>>
857JITDylib::delegate(MaterializationResponsibility &FromMR,
858 SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol) {
859
860 return ES.runSessionLocked(
861 [&]() -> Expected<std::unique_ptr<MaterializationResponsibility>> {
862 if (FromMR.RT->isDefunct())
863 return make_error<ResourceTrackerDefunct>(std::move(FromMR.RT));
864
865 return ES.createMaterializationResponsibility(
866 *FromMR.RT, std::move(SymbolFlags), std::move(InitSymbol));
867 });
868}
869
871JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const {
872 return ES.runSessionLocked([&]() {
873 SymbolNameSet RequestedSymbols;
874
875 for (auto &KV : SymbolFlags) {
876 assert(Symbols.count(KV.first) && "JITDylib does not cover this symbol?");
877 assert(Symbols.find(KV.first)->second.getState() !=
878 SymbolState::NeverSearched &&
879 Symbols.find(KV.first)->second.getState() != SymbolState::Ready &&
880 "getRequestedSymbols can only be called for symbols that have "
881 "started materializing");
882 auto I = MaterializingInfos.find(KV.first);
883 if (I == MaterializingInfos.end())
884 continue;
885
886 if (I->second.hasQueriesPending())
887 RequestedSymbols.insert(KV.first);
888 }
889
890 return RequestedSymbols;
891 });
892}
893
894Error JITDylib::resolve(MaterializationResponsibility &MR,
895 const SymbolMap &Resolved) {
896 AsynchronousSymbolQuerySet CompletedQueries;
897
898 if (auto Err = ES.runSessionLocked([&, this]() -> Error {
899 if (MR.RT->isDefunct())
900 return make_error<ResourceTrackerDefunct>(MR.RT);
901
902 if (State != Open)
903 return make_error<StringError>("JITDylib " + getName() +
904 " is defunct",
905 inconvertibleErrorCode());
906
907 struct WorklistEntry {
908 SymbolTable::iterator SymI;
909 ExecutorSymbolDef ResolvedSym;
910 };
911
912 SymbolNameSet SymbolsInErrorState;
913 std::vector<WorklistEntry> Worklist;
914 Worklist.reserve(Resolved.size());
915
916 // Build worklist and check for any symbols in the error state.
917 for (const auto &KV : Resolved) {
918
919 assert(!KV.second.getFlags().hasError() &&
920 "Resolution result can not have error flag set");
921
922 auto SymI = Symbols.find(KV.first);
923
924 assert(SymI != Symbols.end() && "Symbol not found");
925 assert(!SymI->second.hasMaterializerAttached() &&
926 "Resolving symbol with materializer attached?");
927 assert(SymI->second.getState() == SymbolState::Materializing &&
928 "Symbol should be materializing");
929 assert(SymI->second.getAddress() == ExecutorAddr() &&
930 "Symbol has already been resolved");
931
932 if (SymI->second.getFlags().hasError())
933 SymbolsInErrorState.insert(KV.first);
934 else {
935 auto Flags = KV.second.getFlags();
936 Flags &= ~JITSymbolFlags::Common;
937 assert(Flags ==
938 (SymI->second.getFlags() & ~JITSymbolFlags::Common) &&
939 "Resolved flags should match the declared flags");
940
941 Worklist.push_back({SymI, {KV.second.getAddress(), Flags}});
942 }
943 }
944
945 // If any symbols were in the error state then bail out.
946 if (!SymbolsInErrorState.empty()) {
947 auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>();
948 (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState);
949 return make_error<FailedToMaterialize>(
950 getExecutionSession().getSymbolStringPool(),
951 std::move(FailedSymbolsDepMap));
952 }
953
954 while (!Worklist.empty()) {
955 auto SymI = Worklist.back().SymI;
956 auto ResolvedSym = Worklist.back().ResolvedSym;
957 Worklist.pop_back();
958
959 auto &Name = SymI->first;
960
961 // Resolved symbols can not be weak: discard the weak flag.
962 JITSymbolFlags ResolvedFlags = ResolvedSym.getFlags();
963 SymI->second.setAddress(ResolvedSym.getAddress());
964 SymI->second.setFlags(ResolvedFlags);
965 SymI->second.setState(SymbolState::Resolved);
966
967 auto MII = MaterializingInfos.find(Name);
968 if (MII == MaterializingInfos.end())
969 continue;
970
971 auto &MI = MII->second;
972 for (auto &Q : MI.takeQueriesMeeting(SymbolState::Resolved)) {
973 Q->notifySymbolMetRequiredState(Name, ResolvedSym);
974 Q->removeQueryDependence(*this, Name);
975 if (Q->isComplete())
976 CompletedQueries.insert(std::move(Q));
977 }
978 }
979
980 return Error::success();
981 }))
982 return Err;
983
984 // Otherwise notify all the completed queries.
985 for (auto &Q : CompletedQueries) {
986 assert(Q->isComplete() && "Q not completed");
987 Q->handleComplete(ES);
988 }
989
990 return Error::success();
991}
992
993void JITDylib::unlinkMaterializationResponsibility(
994 MaterializationResponsibility &MR) {
995 ES.runSessionLocked([&]() {
996 auto I = TrackerMRs.find(MR.RT.get());
997 assert(I != TrackerMRs.end() && "No MRs in TrackerMRs list for RT");
998 assert(I->second.count(&MR) && "MR not in TrackerMRs list for RT");
999 I->second.erase(&MR);
1000 if (I->second.empty())
1001 TrackerMRs.erase(MR.RT.get());
1002 });
1003}
1004
1005void JITDylib::shrinkMaterializationInfoMemory() {
1006 // DenseMap::erase never shrinks its storage; use clear to heuristically free
1007 // memory since we may have long-lived JDs after linking is done.
1008
1009 if (UnmaterializedInfos.empty())
1010 UnmaterializedInfos.clear();
1011
1012 if (MaterializingInfos.empty())
1013 MaterializingInfos.clear();
1014}
1015
1016void JITDylib::setLinkOrder(JITDylibSearchOrder NewLinkOrder,
1017 bool LinkAgainstThisJITDylibFirst) {
1018 ES.runSessionLocked([&]() {
1019 assert(State == Open && "JD is defunct");
1020 if (LinkAgainstThisJITDylibFirst) {
1021 LinkOrder.clear();
1022 if (NewLinkOrder.empty() || NewLinkOrder.front().first != this)
1023 LinkOrder.push_back(
1024 std::make_pair(this, JITDylibLookupFlags::MatchAllSymbols));
1025 llvm::append_range(LinkOrder, NewLinkOrder);
1026 } else
1027 LinkOrder = std::move(NewLinkOrder);
1028 });
1029}
1030
1031void JITDylib::addToLinkOrder(const JITDylibSearchOrder &NewLinks) {
1032 ES.runSessionLocked([&]() {
1033 for (auto &KV : NewLinks) {
1034 // Skip elements of NewLinks that are already in the link order.
1035 if (llvm::is_contained(LinkOrder, KV))
1036 continue;
1037
1038 LinkOrder.push_back(std::move(KV));
1039 }
1040 });
1041}
1042
1043void JITDylib::addToLinkOrder(JITDylib &JD, JITDylibLookupFlags JDLookupFlags) {
1044 ES.runSessionLocked([&]() { LinkOrder.push_back({&JD, JDLookupFlags}); });
1045}
1046
1047void JITDylib::replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD,
1048 JITDylibLookupFlags JDLookupFlags) {
1049 ES.runSessionLocked([&]() {
1050 assert(State == Open && "JD is defunct");
1051 for (auto &KV : LinkOrder)
1052 if (KV.first == &OldJD) {
1053 KV = {&NewJD, JDLookupFlags};
1054 break;
1055 }
1056 });
1057}
1058
1059void JITDylib::removeFromLinkOrder(JITDylib &JD) {
1060 ES.runSessionLocked([&]() {
1061 assert(State == Open && "JD is defunct");
1062 auto I = llvm::find_if(LinkOrder,
1063 [&](const JITDylibSearchOrder::value_type &KV) {
1064 return KV.first == &JD;
1065 });
1066 if (I != LinkOrder.end())
1067 LinkOrder.erase(I);
1068 });
1069}
1070
1071Error JITDylib::remove(const SymbolNameSet &Names) {
1072 return ES.runSessionLocked([&]() -> Error {
1073 assert(State == Open && "JD is defunct");
1074 using SymbolMaterializerItrPair =
1075 std::pair<SymbolTable::iterator, UnmaterializedInfosMap::iterator>;
1076 std::vector<SymbolMaterializerItrPair> SymbolsToRemove;
1077 SymbolNameSet Missing;
1079
1080 for (auto &Name : Names) {
1081 auto I = Symbols.find(Name);
1082
1083 // Note symbol missing.
1084 if (I == Symbols.end()) {
1085 Missing.insert(Name);
1086 continue;
1087 }
1088
1089 // Note symbol materializing.
1090 if (I->second.getState() != SymbolState::NeverSearched &&
1091 I->second.getState() != SymbolState::Ready) {
1092 Materializing.insert(Name);
1093 continue;
1094 }
1095
1096 auto UMII = I->second.hasMaterializerAttached()
1097 ? UnmaterializedInfos.find(Name)
1098 : UnmaterializedInfos.end();
1099 SymbolsToRemove.push_back(std::make_pair(I, UMII));
1100 }
1101
1102 // If any of the symbols are not defined, return an error.
1103 if (!Missing.empty())
1104 return make_error<SymbolsNotFound>(ES.getSymbolStringPool(),
1105 std::move(Missing));
1106
1107 // If any of the symbols are currently materializing, return an error.
1108 if (!Materializing.empty())
1109 return make_error<SymbolsCouldNotBeRemoved>(ES.getSymbolStringPool(),
1110 std::move(Materializing));
1111
1112 // Remove the symbols.
1113 for (auto &SymbolMaterializerItrPair : SymbolsToRemove) {
1114 auto UMII = SymbolMaterializerItrPair.second;
1115
1116 // If there is a materializer attached, call discard.
1117 if (UMII != UnmaterializedInfos.end()) {
1118 UMII->second->MU->doDiscard(*this, UMII->first);
1119 UnmaterializedInfos.erase(UMII);
1120 }
1121
1122 auto SymI = SymbolMaterializerItrPair.first;
1123 Symbols.erase(SymI);
1124 }
1125
1126 shrinkMaterializationInfoMemory();
1127
1128 return Error::success();
1129 });
1130}
1131
1132void JITDylib::dump(raw_ostream &OS) {
1133 ES.runSessionLocked([&, this]() {
1134 OS << "JITDylib \"" << getName() << "\" (ES: "
1135 << format("0x%016" PRIx64, reinterpret_cast<uintptr_t>(&ES))
1136 << ", State = ";
1137 switch (State) {
1138 case Open:
1139 OS << "Open";
1140 break;
1141 case Closing:
1142 OS << "Closing";
1143 break;
1144 case Closed:
1145 OS << "Closed";
1146 break;
1147 }
1148 OS << ")\n";
1149 if (State == Closed)
1150 return;
1151 OS << "Link order: " << LinkOrder << "\n"
1152 << "Symbol table:\n";
1153
1154 // Sort symbols so we get a deterministic order and can check them in tests.
1155 std::vector<std::pair<SymbolStringPtr, SymbolTableEntry *>> SymbolsSorted;
1156 for (auto &KV : Symbols)
1157 SymbolsSorted.emplace_back(KV.first, &KV.second);
1158 llvm::sort(SymbolsSorted, [](const auto &L, const auto &R) {
1159 return *L.first < *R.first;
1160 });
1161
1162 for (auto &KV : SymbolsSorted) {
1163 OS << " \"" << *KV.first << "\": ";
1164 if (auto Addr = KV.second->getAddress())
1165 OS << Addr;
1166 else
1167 OS << "<not resolved> ";
1168
1169 OS << " " << KV.second->getFlags() << " " << KV.second->getState();
1170
1171 if (KV.second->hasMaterializerAttached()) {
1172 OS << " (Materializer ";
1173 auto I = UnmaterializedInfos.find(KV.first);
1174 assert(I != UnmaterializedInfos.end() &&
1175 "Lazy symbol should have UnmaterializedInfo");
1176 OS << I->second->MU.get() << ", " << I->second->MU->getName() << ")\n";
1177 } else
1178 OS << "\n";
1179 }
1180
1181 if (!MaterializingInfos.empty())
1182 OS << " MaterializingInfos entries:\n";
1183 for (auto &KV : MaterializingInfos) {
1184 OS << " \"" << *KV.first << "\":\n"
1185 << " " << KV.second.pendingQueries().size()
1186 << " pending queries: { ";
1187 for (const auto &Q : KV.second.pendingQueries())
1188 OS << Q.get() << " (" << Q->getRequiredState() << ") ";
1189 OS << "}\n Defining EDU: ";
1190 if (KV.second.DefiningEDU) {
1191 OS << KV.second.DefiningEDU.get() << " { ";
1192 for (auto &[Name, Flags] : KV.second.DefiningEDU->Symbols)
1193 OS << Name << " ";
1194 OS << "}\n";
1195 OS << " Dependencies:\n";
1196 if (!KV.second.DefiningEDU->Dependencies.empty()) {
1197 for (auto &[DepJD, Deps] : KV.second.DefiningEDU->Dependencies) {
1198 OS << " " << DepJD->getName() << ": [ ";
1199 for (auto &Dep : Deps)
1200 OS << Dep << " ";
1201 OS << "]\n";
1202 }
1203 } else
1204 OS << " none\n";
1205 } else
1206 OS << "none\n";
1207 OS << " Dependant EDUs:\n";
1208 if (!KV.second.DependantEDUs.empty()) {
1209 for (auto &DependantEDU : KV.second.DependantEDUs) {
1210 OS << " " << DependantEDU << ": "
1211 << DependantEDU->JD->getName() << " { ";
1212 for (auto &[Name, Flags] : DependantEDU->Symbols)
1213 OS << Name << " ";
1214 OS << "}\n";
1215 }
1216 } else
1217 OS << " none\n";
1218 assert((Symbols[KV.first].getState() != SymbolState::Ready ||
1219 (KV.second.pendingQueries().empty() && !KV.second.DefiningEDU &&
1220 !KV.second.DependantEDUs.empty())) &&
1221 "Stale materializing info entry");
1222 }
1223 });
1224}
1225
1226void JITDylib::MaterializingInfo::addQuery(
1227 std::shared_ptr<AsynchronousSymbolQuery> Q) {
1228
1229 auto I = llvm::lower_bound(
1230 llvm::reverse(PendingQueries), Q->getRequiredState(),
1231 [](const std::shared_ptr<AsynchronousSymbolQuery> &V, SymbolState S) {
1232 return V->getRequiredState() <= S;
1233 });
1234 PendingQueries.insert(I.base(), std::move(Q));
1235}
1236
1237void JITDylib::MaterializingInfo::removeQuery(
1238 const AsynchronousSymbolQuery &Q) {
1239 // FIXME: Implement 'find_as' for shared_ptr<T>/T*.
1240 auto I = llvm::find_if(
1241 PendingQueries, [&Q](const std::shared_ptr<AsynchronousSymbolQuery> &V) {
1242 return V.get() == &Q;
1243 });
1244 assert(I != PendingQueries.end() &&
1245 "Query is not attached to this MaterializingInfo");
1246 PendingQueries.erase(I);
1247}
1248
1249JITDylib::AsynchronousSymbolQueryList
1250JITDylib::MaterializingInfo::takeQueriesMeeting(SymbolState RequiredState) {
1251 AsynchronousSymbolQueryList Result;
1252 while (!PendingQueries.empty()) {
1253 if (PendingQueries.back()->getRequiredState() > RequiredState)
1254 break;
1255
1256 Result.push_back(std::move(PendingQueries.back()));
1257 PendingQueries.pop_back();
1258 }
1259
1260 return Result;
1261}
1262
1263JITDylib::JITDylib(ExecutionSession &ES, std::string Name)
1264 : JITLinkDylib(std::move(Name)), ES(ES) {
1265 LinkOrder.push_back({this, JITDylibLookupFlags::MatchAllSymbols});
1266}
1267
1268std::pair<JITDylib::AsynchronousSymbolQuerySet,
1269 std::shared_ptr<SymbolDependenceMap>>
1270JITDylib::removeTracker(ResourceTracker &RT) {
1271 // Note: Should be called under the session lock.
1272 assert(State != Closed && "JD is defunct");
1273
1274 SymbolNameVector SymbolsToRemove;
1275 SymbolNameVector SymbolsToFail;
1276
1277 if (&RT == DefaultTracker.get()) {
1278 SymbolNameSet TrackedSymbols;
1279 for (auto &KV : TrackerSymbols)
1280 for (auto &Sym : KV.second)
1281 TrackedSymbols.insert(Sym);
1282
1283 for (auto &KV : Symbols) {
1284 auto &Sym = KV.first;
1285 if (!TrackedSymbols.count(Sym))
1286 SymbolsToRemove.push_back(Sym);
1287 }
1288
1289 DefaultTracker.reset();
1290 } else {
1291 /// Check for a non-default tracker.
1292 auto I = TrackerSymbols.find(&RT);
1293 if (I != TrackerSymbols.end()) {
1294 SymbolsToRemove = std::move(I->second);
1295 TrackerSymbols.erase(I);
1296 }
1297 // ... if not found this tracker was already defunct. Nothing to do.
1298 }
1299
1300 for (auto &Sym : SymbolsToRemove) {
1301 assert(Symbols.count(Sym) && "Symbol not in symbol table");
1302
1303 // If there is a MaterializingInfo then collect any queries to fail.
1304 auto MII = MaterializingInfos.find(Sym);
1305 if (MII != MaterializingInfos.end())
1306 SymbolsToFail.push_back(Sym);
1307 }
1308
1309 AsynchronousSymbolQuerySet QueriesToFail;
1310 auto Result = ES.runSessionLocked(
1311 [&]() { return ES.IL_failSymbols(*this, std::move(SymbolsToFail)); });
1312
1313 // Removed symbols should be taken out of the table altogether.
1314 for (auto &Sym : SymbolsToRemove) {
1315 auto I = Symbols.find(Sym);
1316 assert(I != Symbols.end() && "Symbol not present in table");
1317
1318 // Remove Materializer if present.
1319 if (I->second.hasMaterializerAttached()) {
1320 // FIXME: Should this discard the symbols?
1321 UnmaterializedInfos.erase(Sym);
1322 } else {
1323 assert(!UnmaterializedInfos.count(Sym) &&
1324 "Symbol has materializer attached");
1325 }
1326
1327 Symbols.erase(I);
1328 }
1329
1330 shrinkMaterializationInfoMemory();
1331
1332 return Result;
1333}
1334
1335void JITDylib::transferTracker(ResourceTracker &DstRT, ResourceTracker &SrcRT) {
1336 assert(State != Closed && "JD is defunct");
1337 assert(&DstRT != &SrcRT && "No-op transfers shouldn't call transferTracker");
1338 assert(&DstRT.getJITDylib() == this && "DstRT is not for this JITDylib");
1339 assert(&SrcRT.getJITDylib() == this && "SrcRT is not for this JITDylib");
1340
1341 // Update trackers for any not-yet materialized units.
1342 for (auto &KV : UnmaterializedInfos) {
1343 if (KV.second->RT == &SrcRT)
1344 KV.second->RT = &DstRT;
1345 }
1346
1347 // Update trackers for any active materialization responsibilities.
1348 {
1349 auto I = TrackerMRs.find(&SrcRT);
1350 if (I != TrackerMRs.end()) {
1351 auto &SrcMRs = I->second;
1352 auto &DstMRs = TrackerMRs[&DstRT];
1353 for (auto *MR : SrcMRs)
1354 MR->RT = &DstRT;
1355 if (DstMRs.empty())
1356 DstMRs = std::move(SrcMRs);
1357 else
1358 for (auto *MR : SrcMRs)
1359 DstMRs.insert(MR);
1360 // Erase SrcRT entry in TrackerMRs. Use &SrcRT key rather than iterator I
1361 // for this, since I may have been invalidated by 'TrackerMRs[&DstRT]'.
1362 TrackerMRs.erase(&SrcRT);
1363 }
1364 }
1365
1366 // If we're transfering to the default tracker we just need to delete the
1367 // tracked symbols for the source tracker.
1368 if (&DstRT == DefaultTracker.get()) {
1369 TrackerSymbols.erase(&SrcRT);
1370 return;
1371 }
1372
1373 // If we're transferring from the default tracker we need to find all
1374 // currently untracked symbols.
1375 if (&SrcRT == DefaultTracker.get()) {
1376 assert(!TrackerSymbols.count(&SrcRT) &&
1377 "Default tracker should not appear in TrackerSymbols");
1378
1379 SymbolNameVector SymbolsToTrack;
1380
1381 SymbolNameSet CurrentlyTrackedSymbols;
1382 for (auto &KV : TrackerSymbols)
1383 for (auto &Sym : KV.second)
1384 CurrentlyTrackedSymbols.insert(Sym);
1385
1386 for (auto &KV : Symbols) {
1387 auto &Sym = KV.first;
1388 if (!CurrentlyTrackedSymbols.count(Sym))
1389 SymbolsToTrack.push_back(Sym);
1390 }
1391
1392 TrackerSymbols[&DstRT] = std::move(SymbolsToTrack);
1393 return;
1394 }
1395
1396 auto &DstTrackedSymbols = TrackerSymbols[&DstRT];
1397
1398 // Finally if neither SrtRT or DstRT are the default tracker then
1399 // just append DstRT's tracked symbols to SrtRT's.
1400 auto SI = TrackerSymbols.find(&SrcRT);
1401 if (SI == TrackerSymbols.end())
1402 return;
1403
1404 DstTrackedSymbols.reserve(DstTrackedSymbols.size() + SI->second.size());
1405 for (auto &Sym : SI->second)
1406 DstTrackedSymbols.push_back(std::move(Sym));
1407 TrackerSymbols.erase(SI);
1408}
1409
1410Error JITDylib::defineImpl(MaterializationUnit &MU) {
1411 LLVM_DEBUG({ dbgs() << " " << MU.getSymbols() << "\n"; });
1412
1413 SymbolNameSet Duplicates;
1414 std::vector<SymbolStringPtr> ExistingDefsOverridden;
1415 std::vector<SymbolStringPtr> MUDefsOverridden;
1416
1417 for (const auto &KV : MU.getSymbols()) {
1418 auto I = Symbols.find(KV.first);
1419
1420 if (I != Symbols.end()) {
1421 if (KV.second.isStrong()) {
1422 if (I->second.getFlags().isStrong() ||
1423 I->second.getState() > SymbolState::NeverSearched)
1424 Duplicates.insert(KV.first);
1425 else {
1426 assert(I->second.getState() == SymbolState::NeverSearched &&
1427 "Overridden existing def should be in the never-searched "
1428 "state");
1429 ExistingDefsOverridden.push_back(KV.first);
1430 }
1431 } else
1432 MUDefsOverridden.push_back(KV.first);
1433 }
1434 }
1435
1436 // If there were any duplicate definitions then bail out.
1437 if (!Duplicates.empty()) {
1438 LLVM_DEBUG(
1439 { dbgs() << " Error: Duplicate symbols " << Duplicates << "\n"; });
1440 return make_error<DuplicateDefinition>(std::string(**Duplicates.begin()));
1441 }
1442
1443 // Discard any overridden defs in this MU.
1444 LLVM_DEBUG({
1445 if (!MUDefsOverridden.empty())
1446 dbgs() << " Defs in this MU overridden: " << MUDefsOverridden << "\n";
1447 });
1448 for (auto &S : MUDefsOverridden)
1449 MU.doDiscard(*this, S);
1450
1451 // Discard existing overridden defs.
1452 LLVM_DEBUG({
1453 if (!ExistingDefsOverridden.empty())
1454 dbgs() << " Existing defs overridden by this MU: " << MUDefsOverridden
1455 << "\n";
1456 });
1457 for (auto &S : ExistingDefsOverridden) {
1458
1459 auto UMII = UnmaterializedInfos.find(S);
1460 assert(UMII != UnmaterializedInfos.end() &&
1461 "Overridden existing def should have an UnmaterializedInfo");
1462 UMII->second->MU->doDiscard(*this, S);
1463 }
1464
1465 // Finally, add the defs from this MU.
1466 for (auto &KV : MU.getSymbols()) {
1467 auto &SymEntry = Symbols[KV.first];
1468 SymEntry.setFlags(KV.second);
1469 SymEntry.setState(SymbolState::NeverSearched);
1470 SymEntry.setMaterializerAttached(true);
1471 }
1472
1473 return Error::success();
1474}
1475
1476void JITDylib::installMaterializationUnit(
1477 std::unique_ptr<MaterializationUnit> MU, ResourceTracker &RT) {
1478
1479 /// defineImpl succeeded.
1480 if (&RT != DefaultTracker.get()) {
1481 auto &TS = TrackerSymbols[&RT];
1482 TS.reserve(TS.size() + MU->getSymbols().size());
1483 for (auto &KV : MU->getSymbols())
1484 TS.push_back(KV.first);
1485 }
1486
1487 auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU), &RT);
1488 for (auto &KV : UMI->MU->getSymbols())
1489 UnmaterializedInfos[KV.first] = UMI;
1490}
1491
1492void JITDylib::detachQueryHelper(AsynchronousSymbolQuery &Q,
1493 const SymbolNameSet &QuerySymbols) {
1494 for (auto &QuerySymbol : QuerySymbols) {
1495 assert(MaterializingInfos.count(QuerySymbol) &&
1496 "QuerySymbol does not have MaterializingInfo");
1497 auto &MI = MaterializingInfos[QuerySymbol];
1498 MI.removeQuery(Q);
1499 }
1500}
1501
1502Platform::~Platform() = default;
1503
1505 ExecutionSession &ES,
1506 const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {
1507
1508 DenseMap<JITDylib *, SymbolMap> CompoundResult;
1509 Error CompoundErr = Error::success();
1510 std::mutex LookupMutex;
1511 std::condition_variable CV;
1512 uint64_t Count = InitSyms.size();
1513
1514 LLVM_DEBUG({
1515 dbgs() << "Issuing init-symbol lookup:\n";
1516 for (auto &KV : InitSyms)
1517 dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
1518 });
1519
1520 for (auto &KV : InitSyms) {
1521 auto *JD = KV.first;
1522 auto Names = std::move(KV.second);
1523 ES.lookup(
1526 std::move(Names), SymbolState::Ready,
1527 [&, JD](Expected<SymbolMap> Result) {
1528 {
1529 std::lock_guard<std::mutex> Lock(LookupMutex);
1530 --Count;
1531 if (Result) {
1532 assert(!CompoundResult.count(JD) &&
1533 "Duplicate JITDylib in lookup?");
1534 CompoundResult[JD] = std::move(*Result);
1535 } else
1536 CompoundErr =
1537 joinErrors(std::move(CompoundErr), Result.takeError());
1538 }
1539 CV.notify_one();
1540 },
1542 }
1543
1544 std::unique_lock<std::mutex> Lock(LookupMutex);
1545 CV.wait(Lock, [&] { return Count == 0 || CompoundErr; });
1546
1547 if (CompoundErr)
1548 return std::move(CompoundErr);
1549
1550 return std::move(CompoundResult);
1551}
1552
1554 unique_function<void(Error)> OnComplete, ExecutionSession &ES,
1555 const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {
1556
1557 class TriggerOnComplete {
1558 public:
1559 using OnCompleteFn = unique_function<void(Error)>;
1560 TriggerOnComplete(OnCompleteFn OnComplete)
1561 : OnComplete(std::move(OnComplete)) {}
1562 ~TriggerOnComplete() { OnComplete(std::move(LookupResult)); }
1563 void reportResult(Error Err) {
1564 std::lock_guard<std::mutex> Lock(ResultMutex);
1565 LookupResult = joinErrors(std::move(LookupResult), std::move(Err));
1566 }
1567
1568 private:
1569 std::mutex ResultMutex;
1570 Error LookupResult{Error::success()};
1571 OnCompleteFn OnComplete;
1572 };
1573
1574 LLVM_DEBUG({
1575 dbgs() << "Issuing init-symbol lookup:\n";
1576 for (auto &KV : InitSyms)
1577 dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
1578 });
1579
1580 auto TOC = std::make_shared<TriggerOnComplete>(std::move(OnComplete));
1581
1582 for (auto &KV : InitSyms) {
1583 auto *JD = KV.first;
1584 auto Names = std::move(KV.second);
1585 ES.lookup(
1588 std::move(Names), SymbolState::Ready,
1590 TOC->reportResult(Result.takeError());
1591 },
1593 }
1594}
1595
1597 OS << "Materialization task: " << MU->getName() << " in "
1598 << MR->getTargetJITDylib().getName();
1599}
1600
1601void MaterializationTask::run() { MU->materialize(std::move(MR)); }
1602
1603void LookupTask::printDescription(raw_ostream &OS) { OS << "Lookup task"; }
1604
1606
1607ExecutionSession::ExecutionSession(std::unique_ptr<ExecutorProcessControl> EPC)
1608 : EPC(std::move(EPC)) {
1609 // Associated EPC and this.
1610 this->EPC->ES = this;
1611}
1612
1614 // You must call endSession prior to destroying the session.
1615 assert(!SessionOpen &&
1616 "Session still open. Did you forget to call endSession?");
1617}
1618
1620 LLVM_DEBUG(dbgs() << "Ending ExecutionSession " << this << "\n");
1621
1622 auto JDsToRemove = runSessionLocked([&] {
1623
1624#ifdef EXPENSIVE_CHECKS
1625 verifySessionState("Entering ExecutionSession::endSession");
1626#endif
1627
1628 SessionOpen = false;
1629 return JDs;
1630 });
1631
1632 std::reverse(JDsToRemove.begin(), JDsToRemove.end());
1633
1634 auto Err = removeJITDylibs(std::move(JDsToRemove));
1635
1636 Err = joinErrors(std::move(Err), EPC->disconnect());
1637
1638 return Err;
1639}
1640
1642 runSessionLocked([&] { ResourceManagers.push_back(&RM); });
1643}
1644
1646 runSessionLocked([&] {
1647 assert(!ResourceManagers.empty() && "No managers registered");
1648 if (ResourceManagers.back() == &RM)
1649 ResourceManagers.pop_back();
1650 else {
1651 auto I = llvm::find(ResourceManagers, &RM);
1652 assert(I != ResourceManagers.end() && "RM not registered");
1653 ResourceManagers.erase(I);
1654 }
1655 });
1656}
1657
1659 return runSessionLocked([&, this]() -> JITDylib * {
1660 for (auto &JD : JDs)
1661 if (JD->getName() == Name)
1662 return JD.get();
1663 return nullptr;
1664 });
1665}
1666
1668 assert(!getJITDylibByName(Name) && "JITDylib with that name already exists");
1669 return runSessionLocked([&, this]() -> JITDylib & {
1670 assert(SessionOpen && "Cannot create JITDylib after session is closed");
1671 JDs.push_back(new JITDylib(*this, std::move(Name)));
1672 return *JDs.back();
1673 });
1674}
1675
1677 auto &JD = createBareJITDylib(Name);
1678 if (P)
1679 if (auto Err = P->setupJITDylib(JD))
1680 return std::move(Err);
1681 return JD;
1682}
1683
1684Error ExecutionSession::removeJITDylibs(std::vector<JITDylibSP> JDsToRemove) {
1685 // Set JD to 'Closing' state and remove JD from the ExecutionSession.
1686 runSessionLocked([&] {
1687 for (auto &JD : JDsToRemove) {
1688 assert(JD->State == JITDylib::Open && "JD already closed");
1689 JD->State = JITDylib::Closing;
1690 auto I = llvm::find(JDs, JD);
1691 assert(I != JDs.end() && "JD does not appear in session JDs");
1692 JDs.erase(I);
1693 }
1694 });
1695
1696 // Clear JITDylibs and notify the platform.
1697 Error Err = Error::success();
1698 for (auto JD : JDsToRemove) {
1699 Err = joinErrors(std::move(Err), JD->clear());
1700 if (P)
1701 Err = joinErrors(std::move(Err), P->teardownJITDylib(*JD));
1702 }
1703
1704 // Set JD to closed state. Clear remaining data structures.
1705 runSessionLocked([&] {
1706 for (auto &JD : JDsToRemove) {
1707 assert(JD->State == JITDylib::Closing && "JD should be closing");
1708 JD->State = JITDylib::Closed;
1709 assert(JD->Symbols.empty() && "JD.Symbols is not empty after clear");
1710 assert(JD->UnmaterializedInfos.empty() &&
1711 "JD.UnmaterializedInfos is not empty after clear");
1712 assert(JD->MaterializingInfos.empty() &&
1713 "JD.MaterializingInfos is not empty after clear");
1714 assert(JD->TrackerSymbols.empty() &&
1715 "TrackerSymbols is not empty after clear");
1716 JD->DefGenerators.clear();
1717 JD->LinkOrder.clear();
1718 }
1719 });
1720
1721 return Err;
1722}
1723
1726 if (JDs.empty())
1727 return std::vector<JITDylibSP>();
1728
1729 auto &ES = JDs.front()->getExecutionSession();
1730 return ES.runSessionLocked([&]() -> Expected<std::vector<JITDylibSP>> {
1731 DenseSet<JITDylib *> Visited;
1732 std::vector<JITDylibSP> Result;
1733
1734 for (auto &JD : JDs) {
1735
1736 if (JD->State != Open)
1737 return make_error<StringError>(
1738 "Error building link order: " + JD->getName() + " is defunct",
1740 if (Visited.count(JD.get()))
1741 continue;
1742
1744 WorkStack.push_back(JD);
1745 Visited.insert(JD.get());
1746
1747 while (!WorkStack.empty()) {
1748 Result.push_back(std::move(WorkStack.back()));
1749 WorkStack.pop_back();
1750
1751 for (auto &KV : llvm::reverse(Result.back()->LinkOrder)) {
1752 auto &JD = *KV.first;
1753 if (!Visited.insert(&JD).second)
1754 continue;
1755 WorkStack.push_back(&JD);
1756 }
1757 }
1758 }
1759 return Result;
1760 });
1761}
1762
1765 auto Result = getDFSLinkOrder(JDs);
1766 if (Result)
1767 std::reverse(Result->begin(), Result->end());
1768 return Result;
1769}
1770
1772 return getDFSLinkOrder({this});
1773}
1774
1776 return getReverseDFSLinkOrder({this});
1777}
1778
1780 LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet,
1781 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) {
1782
1783 OL_applyQueryPhase1(std::make_unique<InProgressLookupFlagsState>(
1784 K, std::move(SearchOrder), std::move(LookupSet),
1785 std::move(OnComplete)),
1786 Error::success());
1787}
1788
1791 SymbolLookupSet LookupSet) {
1792
1793 std::promise<MSVCPExpected<SymbolFlagsMap>> ResultP;
1794 OL_applyQueryPhase1(std::make_unique<InProgressLookupFlagsState>(
1795 K, std::move(SearchOrder), std::move(LookupSet),
1796 [&ResultP](Expected<SymbolFlagsMap> Result) {
1797 ResultP.set_value(std::move(Result));
1798 }),
1799 Error::success());
1800
1801 auto ResultF = ResultP.get_future();
1802 return ResultF.get();
1803}
1804
1806 LookupKind K, const JITDylibSearchOrder &SearchOrder,
1807 SymbolLookupSet Symbols, SymbolState RequiredState,
1808 SymbolsResolvedCallback NotifyComplete,
1809 RegisterDependenciesFunction RegisterDependencies) {
1810
1811 LLVM_DEBUG({
1812 runSessionLocked([&]() {
1813 dbgs() << "Looking up " << Symbols << " in " << SearchOrder
1814 << " (required state: " << RequiredState << ")\n";
1815 });
1816 });
1817
1818 // lookup can be re-entered recursively if running on a single thread. Run any
1819 // outstanding MUs in case this query depends on them, otherwise this lookup
1820 // will starve waiting for a result from an MU that is stuck in the queue.
1821 dispatchOutstandingMUs();
1822
1823 auto Unresolved = std::move(Symbols);
1824 auto Q = std::make_shared<AsynchronousSymbolQuery>(Unresolved, RequiredState,
1825 std::move(NotifyComplete));
1826
1827 auto IPLS = std::make_unique<InProgressFullLookupState>(
1828 K, SearchOrder, std::move(Unresolved), RequiredState, std::move(Q),
1829 std::move(RegisterDependencies));
1830
1831 OL_applyQueryPhase1(std::move(IPLS), Error::success());
1832}
1833
1836 SymbolLookupSet Symbols, LookupKind K,
1837 SymbolState RequiredState,
1838 RegisterDependenciesFunction RegisterDependencies) {
1839#if LLVM_ENABLE_THREADS
1840 // In the threaded case we use promises to return the results.
1841 std::promise<SymbolMap> PromisedResult;
1842 Error ResolutionError = Error::success();
1843
1844 auto NotifyComplete = [&](Expected<SymbolMap> R) {
1845 if (R)
1846 PromisedResult.set_value(std::move(*R));
1847 else {
1848 ErrorAsOutParameter _(&ResolutionError);
1849 ResolutionError = R.takeError();
1850 PromisedResult.set_value(SymbolMap());
1851 }
1852 };
1853
1854#else
1856 Error ResolutionError = Error::success();
1857
1858 auto NotifyComplete = [&](Expected<SymbolMap> R) {
1859 ErrorAsOutParameter _(&ResolutionError);
1860 if (R)
1861 Result = std::move(*R);
1862 else
1863 ResolutionError = R.takeError();
1864 };
1865#endif
1866
1867 // Perform the asynchronous lookup.
1868 lookup(K, SearchOrder, std::move(Symbols), RequiredState, NotifyComplete,
1869 RegisterDependencies);
1870
1871#if LLVM_ENABLE_THREADS
1872 auto ResultFuture = PromisedResult.get_future();
1873 auto Result = ResultFuture.get();
1874
1875 if (ResolutionError)
1876 return std::move(ResolutionError);
1877
1878 return std::move(Result);
1879
1880#else
1881 if (ResolutionError)
1882 return std::move(ResolutionError);
1883
1884 return Result;
1885#endif
1886}
1887
1890 SymbolStringPtr Name, SymbolState RequiredState) {
1891 SymbolLookupSet Names({Name});
1892
1893 if (auto ResultMap = lookup(SearchOrder, std::move(Names), LookupKind::Static,
1894 RequiredState, NoDependenciesToRegister)) {
1895 assert(ResultMap->size() == 1 && "Unexpected number of results");
1896 assert(ResultMap->count(Name) && "Missing result for symbol");
1897 return std::move(ResultMap->begin()->second);
1898 } else
1899 return ResultMap.takeError();
1900}
1901
1904 SymbolState RequiredState) {
1905 return lookup(makeJITDylibSearchOrder(SearchOrder), Name, RequiredState);
1906}
1907
1910 SymbolState RequiredState) {
1911 return lookup(SearchOrder, intern(Name), RequiredState);
1912}
1913
1916
1917 auto TagAddrs = lookup({{&JD, JITDylibLookupFlags::MatchAllSymbols}},
1920 if (!TagAddrs)
1921 return TagAddrs.takeError();
1922
1923 // Associate tag addresses with implementations.
1924 std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex);
1925 for (auto &KV : *TagAddrs) {
1926 auto TagAddr = KV.second.getAddress();
1927 if (JITDispatchHandlers.count(TagAddr))
1928 return make_error<StringError>("Tag " + formatv("{0:x16}", TagAddr) +
1929 " (for " + *KV.first +
1930 ") already registered",
1932 auto I = WFs.find(KV.first);
1933 assert(I != WFs.end() && I->second &&
1934 "JITDispatchHandler implementation missing");
1935 JITDispatchHandlers[KV.second.getAddress()] =
1936 std::make_shared<JITDispatchHandlerFunction>(std::move(I->second));
1937 LLVM_DEBUG({
1938 dbgs() << "Associated function tag \"" << *KV.first << "\" ("
1939 << formatv("{0:x}", KV.second.getAddress()) << ") with handler\n";
1940 });
1941 }
1942 return Error::success();
1943}
1944
1946 ExecutorAddr HandlerFnTagAddr,
1947 ArrayRef<char> ArgBuffer) {
1948
1949 std::shared_ptr<JITDispatchHandlerFunction> F;
1950 {
1951 std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex);
1952 auto I = JITDispatchHandlers.find(HandlerFnTagAddr);
1953 if (I != JITDispatchHandlers.end())
1954 F = I->second;
1955 }
1956
1957 if (F)
1958 (*F)(std::move(SendResult), ArgBuffer.data(), ArgBuffer.size());
1959 else
1961 ("No function registered for tag " +
1962 formatv("{0:x16}", HandlerFnTagAddr))
1963 .str()));
1964}
1965
1967 runSessionLocked([this, &OS]() {
1968 for (auto &JD : JDs)
1969 JD->dump(OS);
1970 });
1971}
1972
1973#ifdef EXPENSIVE_CHECKS
1974bool ExecutionSession::verifySessionState(Twine Phase) {
1975 return runSessionLocked([&]() {
1976 bool AllOk = true;
1977
1978 // We'll collect these and verify them later to avoid redundant checks.
1980
1981 for (auto &JD : JDs) {
1982
1983 auto LogFailure = [&]() -> raw_fd_ostream & {
1984 auto &Stream = errs();
1985 if (AllOk)
1986 Stream << "ERROR: Bad ExecutionSession state detected " << Phase
1987 << "\n";
1988 Stream << " In JITDylib " << JD->getName() << ", ";
1989 AllOk = false;
1990 return Stream;
1991 };
1992
1993 if (JD->State != JITDylib::Open) {
1994 LogFailure()
1995 << "state is not Open, but JD is in ExecutionSession list.";
1996 }
1997
1998 // Check symbol table.
1999 // 1. If the entry state isn't resolved then check that no address has
2000 // been set.
2001 // 2. Check that if the hasMaterializerAttached flag is set then there is
2002 // an UnmaterializedInfo entry, and vice-versa.
2003 for (auto &[Sym, Entry] : JD->Symbols) {
2004 // Check that unresolved symbols have null addresses.
2005 if (Entry.getState() < SymbolState::Resolved) {
2006 if (Entry.getAddress()) {
2007 LogFailure() << "symbol " << Sym << " has state "
2008 << Entry.getState()
2009 << " (not-yet-resolved) but non-null address "
2010 << Entry.getAddress() << ".\n";
2011 }
2012 }
2013
2014 // Check that the hasMaterializerAttached flag is correct.
2015 auto UMIItr = JD->UnmaterializedInfos.find(Sym);
2016 if (Entry.hasMaterializerAttached()) {
2017 if (UMIItr == JD->UnmaterializedInfos.end()) {
2018 LogFailure() << "symbol " << Sym
2019 << " entry claims materializer attached, but "
2020 "UnmaterializedInfos has no corresponding entry.\n";
2021 }
2022 } else if (UMIItr != JD->UnmaterializedInfos.end()) {
2023 LogFailure()
2024 << "symbol " << Sym
2025 << " entry claims no materializer attached, but "
2026 "UnmaterializedInfos has an unexpected entry for it.\n";
2027 }
2028 }
2029
2030 // Check that every UnmaterializedInfo entry has a corresponding entry
2031 // in the Symbols table.
2032 for (auto &[Sym, UMI] : JD->UnmaterializedInfos) {
2033 auto SymItr = JD->Symbols.find(Sym);
2034 if (SymItr == JD->Symbols.end()) {
2035 LogFailure()
2036 << "symbol " << Sym
2037 << " has UnmaterializedInfos entry, but no Symbols entry.\n";
2038 }
2039 }
2040
2041 // Check consistency of the MaterializingInfos table.
2042 for (auto &[Sym, MII] : JD->MaterializingInfos) {
2043
2044 auto SymItr = JD->Symbols.find(Sym);
2045 if (SymItr == JD->Symbols.end()) {
2046 // If there's no Symbols entry for this MaterializingInfos entry then
2047 // report that.
2048 LogFailure()
2049 << "symbol " << Sym
2050 << " has MaterializingInfos entry, but no Symbols entry.\n";
2051 } else {
2052 // Otherwise check consistency between Symbols and MaterializingInfos.
2053
2054 // Ready symbols should not have MaterializingInfos.
2055 if (SymItr->second.getState() == SymbolState::Ready) {
2056 LogFailure()
2057 << "symbol " << Sym
2058 << " is in Ready state, should not have MaterializingInfo.\n";
2059 }
2060
2061 // Pending queries should be for subsequent states.
2062 auto CurState = static_cast<SymbolState>(
2063 static_cast<std::underlying_type_t<SymbolState>>(
2064 SymItr->second.getState()) + 1);
2065 for (auto &Q : MII.PendingQueries) {
2066 if (Q->getRequiredState() != CurState) {
2067 if (Q->getRequiredState() > CurState)
2068 CurState = Q->getRequiredState();
2069 else
2070 LogFailure() << "symbol " << Sym
2071 << " has stale or misordered queries.\n";
2072 }
2073 }
2074
2075 // If there's a DefiningEDU then check that...
2076 // 1. The JD matches.
2077 // 2. The symbol is in the EDU's Symbols map.
2078 // 3. The symbol table entry is in the Emitted state.
2079 if (MII.DefiningEDU) {
2080
2081 EDUsToCheck.insert(MII.DefiningEDU.get());
2082
2083 if (MII.DefiningEDU->JD != JD.get()) {
2084 LogFailure() << "symbol " << Sym
2085 << " has DefiningEDU with incorrect JD"
2086 << (llvm::is_contained(JDs, MII.DefiningEDU->JD)
2087 ? " (JD not currently in ExecutionSession"
2088 : "")
2089 << "\n";
2090 }
2091
2092 if (SymItr->second.getState() != SymbolState::Emitted) {
2093 LogFailure()
2094 << "symbol " << Sym
2095 << " has DefiningEDU, but is not in Emitted state.\n";
2096 }
2097 }
2098
2099 // Check that JDs for any DependantEDUs are also in the session --
2100 // that guarantees that we'll also visit them during this loop.
2101 for (auto &DepEDU : MII.DependantEDUs) {
2102 if (!llvm::is_contained(JDs, DepEDU->JD)) {
2103 LogFailure() << "symbol " << Sym << " has DependantEDU "
2104 << (void *)DepEDU << " with JD (" << DepEDU->JD
2105 << ") that isn't in ExecutionSession.\n";
2106 }
2107 }
2108 }
2109 }
2110 }
2111
2112 // Check EDUs.
2113 for (auto *EDU : EDUsToCheck) {
2114 assert(EDU->JD->State == JITDylib::Open && "EDU->JD is not Open");
2115
2116 auto LogFailure = [&]() -> raw_fd_ostream & {
2117 AllOk = false;
2118 auto &Stream = errs();
2119 Stream << "In EDU defining " << EDU->JD->getName() << ": { ";
2120 for (auto &[Sym, Flags] : EDU->Symbols)
2121 Stream << Sym << " ";
2122 Stream << "}, ";
2123 return Stream;
2124 };
2125
2126 if (EDU->Symbols.empty())
2127 LogFailure() << "no symbols defined.\n";
2128 else {
2129 for (auto &[Sym, Flags] : EDU->Symbols) {
2130 if (!Sym)
2131 LogFailure() << "null symbol defined.\n";
2132 else {
2133 if (!EDU->JD->Symbols.count(SymbolStringPtr(Sym))) {
2134 LogFailure() << "symbol " << Sym
2135 << " isn't present in JD's symbol table.\n";
2136 }
2137 }
2138 }
2139 }
2140
2141 for (auto &[DepJD, Symbols] : EDU->Dependencies) {
2142 if (!llvm::is_contained(JDs, DepJD)) {
2143 LogFailure() << "dependant symbols listed for JD that isn't in "
2144 "ExecutionSession.\n";
2145 } else {
2146 for (auto &DepSym : Symbols) {
2147 if (!DepJD->Symbols.count(SymbolStringPtr(DepSym))) {
2148 LogFailure()
2149 << "dependant symbol " << DepSym
2150 << " does not appear in symbol table for dependant JD "
2151 << DepJD->getName() << ".\n";
2152 }
2153 }
2154 }
2155 }
2156 }
2157
2158 return AllOk;
2159 });
2160}
2161#endif // EXPENSIVE_CHECKS
2162
2163void ExecutionSession::dispatchOutstandingMUs() {
2164 LLVM_DEBUG(dbgs() << "Dispatching MaterializationUnits...\n");
2165 while (true) {
2166 std::optional<std::pair<std::unique_ptr<MaterializationUnit>,
2167 std::unique_ptr<MaterializationResponsibility>>>
2168 JMU;
2169
2170 {
2171 std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
2172 if (!OutstandingMUs.empty()) {
2173 JMU.emplace(std::move(OutstandingMUs.back()));
2174 OutstandingMUs.pop_back();
2175 }
2176 }
2177
2178 if (!JMU)
2179 break;
2180
2181 assert(JMU->first && "No MU?");
2182 LLVM_DEBUG(dbgs() << " Dispatching \"" << JMU->first->getName() << "\"\n");
2183 dispatchTask(std::make_unique<MaterializationTask>(std::move(JMU->first),
2184 std::move(JMU->second)));
2185 }
2186 LLVM_DEBUG(dbgs() << "Done dispatching MaterializationUnits.\n");
2187}
2188
2189Error ExecutionSession::removeResourceTracker(ResourceTracker &RT) {
2190 LLVM_DEBUG({
2191 dbgs() << "In " << RT.getJITDylib().getName() << " removing tracker "
2192 << formatv("{0:x}", RT.getKeyUnsafe()) << "\n";
2193 });
2194 std::vector<ResourceManager *> CurrentResourceManagers;
2195
2196 JITDylib::AsynchronousSymbolQuerySet QueriesToFail;
2197 std::shared_ptr<SymbolDependenceMap> FailedSymbols;
2198
2199 runSessionLocked([&] {
2200 CurrentResourceManagers = ResourceManagers;
2201 RT.makeDefunct();
2202 std::tie(QueriesToFail, FailedSymbols) = RT.getJITDylib().removeTracker(RT);
2203 });
2204
2205 Error Err = Error::success();
2206
2207 auto &JD = RT.getJITDylib();
2208 for (auto *L : reverse(CurrentResourceManagers))
2209 Err = joinErrors(std::move(Err),
2210 L->handleRemoveResources(JD, RT.getKeyUnsafe()));
2211
2212 for (auto &Q : QueriesToFail)
2213 Q->handleFailed(
2214 make_error<FailedToMaterialize>(getSymbolStringPool(), FailedSymbols));
2215
2216 return Err;
2217}
2218
2219void ExecutionSession::transferResourceTracker(ResourceTracker &DstRT,
2220 ResourceTracker &SrcRT) {
2221 LLVM_DEBUG({
2222 dbgs() << "In " << SrcRT.getJITDylib().getName()
2223 << " transfering resources from tracker "
2224 << formatv("{0:x}", SrcRT.getKeyUnsafe()) << " to tracker "
2225 << formatv("{0:x}", DstRT.getKeyUnsafe()) << "\n";
2226 });
2227
2228 // No-op transfers are allowed and do not invalidate the source.
2229 if (&DstRT == &SrcRT)
2230 return;
2231
2232 assert(&DstRT.getJITDylib() == &SrcRT.getJITDylib() &&
2233 "Can't transfer resources between JITDylibs");
2234 runSessionLocked([&]() {
2235 SrcRT.makeDefunct();
2236 auto &JD = DstRT.getJITDylib();
2237 JD.transferTracker(DstRT, SrcRT);
2238 for (auto *L : reverse(ResourceManagers))
2239 L->handleTransferResources(JD, DstRT.getKeyUnsafe(),
2240 SrcRT.getKeyUnsafe());
2241 });
2242}
2243
2244void ExecutionSession::destroyResourceTracker(ResourceTracker &RT) {
2245 runSessionLocked([&]() {
2246 LLVM_DEBUG({
2247 dbgs() << "In " << RT.getJITDylib().getName() << " destroying tracker "
2248 << formatv("{0:x}", RT.getKeyUnsafe()) << "\n";
2249 });
2250 if (!RT.isDefunct())
2251 transferResourceTracker(*RT.getJITDylib().getDefaultResourceTracker(),
2252 RT);
2253 });
2254}
2255
2256Error ExecutionSession::IL_updateCandidatesFor(
2257 JITDylib &JD, JITDylibLookupFlags JDLookupFlags,
2258 SymbolLookupSet &Candidates, SymbolLookupSet *NonCandidates) {
2259 return Candidates.forEachWithRemoval(
2260 [&](const SymbolStringPtr &Name,
2261 SymbolLookupFlags SymLookupFlags) -> Expected<bool> {
2262 /// Search for the symbol. If not found then continue without
2263 /// removal.
2264 auto SymI = JD.Symbols.find(Name);
2265 if (SymI == JD.Symbols.end())
2266 return false;
2267
2268 // If this is a non-exported symbol and we're matching exported
2269 // symbols only then remove this symbol from the candidates list.
2270 //
2271 // If we're tracking non-candidates then add this to the non-candidate
2272 // list.
2273 if (!SymI->second.getFlags().isExported() &&
2275 if (NonCandidates)
2276 NonCandidates->add(Name, SymLookupFlags);
2277 return true;
2278 }
2279
2280 // If we match against a materialization-side-effects only symbol
2281 // then make sure it is weakly-referenced. Otherwise bail out with
2282 // an error.
2283 // FIXME: Use a "materialization-side-effects-only symbols must be
2284 // weakly referenced" specific error here to reduce confusion.
2285 if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
2287 return make_error<SymbolsNotFound>(getSymbolStringPool(),
2289
2290 // If we matched against this symbol but it is in the error state
2291 // then bail out and treat it as a failure to materialize.
2292 if (SymI->second.getFlags().hasError()) {
2293 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
2294 (*FailedSymbolsMap)[&JD] = {Name};
2295 return make_error<FailedToMaterialize>(getSymbolStringPool(),
2296 std::move(FailedSymbolsMap));
2297 }
2298
2299 // Otherwise this is a match. Remove it from the candidate set.
2300 return true;
2301 });
2302}
2303
2304void ExecutionSession::OL_resumeLookupAfterGeneration(
2305 InProgressLookupState &IPLS) {
2306
2308 "Should not be called for not-in-generator lookups");
2310
2312
2313 if (auto DG = IPLS.CurDefGeneratorStack.back().lock()) {
2314 IPLS.CurDefGeneratorStack.pop_back();
2315 std::lock_guard<std::mutex> Lock(DG->M);
2316
2317 // If there are no pending lookups then mark the generator as free and
2318 // return.
2319 if (DG->PendingLookups.empty()) {
2320 DG->InUse = false;
2321 return;
2322 }
2323
2324 // Otherwise resume the next lookup.
2325 LS = std::move(DG->PendingLookups.front());
2326 DG->PendingLookups.pop_front();
2327 }
2328
2329 if (LS.IPLS) {
2331 dispatchTask(std::make_unique<LookupTask>(std::move(LS)));
2332 }
2333}
2334
2335void ExecutionSession::OL_applyQueryPhase1(
2336 std::unique_ptr<InProgressLookupState> IPLS, Error Err) {
2337
2338 LLVM_DEBUG({
2339 dbgs() << "Entering OL_applyQueryPhase1:\n"
2340 << " Lookup kind: " << IPLS->K << "\n"
2341 << " Search order: " << IPLS->SearchOrder
2342 << ", Current index = " << IPLS->CurSearchOrderIndex
2343 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2344 << " Lookup set: " << IPLS->LookupSet << "\n"
2345 << " Definition generator candidates: "
2346 << IPLS->DefGeneratorCandidates << "\n"
2347 << " Definition generator non-candidates: "
2348 << IPLS->DefGeneratorNonCandidates << "\n";
2349 });
2350
2351 if (IPLS->GenState == InProgressLookupState::InGenerator)
2352 OL_resumeLookupAfterGeneration(*IPLS);
2353
2354 assert(IPLS->GenState != InProgressLookupState::InGenerator &&
2355 "Lookup should not be in InGenerator state here");
2356
2357 // FIXME: We should attach the query as we go: This provides a result in a
2358 // single pass in the common case where all symbols have already reached the
2359 // required state. The query could be detached again in the 'fail' method on
2360 // IPLS. Phase 2 would be reduced to collecting and dispatching the MUs.
2361
2362 while (IPLS->CurSearchOrderIndex != IPLS->SearchOrder.size()) {
2363
2364 // If we've been handed an error or received one back from a generator then
2365 // fail the query. We don't need to unlink: At this stage the query hasn't
2366 // actually been lodged.
2367 if (Err)
2368 return IPLS->fail(std::move(Err));
2369
2370 // Get the next JITDylib and lookup flags.
2371 auto &KV = IPLS->SearchOrder[IPLS->CurSearchOrderIndex];
2372 auto &JD = *KV.first;
2373 auto JDLookupFlags = KV.second;
2374
2375 LLVM_DEBUG({
2376 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2377 << ") with lookup set " << IPLS->LookupSet << ":\n";
2378 });
2379
2380 // If we've just reached a new JITDylib then perform some setup.
2381 if (IPLS->NewJITDylib) {
2382 // Add any non-candidates from the last JITDylib (if any) back on to the
2383 // list of definition candidates for this JITDylib, reset definition
2384 // non-candidates to the empty set.
2385 SymbolLookupSet Tmp;
2386 std::swap(IPLS->DefGeneratorNonCandidates, Tmp);
2387 IPLS->DefGeneratorCandidates.append(std::move(Tmp));
2388
2389 LLVM_DEBUG({
2390 dbgs() << " First time visiting " << JD.getName()
2391 << ", resetting candidate sets and building generator stack\n";
2392 });
2393
2394 // Build the definition generator stack for this JITDylib.
2395 runSessionLocked([&] {
2396 IPLS->CurDefGeneratorStack.reserve(JD.DefGenerators.size());
2397 for (auto &DG : reverse(JD.DefGenerators))
2398 IPLS->CurDefGeneratorStack.push_back(DG);
2399 });
2400
2401 // Flag that we've done our initialization.
2402 IPLS->NewJITDylib = false;
2403 }
2404
2405 // Remove any generation candidates that are already defined (and match) in
2406 // this JITDylib.
2407 runSessionLocked([&] {
2408 // Update the list of candidates (and non-candidates) for definition
2409 // generation.
2410 LLVM_DEBUG(dbgs() << " Updating candidate set...\n");
2411 Err = IL_updateCandidatesFor(
2412 JD, JDLookupFlags, IPLS->DefGeneratorCandidates,
2413 JD.DefGenerators.empty() ? nullptr
2414 : &IPLS->DefGeneratorNonCandidates);
2415 LLVM_DEBUG({
2416 dbgs() << " Remaining candidates = " << IPLS->DefGeneratorCandidates
2417 << "\n";
2418 });
2419
2420 // If this lookup was resumed after auto-suspension but all candidates
2421 // have already been generated (by some previous call to the generator)
2422 // treat the lookup as if it had completed generation.
2423 if (IPLS->GenState == InProgressLookupState::ResumedForGenerator &&
2424 IPLS->DefGeneratorCandidates.empty())
2425 OL_resumeLookupAfterGeneration(*IPLS);
2426 });
2427
2428 // If we encountered an error while filtering generation candidates then
2429 // bail out.
2430 if (Err)
2431 return IPLS->fail(std::move(Err));
2432
2433 /// Apply any definition generators on the stack.
2434 LLVM_DEBUG({
2435 if (IPLS->CurDefGeneratorStack.empty())
2436 LLVM_DEBUG(dbgs() << " No generators to run for this JITDylib.\n");
2437 else if (IPLS->DefGeneratorCandidates.empty())
2438 LLVM_DEBUG(dbgs() << " No candidates to generate.\n");
2439 else
2440 dbgs() << " Running " << IPLS->CurDefGeneratorStack.size()
2441 << " remaining generators for "
2442 << IPLS->DefGeneratorCandidates.size() << " candidates\n";
2443 });
2444 while (!IPLS->CurDefGeneratorStack.empty() &&
2445 !IPLS->DefGeneratorCandidates.empty()) {
2446 auto DG = IPLS->CurDefGeneratorStack.back().lock();
2447
2448 if (!DG)
2449 return IPLS->fail(make_error<StringError>(
2450 "DefinitionGenerator removed while lookup in progress",
2452
2453 // At this point the lookup is in either the NotInGenerator state, or in
2454 // the ResumedForGenerator state.
2455 // If this lookup is in the NotInGenerator state then check whether the
2456 // generator is in use. If the generator is not in use then move the
2457 // lookup to the InGenerator state and continue. If the generator is
2458 // already in use then just add this lookup to the pending lookups list
2459 // and bail out.
2460 // If this lookup is in the ResumedForGenerator state then just move it
2461 // to InGenerator and continue.
2462 if (IPLS->GenState == InProgressLookupState::NotInGenerator) {
2463 std::lock_guard<std::mutex> Lock(DG->M);
2464 if (DG->InUse) {
2465 DG->PendingLookups.push_back(std::move(IPLS));
2466 return;
2467 }
2468 DG->InUse = true;
2469 }
2470
2471 IPLS->GenState = InProgressLookupState::InGenerator;
2472
2473 auto K = IPLS->K;
2474 auto &LookupSet = IPLS->DefGeneratorCandidates;
2475
2476 // Run the generator. If the generator takes ownership of QA then this
2477 // will break the loop.
2478 {
2479 LLVM_DEBUG(dbgs() << " Attempting to generate " << LookupSet << "\n");
2480 LookupState LS(std::move(IPLS));
2481 Err = DG->tryToGenerate(LS, K, JD, JDLookupFlags, LookupSet);
2482 IPLS = std::move(LS.IPLS);
2483 }
2484
2485 // If the lookup returned then pop the generator stack and unblock the
2486 // next lookup on this generator (if any).
2487 if (IPLS)
2488 OL_resumeLookupAfterGeneration(*IPLS);
2489
2490 // If there was an error then fail the query.
2491 if (Err) {
2492 LLVM_DEBUG({
2493 dbgs() << " Error attempting to generate " << LookupSet << "\n";
2494 });
2495 assert(IPLS && "LS cannot be retained if error is returned");
2496 return IPLS->fail(std::move(Err));
2497 }
2498
2499 // Otherwise if QA was captured then break the loop.
2500 if (!IPLS) {
2501 LLVM_DEBUG(
2502 { dbgs() << " LookupState captured. Exiting phase1 for now.\n"; });
2503 return;
2504 }
2505
2506 // Otherwise if we're continuing around the loop then update candidates
2507 // for the next round.
2508 runSessionLocked([&] {
2509 LLVM_DEBUG(dbgs() << " Updating candidate set post-generation\n");
2510 Err = IL_updateCandidatesFor(
2511 JD, JDLookupFlags, IPLS->DefGeneratorCandidates,
2512 JD.DefGenerators.empty() ? nullptr
2513 : &IPLS->DefGeneratorNonCandidates);
2514 });
2515
2516 // If updating candidates failed then fail the query.
2517 if (Err) {
2518 LLVM_DEBUG(dbgs() << " Error encountered while updating candidates\n");
2519 return IPLS->fail(std::move(Err));
2520 }
2521 }
2522
2523 if (IPLS->DefGeneratorCandidates.empty() &&
2524 IPLS->DefGeneratorNonCandidates.empty()) {
2525 // Early out if there are no remaining symbols.
2526 LLVM_DEBUG(dbgs() << "All symbols matched.\n");
2527 IPLS->CurSearchOrderIndex = IPLS->SearchOrder.size();
2528 break;
2529 } else {
2530 // If we get here then we've moved on to the next JITDylib with candidates
2531 // remaining.
2532 LLVM_DEBUG(dbgs() << "Phase 1 moving to next JITDylib.\n");
2533 ++IPLS->CurSearchOrderIndex;
2534 IPLS->NewJITDylib = true;
2535 }
2536 }
2537
2538 // Remove any weakly referenced candidates that could not be found/generated.
2539 IPLS->DefGeneratorCandidates.remove_if(
2540 [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2541 return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol;
2542 });
2543
2544 // If we get here then we've finished searching all JITDylibs.
2545 // If we matched all symbols then move to phase 2, otherwise fail the query
2546 // with a SymbolsNotFound error.
2547 if (IPLS->DefGeneratorCandidates.empty()) {
2548 LLVM_DEBUG(dbgs() << "Phase 1 succeeded.\n");
2549 IPLS->complete(std::move(IPLS));
2550 } else {
2551 LLVM_DEBUG(dbgs() << "Phase 1 failed with unresolved symbols.\n");
2552 IPLS->fail(make_error<SymbolsNotFound>(
2553 getSymbolStringPool(), IPLS->DefGeneratorCandidates.getSymbolNames()));
2554 }
2555}
2556
2557void ExecutionSession::OL_completeLookup(
2558 std::unique_ptr<InProgressLookupState> IPLS,
2559 std::shared_ptr<AsynchronousSymbolQuery> Q,
2560 RegisterDependenciesFunction RegisterDependencies) {
2561
2562 LLVM_DEBUG({
2563 dbgs() << "Entering OL_completeLookup:\n"
2564 << " Lookup kind: " << IPLS->K << "\n"
2565 << " Search order: " << IPLS->SearchOrder
2566 << ", Current index = " << IPLS->CurSearchOrderIndex
2567 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2568 << " Lookup set: " << IPLS->LookupSet << "\n"
2569 << " Definition generator candidates: "
2570 << IPLS->DefGeneratorCandidates << "\n"
2571 << " Definition generator non-candidates: "
2572 << IPLS->DefGeneratorNonCandidates << "\n";
2573 });
2574
2575 bool QueryComplete = false;
2576 DenseMap<JITDylib *, JITDylib::UnmaterializedInfosList> CollectedUMIs;
2577
2578 auto LodgingErr = runSessionLocked([&]() -> Error {
2579 for (auto &KV : IPLS->SearchOrder) {
2580 auto &JD = *KV.first;
2581 auto JDLookupFlags = KV.second;
2582 LLVM_DEBUG({
2583 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2584 << ") with lookup set " << IPLS->LookupSet << ":\n";
2585 });
2586
2587 auto Err = IPLS->LookupSet.forEachWithRemoval(
2588 [&](const SymbolStringPtr &Name,
2589 SymbolLookupFlags SymLookupFlags) -> Expected<bool> {
2590 LLVM_DEBUG({
2591 dbgs() << " Attempting to match \"" << Name << "\" ("
2592 << SymLookupFlags << ")... ";
2593 });
2594
2595 /// Search for the symbol. If not found then continue without
2596 /// removal.
2597 auto SymI = JD.Symbols.find(Name);
2598 if (SymI == JD.Symbols.end()) {
2599 LLVM_DEBUG(dbgs() << "skipping: not present\n");
2600 return false;
2601 }
2602
2603 // If this is a non-exported symbol and we're matching exported
2604 // symbols only then skip this symbol without removal.
2605 if (!SymI->second.getFlags().isExported() &&
2606 JDLookupFlags ==
2608 LLVM_DEBUG(dbgs() << "skipping: not exported\n");
2609 return false;
2610 }
2611
2612 // If we match against a materialization-side-effects only symbol
2613 // then make sure it is weakly-referenced. Otherwise bail out with
2614 // an error.
2615 // FIXME: Use a "materialization-side-effects-only symbols must be
2616 // weakly referenced" specific error here to reduce confusion.
2617 if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
2619 LLVM_DEBUG({
2620 dbgs() << "error: "
2621 "required, but symbol is has-side-effects-only\n";
2622 });
2623 return make_error<SymbolsNotFound>(getSymbolStringPool(),
2625 }
2626
2627 // If we matched against this symbol but it is in the error state
2628 // then bail out and treat it as a failure to materialize.
2629 if (SymI->second.getFlags().hasError()) {
2630 LLVM_DEBUG(dbgs() << "error: symbol is in error state\n");
2631 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
2632 (*FailedSymbolsMap)[&JD] = {Name};
2633 return make_error<FailedToMaterialize>(
2634 getSymbolStringPool(), std::move(FailedSymbolsMap));
2635 }
2636
2637 // Otherwise this is a match.
2638
2639 // If this symbol is already in the required state then notify the
2640 // query, remove the symbol and continue.
2641 if (SymI->second.getState() >= Q->getRequiredState()) {
2643 << "matched, symbol already in required state\n");
2644 Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol());
2645 return true;
2646 }
2647
2648 // Otherwise this symbol does not yet meet the required state. Check
2649 // whether it has a materializer attached, and if so prepare to run
2650 // it.
2651 if (SymI->second.hasMaterializerAttached()) {
2652 assert(SymI->second.getAddress() == ExecutorAddr() &&
2653 "Symbol not resolved but already has address?");
2654 auto UMII = JD.UnmaterializedInfos.find(Name);
2655 assert(UMII != JD.UnmaterializedInfos.end() &&
2656 "Lazy symbol should have UnmaterializedInfo");
2657
2658 auto UMI = UMII->second;
2659 assert(UMI->MU && "Materializer should not be null");
2660 assert(UMI->RT && "Tracker should not be null");
2661 LLVM_DEBUG({
2662 dbgs() << "matched, preparing to dispatch MU@" << UMI->MU.get()
2663 << " (" << UMI->MU->getName() << ")\n";
2664 });
2665
2666 // Move all symbols associated with this MaterializationUnit into
2667 // materializing state.
2668 for (auto &KV : UMI->MU->getSymbols()) {
2669 auto SymK = JD.Symbols.find(KV.first);
2670 assert(SymK != JD.Symbols.end() &&
2671 "No entry for symbol covered by MaterializationUnit");
2672 SymK->second.setMaterializerAttached(false);
2673 SymK->second.setState(SymbolState::Materializing);
2674 JD.UnmaterializedInfos.erase(KV.first);
2675 }
2676
2677 // Add MU to the list of MaterializationUnits to be materialized.
2678 CollectedUMIs[&JD].push_back(std::move(UMI));
2679 } else
2680 LLVM_DEBUG(dbgs() << "matched, registering query");
2681
2682 // Add the query to the PendingQueries list and continue, deleting
2683 // the element from the lookup set.
2684 assert(SymI->second.getState() != SymbolState::NeverSearched &&
2685 SymI->second.getState() != SymbolState::Ready &&
2686 "By this line the symbol should be materializing");
2687 auto &MI = JD.MaterializingInfos[Name];
2688 MI.addQuery(Q);
2689 Q->addQueryDependence(JD, Name);
2690
2691 return true;
2692 });
2693
2694 JD.shrinkMaterializationInfoMemory();
2695
2696 // Handle failure.
2697 if (Err) {
2698
2699 LLVM_DEBUG({
2700 dbgs() << "Lookup failed. Detaching query and replacing MUs.\n";
2701 });
2702
2703 // Detach the query.
2704 Q->detach();
2705
2706 // Replace the MUs.
2707 for (auto &KV : CollectedUMIs) {
2708 auto &JD = *KV.first;
2709 for (auto &UMI : KV.second)
2710 for (auto &KV2 : UMI->MU->getSymbols()) {
2711 assert(!JD.UnmaterializedInfos.count(KV2.first) &&
2712 "Unexpected materializer in map");
2713 auto SymI = JD.Symbols.find(KV2.first);
2714 assert(SymI != JD.Symbols.end() && "Missing symbol entry");
2715 assert(SymI->second.getState() == SymbolState::Materializing &&
2716 "Can not replace symbol that is not materializing");
2717 assert(!SymI->second.hasMaterializerAttached() &&
2718 "MaterializerAttached flag should not be set");
2719 SymI->second.setMaterializerAttached(true);
2720 JD.UnmaterializedInfos[KV2.first] = UMI;
2721 }
2722 }
2723
2724 return Err;
2725 }
2726 }
2727
2728 LLVM_DEBUG(dbgs() << "Stripping unmatched weakly-referenced symbols\n");
2729 IPLS->LookupSet.forEachWithRemoval(
2730 [&](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2731 if (SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol) {
2732 Q->dropSymbol(Name);
2733 return true;
2734 } else
2735 return false;
2736 });
2737
2738 if (!IPLS->LookupSet.empty()) {
2739 LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n");
2740 return make_error<SymbolsNotFound>(getSymbolStringPool(),
2741 IPLS->LookupSet.getSymbolNames());
2742 }
2743
2744 // Record whether the query completed.
2745 QueryComplete = Q->isComplete();
2746
2747 LLVM_DEBUG({
2748 dbgs() << "Query successfully "
2749 << (QueryComplete ? "completed" : "lodged") << "\n";
2750 });
2751
2752 // Move the collected MUs to the OutstandingMUs list.
2753 if (!CollectedUMIs.empty()) {
2754 std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
2755
2756 LLVM_DEBUG(dbgs() << "Adding MUs to dispatch:\n");
2757 for (auto &KV : CollectedUMIs) {
2758 LLVM_DEBUG({
2759 auto &JD = *KV.first;
2760 dbgs() << " For " << JD.getName() << ": Adding " << KV.second.size()
2761 << " MUs.\n";
2762 });
2763 for (auto &UMI : KV.second) {
2764 auto MR = createMaterializationResponsibility(
2765 *UMI->RT, std::move(UMI->MU->SymbolFlags),
2766 std::move(UMI->MU->InitSymbol));
2767 OutstandingMUs.push_back(
2768 std::make_pair(std::move(UMI->MU), std::move(MR)));
2769 }
2770 }
2771 } else
2772 LLVM_DEBUG(dbgs() << "No MUs to dispatch.\n");
2773
2774 if (RegisterDependencies && !Q->QueryRegistrations.empty()) {
2775 LLVM_DEBUG(dbgs() << "Registering dependencies\n");
2776 RegisterDependencies(Q->QueryRegistrations);
2777 } else
2778 LLVM_DEBUG(dbgs() << "No dependencies to register\n");
2779
2780 return Error::success();
2781 });
2782
2783 if (LodgingErr) {
2784 LLVM_DEBUG(dbgs() << "Failing query\n");
2785 Q->detach();
2786 Q->handleFailed(std::move(LodgingErr));
2787 return;
2788 }
2789
2790 if (QueryComplete) {
2791 LLVM_DEBUG(dbgs() << "Completing query\n");
2792 Q->handleComplete(*this);
2793 }
2794
2795 dispatchOutstandingMUs();
2796}
2797
2798void ExecutionSession::OL_completeLookupFlags(
2799 std::unique_ptr<InProgressLookupState> IPLS,
2800 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) {
2801
2802 auto Result = runSessionLocked([&]() -> Expected<SymbolFlagsMap> {
2803 LLVM_DEBUG({
2804 dbgs() << "Entering OL_completeLookupFlags:\n"
2805 << " Lookup kind: " << IPLS->K << "\n"
2806 << " Search order: " << IPLS->SearchOrder
2807 << ", Current index = " << IPLS->CurSearchOrderIndex
2808 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2809 << " Lookup set: " << IPLS->LookupSet << "\n"
2810 << " Definition generator candidates: "
2811 << IPLS->DefGeneratorCandidates << "\n"
2812 << " Definition generator non-candidates: "
2813 << IPLS->DefGeneratorNonCandidates << "\n";
2814 });
2815
2817
2818 // Attempt to find flags for each symbol.
2819 for (auto &KV : IPLS->SearchOrder) {
2820 auto &JD = *KV.first;
2821 auto JDLookupFlags = KV.second;
2822 LLVM_DEBUG({
2823 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2824 << ") with lookup set " << IPLS->LookupSet << ":\n";
2825 });
2826
2827 IPLS->LookupSet.forEachWithRemoval([&](const SymbolStringPtr &Name,
2828 SymbolLookupFlags SymLookupFlags) {
2829 LLVM_DEBUG({
2830 dbgs() << " Attempting to match \"" << Name << "\" ("
2831 << SymLookupFlags << ")... ";
2832 });
2833
2834 // Search for the symbol. If not found then continue without removing
2835 // from the lookup set.
2836 auto SymI = JD.Symbols.find(Name);
2837 if (SymI == JD.Symbols.end()) {
2838 LLVM_DEBUG(dbgs() << "skipping: not present\n");
2839 return false;
2840 }
2841
2842 // If this is a non-exported symbol then it doesn't match. Skip it.
2843 if (!SymI->second.getFlags().isExported() &&
2845 LLVM_DEBUG(dbgs() << "skipping: not exported\n");
2846 return false;
2847 }
2848
2849 LLVM_DEBUG({
2850 dbgs() << "matched, \"" << Name << "\" -> " << SymI->second.getFlags()
2851 << "\n";
2852 });
2853 Result[Name] = SymI->second.getFlags();
2854 return true;
2855 });
2856 }
2857
2858 // Remove any weakly referenced symbols that haven't been resolved.
2859 IPLS->LookupSet.remove_if(
2860 [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2861 return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol;
2862 });
2863
2864 if (!IPLS->LookupSet.empty()) {
2865 LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n");
2866 return make_error<SymbolsNotFound>(getSymbolStringPool(),
2867 IPLS->LookupSet.getSymbolNames());
2868 }
2869
2870 LLVM_DEBUG(dbgs() << "Succeded, result = " << Result << "\n");
2871 return Result;
2872 });
2873
2874 // Run the callback on the result.
2875 LLVM_DEBUG(dbgs() << "Sending result to handler.\n");
2876 OnComplete(std::move(Result));
2877}
2878
2879void ExecutionSession::OL_destroyMaterializationResponsibility(
2880 MaterializationResponsibility &MR) {
2881
2882 assert(MR.SymbolFlags.empty() &&
2883 "All symbols should have been explicitly materialized or failed");
2884 MR.JD.unlinkMaterializationResponsibility(MR);
2885}
2886
2887SymbolNameSet ExecutionSession::OL_getRequestedSymbols(
2888 const MaterializationResponsibility &MR) {
2889 return MR.JD.getRequestedSymbols(MR.SymbolFlags);
2890}
2891
2892Error ExecutionSession::OL_notifyResolved(MaterializationResponsibility &MR,
2893 const SymbolMap &Symbols) {
2894 LLVM_DEBUG({
2895 dbgs() << "In " << MR.JD.getName() << " resolving " << Symbols << "\n";
2896 });
2897#ifndef NDEBUG
2898 for (auto &KV : Symbols) {
2899 auto I = MR.SymbolFlags.find(KV.first);
2900 assert(I != MR.SymbolFlags.end() &&
2901 "Resolving symbol outside this responsibility set");
2902 assert(!I->second.hasMaterializationSideEffectsOnly() &&
2903 "Can't resolve materialization-side-effects-only symbol");
2904 assert((KV.second.getFlags() & ~JITSymbolFlags::Common) ==
2905 (I->second & ~JITSymbolFlags::Common) &&
2906 "Resolving symbol with incorrect flags");
2907 }
2908#endif
2909
2910 return MR.JD.resolve(MR, Symbols);
2911}
2912
2913template <typename HandleNewDepFn>
2914void ExecutionSession::propagateExtraEmitDeps(
2915 std::deque<JITDylib::EmissionDepUnit *> Worklist, EDUInfosMap &EDUInfos,
2916 HandleNewDepFn HandleNewDep) {
2917
2918 // Iterate to a fixed-point to propagate extra-emit dependencies through the
2919 // EDU graph.
2920 while (!Worklist.empty()) {
2921 auto &EDU = *Worklist.front();
2922 Worklist.pop_front();
2923
2924 assert(EDUInfos.count(&EDU) && "No info entry for EDU");
2925 auto &EDUInfo = EDUInfos[&EDU];
2926
2927 // Propagate new dependencies to users.
2928 for (auto *UserEDU : EDUInfo.IntraEmitUsers) {
2929
2930 // UserEDUInfo only present if UserEDU has its own users.
2931 JITDylib::EmissionDepUnitInfo *UserEDUInfo = nullptr;
2932 {
2933 auto UserEDUInfoItr = EDUInfos.find(UserEDU);
2934 if (UserEDUInfoItr != EDUInfos.end())
2935 UserEDUInfo = &UserEDUInfoItr->second;
2936 }
2937
2938 for (auto &[DepJD, Deps] : EDUInfo.NewDeps) {
2939 auto &UserEDUDepsForJD = UserEDU->Dependencies[DepJD];
2940 DenseSet<NonOwningSymbolStringPtr> *UserEDUNewDepsForJD = nullptr;
2941 for (auto Dep : Deps) {
2942 if (UserEDUDepsForJD.insert(Dep).second) {
2943 HandleNewDep(*UserEDU, *DepJD, Dep);
2944 if (UserEDUInfo) {
2945 if (!UserEDUNewDepsForJD) {
2946 // If UserEDU has no new deps then it's not in the worklist
2947 // yet, so add it.
2948 if (UserEDUInfo->NewDeps.empty())
2949 Worklist.push_back(UserEDU);
2950 UserEDUNewDepsForJD = &UserEDUInfo->NewDeps[DepJD];
2951 }
2952 // Add (DepJD, Dep) to NewDeps.
2953 UserEDUNewDepsForJD->insert(Dep);
2954 }
2955 }
2956 }
2957 }
2958 }
2959
2960 EDUInfo.NewDeps.clear();
2961 }
2962}
2963
2964// Note: This method modifies the emitted set.
2965ExecutionSession::EDUInfosMap ExecutionSession::simplifyDepGroups(
2966 MaterializationResponsibility &MR,
2967 ArrayRef<SymbolDependenceGroup> EmittedDeps) {
2968
2969 auto &TargetJD = MR.getTargetJITDylib();
2970
2971 // 1. Build initial EmissionDepUnit -> EmissionDepUnitInfo and
2972 // Symbol -> EmissionDepUnit mappings.
2973 DenseMap<JITDylib::EmissionDepUnit *, JITDylib::EmissionDepUnitInfo> EDUInfos;
2974 EDUInfos.reserve(EmittedDeps.size());
2975 DenseMap<NonOwningSymbolStringPtr, JITDylib::EmissionDepUnit *> EDUForSymbol;
2976 for (auto &DG : EmittedDeps) {
2977 assert(!DG.Symbols.empty() && "DepGroup does not cover any symbols");
2978
2979 // Skip empty EDUs.
2980 if (DG.Dependencies.empty())
2981 continue;
2982
2983 auto TmpEDU = std::make_shared<JITDylib::EmissionDepUnit>(TargetJD);
2984 auto &EDUInfo = EDUInfos[TmpEDU.get()];
2985 EDUInfo.EDU = std::move(TmpEDU);
2986 for (const auto &Symbol : DG.Symbols) {
2987 NonOwningSymbolStringPtr NonOwningSymbol(Symbol);
2988 assert(!EDUForSymbol.count(NonOwningSymbol) &&
2989 "Symbol should not appear in more than one SymbolDependenceGroup");
2990 assert(MR.getSymbols().count(Symbol) &&
2991 "Symbol in DepGroups not in the emitted set");
2992 auto NewlyEmittedItr = MR.getSymbols().find(Symbol);
2993 EDUInfo.EDU->Symbols[NonOwningSymbol] = NewlyEmittedItr->second;
2994 EDUForSymbol[NonOwningSymbol] = EDUInfo.EDU.get();
2995 }
2996 }
2997
2998 // 2. Build a "residual" EDU to cover all symbols that have no dependencies.
2999 {
3000 DenseMap<NonOwningSymbolStringPtr, JITSymbolFlags> ResidualSymbolFlags;
3001 for (auto &[Sym, Flags] : MR.getSymbols()) {
3002 if (!EDUForSymbol.count(NonOwningSymbolStringPtr(Sym)))
3003 ResidualSymbolFlags[NonOwningSymbolStringPtr(Sym)] = Flags;
3004 }
3005 if (!ResidualSymbolFlags.empty()) {
3006 auto ResidualEDU = std::make_shared<JITDylib::EmissionDepUnit>(TargetJD);
3007 ResidualEDU->Symbols = std::move(ResidualSymbolFlags);
3008 auto &ResidualEDUInfo = EDUInfos[ResidualEDU.get()];
3009 ResidualEDUInfo.EDU = std::move(ResidualEDU);
3010
3011 // If the residual EDU is the only one then bail out early.
3012 if (EDUInfos.size() == 1)
3013 return EDUInfos;
3014
3015 // Otherwise add the residual EDU to the EDUForSymbol map.
3016 for (auto &[Sym, Flags] : ResidualEDUInfo.EDU->Symbols)
3017 EDUForSymbol[Sym] = ResidualEDUInfo.EDU.get();
3018 }
3019 }
3020
3021#ifndef NDEBUG
3022 assert(EDUForSymbol.size() == MR.getSymbols().size() &&
3023 "MR symbols not fully covered by EDUs?");
3024 for (auto &[Sym, Flags] : MR.getSymbols()) {
3025 assert(EDUForSymbol.count(NonOwningSymbolStringPtr(Sym)) &&
3026 "Sym in MR not covered by EDU");
3027 }
3028#endif // NDEBUG
3029
3030 // 3. Use the DepGroups array to build a graph of dependencies between
3031 // EmissionDepUnits in this finalization. We want to remove these
3032 // intra-finalization uses, propagating dependencies on symbols outside
3033 // this finalization. Add EDUs to the worklist.
3034 for (auto &DG : EmittedDeps) {
3035
3036 // Skip SymbolDependenceGroups with no dependencies.
3037 if (DG.Dependencies.empty())
3038 continue;
3039
3040 assert(EDUForSymbol.count(NonOwningSymbolStringPtr(*DG.Symbols.begin())) &&
3041 "No EDU for DG");
3042 auto &EDU =
3043 *EDUForSymbol.find(NonOwningSymbolStringPtr(*DG.Symbols.begin()))
3044 ->second;
3045
3046 for (auto &[DepJD, Deps] : DG.Dependencies) {
3047 DenseSet<NonOwningSymbolStringPtr> NewDepsForJD;
3048
3049 assert(!Deps.empty() && "Dependence set for DepJD is empty");
3050
3051 if (DepJD != &TargetJD) {
3052 // DepJD is some other JITDylib.There can't be any intra-finalization
3053 // edges here, so just skip.
3054 for (auto &Dep : Deps)
3055 NewDepsForJD.insert(NonOwningSymbolStringPtr(Dep));
3056 } else {
3057 // DepJD is the Target JITDylib. Check for intra-finaliztaion edges,
3058 // skipping any and recording the intra-finalization use instead.
3059 for (auto &Dep : Deps) {
3060 NonOwningSymbolStringPtr NonOwningDep(Dep);
3061 auto I = EDUForSymbol.find(NonOwningDep);
3062 if (I == EDUForSymbol.end()) {
3063 if (!MR.getSymbols().count(Dep))
3064 NewDepsForJD.insert(NonOwningDep);
3065 continue;
3066 }
3067
3068 if (I->second != &EDU)
3069 EDUInfos[I->second].IntraEmitUsers.insert(&EDU);
3070 }
3071 }
3072
3073 if (!NewDepsForJD.empty())
3074 EDU.Dependencies[DepJD] = std::move(NewDepsForJD);
3075 }
3076 }
3077
3078 // 4. Build the worklist.
3079 std::deque<JITDylib::EmissionDepUnit *> Worklist;
3080 for (auto &[EDU, EDUInfo] : EDUInfos) {
3081 // If this EDU has extra-finalization dependencies and intra-finalization
3082 // users then add it to the worklist.
3083 if (!EDU->Dependencies.empty()) {
3084 auto I = EDUInfos.find(EDU);
3085 if (I != EDUInfos.end()) {
3086 auto &EDUInfo = I->second;
3087 if (!EDUInfo.IntraEmitUsers.empty()) {
3088 EDUInfo.NewDeps = EDU->Dependencies;
3089 Worklist.push_back(EDU);
3090 }
3091 }
3092 }
3093 }
3094
3095 // 4. Propagate dependencies through the EDU graph.
3096 propagateExtraEmitDeps(
3097 Worklist, EDUInfos,
3098 [](JITDylib::EmissionDepUnit &, JITDylib &, NonOwningSymbolStringPtr) {});
3099
3100 return EDUInfos;
3101}
3102
3103void ExecutionSession::IL_makeEDUReady(
3104 std::shared_ptr<JITDylib::EmissionDepUnit> EDU,
3105 JITDylib::AsynchronousSymbolQuerySet &Queries) {
3106
3107 // The symbols for this EDU are ready.
3108 auto &JD = *EDU->JD;
3109
3110 for (auto &[Sym, Flags] : EDU->Symbols) {
3111 assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
3112 "JD does not have an entry for Sym");
3113 auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
3114
3115 assert(((Entry.getFlags().hasMaterializationSideEffectsOnly() &&
3116 Entry.getState() == SymbolState::Materializing) ||
3117 Entry.getState() == SymbolState::Resolved ||
3118 Entry.getState() == SymbolState::Emitted) &&
3119 "Emitting from state other than Resolved");
3120
3121 Entry.setState(SymbolState::Ready);
3122
3123 auto MII = JD.MaterializingInfos.find(SymbolStringPtr(Sym));
3124
3125 // Check for pending queries.
3126 if (MII == JD.MaterializingInfos.end())
3127 continue;
3128 auto &MI = MII->second;
3129
3130 for (auto &Q : MI.takeQueriesMeeting(SymbolState::Ready)) {
3131 Q->notifySymbolMetRequiredState(SymbolStringPtr(Sym), Entry.getSymbol());
3132 if (Q->isComplete())
3133 Queries.insert(Q);
3134 Q->removeQueryDependence(JD, SymbolStringPtr(Sym));
3135 }
3136
3137 JD.MaterializingInfos.erase(MII);
3138 }
3139
3140 JD.shrinkMaterializationInfoMemory();
3141}
3142
3143void ExecutionSession::IL_makeEDUEmitted(
3144 std::shared_ptr<JITDylib::EmissionDepUnit> EDU,
3145 JITDylib::AsynchronousSymbolQuerySet &Queries) {
3146
3147 // The symbols for this EDU are emitted, but not ready.
3148 auto &JD = *EDU->JD;
3149
3150 for (auto &[Sym, Flags] : EDU->Symbols) {
3151 assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
3152 "JD does not have an entry for Sym");
3153 auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
3154
3155 assert(((Entry.getFlags().hasMaterializationSideEffectsOnly() &&
3156 Entry.getState() == SymbolState::Materializing) ||
3157 Entry.getState() == SymbolState::Resolved ||
3158 Entry.getState() == SymbolState::Emitted) &&
3159 "Emitting from state other than Resolved");
3160
3161 if (Entry.getState() == SymbolState::Emitted) {
3162 // This was already emitted, so we can skip the rest of this loop.
3163#ifndef NDEBUG
3164 for (auto &[Sym, Flags] : EDU->Symbols) {
3165 assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
3166 "JD does not have an entry for Sym");
3167 auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
3168 assert(Entry.getState() == SymbolState::Emitted &&
3169 "Symbols for EDU in inconsistent state");
3170 assert(JD.MaterializingInfos.count(SymbolStringPtr(Sym)) &&
3171 "Emitted symbol has no MI");
3172 auto MI = JD.MaterializingInfos[SymbolStringPtr(Sym)];
3173 assert(MI.takeQueriesMeeting(SymbolState::Emitted).empty() &&
3174 "Already-emitted symbol has waiting-on-emitted queries");
3175 }
3176#endif // NDEBUG
3177 break;
3178 }
3179
3180 Entry.setState(SymbolState::Emitted);
3181 auto &MI = JD.MaterializingInfos[SymbolStringPtr(Sym)];
3182 MI.DefiningEDU = EDU;
3183
3184 for (auto &Q : MI.takeQueriesMeeting(SymbolState::Emitted)) {
3185 Q->notifySymbolMetRequiredState(SymbolStringPtr(Sym), Entry.getSymbol());
3186 if (Q->isComplete())
3187 Queries.insert(Q);
3188 Q->removeQueryDependence(JD, SymbolStringPtr(Sym));
3189 }
3190 }
3191
3192 for (auto &[DepJD, Deps] : EDU->Dependencies) {
3193 for (auto &Dep : Deps)
3194 DepJD->MaterializingInfos[SymbolStringPtr(Dep)].DependantEDUs.insert(
3195 EDU.get());
3196 }
3197}
3198
3199/// Removes the given dependence from EDU. If EDU's dependence set becomes
3200/// empty then this function adds an entry for it to the EDUInfos map.
3201/// Returns true if a new EDUInfosMap entry is added.
3202bool ExecutionSession::IL_removeEDUDependence(JITDylib::EmissionDepUnit &EDU,
3203 JITDylib &DepJD,
3204 NonOwningSymbolStringPtr DepSym,
3205 EDUInfosMap &EDUInfos) {
3206 assert(EDU.Dependencies.count(&DepJD) &&
3207 "JD does not appear in Dependencies of DependantEDU");
3208 assert(EDU.Dependencies[&DepJD].count(DepSym) &&
3209 "Symbol does not appear in Dependencies of DependantEDU");
3210 auto &JDDeps = EDU.Dependencies[&DepJD];
3211 JDDeps.erase(DepSym);
3212 if (JDDeps.empty()) {
3213 EDU.Dependencies.erase(&DepJD);
3214 if (EDU.Dependencies.empty()) {
3215 // If the dependencies set has become empty then EDU _may_ be ready
3216 // (we won't know for sure until we've propagated the extra-emit deps).
3217 // Create an EDUInfo for it (if it doesn't have one already) so that
3218 // it'll be visited after propagation.
3219 auto &DepEDUInfo = EDUInfos[&EDU];
3220 if (!DepEDUInfo.EDU) {
3221 assert(EDU.JD->Symbols.count(
3222 SymbolStringPtr(EDU.Symbols.begin()->first)) &&
3223 "Missing symbol entry for first symbol in EDU");
3224 auto DepEDUFirstMI = EDU.JD->MaterializingInfos.find(
3225 SymbolStringPtr(EDU.Symbols.begin()->first));
3226 assert(DepEDUFirstMI != EDU.JD->MaterializingInfos.end() &&
3227 "Missing MI for first symbol in DependantEDU");
3228 DepEDUInfo.EDU = DepEDUFirstMI->second.DefiningEDU;
3229 return true;
3230 }
3231 }
3232 }
3233 return false;
3234}
3235
3236Error ExecutionSession::makeJDClosedError(JITDylib::EmissionDepUnit &EDU,
3237 JITDylib &ClosedJD) {
3238 SymbolNameSet FailedSymbols;
3239 for (auto &[Sym, Flags] : EDU.Symbols)
3240 FailedSymbols.insert(SymbolStringPtr(Sym));
3241 SymbolDependenceMap BadDeps;
3242 for (auto &Dep : EDU.Dependencies[&ClosedJD])
3243 BadDeps[&ClosedJD].insert(SymbolStringPtr(Dep));
3244 return make_error<UnsatisfiedSymbolDependencies>(
3245 ClosedJD.getExecutionSession().getSymbolStringPool(), EDU.JD,
3246 std::move(FailedSymbols), std::move(BadDeps),
3247 ClosedJD.getName() + " is closed");
3248}
3249
3250Error ExecutionSession::makeUnsatisfiedDepsError(JITDylib::EmissionDepUnit &EDU,
3251 JITDylib &BadJD,
3252 SymbolNameSet BadDeps) {
3253 SymbolNameSet FailedSymbols;
3254 for (auto &[Sym, Flags] : EDU.Symbols)
3255 FailedSymbols.insert(SymbolStringPtr(Sym));
3256 SymbolDependenceMap BadDepsMap;
3257 BadDepsMap[&BadJD] = std::move(BadDeps);
3258 return make_error<UnsatisfiedSymbolDependencies>(
3259 BadJD.getExecutionSession().getSymbolStringPool(), &BadJD,
3260 std::move(FailedSymbols), std::move(BadDepsMap),
3261 "dependencies removed or in error state");
3262}
3263
3264Expected<JITDylib::AsynchronousSymbolQuerySet>
3265ExecutionSession::IL_emit(MaterializationResponsibility &MR,
3266 EDUInfosMap EDUInfos) {
3267
3268 if (MR.RT->isDefunct())
3269 return make_error<ResourceTrackerDefunct>(MR.RT);
3270
3271 auto &TargetJD = MR.getTargetJITDylib();
3272 if (TargetJD.State != JITDylib::Open)
3273 return make_error<StringError>("JITDylib " + TargetJD.getName() +
3274 " is defunct",
3276#ifdef EXPENSIVE_CHECKS
3277 verifySessionState("entering ExecutionSession::IL_emit");
3278#endif
3279
3280 // Walk all EDUs:
3281 // 1. Verifying that dependencies are available (not removed or in the error
3282 // state.
3283 // 2. Removing any dependencies that are already Ready.
3284 // 3. Lifting any EDUs for Emitted symbols into the EDUInfos map.
3285 // 4. Finding any dependant EDUs and lifting them into the EDUInfos map.
3286 std::deque<JITDylib::EmissionDepUnit *> Worklist;
3287 for (auto &[EDU, _] : EDUInfos)
3288 Worklist.push_back(EDU);
3289
3290 for (auto *EDU : Worklist) {
3291 auto *EDUInfo = &EDUInfos[EDU];
3292
3293 SmallVector<JITDylib *> DepJDsToRemove;
3294 for (auto &[DepJD, Deps] : EDU->Dependencies) {
3295 if (DepJD->State != JITDylib::Open)
3296 return makeJDClosedError(*EDU, *DepJD);
3297
3298 SymbolNameSet BadDeps;
3299 SmallVector<NonOwningSymbolStringPtr> DepsToRemove;
3300 for (auto &Dep : Deps) {
3301 auto DepEntryItr = DepJD->Symbols.find(SymbolStringPtr(Dep));
3302
3303 // If this dep has been removed or moved to the error state then add it
3304 // to the bad deps set. We aggregate these bad deps for more
3305 // comprehensive error messages.
3306 if (DepEntryItr == DepJD->Symbols.end() ||
3307 DepEntryItr->second.getFlags().hasError()) {
3308 BadDeps.insert(SymbolStringPtr(Dep));
3309 continue;
3310 }
3311
3312 // If this dep isn't emitted yet then just add it to the NewDeps set to
3313 // be propagated.
3314 auto &DepEntry = DepEntryItr->second;
3315 if (DepEntry.getState() < SymbolState::Emitted) {
3316 EDUInfo->NewDeps[DepJD].insert(Dep);
3317 continue;
3318 }
3319
3320 // This dep has been emitted, so add it to the list to be removed from
3321 // EDU.
3322 DepsToRemove.push_back(Dep);
3323
3324 // If Dep is Ready then there's nothing further to do.
3325 if (DepEntry.getState() == SymbolState::Ready) {
3326 assert(!DepJD->MaterializingInfos.count(SymbolStringPtr(Dep)) &&
3327 "Unexpected MaterializationInfo attached to ready symbol");
3328 continue;
3329 }
3330
3331 // If we get here thene Dep is Emitted. We need to look up its defining
3332 // EDU and add this EDU to the defining EDU's list of users (this means
3333 // creating an EDUInfos entry if the defining EDU doesn't have one
3334 // already).
3335 assert(DepJD->MaterializingInfos.count(SymbolStringPtr(Dep)) &&
3336 "Expected MaterializationInfo for emitted dependency");
3337 auto &DepMI = DepJD->MaterializingInfos[SymbolStringPtr(Dep)];
3338 assert(DepMI.DefiningEDU &&
3339 "Emitted symbol does not have a defining EDU");
3340 assert(!DepMI.DefiningEDU->Dependencies.empty() &&
3341 "Emitted symbol has empty dependencies (should be ready)");
3342 assert(DepMI.DependantEDUs.empty() &&
3343 "Already-emitted symbol has dependant EDUs?");
3344 auto &DepEDUInfo = EDUInfos[DepMI.DefiningEDU.get()];
3345 if (!DepEDUInfo.EDU) {
3346 // No EDUInfo yet -- build initial entry, and reset the EDUInfo
3347 // pointer, which we will have invalidated.
3348 EDUInfo = &EDUInfos[EDU];
3349 DepEDUInfo.EDU = DepMI.DefiningEDU;
3350 for (auto &[DepDepJD, DepDeps] : DepEDUInfo.EDU->Dependencies) {
3351 if (DepDepJD == &TargetJD) {
3352 for (auto &DepDep : DepDeps)
3353 if (!MR.getSymbols().count(SymbolStringPtr(DepDep)))
3354 DepEDUInfo.NewDeps[DepDepJD].insert(DepDep);
3355 } else
3356 DepEDUInfo.NewDeps[DepDepJD] = DepDeps;
3357 }
3358 }
3359 DepEDUInfo.IntraEmitUsers.insert(EDU);
3360 }
3361
3362 // Some dependencies were removed or in an error state -- error out.
3363 if (!BadDeps.empty())
3364 return makeUnsatisfiedDepsError(*EDU, *DepJD, std::move(BadDeps));
3365
3366 // Remove the emitted / ready deps from DepJD.
3367 for (auto &Dep : DepsToRemove)
3368 Deps.erase(Dep);
3369
3370 // If there are no further deps in DepJD then flag it for removal too.
3371 if (Deps.empty())
3372 DepJDsToRemove.push_back(DepJD);
3373 }
3374
3375 // Remove any JDs whose dependence sets have become empty.
3376 for (auto &DepJD : DepJDsToRemove) {
3377 assert(EDU->Dependencies.count(DepJD) &&
3378 "Trying to remove non-existent dep entries");
3379 EDU->Dependencies.erase(DepJD);
3380 }
3381
3382 // Now look for users of this EDU.
3383 for (auto &[Sym, Flags] : EDU->Symbols) {
3384 assert(TargetJD.Symbols.count(SymbolStringPtr(Sym)) &&
3385 "Sym not present in symbol table");
3386 assert((TargetJD.Symbols[SymbolStringPtr(Sym)].getState() ==
3388 TargetJD.Symbols[SymbolStringPtr(Sym)]
3389 .getFlags()
3390 .hasMaterializationSideEffectsOnly()) &&
3391 "Emitting symbol not in the resolved state");
3392 assert(!TargetJD.Symbols[SymbolStringPtr(Sym)].getFlags().hasError() &&
3393 "Symbol is already in an error state");
3394
3395 auto MII = TargetJD.MaterializingInfos.find(SymbolStringPtr(Sym));
3396 if (MII == TargetJD.MaterializingInfos.end() ||
3397 MII->second.DependantEDUs.empty())
3398 continue;
3399
3400 for (auto &DependantEDU : MII->second.DependantEDUs) {
3401 if (IL_removeEDUDependence(*DependantEDU, TargetJD, Sym, EDUInfos))
3402 EDUInfo = &EDUInfos[EDU];
3403 EDUInfo->IntraEmitUsers.insert(DependantEDU);
3404 }
3405 MII->second.DependantEDUs.clear();
3406 }
3407 }
3408
3409 Worklist.clear();
3410 for (auto &[EDU, EDUInfo] : EDUInfos) {
3411 if (!EDUInfo.IntraEmitUsers.empty() && !EDU->Dependencies.empty()) {
3412 if (EDUInfo.NewDeps.empty())
3413 EDUInfo.NewDeps = EDU->Dependencies;
3414 Worklist.push_back(EDU);
3415 }
3416 }
3417
3418 propagateExtraEmitDeps(
3419 Worklist, EDUInfos,
3420 [](JITDylib::EmissionDepUnit &EDU, JITDylib &JD,
3421 NonOwningSymbolStringPtr Sym) {
3422 JD.MaterializingInfos[SymbolStringPtr(Sym)].DependantEDUs.insert(&EDU);
3423 });
3424
3425 JITDylib::AsynchronousSymbolQuerySet CompletedQueries;
3426
3427 // Extract completed queries and lodge not-yet-ready EDUs in the
3428 // session.
3429 for (auto &[EDU, EDUInfo] : EDUInfos) {
3430 if (EDU->Dependencies.empty())
3431 IL_makeEDUReady(std::move(EDUInfo.EDU), CompletedQueries);
3432 else
3433 IL_makeEDUEmitted(std::move(EDUInfo.EDU), CompletedQueries);
3434 }
3435
3436#ifdef EXPENSIVE_CHECKS
3437 verifySessionState("exiting ExecutionSession::IL_emit");
3438#endif
3439
3440 return std::move(CompletedQueries);
3441}
3442
3443Error ExecutionSession::OL_notifyEmitted(
3444 MaterializationResponsibility &MR,
3445 ArrayRef<SymbolDependenceGroup> DepGroups) {
3446 LLVM_DEBUG({
3447 dbgs() << "In " << MR.JD.getName() << " emitting " << MR.SymbolFlags
3448 << "\n";
3449 if (!DepGroups.empty()) {
3450 dbgs() << " Initial dependencies:\n";
3451 for (auto &SDG : DepGroups) {
3452 dbgs() << " Symbols: " << SDG.Symbols
3453 << ", Dependencies: " << SDG.Dependencies << "\n";
3454 }
3455 }
3456 });
3457
3458#ifndef NDEBUG
3459 SymbolNameSet Visited;
3460 for (auto &DG : DepGroups) {
3461 for (auto &Sym : DG.Symbols) {
3462 assert(MR.SymbolFlags.count(Sym) &&
3463 "DG contains dependence for symbol outside this MR");
3464 assert(Visited.insert(Sym).second &&
3465 "DG contains duplicate entries for Name");
3466 }
3467 }
3468#endif // NDEBUG
3469
3470 auto EDUInfos = simplifyDepGroups(MR, DepGroups);
3471
3472 LLVM_DEBUG({
3473 dbgs() << " Simplified dependencies:\n";
3474 for (auto &[EDU, EDUInfo] : EDUInfos) {
3475 dbgs() << " Symbols: { ";
3476 for (auto &[Sym, Flags] : EDU->Symbols)
3477 dbgs() << Sym << " ";
3478 dbgs() << "}, Dependencies: { ";
3479 for (auto &[DepJD, Deps] : EDU->Dependencies) {
3480 dbgs() << "(" << DepJD->getName() << ", { ";
3481 for (auto &Dep : Deps)
3482 dbgs() << Dep << " ";
3483 dbgs() << "}) ";
3484 }
3485 dbgs() << "}\n";
3486 }
3487 });
3488
3489 auto CompletedQueries =
3490 runSessionLocked([&]() { return IL_emit(MR, EDUInfos); });
3491
3492 // On error bail out.
3493 if (!CompletedQueries)
3494 return CompletedQueries.takeError();
3495
3496 MR.SymbolFlags.clear();
3497
3498 // Otherwise notify all the completed queries.
3499 for (auto &Q : *CompletedQueries) {
3500 assert(Q->isComplete() && "Q is not complete");
3501 Q->handleComplete(*this);
3502 }
3503
3504 return Error::success();
3505}
3506
3507Error ExecutionSession::OL_defineMaterializing(
3508 MaterializationResponsibility &MR, SymbolFlagsMap NewSymbolFlags) {
3509
3510 LLVM_DEBUG({
3511 dbgs() << "In " << MR.JD.getName() << " defining materializing symbols "
3512 << NewSymbolFlags << "\n";
3513 });
3514 if (auto AcceptedDefs =
3515 MR.JD.defineMaterializing(MR, std::move(NewSymbolFlags))) {
3516 // Add all newly accepted symbols to this responsibility object.
3517 for (auto &KV : *AcceptedDefs)
3518 MR.SymbolFlags.insert(KV);
3519 return Error::success();
3520 } else
3521 return AcceptedDefs.takeError();
3522}
3523
3524std::pair<JITDylib::AsynchronousSymbolQuerySet,
3525 std::shared_ptr<SymbolDependenceMap>>
3526ExecutionSession::IL_failSymbols(JITDylib &JD,
3527 const SymbolNameVector &SymbolsToFail) {
3528
3529#ifdef EXPENSIVE_CHECKS
3530 verifySessionState("entering ExecutionSession::IL_failSymbols");
3531#endif
3532
3533 JITDylib::AsynchronousSymbolQuerySet FailedQueries;
3534 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
3535 auto ExtractFailedQueries = [&](JITDylib::MaterializingInfo &MI) {
3536 JITDylib::AsynchronousSymbolQueryList ToDetach;
3537 for (auto &Q : MI.pendingQueries()) {
3538 // Add the query to the list to be failed and detach it.
3539 FailedQueries.insert(Q);
3540 ToDetach.push_back(Q);
3541 }
3542 for (auto &Q : ToDetach)
3543 Q->detach();
3544 assert(!MI.hasQueriesPending() && "Queries still pending after detach");
3545 };
3546
3547 for (auto &Name : SymbolsToFail) {
3548 (*FailedSymbolsMap)[&JD].insert(Name);
3549
3550 // Look up the symbol to fail.
3551 auto SymI = JD.Symbols.find(Name);
3552
3553 // FIXME: Revisit this. We should be able to assert sequencing between
3554 // ResourceTracker removal and symbol failure.
3555 //
3556 // It's possible that this symbol has already been removed, e.g. if a
3557 // materialization failure happens concurrently with a ResourceTracker or
3558 // JITDylib removal. In that case we can safely skip this symbol and
3559 // continue.
3560 if (SymI == JD.Symbols.end())
3561 continue;
3562 auto &Sym = SymI->second;
3563
3564 // If the symbol is already in the error state then we must have visited
3565 // it earlier.
3566 if (Sym.getFlags().hasError()) {
3567 assert(!JD.MaterializingInfos.count(Name) &&
3568 "Symbol in error state still has MaterializingInfo");
3569 continue;
3570 }
3571
3572 // Move the symbol into the error state.
3573 Sym.setFlags(Sym.getFlags() | JITSymbolFlags::HasError);
3574
3575 // FIXME: Come up with a sane mapping of state to
3576 // presence-of-MaterializingInfo so that we can assert presence / absence
3577 // here, rather than testing it.
3578 auto MII = JD.MaterializingInfos.find(Name);
3579 if (MII == JD.MaterializingInfos.end())
3580 continue;
3581
3582 auto &MI = MII->second;
3583
3584 // Collect queries to be failed for this MII.
3585 ExtractFailedQueries(MI);
3586
3587 if (MI.DefiningEDU) {
3588 // If there is a DefiningEDU for this symbol then remove this
3589 // symbol from it.
3590 assert(MI.DependantEDUs.empty() &&
3591 "Symbol with DefiningEDU should not have DependantEDUs");
3592 assert(Sym.getState() >= SymbolState::Emitted &&
3593 "Symbol has EDU, should have been emitted");
3594 assert(MI.DefiningEDU->Symbols.count(NonOwningSymbolStringPtr(Name)) &&
3595 "Symbol does not appear in its DefiningEDU");
3596 MI.DefiningEDU->Symbols.erase(NonOwningSymbolStringPtr(Name));
3597 MI.DefiningEDU = nullptr;
3598 } else {
3599 // Otherwise if there are any EDUs waiting on this symbol then move
3600 // those symbols to the error state too, and deregister them from the
3601 // symbols that they depend on.
3602 // Note: We use a copy of DependantEDUs here since we'll be removing
3603 // from the original set as we go.
3604 for (auto &DependantEDU : MI.DependantEDUs) {
3605
3606 // Remove DependantEDU from all of its users DependantEDUs lists.
3607 for (auto &[DepJD, DepSyms] : DependantEDU->Dependencies) {
3608 for (auto DepSym : DepSyms) {
3609 // Skip self-reference to avoid invalidating the MI.DependantEDUs
3610 // map. We'll clear this later.
3611 if (DepJD == &JD && DepSym == Name)
3612 continue;
3613 assert(DepJD->Symbols.count(SymbolStringPtr(DepSym)) &&
3614 "DepSym not in DepJD?");
3615 assert(DepJD->MaterializingInfos.count(SymbolStringPtr(DepSym)) &&
3616 "DependantEDU not registered with symbol it depends on");
3617 auto &SymMI = DepJD->MaterializingInfos[SymbolStringPtr(DepSym)];
3618 assert(SymMI.DependantEDUs.count(DependantEDU) &&
3619 "DependantEDU missing from DependantEDUs list");
3620 SymMI.DependantEDUs.erase(DependantEDU);
3621 }
3622 }
3623
3624 // Move any symbols defined by DependantEDU into the error state and
3625 // fail any queries waiting on them.
3626 auto &DepJD = *DependantEDU->JD;
3627 auto DepEDUSymbols = std::move(DependantEDU->Symbols);
3628 for (auto &[DepName, Flags] : DepEDUSymbols) {
3629 auto DepSymItr = DepJD.Symbols.find(SymbolStringPtr(DepName));
3630 assert(DepSymItr != DepJD.Symbols.end() &&
3631 "Symbol not present in table");
3632 auto &DepSym = DepSymItr->second;
3633
3634 assert(DepSym.getState() >= SymbolState::Emitted &&
3635 "Symbol has EDU, should have been emitted");
3636 assert(!DepSym.getFlags().hasError() &&
3637 "Symbol is already in the error state?");
3638 DepSym.setFlags(DepSym.getFlags() | JITSymbolFlags::HasError);
3639 (*FailedSymbolsMap)[&DepJD].insert(SymbolStringPtr(DepName));
3640
3641 // This symbol has a defining EDU so its MaterializingInfo object must
3642 // exist.
3643 auto DepMIItr =
3644 DepJD.MaterializingInfos.find(SymbolStringPtr(DepName));
3645 assert(DepMIItr != DepJD.MaterializingInfos.end() &&
3646 "Symbol has defining EDU but not MaterializingInfo");
3647 auto &DepMI = DepMIItr->second;
3648 assert(DepMI.DefiningEDU.get() == DependantEDU &&
3649 "Bad EDU dependence edge");
3650 assert(DepMI.DependantEDUs.empty() &&
3651 "Symbol was emitted, should not have any DependantEDUs");
3652 ExtractFailedQueries(DepMI);
3653 DepJD.MaterializingInfos.erase(SymbolStringPtr(DepName));
3654 }
3655
3656 DepJD.shrinkMaterializationInfoMemory();
3657 }
3658
3659 MI.DependantEDUs.clear();
3660 }
3661
3662 assert(!MI.DefiningEDU && "DefiningEDU should have been reset");
3663 assert(MI.DependantEDUs.empty() &&
3664 "DependantEDUs should have been removed above");
3665 assert(!MI.hasQueriesPending() &&
3666 "Can not delete MaterializingInfo with queries pending");
3667 JD.MaterializingInfos.erase(Name);
3668 }
3669
3670 JD.shrinkMaterializationInfoMemory();
3671
3672#ifdef EXPENSIVE_CHECKS
3673 verifySessionState("exiting ExecutionSession::IL_failSymbols");
3674#endif
3675
3676 return std::make_pair(std::move(FailedQueries), std::move(FailedSymbolsMap));
3677}
3678
3679void ExecutionSession::OL_notifyFailed(MaterializationResponsibility &MR) {
3680
3681 LLVM_DEBUG({
3682 dbgs() << "In " << MR.JD.getName() << " failing materialization for "
3683 << MR.SymbolFlags << "\n";
3684 });
3685
3686 if (MR.SymbolFlags.empty())
3687 return;
3688
3689 SymbolNameVector SymbolsToFail;
3690 for (auto &[Name, Flags] : MR.SymbolFlags)
3691 SymbolsToFail.push_back(Name);
3692 MR.SymbolFlags.clear();
3693
3694 JITDylib::AsynchronousSymbolQuerySet FailedQueries;
3695 std::shared_ptr<SymbolDependenceMap> FailedSymbols;
3696
3697 std::tie(FailedQueries, FailedSymbols) = runSessionLocked([&]() {
3698 // If the tracker is defunct then there's nothing to do here.
3699 if (MR.RT->isDefunct())
3700 return std::pair<JITDylib::AsynchronousSymbolQuerySet,
3701 std::shared_ptr<SymbolDependenceMap>>();
3702 return IL_failSymbols(MR.getTargetJITDylib(), SymbolsToFail);
3703 });
3704
3705 for (auto &Q : FailedQueries)
3706 Q->handleFailed(
3707 make_error<FailedToMaterialize>(getSymbolStringPool(), FailedSymbols));
3708}
3709
3710Error ExecutionSession::OL_replace(MaterializationResponsibility &MR,
3711 std::unique_ptr<MaterializationUnit> MU) {
3712 for (auto &KV : MU->getSymbols()) {
3713 assert(MR.SymbolFlags.count(KV.first) &&
3714 "Replacing definition outside this responsibility set");
3715 MR.SymbolFlags.erase(KV.first);
3716 }
3717
3718 if (MU->getInitializerSymbol() == MR.InitSymbol)
3719 MR.InitSymbol = nullptr;
3720
3721 LLVM_DEBUG(MR.JD.getExecutionSession().runSessionLocked([&]() {
3722 dbgs() << "In " << MR.JD.getName() << " replacing symbols with " << *MU
3723 << "\n";
3724 }););
3725
3726 return MR.JD.replace(MR, std::move(MU));
3727}
3728
3729Expected<std::unique_ptr<MaterializationResponsibility>>
3730ExecutionSession::OL_delegate(MaterializationResponsibility &MR,
3731 const SymbolNameSet &Symbols) {
3732
3733 SymbolStringPtr DelegatedInitSymbol;
3734 SymbolFlagsMap DelegatedFlags;
3735
3736 for (auto &Name : Symbols) {
3737 auto I = MR.SymbolFlags.find(Name);
3738 assert(I != MR.SymbolFlags.end() &&
3739 "Symbol is not tracked by this MaterializationResponsibility "
3740 "instance");
3741
3742 DelegatedFlags[Name] = std::move(I->second);
3743 if (Name == MR.InitSymbol)
3744 std::swap(MR.InitSymbol, DelegatedInitSymbol);
3745
3746 MR.SymbolFlags.erase(I);
3747 }
3748
3749 return MR.JD.delegate(MR, std::move(DelegatedFlags),
3750 std::move(DelegatedInitSymbol));
3751}
3752
3753#ifndef NDEBUG
3754void ExecutionSession::dumpDispatchInfo(Task &T) {
3755 runSessionLocked([&]() {
3756 dbgs() << "Dispatching: ";
3757 T.printDescription(dbgs());
3758 dbgs() << "\n";
3759 });
3760}
3761#endif // NDEBUG
3762
3763} // End namespace orc.
3764} // End namespace llvm.
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
aarch64 falkor hwpf fix Falkor HW Prefetch Fix Late Phase
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Addr
std::string Name
Symbol * Sym
Definition: ELF_riscv.cpp:479
#define _
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
#define H(x, y, z)
Definition: MD5.cpp:57
while(!ToSimplify.empty())
if(VerifyEach)
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
raw_pwrite_stream & OS
static uint32_t getFlags(const Symbol *Sym)
Definition: TapiFile.cpp:27
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
const T & front() const
front - Get the first element.
Definition: ArrayRef.h:168
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
const T * data() const
Definition: ArrayRef.h:162
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
bool erase(const KeyT &Val)
Definition: DenseMap.h:345
unsigned size() const
Definition: DenseMap.h:99
bool empty() const
Definition: DenseMap.h:98
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
iterator end()
Definition: DenseMap.h:84
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:220
Helper for Errors used as out-parameters.
Definition: Error.h:1130
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:337
Tagged union holding either a T or a Error.
Definition: Error.h:481
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Symbol info for RuntimeDyld.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
iterator find(const_arg_type_t< ValueT > V)
Definition: DenseSet.h:179
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:97
StringRef getName() const override
Return the name of this materialization unit.
Definition: Core.cpp:283
AbsoluteSymbolsMaterializationUnit(SymbolMap Symbols)
Definition: Core.cpp:279
AsynchronousSymbolQuery(const SymbolLookupSet &Symbols, SymbolState RequiredState, SymbolsResolvedCallback NotifyComplete)
Create a query for the given symbols.
Definition: Core.cpp:180
void notifySymbolMetRequiredState(const SymbolStringPtr &Name, ExecutorSymbolDef Sym)
Notify the query that a requested symbol has reached the required state.
Definition: Core.cpp:194
Definition generators can be attached to JITDylibs to generate new definitions for otherwise unresolv...
Definition: Core.h:946
An ExecutionSession represents a running JIT program.
Definition: Core.h:1431
Error endSession()
End the session.
Definition: Core.cpp:1619
void reportError(Error Err)
Report a error for this execution session.
Definition: Core.h:1566
friend class JITDylib
Definition: Core.h:1434
void lookupFlags(LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet Symbols, unique_function< void(Expected< SymbolFlagsMap >)> OnComplete)
Search the given JITDylibs to find the flags associated with each of the given symbols.
Definition: Core.cpp:1779
SymbolStringPtr intern(StringRef SymName)
Add a symbol name to the SymbolStringPool and return a pointer to it.
Definition: Core.h:1485
JITDylib * getJITDylibByName(StringRef Name)
Return a pointer to the "name" JITDylib.
Definition: Core.cpp:1658
friend class LookupState
Definition: Core.h:1435
JITDylib & createBareJITDylib(std::string Name)
Add a new bare JITDylib to this ExecutionSession.
Definition: Core.cpp:1667
std::shared_ptr< SymbolStringPool > getSymbolStringPool()
Get the SymbolStringPool for this instance.
Definition: Core.h:1480
void lookup(LookupKind K, const JITDylibSearchOrder &SearchOrder, SymbolLookupSet Symbols, SymbolState RequiredState, SymbolsResolvedCallback NotifyComplete, RegisterDependenciesFunction RegisterDependencies)
Search the given JITDylibs for the given symbols.
Definition: Core.cpp:1805
Error registerJITDispatchHandlers(JITDylib &JD, JITDispatchHandlerAssociationMap WFs)
For each tag symbol name, associate the corresponding AsyncHandlerWrapperFunction with the address of...
Definition: Core.cpp:1914
void registerResourceManager(ResourceManager &RM)
Register the given ResourceManager with this ExecutionSession.
Definition: Core.cpp:1641
~ExecutionSession()
Destroy an ExecutionSession.
Definition: Core.cpp:1613
void runJITDispatchHandler(SendResultFunction SendResult, ExecutorAddr HandlerFnTagAddr, ArrayRef< char > ArgBuffer)
Run a registered jit-side wrapper function.
Definition: Core.cpp:1945
void deregisterResourceManager(ResourceManager &RM)
Deregister the given ResourceManager with this ExecutionSession.
Definition: Core.cpp:1645
ExecutionSession(std::unique_ptr< ExecutorProcessControl > EPC)
Construct an ExecutionSession with the given ExecutorProcessControl object.
Definition: Core.cpp:1607
decltype(auto) runSessionLocked(Func &&F)
Run the given lambda with the session mutex locked.
Definition: Core.h:1495
void dump(raw_ostream &OS)
Dump the state of all the JITDylibs in this session.
Definition: Core.cpp:1966
Error removeJITDylibs(std::vector< JITDylibSP > JDsToRemove)
Removes the given JITDylibs from the ExecutionSession.
Definition: Core.cpp:1684
Expected< JITDylib & > createJITDylib(std::string Name)
Add a new JITDylib to this ExecutionSession.
Definition: Core.cpp:1676
void dispatchTask(std::unique_ptr< Task > T)
Materialize the given unit.
Definition: Core.h:1639
Represents an address in the executor process.
Represents a defining location for a JIT symbol.
FailedToMaterialize(std::shared_ptr< SymbolStringPool > SSP, std::shared_ptr< SymbolDependenceMap > Symbols)
Definition: Core.cpp:81
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:99
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:103
InProgressFullLookupState(LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet, SymbolState RequiredState, std::shared_ptr< AsynchronousSymbolQuery > Q, RegisterDependenciesFunction RegisterDependencies)
Definition: Core.cpp:585
void complete(std::unique_ptr< InProgressLookupState > IPLS) override
Definition: Core.cpp:595
void fail(Error Err) override
Definition: Core.cpp:601
void complete(std::unique_ptr< InProgressLookupState > IPLS) override
Definition: Core.cpp:572
void fail(Error Err) override
Definition: Core.cpp:577
InProgressLookupFlagsState(LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet, unique_function< void(Expected< SymbolFlagsMap >)> OnComplete)
Definition: Core.cpp:565
virtual ~InProgressLookupState()=default
SymbolLookupSet DefGeneratorCandidates
Definition: Core.cpp:552
JITDylibSearchOrder SearchOrder
Definition: Core.cpp:546
std::vector< std::weak_ptr< DefinitionGenerator > > CurDefGeneratorStack
Definition: Core.cpp:560
SymbolLookupSet LookupSet
Definition: Core.cpp:547
virtual void complete(std::unique_ptr< InProgressLookupState > IPLS)=0
InProgressLookupState(LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet, SymbolState RequiredState)
Definition: Core.cpp:535
SymbolLookupSet DefGeneratorNonCandidates
Definition: Core.cpp:553
virtual void fail(Error Err)=0
enum llvm::orc::InProgressLookupState::@471 GenState
Represents a JIT'd dynamic library.
Definition: Core.h:989
Error clear()
Calls remove on all trackers currently associated with this JITDylib.
Definition: Core.cpp:676
Error define(std::unique_ptr< MaterializationUnitType > &&MU, ResourceTrackerSP RT=nullptr)
Define all symbols provided by the materialization unit to be part of this JITDylib.
Definition: Core.h:1910
ExecutionSession & getExecutionSession() const
Get a reference to the ExecutionSession for this JITDylib.
Definition: Core.h:1008
ResourceTrackerSP createResourceTracker()
Create a resource tracker for this JITDylib.
Definition: Core.cpp:700
Expected< std::vector< JITDylibSP > > getReverseDFSLinkOrder()
Rteurn this JITDylib and its transitive dependencies in reverse DFS order based on linkage relationsh...
Definition: Core.cpp:1775
ResourceTrackerSP getDefaultResourceTracker()
Get the default resource tracker for this JITDylib.
Definition: Core.cpp:691
void removeGenerator(DefinitionGenerator &G)
Remove a definition generator from this JITDylib.
Definition: Core.cpp:708
Expected< std::vector< JITDylibSP > > getDFSLinkOrder()
Return this JITDylib and its transitive dependencies in DFS order based on linkage relationships.
Definition: Core.cpp:1771
Wraps state for a lookup-in-progress.
Definition: Core.h:921
void continueLookup(Error Err)
Continue the lookup.
Definition: Core.cpp:652
LookupState & operator=(LookupState &&)
void run() override
Definition: Core.cpp:1605
static char ID
Definition: Core.h:1420
void printDescription(raw_ostream &OS) override
Definition: Core.cpp:1603
Tracks responsibility for materialization, and mediates interactions between MaterializationUnits and...
Definition: Core.h:555
void printDescription(raw_ostream &OS) override
Definition: Core.cpp:1596
A MaterializationUnit represents a set of symbol definitions that can be materialized as a group,...
Definition: Core.h:693
SymbolFlagsMap SymbolFlags
Definition: Core.h:749
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:162
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:166
Non-owning SymbolStringPool entry pointer.
static void lookupInitSymbolsAsync(unique_function< void(Error)> OnComplete, ExecutionSession &ES, const DenseMap< JITDylib *, SymbolLookupSet > &InitSyms)
Performs an async lookup for the given symbols in each of the given JITDylibs, calling the given hand...
Definition: Core.cpp:1553
static Expected< DenseMap< JITDylib *, SymbolMap > > lookupInitSymbols(ExecutionSession &ES, const DenseMap< JITDylib *, SymbolLookupSet > &InitSyms)
A utility function for looking up initializer symbols.
Definition: Core.cpp:1504
StringRef getName() const override
Return the name of this materialization unit.
Definition: Core.cpp:326
ReExportsMaterializationUnit(JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags, SymbolAliasMap Aliases)
SourceJD is allowed to be nullptr, in which case the source JITDylib is taken to be whatever JITDylib...
Definition: Core.cpp:320
std::function< bool(SymbolStringPtr)> SymbolPredicate
Definition: Core.h:1999
Error tryToGenerate(LookupState &LS, LookupKind K, JITDylib &JD, JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &LookupSet) override
DefinitionGenerators should override this method to insert new definitions into the parent JITDylib.
Definition: Core.cpp:617
ReexportsGenerator(JITDylib &SourceJD, JITDylibLookupFlags SourceJDLookupFlags, SymbolPredicate Allow=SymbolPredicate())
Create a reexports generator.
Definition: Core.cpp:611
Listens for ResourceTracker operations.
Definition: Core.h:104
ResourceTrackerDefunct(ResourceTrackerSP RT)
Definition: Core.cpp:70
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:77
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:73
API to remove / transfer ownership of JIT resources.
Definition: Core.h:56
JITDylib & getJITDylib() const
Return the JITDylib targeted by this tracker.
Definition: Core.h:71
void transferTo(ResourceTracker &DstRT)
Transfer all resources associated with this key to the given tracker, which must target the same JITD...
Definition: Core.cpp:58
ResourceTracker(const ResourceTracker &)=delete
Error remove()
Remove all resources associated with this key.
Definition: Core.cpp:54
A set of symbols to look up, each associated with a SymbolLookupFlags value.
Definition: Core.h:183
static SymbolLookupSet fromMapKeys(const DenseMap< SymbolStringPtr, ValT > &M, SymbolLookupFlags Flags=SymbolLookupFlags::RequiredSymbol)
Construct a SymbolLookupSet from DenseMap keys.
Definition: Core.h:232
Pointer to a pooled string representing a symbol name.
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:154
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:158
SymbolsCouldNotBeRemoved(std::shared_ptr< SymbolStringPool > SSP, SymbolNameSet Symbols)
Definition: Core.cpp:148
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:144
SymbolsNotFound(std::shared_ptr< SymbolStringPool > SSP, SymbolNameSet Symbols)
Definition: Core.cpp:126
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:140
Represents an abstract task for ORC to run.
Definition: TaskDispatch.h:35
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:171
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:175
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:119
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:115
UnsatisfiedSymbolDependencies(std::shared_ptr< SymbolStringPool > SSP, JITDylibSP JD, SymbolNameSet FailedSymbols, SymbolDependenceMap BadDeps, std::string Explanation)
Definition: Core.cpp:107
static WrapperFunctionResult createOutOfBandError(const char *Msg)
Create an out-of-band error by copying the given string.
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:471
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
@ Entry
Definition: COFF.h:811
IntrusiveRefCntPtr< JITDylib > JITDylibSP
Definition: Core.h:51
JITDylibSearchOrder makeJITDylibSearchOrder(ArrayRef< JITDylib * > JDs, JITDylibLookupFlags Flags=JITDylibLookupFlags::MatchExportedSymbolsOnly)
Convenience function for creating a search order from an ArrayRef of JITDylib*, all with the same fla...
Definition: Core.h:166
std::vector< std::pair< JITDylib *, JITDylibLookupFlags > > JITDylibSearchOrder
A list of (JITDylib*, JITDylibLookupFlags) pairs to be used as a search order during symbol lookup.
Definition: Core.h:162
std::function< void(const SymbolDependenceMap &)> RegisterDependenciesFunction
Callback to register the dependencies for a given query.
Definition: Core.h:403
std::unique_ptr< ReExportsMaterializationUnit > symbolAliases(SymbolAliasMap Aliases)
Create a ReExportsMaterializationUnit with the given aliases.
Definition: Core.h:837
SymbolLookupFlags
Lookup flags that apply to each symbol in a lookup.
Definition: Core.h:145
std::unique_ptr< ReExportsMaterializationUnit > reexports(JITDylib &SourceJD, SymbolAliasMap Aliases, JITDylibLookupFlags SourceJDLookupFlags=JITDylibLookupFlags::MatchExportedSymbolsOnly)
Create a materialization unit for re-exporting symbols from another JITDylib with alternative names/f...
Definition: Core.h:846
DenseMap< SymbolStringPtr, SymbolAliasMapEntry > SymbolAliasMap
A map of Symbols to (Symbol, Flags) pairs.
Definition: Core.h:396
JITDylibLookupFlags
Lookup flags that apply to each dylib in the search order for a lookup.
Definition: Core.h:135
DenseMap< SymbolStringPtr, ExecutorSymbolDef > SymbolMap
A map from symbol names (as SymbolStringPtrs) to JITSymbols (address/flags pairs).
Definition: Core.h:121
DenseMap< SymbolStringPtr, JITSymbolFlags > SymbolFlagsMap
A map from symbol names (as SymbolStringPtrs) to JITSymbolFlags.
Definition: Core.h:124
unique_function< void(Expected< SymbolMap >)> SymbolsResolvedCallback
Callback to notify client that symbols have been resolved.
Definition: Core.h:399
DenseSet< SymbolStringPtr > SymbolNameSet
A set of symbol names (represented by SymbolStringPtrs for.
Definition: Core.h:114
LookupKind
Describes the kind of lookup being performed.
Definition: Core.h:157
RegisterDependenciesFunction NoDependenciesToRegister
This can be used as the value for a RegisterDependenciesFunction if there are no dependants to regist...
Definition: Core.cpp:37
std::vector< SymbolStringPtr > SymbolNameVector
A vector of symbol names.
Definition: Core.h:117
SymbolState
Represents the state that a symbol has reached during materialization.
Definition: Core.h:859
@ Materializing
Added to the symbol table, never queried.
@ NeverSearched
No symbol should be in this state.
@ Ready
Emitted to memory, but waiting on transitive dependencies.
@ Emitted
Assigned address, still materializing.
@ Resolved
Queried, materialization begun.
std::error_code orcError(OrcErrorCode ErrCode)
Definition: OrcError.cpp:84
DenseMap< JITDylib *, SymbolNameSet > SymbolDependenceMap
A map from JITDylibs to sets of symbols.
Definition: Core.h:127
Expected< SymbolAliasMap > buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols)
Build a SymbolAliasMap for the common case where you want to re-export symbols from another JITDylib ...
NodeAddr< DefNode * > Def
Definition: RDFGraph.h:384
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1742
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:98
auto formatv(const char *Fmt, Ts &&...Vals) -> formatv_object< decltype(std::make_tuple(support::detail::build_format_adapter(std::forward< Ts >(Vals))...))>
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition: STLExtras.h:2067
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:419
Error joinErrors(Error E1, Error E2)
Concatenate errors.
Definition: Error.h:438
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1647
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
Definition: Format.h:125
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1954
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1849
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1749
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition: STLExtras.h:1879
Implement std::hash so that hash_code can be used in STL containers.
Definition: BitVector.h:858
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define NDEBUG
Definition: regutils.h:48