LLVM 17.0.0git
LTO.cpp
Go to the documentation of this file.
1//===-LTO.cpp - LLVM Link Time Optimizer ----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements functions and classes used to support LTO.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/LTO/LTO.h"
14#include "llvm/ADT/ScopeExit.h"
15#include "llvm/ADT/SmallSet.h"
16#include "llvm/ADT/Statistic.h"
25#include "llvm/Config/llvm-config.h"
26#include "llvm/IR/AutoUpgrade.h"
28#include "llvm/IR/Intrinsics.h"
31#include "llvm/IR/Mangler.h"
32#include "llvm/IR/Metadata.h"
33#include "llvm/LTO/LTOBackend.h"
35#include "llvm/Linker/IRMover.h"
39#include "llvm/Support/Error.h"
43#include "llvm/Support/Path.h"
44#include "llvm/Support/SHA1.h"
50#include "llvm/Support/VCSRevision.h"
53#include "llvm/Transforms/IPO.h"
58
59#include <optional>
60#include <set>
61
62using namespace llvm;
63using namespace lto;
64using namespace object;
65
66#define DEBUG_TYPE "lto"
67
68static cl::opt<bool>
69 DumpThinCGSCCs("dump-thin-cg-sccs", cl::init(false), cl::Hidden,
70 cl::desc("Dump the SCCs in the ThinLTO index's callgraph"));
71
72namespace llvm {
73/// Enable global value internalization in LTO.
75 "enable-lto-internalization", cl::init(true), cl::Hidden,
76 cl::desc("Enable global value internalization in LTO"));
77}
78
79/// Indicate we are linking with an allocator that supports hot/cold operator
80/// new interfaces.
82
83/// Enable MemProf context disambiguation for thin link.
85
86// Computes a unique hash for the Module considering the current list of
87// export/import and other global analysis results.
88// The hash is produced in \p Key.
90 SmallString<40> &Key, const Config &Conf, const ModuleSummaryIndex &Index,
91 StringRef ModuleID, const FunctionImporter::ImportMapTy &ImportList,
92 const FunctionImporter::ExportSetTy &ExportList,
93 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
94 const GVSummaryMapTy &DefinedGlobals,
95 const std::set<GlobalValue::GUID> &CfiFunctionDefs,
96 const std::set<GlobalValue::GUID> &CfiFunctionDecls) {
97 // Compute the unique hash for this entry.
98 // This is based on the current compiler version, the module itself, the
99 // export list, the hash for every single module in the import list, the
100 // list of ResolvedODR for the module, and the list of preserved symbols.
101 SHA1 Hasher;
102
103 // Start with the compiler revision
104 Hasher.update(LLVM_VERSION_STRING);
105#ifdef LLVM_REVISION
106 Hasher.update(LLVM_REVISION);
107#endif
108
109 // Include the parts of the LTO configuration that affect code generation.
110 auto AddString = [&](StringRef Str) {
111 Hasher.update(Str);
112 Hasher.update(ArrayRef<uint8_t>{0});
113 };
114 auto AddUnsigned = [&](unsigned I) {
115 uint8_t Data[4];
117 Hasher.update(ArrayRef<uint8_t>{Data, 4});
118 };
119 auto AddUint64 = [&](uint64_t I) {
120 uint8_t Data[8];
122 Hasher.update(ArrayRef<uint8_t>{Data, 8});
123 };
124 AddString(Conf.CPU);
125 // FIXME: Hash more of Options. For now all clients initialize Options from
126 // command-line flags (which is unsupported in production), but may set
127 // RelaxELFRelocations. The clang driver can also pass FunctionSections,
128 // DataSections and DebuggerTuning via command line flags.
129 AddUnsigned(Conf.Options.RelaxELFRelocations);
130 AddUnsigned(Conf.Options.FunctionSections);
131 AddUnsigned(Conf.Options.DataSections);
132 AddUnsigned((unsigned)Conf.Options.DebuggerTuning);
133 for (auto &A : Conf.MAttrs)
134 AddString(A);
135 if (Conf.RelocModel)
136 AddUnsigned(*Conf.RelocModel);
137 else
138 AddUnsigned(-1);
139 if (Conf.CodeModel)
140 AddUnsigned(*Conf.CodeModel);
141 else
142 AddUnsigned(-1);
143 for (const auto &S : Conf.MllvmArgs)
144 AddString(S);
145 AddUnsigned(Conf.CGOptLevel);
146 AddUnsigned(Conf.CGFileType);
147 AddUnsigned(Conf.OptLevel);
148 AddUnsigned(Conf.Freestanding);
149 AddString(Conf.OptPipeline);
150 AddString(Conf.AAPipeline);
151 AddString(Conf.OverrideTriple);
152 AddString(Conf.DefaultTriple);
153 AddString(Conf.DwoDir);
154
155 // Include the hash for the current module
156 auto ModHash = Index.getModuleHash(ModuleID);
157 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
158
159 std::vector<uint64_t> ExportsGUID;
160 ExportsGUID.reserve(ExportList.size());
161 for (const auto &VI : ExportList) {
162 auto GUID = VI.getGUID();
163 ExportsGUID.push_back(GUID);
164 }
165
166 // Sort the export list elements GUIDs.
167 llvm::sort(ExportsGUID);
168 for (uint64_t GUID : ExportsGUID) {
169 // The export list can impact the internalization, be conservative here
170 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&GUID, sizeof(GUID)));
171 }
172
173 // Include the hash for every module we import functions from. The set of
174 // imported symbols for each module may affect code generation and is
175 // sensitive to link order, so include that as well.
176 using ImportMapIteratorTy = FunctionImporter::ImportMapTy::const_iterator;
177 struct ImportModule {
178 ImportMapIteratorTy ModIt;
179 const ModuleSummaryIndex::ModuleInfo *ModInfo;
180
181 StringRef getIdentifier() const { return ModIt->getKey(); }
182 const FunctionImporter::FunctionsToImportTy &getFunctions() const {
183 return ModIt->second;
184 }
185
186 const ModuleHash &getHash() const { return ModInfo->second.second; }
187 uint64_t getId() const { return ModInfo->second.first; }
188 };
189
190 std::vector<ImportModule> ImportModulesVector;
191 ImportModulesVector.reserve(ImportList.size());
192
193 for (ImportMapIteratorTy It = ImportList.begin(); It != ImportList.end();
194 ++It) {
195 ImportModulesVector.push_back({It, Index.getModule(It->getKey())});
196 }
197 // Order using moduleId integer which is based on the order the module was
198 // added.
199 llvm::sort(ImportModulesVector,
200 [](const ImportModule &Lhs, const ImportModule &Rhs) -> bool {
201 return Lhs.getId() < Rhs.getId();
202 });
203 for (const ImportModule &Entry : ImportModulesVector) {
204 auto ModHash = Entry.getHash();
205 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
206
207 AddUint64(Entry.getFunctions().size());
208 for (auto &Fn : Entry.getFunctions())
209 AddUint64(Fn);
210 }
211
212 // Include the hash for the resolved ODR.
213 for (auto &Entry : ResolvedODR) {
214 Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.first,
215 sizeof(GlobalValue::GUID)));
216 Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.second,
218 }
219
220 // Members of CfiFunctionDefs and CfiFunctionDecls that are referenced or
221 // defined in this module.
222 std::set<GlobalValue::GUID> UsedCfiDefs;
223 std::set<GlobalValue::GUID> UsedCfiDecls;
224
225 // Typeids used in this module.
226 std::set<GlobalValue::GUID> UsedTypeIds;
227
228 auto AddUsedCfiGlobal = [&](GlobalValue::GUID ValueGUID) {
229 if (CfiFunctionDefs.count(ValueGUID))
230 UsedCfiDefs.insert(ValueGUID);
231 if (CfiFunctionDecls.count(ValueGUID))
232 UsedCfiDecls.insert(ValueGUID);
233 };
234
235 auto AddUsedThings = [&](GlobalValueSummary *GS) {
236 if (!GS) return;
237 AddUnsigned(GS->getVisibility());
238 AddUnsigned(GS->isLive());
239 AddUnsigned(GS->canAutoHide());
240 for (const ValueInfo &VI : GS->refs()) {
241 AddUnsigned(VI.isDSOLocal(Index.withDSOLocalPropagation()));
242 AddUsedCfiGlobal(VI.getGUID());
243 }
244 if (auto *GVS = dyn_cast<GlobalVarSummary>(GS)) {
245 AddUnsigned(GVS->maybeReadOnly());
246 AddUnsigned(GVS->maybeWriteOnly());
247 }
248 if (auto *FS = dyn_cast<FunctionSummary>(GS)) {
249 for (auto &TT : FS->type_tests())
250 UsedTypeIds.insert(TT);
251 for (auto &TT : FS->type_test_assume_vcalls())
252 UsedTypeIds.insert(TT.GUID);
253 for (auto &TT : FS->type_checked_load_vcalls())
254 UsedTypeIds.insert(TT.GUID);
255 for (auto &TT : FS->type_test_assume_const_vcalls())
256 UsedTypeIds.insert(TT.VFunc.GUID);
257 for (auto &TT : FS->type_checked_load_const_vcalls())
258 UsedTypeIds.insert(TT.VFunc.GUID);
259 for (auto &ET : FS->calls()) {
260 AddUnsigned(ET.first.isDSOLocal(Index.withDSOLocalPropagation()));
261 AddUsedCfiGlobal(ET.first.getGUID());
262 }
263 }
264 };
265
266 // Include the hash for the linkage type to reflect internalization and weak
267 // resolution, and collect any used type identifier resolutions.
268 for (auto &GS : DefinedGlobals) {
269 GlobalValue::LinkageTypes Linkage = GS.second->linkage();
270 Hasher.update(
271 ArrayRef<uint8_t>((const uint8_t *)&Linkage, sizeof(Linkage)));
272 AddUsedCfiGlobal(GS.first);
273 AddUsedThings(GS.second);
274 }
275
276 // Imported functions may introduce new uses of type identifier resolutions,
277 // so we need to collect their used resolutions as well.
278 for (const ImportModule &ImpM : ImportModulesVector)
279 for (auto &ImpF : ImpM.getFunctions()) {
281 Index.findSummaryInModule(ImpF, ImpM.getIdentifier());
282 AddUsedThings(S);
283 // If this is an alias, we also care about any types/etc. that the aliasee
284 // may reference.
285 if (auto *AS = dyn_cast_or_null<AliasSummary>(S))
286 AddUsedThings(AS->getBaseObject());
287 }
288
289 auto AddTypeIdSummary = [&](StringRef TId, const TypeIdSummary &S) {
290 AddString(TId);
291
292 AddUnsigned(S.TTRes.TheKind);
293 AddUnsigned(S.TTRes.SizeM1BitWidth);
294
295 AddUint64(S.TTRes.AlignLog2);
296 AddUint64(S.TTRes.SizeM1);
297 AddUint64(S.TTRes.BitMask);
298 AddUint64(S.TTRes.InlineBits);
299
300 AddUint64(S.WPDRes.size());
301 for (auto &WPD : S.WPDRes) {
302 AddUnsigned(WPD.first);
303 AddUnsigned(WPD.second.TheKind);
304 AddString(WPD.second.SingleImplName);
305
306 AddUint64(WPD.second.ResByArg.size());
307 for (auto &ByArg : WPD.second.ResByArg) {
308 AddUint64(ByArg.first.size());
309 for (uint64_t Arg : ByArg.first)
310 AddUint64(Arg);
311 AddUnsigned(ByArg.second.TheKind);
312 AddUint64(ByArg.second.Info);
313 AddUnsigned(ByArg.second.Byte);
314 AddUnsigned(ByArg.second.Bit);
315 }
316 }
317 };
318
319 // Include the hash for all type identifiers used by this module.
320 for (GlobalValue::GUID TId : UsedTypeIds) {
321 auto TidIter = Index.typeIds().equal_range(TId);
322 for (auto It = TidIter.first; It != TidIter.second; ++It)
323 AddTypeIdSummary(It->second.first, It->second.second);
324 }
325
326 AddUnsigned(UsedCfiDefs.size());
327 for (auto &V : UsedCfiDefs)
328 AddUint64(V);
329
330 AddUnsigned(UsedCfiDecls.size());
331 for (auto &V : UsedCfiDecls)
332 AddUint64(V);
333
334 if (!Conf.SampleProfile.empty()) {
335 auto FileOrErr = MemoryBuffer::getFile(Conf.SampleProfile);
336 if (FileOrErr) {
337 Hasher.update(FileOrErr.get()->getBuffer());
338
339 if (!Conf.ProfileRemapping.empty()) {
340 FileOrErr = MemoryBuffer::getFile(Conf.ProfileRemapping);
341 if (FileOrErr)
342 Hasher.update(FileOrErr.get()->getBuffer());
343 }
344 }
345 }
346
347 Key = toHex(Hasher.result());
348}
349
351 const Config &C, ValueInfo VI,
352 DenseSet<GlobalValueSummary *> &GlobalInvolvedWithAlias,
354 isPrevailing,
356 recordNewLinkage,
357 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
359 C.VisibilityScheme == Config::ELF ? VI.getELFVisibility()
361 for (auto &S : VI.getSummaryList()) {
362 GlobalValue::LinkageTypes OriginalLinkage = S->linkage();
363 // Ignore local and appending linkage values since the linker
364 // doesn't resolve them.
365 if (GlobalValue::isLocalLinkage(OriginalLinkage) ||
367 continue;
368 // We need to emit only one of these. The prevailing module will keep it,
369 // but turned into a weak, while the others will drop it when possible.
370 // This is both a compile-time optimization and a correctness
371 // transformation. This is necessary for correctness when we have exported
372 // a reference - we need to convert the linkonce to weak to
373 // ensure a copy is kept to satisfy the exported reference.
374 // FIXME: We may want to split the compile time and correctness
375 // aspects into separate routines.
376 if (isPrevailing(VI.getGUID(), S.get())) {
377 if (GlobalValue::isLinkOnceLinkage(OriginalLinkage)) {
378 S->setLinkage(GlobalValue::getWeakLinkage(
379 GlobalValue::isLinkOnceODRLinkage(OriginalLinkage)));
380 // The kept copy is eligible for auto-hiding (hidden visibility) if all
381 // copies were (i.e. they were all linkonce_odr global unnamed addr).
382 // If any copy is not (e.g. it was originally weak_odr), then the symbol
383 // must remain externally available (e.g. a weak_odr from an explicitly
384 // instantiated template). Additionally, if it is in the
385 // GUIDPreservedSymbols set, that means that it is visibile outside
386 // the summary (e.g. in a native object or a bitcode file without
387 // summary), and in that case we cannot hide it as it isn't possible to
388 // check all copies.
389 S->setCanAutoHide(VI.canAutoHide() &&
390 !GUIDPreservedSymbols.count(VI.getGUID()));
391 }
392 if (C.VisibilityScheme == Config::FromPrevailing)
393 Visibility = S->getVisibility();
394 }
395 // Alias and aliasee can't be turned into available_externally.
396 else if (!isa<AliasSummary>(S.get()) &&
397 !GlobalInvolvedWithAlias.count(S.get()))
399
400 // For ELF, set visibility to the computed visibility from summaries. We
401 // don't track visibility from declarations so this may be more relaxed than
402 // the most constraining one.
403 if (C.VisibilityScheme == Config::ELF)
404 S->setVisibility(Visibility);
405
406 if (S->linkage() != OriginalLinkage)
407 recordNewLinkage(S->modulePath(), VI.getGUID(), S->linkage());
408 }
409
410 if (C.VisibilityScheme == Config::FromPrevailing) {
411 for (auto &S : VI.getSummaryList()) {
412 GlobalValue::LinkageTypes OriginalLinkage = S->linkage();
413 if (GlobalValue::isLocalLinkage(OriginalLinkage) ||
415 continue;
416 S->setVisibility(Visibility);
417 }
418 }
419}
420
421/// Resolve linkage for prevailing symbols in the \p Index.
422//
423// We'd like to drop these functions if they are no longer referenced in the
424// current module. However there is a chance that another module is still
425// referencing them because of the import. We make sure we always emit at least
426// one copy.
430 isPrevailing,
432 recordNewLinkage,
433 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
434 // We won't optimize the globals that are referenced by an alias for now
435 // Ideally we should turn the alias into a global and duplicate the definition
436 // when needed.
437 DenseSet<GlobalValueSummary *> GlobalInvolvedWithAlias;
438 for (auto &I : Index)
439 for (auto &S : I.second.SummaryList)
440 if (auto AS = dyn_cast<AliasSummary>(S.get()))
441 GlobalInvolvedWithAlias.insert(&AS->getAliasee());
442
443 for (auto &I : Index)
444 thinLTOResolvePrevailingGUID(C, Index.getValueInfo(I),
445 GlobalInvolvedWithAlias, isPrevailing,
446 recordNewLinkage, GUIDPreservedSymbols);
447}
448
450 ValueInfo VI, function_ref<bool(StringRef, ValueInfo)> isExported,
452 isPrevailing) {
453 auto ExternallyVisibleCopies =
454 llvm::count_if(VI.getSummaryList(),
455 [](const std::unique_ptr<GlobalValueSummary> &Summary) {
456 return !GlobalValue::isLocalLinkage(Summary->linkage());
457 });
458
459 for (auto &S : VI.getSummaryList()) {
460 // First see if we need to promote an internal value because it is not
461 // exported.
462 if (isExported(S->modulePath(), VI)) {
463 if (GlobalValue::isLocalLinkage(S->linkage()))
464 S->setLinkage(GlobalValue::ExternalLinkage);
465 continue;
466 }
467
468 // Otherwise, see if we can internalize.
470 continue;
471
472 // Ignore local and appending linkage values since the linker
473 // doesn't resolve them (and there is no need to internalize if this is
474 // already internal).
475 if (GlobalValue::isLocalLinkage(S->linkage()) ||
476 S->linkage() == GlobalValue::AppendingLinkage)
477 continue;
478
479 // We can't internalize available_externally globals because this
480 // can break function pointer equality.
481 if (S->linkage() == GlobalValue::AvailableExternallyLinkage)
482 continue;
483
484 bool IsPrevailing = isPrevailing(VI.getGUID(), S.get());
485
486 if (GlobalValue::isInterposableLinkage(S->linkage()) && !IsPrevailing)
487 continue;
488
489 // Non-exported functions and variables with linkonce_odr or weak_odr
490 // linkage can be internalized in certain cases. The minimum legality
491 // requirements would be that they are not address taken to ensure that we
492 // don't break pointer equality checks, and that variables are either read-
493 // or write-only. For functions, this is the case if either all copies are
494 // [local_]unnamed_addr, or we can propagate reference edge attributes
495 // (which is how this is guaranteed for variables, when analyzing whether
496 // they are read or write-only).
497 //
498 // However, we only get to this code for weak/linkonce ODR values in one of
499 // two cases:
500 // 1) The prevailing copy is not in IR (it is in native code).
501 // 2) The prevailing copy in IR is not exported from its module.
502 // Additionally, at least for the new LTO API, case 2 will only happen if
503 // there is exactly one definition of the value (i.e. in exactly one
504 // module), as duplicate defs are result in the value being marked exported.
505 // Likely, users of the legacy LTO API are similar, however, currently there
506 // are llvm-lto based tests of the legacy LTO API that do not mark
507 // duplicate linkonce_odr copies as exported via the tool, so we need
508 // to handle that case below by checking the number of copies.
509 //
510 // Generally, we only want to internalize a linkonce/weak ODR value in case
511 // 2, because in case 1 we cannot see how the value is used to know if it
512 // is read or write-only. We also don't want to bloat the binary with
513 // multiple internalized copies of non-prevailing linkonce_odr functions.
514 // Note if we don't internalize, we will convert non-prevailing copies to
515 // available_externally anyway, so that we drop them after inlining. The
516 // only reason to internalize such a function is if we indeed have a single
517 // copy, because internalizing it won't increase binary size, and enables
518 // use of inliner heuristics that are more aggressive in the face of a
519 // single call to a static (local). For variables, internalizing a read or
520 // write only variable can enable more aggressive optimization. However, we
521 // already perform this elsewhere in the ThinLTO backend handling for
522 // read or write-only variables (processGlobalForThinLTO).
523 //
524 // Therefore, only internalize linkonce/weak ODR if there is a single copy,
525 // that is prevailing in this IR module. We can do so aggressively, without
526 // requiring the address to be insignificant, or that a variable be read or
527 // write-only.
528 if ((S->linkage() == GlobalValue::WeakODRLinkage ||
529 S->linkage() == GlobalValue::LinkOnceODRLinkage) &&
530 // We can have only one copy in ThinLTO that isn't prevailing, if the
531 // prevailing copy is in a native object.
532 (!IsPrevailing || ExternallyVisibleCopies > 1))
533 continue;
534
535 S->setLinkage(GlobalValue::InternalLinkage);
536 }
537}
538
539// Update the linkages in the given \p Index to mark exported values
540// as external and non-exported values as internal.
543 function_ref<bool(StringRef, ValueInfo)> isExported,
545 isPrevailing) {
546 for (auto &I : Index)
547 thinLTOInternalizeAndPromoteGUID(Index.getValueInfo(I), isExported,
548 isPrevailing);
549}
550
551// Requires a destructor for std::vector<InputModule>.
552InputFile::~InputFile() = default;
553
555 std::unique_ptr<InputFile> File(new InputFile);
556
557 Expected<IRSymtabFile> FOrErr = readIRSymtab(Object);
558 if (!FOrErr)
559 return FOrErr.takeError();
560
561 File->TargetTriple = FOrErr->TheReader.getTargetTriple();
562 File->SourceFileName = FOrErr->TheReader.getSourceFileName();
563 File->COFFLinkerOpts = FOrErr->TheReader.getCOFFLinkerOpts();
564 File->DependentLibraries = FOrErr->TheReader.getDependentLibraries();
565 File->ComdatTable = FOrErr->TheReader.getComdatTable();
566
567 for (unsigned I = 0; I != FOrErr->Mods.size(); ++I) {
568 size_t Begin = File->Symbols.size();
569 for (const irsymtab::Reader::SymbolRef &Sym :
570 FOrErr->TheReader.module_symbols(I))
571 // Skip symbols that are irrelevant to LTO. Note that this condition needs
572 // to match the one in Skip() in LTO::addRegularLTO().
573 if (Sym.isGlobal() && !Sym.isFormatSpecific())
574 File->Symbols.push_back(Sym);
575 File->ModuleSymIndices.push_back({Begin, File->Symbols.size()});
576 }
577
578 File->Mods = FOrErr->Mods;
579 File->Strtab = std::move(FOrErr->Strtab);
580 return std::move(File);
581}
582
584 return Mods[0].getModuleIdentifier();
585}
586
588 assert(Mods.size() == 1 && "Expect only one bitcode module");
589 return Mods[0];
590}
591
592LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel,
593 const Config &Conf)
594 : ParallelCodeGenParallelismLevel(ParallelCodeGenParallelismLevel),
595 Ctx(Conf), CombinedModule(std::make_unique<Module>("ld-temp.o", Ctx)),
596 Mover(std::make_unique<IRMover>(*CombinedModule)) {}
597
598LTO::ThinLTOState::ThinLTOState(ThinBackend Backend)
599 : Backend(Backend), CombinedIndex(/*HaveGVs*/ false) {
600 if (!Backend)
601 this->Backend =
603}
604
606 unsigned ParallelCodeGenParallelismLevel)
607 : Conf(std::move(Conf)),
608 RegularLTO(ParallelCodeGenParallelismLevel, this->Conf),
609 ThinLTO(std::move(Backend)) {}
610
611// Requires a destructor for MapVector<BitcodeModule>.
612LTO::~LTO() = default;
613
614// Add the symbols in the given module to the GlobalResolutions map, and resolve
615// their partitions.
616void LTO::addModuleToGlobalRes(ArrayRef<InputFile::Symbol> Syms,
618 unsigned Partition, bool InSummary) {
619 auto *ResI = Res.begin();
620 auto *ResE = Res.end();
621 (void)ResE;
622 const Triple TT(RegularLTO.CombinedModule->getTargetTriple());
623 for (const InputFile::Symbol &Sym : Syms) {
624 assert(ResI != ResE);
625 SymbolResolution Res = *ResI++;
626
627 StringRef Name = Sym.getName();
628 // Strip the __imp_ prefix from COFF dllimport symbols (similar to the
629 // way they are handled by lld), otherwise we can end up with two
630 // global resolutions (one with and one for a copy of the symbol without).
631 if (TT.isOSBinFormatCOFF() && Name.startswith("__imp_"))
632 Name = Name.substr(strlen("__imp_"));
633 auto &GlobalRes = GlobalResolutions[Name];
634 GlobalRes.UnnamedAddr &= Sym.isUnnamedAddr();
635 if (Res.Prevailing) {
636 assert(!GlobalRes.Prevailing &&
637 "Multiple prevailing defs are not allowed");
638 GlobalRes.Prevailing = true;
639 GlobalRes.IRName = std::string(Sym.getIRName());
640 } else if (!GlobalRes.Prevailing && GlobalRes.IRName.empty()) {
641 // Sometimes it can be two copies of symbol in a module and prevailing
642 // symbol can have no IR name. That might happen if symbol is defined in
643 // module level inline asm block. In case we have multiple modules with
644 // the same symbol we want to use IR name of the prevailing symbol.
645 // Otherwise, if we haven't seen a prevailing symbol, set the name so that
646 // we can later use it to check if there is any prevailing copy in IR.
647 GlobalRes.IRName = std::string(Sym.getIRName());
648 }
649
650 // In rare occasion, the symbol used to initialize GlobalRes has a different
651 // IRName from the inspected Symbol. This can happen on macOS + iOS, when a
652 // symbol is referenced through its mangled name, say @"\01_symbol" while
653 // the IRName is @symbol (the prefix underscore comes from MachO mangling).
654 // In that case, we have the same actual Symbol that can get two different
655 // GUID, leading to some invalid internalization. Workaround this by marking
656 // the GlobalRes external.
657
658 // FIXME: instead of this check, it would be desirable to compute GUIDs
659 // based on mangled name, but this requires an access to the Target Triple
660 // and would be relatively invasive on the codebase.
661 if (GlobalRes.IRName != Sym.getIRName()) {
662 GlobalRes.Partition = GlobalResolution::External;
663 GlobalRes.VisibleOutsideSummary = true;
664 }
665
666 // Set the partition to external if we know it is re-defined by the linker
667 // with -defsym or -wrap options, used elsewhere, e.g. it is visible to a
668 // regular object, is referenced from llvm.compiler.used/llvm.used, or was
669 // already recorded as being referenced from a different partition.
670 if (Res.LinkerRedefined || Res.VisibleToRegularObj || Sym.isUsed() ||
671 (GlobalRes.Partition != GlobalResolution::Unknown &&
672 GlobalRes.Partition != Partition)) {
673 GlobalRes.Partition = GlobalResolution::External;
674 } else
675 // First recorded reference, save the current partition.
676 GlobalRes.Partition = Partition;
677
678 // Flag as visible outside of summary if visible from a regular object or
679 // from a module that does not have a summary.
680 GlobalRes.VisibleOutsideSummary |=
681 (Res.VisibleToRegularObj || Sym.isUsed() || !InSummary);
682
683 GlobalRes.ExportDynamic |= Res.ExportDynamic;
684 }
685}
686
689 StringRef Path = Input->getName();
690 OS << Path << '\n';
691 auto ResI = Res.begin();
692 for (const InputFile::Symbol &Sym : Input->symbols()) {
693 assert(ResI != Res.end());
694 SymbolResolution Res = *ResI++;
695
696 OS << "-r=" << Path << ',' << Sym.getName() << ',';
697 if (Res.Prevailing)
698 OS << 'p';
700 OS << 'l';
701 if (Res.VisibleToRegularObj)
702 OS << 'x';
703 if (Res.LinkerRedefined)
704 OS << 'r';
705 OS << '\n';
706 }
707 OS.flush();
708 assert(ResI == Res.end());
709}
710
711Error LTO::add(std::unique_ptr<InputFile> Input,
713 assert(!CalledGetMaxTasks);
714
715 if (Conf.ResolutionFile)
716 writeToResolutionFile(*Conf.ResolutionFile, Input.get(), Res);
717
718 if (RegularLTO.CombinedModule->getTargetTriple().empty()) {
719 RegularLTO.CombinedModule->setTargetTriple(Input->getTargetTriple());
720 if (Triple(Input->getTargetTriple()).isOSBinFormatELF())
722 }
723
724 const SymbolResolution *ResI = Res.begin();
725 for (unsigned I = 0; I != Input->Mods.size(); ++I)
726 if (Error Err = addModule(*Input, I, ResI, Res.end()))
727 return Err;
728
729 assert(ResI == Res.end());
730 return Error::success();
731}
732
733Error LTO::addModule(InputFile &Input, unsigned ModI,
734 const SymbolResolution *&ResI,
735 const SymbolResolution *ResE) {
736 Expected<BitcodeLTOInfo> LTOInfo = Input.Mods[ModI].getLTOInfo();
737 if (!LTOInfo)
738 return LTOInfo.takeError();
739
740 if (EnableSplitLTOUnit) {
741 // If only some modules were split, flag this in the index so that
742 // we can skip or error on optimizations that need consistently split
743 // modules (whole program devirt and lower type tests).
744 if (*EnableSplitLTOUnit != LTOInfo->EnableSplitLTOUnit)
745 ThinLTO.CombinedIndex.setPartiallySplitLTOUnits();
746 } else
747 EnableSplitLTOUnit = LTOInfo->EnableSplitLTOUnit;
748
749 BitcodeModule BM = Input.Mods[ModI];
750 auto ModSyms = Input.module_symbols(ModI);
751 addModuleToGlobalRes(ModSyms, {ResI, ResE},
752 LTOInfo->IsThinLTO ? ThinLTO.ModuleMap.size() + 1 : 0,
753 LTOInfo->HasSummary);
754
755 if (LTOInfo->IsThinLTO)
756 return addThinLTO(BM, ModSyms, ResI, ResE);
757
758 RegularLTO.EmptyCombinedModule = false;
760 addRegularLTO(BM, ModSyms, ResI, ResE);
761 if (!ModOrErr)
762 return ModOrErr.takeError();
763
764 if (!LTOInfo->HasSummary)
765 return linkRegularLTO(std::move(*ModOrErr), /*LivenessFromIndex=*/false);
766
767 // Regular LTO module summaries are added to a dummy module that represents
768 // the combined regular LTO module.
769 if (Error Err = BM.readSummary(ThinLTO.CombinedIndex, "", -1ull))
770 return Err;
771 RegularLTO.ModsWithSummaries.push_back(std::move(*ModOrErr));
772 return Error::success();
773}
774
775// Checks whether the given global value is in a non-prevailing comdat
776// (comdat containing values the linker indicated were not prevailing,
777// which we then dropped to available_externally), and if so, removes
778// it from the comdat. This is called for all global values to ensure the
779// comdat is empty rather than leaving an incomplete comdat. It is needed for
780// regular LTO modules, in case we are in a mixed-LTO mode (both regular
781// and thin LTO modules) compilation. Since the regular LTO module will be
782// linked first in the final native link, we want to make sure the linker
783// doesn't select any of these incomplete comdats that would be left
784// in the regular LTO module without this cleanup.
785static void
787 std::set<const Comdat *> &NonPrevailingComdats) {
788 Comdat *C = GV.getComdat();
789 if (!C)
790 return;
791
792 if (!NonPrevailingComdats.count(C))
793 return;
794
795 // Additionally need to drop all global values from the comdat to
796 // available_externally, to satisfy the COMDAT requirement that all members
797 // are discarded as a unit. The non-local linkage global values avoid
798 // duplicate definition linker errors.
800
801 if (auto GO = dyn_cast<GlobalObject>(&GV))
802 GO->setComdat(nullptr);
803}
804
805// Add a regular LTO object to the link.
806// The resulting module needs to be linked into the combined LTO module with
807// linkRegularLTO.
809LTO::addRegularLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
810 const SymbolResolution *&ResI,
811 const SymbolResolution *ResE) {
812 RegularLTOState::AddedModule Mod;
814 BM.getLazyModule(RegularLTO.Ctx, /*ShouldLazyLoadMetadata*/ true,
815 /*IsImporting*/ false);
816 if (!MOrErr)
817 return MOrErr.takeError();
818 Module &M = **MOrErr;
819 Mod.M = std::move(*MOrErr);
820
821 if (Error Err = M.materializeMetadata())
822 return std::move(Err);
824
825 ModuleSymbolTable SymTab;
826 SymTab.addModule(&M);
827
828 for (GlobalVariable &GV : M.globals())
829 if (GV.hasAppendingLinkage())
830 Mod.Keep.push_back(&GV);
831
832 DenseSet<GlobalObject *> AliasedGlobals;
833 for (auto &GA : M.aliases())
834 if (GlobalObject *GO = GA.getAliaseeObject())
835 AliasedGlobals.insert(GO);
836
837 // In this function we need IR GlobalValues matching the symbols in Syms
838 // (which is not backed by a module), so we need to enumerate them in the same
839 // order. The symbol enumeration order of a ModuleSymbolTable intentionally
840 // matches the order of an irsymtab, but when we read the irsymtab in
841 // InputFile::create we omit some symbols that are irrelevant to LTO. The
842 // Skip() function skips the same symbols from the module as InputFile does
843 // from the symbol table.
844 auto MsymI = SymTab.symbols().begin(), MsymE = SymTab.symbols().end();
845 auto Skip = [&]() {
846 while (MsymI != MsymE) {
847 auto Flags = SymTab.getSymbolFlags(*MsymI);
848 if ((Flags & object::BasicSymbolRef::SF_Global) &&
850 return;
851 ++MsymI;
852 }
853 };
854 Skip();
855
856 std::set<const Comdat *> NonPrevailingComdats;
857 SmallSet<StringRef, 2> NonPrevailingAsmSymbols;
858 for (const InputFile::Symbol &Sym : Syms) {
859 assert(ResI != ResE);
860 SymbolResolution Res = *ResI++;
861
862 assert(MsymI != MsymE);
863 ModuleSymbolTable::Symbol Msym = *MsymI++;
864 Skip();
865
866 if (GlobalValue *GV = dyn_cast_if_present<GlobalValue *>(Msym)) {
867 if (Res.Prevailing) {
868 if (Sym.isUndefined())
869 continue;
870 Mod.Keep.push_back(GV);
871 // For symbols re-defined with linker -wrap and -defsym options,
872 // set the linkage to weak to inhibit IPO. The linkage will be
873 // restored by the linker.
874 if (Res.LinkerRedefined)
875 GV->setLinkage(GlobalValue::WeakAnyLinkage);
876
877 GlobalValue::LinkageTypes OriginalLinkage = GV->getLinkage();
878 if (GlobalValue::isLinkOnceLinkage(OriginalLinkage))
879 GV->setLinkage(GlobalValue::getWeakLinkage(
880 GlobalValue::isLinkOnceODRLinkage(OriginalLinkage)));
881 } else if (isa<GlobalObject>(GV) &&
882 (GV->hasLinkOnceODRLinkage() || GV->hasWeakODRLinkage() ||
883 GV->hasAvailableExternallyLinkage()) &&
884 !AliasedGlobals.count(cast<GlobalObject>(GV))) {
885 // Any of the above three types of linkage indicates that the
886 // chosen prevailing symbol will have the same semantics as this copy of
887 // the symbol, so we may be able to link it with available_externally
888 // linkage. We will decide later whether to do that when we link this
889 // module (in linkRegularLTO), based on whether it is undefined.
890 Mod.Keep.push_back(GV);
892 if (GV->hasComdat())
893 NonPrevailingComdats.insert(GV->getComdat());
894 cast<GlobalObject>(GV)->setComdat(nullptr);
895 }
896
897 // Set the 'local' flag based on the linker resolution for this symbol.
899 GV->setDSOLocal(true);
900 if (GV->hasDLLImportStorageClass())
901 GV->setDLLStorageClass(GlobalValue::DLLStorageClassTypes::
902 DefaultStorageClass);
903 }
904 } else if (auto *AS =
905 dyn_cast_if_present<ModuleSymbolTable::AsmSymbol *>(Msym)) {
906 // Collect non-prevailing symbols.
907 if (!Res.Prevailing)
908 NonPrevailingAsmSymbols.insert(AS->first);
909 } else {
910 llvm_unreachable("unknown symbol type");
911 }
912
913 // Common resolution: collect the maximum size/alignment over all commons.
914 // We also record if we see an instance of a common as prevailing, so that
915 // if none is prevailing we can ignore it later.
916 if (Sym.isCommon()) {
917 // FIXME: We should figure out what to do about commons defined by asm.
918 // For now they aren't reported correctly by ModuleSymbolTable.
919 auto &CommonRes = RegularLTO.Commons[std::string(Sym.getIRName())];
920 CommonRes.Size = std::max(CommonRes.Size, Sym.getCommonSize());
921 if (uint32_t SymAlignValue = Sym.getCommonAlignment()) {
922 CommonRes.Alignment =
923 std::max(Align(SymAlignValue), CommonRes.Alignment);
924 }
925 CommonRes.Prevailing |= Res.Prevailing;
926 }
927 }
928
929 if (!M.getComdatSymbolTable().empty())
930 for (GlobalValue &GV : M.global_values())
931 handleNonPrevailingComdat(GV, NonPrevailingComdats);
932
933 // Prepend ".lto_discard <sym>, <sym>*" directive to each module inline asm
934 // block.
935 if (!M.getModuleInlineAsm().empty()) {
936 std::string NewIA = ".lto_discard";
937 if (!NonPrevailingAsmSymbols.empty()) {
938 // Don't dicard a symbol if there is a live .symver for it.
940 M, [&](StringRef Name, StringRef Alias) {
941 if (!NonPrevailingAsmSymbols.count(Alias))
942 NonPrevailingAsmSymbols.erase(Name);
943 });
944 NewIA += " " + llvm::join(NonPrevailingAsmSymbols, ", ");
945 }
946 NewIA += "\n";
947 M.setModuleInlineAsm(NewIA + M.getModuleInlineAsm());
948 }
949
950 assert(MsymI == MsymE);
951 return std::move(Mod);
952}
953
954Error LTO::linkRegularLTO(RegularLTOState::AddedModule Mod,
955 bool LivenessFromIndex) {
956 std::vector<GlobalValue *> Keep;
957 for (GlobalValue *GV : Mod.Keep) {
958 if (LivenessFromIndex && !ThinLTO.CombinedIndex.isGUIDLive(GV->getGUID())) {
959 if (Function *F = dyn_cast<Function>(GV)) {
960 if (DiagnosticOutputFile) {
961 if (Error Err = F->materialize())
962 return Err;
963 OptimizationRemarkEmitter ORE(F, nullptr);
964 ORE.emit(OptimizationRemark(DEBUG_TYPE, "deadfunction", F)
965 << ore::NV("Function", F)
966 << " not added to the combined module ");
967 }
968 }
969 continue;
970 }
971
973 Keep.push_back(GV);
974 continue;
975 }
976
977 // Only link available_externally definitions if we don't already have a
978 // definition.
979 GlobalValue *CombinedGV =
980 RegularLTO.CombinedModule->getNamedValue(GV->getName());
981 if (CombinedGV && !CombinedGV->isDeclaration())
982 continue;
983
984 Keep.push_back(GV);
985 }
986
987 return RegularLTO.Mover->move(std::move(Mod.M), Keep, nullptr,
988 /* IsPerformingImport */ false);
989}
990
991// Add a ThinLTO module to the link.
992Error LTO::addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
993 const SymbolResolution *&ResI,
994 const SymbolResolution *ResE) {
995 const SymbolResolution *ResITmp = ResI;
996 for (const InputFile::Symbol &Sym : Syms) {
997 assert(ResITmp != ResE);
998 SymbolResolution Res = *ResITmp++;
999
1000 if (!Sym.getIRName().empty()) {
1002 Sym.getIRName(), GlobalValue::ExternalLinkage, ""));
1003 if (Res.Prevailing)
1004 ThinLTO.PrevailingModuleForGUID[GUID] = BM.getModuleIdentifier();
1005 }
1006 }
1007
1008 uint64_t ModuleId = ThinLTO.ModuleMap.size();
1009 if (Error Err =
1010 BM.readSummary(ThinLTO.CombinedIndex, BM.getModuleIdentifier(),
1011 ModuleId, [&](GlobalValue::GUID GUID) {
1012 return ThinLTO.PrevailingModuleForGUID[GUID] ==
1013 BM.getModuleIdentifier();
1014 }))
1015 return Err;
1016 LLVM_DEBUG(dbgs() << "Module " << ModuleId << ": " << BM.getModuleIdentifier()
1017 << "\n");
1018
1019 for (const InputFile::Symbol &Sym : Syms) {
1020 assert(ResI != ResE);
1021 SymbolResolution Res = *ResI++;
1022
1023 if (!Sym.getIRName().empty()) {
1025 Sym.getIRName(), GlobalValue::ExternalLinkage, ""));
1026 if (Res.Prevailing) {
1027 assert(ThinLTO.PrevailingModuleForGUID[GUID] ==
1028 BM.getModuleIdentifier());
1029
1030 // For linker redefined symbols (via --wrap or --defsym) we want to
1031 // switch the linkage to `weak` to prevent IPOs from happening.
1032 // Find the summary in the module for this very GV and record the new
1033 // linkage so that we can switch it when we import the GV.
1034 if (Res.LinkerRedefined)
1035 if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(
1036 GUID, BM.getModuleIdentifier()))
1037 S->setLinkage(GlobalValue::WeakAnyLinkage);
1038 }
1039
1040 // If the linker resolved the symbol to a local definition then mark it
1041 // as local in the summary for the module we are adding.
1043 if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(
1044 GUID, BM.getModuleIdentifier())) {
1045 S->setDSOLocal(true);
1046 }
1047 }
1048 }
1049 }
1050
1051 if (!ThinLTO.ModuleMap.insert({BM.getModuleIdentifier(), BM}).second)
1052 return make_error<StringError>(
1053 "Expected at most one ThinLTO module per bitcode file",
1055
1056 if (!Conf.ThinLTOModulesToCompile.empty()) {
1057 if (!ThinLTO.ModulesToCompile)
1058 ThinLTO.ModulesToCompile = ModuleMapType();
1059 // This is a fuzzy name matching where only modules with name containing the
1060 // specified switch values are going to be compiled.
1061 for (const std::string &Name : Conf.ThinLTOModulesToCompile) {
1062 if (BM.getModuleIdentifier().contains(Name)) {
1063 ThinLTO.ModulesToCompile->insert({BM.getModuleIdentifier(), BM});
1064 llvm::errs() << "[ThinLTO] Selecting " << BM.getModuleIdentifier()
1065 << " to compile\n";
1066 }
1067 }
1068 }
1069
1070 return Error::success();
1071}
1072
1073unsigned LTO::getMaxTasks() const {
1074 CalledGetMaxTasks = true;
1075 auto ModuleCount = ThinLTO.ModulesToCompile ? ThinLTO.ModulesToCompile->size()
1076 : ThinLTO.ModuleMap.size();
1077 return RegularLTO.ParallelCodeGenParallelismLevel + ModuleCount;
1078}
1079
1080// If only some of the modules were split, we cannot correctly handle
1081// code that contains type tests or type checked loads.
1082Error LTO::checkPartiallySplit() {
1083 if (!ThinLTO.CombinedIndex.partiallySplitLTOUnits())
1084 return Error::success();
1085
1086 Function *TypeTestFunc = RegularLTO.CombinedModule->getFunction(
1087 Intrinsic::getName(Intrinsic::type_test));
1088 Function *TypeCheckedLoadFunc = RegularLTO.CombinedModule->getFunction(
1089 Intrinsic::getName(Intrinsic::type_checked_load));
1090
1091 // First check if there are type tests / type checked loads in the
1092 // merged regular LTO module IR.
1093 if ((TypeTestFunc && !TypeTestFunc->use_empty()) ||
1094 (TypeCheckedLoadFunc && !TypeCheckedLoadFunc->use_empty()))
1095 return make_error<StringError>(
1096 "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",
1098
1099 // Otherwise check if there are any recorded in the combined summary from the
1100 // ThinLTO modules.
1101 for (auto &P : ThinLTO.CombinedIndex) {
1102 for (auto &S : P.second.SummaryList) {
1103 auto *FS = dyn_cast<FunctionSummary>(S.get());
1104 if (!FS)
1105 continue;
1106 if (!FS->type_test_assume_vcalls().empty() ||
1107 !FS->type_checked_load_vcalls().empty() ||
1108 !FS->type_test_assume_const_vcalls().empty() ||
1109 !FS->type_checked_load_const_vcalls().empty() ||
1110 !FS->type_tests().empty())
1111 return make_error<StringError>(
1112 "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",
1114 }
1115 }
1116 return Error::success();
1117}
1118
1120 // Compute "dead" symbols, we don't want to import/export these!
1121 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1122 DenseMap<GlobalValue::GUID, PrevailingType> GUIDPrevailingResolutions;
1123 for (auto &Res : GlobalResolutions) {
1124 // Normally resolution have IR name of symbol. We can do nothing here
1125 // otherwise. See comments in GlobalResolution struct for more details.
1126 if (Res.second.IRName.empty())
1127 continue;
1128
1130 GlobalValue::dropLLVMManglingEscape(Res.second.IRName));
1131
1132 if (Res.second.VisibleOutsideSummary && Res.second.Prevailing)
1133 GUIDPreservedSymbols.insert(GUID);
1134
1135 if (Res.second.ExportDynamic)
1136 DynamicExportSymbols.insert(GUID);
1137
1138 GUIDPrevailingResolutions[GUID] =
1140 }
1141
1142 auto isPrevailing = [&](GlobalValue::GUID G) {
1143 auto It = GUIDPrevailingResolutions.find(G);
1144 if (It == GUIDPrevailingResolutions.end())
1146 return It->second;
1147 };
1148 computeDeadSymbolsWithConstProp(ThinLTO.CombinedIndex, GUIDPreservedSymbols,
1149 isPrevailing, Conf.OptLevel > 0);
1150
1151 // Setup output file to emit statistics.
1152 auto StatsFileOrErr = setupStatsFile(Conf.StatsFile);
1153 if (!StatsFileOrErr)
1154 return StatsFileOrErr.takeError();
1155 std::unique_ptr<ToolOutputFile> StatsFile = std::move(StatsFileOrErr.get());
1156
1157 // TODO: Ideally this would be controlled automatically by detecting that we
1158 // are linking with an allocator that supports these interfaces, rather than
1159 // an internal option (which would still be needed for tests, however). For
1160 // example, if the library exported a symbol like __malloc_hot_cold the linker
1161 // could recognize that and set a flag in the lto::Config.
1163 ThinLTO.CombinedIndex.setWithSupportsHotColdNew();
1164
1165 Error Result = runRegularLTO(AddStream);
1166 if (!Result)
1167 Result = runThinLTO(AddStream, Cache, GUIDPreservedSymbols);
1168
1169 if (StatsFile)
1170 PrintStatisticsJSON(StatsFile->os());
1171
1172 return Result;
1173}
1174
1176 const ModuleSummaryIndex &Index) {
1177 if (Index.withSupportsHotColdNew())
1178 return;
1179
1180 // The profile matcher applies hotness attributes directly for allocations,
1181 // and those will cause us to generate calls to the hot/cold interfaces
1182 // unconditionally. If supports-hot-cold-new was not enabled in the LTO
1183 // link then assume we don't want these calls (e.g. not linking with
1184 // the appropriate library, or otherwise trying to disable this behavior).
1185 for (auto &F : Mod) {
1186 for (auto &BB : F) {
1187 for (auto &I : BB) {
1188 auto *CI = dyn_cast<CallBase>(&I);
1189 if (!CI)
1190 continue;
1191 if (CI->hasFnAttr("memprof"))
1192 CI->removeFnAttr("memprof");
1193 // Strip off all memprof metadata as it is no longer needed.
1194 // Importantly, this avoids the addition of new memprof attributes
1195 // after inlining propagation.
1196 // TODO: If we support additional types of MemProf metadata beyond hot
1197 // and cold, we will need to update the metadata based on the allocator
1198 // APIs supported instead of completely stripping all.
1199 CI->setMetadata(LLVMContext::MD_memprof, nullptr);
1200 CI->setMetadata(LLVMContext::MD_callsite, nullptr);
1201 }
1202 }
1203 }
1204}
1205
1206Error LTO::runRegularLTO(AddStreamFn AddStream) {
1207 // Setup optimization remarks.
1208 auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
1209 RegularLTO.CombinedModule->getContext(), Conf.RemarksFilename,
1212 if (!DiagFileOrErr)
1213 return DiagFileOrErr.takeError();
1214 DiagnosticOutputFile = std::move(*DiagFileOrErr);
1215
1216 // Finalize linking of regular LTO modules containing summaries now that
1217 // we have computed liveness information.
1218 for (auto &M : RegularLTO.ModsWithSummaries)
1219 if (Error Err = linkRegularLTO(std::move(M),
1220 /*LivenessFromIndex=*/true))
1221 return Err;
1222
1223 // Ensure we don't have inconsistently split LTO units with type tests.
1224 // FIXME: this checks both LTO and ThinLTO. It happens to work as we take
1225 // this path both cases but eventually this should be split into two and
1226 // do the ThinLTO checks in `runThinLTO`.
1227 if (Error Err = checkPartiallySplit())
1228 return Err;
1229
1230 // Make sure commons have the right size/alignment: we kept the largest from
1231 // all the prevailing when adding the inputs, and we apply it here.
1232 const DataLayout &DL = RegularLTO.CombinedModule->getDataLayout();
1233 for (auto &I : RegularLTO.Commons) {
1234 if (!I.second.Prevailing)
1235 // Don't do anything if no instance of this common was prevailing.
1236 continue;
1237 GlobalVariable *OldGV = RegularLTO.CombinedModule->getNamedGlobal(I.first);
1238 if (OldGV && DL.getTypeAllocSize(OldGV->getValueType()) == I.second.Size) {
1239 // Don't create a new global if the type is already correct, just make
1240 // sure the alignment is correct.
1241 OldGV->setAlignment(I.second.Alignment);
1242 continue;
1243 }
1244 ArrayType *Ty =
1245 ArrayType::get(Type::getInt8Ty(RegularLTO.Ctx), I.second.Size);
1246 auto *GV = new GlobalVariable(*RegularLTO.CombinedModule, Ty, false,
1249 GV->setAlignment(I.second.Alignment);
1250 if (OldGV) {
1252 GV->takeName(OldGV);
1253 OldGV->eraseFromParent();
1254 } else {
1255 GV->setName(I.first);
1256 }
1257 }
1258
1259 updateMemProfAttributes(*RegularLTO.CombinedModule, ThinLTO.CombinedIndex);
1260
1261 // If allowed, upgrade public vcall visibility metadata to linkage unit
1262 // visibility before whole program devirtualization in the optimizer.
1263 updateVCallVisibilityInModule(*RegularLTO.CombinedModule,
1265 DynamicExportSymbols);
1266 updatePublicTypeTestCalls(*RegularLTO.CombinedModule,
1268
1269 if (Conf.PreOptModuleHook &&
1270 !Conf.PreOptModuleHook(0, *RegularLTO.CombinedModule))
1271 return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
1272
1273 if (!Conf.CodeGenOnly) {
1274 for (const auto &R : GlobalResolutions) {
1275 if (!R.second.isPrevailingIRSymbol())
1276 continue;
1277 if (R.second.Partition != 0 &&
1278 R.second.Partition != GlobalResolution::External)
1279 continue;
1280
1281 GlobalValue *GV =
1282 RegularLTO.CombinedModule->getNamedValue(R.second.IRName);
1283 // Ignore symbols defined in other partitions.
1284 // Also skip declarations, which are not allowed to have internal linkage.
1285 if (!GV || GV->hasLocalLinkage() || GV->isDeclaration())
1286 continue;
1287 GV->setUnnamedAddr(R.second.UnnamedAddr ? GlobalValue::UnnamedAddr::Global
1289 if (EnableLTOInternalization && R.second.Partition == 0)
1291 }
1292
1293 RegularLTO.CombinedModule->addModuleFlag(Module::Error, "LTOPostLink", 1);
1294
1295 if (Conf.PostInternalizeModuleHook &&
1296 !Conf.PostInternalizeModuleHook(0, *RegularLTO.CombinedModule))
1297 return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
1298 }
1299
1300 if (!RegularLTO.EmptyCombinedModule || Conf.AlwaysEmitRegularLTOObj) {
1301 if (Error Err =
1302 backend(Conf, AddStream, RegularLTO.ParallelCodeGenParallelismLevel,
1303 *RegularLTO.CombinedModule, ThinLTO.CombinedIndex))
1304 return Err;
1305 }
1306
1307 return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
1308}
1309
1310static const char *libcallRoutineNames[] = {
1311#define HANDLE_LIBCALL(code, name) name,
1312#include "llvm/IR/RuntimeLibcalls.def"
1313#undef HANDLE_LIBCALL
1314};
1315
1318}
1319
1320/// This class defines the interface to the ThinLTO backend.
1322protected:
1323 const Config &Conf;
1328
1329public:
1330 ThinBackendProc(const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1331 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1332 lto::IndexWriteCallback OnWrite, bool ShouldEmitImportsFiles)
1333 : Conf(Conf), CombinedIndex(CombinedIndex),
1334 ModuleToDefinedGVSummaries(ModuleToDefinedGVSummaries),
1335 OnWrite(OnWrite), ShouldEmitImportsFiles(ShouldEmitImportsFiles) {}
1336
1337 virtual ~ThinBackendProc() = default;
1338 virtual Error start(
1339 unsigned Task, BitcodeModule BM,
1340 const FunctionImporter::ImportMapTy &ImportList,
1341 const FunctionImporter::ExportSetTy &ExportList,
1342 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1344 virtual Error wait() = 0;
1345 virtual unsigned getThreadCount() = 0;
1346
1347 // Write sharded indices and (optionally) imports to disk
1349 llvm::StringRef ModulePath,
1350 const std::string &NewModulePath) {
1351 std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
1352 std::error_code EC;
1353 gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries,
1354 ImportList, ModuleToSummariesForIndex);
1355
1356 raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC,
1357 sys::fs::OpenFlags::OF_None);
1358 if (EC)
1359 return errorCodeToError(EC);
1360 writeIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex);
1361
1362 if (ShouldEmitImportsFiles) {
1363 EC = EmitImportsFiles(ModulePath, NewModulePath + ".imports",
1364 ModuleToSummariesForIndex);
1365 if (EC)
1366 return errorCodeToError(EC);
1367 }
1368 return Error::success();
1369 }
1370};
1371
1372namespace {
1373class InProcessThinBackend : public ThinBackendProc {
1374 ThreadPool BackendThreadPool;
1375 AddStreamFn AddStream;
1376 FileCache Cache;
1377 std::set<GlobalValue::GUID> CfiFunctionDefs;
1378 std::set<GlobalValue::GUID> CfiFunctionDecls;
1379
1380 std::optional<Error> Err;
1381 std::mutex ErrMu;
1382
1383 bool ShouldEmitIndexFiles;
1384
1385public:
1386 InProcessThinBackend(
1387 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1388 ThreadPoolStrategy ThinLTOParallelism,
1389 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1390 AddStreamFn AddStream, FileCache Cache, lto::IndexWriteCallback OnWrite,
1391 bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles)
1392 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries,
1393 OnWrite, ShouldEmitImportsFiles),
1394 BackendThreadPool(ThinLTOParallelism), AddStream(std::move(AddStream)),
1395 Cache(std::move(Cache)), ShouldEmitIndexFiles(ShouldEmitIndexFiles) {
1396 for (auto &Name : CombinedIndex.cfiFunctionDefs())
1397 CfiFunctionDefs.insert(
1399 for (auto &Name : CombinedIndex.cfiFunctionDecls())
1400 CfiFunctionDecls.insert(
1402 }
1403
1404 Error runThinLTOBackendThread(
1405 AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM,
1406 ModuleSummaryIndex &CombinedIndex,
1407 const FunctionImporter::ImportMapTy &ImportList,
1408 const FunctionImporter::ExportSetTy &ExportList,
1409 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1410 const GVSummaryMapTy &DefinedGlobals,
1412 auto RunThinBackend = [&](AddStreamFn AddStream) {
1413 LTOLLVMContext BackendContext(Conf);
1414 Expected<std::unique_ptr<Module>> MOrErr = BM.parseModule(BackendContext);
1415 if (!MOrErr)
1416 return MOrErr.takeError();
1417
1418 return thinBackend(Conf, Task, AddStream, **MOrErr, CombinedIndex,
1419 ImportList, DefinedGlobals, &ModuleMap);
1420 };
1421
1422 auto ModuleID = BM.getModuleIdentifier();
1423
1424 if (ShouldEmitIndexFiles) {
1425 if (auto E = emitFiles(ImportList, ModuleID, ModuleID.str()))
1426 return E;
1427 }
1428
1429 if (!Cache || !CombinedIndex.modulePaths().count(ModuleID) ||
1430 all_of(CombinedIndex.getModuleHash(ModuleID),
1431 [](uint32_t V) { return V == 0; }))
1432 // Cache disabled or no entry for this module in the combined index or
1433 // no module hash.
1434 return RunThinBackend(AddStream);
1435
1437 // The module may be cached, this helps handling it.
1438 computeLTOCacheKey(Key, Conf, CombinedIndex, ModuleID, ImportList,
1439 ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs,
1440 CfiFunctionDecls);
1441 Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, Key, ModuleID);
1442 if (Error Err = CacheAddStreamOrErr.takeError())
1443 return Err;
1444 AddStreamFn &CacheAddStream = *CacheAddStreamOrErr;
1445 if (CacheAddStream)
1446 return RunThinBackend(CacheAddStream);
1447
1448 return Error::success();
1449 }
1450
1451 Error start(
1452 unsigned Task, BitcodeModule BM,
1453 const FunctionImporter::ImportMapTy &ImportList,
1454 const FunctionImporter::ExportSetTy &ExportList,
1455 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1456 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1457 StringRef ModulePath = BM.getModuleIdentifier();
1458 assert(ModuleToDefinedGVSummaries.count(ModulePath));
1459 const GVSummaryMapTy &DefinedGlobals =
1460 ModuleToDefinedGVSummaries.find(ModulePath)->second;
1461 BackendThreadPool.async(
1462 [=](BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
1463 const FunctionImporter::ImportMapTy &ImportList,
1464 const FunctionImporter::ExportSetTy &ExportList,
1465 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>
1466 &ResolvedODR,
1467 const GVSummaryMapTy &DefinedGlobals,
1469 if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled)
1471 "thin backend");
1472 Error E = runThinLTOBackendThread(
1473 AddStream, Cache, Task, BM, CombinedIndex, ImportList, ExportList,
1474 ResolvedODR, DefinedGlobals, ModuleMap);
1475 if (E) {
1476 std::unique_lock<std::mutex> L(ErrMu);
1477 if (Err)
1478 Err = joinErrors(std::move(*Err), std::move(E));
1479 else
1480 Err = std::move(E);
1481 }
1482 if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled)
1484 },
1485 BM, std::ref(CombinedIndex), std::ref(ImportList), std::ref(ExportList),
1486 std::ref(ResolvedODR), std::ref(DefinedGlobals), std::ref(ModuleMap));
1487
1488 if (OnWrite)
1489 OnWrite(std::string(ModulePath));
1490 return Error::success();
1491 }
1492
1493 Error wait() override {
1494 BackendThreadPool.wait();
1495 if (Err)
1496 return std::move(*Err);
1497 else
1498 return Error::success();
1499 }
1500
1501 unsigned getThreadCount() override {
1502 return BackendThreadPool.getThreadCount();
1503 }
1504};
1505} // end anonymous namespace
1506
1509 bool ShouldEmitIndexFiles,
1510 bool ShouldEmitImportsFiles) {
1511 return [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1512 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1513 AddStreamFn AddStream, FileCache Cache) {
1514 return std::make_unique<InProcessThinBackend>(
1515 Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries, AddStream,
1516 Cache, OnWrite, ShouldEmitIndexFiles, ShouldEmitImportsFiles);
1517 };
1518}
1519
1520// Given the original \p Path to an output file, replace any path
1521// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
1522// resulting directory if it does not yet exist.
1524 StringRef NewPrefix) {
1525 if (OldPrefix.empty() && NewPrefix.empty())
1526 return std::string(Path);
1527 SmallString<128> NewPath(Path);
1528 llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix);
1529 StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str());
1530 if (!ParentPath.empty()) {
1531 // Make sure the new directory exists, creating it if necessary.
1532 if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath))
1533 llvm::errs() << "warning: could not create directory '" << ParentPath
1534 << "': " << EC.message() << '\n';
1535 }
1536 return std::string(NewPath.str());
1537}
1538
1539namespace {
1540class WriteIndexesThinBackend : public ThinBackendProc {
1541 std::string OldPrefix, NewPrefix, NativeObjectPrefix;
1542 raw_fd_ostream *LinkedObjectsFile;
1543
1544public:
1545 WriteIndexesThinBackend(
1546 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1547 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1548 std::string OldPrefix, std::string NewPrefix,
1549 std::string NativeObjectPrefix, bool ShouldEmitImportsFiles,
1550 raw_fd_ostream *LinkedObjectsFile, lto::IndexWriteCallback OnWrite)
1551 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries,
1552 OnWrite, ShouldEmitImportsFiles),
1553 OldPrefix(OldPrefix), NewPrefix(NewPrefix),
1554 NativeObjectPrefix(NativeObjectPrefix),
1555 LinkedObjectsFile(LinkedObjectsFile) {}
1556
1557 Error start(
1558 unsigned Task, BitcodeModule BM,
1559 const FunctionImporter::ImportMapTy &ImportList,
1560 const FunctionImporter::ExportSetTy &ExportList,
1561 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1562 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1563 StringRef ModulePath = BM.getModuleIdentifier();
1564 std::string NewModulePath =
1565 getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix);
1566
1567 if (LinkedObjectsFile) {
1568 std::string ObjectPrefix =
1569 NativeObjectPrefix.empty() ? NewPrefix : NativeObjectPrefix;
1570 std::string LinkedObjectsFilePath =
1571 getThinLTOOutputFile(ModulePath, OldPrefix, ObjectPrefix);
1572 *LinkedObjectsFile << LinkedObjectsFilePath << '\n';
1573 }
1574
1575 if (auto E = emitFiles(ImportList, ModulePath, NewModulePath))
1576 return E;
1577
1578 if (OnWrite)
1579 OnWrite(std::string(ModulePath));
1580 return Error::success();
1581 }
1582
1583 Error wait() override { return Error::success(); }
1584
1585 // WriteIndexesThinBackend should always return 1 to prevent module
1586 // re-ordering and avoid non-determinism in the final link.
1587 unsigned getThreadCount() override { return 1; }
1588};
1589} // end anonymous namespace
1590
1592 std::string OldPrefix, std::string NewPrefix,
1593 std::string NativeObjectPrefix, bool ShouldEmitImportsFiles,
1594 raw_fd_ostream *LinkedObjectsFile, IndexWriteCallback OnWrite) {
1595 return [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1596 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1597 AddStreamFn AddStream, FileCache Cache) {
1598 return std::make_unique<WriteIndexesThinBackend>(
1599 Conf, CombinedIndex, ModuleToDefinedGVSummaries, OldPrefix, NewPrefix,
1600 NativeObjectPrefix, ShouldEmitImportsFiles, LinkedObjectsFile, OnWrite);
1601 };
1602}
1603
1604Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache,
1605 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
1606 ThinLTO.CombinedIndex.releaseTemporaryMemory();
1607 timeTraceProfilerBegin("ThinLink", StringRef(""));
1608 auto TimeTraceScopeExit = llvm::make_scope_exit([]() {
1611 });
1612 if (ThinLTO.ModuleMap.empty())
1613 return Error::success();
1614
1615 if (ThinLTO.ModulesToCompile && ThinLTO.ModulesToCompile->empty()) {
1616 llvm::errs() << "warning: [ThinLTO] No module compiled\n";
1617 return Error::success();
1618 }
1619
1620 if (Conf.CombinedIndexHook &&
1621 !Conf.CombinedIndexHook(ThinLTO.CombinedIndex, GUIDPreservedSymbols))
1622 return Error::success();
1623
1624 // Collect for each module the list of function it defines (GUID ->
1625 // Summary).
1627 ModuleToDefinedGVSummaries(ThinLTO.ModuleMap.size());
1628 ThinLTO.CombinedIndex.collectDefinedGVSummariesPerModule(
1629 ModuleToDefinedGVSummaries);
1630 // Create entries for any modules that didn't have any GV summaries
1631 // (either they didn't have any GVs to start with, or we suppressed
1632 // generation of the summaries because they e.g. had inline assembly
1633 // uses that couldn't be promoted/renamed on export). This is so
1634 // InProcessThinBackend::start can still launch a backend thread, which
1635 // is passed the map of summaries for the module, without any special
1636 // handling for this case.
1637 for (auto &Mod : ThinLTO.ModuleMap)
1638 if (!ModuleToDefinedGVSummaries.count(Mod.first))
1639 ModuleToDefinedGVSummaries.try_emplace(Mod.first);
1640
1641 // Synthesize entry counts for functions in the CombinedIndex.
1642 computeSyntheticCounts(ThinLTO.CombinedIndex);
1643
1645 ThinLTO.ModuleMap.size());
1647 ThinLTO.ModuleMap.size());
1649
1650 if (DumpThinCGSCCs)
1651 ThinLTO.CombinedIndex.dumpSCCs(outs());
1652
1653 std::set<GlobalValue::GUID> ExportedGUIDs;
1654
1656 ThinLTO.CombinedIndex.setWithWholeProgramVisibility();
1657 // If allowed, upgrade public vcall visibility to linkage unit visibility in
1658 // the summaries before whole program devirtualization below.
1659 updateVCallVisibilityInIndex(ThinLTO.CombinedIndex,
1661 DynamicExportSymbols);
1662
1663 // Perform index-based WPD. This will return immediately if there are
1664 // no index entries in the typeIdMetadata map (e.g. if we are instead
1665 // performing IR-based WPD in hybrid regular/thin LTO mode).
1666 std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
1667 runWholeProgramDevirtOnIndex(ThinLTO.CombinedIndex, ExportedGUIDs,
1668 LocalWPDTargetsMap);
1669
1670 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1671 return ThinLTO.PrevailingModuleForGUID[GUID] == S->modulePath();
1672 };
1674 MemProfContextDisambiguation ContextDisambiguation;
1675 ContextDisambiguation.run(ThinLTO.CombinedIndex, isPrevailing);
1676 }
1677
1678 if (Conf.OptLevel > 0)
1679 ComputeCrossModuleImport(ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
1680 isPrevailing, ImportLists, ExportLists);
1681
1682 // Figure out which symbols need to be internalized. This also needs to happen
1683 // at -O0 because summary-based DCE is implemented using internalization, and
1684 // we must apply DCE consistently with the full LTO module in order to avoid
1685 // undefined references during the final link.
1686 for (auto &Res : GlobalResolutions) {
1687 // If the symbol does not have external references or it is not prevailing,
1688 // then not need to mark it as exported from a ThinLTO partition.
1689 if (Res.second.Partition != GlobalResolution::External ||
1690 !Res.second.isPrevailingIRSymbol())
1691 continue;
1692 auto GUID = GlobalValue::getGUID(
1693 GlobalValue::dropLLVMManglingEscape(Res.second.IRName));
1694 // Mark exported unless index-based analysis determined it to be dead.
1695 if (ThinLTO.CombinedIndex.isGUIDLive(GUID))
1696 ExportedGUIDs.insert(GUID);
1697 }
1698
1699 // Any functions referenced by the jump table in the regular LTO object must
1700 // be exported.
1701 for (auto &Def : ThinLTO.CombinedIndex.cfiFunctionDefs())
1702 ExportedGUIDs.insert(
1704 for (auto &Decl : ThinLTO.CombinedIndex.cfiFunctionDecls())
1705 ExportedGUIDs.insert(
1707
1708 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1709 const auto &ExportList = ExportLists.find(ModuleIdentifier);
1710 return (ExportList != ExportLists.end() && ExportList->second.count(VI)) ||
1711 ExportedGUIDs.count(VI.getGUID());
1712 };
1713
1714 // Update local devirtualized targets that were exported by cross-module
1715 // importing or by other devirtualizations marked in the ExportedGUIDs set.
1716 updateIndexWPDForExports(ThinLTO.CombinedIndex, isExported,
1717 LocalWPDTargetsMap);
1718
1719 thinLTOInternalizeAndPromoteInIndex(ThinLTO.CombinedIndex, isExported,
1720 isPrevailing);
1721
1722 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1723 GlobalValue::GUID GUID,
1724 GlobalValue::LinkageTypes NewLinkage) {
1725 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1726 };
1727 thinLTOResolvePrevailingInIndex(Conf, ThinLTO.CombinedIndex, isPrevailing,
1728 recordNewLinkage, GUIDPreservedSymbols);
1729
1730 thinLTOPropagateFunctionAttrs(ThinLTO.CombinedIndex, isPrevailing);
1731
1732 generateParamAccessSummary(ThinLTO.CombinedIndex);
1733
1736
1737 TimeTraceScopeExit.release();
1738
1739 std::unique_ptr<ThinBackendProc> BackendProc =
1740 ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
1741 AddStream, Cache);
1742
1743 auto &ModuleMap =
1744 ThinLTO.ModulesToCompile ? *ThinLTO.ModulesToCompile : ThinLTO.ModuleMap;
1745
1746 auto ProcessOneModule = [&](int I) -> Error {
1747 auto &Mod = *(ModuleMap.begin() + I);
1748 // Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for
1749 // combined module and parallel code generation partitions.
1750 return BackendProc->start(RegularLTO.ParallelCodeGenParallelismLevel + I,
1751 Mod.second, ImportLists[Mod.first],
1752 ExportLists[Mod.first], ResolvedODR[Mod.first],
1753 ThinLTO.ModuleMap);
1754 };
1755
1756 if (BackendProc->getThreadCount() == 1) {
1757 // Process the modules in the order they were provided on the command-line.
1758 // It is important for this codepath to be used for WriteIndexesThinBackend,
1759 // to ensure the emitted LinkedObjectsFile lists ThinLTO objects in the same
1760 // order as the inputs, which otherwise would affect the final link order.
1761 for (int I = 0, E = ModuleMap.size(); I != E; ++I)
1762 if (Error E = ProcessOneModule(I))
1763 return E;
1764 } else {
1765 // When executing in parallel, process largest bitsize modules first to
1766 // improve parallelism, and avoid starving the thread pool near the end.
1767 // This saves about 15 sec on a 36-core machine while link `clang.exe` (out
1768 // of 100 sec).
1769 std::vector<BitcodeModule *> ModulesVec;
1770 ModulesVec.reserve(ModuleMap.size());
1771 for (auto &Mod : ModuleMap)
1772 ModulesVec.push_back(&Mod.second);
1773 for (int I : generateModulesOrdering(ModulesVec))
1774 if (Error E = ProcessOneModule(I))
1775 return E;
1776 }
1777 return BackendProc->wait();
1778}
1779
1783 std::optional<uint64_t> RemarksHotnessThreshold, int Count) {
1784 std::string Filename = std::string(RemarksFilename);
1785 // For ThinLTO, file.opt.<format> becomes
1786 // file.opt.<format>.thin.<num>.<format>.
1787 if (!Filename.empty() && Count != -1)
1788 Filename =
1789 (Twine(Filename) + ".thin." + llvm::utostr(Count) + "." + RemarksFormat)
1790 .str();
1791
1792 auto ResultOrErr = llvm::setupLLVMOptimizationRemarks(
1795 if (Error E = ResultOrErr.takeError())
1796 return std::move(E);
1797
1798 if (*ResultOrErr)
1799 (*ResultOrErr)->keep();
1800
1801 return ResultOrErr;
1802}
1803
1806 // Setup output file to emit statistics.
1807 if (StatsFilename.empty())
1808 return nullptr;
1809
1811 std::error_code EC;
1812 auto StatsFile =
1813 std::make_unique<ToolOutputFile>(StatsFilename, EC, sys::fs::OF_None);
1814 if (EC)
1815 return errorCodeToError(EC);
1816
1817 StatsFile->keep();
1818 return std::move(StatsFile);
1819}
1820
1821// Compute the ordering we will process the inputs: the rough heuristic here
1822// is to sort them per size so that the largest module get schedule as soon as
1823// possible. This is purely a compile-time optimization.
1825 auto Seq = llvm::seq<int>(0, R.size());
1826 std::vector<int> ModulesOrdering(Seq.begin(), Seq.end());
1827 llvm::sort(ModulesOrdering, [&](int LeftIndex, int RightIndex) {
1828 auto LSize = R[LeftIndex]->getBuffer().size();
1829 auto RSize = R[RightIndex]->getBuffer().size();
1830 return LSize > RSize;
1831 });
1832 return ModulesOrdering;
1833}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DEBUG(X)
Definition: Debug.h:101
std::string Name
Symbol * Sym
Definition: ELF_riscv.cpp:463
#define DEBUG_TYPE
static void writeToResolutionFile(raw_ostream &OS, InputFile *Input, ArrayRef< SymbolResolution > Res)
Definition: LTO.cpp:687
static void thinLTOResolvePrevailingGUID(const Config &C, ValueInfo VI, DenseSet< GlobalValueSummary * > &GlobalInvolvedWithAlias, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, function_ref< void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)> recordNewLinkage, const DenseSet< GlobalValue::GUID > &GUIDPreservedSymbols)
Definition: LTO.cpp:350
static void handleNonPrevailingComdat(GlobalValue &GV, std::set< const Comdat * > &NonPrevailingComdats)
Definition: LTO.cpp:786
cl::opt< bool > SupportsHotColdNew
Indicate we are linking with an allocator that supports hot/cold operator new interfaces.
cl::opt< bool > EnableMemProfContextDisambiguation
Enable MemProf context disambiguation for thin link.
static cl::opt< bool > DumpThinCGSCCs("dump-thin-cg-sccs", cl::init(false), cl::Hidden, cl::desc("Dump the SCCs in the ThinLTO index's callgraph"))
static void thinLTOInternalizeAndPromoteGUID(ValueInfo VI, function_ref< bool(StringRef, ValueInfo)> isExported, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing)
Definition: LTO.cpp:449
static const char * libcallRoutineNames[]
Definition: LTO.cpp:1310
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
This file contains the declarations for metadata subclasses.
LLVMContext & Context
#define P(N)
@ VI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
raw_pwrite_stream & OS
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
This file contains some functions that are useful when dealing with strings.
This pass exposes codegen information to IR-level passes.
@ Flags
Definition: TextStubV5.cpp:93
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:152
iterator begin() const
Definition: ArrayRef.h:151
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:708
Represents a module in a bitcode file.
StringRef getModuleIdentifier() const
Expected< std::unique_ptr< Module > > parseModule(LLVMContext &Context, ParserCallbacks Callbacks={})
Read the entire bitcode module and return it.
Error readSummary(ModuleSummaryIndex &CombinedIndex, StringRef ModulePath, uint64_t ModuleId, std::function< bool(GlobalValue::GUID)> IsPrevailing=nullptr)
Parse the specified bitcode buffer and merge its module summary index into CombinedIndex.
Expected< std::unique_ptr< Module > > getLazyModule(LLVMContext &Context, bool ShouldLazyLoadMetadata, bool IsImporting, ParserCallbacks Callbacks={})
Read the bitcode module and prepare for lazy deserialization of function bodies.
static ConstantAggregateZero * get(Type *Ty)
Definition: Constants.cpp:1579
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2213
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
iterator end()
Definition: DenseMap.h:84
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
Lightweight error class with error context and mandatory checking.
Definition: Error.h:156
static ErrorSuccess success()
Create a success value.
Definition: Error.h:330
Tagged union holding either a T or a Error.
Definition: Error.h:470
Error takeError()
Take ownership of the stored error.
Definition: Error.h:597
std::unordered_set< GlobalValue::GUID > FunctionsToImportTy
Set of functions to import from a source module.
void setAlignment(Align Align)
Sets the alignment attribute of the GlobalObject.
Definition: Globals.cpp:128
Function and variable summary information to aid decisions and implementation of importing.
static bool isAppendingLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:395
static bool isLocalLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:404
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:273
void setUnnamedAddr(UnnamedAddr Val)
Definition: GlobalValue.h:227
bool hasLocalLinkage() const
Definition: GlobalValue.h:523
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition: GlobalValue.h:562
const Comdat * getComdat() const
Definition: Globals.cpp:183
static bool isLinkOnceLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:383
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:532
DLLStorageClassTypes
Storage classes of global values for PE targets.
Definition: GlobalValue.h:69
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:591
const GlobalObject * getAliaseeObject() const
Definition: Globals.cpp:367
PointerType * getType() const
Global values are always pointers.
Definition: GlobalValue.h:290
VisibilityTypes
An enumeration for the kinds of visibility of global values.
Definition: GlobalValue.h:62
@ DefaultVisibility
The GV is visible.
Definition: GlobalValue.h:63
static GUID getGUID(StringRef GlobalName)
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:587
static bool isInterposableLinkage(LinkageTypes Linkage)
Whether the definition of this global may be replaced by something non-equivalent at link time.
Definition: GlobalValue.h:420
static LinkageTypes getWeakLinkage(bool ODR)
Definition: GlobalValue.h:367
bool hasAvailableExternallyLinkage() const
Definition: GlobalValue.h:507
std::string getGlobalIdentifier() const
Return the modified name for this global value suitable to be used as the key for a global lookup (e....
Definition: Globals.cpp:168
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:47
@ CommonLinkage
Tentative definitions.
Definition: GlobalValue.h:58
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:55
@ WeakODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:53
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
Definition: GlobalValue.h:52
@ AppendingLinkage
Special purpose, only applies to global arrays.
Definition: GlobalValue.h:54
@ AvailableExternallyLinkage
Available for inspection, not emission.
Definition: GlobalValue.h:49
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:51
Type * getValueType() const
Definition: GlobalValue.h:292
static bool isLinkOnceODRLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:380
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition: Globals.cpp:454
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
This class implements a map that also provides access to all stored values in a deterministic order.
Definition: MapVector.h:37
iterator begin()
Definition: MapVector.h:70
size_type size() const
Definition: MapVector.h:61
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFile(const Twine &Filename, bool IsText=false, bool RequiresNullTerminator=true, bool IsVolatile=false, std::optional< Align > Alignment=std::nullopt)
Open the specified file as a MemoryBuffer, returning a new MemoryBuffer if successful,...
Class to hold module path string table and global value map, and encapsulate methods for operating on...
std::set< std::string > & cfiFunctionDecls()
const ModuleHash & getModuleHash(const StringRef ModPath) const
Get the module SHA1 hash recorded for the given module path.
const StringMap< std::pair< uint64_t, ModuleHash > > & modulePaths() const
Table of modules, containing module hash and id.
std::set< std::string > & cfiFunctionDefs()
static void CollectAsmSymvers(const Module &M, function_ref< void(StringRef, StringRef)> AsmSymver)
Parse inline ASM and collect the symvers directives that are defined in the current module.
uint32_t getSymbolFlags(Symbol S) const
ArrayRef< Symbol > symbols() const
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
Definition: Module.h:118
The optimization diagnostic interface.
Diagnostic information for applied optimization remarks.
A discriminated union of two or more pointer types, with the discriminator in the low bit of the poin...
Definition: PointerUnion.h:118
A class that wrap the SHA1 algorithm.
Definition: SHA1.h:26
void update(ArrayRef< uint8_t > Data)
Digest more data.
Definition: SHA1.cpp:208
std::array< uint8_t, 20 > result()
Return the current raw 160-bits SHA1 for the digested data since the last call to init().
Definition: SHA1.cpp:288
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:166
bool empty() const
Definition: SmallSet.h:159
bool erase(const T &V)
Definition: SmallSet.h:207
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:179
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition: SmallString.h:26
StringRef str() const
Explicit conversion to StringRef.
Definition: SmallString.h:261
StringMapEntry - This is used to represent one value that is inserted into a StringMap.
unsigned size() const
Definition: StringMap.h:95
StringMap - This is an unconventional map that is specialized for handling keys that are "strings",...
Definition: StringMap.h:111
StringMapConstIterator< ValueTy > const_iterator
Definition: StringMap.h:200
iterator end()
Definition: StringMap.h:204
iterator begin()
Definition: StringMap.h:203
iterator find(StringRef Key)
Definition: StringMap.h:217
size_type count(StringRef Key) const
count - Return 1 if the element is in the map, 0 otherwise.
Definition: StringMap.h:256
std::pair< iterator, bool > try_emplace(StringRef Key, ArgsTy &&...Args)
Emplace a new element for the specified key into the map if the key isn't already in the map.
Definition: StringMap.h:340
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
bool contains(StringRef Other) const
Return true if the given string is a substring of *this, and false otherwise.
Definition: StringRef.h:428
unsigned RelaxELFRelocations
DebuggerKind DebuggerTuning
Which debugger to tune for.
unsigned FunctionSections
Emit functions into separate sections.
unsigned DataSections
Emit data into separate sections.
This tells how a thread pool will be used.
Definition: Threading.h:116
A ThreadPool for asynchronous parallel execution on a defined number of threads.
Definition: ThreadPool.h:52
void wait()
Blocking wait for all the threads to complete and the queue to be empty.
Definition: ThreadPool.cpp:202
unsigned getThreadCount() const
Definition: ThreadPool.h:110
auto async(Function &&F, Args &&...ArgList)
Asynchronous submission of a task to the pool.
Definition: ThreadPool.h:66
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
bool isOSBinFormatELF() const
Tests whether the OS uses the ELF binary format.
Definition: Triple.h:681
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
static IntegerType * getInt8Ty(LLVMContext &C)
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:378
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:535
bool use_empty() const
Definition: Value.h:344
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:384
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
iterator find(const_arg_type_t< ValueT > V)
Definition: DenseSet.h:179
size_type size() const
Definition: DenseSet.h:81
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:97
An efficient, type-erasing, non-owning reference to a callable.
Ephemeral symbols produced by Reader::symbols() and Reader::module_symbols().
Definition: IRSymtab.h:313
The purpose of this class is to only expose the symbol information that an LTO client should need in ...
Definition: LTO.h:137
An input file.
Definition: LTO.h:109
static Expected< std::unique_ptr< InputFile > > create(MemoryBufferRef Object)
Create an InputFile.
Definition: LTO.cpp:554
ArrayRef< Symbol > symbols() const
A range over the symbols in this InputFile.
Definition: LTO.h:162
StringRef getName() const
Returns the path to the InputFile.
Definition: LTO.cpp:583
BitcodeModule & getSingleBitcodeModule()
Definition: LTO.cpp:587
Error add(std::unique_ptr< InputFile > Obj, ArrayRef< SymbolResolution > Res)
Add an input file to the LTO link, using the provided symbol resolutions.
Definition: LTO.cpp:711
static ArrayRef< const char * > getRuntimeLibcallSymbols()
Static method that returns a list of libcall symbols that can be generated by LTO but might not be vi...
Definition: LTO.cpp:1316
Error run(AddStreamFn AddStream, FileCache Cache=nullptr)
Runs the LTO pipeline.
Definition: LTO.cpp:1119
unsigned getMaxTasks() const
Returns an upper bound on the number of tasks that the client may expect.
Definition: LTO.cpp:1073
LTO(Config Conf, ThinBackend Backend=nullptr, unsigned ParallelCodeGenParallelismLevel=1)
Create an LTO object.
Definition: LTO.cpp:605
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:454
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
This class defines the interface to the ThinLTO backend.
Definition: LTO.cpp:1321
virtual unsigned getThreadCount()=0
const Config & Conf
Definition: LTO.cpp:1323
lto::IndexWriteCallback OnWrite
Definition: LTO.cpp:1326
ThinBackendProc(const Config &Conf, ModuleSummaryIndex &CombinedIndex, const StringMap< GVSummaryMapTy > &ModuleToDefinedGVSummaries, lto::IndexWriteCallback OnWrite, bool ShouldEmitImportsFiles)
Definition: LTO.cpp:1330
Error emitFiles(const FunctionImporter::ImportMapTy &ImportList, llvm::StringRef ModulePath, const std::string &NewModulePath)
Definition: LTO.cpp:1348
virtual Error wait()=0
ModuleSummaryIndex & CombinedIndex
Definition: LTO.cpp:1324
virtual ~ThinBackendProc()=default
virtual Error start(unsigned Task, BitcodeModule BM, const FunctionImporter::ImportMapTy &ImportList, const FunctionImporter::ExportSetTy &ExportList, const std::map< GlobalValue::GUID, GlobalValue::LinkageTypes > &ResolvedODR, MapVector< StringRef, BitcodeModule > &ModuleMap)=0
const StringMap< GVSummaryMapTy > & ModuleToDefinedGVSummaries
Definition: LTO.cpp:1325
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Key
PAL metadata keys.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Definition: Function.cpp:992
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
ThinBackend createInProcessThinBackend(ThreadPoolStrategy Parallelism, IndexWriteCallback OnWrite=nullptr, bool ShouldEmitIndexFiles=false, bool ShouldEmitImportsFiles=false)
Definition: LTO.cpp:1507
Error thinBackend(const Config &C, unsigned Task, AddStreamFn AddStream, Module &M, const ModuleSummaryIndex &CombinedIndex, const FunctionImporter::ImportMapTy &ImportList, const GVSummaryMapTy &DefinedGlobals, MapVector< StringRef, BitcodeModule > *ModuleMap, const std::vector< uint8_t > &CmdArgs=std::vector< uint8_t >())
Runs a ThinLTO backend.
Definition: LTOBackend.cpp:543
std::string getThinLTOOutputFile(StringRef Path, StringRef OldPrefix, StringRef NewPrefix)
Given the original Path to an output file, replace any path prefix matching OldPrefix with NewPrefix.
Definition: LTO.cpp:1523
std::function< std::unique_ptr< ThinBackendProc >(const Config &C, ModuleSummaryIndex &CombinedIndex, StringMap< GVSummaryMapTy > &ModuleToDefinedGVSummaries, AddStreamFn AddStream, FileCache Cache)> ThinBackend
A ThinBackend defines what happens after the thin-link phase during ThinLTO.
Definition: LTO.h:200
std::function< void(const std::string &)> IndexWriteCallback
This ThinBackend runs the individual backend jobs in-process.
Definition: LTO.h:210
Expected< std::unique_ptr< ToolOutputFile > > setupStatsFile(StringRef StatsFilename)
Setups the output file for saving statistics.
Definition: LTO.cpp:1805
Error backend(const Config &C, AddStreamFn AddStream, unsigned ParallelCodeGenParallelismLevel, Module &M, ModuleSummaryIndex &CombinedIndex)
Runs a regular LTO backend.
Definition: LTOBackend.cpp:497
Error finalizeOptimizationRemarks(std::unique_ptr< ToolOutputFile > DiagOutputFile)
Definition: LTOBackend.cpp:486
ThinBackend createWriteIndexesThinBackend(std::string OldPrefix, std::string NewPrefix, std::string NativeObjectPrefix, bool ShouldEmitImportsFiles, raw_fd_ostream *LinkedObjectsFile, IndexWriteCallback OnWrite)
This ThinBackend writes individual module indexes to files, instead of running the individual backend...
Definition: LTO.cpp:1591
std::vector< int > generateModulesOrdering(ArrayRef< BitcodeModule * > R)
Produces a container ordering for optimal multi-threaded processing.
Definition: LTO.cpp:1824
Expected< std::unique_ptr< ToolOutputFile > > setupLLVMOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses, StringRef RemarksFormat, bool RemarksWithHotness, std::optional< uint64_t > RemarksHotnessThreshold=0, int Count=-1)
Setup optimization remarks.
Definition: LTO.cpp:1780
void updateMemProfAttributes(Module &Mod, const ModuleSummaryIndex &Index)
Updates MemProf attributes (and metadata) based on whether the index has recorded that we are linking...
Definition: LTO.cpp:1175
Expected< IRSymtabFile > readIRSymtab(MemoryBufferRef MBRef)
Reads a bitcode file, creating its irsymtab if necessary.
size_t getThreadCount()
Definition: Parallel.h:55
void write64le(void *P, uint64_t V)
Definition: Endian.h:417
void write32le(void *P, uint32_t V)
Definition: Endian.h:416
std::error_code create_directories(const Twine &path, bool IgnoreExisting=true, perms Perms=owner_all|group_all)
Create all the non-existent directories in path.
Definition: Path.cpp:968
StringRef parent_path(StringRef path, Style style=Style::native)
Get parent path.
Definition: Path.cpp:468
bool replace_path_prefix(SmallVectorImpl< char > &Path, StringRef OldPrefix, StringRef NewPrefix, Style style=Style::native)
Replace matching path prefix with another path.
Definition: Path.cpp:519
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
ThreadPoolStrategy heavyweight_hardware_concurrency(unsigned ThreadCount=0)
Returns a thread strategy for tasks requiring significant memory or other resources.
Definition: Threading.h:162
cl::opt< std::string > RemarksFormat("lto-pass-remarks-format", cl::desc("The format used for serializing remarks (default: YAML)"), cl::value_desc("format"), cl::init("yaml"))
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1819
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1777
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:59
void timeTraceProfilerInitialize(unsigned TimeTraceGranularity, StringRef ProcName)
Initialize the time trace profiler.
void generateParamAccessSummary(ModuleSummaryIndex &Index)
void updateVCallVisibilityInModule(Module &M, bool WholeProgramVisibilityEnabledInLTO, const DenseSet< GlobalValue::GUID > &DynamicExportSymbols)
If whole program visibility asserted, then upgrade all public vcall visibility metadata on vtable def...
cl::opt< std::string > RemarksPasses("lto-pass-remarks-filter", cl::desc("Only record optimization remarks from passes whose " "names match the given regular expression"), cl::value_desc("regex"))
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:79
void writeIndexToFile(const ModuleSummaryIndex &Index, raw_ostream &Out, const std::map< std::string, GVSummaryMapTy > *ModuleToSummariesForIndex=nullptr)
Write the specified module summary index to the given raw output stream, where it will be written in ...
raw_fd_ostream & outs()
This returns a reference to a raw_fd_ostream for standard output.
void gatherImportedSummariesForModule(StringRef ModulePath, const StringMap< GVSummaryMapTy > &ModuleToDefinedGVSummaries, const FunctionImporter::ImportMapTy &ImportList, std::map< std::string, GVSummaryMapTy > &ModuleToSummariesForIndex)
Compute the set of summaries needed for a ThinLTO backend compilation of ModulePath.
std::function< Expected< std::unique_ptr< CachedFileStream > >(unsigned Task, const Twine &ModuleName)> AddStreamFn
This type defines the callback to add a file that is generated on the fly.
Definition: Caching.h:42
std::error_code EmitImportsFiles(StringRef ModulePath, StringRef OutputFilename, const std::map< std::string, GVSummaryMapTy > &ModuleToSummariesForIndex)
Emit into OutputFilename the files module ModulePath will import from.
void ComputeCrossModuleImport(const ModuleSummaryIndex &Index, const StringMap< GVSummaryMapTy > &ModuleToDefinedGVSummaries, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, StringMap< FunctionImporter::ImportMapTy > &ImportLists, StringMap< FunctionImporter::ExportSetTy > &ExportLists)
Compute all the imports and exports for every module in the Index.
bool thinLTOPropagateFunctionAttrs(ModuleSummaryIndex &Index, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing)
Propagate function attributes for function summaries along the index's callgraph during thinlink.
bool hasWholeProgramVisibility(bool WholeProgramVisibilityEnabledInLTO)
void EnableStatistics(bool DoPrintOnExit=true)
Enable the collection and printing of statistics.
Definition: Statistic.cpp:134
void thinLTOInternalizeAndPromoteInIndex(ModuleSummaryIndex &Index, function_ref< bool(StringRef, ValueInfo)> isExported, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing)
Update the linkages in the given Index to mark exported values as external and non-exported values as...
Definition: LTO.cpp:541
void computeLTOCacheKey(SmallString< 40 > &Key, const lto::Config &Conf, const ModuleSummaryIndex &Index, StringRef ModuleID, const FunctionImporter::ImportMapTy &ImportList, const FunctionImporter::ExportSetTy &ExportList, const std::map< GlobalValue::GUID, GlobalValue::LinkageTypes > &ResolvedODR, const GVSummaryMapTy &DefinedGlobals, const std::set< GlobalValue::GUID > &CfiFunctionDefs={}, const std::set< GlobalValue::GUID > &CfiFunctionDecls={})
Computes a unique hash for the Module considering the current list of export/import and other global ...
Definition: LTO.cpp:89
void timeTraceProfilerFinishThread()
Finish a time trace profiler running on a worker thread.
void timeTraceProfilerBegin(StringRef Name, StringRef Detail)
Manually begin a time section, with the given Name and Detail.
Error joinErrors(Error E1, Error E2)
Concatenate errors.
Definition: Error.h:427
void updatePublicTypeTestCalls(Module &M, bool WholeProgramVisibilityEnabledInLTO)
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1744
bool timeTraceProfilerEnabled()
Is the time trace profiler enabled, i.e. initialized?
Definition: TimeProfiler.h:102
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
Expected< std::unique_ptr< ToolOutputFile > > setupLLVMOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses, StringRef RemarksFormat, bool RemarksWithHotness, std::optional< uint64_t > RemarksHotnessThreshold=0)
Setup optimization remarks that output to a file.
cl::opt< bool > EnableLTOInternalization
Enable global value internalization in LTO.
cl::opt< bool > RemarksWithHotness("lto-pass-remarks-with-hotness", cl::desc("With PGO, include profile count in optimization remarks"), cl::Hidden)
void computeSyntheticCounts(ModuleSummaryIndex &Index)
Compute synthetic function entry counts.
void timeTraceProfilerEnd()
Manually end the last time section.
cl::opt< std::string > RemarksFilename("lto-pass-remarks-output", cl::desc("Output filename for pass remarks"), cl::value_desc("filename"))
void updateIndexWPDForExports(ModuleSummaryIndex &Summary, function_ref< bool(StringRef, ValueInfo)> isExported, std::map< ValueInfo, std::vector< VTableSlotSummary > > &LocalWPDTargetsMap)
Call after cross-module importing to update the recorded single impl devirt target names for any loca...
void thinLTOResolvePrevailingInIndex(const lto::Config &C, ModuleSummaryIndex &Index, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, function_ref< void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)> recordNewLinkage, const DenseSet< GlobalValue::GUID > &GUIDPreservedSymbols)
Resolve linkage for prevailing symbols in the Index.
Definition: LTO.cpp:427
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Mod
The access may modify the value stored in memory.
void updateVCallVisibilityInIndex(ModuleSummaryIndex &Index, bool WholeProgramVisibilityEnabledInLTO, const DenseSet< GlobalValue::GUID > &DynamicExportSymbols)
If whole program visibility asserted, then upgrade all public vcall visibility metadata on vtable def...
void runWholeProgramDevirtOnIndex(ModuleSummaryIndex &Summary, std::set< GlobalValue::GUID > &ExportedGUIDs, std::map< ValueInfo, std::vector< VTableSlotSummary > > &LocalWPDTargetsMap)
Perform index-based whole program devirtualization on the Summary index.
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1946
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition: STLExtras.h:2018
cl::opt< std::optional< uint64_t >, false, remarks::HotnessThresholdParser > RemarksHotnessThreshold("lto-pass-remarks-hotness-threshold", cl::desc("Minimum profile count required for an " "optimization remark to be output." " Use 'auto' to apply the threshold from profile summary."), cl::value_desc("uint or 'auto'"), cl::init(0), cl::Hidden)
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
Definition: Error.cpp:92
bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
std::array< uint32_t, 5 > ModuleHash
160 bits SHA1
std::function< Expected< AddStreamFn >(unsigned Task, StringRef Key, const Twine &ModuleName)> FileCache
This is the type of a file cache.
Definition: Caching.h:58
void PrintStatisticsJSON(raw_ostream &OS)
Print statistics in JSON format.
Definition: Statistic.cpp:203
void computeDeadSymbolsWithConstProp(ModuleSummaryIndex &Index, const DenseSet< GlobalValue::GUID > &GUIDPreservedSymbols, function_ref< PrevailingType(GlobalValue::GUID)> isPrevailing, bool ImportEnabled)
Compute dead symbols and run constant propagation in combined index after that.
@ Keep
No function return thunk.
Definition: BitVector.h:858
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Used in the streaming interface as the general argument type.
Struct that holds a reference to a particular GUID in a global value summary.
LTO configuration.
Definition: Config.h:41
bool HasWholeProgramVisibility
Asserts whether we can assume whole program visibility during the LTO link.
Definition: Config.h:80
std::optional< uint64_t > RemarksHotnessThreshold
The minimum hotness value a diagnostic needs in order to be included in optimization diagnostics.
Definition: Config.h:156
std::string StatsFile
Statistics output file path.
Definition: Config.h:165
ModuleHookFn PreOptModuleHook
This module hook is called after linking (regular LTO) or loading (ThinLTO) the module,...
Definition: Config.h:219
CombinedIndexHookFn CombinedIndexHook
Definition: Config.h:251
std::optional< CodeModel::Model > CodeModel
Definition: Config.h:56
std::string AAPipeline
Definition: Config.h:101
bool CodeGenOnly
Disable entirely the optimizer, including importing for ThinLTO.
Definition: Config.h:70
std::vector< std::string > MAttrs
Definition: Config.h:50
std::vector< std::string > MllvmArgs
Definition: Config.h:51
std::vector< std::string > ThinLTOModulesToCompile
Specific thinLTO modules to compile.
Definition: Config.h:168
std::unique_ptr< raw_ostream > ResolutionFile
If this field is set, LTO will write input file paths and symbol resolutions here in llvm-lto2 comman...
Definition: Config.h:186
std::string DefaultTriple
Setting this field will replace unspecified target triples in input files with this triple.
Definition: Config.h:109
bool AlwaysEmitRegularLTOObj
Always emit a Regular LTO object even when it is empty because no Regular LTO modules were linked.
Definition: Config.h:85
std::string CPU
Definition: Config.h:48
std::string DwoDir
The directory to store .dwo files.
Definition: Config.h:121
std::string RemarksFilename
Optimization remarks file path.
Definition: Config.h:135
VisScheme VisibilityScheme
Allows non-imported definitions to get the potentially more constraining visibility from the prevaili...
Definition: Config.h:91
std::string OverrideTriple
Setting this field will replace target triples in input files with this triple.
Definition: Config.h:105
std::string ProfileRemapping
Name remapping file for profile data.
Definition: Config.h:118
TargetOptions Options
Definition: Config.h:49
bool TimeTraceEnabled
Time trace enabled.
Definition: Config.h:171
CodeGenOpt::Level CGOptLevel
Definition: Config.h:57
std::string RemarksPasses
Optimization remarks pass filter.
Definition: Config.h:138
std::string OptPipeline
If this field is set, the set of passes run in the middle-end optimizer will be the one specified by ...
Definition: Config.h:96
ModuleHookFn PostInternalizeModuleHook
This hook is called after internalizing the module.
Definition: Config.h:226
unsigned TimeTraceGranularity
Time trace granularity.
Definition: Config.h:174
unsigned OptLevel
Definition: Config.h:59
bool RemarksWithHotness
Whether to emit optimization remarks with hotness informations.
Definition: Config.h:141
std::optional< Reloc::Model > RelocModel
Definition: Config.h:55
CodeGenFileType CGFileType
Definition: Config.h:58
bool Freestanding
Flag to indicate that the optimizer should not assume builtins are present on the target.
Definition: Config.h:67
std::string SampleProfile
Sample PGO profile path.
Definition: Config.h:115
std::string RemarksFormat
The format used for serializing remarks (default: YAML).
Definition: Config.h:159
A derived class of LLVMContext that initializes itself according to a given Config object.
Definition: Config.h:290
The resolution for a symbol.
Definition: LTO.h:437
unsigned FinalDefinitionInLinkageUnit
The definition of this symbol is unpreemptable at runtime and is known to be in this linkage unit.
Definition: LTO.h:447
unsigned ExportDynamic
The symbol was exported dynamically, and therefore could be referenced by a shared library not visibl...
Definition: LTO.h:454
unsigned Prevailing
The linker has chosen this definition of the symbol.
Definition: LTO.h:443
unsigned LinkerRedefined
Linker redefined version of the symbol which appeared in -wrap or -defsym linker option.
Definition: LTO.h:458
unsigned VisibleToRegularObj
The definition of this symbol is visible outside of the LTO unit.
Definition: LTO.h:450