LLVM 18.0.0git
LTO.cpp
Go to the documentation of this file.
1//===-LTO.cpp - LLVM Link Time Optimizer ----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements functions and classes used to support LTO.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/LTO/LTO.h"
14#include "llvm/ADT/ScopeExit.h"
15#include "llvm/ADT/SmallSet.h"
16#include "llvm/ADT/Statistic.h"
25#include "llvm/Config/llvm-config.h"
26#include "llvm/IR/AutoUpgrade.h"
28#include "llvm/IR/Intrinsics.h"
31#include "llvm/IR/Mangler.h"
32#include "llvm/IR/Metadata.h"
33#include "llvm/LTO/LTOBackend.h"
35#include "llvm/Linker/IRMover.h"
39#include "llvm/Support/Error.h"
43#include "llvm/Support/Path.h"
44#include "llvm/Support/SHA1.h"
50#include "llvm/Support/VCSRevision.h"
53#include "llvm/Transforms/IPO.h"
58
59#include <optional>
60#include <set>
61
62using namespace llvm;
63using namespace lto;
64using namespace object;
65
66#define DEBUG_TYPE "lto"
67
68static cl::opt<bool>
69 DumpThinCGSCCs("dump-thin-cg-sccs", cl::init(false), cl::Hidden,
70 cl::desc("Dump the SCCs in the ThinLTO index's callgraph"));
71
72namespace llvm {
73/// Enable global value internalization in LTO.
75 "enable-lto-internalization", cl::init(true), cl::Hidden,
76 cl::desc("Enable global value internalization in LTO"));
77
78/// Indicate we are linking with an allocator that supports hot/cold operator
79/// new interfaces.
81
82/// Enable MemProf context disambiguation for thin link.
84} // namespace llvm
85
86// Computes a unique hash for the Module considering the current list of
87// export/import and other global analysis results.
88// The hash is produced in \p Key.
90 SmallString<40> &Key, const Config &Conf, const ModuleSummaryIndex &Index,
91 StringRef ModuleID, const FunctionImporter::ImportMapTy &ImportList,
92 const FunctionImporter::ExportSetTy &ExportList,
93 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
94 const GVSummaryMapTy &DefinedGlobals,
95 const std::set<GlobalValue::GUID> &CfiFunctionDefs,
96 const std::set<GlobalValue::GUID> &CfiFunctionDecls) {
97 // Compute the unique hash for this entry.
98 // This is based on the current compiler version, the module itself, the
99 // export list, the hash for every single module in the import list, the
100 // list of ResolvedODR for the module, and the list of preserved symbols.
101 SHA1 Hasher;
102
103 // Start with the compiler revision
104 Hasher.update(LLVM_VERSION_STRING);
105#ifdef LLVM_REVISION
106 Hasher.update(LLVM_REVISION);
107#endif
108
109 // Include the parts of the LTO configuration that affect code generation.
110 auto AddString = [&](StringRef Str) {
111 Hasher.update(Str);
112 Hasher.update(ArrayRef<uint8_t>{0});
113 };
114 auto AddUnsigned = [&](unsigned I) {
115 uint8_t Data[4];
117 Hasher.update(ArrayRef<uint8_t>{Data, 4});
118 };
119 auto AddUint64 = [&](uint64_t I) {
120 uint8_t Data[8];
122 Hasher.update(ArrayRef<uint8_t>{Data, 8});
123 };
124 AddString(Conf.CPU);
125 // FIXME: Hash more of Options. For now all clients initialize Options from
126 // command-line flags (which is unsupported in production), but may set
127 // RelaxELFRelocations. The clang driver can also pass FunctionSections,
128 // DataSections and DebuggerTuning via command line flags.
129 AddUnsigned(Conf.Options.RelaxELFRelocations);
130 AddUnsigned(Conf.Options.FunctionSections);
131 AddUnsigned(Conf.Options.DataSections);
132 AddUnsigned((unsigned)Conf.Options.DebuggerTuning);
133 for (auto &A : Conf.MAttrs)
134 AddString(A);
135 if (Conf.RelocModel)
136 AddUnsigned(*Conf.RelocModel);
137 else
138 AddUnsigned(-1);
139 if (Conf.CodeModel)
140 AddUnsigned(*Conf.CodeModel);
141 else
142 AddUnsigned(-1);
143 for (const auto &S : Conf.MllvmArgs)
144 AddString(S);
145 AddUnsigned(static_cast<int>(Conf.CGOptLevel));
146 AddUnsigned(static_cast<int>(Conf.CGFileType));
147 AddUnsigned(Conf.OptLevel);
148 AddUnsigned(Conf.Freestanding);
149 AddString(Conf.OptPipeline);
150 AddString(Conf.AAPipeline);
151 AddString(Conf.OverrideTriple);
152 AddString(Conf.DefaultTriple);
153 AddString(Conf.DwoDir);
154
155 // Include the hash for the current module
156 auto ModHash = Index.getModuleHash(ModuleID);
157 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
158
159 std::vector<uint64_t> ExportsGUID;
160 ExportsGUID.reserve(ExportList.size());
161 for (const auto &VI : ExportList) {
162 auto GUID = VI.getGUID();
163 ExportsGUID.push_back(GUID);
164 }
165
166 // Sort the export list elements GUIDs.
167 llvm::sort(ExportsGUID);
168 for (uint64_t GUID : ExportsGUID) {
169 // The export list can impact the internalization, be conservative here
170 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&GUID, sizeof(GUID)));
171 }
172
173 // Include the hash for every module we import functions from. The set of
174 // imported symbols for each module may affect code generation and is
175 // sensitive to link order, so include that as well.
176 using ImportMapIteratorTy = FunctionImporter::ImportMapTy::const_iterator;
177 struct ImportModule {
178 ImportMapIteratorTy ModIt;
179 const ModuleSummaryIndex::ModuleInfo *ModInfo;
180
181 StringRef getIdentifier() const { return ModIt->getFirst(); }
182 const FunctionImporter::FunctionsToImportTy &getFunctions() const {
183 return ModIt->second;
184 }
185
186 const ModuleHash &getHash() const { return ModInfo->second; }
187 };
188
189 std::vector<ImportModule> ImportModulesVector;
190 ImportModulesVector.reserve(ImportList.size());
191
192 for (ImportMapIteratorTy It = ImportList.begin(); It != ImportList.end();
193 ++It) {
194 ImportModulesVector.push_back({It, Index.getModule(It->getFirst())});
195 }
196 // Order using module hash, to be both independent of module name and
197 // module order.
198 llvm::sort(ImportModulesVector,
199 [](const ImportModule &Lhs, const ImportModule &Rhs) -> bool {
200 return Lhs.getHash() < Rhs.getHash();
201 });
202 for (const ImportModule &Entry : ImportModulesVector) {
203 auto ModHash = Entry.getHash();
204 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
205
206 AddUint64(Entry.getFunctions().size());
207 for (auto &Fn : Entry.getFunctions())
208 AddUint64(Fn);
209 }
210
211 // Include the hash for the resolved ODR.
212 for (auto &Entry : ResolvedODR) {
213 Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.first,
214 sizeof(GlobalValue::GUID)));
215 Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.second,
217 }
218
219 // Members of CfiFunctionDefs and CfiFunctionDecls that are referenced or
220 // defined in this module.
221 std::set<GlobalValue::GUID> UsedCfiDefs;
222 std::set<GlobalValue::GUID> UsedCfiDecls;
223
224 // Typeids used in this module.
225 std::set<GlobalValue::GUID> UsedTypeIds;
226
227 auto AddUsedCfiGlobal = [&](GlobalValue::GUID ValueGUID) {
228 if (CfiFunctionDefs.count(ValueGUID))
229 UsedCfiDefs.insert(ValueGUID);
230 if (CfiFunctionDecls.count(ValueGUID))
231 UsedCfiDecls.insert(ValueGUID);
232 };
233
234 auto AddUsedThings = [&](GlobalValueSummary *GS) {
235 if (!GS) return;
236 AddUnsigned(GS->getVisibility());
237 AddUnsigned(GS->isLive());
238 AddUnsigned(GS->canAutoHide());
239 for (const ValueInfo &VI : GS->refs()) {
240 AddUnsigned(VI.isDSOLocal(Index.withDSOLocalPropagation()));
241 AddUsedCfiGlobal(VI.getGUID());
242 }
243 if (auto *GVS = dyn_cast<GlobalVarSummary>(GS)) {
244 AddUnsigned(GVS->maybeReadOnly());
245 AddUnsigned(GVS->maybeWriteOnly());
246 }
247 if (auto *FS = dyn_cast<FunctionSummary>(GS)) {
248 for (auto &TT : FS->type_tests())
249 UsedTypeIds.insert(TT);
250 for (auto &TT : FS->type_test_assume_vcalls())
251 UsedTypeIds.insert(TT.GUID);
252 for (auto &TT : FS->type_checked_load_vcalls())
253 UsedTypeIds.insert(TT.GUID);
254 for (auto &TT : FS->type_test_assume_const_vcalls())
255 UsedTypeIds.insert(TT.VFunc.GUID);
256 for (auto &TT : FS->type_checked_load_const_vcalls())
257 UsedTypeIds.insert(TT.VFunc.GUID);
258 for (auto &ET : FS->calls()) {
259 AddUnsigned(ET.first.isDSOLocal(Index.withDSOLocalPropagation()));
260 AddUsedCfiGlobal(ET.first.getGUID());
261 }
262 }
263 };
264
265 // Include the hash for the linkage type to reflect internalization and weak
266 // resolution, and collect any used type identifier resolutions.
267 for (auto &GS : DefinedGlobals) {
268 GlobalValue::LinkageTypes Linkage = GS.second->linkage();
269 Hasher.update(
270 ArrayRef<uint8_t>((const uint8_t *)&Linkage, sizeof(Linkage)));
271 AddUsedCfiGlobal(GS.first);
272 AddUsedThings(GS.second);
273 }
274
275 // Imported functions may introduce new uses of type identifier resolutions,
276 // so we need to collect their used resolutions as well.
277 for (const ImportModule &ImpM : ImportModulesVector)
278 for (auto &ImpF : ImpM.getFunctions()) {
280 Index.findSummaryInModule(ImpF, ImpM.getIdentifier());
281 AddUsedThings(S);
282 // If this is an alias, we also care about any types/etc. that the aliasee
283 // may reference.
284 if (auto *AS = dyn_cast_or_null<AliasSummary>(S))
285 AddUsedThings(AS->getBaseObject());
286 }
287
288 auto AddTypeIdSummary = [&](StringRef TId, const TypeIdSummary &S) {
289 AddString(TId);
290
291 AddUnsigned(S.TTRes.TheKind);
292 AddUnsigned(S.TTRes.SizeM1BitWidth);
293
294 AddUint64(S.TTRes.AlignLog2);
295 AddUint64(S.TTRes.SizeM1);
296 AddUint64(S.TTRes.BitMask);
297 AddUint64(S.TTRes.InlineBits);
298
299 AddUint64(S.WPDRes.size());
300 for (auto &WPD : S.WPDRes) {
301 AddUnsigned(WPD.first);
302 AddUnsigned(WPD.second.TheKind);
303 AddString(WPD.second.SingleImplName);
304
305 AddUint64(WPD.second.ResByArg.size());
306 for (auto &ByArg : WPD.second.ResByArg) {
307 AddUint64(ByArg.first.size());
308 for (uint64_t Arg : ByArg.first)
309 AddUint64(Arg);
310 AddUnsigned(ByArg.second.TheKind);
311 AddUint64(ByArg.second.Info);
312 AddUnsigned(ByArg.second.Byte);
313 AddUnsigned(ByArg.second.Bit);
314 }
315 }
316 };
317
318 // Include the hash for all type identifiers used by this module.
319 for (GlobalValue::GUID TId : UsedTypeIds) {
320 auto TidIter = Index.typeIds().equal_range(TId);
321 for (auto It = TidIter.first; It != TidIter.second; ++It)
322 AddTypeIdSummary(It->second.first, It->second.second);
323 }
324
325 AddUnsigned(UsedCfiDefs.size());
326 for (auto &V : UsedCfiDefs)
327 AddUint64(V);
328
329 AddUnsigned(UsedCfiDecls.size());
330 for (auto &V : UsedCfiDecls)
331 AddUint64(V);
332
333 if (!Conf.SampleProfile.empty()) {
334 auto FileOrErr = MemoryBuffer::getFile(Conf.SampleProfile);
335 if (FileOrErr) {
336 Hasher.update(FileOrErr.get()->getBuffer());
337
338 if (!Conf.ProfileRemapping.empty()) {
339 FileOrErr = MemoryBuffer::getFile(Conf.ProfileRemapping);
340 if (FileOrErr)
341 Hasher.update(FileOrErr.get()->getBuffer());
342 }
343 }
344 }
345
346 Key = toHex(Hasher.result());
347}
348
350 const Config &C, ValueInfo VI,
351 DenseSet<GlobalValueSummary *> &GlobalInvolvedWithAlias,
353 isPrevailing,
355 recordNewLinkage,
356 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
358 C.VisibilityScheme == Config::ELF ? VI.getELFVisibility()
360 for (auto &S : VI.getSummaryList()) {
361 GlobalValue::LinkageTypes OriginalLinkage = S->linkage();
362 // Ignore local and appending linkage values since the linker
363 // doesn't resolve them.
364 if (GlobalValue::isLocalLinkage(OriginalLinkage) ||
366 continue;
367 // We need to emit only one of these. The prevailing module will keep it,
368 // but turned into a weak, while the others will drop it when possible.
369 // This is both a compile-time optimization and a correctness
370 // transformation. This is necessary for correctness when we have exported
371 // a reference - we need to convert the linkonce to weak to
372 // ensure a copy is kept to satisfy the exported reference.
373 // FIXME: We may want to split the compile time and correctness
374 // aspects into separate routines.
375 if (isPrevailing(VI.getGUID(), S.get())) {
376 if (GlobalValue::isLinkOnceLinkage(OriginalLinkage)) {
377 S->setLinkage(GlobalValue::getWeakLinkage(
378 GlobalValue::isLinkOnceODRLinkage(OriginalLinkage)));
379 // The kept copy is eligible for auto-hiding (hidden visibility) if all
380 // copies were (i.e. they were all linkonce_odr global unnamed addr).
381 // If any copy is not (e.g. it was originally weak_odr), then the symbol
382 // must remain externally available (e.g. a weak_odr from an explicitly
383 // instantiated template). Additionally, if it is in the
384 // GUIDPreservedSymbols set, that means that it is visibile outside
385 // the summary (e.g. in a native object or a bitcode file without
386 // summary), and in that case we cannot hide it as it isn't possible to
387 // check all copies.
388 S->setCanAutoHide(VI.canAutoHide() &&
389 !GUIDPreservedSymbols.count(VI.getGUID()));
390 }
391 if (C.VisibilityScheme == Config::FromPrevailing)
392 Visibility = S->getVisibility();
393 }
394 // Alias and aliasee can't be turned into available_externally.
395 else if (!isa<AliasSummary>(S.get()) &&
396 !GlobalInvolvedWithAlias.count(S.get()))
398
399 // For ELF, set visibility to the computed visibility from summaries. We
400 // don't track visibility from declarations so this may be more relaxed than
401 // the most constraining one.
402 if (C.VisibilityScheme == Config::ELF)
403 S->setVisibility(Visibility);
404
405 if (S->linkage() != OriginalLinkage)
406 recordNewLinkage(S->modulePath(), VI.getGUID(), S->linkage());
407 }
408
409 if (C.VisibilityScheme == Config::FromPrevailing) {
410 for (auto &S : VI.getSummaryList()) {
411 GlobalValue::LinkageTypes OriginalLinkage = S->linkage();
412 if (GlobalValue::isLocalLinkage(OriginalLinkage) ||
414 continue;
415 S->setVisibility(Visibility);
416 }
417 }
418}
419
420/// Resolve linkage for prevailing symbols in the \p Index.
421//
422// We'd like to drop these functions if they are no longer referenced in the
423// current module. However there is a chance that another module is still
424// referencing them because of the import. We make sure we always emit at least
425// one copy.
429 isPrevailing,
431 recordNewLinkage,
432 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
433 // We won't optimize the globals that are referenced by an alias for now
434 // Ideally we should turn the alias into a global and duplicate the definition
435 // when needed.
436 DenseSet<GlobalValueSummary *> GlobalInvolvedWithAlias;
437 for (auto &I : Index)
438 for (auto &S : I.second.SummaryList)
439 if (auto AS = dyn_cast<AliasSummary>(S.get()))
440 GlobalInvolvedWithAlias.insert(&AS->getAliasee());
441
442 for (auto &I : Index)
443 thinLTOResolvePrevailingGUID(C, Index.getValueInfo(I),
444 GlobalInvolvedWithAlias, isPrevailing,
445 recordNewLinkage, GUIDPreservedSymbols);
446}
447
449 ValueInfo VI, function_ref<bool(StringRef, ValueInfo)> isExported,
451 isPrevailing) {
452 auto ExternallyVisibleCopies =
453 llvm::count_if(VI.getSummaryList(),
454 [](const std::unique_ptr<GlobalValueSummary> &Summary) {
455 return !GlobalValue::isLocalLinkage(Summary->linkage());
456 });
457
458 for (auto &S : VI.getSummaryList()) {
459 // First see if we need to promote an internal value because it is not
460 // exported.
461 if (isExported(S->modulePath(), VI)) {
462 if (GlobalValue::isLocalLinkage(S->linkage()))
463 S->setLinkage(GlobalValue::ExternalLinkage);
464 continue;
465 }
466
467 // Otherwise, see if we can internalize.
469 continue;
470
471 // Non-exported values with external linkage can be internalized.
472 if (GlobalValue::isExternalLinkage(S->linkage())) {
473 S->setLinkage(GlobalValue::InternalLinkage);
474 continue;
475 }
476
477 // Non-exported function and variable definitions with a weak-for-linker
478 // linkage can be internalized in certain cases. The minimum legality
479 // requirements would be that they are not address taken to ensure that we
480 // don't break pointer equality checks, and that variables are either read-
481 // or write-only. For functions, this is the case if either all copies are
482 // [local_]unnamed_addr, or we can propagate reference edge attributes
483 // (which is how this is guaranteed for variables, when analyzing whether
484 // they are read or write-only).
485 //
486 // However, we only get to this code for weak-for-linkage values in one of
487 // two cases:
488 // 1) The prevailing copy is not in IR (it is in native code).
489 // 2) The prevailing copy in IR is not exported from its module.
490 // Additionally, at least for the new LTO API, case 2 will only happen if
491 // there is exactly one definition of the value (i.e. in exactly one
492 // module), as duplicate defs are result in the value being marked exported.
493 // Likely, users of the legacy LTO API are similar, however, currently there
494 // are llvm-lto based tests of the legacy LTO API that do not mark
495 // duplicate linkonce_odr copies as exported via the tool, so we need
496 // to handle that case below by checking the number of copies.
497 //
498 // Generally, we only want to internalize a weak-for-linker value in case
499 // 2, because in case 1 we cannot see how the value is used to know if it
500 // is read or write-only. We also don't want to bloat the binary with
501 // multiple internalized copies of non-prevailing linkonce/weak functions.
502 // Note if we don't internalize, we will convert non-prevailing copies to
503 // available_externally anyway, so that we drop them after inlining. The
504 // only reason to internalize such a function is if we indeed have a single
505 // copy, because internalizing it won't increase binary size, and enables
506 // use of inliner heuristics that are more aggressive in the face of a
507 // single call to a static (local). For variables, internalizing a read or
508 // write only variable can enable more aggressive optimization. However, we
509 // already perform this elsewhere in the ThinLTO backend handling for
510 // read or write-only variables (processGlobalForThinLTO).
511 //
512 // Therefore, only internalize linkonce/weak if there is a single copy, that
513 // is prevailing in this IR module. We can do so aggressively, without
514 // requiring the address to be insignificant, or that a variable be read or
515 // write-only.
516 if (!GlobalValue::isWeakForLinker(S->linkage()) ||
518 continue;
519
520 if (isPrevailing(VI.getGUID(), S.get()) && ExternallyVisibleCopies == 1)
521 S->setLinkage(GlobalValue::InternalLinkage);
522 }
523}
524
525// Update the linkages in the given \p Index to mark exported values
526// as external and non-exported values as internal.
529 function_ref<bool(StringRef, ValueInfo)> isExported,
531 isPrevailing) {
532 for (auto &I : Index)
533 thinLTOInternalizeAndPromoteGUID(Index.getValueInfo(I), isExported,
534 isPrevailing);
535}
536
537// Requires a destructor for std::vector<InputModule>.
538InputFile::~InputFile() = default;
539
541 std::unique_ptr<InputFile> File(new InputFile);
542
543 Expected<IRSymtabFile> FOrErr = readIRSymtab(Object);
544 if (!FOrErr)
545 return FOrErr.takeError();
546
547 File->TargetTriple = FOrErr->TheReader.getTargetTriple();
548 File->SourceFileName = FOrErr->TheReader.getSourceFileName();
549 File->COFFLinkerOpts = FOrErr->TheReader.getCOFFLinkerOpts();
550 File->DependentLibraries = FOrErr->TheReader.getDependentLibraries();
551 File->ComdatTable = FOrErr->TheReader.getComdatTable();
552
553 for (unsigned I = 0; I != FOrErr->Mods.size(); ++I) {
554 size_t Begin = File->Symbols.size();
555 for (const irsymtab::Reader::SymbolRef &Sym :
556 FOrErr->TheReader.module_symbols(I))
557 // Skip symbols that are irrelevant to LTO. Note that this condition needs
558 // to match the one in Skip() in LTO::addRegularLTO().
559 if (Sym.isGlobal() && !Sym.isFormatSpecific())
560 File->Symbols.push_back(Sym);
561 File->ModuleSymIndices.push_back({Begin, File->Symbols.size()});
562 }
563
564 File->Mods = FOrErr->Mods;
565 File->Strtab = std::move(FOrErr->Strtab);
566 return std::move(File);
567}
568
570 return Mods[0].getModuleIdentifier();
571}
572
574 assert(Mods.size() == 1 && "Expect only one bitcode module");
575 return Mods[0];
576}
577
578LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel,
579 const Config &Conf)
580 : ParallelCodeGenParallelismLevel(ParallelCodeGenParallelismLevel),
581 Ctx(Conf), CombinedModule(std::make_unique<Module>("ld-temp.o", Ctx)),
582 Mover(std::make_unique<IRMover>(*CombinedModule)) {}
583
584LTO::ThinLTOState::ThinLTOState(ThinBackend Backend)
585 : Backend(Backend), CombinedIndex(/*HaveGVs*/ false) {
586 if (!Backend)
587 this->Backend =
589}
590
592 unsigned ParallelCodeGenParallelismLevel, LTOKind LTOMode)
593 : Conf(std::move(Conf)),
594 RegularLTO(ParallelCodeGenParallelismLevel, this->Conf),
595 ThinLTO(std::move(Backend)), LTOMode(LTOMode) {}
596
597// Requires a destructor for MapVector<BitcodeModule>.
598LTO::~LTO() = default;
599
600// Add the symbols in the given module to the GlobalResolutions map, and resolve
601// their partitions.
602void LTO::addModuleToGlobalRes(ArrayRef<InputFile::Symbol> Syms,
604 unsigned Partition, bool InSummary) {
605 auto *ResI = Res.begin();
606 auto *ResE = Res.end();
607 (void)ResE;
608 const Triple TT(RegularLTO.CombinedModule->getTargetTriple());
609 for (const InputFile::Symbol &Sym : Syms) {
610 assert(ResI != ResE);
611 SymbolResolution Res = *ResI++;
612
613 auto &GlobalRes = GlobalResolutions[Sym.getName()];
614 GlobalRes.UnnamedAddr &= Sym.isUnnamedAddr();
615 if (Res.Prevailing) {
616 assert(!GlobalRes.Prevailing &&
617 "Multiple prevailing defs are not allowed");
618 GlobalRes.Prevailing = true;
619 GlobalRes.IRName = std::string(Sym.getIRName());
620 } else if (!GlobalRes.Prevailing && GlobalRes.IRName.empty()) {
621 // Sometimes it can be two copies of symbol in a module and prevailing
622 // symbol can have no IR name. That might happen if symbol is defined in
623 // module level inline asm block. In case we have multiple modules with
624 // the same symbol we want to use IR name of the prevailing symbol.
625 // Otherwise, if we haven't seen a prevailing symbol, set the name so that
626 // we can later use it to check if there is any prevailing copy in IR.
627 GlobalRes.IRName = std::string(Sym.getIRName());
628 }
629
630 // In rare occasion, the symbol used to initialize GlobalRes has a different
631 // IRName from the inspected Symbol. This can happen on macOS + iOS, when a
632 // symbol is referenced through its mangled name, say @"\01_symbol" while
633 // the IRName is @symbol (the prefix underscore comes from MachO mangling).
634 // In that case, we have the same actual Symbol that can get two different
635 // GUID, leading to some invalid internalization. Workaround this by marking
636 // the GlobalRes external.
637
638 // FIXME: instead of this check, it would be desirable to compute GUIDs
639 // based on mangled name, but this requires an access to the Target Triple
640 // and would be relatively invasive on the codebase.
641 if (GlobalRes.IRName != Sym.getIRName()) {
642 GlobalRes.Partition = GlobalResolution::External;
643 GlobalRes.VisibleOutsideSummary = true;
644 }
645
646 // Set the partition to external if we know it is re-defined by the linker
647 // with -defsym or -wrap options, used elsewhere, e.g. it is visible to a
648 // regular object, is referenced from llvm.compiler.used/llvm.used, or was
649 // already recorded as being referenced from a different partition.
650 if (Res.LinkerRedefined || Res.VisibleToRegularObj || Sym.isUsed() ||
651 (GlobalRes.Partition != GlobalResolution::Unknown &&
652 GlobalRes.Partition != Partition)) {
653 GlobalRes.Partition = GlobalResolution::External;
654 } else
655 // First recorded reference, save the current partition.
656 GlobalRes.Partition = Partition;
657
658 // Flag as visible outside of summary if visible from a regular object or
659 // from a module that does not have a summary.
660 GlobalRes.VisibleOutsideSummary |=
661 (Res.VisibleToRegularObj || Sym.isUsed() || !InSummary);
662
663 GlobalRes.ExportDynamic |= Res.ExportDynamic;
664 }
665}
666
669 StringRef Path = Input->getName();
670 OS << Path << '\n';
671 auto ResI = Res.begin();
672 for (const InputFile::Symbol &Sym : Input->symbols()) {
673 assert(ResI != Res.end());
674 SymbolResolution Res = *ResI++;
675
676 OS << "-r=" << Path << ',' << Sym.getName() << ',';
677 if (Res.Prevailing)
678 OS << 'p';
680 OS << 'l';
681 if (Res.VisibleToRegularObj)
682 OS << 'x';
683 if (Res.LinkerRedefined)
684 OS << 'r';
685 OS << '\n';
686 }
687 OS.flush();
688 assert(ResI == Res.end());
689}
690
691Error LTO::add(std::unique_ptr<InputFile> Input,
693 assert(!CalledGetMaxTasks);
694
695 if (Conf.ResolutionFile)
696 writeToResolutionFile(*Conf.ResolutionFile, Input.get(), Res);
697
698 if (RegularLTO.CombinedModule->getTargetTriple().empty()) {
699 RegularLTO.CombinedModule->setTargetTriple(Input->getTargetTriple());
700 if (Triple(Input->getTargetTriple()).isOSBinFormatELF())
702 }
703
704 const SymbolResolution *ResI = Res.begin();
705 for (unsigned I = 0; I != Input->Mods.size(); ++I)
706 if (Error Err = addModule(*Input, I, ResI, Res.end()))
707 return Err;
708
709 assert(ResI == Res.end());
710 return Error::success();
711}
712
713Error LTO::addModule(InputFile &Input, unsigned ModI,
714 const SymbolResolution *&ResI,
715 const SymbolResolution *ResE) {
716 Expected<BitcodeLTOInfo> LTOInfo = Input.Mods[ModI].getLTOInfo();
717 if (!LTOInfo)
718 return LTOInfo.takeError();
719
720 if (EnableSplitLTOUnit) {
721 // If only some modules were split, flag this in the index so that
722 // we can skip or error on optimizations that need consistently split
723 // modules (whole program devirt and lower type tests).
724 if (*EnableSplitLTOUnit != LTOInfo->EnableSplitLTOUnit)
725 ThinLTO.CombinedIndex.setPartiallySplitLTOUnits();
726 } else
727 EnableSplitLTOUnit = LTOInfo->EnableSplitLTOUnit;
728
729 BitcodeModule BM = Input.Mods[ModI];
730
731 if ((LTOMode == LTOK_UnifiedRegular || LTOMode == LTOK_UnifiedThin) &&
732 !LTOInfo->UnifiedLTO)
733 return make_error<StringError>(
734 "unified LTO compilation must use "
735 "compatible bitcode modules (use -funified-lto)",
737
738 if (LTOInfo->UnifiedLTO && LTOMode == LTOK_Default)
739 LTOMode = LTOK_UnifiedThin;
740
741 bool IsThinLTO = LTOInfo->IsThinLTO && (LTOMode != LTOK_UnifiedRegular);
742
743 auto ModSyms = Input.module_symbols(ModI);
744 addModuleToGlobalRes(ModSyms, {ResI, ResE},
745 IsThinLTO ? ThinLTO.ModuleMap.size() + 1 : 0,
746 LTOInfo->HasSummary);
747
748 if (IsThinLTO)
749 return addThinLTO(BM, ModSyms, ResI, ResE);
750
751 RegularLTO.EmptyCombinedModule = false;
753 addRegularLTO(BM, ModSyms, ResI, ResE);
754 if (!ModOrErr)
755 return ModOrErr.takeError();
756
757 if (!LTOInfo->HasSummary)
758 return linkRegularLTO(std::move(*ModOrErr), /*LivenessFromIndex=*/false);
759
760 // Regular LTO module summaries are added to a dummy module that represents
761 // the combined regular LTO module.
762 if (Error Err = BM.readSummary(ThinLTO.CombinedIndex, ""))
763 return Err;
764 RegularLTO.ModsWithSummaries.push_back(std::move(*ModOrErr));
765 return Error::success();
766}
767
768// Checks whether the given global value is in a non-prevailing comdat
769// (comdat containing values the linker indicated were not prevailing,
770// which we then dropped to available_externally), and if so, removes
771// it from the comdat. This is called for all global values to ensure the
772// comdat is empty rather than leaving an incomplete comdat. It is needed for
773// regular LTO modules, in case we are in a mixed-LTO mode (both regular
774// and thin LTO modules) compilation. Since the regular LTO module will be
775// linked first in the final native link, we want to make sure the linker
776// doesn't select any of these incomplete comdats that would be left
777// in the regular LTO module without this cleanup.
778static void
780 std::set<const Comdat *> &NonPrevailingComdats) {
781 Comdat *C = GV.getComdat();
782 if (!C)
783 return;
784
785 if (!NonPrevailingComdats.count(C))
786 return;
787
788 // Additionally need to drop all global values from the comdat to
789 // available_externally, to satisfy the COMDAT requirement that all members
790 // are discarded as a unit. The non-local linkage global values avoid
791 // duplicate definition linker errors.
793
794 if (auto GO = dyn_cast<GlobalObject>(&GV))
795 GO->setComdat(nullptr);
796}
797
798// Add a regular LTO object to the link.
799// The resulting module needs to be linked into the combined LTO module with
800// linkRegularLTO.
802LTO::addRegularLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
803 const SymbolResolution *&ResI,
804 const SymbolResolution *ResE) {
805 RegularLTOState::AddedModule Mod;
807 BM.getLazyModule(RegularLTO.Ctx, /*ShouldLazyLoadMetadata*/ true,
808 /*IsImporting*/ false);
809 if (!MOrErr)
810 return MOrErr.takeError();
811 Module &M = **MOrErr;
812 Mod.M = std::move(*MOrErr);
813
814 if (Error Err = M.materializeMetadata())
815 return std::move(Err);
816
817 // If cfi.functions is present and we are in regular LTO mode, LowerTypeTests
818 // will rename local functions in the merged module as "<function name>.1".
819 // This causes linking errors, since other parts of the module expect the
820 // original function name.
821 if (LTOMode == LTOK_UnifiedRegular)
822 if (NamedMDNode *CfiFunctionsMD = M.getNamedMetadata("cfi.functions"))
823 M.eraseNamedMetadata(CfiFunctionsMD);
824
826
827 ModuleSymbolTable SymTab;
828 SymTab.addModule(&M);
829
830 for (GlobalVariable &GV : M.globals())
831 if (GV.hasAppendingLinkage())
832 Mod.Keep.push_back(&GV);
833
834 DenseSet<GlobalObject *> AliasedGlobals;
835 for (auto &GA : M.aliases())
836 if (GlobalObject *GO = GA.getAliaseeObject())
837 AliasedGlobals.insert(GO);
838
839 // In this function we need IR GlobalValues matching the symbols in Syms
840 // (which is not backed by a module), so we need to enumerate them in the same
841 // order. The symbol enumeration order of a ModuleSymbolTable intentionally
842 // matches the order of an irsymtab, but when we read the irsymtab in
843 // InputFile::create we omit some symbols that are irrelevant to LTO. The
844 // Skip() function skips the same symbols from the module as InputFile does
845 // from the symbol table.
846 auto MsymI = SymTab.symbols().begin(), MsymE = SymTab.symbols().end();
847 auto Skip = [&]() {
848 while (MsymI != MsymE) {
849 auto Flags = SymTab.getSymbolFlags(*MsymI);
850 if ((Flags & object::BasicSymbolRef::SF_Global) &&
852 return;
853 ++MsymI;
854 }
855 };
856 Skip();
857
858 std::set<const Comdat *> NonPrevailingComdats;
859 SmallSet<StringRef, 2> NonPrevailingAsmSymbols;
860 for (const InputFile::Symbol &Sym : Syms) {
861 assert(ResI != ResE);
862 SymbolResolution Res = *ResI++;
863
864 assert(MsymI != MsymE);
865 ModuleSymbolTable::Symbol Msym = *MsymI++;
866 Skip();
867
868 if (GlobalValue *GV = dyn_cast_if_present<GlobalValue *>(Msym)) {
869 if (Res.Prevailing) {
870 if (Sym.isUndefined())
871 continue;
872 Mod.Keep.push_back(GV);
873 // For symbols re-defined with linker -wrap and -defsym options,
874 // set the linkage to weak to inhibit IPO. The linkage will be
875 // restored by the linker.
876 if (Res.LinkerRedefined)
877 GV->setLinkage(GlobalValue::WeakAnyLinkage);
878
879 GlobalValue::LinkageTypes OriginalLinkage = GV->getLinkage();
880 if (GlobalValue::isLinkOnceLinkage(OriginalLinkage))
881 GV->setLinkage(GlobalValue::getWeakLinkage(
882 GlobalValue::isLinkOnceODRLinkage(OriginalLinkage)));
883 } else if (isa<GlobalObject>(GV) &&
884 (GV->hasLinkOnceODRLinkage() || GV->hasWeakODRLinkage() ||
885 GV->hasAvailableExternallyLinkage()) &&
886 !AliasedGlobals.count(cast<GlobalObject>(GV))) {
887 // Any of the above three types of linkage indicates that the
888 // chosen prevailing symbol will have the same semantics as this copy of
889 // the symbol, so we may be able to link it with available_externally
890 // linkage. We will decide later whether to do that when we link this
891 // module (in linkRegularLTO), based on whether it is undefined.
892 Mod.Keep.push_back(GV);
894 if (GV->hasComdat())
895 NonPrevailingComdats.insert(GV->getComdat());
896 cast<GlobalObject>(GV)->setComdat(nullptr);
897 }
898
899 // Set the 'local' flag based on the linker resolution for this symbol.
901 GV->setDSOLocal(true);
902 if (GV->hasDLLImportStorageClass())
903 GV->setDLLStorageClass(GlobalValue::DLLStorageClassTypes::
904 DefaultStorageClass);
905 }
906 } else if (auto *AS =
907 dyn_cast_if_present<ModuleSymbolTable::AsmSymbol *>(Msym)) {
908 // Collect non-prevailing symbols.
909 if (!Res.Prevailing)
910 NonPrevailingAsmSymbols.insert(AS->first);
911 } else {
912 llvm_unreachable("unknown symbol type");
913 }
914
915 // Common resolution: collect the maximum size/alignment over all commons.
916 // We also record if we see an instance of a common as prevailing, so that
917 // if none is prevailing we can ignore it later.
918 if (Sym.isCommon()) {
919 // FIXME: We should figure out what to do about commons defined by asm.
920 // For now they aren't reported correctly by ModuleSymbolTable.
921 auto &CommonRes = RegularLTO.Commons[std::string(Sym.getIRName())];
922 CommonRes.Size = std::max(CommonRes.Size, Sym.getCommonSize());
923 if (uint32_t SymAlignValue = Sym.getCommonAlignment()) {
924 CommonRes.Alignment =
925 std::max(Align(SymAlignValue), CommonRes.Alignment);
926 }
927 CommonRes.Prevailing |= Res.Prevailing;
928 }
929 }
930
931 if (!M.getComdatSymbolTable().empty())
932 for (GlobalValue &GV : M.global_values())
933 handleNonPrevailingComdat(GV, NonPrevailingComdats);
934
935 // Prepend ".lto_discard <sym>, <sym>*" directive to each module inline asm
936 // block.
937 if (!M.getModuleInlineAsm().empty()) {
938 std::string NewIA = ".lto_discard";
939 if (!NonPrevailingAsmSymbols.empty()) {
940 // Don't dicard a symbol if there is a live .symver for it.
942 M, [&](StringRef Name, StringRef Alias) {
943 if (!NonPrevailingAsmSymbols.count(Alias))
944 NonPrevailingAsmSymbols.erase(Name);
945 });
946 NewIA += " " + llvm::join(NonPrevailingAsmSymbols, ", ");
947 }
948 NewIA += "\n";
949 M.setModuleInlineAsm(NewIA + M.getModuleInlineAsm());
950 }
951
952 assert(MsymI == MsymE);
953 return std::move(Mod);
954}
955
956Error LTO::linkRegularLTO(RegularLTOState::AddedModule Mod,
957 bool LivenessFromIndex) {
958 std::vector<GlobalValue *> Keep;
959 for (GlobalValue *GV : Mod.Keep) {
960 if (LivenessFromIndex && !ThinLTO.CombinedIndex.isGUIDLive(GV->getGUID())) {
961 if (Function *F = dyn_cast<Function>(GV)) {
962 if (DiagnosticOutputFile) {
963 if (Error Err = F->materialize())
964 return Err;
965 OptimizationRemarkEmitter ORE(F, nullptr);
966 ORE.emit(OptimizationRemark(DEBUG_TYPE, "deadfunction", F)
967 << ore::NV("Function", F)
968 << " not added to the combined module ");
969 }
970 }
971 continue;
972 }
973
975 Keep.push_back(GV);
976 continue;
977 }
978
979 // Only link available_externally definitions if we don't already have a
980 // definition.
981 GlobalValue *CombinedGV =
982 RegularLTO.CombinedModule->getNamedValue(GV->getName());
983 if (CombinedGV && !CombinedGV->isDeclaration())
984 continue;
985
986 Keep.push_back(GV);
987 }
988
989 return RegularLTO.Mover->move(std::move(Mod.M), Keep, nullptr,
990 /* IsPerformingImport */ false);
991}
992
993// Add a ThinLTO module to the link.
994Error LTO::addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
995 const SymbolResolution *&ResI,
996 const SymbolResolution *ResE) {
997 const SymbolResolution *ResITmp = ResI;
998 for (const InputFile::Symbol &Sym : Syms) {
999 assert(ResITmp != ResE);
1000 SymbolResolution Res = *ResITmp++;
1001
1002 if (!Sym.getIRName().empty()) {
1004 Sym.getIRName(), GlobalValue::ExternalLinkage, ""));
1005 if (Res.Prevailing)
1006 ThinLTO.PrevailingModuleForGUID[GUID] = BM.getModuleIdentifier();
1007 }
1008 }
1009
1010 if (Error Err =
1011 BM.readSummary(ThinLTO.CombinedIndex, BM.getModuleIdentifier(),
1012 [&](GlobalValue::GUID GUID) {
1013 return ThinLTO.PrevailingModuleForGUID[GUID] ==
1014 BM.getModuleIdentifier();
1015 }))
1016 return Err;
1017 LLVM_DEBUG(dbgs() << "Module " << BM.getModuleIdentifier() << "\n");
1018
1019 for (const InputFile::Symbol &Sym : Syms) {
1020 assert(ResI != ResE);
1021 SymbolResolution Res = *ResI++;
1022
1023 if (!Sym.getIRName().empty()) {
1025 Sym.getIRName(), GlobalValue::ExternalLinkage, ""));
1026 if (Res.Prevailing) {
1027 assert(ThinLTO.PrevailingModuleForGUID[GUID] ==
1028 BM.getModuleIdentifier());
1029
1030 // For linker redefined symbols (via --wrap or --defsym) we want to
1031 // switch the linkage to `weak` to prevent IPOs from happening.
1032 // Find the summary in the module for this very GV and record the new
1033 // linkage so that we can switch it when we import the GV.
1034 if (Res.LinkerRedefined)
1035 if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(
1036 GUID, BM.getModuleIdentifier()))
1037 S->setLinkage(GlobalValue::WeakAnyLinkage);
1038 }
1039
1040 // If the linker resolved the symbol to a local definition then mark it
1041 // as local in the summary for the module we are adding.
1043 if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(
1044 GUID, BM.getModuleIdentifier())) {
1045 S->setDSOLocal(true);
1046 }
1047 }
1048 }
1049 }
1050
1051 if (!ThinLTO.ModuleMap.insert({BM.getModuleIdentifier(), BM}).second)
1052 return make_error<StringError>(
1053 "Expected at most one ThinLTO module per bitcode file",
1055
1056 if (!Conf.ThinLTOModulesToCompile.empty()) {
1057 if (!ThinLTO.ModulesToCompile)
1058 ThinLTO.ModulesToCompile = ModuleMapType();
1059 // This is a fuzzy name matching where only modules with name containing the
1060 // specified switch values are going to be compiled.
1061 for (const std::string &Name : Conf.ThinLTOModulesToCompile) {
1062 if (BM.getModuleIdentifier().contains(Name)) {
1063 ThinLTO.ModulesToCompile->insert({BM.getModuleIdentifier(), BM});
1064 llvm::errs() << "[ThinLTO] Selecting " << BM.getModuleIdentifier()
1065 << " to compile\n";
1066 }
1067 }
1068 }
1069
1070 return Error::success();
1071}
1072
1073unsigned LTO::getMaxTasks() const {
1074 CalledGetMaxTasks = true;
1075 auto ModuleCount = ThinLTO.ModulesToCompile ? ThinLTO.ModulesToCompile->size()
1076 : ThinLTO.ModuleMap.size();
1077 return RegularLTO.ParallelCodeGenParallelismLevel + ModuleCount;
1078}
1079
1080// If only some of the modules were split, we cannot correctly handle
1081// code that contains type tests or type checked loads.
1082Error LTO::checkPartiallySplit() {
1083 if (!ThinLTO.CombinedIndex.partiallySplitLTOUnits())
1084 return Error::success();
1085
1086 Function *TypeTestFunc = RegularLTO.CombinedModule->getFunction(
1087 Intrinsic::getName(Intrinsic::type_test));
1088 Function *TypeCheckedLoadFunc = RegularLTO.CombinedModule->getFunction(
1089 Intrinsic::getName(Intrinsic::type_checked_load));
1090 Function *TypeCheckedLoadRelativeFunc =
1091 RegularLTO.CombinedModule->getFunction(
1092 Intrinsic::getName(Intrinsic::type_checked_load_relative));
1093
1094 // First check if there are type tests / type checked loads in the
1095 // merged regular LTO module IR.
1096 if ((TypeTestFunc && !TypeTestFunc->use_empty()) ||
1097 (TypeCheckedLoadFunc && !TypeCheckedLoadFunc->use_empty()) ||
1098 (TypeCheckedLoadRelativeFunc &&
1099 !TypeCheckedLoadRelativeFunc->use_empty()))
1100 return make_error<StringError>(
1101 "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",
1103
1104 // Otherwise check if there are any recorded in the combined summary from the
1105 // ThinLTO modules.
1106 for (auto &P : ThinLTO.CombinedIndex) {
1107 for (auto &S : P.second.SummaryList) {
1108 auto *FS = dyn_cast<FunctionSummary>(S.get());
1109 if (!FS)
1110 continue;
1111 if (!FS->type_test_assume_vcalls().empty() ||
1112 !FS->type_checked_load_vcalls().empty() ||
1113 !FS->type_test_assume_const_vcalls().empty() ||
1114 !FS->type_checked_load_const_vcalls().empty() ||
1115 !FS->type_tests().empty())
1116 return make_error<StringError>(
1117 "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",
1119 }
1120 }
1121 return Error::success();
1122}
1123
1125 // Compute "dead" symbols, we don't want to import/export these!
1126 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1127 DenseMap<GlobalValue::GUID, PrevailingType> GUIDPrevailingResolutions;
1128 for (auto &Res : GlobalResolutions) {
1129 // Normally resolution have IR name of symbol. We can do nothing here
1130 // otherwise. See comments in GlobalResolution struct for more details.
1131 if (Res.second.IRName.empty())
1132 continue;
1133
1135 GlobalValue::dropLLVMManglingEscape(Res.second.IRName));
1136
1137 if (Res.second.VisibleOutsideSummary && Res.second.Prevailing)
1138 GUIDPreservedSymbols.insert(GUID);
1139
1140 if (Res.second.ExportDynamic)
1141 DynamicExportSymbols.insert(GUID);
1142
1143 GUIDPrevailingResolutions[GUID] =
1145 }
1146
1147 auto isPrevailing = [&](GlobalValue::GUID G) {
1148 auto It = GUIDPrevailingResolutions.find(G);
1149 if (It == GUIDPrevailingResolutions.end())
1151 return It->second;
1152 };
1153 computeDeadSymbolsWithConstProp(ThinLTO.CombinedIndex, GUIDPreservedSymbols,
1154 isPrevailing, Conf.OptLevel > 0);
1155
1156 // Setup output file to emit statistics.
1157 auto StatsFileOrErr = setupStatsFile(Conf.StatsFile);
1158 if (!StatsFileOrErr)
1159 return StatsFileOrErr.takeError();
1160 std::unique_ptr<ToolOutputFile> StatsFile = std::move(StatsFileOrErr.get());
1161
1162 // TODO: Ideally this would be controlled automatically by detecting that we
1163 // are linking with an allocator that supports these interfaces, rather than
1164 // an internal option (which would still be needed for tests, however). For
1165 // example, if the library exported a symbol like __malloc_hot_cold the linker
1166 // could recognize that and set a flag in the lto::Config.
1168 ThinLTO.CombinedIndex.setWithSupportsHotColdNew();
1169
1170 Error Result = runRegularLTO(AddStream);
1171 if (!Result)
1172 Result = runThinLTO(AddStream, Cache, GUIDPreservedSymbols);
1173
1174 if (StatsFile)
1175 PrintStatisticsJSON(StatsFile->os());
1176
1177 return Result;
1178}
1179
1181 const ModuleSummaryIndex &Index) {
1182 if (Index.withSupportsHotColdNew())
1183 return;
1184
1185 // The profile matcher applies hotness attributes directly for allocations,
1186 // and those will cause us to generate calls to the hot/cold interfaces
1187 // unconditionally. If supports-hot-cold-new was not enabled in the LTO
1188 // link then assume we don't want these calls (e.g. not linking with
1189 // the appropriate library, or otherwise trying to disable this behavior).
1190 for (auto &F : Mod) {
1191 for (auto &BB : F) {
1192 for (auto &I : BB) {
1193 auto *CI = dyn_cast<CallBase>(&I);
1194 if (!CI)
1195 continue;
1196 if (CI->hasFnAttr("memprof"))
1197 CI->removeFnAttr("memprof");
1198 // Strip off all memprof metadata as it is no longer needed.
1199 // Importantly, this avoids the addition of new memprof attributes
1200 // after inlining propagation.
1201 // TODO: If we support additional types of MemProf metadata beyond hot
1202 // and cold, we will need to update the metadata based on the allocator
1203 // APIs supported instead of completely stripping all.
1204 CI->setMetadata(LLVMContext::MD_memprof, nullptr);
1205 CI->setMetadata(LLVMContext::MD_callsite, nullptr);
1206 }
1207 }
1208 }
1209}
1210
1211Error LTO::runRegularLTO(AddStreamFn AddStream) {
1212 // Setup optimization remarks.
1213 auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
1214 RegularLTO.CombinedModule->getContext(), Conf.RemarksFilename,
1217 LLVM_DEBUG(dbgs() << "Running regular LTO\n");
1218 if (!DiagFileOrErr)
1219 return DiagFileOrErr.takeError();
1220 DiagnosticOutputFile = std::move(*DiagFileOrErr);
1221
1222 // Finalize linking of regular LTO modules containing summaries now that
1223 // we have computed liveness information.
1224 for (auto &M : RegularLTO.ModsWithSummaries)
1225 if (Error Err = linkRegularLTO(std::move(M),
1226 /*LivenessFromIndex=*/true))
1227 return Err;
1228
1229 // Ensure we don't have inconsistently split LTO units with type tests.
1230 // FIXME: this checks both LTO and ThinLTO. It happens to work as we take
1231 // this path both cases but eventually this should be split into two and
1232 // do the ThinLTO checks in `runThinLTO`.
1233 if (Error Err = checkPartiallySplit())
1234 return Err;
1235
1236 // Make sure commons have the right size/alignment: we kept the largest from
1237 // all the prevailing when adding the inputs, and we apply it here.
1238 const DataLayout &DL = RegularLTO.CombinedModule->getDataLayout();
1239 for (auto &I : RegularLTO.Commons) {
1240 if (!I.second.Prevailing)
1241 // Don't do anything if no instance of this common was prevailing.
1242 continue;
1243 GlobalVariable *OldGV = RegularLTO.CombinedModule->getNamedGlobal(I.first);
1244 if (OldGV && DL.getTypeAllocSize(OldGV->getValueType()) == I.second.Size) {
1245 // Don't create a new global if the type is already correct, just make
1246 // sure the alignment is correct.
1247 OldGV->setAlignment(I.second.Alignment);
1248 continue;
1249 }
1250 ArrayType *Ty =
1251 ArrayType::get(Type::getInt8Ty(RegularLTO.Ctx), I.second.Size);
1252 auto *GV = new GlobalVariable(*RegularLTO.CombinedModule, Ty, false,
1255 GV->setAlignment(I.second.Alignment);
1256 if (OldGV) {
1257 OldGV->replaceAllUsesWith(GV);
1258 GV->takeName(OldGV);
1259 OldGV->eraseFromParent();
1260 } else {
1261 GV->setName(I.first);
1262 }
1263 }
1264
1265 updateMemProfAttributes(*RegularLTO.CombinedModule, ThinLTO.CombinedIndex);
1266
1267 bool WholeProgramVisibilityEnabledInLTO =
1269 // If validation is enabled, upgrade visibility only when all vtables
1270 // have typeinfos.
1272
1273 // This returns true when the name is local or not defined. Locals are
1274 // expected to be handled separately.
1275 auto IsVisibleToRegularObj = [&](StringRef name) {
1276 auto It = GlobalResolutions.find(name);
1277 return (It == GlobalResolutions.end() || It->second.VisibleOutsideSummary);
1278 };
1279
1280 // If allowed, upgrade public vcall visibility metadata to linkage unit
1281 // visibility before whole program devirtualization in the optimizer.
1283 *RegularLTO.CombinedModule, WholeProgramVisibilityEnabledInLTO,
1284 DynamicExportSymbols, Conf.ValidateAllVtablesHaveTypeInfos,
1285 IsVisibleToRegularObj);
1286 updatePublicTypeTestCalls(*RegularLTO.CombinedModule,
1287 WholeProgramVisibilityEnabledInLTO);
1288
1289 if (Conf.PreOptModuleHook &&
1290 !Conf.PreOptModuleHook(0, *RegularLTO.CombinedModule))
1291 return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
1292
1293 if (!Conf.CodeGenOnly) {
1294 for (const auto &R : GlobalResolutions) {
1295 GlobalValue *GV =
1296 RegularLTO.CombinedModule->getNamedValue(R.second.IRName);
1297 if (!R.second.isPrevailingIRSymbol())
1298 continue;
1299 if (R.second.Partition != 0 &&
1300 R.second.Partition != GlobalResolution::External)
1301 continue;
1302
1303 // Ignore symbols defined in other partitions.
1304 // Also skip declarations, which are not allowed to have internal linkage.
1305 if (!GV || GV->hasLocalLinkage() || GV->isDeclaration())
1306 continue;
1307
1308 // Symbols that are marked DLLImport or DLLExport should not be
1309 // internalized, as they are either externally visible or referencing
1310 // external symbols. Symbols that have AvailableExternally or Appending
1311 // linkage might be used by future passes and should be kept as is.
1312 // These linkages are seen in Unified regular LTO, because the process
1313 // of creating split LTO units introduces symbols with that linkage into
1314 // one of the created modules. Normally, only the ThinLTO backend would
1315 // compile this module, but Unified Regular LTO processes both
1316 // modules created by the splitting process as regular LTO modules.
1317 if ((LTOMode == LTOKind::LTOK_UnifiedRegular) &&
1320 continue;
1321
1322 GV->setUnnamedAddr(R.second.UnnamedAddr ? GlobalValue::UnnamedAddr::Global
1324 if (EnableLTOInternalization && R.second.Partition == 0)
1326 }
1327
1328 if (Conf.PostInternalizeModuleHook &&
1329 !Conf.PostInternalizeModuleHook(0, *RegularLTO.CombinedModule))
1330 return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
1331 }
1332
1333 if (!RegularLTO.EmptyCombinedModule || Conf.AlwaysEmitRegularLTOObj) {
1334 if (Error Err =
1335 backend(Conf, AddStream, RegularLTO.ParallelCodeGenParallelismLevel,
1336 *RegularLTO.CombinedModule, ThinLTO.CombinedIndex))
1337 return Err;
1338 }
1339
1340 return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
1341}
1342
1343static const char *libcallRoutineNames[] = {
1344#define HANDLE_LIBCALL(code, name) name,
1345#include "llvm/IR/RuntimeLibcalls.def"
1346#undef HANDLE_LIBCALL
1347};
1348
1351}
1352
1353/// This class defines the interface to the ThinLTO backend.
1355protected:
1356 const Config &Conf;
1361
1362public:
1364 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1365 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1366 lto::IndexWriteCallback OnWrite, bool ShouldEmitImportsFiles)
1367 : Conf(Conf), CombinedIndex(CombinedIndex),
1368 ModuleToDefinedGVSummaries(ModuleToDefinedGVSummaries),
1369 OnWrite(OnWrite), ShouldEmitImportsFiles(ShouldEmitImportsFiles) {}
1370
1371 virtual ~ThinBackendProc() = default;
1372 virtual Error start(
1373 unsigned Task, BitcodeModule BM,
1374 const FunctionImporter::ImportMapTy &ImportList,
1375 const FunctionImporter::ExportSetTy &ExportList,
1376 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1378 virtual Error wait() = 0;
1379 virtual unsigned getThreadCount() = 0;
1380
1381 // Write sharded indices and (optionally) imports to disk
1383 llvm::StringRef ModulePath,
1384 const std::string &NewModulePath) {
1385 std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
1386 std::error_code EC;
1387 gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries,
1388 ImportList, ModuleToSummariesForIndex);
1389
1390 raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC,
1391 sys::fs::OpenFlags::OF_None);
1392 if (EC)
1393 return errorCodeToError(EC);
1394 writeIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex);
1395
1396 if (ShouldEmitImportsFiles) {
1397 EC = EmitImportsFiles(ModulePath, NewModulePath + ".imports",
1398 ModuleToSummariesForIndex);
1399 if (EC)
1400 return errorCodeToError(EC);
1401 }
1402 return Error::success();
1403 }
1404};
1405
1406namespace {
1407class InProcessThinBackend : public ThinBackendProc {
1408 ThreadPool BackendThreadPool;
1409 AddStreamFn AddStream;
1410 FileCache Cache;
1411 std::set<GlobalValue::GUID> CfiFunctionDefs;
1412 std::set<GlobalValue::GUID> CfiFunctionDecls;
1413
1414 std::optional<Error> Err;
1415 std::mutex ErrMu;
1416
1417 bool ShouldEmitIndexFiles;
1418
1419public:
1420 InProcessThinBackend(
1421 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1422 ThreadPoolStrategy ThinLTOParallelism,
1423 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1424 AddStreamFn AddStream, FileCache Cache, lto::IndexWriteCallback OnWrite,
1425 bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles)
1426 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries,
1427 OnWrite, ShouldEmitImportsFiles),
1428 BackendThreadPool(ThinLTOParallelism), AddStream(std::move(AddStream)),
1429 Cache(std::move(Cache)), ShouldEmitIndexFiles(ShouldEmitIndexFiles) {
1430 for (auto &Name : CombinedIndex.cfiFunctionDefs())
1431 CfiFunctionDefs.insert(
1433 for (auto &Name : CombinedIndex.cfiFunctionDecls())
1434 CfiFunctionDecls.insert(
1436 }
1437
1438 Error runThinLTOBackendThread(
1439 AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM,
1440 ModuleSummaryIndex &CombinedIndex,
1441 const FunctionImporter::ImportMapTy &ImportList,
1442 const FunctionImporter::ExportSetTy &ExportList,
1443 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1444 const GVSummaryMapTy &DefinedGlobals,
1446 auto RunThinBackend = [&](AddStreamFn AddStream) {
1447 LTOLLVMContext BackendContext(Conf);
1448 Expected<std::unique_ptr<Module>> MOrErr = BM.parseModule(BackendContext);
1449 if (!MOrErr)
1450 return MOrErr.takeError();
1451
1452 return thinBackend(Conf, Task, AddStream, **MOrErr, CombinedIndex,
1453 ImportList, DefinedGlobals, &ModuleMap);
1454 };
1455
1456 auto ModuleID = BM.getModuleIdentifier();
1457
1458 if (ShouldEmitIndexFiles) {
1459 if (auto E = emitFiles(ImportList, ModuleID, ModuleID.str()))
1460 return E;
1461 }
1462
1463 if (!Cache || !CombinedIndex.modulePaths().count(ModuleID) ||
1464 all_of(CombinedIndex.getModuleHash(ModuleID),
1465 [](uint32_t V) { return V == 0; }))
1466 // Cache disabled or no entry for this module in the combined index or
1467 // no module hash.
1468 return RunThinBackend(AddStream);
1469
1471 // The module may be cached, this helps handling it.
1472 computeLTOCacheKey(Key, Conf, CombinedIndex, ModuleID, ImportList,
1473 ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs,
1474 CfiFunctionDecls);
1475 Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, Key, ModuleID);
1476 if (Error Err = CacheAddStreamOrErr.takeError())
1477 return Err;
1478 AddStreamFn &CacheAddStream = *CacheAddStreamOrErr;
1479 if (CacheAddStream)
1480 return RunThinBackend(CacheAddStream);
1481
1482 return Error::success();
1483 }
1484
1485 Error start(
1486 unsigned Task, BitcodeModule BM,
1487 const FunctionImporter::ImportMapTy &ImportList,
1488 const FunctionImporter::ExportSetTy &ExportList,
1489 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1490 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1491 StringRef ModulePath = BM.getModuleIdentifier();
1492 assert(ModuleToDefinedGVSummaries.count(ModulePath));
1493 const GVSummaryMapTy &DefinedGlobals =
1494 ModuleToDefinedGVSummaries.find(ModulePath)->second;
1495 BackendThreadPool.async(
1496 [=](BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
1497 const FunctionImporter::ImportMapTy &ImportList,
1498 const FunctionImporter::ExportSetTy &ExportList,
1499 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>
1500 &ResolvedODR,
1501 const GVSummaryMapTy &DefinedGlobals,
1503 if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled)
1505 "thin backend");
1506 Error E = runThinLTOBackendThread(
1507 AddStream, Cache, Task, BM, CombinedIndex, ImportList, ExportList,
1508 ResolvedODR, DefinedGlobals, ModuleMap);
1509 if (E) {
1510 std::unique_lock<std::mutex> L(ErrMu);
1511 if (Err)
1512 Err = joinErrors(std::move(*Err), std::move(E));
1513 else
1514 Err = std::move(E);
1515 }
1516 if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled)
1518 },
1519 BM, std::ref(CombinedIndex), std::ref(ImportList), std::ref(ExportList),
1520 std::ref(ResolvedODR), std::ref(DefinedGlobals), std::ref(ModuleMap));
1521
1522 if (OnWrite)
1523 OnWrite(std::string(ModulePath));
1524 return Error::success();
1525 }
1526
1527 Error wait() override {
1528 BackendThreadPool.wait();
1529 if (Err)
1530 return std::move(*Err);
1531 else
1532 return Error::success();
1533 }
1534
1535 unsigned getThreadCount() override {
1536 return BackendThreadPool.getThreadCount();
1537 }
1538};
1539} // end anonymous namespace
1540
1543 bool ShouldEmitIndexFiles,
1544 bool ShouldEmitImportsFiles) {
1545 return
1546 [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1547 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1548 AddStreamFn AddStream, FileCache Cache) {
1549 return std::make_unique<InProcessThinBackend>(
1550 Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries,
1551 AddStream, Cache, OnWrite, ShouldEmitIndexFiles,
1552 ShouldEmitImportsFiles);
1553 };
1554}
1555
1556// Given the original \p Path to an output file, replace any path
1557// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
1558// resulting directory if it does not yet exist.
1560 StringRef NewPrefix) {
1561 if (OldPrefix.empty() && NewPrefix.empty())
1562 return std::string(Path);
1563 SmallString<128> NewPath(Path);
1564 llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix);
1565 StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str());
1566 if (!ParentPath.empty()) {
1567 // Make sure the new directory exists, creating it if necessary.
1568 if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath))
1569 llvm::errs() << "warning: could not create directory '" << ParentPath
1570 << "': " << EC.message() << '\n';
1571 }
1572 return std::string(NewPath.str());
1573}
1574
1575namespace {
1576class WriteIndexesThinBackend : public ThinBackendProc {
1577 std::string OldPrefix, NewPrefix, NativeObjectPrefix;
1578 raw_fd_ostream *LinkedObjectsFile;
1579
1580public:
1581 WriteIndexesThinBackend(
1582 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1583 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1584 std::string OldPrefix, std::string NewPrefix,
1585 std::string NativeObjectPrefix, bool ShouldEmitImportsFiles,
1586 raw_fd_ostream *LinkedObjectsFile, lto::IndexWriteCallback OnWrite)
1587 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries,
1588 OnWrite, ShouldEmitImportsFiles),
1589 OldPrefix(OldPrefix), NewPrefix(NewPrefix),
1590 NativeObjectPrefix(NativeObjectPrefix),
1591 LinkedObjectsFile(LinkedObjectsFile) {}
1592
1593 Error start(
1594 unsigned Task, BitcodeModule BM,
1595 const FunctionImporter::ImportMapTy &ImportList,
1596 const FunctionImporter::ExportSetTy &ExportList,
1597 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1598 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1599 StringRef ModulePath = BM.getModuleIdentifier();
1600 std::string NewModulePath =
1601 getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix);
1602
1603 if (LinkedObjectsFile) {
1604 std::string ObjectPrefix =
1605 NativeObjectPrefix.empty() ? NewPrefix : NativeObjectPrefix;
1606 std::string LinkedObjectsFilePath =
1607 getThinLTOOutputFile(ModulePath, OldPrefix, ObjectPrefix);
1608 *LinkedObjectsFile << LinkedObjectsFilePath << '\n';
1609 }
1610
1611 if (auto E = emitFiles(ImportList, ModulePath, NewModulePath))
1612 return E;
1613
1614 if (OnWrite)
1615 OnWrite(std::string(ModulePath));
1616 return Error::success();
1617 }
1618
1619 Error wait() override { return Error::success(); }
1620
1621 // WriteIndexesThinBackend should always return 1 to prevent module
1622 // re-ordering and avoid non-determinism in the final link.
1623 unsigned getThreadCount() override { return 1; }
1624};
1625} // end anonymous namespace
1626
1628 std::string OldPrefix, std::string NewPrefix,
1629 std::string NativeObjectPrefix, bool ShouldEmitImportsFiles,
1630 raw_fd_ostream *LinkedObjectsFile, IndexWriteCallback OnWrite) {
1631 return
1632 [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1633 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1634 AddStreamFn AddStream, FileCache Cache) {
1635 return std::make_unique<WriteIndexesThinBackend>(
1636 Conf, CombinedIndex, ModuleToDefinedGVSummaries, OldPrefix,
1637 NewPrefix, NativeObjectPrefix, ShouldEmitImportsFiles,
1638 LinkedObjectsFile, OnWrite);
1639 };
1640}
1641
1642Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache,
1643 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
1644 LLVM_DEBUG(dbgs() << "Running ThinLTO\n");
1645 ThinLTO.CombinedIndex.releaseTemporaryMemory();
1646 timeTraceProfilerBegin("ThinLink", StringRef(""));
1647 auto TimeTraceScopeExit = llvm::make_scope_exit([]() {
1650 });
1651 if (ThinLTO.ModuleMap.empty())
1652 return Error::success();
1653
1654 if (ThinLTO.ModulesToCompile && ThinLTO.ModulesToCompile->empty()) {
1655 llvm::errs() << "warning: [ThinLTO] No module compiled\n";
1656 return Error::success();
1657 }
1658
1659 if (Conf.CombinedIndexHook &&
1660 !Conf.CombinedIndexHook(ThinLTO.CombinedIndex, GUIDPreservedSymbols))
1661 return Error::success();
1662
1663 // Collect for each module the list of function it defines (GUID ->
1664 // Summary).
1665 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(
1666 ThinLTO.ModuleMap.size());
1667 ThinLTO.CombinedIndex.collectDefinedGVSummariesPerModule(
1668 ModuleToDefinedGVSummaries);
1669 // Create entries for any modules that didn't have any GV summaries
1670 // (either they didn't have any GVs to start with, or we suppressed
1671 // generation of the summaries because they e.g. had inline assembly
1672 // uses that couldn't be promoted/renamed on export). This is so
1673 // InProcessThinBackend::start can still launch a backend thread, which
1674 // is passed the map of summaries for the module, without any special
1675 // handling for this case.
1676 for (auto &Mod : ThinLTO.ModuleMap)
1677 if (!ModuleToDefinedGVSummaries.count(Mod.first))
1678 ModuleToDefinedGVSummaries.try_emplace(Mod.first);
1679
1680 // Synthesize entry counts for functions in the CombinedIndex.
1681 computeSyntheticCounts(ThinLTO.CombinedIndex);
1682
1684 ThinLTO.ModuleMap.size());
1686 ThinLTO.ModuleMap.size());
1688
1689 if (DumpThinCGSCCs)
1690 ThinLTO.CombinedIndex.dumpSCCs(outs());
1691
1692 std::set<GlobalValue::GUID> ExportedGUIDs;
1693
1694 bool WholeProgramVisibilityEnabledInLTO =
1696 // If validation is enabled, upgrade visibility only when all vtables
1697 // have typeinfos.
1699 if (hasWholeProgramVisibility(WholeProgramVisibilityEnabledInLTO))
1700 ThinLTO.CombinedIndex.setWithWholeProgramVisibility();
1701
1702 // If we're validating, get the vtable symbols that should not be
1703 // upgraded because they correspond to typeIDs outside of index-based
1704 // WPD info.
1705 DenseSet<GlobalValue::GUID> VisibleToRegularObjSymbols;
1706 if (WholeProgramVisibilityEnabledInLTO &&
1708 // This returns true when the name is local or not defined. Locals are
1709 // expected to be handled separately.
1710 auto IsVisibleToRegularObj = [&](StringRef name) {
1711 auto It = GlobalResolutions.find(name);
1712 return (It == GlobalResolutions.end() ||
1713 It->second.VisibleOutsideSummary);
1714 };
1715
1716 getVisibleToRegularObjVtableGUIDs(ThinLTO.CombinedIndex,
1717 VisibleToRegularObjSymbols,
1718 IsVisibleToRegularObj);
1719 }
1720
1721 // If allowed, upgrade public vcall visibility to linkage unit visibility in
1722 // the summaries before whole program devirtualization below.
1724 ThinLTO.CombinedIndex, WholeProgramVisibilityEnabledInLTO,
1725 DynamicExportSymbols, VisibleToRegularObjSymbols);
1726
1727 // Perform index-based WPD. This will return immediately if there are
1728 // no index entries in the typeIdMetadata map (e.g. if we are instead
1729 // performing IR-based WPD in hybrid regular/thin LTO mode).
1730 std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
1731 runWholeProgramDevirtOnIndex(ThinLTO.CombinedIndex, ExportedGUIDs,
1732 LocalWPDTargetsMap);
1733
1734 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1735 return ThinLTO.PrevailingModuleForGUID[GUID] == S->modulePath();
1736 };
1738 MemProfContextDisambiguation ContextDisambiguation;
1739 ContextDisambiguation.run(ThinLTO.CombinedIndex, isPrevailing);
1740 }
1741
1742 if (Conf.OptLevel > 0)
1743 ComputeCrossModuleImport(ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
1744 isPrevailing, ImportLists, ExportLists);
1745
1746 // Figure out which symbols need to be internalized. This also needs to happen
1747 // at -O0 because summary-based DCE is implemented using internalization, and
1748 // we must apply DCE consistently with the full LTO module in order to avoid
1749 // undefined references during the final link.
1750 for (auto &Res : GlobalResolutions) {
1751 // If the symbol does not have external references or it is not prevailing,
1752 // then not need to mark it as exported from a ThinLTO partition.
1753 if (Res.second.Partition != GlobalResolution::External ||
1754 !Res.second.isPrevailingIRSymbol())
1755 continue;
1756 auto GUID = GlobalValue::getGUID(
1757 GlobalValue::dropLLVMManglingEscape(Res.second.IRName));
1758 // Mark exported unless index-based analysis determined it to be dead.
1759 if (ThinLTO.CombinedIndex.isGUIDLive(GUID))
1760 ExportedGUIDs.insert(GUID);
1761 }
1762
1763 // Any functions referenced by the jump table in the regular LTO object must
1764 // be exported.
1765 for (auto &Def : ThinLTO.CombinedIndex.cfiFunctionDefs())
1766 ExportedGUIDs.insert(
1768 for (auto &Decl : ThinLTO.CombinedIndex.cfiFunctionDecls())
1769 ExportedGUIDs.insert(
1771
1772 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1773 const auto &ExportList = ExportLists.find(ModuleIdentifier);
1774 return (ExportList != ExportLists.end() && ExportList->second.count(VI)) ||
1775 ExportedGUIDs.count(VI.getGUID());
1776 };
1777
1778 // Update local devirtualized targets that were exported by cross-module
1779 // importing or by other devirtualizations marked in the ExportedGUIDs set.
1780 updateIndexWPDForExports(ThinLTO.CombinedIndex, isExported,
1781 LocalWPDTargetsMap);
1782
1783 thinLTOInternalizeAndPromoteInIndex(ThinLTO.CombinedIndex, isExported,
1784 isPrevailing);
1785
1786 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1787 GlobalValue::GUID GUID,
1788 GlobalValue::LinkageTypes NewLinkage) {
1789 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1790 };
1791 thinLTOResolvePrevailingInIndex(Conf, ThinLTO.CombinedIndex, isPrevailing,
1792 recordNewLinkage, GUIDPreservedSymbols);
1793
1794 thinLTOPropagateFunctionAttrs(ThinLTO.CombinedIndex, isPrevailing);
1795
1796 generateParamAccessSummary(ThinLTO.CombinedIndex);
1797
1800
1801 TimeTraceScopeExit.release();
1802
1803 std::unique_ptr<ThinBackendProc> BackendProc =
1804 ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
1805 AddStream, Cache);
1806
1807 auto &ModuleMap =
1808 ThinLTO.ModulesToCompile ? *ThinLTO.ModulesToCompile : ThinLTO.ModuleMap;
1809
1810 auto ProcessOneModule = [&](int I) -> Error {
1811 auto &Mod = *(ModuleMap.begin() + I);
1812 // Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for
1813 // combined module and parallel code generation partitions.
1814 return BackendProc->start(RegularLTO.ParallelCodeGenParallelismLevel + I,
1815 Mod.second, ImportLists[Mod.first],
1816 ExportLists[Mod.first], ResolvedODR[Mod.first],
1817 ThinLTO.ModuleMap);
1818 };
1819
1820 if (BackendProc->getThreadCount() == 1) {
1821 // Process the modules in the order they were provided on the command-line.
1822 // It is important for this codepath to be used for WriteIndexesThinBackend,
1823 // to ensure the emitted LinkedObjectsFile lists ThinLTO objects in the same
1824 // order as the inputs, which otherwise would affect the final link order.
1825 for (int I = 0, E = ModuleMap.size(); I != E; ++I)
1826 if (Error E = ProcessOneModule(I))
1827 return E;
1828 } else {
1829 // When executing in parallel, process largest bitsize modules first to
1830 // improve parallelism, and avoid starving the thread pool near the end.
1831 // This saves about 15 sec on a 36-core machine while link `clang.exe` (out
1832 // of 100 sec).
1833 std::vector<BitcodeModule *> ModulesVec;
1834 ModulesVec.reserve(ModuleMap.size());
1835 for (auto &Mod : ModuleMap)
1836 ModulesVec.push_back(&Mod.second);
1837 for (int I : generateModulesOrdering(ModulesVec))
1838 if (Error E = ProcessOneModule(I))
1839 return E;
1840 }
1841 return BackendProc->wait();
1842}
1843
1847 std::optional<uint64_t> RemarksHotnessThreshold, int Count) {
1848 std::string Filename = std::string(RemarksFilename);
1849 // For ThinLTO, file.opt.<format> becomes
1850 // file.opt.<format>.thin.<num>.<format>.
1851 if (!Filename.empty() && Count != -1)
1852 Filename =
1853 (Twine(Filename) + ".thin." + llvm::utostr(Count) + "." + RemarksFormat)
1854 .str();
1855
1856 auto ResultOrErr = llvm::setupLLVMOptimizationRemarks(
1859 if (Error E = ResultOrErr.takeError())
1860 return std::move(E);
1861
1862 if (*ResultOrErr)
1863 (*ResultOrErr)->keep();
1864
1865 return ResultOrErr;
1866}
1867
1870 // Setup output file to emit statistics.
1871 if (StatsFilename.empty())
1872 return nullptr;
1873
1875 std::error_code EC;
1876 auto StatsFile =
1877 std::make_unique<ToolOutputFile>(StatsFilename, EC, sys::fs::OF_None);
1878 if (EC)
1879 return errorCodeToError(EC);
1880
1881 StatsFile->keep();
1882 return std::move(StatsFile);
1883}
1884
1885// Compute the ordering we will process the inputs: the rough heuristic here
1886// is to sort them per size so that the largest module get schedule as soon as
1887// possible. This is purely a compile-time optimization.
1889 auto Seq = llvm::seq<int>(0, R.size());
1890 std::vector<int> ModulesOrdering(Seq.begin(), Seq.end());
1891 llvm::sort(ModulesOrdering, [&](int LeftIndex, int RightIndex) {
1892 auto LSize = R[LeftIndex]->getBuffer().size();
1893 auto RSize = R[RightIndex]->getBuffer().size();
1894 return LSize > RSize;
1895 });
1896 return ModulesOrdering;
1897}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DEBUG(X)
Definition: Debug.h:101
std::string Name
Symbol * Sym
Definition: ELF_riscv.cpp:477
#define DEBUG_TYPE
static void writeToResolutionFile(raw_ostream &OS, InputFile *Input, ArrayRef< SymbolResolution > Res)
Definition: LTO.cpp:667
static void thinLTOResolvePrevailingGUID(const Config &C, ValueInfo VI, DenseSet< GlobalValueSummary * > &GlobalInvolvedWithAlias, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, function_ref< void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)> recordNewLinkage, const DenseSet< GlobalValue::GUID > &GUIDPreservedSymbols)
Definition: LTO.cpp:349
static void handleNonPrevailingComdat(GlobalValue &GV, std::set< const Comdat * > &NonPrevailingComdats)
Definition: LTO.cpp:779
static cl::opt< bool > DumpThinCGSCCs("dump-thin-cg-sccs", cl::init(false), cl::Hidden, cl::desc("Dump the SCCs in the ThinLTO index's callgraph"))
static void thinLTOInternalizeAndPromoteGUID(ValueInfo VI, function_ref< bool(StringRef, ValueInfo)> isExported, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing)
Definition: LTO.cpp:448
static const char * libcallRoutineNames[]
Definition: LTO.cpp:1343
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
This file contains the declarations for metadata subclasses.
LLVMContext & Context
#define P(N)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static const char * name
Definition: SMEABIPass.cpp:49
raw_pwrite_stream & OS
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
This file contains some functions that are useful when dealing with strings.
This pass exposes codegen information to IR-level passes.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
iterator begin() const
Definition: ArrayRef.h:153
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:654
Represents a module in a bitcode file.
StringRef getModuleIdentifier() const
Error readSummary(ModuleSummaryIndex &CombinedIndex, StringRef ModulePath, std::function< bool(GlobalValue::GUID)> IsPrevailing=nullptr)
Parse the specified bitcode buffer and merge its module summary index into CombinedIndex.
Expected< std::unique_ptr< Module > > parseModule(LLVMContext &Context, ParserCallbacks Callbacks={})
Read the entire bitcode module and return it.
Expected< std::unique_ptr< Module > > getLazyModule(LLVMContext &Context, bool ShouldLazyLoadMetadata, bool IsImporting, ParserCallbacks Callbacks={})
Read the bitcode module and prepare for lazy deserialization of function bodies.
static ConstantAggregateZero * get(Type *Ty)
Definition: Constants.cpp:1579
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
Definition: DenseMap.h:235
unsigned size() const
Definition: DenseMap.h:99
iterator begin()
Definition: DenseMap.h:75
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
iterator end()
Definition: DenseMap.h:84
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:334
Tagged union holding either a T or a Error.
Definition: Error.h:474
Error takeError()
Take ownership of the stored error.
Definition: Error.h:601
std::unordered_set< GlobalValue::GUID > FunctionsToImportTy
Set of functions to import from a source module.
void setAlignment(Align Align)
Sets the alignment attribute of the GlobalObject.
Definition: Globals.cpp:128
Function and variable summary information to aid decisions and implementation of importing.
static bool isAppendingLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:395
static bool isExternalWeakLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:407
static bool isLocalLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:404
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:273
void setUnnamedAddr(UnnamedAddr Val)
Definition: GlobalValue.h:227
bool hasLocalLinkage() const
Definition: GlobalValue.h:523
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition: GlobalValue.h:562
const Comdat * getComdat() const
Definition: Globals.cpp:183
static bool isLinkOnceLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:383
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:532
DLLStorageClassTypes
Storage classes of global values for PE targets.
Definition: GlobalValue.h:69
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:591
const GlobalObject * getAliaseeObject() const
Definition: Globals.cpp:367
static bool isExternalLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:371
VisibilityTypes
An enumeration for the kinds of visibility of global values.
Definition: GlobalValue.h:62
@ DefaultVisibility
The GV is visible.
Definition: GlobalValue.h:63
static GUID getGUID(StringRef GlobalName)
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:587
static LinkageTypes getWeakLinkage(bool ODR)
Definition: GlobalValue.h:367
bool isWeakForLinker() const
Definition: GlobalValue.h:547
bool hasAppendingLinkage() const
Definition: GlobalValue.h:520
bool hasAvailableExternallyLinkage() const
Definition: GlobalValue.h:507
std::string getGlobalIdentifier() const
Return the modified name for this global value suitable to be used as the key for a global lookup (e....
Definition: Globals.cpp:168
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:47
@ CommonLinkage
Tentative definitions.
Definition: GlobalValue.h:58
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:55
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
Definition: GlobalValue.h:52
@ AvailableExternallyLinkage
Available for inspection, not emission.
Definition: GlobalValue.h:49
DLLStorageClassTypes getDLLStorageClass() const
Definition: GlobalValue.h:271
Type * getValueType() const
Definition: GlobalValue.h:292
static bool isLinkOnceODRLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:380
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition: Globals.cpp:454
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
This class implements a map that also provides access to all stored values in a deterministic order.
Definition: MapVector.h:36
iterator begin()
Definition: MapVector.h:69
size_type size() const
Definition: MapVector.h:60
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFile(const Twine &Filename, bool IsText=false, bool RequiresNullTerminator=true, bool IsVolatile=false, std::optional< Align > Alignment=std::nullopt)
Open the specified file as a MemoryBuffer, returning a new MemoryBuffer if successful,...
Class to hold module path string table and global value map, and encapsulate methods for operating on...
std::set< std::string > & cfiFunctionDecls()
const ModuleHash & getModuleHash(const StringRef ModPath) const
Get the module SHA1 hash recorded for the given module path.
const StringMap< ModuleHash > & modulePaths() const
Table of modules, containing module hash and id.
std::set< std::string > & cfiFunctionDefs()
static void CollectAsmSymvers(const Module &M, function_ref< void(StringRef, StringRef)> AsmSymver)
Parse inline ASM and collect the symvers directives that are defined in the current module.
uint32_t getSymbolFlags(Symbol S) const
ArrayRef< Symbol > symbols() const
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
A tuple of MDNodes.
Definition: Metadata.h:1692
The optimization diagnostic interface.
Diagnostic information for applied optimization remarks.
A discriminated union of two or more pointer types, with the discriminator in the low bit of the poin...
Definition: PointerUnion.h:118
A class that wrap the SHA1 algorithm.
Definition: SHA1.h:26
void update(ArrayRef< uint8_t > Data)
Digest more data.
Definition: SHA1.cpp:208
std::array< uint8_t, 20 > result()
Return the current raw 160-bits SHA1 for the digested data since the last call to init().
Definition: SHA1.cpp:288
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:166
bool empty() const
Definition: SmallSet.h:159
bool erase(const T &V)
Definition: SmallSet.h:207
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:179
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition: SmallString.h:26
StringRef str() const
Explicit conversion to StringRef.
Definition: SmallString.h:261
StringMapEntry - This is used to represent one value that is inserted into a StringMap.
StringMap - This is an unconventional map that is specialized for handling keys that are "strings",...
Definition: StringMap.h:112
iterator end()
Definition: StringMap.h:205
iterator find(StringRef Key)
Definition: StringMap.h:218
size_type count(StringRef Key) const
count - Return 1 if the element is in the map, 0 otherwise.
Definition: StringMap.h:257
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
bool contains(StringRef Other) const
Return true if the given string is a substring of *this, and false otherwise.
Definition: StringRef.h:428
unsigned RelaxELFRelocations
DebuggerKind DebuggerTuning
Which debugger to tune for.
unsigned FunctionSections
Emit functions into separate sections.
unsigned DataSections
Emit data into separate sections.
This tells how a thread pool will be used.
Definition: Threading.h:116
A ThreadPool for asynchronous parallel execution on a defined number of threads.
Definition: ThreadPool.h:52
void wait()
Blocking wait for all the threads to complete and the queue to be empty.
Definition: ThreadPool.cpp:202
unsigned getThreadCount() const
Definition: ThreadPool.h:110
auto async(Function &&F, Args &&...ArgList)
Asynchronous submission of a task to the pool.
Definition: ThreadPool.h:66
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
bool isOSBinFormatELF() const
Tests whether the OS uses the ELF binary format.
Definition: Triple.h:678
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
static IntegerType * getInt8Ty(LLVMContext &C)
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:377
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:534
bool use_empty() const
Definition: Value.h:344
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:383
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
iterator find(const_arg_type_t< ValueT > V)
Definition: DenseSet.h:179
size_type size() const
Definition: DenseSet.h:81
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:97
An efficient, type-erasing, non-owning reference to a callable.
Ephemeral symbols produced by Reader::symbols() and Reader::module_symbols().
Definition: IRSymtab.h:313
The purpose of this class is to only expose the symbol information that an LTO client should need in ...
Definition: LTO.h:137
An input file.
Definition: LTO.h:109
static Expected< std::unique_ptr< InputFile > > create(MemoryBufferRef Object)
Create an InputFile.
Definition: LTO.cpp:540
ArrayRef< Symbol > symbols() const
A range over the symbols in this InputFile.
Definition: LTO.h:162
StringRef getName() const
Returns the path to the InputFile.
Definition: LTO.cpp:569
BitcodeModule & getSingleBitcodeModule()
Definition: LTO.cpp:573
Error add(std::unique_ptr< InputFile > Obj, ArrayRef< SymbolResolution > Res)
Add an input file to the LTO link, using the provided symbol resolutions.
Definition: LTO.cpp:691
LTO(Config Conf, ThinBackend Backend=nullptr, unsigned ParallelCodeGenParallelismLevel=1, LTOKind LTOMode=LTOK_Default)
Create an LTO object.
Definition: LTO.cpp:591
LTOKind
Unified LTO modes.
Definition: LTO.h:259
@ LTOK_UnifiedRegular
Regular LTO, with Unified LTO enabled.
Definition: LTO.h:264
@ LTOK_Default
Any LTO mode without Unified LTO. The default mode.
Definition: LTO.h:261
@ LTOK_UnifiedThin
ThinLTO, with Unified LTO enabled.
Definition: LTO.h:267
static ArrayRef< const char * > getRuntimeLibcallSymbols()
Static method that returns a list of libcall symbols that can be generated by LTO but might not be vi...
Definition: LTO.cpp:1349
Error run(AddStreamFn AddStream, FileCache Cache=nullptr)
Runs the LTO pipeline.
Definition: LTO.cpp:1124
unsigned getMaxTasks() const
Returns an upper bound on the number of tasks that the client may expect.
Definition: LTO.cpp:1073
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:454
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
This class defines the interface to the ThinLTO backend.
Definition: LTO.cpp:1354
virtual unsigned getThreadCount()=0
const Config & Conf
Definition: LTO.cpp:1356
lto::IndexWriteCallback OnWrite
Definition: LTO.cpp:1359
Error emitFiles(const FunctionImporter::ImportMapTy &ImportList, llvm::StringRef ModulePath, const std::string &NewModulePath)
Definition: LTO.cpp:1382
virtual Error wait()=0
ThinBackendProc(const Config &Conf, ModuleSummaryIndex &CombinedIndex, const DenseMap< StringRef, GVSummaryMapTy > &ModuleToDefinedGVSummaries, lto::IndexWriteCallback OnWrite, bool ShouldEmitImportsFiles)
Definition: LTO.cpp:1363
const DenseMap< StringRef, GVSummaryMapTy > & ModuleToDefinedGVSummaries
Definition: LTO.cpp:1358
ModuleSummaryIndex & CombinedIndex
Definition: LTO.cpp:1357
virtual ~ThinBackendProc()=default
virtual Error start(unsigned Task, BitcodeModule BM, const FunctionImporter::ImportMapTy &ImportList, const FunctionImporter::ExportSetTy &ExportList, const std::map< GlobalValue::GUID, GlobalValue::LinkageTypes > &ResolvedODR, MapVector< StringRef, BitcodeModule > &ModuleMap)=0
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Key
PAL metadata keys.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Definition: Function.cpp:1010
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
ThinBackend createInProcessThinBackend(ThreadPoolStrategy Parallelism, IndexWriteCallback OnWrite=nullptr, bool ShouldEmitIndexFiles=false, bool ShouldEmitImportsFiles=false)
Definition: LTO.cpp:1541
Error thinBackend(const Config &C, unsigned Task, AddStreamFn AddStream, Module &M, const ModuleSummaryIndex &CombinedIndex, const FunctionImporter::ImportMapTy &ImportList, const GVSummaryMapTy &DefinedGlobals, MapVector< StringRef, BitcodeModule > *ModuleMap, const std::vector< uint8_t > &CmdArgs=std::vector< uint8_t >())
Runs a ThinLTO backend.
Definition: LTOBackend.cpp:551
std::string getThinLTOOutputFile(StringRef Path, StringRef OldPrefix, StringRef NewPrefix)
Given the original Path to an output file, replace any path prefix matching OldPrefix with NewPrefix.
Definition: LTO.cpp:1559
std::function< std::unique_ptr< ThinBackendProc >(const Config &C, ModuleSummaryIndex &CombinedIndex, DenseMap< StringRef, GVSummaryMapTy > &ModuleToDefinedGVSummaries, AddStreamFn AddStream, FileCache Cache)> ThinBackend
A ThinBackend defines what happens after the thin-link phase during ThinLTO.
Definition: LTO.h:200
std::function< void(const std::string &)> IndexWriteCallback
This ThinBackend runs the individual backend jobs in-process.
Definition: LTO.h:210
Expected< std::unique_ptr< ToolOutputFile > > setupStatsFile(StringRef StatsFilename)
Setups the output file for saving statistics.
Definition: LTO.cpp:1869
Error backend(const Config &C, AddStreamFn AddStream, unsigned ParallelCodeGenParallelismLevel, Module &M, ModuleSummaryIndex &CombinedIndex)
Runs a regular LTO backend.
Definition: LTOBackend.cpp:504
Error finalizeOptimizationRemarks(std::unique_ptr< ToolOutputFile > DiagOutputFile)
Definition: LTOBackend.cpp:493
ThinBackend createWriteIndexesThinBackend(std::string OldPrefix, std::string NewPrefix, std::string NativeObjectPrefix, bool ShouldEmitImportsFiles, raw_fd_ostream *LinkedObjectsFile, IndexWriteCallback OnWrite)
This ThinBackend writes individual module indexes to files, instead of running the individual backend...
Definition: LTO.cpp:1627
std::vector< int > generateModulesOrdering(ArrayRef< BitcodeModule * > R)
Produces a container ordering for optimal multi-threaded processing.
Definition: LTO.cpp:1888
Expected< std::unique_ptr< ToolOutputFile > > setupLLVMOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses, StringRef RemarksFormat, bool RemarksWithHotness, std::optional< uint64_t > RemarksHotnessThreshold=0, int Count=-1)
Setup optimization remarks.
Definition: LTO.cpp:1844
void updateMemProfAttributes(Module &Mod, const ModuleSummaryIndex &Index)
Updates MemProf attributes (and metadata) based on whether the index has recorded that we are linking...
Definition: LTO.cpp:1180
Expected< IRSymtabFile > readIRSymtab(MemoryBufferRef MBRef)
Reads a bitcode file, creating its irsymtab if necessary.
size_t getThreadCount()
Definition: Parallel.h:55
void write64le(void *P, uint64_t V)
Definition: Endian.h:455
void write32le(void *P, uint32_t V)
Definition: Endian.h:452
std::error_code create_directories(const Twine &path, bool IgnoreExisting=true, perms Perms=owner_all|group_all)
Create all the non-existent directories in path.
Definition: Path.cpp:969
StringRef parent_path(StringRef path, Style style=Style::native)
Get parent path.
Definition: Path.cpp:469
bool replace_path_prefix(SmallVectorImpl< char > &Path, StringRef OldPrefix, StringRef NewPrefix, Style style=Style::native)
Replace matching path prefix with another path.
Definition: Path.cpp:520
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
ThreadPoolStrategy heavyweight_hardware_concurrency(unsigned ThreadCount=0)
Returns a thread strategy for tasks requiring significant memory or other resources.
Definition: Threading.h:162
cl::opt< std::string > RemarksFormat("lto-pass-remarks-format", cl::desc("The format used for serializing remarks (default: YAML)"), cl::value_desc("format"), cl::init("yaml"))
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1726
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1684
void ComputeCrossModuleImport(const ModuleSummaryIndex &Index, const DenseMap< StringRef, GVSummaryMapTy > &ModuleToDefinedGVSummaries, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, DenseMap< StringRef, FunctionImporter::ImportMapTy > &ImportLists, DenseMap< StringRef, FunctionImporter::ExportSetTy > &ExportLists)
Compute all the imports and exports for every module in the Index.
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:59
void timeTraceProfilerInitialize(unsigned TimeTraceGranularity, StringRef ProcName)
Initialize the time trace profiler.
void generateParamAccessSummary(ModuleSummaryIndex &Index)
cl::opt< std::string > RemarksPasses("lto-pass-remarks-filter", cl::desc("Only record optimization remarks from passes whose " "names match the given regular expression"), cl::value_desc("regex"))
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:90
void writeIndexToFile(const ModuleSummaryIndex &Index, raw_ostream &Out, const std::map< std::string, GVSummaryMapTy > *ModuleToSummariesForIndex=nullptr)
Write the specified module summary index to the given raw output stream, where it will be written in ...
raw_fd_ostream & outs()
This returns a reference to a raw_fd_ostream for standard output.
std::function< Expected< std::unique_ptr< CachedFileStream > >(unsigned Task, const Twine &ModuleName)> AddStreamFn
This type defines the callback to add a file that is generated on the fly.
Definition: Caching.h:42
std::error_code EmitImportsFiles(StringRef ModulePath, StringRef OutputFilename, const std::map< std::string, GVSummaryMapTy > &ModuleToSummariesForIndex)
Emit into OutputFilename the files module ModulePath will import from.
bool thinLTOPropagateFunctionAttrs(ModuleSummaryIndex &Index, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing)
Propagate function attributes for function summaries along the index's callgraph during thinlink.
bool hasWholeProgramVisibility(bool WholeProgramVisibilityEnabledInLTO)
void EnableStatistics(bool DoPrintOnExit=true)
Enable the collection and printing of statistics.
Definition: Statistic.cpp:134
void thinLTOInternalizeAndPromoteInIndex(ModuleSummaryIndex &Index, function_ref< bool(StringRef, ValueInfo)> isExported, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing)
Update the linkages in the given Index to mark exported values as external and non-exported values as...
Definition: LTO.cpp:527
void computeLTOCacheKey(SmallString< 40 > &Key, const lto::Config &Conf, const ModuleSummaryIndex &Index, StringRef ModuleID, const FunctionImporter::ImportMapTy &ImportList, const FunctionImporter::ExportSetTy &ExportList, const std::map< GlobalValue::GUID, GlobalValue::LinkageTypes > &ResolvedODR, const GVSummaryMapTy &DefinedGlobals, const std::set< GlobalValue::GUID > &CfiFunctionDefs={}, const std::set< GlobalValue::GUID > &CfiFunctionDecls={})
Computes a unique hash for the Module considering the current list of export/import and other global ...
Definition: LTO.cpp:89
void timeTraceProfilerFinishThread()
Finish a time trace profiler running on a worker thread.
void timeTraceProfilerBegin(StringRef Name, StringRef Detail)
Manually begin a time section, with the given Name and Detail.
Error joinErrors(Error E1, Error E2)
Concatenate errors.
Definition: Error.h:431
void updatePublicTypeTestCalls(Module &M, bool WholeProgramVisibilityEnabledInLTO)
void getVisibleToRegularObjVtableGUIDs(ModuleSummaryIndex &Index, DenseSet< GlobalValue::GUID > &VisibleToRegularObjSymbols, function_ref< bool(StringRef)> IsVisibleToRegularObj)
Based on typeID string, get all associated vtable GUIDS that are visible to regular objects.
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1651
bool timeTraceProfilerEnabled()
Is the time trace profiler enabled, i.e. initialized?
Definition: TimeProfiler.h:102
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
cl::opt< bool > EnableMemProfContextDisambiguation
Enable MemProf context disambiguation for thin link.
Expected< std::unique_ptr< ToolOutputFile > > setupLLVMOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses, StringRef RemarksFormat, bool RemarksWithHotness, std::optional< uint64_t > RemarksHotnessThreshold=0)
Setup optimization remarks that output to a file.
cl::opt< bool > EnableLTOInternalization
Enable global value internalization in LTO.
cl::opt< bool > RemarksWithHotness("lto-pass-remarks-with-hotness", cl::desc("With PGO, include profile count in optimization remarks"), cl::Hidden)
void computeSyntheticCounts(ModuleSummaryIndex &Index)
Compute synthetic function entry counts.
void timeTraceProfilerEnd()
Manually end the last time section.
cl::opt< std::string > RemarksFilename("lto-pass-remarks-output", cl::desc("Output filename for pass remarks"), cl::value_desc("filename"))
cl::opt< bool > SupportsHotColdNew
Indicate we are linking with an allocator that supports hot/cold operator new interfaces.
void updateIndexWPDForExports(ModuleSummaryIndex &Summary, function_ref< bool(StringRef, ValueInfo)> isExported, std::map< ValueInfo, std::vector< VTableSlotSummary > > &LocalWPDTargetsMap)
Call after cross-module importing to update the recorded single impl devirt target names for any loca...
void thinLTOResolvePrevailingInIndex(const lto::Config &C, ModuleSummaryIndex &Index, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, function_ref< void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)> recordNewLinkage, const DenseSet< GlobalValue::GUID > &GUIDPreservedSymbols)
Resolve linkage for prevailing symbols in the Index.
Definition: LTO.cpp:426
void gatherImportedSummariesForModule(StringRef ModulePath, const DenseMap< StringRef, GVSummaryMapTy > &ModuleToDefinedGVSummaries, const FunctionImporter::ImportMapTy &ImportList, std::map< std::string, GVSummaryMapTy > &ModuleToSummariesForIndex)
Compute the set of summaries needed for a ThinLTO backend compilation of ModulePath.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Mod
The access may modify the value stored in memory.
void runWholeProgramDevirtOnIndex(ModuleSummaryIndex &Summary, std::set< GlobalValue::GUID > &ExportedGUIDs, std::map< ValueInfo, std::vector< VTableSlotSummary > > &LocalWPDTargetsMap)
Perform index-based whole program devirtualization on the Summary index.
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1853
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition: STLExtras.h:1925
cl::opt< std::optional< uint64_t >, false, remarks::HotnessThresholdParser > RemarksHotnessThreshold("lto-pass-remarks-hotness-threshold", cl::desc("Minimum profile count required for an " "optimization remark to be output." " Use 'auto' to apply the threshold from profile summary."), cl::value_desc("uint or 'auto'"), cl::init(0), cl::Hidden)
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
Definition: Error.cpp:103
bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
std::array< uint32_t, 5 > ModuleHash
160 bits SHA1
std::function< Expected< AddStreamFn >(unsigned Task, StringRef Key, const Twine &ModuleName)> FileCache
This is the type of a file cache.
Definition: Caching.h:58
void PrintStatisticsJSON(raw_ostream &OS)
Print statistics in JSON format.
Definition: Statistic.cpp:203
void computeDeadSymbolsWithConstProp(ModuleSummaryIndex &Index, const DenseSet< GlobalValue::GUID > &GUIDPreservedSymbols, function_ref< PrevailingType(GlobalValue::GUID)> isPrevailing, bool ImportEnabled)
Compute dead symbols and run constant propagation in combined index after that.
@ Keep
No function return thunk.
void updateVCallVisibilityInModule(Module &M, bool WholeProgramVisibilityEnabledInLTO, const DenseSet< GlobalValue::GUID > &DynamicExportSymbols, bool ValidateAllVtablesHaveTypeInfos, function_ref< bool(StringRef)> IsVisibleToRegularObj)
If whole program visibility asserted, then upgrade all public vcall visibility metadata on vtable def...
void updateVCallVisibilityInIndex(ModuleSummaryIndex &Index, bool WholeProgramVisibilityEnabledInLTO, const DenseSet< GlobalValue::GUID > &DynamicExportSymbols, const DenseSet< GlobalValue::GUID > &VisibleToRegularObjSymbols)
If whole program visibility asserted, then upgrade all public vcall visibility metadata on vtable def...
Implement std::hash so that hash_code can be used in STL containers.
Definition: BitVector.h:858
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Used in the streaming interface as the general argument type.
Struct that holds a reference to a particular GUID in a global value summary.
LTO configuration.
Definition: Config.h:41
bool HasWholeProgramVisibility
Asserts whether we can assume whole program visibility during the LTO link.
Definition: Config.h:81
bool ValidateAllVtablesHaveTypeInfos
We're validating that all native vtables have corresponding type infos.
Definition: Config.h:84
std::optional< uint64_t > RemarksHotnessThreshold
The minimum hotness value a diagnostic needs in order to be included in optimization diagnostics.
Definition: Config.h:163
std::string StatsFile
Statistics output file path.
Definition: Config.h:172
ModuleHookFn PreOptModuleHook
This module hook is called after linking (regular LTO) or loading (ThinLTO) the module,...
Definition: Config.h:226
CombinedIndexHookFn CombinedIndexHook
Definition: Config.h:258
std::optional< CodeModel::Model > CodeModel
Definition: Config.h:56
std::string AAPipeline
Definition: Config.h:108
bool CodeGenOnly
Disable entirely the optimizer, including importing for ThinLTO.
Definition: Config.h:71
std::vector< std::string > MAttrs
Definition: Config.h:50
std::vector< std::string > MllvmArgs
Definition: Config.h:51
std::vector< std::string > ThinLTOModulesToCompile
Specific thinLTO modules to compile.
Definition: Config.h:175
CodeGenOptLevel CGOptLevel
Definition: Config.h:57
std::unique_ptr< raw_ostream > ResolutionFile
If this field is set, LTO will write input file paths and symbol resolutions here in llvm-lto2 comman...
Definition: Config.h:193
std::string DefaultTriple
Setting this field will replace unspecified target triples in input files with this triple.
Definition: Config.h:116
bool AlwaysEmitRegularLTOObj
Always emit a Regular LTO object even when it is empty because no Regular LTO modules were linked.
Definition: Config.h:92
std::string CPU
Definition: Config.h:48
std::string DwoDir
The directory to store .dwo files.
Definition: Config.h:128
std::string RemarksFilename
Optimization remarks file path.
Definition: Config.h:142
VisScheme VisibilityScheme
Allows non-imported definitions to get the potentially more constraining visibility from the prevaili...
Definition: Config.h:98
std::string OverrideTriple
Setting this field will replace target triples in input files with this triple.
Definition: Config.h:112
std::string ProfileRemapping
Name remapping file for profile data.
Definition: Config.h:125
bool AllVtablesHaveTypeInfos
If all native vtables have corresponding type infos, allow usage of RTTI to block devirtualization on...
Definition: Config.h:87
TargetOptions Options
Definition: Config.h:49
bool TimeTraceEnabled
Time trace enabled.
Definition: Config.h:178
std::string RemarksPasses
Optimization remarks pass filter.
Definition: Config.h:145
std::string OptPipeline
If this field is set, the set of passes run in the middle-end optimizer will be the one specified by ...
Definition: Config.h:103
ModuleHookFn PostInternalizeModuleHook
This hook is called after internalizing the module.
Definition: Config.h:233
unsigned TimeTraceGranularity
Time trace granularity.
Definition: Config.h:181
unsigned OptLevel
Definition: Config.h:59
bool RemarksWithHotness
Whether to emit optimization remarks with hotness informations.
Definition: Config.h:148
std::optional< Reloc::Model > RelocModel
Definition: Config.h:55
CodeGenFileType CGFileType
Definition: Config.h:58
bool Freestanding
Flag to indicate that the optimizer should not assume builtins are present on the target.
Definition: Config.h:68
std::string SampleProfile
Sample PGO profile path.
Definition: Config.h:122
std::string RemarksFormat
The format used for serializing remarks (default: YAML).
Definition: Config.h:166
A derived class of LLVMContext that initializes itself according to a given Config object.
Definition: Config.h:297
The resolution for a symbol.
Definition: LTO.h:453
unsigned FinalDefinitionInLinkageUnit
The definition of this symbol is unpreemptable at runtime and is known to be in this linkage unit.
Definition: LTO.h:463
unsigned ExportDynamic
The symbol was exported dynamically, and therefore could be referenced by a shared library not visibl...
Definition: LTO.h:470
unsigned Prevailing
The linker has chosen this definition of the symbol.
Definition: LTO.h:459
unsigned LinkerRedefined
Linker redefined version of the symbol which appeared in -wrap or -defsym linker option.
Definition: LTO.h:474
unsigned VisibleToRegularObj
The definition of this symbol is visible outside of the LTO unit.
Definition: LTO.h:466