LLVM 19.0.0git
LTO.cpp
Go to the documentation of this file.
1//===-LTO.cpp - LLVM Link Time Optimizer ----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements functions and classes used to support LTO.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/LTO/LTO.h"
14#include "llvm/ADT/ScopeExit.h"
15#include "llvm/ADT/SmallSet.h"
16#include "llvm/ADT/Statistic.h"
25#include "llvm/Config/llvm-config.h"
26#include "llvm/IR/AutoUpgrade.h"
28#include "llvm/IR/Intrinsics.h"
31#include "llvm/IR/Mangler.h"
32#include "llvm/IR/Metadata.h"
33#include "llvm/LTO/LTOBackend.h"
35#include "llvm/Linker/IRMover.h"
39#include "llvm/Support/Error.h"
43#include "llvm/Support/Path.h"
44#include "llvm/Support/SHA1.h"
50#include "llvm/Support/VCSRevision.h"
53#include "llvm/Transforms/IPO.h"
58
59#include <optional>
60#include <set>
61
62using namespace llvm;
63using namespace lto;
64using namespace object;
65
66#define DEBUG_TYPE "lto"
67
68static cl::opt<bool>
69 DumpThinCGSCCs("dump-thin-cg-sccs", cl::init(false), cl::Hidden,
70 cl::desc("Dump the SCCs in the ThinLTO index's callgraph"));
71
72namespace llvm {
73/// Enable global value internalization in LTO.
75 "enable-lto-internalization", cl::init(true), cl::Hidden,
76 cl::desc("Enable global value internalization in LTO"));
77
78/// Indicate we are linking with an allocator that supports hot/cold operator
79/// new interfaces.
81
82/// Enable MemProf context disambiguation for thin link.
84} // namespace llvm
85
86// Computes a unique hash for the Module considering the current list of
87// export/import and other global analysis results.
88// The hash is produced in \p Key.
90 SmallString<40> &Key, const Config &Conf, const ModuleSummaryIndex &Index,
91 StringRef ModuleID, const FunctionImporter::ImportMapTy &ImportList,
92 const FunctionImporter::ExportSetTy &ExportList,
93 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
94 const GVSummaryMapTy &DefinedGlobals,
95 const std::set<GlobalValue::GUID> &CfiFunctionDefs,
96 const std::set<GlobalValue::GUID> &CfiFunctionDecls) {
97 // Compute the unique hash for this entry.
98 // This is based on the current compiler version, the module itself, the
99 // export list, the hash for every single module in the import list, the
100 // list of ResolvedODR for the module, and the list of preserved symbols.
101 SHA1 Hasher;
102
103 // Start with the compiler revision
104 Hasher.update(LLVM_VERSION_STRING);
105#ifdef LLVM_REVISION
106 Hasher.update(LLVM_REVISION);
107#endif
108
109 // Include the parts of the LTO configuration that affect code generation.
110 auto AddString = [&](StringRef Str) {
111 Hasher.update(Str);
112 Hasher.update(ArrayRef<uint8_t>{0});
113 };
114 auto AddUnsigned = [&](unsigned I) {
115 uint8_t Data[4];
117 Hasher.update(ArrayRef<uint8_t>{Data, 4});
118 };
119 auto AddUint64 = [&](uint64_t I) {
120 uint8_t Data[8];
122 Hasher.update(ArrayRef<uint8_t>{Data, 8});
123 };
124 AddString(Conf.CPU);
125 // FIXME: Hash more of Options. For now all clients initialize Options from
126 // command-line flags (which is unsupported in production), but may set
127 // X86RelaxRelocations. The clang driver can also pass FunctionSections,
128 // DataSections and DebuggerTuning via command line flags.
129 AddUnsigned(Conf.Options.MCOptions.X86RelaxRelocations);
130 AddUnsigned(Conf.Options.FunctionSections);
131 AddUnsigned(Conf.Options.DataSections);
132 AddUnsigned((unsigned)Conf.Options.DebuggerTuning);
133 for (auto &A : Conf.MAttrs)
134 AddString(A);
135 if (Conf.RelocModel)
136 AddUnsigned(*Conf.RelocModel);
137 else
138 AddUnsigned(-1);
139 if (Conf.CodeModel)
140 AddUnsigned(*Conf.CodeModel);
141 else
142 AddUnsigned(-1);
143 for (const auto &S : Conf.MllvmArgs)
144 AddString(S);
145 AddUnsigned(static_cast<int>(Conf.CGOptLevel));
146 AddUnsigned(static_cast<int>(Conf.CGFileType));
147 AddUnsigned(Conf.OptLevel);
148 AddUnsigned(Conf.Freestanding);
149 AddString(Conf.OptPipeline);
150 AddString(Conf.AAPipeline);
151 AddString(Conf.OverrideTriple);
152 AddString(Conf.DefaultTriple);
153 AddString(Conf.DwoDir);
154
155 // Include the hash for the current module
156 auto ModHash = Index.getModuleHash(ModuleID);
157 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
158
159 std::vector<uint64_t> ExportsGUID;
160 ExportsGUID.reserve(ExportList.size());
161 for (const auto &VI : ExportList) {
162 auto GUID = VI.getGUID();
163 ExportsGUID.push_back(GUID);
164 }
165
166 // Sort the export list elements GUIDs.
167 llvm::sort(ExportsGUID);
168 for (uint64_t GUID : ExportsGUID) {
169 // The export list can impact the internalization, be conservative here
170 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&GUID, sizeof(GUID)));
171 }
172
173 // Include the hash for every module we import functions from. The set of
174 // imported symbols for each module may affect code generation and is
175 // sensitive to link order, so include that as well.
176 using ImportMapIteratorTy = FunctionImporter::ImportMapTy::const_iterator;
177 struct ImportModule {
178 ImportMapIteratorTy ModIt;
179 const ModuleSummaryIndex::ModuleInfo *ModInfo;
180
181 StringRef getIdentifier() const { return ModIt->getFirst(); }
182 const FunctionImporter::FunctionsToImportTy &getFunctions() const {
183 return ModIt->second;
184 }
185
186 const ModuleHash &getHash() const { return ModInfo->second; }
187 };
188
189 std::vector<ImportModule> ImportModulesVector;
190 ImportModulesVector.reserve(ImportList.size());
191
192 for (ImportMapIteratorTy It = ImportList.begin(); It != ImportList.end();
193 ++It) {
194 ImportModulesVector.push_back({It, Index.getModule(It->getFirst())});
195 }
196 // Order using module hash, to be both independent of module name and
197 // module order.
198 llvm::sort(ImportModulesVector,
199 [](const ImportModule &Lhs, const ImportModule &Rhs) -> bool {
200 return Lhs.getHash() < Rhs.getHash();
201 });
202 for (const ImportModule &Entry : ImportModulesVector) {
203 auto ModHash = Entry.getHash();
204 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
205
206 AddUint64(Entry.getFunctions().size());
207 for (auto &Fn : Entry.getFunctions())
208 AddUint64(Fn);
209 }
210
211 // Include the hash for the resolved ODR.
212 for (auto &Entry : ResolvedODR) {
213 Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.first,
214 sizeof(GlobalValue::GUID)));
215 Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.second,
217 }
218
219 // Members of CfiFunctionDefs and CfiFunctionDecls that are referenced or
220 // defined in this module.
221 std::set<GlobalValue::GUID> UsedCfiDefs;
222 std::set<GlobalValue::GUID> UsedCfiDecls;
223
224 // Typeids used in this module.
225 std::set<GlobalValue::GUID> UsedTypeIds;
226
227 auto AddUsedCfiGlobal = [&](GlobalValue::GUID ValueGUID) {
228 if (CfiFunctionDefs.count(ValueGUID))
229 UsedCfiDefs.insert(ValueGUID);
230 if (CfiFunctionDecls.count(ValueGUID))
231 UsedCfiDecls.insert(ValueGUID);
232 };
233
234 auto AddUsedThings = [&](GlobalValueSummary *GS) {
235 if (!GS) return;
236 AddUnsigned(GS->getVisibility());
237 AddUnsigned(GS->isLive());
238 AddUnsigned(GS->canAutoHide());
239 for (const ValueInfo &VI : GS->refs()) {
240 AddUnsigned(VI.isDSOLocal(Index.withDSOLocalPropagation()));
241 AddUsedCfiGlobal(VI.getGUID());
242 }
243 if (auto *GVS = dyn_cast<GlobalVarSummary>(GS)) {
244 AddUnsigned(GVS->maybeReadOnly());
245 AddUnsigned(GVS->maybeWriteOnly());
246 }
247 if (auto *FS = dyn_cast<FunctionSummary>(GS)) {
248 for (auto &TT : FS->type_tests())
249 UsedTypeIds.insert(TT);
250 for (auto &TT : FS->type_test_assume_vcalls())
251 UsedTypeIds.insert(TT.GUID);
252 for (auto &TT : FS->type_checked_load_vcalls())
253 UsedTypeIds.insert(TT.GUID);
254 for (auto &TT : FS->type_test_assume_const_vcalls())
255 UsedTypeIds.insert(TT.VFunc.GUID);
256 for (auto &TT : FS->type_checked_load_const_vcalls())
257 UsedTypeIds.insert(TT.VFunc.GUID);
258 for (auto &ET : FS->calls()) {
259 AddUnsigned(ET.first.isDSOLocal(Index.withDSOLocalPropagation()));
260 AddUsedCfiGlobal(ET.first.getGUID());
261 }
262 }
263 };
264
265 // Include the hash for the linkage type to reflect internalization and weak
266 // resolution, and collect any used type identifier resolutions.
267 for (auto &GS : DefinedGlobals) {
268 GlobalValue::LinkageTypes Linkage = GS.second->linkage();
269 Hasher.update(
270 ArrayRef<uint8_t>((const uint8_t *)&Linkage, sizeof(Linkage)));
271 AddUsedCfiGlobal(GS.first);
272 AddUsedThings(GS.second);
273 }
274
275 // Imported functions may introduce new uses of type identifier resolutions,
276 // so we need to collect their used resolutions as well.
277 for (const ImportModule &ImpM : ImportModulesVector)
278 for (auto &ImpF : ImpM.getFunctions()) {
280 Index.findSummaryInModule(ImpF, ImpM.getIdentifier());
281 AddUsedThings(S);
282 // If this is an alias, we also care about any types/etc. that the aliasee
283 // may reference.
284 if (auto *AS = dyn_cast_or_null<AliasSummary>(S))
285 AddUsedThings(AS->getBaseObject());
286 }
287
288 auto AddTypeIdSummary = [&](StringRef TId, const TypeIdSummary &S) {
289 AddString(TId);
290
291 AddUnsigned(S.TTRes.TheKind);
292 AddUnsigned(S.TTRes.SizeM1BitWidth);
293
294 AddUint64(S.TTRes.AlignLog2);
295 AddUint64(S.TTRes.SizeM1);
296 AddUint64(S.TTRes.BitMask);
297 AddUint64(S.TTRes.InlineBits);
298
299 AddUint64(S.WPDRes.size());
300 for (auto &WPD : S.WPDRes) {
301 AddUnsigned(WPD.first);
302 AddUnsigned(WPD.second.TheKind);
303 AddString(WPD.second.SingleImplName);
304
305 AddUint64(WPD.second.ResByArg.size());
306 for (auto &ByArg : WPD.second.ResByArg) {
307 AddUint64(ByArg.first.size());
308 for (uint64_t Arg : ByArg.first)
309 AddUint64(Arg);
310 AddUnsigned(ByArg.second.TheKind);
311 AddUint64(ByArg.second.Info);
312 AddUnsigned(ByArg.second.Byte);
313 AddUnsigned(ByArg.second.Bit);
314 }
315 }
316 };
317
318 // Include the hash for all type identifiers used by this module.
319 for (GlobalValue::GUID TId : UsedTypeIds) {
320 auto TidIter = Index.typeIds().equal_range(TId);
321 for (auto It = TidIter.first; It != TidIter.second; ++It)
322 AddTypeIdSummary(It->second.first, It->second.second);
323 }
324
325 AddUnsigned(UsedCfiDefs.size());
326 for (auto &V : UsedCfiDefs)
327 AddUint64(V);
328
329 AddUnsigned(UsedCfiDecls.size());
330 for (auto &V : UsedCfiDecls)
331 AddUint64(V);
332
333 if (!Conf.SampleProfile.empty()) {
334 auto FileOrErr = MemoryBuffer::getFile(Conf.SampleProfile);
335 if (FileOrErr) {
336 Hasher.update(FileOrErr.get()->getBuffer());
337
338 if (!Conf.ProfileRemapping.empty()) {
339 FileOrErr = MemoryBuffer::getFile(Conf.ProfileRemapping);
340 if (FileOrErr)
341 Hasher.update(FileOrErr.get()->getBuffer());
342 }
343 }
344 }
345
346 Key = toHex(Hasher.result());
347}
348
350 const Config &C, ValueInfo VI,
351 DenseSet<GlobalValueSummary *> &GlobalInvolvedWithAlias,
353 isPrevailing,
355 recordNewLinkage,
356 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
358 C.VisibilityScheme == Config::ELF ? VI.getELFVisibility()
360 for (auto &S : VI.getSummaryList()) {
361 GlobalValue::LinkageTypes OriginalLinkage = S->linkage();
362 // Ignore local and appending linkage values since the linker
363 // doesn't resolve them.
364 if (GlobalValue::isLocalLinkage(OriginalLinkage) ||
366 continue;
367 // We need to emit only one of these. The prevailing module will keep it,
368 // but turned into a weak, while the others will drop it when possible.
369 // This is both a compile-time optimization and a correctness
370 // transformation. This is necessary for correctness when we have exported
371 // a reference - we need to convert the linkonce to weak to
372 // ensure a copy is kept to satisfy the exported reference.
373 // FIXME: We may want to split the compile time and correctness
374 // aspects into separate routines.
375 if (isPrevailing(VI.getGUID(), S.get())) {
376 if (GlobalValue::isLinkOnceLinkage(OriginalLinkage)) {
377 S->setLinkage(GlobalValue::getWeakLinkage(
378 GlobalValue::isLinkOnceODRLinkage(OriginalLinkage)));
379 // The kept copy is eligible for auto-hiding (hidden visibility) if all
380 // copies were (i.e. they were all linkonce_odr global unnamed addr).
381 // If any copy is not (e.g. it was originally weak_odr), then the symbol
382 // must remain externally available (e.g. a weak_odr from an explicitly
383 // instantiated template). Additionally, if it is in the
384 // GUIDPreservedSymbols set, that means that it is visibile outside
385 // the summary (e.g. in a native object or a bitcode file without
386 // summary), and in that case we cannot hide it as it isn't possible to
387 // check all copies.
388 S->setCanAutoHide(VI.canAutoHide() &&
389 !GUIDPreservedSymbols.count(VI.getGUID()));
390 }
391 if (C.VisibilityScheme == Config::FromPrevailing)
392 Visibility = S->getVisibility();
393 }
394 // Alias and aliasee can't be turned into available_externally.
395 else if (!isa<AliasSummary>(S.get()) &&
396 !GlobalInvolvedWithAlias.count(S.get()))
398
399 // For ELF, set visibility to the computed visibility from summaries. We
400 // don't track visibility from declarations so this may be more relaxed than
401 // the most constraining one.
402 if (C.VisibilityScheme == Config::ELF)
403 S->setVisibility(Visibility);
404
405 if (S->linkage() != OriginalLinkage)
406 recordNewLinkage(S->modulePath(), VI.getGUID(), S->linkage());
407 }
408
409 if (C.VisibilityScheme == Config::FromPrevailing) {
410 for (auto &S : VI.getSummaryList()) {
411 GlobalValue::LinkageTypes OriginalLinkage = S->linkage();
412 if (GlobalValue::isLocalLinkage(OriginalLinkage) ||
414 continue;
415 S->setVisibility(Visibility);
416 }
417 }
418}
419
420/// Resolve linkage for prevailing symbols in the \p Index.
421//
422// We'd like to drop these functions if they are no longer referenced in the
423// current module. However there is a chance that another module is still
424// referencing them because of the import. We make sure we always emit at least
425// one copy.
429 isPrevailing,
431 recordNewLinkage,
432 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
433 // We won't optimize the globals that are referenced by an alias for now
434 // Ideally we should turn the alias into a global and duplicate the definition
435 // when needed.
436 DenseSet<GlobalValueSummary *> GlobalInvolvedWithAlias;
437 for (auto &I : Index)
438 for (auto &S : I.second.SummaryList)
439 if (auto AS = dyn_cast<AliasSummary>(S.get()))
440 GlobalInvolvedWithAlias.insert(&AS->getAliasee());
441
442 for (auto &I : Index)
443 thinLTOResolvePrevailingGUID(C, Index.getValueInfo(I),
444 GlobalInvolvedWithAlias, isPrevailing,
445 recordNewLinkage, GUIDPreservedSymbols);
446}
447
449 ValueInfo VI, function_ref<bool(StringRef, ValueInfo)> isExported,
451 isPrevailing) {
452 auto ExternallyVisibleCopies =
453 llvm::count_if(VI.getSummaryList(),
454 [](const std::unique_ptr<GlobalValueSummary> &Summary) {
455 return !GlobalValue::isLocalLinkage(Summary->linkage());
456 });
457
458 for (auto &S : VI.getSummaryList()) {
459 // First see if we need to promote an internal value because it is not
460 // exported.
461 if (isExported(S->modulePath(), VI)) {
462 if (GlobalValue::isLocalLinkage(S->linkage()))
463 S->setLinkage(GlobalValue::ExternalLinkage);
464 continue;
465 }
466
467 // Otherwise, see if we can internalize.
469 continue;
470
471 // Non-exported values with external linkage can be internalized.
472 if (GlobalValue::isExternalLinkage(S->linkage())) {
473 S->setLinkage(GlobalValue::InternalLinkage);
474 continue;
475 }
476
477 // Non-exported function and variable definitions with a weak-for-linker
478 // linkage can be internalized in certain cases. The minimum legality
479 // requirements would be that they are not address taken to ensure that we
480 // don't break pointer equality checks, and that variables are either read-
481 // or write-only. For functions, this is the case if either all copies are
482 // [local_]unnamed_addr, or we can propagate reference edge attributes
483 // (which is how this is guaranteed for variables, when analyzing whether
484 // they are read or write-only).
485 //
486 // However, we only get to this code for weak-for-linkage values in one of
487 // two cases:
488 // 1) The prevailing copy is not in IR (it is in native code).
489 // 2) The prevailing copy in IR is not exported from its module.
490 // Additionally, at least for the new LTO API, case 2 will only happen if
491 // there is exactly one definition of the value (i.e. in exactly one
492 // module), as duplicate defs are result in the value being marked exported.
493 // Likely, users of the legacy LTO API are similar, however, currently there
494 // are llvm-lto based tests of the legacy LTO API that do not mark
495 // duplicate linkonce_odr copies as exported via the tool, so we need
496 // to handle that case below by checking the number of copies.
497 //
498 // Generally, we only want to internalize a weak-for-linker value in case
499 // 2, because in case 1 we cannot see how the value is used to know if it
500 // is read or write-only. We also don't want to bloat the binary with
501 // multiple internalized copies of non-prevailing linkonce/weak functions.
502 // Note if we don't internalize, we will convert non-prevailing copies to
503 // available_externally anyway, so that we drop them after inlining. The
504 // only reason to internalize such a function is if we indeed have a single
505 // copy, because internalizing it won't increase binary size, and enables
506 // use of inliner heuristics that are more aggressive in the face of a
507 // single call to a static (local). For variables, internalizing a read or
508 // write only variable can enable more aggressive optimization. However, we
509 // already perform this elsewhere in the ThinLTO backend handling for
510 // read or write-only variables (processGlobalForThinLTO).
511 //
512 // Therefore, only internalize linkonce/weak if there is a single copy, that
513 // is prevailing in this IR module. We can do so aggressively, without
514 // requiring the address to be insignificant, or that a variable be read or
515 // write-only.
516 if (!GlobalValue::isWeakForLinker(S->linkage()) ||
518 continue;
519
520 if (isPrevailing(VI.getGUID(), S.get()) && ExternallyVisibleCopies == 1)
521 S->setLinkage(GlobalValue::InternalLinkage);
522 }
523}
524
525// Update the linkages in the given \p Index to mark exported values
526// as external and non-exported values as internal.
529 function_ref<bool(StringRef, ValueInfo)> isExported,
531 isPrevailing) {
532 for (auto &I : Index)
533 thinLTOInternalizeAndPromoteGUID(Index.getValueInfo(I), isExported,
534 isPrevailing);
535}
536
537// Requires a destructor for std::vector<InputModule>.
538InputFile::~InputFile() = default;
539
541 std::unique_ptr<InputFile> File(new InputFile);
542
543 Expected<IRSymtabFile> FOrErr = readIRSymtab(Object);
544 if (!FOrErr)
545 return FOrErr.takeError();
546
547 File->TargetTriple = FOrErr->TheReader.getTargetTriple();
548 File->SourceFileName = FOrErr->TheReader.getSourceFileName();
549 File->COFFLinkerOpts = FOrErr->TheReader.getCOFFLinkerOpts();
550 File->DependentLibraries = FOrErr->TheReader.getDependentLibraries();
551 File->ComdatTable = FOrErr->TheReader.getComdatTable();
552
553 for (unsigned I = 0; I != FOrErr->Mods.size(); ++I) {
554 size_t Begin = File->Symbols.size();
555 for (const irsymtab::Reader::SymbolRef &Sym :
556 FOrErr->TheReader.module_symbols(I))
557 // Skip symbols that are irrelevant to LTO. Note that this condition needs
558 // to match the one in Skip() in LTO::addRegularLTO().
559 if (Sym.isGlobal() && !Sym.isFormatSpecific())
560 File->Symbols.push_back(Sym);
561 File->ModuleSymIndices.push_back({Begin, File->Symbols.size()});
562 }
563
564 File->Mods = FOrErr->Mods;
565 File->Strtab = std::move(FOrErr->Strtab);
566 return std::move(File);
567}
568
570 return Mods[0].getModuleIdentifier();
571}
572
574 assert(Mods.size() == 1 && "Expect only one bitcode module");
575 return Mods[0];
576}
577
578LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel,
579 const Config &Conf)
580 : ParallelCodeGenParallelismLevel(ParallelCodeGenParallelismLevel),
581 Ctx(Conf), CombinedModule(std::make_unique<Module>("ld-temp.o", Ctx)),
582 Mover(std::make_unique<IRMover>(*CombinedModule)) {
583 CombinedModule->IsNewDbgInfoFormat = UseNewDbgInfoFormat;
584}
585
586LTO::ThinLTOState::ThinLTOState(ThinBackend Backend)
587 : Backend(Backend), CombinedIndex(/*HaveGVs*/ false) {
588 if (!Backend)
589 this->Backend =
591}
592
594 unsigned ParallelCodeGenParallelismLevel, LTOKind LTOMode)
595 : Conf(std::move(Conf)),
596 RegularLTO(ParallelCodeGenParallelismLevel, this->Conf),
597 ThinLTO(std::move(Backend)),
598 GlobalResolutions(std::make_optional<StringMap<GlobalResolution>>()),
599 LTOMode(LTOMode) {}
600
601// Requires a destructor for MapVector<BitcodeModule>.
602LTO::~LTO() = default;
603
604// Add the symbols in the given module to the GlobalResolutions map, and resolve
605// their partitions.
606void LTO::addModuleToGlobalRes(ArrayRef<InputFile::Symbol> Syms,
608 unsigned Partition, bool InSummary) {
609 auto *ResI = Res.begin();
610 auto *ResE = Res.end();
611 (void)ResE;
612 const Triple TT(RegularLTO.CombinedModule->getTargetTriple());
613 for (const InputFile::Symbol &Sym : Syms) {
614 assert(ResI != ResE);
615 SymbolResolution Res = *ResI++;
616
617 auto &GlobalRes = (*GlobalResolutions)[Sym.getName()];
618 GlobalRes.UnnamedAddr &= Sym.isUnnamedAddr();
619 if (Res.Prevailing) {
620 assert(!GlobalRes.Prevailing &&
621 "Multiple prevailing defs are not allowed");
622 GlobalRes.Prevailing = true;
623 GlobalRes.IRName = std::string(Sym.getIRName());
624 } else if (!GlobalRes.Prevailing && GlobalRes.IRName.empty()) {
625 // Sometimes it can be two copies of symbol in a module and prevailing
626 // symbol can have no IR name. That might happen if symbol is defined in
627 // module level inline asm block. In case we have multiple modules with
628 // the same symbol we want to use IR name of the prevailing symbol.
629 // Otherwise, if we haven't seen a prevailing symbol, set the name so that
630 // we can later use it to check if there is any prevailing copy in IR.
631 GlobalRes.IRName = std::string(Sym.getIRName());
632 }
633
634 // In rare occasion, the symbol used to initialize GlobalRes has a different
635 // IRName from the inspected Symbol. This can happen on macOS + iOS, when a
636 // symbol is referenced through its mangled name, say @"\01_symbol" while
637 // the IRName is @symbol (the prefix underscore comes from MachO mangling).
638 // In that case, we have the same actual Symbol that can get two different
639 // GUID, leading to some invalid internalization. Workaround this by marking
640 // the GlobalRes external.
641
642 // FIXME: instead of this check, it would be desirable to compute GUIDs
643 // based on mangled name, but this requires an access to the Target Triple
644 // and would be relatively invasive on the codebase.
645 if (GlobalRes.IRName != Sym.getIRName()) {
646 GlobalRes.Partition = GlobalResolution::External;
647 GlobalRes.VisibleOutsideSummary = true;
648 }
649
650 // Set the partition to external if we know it is re-defined by the linker
651 // with -defsym or -wrap options, used elsewhere, e.g. it is visible to a
652 // regular object, is referenced from llvm.compiler.used/llvm.used, or was
653 // already recorded as being referenced from a different partition.
654 if (Res.LinkerRedefined || Res.VisibleToRegularObj || Sym.isUsed() ||
655 (GlobalRes.Partition != GlobalResolution::Unknown &&
656 GlobalRes.Partition != Partition)) {
657 GlobalRes.Partition = GlobalResolution::External;
658 } else
659 // First recorded reference, save the current partition.
660 GlobalRes.Partition = Partition;
661
662 // Flag as visible outside of summary if visible from a regular object or
663 // from a module that does not have a summary.
664 GlobalRes.VisibleOutsideSummary |=
665 (Res.VisibleToRegularObj || Sym.isUsed() || !InSummary);
666
667 GlobalRes.ExportDynamic |= Res.ExportDynamic;
668 }
669}
670
673 StringRef Path = Input->getName();
674 OS << Path << '\n';
675 auto ResI = Res.begin();
676 for (const InputFile::Symbol &Sym : Input->symbols()) {
677 assert(ResI != Res.end());
678 SymbolResolution Res = *ResI++;
679
680 OS << "-r=" << Path << ',' << Sym.getName() << ',';
681 if (Res.Prevailing)
682 OS << 'p';
684 OS << 'l';
685 if (Res.VisibleToRegularObj)
686 OS << 'x';
687 if (Res.LinkerRedefined)
688 OS << 'r';
689 OS << '\n';
690 }
691 OS.flush();
692 assert(ResI == Res.end());
693}
694
695Error LTO::add(std::unique_ptr<InputFile> Input,
697 assert(!CalledGetMaxTasks);
698
699 if (Conf.ResolutionFile)
700 writeToResolutionFile(*Conf.ResolutionFile, Input.get(), Res);
701
702 if (RegularLTO.CombinedModule->getTargetTriple().empty()) {
703 RegularLTO.CombinedModule->setTargetTriple(Input->getTargetTriple());
704 if (Triple(Input->getTargetTriple()).isOSBinFormatELF())
706 }
707
708 const SymbolResolution *ResI = Res.begin();
709 for (unsigned I = 0; I != Input->Mods.size(); ++I)
710 if (Error Err = addModule(*Input, I, ResI, Res.end()))
711 return Err;
712
713 assert(ResI == Res.end());
714 return Error::success();
715}
716
717Error LTO::addModule(InputFile &Input, unsigned ModI,
718 const SymbolResolution *&ResI,
719 const SymbolResolution *ResE) {
720 Expected<BitcodeLTOInfo> LTOInfo = Input.Mods[ModI].getLTOInfo();
721 if (!LTOInfo)
722 return LTOInfo.takeError();
723
724 if (EnableSplitLTOUnit) {
725 // If only some modules were split, flag this in the index so that
726 // we can skip or error on optimizations that need consistently split
727 // modules (whole program devirt and lower type tests).
728 if (*EnableSplitLTOUnit != LTOInfo->EnableSplitLTOUnit)
729 ThinLTO.CombinedIndex.setPartiallySplitLTOUnits();
730 } else
731 EnableSplitLTOUnit = LTOInfo->EnableSplitLTOUnit;
732
733 BitcodeModule BM = Input.Mods[ModI];
734
735 if ((LTOMode == LTOK_UnifiedRegular || LTOMode == LTOK_UnifiedThin) &&
736 !LTOInfo->UnifiedLTO)
737 return make_error<StringError>(
738 "unified LTO compilation must use "
739 "compatible bitcode modules (use -funified-lto)",
741
742 if (LTOInfo->UnifiedLTO && LTOMode == LTOK_Default)
743 LTOMode = LTOK_UnifiedThin;
744
745 bool IsThinLTO = LTOInfo->IsThinLTO && (LTOMode != LTOK_UnifiedRegular);
746
747 auto ModSyms = Input.module_symbols(ModI);
748 addModuleToGlobalRes(ModSyms, {ResI, ResE},
749 IsThinLTO ? ThinLTO.ModuleMap.size() + 1 : 0,
750 LTOInfo->HasSummary);
751
752 if (IsThinLTO)
753 return addThinLTO(BM, ModSyms, ResI, ResE);
754
755 RegularLTO.EmptyCombinedModule = false;
757 addRegularLTO(BM, ModSyms, ResI, ResE);
758 if (!ModOrErr)
759 return ModOrErr.takeError();
760
761 if (!LTOInfo->HasSummary)
762 return linkRegularLTO(std::move(*ModOrErr), /*LivenessFromIndex=*/false);
763
764 // Regular LTO module summaries are added to a dummy module that represents
765 // the combined regular LTO module.
766 if (Error Err = BM.readSummary(ThinLTO.CombinedIndex, ""))
767 return Err;
768 RegularLTO.ModsWithSummaries.push_back(std::move(*ModOrErr));
769 return Error::success();
770}
771
772// Checks whether the given global value is in a non-prevailing comdat
773// (comdat containing values the linker indicated were not prevailing,
774// which we then dropped to available_externally), and if so, removes
775// it from the comdat. This is called for all global values to ensure the
776// comdat is empty rather than leaving an incomplete comdat. It is needed for
777// regular LTO modules, in case we are in a mixed-LTO mode (both regular
778// and thin LTO modules) compilation. Since the regular LTO module will be
779// linked first in the final native link, we want to make sure the linker
780// doesn't select any of these incomplete comdats that would be left
781// in the regular LTO module without this cleanup.
782static void
784 std::set<const Comdat *> &NonPrevailingComdats) {
785 Comdat *C = GV.getComdat();
786 if (!C)
787 return;
788
789 if (!NonPrevailingComdats.count(C))
790 return;
791
792 // Additionally need to drop all global values from the comdat to
793 // available_externally, to satisfy the COMDAT requirement that all members
794 // are discarded as a unit. The non-local linkage global values avoid
795 // duplicate definition linker errors.
797
798 if (auto GO = dyn_cast<GlobalObject>(&GV))
799 GO->setComdat(nullptr);
800}
801
802// Add a regular LTO object to the link.
803// The resulting module needs to be linked into the combined LTO module with
804// linkRegularLTO.
806LTO::addRegularLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
807 const SymbolResolution *&ResI,
808 const SymbolResolution *ResE) {
809 RegularLTOState::AddedModule Mod;
811 BM.getLazyModule(RegularLTO.Ctx, /*ShouldLazyLoadMetadata*/ true,
812 /*IsImporting*/ false);
813 if (!MOrErr)
814 return MOrErr.takeError();
815 Module &M = **MOrErr;
816 Mod.M = std::move(*MOrErr);
817
818 if (Error Err = M.materializeMetadata())
819 return std::move(Err);
820
821 // If cfi.functions is present and we are in regular LTO mode, LowerTypeTests
822 // will rename local functions in the merged module as "<function name>.1".
823 // This causes linking errors, since other parts of the module expect the
824 // original function name.
825 if (LTOMode == LTOK_UnifiedRegular)
826 if (NamedMDNode *CfiFunctionsMD = M.getNamedMetadata("cfi.functions"))
827 M.eraseNamedMetadata(CfiFunctionsMD);
828
830
831 ModuleSymbolTable SymTab;
832 SymTab.addModule(&M);
833
834 for (GlobalVariable &GV : M.globals())
835 if (GV.hasAppendingLinkage())
836 Mod.Keep.push_back(&GV);
837
838 DenseSet<GlobalObject *> AliasedGlobals;
839 for (auto &GA : M.aliases())
840 if (GlobalObject *GO = GA.getAliaseeObject())
841 AliasedGlobals.insert(GO);
842
843 // In this function we need IR GlobalValues matching the symbols in Syms
844 // (which is not backed by a module), so we need to enumerate them in the same
845 // order. The symbol enumeration order of a ModuleSymbolTable intentionally
846 // matches the order of an irsymtab, but when we read the irsymtab in
847 // InputFile::create we omit some symbols that are irrelevant to LTO. The
848 // Skip() function skips the same symbols from the module as InputFile does
849 // from the symbol table.
850 auto MsymI = SymTab.symbols().begin(), MsymE = SymTab.symbols().end();
851 auto Skip = [&]() {
852 while (MsymI != MsymE) {
853 auto Flags = SymTab.getSymbolFlags(*MsymI);
854 if ((Flags & object::BasicSymbolRef::SF_Global) &&
856 return;
857 ++MsymI;
858 }
859 };
860 Skip();
861
862 std::set<const Comdat *> NonPrevailingComdats;
863 SmallSet<StringRef, 2> NonPrevailingAsmSymbols;
864 for (const InputFile::Symbol &Sym : Syms) {
865 assert(ResI != ResE);
866 SymbolResolution Res = *ResI++;
867
868 assert(MsymI != MsymE);
869 ModuleSymbolTable::Symbol Msym = *MsymI++;
870 Skip();
871
872 if (GlobalValue *GV = dyn_cast_if_present<GlobalValue *>(Msym)) {
873 if (Res.Prevailing) {
874 if (Sym.isUndefined())
875 continue;
876 Mod.Keep.push_back(GV);
877 // For symbols re-defined with linker -wrap and -defsym options,
878 // set the linkage to weak to inhibit IPO. The linkage will be
879 // restored by the linker.
880 if (Res.LinkerRedefined)
881 GV->setLinkage(GlobalValue::WeakAnyLinkage);
882
883 GlobalValue::LinkageTypes OriginalLinkage = GV->getLinkage();
884 if (GlobalValue::isLinkOnceLinkage(OriginalLinkage))
885 GV->setLinkage(GlobalValue::getWeakLinkage(
886 GlobalValue::isLinkOnceODRLinkage(OriginalLinkage)));
887 } else if (isa<GlobalObject>(GV) &&
888 (GV->hasLinkOnceODRLinkage() || GV->hasWeakODRLinkage() ||
889 GV->hasAvailableExternallyLinkage()) &&
890 !AliasedGlobals.count(cast<GlobalObject>(GV))) {
891 // Any of the above three types of linkage indicates that the
892 // chosen prevailing symbol will have the same semantics as this copy of
893 // the symbol, so we may be able to link it with available_externally
894 // linkage. We will decide later whether to do that when we link this
895 // module (in linkRegularLTO), based on whether it is undefined.
896 Mod.Keep.push_back(GV);
898 if (GV->hasComdat())
899 NonPrevailingComdats.insert(GV->getComdat());
900 cast<GlobalObject>(GV)->setComdat(nullptr);
901 }
902
903 // Set the 'local' flag based on the linker resolution for this symbol.
905 GV->setDSOLocal(true);
906 if (GV->hasDLLImportStorageClass())
907 GV->setDLLStorageClass(GlobalValue::DLLStorageClassTypes::
908 DefaultStorageClass);
909 }
910 } else if (auto *AS =
911 dyn_cast_if_present<ModuleSymbolTable::AsmSymbol *>(Msym)) {
912 // Collect non-prevailing symbols.
913 if (!Res.Prevailing)
914 NonPrevailingAsmSymbols.insert(AS->first);
915 } else {
916 llvm_unreachable("unknown symbol type");
917 }
918
919 // Common resolution: collect the maximum size/alignment over all commons.
920 // We also record if we see an instance of a common as prevailing, so that
921 // if none is prevailing we can ignore it later.
922 if (Sym.isCommon()) {
923 // FIXME: We should figure out what to do about commons defined by asm.
924 // For now they aren't reported correctly by ModuleSymbolTable.
925 auto &CommonRes = RegularLTO.Commons[std::string(Sym.getIRName())];
926 CommonRes.Size = std::max(CommonRes.Size, Sym.getCommonSize());
927 if (uint32_t SymAlignValue = Sym.getCommonAlignment()) {
928 CommonRes.Alignment =
929 std::max(Align(SymAlignValue), CommonRes.Alignment);
930 }
931 CommonRes.Prevailing |= Res.Prevailing;
932 }
933 }
934
935 if (!M.getComdatSymbolTable().empty())
936 for (GlobalValue &GV : M.global_values())
937 handleNonPrevailingComdat(GV, NonPrevailingComdats);
938
939 // Prepend ".lto_discard <sym>, <sym>*" directive to each module inline asm
940 // block.
941 if (!M.getModuleInlineAsm().empty()) {
942 std::string NewIA = ".lto_discard";
943 if (!NonPrevailingAsmSymbols.empty()) {
944 // Don't dicard a symbol if there is a live .symver for it.
946 M, [&](StringRef Name, StringRef Alias) {
947 if (!NonPrevailingAsmSymbols.count(Alias))
948 NonPrevailingAsmSymbols.erase(Name);
949 });
950 NewIA += " " + llvm::join(NonPrevailingAsmSymbols, ", ");
951 }
952 NewIA += "\n";
953 M.setModuleInlineAsm(NewIA + M.getModuleInlineAsm());
954 }
955
956 assert(MsymI == MsymE);
957 return std::move(Mod);
958}
959
960Error LTO::linkRegularLTO(RegularLTOState::AddedModule Mod,
961 bool LivenessFromIndex) {
962 std::vector<GlobalValue *> Keep;
963 for (GlobalValue *GV : Mod.Keep) {
964 if (LivenessFromIndex && !ThinLTO.CombinedIndex.isGUIDLive(GV->getGUID())) {
965 if (Function *F = dyn_cast<Function>(GV)) {
966 if (DiagnosticOutputFile) {
967 if (Error Err = F->materialize())
968 return Err;
969 OptimizationRemarkEmitter ORE(F, nullptr);
970 ORE.emit(OptimizationRemark(DEBUG_TYPE, "deadfunction", F)
971 << ore::NV("Function", F)
972 << " not added to the combined module ");
973 }
974 }
975 continue;
976 }
977
979 Keep.push_back(GV);
980 continue;
981 }
982
983 // Only link available_externally definitions if we don't already have a
984 // definition.
985 GlobalValue *CombinedGV =
986 RegularLTO.CombinedModule->getNamedValue(GV->getName());
987 if (CombinedGV && !CombinedGV->isDeclaration())
988 continue;
989
990 Keep.push_back(GV);
991 }
992
993 return RegularLTO.Mover->move(std::move(Mod.M), Keep, nullptr,
994 /* IsPerformingImport */ false);
995}
996
997// Add a ThinLTO module to the link.
998Error LTO::addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
999 const SymbolResolution *&ResI,
1000 const SymbolResolution *ResE) {
1001 const SymbolResolution *ResITmp = ResI;
1002 for (const InputFile::Symbol &Sym : Syms) {
1003 assert(ResITmp != ResE);
1004 SymbolResolution Res = *ResITmp++;
1005
1006 if (!Sym.getIRName().empty()) {
1008 Sym.getIRName(), GlobalValue::ExternalLinkage, ""));
1009 if (Res.Prevailing)
1010 ThinLTO.PrevailingModuleForGUID[GUID] = BM.getModuleIdentifier();
1011 }
1012 }
1013
1014 if (Error Err =
1015 BM.readSummary(ThinLTO.CombinedIndex, BM.getModuleIdentifier(),
1016 [&](GlobalValue::GUID GUID) {
1017 return ThinLTO.PrevailingModuleForGUID[GUID] ==
1018 BM.getModuleIdentifier();
1019 }))
1020 return Err;
1021 LLVM_DEBUG(dbgs() << "Module " << BM.getModuleIdentifier() << "\n");
1022
1023 for (const InputFile::Symbol &Sym : Syms) {
1024 assert(ResI != ResE);
1025 SymbolResolution Res = *ResI++;
1026
1027 if (!Sym.getIRName().empty()) {
1029 Sym.getIRName(), GlobalValue::ExternalLinkage, ""));
1030 if (Res.Prevailing) {
1031 assert(ThinLTO.PrevailingModuleForGUID[GUID] ==
1032 BM.getModuleIdentifier());
1033
1034 // For linker redefined symbols (via --wrap or --defsym) we want to
1035 // switch the linkage to `weak` to prevent IPOs from happening.
1036 // Find the summary in the module for this very GV and record the new
1037 // linkage so that we can switch it when we import the GV.
1038 if (Res.LinkerRedefined)
1039 if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(
1040 GUID, BM.getModuleIdentifier()))
1041 S->setLinkage(GlobalValue::WeakAnyLinkage);
1042 }
1043
1044 // If the linker resolved the symbol to a local definition then mark it
1045 // as local in the summary for the module we are adding.
1047 if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(
1048 GUID, BM.getModuleIdentifier())) {
1049 S->setDSOLocal(true);
1050 }
1051 }
1052 }
1053 }
1054
1055 if (!ThinLTO.ModuleMap.insert({BM.getModuleIdentifier(), BM}).second)
1056 return make_error<StringError>(
1057 "Expected at most one ThinLTO module per bitcode file",
1059
1060 if (!Conf.ThinLTOModulesToCompile.empty()) {
1061 if (!ThinLTO.ModulesToCompile)
1062 ThinLTO.ModulesToCompile = ModuleMapType();
1063 // This is a fuzzy name matching where only modules with name containing the
1064 // specified switch values are going to be compiled.
1065 for (const std::string &Name : Conf.ThinLTOModulesToCompile) {
1066 if (BM.getModuleIdentifier().contains(Name)) {
1067 ThinLTO.ModulesToCompile->insert({BM.getModuleIdentifier(), BM});
1068 llvm::errs() << "[ThinLTO] Selecting " << BM.getModuleIdentifier()
1069 << " to compile\n";
1070 }
1071 }
1072 }
1073
1074 return Error::success();
1075}
1076
1077unsigned LTO::getMaxTasks() const {
1078 CalledGetMaxTasks = true;
1079 auto ModuleCount = ThinLTO.ModulesToCompile ? ThinLTO.ModulesToCompile->size()
1080 : ThinLTO.ModuleMap.size();
1081 return RegularLTO.ParallelCodeGenParallelismLevel + ModuleCount;
1082}
1083
1084// If only some of the modules were split, we cannot correctly handle
1085// code that contains type tests or type checked loads.
1086Error LTO::checkPartiallySplit() {
1087 if (!ThinLTO.CombinedIndex.partiallySplitLTOUnits())
1088 return Error::success();
1089
1090 Function *TypeTestFunc = RegularLTO.CombinedModule->getFunction(
1091 Intrinsic::getName(Intrinsic::type_test));
1092 Function *TypeCheckedLoadFunc = RegularLTO.CombinedModule->getFunction(
1093 Intrinsic::getName(Intrinsic::type_checked_load));
1094 Function *TypeCheckedLoadRelativeFunc =
1095 RegularLTO.CombinedModule->getFunction(
1096 Intrinsic::getName(Intrinsic::type_checked_load_relative));
1097
1098 // First check if there are type tests / type checked loads in the
1099 // merged regular LTO module IR.
1100 if ((TypeTestFunc && !TypeTestFunc->use_empty()) ||
1101 (TypeCheckedLoadFunc && !TypeCheckedLoadFunc->use_empty()) ||
1102 (TypeCheckedLoadRelativeFunc &&
1103 !TypeCheckedLoadRelativeFunc->use_empty()))
1104 return make_error<StringError>(
1105 "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",
1107
1108 // Otherwise check if there are any recorded in the combined summary from the
1109 // ThinLTO modules.
1110 for (auto &P : ThinLTO.CombinedIndex) {
1111 for (auto &S : P.second.SummaryList) {
1112 auto *FS = dyn_cast<FunctionSummary>(S.get());
1113 if (!FS)
1114 continue;
1115 if (!FS->type_test_assume_vcalls().empty() ||
1116 !FS->type_checked_load_vcalls().empty() ||
1117 !FS->type_test_assume_const_vcalls().empty() ||
1118 !FS->type_checked_load_const_vcalls().empty() ||
1119 !FS->type_tests().empty())
1120 return make_error<StringError>(
1121 "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",
1123 }
1124 }
1125 return Error::success();
1126}
1127
1129 // Compute "dead" symbols, we don't want to import/export these!
1130 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1131 DenseMap<GlobalValue::GUID, PrevailingType> GUIDPrevailingResolutions;
1132 for (auto &Res : *GlobalResolutions) {
1133 // Normally resolution have IR name of symbol. We can do nothing here
1134 // otherwise. See comments in GlobalResolution struct for more details.
1135 if (Res.second.IRName.empty())
1136 continue;
1137
1139 GlobalValue::dropLLVMManglingEscape(Res.second.IRName));
1140
1141 if (Res.second.VisibleOutsideSummary && Res.second.Prevailing)
1142 GUIDPreservedSymbols.insert(GUID);
1143
1144 if (Res.second.ExportDynamic)
1145 DynamicExportSymbols.insert(GUID);
1146
1147 GUIDPrevailingResolutions[GUID] =
1149 }
1150
1151 auto isPrevailing = [&](GlobalValue::GUID G) {
1152 auto It = GUIDPrevailingResolutions.find(G);
1153 if (It == GUIDPrevailingResolutions.end())
1155 return It->second;
1156 };
1157 computeDeadSymbolsWithConstProp(ThinLTO.CombinedIndex, GUIDPreservedSymbols,
1158 isPrevailing, Conf.OptLevel > 0);
1159
1160 // Setup output file to emit statistics.
1161 auto StatsFileOrErr = setupStatsFile(Conf.StatsFile);
1162 if (!StatsFileOrErr)
1163 return StatsFileOrErr.takeError();
1164 std::unique_ptr<ToolOutputFile> StatsFile = std::move(StatsFileOrErr.get());
1165
1166 // TODO: Ideally this would be controlled automatically by detecting that we
1167 // are linking with an allocator that supports these interfaces, rather than
1168 // an internal option (which would still be needed for tests, however). For
1169 // example, if the library exported a symbol like __malloc_hot_cold the linker
1170 // could recognize that and set a flag in the lto::Config.
1172 ThinLTO.CombinedIndex.setWithSupportsHotColdNew();
1173
1174 Error Result = runRegularLTO(AddStream);
1175 if (!Result)
1176 // This will reset the GlobalResolutions optional once done with it to
1177 // reduce peak memory before importing.
1178 Result = runThinLTO(AddStream, Cache, GUIDPreservedSymbols);
1179
1180 if (StatsFile)
1181 PrintStatisticsJSON(StatsFile->os());
1182
1183 return Result;
1184}
1185
1187 const ModuleSummaryIndex &Index) {
1188 if (Index.withSupportsHotColdNew())
1189 return;
1190
1191 // The profile matcher applies hotness attributes directly for allocations,
1192 // and those will cause us to generate calls to the hot/cold interfaces
1193 // unconditionally. If supports-hot-cold-new was not enabled in the LTO
1194 // link then assume we don't want these calls (e.g. not linking with
1195 // the appropriate library, or otherwise trying to disable this behavior).
1196 for (auto &F : Mod) {
1197 for (auto &BB : F) {
1198 for (auto &I : BB) {
1199 auto *CI = dyn_cast<CallBase>(&I);
1200 if (!CI)
1201 continue;
1202 if (CI->hasFnAttr("memprof"))
1203 CI->removeFnAttr("memprof");
1204 // Strip off all memprof metadata as it is no longer needed.
1205 // Importantly, this avoids the addition of new memprof attributes
1206 // after inlining propagation.
1207 // TODO: If we support additional types of MemProf metadata beyond hot
1208 // and cold, we will need to update the metadata based on the allocator
1209 // APIs supported instead of completely stripping all.
1210 CI->setMetadata(LLVMContext::MD_memprof, nullptr);
1211 CI->setMetadata(LLVMContext::MD_callsite, nullptr);
1212 }
1213 }
1214 }
1215}
1216
1217Error LTO::runRegularLTO(AddStreamFn AddStream) {
1218 // Setup optimization remarks.
1219 auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
1220 RegularLTO.CombinedModule->getContext(), Conf.RemarksFilename,
1223 LLVM_DEBUG(dbgs() << "Running regular LTO\n");
1224 if (!DiagFileOrErr)
1225 return DiagFileOrErr.takeError();
1226 DiagnosticOutputFile = std::move(*DiagFileOrErr);
1227
1228 // Finalize linking of regular LTO modules containing summaries now that
1229 // we have computed liveness information.
1230 for (auto &M : RegularLTO.ModsWithSummaries)
1231 if (Error Err = linkRegularLTO(std::move(M),
1232 /*LivenessFromIndex=*/true))
1233 return Err;
1234
1235 // Ensure we don't have inconsistently split LTO units with type tests.
1236 // FIXME: this checks both LTO and ThinLTO. It happens to work as we take
1237 // this path both cases but eventually this should be split into two and
1238 // do the ThinLTO checks in `runThinLTO`.
1239 if (Error Err = checkPartiallySplit())
1240 return Err;
1241
1242 // Make sure commons have the right size/alignment: we kept the largest from
1243 // all the prevailing when adding the inputs, and we apply it here.
1244 const DataLayout &DL = RegularLTO.CombinedModule->getDataLayout();
1245 for (auto &I : RegularLTO.Commons) {
1246 if (!I.second.Prevailing)
1247 // Don't do anything if no instance of this common was prevailing.
1248 continue;
1249 GlobalVariable *OldGV = RegularLTO.CombinedModule->getNamedGlobal(I.first);
1250 if (OldGV && DL.getTypeAllocSize(OldGV->getValueType()) == I.second.Size) {
1251 // Don't create a new global if the type is already correct, just make
1252 // sure the alignment is correct.
1253 OldGV->setAlignment(I.second.Alignment);
1254 continue;
1255 }
1256 ArrayType *Ty =
1257 ArrayType::get(Type::getInt8Ty(RegularLTO.Ctx), I.second.Size);
1258 auto *GV = new GlobalVariable(*RegularLTO.CombinedModule, Ty, false,
1261 GV->setAlignment(I.second.Alignment);
1262 if (OldGV) {
1263 OldGV->replaceAllUsesWith(GV);
1264 GV->takeName(OldGV);
1265 OldGV->eraseFromParent();
1266 } else {
1267 GV->setName(I.first);
1268 }
1269 }
1270
1271 updateMemProfAttributes(*RegularLTO.CombinedModule, ThinLTO.CombinedIndex);
1272
1273 bool WholeProgramVisibilityEnabledInLTO =
1275 // If validation is enabled, upgrade visibility only when all vtables
1276 // have typeinfos.
1278
1279 // This returns true when the name is local or not defined. Locals are
1280 // expected to be handled separately.
1281 auto IsVisibleToRegularObj = [&](StringRef name) {
1282 auto It = GlobalResolutions->find(name);
1283 return (It == GlobalResolutions->end() || It->second.VisibleOutsideSummary);
1284 };
1285
1286 // If allowed, upgrade public vcall visibility metadata to linkage unit
1287 // visibility before whole program devirtualization in the optimizer.
1289 *RegularLTO.CombinedModule, WholeProgramVisibilityEnabledInLTO,
1290 DynamicExportSymbols, Conf.ValidateAllVtablesHaveTypeInfos,
1291 IsVisibleToRegularObj);
1292 updatePublicTypeTestCalls(*RegularLTO.CombinedModule,
1293 WholeProgramVisibilityEnabledInLTO);
1294
1295 if (Conf.PreOptModuleHook &&
1296 !Conf.PreOptModuleHook(0, *RegularLTO.CombinedModule))
1297 return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
1298
1299 if (!Conf.CodeGenOnly) {
1300 for (const auto &R : *GlobalResolutions) {
1301 GlobalValue *GV =
1302 RegularLTO.CombinedModule->getNamedValue(R.second.IRName);
1303 if (!R.second.isPrevailingIRSymbol())
1304 continue;
1305 if (R.second.Partition != 0 &&
1306 R.second.Partition != GlobalResolution::External)
1307 continue;
1308
1309 // Ignore symbols defined in other partitions.
1310 // Also skip declarations, which are not allowed to have internal linkage.
1311 if (!GV || GV->hasLocalLinkage() || GV->isDeclaration())
1312 continue;
1313
1314 // Symbols that are marked DLLImport or DLLExport should not be
1315 // internalized, as they are either externally visible or referencing
1316 // external symbols. Symbols that have AvailableExternally or Appending
1317 // linkage might be used by future passes and should be kept as is.
1318 // These linkages are seen in Unified regular LTO, because the process
1319 // of creating split LTO units introduces symbols with that linkage into
1320 // one of the created modules. Normally, only the ThinLTO backend would
1321 // compile this module, but Unified Regular LTO processes both
1322 // modules created by the splitting process as regular LTO modules.
1323 if ((LTOMode == LTOKind::LTOK_UnifiedRegular) &&
1326 continue;
1327
1328 GV->setUnnamedAddr(R.second.UnnamedAddr ? GlobalValue::UnnamedAddr::Global
1330 if (EnableLTOInternalization && R.second.Partition == 0)
1332 }
1333
1334 if (Conf.PostInternalizeModuleHook &&
1335 !Conf.PostInternalizeModuleHook(0, *RegularLTO.CombinedModule))
1336 return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
1337 }
1338
1339 if (!RegularLTO.EmptyCombinedModule || Conf.AlwaysEmitRegularLTOObj) {
1340 if (Error Err =
1341 backend(Conf, AddStream, RegularLTO.ParallelCodeGenParallelismLevel,
1342 *RegularLTO.CombinedModule, ThinLTO.CombinedIndex))
1343 return Err;
1344 }
1345
1346 return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
1347}
1348
1349static const char *libcallRoutineNames[] = {
1350#define HANDLE_LIBCALL(code, name) name,
1351#include "llvm/IR/RuntimeLibcalls.def"
1352#undef HANDLE_LIBCALL
1353};
1354
1357}
1358
1359/// This class defines the interface to the ThinLTO backend.
1361protected:
1362 const Config &Conf;
1367
1368public:
1370 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1371 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1372 lto::IndexWriteCallback OnWrite, bool ShouldEmitImportsFiles)
1373 : Conf(Conf), CombinedIndex(CombinedIndex),
1374 ModuleToDefinedGVSummaries(ModuleToDefinedGVSummaries),
1375 OnWrite(OnWrite), ShouldEmitImportsFiles(ShouldEmitImportsFiles) {}
1376
1377 virtual ~ThinBackendProc() = default;
1378 virtual Error start(
1379 unsigned Task, BitcodeModule BM,
1380 const FunctionImporter::ImportMapTy &ImportList,
1381 const FunctionImporter::ExportSetTy &ExportList,
1382 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1384 virtual Error wait() = 0;
1385 virtual unsigned getThreadCount() = 0;
1386
1387 // Write sharded indices and (optionally) imports to disk
1389 llvm::StringRef ModulePath,
1390 const std::string &NewModulePath) {
1391 std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
1392 std::error_code EC;
1393 gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries,
1394 ImportList, ModuleToSummariesForIndex);
1395
1396 raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC,
1397 sys::fs::OpenFlags::OF_None);
1398 if (EC)
1399 return errorCodeToError(EC);
1400 writeIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex);
1401
1402 if (ShouldEmitImportsFiles) {
1403 EC = EmitImportsFiles(ModulePath, NewModulePath + ".imports",
1404 ModuleToSummariesForIndex);
1405 if (EC)
1406 return errorCodeToError(EC);
1407 }
1408 return Error::success();
1409 }
1410};
1411
1412namespace {
1413class InProcessThinBackend : public ThinBackendProc {
1414 DefaultThreadPool BackendThreadPool;
1415 AddStreamFn AddStream;
1416 FileCache Cache;
1417 std::set<GlobalValue::GUID> CfiFunctionDefs;
1418 std::set<GlobalValue::GUID> CfiFunctionDecls;
1419
1420 std::optional<Error> Err;
1421 std::mutex ErrMu;
1422
1423 bool ShouldEmitIndexFiles;
1424
1425public:
1426 InProcessThinBackend(
1427 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1428 ThreadPoolStrategy ThinLTOParallelism,
1429 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1430 AddStreamFn AddStream, FileCache Cache, lto::IndexWriteCallback OnWrite,
1431 bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles)
1432 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries,
1433 OnWrite, ShouldEmitImportsFiles),
1434 BackendThreadPool(ThinLTOParallelism), AddStream(std::move(AddStream)),
1435 Cache(std::move(Cache)), ShouldEmitIndexFiles(ShouldEmitIndexFiles) {
1436 for (auto &Name : CombinedIndex.cfiFunctionDefs())
1437 CfiFunctionDefs.insert(
1439 for (auto &Name : CombinedIndex.cfiFunctionDecls())
1440 CfiFunctionDecls.insert(
1442 }
1443
1444 Error runThinLTOBackendThread(
1445 AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM,
1446 ModuleSummaryIndex &CombinedIndex,
1447 const FunctionImporter::ImportMapTy &ImportList,
1448 const FunctionImporter::ExportSetTy &ExportList,
1449 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1450 const GVSummaryMapTy &DefinedGlobals,
1452 auto RunThinBackend = [&](AddStreamFn AddStream) {
1453 LTOLLVMContext BackendContext(Conf);
1454 Expected<std::unique_ptr<Module>> MOrErr = BM.parseModule(BackendContext);
1455 if (!MOrErr)
1456 return MOrErr.takeError();
1457
1458 return thinBackend(Conf, Task, AddStream, **MOrErr, CombinedIndex,
1459 ImportList, DefinedGlobals, &ModuleMap);
1460 };
1461
1462 auto ModuleID = BM.getModuleIdentifier();
1463
1464 if (ShouldEmitIndexFiles) {
1465 if (auto E = emitFiles(ImportList, ModuleID, ModuleID.str()))
1466 return E;
1467 }
1468
1469 if (!Cache || !CombinedIndex.modulePaths().count(ModuleID) ||
1470 all_of(CombinedIndex.getModuleHash(ModuleID),
1471 [](uint32_t V) { return V == 0; }))
1472 // Cache disabled or no entry for this module in the combined index or
1473 // no module hash.
1474 return RunThinBackend(AddStream);
1475
1477 // The module may be cached, this helps handling it.
1478 computeLTOCacheKey(Key, Conf, CombinedIndex, ModuleID, ImportList,
1479 ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs,
1480 CfiFunctionDecls);
1481 Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, Key, ModuleID);
1482 if (Error Err = CacheAddStreamOrErr.takeError())
1483 return Err;
1484 AddStreamFn &CacheAddStream = *CacheAddStreamOrErr;
1485 if (CacheAddStream)
1486 return RunThinBackend(CacheAddStream);
1487
1488 return Error::success();
1489 }
1490
1491 Error start(
1492 unsigned Task, BitcodeModule BM,
1493 const FunctionImporter::ImportMapTy &ImportList,
1494 const FunctionImporter::ExportSetTy &ExportList,
1495 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1496 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1497 StringRef ModulePath = BM.getModuleIdentifier();
1498 assert(ModuleToDefinedGVSummaries.count(ModulePath));
1499 const GVSummaryMapTy &DefinedGlobals =
1500 ModuleToDefinedGVSummaries.find(ModulePath)->second;
1501 BackendThreadPool.async(
1502 [=](BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
1503 const FunctionImporter::ImportMapTy &ImportList,
1504 const FunctionImporter::ExportSetTy &ExportList,
1505 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>
1506 &ResolvedODR,
1507 const GVSummaryMapTy &DefinedGlobals,
1509 if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled)
1511 "thin backend");
1512 Error E = runThinLTOBackendThread(
1513 AddStream, Cache, Task, BM, CombinedIndex, ImportList, ExportList,
1514 ResolvedODR, DefinedGlobals, ModuleMap);
1515 if (E) {
1516 std::unique_lock<std::mutex> L(ErrMu);
1517 if (Err)
1518 Err = joinErrors(std::move(*Err), std::move(E));
1519 else
1520 Err = std::move(E);
1521 }
1522 if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled)
1524 },
1525 BM, std::ref(CombinedIndex), std::ref(ImportList), std::ref(ExportList),
1526 std::ref(ResolvedODR), std::ref(DefinedGlobals), std::ref(ModuleMap));
1527
1528 if (OnWrite)
1529 OnWrite(std::string(ModulePath));
1530 return Error::success();
1531 }
1532
1533 Error wait() override {
1534 BackendThreadPool.wait();
1535 if (Err)
1536 return std::move(*Err);
1537 else
1538 return Error::success();
1539 }
1540
1541 unsigned getThreadCount() override {
1542 return BackendThreadPool.getMaxConcurrency();
1543 }
1544};
1545} // end anonymous namespace
1546
1549 bool ShouldEmitIndexFiles,
1550 bool ShouldEmitImportsFiles) {
1551 return
1552 [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1553 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1554 AddStreamFn AddStream, FileCache Cache) {
1555 return std::make_unique<InProcessThinBackend>(
1556 Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries,
1557 AddStream, Cache, OnWrite, ShouldEmitIndexFiles,
1558 ShouldEmitImportsFiles);
1559 };
1560}
1561
1563 if (!TheTriple.isOSDarwin())
1564 return "";
1565 if (TheTriple.getArch() == Triple::x86_64)
1566 return "core2";
1567 if (TheTriple.getArch() == Triple::x86)
1568 return "yonah";
1569 if (TheTriple.isArm64e())
1570 return "apple-a12";
1571 if (TheTriple.getArch() == Triple::aarch64 ||
1572 TheTriple.getArch() == Triple::aarch64_32)
1573 return "cyclone";
1574 return "";
1575}
1576
1577// Given the original \p Path to an output file, replace any path
1578// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
1579// resulting directory if it does not yet exist.
1581 StringRef NewPrefix) {
1582 if (OldPrefix.empty() && NewPrefix.empty())
1583 return std::string(Path);
1584 SmallString<128> NewPath(Path);
1585 llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix);
1586 StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str());
1587 if (!ParentPath.empty()) {
1588 // Make sure the new directory exists, creating it if necessary.
1589 if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath))
1590 llvm::errs() << "warning: could not create directory '" << ParentPath
1591 << "': " << EC.message() << '\n';
1592 }
1593 return std::string(NewPath);
1594}
1595
1596namespace {
1597class WriteIndexesThinBackend : public ThinBackendProc {
1598 std::string OldPrefix, NewPrefix, NativeObjectPrefix;
1599 raw_fd_ostream *LinkedObjectsFile;
1600
1601public:
1602 WriteIndexesThinBackend(
1603 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1604 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1605 std::string OldPrefix, std::string NewPrefix,
1606 std::string NativeObjectPrefix, bool ShouldEmitImportsFiles,
1607 raw_fd_ostream *LinkedObjectsFile, lto::IndexWriteCallback OnWrite)
1608 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries,
1609 OnWrite, ShouldEmitImportsFiles),
1610 OldPrefix(OldPrefix), NewPrefix(NewPrefix),
1611 NativeObjectPrefix(NativeObjectPrefix),
1612 LinkedObjectsFile(LinkedObjectsFile) {}
1613
1614 Error start(
1615 unsigned Task, BitcodeModule BM,
1616 const FunctionImporter::ImportMapTy &ImportList,
1617 const FunctionImporter::ExportSetTy &ExportList,
1618 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1619 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1620 StringRef ModulePath = BM.getModuleIdentifier();
1621 std::string NewModulePath =
1622 getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix);
1623
1624 if (LinkedObjectsFile) {
1625 std::string ObjectPrefix =
1626 NativeObjectPrefix.empty() ? NewPrefix : NativeObjectPrefix;
1627 std::string LinkedObjectsFilePath =
1628 getThinLTOOutputFile(ModulePath, OldPrefix, ObjectPrefix);
1629 *LinkedObjectsFile << LinkedObjectsFilePath << '\n';
1630 }
1631
1632 if (auto E = emitFiles(ImportList, ModulePath, NewModulePath))
1633 return E;
1634
1635 if (OnWrite)
1636 OnWrite(std::string(ModulePath));
1637 return Error::success();
1638 }
1639
1640 Error wait() override { return Error::success(); }
1641
1642 // WriteIndexesThinBackend should always return 1 to prevent module
1643 // re-ordering and avoid non-determinism in the final link.
1644 unsigned getThreadCount() override { return 1; }
1645};
1646} // end anonymous namespace
1647
1649 std::string OldPrefix, std::string NewPrefix,
1650 std::string NativeObjectPrefix, bool ShouldEmitImportsFiles,
1651 raw_fd_ostream *LinkedObjectsFile, IndexWriteCallback OnWrite) {
1652 return
1653 [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1654 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1655 AddStreamFn AddStream, FileCache Cache) {
1656 return std::make_unique<WriteIndexesThinBackend>(
1657 Conf, CombinedIndex, ModuleToDefinedGVSummaries, OldPrefix,
1658 NewPrefix, NativeObjectPrefix, ShouldEmitImportsFiles,
1659 LinkedObjectsFile, OnWrite);
1660 };
1661}
1662
1663Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache,
1664 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
1665 LLVM_DEBUG(dbgs() << "Running ThinLTO\n");
1666 ThinLTO.CombinedIndex.releaseTemporaryMemory();
1667 timeTraceProfilerBegin("ThinLink", StringRef(""));
1668 auto TimeTraceScopeExit = llvm::make_scope_exit([]() {
1671 });
1672 if (ThinLTO.ModuleMap.empty())
1673 return Error::success();
1674
1675 if (ThinLTO.ModulesToCompile && ThinLTO.ModulesToCompile->empty()) {
1676 llvm::errs() << "warning: [ThinLTO] No module compiled\n";
1677 return Error::success();
1678 }
1679
1680 if (Conf.CombinedIndexHook &&
1681 !Conf.CombinedIndexHook(ThinLTO.CombinedIndex, GUIDPreservedSymbols))
1682 return Error::success();
1683
1684 // Collect for each module the list of function it defines (GUID ->
1685 // Summary).
1686 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(
1687 ThinLTO.ModuleMap.size());
1688 ThinLTO.CombinedIndex.collectDefinedGVSummariesPerModule(
1689 ModuleToDefinedGVSummaries);
1690 // Create entries for any modules that didn't have any GV summaries
1691 // (either they didn't have any GVs to start with, or we suppressed
1692 // generation of the summaries because they e.g. had inline assembly
1693 // uses that couldn't be promoted/renamed on export). This is so
1694 // InProcessThinBackend::start can still launch a backend thread, which
1695 // is passed the map of summaries for the module, without any special
1696 // handling for this case.
1697 for (auto &Mod : ThinLTO.ModuleMap)
1698 if (!ModuleToDefinedGVSummaries.count(Mod.first))
1699 ModuleToDefinedGVSummaries.try_emplace(Mod.first);
1700
1701 // Synthesize entry counts for functions in the CombinedIndex.
1702 computeSyntheticCounts(ThinLTO.CombinedIndex);
1703
1705 ThinLTO.ModuleMap.size());
1707 ThinLTO.ModuleMap.size());
1709
1710 if (DumpThinCGSCCs)
1711 ThinLTO.CombinedIndex.dumpSCCs(outs());
1712
1713 std::set<GlobalValue::GUID> ExportedGUIDs;
1714
1715 bool WholeProgramVisibilityEnabledInLTO =
1717 // If validation is enabled, upgrade visibility only when all vtables
1718 // have typeinfos.
1720 if (hasWholeProgramVisibility(WholeProgramVisibilityEnabledInLTO))
1721 ThinLTO.CombinedIndex.setWithWholeProgramVisibility();
1722
1723 // If we're validating, get the vtable symbols that should not be
1724 // upgraded because they correspond to typeIDs outside of index-based
1725 // WPD info.
1726 DenseSet<GlobalValue::GUID> VisibleToRegularObjSymbols;
1727 if (WholeProgramVisibilityEnabledInLTO &&
1729 // This returns true when the name is local or not defined. Locals are
1730 // expected to be handled separately.
1731 auto IsVisibleToRegularObj = [&](StringRef name) {
1732 auto It = GlobalResolutions->find(name);
1733 return (It == GlobalResolutions->end() ||
1734 It->second.VisibleOutsideSummary);
1735 };
1736
1737 getVisibleToRegularObjVtableGUIDs(ThinLTO.CombinedIndex,
1738 VisibleToRegularObjSymbols,
1739 IsVisibleToRegularObj);
1740 }
1741
1742 // If allowed, upgrade public vcall visibility to linkage unit visibility in
1743 // the summaries before whole program devirtualization below.
1745 ThinLTO.CombinedIndex, WholeProgramVisibilityEnabledInLTO,
1746 DynamicExportSymbols, VisibleToRegularObjSymbols);
1747
1748 // Perform index-based WPD. This will return immediately if there are
1749 // no index entries in the typeIdMetadata map (e.g. if we are instead
1750 // performing IR-based WPD in hybrid regular/thin LTO mode).
1751 std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
1752 runWholeProgramDevirtOnIndex(ThinLTO.CombinedIndex, ExportedGUIDs,
1753 LocalWPDTargetsMap);
1754
1755 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1756 return ThinLTO.PrevailingModuleForGUID[GUID] == S->modulePath();
1757 };
1759 MemProfContextDisambiguation ContextDisambiguation;
1760 ContextDisambiguation.run(ThinLTO.CombinedIndex, isPrevailing);
1761 }
1762
1763 // Figure out which symbols need to be internalized. This also needs to happen
1764 // at -O0 because summary-based DCE is implemented using internalization, and
1765 // we must apply DCE consistently with the full LTO module in order to avoid
1766 // undefined references during the final link.
1767 for (auto &Res : *GlobalResolutions) {
1768 // If the symbol does not have external references or it is not prevailing,
1769 // then not need to mark it as exported from a ThinLTO partition.
1770 if (Res.second.Partition != GlobalResolution::External ||
1771 !Res.second.isPrevailingIRSymbol())
1772 continue;
1773 auto GUID = GlobalValue::getGUID(
1774 GlobalValue::dropLLVMManglingEscape(Res.second.IRName));
1775 // Mark exported unless index-based analysis determined it to be dead.
1776 if (ThinLTO.CombinedIndex.isGUIDLive(GUID))
1777 ExportedGUIDs.insert(GUID);
1778 }
1779
1780 // Reset the GlobalResolutions to deallocate the associated memory, as there
1781 // are no further accesses. We specifically want to do this before computing
1782 // cross module importing, which adds to peak memory via the computed import
1783 // and export lists.
1784 GlobalResolutions.reset();
1785
1786 if (Conf.OptLevel > 0)
1787 ComputeCrossModuleImport(ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
1788 isPrevailing, ImportLists, ExportLists);
1789
1790 // Any functions referenced by the jump table in the regular LTO object must
1791 // be exported.
1792 for (auto &Def : ThinLTO.CombinedIndex.cfiFunctionDefs())
1793 ExportedGUIDs.insert(
1795 for (auto &Decl : ThinLTO.CombinedIndex.cfiFunctionDecls())
1796 ExportedGUIDs.insert(
1798
1799 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1800 const auto &ExportList = ExportLists.find(ModuleIdentifier);
1801 return (ExportList != ExportLists.end() && ExportList->second.count(VI)) ||
1802 ExportedGUIDs.count(VI.getGUID());
1803 };
1804
1805 // Update local devirtualized targets that were exported by cross-module
1806 // importing or by other devirtualizations marked in the ExportedGUIDs set.
1807 updateIndexWPDForExports(ThinLTO.CombinedIndex, isExported,
1808 LocalWPDTargetsMap);
1809
1810 thinLTOInternalizeAndPromoteInIndex(ThinLTO.CombinedIndex, isExported,
1811 isPrevailing);
1812
1813 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1814 GlobalValue::GUID GUID,
1815 GlobalValue::LinkageTypes NewLinkage) {
1816 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1817 };
1818 thinLTOResolvePrevailingInIndex(Conf, ThinLTO.CombinedIndex, isPrevailing,
1819 recordNewLinkage, GUIDPreservedSymbols);
1820
1821 thinLTOPropagateFunctionAttrs(ThinLTO.CombinedIndex, isPrevailing);
1822
1823 generateParamAccessSummary(ThinLTO.CombinedIndex);
1824
1827
1828 TimeTraceScopeExit.release();
1829
1830 std::unique_ptr<ThinBackendProc> BackendProc =
1831 ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
1832 AddStream, Cache);
1833
1834 auto &ModuleMap =
1835 ThinLTO.ModulesToCompile ? *ThinLTO.ModulesToCompile : ThinLTO.ModuleMap;
1836
1837 auto ProcessOneModule = [&](int I) -> Error {
1838 auto &Mod = *(ModuleMap.begin() + I);
1839 // Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for
1840 // combined module and parallel code generation partitions.
1841 return BackendProc->start(RegularLTO.ParallelCodeGenParallelismLevel + I,
1842 Mod.second, ImportLists[Mod.first],
1843 ExportLists[Mod.first], ResolvedODR[Mod.first],
1844 ThinLTO.ModuleMap);
1845 };
1846
1847 if (BackendProc->getThreadCount() == 1) {
1848 // Process the modules in the order they were provided on the command-line.
1849 // It is important for this codepath to be used for WriteIndexesThinBackend,
1850 // to ensure the emitted LinkedObjectsFile lists ThinLTO objects in the same
1851 // order as the inputs, which otherwise would affect the final link order.
1852 for (int I = 0, E = ModuleMap.size(); I != E; ++I)
1853 if (Error E = ProcessOneModule(I))
1854 return E;
1855 } else {
1856 // When executing in parallel, process largest bitsize modules first to
1857 // improve parallelism, and avoid starving the thread pool near the end.
1858 // This saves about 15 sec on a 36-core machine while link `clang.exe` (out
1859 // of 100 sec).
1860 std::vector<BitcodeModule *> ModulesVec;
1861 ModulesVec.reserve(ModuleMap.size());
1862 for (auto &Mod : ModuleMap)
1863 ModulesVec.push_back(&Mod.second);
1864 for (int I : generateModulesOrdering(ModulesVec))
1865 if (Error E = ProcessOneModule(I))
1866 return E;
1867 }
1868 return BackendProc->wait();
1869}
1870
1874 std::optional<uint64_t> RemarksHotnessThreshold, int Count) {
1875 std::string Filename = std::string(RemarksFilename);
1876 // For ThinLTO, file.opt.<format> becomes
1877 // file.opt.<format>.thin.<num>.<format>.
1878 if (!Filename.empty() && Count != -1)
1879 Filename =
1880 (Twine(Filename) + ".thin." + llvm::utostr(Count) + "." + RemarksFormat)
1881 .str();
1882
1883 auto ResultOrErr = llvm::setupLLVMOptimizationRemarks(
1886 if (Error E = ResultOrErr.takeError())
1887 return std::move(E);
1888
1889 if (*ResultOrErr)
1890 (*ResultOrErr)->keep();
1891
1892 return ResultOrErr;
1893}
1894
1897 // Setup output file to emit statistics.
1898 if (StatsFilename.empty())
1899 return nullptr;
1900
1902 std::error_code EC;
1903 auto StatsFile =
1904 std::make_unique<ToolOutputFile>(StatsFilename, EC, sys::fs::OF_None);
1905 if (EC)
1906 return errorCodeToError(EC);
1907
1908 StatsFile->keep();
1909 return std::move(StatsFile);
1910}
1911
1912// Compute the ordering we will process the inputs: the rough heuristic here
1913// is to sort them per size so that the largest module get schedule as soon as
1914// possible. This is purely a compile-time optimization.
1916 auto Seq = llvm::seq<int>(0, R.size());
1917 std::vector<int> ModulesOrdering(Seq.begin(), Seq.end());
1918 llvm::sort(ModulesOrdering, [&](int LeftIndex, int RightIndex) {
1919 auto LSize = R[LeftIndex]->getBuffer().size();
1920 auto RSize = R[RightIndex]->getBuffer().size();
1921 return LSize > RSize;
1922 });
1923 return ModulesOrdering;
1924}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DEBUG(X)
Definition: Debug.h:101
std::string Name
Symbol * Sym
Definition: ELF_riscv.cpp:479
#define DEBUG_TYPE
static void writeToResolutionFile(raw_ostream &OS, InputFile *Input, ArrayRef< SymbolResolution > Res)
Definition: LTO.cpp:671
static void thinLTOResolvePrevailingGUID(const Config &C, ValueInfo VI, DenseSet< GlobalValueSummary * > &GlobalInvolvedWithAlias, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, function_ref< void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)> recordNewLinkage, const DenseSet< GlobalValue::GUID > &GUIDPreservedSymbols)
Definition: LTO.cpp:349
static void handleNonPrevailingComdat(GlobalValue &GV, std::set< const Comdat * > &NonPrevailingComdats)
Definition: LTO.cpp:783
static cl::opt< bool > DumpThinCGSCCs("dump-thin-cg-sccs", cl::init(false), cl::Hidden, cl::desc("Dump the SCCs in the ThinLTO index's callgraph"))
static void thinLTOInternalizeAndPromoteGUID(ValueInfo VI, function_ref< bool(StringRef, ValueInfo)> isExported, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing)
Definition: LTO.cpp:448
static const char * libcallRoutineNames[]
Definition: LTO.cpp:1349
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
This file contains the declarations for metadata subclasses.
LLVMContext & Context
#define P(N)
llvm::cl::opt< bool > UseNewDbgInfoFormat
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static const char * name
Definition: SMEABIPass.cpp:49
raw_pwrite_stream & OS
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
This file contains some functions that are useful when dealing with strings.
This pass exposes codegen information to IR-level passes.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
iterator begin() const
Definition: ArrayRef.h:153
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:647
Represents a module in a bitcode file.
StringRef getModuleIdentifier() const
Error readSummary(ModuleSummaryIndex &CombinedIndex, StringRef ModulePath, std::function< bool(GlobalValue::GUID)> IsPrevailing=nullptr)
Parse the specified bitcode buffer and merge its module summary index into CombinedIndex.
Expected< std::unique_ptr< Module > > parseModule(LLVMContext &Context, ParserCallbacks Callbacks={})
Read the entire bitcode module and return it.
Expected< std::unique_ptr< Module > > getLazyModule(LLVMContext &Context, bool ShouldLazyLoadMetadata, bool IsImporting, ParserCallbacks Callbacks={})
Read the bitcode module and prepare for lazy deserialization of function bodies.
static ConstantAggregateZero * get(Type *Ty)
Definition: Constants.cpp:1663
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
Definition: DenseMap.h:235
unsigned size() const
Definition: DenseMap.h:99
iterator begin()
Definition: DenseMap.h:75
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
iterator end()
Definition: DenseMap.h:84
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:334
Tagged union holding either a T or a Error.
Definition: Error.h:474
Error takeError()
Take ownership of the stored error.
Definition: Error.h:601
std::unordered_set< GlobalValue::GUID > FunctionsToImportTy
Set of functions to import from a source module.
void setAlignment(Align Align)
Sets the alignment attribute of the GlobalObject.
Definition: Globals.cpp:128
Function and variable summary information to aid decisions and implementation of importing.
static bool isAppendingLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:400
static bool isExternalWeakLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:412
static bool isLocalLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:409
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:281
void setUnnamedAddr(UnnamedAddr Val)
Definition: GlobalValue.h:231
bool hasLocalLinkage() const
Definition: GlobalValue.h:528
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition: GlobalValue.h:567
const Comdat * getComdat() const
Definition: Globals.cpp:184
static bool isLinkOnceLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:388
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:537
DLLStorageClassTypes
Storage classes of global values for PE targets.
Definition: GlobalValue.h:73
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:595
const GlobalObject * getAliaseeObject() const
Definition: Globals.cpp:375
static bool isExternalLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:376
VisibilityTypes
An enumeration for the kinds of visibility of global values.
Definition: GlobalValue.h:66
@ DefaultVisibility
The GV is visible.
Definition: GlobalValue.h:67
static GUID getGUID(StringRef GlobalName)
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:591
static LinkageTypes getWeakLinkage(bool ODR)
Definition: GlobalValue.h:372
bool isWeakForLinker() const
Definition: GlobalValue.h:552
bool hasAppendingLinkage() const
Definition: GlobalValue.h:525
bool hasAvailableExternallyLinkage() const
Definition: GlobalValue.h:512
std::string getGlobalIdentifier() const
Return the modified name for this global value suitable to be used as the key for a global lookup (e....
Definition: Globals.cpp:169
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:51
@ CommonLinkage
Tentative definitions.
Definition: GlobalValue.h:62
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:59
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:52
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
Definition: GlobalValue.h:56
@ AvailableExternallyLinkage
Available for inspection, not emission.
Definition: GlobalValue.h:53
DLLStorageClassTypes getDLLStorageClass() const
Definition: GlobalValue.h:275
Type * getValueType() const
Definition: GlobalValue.h:296
static bool isLinkOnceODRLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:385
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition: Globals.cpp:462
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
This class implements a map that also provides access to all stored values in a deterministic order.
Definition: MapVector.h:36
iterator begin()
Definition: MapVector.h:69
size_type size() const
Definition: MapVector.h:60
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFile(const Twine &Filename, bool IsText=false, bool RequiresNullTerminator=true, bool IsVolatile=false, std::optional< Align > Alignment=std::nullopt)
Open the specified file as a MemoryBuffer, returning a new MemoryBuffer if successful,...
Class to hold module path string table and global value map, and encapsulate methods for operating on...
std::set< std::string > & cfiFunctionDecls()
const ModuleHash & getModuleHash(const StringRef ModPath) const
Get the module SHA1 hash recorded for the given module path.
const StringMap< ModuleHash > & modulePaths() const
Table of modules, containing module hash and id.
std::set< std::string > & cfiFunctionDefs()
static void CollectAsmSymvers(const Module &M, function_ref< void(StringRef, StringRef)> AsmSymver)
Parse inline ASM and collect the symvers directives that are defined in the current module.
uint32_t getSymbolFlags(Symbol S) const
ArrayRef< Symbol > symbols() const
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
A tuple of MDNodes.
Definition: Metadata.h:1729
The optimization diagnostic interface.
Diagnostic information for applied optimization remarks.
A discriminated union of two or more pointer types, with the discriminator in the low bit of the poin...
Definition: PointerUnion.h:118
A class that wrap the SHA1 algorithm.
Definition: SHA1.h:26
void update(ArrayRef< uint8_t > Data)
Digest more data.
Definition: SHA1.cpp:208
std::array< uint8_t, 20 > result()
Return the current raw 160-bits SHA1 for the digested data since the last call to init().
Definition: SHA1.cpp:288
A non-threaded implementation.
Definition: ThreadPool.h:218
void wait() override
Blocking wait for all the tasks to execute first.
Definition: ThreadPool.cpp:201
unsigned getMaxConcurrency() const override
Returns always 1: there is no concurrency.
Definition: ThreadPool.h:233
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:166
bool empty() const
Definition: SmallSet.h:159
bool erase(const T &V)
Definition: SmallSet.h:207
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:179
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition: SmallString.h:26
StringRef str() const
Explicit conversion to StringRef.
Definition: SmallString.h:254
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Definition: StringRef.h:849
StringMapEntry - This is used to represent one value that is inserted into a StringMap.
StringMap - This is an unconventional map that is specialized for handling keys that are "strings",...
Definition: StringMap.h:127
size_type count(StringRef Key) const
count - Return 1 if the element is in the map, 0 otherwise.
Definition: StringMap.h:276
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
bool contains(StringRef Other) const
Return true if the given string is a substring of *this, and false otherwise.
Definition: StringRef.h:420
MCTargetOptions MCOptions
Machine level options.
DebuggerKind DebuggerTuning
Which debugger to tune for.
unsigned FunctionSections
Emit functions into separate sections.
unsigned DataSections
Emit data into separate sections.
auto async(Function &&F, Args &&...ArgList)
Asynchronous submission of a task to the pool.
Definition: ThreadPool.h:78
This tells how a thread pool will be used.
Definition: Threading.h:116
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
bool isArm64e() const
Tests whether the target is the Apple "arm64e" AArch64 subarch.
Definition: Triple.h:1015
@ aarch64_32
Definition: Triple.h:53
ArchType getArch() const
Get the parsed architecture type of this triple.
Definition: Triple.h:361
bool isOSDarwin() const
Is this a "Darwin" OS (macOS, iOS, tvOS, watchOS, XROS, or DriverKit).
Definition: Triple.h:542
bool isOSBinFormatELF() const
Tests whether the OS uses the ELF binary format.
Definition: Triple.h:703
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
static IntegerType * getInt8Ty(LLVMContext &C)
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:377
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:534
bool use_empty() const
Definition: Value.h:344
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:383
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
iterator find(const_arg_type_t< ValueT > V)
Definition: DenseSet.h:179
size_type size() const
Definition: DenseSet.h:81
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:97
An efficient, type-erasing, non-owning reference to a callable.
Ephemeral symbols produced by Reader::symbols() and Reader::module_symbols().
Definition: IRSymtab.h:313
The purpose of this class is to only expose the symbol information that an LTO client should need in ...
Definition: LTO.h:139
An input file.
Definition: LTO.h:111
static Expected< std::unique_ptr< InputFile > > create(MemoryBufferRef Object)
Create an InputFile.
Definition: LTO.cpp:540
ArrayRef< Symbol > symbols() const
A range over the symbols in this InputFile.
Definition: LTO.h:164
StringRef getName() const
Returns the path to the InputFile.
Definition: LTO.cpp:569
BitcodeModule & getSingleBitcodeModule()
Definition: LTO.cpp:573
Error add(std::unique_ptr< InputFile > Obj, ArrayRef< SymbolResolution > Res)
Add an input file to the LTO link, using the provided symbol resolutions.
Definition: LTO.cpp:695
LTO(Config Conf, ThinBackend Backend=nullptr, unsigned ParallelCodeGenParallelismLevel=1, LTOKind LTOMode=LTOK_Default)
Create an LTO object.
Definition: LTO.cpp:593
LTOKind
Unified LTO modes.
Definition: LTO.h:261
@ LTOK_UnifiedRegular
Regular LTO, with Unified LTO enabled.
Definition: LTO.h:266
@ LTOK_Default
Any LTO mode without Unified LTO. The default mode.
Definition: LTO.h:263
@ LTOK_UnifiedThin
ThinLTO, with Unified LTO enabled.
Definition: LTO.h:269
static ArrayRef< const char * > getRuntimeLibcallSymbols()
Static method that returns a list of libcall symbols that can be generated by LTO but might not be vi...
Definition: LTO.cpp:1355
Error run(AddStreamFn AddStream, FileCache Cache=nullptr)
Runs the LTO pipeline.
Definition: LTO.cpp:1128
unsigned getMaxTasks() const
Returns an upper bound on the number of tasks that the client may expect.
Definition: LTO.cpp:1077
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:470
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
This class defines the interface to the ThinLTO backend.
Definition: LTO.cpp:1360
virtual unsigned getThreadCount()=0
const Config & Conf
Definition: LTO.cpp:1362
lto::IndexWriteCallback OnWrite
Definition: LTO.cpp:1365
Error emitFiles(const FunctionImporter::ImportMapTy &ImportList, llvm::StringRef ModulePath, const std::string &NewModulePath)
Definition: LTO.cpp:1388
virtual Error wait()=0
ThinBackendProc(const Config &Conf, ModuleSummaryIndex &CombinedIndex, const DenseMap< StringRef, GVSummaryMapTy > &ModuleToDefinedGVSummaries, lto::IndexWriteCallback OnWrite, bool ShouldEmitImportsFiles)
Definition: LTO.cpp:1369
const DenseMap< StringRef, GVSummaryMapTy > & ModuleToDefinedGVSummaries
Definition: LTO.cpp:1364
ModuleSummaryIndex & CombinedIndex
Definition: LTO.cpp:1363
virtual ~ThinBackendProc()=default
virtual Error start(unsigned Task, BitcodeModule BM, const FunctionImporter::ImportMapTy &ImportList, const FunctionImporter::ExportSetTy &ExportList, const std::map< GlobalValue::GUID, GlobalValue::LinkageTypes > &ResolvedODR, MapVector< StringRef, BitcodeModule > &ModuleMap)=0
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Key
PAL metadata keys.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Definition: Function.cpp:1031
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
ThinBackend createInProcessThinBackend(ThreadPoolStrategy Parallelism, IndexWriteCallback OnWrite=nullptr, bool ShouldEmitIndexFiles=false, bool ShouldEmitImportsFiles=false)
Definition: LTO.cpp:1547
Error thinBackend(const Config &C, unsigned Task, AddStreamFn AddStream, Module &M, const ModuleSummaryIndex &CombinedIndex, const FunctionImporter::ImportMapTy &ImportList, const GVSummaryMapTy &DefinedGlobals, MapVector< StringRef, BitcodeModule > *ModuleMap, const std::vector< uint8_t > &CmdArgs=std::vector< uint8_t >())
Runs a ThinLTO backend.
Definition: LTOBackend.cpp:553
std::string getThinLTOOutputFile(StringRef Path, StringRef OldPrefix, StringRef NewPrefix)
Given the original Path to an output file, replace any path prefix matching OldPrefix with NewPrefix.
Definition: LTO.cpp:1580
std::function< std::unique_ptr< ThinBackendProc >(const Config &C, ModuleSummaryIndex &CombinedIndex, DenseMap< StringRef, GVSummaryMapTy > &ModuleToDefinedGVSummaries, AddStreamFn AddStream, FileCache Cache)> ThinBackend
A ThinBackend defines what happens after the thin-link phase during ThinLTO.
Definition: LTO.h:202
std::function< void(const std::string &)> IndexWriteCallback
This ThinBackend runs the individual backend jobs in-process.
Definition: LTO.h:212
StringLiteral getThinLTODefaultCPU(const Triple &TheTriple)
Definition: LTO.cpp:1562
Expected< std::unique_ptr< ToolOutputFile > > setupStatsFile(StringRef StatsFilename)
Setups the output file for saving statistics.
Definition: LTO.cpp:1896
Error backend(const Config &C, AddStreamFn AddStream, unsigned ParallelCodeGenParallelismLevel, Module &M, ModuleSummaryIndex &CombinedIndex)
Runs a regular LTO backend.
Definition: LTOBackend.cpp:506
Error finalizeOptimizationRemarks(std::unique_ptr< ToolOutputFile > DiagOutputFile)
Definition: LTOBackend.cpp:495
ThinBackend createWriteIndexesThinBackend(std::string OldPrefix, std::string NewPrefix, std::string NativeObjectPrefix, bool ShouldEmitImportsFiles, raw_fd_ostream *LinkedObjectsFile, IndexWriteCallback OnWrite)
This ThinBackend writes individual module indexes to files, instead of running the individual backend...
Definition: LTO.cpp:1648
std::vector< int > generateModulesOrdering(ArrayRef< BitcodeModule * > R)
Produces a container ordering for optimal multi-threaded processing.
Definition: LTO.cpp:1915
Expected< std::unique_ptr< ToolOutputFile > > setupLLVMOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses, StringRef RemarksFormat, bool RemarksWithHotness, std::optional< uint64_t > RemarksHotnessThreshold=0, int Count=-1)
Setup optimization remarks.
Definition: LTO.cpp:1871
void updateMemProfAttributes(Module &Mod, const ModuleSummaryIndex &Index)
Updates MemProf attributes (and metadata) based on whether the index has recorded that we are linking...
Definition: LTO.cpp:1186
Expected< IRSymtabFile > readIRSymtab(MemoryBufferRef MBRef)
Reads a bitcode file, creating its irsymtab if necessary.
size_t getThreadCount()
Definition: Parallel.h:55
void write64le(void *P, uint64_t V)
Definition: Endian.h:470
void write32le(void *P, uint32_t V)
Definition: Endian.h:467
std::error_code create_directories(const Twine &path, bool IgnoreExisting=true, perms Perms=owner_all|group_all)
Create all the non-existent directories in path.
Definition: Path.cpp:968
StringRef parent_path(StringRef path, Style style=Style::native)
Get parent path.
Definition: Path.cpp:468
bool replace_path_prefix(SmallVectorImpl< char > &Path, StringRef OldPrefix, StringRef NewPrefix, Style style=Style::native)
Replace matching path prefix with another path.
Definition: Path.cpp:519
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
ThreadPoolStrategy heavyweight_hardware_concurrency(unsigned ThreadCount=0)
Returns a thread strategy for tasks requiring significant memory or other resources.
Definition: Threading.h:162
cl::opt< std::string > RemarksFormat("lto-pass-remarks-format", cl::desc("The format used for serializing remarks (default: YAML)"), cl::value_desc("format"), cl::init("yaml"))
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1722
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1680
void ComputeCrossModuleImport(const ModuleSummaryIndex &Index, const DenseMap< StringRef, GVSummaryMapTy > &ModuleToDefinedGVSummaries, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, DenseMap< StringRef, FunctionImporter::ImportMapTy > &ImportLists, DenseMap< StringRef, FunctionImporter::ExportSetTy > &ExportLists)
Compute all the imports and exports for every module in the Index.
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:59
void timeTraceProfilerInitialize(unsigned TimeTraceGranularity, StringRef ProcName)
Initialize the time trace profiler.
void generateParamAccessSummary(ModuleSummaryIndex &Index)
cl::opt< std::string > RemarksPasses("lto-pass-remarks-filter", cl::desc("Only record optimization remarks from passes whose " "names match the given regular expression"), cl::value_desc("regex"))
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:90
void writeIndexToFile(const ModuleSummaryIndex &Index, raw_ostream &Out, const std::map< std::string, GVSummaryMapTy > *ModuleToSummariesForIndex=nullptr)
Write the specified module summary index to the given raw output stream, where it will be written in ...
raw_fd_ostream & outs()
This returns a reference to a raw_fd_ostream for standard output.
std::function< Expected< std::unique_ptr< CachedFileStream > >(unsigned Task, const Twine &ModuleName)> AddStreamFn
This type defines the callback to add a file that is generated on the fly.
Definition: Caching.h:42
std::error_code EmitImportsFiles(StringRef ModulePath, StringRef OutputFilename, const std::map< std::string, GVSummaryMapTy > &ModuleToSummariesForIndex)
Emit into OutputFilename the files module ModulePath will import from.
bool thinLTOPropagateFunctionAttrs(ModuleSummaryIndex &Index, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing)
Propagate function attributes for function summaries along the index's callgraph during thinlink.
bool hasWholeProgramVisibility(bool WholeProgramVisibilityEnabledInLTO)
void EnableStatistics(bool DoPrintOnExit=true)
Enable the collection and printing of statistics.
Definition: Statistic.cpp:134
void thinLTOInternalizeAndPromoteInIndex(ModuleSummaryIndex &Index, function_ref< bool(StringRef, ValueInfo)> isExported, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing)
Update the linkages in the given Index to mark exported values as external and non-exported values as...
Definition: LTO.cpp:527
void computeLTOCacheKey(SmallString< 40 > &Key, const lto::Config &Conf, const ModuleSummaryIndex &Index, StringRef ModuleID, const FunctionImporter::ImportMapTy &ImportList, const FunctionImporter::ExportSetTy &ExportList, const std::map< GlobalValue::GUID, GlobalValue::LinkageTypes > &ResolvedODR, const GVSummaryMapTy &DefinedGlobals, const std::set< GlobalValue::GUID > &CfiFunctionDefs={}, const std::set< GlobalValue::GUID > &CfiFunctionDecls={})
Computes a unique hash for the Module considering the current list of export/import and other global ...
Definition: LTO.cpp:89
void timeTraceProfilerFinishThread()
Finish a time trace profiler running on a worker thread.
Error joinErrors(Error E1, Error E2)
Concatenate errors.
Definition: Error.h:431
void updatePublicTypeTestCalls(Module &M, bool WholeProgramVisibilityEnabledInLTO)
void getVisibleToRegularObjVtableGUIDs(ModuleSummaryIndex &Index, DenseSet< GlobalValue::GUID > &VisibleToRegularObjSymbols, function_ref< bool(StringRef)> IsVisibleToRegularObj)
Based on typeID string, get all associated vtable GUIDS that are visible to regular objects.
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1647
bool timeTraceProfilerEnabled()
Is the time trace profiler enabled, i.e. initialized?
Definition: TimeProfiler.h:104
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
Expected< std::unique_ptr< ToolOutputFile > > setupLLVMOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses, StringRef RemarksFormat, bool RemarksWithHotness, std::optional< uint64_t > RemarksHotnessThreshold=0)
Setup optimization remarks that output to a file.
cl::opt< bool > EnableLTOInternalization
Enable global value internalization in LTO.
cl::opt< bool > RemarksWithHotness("lto-pass-remarks-with-hotness", cl::desc("With PGO, include profile count in optimization remarks"), cl::Hidden)
void computeSyntheticCounts(ModuleSummaryIndex &Index)
Compute synthetic function entry counts.
void timeTraceProfilerEnd()
Manually end the last time section.
cl::opt< std::string > RemarksFilename("lto-pass-remarks-output", cl::desc("Output filename for pass remarks"), cl::value_desc("filename"))
cl::opt< bool > SupportsHotColdNew
Indicate we are linking with an allocator that supports hot/cold operator new interfaces.
void updateIndexWPDForExports(ModuleSummaryIndex &Summary, function_ref< bool(StringRef, ValueInfo)> isExported, std::map< ValueInfo, std::vector< VTableSlotSummary > > &LocalWPDTargetsMap)
Call after cross-module importing to update the recorded single impl devirt target names for any loca...
void thinLTOResolvePrevailingInIndex(const lto::Config &C, ModuleSummaryIndex &Index, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, function_ref< void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)> recordNewLinkage, const DenseSet< GlobalValue::GUID > &GUIDPreservedSymbols)
Resolve linkage for prevailing symbols in the Index.
Definition: LTO.cpp:426
void gatherImportedSummariesForModule(StringRef ModulePath, const DenseMap< StringRef, GVSummaryMapTy > &ModuleToDefinedGVSummaries, const FunctionImporter::ImportMapTy &ImportList, std::map< std::string, GVSummaryMapTy > &ModuleToSummariesForIndex)
Compute the set of summaries needed for a ThinLTO backend compilation of ModulePath.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Mod
The access may modify the value stored in memory.
cl::opt< bool > EnableMemProfContextDisambiguation
Enable MemProf context disambiguation for thin link.
void runWholeProgramDevirtOnIndex(ModuleSummaryIndex &Summary, std::set< GlobalValue::GUID > &ExportedGUIDs, std::map< ValueInfo, std::vector< VTableSlotSummary > > &LocalWPDTargetsMap)
Perform index-based whole program devirtualization on the Summary index.
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1849
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition: STLExtras.h:1921
cl::opt< std::optional< uint64_t >, false, remarks::HotnessThresholdParser > RemarksHotnessThreshold("lto-pass-remarks-hotness-threshold", cl::desc("Minimum profile count required for an " "optimization remark to be output." " Use 'auto' to apply the threshold from profile summary."), cl::value_desc("uint or 'auto'"), cl::init(0), cl::Hidden)
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
Definition: Error.cpp:103
bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
std::array< uint32_t, 5 > ModuleHash
160 bits SHA1
std::function< Expected< AddStreamFn >(unsigned Task, StringRef Key, const Twine &ModuleName)> FileCache
This is the type of a file cache.
Definition: Caching.h:58
void PrintStatisticsJSON(raw_ostream &OS)
Print statistics in JSON format.
Definition: Statistic.cpp:203
void computeDeadSymbolsWithConstProp(ModuleSummaryIndex &Index, const DenseSet< GlobalValue::GUID > &GUIDPreservedSymbols, function_ref< PrevailingType(GlobalValue::GUID)> isPrevailing, bool ImportEnabled)
Compute dead symbols and run constant propagation in combined index after that.
@ Keep
No function return thunk.
void updateVCallVisibilityInModule(Module &M, bool WholeProgramVisibilityEnabledInLTO, const DenseSet< GlobalValue::GUID > &DynamicExportSymbols, bool ValidateAllVtablesHaveTypeInfos, function_ref< bool(StringRef)> IsVisibleToRegularObj)
If whole program visibility asserted, then upgrade all public vcall visibility metadata on vtable def...
TimeTraceProfilerEntry * timeTraceProfilerBegin(StringRef Name, StringRef Detail)
Manually begin a time section, with the given Name and Detail.
void updateVCallVisibilityInIndex(ModuleSummaryIndex &Index, bool WholeProgramVisibilityEnabledInLTO, const DenseSet< GlobalValue::GUID > &DynamicExportSymbols, const DenseSet< GlobalValue::GUID > &VisibleToRegularObjSymbols)
If whole program visibility asserted, then upgrade all public vcall visibility metadata on vtable def...
Implement std::hash so that hash_code can be used in STL containers.
Definition: BitVector.h:858
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Used in the streaming interface as the general argument type.
Struct that holds a reference to a particular GUID in a global value summary.
LTO configuration.
Definition: Config.h:41
bool HasWholeProgramVisibility
Asserts whether we can assume whole program visibility during the LTO link.
Definition: Config.h:78
bool ValidateAllVtablesHaveTypeInfos
We're validating that all native vtables have corresponding type infos.
Definition: Config.h:81
std::optional< uint64_t > RemarksHotnessThreshold
The minimum hotness value a diagnostic needs in order to be included in optimization diagnostics.
Definition: Config.h:160
std::string StatsFile
Statistics output file path.
Definition: Config.h:169
ModuleHookFn PreOptModuleHook
This module hook is called after linking (regular LTO) or loading (ThinLTO) the module,...
Definition: Config.h:223
CombinedIndexHookFn CombinedIndexHook
Definition: Config.h:255
std::optional< CodeModel::Model > CodeModel
Definition: Config.h:56
std::string AAPipeline
Definition: Config.h:105
bool CodeGenOnly
Disable entirely the optimizer, including importing for ThinLTO.
Definition: Config.h:68
std::vector< std::string > MAttrs
Definition: Config.h:50
std::vector< std::string > MllvmArgs
Definition: Config.h:51
std::vector< std::string > ThinLTOModulesToCompile
Specific thinLTO modules to compile.
Definition: Config.h:172
CodeGenOptLevel CGOptLevel
Definition: Config.h:57
std::unique_ptr< raw_ostream > ResolutionFile
If this field is set, LTO will write input file paths and symbol resolutions here in llvm-lto2 comman...
Definition: Config.h:190
std::string DefaultTriple
Setting this field will replace unspecified target triples in input files with this triple.
Definition: Config.h:113
bool AlwaysEmitRegularLTOObj
Always emit a Regular LTO object even when it is empty because no Regular LTO modules were linked.
Definition: Config.h:89
std::string CPU
Definition: Config.h:48
std::string DwoDir
The directory to store .dwo files.
Definition: Config.h:125
std::string RemarksFilename
Optimization remarks file path.
Definition: Config.h:139
VisScheme VisibilityScheme
Allows non-imported definitions to get the potentially more constraining visibility from the prevaili...
Definition: Config.h:95
std::string OverrideTriple
Setting this field will replace target triples in input files with this triple.
Definition: Config.h:109
std::string ProfileRemapping
Name remapping file for profile data.
Definition: Config.h:122
bool AllVtablesHaveTypeInfos
If all native vtables have corresponding type infos, allow usage of RTTI to block devirtualization on...
Definition: Config.h:84
TargetOptions Options
Definition: Config.h:49
bool TimeTraceEnabled
Time trace enabled.
Definition: Config.h:175
std::string RemarksPasses
Optimization remarks pass filter.
Definition: Config.h:142
std::string OptPipeline
If this field is set, the set of passes run in the middle-end optimizer will be the one specified by ...
Definition: Config.h:100
ModuleHookFn PostInternalizeModuleHook
This hook is called after internalizing the module.
Definition: Config.h:230
unsigned TimeTraceGranularity
Time trace granularity.
Definition: Config.h:178
unsigned OptLevel
Definition: Config.h:59
bool RemarksWithHotness
Whether to emit optimization remarks with hotness informations.
Definition: Config.h:145
std::optional< Reloc::Model > RelocModel
Definition: Config.h:55
CodeGenFileType CGFileType
Definition: Config.h:58
bool Freestanding
Flag to indicate that the optimizer should not assume builtins are present on the target.
Definition: Config.h:65
std::string SampleProfile
Sample PGO profile path.
Definition: Config.h:119
std::string RemarksFormat
The format used for serializing remarks (default: YAML).
Definition: Config.h:163
A derived class of LLVMContext that initializes itself according to a given Config object.
Definition: Config.h:294
The resolution for a symbol.
Definition: LTO.h:457
unsigned FinalDefinitionInLinkageUnit
The definition of this symbol is unpreemptable at runtime and is known to be in this linkage unit.
Definition: LTO.h:467
unsigned ExportDynamic
The symbol was exported dynamically, and therefore could be referenced by a shared library not visibl...
Definition: LTO.h:474
unsigned Prevailing
The linker has chosen this definition of the symbol.
Definition: LTO.h:463
unsigned LinkerRedefined
Linker redefined version of the symbol which appeared in -wrap or -defsym linker option.
Definition: LTO.h:478
unsigned VisibleToRegularObj
The definition of this symbol is visible outside of the LTO unit.
Definition: LTO.h:470