LLVM 20.0.0git
ModuleSummaryIndex.h
Go to the documentation of this file.
1//===- llvm/ModuleSummaryIndex.h - Module Summary Index ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// @file
10/// ModuleSummaryIndex.h This file contains the declarations the classes that
11/// hold the module index and summary for function importing.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_IR_MODULESUMMARYINDEX_H
16#define LLVM_IR_MODULESUMMARYINDEX_H
17
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/DenseMap.h"
20#include "llvm/ADT/STLExtras.h"
24#include "llvm/ADT/StringMap.h"
25#include "llvm/ADT/StringRef.h"
27#include "llvm/IR/GlobalValue.h"
28#include "llvm/IR/Module.h"
34#include <algorithm>
35#include <array>
36#include <cassert>
37#include <cstddef>
38#include <cstdint>
39#include <map>
40#include <memory>
41#include <optional>
42#include <set>
43#include <string>
44#include <unordered_set>
45#include <utility>
46#include <vector>
47
48namespace llvm {
49
50template <class GraphType> struct GraphTraits;
51
52namespace yaml {
53
54template <typename T> struct MappingTraits;
55
56} // end namespace yaml
57
58/// Class to accumulate and hold information about a callee.
59struct CalleeInfo {
60 enum class HotnessType : uint8_t {
61 Unknown = 0,
62 Cold = 1,
63 None = 2,
64 Hot = 3,
65 Critical = 4
66 };
67
68 // The size of the bit-field might need to be adjusted if more values are
69 // added to HotnessType enum.
71
72 // True if at least one of the calls to the callee is a tail call.
73 bool HasTailCall : 1;
74
75 /// The value stored in RelBlockFreq has to be interpreted as the digits of
76 /// a scaled number with a scale of \p -ScaleShift.
77 static constexpr unsigned RelBlockFreqBits = 28;
79 static constexpr int32_t ScaleShift = 8;
80 static constexpr uint64_t MaxRelBlockFreq = (1 << RelBlockFreqBits) - 1;
81
83 : Hotness(static_cast<uint32_t>(HotnessType::Unknown)),
85 explicit CalleeInfo(HotnessType Hotness, bool HasTC, uint64_t RelBF)
86 : Hotness(static_cast<uint32_t>(Hotness)), HasTailCall(HasTC),
87 RelBlockFreq(RelBF) {}
88
89 void updateHotness(const HotnessType OtherHotness) {
90 Hotness = std::max(Hotness, static_cast<uint32_t>(OtherHotness));
91 }
92
93 bool hasTailCall() const { return HasTailCall; }
94
95 void setHasTailCall(const bool HasTC) { HasTailCall = HasTC; }
96
98
99 /// Update \p RelBlockFreq from \p BlockFreq and \p EntryFreq
100 ///
101 /// BlockFreq is divided by EntryFreq and added to RelBlockFreq. To represent
102 /// fractional values, the result is represented as a fixed point number with
103 /// scale of -ScaleShift.
104 void updateRelBlockFreq(uint64_t BlockFreq, uint64_t EntryFreq) {
105 if (EntryFreq == 0)
106 return;
108 Scaled64 Temp(BlockFreq, ScaleShift);
109 Temp /= Scaled64::get(EntryFreq);
110
111 uint64_t Sum =
112 SaturatingAdd<uint64_t>(Temp.toInt<uint64_t>(), RelBlockFreq);
113 Sum = std::min(Sum, uint64_t(MaxRelBlockFreq));
114 RelBlockFreq = static_cast<uint32_t>(Sum);
115 }
116};
117
119 switch (HT) {
121 return "unknown";
123 return "cold";
125 return "none";
127 return "hot";
129 return "critical";
130 }
131 llvm_unreachable("invalid hotness");
132}
133
134class GlobalValueSummary;
135
136using GlobalValueSummaryList = std::vector<std::unique_ptr<GlobalValueSummary>>;
137
138struct alignas(8) GlobalValueSummaryInfo {
139 union NameOrGV {
140 NameOrGV(bool HaveGVs) {
141 if (HaveGVs)
142 GV = nullptr;
143 else
144 Name = "";
145 }
146
147 /// The GlobalValue corresponding to this summary. This is only used in
148 /// per-module summaries and when the IR is available. E.g. when module
149 /// analysis is being run, or when parsing both the IR and the summary
150 /// from assembly.
152
153 /// Summary string representation. This StringRef points to BC module
154 /// string table and is valid until module data is stored in memory.
155 /// This is guaranteed to happen until runThinLTOBackend function is
156 /// called, so it is safe to use this field during thin link. This field
157 /// is only valid if summary index was loaded from BC file.
159 } U;
160
161 inline GlobalValueSummaryInfo(bool HaveGVs);
162
163 /// List of global value summary structures for a particular value held
164 /// in the GlobalValueMap. Requires a vector in the case of multiple
165 /// COMDAT values of the same name.
167};
168
169/// Map from global value GUID to corresponding summary structures. Use a
170/// std::map rather than a DenseMap so that pointers to the map's value_type
171/// (which are used by ValueInfo) are not invalidated by insertion. Also it will
172/// likely incur less overhead, as the value type is not very small and the size
173/// of the map is unknown, resulting in inefficiencies due to repeated
174/// insertions and resizing.
176 std::map<GlobalValue::GUID, GlobalValueSummaryInfo>;
177
178/// Struct that holds a reference to a particular GUID in a global value
179/// summary.
180struct ValueInfo {
181 enum Flags { HaveGV = 1, ReadOnly = 2, WriteOnly = 4 };
184
185 ValueInfo() = default;
186 ValueInfo(bool HaveGVs, const GlobalValueSummaryMapTy::value_type *R) {
188 RefAndFlags.setInt(HaveGVs);
189 }
190
191 explicit operator bool() const { return getRef(); }
192
193 GlobalValue::GUID getGUID() const { return getRef()->first; }
194 const GlobalValue *getValue() const {
195 assert(haveGVs());
196 return getRef()->second.U.GV;
197 }
198
200 return getRef()->second.SummaryList;
201 }
202
203 StringRef name() const {
204 return haveGVs() ? getRef()->second.U.GV->getName()
205 : getRef()->second.U.Name;
206 }
207
208 bool haveGVs() const { return RefAndFlags.getInt() & HaveGV; }
209 bool isReadOnly() const {
211 return RefAndFlags.getInt() & ReadOnly;
212 }
213 bool isWriteOnly() const {
215 return RefAndFlags.getInt() & WriteOnly;
216 }
217 unsigned getAccessSpecifier() const {
219 return RefAndFlags.getInt() & (ReadOnly | WriteOnly);
220 }
222 unsigned BadAccessMask = ReadOnly | WriteOnly;
223 return (RefAndFlags.getInt() & BadAccessMask) != BadAccessMask;
224 }
225 void setReadOnly() {
226 // We expect ro/wo attribute to set only once during
227 // ValueInfo lifetime.
230 }
234 }
235
236 const GlobalValueSummaryMapTy::value_type *getRef() const {
237 return RefAndFlags.getPointer();
238 }
239
240 /// Returns the most constraining visibility among summaries. The
241 /// visibilities, ordered from least to most constraining, are: default,
242 /// protected and hidden.
244
245 /// Checks if all summaries are DSO local (have the flag set). When DSOLocal
246 /// propagation has been done, set the parameter to enable fast check.
247 bool isDSOLocal(bool WithDSOLocalPropagation = false) const;
248
249 /// Checks if all copies are eligible for auto-hiding (have flag set).
250 bool canAutoHide() const;
251};
252
254 OS << VI.getGUID();
255 if (!VI.name().empty())
256 OS << " (" << VI.name() << ")";
257 return OS;
258}
259
260inline bool operator==(const ValueInfo &A, const ValueInfo &B) {
261 assert(A.getRef() && B.getRef() &&
262 "Need ValueInfo with non-null Ref for comparison");
263 return A.getRef() == B.getRef();
264}
265
266inline bool operator!=(const ValueInfo &A, const ValueInfo &B) {
267 assert(A.getRef() && B.getRef() &&
268 "Need ValueInfo with non-null Ref for comparison");
269 return A.getRef() != B.getRef();
270}
271
272inline bool operator<(const ValueInfo &A, const ValueInfo &B) {
273 assert(A.getRef() && B.getRef() &&
274 "Need ValueInfo with non-null Ref to compare GUIDs");
275 return A.getGUID() < B.getGUID();
276}
277
278template <> struct DenseMapInfo<ValueInfo> {
279 static inline ValueInfo getEmptyKey() {
280 return ValueInfo(false, (GlobalValueSummaryMapTy::value_type *)-8);
281 }
282
283 static inline ValueInfo getTombstoneKey() {
284 return ValueInfo(false, (GlobalValueSummaryMapTy::value_type *)-16);
285 }
286
287 static inline bool isSpecialKey(ValueInfo V) {
288 return V == getTombstoneKey() || V == getEmptyKey();
289 }
290
291 static bool isEqual(ValueInfo L, ValueInfo R) {
292 // We are not supposed to mix ValueInfo(s) with different HaveGVs flag
293 // in a same container.
294 assert(isSpecialKey(L) || isSpecialKey(R) || (L.haveGVs() == R.haveGVs()));
295 return L.getRef() == R.getRef();
296 }
297 static unsigned getHashValue(ValueInfo I) { return (uintptr_t)I.getRef(); }
298};
299
300/// Summary of memprof callsite metadata.
302 // Actual callee function.
304
305 // Used to record whole program analysis cloning decisions.
306 // The ThinLTO backend will need to create as many clones as there are entries
307 // in the vector (it is expected and should be confirmed that all such
308 // summaries in the same FunctionSummary have the same number of entries).
309 // Each index records version info for the corresponding clone of this
310 // function. The value is the callee clone it calls (becomes the appended
311 // suffix id). Index 0 is the original version, and a value of 0 calls the
312 // original callee.
314
315 // Represents stack ids in this context, recorded as indices into the
316 // StackIds vector in the summary index, which in turn holds the full 64-bit
317 // stack ids. This reduces memory as there are in practice far fewer unique
318 // stack ids than stack id references.
320
327};
328
330 OS << "Callee: " << SNI.Callee;
331 bool First = true;
332 OS << " Clones: ";
333 for (auto V : SNI.Clones) {
334 if (!First)
335 OS << ", ";
336 First = false;
337 OS << V;
338 }
339 First = true;
340 OS << " StackIds: ";
341 for (auto Id : SNI.StackIdIndices) {
342 if (!First)
343 OS << ", ";
344 First = false;
345 OS << Id;
346 }
347 return OS;
348}
349
350// Allocation type assigned to an allocation reached by a given context.
351// More can be added, now this is cold, notcold and hot.
352// Values should be powers of two so that they can be ORed, in particular to
353// track allocations that have different behavior with different calling
354// contexts.
355enum class AllocationType : uint8_t {
356 None = 0,
357 NotCold = 1,
358 Cold = 2,
359 Hot = 4,
360 All = 7 // This should always be set to the OR of all values.
361};
362
363/// Summary of a single MIB in a memprof metadata on allocations.
364struct MIBInfo {
365 // The allocation type for this profiled context.
367
368 // Represents stack ids in this context, recorded as indices into the
369 // StackIds vector in the summary index, which in turn holds the full 64-bit
370 // stack ids. This reduces memory as there are in practice far fewer unique
371 // stack ids than stack id references.
373
376};
377
379 OS << "AllocType " << (unsigned)MIB.AllocType;
380 bool First = true;
381 OS << " StackIds: ";
382 for (auto Id : MIB.StackIdIndices) {
383 if (!First)
384 OS << ", ";
385 First = false;
386 OS << Id;
387 }
388 return OS;
389}
390
391/// Summary of memprof metadata on allocations.
392struct AllocInfo {
393 // Used to record whole program analysis cloning decisions.
394 // The ThinLTO backend will need to create as many clones as there are entries
395 // in the vector (it is expected and should be confirmed that all such
396 // summaries in the same FunctionSummary have the same number of entries).
397 // Each index records version info for the corresponding clone of this
398 // function. The value is the allocation type of the corresponding allocation.
399 // Index 0 is the original version. Before cloning, index 0 may have more than
400 // one allocation type.
402
403 // Vector of MIBs in this memprof metadata.
404 std::vector<MIBInfo> MIBs;
405
406 // If requested, keep track of total profiled sizes for each MIB. This will be
407 // a vector of the same length and order as the MIBs vector, if non-empty.
408 std::vector<uint64_t> TotalSizes;
409
410 AllocInfo(std::vector<MIBInfo> MIBs) : MIBs(std::move(MIBs)) {
412 }
415};
416
418 bool First = true;
419 OS << "Versions: ";
420 for (auto V : AE.Versions) {
421 if (!First)
422 OS << ", ";
423 First = false;
424 OS << (unsigned)V;
425 }
426 OS << " MIB:\n";
427 for (auto &M : AE.MIBs) {
428 OS << "\t\t" << M << "\n";
429 }
430 if (!AE.TotalSizes.empty()) {
431 OS << " TotalSizes per MIB:\n\t\t";
432 First = true;
433 for (uint64_t TS : AE.TotalSizes) {
434 if (!First)
435 OS << ", ";
436 First = false;
437 OS << TS << "\n";
438 }
439 }
440 return OS;
441}
442
443/// Function and variable summary information to aid decisions and
444/// implementation of importing.
446public:
447 /// Sububclass discriminator (for dyn_cast<> et al.)
449
450 enum ImportKind : unsigned {
451 // The global value definition corresponding to the summary should be
452 // imported from source module
454
455 // When its definition doesn't exist in the destination module and not
456 // imported (e.g., function is too large to be inlined), the global value
457 // declaration corresponding to the summary should be imported, or the
458 // attributes from summary should be annotated on the function declaration.
460 };
461
462 /// Group flags (Linkage, NotEligibleToImport, etc.) as a bitfield.
463 struct GVFlags {
464 /// The linkage type of the associated global value.
465 ///
466 /// One use is to flag values that have local linkage types and need to
467 /// have module identifier appended before placing into the combined
468 /// index, to disambiguate from other values with the same name.
469 /// In the future this will be used to update and optimize linkage
470 /// types based on global summary-based analysis.
471 unsigned Linkage : 4;
472
473 /// Indicates the visibility.
474 unsigned Visibility : 2;
475
476 /// Indicate if the global value cannot be imported (e.g. it cannot
477 /// be renamed or references something that can't be renamed).
479
480 /// In per-module summary, indicate that the global value must be considered
481 /// a live root for index-based liveness analysis. Used for special LLVM
482 /// values such as llvm.global_ctors that the linker does not know about.
483 ///
484 /// In combined summary, indicate that the global value is live.
485 unsigned Live : 1;
486
487 /// Indicates that the linker resolved the symbol to a definition from
488 /// within the same linkage unit.
489 unsigned DSOLocal : 1;
490
491 /// In the per-module summary, indicates that the global value is
492 /// linkonce_odr and global unnamed addr (so eligible for auto-hiding
493 /// via hidden visibility). In the combined summary, indicates that the
494 /// prevailing linkonce_odr copy can be auto-hidden via hidden visibility
495 /// when it is upgraded to weak_odr in the backend. This is legal when
496 /// all copies are eligible for auto-hiding (i.e. all copies were
497 /// linkonce_odr global unnamed addr. If any copy is not (e.g. it was
498 /// originally weak_odr, we cannot auto-hide the prevailing copy as it
499 /// means the symbol was externally visible.
500 unsigned CanAutoHide : 1;
501
502 /// This field is written by the ThinLTO indexing step to postlink combined
503 /// summary. The value is interpreted as 'ImportKind' enum defined above.
504 unsigned ImportType : 1;
505
506 /// Convenience Constructors
509 bool NotEligibleToImport, bool Live, bool IsLocal,
514 ImportType(static_cast<unsigned>(ImportType)) {}
515 };
516
517private:
518 /// Kind of summary for use in dyn_cast<> et al.
519 SummaryKind Kind;
520
521 GVFlags Flags;
522
523 /// This is the hash of the name of the symbol in the original file. It is
524 /// identical to the GUID for global symbols, but differs for local since the
525 /// GUID includes the module level id in the hash.
526 GlobalValue::GUID OriginalName = 0;
527
528 /// Path of module IR containing value's definition, used to locate
529 /// module during importing.
530 ///
531 /// This is only used during parsing of the combined index, or when
532 /// parsing the per-module index for creation of the combined summary index,
533 /// not during writing of the per-module index which doesn't contain a
534 /// module path string table.
535 StringRef ModulePath;
536
537 /// List of values referenced by this global value's definition
538 /// (either by the initializer of a global variable, or referenced
539 /// from within a function). This does not include functions called, which
540 /// are listed in the derived FunctionSummary object.
541 std::vector<ValueInfo> RefEdgeList;
542
543protected:
544 GlobalValueSummary(SummaryKind K, GVFlags Flags, std::vector<ValueInfo> Refs)
545 : Kind(K), Flags(Flags), RefEdgeList(std::move(Refs)) {
546 assert((K != AliasKind || Refs.empty()) &&
547 "Expect no references for AliasSummary");
548 }
549
550public:
551 virtual ~GlobalValueSummary() = default;
552
553 /// Returns the hash of the original name, it is identical to the GUID for
554 /// externally visible symbols, but not for local ones.
555 GlobalValue::GUID getOriginalName() const { return OriginalName; }
556
557 /// Initialize the original name hash in this summary.
558 void setOriginalName(GlobalValue::GUID Name) { OriginalName = Name; }
559
560 /// Which kind of summary subclass this is.
561 SummaryKind getSummaryKind() const { return Kind; }
562
563 /// Set the path to the module containing this function, for use in
564 /// the combined index.
565 void setModulePath(StringRef ModPath) { ModulePath = ModPath; }
566
567 /// Get the path to the module containing this function.
568 StringRef modulePath() const { return ModulePath; }
569
570 /// Get the flags for this GlobalValue (see \p struct GVFlags).
571 GVFlags flags() const { return Flags; }
572
573 /// Return linkage type recorded for this global value.
575 return static_cast<GlobalValue::LinkageTypes>(Flags.Linkage);
576 }
577
578 /// Sets the linkage to the value determined by global summary-based
579 /// optimization. Will be applied in the ThinLTO backends.
581 Flags.Linkage = Linkage;
582 }
583
584 /// Return true if this global value can't be imported.
585 bool notEligibleToImport() const { return Flags.NotEligibleToImport; }
586
587 bool isLive() const { return Flags.Live; }
588
589 void setLive(bool Live) { Flags.Live = Live; }
590
591 void setDSOLocal(bool Local) { Flags.DSOLocal = Local; }
592
593 bool isDSOLocal() const { return Flags.DSOLocal; }
594
595 void setCanAutoHide(bool CanAutoHide) { Flags.CanAutoHide = CanAutoHide; }
596
597 bool canAutoHide() const { return Flags.CanAutoHide; }
598
599 bool shouldImportAsDecl() const {
600 return Flags.ImportType == GlobalValueSummary::ImportKind::Declaration;
601 }
602
603 void setImportKind(ImportKind IK) { Flags.ImportType = IK; }
604
606 return static_cast<ImportKind>(Flags.ImportType);
607 }
608
610 return (GlobalValue::VisibilityTypes)Flags.Visibility;
611 }
613 Flags.Visibility = (unsigned)Vis;
614 }
615
616 /// Flag that this global value cannot be imported.
617 void setNotEligibleToImport() { Flags.NotEligibleToImport = true; }
618
619 /// Return the list of values referenced by this global value definition.
620 ArrayRef<ValueInfo> refs() const { return RefEdgeList; }
621
622 /// If this is an alias summary, returns the summary of the aliased object (a
623 /// global variable or function), otherwise returns itself.
625 const GlobalValueSummary *getBaseObject() const;
626
627 friend class ModuleSummaryIndex;
628};
629
631
632/// Alias summary information.
634 ValueInfo AliaseeValueInfo;
635
636 /// This is the Aliasee in the same module as alias (could get from VI, trades
637 /// memory for time). Note that this pointer may be null (and the value info
638 /// empty) when we have a distributed index where the alias is being imported
639 /// (as a copy of the aliasee), but the aliasee is not.
640 GlobalValueSummary *AliaseeSummary;
641
642public:
645 AliaseeSummary(nullptr) {}
646
647 /// Check if this is an alias summary.
648 static bool classof(const GlobalValueSummary *GVS) {
649 return GVS->getSummaryKind() == AliasKind;
650 }
651
652 void setAliasee(ValueInfo &AliaseeVI, GlobalValueSummary *Aliasee) {
653 AliaseeValueInfo = AliaseeVI;
654 AliaseeSummary = Aliasee;
655 }
656
657 bool hasAliasee() const {
658 assert(!!AliaseeSummary == (AliaseeValueInfo &&
659 !AliaseeValueInfo.getSummaryList().empty()) &&
660 "Expect to have both aliasee summary and summary list or neither");
661 return !!AliaseeSummary;
662 }
663
665 assert(AliaseeSummary && "Unexpected missing aliasee summary");
666 return *AliaseeSummary;
667 }
668
670 return const_cast<GlobalValueSummary &>(
671 static_cast<const AliasSummary *>(this)->getAliasee());
672 }
674 assert(AliaseeValueInfo && "Unexpected missing aliasee");
675 return AliaseeValueInfo;
676 }
678 assert(AliaseeValueInfo && "Unexpected missing aliasee");
679 return AliaseeValueInfo.getGUID();
680 }
681};
682
684 if (auto *AS = dyn_cast<AliasSummary>(this))
685 return &AS->getAliasee();
686 return this;
687}
688
690 if (auto *AS = dyn_cast<AliasSummary>(this))
691 return &AS->getAliasee();
692 return this;
693}
694
695/// Function summary information to aid decisions and implementation of
696/// importing.
698public:
699 /// <CalleeValueInfo, CalleeInfo> call edge pair.
700 using EdgeTy = std::pair<ValueInfo, CalleeInfo>;
701
702 /// Types for -force-summary-edges-cold debugging option.
703 enum ForceSummaryHotnessType : unsigned {
707 };
708
709 /// An "identifier" for a virtual function. This contains the type identifier
710 /// represented as a GUID and the offset from the address point to the virtual
711 /// function pointer, where "address point" is as defined in the Itanium ABI:
712 /// https://itanium-cxx-abi.github.io/cxx-abi/abi.html#vtable-general
713 struct VFuncId {
716 };
717
718 /// A specification for a virtual function call with all constant integer
719 /// arguments. This is used to perform virtual constant propagation on the
720 /// summary.
721 struct ConstVCall {
723 std::vector<uint64_t> Args;
724 };
725
726 /// All type identifier related information. Because these fields are
727 /// relatively uncommon we only allocate space for them if necessary.
728 struct TypeIdInfo {
729 /// List of type identifiers used by this function in llvm.type.test
730 /// intrinsics referenced by something other than an llvm.assume intrinsic,
731 /// represented as GUIDs.
732 std::vector<GlobalValue::GUID> TypeTests;
733
734 /// List of virtual calls made by this function using (respectively)
735 /// llvm.assume(llvm.type.test) or llvm.type.checked.load intrinsics that do
736 /// not have all constant integer arguments.
738
739 /// List of virtual calls made by this function using (respectively)
740 /// llvm.assume(llvm.type.test) or llvm.type.checked.load intrinsics with
741 /// all constant integer arguments.
742 std::vector<ConstVCall> TypeTestAssumeConstVCalls,
744 };
745
746 /// Flags specific to function summaries.
747 struct FFlags {
748 // Function attribute flags. Used to track if a function accesses memory,
749 // recurses or aliases.
750 unsigned ReadNone : 1;
751 unsigned ReadOnly : 1;
752 unsigned NoRecurse : 1;
753 unsigned ReturnDoesNotAlias : 1;
754
755 // Indicate if the global value cannot be inlined.
756 unsigned NoInline : 1;
757 // Indicate if function should be always inlined.
758 unsigned AlwaysInline : 1;
759 // Indicate if function never raises an exception. Can be modified during
760 // thinlink function attribute propagation
761 unsigned NoUnwind : 1;
762 // Indicate if function contains instructions that mayThrow
763 unsigned MayThrow : 1;
764
765 // If there are calls to unknown targets (e.g. indirect)
766 unsigned HasUnknownCall : 1;
767
768 // Indicate if a function must be an unreachable function.
769 //
770 // This bit is sufficient but not necessary;
771 // if this bit is on, the function must be regarded as unreachable;
772 // if this bit is off, the function might be reachable or unreachable.
773 unsigned MustBeUnreachable : 1;
774
776 this->ReadNone &= RHS.ReadNone;
777 this->ReadOnly &= RHS.ReadOnly;
778 this->NoRecurse &= RHS.NoRecurse;
780 this->NoInline &= RHS.NoInline;
781 this->AlwaysInline &= RHS.AlwaysInline;
782 this->NoUnwind &= RHS.NoUnwind;
783 this->MayThrow &= RHS.MayThrow;
784 this->HasUnknownCall &= RHS.HasUnknownCall;
786 return *this;
787 }
788
789 bool anyFlagSet() {
790 return this->ReadNone | this->ReadOnly | this->NoRecurse |
791 this->ReturnDoesNotAlias | this->NoInline | this->AlwaysInline |
792 this->NoUnwind | this->MayThrow | this->HasUnknownCall |
793 this->MustBeUnreachable;
794 }
795
796 operator std::string() {
797 std::string Output;
798 raw_string_ostream OS(Output);
799 OS << "funcFlags: (";
800 OS << "readNone: " << this->ReadNone;
801 OS << ", readOnly: " << this->ReadOnly;
802 OS << ", noRecurse: " << this->NoRecurse;
803 OS << ", returnDoesNotAlias: " << this->ReturnDoesNotAlias;
804 OS << ", noInline: " << this->NoInline;
805 OS << ", alwaysInline: " << this->AlwaysInline;
806 OS << ", noUnwind: " << this->NoUnwind;
807 OS << ", mayThrow: " << this->MayThrow;
808 OS << ", hasUnknownCall: " << this->HasUnknownCall;
809 OS << ", mustBeUnreachable: " << this->MustBeUnreachable;
810 OS << ")";
811 return Output;
812 }
813 };
814
815 /// Describes the uses of a parameter by the function.
816 struct ParamAccess {
817 static constexpr uint32_t RangeWidth = 64;
818
819 /// Describes the use of a value in a call instruction, specifying the
820 /// call's target, the value's parameter number, and the possible range of
821 /// offsets from the beginning of the value that are passed.
822 struct Call {
825 ConstantRange Offsets{/*BitWidth=*/RangeWidth, /*isFullSet=*/true};
826
827 Call() = default;
830 };
831
833 /// The range contains byte offsets from the parameter pointer which
834 /// accessed by the function. In the per-module summary, it only includes
835 /// accesses made by the function instructions. In the combined summary, it
836 /// also includes accesses by nested function calls.
837 ConstantRange Use{/*BitWidth=*/RangeWidth, /*isFullSet=*/true};
838 /// In the per-module summary, it summarizes the byte offset applied to each
839 /// pointer parameter before passing to each corresponding callee.
840 /// In the combined summary, it's empty and information is propagated by
841 /// inter-procedural analysis and applied to the Use field.
842 std::vector<Call> Calls;
843
844 ParamAccess() = default;
846 : ParamNo(ParamNo), Use(Use) {}
847 };
848
849 /// Create an empty FunctionSummary (with specified call edges).
850 /// Used to represent external nodes and the dummy root node.
851 static FunctionSummary
852 makeDummyFunctionSummary(std::vector<FunctionSummary::EdgeTy> Edges) {
853 return FunctionSummary(
857 /*NotEligibleToImport=*/true, /*Live=*/true, /*IsLocal=*/false,
858 /*CanAutoHide=*/false, GlobalValueSummary::ImportKind::Definition),
859 /*NumInsts=*/0, FunctionSummary::FFlags{}, /*EntryCount=*/0,
860 std::vector<ValueInfo>(), std::move(Edges),
861 std::vector<GlobalValue::GUID>(),
862 std::vector<FunctionSummary::VFuncId>(),
863 std::vector<FunctionSummary::VFuncId>(),
864 std::vector<FunctionSummary::ConstVCall>(),
865 std::vector<FunctionSummary::ConstVCall>(),
866 std::vector<FunctionSummary::ParamAccess>(),
867 std::vector<CallsiteInfo>(), std::vector<AllocInfo>());
868 }
869
870 /// A dummy node to reference external functions that aren't in the index
872
873private:
874 /// Number of instructions (ignoring debug instructions, e.g.) computed
875 /// during the initial compile step when the summary index is first built.
876 unsigned InstCount;
877
878 /// Function summary specific flags.
879 FFlags FunFlags;
880
881 /// The synthesized entry count of the function.
882 /// This is only populated during ThinLink phase and remains unused while
883 /// generating per-module summaries.
884 uint64_t EntryCount = 0;
885
886 /// List of <CalleeValueInfo, CalleeInfo> call edge pairs from this function.
887 std::vector<EdgeTy> CallGraphEdgeList;
888
889 std::unique_ptr<TypeIdInfo> TIdInfo;
890
891 /// Uses for every parameter to this function.
892 using ParamAccessesTy = std::vector<ParamAccess>;
893 std::unique_ptr<ParamAccessesTy> ParamAccesses;
894
895 /// Optional list of memprof callsite metadata summaries. The correspondence
896 /// between the callsite summary and the callsites in the function is implied
897 /// by the order in the vector (and can be validated by comparing the stack
898 /// ids in the CallsiteInfo to those in the instruction callsite metadata).
899 /// As a memory savings optimization, we only create these for the prevailing
900 /// copy of a symbol when creating the combined index during LTO.
901 using CallsitesTy = std::vector<CallsiteInfo>;
902 std::unique_ptr<CallsitesTy> Callsites;
903
904 /// Optional list of allocation memprof metadata summaries. The correspondence
905 /// between the alloc memprof summary and the allocation callsites in the
906 /// function is implied by the order in the vector (and can be validated by
907 /// comparing the stack ids in the AllocInfo to those in the instruction
908 /// memprof metadata).
909 /// As a memory savings optimization, we only create these for the prevailing
910 /// copy of a symbol when creating the combined index during LTO.
911 using AllocsTy = std::vector<AllocInfo>;
912 std::unique_ptr<AllocsTy> Allocs;
913
914public:
915 FunctionSummary(GVFlags Flags, unsigned NumInsts, FFlags FunFlags,
916 uint64_t EntryCount, std::vector<ValueInfo> Refs,
917 std::vector<EdgeTy> CGEdges,
918 std::vector<GlobalValue::GUID> TypeTests,
919 std::vector<VFuncId> TypeTestAssumeVCalls,
920 std::vector<VFuncId> TypeCheckedLoadVCalls,
921 std::vector<ConstVCall> TypeTestAssumeConstVCalls,
922 std::vector<ConstVCall> TypeCheckedLoadConstVCalls,
923 std::vector<ParamAccess> Params, CallsitesTy CallsiteList,
924 AllocsTy AllocList)
925 : GlobalValueSummary(FunctionKind, Flags, std::move(Refs)),
926 InstCount(NumInsts), FunFlags(FunFlags), EntryCount(EntryCount),
927 CallGraphEdgeList(std::move(CGEdges)) {
928 if (!TypeTests.empty() || !TypeTestAssumeVCalls.empty() ||
929 !TypeCheckedLoadVCalls.empty() || !TypeTestAssumeConstVCalls.empty() ||
930 !TypeCheckedLoadConstVCalls.empty())
931 TIdInfo = std::make_unique<TypeIdInfo>(
932 TypeIdInfo{std::move(TypeTests), std::move(TypeTestAssumeVCalls),
933 std::move(TypeCheckedLoadVCalls),
934 std::move(TypeTestAssumeConstVCalls),
935 std::move(TypeCheckedLoadConstVCalls)});
936 if (!Params.empty())
937 ParamAccesses = std::make_unique<ParamAccessesTy>(std::move(Params));
938 if (!CallsiteList.empty())
939 Callsites = std::make_unique<CallsitesTy>(std::move(CallsiteList));
940 if (!AllocList.empty())
941 Allocs = std::make_unique<AllocsTy>(std::move(AllocList));
942 }
943 // Gets the number of readonly and writeonly refs in RefEdgeList
944 std::pair<unsigned, unsigned> specialRefCounts() const;
945
946 /// Check if this is a function summary.
947 static bool classof(const GlobalValueSummary *GVS) {
948 return GVS->getSummaryKind() == FunctionKind;
949 }
950
951 /// Get function summary flags.
952 FFlags fflags() const { return FunFlags; }
953
954 void setNoRecurse() { FunFlags.NoRecurse = true; }
955
956 void setNoUnwind() { FunFlags.NoUnwind = true; }
957
958 /// Get the instruction count recorded for this function.
959 unsigned instCount() const { return InstCount; }
960
961 /// Get the synthetic entry count for this function.
962 uint64_t entryCount() const { return EntryCount; }
963
964 /// Set the synthetic entry count for this function.
965 void setEntryCount(uint64_t EC) { EntryCount = EC; }
966
967 /// Return the list of <CalleeValueInfo, CalleeInfo> pairs.
968 ArrayRef<EdgeTy> calls() const { return CallGraphEdgeList; }
969
970 std::vector<EdgeTy> &mutableCalls() { return CallGraphEdgeList; }
971
972 void addCall(EdgeTy E) { CallGraphEdgeList.push_back(E); }
973
974 /// Returns the list of type identifiers used by this function in
975 /// llvm.type.test intrinsics other than by an llvm.assume intrinsic,
976 /// represented as GUIDs.
978 if (TIdInfo)
979 return TIdInfo->TypeTests;
980 return {};
981 }
982
983 /// Returns the list of virtual calls made by this function using
984 /// llvm.assume(llvm.type.test) intrinsics that do not have all constant
985 /// integer arguments.
987 if (TIdInfo)
988 return TIdInfo->TypeTestAssumeVCalls;
989 return {};
990 }
991
992 /// Returns the list of virtual calls made by this function using
993 /// llvm.type.checked.load intrinsics that do not have all constant integer
994 /// arguments.
996 if (TIdInfo)
997 return TIdInfo->TypeCheckedLoadVCalls;
998 return {};
999 }
1000
1001 /// Returns the list of virtual calls made by this function using
1002 /// llvm.assume(llvm.type.test) intrinsics with all constant integer
1003 /// arguments.
1005 if (TIdInfo)
1006 return TIdInfo->TypeTestAssumeConstVCalls;
1007 return {};
1008 }
1009
1010 /// Returns the list of virtual calls made by this function using
1011 /// llvm.type.checked.load intrinsics with all constant integer arguments.
1013 if (TIdInfo)
1014 return TIdInfo->TypeCheckedLoadConstVCalls;
1015 return {};
1016 }
1017
1018 /// Returns the list of known uses of pointer parameters.
1020 if (ParamAccesses)
1021 return *ParamAccesses;
1022 return {};
1023 }
1024
1025 /// Sets the list of known uses of pointer parameters.
1026 void setParamAccesses(std::vector<ParamAccess> NewParams) {
1027 if (NewParams.empty())
1028 ParamAccesses.reset();
1029 else if (ParamAccesses)
1030 *ParamAccesses = std::move(NewParams);
1031 else
1032 ParamAccesses = std::make_unique<ParamAccessesTy>(std::move(NewParams));
1033 }
1034
1035 /// Add a type test to the summary. This is used by WholeProgramDevirt if we
1036 /// were unable to devirtualize a checked call.
1038 if (!TIdInfo)
1039 TIdInfo = std::make_unique<TypeIdInfo>();
1040 TIdInfo->TypeTests.push_back(Guid);
1041 }
1042
1043 const TypeIdInfo *getTypeIdInfo() const { return TIdInfo.get(); };
1044
1046 if (Callsites)
1047 return *Callsites;
1048 return {};
1049 }
1050
1051 CallsitesTy &mutableCallsites() {
1052 assert(Callsites);
1053 return *Callsites;
1054 }
1055
1056 void addCallsite(CallsiteInfo &Callsite) {
1057 if (!Callsites)
1058 Callsites = std::make_unique<CallsitesTy>();
1059 Callsites->push_back(Callsite);
1060 }
1061
1063 if (Allocs)
1064 return *Allocs;
1065 return {};
1066 }
1067
1068 AllocsTy &mutableAllocs() {
1069 assert(Allocs);
1070 return *Allocs;
1071 }
1072
1073 friend struct GraphTraits<ValueInfo>;
1074};
1075
1076template <> struct DenseMapInfo<FunctionSummary::VFuncId> {
1077 static FunctionSummary::VFuncId getEmptyKey() { return {0, uint64_t(-1)}; }
1078
1080 return {0, uint64_t(-2)};
1081 }
1082
1084 return L.GUID == R.GUID && L.Offset == R.Offset;
1085 }
1086
1087 static unsigned getHashValue(FunctionSummary::VFuncId I) { return I.GUID; }
1088};
1089
1090template <> struct DenseMapInfo<FunctionSummary::ConstVCall> {
1092 return {{0, uint64_t(-1)}, {}};
1093 }
1094
1096 return {{0, uint64_t(-2)}, {}};
1097 }
1098
1101 return DenseMapInfo<FunctionSummary::VFuncId>::isEqual(L.VFunc, R.VFunc) &&
1102 L.Args == R.Args;
1103 }
1104
1106 return I.VFunc.GUID;
1107 }
1108};
1109
1110/// The ValueInfo and offset for a function within a vtable definition
1111/// initializer array.
1114 : FuncVI(VI), VTableOffset(Offset) {}
1115
1118};
1119/// List of functions referenced by a particular vtable definition.
1120using VTableFuncList = std::vector<VirtFuncOffset>;
1121
1122/// Global variable summary information to aid decisions and
1123/// implementation of importing.
1124///
1125/// Global variable summary has two extra flag, telling if it is
1126/// readonly or writeonly. Both readonly and writeonly variables
1127/// can be optimized in the backed: readonly variables can be
1128/// const-folded, while writeonly vars can be completely eliminated
1129/// together with corresponding stores. We let both things happen
1130/// by means of internalizing such variables after ThinLTO import.
1132private:
1133 /// For vtable definitions this holds the list of functions and
1134 /// their corresponding offsets within the initializer array.
1135 std::unique_ptr<VTableFuncList> VTableFuncs;
1136
1137public:
1138 struct GVarFlags {
1139 GVarFlags(bool ReadOnly, bool WriteOnly, bool Constant,
1141 : MaybeReadOnly(ReadOnly), MaybeWriteOnly(WriteOnly),
1143
1144 // If true indicates that this global variable might be accessed
1145 // purely by non-volatile load instructions. This in turn means
1146 // it can be internalized in source and destination modules during
1147 // thin LTO import because it neither modified nor its address
1148 // is taken.
1149 unsigned MaybeReadOnly : 1;
1150 // If true indicates that variable is possibly only written to, so
1151 // its value isn't loaded and its address isn't taken anywhere.
1152 // False, when 'Constant' attribute is set.
1153 unsigned MaybeWriteOnly : 1;
1154 // Indicates that value is a compile-time constant. Global variable
1155 // can be 'Constant' while not being 'ReadOnly' on several occasions:
1156 // - it is volatile, (e.g mapped device address)
1157 // - its address is taken, meaning that unlike 'ReadOnly' vars we can't
1158 // internalize it.
1159 // Constant variables are always imported thus giving compiler an
1160 // opportunity to make some extra optimizations. Readonly constants
1161 // are also internalized.
1162 unsigned Constant : 1;
1163 // Set from metadata on vtable definitions during the module summary
1164 // analysis.
1165 unsigned VCallVisibility : 2;
1167
1169 std::vector<ValueInfo> Refs)
1170 : GlobalValueSummary(GlobalVarKind, Flags, std::move(Refs)),
1171 VarFlags(VarFlags) {}
1172
1173 /// Check if this is a global variable summary.
1174 static bool classof(const GlobalValueSummary *GVS) {
1175 return GVS->getSummaryKind() == GlobalVarKind;
1176 }
1177
1178 GVarFlags varflags() const { return VarFlags; }
1179 void setReadOnly(bool RO) { VarFlags.MaybeReadOnly = RO; }
1180 void setWriteOnly(bool WO) { VarFlags.MaybeWriteOnly = WO; }
1181 bool maybeReadOnly() const { return VarFlags.MaybeReadOnly; }
1182 bool maybeWriteOnly() const { return VarFlags.MaybeWriteOnly; }
1183 bool isConstant() const { return VarFlags.Constant; }
1186 }
1189 }
1190
1192 assert(!VTableFuncs);
1193 VTableFuncs = std::make_unique<VTableFuncList>(std::move(Funcs));
1194 }
1195
1197 if (VTableFuncs)
1198 return *VTableFuncs;
1199 return {};
1200 }
1201};
1202
1204 /// Specifies which kind of type check we should emit for this byte array.
1205 /// See http://clang.llvm.org/docs/ControlFlowIntegrityDesign.html for full
1206 /// details on each kind of check; the enumerators are described with
1207 /// reference to that document.
1208 enum Kind {
1209 Unsat, ///< Unsatisfiable type (i.e. no global has this type metadata)
1210 ByteArray, ///< Test a byte array (first example)
1211 Inline, ///< Inlined bit vector ("Short Inline Bit Vectors")
1212 Single, ///< Single element (last example in "Short Inline Bit Vectors")
1213 AllOnes, ///< All-ones bit vector ("Eliminating Bit Vector Checks for
1214 /// All-Ones Bit Vectors")
1215 Unknown, ///< Unknown (analysis not performed, don't lower)
1217
1218 /// Range of size-1 expressed as a bit width. For example, if the size is in
1219 /// range [1,256], this number will be 8. This helps generate the most compact
1220 /// instruction sequences.
1221 unsigned SizeM1BitWidth = 0;
1222
1223 // The following fields are only used if the target does not support the use
1224 // of absolute symbols to store constants. Their meanings are the same as the
1225 // corresponding fields in LowerTypeTestsModule::TypeIdLowering in
1226 // LowerTypeTests.cpp.
1227
1230 uint8_t BitMask = 0;
1232};
1233
1235 enum Kind {
1236 Indir, ///< Just do a regular virtual call
1237 SingleImpl, ///< Single implementation devirtualization
1238 BranchFunnel, ///< When retpoline mitigation is enabled, use a branch funnel
1239 ///< that is defined in the merged module. Otherwise same as
1240 ///< Indir.
1242
1243 std::string SingleImplName;
1244
1245 struct ByArg {
1246 enum Kind {
1247 Indir, ///< Just do a regular virtual call
1248 UniformRetVal, ///< Uniform return value optimization
1249 UniqueRetVal, ///< Unique return value optimization
1250 VirtualConstProp, ///< Virtual constant propagation
1252
1253 /// Additional information for the resolution:
1254 /// - UniformRetVal: the uniform return value.
1255 /// - UniqueRetVal: the return value associated with the unique vtable (0 or
1256 /// 1).
1258
1259 // The following fields are only used if the target does not support the use
1260 // of absolute symbols to store constants.
1261
1264 };
1265
1266 /// Resolutions for calls with all constant integer arguments (excluding the
1267 /// first argument, "this"), where the key is the argument vector.
1268 std::map<std::vector<uint64_t>, ByArg> ResByArg;
1269};
1270
1273
1274 /// Mapping from byte offset to whole-program devirt resolution for that
1275 /// (typeid, byte offset) pair.
1276 std::map<uint64_t, WholeProgramDevirtResolution> WPDRes;
1277};
1278
1279/// 160 bits SHA1
1280using ModuleHash = std::array<uint32_t, 5>;
1281
1282/// Type used for iterating through the global value summary map.
1283using const_gvsummary_iterator = GlobalValueSummaryMapTy::const_iterator;
1284using gvsummary_iterator = GlobalValueSummaryMapTy::iterator;
1285
1286/// String table to hold/own module path strings, as well as a hash
1287/// of the module. The StringMap makes a copy of and owns inserted strings.
1289
1290/// Map of global value GUID to its summary, used to identify values defined in
1291/// a particular module, and provide efficient access to their summary.
1293
1294/// A set of global value summary pointers.
1295using GVSummaryPtrSet = std::unordered_set<GlobalValueSummary *>;
1296
1297/// Map of a type GUID to type id string and summary (multimap used
1298/// in case of GUID conflicts).
1300 std::multimap<GlobalValue::GUID, std::pair<std::string, TypeIdSummary>>;
1301
1302/// The following data structures summarize type metadata information.
1303/// For type metadata overview see https://llvm.org/docs/TypeMetadata.html.
1304/// Each type metadata includes both the type identifier and the offset of
1305/// the address point of the type (the address held by objects of that type
1306/// which may not be the beginning of the virtual table). Vtable definitions
1307/// are decorated with type metadata for the types they are compatible with.
1308///
1309/// Holds information about vtable definitions decorated with type metadata:
1310/// the vtable definition value and its address point offset in a type
1311/// identifier metadata it is decorated (compatible) with.
1315
1318};
1319/// List of vtable definitions decorated by a particular type identifier,
1320/// and their corresponding offsets in that type identifier's metadata.
1321/// Note that each type identifier may be compatible with multiple vtables, due
1322/// to inheritance, which is why this is a vector.
1323using TypeIdCompatibleVtableInfo = std::vector<TypeIdOffsetVtableInfo>;
1324
1325/// Class to hold module path string table and global value map,
1326/// and encapsulate methods for operating on them.
1328private:
1329 /// Map from value name to list of summary instances for values of that
1330 /// name (may be duplicates in the COMDAT case, e.g.).
1331 GlobalValueSummaryMapTy GlobalValueMap;
1332
1333 /// Holds strings for combined index, mapping to the corresponding module ID.
1334 ModulePathStringTableTy ModulePathStringTable;
1335
1336 /// Mapping from type identifier GUIDs to type identifier and its summary
1337 /// information. Produced by thin link.
1338 TypeIdSummaryMapTy TypeIdMap;
1339
1340 /// Mapping from type identifier to information about vtables decorated
1341 /// with that type identifier's metadata. Produced by per module summary
1342 /// analysis and consumed by thin link. For more information, see description
1343 /// above where TypeIdCompatibleVtableInfo is defined.
1344 std::map<std::string, TypeIdCompatibleVtableInfo, std::less<>>
1345 TypeIdCompatibleVtableMap;
1346
1347 /// Mapping from original ID to GUID. If original ID can map to multiple
1348 /// GUIDs, it will be mapped to 0.
1349 std::map<GlobalValue::GUID, GlobalValue::GUID> OidGuidMap;
1350
1351 /// Indicates that summary-based GlobalValue GC has run, and values with
1352 /// GVFlags::Live==false are really dead. Otherwise, all values must be
1353 /// considered live.
1354 bool WithGlobalValueDeadStripping = false;
1355
1356 /// Indicates that summary-based attribute propagation has run and
1357 /// GVarFlags::MaybeReadonly / GVarFlags::MaybeWriteonly are really
1358 /// read/write only.
1359 bool WithAttributePropagation = false;
1360
1361 /// Indicates that summary-based DSOLocal propagation has run and the flag in
1362 /// every summary of a GV is synchronized.
1363 bool WithDSOLocalPropagation = false;
1364
1365 /// Indicates that we have whole program visibility.
1366 bool WithWholeProgramVisibility = false;
1367
1368 /// Indicates that summary-based synthetic entry count propagation has run
1369 bool HasSyntheticEntryCounts = false;
1370
1371 /// Indicates that we linked with allocator supporting hot/cold new operators.
1372 bool WithSupportsHotColdNew = false;
1373
1374 /// Indicates that distributed backend should skip compilation of the
1375 /// module. Flag is suppose to be set by distributed ThinLTO indexing
1376 /// when it detected that the module is not needed during the final
1377 /// linking. As result distributed backend should just output a minimal
1378 /// valid object file.
1379 bool SkipModuleByDistributedBackend = false;
1380
1381 /// If true then we're performing analysis of IR module, or parsing along with
1382 /// the IR from assembly. The value of 'false' means we're reading summary
1383 /// from BC or YAML source. Affects the type of value stored in NameOrGV
1384 /// union.
1385 bool HaveGVs;
1386
1387 // True if the index was created for a module compiled with -fsplit-lto-unit.
1388 bool EnableSplitLTOUnit;
1389
1390 // True if the index was created for a module compiled with -funified-lto
1391 bool UnifiedLTO;
1392
1393 // True if some of the modules were compiled with -fsplit-lto-unit and
1394 // some were not. Set when the combined index is created during the thin link.
1395 bool PartiallySplitLTOUnits = false;
1396
1397 /// True if some of the FunctionSummary contains a ParamAccess.
1398 bool HasParamAccess = false;
1399
1400 std::set<std::string> CfiFunctionDefs;
1401 std::set<std::string> CfiFunctionDecls;
1402
1403 // Used in cases where we want to record the name of a global, but
1404 // don't have the string owned elsewhere (e.g. the Strtab on a module).
1405 BumpPtrAllocator Alloc;
1406 StringSaver Saver;
1407
1408 // The total number of basic blocks in the module in the per-module summary or
1409 // the total number of basic blocks in the LTO unit in the combined index.
1410 // FIXME: Putting this in the distributed ThinLTO index files breaks LTO
1411 // backend caching on any BB change to any linked file. It is currently not
1412 // used except in the case of a SamplePGO partial profile, and should be
1413 // reevaluated/redesigned to allow more effective incremental builds in that
1414 // case.
1415 uint64_t BlockCount;
1416
1417 // List of unique stack ids (hashes). We use a 4B index of the id in the
1418 // stack id lists on the alloc and callsite summaries for memory savings,
1419 // since the number of unique ids is in practice much smaller than the
1420 // number of stack id references in the summaries.
1421 std::vector<uint64_t> StackIds;
1422
1423 // Temporary map while building StackIds list. Clear when index is completely
1424 // built via releaseTemporaryMemory.
1425 DenseMap<uint64_t, unsigned> StackIdToIndex;
1426
1427 // YAML I/O support.
1429
1430 GlobalValueSummaryMapTy::value_type *
1431 getOrInsertValuePtr(GlobalValue::GUID GUID) {
1432 return &*GlobalValueMap.emplace(GUID, GlobalValueSummaryInfo(HaveGVs))
1433 .first;
1434 }
1435
1436public:
1437 // See HaveGVs variable comment.
1438 ModuleSummaryIndex(bool HaveGVs, bool EnableSplitLTOUnit = false,
1439 bool UnifiedLTO = false)
1440 : HaveGVs(HaveGVs), EnableSplitLTOUnit(EnableSplitLTOUnit),
1441 UnifiedLTO(UnifiedLTO), Saver(Alloc), BlockCount(0) {}
1442
1443 // Current version for the module summary in bitcode files.
1444 // The BitcodeSummaryVersion should be bumped whenever we introduce changes
1445 // in the way some record are interpreted, like flags for instance.
1446 // Note that incrementing this may require changes in both BitcodeReader.cpp
1447 // and BitcodeWriter.cpp.
1448 static constexpr uint64_t BitcodeSummaryVersion = 10;
1449
1450 // Regular LTO module name for ASM writer
1451 static constexpr const char *getRegularLTOModuleName() {
1452 return "[Regular LTO]";
1453 }
1454
1455 bool haveGVs() const { return HaveGVs; }
1456
1457 uint64_t getFlags() const;
1458 void setFlags(uint64_t Flags);
1459
1460 uint64_t getBlockCount() const { return BlockCount; }
1461 void addBlockCount(uint64_t C) { BlockCount += C; }
1462 void setBlockCount(uint64_t C) { BlockCount = C; }
1463
1464 gvsummary_iterator begin() { return GlobalValueMap.begin(); }
1465 const_gvsummary_iterator begin() const { return GlobalValueMap.begin(); }
1466 gvsummary_iterator end() { return GlobalValueMap.end(); }
1467 const_gvsummary_iterator end() const { return GlobalValueMap.end(); }
1468 size_t size() const { return GlobalValueMap.size(); }
1469
1470 const std::vector<uint64_t> &stackIds() const { return StackIds; }
1471
1472 unsigned addOrGetStackIdIndex(uint64_t StackId) {
1473 auto Inserted = StackIdToIndex.insert({StackId, StackIds.size()});
1474 if (Inserted.second)
1475 StackIds.push_back(StackId);
1476 return Inserted.first->second;
1477 }
1478
1480 assert(StackIds.size() > Index);
1481 return StackIds[Index];
1482 }
1483
1484 // Facility to release memory from data structures only needed during index
1485 // construction (including while building combined index). Currently this only
1486 // releases the temporary map used while constructing a correspondence between
1487 // stack ids and their index in the StackIds vector. Mostly impactful when
1488 // building a large combined index.
1490 assert(StackIdToIndex.size() == StackIds.size());
1491 StackIdToIndex.clear();
1492 StackIds.shrink_to_fit();
1493 }
1494
1495 /// Convenience function for doing a DFS on a ValueInfo. Marks the function in
1496 /// the FunctionHasParent map.
1498 std::map<ValueInfo, bool> &FunctionHasParent) {
1499 if (!V.getSummaryList().size())
1500 return; // skip external functions that don't have summaries
1501
1502 // Mark discovered if we haven't yet
1503 auto S = FunctionHasParent.emplace(V, false);
1504
1505 // Stop if we've already discovered this node
1506 if (!S.second)
1507 return;
1508
1510 dyn_cast<FunctionSummary>(V.getSummaryList().front().get());
1511 assert(F != nullptr && "Expected FunctionSummary node");
1512
1513 for (const auto &C : F->calls()) {
1514 // Insert node if necessary
1515 auto S = FunctionHasParent.emplace(C.first, true);
1516
1517 // Skip nodes that we're sure have parents
1518 if (!S.second && S.first->second)
1519 continue;
1520
1521 if (S.second)
1522 discoverNodes(C.first, FunctionHasParent);
1523 else
1524 S.first->second = true;
1525 }
1526 }
1527
1528 // Calculate the callgraph root
1530 // Functions that have a parent will be marked in FunctionHasParent pair.
1531 // Once we've marked all functions, the functions in the map that are false
1532 // have no parent (so they're the roots)
1533 std::map<ValueInfo, bool> FunctionHasParent;
1534
1535 for (auto &S : *this) {
1536 // Skip external functions
1537 if (!S.second.SummaryList.size() ||
1538 !isa<FunctionSummary>(S.second.SummaryList.front().get()))
1539 continue;
1540 discoverNodes(ValueInfo(HaveGVs, &S), FunctionHasParent);
1541 }
1542
1543 std::vector<FunctionSummary::EdgeTy> Edges;
1544 // create edges to all roots in the Index
1545 for (auto &P : FunctionHasParent) {
1546 if (P.second)
1547 continue; // skip over non-root nodes
1548 Edges.push_back(std::make_pair(P.first, CalleeInfo{}));
1549 }
1550 if (Edges.empty()) {
1551 // Failed to find root - return an empty node
1553 }
1554 auto CallGraphRoot = FunctionSummary::makeDummyFunctionSummary(Edges);
1555 return CallGraphRoot;
1556 }
1557
1559 return WithGlobalValueDeadStripping;
1560 }
1562 WithGlobalValueDeadStripping = true;
1563 }
1564
1565 bool withAttributePropagation() const { return WithAttributePropagation; }
1567 WithAttributePropagation = true;
1568 }
1569
1570 bool withDSOLocalPropagation() const { return WithDSOLocalPropagation; }
1571 void setWithDSOLocalPropagation() { WithDSOLocalPropagation = true; }
1572
1573 bool withWholeProgramVisibility() const { return WithWholeProgramVisibility; }
1574 void setWithWholeProgramVisibility() { WithWholeProgramVisibility = true; }
1575
1576 bool isReadOnly(const GlobalVarSummary *GVS) const {
1577 return WithAttributePropagation && GVS->maybeReadOnly();
1578 }
1579 bool isWriteOnly(const GlobalVarSummary *GVS) const {
1580 return WithAttributePropagation && GVS->maybeWriteOnly();
1581 }
1582
1583 bool hasSyntheticEntryCounts() const { return HasSyntheticEntryCounts; }
1584 void setHasSyntheticEntryCounts() { HasSyntheticEntryCounts = true; }
1585
1586 bool withSupportsHotColdNew() const { return WithSupportsHotColdNew; }
1587 void setWithSupportsHotColdNew() { WithSupportsHotColdNew = true; }
1588
1590 return SkipModuleByDistributedBackend;
1591 }
1593 SkipModuleByDistributedBackend = true;
1594 }
1595
1596 bool enableSplitLTOUnit() const { return EnableSplitLTOUnit; }
1597 void setEnableSplitLTOUnit() { EnableSplitLTOUnit = true; }
1598
1599 bool hasUnifiedLTO() const { return UnifiedLTO; }
1600 void setUnifiedLTO() { UnifiedLTO = true; }
1601
1602 bool partiallySplitLTOUnits() const { return PartiallySplitLTOUnits; }
1603 void setPartiallySplitLTOUnits() { PartiallySplitLTOUnits = true; }
1604
1605 bool hasParamAccess() const { return HasParamAccess; }
1606
1607 bool isGlobalValueLive(const GlobalValueSummary *GVS) const {
1608 return !WithGlobalValueDeadStripping || GVS->isLive();
1609 }
1610 bool isGUIDLive(GlobalValue::GUID GUID) const;
1611
1612 /// Return a ValueInfo for the index value_type (convenient when iterating
1613 /// index).
1614 ValueInfo getValueInfo(const GlobalValueSummaryMapTy::value_type &R) const {
1615 return ValueInfo(HaveGVs, &R);
1616 }
1617
1618 /// Return a ValueInfo for GUID if it exists, otherwise return ValueInfo().
1620 auto I = GlobalValueMap.find(GUID);
1621 return ValueInfo(HaveGVs, I == GlobalValueMap.end() ? nullptr : &*I);
1622 }
1623
1624 /// Return a ValueInfo for \p GUID.
1626 return ValueInfo(HaveGVs, getOrInsertValuePtr(GUID));
1627 }
1628
1629 // Save a string in the Index. Use before passing Name to
1630 // getOrInsertValueInfo when the string isn't owned elsewhere (e.g. on the
1631 // module's Strtab).
1633
1634 /// Return a ValueInfo for \p GUID setting value \p Name.
1636 assert(!HaveGVs);
1637 auto VP = getOrInsertValuePtr(GUID);
1638 VP->second.U.Name = Name;
1639 return ValueInfo(HaveGVs, VP);
1640 }
1641
1642 /// Return a ValueInfo for \p GV and mark it as belonging to GV.
1644 assert(HaveGVs);
1645 auto VP = getOrInsertValuePtr(GV->getGUID());
1646 VP->second.U.GV = GV;
1647 return ValueInfo(HaveGVs, VP);
1648 }
1649
1650 /// Return the GUID for \p OriginalId in the OidGuidMap.
1652 const auto I = OidGuidMap.find(OriginalID);
1653 return I == OidGuidMap.end() ? 0 : I->second;
1654 }
1655
1656 std::set<std::string> &cfiFunctionDefs() { return CfiFunctionDefs; }
1657 const std::set<std::string> &cfiFunctionDefs() const { return CfiFunctionDefs; }
1658
1659 std::set<std::string> &cfiFunctionDecls() { return CfiFunctionDecls; }
1660 const std::set<std::string> &cfiFunctionDecls() const { return CfiFunctionDecls; }
1661
1662 /// Add a global value summary for a value.
1664 std::unique_ptr<GlobalValueSummary> Summary) {
1665 addGlobalValueSummary(getOrInsertValueInfo(&GV), std::move(Summary));
1666 }
1667
1668 /// Add a global value summary for a value of the given name.
1670 std::unique_ptr<GlobalValueSummary> Summary) {
1672 std::move(Summary));
1673 }
1674
1675 /// Add a global value summary for the given ValueInfo.
1677 std::unique_ptr<GlobalValueSummary> Summary) {
1678 if (const FunctionSummary *FS = dyn_cast<FunctionSummary>(Summary.get()))
1679 HasParamAccess |= !FS->paramAccesses().empty();
1680 addOriginalName(VI.getGUID(), Summary->getOriginalName());
1681 // Here we have a notionally const VI, but the value it points to is owned
1682 // by the non-const *this.
1683 const_cast<GlobalValueSummaryMapTy::value_type *>(VI.getRef())
1684 ->second.SummaryList.push_back(std::move(Summary));
1685 }
1686
1687 /// Add an original name for the value of the given GUID.
1689 GlobalValue::GUID OrigGUID) {
1690 if (OrigGUID == 0 || ValueGUID == OrigGUID)
1691 return;
1692 if (OidGuidMap.count(OrigGUID) && OidGuidMap[OrigGUID] != ValueGUID)
1693 OidGuidMap[OrigGUID] = 0;
1694 else
1695 OidGuidMap[OrigGUID] = ValueGUID;
1696 }
1697
1698 /// Find the summary for ValueInfo \p VI in module \p ModuleId, or nullptr if
1699 /// not found.
1701 auto SummaryList = VI.getSummaryList();
1702 auto Summary =
1703 llvm::find_if(SummaryList,
1704 [&](const std::unique_ptr<GlobalValueSummary> &Summary) {
1705 return Summary->modulePath() == ModuleId;
1706 });
1707 if (Summary == SummaryList.end())
1708 return nullptr;
1709 return Summary->get();
1710 }
1711
1712 /// Find the summary for global \p GUID in module \p ModuleId, or nullptr if
1713 /// not found.
1715 StringRef ModuleId) const {
1716 auto CalleeInfo = getValueInfo(ValueGUID);
1717 if (!CalleeInfo)
1718 return nullptr; // This function does not have a summary
1719 return findSummaryInModule(CalleeInfo, ModuleId);
1720 }
1721
1722 /// Returns the first GlobalValueSummary for \p GV, asserting that there
1723 /// is only one if \p PerModuleIndex.
1725 bool PerModuleIndex = true) const {
1726 assert(GV.hasName() && "Can't get GlobalValueSummary for GV with no name");
1727 return getGlobalValueSummary(GV.getGUID(), PerModuleIndex);
1728 }
1729
1730 /// Returns the first GlobalValueSummary for \p ValueGUID, asserting that
1731 /// there
1732 /// is only one if \p PerModuleIndex.
1734 bool PerModuleIndex = true) const;
1735
1736 /// Table of modules, containing module hash and id.
1738 return ModulePathStringTable;
1739 }
1740
1741 /// Table of modules, containing hash and id.
1742 StringMap<ModuleHash> &modulePaths() { return ModulePathStringTable; }
1743
1744 /// Get the module SHA1 hash recorded for the given module path.
1745 const ModuleHash &getModuleHash(const StringRef ModPath) const {
1746 auto It = ModulePathStringTable.find(ModPath);
1747 assert(It != ModulePathStringTable.end() && "Module not registered");
1748 return It->second;
1749 }
1750
1751 /// Convenience method for creating a promoted global name
1752 /// for the given value name of a local, and its original module's ID.
1753 static std::string getGlobalNameForLocal(StringRef Name, ModuleHash ModHash) {
1754 std::string Suffix = utostr((uint64_t(ModHash[0]) << 32) |
1755 ModHash[1]); // Take the first 64 bits
1756 return getGlobalNameForLocal(Name, Suffix);
1757 }
1758
1759 static std::string getGlobalNameForLocal(StringRef Name, StringRef Suffix) {
1760 SmallString<256> NewName(Name);
1761 NewName += ".llvm.";
1762 NewName += Suffix;
1763 return std::string(NewName);
1764 }
1765
1766 /// Helper to obtain the unpromoted name for a global value (or the original
1767 /// name if not promoted). Split off the rightmost ".llvm.${hash}" suffix,
1768 /// because it is possible in certain clients (not clang at the moment) for
1769 /// two rounds of ThinLTO optimization and therefore promotion to occur.
1771 std::pair<StringRef, StringRef> Pair = Name.rsplit(".llvm.");
1772 return Pair.first;
1773 }
1774
1776
1777 /// Add a new module with the given \p Hash, mapped to the given \p
1778 /// ModID, and return a reference to the module.
1780 return &*ModulePathStringTable.insert({ModPath, Hash}).first;
1781 }
1782
1783 /// Return module entry for module with the given \p ModPath.
1785 auto It = ModulePathStringTable.find(ModPath);
1786 assert(It != ModulePathStringTable.end() && "Module not registered");
1787 return &*It;
1788 }
1789
1790 /// Return module entry for module with the given \p ModPath.
1791 const ModuleInfo *getModule(StringRef ModPath) const {
1792 auto It = ModulePathStringTable.find(ModPath);
1793 assert(It != ModulePathStringTable.end() && "Module not registered");
1794 return &*It;
1795 }
1796
1797 /// Check if the given Module has any functions available for exporting
1798 /// in the index. We consider any module present in the ModulePathStringTable
1799 /// to have exported functions.
1800 bool hasExportedFunctions(const Module &M) const {
1801 return ModulePathStringTable.count(M.getModuleIdentifier());
1802 }
1803
1804 const TypeIdSummaryMapTy &typeIds() const { return TypeIdMap; }
1805
1806 /// Return an existing or new TypeIdSummary entry for \p TypeId.
1807 /// This accessor can mutate the map and therefore should not be used in
1808 /// the ThinLTO backends.
1810 auto TidIter = TypeIdMap.equal_range(GlobalValue::getGUID(TypeId));
1811 for (auto It = TidIter.first; It != TidIter.second; ++It)
1812 if (It->second.first == TypeId)
1813 return It->second.second;
1814 auto It = TypeIdMap.insert(
1815 {GlobalValue::getGUID(TypeId), {std::string(TypeId), TypeIdSummary()}});
1816 return It->second.second;
1817 }
1818
1819 /// This returns either a pointer to the type id summary (if present in the
1820 /// summary map) or null (if not present). This may be used when importing.
1822 auto TidIter = TypeIdMap.equal_range(GlobalValue::getGUID(TypeId));
1823 for (auto It = TidIter.first; It != TidIter.second; ++It)
1824 if (It->second.first == TypeId)
1825 return &It->second.second;
1826 return nullptr;
1827 }
1828
1830 return const_cast<TypeIdSummary *>(
1831 static_cast<const ModuleSummaryIndex *>(this)->getTypeIdSummary(
1832 TypeId));
1833 }
1834
1835 const auto &typeIdCompatibleVtableMap() const {
1836 return TypeIdCompatibleVtableMap;
1837 }
1838
1839 /// Return an existing or new TypeIdCompatibleVtableMap entry for \p TypeId.
1840 /// This accessor can mutate the map and therefore should not be used in
1841 /// the ThinLTO backends.
1844 return TypeIdCompatibleVtableMap[std::string(TypeId)];
1845 }
1846
1847 /// For the given \p TypeId, this returns the TypeIdCompatibleVtableMap
1848 /// entry if present in the summary map. This may be used when importing.
1849 std::optional<TypeIdCompatibleVtableInfo>
1851 auto I = TypeIdCompatibleVtableMap.find(TypeId);
1852 if (I == TypeIdCompatibleVtableMap.end())
1853 return std::nullopt;
1854 return I->second;
1855 }
1856
1857 /// Collect for the given module the list of functions it defines
1858 /// (GUID -> Summary).
1860 GVSummaryMapTy &GVSummaryMap) const;
1861
1862 /// Collect for each module the list of Summaries it defines (GUID ->
1863 /// Summary).
1864 template <class Map>
1865 void
1866 collectDefinedGVSummariesPerModule(Map &ModuleToDefinedGVSummaries) const {
1867 for (const auto &GlobalList : *this) {
1868 auto GUID = GlobalList.first;
1869 for (const auto &Summary : GlobalList.second.SummaryList) {
1870 ModuleToDefinedGVSummaries[Summary->modulePath()][GUID] = Summary.get();
1871 }
1872 }
1873 }
1874
1875 /// Print to an output stream.
1876 void print(raw_ostream &OS, bool IsForDebug = false) const;
1877
1878 /// Dump to stderr (for debugging).
1879 void dump() const;
1880
1881 /// Export summary to dot file for GraphViz.
1882 void
1884 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) const;
1885
1886 /// Print out strongly connected components for debugging.
1887 void dumpSCCs(raw_ostream &OS);
1888
1889 /// Do the access attribute and DSOLocal propagation in combined index.
1891
1892 /// Checks if we can import global variable from another module.
1893 bool canImportGlobalVar(const GlobalValueSummary *S, bool AnalyzeRefs) const;
1894};
1895
1896/// GraphTraits definition to build SCC for the index
1897template <> struct GraphTraits<ValueInfo> {
1900
1902 return P.first;
1903 }
1906 decltype(&valueInfoFromEdge)>;
1907
1908 using ChildEdgeIteratorType = std::vector<FunctionSummary::EdgeTy>::iterator;
1909
1910 static NodeRef getEntryNode(ValueInfo V) { return V; }
1911
1913 if (!N.getSummaryList().size()) // handle external function
1914 return ChildIteratorType(
1915 FunctionSummary::ExternalNode.CallGraphEdgeList.begin(),
1916 &valueInfoFromEdge);
1918 cast<FunctionSummary>(N.getSummaryList().front()->getBaseObject());
1919 return ChildIteratorType(F->CallGraphEdgeList.begin(), &valueInfoFromEdge);
1920 }
1921
1923 if (!N.getSummaryList().size()) // handle external function
1924 return ChildIteratorType(
1925 FunctionSummary::ExternalNode.CallGraphEdgeList.end(),
1926 &valueInfoFromEdge);
1928 cast<FunctionSummary>(N.getSummaryList().front()->getBaseObject());
1929 return ChildIteratorType(F->CallGraphEdgeList.end(), &valueInfoFromEdge);
1930 }
1931
1933 if (!N.getSummaryList().size()) // handle external function
1934 return FunctionSummary::ExternalNode.CallGraphEdgeList.begin();
1935
1937 cast<FunctionSummary>(N.getSummaryList().front()->getBaseObject());
1938 return F->CallGraphEdgeList.begin();
1939 }
1940
1942 if (!N.getSummaryList().size()) // handle external function
1943 return FunctionSummary::ExternalNode.CallGraphEdgeList.end();
1944
1946 cast<FunctionSummary>(N.getSummaryList().front()->getBaseObject());
1947 return F->CallGraphEdgeList.end();
1948 }
1949
1950 static NodeRef edge_dest(EdgeRef E) { return E.first; }
1951};
1952
1953template <>
1956 std::unique_ptr<GlobalValueSummary> Root =
1957 std::make_unique<FunctionSummary>(I->calculateCallGraphRoot());
1958 GlobalValueSummaryInfo G(I->haveGVs());
1959 G.SummaryList.push_back(std::move(Root));
1960 static auto P =
1961 GlobalValueSummaryMapTy::value_type(GlobalValue::GUID(0), std::move(G));
1962 return ValueInfo(I->haveGVs(), &P);
1963 }
1964};
1965} // end namespace llvm
1966
1967#endif // LLVM_IR_MODULESUMMARYINDEX_H
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
This file defines the StringMap class.
This file defines the BumpPtrAllocator interface.
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
DXIL Finalize Linkage
Looks at all the uses of the given value Returns the Liveness deduced from the uses of this value Adds all uses that cause the result to be MaybeLive to MaybeLiveRetUses If the result is Live
This file defines the DenseMap class.
std::string Name
static const char * PreservedSymbols[]
Definition: IRSymtab.cpp:49
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
AllocType
Module.h This file contains the declarations for the Module class.
#define P(N)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
raw_pwrite_stream & OS
This file defines the SmallString class.
This file defines the SmallVector class.
This file contains some functions that are useful when dealing with strings.
ScaledNumber< uint64_t > Scaled64
Value * RHS
Alias summary information.
GlobalValue::GUID getAliaseeGUID() const
const GlobalValueSummary & getAliasee() const
ValueInfo getAliaseeVI() const
static bool classof(const GlobalValueSummary *GVS)
Check if this is an alias summary.
AliasSummary(GVFlags Flags)
GlobalValueSummary & getAliasee()
void setAliasee(ValueInfo &AliaseeVI, GlobalValueSummary *Aliasee)
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
Allocate memory in an ever growing pool, as if by bump-pointer.
Definition: Allocator.h:66
This class represents a range of values.
Definition: ConstantRange.h:47
This is an important base class in LLVM.
Definition: Constant.h:42
unsigned size() const
Definition: DenseMap.h:99
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:211
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
Function summary information to aid decisions and implementation of importing.
static FunctionSummary ExternalNode
A dummy node to reference external functions that aren't in the index.
void addCallsite(CallsiteInfo &Callsite)
FunctionSummary(GVFlags Flags, unsigned NumInsts, FFlags FunFlags, uint64_t EntryCount, std::vector< ValueInfo > Refs, std::vector< EdgeTy > CGEdges, std::vector< GlobalValue::GUID > TypeTests, std::vector< VFuncId > TypeTestAssumeVCalls, std::vector< VFuncId > TypeCheckedLoadVCalls, std::vector< ConstVCall > TypeTestAssumeConstVCalls, std::vector< ConstVCall > TypeCheckedLoadConstVCalls, std::vector< ParamAccess > Params, CallsitesTy CallsiteList, AllocsTy AllocList)
ArrayRef< VFuncId > type_test_assume_vcalls() const
Returns the list of virtual calls made by this function using llvm.assume(llvm.type....
ArrayRef< ConstVCall > type_test_assume_const_vcalls() const
Returns the list of virtual calls made by this function using llvm.assume(llvm.type....
static FunctionSummary makeDummyFunctionSummary(std::vector< FunctionSummary::EdgeTy > Edges)
Create an empty FunctionSummary (with specified call edges).
std::pair< ValueInfo, CalleeInfo > EdgeTy
<CalleeValueInfo, CalleeInfo> call edge pair.
std::pair< unsigned, unsigned > specialRefCounts() const
void setEntryCount(uint64_t EC)
Set the synthetic entry count for this function.
ArrayRef< AllocInfo > allocs() const
ArrayRef< CallsiteInfo > callsites() const
void addTypeTest(GlobalValue::GUID Guid)
Add a type test to the summary.
uint64_t entryCount() const
Get the synthetic entry count for this function.
std::vector< EdgeTy > & mutableCalls()
ArrayRef< VFuncId > type_checked_load_vcalls() const
Returns the list of virtual calls made by this function using llvm.type.checked.load intrinsics that ...
void setParamAccesses(std::vector< ParamAccess > NewParams)
Sets the list of known uses of pointer parameters.
unsigned instCount() const
Get the instruction count recorded for this function.
const TypeIdInfo * getTypeIdInfo() const
ArrayRef< ConstVCall > type_checked_load_const_vcalls() const
Returns the list of virtual calls made by this function using llvm.type.checked.load intrinsics with ...
ArrayRef< EdgeTy > calls() const
Return the list of <CalleeValueInfo, CalleeInfo> pairs.
ArrayRef< ParamAccess > paramAccesses() const
Returns the list of known uses of pointer parameters.
CallsitesTy & mutableCallsites()
ForceSummaryHotnessType
Types for -force-summary-edges-cold debugging option.
FFlags fflags() const
Get function summary flags.
ArrayRef< GlobalValue::GUID > type_tests() const
Returns the list of type identifiers used by this function in llvm.type.test intrinsics other than by...
static bool classof(const GlobalValueSummary *GVS)
Check if this is a function summary.
Function and variable summary information to aid decisions and implementation of importing.
SummaryKind
Sububclass discriminator (for dyn_cast<> et al.)
GVFlags flags() const
Get the flags for this GlobalValue (see struct GVFlags).
StringRef modulePath() const
Get the path to the module containing this function.
GlobalValueSummary * getBaseObject()
If this is an alias summary, returns the summary of the aliased object (a global variable or function...
SummaryKind getSummaryKind() const
Which kind of summary subclass this is.
GlobalValue::GUID getOriginalName() const
Returns the hash of the original name, it is identical to the GUID for externally visible symbols,...
GlobalValue::VisibilityTypes getVisibility() const
ArrayRef< ValueInfo > refs() const
Return the list of values referenced by this global value definition.
GlobalValueSummary(SummaryKind K, GVFlags Flags, std::vector< ValueInfo > Refs)
void setLinkage(GlobalValue::LinkageTypes Linkage)
Sets the linkage to the value determined by global summary-based optimization.
void setVisibility(GlobalValue::VisibilityTypes Vis)
virtual ~GlobalValueSummary()=default
GlobalValueSummary::ImportKind importType() const
void setModulePath(StringRef ModPath)
Set the path to the module containing this function, for use in the combined index.
void setNotEligibleToImport()
Flag that this global value cannot be imported.
void setCanAutoHide(bool CanAutoHide)
GlobalValue::LinkageTypes linkage() const
Return linkage type recorded for this global value.
bool notEligibleToImport() const
Return true if this global value can't be imported.
void setImportKind(ImportKind IK)
void setOriginalName(GlobalValue::GUID Name)
Initialize the original name hash in this summary.
static GUID getGUID(StringRef GlobalName)
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: Globals.cpp:75
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:595
VisibilityTypes
An enumeration for the kinds of visibility of global values.
Definition: GlobalValue.h:66
@ DefaultVisibility
The GV is visible.
Definition: GlobalValue.h:67
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:51
@ AvailableExternallyLinkage
Available for inspection, not emission.
Definition: GlobalValue.h:53
Global variable summary information to aid decisions and implementation of importing.
void setVCallVisibility(GlobalObject::VCallVisibility Vis)
struct llvm::GlobalVarSummary::GVarFlags VarFlags
GVarFlags varflags() const
ArrayRef< VirtFuncOffset > vTableFuncs() const
GlobalObject::VCallVisibility getVCallVisibility() const
static bool classof(const GlobalValueSummary *GVS)
Check if this is a global variable summary.
GlobalVarSummary(GVFlags Flags, GVarFlags VarFlags, std::vector< ValueInfo > Refs)
void setVTableFuncs(VTableFuncList Funcs)
Class to hold module path string table and global value map, and encapsulate methods for operating on...
std::set< std::string > & cfiFunctionDecls()
const std::set< std::string > & cfiFunctionDecls() const
TypeIdSummary & getOrInsertTypeIdSummary(StringRef TypeId)
Return an existing or new TypeIdSummary entry for TypeId.
std::optional< TypeIdCompatibleVtableInfo > getTypeIdCompatibleVtableSummary(StringRef TypeId) const
For the given TypeId, this returns the TypeIdCompatibleVtableMap entry if present in the summary map.
gvsummary_iterator end()
void addGlobalValueSummary(ValueInfo VI, std::unique_ptr< GlobalValueSummary > Summary)
Add a global value summary for the given ValueInfo.
ModulePathStringTableTy::value_type ModuleInfo
ValueInfo getOrInsertValueInfo(GlobalValue::GUID GUID)
Return a ValueInfo for GUID.
bool withGlobalValueDeadStripping() const
const std::set< std::string > & cfiFunctionDefs() const
static void discoverNodes(ValueInfo V, std::map< ValueInfo, bool > &FunctionHasParent)
Convenience function for doing a DFS on a ValueInfo.
StringRef saveString(StringRef String)
const TypeIdSummaryMapTy & typeIds() const
static StringRef getOriginalNameBeforePromote(StringRef Name)
Helper to obtain the unpromoted name for a global value (or the original name if not promoted).
const TypeIdSummary * getTypeIdSummary(StringRef TypeId) const
This returns either a pointer to the type id summary (if present in the summary map) or null (if not ...
bool isGUIDLive(GlobalValue::GUID GUID) const
gvsummary_iterator begin()
const_gvsummary_iterator end() const
bool isReadOnly(const GlobalVarSummary *GVS) const
void setFlags(uint64_t Flags)
const_gvsummary_iterator begin() const
bool isWriteOnly(const GlobalVarSummary *GVS) const
const std::vector< uint64_t > & stackIds() const
GlobalValueSummary * findSummaryInModule(GlobalValue::GUID ValueGUID, StringRef ModuleId) const
Find the summary for global GUID in module ModuleId, or nullptr if not found.
ValueInfo getValueInfo(const GlobalValueSummaryMapTy::value_type &R) const
Return a ValueInfo for the index value_type (convenient when iterating index).
const ModuleHash & getModuleHash(const StringRef ModPath) const
Get the module SHA1 hash recorded for the given module path.
static constexpr const char * getRegularLTOModuleName()
void addGlobalValueSummary(StringRef ValueName, std::unique_ptr< GlobalValueSummary > Summary)
Add a global value summary for a value of the given name.
ModuleSummaryIndex(bool HaveGVs, bool EnableSplitLTOUnit=false, bool UnifiedLTO=false)
void collectDefinedFunctionsForModule(StringRef ModulePath, GVSummaryMapTy &GVSummaryMap) const
Collect for the given module the list of functions it defines (GUID -> Summary).
const auto & typeIdCompatibleVtableMap() const
void dumpSCCs(raw_ostream &OS)
Print out strongly connected components for debugging.
bool isGlobalValueLive(const GlobalValueSummary *GVS) const
const ModuleInfo * getModule(StringRef ModPath) const
Return module entry for module with the given ModPath.
void propagateAttributes(const DenseSet< GlobalValue::GUID > &PreservedSymbols)
Do the access attribute and DSOLocal propagation in combined index.
const StringMap< ModuleHash > & modulePaths() const
Table of modules, containing module hash and id.
void dump() const
Dump to stderr (for debugging).
Definition: AsmWriter.cpp:5318
ModuleInfo * addModule(StringRef ModPath, ModuleHash Hash=ModuleHash{{0}})
Add a new module with the given Hash, mapped to the given ModID, and return a reference to the module...
void collectDefinedGVSummariesPerModule(Map &ModuleToDefinedGVSummaries) const
Collect for each module the list of Summaries it defines (GUID -> Summary).
void addGlobalValueSummary(const GlobalValue &GV, std::unique_ptr< GlobalValueSummary > Summary)
Add a global value summary for a value.
bool hasExportedFunctions(const Module &M) const
Check if the given Module has any functions available for exporting in the index.
static std::string getGlobalNameForLocal(StringRef Name, ModuleHash ModHash)
Convenience method for creating a promoted global name for the given value name of a local,...
static constexpr uint64_t BitcodeSummaryVersion
void exportToDot(raw_ostream &OS, const DenseSet< GlobalValue::GUID > &GUIDPreservedSymbols) const
Export summary to dot file for GraphViz.
uint64_t getStackIdAtIndex(unsigned Index) const
StringMap< ModuleHash > & modulePaths()
Table of modules, containing hash and id.
void print(raw_ostream &OS, bool IsForDebug=false) const
Print to an output stream.
Definition: AsmWriter.cpp:5245
bool skipModuleByDistributedBackend() const
ValueInfo getOrInsertValueInfo(const GlobalValue *GV)
Return a ValueInfo for GV and mark it as belonging to GV.
GlobalValueSummary * findSummaryInModule(ValueInfo VI, StringRef ModuleId) const
Find the summary for ValueInfo VI in module ModuleId, or nullptr if not found.
ValueInfo getValueInfo(GlobalValue::GUID GUID) const
Return a ValueInfo for GUID if it exists, otherwise return ValueInfo().
unsigned addOrGetStackIdIndex(uint64_t StackId)
GlobalValue::GUID getGUIDFromOriginalID(GlobalValue::GUID OriginalID) const
Return the GUID for OriginalId in the OidGuidMap.
GlobalValueSummary * getGlobalValueSummary(const GlobalValue &GV, bool PerModuleIndex=true) const
Returns the first GlobalValueSummary for GV, asserting that there is only one if PerModuleIndex.
std::set< std::string > & cfiFunctionDefs()
ModuleInfo * getModule(StringRef ModPath)
Return module entry for module with the given ModPath.
ValueInfo getOrInsertValueInfo(GlobalValue::GUID GUID, StringRef Name)
Return a ValueInfo for GUID setting value Name.
bool canImportGlobalVar(const GlobalValueSummary *S, bool AnalyzeRefs) const
Checks if we can import global variable from another module.
static std::string getGlobalNameForLocal(StringRef Name, StringRef Suffix)
void addOriginalName(GlobalValue::GUID ValueGUID, GlobalValue::GUID OrigGUID)
Add an original name for the value of the given GUID.
FunctionSummary calculateCallGraphRoot()
TypeIdSummary * getTypeIdSummary(StringRef TypeId)
TypeIdCompatibleVtableInfo & getOrInsertTypeIdCompatibleVtableSummary(StringRef TypeId)
Return an existing or new TypeIdCompatibleVtableMap entry for TypeId.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
PointerIntPair - This class implements a pair of a pointer and small integer.
void setPointer(PointerTy PtrVal) &
IntType getInt() const
void setInt(IntType IntVal) &
PointerTy getPointer() const
Simple representation of a scaled number.
Definition: ScaledNumber.h:493
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition: SmallString.h:26
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StringMapEntry - This is used to represent one value that is inserted into a StringMap.
iterator end()
Definition: StringMap.h:220
iterator find(StringRef Key)
Definition: StringMap.h:233
size_type count(StringRef Key) const
count - Return 1 if the element is in the map, 0 otherwise.
Definition: StringMap.h:276
bool insert(MapEntryTy *KeyValue)
insert - Insert the specified key/value pair into the map.
Definition: StringMap.h:308
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Saves strings in the provided stable storage and returns a StringRef with a stable character pointer.
Definition: StringSaver.h:21
StringRef save(const char *S)
Definition: StringSaver.h:30
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
bool hasName() const
Definition: Value.h:261
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:661
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Cold
Attempts to make code in the caller as efficient as possible under the assumption that the call is no...
Definition: CallingConv.h:47
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
GlobalValueSummaryMapTy::iterator gvsummary_iterator
@ Offset
Definition: DWP.cpp:480
bool operator<(int64_t V1, const APSInt &V2)
Definition: APSInt.h:361
const char * getHotnessName(CalleeInfo::HotnessType HT)
bool operator!=(uint64_t V1, const APInt &V2)
Definition: APInt.h:2052
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
std::vector< VirtFuncOffset > VTableFuncList
List of functions referenced by a particular vtable definition.
@ None
Definition: CodeGenData.h:101
std::vector< std::unique_ptr< GlobalValueSummary > > GlobalValueSummaryList
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
std::unordered_set< GlobalValueSummary * > GVSummaryPtrSet
A set of global value summary pointers.
GlobalValueSummaryMapTy::const_iterator const_gvsummary_iterator
Type used for iterating through the global value summary map.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
Definition: APFixedPoint.h:292
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1856
std::multimap< GlobalValue::GUID, std::pair< std::string, TypeIdSummary > > TypeIdSummaryMapTy
Map of a type GUID to type id string and summary (multimap used in case of GUID conflicts).
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1749
std::array< uint32_t, 5 > ModuleHash
160 bits SHA1
std::map< GlobalValue::GUID, GlobalValueSummaryInfo > GlobalValueSummaryMapTy
Map from global value GUID to corresponding summary structures.
std::vector< TypeIdOffsetVtableInfo > TypeIdCompatibleVtableInfo
List of vtable definitions decorated by a particular type identifier, and their corresponding offsets...
Implement std::hash so that hash_code can be used in STL containers.
Definition: BitVector.h:858
#define N
Summary of memprof metadata on allocations.
AllocInfo(std::vector< MIBInfo > MIBs)
AllocInfo(SmallVector< uint8_t > Versions, std::vector< MIBInfo > MIBs)
std::vector< uint64_t > TotalSizes
SmallVector< uint8_t > Versions
std::vector< MIBInfo > MIBs
Class to accumulate and hold information about a callee.
static constexpr uint64_t MaxRelBlockFreq
bool hasTailCall() const
void updateHotness(const HotnessType OtherHotness)
CalleeInfo(HotnessType Hotness, bool HasTC, uint64_t RelBF)
HotnessType getHotness() const
static constexpr int32_t ScaleShift
void setHasTailCall(const bool HasTC)
void updateRelBlockFreq(uint64_t BlockFreq, uint64_t EntryFreq)
Update RelBlockFreq from BlockFreq and EntryFreq.
static constexpr unsigned RelBlockFreqBits
The value stored in RelBlockFreq has to be interpreted as the digits of a scaled number with a scale ...
Summary of memprof callsite metadata.
SmallVector< unsigned > StackIdIndices
SmallVector< unsigned > Clones
CallsiteInfo(ValueInfo Callee, SmallVector< unsigned > StackIdIndices)
CallsiteInfo(ValueInfo Callee, SmallVector< unsigned > Clones, SmallVector< unsigned > StackIdIndices)
static FunctionSummary::ConstVCall getEmptyKey()
static FunctionSummary::ConstVCall getTombstoneKey()
static unsigned getHashValue(FunctionSummary::ConstVCall I)
static bool isEqual(FunctionSummary::ConstVCall L, FunctionSummary::ConstVCall R)
static FunctionSummary::VFuncId getEmptyKey()
static bool isEqual(FunctionSummary::VFuncId L, FunctionSummary::VFuncId R)
static FunctionSummary::VFuncId getTombstoneKey()
static unsigned getHashValue(FunctionSummary::VFuncId I)
static bool isEqual(ValueInfo L, ValueInfo R)
static bool isSpecialKey(ValueInfo V)
static unsigned getHashValue(ValueInfo I)
An information struct used to provide DenseMap with the various necessary components for a given valu...
Definition: DenseMapInfo.h:52
A specification for a virtual function call with all constant integer arguments.
Flags specific to function summaries.
FFlags & operator&=(const FFlags &RHS)
Describes the use of a value in a call instruction, specifying the call's target, the value's paramet...
Call(uint64_t ParamNo, ValueInfo Callee, const ConstantRange &Offsets)
Describes the uses of a parameter by the function.
ParamAccess(uint64_t ParamNo, const ConstantRange &Use)
std::vector< Call > Calls
In the per-module summary, it summarizes the byte offset applied to each pointer parameter before pas...
static constexpr uint32_t RangeWidth
All type identifier related information.
std::vector< ConstVCall > TypeCheckedLoadConstVCalls
std::vector< VFuncId > TypeCheckedLoadVCalls
std::vector< ConstVCall > TypeTestAssumeConstVCalls
List of virtual calls made by this function using (respectively) llvm.assume(llvm....
std::vector< GlobalValue::GUID > TypeTests
List of type identifiers used by this function in llvm.type.test intrinsics referenced by something o...
std::vector< VFuncId > TypeTestAssumeVCalls
List of virtual calls made by this function using (respectively) llvm.assume(llvm....
An "identifier" for a virtual function.
GlobalValueSummaryList SummaryList
List of global value summary structures for a particular value held in the GlobalValueMap.
union llvm::GlobalValueSummaryInfo::NameOrGV U
Group flags (Linkage, NotEligibleToImport, etc.) as a bitfield.
GVFlags(GlobalValue::LinkageTypes Linkage, GlobalValue::VisibilityTypes Visibility, bool NotEligibleToImport, bool Live, bool IsLocal, bool CanAutoHide, ImportKind ImportType)
Convenience Constructors.
unsigned DSOLocal
Indicates that the linker resolved the symbol to a definition from within the same linkage unit.
unsigned CanAutoHide
In the per-module summary, indicates that the global value is linkonce_odr and global unnamed addr (s...
unsigned ImportType
This field is written by the ThinLTO indexing step to postlink combined summary.
unsigned NotEligibleToImport
Indicate if the global value cannot be imported (e.g.
unsigned Linkage
The linkage type of the associated global value.
unsigned Visibility
Indicates the visibility.
unsigned Live
In per-module summary, indicate that the global value must be considered a live root for index-based ...
GVarFlags(bool ReadOnly, bool WriteOnly, bool Constant, GlobalObject::VCallVisibility Vis)
static NodeRef getEntryNode(ModuleSummaryIndex *I)
static NodeRef valueInfoFromEdge(FunctionSummary::EdgeTy &P)
static ChildIteratorType child_begin(NodeRef N)
static ChildEdgeIteratorType child_edge_begin(NodeRef N)
static NodeRef edge_dest(EdgeRef E)
std::vector< FunctionSummary::EdgeTy >::iterator ChildEdgeIteratorType
static NodeRef getEntryNode(ValueInfo V)
static ChildIteratorType child_end(NodeRef N)
static ChildEdgeIteratorType child_edge_end(NodeRef N)
FunctionSummary::EdgeTy & EdgeRef
Summary of a single MIB in a memprof metadata on allocations.
MIBInfo(AllocationType AllocType, SmallVector< unsigned > StackIdIndices)
AllocationType AllocType
SmallVector< unsigned > StackIdIndices
The following data structures summarize type metadata information.
TypeIdOffsetVtableInfo(uint64_t Offset, ValueInfo VI)
std::map< uint64_t, WholeProgramDevirtResolution > WPDRes
Mapping from byte offset to whole-program devirt resolution for that (typeid, byte offset) pair.
TypeTestResolution TTRes
Kind
Specifies which kind of type check we should emit for this byte array.
@ Unknown
Unknown (analysis not performed, don't lower)
@ Single
Single element (last example in "Short Inline Bit Vectors")
@ Inline
Inlined bit vector ("Short Inline Bit Vectors")
@ Unsat
Unsatisfiable type (i.e. no global has this type metadata)
@ AllOnes
All-ones bit vector ("Eliminating Bit Vector Checks for All-Ones Bit Vectors")
@ ByteArray
Test a byte array (first example)
unsigned SizeM1BitWidth
Range of size-1 expressed as a bit width.
enum llvm::TypeTestResolution::Kind TheKind
Struct that holds a reference to a particular GUID in a global value summary.
PointerIntPair< const GlobalValueSummaryMapTy::value_type *, 3, int > RefAndFlags
GlobalValue::VisibilityTypes getELFVisibility() const
Returns the most constraining visibility among summaries.
bool isValidAccessSpecifier() const
const GlobalValueSummaryMapTy::value_type * getRef() const
ArrayRef< std::unique_ptr< GlobalValueSummary > > getSummaryList() const
StringRef name() const
bool isWriteOnly() const
const GlobalValue * getValue() const
ValueInfo(bool HaveGVs, const GlobalValueSummaryMapTy::value_type *R)
bool isReadOnly() const
bool canAutoHide() const
Checks if all copies are eligible for auto-hiding (have flag set).
unsigned getAccessSpecifier() const
ValueInfo()=default
bool isDSOLocal(bool WithDSOLocalPropagation=false) const
Checks if all summaries are DSO local (have the flag set).
GlobalValue::GUID getGUID() const
bool haveGVs() const
The ValueInfo and offset for a function within a vtable definition initializer array.
VirtFuncOffset(ValueInfo VI, uint64_t Offset)
@ UniformRetVal
Uniform return value optimization.
@ VirtualConstProp
Virtual constant propagation.
@ UniqueRetVal
Unique return value optimization.
@ Indir
Just do a regular virtual call.
uint64_t Info
Additional information for the resolution:
enum llvm::WholeProgramDevirtResolution::ByArg::Kind TheKind
enum llvm::WholeProgramDevirtResolution::Kind TheKind
std::map< std::vector< uint64_t >, ByArg > ResByArg
Resolutions for calls with all constant integer arguments (excluding the first argument,...
@ SingleImpl
Single implementation devirtualization.
@ Indir
Just do a regular virtual call.
@ BranchFunnel
When retpoline mitigation is enabled, use a branch funnel that is defined in the merged module.
const GlobalValue * GV
The GlobalValue corresponding to this summary.
StringRef Name
Summary string representation.