LLVM 20.0.0git
ModuleSummaryAnalysis.cpp
Go to the documentation of this file.
1//===- ModuleSummaryAnalysis.cpp - Module summary index builder -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass builds a ModuleSummaryIndex object for the module, to be written
10// to bitcode or LLVM assembly.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/DenseSet.h"
17#include "llvm/ADT/MapVector.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/SetVector.h"
22#include "llvm/ADT/StringRef.h"
32#include "llvm/IR/Attributes.h"
33#include "llvm/IR/BasicBlock.h"
34#include "llvm/IR/Constant.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/Dominators.h"
37#include "llvm/IR/Function.h"
38#include "llvm/IR/GlobalAlias.h"
39#include "llvm/IR/GlobalValue.h"
43#include "llvm/IR/Metadata.h"
44#include "llvm/IR/Module.h"
46#include "llvm/IR/Use.h"
47#include "llvm/IR/User.h"
51#include "llvm/Pass.h"
55#include <algorithm>
56#include <cassert>
57#include <cstdint>
58#include <vector>
59
60using namespace llvm;
61using namespace llvm::memprof;
62
63#define DEBUG_TYPE "module-summary-analysis"
64
65// Option to force edges cold which will block importing when the
66// -import-cold-multiplier is set to 0. Useful for debugging.
67namespace llvm {
70} // namespace llvm
71
73 "force-summary-edges-cold", cl::Hidden, cl::location(ForceSummaryEdgesCold),
74 cl::desc("Force all edges in the function summary to cold"),
77 "all-non-critical", "All non-critical edges."),
78 clEnumValN(FunctionSummary::FSHT_All, "all", "All edges.")));
79
81 "module-summary-dot-file", cl::Hidden, cl::value_desc("filename"),
82 cl::desc("File to emit dot graph of new summary into"));
83
85
87
89
90// Walk through the operands of a given User via worklist iteration and populate
91// the set of GlobalValue references encountered. Invoked either on an
92// Instruction or a GlobalVariable (which walks its initializer).
93// Return true if any of the operands contains blockaddress. This is important
94// to know when computing summary for global var, because if global variable
95// references basic block address we can't import it separately from function
96// containing that basic block. For simplicity we currently don't import such
97// global vars at all. When importing function we aren't interested if any
98// instruction in it takes an address of any basic block, because instruction
99// can only take an address of basic block located in the same function.
100// Set `RefLocalLinkageIFunc` to true if the analyzed value references a
101// local-linkage ifunc.
102static bool findRefEdges(ModuleSummaryIndex &Index, const User *CurUser,
103 SetVector<ValueInfo, std::vector<ValueInfo>> &RefEdges,
105 bool &RefLocalLinkageIFunc) {
106 bool HasBlockAddress = false;
108 if (Visited.insert(CurUser).second)
109 Worklist.push_back(CurUser);
110
111 while (!Worklist.empty()) {
112 const User *U = Worklist.pop_back_val();
113 const auto *CB = dyn_cast<CallBase>(U);
114
115 for (const auto &OI : U->operands()) {
116 const User *Operand = dyn_cast<User>(OI);
117 if (!Operand)
118 continue;
119 if (isa<BlockAddress>(Operand)) {
120 HasBlockAddress = true;
121 continue;
122 }
123 if (auto *GV = dyn_cast<GlobalValue>(Operand)) {
124 // We have a reference to a global value. This should be added to
125 // the reference set unless it is a callee. Callees are handled
126 // specially by WriteFunction and are added to a separate list.
127 if (!(CB && CB->isCallee(&OI))) {
128 // If an ifunc has local linkage, do not add it into ref edges, and
129 // sets `RefLocalLinkageIFunc` to true. The referencer is not eligible
130 // for import. An ifunc doesn't have summary and ThinLTO cannot
131 // promote it; importing the referencer may cause linkage errors.
132 if (auto *GI = dyn_cast_if_present<GlobalIFunc>(GV);
133 GI && GI->hasLocalLinkage()) {
134 RefLocalLinkageIFunc = true;
135 continue;
136 }
137 RefEdges.insert(Index.getOrInsertValueInfo(GV));
138 }
139 continue;
140 }
141 if (Visited.insert(Operand).second)
142 Worklist.push_back(Operand);
143 }
144 }
145
146 const Instruction *I = dyn_cast<Instruction>(CurUser);
147 if (I) {
148 uint64_t TotalCount = 0;
149 // MaxNumVTableAnnotations is the maximum number of vtables annotated on
150 // the instruction.
151 auto ValueDataArray = getValueProfDataFromInst(
152 *I, IPVK_VTableTarget, MaxNumVTableAnnotations, TotalCount);
153
154 for (const auto &V : ValueDataArray)
155 RefEdges.insert(Index.getOrInsertValueInfo(/* VTableGUID = */
156 V.Value));
157 }
158 return HasBlockAddress;
159}
160
162 ProfileSummaryInfo *PSI) {
163 if (!PSI)
164 return CalleeInfo::HotnessType::Unknown;
165 if (PSI->isHotCount(ProfileCount))
166 return CalleeInfo::HotnessType::Hot;
167 if (PSI->isColdCount(ProfileCount))
168 return CalleeInfo::HotnessType::Cold;
169 return CalleeInfo::HotnessType::None;
170}
171
172static bool isNonRenamableLocal(const GlobalValue &GV) {
173 return GV.hasSection() && GV.hasLocalLinkage();
174}
175
176/// Determine whether this call has all constant integer arguments (excluding
177/// "this") and summarize it to VCalls or ConstVCalls as appropriate.
178static void addVCallToSet(
180 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
181 &VCalls,
183 std::vector<FunctionSummary::ConstVCall>> &ConstVCalls) {
184 std::vector<uint64_t> Args;
185 // Start from the second argument to skip the "this" pointer.
186 for (auto &Arg : drop_begin(Call.CB.args())) {
187 auto *CI = dyn_cast<ConstantInt>(Arg);
188 if (!CI || CI->getBitWidth() > 64) {
189 VCalls.insert({Guid, Call.Offset});
190 return;
191 }
192 Args.push_back(CI->getZExtValue());
193 }
194 ConstVCalls.insert({{Guid, Call.Offset}, std::move(Args)});
195}
196
197/// If this intrinsic call requires that we add information to the function
198/// summary, do so via the non-constant reference arguments.
200 const CallInst *CI,
201 SetVector<GlobalValue::GUID, std::vector<GlobalValue::GUID>> &TypeTests,
202 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
203 &TypeTestAssumeVCalls,
204 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
205 &TypeCheckedLoadVCalls,
207 std::vector<FunctionSummary::ConstVCall>>
208 &TypeTestAssumeConstVCalls,
210 std::vector<FunctionSummary::ConstVCall>>
211 &TypeCheckedLoadConstVCalls,
212 DominatorTree &DT) {
213 switch (CI->getCalledFunction()->getIntrinsicID()) {
214 case Intrinsic::type_test:
215 case Intrinsic::public_type_test: {
216 auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(1));
217 auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
218 if (!TypeId)
219 break;
220 GlobalValue::GUID Guid = GlobalValue::getGUID(TypeId->getString());
221
222 // Produce a summary from type.test intrinsics. We only summarize type.test
223 // intrinsics that are used other than by an llvm.assume intrinsic.
224 // Intrinsics that are assumed are relevant only to the devirtualization
225 // pass, not the type test lowering pass.
226 bool HasNonAssumeUses = llvm::any_of(CI->uses(), [](const Use &CIU) {
227 return !isa<AssumeInst>(CIU.getUser());
228 });
229 if (HasNonAssumeUses)
230 TypeTests.insert(Guid);
231
234 findDevirtualizableCallsForTypeTest(DevirtCalls, Assumes, CI, DT);
235 for (auto &Call : DevirtCalls)
236 addVCallToSet(Call, Guid, TypeTestAssumeVCalls,
237 TypeTestAssumeConstVCalls);
238
239 break;
240 }
241
242 case Intrinsic::type_checked_load_relative:
243 case Intrinsic::type_checked_load: {
244 auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(2));
245 auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
246 if (!TypeId)
247 break;
248 GlobalValue::GUID Guid = GlobalValue::getGUID(TypeId->getString());
249
253 bool HasNonCallUses = false;
254 findDevirtualizableCallsForTypeCheckedLoad(DevirtCalls, LoadedPtrs, Preds,
255 HasNonCallUses, CI, DT);
256 // Any non-call uses of the result of llvm.type.checked.load will
257 // prevent us from optimizing away the llvm.type.test.
258 if (HasNonCallUses)
259 TypeTests.insert(Guid);
260 for (auto &Call : DevirtCalls)
261 addVCallToSet(Call, Guid, TypeCheckedLoadVCalls,
262 TypeCheckedLoadConstVCalls);
263
264 break;
265 }
266 default:
267 break;
268 }
269}
270
271static bool isNonVolatileLoad(const Instruction *I) {
272 if (const auto *LI = dyn_cast<LoadInst>(I))
273 return !LI->isVolatile();
274
275 return false;
276}
277
278static bool isNonVolatileStore(const Instruction *I) {
279 if (const auto *SI = dyn_cast<StoreInst>(I))
280 return !SI->isVolatile();
281
282 return false;
283}
284
285// Returns true if the function definition must be unreachable.
286//
287// Note if this helper function returns true, `F` is guaranteed
288// to be unreachable; if it returns false, `F` might still
289// be unreachable but not covered by this helper function.
291 // A function must be unreachable if its entry block ends with an
292 // 'unreachable'.
293 assert(!F.isDeclaration());
294 return isa<UnreachableInst>(F.getEntryBlock().getTerminator());
295}
296
298 ModuleSummaryIndex &Index, const Module &M, const Function &F,
300 bool HasLocalsInUsedOrAsm, DenseSet<GlobalValue::GUID> &CantBePromoted,
301 bool IsThinLTO,
302 std::function<const StackSafetyInfo *(const Function &F)> GetSSICallback) {
303 // Summary not currently supported for anonymous functions, they should
304 // have been named.
305 assert(F.hasName());
306
307 unsigned NumInsts = 0;
308 // Map from callee ValueId to profile count. Used to accumulate profile
309 // counts for all static calls to a given callee.
311 std::vector<std::pair<ValueInfo, CalleeInfo>>>
312 CallGraphEdges;
313 SetVector<ValueInfo, std::vector<ValueInfo>> RefEdges, LoadRefEdges,
314 StoreRefEdges;
317 TypeTestAssumeVCalls, TypeCheckedLoadVCalls;
319 std::vector<FunctionSummary::ConstVCall>>
320 TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls;
321 ICallPromotionAnalysis ICallAnalysis;
323
324 // Add personality function, prefix data and prologue data to function's ref
325 // list.
326 bool HasLocalIFuncCallOrRef = false;
327 findRefEdges(Index, &F, RefEdges, Visited, HasLocalIFuncCallOrRef);
328 std::vector<const Instruction *> NonVolatileLoads;
329 std::vector<const Instruction *> NonVolatileStores;
330
331 std::vector<CallsiteInfo> Callsites;
332 std::vector<AllocInfo> Allocs;
333
334#ifndef NDEBUG
335 DenseSet<const CallBase *> CallsThatMayHaveMemprofSummary;
336#endif
337
338 bool HasInlineAsmMaybeReferencingInternal = false;
339 bool HasIndirBranchToBlockAddress = false;
340 bool HasUnknownCall = false;
341 bool MayThrow = false;
342 for (const BasicBlock &BB : F) {
343 // We don't allow inlining of function with indirect branch to blockaddress.
344 // If the blockaddress escapes the function, e.g., via a global variable,
345 // inlining may lead to an invalid cross-function reference. So we shouldn't
346 // import such function either.
347 if (BB.hasAddressTaken()) {
348 for (User *U : BlockAddress::get(const_cast<BasicBlock *>(&BB))->users())
349 if (!isa<CallBrInst>(*U)) {
350 HasIndirBranchToBlockAddress = true;
351 break;
352 }
353 }
354
355 for (const Instruction &I : BB) {
356 if (I.isDebugOrPseudoInst())
357 continue;
358 ++NumInsts;
359
360 // Regular LTO module doesn't participate in ThinLTO import,
361 // so no reference from it can be read/writeonly, since this
362 // would require importing variable as local copy
363 if (IsThinLTO) {
364 if (isNonVolatileLoad(&I)) {
365 // Postpone processing of non-volatile load instructions
366 // See comments below
367 Visited.insert(&I);
368 NonVolatileLoads.push_back(&I);
369 continue;
370 } else if (isNonVolatileStore(&I)) {
371 Visited.insert(&I);
372 NonVolatileStores.push_back(&I);
373 // All references from second operand of store (destination address)
374 // can be considered write-only if they're not referenced by any
375 // non-store instruction. References from first operand of store
376 // (stored value) can't be treated either as read- or as write-only
377 // so we add them to RefEdges as we do with all other instructions
378 // except non-volatile load.
379 Value *Stored = I.getOperand(0);
380 if (auto *GV = dyn_cast<GlobalValue>(Stored))
381 // findRefEdges will try to examine GV operands, so instead
382 // of calling it we should add GV to RefEdges directly.
383 RefEdges.insert(Index.getOrInsertValueInfo(GV));
384 else if (auto *U = dyn_cast<User>(Stored))
385 findRefEdges(Index, U, RefEdges, Visited, HasLocalIFuncCallOrRef);
386 continue;
387 }
388 }
389 findRefEdges(Index, &I, RefEdges, Visited, HasLocalIFuncCallOrRef);
390 const auto *CB = dyn_cast<CallBase>(&I);
391 if (!CB) {
392 if (I.mayThrow())
393 MayThrow = true;
394 continue;
395 }
396
397 const auto *CI = dyn_cast<CallInst>(&I);
398 // Since we don't know exactly which local values are referenced in inline
399 // assembly, conservatively mark the function as possibly referencing
400 // a local value from inline assembly to ensure we don't export a
401 // reference (which would require renaming and promotion of the
402 // referenced value).
403 if (HasLocalsInUsedOrAsm && CI && CI->isInlineAsm())
404 HasInlineAsmMaybeReferencingInternal = true;
405
406 auto *CalledValue = CB->getCalledOperand();
407 auto *CalledFunction = CB->getCalledFunction();
408 if (CalledValue && !CalledFunction) {
409 CalledValue = CalledValue->stripPointerCasts();
410 // Stripping pointer casts can reveal a called function.
411 CalledFunction = dyn_cast<Function>(CalledValue);
412 }
413 // Check if this is an alias to a function. If so, get the
414 // called aliasee for the checks below.
415 if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
416 assert(!CalledFunction && "Expected null called function in callsite for alias");
417 CalledFunction = dyn_cast<Function>(GA->getAliaseeObject());
418 }
419 // Check if this is a direct call to a known function or a known
420 // intrinsic, or an indirect call with profile data.
421 if (CalledFunction) {
422 if (CI && CalledFunction->isIntrinsic()) {
424 CI, TypeTests, TypeTestAssumeVCalls, TypeCheckedLoadVCalls,
425 TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls, DT);
426 continue;
427 }
428 // We should have named any anonymous globals
429 assert(CalledFunction->hasName());
430 auto ScaledCount = PSI->getProfileCount(*CB, BFI);
431 auto Hotness = ScaledCount ? getHotness(*ScaledCount, PSI)
432 : CalleeInfo::HotnessType::Unknown;
434 Hotness = CalleeInfo::HotnessType::Cold;
435
436 // Use the original CalledValue, in case it was an alias. We want
437 // to record the call edge to the alias in that case. Eventually
438 // an alias summary will be created to associate the alias and
439 // aliasee.
440 auto &ValueInfo = CallGraphEdges[Index.getOrInsertValueInfo(
441 cast<GlobalValue>(CalledValue))];
442 ValueInfo.updateHotness(Hotness);
443 if (CB->isTailCall())
444 ValueInfo.setHasTailCall(true);
445 // Add the relative block frequency to CalleeInfo if there is no profile
446 // information.
447 if (BFI != nullptr && Hotness == CalleeInfo::HotnessType::Unknown) {
448 uint64_t BBFreq = BFI->getBlockFreq(&BB).getFrequency();
449 uint64_t EntryFreq = BFI->getEntryFreq().getFrequency();
450 ValueInfo.updateRelBlockFreq(BBFreq, EntryFreq);
451 }
452 } else {
453 HasUnknownCall = true;
454 // If F is imported, a local linkage ifunc (e.g. target_clones on a
455 // static function) called by F will be cloned. Since summaries don't
456 // track ifunc, we do not know implementation functions referenced by
457 // the ifunc resolver need to be promoted in the exporter, and we will
458 // get linker errors due to cloned declarations for implementation
459 // functions. As a simple fix, just mark F as not eligible for import.
460 // Non-local ifunc is not cloned and does not have the issue.
461 if (auto *GI = dyn_cast_if_present<GlobalIFunc>(CalledValue))
462 if (GI->hasLocalLinkage())
463 HasLocalIFuncCallOrRef = true;
464 // Skip inline assembly calls.
465 if (CI && CI->isInlineAsm())
466 continue;
467 // Skip direct calls.
468 if (!CalledValue || isa<Constant>(CalledValue))
469 continue;
470
471 // Check if the instruction has a callees metadata. If so, add callees
472 // to CallGraphEdges to reflect the references from the metadata, and
473 // to enable importing for subsequent indirect call promotion and
474 // inlining.
475 if (auto *MD = I.getMetadata(LLVMContext::MD_callees)) {
476 for (const auto &Op : MD->operands()) {
477 Function *Callee = mdconst::extract_or_null<Function>(Op);
478 if (Callee)
479 CallGraphEdges[Index.getOrInsertValueInfo(Callee)];
480 }
481 }
482
483 uint32_t NumCandidates;
484 uint64_t TotalCount;
485 auto CandidateProfileData =
486 ICallAnalysis.getPromotionCandidatesForInstruction(&I, TotalCount,
487 NumCandidates);
488 for (const auto &Candidate : CandidateProfileData)
489 CallGraphEdges[Index.getOrInsertValueInfo(Candidate.Value)]
490 .updateHotness(getHotness(Candidate.Count, PSI));
491 }
492
493 // Summarize memprof related metadata. This is only needed for ThinLTO.
494 if (!IsThinLTO)
495 continue;
496
497 // TODO: Skip indirect calls for now. Need to handle these better, likely
498 // by creating multiple Callsites, one per target, then speculatively
499 // devirtualize while applying clone info in the ThinLTO backends. This
500 // will also be important because we will have a different set of clone
501 // versions per target. This handling needs to match that in the ThinLTO
502 // backend so we handle things consistently for matching of callsite
503 // summaries to instructions.
504 if (!CalledFunction)
505 continue;
506
507 // Ensure we keep this analysis in sync with the handling in the ThinLTO
508 // backend (see MemProfContextDisambiguation::applyImport). Save this call
509 // so that we can skip it in checking the reverse case later.
511#ifndef NDEBUG
512 CallsThatMayHaveMemprofSummary.insert(CB);
513#endif
514
515 // Compute the list of stack ids first (so we can trim them from the stack
516 // ids on any MIBs).
518 I.getMetadata(LLVMContext::MD_callsite));
519 auto *MemProfMD = I.getMetadata(LLVMContext::MD_memprof);
520 if (MemProfMD) {
521 std::vector<MIBInfo> MIBs;
522 std::vector<uint64_t> TotalSizes;
523 for (auto &MDOp : MemProfMD->operands()) {
524 auto *MIBMD = cast<const MDNode>(MDOp);
527 SmallVector<unsigned> StackIdIndices;
529 // Collapse out any on the allocation call (inlining).
530 for (auto ContextIter =
531 StackContext.beginAfterSharedPrefix(InstCallsite);
532 ContextIter != StackContext.end(); ++ContextIter) {
533 unsigned StackIdIdx = Index.addOrGetStackIdIndex(*ContextIter);
534 // If this is a direct recursion, simply skip the duplicate
535 // entries. If this is mutual recursion, handling is left to
536 // the LTO link analysis client.
537 if (StackIdIndices.empty() || StackIdIndices.back() != StackIdIdx)
538 StackIdIndices.push_back(StackIdIdx);
539 }
540 MIBs.push_back(
541 MIBInfo(getMIBAllocType(MIBMD), std::move(StackIdIndices)));
543 auto TotalSize = getMIBTotalSize(MIBMD);
544 assert(TotalSize);
545 TotalSizes.push_back(TotalSize);
546 }
547 }
548 Allocs.push_back(AllocInfo(std::move(MIBs)));
550 assert(Allocs.back().MIBs.size() == TotalSizes.size());
551 Allocs.back().TotalSizes = std::move(TotalSizes);
552 }
553 } else if (!InstCallsite.empty()) {
554 SmallVector<unsigned> StackIdIndices;
555 for (auto StackId : InstCallsite)
556 StackIdIndices.push_back(Index.addOrGetStackIdIndex(StackId));
557 // Use the original CalledValue, in case it was an alias. We want
558 // to record the call edge to the alias in that case. Eventually
559 // an alias summary will be created to associate the alias and
560 // aliasee.
561 auto CalleeValueInfo =
562 Index.getOrInsertValueInfo(cast<GlobalValue>(CalledValue));
563 Callsites.push_back({CalleeValueInfo, StackIdIndices});
564 }
565 }
566 }
567
569 Index.addBlockCount(F.size());
570
571 std::vector<ValueInfo> Refs;
572 if (IsThinLTO) {
573 auto AddRefEdges = [&](const std::vector<const Instruction *> &Instrs,
576 for (const auto *I : Instrs) {
577 Cache.erase(I);
578 findRefEdges(Index, I, Edges, Cache, HasLocalIFuncCallOrRef);
579 }
580 };
581
582 // By now we processed all instructions in a function, except
583 // non-volatile loads and non-volatile value stores. Let's find
584 // ref edges for both of instruction sets
585 AddRefEdges(NonVolatileLoads, LoadRefEdges, Visited);
586 // We can add some values to the Visited set when processing load
587 // instructions which are also used by stores in NonVolatileStores.
588 // For example this can happen if we have following code:
589 //
590 // store %Derived* @foo, %Derived** bitcast (%Base** @bar to %Derived**)
591 // %42 = load %Derived*, %Derived** bitcast (%Base** @bar to %Derived**)
592 //
593 // After processing loads we'll add bitcast to the Visited set, and if
594 // we use the same set while processing stores, we'll never see store
595 // to @bar and @bar will be mistakenly treated as readonly.
597 AddRefEdges(NonVolatileStores, StoreRefEdges, StoreCache);
598
599 // If both load and store instruction reference the same variable
600 // we won't be able to optimize it. Add all such reference edges
601 // to RefEdges set.
602 for (const auto &VI : StoreRefEdges)
603 if (LoadRefEdges.remove(VI))
604 RefEdges.insert(VI);
605
606 unsigned RefCnt = RefEdges.size();
607 // All new reference edges inserted in two loops below are either
608 // read or write only. They will be grouped in the end of RefEdges
609 // vector, so we can use a single integer value to identify them.
610 for (const auto &VI : LoadRefEdges)
611 RefEdges.insert(VI);
612
613 unsigned FirstWORef = RefEdges.size();
614 for (const auto &VI : StoreRefEdges)
615 RefEdges.insert(VI);
616
617 Refs = RefEdges.takeVector();
618 for (; RefCnt < FirstWORef; ++RefCnt)
619 Refs[RefCnt].setReadOnly();
620
621 for (; RefCnt < Refs.size(); ++RefCnt)
622 Refs[RefCnt].setWriteOnly();
623 } else {
624 Refs = RefEdges.takeVector();
625 }
626 // Explicit add hot edges to enforce importing for designated GUIDs for
627 // sample PGO, to enable the same inlines as the profiled optimized binary.
628 for (auto &I : F.getImportGUIDs())
629 CallGraphEdges[Index.getOrInsertValueInfo(I)].updateHotness(
631 ? CalleeInfo::HotnessType::Cold
632 : CalleeInfo::HotnessType::Critical);
633
634#ifndef NDEBUG
635 // Make sure that all calls we decided could not have memprof summaries get a
636 // false value for mayHaveMemprofSummary, to ensure that this handling remains
637 // in sync with the ThinLTO backend handling.
638 if (IsThinLTO) {
639 for (const BasicBlock &BB : F) {
640 for (const Instruction &I : BB) {
641 const auto *CB = dyn_cast<CallBase>(&I);
642 if (!CB)
643 continue;
644 // We already checked these above.
645 if (CallsThatMayHaveMemprofSummary.count(CB))
646 continue;
648 }
649 }
650 }
651#endif
652
653 bool NonRenamableLocal = isNonRenamableLocal(F);
654 bool NotEligibleForImport =
655 NonRenamableLocal || HasInlineAsmMaybeReferencingInternal ||
656 HasIndirBranchToBlockAddress || HasLocalIFuncCallOrRef;
658 F.getLinkage(), F.getVisibility(), NotEligibleForImport,
659 /* Live = */ false, F.isDSOLocal(), F.canBeOmittedFromSymbolTable(),
660 GlobalValueSummary::ImportKind::Definition);
662 F.doesNotAccessMemory(), F.onlyReadsMemory() && !F.doesNotAccessMemory(),
663 F.hasFnAttribute(Attribute::NoRecurse), F.returnDoesNotAlias(),
664 // FIXME: refactor this to use the same code that inliner is using.
665 // Don't try to import functions with noinline attribute.
666 F.getAttributes().hasFnAttr(Attribute::NoInline),
667 F.hasFnAttribute(Attribute::AlwaysInline),
668 F.hasFnAttribute(Attribute::NoUnwind), MayThrow, HasUnknownCall,
670 std::vector<FunctionSummary::ParamAccess> ParamAccesses;
671 if (auto *SSI = GetSSICallback(F))
672 ParamAccesses = SSI->getParamAccesses(Index);
673 auto FuncSummary = std::make_unique<FunctionSummary>(
674 Flags, NumInsts, FunFlags, /*EntryCount=*/0, std::move(Refs),
675 CallGraphEdges.takeVector(), TypeTests.takeVector(),
676 TypeTestAssumeVCalls.takeVector(), TypeCheckedLoadVCalls.takeVector(),
677 TypeTestAssumeConstVCalls.takeVector(),
678 TypeCheckedLoadConstVCalls.takeVector(), std::move(ParamAccesses),
679 std::move(Callsites), std::move(Allocs));
680 if (NonRenamableLocal)
681 CantBePromoted.insert(F.getGUID());
682 Index.addGlobalValueSummary(F, std::move(FuncSummary));
683}
684
685/// Find function pointers referenced within the given vtable initializer
686/// (or subset of an initializer) \p I. The starting offset of \p I within
687/// the vtable initializer is \p StartingOffset. Any discovered function
688/// pointers are added to \p VTableFuncs along with their cumulative offset
689/// within the initializer.
690static void findFuncPointers(const Constant *I, uint64_t StartingOffset,
692 VTableFuncList &VTableFuncs,
693 const GlobalVariable &OrigGV) {
694 // First check if this is a function pointer.
695 if (I->getType()->isPointerTy()) {
696 auto C = I->stripPointerCasts();
697 auto A = dyn_cast<GlobalAlias>(C);
698 if (isa<Function>(C) || (A && isa<Function>(A->getAliasee()))) {
699 auto GV = dyn_cast<GlobalValue>(C);
700 assert(GV);
701 // We can disregard __cxa_pure_virtual as a possible call target, as
702 // calls to pure virtuals are UB.
703 if (GV && GV->getName() != "__cxa_pure_virtual")
704 VTableFuncs.push_back({Index.getOrInsertValueInfo(GV), StartingOffset});
705 return;
706 }
707 }
708
709 // Walk through the elements in the constant struct or array and recursively
710 // look for virtual function pointers.
711 const DataLayout &DL = M.getDataLayout();
712 if (auto *C = dyn_cast<ConstantStruct>(I)) {
713 StructType *STy = dyn_cast<StructType>(C->getType());
714 assert(STy);
715 const StructLayout *SL = DL.getStructLayout(C->getType());
716
717 for (auto EI : llvm::enumerate(STy->elements())) {
718 auto Offset = SL->getElementOffset(EI.index());
719 unsigned Op = SL->getElementContainingOffset(Offset);
720 findFuncPointers(cast<Constant>(I->getOperand(Op)),
721 StartingOffset + Offset, M, Index, VTableFuncs, OrigGV);
722 }
723 } else if (auto *C = dyn_cast<ConstantArray>(I)) {
724 ArrayType *ATy = C->getType();
725 Type *EltTy = ATy->getElementType();
726 uint64_t EltSize = DL.getTypeAllocSize(EltTy);
727 for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) {
728 findFuncPointers(cast<Constant>(I->getOperand(i)),
729 StartingOffset + i * EltSize, M, Index, VTableFuncs,
730 OrigGV);
731 }
732 } else if (const auto *CE = dyn_cast<ConstantExpr>(I)) {
733 // For relative vtables, the next sub-component should be a trunc.
734 if (CE->getOpcode() != Instruction::Trunc ||
735 !(CE = dyn_cast<ConstantExpr>(CE->getOperand(0))))
736 return;
737
738 // If this constant can be reduced to the offset between a function and a
739 // global, then we know this is a valid virtual function if the RHS is the
740 // original vtable we're scanning through.
741 if (CE->getOpcode() == Instruction::Sub) {
743 APSInt LHSOffset, RHSOffset;
744 if (IsConstantOffsetFromGlobal(CE->getOperand(0), LHS, LHSOffset, DL) &&
745 IsConstantOffsetFromGlobal(CE->getOperand(1), RHS, RHSOffset, DL) &&
746 RHS == &OrigGV &&
747
748 // For relative vtables, this component should point to the callable
749 // function without any offsets.
750 LHSOffset == 0 &&
751
752 // Also, the RHS should always point to somewhere within the vtable.
753 RHSOffset <=
754 static_cast<uint64_t>(DL.getTypeAllocSize(OrigGV.getInitializer()->getType()))) {
755 findFuncPointers(LHS, StartingOffset, M, Index, VTableFuncs, OrigGV);
756 }
757 }
758 }
759}
760
761// Identify the function pointers referenced by vtable definition \p V.
763 const GlobalVariable &V, const Module &M,
764 VTableFuncList &VTableFuncs) {
765 if (!V.isConstant())
766 return;
767
768 findFuncPointers(V.getInitializer(), /*StartingOffset=*/0, M, Index,
769 VTableFuncs, V);
770
771#ifndef NDEBUG
772 // Validate that the VTableFuncs list is ordered by offset.
773 uint64_t PrevOffset = 0;
774 for (auto &P : VTableFuncs) {
775 // The findVFuncPointers traversal should have encountered the
776 // functions in offset order. We need to use ">=" since PrevOffset
777 // starts at 0.
778 assert(P.VTableOffset >= PrevOffset);
779 PrevOffset = P.VTableOffset;
780 }
781#endif
782}
783
784/// Record vtable definition \p V for each type metadata it references.
785static void
787 const GlobalVariable &V,
789 for (MDNode *Type : Types) {
790 auto TypeID = Type->getOperand(1).get();
791
793 cast<ConstantInt>(
794 cast<ConstantAsMetadata>(Type->getOperand(0))->getValue())
795 ->getZExtValue();
796
797 if (auto *TypeId = dyn_cast<MDString>(TypeID))
798 Index.getOrInsertTypeIdCompatibleVtableSummary(TypeId->getString())
799 .push_back({Offset, Index.getOrInsertValueInfo(&V)});
800 }
801}
802
804 const GlobalVariable &V,
805 DenseSet<GlobalValue::GUID> &CantBePromoted,
806 const Module &M,
810 bool RefLocalIFunc = false;
811 bool HasBlockAddress =
812 findRefEdges(Index, &V, RefEdges, Visited, RefLocalIFunc);
813 const bool NotEligibleForImport = (HasBlockAddress || RefLocalIFunc);
814 bool NonRenamableLocal = isNonRenamableLocal(V);
816 V.getLinkage(), V.getVisibility(), NonRenamableLocal,
817 /* Live = */ false, V.isDSOLocal(), V.canBeOmittedFromSymbolTable(),
819
820 VTableFuncList VTableFuncs;
821 // If splitting is not enabled, then we compute the summary information
822 // necessary for index-based whole program devirtualization.
823 if (!Index.enableSplitLTOUnit()) {
824 Types.clear();
825 V.getMetadata(LLVMContext::MD_type, Types);
826 if (!Types.empty()) {
827 // Identify the function pointers referenced by this vtable definition.
828 computeVTableFuncs(Index, V, M, VTableFuncs);
829
830 // Record this vtable definition for each type metadata it references.
832 }
833 }
834
835 // Don't mark variables we won't be able to internalize as read/write-only.
836 bool CanBeInternalized =
837 !V.hasComdat() && !V.hasAppendingLinkage() && !V.isInterposable() &&
838 !V.hasAvailableExternallyLinkage() && !V.hasDLLExportStorageClass();
839 bool Constant = V.isConstant();
840 GlobalVarSummary::GVarFlags VarFlags(CanBeInternalized,
841 Constant ? false : CanBeInternalized,
842 Constant, V.getVCallVisibility());
843 auto GVarSummary = std::make_unique<GlobalVarSummary>(Flags, VarFlags,
844 RefEdges.takeVector());
845 if (NonRenamableLocal)
846 CantBePromoted.insert(V.getGUID());
847 if (NotEligibleForImport)
848 GVarSummary->setNotEligibleToImport();
849 if (!VTableFuncs.empty())
850 GVarSummary->setVTableFuncs(VTableFuncs);
851 Index.addGlobalValueSummary(V, std::move(GVarSummary));
852}
853
855 DenseSet<GlobalValue::GUID> &CantBePromoted) {
856 // Skip summary for indirect function aliases as summary for aliasee will not
857 // be emitted.
858 const GlobalObject *Aliasee = A.getAliaseeObject();
859 if (isa<GlobalIFunc>(Aliasee))
860 return;
861 bool NonRenamableLocal = isNonRenamableLocal(A);
863 A.getLinkage(), A.getVisibility(), NonRenamableLocal,
864 /* Live = */ false, A.isDSOLocal(), A.canBeOmittedFromSymbolTable(),
866 auto AS = std::make_unique<AliasSummary>(Flags);
867 auto AliaseeVI = Index.getValueInfo(Aliasee->getGUID());
868 assert(AliaseeVI && "Alias expects aliasee summary to be available");
869 assert(AliaseeVI.getSummaryList().size() == 1 &&
870 "Expected a single entry per aliasee in per-module index");
871 AS->setAliasee(AliaseeVI, AliaseeVI.getSummaryList()[0].get());
872 if (NonRenamableLocal)
873 CantBePromoted.insert(A.getGUID());
874 Index.addGlobalValueSummary(A, std::move(AS));
875}
876
877// Set LiveRoot flag on entries matching the given value name.
879 if (ValueInfo VI = Index.getValueInfo(GlobalValue::getGUID(Name)))
880 for (const auto &Summary : VI.getSummaryList())
881 Summary->setLive(true);
882}
883
885 const Module &M,
886 std::function<BlockFrequencyInfo *(const Function &F)> GetBFICallback,
888 std::function<const StackSafetyInfo *(const Function &F)> GetSSICallback) {
889 assert(PSI);
890 bool EnableSplitLTOUnit = false;
891 bool UnifiedLTO = false;
892 if (auto *MD = mdconst::extract_or_null<ConstantInt>(
893 M.getModuleFlag("EnableSplitLTOUnit")))
894 EnableSplitLTOUnit = MD->getZExtValue();
895 if (auto *MD =
896 mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("UnifiedLTO")))
897 UnifiedLTO = MD->getZExtValue();
898 ModuleSummaryIndex Index(/*HaveGVs=*/true, EnableSplitLTOUnit, UnifiedLTO);
899
900 // Identify the local values in the llvm.used and llvm.compiler.used sets,
901 // which should not be exported as they would then require renaming and
902 // promotion, but we may have opaque uses e.g. in inline asm. We collect them
903 // here because we use this information to mark functions containing inline
904 // assembly calls as not importable.
907 // First collect those in the llvm.used set.
908 collectUsedGlobalVariables(M, Used, /*CompilerUsed=*/false);
909 // Next collect those in the llvm.compiler.used set.
910 collectUsedGlobalVariables(M, Used, /*CompilerUsed=*/true);
911 DenseSet<GlobalValue::GUID> CantBePromoted;
912 for (auto *V : Used) {
913 if (V->hasLocalLinkage()) {
914 LocalsUsed.insert(V);
915 CantBePromoted.insert(V->getGUID());
916 }
917 }
918
919 bool HasLocalInlineAsmSymbol = false;
920 if (!M.getModuleInlineAsm().empty()) {
921 // Collect the local values defined by module level asm, and set up
922 // summaries for these symbols so that they can be marked as NoRename,
923 // to prevent export of any use of them in regular IR that would require
924 // renaming within the module level asm. Note we don't need to create a
925 // summary for weak or global defs, as they don't need to be flagged as
926 // NoRename, and defs in module level asm can't be imported anyway.
927 // Also, any values used but not defined within module level asm should
928 // be listed on the llvm.used or llvm.compiler.used global and marked as
929 // referenced from there.
932 // Symbols not marked as Weak or Global are local definitions.
935 return;
936 HasLocalInlineAsmSymbol = true;
937 GlobalValue *GV = M.getNamedValue(Name);
938 if (!GV)
939 return;
940 assert(GV->isDeclaration() && "Def in module asm already has definition");
943 /* NotEligibleToImport = */ true,
944 /* Live = */ true,
945 /* Local */ GV->isDSOLocal(), GV->canBeOmittedFromSymbolTable(),
947 CantBePromoted.insert(GV->getGUID());
948 // Create the appropriate summary type.
949 if (Function *F = dyn_cast<Function>(GV)) {
950 std::unique_ptr<FunctionSummary> Summary =
951 std::make_unique<FunctionSummary>(
952 GVFlags, /*InstCount=*/0,
954 F->hasFnAttribute(Attribute::ReadNone),
955 F->hasFnAttribute(Attribute::ReadOnly),
956 F->hasFnAttribute(Attribute::NoRecurse),
957 F->returnDoesNotAlias(),
958 /* NoInline = */ false,
959 F->hasFnAttribute(Attribute::AlwaysInline),
960 F->hasFnAttribute(Attribute::NoUnwind),
961 /* MayThrow */ true,
962 /* HasUnknownCall */ true,
963 /* MustBeUnreachable */ false},
964 /*EntryCount=*/0, ArrayRef<ValueInfo>{},
973 Index.addGlobalValueSummary(*GV, std::move(Summary));
974 } else {
975 std::unique_ptr<GlobalVarSummary> Summary =
976 std::make_unique<GlobalVarSummary>(
977 GVFlags,
979 false, false, cast<GlobalVariable>(GV)->isConstant(),
982 Index.addGlobalValueSummary(*GV, std::move(Summary));
983 }
984 });
985 }
986
987 bool IsThinLTO = true;
988 if (auto *MD =
989 mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("ThinLTO")))
990 IsThinLTO = MD->getZExtValue();
991
992 // Compute summaries for all functions defined in module, and save in the
993 // index.
994 for (const auto &F : M) {
995 if (F.isDeclaration())
996 continue;
997
998 DominatorTree DT(const_cast<Function &>(F));
999 BlockFrequencyInfo *BFI = nullptr;
1000 std::unique_ptr<BlockFrequencyInfo> BFIPtr;
1001 if (GetBFICallback)
1002 BFI = GetBFICallback(F);
1003 else if (F.hasProfileData()) {
1004 LoopInfo LI{DT};
1005 BranchProbabilityInfo BPI{F, LI};
1006 BFIPtr = std::make_unique<BlockFrequencyInfo>(F, BPI, LI);
1007 BFI = BFIPtr.get();
1008 }
1009
1010 computeFunctionSummary(Index, M, F, BFI, PSI, DT,
1011 !LocalsUsed.empty() || HasLocalInlineAsmSymbol,
1012 CantBePromoted, IsThinLTO, GetSSICallback);
1013 }
1014
1015 // Compute summaries for all variables defined in module, and save in the
1016 // index.
1018 for (const GlobalVariable &G : M.globals()) {
1019 if (G.isDeclaration())
1020 continue;
1021 computeVariableSummary(Index, G, CantBePromoted, M, Types);
1022 }
1023
1024 // Compute summaries for all aliases defined in module, and save in the
1025 // index.
1026 for (const GlobalAlias &A : M.aliases())
1027 computeAliasSummary(Index, A, CantBePromoted);
1028
1029 // Iterate through ifuncs, set their resolvers all alive.
1030 for (const GlobalIFunc &I : M.ifuncs()) {
1031 I.applyAlongResolverPath([&Index](const GlobalValue &GV) {
1032 Index.getGlobalValueSummary(GV)->setLive(true);
1033 });
1034 }
1035
1036 for (auto *V : LocalsUsed) {
1037 auto *Summary = Index.getGlobalValueSummary(*V);
1038 assert(Summary && "Missing summary for global value");
1039 Summary->setNotEligibleToImport();
1040 }
1041
1042 // The linker doesn't know about these LLVM produced values, so we need
1043 // to flag them as live in the index to ensure index-based dead value
1044 // analysis treats them as live roots of the analysis.
1045 setLiveRoot(Index, "llvm.used");
1046 setLiveRoot(Index, "llvm.compiler.used");
1047 setLiveRoot(Index, "llvm.global_ctors");
1048 setLiveRoot(Index, "llvm.global_dtors");
1049 setLiveRoot(Index, "llvm.global.annotations");
1050
1051 for (auto &GlobalList : Index) {
1052 // Ignore entries for references that are undefined in the current module.
1053 if (GlobalList.second.SummaryList.empty())
1054 continue;
1055
1056 assert(GlobalList.second.SummaryList.size() == 1 &&
1057 "Expected module's index to have one summary per GUID");
1058 auto &Summary = GlobalList.second.SummaryList[0];
1059 if (!IsThinLTO) {
1060 Summary->setNotEligibleToImport();
1061 continue;
1062 }
1063
1064 bool AllRefsCanBeExternallyReferenced =
1065 llvm::all_of(Summary->refs(), [&](const ValueInfo &VI) {
1066 return !CantBePromoted.count(VI.getGUID());
1067 });
1068 if (!AllRefsCanBeExternallyReferenced) {
1069 Summary->setNotEligibleToImport();
1070 continue;
1071 }
1072
1073 if (auto *FuncSummary = dyn_cast<FunctionSummary>(Summary.get())) {
1074 bool AllCallsCanBeExternallyReferenced = llvm::all_of(
1075 FuncSummary->calls(), [&](const FunctionSummary::EdgeTy &Edge) {
1076 return !CantBePromoted.count(Edge.first.getGUID());
1077 });
1078 if (!AllCallsCanBeExternallyReferenced)
1079 Summary->setNotEligibleToImport();
1080 }
1081 }
1082
1083 if (!ModuleSummaryDotFile.empty()) {
1084 std::error_code EC;
1085 raw_fd_ostream OSDot(ModuleSummaryDotFile, EC, sys::fs::OpenFlags::OF_Text);
1086 if (EC)
1087 report_fatal_error(Twine("Failed to open dot file ") +
1088 ModuleSummaryDotFile + ": " + EC.message() + "\n");
1089 Index.exportToDot(OSDot, {});
1090 }
1091
1092 return Index;
1093}
1094
1095AnalysisKey ModuleSummaryIndexAnalysis::Key;
1096
1100 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
1101 bool NeedSSI = needsParamAccessSummary(M);
1103 M,
1104 [&FAM](const Function &F) {
1106 *const_cast<Function *>(&F));
1107 },
1108 &PSI,
1109 [&FAM, NeedSSI](const Function &F) -> const StackSafetyInfo * {
1110 return NeedSSI ? &FAM.getResult<StackSafetyAnalysis>(
1111 const_cast<Function &>(F))
1112 : nullptr;
1113 });
1114}
1115
1117
1119 "Module Summary Analysis", false, true)
1125
1127 return new ModuleSummaryIndexWrapperPass();
1128}
1129
1131 : ModulePass(ID) {
1133}
1134
1136 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
1137 bool NeedSSI = needsParamAccessSummary(M);
1139 M,
1140 [this](const Function &F) {
1141 return &(this->getAnalysis<BlockFrequencyInfoWrapperPass>(
1142 *const_cast<Function *>(&F))
1143 .getBFI());
1144 },
1145 PSI,
1146 [&](const Function &F) -> const StackSafetyInfo * {
1147 return NeedSSI ? &getAnalysis<StackSafetyInfoWrapperPass>(
1148 const_cast<Function &>(F))
1149 .getResult()
1150 : nullptr;
1151 }));
1152 return false;
1153}
1154
1156 Index.reset();
1157 return false;
1158}
1159
1161 AU.setPreservesAll();
1165}
1166
1168
1174}
1175
1177 AnalysisUsage &AU) const {
1178 AU.setPreservesAll();
1179}
1180
1182 const ModuleSummaryIndex *Index) {
1184}
1185
1187 "Module summary info", false, true)
1188
1190 if (!CB)
1191 return false;
1192 if (CB->isDebugOrPseudoInst())
1193 return false;
1194 auto *CI = dyn_cast<CallInst>(CB);
1195 auto *CalledValue = CB->getCalledOperand();
1196 auto *CalledFunction = CB->getCalledFunction();
1197 if (CalledValue && !CalledFunction) {
1198 CalledValue = CalledValue->stripPointerCasts();
1199 // Stripping pointer casts can reveal a called function.
1200 CalledFunction = dyn_cast<Function>(CalledValue);
1201 }
1202 // Check if this is an alias to a function. If so, get the
1203 // called aliasee for the checks below.
1204 if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
1205 assert(!CalledFunction &&
1206 "Expected null called function in callsite for alias");
1207 CalledFunction = dyn_cast<Function>(GA->getAliaseeObject());
1208 }
1209 // Check if this is a direct call to a known function or a known
1210 // intrinsic, or an indirect call with profile data.
1211 if (CalledFunction) {
1212 if (CI && CalledFunction->isIntrinsic())
1213 return false;
1214 } else {
1215 // TODO: For now skip indirect calls. See comments in
1216 // computeFunctionSummary for what is needed to handle this.
1217 return false;
1218 }
1219 return true;
1220}
aarch64 promote const
static bool isConstant(const MachineInstr &MI)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file contains the simple types necessary to represent the attributes associated with functions a...
basic Basic Alias true
block Block Frequency Analysis
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Definition: CommandLine.h:686
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file defines the DenseSet and SmallDenseSet classes.
std::string Name
This defines the Use class.
iv users
Definition: IVUsers.cpp:48
Interface to identify indirect call promotion candidates.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
This file implements a map that provides insertion order iteration.
This file contains the declarations for metadata subclasses.
static void addVCallToSet(DevirtCallSite Call, GlobalValue::GUID Guid, SetVector< FunctionSummary::VFuncId, std::vector< FunctionSummary::VFuncId > > &VCalls, SetVector< FunctionSummary::ConstVCall, std::vector< FunctionSummary::ConstVCall > > &ConstVCalls)
Determine whether this call has all constant integer arguments (excluding "this") and summarize it to...
cl::opt< unsigned > MaxNumVTableAnnotations
static void computeVTableFuncs(ModuleSummaryIndex &Index, const GlobalVariable &V, const Module &M, VTableFuncList &VTableFuncs)
static void computeAliasSummary(ModuleSummaryIndex &Index, const GlobalAlias &A, DenseSet< GlobalValue::GUID > &CantBePromoted)
static void findFuncPointers(const Constant *I, uint64_t StartingOffset, const Module &M, ModuleSummaryIndex &Index, VTableFuncList &VTableFuncs, const GlobalVariable &OrigGV)
Find function pointers referenced within the given vtable initializer (or subset of an initializer) I...
static void computeVariableSummary(ModuleSummaryIndex &Index, const GlobalVariable &V, DenseSet< GlobalValue::GUID > &CantBePromoted, const Module &M, SmallVectorImpl< MDNode * > &Types)
static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name)
static CalleeInfo::HotnessType getHotness(uint64_t ProfileCount, ProfileSummaryInfo *PSI)
static bool isNonVolatileLoad(const Instruction *I)
cl::opt< bool > ScalePartialSampleProfileWorkingSetSize
cl::opt< bool > MemProfReportHintedSizes
static bool isNonRenamableLocal(const GlobalValue &GV)
static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M, const Function &F, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, DominatorTree &DT, bool HasLocalsInUsedOrAsm, DenseSet< GlobalValue::GUID > &CantBePromoted, bool IsThinLTO, std::function< const StackSafetyInfo *(const Function &F)> GetSSICallback)
static bool findRefEdges(ModuleSummaryIndex &Index, const User *CurUser, SetVector< ValueInfo, std::vector< ValueInfo > > &RefEdges, SmallPtrSet< const User *, 8 > &Visited, bool &RefLocalLinkageIFunc)
static cl::opt< FunctionSummary::ForceSummaryHotnessType, true > FSEC("force-summary-edges-cold", cl::Hidden, cl::location(ForceSummaryEdgesCold), cl::desc("Force all edges in the function summary to cold"), cl::values(clEnumValN(FunctionSummary::FSHT_None, "none", "None."), clEnumValN(FunctionSummary::FSHT_AllNonCritical, "all-non-critical", "All non-critical edges."), clEnumValN(FunctionSummary::FSHT_All, "all", "All edges.")))
static bool mustBeUnreachableFunction(const Function &F)
static bool isNonVolatileStore(const Instruction *I)
module summary analysis
static cl::opt< std::string > ModuleSummaryDotFile("module-summary-dot-file", cl::Hidden, cl::value_desc("filename"), cl::desc("File to emit dot graph of new summary into"))
static void addIntrinsicToSummary(const CallInst *CI, SetVector< GlobalValue::GUID, std::vector< GlobalValue::GUID > > &TypeTests, SetVector< FunctionSummary::VFuncId, std::vector< FunctionSummary::VFuncId > > &TypeTestAssumeVCalls, SetVector< FunctionSummary::VFuncId, std::vector< FunctionSummary::VFuncId > > &TypeCheckedLoadVCalls, SetVector< FunctionSummary::ConstVCall, std::vector< FunctionSummary::ConstVCall > > &TypeTestAssumeConstVCalls, SetVector< FunctionSummary::ConstVCall, std::vector< FunctionSummary::ConstVCall > > &TypeCheckedLoadConstVCalls, DominatorTree &DT)
If this intrinsic call requires that we add information to the function summary, do so via the non-co...
static void recordTypeIdCompatibleVtableReferences(ModuleSummaryIndex &Index, const GlobalVariable &V, SmallVectorImpl< MDNode * > &Types)
Record vtable definition V for each type metadata it references.
This is the interface to build a ModuleSummaryIndex for a module.
ModuleSummaryIndex.h This file contains the declarations the classes that hold the module index and s...
Module.h This file contains the declarations for the Module class.
#define P(N)
FunctionAnalysisManager FAM
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:57
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
Value * RHS
Value * LHS
An arbitrary precision integer that knows its signedness.
Definition: APSInt.h:23
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:253
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:405
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesAll()
Set by analyses that do not transform their input at all.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
LLVM Basic Block Representation.
Definition: BasicBlock.h:61
static BlockAddress * get(Function *F, BasicBlock *BB)
Return a BlockAddress for the specified function and basic block.
Definition: Constants.cpp:1871
Analysis pass which computes BlockFrequencyInfo.
Legacy analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Analysis providing branch probability information.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1236
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1465
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1410
This class represents a function call, abstracting a target machine's calling convention.
This is an important base class in LLVM.
Definition: Constant.h:42
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:63
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:162
std::pair< ValueInfo, CalleeInfo > EdgeTy
<CalleeValueInfo, CalleeInfo> call edge pair.
ForceSummaryHotnessType
Types for -force-summary-edges-cold debugging option.
Class to represent profile counts.
Definition: Function.h:296
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:249
bool isDSOLocal() const
Definition: GlobalValue.h:305
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:290
bool hasLocalLinkage() const
Definition: GlobalValue.h:528
static GUID getGUID(StringRef GlobalName)
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: Globals.cpp:75
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:595
@ DefaultVisibility
The GV is visible.
Definition: GlobalValue.h:67
bool hasSection() const
Definition: GlobalValue.h:290
bool canBeOmittedFromSymbolTable() const
True if GV can be left out of the object symbol table.
Definition: Globals.cpp:419
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:59
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
MutableArrayRef< InstrProfValueData > getPromotionCandidatesForInstruction(const Instruction *I, uint64_t &TotalCount, uint32_t &NumCandidates)
Returns reference to array of InstrProfValueData for the given instruction I.
Legacy wrapper pass to provide the ModuleSummaryIndex object.
ImmutableModuleSummaryIndexWrapperPass(const ModuleSummaryIndex *Index=nullptr)
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
ImmutablePass class - This class is used to provide information that does not need to be run.
Definition: Pass.h:281
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:563
Metadata node.
Definition: Metadata.h:1069
This class implements a map that also provides access to all stored values in a deterministic order.
Definition: MapVector.h:36
VectorType takeVector()
Clear the MapVector and return the underlying vector.
Definition: MapVector.h:55
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:251
Result run(Module &M, ModuleAnalysisManager &AM)
Legacy wrapper pass to provide the ModuleSummaryIndex object.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
bool doFinalization(Module &M) override
doFinalization - Virtual method overriden by subclasses to do any necessary clean up after all passes...
bool runOnModule(Module &M) override
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
Class to hold module path string table and global value map, and encapsulate methods for operating on...
static void CollectAsmSymbols(const Module &M, function_ref< void(StringRef, object::BasicSymbolRef::Flags)> AsmSymbol)
Parse inline ASM and collect the symbols that are defined or referenced in the current module.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
Analysis providing profile information.
std::optional< uint64_t > getProfileCount(const CallBase &CallInst, BlockFrequencyInfo *BFI, bool AllowSynthetic=false) const
Returns the profile count for CallInst.
bool isColdCount(uint64_t C) const
Returns true if count C is considered cold.
bool hasPartialSampleProfile() const
Returns true if module M has partial-profile sample profile.
bool isHotCount(uint64_t C) const
Returns true if count C is considered hot.
A vector that has set insertion semantics.
Definition: SetVector.h:57
bool remove(const value_type &X)
Remove an item from the set vector.
Definition: SetVector.h:188
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:98
Vector takeVector()
Clear the SetVector and return the underlying vector.
Definition: SetVector.h:87
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:367
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:502
bool empty() const
Definition: SmallVector.h:94
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StackSafetyInfo wrapper for the new pass manager.
StackSafetyInfo wrapper for the legacy pass manager.
Interface to access stack safety analysis results for single function.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
Definition: DataLayout.h:571
unsigned getElementContainingOffset(uint64_t FixedOffset) const
Given a valid byte offset into the structure, returns the structure index that contains it.
Definition: DataLayout.cpp:92
TypeSize getElementOffset(unsigned Idx) const
Definition: DataLayout.h:600
Class to represent struct types.
Definition: DerivedTypes.h:216
ArrayRef< Type * > elements() const
Definition: DerivedTypes.h:333
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
iterator_range< use_iterator > uses()
Definition: Value.h:376
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:97
Helper class to iterate through stack ids in both metadata (memprof MIB and callsite) and the corresp...
CallStackIterator end() const
CallStackIterator beginAfterSharedPrefix(CallStack &Other)
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:460
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
Definition: CommandLine.h:711
LocationClass< Ty > location(Ty &L)
Definition: CommandLine.h:463
AllocationType getMIBAllocType(const MDNode *MIB)
Returns the allocation type from an MIB metadata node.
uint64_t getMIBTotalSize(const MDNode *MIB)
Returns the total size from an MIB metadata node, or 0 if it was not recorded.
MDNode * getMIBStackNode(const MDNode *MIB)
Returns the stack node from an MIB metadata node.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:329
@ Offset
Definition: DWP.cpp:480
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1722
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition: STLExtras.h:2431
bool IsConstantOffsetFromGlobal(Constant *C, GlobalValue *&GV, APInt &Offset, const DataLayout &DL, DSOLocalEquivalent **DSOEquiv=nullptr)
If this constant is a constant offset from a global, return the global and the constant.
FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold
bool needsParamAccessSummary(const Module &M)
ModuleSummaryIndex buildModuleSummaryIndex(const Module &M, std::function< BlockFrequencyInfo *(const Function &F)> GetBFICallback, ProfileSummaryInfo *PSI, std::function< const StackSafetyInfo *(const Function &F)> GetSSICallback=[](const Function &F) -> const StackSafetyInfo *{ return nullptr;})
Direct function to compute a ModuleSummaryIndex from a given module.
void initializeModuleSummaryIndexWrapperPassPass(PassRegistry &)
std::vector< VirtFuncOffset > VTableFuncList
List of functions referenced by a particular vtable definition.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1729
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:167
void findDevirtualizableCallsForTypeCheckedLoad(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< Instruction * > &LoadedPtrs, SmallVectorImpl< Instruction * > &Preds, bool &HasNonCallUses, const CallInst *CI, DominatorTree &DT)
Given a call to the intrinsic @llvm.type.checked.load, find all devirtualizable call sites based on t...
SmallVector< InstrProfValueData, 4 > getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst and returns them if Inst is annotated with value profile dat...
Definition: InstrProf.cpp:1365
ModulePass * createModuleSummaryIndexWrapperPass()
ImmutablePass * createImmutableModuleSummaryIndexWrapperPass(const ModuleSummaryIndex *Index)
void initializeImmutableModuleSummaryIndexWrapperPassPass(PassRegistry &)
bool mayHaveMemprofSummary(const CallBase *CB)
Returns true if the instruction could have memprof metadata, used to ensure consistency between summa...
void findDevirtualizableCallsForTypeTest(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< CallInst * > &Assumes, const CallInst *CI, DominatorTree &DT)
Given a call to the intrinsic @llvm.type.test, find all devirtualizable call sites based on the call ...
GlobalVariable * collectUsedGlobalVariables(const Module &M, SmallVectorImpl< GlobalValue * > &Vec, bool CompilerUsed)
Given "llvm.used" or "llvm.compiler.used" as a global name, collect the initializer elements of that ...
Definition: Module.cpp:830
Summary of memprof metadata on allocations.
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition: Analysis.h:28
A call site that could be devirtualized.
A specification for a virtual function call with all constant integer arguments.
Flags specific to function summaries.
An "identifier" for a virtual function.
Group flags (Linkage, NotEligibleToImport, etc.) as a bitfield.
Summary of a single MIB in a memprof metadata on allocations.
Struct that holds a reference to a particular GUID in a global value summary.