LLVM 19.0.0git
ModuleSummaryAnalysis.cpp
Go to the documentation of this file.
1//===- ModuleSummaryAnalysis.cpp - Module summary index builder -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass builds a ModuleSummaryIndex object for the module, to be written
10// to bitcode or LLVM assembly.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/DenseSet.h"
17#include "llvm/ADT/MapVector.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/SetVector.h"
22#include "llvm/ADT/StringRef.h"
31#include "llvm/IR/Attributes.h"
32#include "llvm/IR/BasicBlock.h"
33#include "llvm/IR/Constant.h"
34#include "llvm/IR/Constants.h"
35#include "llvm/IR/Dominators.h"
36#include "llvm/IR/Function.h"
37#include "llvm/IR/GlobalAlias.h"
38#include "llvm/IR/GlobalValue.h"
42#include "llvm/IR/Metadata.h"
43#include "llvm/IR/Module.h"
45#include "llvm/IR/Use.h"
46#include "llvm/IR/User.h"
50#include "llvm/Pass.h"
54#include <algorithm>
55#include <cassert>
56#include <cstdint>
57#include <vector>
58
59using namespace llvm;
60using namespace llvm::memprof;
61
62#define DEBUG_TYPE "module-summary-analysis"
63
64// Option to force edges cold which will block importing when the
65// -import-cold-multiplier is set to 0. Useful for debugging.
66namespace llvm {
69} // namespace llvm
70
72 "force-summary-edges-cold", cl::Hidden, cl::location(ForceSummaryEdgesCold),
73 cl::desc("Force all edges in the function summary to cold"),
76 "all-non-critical", "All non-critical edges."),
77 clEnumValN(FunctionSummary::FSHT_All, "all", "All edges.")));
78
80 "module-summary-dot-file", cl::Hidden, cl::value_desc("filename"),
81 cl::desc("File to emit dot graph of new summary into"));
82
84
85// Walk through the operands of a given User via worklist iteration and populate
86// the set of GlobalValue references encountered. Invoked either on an
87// Instruction or a GlobalVariable (which walks its initializer).
88// Return true if any of the operands contains blockaddress. This is important
89// to know when computing summary for global var, because if global variable
90// references basic block address we can't import it separately from function
91// containing that basic block. For simplicity we currently don't import such
92// global vars at all. When importing function we aren't interested if any
93// instruction in it takes an address of any basic block, because instruction
94// can only take an address of basic block located in the same function.
95static bool findRefEdges(ModuleSummaryIndex &Index, const User *CurUser,
96 SetVector<ValueInfo, std::vector<ValueInfo>> &RefEdges,
98 bool HasBlockAddress = false;
100 if (Visited.insert(CurUser).second)
101 Worklist.push_back(CurUser);
102
103 while (!Worklist.empty()) {
104 const User *U = Worklist.pop_back_val();
105 const auto *CB = dyn_cast<CallBase>(U);
106
107 for (const auto &OI : U->operands()) {
108 const User *Operand = dyn_cast<User>(OI);
109 if (!Operand)
110 continue;
111 if (isa<BlockAddress>(Operand)) {
112 HasBlockAddress = true;
113 continue;
114 }
115 if (auto *GV = dyn_cast<GlobalValue>(Operand)) {
116 // We have a reference to a global value. This should be added to
117 // the reference set unless it is a callee. Callees are handled
118 // specially by WriteFunction and are added to a separate list.
119 if (!(CB && CB->isCallee(&OI)))
120 RefEdges.insert(Index.getOrInsertValueInfo(GV));
121 continue;
122 }
123 if (Visited.insert(Operand).second)
124 Worklist.push_back(Operand);
125 }
126 }
127 return HasBlockAddress;
128}
129
131 ProfileSummaryInfo *PSI) {
132 if (!PSI)
133 return CalleeInfo::HotnessType::Unknown;
134 if (PSI->isHotCount(ProfileCount))
135 return CalleeInfo::HotnessType::Hot;
136 if (PSI->isColdCount(ProfileCount))
137 return CalleeInfo::HotnessType::Cold;
138 return CalleeInfo::HotnessType::None;
139}
140
141static bool isNonRenamableLocal(const GlobalValue &GV) {
142 return GV.hasSection() && GV.hasLocalLinkage();
143}
144
145/// Determine whether this call has all constant integer arguments (excluding
146/// "this") and summarize it to VCalls or ConstVCalls as appropriate.
147static void addVCallToSet(
149 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
150 &VCalls,
152 std::vector<FunctionSummary::ConstVCall>> &ConstVCalls) {
153 std::vector<uint64_t> Args;
154 // Start from the second argument to skip the "this" pointer.
155 for (auto &Arg : drop_begin(Call.CB.args())) {
156 auto *CI = dyn_cast<ConstantInt>(Arg);
157 if (!CI || CI->getBitWidth() > 64) {
158 VCalls.insert({Guid, Call.Offset});
159 return;
160 }
161 Args.push_back(CI->getZExtValue());
162 }
163 ConstVCalls.insert({{Guid, Call.Offset}, std::move(Args)});
164}
165
166/// If this intrinsic call requires that we add information to the function
167/// summary, do so via the non-constant reference arguments.
169 const CallInst *CI,
170 SetVector<GlobalValue::GUID, std::vector<GlobalValue::GUID>> &TypeTests,
171 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
172 &TypeTestAssumeVCalls,
173 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
174 &TypeCheckedLoadVCalls,
176 std::vector<FunctionSummary::ConstVCall>>
177 &TypeTestAssumeConstVCalls,
179 std::vector<FunctionSummary::ConstVCall>>
180 &TypeCheckedLoadConstVCalls,
181 DominatorTree &DT) {
182 switch (CI->getCalledFunction()->getIntrinsicID()) {
183 case Intrinsic::type_test:
184 case Intrinsic::public_type_test: {
185 auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(1));
186 auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
187 if (!TypeId)
188 break;
189 GlobalValue::GUID Guid = GlobalValue::getGUID(TypeId->getString());
190
191 // Produce a summary from type.test intrinsics. We only summarize type.test
192 // intrinsics that are used other than by an llvm.assume intrinsic.
193 // Intrinsics that are assumed are relevant only to the devirtualization
194 // pass, not the type test lowering pass.
195 bool HasNonAssumeUses = llvm::any_of(CI->uses(), [](const Use &CIU) {
196 return !isa<AssumeInst>(CIU.getUser());
197 });
198 if (HasNonAssumeUses)
199 TypeTests.insert(Guid);
200
203 findDevirtualizableCallsForTypeTest(DevirtCalls, Assumes, CI, DT);
204 for (auto &Call : DevirtCalls)
205 addVCallToSet(Call, Guid, TypeTestAssumeVCalls,
206 TypeTestAssumeConstVCalls);
207
208 break;
209 }
210
211 case Intrinsic::type_checked_load_relative:
212 case Intrinsic::type_checked_load: {
213 auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(2));
214 auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
215 if (!TypeId)
216 break;
217 GlobalValue::GUID Guid = GlobalValue::getGUID(TypeId->getString());
218
222 bool HasNonCallUses = false;
223 findDevirtualizableCallsForTypeCheckedLoad(DevirtCalls, LoadedPtrs, Preds,
224 HasNonCallUses, CI, DT);
225 // Any non-call uses of the result of llvm.type.checked.load will
226 // prevent us from optimizing away the llvm.type.test.
227 if (HasNonCallUses)
228 TypeTests.insert(Guid);
229 for (auto &Call : DevirtCalls)
230 addVCallToSet(Call, Guid, TypeCheckedLoadVCalls,
231 TypeCheckedLoadConstVCalls);
232
233 break;
234 }
235 default:
236 break;
237 }
238}
239
240static bool isNonVolatileLoad(const Instruction *I) {
241 if (const auto *LI = dyn_cast<LoadInst>(I))
242 return !LI->isVolatile();
243
244 return false;
245}
246
247static bool isNonVolatileStore(const Instruction *I) {
248 if (const auto *SI = dyn_cast<StoreInst>(I))
249 return !SI->isVolatile();
250
251 return false;
252}
253
254// Returns true if the function definition must be unreachable.
255//
256// Note if this helper function returns true, `F` is guaranteed
257// to be unreachable; if it returns false, `F` might still
258// be unreachable but not covered by this helper function.
260 // A function must be unreachable if its entry block ends with an
261 // 'unreachable'.
262 assert(!F.isDeclaration());
263 return isa<UnreachableInst>(F.getEntryBlock().getTerminator());
264}
265
267 ModuleSummaryIndex &Index, const Module &M, const Function &F,
269 bool HasLocalsInUsedOrAsm, DenseSet<GlobalValue::GUID> &CantBePromoted,
270 bool IsThinLTO,
271 std::function<const StackSafetyInfo *(const Function &F)> GetSSICallback) {
272 // Summary not currently supported for anonymous functions, they should
273 // have been named.
274 assert(F.hasName());
275
276 unsigned NumInsts = 0;
277 // Map from callee ValueId to profile count. Used to accumulate profile
278 // counts for all static calls to a given callee.
280 std::vector<std::pair<ValueInfo, CalleeInfo>>>
281 CallGraphEdges;
282 SetVector<ValueInfo, std::vector<ValueInfo>> RefEdges, LoadRefEdges,
283 StoreRefEdges;
286 TypeTestAssumeVCalls, TypeCheckedLoadVCalls;
288 std::vector<FunctionSummary::ConstVCall>>
289 TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls;
290 ICallPromotionAnalysis ICallAnalysis;
292
293 // Add personality function, prefix data and prologue data to function's ref
294 // list.
295 findRefEdges(Index, &F, RefEdges, Visited);
296 std::vector<const Instruction *> NonVolatileLoads;
297 std::vector<const Instruction *> NonVolatileStores;
298
299 std::vector<CallsiteInfo> Callsites;
300 std::vector<AllocInfo> Allocs;
301
302#ifndef NDEBUG
303 DenseSet<const CallBase *> CallsThatMayHaveMemprofSummary;
304#endif
305
306 bool HasInlineAsmMaybeReferencingInternal = false;
307 bool HasIndirBranchToBlockAddress = false;
308 bool HasIFuncCall = false;
309 bool HasUnknownCall = false;
310 bool MayThrow = false;
311 for (const BasicBlock &BB : F) {
312 // We don't allow inlining of function with indirect branch to blockaddress.
313 // If the blockaddress escapes the function, e.g., via a global variable,
314 // inlining may lead to an invalid cross-function reference. So we shouldn't
315 // import such function either.
316 if (BB.hasAddressTaken()) {
317 for (User *U : BlockAddress::get(const_cast<BasicBlock *>(&BB))->users())
318 if (!isa<CallBrInst>(*U)) {
319 HasIndirBranchToBlockAddress = true;
320 break;
321 }
322 }
323
324 for (const Instruction &I : BB) {
325 if (I.isDebugOrPseudoInst())
326 continue;
327 ++NumInsts;
328
329 // Regular LTO module doesn't participate in ThinLTO import,
330 // so no reference from it can be read/writeonly, since this
331 // would require importing variable as local copy
332 if (IsThinLTO) {
333 if (isNonVolatileLoad(&I)) {
334 // Postpone processing of non-volatile load instructions
335 // See comments below
336 Visited.insert(&I);
337 NonVolatileLoads.push_back(&I);
338 continue;
339 } else if (isNonVolatileStore(&I)) {
340 Visited.insert(&I);
341 NonVolatileStores.push_back(&I);
342 // All references from second operand of store (destination address)
343 // can be considered write-only if they're not referenced by any
344 // non-store instruction. References from first operand of store
345 // (stored value) can't be treated either as read- or as write-only
346 // so we add them to RefEdges as we do with all other instructions
347 // except non-volatile load.
348 Value *Stored = I.getOperand(0);
349 if (auto *GV = dyn_cast<GlobalValue>(Stored))
350 // findRefEdges will try to examine GV operands, so instead
351 // of calling it we should add GV to RefEdges directly.
352 RefEdges.insert(Index.getOrInsertValueInfo(GV));
353 else if (auto *U = dyn_cast<User>(Stored))
354 findRefEdges(Index, U, RefEdges, Visited);
355 continue;
356 }
357 }
358 findRefEdges(Index, &I, RefEdges, Visited);
359 const auto *CB = dyn_cast<CallBase>(&I);
360 if (!CB) {
361 if (I.mayThrow())
362 MayThrow = true;
363 continue;
364 }
365
366 const auto *CI = dyn_cast<CallInst>(&I);
367 // Since we don't know exactly which local values are referenced in inline
368 // assembly, conservatively mark the function as possibly referencing
369 // a local value from inline assembly to ensure we don't export a
370 // reference (which would require renaming and promotion of the
371 // referenced value).
372 if (HasLocalsInUsedOrAsm && CI && CI->isInlineAsm())
373 HasInlineAsmMaybeReferencingInternal = true;
374
375 auto *CalledValue = CB->getCalledOperand();
376 auto *CalledFunction = CB->getCalledFunction();
377 if (CalledValue && !CalledFunction) {
378 CalledValue = CalledValue->stripPointerCasts();
379 // Stripping pointer casts can reveal a called function.
380 CalledFunction = dyn_cast<Function>(CalledValue);
381 }
382 // Check if this is an alias to a function. If so, get the
383 // called aliasee for the checks below.
384 if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
385 assert(!CalledFunction && "Expected null called function in callsite for alias");
386 CalledFunction = dyn_cast<Function>(GA->getAliaseeObject());
387 }
388 // Check if this is a direct call to a known function or a known
389 // intrinsic, or an indirect call with profile data.
390 if (CalledFunction) {
391 if (CI && CalledFunction->isIntrinsic()) {
393 CI, TypeTests, TypeTestAssumeVCalls, TypeCheckedLoadVCalls,
394 TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls, DT);
395 continue;
396 }
397 // We should have named any anonymous globals
398 assert(CalledFunction->hasName());
399 auto ScaledCount = PSI->getProfileCount(*CB, BFI);
400 auto Hotness = ScaledCount ? getHotness(*ScaledCount, PSI)
401 : CalleeInfo::HotnessType::Unknown;
403 Hotness = CalleeInfo::HotnessType::Cold;
404
405 // Use the original CalledValue, in case it was an alias. We want
406 // to record the call edge to the alias in that case. Eventually
407 // an alias summary will be created to associate the alias and
408 // aliasee.
409 auto &ValueInfo = CallGraphEdges[Index.getOrInsertValueInfo(
410 cast<GlobalValue>(CalledValue))];
411 ValueInfo.updateHotness(Hotness);
412 if (CB->isTailCall())
413 ValueInfo.setHasTailCall(true);
414 // Add the relative block frequency to CalleeInfo if there is no profile
415 // information.
416 if (BFI != nullptr && Hotness == CalleeInfo::HotnessType::Unknown) {
417 uint64_t BBFreq = BFI->getBlockFreq(&BB).getFrequency();
418 uint64_t EntryFreq = BFI->getEntryFreq().getFrequency();
419 ValueInfo.updateRelBlockFreq(BBFreq, EntryFreq);
420 }
421 } else {
422 HasUnknownCall = true;
423 // If F is imported, a local linkage ifunc (e.g. target_clones on a
424 // static function) called by F will be cloned. Since summaries don't
425 // track ifunc, we do not know implementation functions referenced by
426 // the ifunc resolver need to be promoted in the exporter, and we will
427 // get linker errors due to cloned declarations for implementation
428 // functions. As a simple fix, just mark F as not eligible for import.
429 // Non-local ifunc is not cloned and does not have the issue.
430 if (auto *GI = dyn_cast_if_present<GlobalIFunc>(CalledValue))
431 if (GI->hasLocalLinkage())
432 HasIFuncCall = true;
433 // Skip inline assembly calls.
434 if (CI && CI->isInlineAsm())
435 continue;
436 // Skip direct calls.
437 if (!CalledValue || isa<Constant>(CalledValue))
438 continue;
439
440 // Check if the instruction has a callees metadata. If so, add callees
441 // to CallGraphEdges to reflect the references from the metadata, and
442 // to enable importing for subsequent indirect call promotion and
443 // inlining.
444 if (auto *MD = I.getMetadata(LLVMContext::MD_callees)) {
445 for (const auto &Op : MD->operands()) {
446 Function *Callee = mdconst::extract_or_null<Function>(Op);
447 if (Callee)
448 CallGraphEdges[Index.getOrInsertValueInfo(Callee)];
449 }
450 }
451
452 uint32_t NumVals, NumCandidates;
453 uint64_t TotalCount;
454 auto CandidateProfileData =
456 &I, NumVals, TotalCount, NumCandidates);
457 for (const auto &Candidate : CandidateProfileData)
458 CallGraphEdges[Index.getOrInsertValueInfo(Candidate.Value)]
459 .updateHotness(getHotness(Candidate.Count, PSI));
460 }
461
462 // Summarize memprof related metadata. This is only needed for ThinLTO.
463 if (!IsThinLTO)
464 continue;
465
466 // TODO: Skip indirect calls for now. Need to handle these better, likely
467 // by creating multiple Callsites, one per target, then speculatively
468 // devirtualize while applying clone info in the ThinLTO backends. This
469 // will also be important because we will have a different set of clone
470 // versions per target. This handling needs to match that in the ThinLTO
471 // backend so we handle things consistently for matching of callsite
472 // summaries to instructions.
473 if (!CalledFunction)
474 continue;
475
476 // Ensure we keep this analysis in sync with the handling in the ThinLTO
477 // backend (see MemProfContextDisambiguation::applyImport). Save this call
478 // so that we can skip it in checking the reverse case later.
480#ifndef NDEBUG
481 CallsThatMayHaveMemprofSummary.insert(CB);
482#endif
483
484 // Compute the list of stack ids first (so we can trim them from the stack
485 // ids on any MIBs).
487 I.getMetadata(LLVMContext::MD_callsite));
488 auto *MemProfMD = I.getMetadata(LLVMContext::MD_memprof);
489 if (MemProfMD) {
490 std::vector<MIBInfo> MIBs;
491 for (auto &MDOp : MemProfMD->operands()) {
492 auto *MIBMD = cast<const MDNode>(MDOp);
493 MDNode *StackNode = getMIBStackNode(MIBMD);
494 assert(StackNode);
495 SmallVector<unsigned> StackIdIndices;
496 CallStack<MDNode, MDNode::op_iterator> StackContext(StackNode);
497 // Collapse out any on the allocation call (inlining).
498 for (auto ContextIter =
499 StackContext.beginAfterSharedPrefix(InstCallsite);
500 ContextIter != StackContext.end(); ++ContextIter) {
501 unsigned StackIdIdx = Index.addOrGetStackIdIndex(*ContextIter);
502 // If this is a direct recursion, simply skip the duplicate
503 // entries. If this is mutual recursion, handling is left to
504 // the LTO link analysis client.
505 if (StackIdIndices.empty() || StackIdIndices.back() != StackIdIdx)
506 StackIdIndices.push_back(StackIdIdx);
507 }
508 MIBs.push_back(
509 MIBInfo(getMIBAllocType(MIBMD), std::move(StackIdIndices)));
510 }
511 Allocs.push_back(AllocInfo(std::move(MIBs)));
512 } else if (!InstCallsite.empty()) {
513 SmallVector<unsigned> StackIdIndices;
514 for (auto StackId : InstCallsite)
515 StackIdIndices.push_back(Index.addOrGetStackIdIndex(StackId));
516 // Use the original CalledValue, in case it was an alias. We want
517 // to record the call edge to the alias in that case. Eventually
518 // an alias summary will be created to associate the alias and
519 // aliasee.
520 auto CalleeValueInfo =
521 Index.getOrInsertValueInfo(cast<GlobalValue>(CalledValue));
522 Callsites.push_back({CalleeValueInfo, StackIdIndices});
523 }
524 }
525 }
526
528 Index.addBlockCount(F.size());
529
530 std::vector<ValueInfo> Refs;
531 if (IsThinLTO) {
532 auto AddRefEdges = [&](const std::vector<const Instruction *> &Instrs,
535 for (const auto *I : Instrs) {
536 Cache.erase(I);
537 findRefEdges(Index, I, Edges, Cache);
538 }
539 };
540
541 // By now we processed all instructions in a function, except
542 // non-volatile loads and non-volatile value stores. Let's find
543 // ref edges for both of instruction sets
544 AddRefEdges(NonVolatileLoads, LoadRefEdges, Visited);
545 // We can add some values to the Visited set when processing load
546 // instructions which are also used by stores in NonVolatileStores.
547 // For example this can happen if we have following code:
548 //
549 // store %Derived* @foo, %Derived** bitcast (%Base** @bar to %Derived**)
550 // %42 = load %Derived*, %Derived** bitcast (%Base** @bar to %Derived**)
551 //
552 // After processing loads we'll add bitcast to the Visited set, and if
553 // we use the same set while processing stores, we'll never see store
554 // to @bar and @bar will be mistakenly treated as readonly.
556 AddRefEdges(NonVolatileStores, StoreRefEdges, StoreCache);
557
558 // If both load and store instruction reference the same variable
559 // we won't be able to optimize it. Add all such reference edges
560 // to RefEdges set.
561 for (const auto &VI : StoreRefEdges)
562 if (LoadRefEdges.remove(VI))
563 RefEdges.insert(VI);
564
565 unsigned RefCnt = RefEdges.size();
566 // All new reference edges inserted in two loops below are either
567 // read or write only. They will be grouped in the end of RefEdges
568 // vector, so we can use a single integer value to identify them.
569 for (const auto &VI : LoadRefEdges)
570 RefEdges.insert(VI);
571
572 unsigned FirstWORef = RefEdges.size();
573 for (const auto &VI : StoreRefEdges)
574 RefEdges.insert(VI);
575
576 Refs = RefEdges.takeVector();
577 for (; RefCnt < FirstWORef; ++RefCnt)
578 Refs[RefCnt].setReadOnly();
579
580 for (; RefCnt < Refs.size(); ++RefCnt)
581 Refs[RefCnt].setWriteOnly();
582 } else {
583 Refs = RefEdges.takeVector();
584 }
585 // Explicit add hot edges to enforce importing for designated GUIDs for
586 // sample PGO, to enable the same inlines as the profiled optimized binary.
587 for (auto &I : F.getImportGUIDs())
588 CallGraphEdges[Index.getOrInsertValueInfo(I)].updateHotness(
590 ? CalleeInfo::HotnessType::Cold
591 : CalleeInfo::HotnessType::Critical);
592
593#ifndef NDEBUG
594 // Make sure that all calls we decided could not have memprof summaries get a
595 // false value for mayHaveMemprofSummary, to ensure that this handling remains
596 // in sync with the ThinLTO backend handling.
597 if (IsThinLTO) {
598 for (const BasicBlock &BB : F) {
599 for (const Instruction &I : BB) {
600 const auto *CB = dyn_cast<CallBase>(&I);
601 if (!CB)
602 continue;
603 // We already checked these above.
604 if (CallsThatMayHaveMemprofSummary.count(CB))
605 continue;
607 }
608 }
609 }
610#endif
611
612 bool NonRenamableLocal = isNonRenamableLocal(F);
613 bool NotEligibleForImport = NonRenamableLocal ||
614 HasInlineAsmMaybeReferencingInternal ||
615 HasIndirBranchToBlockAddress || HasIFuncCall;
617 F.getLinkage(), F.getVisibility(), NotEligibleForImport,
618 /* Live = */ false, F.isDSOLocal(), F.canBeOmittedFromSymbolTable());
620 F.doesNotAccessMemory(), F.onlyReadsMemory() && !F.doesNotAccessMemory(),
621 F.hasFnAttribute(Attribute::NoRecurse), F.returnDoesNotAlias(),
622 // FIXME: refactor this to use the same code that inliner is using.
623 // Don't try to import functions with noinline attribute.
624 F.getAttributes().hasFnAttr(Attribute::NoInline),
625 F.hasFnAttribute(Attribute::AlwaysInline),
626 F.hasFnAttribute(Attribute::NoUnwind), MayThrow, HasUnknownCall,
628 std::vector<FunctionSummary::ParamAccess> ParamAccesses;
629 if (auto *SSI = GetSSICallback(F))
630 ParamAccesses = SSI->getParamAccesses(Index);
631 auto FuncSummary = std::make_unique<FunctionSummary>(
632 Flags, NumInsts, FunFlags, /*EntryCount=*/0, std::move(Refs),
633 CallGraphEdges.takeVector(), TypeTests.takeVector(),
634 TypeTestAssumeVCalls.takeVector(), TypeCheckedLoadVCalls.takeVector(),
635 TypeTestAssumeConstVCalls.takeVector(),
636 TypeCheckedLoadConstVCalls.takeVector(), std::move(ParamAccesses),
637 std::move(Callsites), std::move(Allocs));
638 if (NonRenamableLocal)
639 CantBePromoted.insert(F.getGUID());
640 Index.addGlobalValueSummary(F, std::move(FuncSummary));
641}
642
643/// Find function pointers referenced within the given vtable initializer
644/// (or subset of an initializer) \p I. The starting offset of \p I within
645/// the vtable initializer is \p StartingOffset. Any discovered function
646/// pointers are added to \p VTableFuncs along with their cumulative offset
647/// within the initializer.
648static void findFuncPointers(const Constant *I, uint64_t StartingOffset,
650 VTableFuncList &VTableFuncs) {
651 // First check if this is a function pointer.
652 if (I->getType()->isPointerTy()) {
653 auto C = I->stripPointerCasts();
654 auto A = dyn_cast<GlobalAlias>(C);
655 if (isa<Function>(C) || (A && isa<Function>(A->getAliasee()))) {
656 auto GV = dyn_cast<GlobalValue>(C);
657 assert(GV);
658 // We can disregard __cxa_pure_virtual as a possible call target, as
659 // calls to pure virtuals are UB.
660 if (GV && GV->getName() != "__cxa_pure_virtual")
661 VTableFuncs.push_back({Index.getOrInsertValueInfo(GV), StartingOffset});
662 return;
663 }
664 }
665
666 // Walk through the elements in the constant struct or array and recursively
667 // look for virtual function pointers.
668 const DataLayout &DL = M.getDataLayout();
669 if (auto *C = dyn_cast<ConstantStruct>(I)) {
670 StructType *STy = dyn_cast<StructType>(C->getType());
671 assert(STy);
672 const StructLayout *SL = DL.getStructLayout(C->getType());
673
674 for (auto EI : llvm::enumerate(STy->elements())) {
675 auto Offset = SL->getElementOffset(EI.index());
676 unsigned Op = SL->getElementContainingOffset(Offset);
677 findFuncPointers(cast<Constant>(I->getOperand(Op)),
678 StartingOffset + Offset, M, Index, VTableFuncs);
679 }
680 } else if (auto *C = dyn_cast<ConstantArray>(I)) {
681 ArrayType *ATy = C->getType();
682 Type *EltTy = ATy->getElementType();
683 uint64_t EltSize = DL.getTypeAllocSize(EltTy);
684 for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) {
685 findFuncPointers(cast<Constant>(I->getOperand(i)),
686 StartingOffset + i * EltSize, M, Index, VTableFuncs);
687 }
688 }
689}
690
691// Identify the function pointers referenced by vtable definition \p V.
693 const GlobalVariable &V, const Module &M,
694 VTableFuncList &VTableFuncs) {
695 if (!V.isConstant())
696 return;
697
698 findFuncPointers(V.getInitializer(), /*StartingOffset=*/0, M, Index,
699 VTableFuncs);
700
701#ifndef NDEBUG
702 // Validate that the VTableFuncs list is ordered by offset.
703 uint64_t PrevOffset = 0;
704 for (auto &P : VTableFuncs) {
705 // The findVFuncPointers traversal should have encountered the
706 // functions in offset order. We need to use ">=" since PrevOffset
707 // starts at 0.
708 assert(P.VTableOffset >= PrevOffset);
709 PrevOffset = P.VTableOffset;
710 }
711#endif
712}
713
714/// Record vtable definition \p V for each type metadata it references.
715static void
717 const GlobalVariable &V,
719 for (MDNode *Type : Types) {
720 auto TypeID = Type->getOperand(1).get();
721
723 cast<ConstantInt>(
724 cast<ConstantAsMetadata>(Type->getOperand(0))->getValue())
725 ->getZExtValue();
726
727 if (auto *TypeId = dyn_cast<MDString>(TypeID))
728 Index.getOrInsertTypeIdCompatibleVtableSummary(TypeId->getString())
729 .push_back({Offset, Index.getOrInsertValueInfo(&V)});
730 }
731}
732
734 const GlobalVariable &V,
735 DenseSet<GlobalValue::GUID> &CantBePromoted,
736 const Module &M,
740 bool HasBlockAddress = findRefEdges(Index, &V, RefEdges, Visited);
741 bool NonRenamableLocal = isNonRenamableLocal(V);
743 V.getLinkage(), V.getVisibility(), NonRenamableLocal,
744 /* Live = */ false, V.isDSOLocal(), V.canBeOmittedFromSymbolTable());
745
746 VTableFuncList VTableFuncs;
747 // If splitting is not enabled, then we compute the summary information
748 // necessary for index-based whole program devirtualization.
749 if (!Index.enableSplitLTOUnit()) {
750 Types.clear();
751 V.getMetadata(LLVMContext::MD_type, Types);
752 if (!Types.empty()) {
753 // Identify the function pointers referenced by this vtable definition.
754 computeVTableFuncs(Index, V, M, VTableFuncs);
755
756 // Record this vtable definition for each type metadata it references.
758 }
759 }
760
761 // Don't mark variables we won't be able to internalize as read/write-only.
762 bool CanBeInternalized =
763 !V.hasComdat() && !V.hasAppendingLinkage() && !V.isInterposable() &&
764 !V.hasAvailableExternallyLinkage() && !V.hasDLLExportStorageClass();
765 bool Constant = V.isConstant();
766 GlobalVarSummary::GVarFlags VarFlags(CanBeInternalized,
767 Constant ? false : CanBeInternalized,
768 Constant, V.getVCallVisibility());
769 auto GVarSummary = std::make_unique<GlobalVarSummary>(Flags, VarFlags,
770 RefEdges.takeVector());
771 if (NonRenamableLocal)
772 CantBePromoted.insert(V.getGUID());
773 if (HasBlockAddress)
774 GVarSummary->setNotEligibleToImport();
775 if (!VTableFuncs.empty())
776 GVarSummary->setVTableFuncs(VTableFuncs);
777 Index.addGlobalValueSummary(V, std::move(GVarSummary));
778}
779
781 DenseSet<GlobalValue::GUID> &CantBePromoted) {
782 // Skip summary for indirect function aliases as summary for aliasee will not
783 // be emitted.
784 const GlobalObject *Aliasee = A.getAliaseeObject();
785 if (isa<GlobalIFunc>(Aliasee))
786 return;
787 bool NonRenamableLocal = isNonRenamableLocal(A);
789 A.getLinkage(), A.getVisibility(), NonRenamableLocal,
790 /* Live = */ false, A.isDSOLocal(), A.canBeOmittedFromSymbolTable());
791 auto AS = std::make_unique<AliasSummary>(Flags);
792 auto AliaseeVI = Index.getValueInfo(Aliasee->getGUID());
793 assert(AliaseeVI && "Alias expects aliasee summary to be available");
794 assert(AliaseeVI.getSummaryList().size() == 1 &&
795 "Expected a single entry per aliasee in per-module index");
796 AS->setAliasee(AliaseeVI, AliaseeVI.getSummaryList()[0].get());
797 if (NonRenamableLocal)
798 CantBePromoted.insert(A.getGUID());
799 Index.addGlobalValueSummary(A, std::move(AS));
800}
801
802// Set LiveRoot flag on entries matching the given value name.
804 if (ValueInfo VI = Index.getValueInfo(GlobalValue::getGUID(Name)))
805 for (const auto &Summary : VI.getSummaryList())
806 Summary->setLive(true);
807}
808
810 const Module &M,
811 std::function<BlockFrequencyInfo *(const Function &F)> GetBFICallback,
813 std::function<const StackSafetyInfo *(const Function &F)> GetSSICallback) {
814 assert(PSI);
815 bool EnableSplitLTOUnit = false;
816 bool UnifiedLTO = false;
817 if (auto *MD = mdconst::extract_or_null<ConstantInt>(
818 M.getModuleFlag("EnableSplitLTOUnit")))
819 EnableSplitLTOUnit = MD->getZExtValue();
820 if (auto *MD =
821 mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("UnifiedLTO")))
822 UnifiedLTO = MD->getZExtValue();
823 ModuleSummaryIndex Index(/*HaveGVs=*/true, EnableSplitLTOUnit, UnifiedLTO);
824
825 // Identify the local values in the llvm.used and llvm.compiler.used sets,
826 // which should not be exported as they would then require renaming and
827 // promotion, but we may have opaque uses e.g. in inline asm. We collect them
828 // here because we use this information to mark functions containing inline
829 // assembly calls as not importable.
832 // First collect those in the llvm.used set.
833 collectUsedGlobalVariables(M, Used, /*CompilerUsed=*/false);
834 // Next collect those in the llvm.compiler.used set.
835 collectUsedGlobalVariables(M, Used, /*CompilerUsed=*/true);
836 DenseSet<GlobalValue::GUID> CantBePromoted;
837 for (auto *V : Used) {
838 if (V->hasLocalLinkage()) {
839 LocalsUsed.insert(V);
840 CantBePromoted.insert(V->getGUID());
841 }
842 }
843
844 bool HasLocalInlineAsmSymbol = false;
845 if (!M.getModuleInlineAsm().empty()) {
846 // Collect the local values defined by module level asm, and set up
847 // summaries for these symbols so that they can be marked as NoRename,
848 // to prevent export of any use of them in regular IR that would require
849 // renaming within the module level asm. Note we don't need to create a
850 // summary for weak or global defs, as they don't need to be flagged as
851 // NoRename, and defs in module level asm can't be imported anyway.
852 // Also, any values used but not defined within module level asm should
853 // be listed on the llvm.used or llvm.compiler.used global and marked as
854 // referenced from there.
857 // Symbols not marked as Weak or Global are local definitions.
860 return;
861 HasLocalInlineAsmSymbol = true;
862 GlobalValue *GV = M.getNamedValue(Name);
863 if (!GV)
864 return;
865 assert(GV->isDeclaration() && "Def in module asm already has definition");
868 /* NotEligibleToImport = */ true,
869 /* Live = */ true,
870 /* Local */ GV->isDSOLocal(), GV->canBeOmittedFromSymbolTable());
871 CantBePromoted.insert(GV->getGUID());
872 // Create the appropriate summary type.
873 if (Function *F = dyn_cast<Function>(GV)) {
874 std::unique_ptr<FunctionSummary> Summary =
875 std::make_unique<FunctionSummary>(
876 GVFlags, /*InstCount=*/0,
878 F->hasFnAttribute(Attribute::ReadNone),
879 F->hasFnAttribute(Attribute::ReadOnly),
880 F->hasFnAttribute(Attribute::NoRecurse),
881 F->returnDoesNotAlias(),
882 /* NoInline = */ false,
883 F->hasFnAttribute(Attribute::AlwaysInline),
884 F->hasFnAttribute(Attribute::NoUnwind),
885 /* MayThrow */ true,
886 /* HasUnknownCall */ true,
887 /* MustBeUnreachable */ false},
888 /*EntryCount=*/0, ArrayRef<ValueInfo>{},
897 Index.addGlobalValueSummary(*GV, std::move(Summary));
898 } else {
899 std::unique_ptr<GlobalVarSummary> Summary =
900 std::make_unique<GlobalVarSummary>(
901 GVFlags,
903 false, false, cast<GlobalVariable>(GV)->isConstant(),
906 Index.addGlobalValueSummary(*GV, std::move(Summary));
907 }
908 });
909 }
910
911 bool IsThinLTO = true;
912 if (auto *MD =
913 mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("ThinLTO")))
914 IsThinLTO = MD->getZExtValue();
915
916 // Compute summaries for all functions defined in module, and save in the
917 // index.
918 for (const auto &F : M) {
919 if (F.isDeclaration())
920 continue;
921
922 DominatorTree DT(const_cast<Function &>(F));
923 BlockFrequencyInfo *BFI = nullptr;
924 std::unique_ptr<BlockFrequencyInfo> BFIPtr;
925 if (GetBFICallback)
926 BFI = GetBFICallback(F);
927 else if (F.hasProfileData()) {
928 LoopInfo LI{DT};
929 BranchProbabilityInfo BPI{F, LI};
930 BFIPtr = std::make_unique<BlockFrequencyInfo>(F, BPI, LI);
931 BFI = BFIPtr.get();
932 }
933
934 computeFunctionSummary(Index, M, F, BFI, PSI, DT,
935 !LocalsUsed.empty() || HasLocalInlineAsmSymbol,
936 CantBePromoted, IsThinLTO, GetSSICallback);
937 }
938
939 // Compute summaries for all variables defined in module, and save in the
940 // index.
942 for (const GlobalVariable &G : M.globals()) {
943 if (G.isDeclaration())
944 continue;
945 computeVariableSummary(Index, G, CantBePromoted, M, Types);
946 }
947
948 // Compute summaries for all aliases defined in module, and save in the
949 // index.
950 for (const GlobalAlias &A : M.aliases())
951 computeAliasSummary(Index, A, CantBePromoted);
952
953 // Iterate through ifuncs, set their resolvers all alive.
954 for (const GlobalIFunc &I : M.ifuncs()) {
955 I.applyAlongResolverPath([&Index](const GlobalValue &GV) {
956 Index.getGlobalValueSummary(GV)->setLive(true);
957 });
958 }
959
960 for (auto *V : LocalsUsed) {
961 auto *Summary = Index.getGlobalValueSummary(*V);
962 assert(Summary && "Missing summary for global value");
963 Summary->setNotEligibleToImport();
964 }
965
966 // The linker doesn't know about these LLVM produced values, so we need
967 // to flag them as live in the index to ensure index-based dead value
968 // analysis treats them as live roots of the analysis.
969 setLiveRoot(Index, "llvm.used");
970 setLiveRoot(Index, "llvm.compiler.used");
971 setLiveRoot(Index, "llvm.global_ctors");
972 setLiveRoot(Index, "llvm.global_dtors");
973 setLiveRoot(Index, "llvm.global.annotations");
974
975 for (auto &GlobalList : Index) {
976 // Ignore entries for references that are undefined in the current module.
977 if (GlobalList.second.SummaryList.empty())
978 continue;
979
980 assert(GlobalList.second.SummaryList.size() == 1 &&
981 "Expected module's index to have one summary per GUID");
982 auto &Summary = GlobalList.second.SummaryList[0];
983 if (!IsThinLTO) {
984 Summary->setNotEligibleToImport();
985 continue;
986 }
987
988 bool AllRefsCanBeExternallyReferenced =
989 llvm::all_of(Summary->refs(), [&](const ValueInfo &VI) {
990 return !CantBePromoted.count(VI.getGUID());
991 });
992 if (!AllRefsCanBeExternallyReferenced) {
993 Summary->setNotEligibleToImport();
994 continue;
995 }
996
997 if (auto *FuncSummary = dyn_cast<FunctionSummary>(Summary.get())) {
998 bool AllCallsCanBeExternallyReferenced = llvm::all_of(
999 FuncSummary->calls(), [&](const FunctionSummary::EdgeTy &Edge) {
1000 return !CantBePromoted.count(Edge.first.getGUID());
1001 });
1002 if (!AllCallsCanBeExternallyReferenced)
1003 Summary->setNotEligibleToImport();
1004 }
1005 }
1006
1007 if (!ModuleSummaryDotFile.empty()) {
1008 std::error_code EC;
1009 raw_fd_ostream OSDot(ModuleSummaryDotFile, EC, sys::fs::OpenFlags::OF_None);
1010 if (EC)
1011 report_fatal_error(Twine("Failed to open dot file ") +
1012 ModuleSummaryDotFile + ": " + EC.message() + "\n");
1013 Index.exportToDot(OSDot, {});
1014 }
1015
1016 return Index;
1017}
1018
1019AnalysisKey ModuleSummaryIndexAnalysis::Key;
1020
1024 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
1025 bool NeedSSI = needsParamAccessSummary(M);
1027 M,
1028 [&FAM](const Function &F) {
1030 *const_cast<Function *>(&F));
1031 },
1032 &PSI,
1033 [&FAM, NeedSSI](const Function &F) -> const StackSafetyInfo * {
1034 return NeedSSI ? &FAM.getResult<StackSafetyAnalysis>(
1035 const_cast<Function &>(F))
1036 : nullptr;
1037 });
1038}
1039
1041
1043 "Module Summary Analysis", false, true)
1049
1051 return new ModuleSummaryIndexWrapperPass();
1052}
1053
1055 : ModulePass(ID) {
1057}
1058
1060 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
1061 bool NeedSSI = needsParamAccessSummary(M);
1063 M,
1064 [this](const Function &F) {
1065 return &(this->getAnalysis<BlockFrequencyInfoWrapperPass>(
1066 *const_cast<Function *>(&F))
1067 .getBFI());
1068 },
1069 PSI,
1070 [&](const Function &F) -> const StackSafetyInfo * {
1071 return NeedSSI ? &getAnalysis<StackSafetyInfoWrapperPass>(
1072 const_cast<Function &>(F))
1073 .getResult()
1074 : nullptr;
1075 }));
1076 return false;
1077}
1078
1080 Index.reset();
1081 return false;
1082}
1083
1085 AU.setPreservesAll();
1089}
1090
1092
1098}
1099
1101 AnalysisUsage &AU) const {
1102 AU.setPreservesAll();
1103}
1104
1106 const ModuleSummaryIndex *Index) {
1108}
1109
1111 "Module summary info", false, true)
1112
1114 if (!CB)
1115 return false;
1116 if (CB->isDebugOrPseudoInst())
1117 return false;
1118 auto *CI = dyn_cast<CallInst>(CB);
1119 auto *CalledValue = CB->getCalledOperand();
1120 auto *CalledFunction = CB->getCalledFunction();
1121 if (CalledValue && !CalledFunction) {
1122 CalledValue = CalledValue->stripPointerCasts();
1123 // Stripping pointer casts can reveal a called function.
1124 CalledFunction = dyn_cast<Function>(CalledValue);
1125 }
1126 // Check if this is an alias to a function. If so, get the
1127 // called aliasee for the checks below.
1128 if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
1129 assert(!CalledFunction &&
1130 "Expected null called function in callsite for alias");
1131 CalledFunction = dyn_cast<Function>(GA->getAliaseeObject());
1132 }
1133 // Check if this is a direct call to a known function or a known
1134 // intrinsic, or an indirect call with profile data.
1135 if (CalledFunction) {
1136 if (CI && CalledFunction->isIntrinsic())
1137 return false;
1138 } else {
1139 // TODO: For now skip indirect calls. See comments in
1140 // computeFunctionSummary for what is needed to handle this.
1141 return false;
1142 }
1143 return true;
1144}
aarch64 promote const
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isConstant(const MachineInstr &MI)
This file contains the simple types necessary to represent the attributes associated with functions a...
basic Basic Alias true
block Block Frequency Analysis
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Definition: CommandLine.h:693
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file defines the DenseSet and SmallDenseSet classes.
std::string Name
iv users
Definition: IVUsers.cpp:48
Interface to identify indirect call promotion candidates.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
This file implements a map that provides insertion order iteration.
This file contains the declarations for metadata subclasses.
static void addVCallToSet(DevirtCallSite Call, GlobalValue::GUID Guid, SetVector< FunctionSummary::VFuncId, std::vector< FunctionSummary::VFuncId > > &VCalls, SetVector< FunctionSummary::ConstVCall, std::vector< FunctionSummary::ConstVCall > > &ConstVCalls)
Determine whether this call has all constant integer arguments (excluding "this") and summarize it to...
static void computeVTableFuncs(ModuleSummaryIndex &Index, const GlobalVariable &V, const Module &M, VTableFuncList &VTableFuncs)
static void computeAliasSummary(ModuleSummaryIndex &Index, const GlobalAlias &A, DenseSet< GlobalValue::GUID > &CantBePromoted)
static bool findRefEdges(ModuleSummaryIndex &Index, const User *CurUser, SetVector< ValueInfo, std::vector< ValueInfo > > &RefEdges, SmallPtrSet< const User *, 8 > &Visited)
static void findFuncPointers(const Constant *I, uint64_t StartingOffset, const Module &M, ModuleSummaryIndex &Index, VTableFuncList &VTableFuncs)
Find function pointers referenced within the given vtable initializer (or subset of an initializer) I...
static void computeVariableSummary(ModuleSummaryIndex &Index, const GlobalVariable &V, DenseSet< GlobalValue::GUID > &CantBePromoted, const Module &M, SmallVectorImpl< MDNode * > &Types)
static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name)
static CalleeInfo::HotnessType getHotness(uint64_t ProfileCount, ProfileSummaryInfo *PSI)
static bool isNonVolatileLoad(const Instruction *I)
cl::opt< bool > ScalePartialSampleProfileWorkingSetSize
static bool isNonRenamableLocal(const GlobalValue &GV)
static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M, const Function &F, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, DominatorTree &DT, bool HasLocalsInUsedOrAsm, DenseSet< GlobalValue::GUID > &CantBePromoted, bool IsThinLTO, std::function< const StackSafetyInfo *(const Function &F)> GetSSICallback)
static cl::opt< FunctionSummary::ForceSummaryHotnessType, true > FSEC("force-summary-edges-cold", cl::Hidden, cl::location(ForceSummaryEdgesCold), cl::desc("Force all edges in the function summary to cold"), cl::values(clEnumValN(FunctionSummary::FSHT_None, "none", "None."), clEnumValN(FunctionSummary::FSHT_AllNonCritical, "all-non-critical", "All non-critical edges."), clEnumValN(FunctionSummary::FSHT_All, "all", "All edges.")))
static bool mustBeUnreachableFunction(const Function &F)
static bool isNonVolatileStore(const Instruction *I)
module summary analysis
static cl::opt< std::string > ModuleSummaryDotFile("module-summary-dot-file", cl::Hidden, cl::value_desc("filename"), cl::desc("File to emit dot graph of new summary into"))
static void addIntrinsicToSummary(const CallInst *CI, SetVector< GlobalValue::GUID, std::vector< GlobalValue::GUID > > &TypeTests, SetVector< FunctionSummary::VFuncId, std::vector< FunctionSummary::VFuncId > > &TypeTestAssumeVCalls, SetVector< FunctionSummary::VFuncId, std::vector< FunctionSummary::VFuncId > > &TypeCheckedLoadVCalls, SetVector< FunctionSummary::ConstVCall, std::vector< FunctionSummary::ConstVCall > > &TypeTestAssumeConstVCalls, SetVector< FunctionSummary::ConstVCall, std::vector< FunctionSummary::ConstVCall > > &TypeCheckedLoadConstVCalls, DominatorTree &DT)
If this intrinsic call requires that we add information to the function summary, do so via the non-co...
static void recordTypeIdCompatibleVtableReferences(ModuleSummaryIndex &Index, const GlobalVariable &V, SmallVectorImpl< MDNode * > &Types)
Record vtable definition V for each type metadata it references.
This is the interface to build a ModuleSummaryIndex for a module.
ModuleSummaryIndex.h This file contains the declarations the classes that hold the module index and s...
Module.h This file contains the declarations for the Module class.
#define P(N)
FunctionAnalysisManager FAM
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:59
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This defines the Use class.
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:348
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:500
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesAll()
Set by analyses that do not transform their input at all.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
static BlockAddress * get(Function *F, BasicBlock *BB)
Return a BlockAddress for the specified function and basic block.
Definition: Constants.cpp:1762
Analysis pass which computes BlockFrequencyInfo.
Legacy analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Analysis providing branch probability information.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1259
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1481
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1426
This class represents a function call, abstracting a target machine's calling convention.
This is an important base class in LLVM.
Definition: Constant.h:41
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:162
std::pair< ValueInfo, CalleeInfo > EdgeTy
<CalleeValueInfo, CalleeInfo> call edge pair.
ForceSummaryHotnessType
Types for -force-summary-edges-cold debugging option.
Class to represent profile counts.
Definition: Function.h:277
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:230
bool isDSOLocal() const
Definition: GlobalValue.h:305
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:274
bool hasLocalLinkage() const
Definition: GlobalValue.h:527
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:594
@ DefaultVisibility
The GV is visible.
Definition: GlobalValue.h:67
static GUID getGUID(StringRef GlobalName)
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:590
bool hasSection() const
Definition: GlobalValue.h:290
bool canBeOmittedFromSymbolTable() const
True if GV can be left out of the object symbol table.
Definition: Globals.cpp:393
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:59
ArrayRef< InstrProfValueData > getPromotionCandidatesForInstruction(const Instruction *I, uint32_t &NumVals, uint64_t &TotalCount, uint32_t &NumCandidates)
Returns reference to array of InstrProfValueData for the given instruction I.
Legacy wrapper pass to provide the ModuleSummaryIndex object.
ImmutableModuleSummaryIndexWrapperPass(const ModuleSummaryIndex *Index=nullptr)
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
ImmutablePass class - This class is used to provide information that does not need to be run.
Definition: Pass.h:282
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:658
Metadata node.
Definition: Metadata.h:1067
This class implements a map that also provides access to all stored values in a deterministic order.
Definition: MapVector.h:36
VectorType takeVector()
Clear the MapVector and return the underlying vector.
Definition: MapVector.h:55
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:251
Result run(Module &M, ModuleAnalysisManager &AM)
Legacy wrapper pass to provide the ModuleSummaryIndex object.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
bool doFinalization(Module &M) override
doFinalization - Virtual method overriden by subclasses to do any necessary clean up after all passes...
bool runOnModule(Module &M) override
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
Class to hold module path string table and global value map, and encapsulate methods for operating on...
static void CollectAsmSymbols(const Module &M, function_ref< void(StringRef, object::BasicSymbolRef::Flags)> AsmSymbol)
Parse inline ASM and collect the symbols that are defined or referenced in the current module.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
Analysis providing profile information.
std::optional< uint64_t > getProfileCount(const CallBase &CallInst, BlockFrequencyInfo *BFI, bool AllowSynthetic=false) const
Returns the profile count for CallInst.
bool isColdCount(uint64_t C) const
Returns true if count C is considered cold.
bool hasPartialSampleProfile() const
Returns true if module M has partial-profile sample profile.
bool isHotCount(uint64_t C) const
Returns true if count C is considered hot.
A vector that has set insertion semantics.
Definition: SetVector.h:57
bool remove(const value_type &X)
Remove an item from the set vector.
Definition: SetVector.h:188
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:98
Vector takeVector()
Clear the SetVector and return the underlying vector.
Definition: SetVector.h:87
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
bool empty() const
Definition: SmallVector.h:94
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StackSafetyInfo wrapper for the new pass manager.
StackSafetyInfo wrapper for the legacy pass manager.
Interface to access stack safety analysis results for single function.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
Definition: DataLayout.h:622
unsigned getElementContainingOffset(uint64_t FixedOffset) const
Given a valid byte offset into the structure, returns the structure index that contains it.
Definition: DataLayout.cpp:92
TypeSize getElementOffset(unsigned Idx) const
Definition: DataLayout.h:651
Class to represent struct types.
Definition: DerivedTypes.h:216
ArrayRef< Type * > elements() const
Definition: DerivedTypes.h:333
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
LLVM Value Representation.
Definition: Value.h:74
iterator_range< use_iterator > uses()
Definition: Value.h:376
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:97
Helper class to iterate through stack ids in both metadata (memprof MIB and callsite) and the corresp...
CallStackIterator end() const
CallStackIterator beginAfterSharedPrefix(CallStack &Other)
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:470
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
Definition: CommandLine.h:718
LocationClass< Ty > location(Ty &L)
Definition: CommandLine.h:470
AllocationType getMIBAllocType(const MDNode *MIB)
Returns the allocation type from an MIB metadata node.
MDNode * getMIBStackNode(const MDNode *MIB)
Returns the stack node from an MIB metadata node.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:329
@ Offset
Definition: DWP.cpp:456
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1731
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
Definition: STLExtras.h:2380
FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold
bool needsParamAccessSummary(const Module &M)
ModuleSummaryIndex buildModuleSummaryIndex(const Module &M, std::function< BlockFrequencyInfo *(const Function &F)> GetBFICallback, ProfileSummaryInfo *PSI, std::function< const StackSafetyInfo *(const Function &F)> GetSSICallback=[](const Function &F) -> const StackSafetyInfo *{ return nullptr;})
Direct function to compute a ModuleSummaryIndex from a given module.
void initializeModuleSummaryIndexWrapperPassPass(PassRegistry &)
std::vector< VirtFuncOffset > VTableFuncList
List of functions referenced by a particular vtable definition.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1738
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
void findDevirtualizableCallsForTypeCheckedLoad(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< Instruction * > &LoadedPtrs, SmallVectorImpl< Instruction * > &Preds, bool &HasNonCallUses, const CallInst *CI, DominatorTree &DT)
Given a call to the intrinsic @llvm.type.checked.load, find all devirtualizable call sites based on t...
ModulePass * createModuleSummaryIndexWrapperPass()
ImmutablePass * createImmutableModuleSummaryIndexWrapperPass(const ModuleSummaryIndex *Index)
void initializeImmutableModuleSummaryIndexWrapperPassPass(PassRegistry &)
bool mayHaveMemprofSummary(const CallBase *CB)
Returns true if the instruction could have memprof metadata, used to ensure consistency between summa...
void findDevirtualizableCallsForTypeTest(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< CallInst * > &Assumes, const CallInst *CI, DominatorTree &DT)
Given a call to the intrinsic @llvm.type.test, find all devirtualizable call sites based on the call ...
GlobalVariable * collectUsedGlobalVariables(const Module &M, SmallVectorImpl< GlobalValue * > &Vec, bool CompilerUsed)
Given "llvm.used" or "llvm.compiler.used" as a global name, collect the initializer elements of that ...
Definition: Module.cpp:822
Summary of memprof metadata on allocations.
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition: Analysis.h:26
A call site that could be devirtualized.
A specification for a virtual function call with all constant integer arguments.
Flags specific to function summaries.
An "identifier" for a virtual function.
Group flags (Linkage, NotEligibleToImport, etc.) as a bitfield.
Summary of a single MIB in a memprof metadata on allocations.
Struct that holds a reference to a particular GUID in a global value summary.