LLVM 17.0.0git
Inliner.cpp
Go to the documentation of this file.
1//===- Inliner.cpp - Code common to all inliners --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the mechanics required to implement inlining without
10// missing any calls and updating the call graph. The decisions of which calls
11// are profitable to inline are implemented elsewhere.
12//
13//===----------------------------------------------------------------------===//
14
16#include "llvm/ADT/DenseMap.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/ScopeExit.h"
20#include "llvm/ADT/SetVector.h"
23#include "llvm/ADT/Statistic.h"
25#include "llvm/ADT/StringRef.h"
38#include "llvm/IR/Attributes.h"
39#include "llvm/IR/BasicBlock.h"
40#include "llvm/IR/DebugLoc.h"
43#include "llvm/IR/Function.h"
45#include "llvm/IR/Instruction.h"
48#include "llvm/IR/Metadata.h"
49#include "llvm/IR/Module.h"
50#include "llvm/IR/PassManager.h"
51#include "llvm/IR/User.h"
52#include "llvm/IR/Value.h"
53#include "llvm/Pass.h"
56#include "llvm/Support/Debug.h"
62#include <algorithm>
63#include <cassert>
64#include <functional>
65#include <utility>
66#include <vector>
67
68using namespace llvm;
69
70#define DEBUG_TYPE "inline"
71
72STATISTIC(NumInlined, "Number of functions inlined");
73STATISTIC(NumDeleted, "Number of functions deleted because all callers found");
74
76 "intra-scc-cost-multiplier", cl::init(2), cl::Hidden,
78 "Cost multiplier to multiply onto inlined call sites where the "
79 "new call was previously an intra-SCC call (not relevant when the "
80 "original call was already intra-SCC). This can accumulate over "
81 "multiple inlinings (e.g. if a call site already had a cost "
82 "multiplier and one of its inlined calls was also subject to "
83 "this, the inlined call would have the original multiplier "
84 "multiplied by intra-scc-cost-multiplier). This is to prevent tons of "
85 "inlining through a child SCC which can cause terrible compile times"));
86
87/// A flag for test, so we can print the content of the advisor when running it
88/// as part of the default (e.g. -O3) pipeline.
89static cl::opt<bool> KeepAdvisorForPrinting("keep-inline-advisor-for-printing",
90 cl::init(false), cl::Hidden);
91
92/// Allows printing the contents of the advisor after each SCC inliner pass.
93static cl::opt<bool>
94 EnablePostSCCAdvisorPrinting("enable-scc-inline-advisor-printing",
95 cl::init(false), cl::Hidden);
96
97
99 "cgscc-inline-replay", cl::init(""), cl::value_desc("filename"),
100 cl::desc(
101 "Optimization remarks file containing inline remarks to be replayed "
102 "by cgscc inlining."),
103 cl::Hidden);
104
106 "cgscc-inline-replay-scope",
107 cl::init(ReplayInlinerSettings::Scope::Function),
108 cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function",
109 "Replay on functions that have remarks associated "
110 "with them (default)"),
111 clEnumValN(ReplayInlinerSettings::Scope::Module, "Module",
112 "Replay on the entire module")),
113 cl::desc("Whether inline replay should be applied to the entire "
114 "Module or just the Functions (default) that are present as "
115 "callers in remarks during cgscc inlining."),
116 cl::Hidden);
117
119 "cgscc-inline-replay-fallback",
120 cl::init(ReplayInlinerSettings::Fallback::Original),
123 ReplayInlinerSettings::Fallback::Original, "Original",
124 "All decisions not in replay send to original advisor (default)"),
125 clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline,
126 "AlwaysInline", "All decisions not in replay are inlined"),
127 clEnumValN(ReplayInlinerSettings::Fallback::NeverInline, "NeverInline",
128 "All decisions not in replay are not inlined")),
129 cl::desc(
130 "How cgscc inline replay treats sites that don't come from the replay. "
131 "Original: defers to original advisor, AlwaysInline: inline all sites "
132 "not in replay, NeverInline: inline no sites not in replay"),
133 cl::Hidden);
134
136 "cgscc-inline-replay-format",
137 cl::init(CallSiteFormat::Format::LineColumnDiscriminator),
139 clEnumValN(CallSiteFormat::Format::Line, "Line", "<Line Number>"),
140 clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn",
141 "<Line Number>:<Column Number>"),
142 clEnumValN(CallSiteFormat::Format::LineDiscriminator,
143 "LineDiscriminator", "<Line Number>.<Discriminator>"),
144 clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator,
145 "LineColumnDiscriminator",
146 "<Line Number>:<Column Number>.<Discriminator> (default)")),
147 cl::desc("How cgscc inline replay file is formatted"), cl::Hidden);
148
149/// Return true if the specified inline history ID
150/// indicates an inline history that includes the specified function.
152 Function *F, int InlineHistoryID,
153 const SmallVectorImpl<std::pair<Function *, int>> &InlineHistory) {
154 while (InlineHistoryID != -1) {
155 assert(unsigned(InlineHistoryID) < InlineHistory.size() &&
156 "Invalid inline history ID");
157 if (InlineHistory[InlineHistoryID].first == F)
158 return true;
159 InlineHistoryID = InlineHistory[InlineHistoryID].second;
160 }
161 return false;
162}
163
165InlinerPass::getAdvisor(const ModuleAnalysisManagerCGSCCProxy::Result &MAM,
167 if (OwnedAdvisor)
168 return *OwnedAdvisor;
169
171 if (!IAA) {
172 // It should still be possible to run the inliner as a stand-alone SCC pass,
173 // for test scenarios. In that case, we default to the
174 // DefaultInlineAdvisor, which doesn't need to keep state between SCC pass
175 // runs. It also uses just the default InlineParams.
176 // In this case, we need to use the provided FAM, which is valid for the
177 // duration of the inliner pass, and thus the lifetime of the owned advisor.
178 // The one we would get from the MAM can be invalidated as a result of the
179 // inliner's activity.
180 OwnedAdvisor = std::make_unique<DefaultInlineAdvisor>(
181 M, FAM, getInlineParams(),
183
184 if (!CGSCCInlineReplayFile.empty())
185 OwnedAdvisor = getReplayInlineAdvisor(
186 M, FAM, M.getContext(), std::move(OwnedAdvisor),
187 ReplayInlinerSettings{CGSCCInlineReplayFile,
188 CGSCCInlineReplayScope,
189 CGSCCInlineReplayFallback,
190 {CGSCCInlineReplayFormat}},
191 /*EmitRemarks=*/true,
193
194 return *OwnedAdvisor;
195 }
196 assert(IAA->getAdvisor() &&
197 "Expected a present InlineAdvisorAnalysis also have an "
198 "InlineAdvisor initialized");
199 return *IAA->getAdvisor();
200}
201
204 CGSCCUpdateResult &UR) {
205 const auto &MAMProxy =
207 bool Changed = false;
208
209 assert(InitialC.size() > 0 && "Cannot handle an empty SCC!");
210 Module &M = *InitialC.begin()->getFunction().getParent();
211 ProfileSummaryInfo *PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(M);
212
215 .getManager();
216
217 InlineAdvisor &Advisor = getAdvisor(MAMProxy, FAM, M);
218 Advisor.onPassEntry(&InitialC);
219
220 auto AdvisorOnExit = make_scope_exit([&] { Advisor.onPassExit(&InitialC); });
221
222 // We use a single common worklist for calls across the entire SCC. We
223 // process these in-order and append new calls introduced during inlining to
224 // the end. The PriorityInlineOrder is optional here, in which the smaller
225 // callee would have a higher priority to inline.
226 //
227 // Note that this particular order of processing is actually critical to
228 // avoid very bad behaviors. Consider *highly connected* call graphs where
229 // each function contains a small amount of code and a couple of calls to
230 // other functions. Because the LLVM inliner is fundamentally a bottom-up
231 // inliner, it can handle gracefully the fact that these all appear to be
232 // reasonable inlining candidates as it will flatten things until they become
233 // too big to inline, and then move on and flatten another batch.
234 //
235 // However, when processing call edges *within* an SCC we cannot rely on this
236 // bottom-up behavior. As a consequence, with heavily connected *SCCs* of
237 // functions we can end up incrementally inlining N calls into each of
238 // N functions because each incremental inlining decision looks good and we
239 // don't have a topological ordering to prevent explosions.
240 //
241 // To compensate for this, we don't process transitive edges made immediate
242 // by inlining until we've done one pass of inlining across the entire SCC.
243 // Large, highly connected SCCs still lead to some amount of code bloat in
244 // this model, but it is uniformly spread across all the functions in the SCC
245 // and eventually they all become too large to inline, rather than
246 // incrementally maknig a single function grow in a super linear fashion.
248
249 // Populate the initial list of calls in this SCC.
250 for (auto &N : InitialC) {
251 auto &ORE =
253 // We want to generally process call sites top-down in order for
254 // simplifications stemming from replacing the call with the returned value
255 // after inlining to be visible to subsequent inlining decisions.
256 // FIXME: Using instructions sequence is a really bad way to do this.
257 // Instead we should do an actual RPO walk of the function body.
258 for (Instruction &I : instructions(N.getFunction()))
259 if (auto *CB = dyn_cast<CallBase>(&I))
260 if (Function *Callee = CB->getCalledFunction()) {
261 if (!Callee->isDeclaration())
262 Calls.push_back({CB, -1});
263 else if (!isa<IntrinsicInst>(I)) {
264 using namespace ore;
265 setInlineRemark(*CB, "unavailable definition");
266 ORE.emit([&]() {
267 return OptimizationRemarkMissed(DEBUG_TYPE, "NoDefinition", &I)
268 << NV("Callee", Callee) << " will not be inlined into "
269 << NV("Caller", CB->getCaller())
270 << " because its definition is unavailable"
271 << setIsVerbose();
272 });
273 }
274 }
275 }
276 if (Calls.empty())
277 return PreservedAnalyses::all();
278
279 // Capture updatable variable for the current SCC.
280 auto *C = &InitialC;
281
282 // When inlining a callee produces new call sites, we want to keep track of
283 // the fact that they were inlined from the callee. This allows us to avoid
284 // infinite inlining in some obscure cases. To represent this, we use an
285 // index into the InlineHistory vector.
287
288 // Track a set vector of inlined callees so that we can augment the caller
289 // with all of their edges in the call graph before pruning out the ones that
290 // got simplified away.
291 SmallSetVector<Function *, 4> InlinedCallees;
292
293 // Track the dead functions to delete once finished with inlining calls. We
294 // defer deleting these to make it easier to handle the call graph updates.
295 SmallVector<Function *, 4> DeadFunctions;
296
297 // Track potentially dead non-local functions with comdats to see if they can
298 // be deleted as a batch after inlining.
299 SmallVector<Function *, 4> DeadFunctionsInComdats;
300
301 // Loop forward over all of the calls. Note that we cannot cache the size as
302 // inlining can introduce new calls that need to be processed.
303 for (int I = 0; I < (int)Calls.size(); ++I) {
304 // We expect the calls to typically be batched with sequences of calls that
305 // have the same caller, so we first set up some shared infrastructure for
306 // this caller. We also do any pruning we can at this layer on the caller
307 // alone.
308 Function &F = *Calls[I].first->getCaller();
309 LazyCallGraph::Node &N = *CG.lookup(F);
310 if (CG.lookupSCC(N) != C)
311 continue;
312
313 LLVM_DEBUG(dbgs() << "Inlining calls in: " << F.getName() << "\n"
314 << " Function size: " << F.getInstructionCount()
315 << "\n");
316
317 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
319 };
320
321 // Now process as many calls as we have within this caller in the sequence.
322 // We bail out as soon as the caller has to change so we can update the
323 // call graph and prepare the context of that new caller.
324 bool DidInline = false;
325 for (; I < (int)Calls.size() && Calls[I].first->getCaller() == &F; ++I) {
326 auto &P = Calls[I];
327 CallBase *CB = P.first;
328 const int InlineHistoryID = P.second;
330
331 if (InlineHistoryID != -1 &&
332 inlineHistoryIncludes(&Callee, InlineHistoryID, InlineHistory)) {
333 LLVM_DEBUG(dbgs() << "Skipping inlining due to history: " << F.getName()
334 << " -> " << Callee.getName() << "\n");
335 setInlineRemark(*CB, "recursive");
336 continue;
337 }
338
339 // Check if this inlining may repeat breaking an SCC apart that has
340 // already been split once before. In that case, inlining here may
341 // trigger infinite inlining, much like is prevented within the inliner
342 // itself by the InlineHistory above, but spread across CGSCC iterations
343 // and thus hidden from the full inline history.
344 LazyCallGraph::SCC *CalleeSCC = CG.lookupSCC(*CG.lookup(Callee));
345 if (CalleeSCC == C && UR.InlinedInternalEdges.count({&N, C})) {
346 LLVM_DEBUG(dbgs() << "Skipping inlining internal SCC edge from a node "
347 "previously split out of this SCC by inlining: "
348 << F.getName() << " -> " << Callee.getName() << "\n");
349 setInlineRemark(*CB, "recursive SCC split");
350 continue;
351 }
352
353 std::unique_ptr<InlineAdvice> Advice =
354 Advisor.getAdvice(*CB, OnlyMandatory);
355
356 // Check whether we want to inline this callsite.
357 if (!Advice)
358 continue;
359
360 if (!Advice->isInliningRecommended()) {
361 Advice->recordUnattemptedInlining();
362 continue;
363 }
364
365 int CBCostMult =
368 .value_or(1);
369
370 // Setup the data structure used to plumb customization into the
371 // `InlineFunction` routine.
373 GetAssumptionCache, PSI,
376
378 InlineFunction(*CB, IFI, /*MergeAttributes=*/true,
379 &FAM.getResult<AAManager>(*CB->getCaller()));
380 if (!IR.isSuccess()) {
381 Advice->recordUnsuccessfulInlining(IR);
382 continue;
383 }
384
385 DidInline = true;
386 InlinedCallees.insert(&Callee);
387 ++NumInlined;
388
389 LLVM_DEBUG(dbgs() << " Size after inlining: "
390 << F.getInstructionCount() << "\n");
391
392 // Add any new callsites to defined functions to the worklist.
393 if (!IFI.InlinedCallSites.empty()) {
394 int NewHistoryID = InlineHistory.size();
395 InlineHistory.push_back({&Callee, InlineHistoryID});
396
397 for (CallBase *ICB : reverse(IFI.InlinedCallSites)) {
398 Function *NewCallee = ICB->getCalledFunction();
399 assert(!(NewCallee && NewCallee->isIntrinsic()) &&
400 "Intrinsic calls should not be tracked.");
401 if (!NewCallee) {
402 // Try to promote an indirect (virtual) call without waiting for
403 // the post-inline cleanup and the next DevirtSCCRepeatedPass
404 // iteration because the next iteration may not happen and we may
405 // miss inlining it.
406 if (tryPromoteCall(*ICB))
407 NewCallee = ICB->getCalledFunction();
408 }
409 if (NewCallee) {
410 if (!NewCallee->isDeclaration()) {
411 Calls.push_back({ICB, NewHistoryID});
412 // Continually inlining through an SCC can result in huge compile
413 // times and bloated code since we arbitrarily stop at some point
414 // when the inliner decides it's not profitable to inline anymore.
415 // We attempt to mitigate this by making these calls exponentially
416 // more expensive.
417 // This doesn't apply to calls in the same SCC since if we do
418 // inline through the SCC the function will end up being
419 // self-recursive which the inliner bails out on, and inlining
420 // within an SCC is necessary for performance.
421 if (CalleeSCC != C &&
422 CalleeSCC == CG.lookupSCC(CG.get(*NewCallee))) {
423 Attribute NewCBCostMult = Attribute::get(
424 M.getContext(),
426 itostr(CBCostMult * IntraSCCCostMultiplier));
427 ICB->addFnAttr(NewCBCostMult);
428 }
429 }
430 }
431 }
432 }
433
434 // For local functions or discardable functions without comdats, check
435 // whether this makes the callee trivially dead. In that case, we can drop
436 // the body of the function eagerly which may reduce the number of callers
437 // of other functions to one, changing inline cost thresholds. Non-local
438 // discardable functions with comdats are checked later on.
439 bool CalleeWasDeleted = false;
440 if (Callee.isDiscardableIfUnused() && Callee.hasZeroLiveUses() &&
441 !CG.isLibFunction(Callee)) {
442 if (Callee.hasLocalLinkage() || !Callee.hasComdat()) {
443 Calls.erase(
444 std::remove_if(Calls.begin() + I + 1, Calls.end(),
445 [&](const std::pair<CallBase *, int> &Call) {
446 return Call.first->getCaller() == &Callee;
447 }),
448 Calls.end());
449
450 // Clear the body and queue the function itself for deletion when we
451 // finish inlining and call graph updates.
452 // Note that after this point, it is an error to do anything other
453 // than use the callee's address or delete it.
454 Callee.dropAllReferences();
455 assert(!is_contained(DeadFunctions, &Callee) &&
456 "Cannot put cause a function to become dead twice!");
457 DeadFunctions.push_back(&Callee);
458 CalleeWasDeleted = true;
459 } else {
460 DeadFunctionsInComdats.push_back(&Callee);
461 }
462 }
463 if (CalleeWasDeleted)
464 Advice->recordInliningWithCalleeDeleted();
465 else
466 Advice->recordInlining();
467 }
468
469 // Back the call index up by one to put us in a good position to go around
470 // the outer loop.
471 --I;
472
473 if (!DidInline)
474 continue;
475 Changed = true;
476
477 // At this point, since we have made changes we have at least removed
478 // a call instruction. However, in the process we do some incremental
479 // simplification of the surrounding code. This simplification can
480 // essentially do all of the same things as a function pass and we can
481 // re-use the exact same logic for updating the call graph to reflect the
482 // change.
483
484 // Inside the update, we also update the FunctionAnalysisManager in the
485 // proxy for this particular SCC. We do this as the SCC may have changed and
486 // as we're going to mutate this particular function we want to make sure
487 // the proxy is in place to forward any invalidation events.
488 LazyCallGraph::SCC *OldC = C;
490 LLVM_DEBUG(dbgs() << "Updated inlining SCC: " << *C << "\n");
491
492 // If this causes an SCC to split apart into multiple smaller SCCs, there
493 // is a subtle risk we need to prepare for. Other transformations may
494 // expose an "infinite inlining" opportunity later, and because of the SCC
495 // mutation, we will revisit this function and potentially re-inline. If we
496 // do, and that re-inlining also has the potentially to mutate the SCC
497 // structure, the infinite inlining problem can manifest through infinite
498 // SCC splits and merges. To avoid this, we capture the originating caller
499 // node and the SCC containing the call edge. This is a slight over
500 // approximation of the possible inlining decisions that must be avoided,
501 // but is relatively efficient to store. We use C != OldC to know when
502 // a new SCC is generated and the original SCC may be generated via merge
503 // in later iterations.
504 //
505 // It is also possible that even if no new SCC is generated
506 // (i.e., C == OldC), the original SCC could be split and then merged
507 // into the same one as itself. and the original SCC will be added into
508 // UR.CWorklist again, we want to catch such cases too.
509 //
510 // FIXME: This seems like a very heavyweight way of retaining the inline
511 // history, we should look for a more efficient way of tracking it.
512 if ((C != OldC || UR.CWorklist.count(OldC)) &&
513 llvm::any_of(InlinedCallees, [&](Function *Callee) {
514 return CG.lookupSCC(*CG.lookup(*Callee)) == OldC;
515 })) {
516 LLVM_DEBUG(dbgs() << "Inlined an internal call edge and split an SCC, "
517 "retaining this to avoid infinite inlining.\n");
518 UR.InlinedInternalEdges.insert({&N, OldC});
519 }
520 InlinedCallees.clear();
521
522 // Invalidate analyses for this function now so that we don't have to
523 // invalidate analyses for all functions in this SCC later.
525 }
526
527 // We must ensure that we only delete functions with comdats if every function
528 // in the comdat is going to be deleted.
529 if (!DeadFunctionsInComdats.empty()) {
530 filterDeadComdatFunctions(DeadFunctionsInComdats);
531 for (auto *Callee : DeadFunctionsInComdats)
532 Callee->dropAllReferences();
533 DeadFunctions.append(DeadFunctionsInComdats);
534 }
535
536 // Now that we've finished inlining all of the calls across this SCC, delete
537 // all of the trivially dead functions, updating the call graph and the CGSCC
538 // pass manager in the process.
539 //
540 // Note that this walks a pointer set which has non-deterministic order but
541 // that is OK as all we do is delete things and add pointers to unordered
542 // sets.
543 for (Function *DeadF : DeadFunctions) {
544 // Get the necessary information out of the call graph and nuke the
545 // function there. Also, clear out any cached analyses.
546 auto &DeadC = *CG.lookupSCC(*CG.lookup(*DeadF));
547 FAM.clear(*DeadF, DeadF->getName());
548 AM.clear(DeadC, DeadC.getName());
549 auto &DeadRC = DeadC.getOuterRefSCC();
550 CG.removeDeadFunction(*DeadF);
551
552 // Mark the relevant parts of the call graph as invalid so we don't visit
553 // them.
554 UR.InvalidatedSCCs.insert(&DeadC);
555 UR.InvalidatedRefSCCs.insert(&DeadRC);
556
557 // If the updated SCC was the one containing the deleted function, clear it.
558 if (&DeadC == UR.UpdatedC)
559 UR.UpdatedC = nullptr;
560
561 // And delete the actual function from the module.
562 M.getFunctionList().erase(DeadF);
563
564 ++NumDeleted;
565 }
566
567 if (!Changed)
568 return PreservedAnalyses::all();
569
571 // Even if we change the IR, we update the core CGSCC data structures and so
572 // can preserve the proxy to the function analysis manager.
574 // We have already invalidated all analyses on modified functions.
576 return PA;
577}
578
580 bool MandatoryFirst,
581 InlineContext IC,
583 unsigned MaxDevirtIterations)
584 : Params(Params), IC(IC), Mode(Mode),
586 // Run the inliner first. The theory is that we are walking bottom-up and so
587 // the callees have already been fully optimized, and we want to inline them
588 // into the callers so that our optimizations can reflect that.
589 // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
590 // because it makes profile annotation in the backend inaccurate.
591 if (MandatoryFirst) {
592 PM.addPass(InlinerPass(/*OnlyMandatory*/ true));
595 }
596 PM.addPass(InlinerPass());
599}
600
603 auto &IAA = MAM.getResult<InlineAdvisorAnalysis>(M);
604 if (!IAA.tryCreate(Params, Mode,
605 {CGSCCInlineReplayFile,
606 CGSCCInlineReplayScope,
607 CGSCCInlineReplayFallback,
608 {CGSCCInlineReplayFormat}},
609 IC)) {
610 M.getContext().emitError(
611 "Could not setup Inlining Advisor for the requested "
612 "mode and/or options");
613 return PreservedAnalyses::all();
614 }
615
616 // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
617 // to detect when we devirtualize indirect calls and iterate the SCC passes
618 // in that case to try and catch knock-on inlining or function attrs
619 // opportunities. Then we add it to the module pipeline by walking the SCCs
620 // in postorder (or bottom-up).
621 // If MaxDevirtIterations is 0, we just don't use the devirtualization
622 // wrapper.
623 if (MaxDevirtIterations == 0)
625 else
628
629 MPM.addPass(std::move(AfterCGMPM));
630 MPM.run(M, MAM);
631
632 // Discard the InlineAdvisor, a subsequent inlining session should construct
633 // its own.
634 auto PA = PreservedAnalyses::all();
636 PA.abandon<InlineAdvisorAnalysis>();
637 return PA;
638}
639
641 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
642 static_cast<PassInfoMixin<InlinerPass> *>(this)->printPipeline(
643 OS, MapClassName2PassName);
644 if (OnlyMandatory)
645 OS << "<only-mandatory>";
646}
647
649 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
650 // Print some info about passes added to the wrapper. This is however
651 // incomplete as InlineAdvisorAnalysis part isn't included (which also depends
652 // on Params and Mode).
653 if (!MPM.isEmpty()) {
654 MPM.printPipeline(OS, MapClassName2PassName);
655 OS << ',';
656 }
657 OS << "cgscc(";
658 if (MaxDevirtIterations != 0)
659 OS << "devirt<" << MaxDevirtIterations << ">(";
660 PM.printPipeline(OS, MapClassName2PassName);
661 if (MaxDevirtIterations != 0)
662 OS << ')';
663 OS << ')';
664}
amdgpu Simplify well known AMD library false FunctionCallee Callee
This file contains the simple types necessary to represent the attributes associated with functions a...
This is the interface for LLVM's primary stateless and local alias analysis.
This header provides classes for managing passes over SCCs of the call graph.
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Definition: CommandLine.h:678
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the DenseMap class.
#define DEBUG_TYPE
static bool inlineHistoryIncludes(Function *F, int InlineHistoryID, const SmallVectorImpl< std::pair< Function *, int > > &InlineHistory)
Return true if the specified inline history ID indicates an inline history that includes the specifie...
Definition: Inliner.cpp:151
static cl::opt< ReplayInlinerSettings::Scope > CGSCCInlineReplayScope("cgscc-inline-replay-scope", cl::init(ReplayInlinerSettings::Scope::Function), cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function", "Replay on functions that have remarks associated " "with them (default)"), clEnumValN(ReplayInlinerSettings::Scope::Module, "Module", "Replay on the entire module")), cl::desc("Whether inline replay should be applied to the entire " "Module or just the Functions (default) that are present as " "callers in remarks during cgscc inlining."), cl::Hidden)
static cl::opt< bool > KeepAdvisorForPrinting("keep-inline-advisor-for-printing", cl::init(false), cl::Hidden)
A flag for test, so we can print the content of the advisor when running it as part of the default (e...
static cl::opt< std::string > CGSCCInlineReplayFile("cgscc-inline-replay", cl::init(""), cl::value_desc("filename"), cl::desc("Optimization remarks file containing inline remarks to be replayed " "by cgscc inlining."), cl::Hidden)
static cl::opt< bool > EnablePostSCCAdvisorPrinting("enable-scc-inline-advisor-printing", cl::init(false), cl::Hidden)
Allows printing the contents of the advisor after each SCC inliner pass.
static cl::opt< int > IntraSCCCostMultiplier("intra-scc-cost-multiplier", cl::init(2), cl::Hidden, cl::desc("Cost multiplier to multiply onto inlined call sites where the " "new call was previously an intra-SCC call (not relevant when the " "original call was already intra-SCC). This can accumulate over " "multiple inlinings (e.g. if a call site already had a cost " "multiplier and one of its inlined calls was also subject to " "this, the inlined call would have the original multiplier " "multiplied by intra-scc-cost-multiplier). This is to prevent tons of " "inlining through a child SCC which can cause terrible compile times"))
static cl::opt< CallSiteFormat::Format > CGSCCInlineReplayFormat("cgscc-inline-replay-format", cl::init(CallSiteFormat::Format::LineColumnDiscriminator), cl::values(clEnumValN(CallSiteFormat::Format::Line, "Line", "<Line Number>"), clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn", "<Line Number>:<Column Number>"), clEnumValN(CallSiteFormat::Format::LineDiscriminator, "LineDiscriminator", "<Line Number>.<Discriminator>"), clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator, "LineColumnDiscriminator", "<Line Number>:<Column Number>.<Discriminator> (default)")), cl::desc("How cgscc inline replay file is formatted"), cl::Hidden)
static cl::opt< ReplayInlinerSettings::Fallback > CGSCCInlineReplayFallback("cgscc-inline-replay-fallback", cl::init(ReplayInlinerSettings::Fallback::Original), cl::values(clEnumValN(ReplayInlinerSettings::Fallback::Original, "Original", "All decisions not in replay send to original advisor (default)"), clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline, "AlwaysInline", "All decisions not in replay are inlined"), clEnumValN(ReplayInlinerSettings::Fallback::NeverInline, "NeverInline", "All decisions not in replay are not inlined")), cl::desc("How cgscc inline replay treats sites that don't come from the replay. " "Original: defers to original advisor, AlwaysInline: inline all sites " "not in replay, NeverInline: inline no sites not in replay"), cl::Hidden)
Implements a lazy call graph analysis and related passes for the new pass manager.
Statically lint checks LLVM IR
Definition: Lint.cpp:746
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
This file contains the declarations for metadata subclasses.
Module.h This file contains the declarations for the Module class.
print must be executed print the must be executed context for all instructions
#define P(N)
ModulePassManager MPM
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
This file provides a priority worklist.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
raw_pwrite_stream & OS
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
This file contains some functions that are useful when dealing with strings.
A manager for alias analyses.
This templated class represents "all analyses that operate over <a particular IR unit>" (e....
Definition: PassManager.h:90
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:620
void clear(IRUnitT &IR, llvm::StringRef Name)
Clear any cached analysis results for a single unit of IR.
void invalidate(IRUnitT &IR, const PreservedAnalyses &PA)
Invalidate cached analyses for an IR unit.
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
Definition: PassManager.h:793
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:774
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition: Attributes.cpp:91
Analysis pass which computes BlockFrequencyInfo.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1186
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1408
Function * getCaller()
Helper to get the caller (the parent function).
A proxy from a FunctionAnalysisManager to an SCC.
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:209
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:275
Printer pass for the FunctionPropertiesAnalysis results.
The InlineAdvisorAnalysis is a module pass because the InlineAdvisor needs to capture state right bef...
Interface for deciding whether to inline a call site or not.
virtual void onPassEntry(LazyCallGraph::SCC *SCC=nullptr)
This must be called when the Inliner pass is entered, to allow the InlineAdvisor update internal stat...
virtual void onPassExit(LazyCallGraph::SCC *SCC=nullptr)
This must be called when the Inliner pass is exited, as function passes may be run subsequently.
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
Definition: Cloning.h:203
InlineResult is basically true or false.
Definition: InlineCost.h:179
The inliner pass for the new pass manager.
Definition: Inliner.h:40
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
Definition: Inliner.cpp:640
PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)
Definition: Inliner.cpp:202
A node in the call graph.
An SCC of the call graph.
iterator begin() const
A lazily constructed view of the call graph of a module.
PreservedAnalyses run(Module &, ModuleAnalysisManager &)
Definition: Inliner.cpp:601
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
Definition: Inliner.cpp:648
ModuleInlinerWrapperPass(InlineParams Params=getInlineParams(), bool MandatoryFirst=true, InlineContext IC={}, InliningAdvisorMode Mode=InliningAdvisorMode::Default, unsigned MaxDevirtIterations=0)
Definition: Inliner.cpp:579
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Diagnostic information for missed-optimization remarks.
Result proxy object for OuterAnalysisManagerProxy.
Definition: PassManager.h:1061
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
Definition: PassManager.h:1058
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
Definition: PassManager.h:486
LLVM_ATTRIBUTE_MINSIZE std::enable_if_t<!std::is_same< PassT, PassManager >::value > addPass(PassT &&Pass)
Definition: PassManager.h:544
PreservedAnalyses run(IRUnitT &IR, AnalysisManagerT &AM, ExtraArgTs... ExtraArgs)
Run all of the passes in this manager over the given unit of IR.
Definition: PassManager.h:498
bool isEmpty() const
Returns if the pass manager contains any passes.
Definition: PassManager.h:568
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:155
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
void preserveSet()
Mark an analysis set as preserved.
Definition: PassManager.h:188
void preserve()
Mark an analysis as preserved.
Definition: PassManager.h:173
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:152
void clear()
Completely clear the SetVector.
Definition: SetVector.h:224
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:312
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:687
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
An efficient, type-erasing, non-owning reference to a callable.
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
const char FunctionInlineCostMultiplierAttributeName[]
Definition: InlineCost.h:59
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
Definition: CommandLine.h:703
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:59
InliningAdvisorMode
There are 4 scenarios we can use the InlineAdvisor:
Definition: InlineAdvisor.h:44
std::optional< int > getStringFnAttrAsInt(CallBase &CB, StringRef AttrKind)
Definition: InlineCost.cpp:175
DevirtSCCRepeatedPass createDevirtSCCRepeatedPass(CGSCCPassT &&Pass, int MaxIterations)
A function to deduce a function pass type and wrap it in the templated adaptor.
LazyCallGraph::SCC & updateCGAndAnalysisManagerForCGSCCPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a CGSCC pass.
void setInlineRemark(CallBase &CB, StringRef Message)
Set the inline-remark attribute.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1826
ModuleToPostOrderCGSCCPassAdaptor createModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT &&Pass)
A function to deduce a function pass type and wrap it in the templated adaptor.
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:511
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
std::unique_ptr< InlineAdvisor > getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr< InlineAdvisor > OriginalAdvisor, const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks, InlineContext IC)
InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)
This function inlines the called function into the basic block of the caller.
InlineParams getInlineParams()
Generate the parameters to tune the inline cost analysis based only on the commandline options.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition: STLExtras.h:1976
void filterDeadComdatFunctions(SmallVectorImpl< Function * > &DeadComdatFunctions)
Filter out potentially dead comdat functions where other entries keep the entire comdat group alive.
bool tryPromoteCall(CallBase &CB)
Try to promote (devirtualize) a virtual call on an Alloca.
cl::opt< unsigned > MaxDevirtIterations("max-devirt-iterations", cl::ReallyHidden, cl::init(4))
#define N
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Provides context on when an inline advisor is constructed in the pipeline (e.g., link phase,...
Definition: InlineAdvisor.h:60
Thresholds to tune inline cost analysis.
Definition: InlineCost.h:205
A CRTP mix-in to automatically provide informational APIs needed for passes.
Definition: PassManager.h:371