LLVM 18.0.0git
Inliner.cpp
Go to the documentation of this file.
1//===- Inliner.cpp - Code common to all inliners --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the mechanics required to implement inlining without
10// missing any calls and updating the call graph. The decisions of which calls
11// are profitable to inline are implemented elsewhere.
12//
13//===----------------------------------------------------------------------===//
14
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/ScopeExit.h"
19#include "llvm/ADT/SetVector.h"
22#include "llvm/ADT/Statistic.h"
24#include "llvm/ADT/StringRef.h"
37#include "llvm/IR/Attributes.h"
38#include "llvm/IR/BasicBlock.h"
39#include "llvm/IR/DebugLoc.h"
42#include "llvm/IR/Function.h"
44#include "llvm/IR/Instruction.h"
47#include "llvm/IR/Metadata.h"
48#include "llvm/IR/Module.h"
49#include "llvm/IR/PassManager.h"
50#include "llvm/IR/User.h"
51#include "llvm/IR/Value.h"
52#include "llvm/Pass.h"
55#include "llvm/Support/Debug.h"
61#include <algorithm>
62#include <cassert>
63#include <functional>
64#include <utility>
65#include <vector>
66
67using namespace llvm;
68
69#define DEBUG_TYPE "inline"
70
71STATISTIC(NumInlined, "Number of functions inlined");
72STATISTIC(NumDeleted, "Number of functions deleted because all callers found");
73
75 "intra-scc-cost-multiplier", cl::init(2), cl::Hidden,
77 "Cost multiplier to multiply onto inlined call sites where the "
78 "new call was previously an intra-SCC call (not relevant when the "
79 "original call was already intra-SCC). This can accumulate over "
80 "multiple inlinings (e.g. if a call site already had a cost "
81 "multiplier and one of its inlined calls was also subject to "
82 "this, the inlined call would have the original multiplier "
83 "multiplied by intra-scc-cost-multiplier). This is to prevent tons of "
84 "inlining through a child SCC which can cause terrible compile times"));
85
86/// A flag for test, so we can print the content of the advisor when running it
87/// as part of the default (e.g. -O3) pipeline.
88static cl::opt<bool> KeepAdvisorForPrinting("keep-inline-advisor-for-printing",
89 cl::init(false), cl::Hidden);
90
91/// Allows printing the contents of the advisor after each SCC inliner pass.
92static cl::opt<bool>
93 EnablePostSCCAdvisorPrinting("enable-scc-inline-advisor-printing",
94 cl::init(false), cl::Hidden);
95
96
98 "cgscc-inline-replay", cl::init(""), cl::value_desc("filename"),
100 "Optimization remarks file containing inline remarks to be replayed "
101 "by cgscc inlining."),
102 cl::Hidden);
103
105 "cgscc-inline-replay-scope",
106 cl::init(ReplayInlinerSettings::Scope::Function),
107 cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function",
108 "Replay on functions that have remarks associated "
109 "with them (default)"),
110 clEnumValN(ReplayInlinerSettings::Scope::Module, "Module",
111 "Replay on the entire module")),
112 cl::desc("Whether inline replay should be applied to the entire "
113 "Module or just the Functions (default) that are present as "
114 "callers in remarks during cgscc inlining."),
115 cl::Hidden);
116
118 "cgscc-inline-replay-fallback",
119 cl::init(ReplayInlinerSettings::Fallback::Original),
122 ReplayInlinerSettings::Fallback::Original, "Original",
123 "All decisions not in replay send to original advisor (default)"),
124 clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline,
125 "AlwaysInline", "All decisions not in replay are inlined"),
126 clEnumValN(ReplayInlinerSettings::Fallback::NeverInline, "NeverInline",
127 "All decisions not in replay are not inlined")),
128 cl::desc(
129 "How cgscc inline replay treats sites that don't come from the replay. "
130 "Original: defers to original advisor, AlwaysInline: inline all sites "
131 "not in replay, NeverInline: inline no sites not in replay"),
132 cl::Hidden);
133
135 "cgscc-inline-replay-format",
136 cl::init(CallSiteFormat::Format::LineColumnDiscriminator),
138 clEnumValN(CallSiteFormat::Format::Line, "Line", "<Line Number>"),
139 clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn",
140 "<Line Number>:<Column Number>"),
141 clEnumValN(CallSiteFormat::Format::LineDiscriminator,
142 "LineDiscriminator", "<Line Number>.<Discriminator>"),
143 clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator,
144 "LineColumnDiscriminator",
145 "<Line Number>:<Column Number>.<Discriminator> (default)")),
146 cl::desc("How cgscc inline replay file is formatted"), cl::Hidden);
147
148/// Return true if the specified inline history ID
149/// indicates an inline history that includes the specified function.
151 Function *F, int InlineHistoryID,
152 const SmallVectorImpl<std::pair<Function *, int>> &InlineHistory) {
153 while (InlineHistoryID != -1) {
154 assert(unsigned(InlineHistoryID) < InlineHistory.size() &&
155 "Invalid inline history ID");
156 if (InlineHistory[InlineHistoryID].first == F)
157 return true;
158 InlineHistoryID = InlineHistory[InlineHistoryID].second;
159 }
160 return false;
161}
162
164InlinerPass::getAdvisor(const ModuleAnalysisManagerCGSCCProxy::Result &MAM,
166 if (OwnedAdvisor)
167 return *OwnedAdvisor;
168
170 if (!IAA) {
171 // It should still be possible to run the inliner as a stand-alone SCC pass,
172 // for test scenarios. In that case, we default to the
173 // DefaultInlineAdvisor, which doesn't need to keep state between SCC pass
174 // runs. It also uses just the default InlineParams.
175 // In this case, we need to use the provided FAM, which is valid for the
176 // duration of the inliner pass, and thus the lifetime of the owned advisor.
177 // The one we would get from the MAM can be invalidated as a result of the
178 // inliner's activity.
179 OwnedAdvisor = std::make_unique<DefaultInlineAdvisor>(
180 M, FAM, getInlineParams(),
182
183 if (!CGSCCInlineReplayFile.empty())
184 OwnedAdvisor = getReplayInlineAdvisor(
185 M, FAM, M.getContext(), std::move(OwnedAdvisor),
186 ReplayInlinerSettings{CGSCCInlineReplayFile,
187 CGSCCInlineReplayScope,
188 CGSCCInlineReplayFallback,
189 {CGSCCInlineReplayFormat}},
190 /*EmitRemarks=*/true,
192
193 return *OwnedAdvisor;
194 }
195 assert(IAA->getAdvisor() &&
196 "Expected a present InlineAdvisorAnalysis also have an "
197 "InlineAdvisor initialized");
198 return *IAA->getAdvisor();
199}
200
203 CGSCCUpdateResult &UR) {
204 const auto &MAMProxy =
206 bool Changed = false;
207
208 assert(InitialC.size() > 0 && "Cannot handle an empty SCC!");
209 Module &M = *InitialC.begin()->getFunction().getParent();
210 ProfileSummaryInfo *PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(M);
211
214 .getManager();
215
216 InlineAdvisor &Advisor = getAdvisor(MAMProxy, FAM, M);
217 Advisor.onPassEntry(&InitialC);
218
219 auto AdvisorOnExit = make_scope_exit([&] { Advisor.onPassExit(&InitialC); });
220
221 // We use a single common worklist for calls across the entire SCC. We
222 // process these in-order and append new calls introduced during inlining to
223 // the end. The PriorityInlineOrder is optional here, in which the smaller
224 // callee would have a higher priority to inline.
225 //
226 // Note that this particular order of processing is actually critical to
227 // avoid very bad behaviors. Consider *highly connected* call graphs where
228 // each function contains a small amount of code and a couple of calls to
229 // other functions. Because the LLVM inliner is fundamentally a bottom-up
230 // inliner, it can handle gracefully the fact that these all appear to be
231 // reasonable inlining candidates as it will flatten things until they become
232 // too big to inline, and then move on and flatten another batch.
233 //
234 // However, when processing call edges *within* an SCC we cannot rely on this
235 // bottom-up behavior. As a consequence, with heavily connected *SCCs* of
236 // functions we can end up incrementally inlining N calls into each of
237 // N functions because each incremental inlining decision looks good and we
238 // don't have a topological ordering to prevent explosions.
239 //
240 // To compensate for this, we don't process transitive edges made immediate
241 // by inlining until we've done one pass of inlining across the entire SCC.
242 // Large, highly connected SCCs still lead to some amount of code bloat in
243 // this model, but it is uniformly spread across all the functions in the SCC
244 // and eventually they all become too large to inline, rather than
245 // incrementally maknig a single function grow in a super linear fashion.
247
248 // Populate the initial list of calls in this SCC.
249 for (auto &N : InitialC) {
250 auto &ORE =
252 // We want to generally process call sites top-down in order for
253 // simplifications stemming from replacing the call with the returned value
254 // after inlining to be visible to subsequent inlining decisions.
255 // FIXME: Using instructions sequence is a really bad way to do this.
256 // Instead we should do an actual RPO walk of the function body.
257 for (Instruction &I : instructions(N.getFunction()))
258 if (auto *CB = dyn_cast<CallBase>(&I))
259 if (Function *Callee = CB->getCalledFunction()) {
260 if (!Callee->isDeclaration())
261 Calls.push_back({CB, -1});
262 else if (!isa<IntrinsicInst>(I)) {
263 using namespace ore;
264 setInlineRemark(*CB, "unavailable definition");
265 ORE.emit([&]() {
266 return OptimizationRemarkMissed(DEBUG_TYPE, "NoDefinition", &I)
267 << NV("Callee", Callee) << " will not be inlined into "
268 << NV("Caller", CB->getCaller())
269 << " because its definition is unavailable"
270 << setIsVerbose();
271 });
272 }
273 }
274 }
275 if (Calls.empty())
276 return PreservedAnalyses::all();
277
278 // Capture updatable variable for the current SCC.
279 auto *C = &InitialC;
280
281 // When inlining a callee produces new call sites, we want to keep track of
282 // the fact that they were inlined from the callee. This allows us to avoid
283 // infinite inlining in some obscure cases. To represent this, we use an
284 // index into the InlineHistory vector.
286
287 // Track a set vector of inlined callees so that we can augment the caller
288 // with all of their edges in the call graph before pruning out the ones that
289 // got simplified away.
290 SmallSetVector<Function *, 4> InlinedCallees;
291
292 // Track the dead functions to delete once finished with inlining calls. We
293 // defer deleting these to make it easier to handle the call graph updates.
294 SmallVector<Function *, 4> DeadFunctions;
295
296 // Track potentially dead non-local functions with comdats to see if they can
297 // be deleted as a batch after inlining.
298 SmallVector<Function *, 4> DeadFunctionsInComdats;
299
300 // Loop forward over all of the calls. Note that we cannot cache the size as
301 // inlining can introduce new calls that need to be processed.
302 for (int I = 0; I < (int)Calls.size(); ++I) {
303 // We expect the calls to typically be batched with sequences of calls that
304 // have the same caller, so we first set up some shared infrastructure for
305 // this caller. We also do any pruning we can at this layer on the caller
306 // alone.
307 Function &F = *Calls[I].first->getCaller();
308 LazyCallGraph::Node &N = *CG.lookup(F);
309 if (CG.lookupSCC(N) != C)
310 continue;
311
312 LLVM_DEBUG(dbgs() << "Inlining calls in: " << F.getName() << "\n"
313 << " Function size: " << F.getInstructionCount()
314 << "\n");
315
316 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
318 };
319
320 // Now process as many calls as we have within this caller in the sequence.
321 // We bail out as soon as the caller has to change so we can update the
322 // call graph and prepare the context of that new caller.
323 bool DidInline = false;
324 for (; I < (int)Calls.size() && Calls[I].first->getCaller() == &F; ++I) {
325 auto &P = Calls[I];
326 CallBase *CB = P.first;
327 const int InlineHistoryID = P.second;
328 Function &Callee = *CB->getCalledFunction();
329
330 if (InlineHistoryID != -1 &&
331 inlineHistoryIncludes(&Callee, InlineHistoryID, InlineHistory)) {
332 LLVM_DEBUG(dbgs() << "Skipping inlining due to history: " << F.getName()
333 << " -> " << Callee.getName() << "\n");
334 setInlineRemark(*CB, "recursive");
335 // Set noinline so that we don't forget this decision across CGSCC
336 // iterations.
337 CB->setIsNoInline();
338 continue;
339 }
340
341 // Check if this inlining may repeat breaking an SCC apart that has
342 // already been split once before. In that case, inlining here may
343 // trigger infinite inlining, much like is prevented within the inliner
344 // itself by the InlineHistory above, but spread across CGSCC iterations
345 // and thus hidden from the full inline history.
346 LazyCallGraph::SCC *CalleeSCC = CG.lookupSCC(*CG.lookup(Callee));
347 if (CalleeSCC == C && UR.InlinedInternalEdges.count({&N, C})) {
348 LLVM_DEBUG(dbgs() << "Skipping inlining internal SCC edge from a node "
349 "previously split out of this SCC by inlining: "
350 << F.getName() << " -> " << Callee.getName() << "\n");
351 setInlineRemark(*CB, "recursive SCC split");
352 continue;
353 }
354
355 std::unique_ptr<InlineAdvice> Advice =
356 Advisor.getAdvice(*CB, OnlyMandatory);
357
358 // Check whether we want to inline this callsite.
359 if (!Advice)
360 continue;
361
362 if (!Advice->isInliningRecommended()) {
363 Advice->recordUnattemptedInlining();
364 continue;
365 }
366
367 int CBCostMult =
370 .value_or(1);
371
372 // Setup the data structure used to plumb customization into the
373 // `InlineFunction` routine.
375 GetAssumptionCache, PSI,
378
380 InlineFunction(*CB, IFI, /*MergeAttributes=*/true,
381 &FAM.getResult<AAManager>(*CB->getCaller()));
382 if (!IR.isSuccess()) {
383 Advice->recordUnsuccessfulInlining(IR);
384 continue;
385 }
386
387 DidInline = true;
388 InlinedCallees.insert(&Callee);
389 ++NumInlined;
390
391 LLVM_DEBUG(dbgs() << " Size after inlining: "
392 << F.getInstructionCount() << "\n");
393
394 // Add any new callsites to defined functions to the worklist.
395 if (!IFI.InlinedCallSites.empty()) {
396 int NewHistoryID = InlineHistory.size();
397 InlineHistory.push_back({&Callee, InlineHistoryID});
398
399 for (CallBase *ICB : reverse(IFI.InlinedCallSites)) {
400 Function *NewCallee = ICB->getCalledFunction();
401 assert(!(NewCallee && NewCallee->isIntrinsic()) &&
402 "Intrinsic calls should not be tracked.");
403 if (!NewCallee) {
404 // Try to promote an indirect (virtual) call without waiting for
405 // the post-inline cleanup and the next DevirtSCCRepeatedPass
406 // iteration because the next iteration may not happen and we may
407 // miss inlining it.
408 if (tryPromoteCall(*ICB))
409 NewCallee = ICB->getCalledFunction();
410 }
411 if (NewCallee) {
412 if (!NewCallee->isDeclaration()) {
413 Calls.push_back({ICB, NewHistoryID});
414 // Continually inlining through an SCC can result in huge compile
415 // times and bloated code since we arbitrarily stop at some point
416 // when the inliner decides it's not profitable to inline anymore.
417 // We attempt to mitigate this by making these calls exponentially
418 // more expensive.
419 // This doesn't apply to calls in the same SCC since if we do
420 // inline through the SCC the function will end up being
421 // self-recursive which the inliner bails out on, and inlining
422 // within an SCC is necessary for performance.
423 if (CalleeSCC != C &&
424 CalleeSCC == CG.lookupSCC(CG.get(*NewCallee))) {
425 Attribute NewCBCostMult = Attribute::get(
426 M.getContext(),
428 itostr(CBCostMult * IntraSCCCostMultiplier));
429 ICB->addFnAttr(NewCBCostMult);
430 }
431 }
432 }
433 }
434 }
435
436 // For local functions or discardable functions without comdats, check
437 // whether this makes the callee trivially dead. In that case, we can drop
438 // the body of the function eagerly which may reduce the number of callers
439 // of other functions to one, changing inline cost thresholds. Non-local
440 // discardable functions with comdats are checked later on.
441 bool CalleeWasDeleted = false;
442 if (Callee.isDiscardableIfUnused() && Callee.hasZeroLiveUses() &&
443 !CG.isLibFunction(Callee)) {
444 if (Callee.hasLocalLinkage() || !Callee.hasComdat()) {
445 Calls.erase(
446 std::remove_if(Calls.begin() + I + 1, Calls.end(),
447 [&](const std::pair<CallBase *, int> &Call) {
448 return Call.first->getCaller() == &Callee;
449 }),
450 Calls.end());
451
452 // Clear the body and queue the function itself for deletion when we
453 // finish inlining and call graph updates.
454 // Note that after this point, it is an error to do anything other
455 // than use the callee's address or delete it.
456 Callee.dropAllReferences();
457 assert(!is_contained(DeadFunctions, &Callee) &&
458 "Cannot put cause a function to become dead twice!");
459 DeadFunctions.push_back(&Callee);
460 CalleeWasDeleted = true;
461 } else {
462 DeadFunctionsInComdats.push_back(&Callee);
463 }
464 }
465 if (CalleeWasDeleted)
466 Advice->recordInliningWithCalleeDeleted();
467 else
468 Advice->recordInlining();
469 }
470
471 // Back the call index up by one to put us in a good position to go around
472 // the outer loop.
473 --I;
474
475 if (!DidInline)
476 continue;
477 Changed = true;
478
479 // At this point, since we have made changes we have at least removed
480 // a call instruction. However, in the process we do some incremental
481 // simplification of the surrounding code. This simplification can
482 // essentially do all of the same things as a function pass and we can
483 // re-use the exact same logic for updating the call graph to reflect the
484 // change.
485
486 // Inside the update, we also update the FunctionAnalysisManager in the
487 // proxy for this particular SCC. We do this as the SCC may have changed and
488 // as we're going to mutate this particular function we want to make sure
489 // the proxy is in place to forward any invalidation events.
490 LazyCallGraph::SCC *OldC = C;
492 LLVM_DEBUG(dbgs() << "Updated inlining SCC: " << *C << "\n");
493
494 // If this causes an SCC to split apart into multiple smaller SCCs, there
495 // is a subtle risk we need to prepare for. Other transformations may
496 // expose an "infinite inlining" opportunity later, and because of the SCC
497 // mutation, we will revisit this function and potentially re-inline. If we
498 // do, and that re-inlining also has the potentially to mutate the SCC
499 // structure, the infinite inlining problem can manifest through infinite
500 // SCC splits and merges. To avoid this, we capture the originating caller
501 // node and the SCC containing the call edge. This is a slight over
502 // approximation of the possible inlining decisions that must be avoided,
503 // but is relatively efficient to store. We use C != OldC to know when
504 // a new SCC is generated and the original SCC may be generated via merge
505 // in later iterations.
506 //
507 // It is also possible that even if no new SCC is generated
508 // (i.e., C == OldC), the original SCC could be split and then merged
509 // into the same one as itself. and the original SCC will be added into
510 // UR.CWorklist again, we want to catch such cases too.
511 //
512 // FIXME: This seems like a very heavyweight way of retaining the inline
513 // history, we should look for a more efficient way of tracking it.
514 if ((C != OldC || UR.CWorklist.count(OldC)) &&
515 llvm::any_of(InlinedCallees, [&](Function *Callee) {
516 return CG.lookupSCC(*CG.lookup(*Callee)) == OldC;
517 })) {
518 LLVM_DEBUG(dbgs() << "Inlined an internal call edge and split an SCC, "
519 "retaining this to avoid infinite inlining.\n");
520 UR.InlinedInternalEdges.insert({&N, OldC});
521 }
522 InlinedCallees.clear();
523
524 // Invalidate analyses for this function now so that we don't have to
525 // invalidate analyses for all functions in this SCC later.
527 }
528
529 // We must ensure that we only delete functions with comdats if every function
530 // in the comdat is going to be deleted.
531 if (!DeadFunctionsInComdats.empty()) {
532 filterDeadComdatFunctions(DeadFunctionsInComdats);
533 for (auto *Callee : DeadFunctionsInComdats)
534 Callee->dropAllReferences();
535 DeadFunctions.append(DeadFunctionsInComdats);
536 }
537
538 // Now that we've finished inlining all of the calls across this SCC, delete
539 // all of the trivially dead functions, updating the call graph and the CGSCC
540 // pass manager in the process.
541 //
542 // Note that this walks a pointer set which has non-deterministic order but
543 // that is OK as all we do is delete things and add pointers to unordered
544 // sets.
545 for (Function *DeadF : DeadFunctions) {
546 // Get the necessary information out of the call graph and nuke the
547 // function there. Also, clear out any cached analyses.
548 auto &DeadC = *CG.lookupSCC(*CG.lookup(*DeadF));
549 FAM.clear(*DeadF, DeadF->getName());
550 AM.clear(DeadC, DeadC.getName());
551 auto &DeadRC = DeadC.getOuterRefSCC();
552 CG.removeDeadFunction(*DeadF);
553
554 // Mark the relevant parts of the call graph as invalid so we don't visit
555 // them.
556 UR.InvalidatedSCCs.insert(&DeadC);
557 UR.InvalidatedRefSCCs.insert(&DeadRC);
558
559 // If the updated SCC was the one containing the deleted function, clear it.
560 if (&DeadC == UR.UpdatedC)
561 UR.UpdatedC = nullptr;
562
563 // And delete the actual function from the module.
564 M.getFunctionList().erase(DeadF);
565
566 ++NumDeleted;
567 }
568
569 if (!Changed)
570 return PreservedAnalyses::all();
571
573 // Even if we change the IR, we update the core CGSCC data structures and so
574 // can preserve the proxy to the function analysis manager.
576 // We have already invalidated all analyses on modified functions.
578 return PA;
579}
580
582 bool MandatoryFirst,
583 InlineContext IC,
585 unsigned MaxDevirtIterations)
586 : Params(Params), IC(IC), Mode(Mode),
588 // Run the inliner first. The theory is that we are walking bottom-up and so
589 // the callees have already been fully optimized, and we want to inline them
590 // into the callers so that our optimizations can reflect that.
591 // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
592 // because it makes profile annotation in the backend inaccurate.
593 if (MandatoryFirst) {
594 PM.addPass(InlinerPass(/*OnlyMandatory*/ true));
597 }
598 PM.addPass(InlinerPass());
601}
602
605 auto &IAA = MAM.getResult<InlineAdvisorAnalysis>(M);
606 if (!IAA.tryCreate(Params, Mode,
607 {CGSCCInlineReplayFile,
608 CGSCCInlineReplayScope,
609 CGSCCInlineReplayFallback,
610 {CGSCCInlineReplayFormat}},
611 IC)) {
612 M.getContext().emitError(
613 "Could not setup Inlining Advisor for the requested "
614 "mode and/or options");
615 return PreservedAnalyses::all();
616 }
617
618 // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
619 // to detect when we devirtualize indirect calls and iterate the SCC passes
620 // in that case to try and catch knock-on inlining or function attrs
621 // opportunities. Then we add it to the module pipeline by walking the SCCs
622 // in postorder (or bottom-up).
623 // If MaxDevirtIterations is 0, we just don't use the devirtualization
624 // wrapper.
625 if (MaxDevirtIterations == 0)
627 else
630
631 MPM.addPass(std::move(AfterCGMPM));
632 MPM.run(M, MAM);
633
634 // Discard the InlineAdvisor, a subsequent inlining session should construct
635 // its own.
636 auto PA = PreservedAnalyses::all();
638 PA.abandon<InlineAdvisorAnalysis>();
639 return PA;
640}
641
643 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
644 static_cast<PassInfoMixin<InlinerPass> *>(this)->printPipeline(
645 OS, MapClassName2PassName);
646 if (OnlyMandatory)
647 OS << "<only-mandatory>";
648}
649
651 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
652 // Print some info about passes added to the wrapper. This is however
653 // incomplete as InlineAdvisorAnalysis part isn't included (which also depends
654 // on Params and Mode).
655 if (!MPM.isEmpty()) {
656 MPM.printPipeline(OS, MapClassName2PassName);
657 OS << ',';
658 }
659 OS << "cgscc(";
660 if (MaxDevirtIterations != 0)
661 OS << "devirt<" << MaxDevirtIterations << ">(";
662 PM.printPipeline(OS, MapClassName2PassName);
663 if (MaxDevirtIterations != 0)
664 OS << ')';
665 OS << ')';
666}
This file contains the simple types necessary to represent the attributes associated with functions a...
This is the interface for LLVM's primary stateless and local alias analysis.
This header provides classes for managing passes over SCCs of the call graph.
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Definition: CommandLine.h:680
#define LLVM_DEBUG(X)
Definition: Debug.h:101
#define DEBUG_TYPE
static bool inlineHistoryIncludes(Function *F, int InlineHistoryID, const SmallVectorImpl< std::pair< Function *, int > > &InlineHistory)
Return true if the specified inline history ID indicates an inline history that includes the specifie...
Definition: Inliner.cpp:150
static cl::opt< ReplayInlinerSettings::Scope > CGSCCInlineReplayScope("cgscc-inline-replay-scope", cl::init(ReplayInlinerSettings::Scope::Function), cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function", "Replay on functions that have remarks associated " "with them (default)"), clEnumValN(ReplayInlinerSettings::Scope::Module, "Module", "Replay on the entire module")), cl::desc("Whether inline replay should be applied to the entire " "Module or just the Functions (default) that are present as " "callers in remarks during cgscc inlining."), cl::Hidden)
static cl::opt< bool > KeepAdvisorForPrinting("keep-inline-advisor-for-printing", cl::init(false), cl::Hidden)
A flag for test, so we can print the content of the advisor when running it as part of the default (e...
static cl::opt< std::string > CGSCCInlineReplayFile("cgscc-inline-replay", cl::init(""), cl::value_desc("filename"), cl::desc("Optimization remarks file containing inline remarks to be replayed " "by cgscc inlining."), cl::Hidden)
static cl::opt< bool > EnablePostSCCAdvisorPrinting("enable-scc-inline-advisor-printing", cl::init(false), cl::Hidden)
Allows printing the contents of the advisor after each SCC inliner pass.
static cl::opt< int > IntraSCCCostMultiplier("intra-scc-cost-multiplier", cl::init(2), cl::Hidden, cl::desc("Cost multiplier to multiply onto inlined call sites where the " "new call was previously an intra-SCC call (not relevant when the " "original call was already intra-SCC). This can accumulate over " "multiple inlinings (e.g. if a call site already had a cost " "multiplier and one of its inlined calls was also subject to " "this, the inlined call would have the original multiplier " "multiplied by intra-scc-cost-multiplier). This is to prevent tons of " "inlining through a child SCC which can cause terrible compile times"))
static cl::opt< CallSiteFormat::Format > CGSCCInlineReplayFormat("cgscc-inline-replay-format", cl::init(CallSiteFormat::Format::LineColumnDiscriminator), cl::values(clEnumValN(CallSiteFormat::Format::Line, "Line", "<Line Number>"), clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn", "<Line Number>:<Column Number>"), clEnumValN(CallSiteFormat::Format::LineDiscriminator, "LineDiscriminator", "<Line Number>.<Discriminator>"), clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator, "LineColumnDiscriminator", "<Line Number>:<Column Number>.<Discriminator> (default)")), cl::desc("How cgscc inline replay file is formatted"), cl::Hidden)
static cl::opt< ReplayInlinerSettings::Fallback > CGSCCInlineReplayFallback("cgscc-inline-replay-fallback", cl::init(ReplayInlinerSettings::Fallback::Original), cl::values(clEnumValN(ReplayInlinerSettings::Fallback::Original, "Original", "All decisions not in replay send to original advisor (default)"), clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline, "AlwaysInline", "All decisions not in replay are inlined"), clEnumValN(ReplayInlinerSettings::Fallback::NeverInline, "NeverInline", "All decisions not in replay are not inlined")), cl::desc("How cgscc inline replay treats sites that don't come from the replay. " "Original: defers to original advisor, AlwaysInline: inline all sites " "not in replay, NeverInline: inline no sites not in replay"), cl::Hidden)
Select target instructions out of generic instructions
Implements a lazy call graph analysis and related passes for the new pass manager.
Legalize the Machine IR a function s Machine IR
Definition: Legalizer.cpp:81
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
This file contains the declarations for metadata subclasses.
Module.h This file contains the declarations for the Module class.
#define P(N)
ModulePassManager MPM
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
This file provides a priority worklist.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
raw_pwrite_stream & OS
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
This file contains some functions that are useful when dealing with strings.
A manager for alias analyses.
This templated class represents "all analyses that operate over <a particular IR unit>" (e....
Definition: PassManager.h:110
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:649
void clear(IRUnitT &IR, llvm::StringRef Name)
Clear any cached analysis results for a single unit of IR.
void invalidate(IRUnitT &IR, const PreservedAnalyses &PA)
Invalidate cached analyses for an IR unit.
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
Definition: PassManager.h:822
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:803
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition: Attributes.cpp:92
Analysis pass which computes BlockFrequencyInfo.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1227
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1449
void setIsNoInline()
Definition: InstrTypes.h:1923
Function * getCaller()
Helper to get the caller (the parent function).
A proxy from a FunctionAnalysisManager to an SCC.
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:235
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:273
Printer pass for the FunctionPropertiesAnalysis results.
The InlineAdvisorAnalysis is a module pass because the InlineAdvisor needs to capture state right bef...
Interface for deciding whether to inline a call site or not.
virtual void onPassEntry(LazyCallGraph::SCC *SCC=nullptr)
This must be called when the Inliner pass is entered, to allow the InlineAdvisor update internal stat...
virtual void onPassExit(LazyCallGraph::SCC *SCC=nullptr)
This must be called when the Inliner pass is exited, as function passes may be run subsequently.
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
Definition: Cloning.h:202
InlineResult is basically true or false.
Definition: InlineCost.h:179
The inliner pass for the new pass manager.
Definition: Inliner.h:35
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
Definition: Inliner.cpp:642
PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)
Definition: Inliner.cpp:201
A node in the call graph.
An SCC of the call graph.
iterator begin() const
A lazily constructed view of the call graph of a module.
PreservedAnalyses run(Module &, ModuleAnalysisManager &)
Definition: Inliner.cpp:603
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
Definition: Inliner.cpp:650
ModuleInlinerWrapperPass(InlineParams Params=getInlineParams(), bool MandatoryFirst=true, InlineContext IC={}, InliningAdvisorMode Mode=InliningAdvisorMode::Default, unsigned MaxDevirtIterations=0)
Definition: Inliner.cpp:581
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Diagnostic information for missed-optimization remarks.
Result proxy object for OuterAnalysisManagerProxy.
Definition: PassManager.h:1090
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
Definition: PassManager.h:1087
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
Definition: PassManager.h:506
LLVM_ATTRIBUTE_MINSIZE std::enable_if_t<!std::is_same< PassT, PassManager >::value > addPass(PassT &&Pass)
Definition: PassManager.h:573
PreservedAnalyses run(IRUnitT &IR, AnalysisManagerT &AM, ExtraArgTs... ExtraArgs)
Run all of the passes in this manager over the given unit of IR.
Definition: PassManager.h:518
bool isEmpty() const
Returns if the pass manager contains any passes.
Definition: PassManager.h:597
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:172
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:175
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:178
void preserveSet()
Mark an analysis set as preserved.
Definition: PassManager.h:208
void preserve()
Mark an analysis as preserved.
Definition: PassManager.h:193
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
void clear()
Completely clear the SetVector.
Definition: SetVector.h:273
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:370
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:687
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
An efficient, type-erasing, non-owning reference to a callable.
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
const char FunctionInlineCostMultiplierAttributeName[]
Definition: InlineCost.h:59
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
Definition: CommandLine.h:705
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:59
InliningAdvisorMode
There are 4 scenarios we can use the InlineAdvisor:
Definition: InlineAdvisor.h:43
std::optional< int > getStringFnAttrAsInt(CallBase &CB, StringRef AttrKind)
Definition: InlineCost.cpp:186
DevirtSCCRepeatedPass createDevirtSCCRepeatedPass(CGSCCPassT &&Pass, int MaxIterations)
A function to deduce a function pass type and wrap it in the templated adaptor.
LazyCallGraph::SCC & updateCGAndAnalysisManagerForCGSCCPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a CGSCC pass.
void setInlineRemark(CallBase &CB, StringRef Message)
Set the inline-remark attribute.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1733
ModuleToPostOrderCGSCCPassAdaptor createModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT &&Pass)
A function to deduce a function pass type and wrap it in the templated adaptor.
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:428
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
std::unique_ptr< InlineAdvisor > getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr< InlineAdvisor > OriginalAdvisor, const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks, InlineContext IC)
InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)
This function inlines the called function into the basic block of the caller.
InlineParams getInlineParams()
Generate the parameters to tune the inline cost analysis based only on the commandline options.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition: STLExtras.h:1883
void filterDeadComdatFunctions(SmallVectorImpl< Function * > &DeadComdatFunctions)
Filter out potentially dead comdat functions where other entries keep the entire comdat group alive.
bool tryPromoteCall(CallBase &CB)
Try to promote (devirtualize) a virtual call on an Alloca.
cl::opt< unsigned > MaxDevirtIterations("max-devirt-iterations", cl::ReallyHidden, cl::init(4))
#define N
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Provides context on when an inline advisor is constructed in the pipeline (e.g., link phase,...
Definition: InlineAdvisor.h:59
Thresholds to tune inline cost analysis.
Definition: InlineCost.h:205
A CRTP mix-in to automatically provide informational APIs needed for passes.
Definition: PassManager.h:391