LLVM  16.0.0git
Inliner.cpp
Go to the documentation of this file.
1 //===- Inliner.cpp - Code common to all inliners --------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the mechanics required to implement inlining without
10 // missing any calls and updating the call graph. The decisions of which calls
11 // are profitable to inline are implemented elsewhere.
12 //
13 //===----------------------------------------------------------------------===//
14 
16 #include "llvm/ADT/DenseMap.h"
17 #include "llvm/ADT/Optional.h"
19 #include "llvm/ADT/STLExtras.h"
20 #include "llvm/ADT/ScopeExit.h"
21 #include "llvm/ADT/SetVector.h"
22 #include "llvm/ADT/SmallPtrSet.h"
23 #include "llvm/ADT/SmallVector.h"
24 #include "llvm/ADT/Statistic.h"
25 #include "llvm/ADT/StringExtras.h"
26 #include "llvm/ADT/StringRef.h"
40 #include "llvm/IR/Attributes.h"
41 #include "llvm/IR/BasicBlock.h"
42 #include "llvm/IR/DebugLoc.h"
43 #include "llvm/IR/DerivedTypes.h"
44 #include "llvm/IR/DiagnosticInfo.h"
45 #include "llvm/IR/Function.h"
46 #include "llvm/IR/InstIterator.h"
47 #include "llvm/IR/Instruction.h"
48 #include "llvm/IR/Instructions.h"
49 #include "llvm/IR/IntrinsicInst.h"
50 #include "llvm/IR/Metadata.h"
51 #include "llvm/IR/Module.h"
52 #include "llvm/IR/PassManager.h"
53 #include "llvm/IR/User.h"
54 #include "llvm/IR/Value.h"
55 #include "llvm/Pass.h"
56 #include "llvm/Support/Casting.h"
58 #include "llvm/Support/Debug.h"
64 #include <algorithm>
65 #include <cassert>
66 #include <functional>
67 #include <utility>
68 #include <vector>
69 
70 using namespace llvm;
71 
72 #define DEBUG_TYPE "inline"
73 
74 STATISTIC(NumInlined, "Number of functions inlined");
75 STATISTIC(NumCallsDeleted, "Number of call sites deleted, not inlined");
76 STATISTIC(NumDeleted, "Number of functions deleted because all callers found");
77 STATISTIC(NumMergedAllocas, "Number of allocas merged together");
78 
79 /// Flag to disable manual alloca merging.
80 ///
81 /// Merging of allocas was originally done as a stack-size saving technique
82 /// prior to LLVM's code generator having support for stack coloring based on
83 /// lifetime markers. It is now in the process of being removed. To experiment
84 /// with disabling it and relying fully on lifetime marker based stack
85 /// coloring, you can pass this flag to LLVM.
86 static cl::opt<bool>
87  DisableInlinedAllocaMerging("disable-inlined-alloca-merging",
88  cl::init(false), cl::Hidden);
89 
91  "intra-scc-cost-multiplier", cl::init(2), cl::Hidden,
92  cl::desc(
93  "Cost multiplier to multiply onto inlined call sites where the "
94  "new call was previously an intra-SCC call (not relevant when the "
95  "original call was already intra-SCC). This can accumulate over "
96  "multiple inlinings (e.g. if a call site already had a cost "
97  "multiplier and one of its inlined calls was also subject to "
98  "this, the inlined call would have the original multiplier "
99  "multiplied by intra-scc-cost-multiplier). This is to prevent tons of "
100  "inlining through a child SCC which can cause terrible compile times"));
101 
102 /// A flag for test, so we can print the content of the advisor when running it
103 /// as part of the default (e.g. -O3) pipeline.
104 static cl::opt<bool> KeepAdvisorForPrinting("keep-inline-advisor-for-printing",
105  cl::init(false), cl::Hidden);
106 
107 /// Allows printing the contents of the advisor after each SCC inliner pass.
108 static cl::opt<bool>
109  EnablePostSCCAdvisorPrinting("enable-scc-inline-advisor-printing",
110  cl::init(false), cl::Hidden);
111 
112 namespace llvm {
114 }
115 
117  "cgscc-inline-replay", cl::init(""), cl::value_desc("filename"),
118  cl::desc(
119  "Optimization remarks file containing inline remarks to be replayed "
120  "by cgscc inlining."),
121  cl::Hidden);
122 
124  "cgscc-inline-replay-scope",
127  "Replay on functions that have remarks associated "
128  "with them (default)"),
130  "Replay on the entire module")),
131  cl::desc("Whether inline replay should be applied to the entire "
132  "Module or just the Functions (default) that are present as "
133  "callers in remarks during cgscc inlining."),
134  cl::Hidden);
135 
137  "cgscc-inline-replay-fallback",
139  cl::values(
140  clEnumValN(
142  "All decisions not in replay send to original advisor (default)"),
144  "AlwaysInline", "All decisions not in replay are inlined"),
146  "All decisions not in replay are not inlined")),
147  cl::desc(
148  "How cgscc inline replay treats sites that don't come from the replay. "
149  "Original: defers to original advisor, AlwaysInline: inline all sites "
150  "not in replay, NeverInline: inline no sites not in replay"),
151  cl::Hidden);
152 
154  "cgscc-inline-replay-format",
156  cl::values(
157  clEnumValN(CallSiteFormat::Format::Line, "Line", "<Line Number>"),
159  "<Line Number>:<Column Number>"),
161  "LineDiscriminator", "<Line Number>.<Discriminator>"),
163  "LineColumnDiscriminator",
164  "<Line Number>:<Column Number>.<Discriminator> (default)")),
165  cl::desc("How cgscc inline replay file is formatted"), cl::Hidden);
166 
168 
169 LegacyInlinerBase::LegacyInlinerBase(char &ID, bool InsertLifetime)
170  : CallGraphSCCPass(ID), InsertLifetime(InsertLifetime) {}
171 
172 /// For this class, we declare that we require and preserve the call graph.
173 /// If the derived class implements this method, it should
174 /// always explicitly call the implementation here.
181 }
182 
184 
185 /// Look at all of the allocas that we inlined through this call site. If we
186 /// have already inlined other allocas through other calls into this function,
187 /// then we know that they have disjoint lifetimes and that we can merge them.
188 ///
189 /// There are many heuristics possible for merging these allocas, and the
190 /// different options have different tradeoffs. One thing that we *really*
191 /// don't want to hurt is SRoA: once inlining happens, often allocas are no
192 /// longer address taken and so they can be promoted.
193 ///
194 /// Our "solution" for that is to only merge allocas whose outermost type is an
195 /// array type. These are usually not promoted because someone is using a
196 /// variable index into them. These are also often the most important ones to
197 /// merge.
198 ///
199 /// A better solution would be to have real memory lifetime markers in the IR
200 /// and not have the inliner do any merging of allocas at all. This would
201 /// allow the backend to do proper stack slot coloring of all allocas that
202 /// *actually make it to the backend*, which is really what we want.
203 ///
204 /// Because we don't have this information, we do this simple and useful hack.
206  InlinedArrayAllocasTy &InlinedArrayAllocas,
207  int InlineHistory) {
208  SmallPtrSet<AllocaInst *, 16> UsedAllocas;
209 
210  // When processing our SCC, check to see if the call site was inlined from
211  // some other call site. For example, if we're processing "A" in this code:
212  // A() { B() }
213  // B() { x = alloca ... C() }
214  // C() { y = alloca ... }
215  // Assume that C was not inlined into B initially, and so we're processing A
216  // and decide to inline B into A. Doing this makes an alloca available for
217  // reuse and makes a callsite (C) available for inlining. When we process
218  // the C call site we don't want to do any alloca merging between X and Y
219  // because their scopes are not disjoint. We could make this smarter by
220  // keeping track of the inline history for each alloca in the
221  // InlinedArrayAllocas but this isn't likely to be a significant win.
222  if (InlineHistory != -1) // Only do merging for top-level call sites in SCC.
223  return;
224 
225  // Loop over all the allocas we have so far and see if they can be merged with
226  // a previously inlined alloca. If not, remember that we had it.
227  for (unsigned AllocaNo = 0, E = IFI.StaticAllocas.size(); AllocaNo != E;
228  ++AllocaNo) {
229  AllocaInst *AI = IFI.StaticAllocas[AllocaNo];
230 
231  // Don't bother trying to merge array allocations (they will usually be
232  // canonicalized to be an allocation *of* an array), or allocations whose
233  // type is not itself an array (because we're afraid of pessimizing SRoA).
234  ArrayType *ATy = dyn_cast<ArrayType>(AI->getAllocatedType());
235  if (!ATy || AI->isArrayAllocation())
236  continue;
237 
238  // Get the list of all available allocas for this array type.
239  std::vector<AllocaInst *> &AllocasForType = InlinedArrayAllocas[ATy];
240 
241  // Loop over the allocas in AllocasForType to see if we can reuse one. Note
242  // that we have to be careful not to reuse the same "available" alloca for
243  // multiple different allocas that we just inlined, we use the 'UsedAllocas'
244  // set to keep track of which "available" allocas are being used by this
245  // function. Also, AllocasForType can be empty of course!
246  bool MergedAwayAlloca = false;
247  for (AllocaInst *AvailableAlloca : AllocasForType) {
248  Align Align1 = AI->getAlign();
249  Align Align2 = AvailableAlloca->getAlign();
250 
251  // The available alloca has to be in the right function, not in some other
252  // function in this SCC.
253  if (AvailableAlloca->getParent() != AI->getParent())
254  continue;
255 
256  // If the inlined function already uses this alloca then we can't reuse
257  // it.
258  if (!UsedAllocas.insert(AvailableAlloca).second)
259  continue;
260 
261  // Otherwise, we *can* reuse it, RAUW AI into AvailableAlloca and declare
262  // success!
263  LLVM_DEBUG(dbgs() << " ***MERGED ALLOCA: " << *AI
264  << "\n\t\tINTO: " << *AvailableAlloca << '\n');
265 
266  // Move affected dbg.declare calls immediately after the new alloca to
267  // avoid the situation when a dbg.declare precedes its alloca.
268  if (auto *L = LocalAsMetadata::getIfExists(AI))
269  if (auto *MDV = MetadataAsValue::getIfExists(AI->getContext(), L))
270  for (User *U : MDV->users())
271  if (DbgDeclareInst *DDI = dyn_cast<DbgDeclareInst>(U))
272  DDI->moveBefore(AvailableAlloca->getNextNode());
273 
274  AI->replaceAllUsesWith(AvailableAlloca);
275 
276  if (Align1 > Align2)
277  AvailableAlloca->setAlignment(AI->getAlign());
278 
279  AI->eraseFromParent();
280  MergedAwayAlloca = true;
281  ++NumMergedAllocas;
282  IFI.StaticAllocas[AllocaNo] = nullptr;
283  break;
284  }
285 
286  // If we already nuked the alloca, we're done with it.
287  if (MergedAwayAlloca)
288  continue;
289 
290  // If we were unable to merge away the alloca either because there are no
291  // allocas of the right type available or because we reused them all
292  // already, remember that this alloca came from an inlined function and mark
293  // it used so we don't reuse it for other allocas from this inline
294  // operation.
295  AllocasForType.push_back(AI);
296  UsedAllocas.insert(AI);
297  }
298 }
299 
300 /// If it is possible to inline the specified call site,
301 /// do so and update the CallGraph for this operation.
302 ///
303 /// This function also does some basic book-keeping to update the IR. The
304 /// InlinedArrayAllocas map keeps track of any allocas that are already
305 /// available from other functions inlined into the caller. If we are able to
306 /// inline this call site we attempt to reuse already available allocas or add
307 /// any new allocas to the set if not possible.
309  CallBase &CB, InlineFunctionInfo &IFI,
310  InlinedArrayAllocasTy &InlinedArrayAllocas, int InlineHistory,
311  bool InsertLifetime, function_ref<AAResults &(Function &)> &AARGetter,
312  ImportedFunctionsInliningStatistics &ImportedFunctionsStats) {
314  Function *Caller = CB.getCaller();
315 
316  AAResults &AAR = AARGetter(*Callee);
317 
318  // Try to inline the function. Get the list of static allocas that were
319  // inlined.
320  InlineResult IR =
321  InlineFunction(CB, IFI,
322  /*MergeAttributes=*/true, &AAR, InsertLifetime);
323  if (!IR.isSuccess())
324  return IR;
325 
327  ImportedFunctionsStats.recordInline(*Caller, *Callee);
328 
330  mergeInlinedArrayAllocas(Caller, IFI, InlinedArrayAllocas, InlineHistory);
331 
332  return IR; // success
333 }
334 
335 /// Return true if the specified inline history ID
336 /// indicates an inline history that includes the specified function.
338  Function *F, int InlineHistoryID,
339  const SmallVectorImpl<std::pair<Function *, int>> &InlineHistory) {
340  while (InlineHistoryID != -1) {
341  assert(unsigned(InlineHistoryID) < InlineHistory.size() &&
342  "Invalid inline history ID");
343  if (InlineHistory[InlineHistoryID].first == F)
344  return true;
345  InlineHistoryID = InlineHistory[InlineHistoryID].second;
346  }
347  return false;
348 }
349 
353  return false; // No changes to CallGraph.
354 }
355 
357  if (skipSCC(SCC))
358  return false;
359  return inlineCalls(SCC);
360 }
361 
362 static bool
364  std::function<AssumptionCache &(Function &)> GetAssumptionCache,
365  ProfileSummaryInfo *PSI,
366  std::function<const TargetLibraryInfo &(Function &)> GetTLI,
367  bool InsertLifetime,
368  function_ref<InlineCost(CallBase &CB)> GetInlineCost,
369  function_ref<AAResults &(Function &)> AARGetter,
370  ImportedFunctionsInliningStatistics &ImportedFunctionsStats) {
371  SmallPtrSet<Function *, 8> SCCFunctions;
372  LLVM_DEBUG(dbgs() << "Inliner visiting SCC:");
373  for (CallGraphNode *Node : SCC) {
374  Function *F = Node->getFunction();
375  if (F)
376  SCCFunctions.insert(F);
377  LLVM_DEBUG(dbgs() << " " << (F ? F->getName() : "INDIRECTNODE"));
378  }
379 
380  // Scan through and identify all call sites ahead of time so that we only
381  // inline call sites in the original functions, not call sites that result
382  // from inlining other functions.
384 
385  // When inlining a callee produces new call sites, we want to keep track of
386  // the fact that they were inlined from the callee. This allows us to avoid
387  // infinite inlining in some obscure cases. To represent this, we use an
388  // index into the InlineHistory vector.
389  SmallVector<std::pair<Function *, int>, 8> InlineHistory;
390 
391  for (CallGraphNode *Node : SCC) {
392  Function *F = Node->getFunction();
393  if (!F || F->isDeclaration())
394  continue;
395 
397  for (BasicBlock &BB : *F)
398  for (Instruction &I : BB) {
399  auto *CB = dyn_cast<CallBase>(&I);
400  // If this isn't a call, or it is a call to an intrinsic, it can
401  // never be inlined.
402  if (!CB || isa<IntrinsicInst>(I))
403  continue;
404 
405  // If this is a direct call to an external function, we can never inline
406  // it. If it is an indirect call, inlining may resolve it to be a
407  // direct call, so we keep it.
408  if (Function *Callee = CB->getCalledFunction())
409  if (Callee->isDeclaration()) {
410  using namespace ore;
411 
412  setInlineRemark(*CB, "unavailable definition");
413  ORE.emit([&]() {
414  return OptimizationRemarkMissed(DEBUG_TYPE, "NoDefinition", &I)
415  << NV("Callee", Callee) << " will not be inlined into "
416  << NV("Caller", CB->getCaller())
417  << " because its definition is unavailable"
418  << setIsVerbose();
419  });
420  continue;
421  }
422 
423  CallSites.push_back(std::make_pair(CB, -1));
424  }
425  }
426 
427  LLVM_DEBUG(dbgs() << ": " << CallSites.size() << " call sites.\n");
428 
429  // If there are no calls in this function, exit early.
430  if (CallSites.empty())
431  return false;
432 
433  // Now that we have all of the call sites, move the ones to functions in the
434  // current SCC to the end of the list.
435  unsigned FirstCallInSCC = CallSites.size();
436  for (unsigned I = 0; I < FirstCallInSCC; ++I)
437  if (Function *F = CallSites[I].first->getCalledFunction())
438  if (SCCFunctions.count(F))
439  std::swap(CallSites[I--], CallSites[--FirstCallInSCC]);
440 
441  InlinedArrayAllocasTy InlinedArrayAllocas;
442  InlineFunctionInfo InlineInfo(&CG, GetAssumptionCache, PSI);
443 
444  // Now that we have all of the call sites, loop over them and inline them if
445  // it looks profitable to do so.
446  bool Changed = false;
447  bool LocalChange;
448  do {
449  LocalChange = false;
450  // Iterate over the outer loop because inlining functions can cause indirect
451  // calls to become direct calls.
452  // CallSites may be modified inside so ranged for loop can not be used.
453  for (unsigned CSi = 0; CSi != CallSites.size(); ++CSi) {
454  auto &P = CallSites[CSi];
455  CallBase &CB = *P.first;
456  const int InlineHistoryID = P.second;
457 
458  Function *Caller = CB.getCaller();
460 
461  // We can only inline direct calls to non-declarations.
462  if (!Callee || Callee->isDeclaration())
463  continue;
464 
465  bool IsTriviallyDead = isInstructionTriviallyDead(&CB, &GetTLI(*Caller));
466 
467  if (!IsTriviallyDead) {
468  // If this call site was obtained by inlining another function, verify
469  // that the include path for the function did not include the callee
470  // itself. If so, we'd be recursively inlining the same function,
471  // which would provide the same callsites, which would cause us to
472  // infinitely inline.
473  if (InlineHistoryID != -1 &&
474  inlineHistoryIncludes(Callee, InlineHistoryID, InlineHistory)) {
475  setInlineRemark(CB, "recursive");
476  continue;
477  }
478  }
479 
480  // FIXME for new PM: because of the old PM we currently generate ORE and
481  // in turn BFI on demand. With the new PM, the ORE dependency should
482  // just become a regular analysis dependency.
483  OptimizationRemarkEmitter ORE(Caller);
484 
485  auto OIC = shouldInline(CB, GetInlineCost, ORE);
486  // If the policy determines that we should inline this function,
487  // delete the call instead.
488  if (!OIC)
489  continue;
490 
491  // If this call site is dead and it is to a readonly function, we should
492  // just delete the call instead of trying to inline it, regardless of
493  // size. This happens because IPSCCP propagates the result out of the
494  // call and then we're left with the dead call.
495  if (IsTriviallyDead) {
496  LLVM_DEBUG(dbgs() << " -> Deleting dead call: " << CB << "\n");
497  // Update the call graph by deleting the edge from Callee to Caller.
498  setInlineRemark(CB, "trivially dead");
499  CG[Caller]->removeCallEdgeFor(CB);
500  CB.eraseFromParent();
501  ++NumCallsDeleted;
502  } else {
503  // Get DebugLoc to report. CB will be invalid after Inliner.
504  DebugLoc DLoc = CB.getDebugLoc();
505  BasicBlock *Block = CB.getParent();
506 
507  // Attempt to inline the function.
508  using namespace ore;
509 
511  CB, InlineInfo, InlinedArrayAllocas, InlineHistoryID,
512  InsertLifetime, AARGetter, ImportedFunctionsStats);
513  if (!IR.isSuccess()) {
514  setInlineRemark(CB, std::string(IR.getFailureReason()) + "; " +
515  inlineCostStr(*OIC));
516  ORE.emit([&]() {
517  return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc,
518  Block)
519  << NV("Callee", Callee) << " will not be inlined into "
520  << NV("Caller", Caller) << ": "
521  << NV("Reason", IR.getFailureReason());
522  });
523  continue;
524  }
525  ++NumInlined;
526 
527  emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC);
528 
529  // If inlining this function gave us any new call sites, throw them
530  // onto our worklist to process. They are useful inline candidates.
531  if (!InlineInfo.InlinedCalls.empty()) {
532  // Create a new inline history entry for this, so that we remember
533  // that these new callsites came about due to inlining Callee.
534  int NewHistoryID = InlineHistory.size();
535  InlineHistory.push_back(std::make_pair(Callee, InlineHistoryID));
536 
537 #ifndef NDEBUG
538  // Make sure no dupplicates in the inline candidates. This could
539  // happen when a callsite is simpilfied to reusing the return value
540  // of another callsite during function cloning, thus the other
541  // callsite will be reconsidered here.
542  DenseSet<CallBase *> DbgCallSites;
543  for (auto &II : CallSites)
544  DbgCallSites.insert(II.first);
545 #endif
546 
547  for (Value *Ptr : InlineInfo.InlinedCalls) {
548 #ifndef NDEBUG
549  assert(DbgCallSites.count(dyn_cast<CallBase>(Ptr)) == 0);
550 #endif
551  CallSites.push_back(
552  std::make_pair(dyn_cast<CallBase>(Ptr), NewHistoryID));
553  }
554  }
555  }
556 
557  // If we inlined or deleted the last possible call site to the function,
558  // delete the function body now.
559  if (Callee && Callee->use_empty() && Callee->hasLocalLinkage() &&
560  // TODO: Can remove if in SCC now.
561  !SCCFunctions.count(Callee) &&
562  // The function may be apparently dead, but if there are indirect
563  // callgraph references to the node, we cannot delete it yet, this
564  // could invalidate the CGSCC iterator.
565  CG[Callee]->getNumReferences() == 0) {
566  LLVM_DEBUG(dbgs() << " -> Deleting dead function: "
567  << Callee->getName() << "\n");
568  CallGraphNode *CalleeNode = CG[Callee];
569 
570  // Remove any call graph edges from the callee to its callees.
571  CalleeNode->removeAllCalledFunctions();
572 
573  // Removing the node for callee from the call graph and delete it.
574  delete CG.removeFunctionFromModule(CalleeNode);
575  ++NumDeleted;
576  }
577 
578  // Remove this call site from the list. If possible, use
579  // swap/pop_back for efficiency, but do not use it if doing so would
580  // move a call site to a function in this SCC before the
581  // 'FirstCallInSCC' barrier.
582  if (SCC.isSingular()) {
583  CallSites[CSi] = CallSites.back();
584  CallSites.pop_back();
585  } else {
586  CallSites.erase(CallSites.begin() + CSi);
587  }
588  --CSi;
589 
590  Changed = true;
591  LocalChange = true;
592  }
593  } while (LocalChange);
594 
595  return Changed;
596 }
597 
599  CallGraph &CG = getAnalysis<CallGraphWrapperPass>().getCallGraph();
600  ACT = &getAnalysis<AssumptionCacheTracker>();
601  PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
602  GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
603  return getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
604  };
605  auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
606  return ACT->getAssumptionCache(F);
607  };
608  return inlineCallsImpl(
609  SCC, CG, GetAssumptionCache, PSI, GetTLI, InsertLifetime,
610  [&](CallBase &CB) { return getInlineCost(CB); }, LegacyAARGetter(*this),
612 }
613 
614 /// Remove now-dead linkonce functions at the end of
615 /// processing to avoid breaking the SCC traversal.
620  return removeDeadFunctions(CG);
621 }
622 
623 /// Remove dead functions that are not included in DNR (Do Not Remove) list.
625  bool AlwaysInlineOnly) {
626  SmallVector<CallGraphNode *, 16> FunctionsToRemove;
627  SmallVector<Function *, 16> DeadFunctionsInComdats;
628 
629  auto RemoveCGN = [&](CallGraphNode *CGN) {
630  // Remove any call graph edges from the function to its callees.
631  CGN->removeAllCalledFunctions();
632 
633  // Remove any edges from the external node to the function's call graph
634  // node. These edges might have been made irrelegant due to
635  // optimization of the program.
637 
638  // Removing the node for callee from the call graph and delete it.
639  FunctionsToRemove.push_back(CGN);
640  };
641 
642  // Scan for all of the functions, looking for ones that should now be removed
643  // from the program. Insert the dead ones in the FunctionsToRemove set.
644  for (const auto &I : CG) {
645  CallGraphNode *CGN = I.second.get();
646  Function *F = CGN->getFunction();
647  if (!F || F->isDeclaration())
648  continue;
649 
650  // Handle the case when this function is called and we only want to care
651  // about always-inline functions. This is a bit of a hack to share code
652  // between here and the InlineAlways pass.
653  if (AlwaysInlineOnly && !F->hasFnAttribute(Attribute::AlwaysInline))
654  continue;
655 
656  // If the only remaining users of the function are dead constants, remove
657  // them.
658  F->removeDeadConstantUsers();
659 
660  if (!F->isDefTriviallyDead())
661  continue;
662 
663  // It is unsafe to drop a function with discardable linkage from a COMDAT
664  // without also dropping the other members of the COMDAT.
665  // The inliner doesn't visit non-function entities which are in COMDAT
666  // groups so it is unsafe to do so *unless* the linkage is local.
667  if (!F->hasLocalLinkage()) {
668  if (F->hasComdat()) {
669  DeadFunctionsInComdats.push_back(F);
670  continue;
671  }
672  }
673 
674  RemoveCGN(CGN);
675  }
676  if (!DeadFunctionsInComdats.empty()) {
677  // Filter out the functions whose comdats remain alive.
678  filterDeadComdatFunctions(DeadFunctionsInComdats);
679  // Remove the rest.
680  for (Function *F : DeadFunctionsInComdats)
681  RemoveCGN(CG[F]);
682  }
683 
684  if (FunctionsToRemove.empty())
685  return false;
686 
687  // Now that we know which functions to delete, do so. We didn't want to do
688  // this inline, because that would invalidate our CallGraph::iterator
689  // objects. :(
690  //
691  // Note that it doesn't matter that we are iterating over a non-stable order
692  // here to do this, it doesn't matter which order the functions are deleted
693  // in.
694  array_pod_sort(FunctionsToRemove.begin(), FunctionsToRemove.end());
695  FunctionsToRemove.erase(
696  std::unique(FunctionsToRemove.begin(), FunctionsToRemove.end()),
697  FunctionsToRemove.end());
698  for (CallGraphNode *CGN : FunctionsToRemove) {
699  delete CG.removeFunctionFromModule(CGN);
700  ++NumDeleted;
701  }
702  return true;
703 }
704 
706 InlinerPass::getAdvisor(const ModuleAnalysisManagerCGSCCProxy::Result &MAM,
708  if (OwnedAdvisor)
709  return *OwnedAdvisor;
710 
712  if (!IAA) {
713  // It should still be possible to run the inliner as a stand-alone SCC pass,
714  // for test scenarios. In that case, we default to the
715  // DefaultInlineAdvisor, which doesn't need to keep state between SCC pass
716  // runs. It also uses just the default InlineParams.
717  // In this case, we need to use the provided FAM, which is valid for the
718  // duration of the inliner pass, and thus the lifetime of the owned advisor.
719  // The one we would get from the MAM can be invalidated as a result of the
720  // inliner's activity.
721  OwnedAdvisor = std::make_unique<DefaultInlineAdvisor>(
722  M, FAM, getInlineParams(),
724 
725  if (!CGSCCInlineReplayFile.empty())
726  OwnedAdvisor = getReplayInlineAdvisor(
727  M, FAM, M.getContext(), std::move(OwnedAdvisor),
728  ReplayInlinerSettings{CGSCCInlineReplayFile,
729  CGSCCInlineReplayScope,
730  CGSCCInlineReplayFallback,
731  {CGSCCInlineReplayFormat}},
732  /*EmitRemarks=*/true,
733  InlineContext{LTOPhase,
735 
736  return *OwnedAdvisor;
737  }
738  assert(IAA->getAdvisor() &&
739  "Expected a present InlineAdvisorAnalysis also have an "
740  "InlineAdvisor initialized");
741  return *IAA->getAdvisor();
742 }
743 
746  CGSCCUpdateResult &UR) {
747  const auto &MAMProxy =
748  AM.getResult<ModuleAnalysisManagerCGSCCProxy>(InitialC, CG);
749  bool Changed = false;
750 
751  assert(InitialC.size() > 0 && "Cannot handle an empty SCC!");
752  Module &M = *InitialC.begin()->getFunction().getParent();
753  ProfileSummaryInfo *PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(M);
754 
757  .getManager();
758 
759  InlineAdvisor &Advisor = getAdvisor(MAMProxy, FAM, M);
760  Advisor.onPassEntry(&InitialC);
761 
762  auto AdvisorOnExit = make_scope_exit([&] { Advisor.onPassExit(&InitialC); });
763 
764  // We use a single common worklist for calls across the entire SCC. We
765  // process these in-order and append new calls introduced during inlining to
766  // the end. The PriorityInlineOrder is optional here, in which the smaller
767  // callee would have a higher priority to inline.
768  //
769  // Note that this particular order of processing is actually critical to
770  // avoid very bad behaviors. Consider *highly connected* call graphs where
771  // each function contains a small amount of code and a couple of calls to
772  // other functions. Because the LLVM inliner is fundamentally a bottom-up
773  // inliner, it can handle gracefully the fact that these all appear to be
774  // reasonable inlining candidates as it will flatten things until they become
775  // too big to inline, and then move on and flatten another batch.
776  //
777  // However, when processing call edges *within* an SCC we cannot rely on this
778  // bottom-up behavior. As a consequence, with heavily connected *SCCs* of
779  // functions we can end up incrementally inlining N calls into each of
780  // N functions because each incremental inlining decision looks good and we
781  // don't have a topological ordering to prevent explosions.
782  //
783  // To compensate for this, we don't process transitive edges made immediate
784  // by inlining until we've done one pass of inlining across the entire SCC.
785  // Large, highly connected SCCs still lead to some amount of code bloat in
786  // this model, but it is uniformly spread across all the functions in the SCC
787  // and eventually they all become too large to inline, rather than
788  // incrementally maknig a single function grow in a super linear fashion.
790 
791  // Populate the initial list of calls in this SCC.
792  for (auto &N : InitialC) {
793  auto &ORE =
795  // We want to generally process call sites top-down in order for
796  // simplifications stemming from replacing the call with the returned value
797  // after inlining to be visible to subsequent inlining decisions.
798  // FIXME: Using instructions sequence is a really bad way to do this.
799  // Instead we should do an actual RPO walk of the function body.
800  for (Instruction &I : instructions(N.getFunction()))
801  if (auto *CB = dyn_cast<CallBase>(&I))
802  if (Function *Callee = CB->getCalledFunction()) {
803  if (!Callee->isDeclaration())
804  Calls.push_back({CB, -1});
805  else if (!isa<IntrinsicInst>(I)) {
806  using namespace ore;
807  setInlineRemark(*CB, "unavailable definition");
808  ORE.emit([&]() {
809  return OptimizationRemarkMissed(DEBUG_TYPE, "NoDefinition", &I)
810  << NV("Callee", Callee) << " will not be inlined into "
811  << NV("Caller", CB->getCaller())
812  << " because its definition is unavailable"
813  << setIsVerbose();
814  });
815  }
816  }
817  }
818  if (Calls.empty())
819  return PreservedAnalyses::all();
820 
821  // Capture updatable variable for the current SCC.
822  auto *C = &InitialC;
823 
824  // When inlining a callee produces new call sites, we want to keep track of
825  // the fact that they were inlined from the callee. This allows us to avoid
826  // infinite inlining in some obscure cases. To represent this, we use an
827  // index into the InlineHistory vector.
828  SmallVector<std::pair<Function *, int>, 16> InlineHistory;
829 
830  // Track a set vector of inlined callees so that we can augment the caller
831  // with all of their edges in the call graph before pruning out the ones that
832  // got simplified away.
833  SmallSetVector<Function *, 4> InlinedCallees;
834 
835  // Track the dead functions to delete once finished with inlining calls. We
836  // defer deleting these to make it easier to handle the call graph updates.
837  SmallVector<Function *, 4> DeadFunctions;
838 
839  // Track potentially dead non-local functions with comdats to see if they can
840  // be deleted as a batch after inlining.
841  SmallVector<Function *, 4> DeadFunctionsInComdats;
842 
843  // Loop forward over all of the calls. Note that we cannot cache the size as
844  // inlining can introduce new calls that need to be processed.
845  for (int I = 0; I < (int)Calls.size(); ++I) {
846  // We expect the calls to typically be batched with sequences of calls that
847  // have the same caller, so we first set up some shared infrastructure for
848  // this caller. We also do any pruning we can at this layer on the caller
849  // alone.
850  Function &F = *Calls[I].first->getCaller();
851  LazyCallGraph::Node &N = *CG.lookup(F);
852  if (CG.lookupSCC(N) != C)
853  continue;
854 
855  LLVM_DEBUG(dbgs() << "Inlining calls in: " << F.getName() << "\n"
856  << " Function size: " << F.getInstructionCount()
857  << "\n");
858 
859  auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
861  };
862 
863  // Now process as many calls as we have within this caller in the sequence.
864  // We bail out as soon as the caller has to change so we can update the
865  // call graph and prepare the context of that new caller.
866  bool DidInline = false;
867  for (; I < (int)Calls.size() && Calls[I].first->getCaller() == &F; ++I) {
868  auto &P = Calls[I];
869  CallBase *CB = P.first;
870  const int InlineHistoryID = P.second;
872 
873  if (InlineHistoryID != -1 &&
874  inlineHistoryIncludes(&Callee, InlineHistoryID, InlineHistory)) {
875  LLVM_DEBUG(dbgs() << "Skipping inlining due to history: "
876  << F.getName() << " -> " << Callee.getName() << "\n");
877  setInlineRemark(*CB, "recursive");
878  continue;
879  }
880 
881  // Check if this inlining may repeat breaking an SCC apart that has
882  // already been split once before. In that case, inlining here may
883  // trigger infinite inlining, much like is prevented within the inliner
884  // itself by the InlineHistory above, but spread across CGSCC iterations
885  // and thus hidden from the full inline history.
886  LazyCallGraph::SCC *CalleeSCC = CG.lookupSCC(*CG.lookup(Callee));
887  if (CalleeSCC == C && UR.InlinedInternalEdges.count({&N, C})) {
888  LLVM_DEBUG(dbgs() << "Skipping inlining internal SCC edge from a node "
889  "previously split out of this SCC by inlining: "
890  << F.getName() << " -> " << Callee.getName() << "\n");
891  setInlineRemark(*CB, "recursive SCC split");
892  continue;
893  }
894 
895  std::unique_ptr<InlineAdvice> Advice =
896  Advisor.getAdvice(*CB, OnlyMandatory);
897 
898  // Check whether we want to inline this callsite.
899  if (!Advice)
900  continue;
901 
902  if (!Advice->isInliningRecommended()) {
903  Advice->recordUnattemptedInlining();
904  continue;
905  }
906 
907  int CBCostMult =
910  .value_or(1);
911 
912  // Setup the data structure used to plumb customization into the
913  // `InlineFunction` routine.
914  InlineFunctionInfo IFI(
915  /*cg=*/nullptr, GetAssumptionCache, PSI,
918 
919  InlineResult IR =
920  InlineFunction(*CB, IFI, /*MergeAttributes=*/true,
921  &FAM.getResult<AAManager>(*CB->getCaller()));
922  if (!IR.isSuccess()) {
923  Advice->recordUnsuccessfulInlining(IR);
924  continue;
925  }
926 
927  DidInline = true;
928  InlinedCallees.insert(&Callee);
929  ++NumInlined;
930 
931  LLVM_DEBUG(dbgs() << " Size after inlining: "
932  << F.getInstructionCount() << "\n");
933 
934  // Add any new callsites to defined functions to the worklist.
935  if (!IFI.InlinedCallSites.empty()) {
936  int NewHistoryID = InlineHistory.size();
937  InlineHistory.push_back({&Callee, InlineHistoryID});
938 
939  for (CallBase *ICB : reverse(IFI.InlinedCallSites)) {
940  Function *NewCallee = ICB->getCalledFunction();
941  assert(!(NewCallee && NewCallee->isIntrinsic()) &&
942  "Intrinsic calls should not be tracked.");
943  if (!NewCallee) {
944  // Try to promote an indirect (virtual) call without waiting for
945  // the post-inline cleanup and the next DevirtSCCRepeatedPass
946  // iteration because the next iteration may not happen and we may
947  // miss inlining it.
948  if (tryPromoteCall(*ICB))
949  NewCallee = ICB->getCalledFunction();
950  }
951  if (NewCallee) {
952  if (!NewCallee->isDeclaration()) {
953  Calls.push_back({ICB, NewHistoryID});
954  // Continually inlining through an SCC can result in huge compile
955  // times and bloated code since we arbitrarily stop at some point
956  // when the inliner decides it's not profitable to inline anymore.
957  // We attempt to mitigate this by making these calls exponentially
958  // more expensive.
959  // This doesn't apply to calls in the same SCC since if we do
960  // inline through the SCC the function will end up being
961  // self-recursive which the inliner bails out on, and inlining
962  // within an SCC is necessary for performance.
963  if (CalleeSCC != C &&
964  CalleeSCC == CG.lookupSCC(CG.get(*NewCallee))) {
965  Attribute NewCBCostMult = Attribute::get(
966  M.getContext(),
968  itostr(CBCostMult * IntraSCCCostMultiplier));
969  ICB->addFnAttr(NewCBCostMult);
970  }
971  }
972  }
973  }
974  }
975 
976  // For local functions or discardable functions without comdats, check
977  // whether this makes the callee trivially dead. In that case, we can drop
978  // the body of the function eagerly which may reduce the number of callers
979  // of other functions to one, changing inline cost thresholds. Non-local
980  // discardable functions with comdats are checked later on.
981  bool CalleeWasDeleted = false;
982  if (Callee.isDiscardableIfUnused() && Callee.hasZeroLiveUses() &&
983  !CG.isLibFunction(Callee)) {
984  if (Callee.hasLocalLinkage() || !Callee.hasComdat()) {
985  Calls.erase(
986  std::remove_if(Calls.begin() + I + 1, Calls.end(),
987  [&](const std::pair<CallBase *, int> &Call) {
988  return Call.first->getCaller() == &Callee;
989  }),
990  Calls.end());
991 
992  // Clear the body and queue the function itself for deletion when we
993  // finish inlining and call graph updates.
994  // Note that after this point, it is an error to do anything other
995  // than use the callee's address or delete it.
996  Callee.dropAllReferences();
997  assert(!is_contained(DeadFunctions, &Callee) &&
998  "Cannot put cause a function to become dead twice!");
999  DeadFunctions.push_back(&Callee);
1000  CalleeWasDeleted = true;
1001  } else {
1002  DeadFunctionsInComdats.push_back(&Callee);
1003  }
1004  }
1005  if (CalleeWasDeleted)
1006  Advice->recordInliningWithCalleeDeleted();
1007  else
1008  Advice->recordInlining();
1009  }
1010 
1011  // Back the call index up by one to put us in a good position to go around
1012  // the outer loop.
1013  --I;
1014 
1015  if (!DidInline)
1016  continue;
1017  Changed = true;
1018 
1019  // At this point, since we have made changes we have at least removed
1020  // a call instruction. However, in the process we do some incremental
1021  // simplification of the surrounding code. This simplification can
1022  // essentially do all of the same things as a function pass and we can
1023  // re-use the exact same logic for updating the call graph to reflect the
1024  // change.
1025 
1026  // Inside the update, we also update the FunctionAnalysisManager in the
1027  // proxy for this particular SCC. We do this as the SCC may have changed and
1028  // as we're going to mutate this particular function we want to make sure
1029  // the proxy is in place to forward any invalidation events.
1030  LazyCallGraph::SCC *OldC = C;
1031  C = &updateCGAndAnalysisManagerForCGSCCPass(CG, *C, N, AM, UR, FAM);
1032  LLVM_DEBUG(dbgs() << "Updated inlining SCC: " << *C << "\n");
1033 
1034  // If this causes an SCC to split apart into multiple smaller SCCs, there
1035  // is a subtle risk we need to prepare for. Other transformations may
1036  // expose an "infinite inlining" opportunity later, and because of the SCC
1037  // mutation, we will revisit this function and potentially re-inline. If we
1038  // do, and that re-inlining also has the potentially to mutate the SCC
1039  // structure, the infinite inlining problem can manifest through infinite
1040  // SCC splits and merges. To avoid this, we capture the originating caller
1041  // node and the SCC containing the call edge. This is a slight over
1042  // approximation of the possible inlining decisions that must be avoided,
1043  // but is relatively efficient to store. We use C != OldC to know when
1044  // a new SCC is generated and the original SCC may be generated via merge
1045  // in later iterations.
1046  //
1047  // It is also possible that even if no new SCC is generated
1048  // (i.e., C == OldC), the original SCC could be split and then merged
1049  // into the same one as itself. and the original SCC will be added into
1050  // UR.CWorklist again, we want to catch such cases too.
1051  //
1052  // FIXME: This seems like a very heavyweight way of retaining the inline
1053  // history, we should look for a more efficient way of tracking it.
1054  if ((C != OldC || UR.CWorklist.count(OldC)) &&
1055  llvm::any_of(InlinedCallees, [&](Function *Callee) {
1056  return CG.lookupSCC(*CG.lookup(*Callee)) == OldC;
1057  })) {
1058  LLVM_DEBUG(dbgs() << "Inlined an internal call edge and split an SCC, "
1059  "retaining this to avoid infinite inlining.\n");
1060  UR.InlinedInternalEdges.insert({&N, OldC});
1061  }
1062  InlinedCallees.clear();
1063 
1064  // Invalidate analyses for this function now so that we don't have to
1065  // invalidate analyses for all functions in this SCC later.
1067  }
1068 
1069  // We must ensure that we only delete functions with comdats if every function
1070  // in the comdat is going to be deleted.
1071  if (!DeadFunctionsInComdats.empty()) {
1072  filterDeadComdatFunctions(DeadFunctionsInComdats);
1073  for (auto *Callee : DeadFunctionsInComdats)
1074  Callee->dropAllReferences();
1075  DeadFunctions.append(DeadFunctionsInComdats);
1076  }
1077 
1078  // Now that we've finished inlining all of the calls across this SCC, delete
1079  // all of the trivially dead functions, updating the call graph and the CGSCC
1080  // pass manager in the process.
1081  //
1082  // Note that this walks a pointer set which has non-deterministic order but
1083  // that is OK as all we do is delete things and add pointers to unordered
1084  // sets.
1085  for (Function *DeadF : DeadFunctions) {
1086  // Get the necessary information out of the call graph and nuke the
1087  // function there. Also, clear out any cached analyses.
1088  auto &DeadC = *CG.lookupSCC(*CG.lookup(*DeadF));
1089  FAM.clear(*DeadF, DeadF->getName());
1090  AM.clear(DeadC, DeadC.getName());
1091  auto &DeadRC = DeadC.getOuterRefSCC();
1092  CG.removeDeadFunction(*DeadF);
1093 
1094  // Mark the relevant parts of the call graph as invalid so we don't visit
1095  // them.
1096  UR.InvalidatedSCCs.insert(&DeadC);
1097  UR.InvalidatedRefSCCs.insert(&DeadRC);
1098 
1099  // If the updated SCC was the one containing the deleted function, clear it.
1100  if (&DeadC == UR.UpdatedC)
1101  UR.UpdatedC = nullptr;
1102 
1103  // And delete the actual function from the module.
1104  M.getFunctionList().erase(DeadF);
1105 
1106  ++NumDeleted;
1107  }
1108 
1109  if (!Changed)
1110  return PreservedAnalyses::all();
1111 
1112  PreservedAnalyses PA;
1113  // Even if we change the IR, we update the core CGSCC data structures and so
1114  // can preserve the proxy to the function analysis manager.
1116  // We have already invalidated all analyses on modified functions.
1118  return PA;
1119 }
1120 
1122  bool MandatoryFirst,
1123  InlineContext IC,
1125  unsigned MaxDevirtIterations)
1126  : Params(Params), IC(IC), Mode(Mode),
1128  // Run the inliner first. The theory is that we are walking bottom-up and so
1129  // the callees have already been fully optimized, and we want to inline them
1130  // into the callers so that our optimizations can reflect that.
1131  // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
1132  // because it makes profile annotation in the backend inaccurate.
1133  if (MandatoryFirst) {
1134  PM.addPass(InlinerPass(/*OnlyMandatory*/ true));
1137  }
1138  PM.addPass(InlinerPass());
1141 }
1142 
1145  auto &IAA = MAM.getResult<InlineAdvisorAnalysis>(M);
1146  if (!IAA.tryCreate(Params, Mode,
1147  {CGSCCInlineReplayFile,
1148  CGSCCInlineReplayScope,
1149  CGSCCInlineReplayFallback,
1150  {CGSCCInlineReplayFormat}},
1151  IC)) {
1152  M.getContext().emitError(
1153  "Could not setup Inlining Advisor for the requested "
1154  "mode and/or options");
1155  return PreservedAnalyses::all();
1156  }
1157 
1158  // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
1159  // to detect when we devirtualize indirect calls and iterate the SCC passes
1160  // in that case to try and catch knock-on inlining or function attrs
1161  // opportunities. Then we add it to the module pipeline by walking the SCCs
1162  // in postorder (or bottom-up).
1163  // If MaxDevirtIterations is 0, we just don't use the devirtualization
1164  // wrapper.
1165  if (MaxDevirtIterations == 0)
1167  else
1170 
1171  MPM.addPass(std::move(AfterCGMPM));
1172  MPM.run(M, MAM);
1173 
1174  // Discard the InlineAdvisor, a subsequent inlining session should construct
1175  // its own.
1176  auto PA = PreservedAnalyses::all();
1178  PA.abandon<InlineAdvisorAnalysis>();
1179  return PA;
1180 }
1181 
1183  raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
1184  static_cast<PassInfoMixin<InlinerPass> *>(this)->printPipeline(
1185  OS, MapClassName2PassName);
1186  if (OnlyMandatory)
1187  OS << "<only-mandatory>";
1188 }
1189 
1191  raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
1192  // Print some info about passes added to the wrapper. This is however
1193  // incomplete as InlineAdvisorAnalysis part isn't included (which also depends
1194  // on Params and Mode).
1195  if (!MPM.isEmpty()) {
1196  MPM.printPipeline(OS, MapClassName2PassName);
1197  OS << ",";
1198  }
1199  OS << "cgscc(";
1200  if (MaxDevirtIterations != 0)
1201  OS << "devirt<" << MaxDevirtIterations << ">(";
1202  PM.printPipeline(OS, MapClassName2PassName);
1203  if (MaxDevirtIterations != 0)
1204  OS << ")";
1205  OS << ")";
1206 }
llvm::CallGraphNode::removeAnyCallEdgeTo
void removeAnyCallEdgeTo(CallGraphNode *Callee)
Removes all call edges from this node to the specified callee function.
Definition: CallGraph.cpp:233
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
llvm::InlineAdvisor::onPassEntry
virtual void onPassEntry(LazyCallGraph::SCC *SCC=nullptr)
This must be called when the Inliner pass is entered, to allow the InlineAdvisor update internal stat...
Definition: InlineAdvisor.h:184
llvm::OuterAnalysisManagerProxy
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
Definition: PassManager.h:1056
llvm::array_pod_sort
void array_pod_sort(IteratorTy Start, IteratorTy End)
array_pod_sort - This sorts an array with the specified start and end extent.
Definition: STLExtras.h:1642
AssumptionCache.h
llvm::AAManager
A manager for alias analyses.
Definition: AliasAnalysis.h:876
llvm::OptimizationRemarkMissed
Diagnostic information for missed-optimization remarks.
Definition: DiagnosticInfo.h:734
llvm::Function::isIntrinsic
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:210
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::SmallVectorImpl::erase
iterator erase(const_iterator CI)
Definition: SmallVector.h:741
llvm::getStringFnAttrAsInt
Optional< int > getStringFnAttrAsInt(CallBase &CB, StringRef AttrKind)
Definition: InlineCost.cpp:174
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::CallGraph::getExternalCallingNode
CallGraphNode * getExternalCallingNode() const
Returns the CallGraphNode which is used to represent undetermined calls into the callgraph.
Definition: CallGraph.h:127
llvm::PassManager::isEmpty
bool isEmpty() const
Returns if the pass manager contains any passes.
Definition: PassManager.h:568
Optional.h
llvm::LegacyInlinerBase::getInlineCost
virtual InlineCost getInlineCost(CallBase &CB)=0
This method must be implemented by the subclass to determine the cost of inlining the specified call ...
Metadata.h
llvm::AllocaInst::getAlign
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:122
IntrinsicInst.h
llvm::AnalysisManager::getResult
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:774
Inliner.h
llvm::PassInfoMixin< InlinerPass >
InstIterator.h
llvm::Function
Definition: Function.h:60
llvm::Attribute
Definition: Attributes.h:66
llvm::AnalysisManager::invalidate
void invalidate(IRUnitT &IR, const PreservedAnalyses &PA)
Invalidate cached analyses for an IR unit.
Definition: PassManagerImpl.h:89
StringRef.h
llvm::PassManager::addPass
LLVM_ATTRIBUTE_MINSIZE std::enable_if_t<!std::is_same< PassT, PassManager >::value > addPass(PassT &&Pass)
Definition: PassManager.h:544
P
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Definition: README-SSE.txt:411
Pass.h
DisableInlinedAllocaMerging
static cl::opt< bool > DisableInlinedAllocaMerging("disable-inlined-alloca-merging", cl::init(false), cl::Hidden)
Flag to disable manual alloca merging.
llvm::InlinerFunctionImportStatsOpts::Verbose
@ Verbose
llvm::Attribute::get
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition: Attributes.cpp:91
llvm::InlineFunction
InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)
This function inlines the called function into the basic block of the caller.
Definition: InlineFunction.cpp:2046
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1199
Statistic.h
llvm::LegacyInlinerBase::inlineCalls
bool inlineCalls(CallGraphSCC &SCC)
This function performs the main work of the pass.
Definition: Inliner.cpp:598
mergeInlinedArrayAllocas
static void mergeInlinedArrayAllocas(Function *Caller, InlineFunctionInfo &IFI, InlinedArrayAllocasTy &InlinedArrayAllocas, int InlineHistory)
Look at all of the allocas that we inlined through this call site.
Definition: Inliner.cpp:205
llvm::LegacyInlinerBase::GetTLI
std::function< const TargetLibraryInfo &(Function &)> GetTLI
Definition: Inliner.h:78
llvm::ImportedFunctionsInliningStatistics
Calculate and dump ThinLTO specific inliner stats.
Definition: ImportedFunctionsInliningStatistics.h:44
llvm::AllAnalysesOn
This templated class represents "all analyses that operate over <a particular IR unit>" (e....
Definition: PassManager.h:90
llvm::ImportedFunctionsInliningStatistics::setModuleInfo
void setModuleInfo(const Module &M)
Set information like AllFunctions, ImportedFunctions, ModuleName.
Definition: ImportedFunctionsInliningStatistics.cpp:75
Local.h
OptimizationRemarkEmitter.h
llvm::CallGraph
The basic data container for the call graph of a Module of IR.
Definition: CallGraph.h:72
FAM
FunctionAnalysisManager FAM
Definition: PassBuilderBindings.cpp:59
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:140
IntraSCCCostMultiplier
static cl::opt< int > IntraSCCCostMultiplier("intra-scc-cost-multiplier", cl::init(2), cl::Hidden, cl::desc("Cost multiplier to multiply onto inlined call sites where the " "new call was previously an intra-SCC call (not relevant when the " "original call was already intra-SCC). This can accumulate over " "multiple inlinings (e.g. if a call site already had a cost " "multiplier and one of its inlined calls was also subject to " "this, the inlined call would have the original multiplier " "multiplied by intra-scc-cost-multiplier). This is to prevent tons of " "inlining through a child SCC which can cause terrible compile times"))
llvm::emitInlinedIntoBasedOnCost
void emitInlinedIntoBasedOnCost(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, const Function &Callee, const Function &Caller, const InlineCost &IC, bool ForProfileContext=false, const char *PassName=nullptr)
Emit ORE message based in cost (default heuristic).
Definition: InlineAdvisor.cpp:502
llvm::PreservedAnalyses::none
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:155
DenseMap.h
Module.h
EnablePostSCCAdvisorPrinting
static cl::opt< bool > EnablePostSCCAdvisorPrinting("enable-scc-inline-advisor-printing", cl::init(false), cl::Hidden)
Allows printing the contents of the advisor after each SCC inliner pass.
llvm::PassManager::printPipeline
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
Definition: PassManager.h:486
llvm::InlineParams
Thresholds to tune inline cost analysis.
Definition: InlineCost.h:204
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:450
llvm::ore::NV
DiagnosticInfoOptimizationBase::Argument NV
Definition: OptimizationRemarkEmitter.h:136
llvm::InlineConstants::FunctionInlineCostMultiplierAttributeName
const char FunctionInlineCostMultiplierAttributeName[]
Definition: InlineCost.h:60
llvm::CallSiteFormat::Format::LineDiscriminator
@ LineDiscriminator
STLExtras.h
llvm::ArrayType
Class to represent array types.
Definition: DerivedTypes.h:357
llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, DenseMapInfo< ValueT >, detail::DenseSetPair< ValueT > >, DenseMapInfo< ValueT > >::insert
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
BasicAliasAnalysis.h
llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, DenseMapInfo< ValueT >, detail::DenseSetPair< ValueT > >, DenseMapInfo< ValueT > >::count
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:97
llvm::ReplayInlinerSettings::Fallback::Original
@ Original
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::getAAResultsAnalysisUsage
void getAAResultsAnalysisUsage(AnalysisUsage &AU)
A helper for the legacy pass manager to populate AU to add uses to make sure the analyses required by...
Definition: AliasAnalysis.cpp:952
F
#define F(x, y, z)
Definition: MD5.cpp:55
llvm::LegacyInlinerBase::LegacyInlinerBase
LegacyInlinerBase(char &ID)
Definition: Inliner.cpp:167
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:55
llvm::InlineAdvisorAnalysisPrinterPass
Printer pass for the FunctionPropertiesAnalysis results.
Definition: InlineAdvisor.h:271
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
PriorityWorklist.h
llvm::remove_if
auto remove_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::remove_if which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1773
Instruction.h
CommandLine.h
ImportedFunctionsInliningStatistics.h
llvm::InlinerFunctionImportStatsOpts::No
@ No
llvm::CallGraphSCC
CallGraphSCC - This is a single SCC that a CallGraphSCCPass is run on.
Definition: CallGraphSCCPass.h:87
llvm::LazyCallGraph::SCC
An SCC of the call graph.
Definition: LazyCallGraph.h:420
llvm::GlobalValue::isDeclaration
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:266
llvm::InlineAdvisor::onPassExit
virtual void onPassExit(LazyCallGraph::SCC *SCC=nullptr)
This must be called when the Inliner pass is exited, as function passes may be run subsequently.
Definition: InlineAdvisor.h:189
inlineCallsImpl
static bool inlineCallsImpl(CallGraphSCC &SCC, CallGraph &CG, std::function< AssumptionCache &(Function &)> GetAssumptionCache, ProfileSummaryInfo *PSI, std::function< const TargetLibraryInfo &(Function &)> GetTLI, bool InsertLifetime, function_ref< InlineCost(CallBase &CB)> GetInlineCost, function_ref< AAResults &(Function &)> AARGetter, ImportedFunctionsInliningStatistics &ImportedFunctionsStats)
Definition: Inliner.cpp:363
llvm::AAResults
Definition: AliasAnalysis.h:294
llvm::AllocaInst::getAllocatedType
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:115
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::User
Definition: User.h:44
CGSCCInlineReplayFormat
static cl::opt< CallSiteFormat::Format > CGSCCInlineReplayFormat("cgscc-inline-replay-format", cl::init(CallSiteFormat::Format::LineColumnDiscriminator), cl::values(clEnumValN(CallSiteFormat::Format::Line, "Line", "<Line Number>"), clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn", "<Line Number>:<Column Number>"), clEnumValN(CallSiteFormat::Format::LineDiscriminator, "LineDiscriminator", "<Line Number>.<Discriminator>"), clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator, "LineColumnDiscriminator", "<Line Number>:<Column Number>.<Discriminator> (default)")), cl::desc("How cgscc inline replay file is formatted"), cl::Hidden)
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1397
int
Clang compiles this i1 i64 store i64 i64 store i64 i64 store i64 i64 store i64 align Which gets codegen d xmm0 movaps rbp movaps rbp movaps rbp movaps rbp rbp rbp rbp rbp It would be better to have movq s of instead of the movaps s LLVM produces ret int
Definition: README.txt:536
llvm::ModuleInlinerWrapperPass::printPipeline
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
Definition: Inliner.cpp:1190
MAM
ModuleAnalysisManager MAM
Definition: PassBuilderBindings.cpp:61
llvm::AnalysisManager::clear
void clear(IRUnitT &IR, llvm::StringRef Name)
Clear any cached analysis results for a single unit of IR.
Definition: PassManagerImpl.h:36
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
llvm::InlineCost
Represents the cost of inlining a function.
Definition: InlineCost.h:90
TargetLibraryInfo.h
InlineAdvisor.h
llvm::CallGraphSCCPass::skipSCC
bool skipSCC(CallGraphSCC &SCC) const
Optional passes call this function to check whether the pass should be skipped.
Definition: CallGraphSCCPass.cpp:751
llvm::CallSiteFormat::Format::LineColumnDiscriminator
@ LineColumnDiscriminator
llvm::Instruction
Definition: Instruction.h:42
llvm::OuterAnalysisManagerProxy::Result
Result proxy object for OuterAnalysisManagerProxy.
Definition: PassManager.h:1061
InlineInfo
@ InlineInfo
Definition: FunctionInfo.cpp:25
llvm::STATISTIC
STATISTIC(NumFunctions, "Total number of functions")
llvm::ReplayInlinerSettings::Fallback::NeverInline
@ NeverInline
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:53
llvm::createDevirtSCCRepeatedPass
DevirtSCCRepeatedPass createDevirtSCCRepeatedPass(CGSCCPassT &&Pass, int MaxIterations)
A function to deduce a function pass type and wrap it in the templated adaptor.
Definition: CGSCCPassManager.h:581
DebugLoc.h
IR
Statically lint checks LLVM IR
Definition: Lint.cpp:746
SmallPtrSet.h
llvm::CallGraphNode
A node in the call graph for a module.
Definition: CallGraph.h:166
llvm::getInlineParams
InlineParams getInlineParams()
Generate the parameters to tune the inline cost analysis based only on the commandline options.
Definition: InlineCost.cpp:3101
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:687
LazyCallGraph.h
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::CallSiteFormat::Format::LineColumn
@ LineColumn
llvm::InlineContext
Provides context on when an inline advisor is constructed in the pipeline (e.g., link phase,...
Definition: InlineAdvisor.h:58
llvm::BlockFrequencyAnalysis
Analysis pass which computes BlockFrequencyInfo.
Definition: BlockFrequencyInfo.h:112
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm::CallBase::getCaller
Function * getCaller()
Helper to get the caller (the parent function).
Definition: Instructions.cpp:284
llvm::LegacyInlinerBase::ACT
AssumptionCacheTracker * ACT
Definition: Inliner.h:76
llvm::LegacyInlinerBase::removeDeadFunctions
bool removeDeadFunctions(CallGraph &CG, bool AlwaysInlineOnly=false)
Remove dead functions.
Definition: Inliner.cpp:624
llvm::ProfileSummaryInfo
Analysis providing profile information.
Definition: ProfileSummaryInfo.h:40
llvm::LocalAsMetadata::getIfExists
static LocalAsMetadata * getIfExists(Value *Local)
Definition: Metadata.h:450
llvm::function_ref
An efficient, type-erasing, non-owning reference to a callable.
Definition: STLFunctionalExtras.h:36
llvm::DenseSet
Implements a dense probed hash-table based set.
Definition: DenseSet.h:268
inlineHistoryIncludes
static bool inlineHistoryIncludes(Function *F, int InlineHistoryID, const SmallVectorImpl< std::pair< Function *, int >> &InlineHistory)
Return true if the specified inline history ID indicates an inline history that includes the specifie...
Definition: Inliner.cpp:337
BasicBlock.h
llvm::cl::opt< bool >
ReplayInlineAdvisor.h
llvm::InliningAdvisorMode
InliningAdvisorMode
There are 3 scenarios we can use the InlineAdvisor:
Definition: InlineAdvisor.h:42
llvm::instructions
inst_range instructions(Function *F)
Definition: InstIterator.h:133
CGSCCInlineReplayFile
static cl::opt< std::string > CGSCCInlineReplayFile("cgscc-inline-replay", cl::init(""), cl::value_desc("filename"), cl::desc("Optimization remarks file containing inline remarks to be replayed " "by cgscc inlining."), cl::Hidden)
CGSCCInlineReplayFallback
static cl::opt< ReplayInlinerSettings::Fallback > CGSCCInlineReplayFallback("cgscc-inline-replay-fallback", cl::init(ReplayInlinerSettings::Fallback::Original), cl::values(clEnumValN(ReplayInlinerSettings::Fallback::Original, "Original", "All decisions not in replay send to original advisor (default)"), clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline, "AlwaysInline", "All decisions not in replay are inlined"), clEnumValN(ReplayInlinerSettings::Fallback::NeverInline, "NeverInline", "All decisions not in replay are not inlined")), cl::desc("How cgscc inline replay treats sites that don't come from the replay. " "Original: defers to original advisor, AlwaysInline: inline all sites " "not in replay, NeverInline: inline no sites not in replay"), cl::Hidden)
llvm::cl::values
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
Definition: CommandLine.h:705
llvm::Instruction::eraseFromParent
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:81
llvm::CallGraphSCCPass::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &Info) const override
getAnalysisUsage - For this class, we declare that we require and preserve the call graph.
Definition: CallGraphSCCPass.cpp:658
llvm::LegacyInlinerBase::runOnSCC
bool runOnSCC(CallGraphSCC &SCC) override
Main run interface method, this implements the interface required by the Pass class.
Definition: Inliner.cpp:356
llvm::TargetLibraryInfoWrapperPass
Definition: TargetLibraryInfo.h:475
ProfileSummaryInfo.h
llvm::CallGraphNode::removeAllCalledFunctions
void removeAllCalledFunctions()
Removes all edges from this CallGraphNode to any functions it calls.
Definition: CallGraph.h:227
llvm::DbgDeclareInst
This represents the llvm.dbg.declare instruction.
Definition: IntrinsicInst.h:348
llvm::AssumptionAnalysis
A function analysis which provides an AssumptionCache.
Definition: AssumptionCache.h:173
llvm::filterDeadComdatFunctions
void filterDeadComdatFunctions(SmallVectorImpl< Function * > &DeadComdatFunctions)
Filter out potentially dead comdat functions where other entries keep the entire comdat group alive.
Definition: ModuleUtils.cpp:182
llvm::PreservedAnalyses::preserve
void preserve()
Mark an analysis as preserved.
Definition: PassManager.h:173
move
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
Definition: README.txt:546
llvm::DenseMap
Definition: DenseMap.h:714
llvm::InlinerFunctionImportStats
cl::opt< InlinerFunctionImportStatsOpts > InlinerFunctionImportStats("inliner-function-import-stats", cl::init(InlinerFunctionImportStatsOpts::No), cl::values(clEnumValN(InlinerFunctionImportStatsOpts::Basic, "basic", "basic statistics"), clEnumValN(InlinerFunctionImportStatsOpts::Verbose, "verbose", "printing of statistics for each inlined function")), cl::Hidden, cl::desc("Enable inliner stats for imported functions"))
Definition: InlineAdvisor.cpp:65
CGSCCPassManager.h
I
#define I(x, y, z)
Definition: MD5.cpp:58
Cloning.h
StringExtras.h
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:447
llvm::ProfileSummaryInfoWrapperPass
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
Definition: ProfileSummaryInfo.h:194
llvm::is_contained
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:1868
llvm::ReplayInlinerSettings::Scope::Module
@ Module
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
DEBUG_TYPE
#define DEBUG_TYPE
Definition: Inliner.cpp:72
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:853
llvm::setInlineRemark
void setInlineRemark(CallBase &CB, StringRef Message)
Set the inline-remark attribute.
Definition: InlineAdvisor.cpp:357
llvm::CallGraph::removeFunctionFromModule
Function * removeFunctionFromModule(CallGraphNode *CGN)
Unlink the function from this module, returning it.
Definition: CallGraph.cpp:161
llvm::OptimizationRemarkEmitter::emit
void emit(DiagnosticInfoOptimizationBase &OptDiag)
Output the remark via the diagnostic handler and to the optimization record file.
Definition: OptimizationRemarkEmitter.cpp:77
llvm::LegacyInlinerBase::PSI
ProfileSummaryInfo * PSI
Definition: Inliner.h:77
InlineCost.h
Ptr
@ Ptr
Definition: TargetLibraryInfo.cpp:60
llvm::InlinePass::CGSCCInliner
@ CGSCCInliner
inlineCallIfPossible
static InlineResult inlineCallIfPossible(CallBase &CB, InlineFunctionInfo &IFI, InlinedArrayAllocasTy &InlinedArrayAllocas, int InlineHistory, bool InsertLifetime, function_ref< AAResults &(Function &)> &AARGetter, ImportedFunctionsInliningStatistics &ImportedFunctionsStats)
If it is possible to inline the specified call site, do so and update the CallGraph for this operatio...
Definition: Inliner.cpp:308
Mode
SI Whole Quad Mode
Definition: SIWholeQuadMode.cpp:264
function
print Print MemDeps of function
Definition: MemDepPrinter.cpp:82
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
llvm::LazyCallGraph::Node
A node in the call graph.
Definition: LazyCallGraph.h:316
llvm::SmallPtrSetImpl::count
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:383
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::insert
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:141
llvm::AMDGPU::CPol::SCC
@ SCC
Definition: SIDefines.h:307
llvm::ProfileSummaryAnalysis
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Definition: ProfileSummaryInfo.h:212
llvm::CallSiteFormat::Format::Line
@ Line
llvm::AssumptionCacheTracker
An immutable pass that tracks lazily created AssumptionCache objects.
Definition: AssumptionCache.h:202
llvm::isInstructionTriviallyDead
bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction is not used, and the instruction will return.
Definition: Local.cpp:396
llvm::OptimizationRemarkEmitter
The optimization diagnostic interface.
Definition: OptimizationRemarkEmitter.h:33
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1741
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
llvm::AssumptionCache
A cache of @llvm.assume calls within a function.
Definition: AssumptionCache.h:42
llvm::InlineAdvisor
Interface for deciding whether to inline a call site or not.
Definition: InlineAdvisor.h:163
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:532
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:994
llvm::AllocaInst::isArrayAllocation
bool isArrayAllocation() const
Return true if there is an allocation size parameter to the allocation instruction that is not 1.
Definition: Instructions.cpp:1498
clEnumValN
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Definition: CommandLine.h:680
llvm::InlinerPass::printPipeline
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
Definition: Inliner.cpp:1182
CallPromotionUtils.h
BlockFrequencyInfo.h
llvm::make_scope_exit
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:59
llvm::InlineFunctionInfo::InlinedCallSites
SmallVector< CallBase *, 8 > InlinedCallSites
All of the new call sites inlined into the caller.
Definition: Cloning.h:235
Callee
amdgpu Simplify well known AMD library false FunctionCallee Callee
Definition: AMDGPULibCalls.cpp:187
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::clear
void clear()
Completely clear the SetVector.
Definition: SetVector.h:220
Attributes.h
llvm::CallGraphNode::getFunction
Function * getFunction() const
Returns the function that this call graph node represents.
Definition: CallGraph.h:197
llvm::ImportedFunctionsInliningStatistics::recordInline
void recordInline(const Function &Caller, const Function &Callee)
Record inline of.
Definition: ImportedFunctionsInliningStatistics.cpp:48
llvm::CGSCCUpdateResult
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Definition: CGSCCPassManager.h:232
llvm::AssumptionCacheTracker::getAssumptionCache
AssumptionCache & getAssumptionCache(Function &F)
Get the cached assumptions for a function.
Definition: AssumptionCache.cpp:285
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
llvm::AnalysisManager::getCachedResult
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
Definition: PassManager.h:793
Casting.h
llvm::ModuleInlinerWrapperPass::ModuleInlinerWrapperPass
ModuleInlinerWrapperPass(InlineParams Params=getInlineParams(), bool MandatoryFirst=true, InlineContext IC={}, InliningAdvisorMode Mode=InliningAdvisorMode::Default, unsigned MaxDevirtIterations=0)
Definition: Inliner.cpp:1121
DiagnosticInfo.h
Function.h
llvm::CallGraphSCCPass
Definition: CallGraphSCCPass.h:34
PassManager.h
llvm::TargetLibraryInfo
Provides information about what library functions are available for the current target.
Definition: TargetLibraryInfo.h:226
llvm::InlineFunctionInfo
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
Definition: Cloning.h:203
KeepAdvisorForPrinting
static cl::opt< bool > KeepAdvisorForPrinting("keep-inline-advisor-for-printing", cl::init(false), cl::Hidden)
A flag for test, so we can print the content of the advisor when running it as part of the default (e...
llvm::unique
auto unique(Range &&R, Predicate P)
Definition: STLExtras.h:1965
llvm::cl::value_desc
Definition: CommandLine.h:422
llvm::inlineCostStr
std::string inlineCostStr(const InlineCost &IC)
Utility for extracting the inline cost message to a string.
Definition: InlineAdvisor.cpp:350
llvm::PassManager::run
PreservedAnalyses run(IRUnitT &IR, AnalysisManagerT &AM, ExtraArgTs... ExtraArgs)
Run all of the passes in this manager over the given unit of IR.
Definition: PassManager.h:498
llvm::CallGraph::getModule
Module & getModule() const
Returns the module the call graph corresponds to.
Definition: CallGraph.h:101
llvm::updateCGAndAnalysisManagerForCGSCCPass
LazyCallGraph::SCC & updateCGAndAnalysisManagerForCGSCCPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a CGSCC pass.
Definition: CGSCCPassManager.cpp:1216
CallGraph.h
llvm::ModuleInlinerWrapperPass::run
PreservedAnalyses run(Module &, ModuleAnalysisManager &)
Definition: Inliner.cpp:1143
llvm::InlineFunctionInfo::StaticAllocas
SmallVector< AllocaInst *, 4 > StaticAllocas
InlineFunction fills this in with all static allocas that get copied into the caller.
Definition: Cloning.h:224
llvm::MetadataAsValue::getIfExists
static MetadataAsValue * getIfExists(LLVMContext &Context, Metadata *MD)
Definition: Metadata.cpp:111
Instructions.h
llvm::PreservedAnalyses::preserveSet
void preserveSet()
Mark an analysis set as preserved.
Definition: PassManager.h:188
llvm::InlineAdvisorAnalysis
The InlineAdvisorAnalysis is a module pass because the InlineAdvisor needs to capture state right bef...
Definition: InlineAdvisor.h:243
SmallVector.h
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:359
User.h
ModuleUtils.h
N
#define N
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:91
llvm::InlinerPass
The inliner pass for the new pass manager.
Definition: Inliner.h:96
llvm::InlinerPass::run
PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)
Definition: Inliner.cpp:744
llvm::ReplayInlinerSettings::Scope::Function
@ Function
ScopeExit.h
llvm::LegacyInlinerBase::doFinalization
bool doFinalization(CallGraph &CG) override
Remove now-dead linkonce functions at the end of processing to avoid breaking the SCC traversal.
Definition: Inliner.cpp:616
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
llvm::reverse
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:485
llvm::LegacyInlinerBase::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &Info) const override
For this class, we declare that we require and preserve the call graph.
Definition: Inliner.cpp:175
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1175
DerivedTypes.h
llvm::SmallSetVector
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:307
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:42
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition: PassAnalysisSupport.h:75
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::ReplayInlinerSettings::Fallback::AlwaysInline
@ AlwaysInline
llvm::AllocaInst
an instruction to allocate memory on the stack
Definition: Instructions.h:59
llvm::cl::desc
Definition: CommandLine.h:413
llvm::LazyCallGraph
A lazily constructed view of the call graph of a module.
Definition: LazyCallGraph.h:111
raw_ostream.h
llvm::MaxDevirtIterations
cl::opt< unsigned > MaxDevirtIterations("max-devirt-iterations", cl::ReallyHidden, cl::init(4))
Definition: PassBuilderPipelines.cpp:301
llvm::FunctionAnalysisManagerCGSCCProxy
A proxy from a FunctionAnalysisManager to an SCC.
Definition: CGSCCPassManager.h:392
Value.h
llvm::LegacyAARGetter
This class is a functor to be used in legacy module or SCC passes for computing AA results for a func...
Definition: BasicAliasAnalysis.h:189
llvm::Optional::value_or
constexpr T value_or(U &&alt) const &
Definition: Optional.h:291
MPM
ModulePassManager MPM
Definition: PassBuilderBindings.cpp:70
llvm::createModuleToPostOrderCGSCCPassAdaptor
ModuleToPostOrderCGSCCPassAdaptor createModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT &&Pass)
A function to deduce a function pass type and wrap it in the templated adaptor.
Definition: CGSCCPassManager.h:373
llvm::OptimizationRemarkEmitterAnalysis
Definition: OptimizationRemarkEmitter.h:164
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::InlineResult
InlineResult is basically true or false.
Definition: InlineCost.h:178
Debug.h
llvm::Value::users
iterator_range< user_iterator > users()
Definition: Value.h:421
llvm::ReplayInlinerSettings
Replay Inliner Setup.
Definition: ReplayInlineAdvisor.h:43
llvm::InlinePass::ReplayCGSCCInliner
@ ReplayCGSCCInliner
llvm::tryPromoteCall
bool tryPromoteCall(CallBase &CB)
Try to promote (devirtualize) a virtual call on an Alloca.
Definition: CallPromotionUtils.cpp:549
CGSCCInlineReplayScope
static cl::opt< ReplayInlinerSettings::Scope > CGSCCInlineReplayScope("cgscc-inline-replay-scope", cl::init(ReplayInlinerSettings::Scope::Function), cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function", "Replay on functions that have remarks associated " "with them (default)"), clEnumValN(ReplayInlinerSettings::Scope::Module, "Module", "Replay on the entire module")), cl::desc("Whether inline replay should be applied to the entire " "Module or just the Functions (default) that are present as " "callers in remarks during cgscc inlining."), cl::Hidden)
SetVector.h
llvm::getReplayInlineAdvisor
std::unique_ptr< InlineAdvisor > getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr< InlineAdvisor > OriginalAdvisor, const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks, InlineContext IC)
Definition: ReplayInlineAdvisor.cpp:80
llvm::ImportedFunctionsInliningStatistics::dump
void dump(bool Verbose)
Dump stats computed with InlinerStatistics class.
Definition: ImportedFunctionsInliningStatistics.cpp:99
llvm::shouldInline
Optional< InlineCost > shouldInline(CallBase &CB, function_ref< InlineCost(CallBase &CB)> GetInlineCost, OptimizationRemarkEmitter &ORE, bool EnableDeferral=true)
Return the cost only if the inliner should attempt to inline at the given CallSite.
Definition: InlineAdvisor.cpp:370
llvm::LegacyInlinerBase::doInitialization
bool doInitialization(CallGraph &CG) override
doInitialization - This method is called before the SCC's of the program has been processed,...
Definition: Inliner.cpp:350
llvm::ore::setIsVerbose
DiagnosticInfoOptimizationBase::setIsVerbose setIsVerbose
Definition: OptimizationRemarkEmitter.h:137
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:365
llvm::LegacyInlinerBase::ImportedFunctionsStats
ImportedFunctionsInliningStatistics ImportedFunctionsStats
Definition: Inliner.h:79