LLVM  9.0.0svn
InlineFunction.cpp
Go to the documentation of this file.
1 //===- InlineFunction.cpp - Code to perform function inlining -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements inlining of a function into a call site, resolving
10 // parameters and the return value as appropriate.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/ADT/DenseMap.h"
15 #include "llvm/ADT/None.h"
16 #include "llvm/ADT/Optional.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/ADT/SetVector.h"
19 #include "llvm/ADT/SmallPtrSet.h"
20 #include "llvm/ADT/SmallVector.h"
21 #include "llvm/ADT/StringExtras.h"
34 #include "llvm/IR/Argument.h"
35 #include "llvm/IR/BasicBlock.h"
36 #include "llvm/IR/CFG.h"
37 #include "llvm/IR/CallSite.h"
38 #include "llvm/IR/Constant.h"
39 #include "llvm/IR/Constants.h"
40 #include "llvm/IR/DIBuilder.h"
41 #include "llvm/IR/DataLayout.h"
43 #include "llvm/IR/DebugLoc.h"
44 #include "llvm/IR/DerivedTypes.h"
45 #include "llvm/IR/Dominators.h"
46 #include "llvm/IR/Function.h"
47 #include "llvm/IR/IRBuilder.h"
48 #include "llvm/IR/InstrTypes.h"
49 #include "llvm/IR/Instruction.h"
50 #include "llvm/IR/Instructions.h"
51 #include "llvm/IR/IntrinsicInst.h"
52 #include "llvm/IR/Intrinsics.h"
53 #include "llvm/IR/LLVMContext.h"
54 #include "llvm/IR/MDBuilder.h"
55 #include "llvm/IR/Metadata.h"
56 #include "llvm/IR/Module.h"
57 #include "llvm/IR/Type.h"
58 #include "llvm/IR/User.h"
59 #include "llvm/IR/Value.h"
60 #include "llvm/Support/Casting.h"
65 #include <algorithm>
66 #include <cassert>
67 #include <cstdint>
68 #include <iterator>
69 #include <limits>
70 #include <string>
71 #include <utility>
72 #include <vector>
73 
74 using namespace llvm;
76 
77 static cl::opt<bool>
78 EnableNoAliasConversion("enable-noalias-to-md-conversion", cl::init(true),
79  cl::Hidden,
80  cl::desc("Convert noalias attributes to metadata during inlining."));
81 
82 static cl::opt<bool>
83 PreserveAlignmentAssumptions("preserve-alignment-assumptions-during-inlining",
84  cl::init(true), cl::Hidden,
85  cl::desc("Convert align attributes to assumptions during inlining."));
86 
88  AAResults *CalleeAAR,
89  bool InsertLifetime) {
90  return InlineFunction(CallSite(CI), IFI, CalleeAAR, InsertLifetime);
91 }
92 
94  AAResults *CalleeAAR,
95  bool InsertLifetime) {
96  return InlineFunction(CallSite(II), IFI, CalleeAAR, InsertLifetime);
97 }
98 
99 namespace {
100 
101  /// A class for recording information about inlining a landing pad.
102  class LandingPadInliningInfo {
103  /// Destination of the invoke's unwind.
104  BasicBlock *OuterResumeDest;
105 
106  /// Destination for the callee's resume.
107  BasicBlock *InnerResumeDest = nullptr;
108 
109  /// LandingPadInst associated with the invoke.
110  LandingPadInst *CallerLPad = nullptr;
111 
112  /// PHI for EH values from landingpad insts.
113  PHINode *InnerEHValuesPHI = nullptr;
114 
115  SmallVector<Value*, 8> UnwindDestPHIValues;
116 
117  public:
118  LandingPadInliningInfo(InvokeInst *II)
119  : OuterResumeDest(II->getUnwindDest()) {
120  // If there are PHI nodes in the unwind destination block, we need to keep
121  // track of which values came into them from the invoke before removing
122  // the edge from this block.
123  BasicBlock *InvokeBB = II->getParent();
124  BasicBlock::iterator I = OuterResumeDest->begin();
125  for (; isa<PHINode>(I); ++I) {
126  // Save the value to use for this edge.
127  PHINode *PHI = cast<PHINode>(I);
128  UnwindDestPHIValues.push_back(PHI->getIncomingValueForBlock(InvokeBB));
129  }
130 
131  CallerLPad = cast<LandingPadInst>(I);
132  }
133 
134  /// The outer unwind destination is the target of
135  /// unwind edges introduced for calls within the inlined function.
136  BasicBlock *getOuterResumeDest() const {
137  return OuterResumeDest;
138  }
139 
140  BasicBlock *getInnerResumeDest();
141 
142  LandingPadInst *getLandingPadInst() const { return CallerLPad; }
143 
144  /// Forward the 'resume' instruction to the caller's landing pad block.
145  /// When the landing pad block has only one predecessor, this is
146  /// a simple branch. When there is more than one predecessor, we need to
147  /// split the landing pad block after the landingpad instruction and jump
148  /// to there.
149  void forwardResume(ResumeInst *RI,
150  SmallPtrSetImpl<LandingPadInst*> &InlinedLPads);
151 
152  /// Add incoming-PHI values to the unwind destination block for the given
153  /// basic block, using the values for the original invoke's source block.
154  void addIncomingPHIValuesFor(BasicBlock *BB) const {
155  addIncomingPHIValuesForInto(BB, OuterResumeDest);
156  }
157 
158  void addIncomingPHIValuesForInto(BasicBlock *src, BasicBlock *dest) const {
159  BasicBlock::iterator I = dest->begin();
160  for (unsigned i = 0, e = UnwindDestPHIValues.size(); i != e; ++i, ++I) {
161  PHINode *phi = cast<PHINode>(I);
162  phi->addIncoming(UnwindDestPHIValues[i], src);
163  }
164  }
165  };
166 
167 } // end anonymous namespace
168 
169 /// Get or create a target for the branch from ResumeInsts.
170 BasicBlock *LandingPadInliningInfo::getInnerResumeDest() {
171  if (InnerResumeDest) return InnerResumeDest;
172 
173  // Split the landing pad.
174  BasicBlock::iterator SplitPoint = ++CallerLPad->getIterator();
175  InnerResumeDest =
176  OuterResumeDest->splitBasicBlock(SplitPoint,
177  OuterResumeDest->getName() + ".body");
178 
179  // The number of incoming edges we expect to the inner landing pad.
180  const unsigned PHICapacity = 2;
181 
182  // Create corresponding new PHIs for all the PHIs in the outer landing pad.
183  Instruction *InsertPoint = &InnerResumeDest->front();
184  BasicBlock::iterator I = OuterResumeDest->begin();
185  for (unsigned i = 0, e = UnwindDestPHIValues.size(); i != e; ++i, ++I) {
186  PHINode *OuterPHI = cast<PHINode>(I);
187  PHINode *InnerPHI = PHINode::Create(OuterPHI->getType(), PHICapacity,
188  OuterPHI->getName() + ".lpad-body",
189  InsertPoint);
190  OuterPHI->replaceAllUsesWith(InnerPHI);
191  InnerPHI->addIncoming(OuterPHI, OuterResumeDest);
192  }
193 
194  // Create a PHI for the exception values.
195  InnerEHValuesPHI = PHINode::Create(CallerLPad->getType(), PHICapacity,
196  "eh.lpad-body", InsertPoint);
197  CallerLPad->replaceAllUsesWith(InnerEHValuesPHI);
198  InnerEHValuesPHI->addIncoming(CallerLPad, OuterResumeDest);
199 
200  // All done.
201  return InnerResumeDest;
202 }
203 
204 /// Forward the 'resume' instruction to the caller's landing pad block.
205 /// When the landing pad block has only one predecessor, this is a simple
206 /// branch. When there is more than one predecessor, we need to split the
207 /// landing pad block after the landingpad instruction and jump to there.
208 void LandingPadInliningInfo::forwardResume(
209  ResumeInst *RI, SmallPtrSetImpl<LandingPadInst *> &InlinedLPads) {
210  BasicBlock *Dest = getInnerResumeDest();
211  BasicBlock *Src = RI->getParent();
212 
213  BranchInst::Create(Dest, Src);
214 
215  // Update the PHIs in the destination. They were inserted in an order which
216  // makes this work.
217  addIncomingPHIValuesForInto(Src, Dest);
218 
219  InnerEHValuesPHI->addIncoming(RI->getOperand(0), Src);
220  RI->eraseFromParent();
221 }
222 
223 /// Helper for getUnwindDestToken/getUnwindDestTokenHelper.
224 static Value *getParentPad(Value *EHPad) {
225  if (auto *FPI = dyn_cast<FuncletPadInst>(EHPad))
226  return FPI->getParentPad();
227  return cast<CatchSwitchInst>(EHPad)->getParentPad();
228 }
229 
231 
232 /// Helper for getUnwindDestToken that does the descendant-ward part of
233 /// the search.
235  UnwindDestMemoTy &MemoMap) {
236  SmallVector<Instruction *, 8> Worklist(1, EHPad);
237 
238  while (!Worklist.empty()) {
239  Instruction *CurrentPad = Worklist.pop_back_val();
240  // We only put pads on the worklist that aren't in the MemoMap. When
241  // we find an unwind dest for a pad we may update its ancestors, but
242  // the queue only ever contains uncles/great-uncles/etc. of CurrentPad,
243  // so they should never get updated while queued on the worklist.
244  assert(!MemoMap.count(CurrentPad));
245  Value *UnwindDestToken = nullptr;
246  if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(CurrentPad)) {
247  if (CatchSwitch->hasUnwindDest()) {
248  UnwindDestToken = CatchSwitch->getUnwindDest()->getFirstNonPHI();
249  } else {
250  // Catchswitch doesn't have a 'nounwind' variant, and one might be
251  // annotated as "unwinds to caller" when really it's nounwind (see
252  // e.g. SimplifyCFGOpt::SimplifyUnreachable), so we can't infer the
253  // parent's unwind dest from this. We can check its catchpads'
254  // descendants, since they might include a cleanuppad with an
255  // "unwinds to caller" cleanupret, which can be trusted.
256  for (auto HI = CatchSwitch->handler_begin(),
257  HE = CatchSwitch->handler_end();
258  HI != HE && !UnwindDestToken; ++HI) {
259  BasicBlock *HandlerBlock = *HI;
260  auto *CatchPad = cast<CatchPadInst>(HandlerBlock->getFirstNonPHI());
261  for (User *Child : CatchPad->users()) {
262  // Intentionally ignore invokes here -- since the catchswitch is
263  // marked "unwind to caller", it would be a verifier error if it
264  // contained an invoke which unwinds out of it, so any invoke we'd
265  // encounter must unwind to some child of the catch.
266  if (!isa<CleanupPadInst>(Child) && !isa<CatchSwitchInst>(Child))
267  continue;
268 
269  Instruction *ChildPad = cast<Instruction>(Child);
270  auto Memo = MemoMap.find(ChildPad);
271  if (Memo == MemoMap.end()) {
272  // Haven't figured out this child pad yet; queue it.
273  Worklist.push_back(ChildPad);
274  continue;
275  }
276  // We've already checked this child, but might have found that
277  // it offers no proof either way.
278  Value *ChildUnwindDestToken = Memo->second;
279  if (!ChildUnwindDestToken)
280  continue;
281  // We already know the child's unwind dest, which can either
282  // be ConstantTokenNone to indicate unwind to caller, or can
283  // be another child of the catchpad. Only the former indicates
284  // the unwind dest of the catchswitch.
285  if (isa<ConstantTokenNone>(ChildUnwindDestToken)) {
286  UnwindDestToken = ChildUnwindDestToken;
287  break;
288  }
289  assert(getParentPad(ChildUnwindDestToken) == CatchPad);
290  }
291  }
292  }
293  } else {
294  auto *CleanupPad = cast<CleanupPadInst>(CurrentPad);
295  for (User *U : CleanupPad->users()) {
296  if (auto *CleanupRet = dyn_cast<CleanupReturnInst>(U)) {
297  if (BasicBlock *RetUnwindDest = CleanupRet->getUnwindDest())
298  UnwindDestToken = RetUnwindDest->getFirstNonPHI();
299  else
300  UnwindDestToken = ConstantTokenNone::get(CleanupPad->getContext());
301  break;
302  }
303  Value *ChildUnwindDestToken;
304  if (auto *Invoke = dyn_cast<InvokeInst>(U)) {
305  ChildUnwindDestToken = Invoke->getUnwindDest()->getFirstNonPHI();
306  } else if (isa<CleanupPadInst>(U) || isa<CatchSwitchInst>(U)) {
307  Instruction *ChildPad = cast<Instruction>(U);
308  auto Memo = MemoMap.find(ChildPad);
309  if (Memo == MemoMap.end()) {
310  // Haven't resolved this child yet; queue it and keep searching.
311  Worklist.push_back(ChildPad);
312  continue;
313  }
314  // We've checked this child, but still need to ignore it if it
315  // had no proof either way.
316  ChildUnwindDestToken = Memo->second;
317  if (!ChildUnwindDestToken)
318  continue;
319  } else {
320  // Not a relevant user of the cleanuppad
321  continue;
322  }
323  // In a well-formed program, the child/invoke must either unwind to
324  // an(other) child of the cleanup, or exit the cleanup. In the
325  // first case, continue searching.
326  if (isa<Instruction>(ChildUnwindDestToken) &&
327  getParentPad(ChildUnwindDestToken) == CleanupPad)
328  continue;
329  UnwindDestToken = ChildUnwindDestToken;
330  break;
331  }
332  }
333  // If we haven't found an unwind dest for CurrentPad, we may have queued its
334  // children, so move on to the next in the worklist.
335  if (!UnwindDestToken)
336  continue;
337 
338  // Now we know that CurrentPad unwinds to UnwindDestToken. It also exits
339  // any ancestors of CurrentPad up to but not including UnwindDestToken's
340  // parent pad. Record this in the memo map, and check to see if the
341  // original EHPad being queried is one of the ones exited.
342  Value *UnwindParent;
343  if (auto *UnwindPad = dyn_cast<Instruction>(UnwindDestToken))
344  UnwindParent = getParentPad(UnwindPad);
345  else
346  UnwindParent = nullptr;
347  bool ExitedOriginalPad = false;
348  for (Instruction *ExitedPad = CurrentPad;
349  ExitedPad && ExitedPad != UnwindParent;
350  ExitedPad = dyn_cast<Instruction>(getParentPad(ExitedPad))) {
351  // Skip over catchpads since they just follow their catchswitches.
352  if (isa<CatchPadInst>(ExitedPad))
353  continue;
354  MemoMap[ExitedPad] = UnwindDestToken;
355  ExitedOriginalPad |= (ExitedPad == EHPad);
356  }
357 
358  if (ExitedOriginalPad)
359  return UnwindDestToken;
360 
361  // Continue the search.
362  }
363 
364  // No definitive information is contained within this funclet.
365  return nullptr;
366 }
367 
368 /// Given an EH pad, find where it unwinds. If it unwinds to an EH pad,
369 /// return that pad instruction. If it unwinds to caller, return
370 /// ConstantTokenNone. If it does not have a definitive unwind destination,
371 /// return nullptr.
372 ///
373 /// This routine gets invoked for calls in funclets in inlinees when inlining
374 /// an invoke. Since many funclets don't have calls inside them, it's queried
375 /// on-demand rather than building a map of pads to unwind dests up front.
376 /// Determining a funclet's unwind dest may require recursively searching its
377 /// descendants, and also ancestors and cousins if the descendants don't provide
378 /// an answer. Since most funclets will have their unwind dest immediately
379 /// available as the unwind dest of a catchswitch or cleanupret, this routine
380 /// searches top-down from the given pad and then up. To avoid worst-case
381 /// quadratic run-time given that approach, it uses a memo map to avoid
382 /// re-processing funclet trees. The callers that rewrite the IR as they go
383 /// take advantage of this, for correctness, by checking/forcing rewritten
384 /// pads' entries to match the original callee view.
386  UnwindDestMemoTy &MemoMap) {
387  // Catchpads unwind to the same place as their catchswitch;
388  // redirct any queries on catchpads so the code below can
389  // deal with just catchswitches and cleanuppads.
390  if (auto *CPI = dyn_cast<CatchPadInst>(EHPad))
391  EHPad = CPI->getCatchSwitch();
392 
393  // Check if we've already determined the unwind dest for this pad.
394  auto Memo = MemoMap.find(EHPad);
395  if (Memo != MemoMap.end())
396  return Memo->second;
397 
398  // Search EHPad and, if necessary, its descendants.
399  Value *UnwindDestToken = getUnwindDestTokenHelper(EHPad, MemoMap);
400  assert((UnwindDestToken == nullptr) != (MemoMap.count(EHPad) != 0));
401  if (UnwindDestToken)
402  return UnwindDestToken;
403 
404  // No information is available for this EHPad from itself or any of its
405  // descendants. An unwind all the way out to a pad in the caller would
406  // need also to agree with the unwind dest of the parent funclet, so
407  // search up the chain to try to find a funclet with information. Put
408  // null entries in the memo map to avoid re-processing as we go up.
409  MemoMap[EHPad] = nullptr;
410 #ifndef NDEBUG
412  TempMemos.insert(EHPad);
413 #endif
414  Instruction *LastUselessPad = EHPad;
415  Value *AncestorToken;
416  for (AncestorToken = getParentPad(EHPad);
417  auto *AncestorPad = dyn_cast<Instruction>(AncestorToken);
418  AncestorToken = getParentPad(AncestorToken)) {
419  // Skip over catchpads since they just follow their catchswitches.
420  if (isa<CatchPadInst>(AncestorPad))
421  continue;
422  // If the MemoMap had an entry mapping AncestorPad to nullptr, since we
423  // haven't yet called getUnwindDestTokenHelper for AncestorPad in this
424  // call to getUnwindDestToken, that would mean that AncestorPad had no
425  // information in itself, its descendants, or its ancestors. If that
426  // were the case, then we should also have recorded the lack of information
427  // for the descendant that we're coming from. So assert that we don't
428  // find a null entry in the MemoMap for AncestorPad.
429  assert(!MemoMap.count(AncestorPad) || MemoMap[AncestorPad]);
430  auto AncestorMemo = MemoMap.find(AncestorPad);
431  if (AncestorMemo == MemoMap.end()) {
432  UnwindDestToken = getUnwindDestTokenHelper(AncestorPad, MemoMap);
433  } else {
434  UnwindDestToken = AncestorMemo->second;
435  }
436  if (UnwindDestToken)
437  break;
438  LastUselessPad = AncestorPad;
439  MemoMap[LastUselessPad] = nullptr;
440 #ifndef NDEBUG
441  TempMemos.insert(LastUselessPad);
442 #endif
443  }
444 
445  // We know that getUnwindDestTokenHelper was called on LastUselessPad and
446  // returned nullptr (and likewise for EHPad and any of its ancestors up to
447  // LastUselessPad), so LastUselessPad has no information from below. Since
448  // getUnwindDestTokenHelper must investigate all downward paths through
449  // no-information nodes to prove that a node has no information like this,
450  // and since any time it finds information it records it in the MemoMap for
451  // not just the immediately-containing funclet but also any ancestors also
452  // exited, it must be the case that, walking downward from LastUselessPad,
453  // visiting just those nodes which have not been mapped to an unwind dest
454  // by getUnwindDestTokenHelper (the nullptr TempMemos notwithstanding, since
455  // they are just used to keep getUnwindDestTokenHelper from repeating work),
456  // any node visited must have been exhaustively searched with no information
457  // for it found.
458  SmallVector<Instruction *, 8> Worklist(1, LastUselessPad);
459  while (!Worklist.empty()) {
460  Instruction *UselessPad = Worklist.pop_back_val();
461  auto Memo = MemoMap.find(UselessPad);
462  if (Memo != MemoMap.end() && Memo->second) {
463  // Here the name 'UselessPad' is a bit of a misnomer, because we've found
464  // that it is a funclet that does have information about unwinding to
465  // a particular destination; its parent was a useless pad.
466  // Since its parent has no information, the unwind edge must not escape
467  // the parent, and must target a sibling of this pad. This local unwind
468  // gives us no information about EHPad. Leave it and the subtree rooted
469  // at it alone.
470  assert(getParentPad(Memo->second) == getParentPad(UselessPad));
471  continue;
472  }
473  // We know we don't have information for UselesPad. If it has an entry in
474  // the MemoMap (mapping it to nullptr), it must be one of the TempMemos
475  // added on this invocation of getUnwindDestToken; if a previous invocation
476  // recorded nullptr, it would have had to prove that the ancestors of
477  // UselessPad, which include LastUselessPad, had no information, and that
478  // in turn would have required proving that the descendants of
479  // LastUselesPad, which include EHPad, have no information about
480  // LastUselessPad, which would imply that EHPad was mapped to nullptr in
481  // the MemoMap on that invocation, which isn't the case if we got here.
482  assert(!MemoMap.count(UselessPad) || TempMemos.count(UselessPad));
483  // Assert as we enumerate users that 'UselessPad' doesn't have any unwind
484  // information that we'd be contradicting by making a map entry for it
485  // (which is something that getUnwindDestTokenHelper must have proved for
486  // us to get here). Just assert on is direct users here; the checks in
487  // this downward walk at its descendants will verify that they don't have
488  // any unwind edges that exit 'UselessPad' either (i.e. they either have no
489  // unwind edges or unwind to a sibling).
490  MemoMap[UselessPad] = UnwindDestToken;
491  if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(UselessPad)) {
492  assert(CatchSwitch->getUnwindDest() == nullptr && "Expected useless pad");
493  for (BasicBlock *HandlerBlock : CatchSwitch->handlers()) {
494  auto *CatchPad = HandlerBlock->getFirstNonPHI();
495  for (User *U : CatchPad->users()) {
496  assert(
497  (!isa<InvokeInst>(U) ||
498  (getParentPad(
499  cast<InvokeInst>(U)->getUnwindDest()->getFirstNonPHI()) ==
500  CatchPad)) &&
501  "Expected useless pad");
502  if (isa<CatchSwitchInst>(U) || isa<CleanupPadInst>(U))
503  Worklist.push_back(cast<Instruction>(U));
504  }
505  }
506  } else {
507  assert(isa<CleanupPadInst>(UselessPad));
508  for (User *U : UselessPad->users()) {
509  assert(!isa<CleanupReturnInst>(U) && "Expected useless pad");
510  assert((!isa<InvokeInst>(U) ||
511  (getParentPad(
512  cast<InvokeInst>(U)->getUnwindDest()->getFirstNonPHI()) ==
513  UselessPad)) &&
514  "Expected useless pad");
515  if (isa<CatchSwitchInst>(U) || isa<CleanupPadInst>(U))
516  Worklist.push_back(cast<Instruction>(U));
517  }
518  }
519  }
520 
521  return UnwindDestToken;
522 }
523 
524 /// When we inline a basic block into an invoke,
525 /// we have to turn all of the calls that can throw into invokes.
526 /// This function analyze BB to see if there are any calls, and if so,
527 /// it rewrites them to be invokes that jump to InvokeDest and fills in the PHI
528 /// nodes in that block with the values specified in InvokeDestPHIValues.
530  BasicBlock *BB, BasicBlock *UnwindEdge,
531  UnwindDestMemoTy *FuncletUnwindMap = nullptr) {
532  for (BasicBlock::iterator BBI = BB->begin(), E = BB->end(); BBI != E; ) {
533  Instruction *I = &*BBI++;
534 
535  // We only need to check for function calls: inlined invoke
536  // instructions require no special handling.
537  CallInst *CI = dyn_cast<CallInst>(I);
538 
539  if (!CI || CI->doesNotThrow() || isa<InlineAsm>(CI->getCalledValue()))
540  continue;
541 
542  // We do not need to (and in fact, cannot) convert possibly throwing calls
543  // to @llvm.experimental_deoptimize (resp. @llvm.experimental.guard) into
544  // invokes. The caller's "segment" of the deoptimization continuation
545  // attached to the newly inlined @llvm.experimental_deoptimize
546  // (resp. @llvm.experimental.guard) call should contain the exception
547  // handling logic, if any.
548  if (auto *F = CI->getCalledFunction())
549  if (F->getIntrinsicID() == Intrinsic::experimental_deoptimize ||
550  F->getIntrinsicID() == Intrinsic::experimental_guard)
551  continue;
552 
553  if (auto FuncletBundle = CI->getOperandBundle(LLVMContext::OB_funclet)) {
554  // This call is nested inside a funclet. If that funclet has an unwind
555  // destination within the inlinee, then unwinding out of this call would
556  // be UB. Rewriting this call to an invoke which targets the inlined
557  // invoke's unwind dest would give the call's parent funclet multiple
558  // unwind destinations, which is something that subsequent EH table
559  // generation can't handle and that the veirifer rejects. So when we
560  // see such a call, leave it as a call.
561  auto *FuncletPad = cast<Instruction>(FuncletBundle->Inputs[0]);
562  Value *UnwindDestToken =
563  getUnwindDestToken(FuncletPad, *FuncletUnwindMap);
564  if (UnwindDestToken && !isa<ConstantTokenNone>(UnwindDestToken))
565  continue;
566 #ifndef NDEBUG
567  Instruction *MemoKey;
568  if (auto *CatchPad = dyn_cast<CatchPadInst>(FuncletPad))
569  MemoKey = CatchPad->getCatchSwitch();
570  else
571  MemoKey = FuncletPad;
572  assert(FuncletUnwindMap->count(MemoKey) &&
573  (*FuncletUnwindMap)[MemoKey] == UnwindDestToken &&
574  "must get memoized to avoid confusing later searches");
575 #endif // NDEBUG
576  }
577 
578  changeToInvokeAndSplitBasicBlock(CI, UnwindEdge);
579  return BB;
580  }
581  return nullptr;
582 }
583 
584 /// If we inlined an invoke site, we need to convert calls
585 /// in the body of the inlined function into invokes.
586 ///
587 /// II is the invoke instruction being inlined. FirstNewBlock is the first
588 /// block of the inlined code (the last block is the end of the function),
589 /// and InlineCodeInfo is information about the code that got inlined.
590 static void HandleInlinedLandingPad(InvokeInst *II, BasicBlock *FirstNewBlock,
591  ClonedCodeInfo &InlinedCodeInfo) {
592  BasicBlock *InvokeDest = II->getUnwindDest();
593 
594  Function *Caller = FirstNewBlock->getParent();
595 
596  // The inlined code is currently at the end of the function, scan from the
597  // start of the inlined code to its end, checking for stuff we need to
598  // rewrite.
599  LandingPadInliningInfo Invoke(II);
600 
601  // Get all of the inlined landing pad instructions.
603  for (Function::iterator I = FirstNewBlock->getIterator(), E = Caller->end();
604  I != E; ++I)
605  if (InvokeInst *II = dyn_cast<InvokeInst>(I->getTerminator()))
606  InlinedLPads.insert(II->getLandingPadInst());
607 
608  // Append the clauses from the outer landing pad instruction into the inlined
609  // landing pad instructions.
610  LandingPadInst *OuterLPad = Invoke.getLandingPadInst();
611  for (LandingPadInst *InlinedLPad : InlinedLPads) {
612  unsigned OuterNum = OuterLPad->getNumClauses();
613  InlinedLPad->reserveClauses(OuterNum);
614  for (unsigned OuterIdx = 0; OuterIdx != OuterNum; ++OuterIdx)
615  InlinedLPad->addClause(OuterLPad->getClause(OuterIdx));
616  if (OuterLPad->isCleanup())
617  InlinedLPad->setCleanup(true);
618  }
619 
620  for (Function::iterator BB = FirstNewBlock->getIterator(), E = Caller->end();
621  BB != E; ++BB) {
622  if (InlinedCodeInfo.ContainsCalls)
624  &*BB, Invoke.getOuterResumeDest()))
625  // Update any PHI nodes in the exceptional block to indicate that there
626  // is now a new entry in them.
627  Invoke.addIncomingPHIValuesFor(NewBB);
628 
629  // Forward any resumes that are remaining here.
630  if (ResumeInst *RI = dyn_cast<ResumeInst>(BB->getTerminator()))
631  Invoke.forwardResume(RI, InlinedLPads);
632  }
633 
634  // Now that everything is happy, we have one final detail. The PHI nodes in
635  // the exception destination block still have entries due to the original
636  // invoke instruction. Eliminate these entries (which might even delete the
637  // PHI node) now.
638  InvokeDest->removePredecessor(II->getParent());
639 }
640 
641 /// If we inlined an invoke site, we need to convert calls
642 /// in the body of the inlined function into invokes.
643 ///
644 /// II is the invoke instruction being inlined. FirstNewBlock is the first
645 /// block of the inlined code (the last block is the end of the function),
646 /// and InlineCodeInfo is information about the code that got inlined.
647 static void HandleInlinedEHPad(InvokeInst *II, BasicBlock *FirstNewBlock,
648  ClonedCodeInfo &InlinedCodeInfo) {
649  BasicBlock *UnwindDest = II->getUnwindDest();
650  Function *Caller = FirstNewBlock->getParent();
651 
652  assert(UnwindDest->getFirstNonPHI()->isEHPad() && "unexpected BasicBlock!");
653 
654  // If there are PHI nodes in the unwind destination block, we need to keep
655  // track of which values came into them from the invoke before removing the
656  // edge from this block.
657  SmallVector<Value *, 8> UnwindDestPHIValues;
658  BasicBlock *InvokeBB = II->getParent();
659  for (Instruction &I : *UnwindDest) {
660  // Save the value to use for this edge.
661  PHINode *PHI = dyn_cast<PHINode>(&I);
662  if (!PHI)
663  break;
664  UnwindDestPHIValues.push_back(PHI->getIncomingValueForBlock(InvokeBB));
665  }
666 
667  // Add incoming-PHI values to the unwind destination block for the given basic
668  // block, using the values for the original invoke's source block.
669  auto UpdatePHINodes = [&](BasicBlock *Src) {
670  BasicBlock::iterator I = UnwindDest->begin();
671  for (Value *V : UnwindDestPHIValues) {
672  PHINode *PHI = cast<PHINode>(I);
673  PHI->addIncoming(V, Src);
674  ++I;
675  }
676  };
677 
678  // This connects all the instructions which 'unwind to caller' to the invoke
679  // destination.
680  UnwindDestMemoTy FuncletUnwindMap;
681  for (Function::iterator BB = FirstNewBlock->getIterator(), E = Caller->end();
682  BB != E; ++BB) {
683  if (auto *CRI = dyn_cast<CleanupReturnInst>(BB->getTerminator())) {
684  if (CRI->unwindsToCaller()) {
685  auto *CleanupPad = CRI->getCleanupPad();
686  CleanupReturnInst::Create(CleanupPad, UnwindDest, CRI);
687  CRI->eraseFromParent();
688  UpdatePHINodes(&*BB);
689  // Finding a cleanupret with an unwind destination would confuse
690  // subsequent calls to getUnwindDestToken, so map the cleanuppad
691  // to short-circuit any such calls and recognize this as an "unwind
692  // to caller" cleanup.
693  assert(!FuncletUnwindMap.count(CleanupPad) ||
694  isa<ConstantTokenNone>(FuncletUnwindMap[CleanupPad]));
695  FuncletUnwindMap[CleanupPad] =
697  }
698  }
699 
700  Instruction *I = BB->getFirstNonPHI();
701  if (!I->isEHPad())
702  continue;
703 
704  Instruction *Replacement = nullptr;
705  if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(I)) {
706  if (CatchSwitch->unwindsToCaller()) {
707  Value *UnwindDestToken;
708  if (auto *ParentPad =
709  dyn_cast<Instruction>(CatchSwitch->getParentPad())) {
710  // This catchswitch is nested inside another funclet. If that
711  // funclet has an unwind destination within the inlinee, then
712  // unwinding out of this catchswitch would be UB. Rewriting this
713  // catchswitch to unwind to the inlined invoke's unwind dest would
714  // give the parent funclet multiple unwind destinations, which is
715  // something that subsequent EH table generation can't handle and
716  // that the veirifer rejects. So when we see such a call, leave it
717  // as "unwind to caller".
718  UnwindDestToken = getUnwindDestToken(ParentPad, FuncletUnwindMap);
719  if (UnwindDestToken && !isa<ConstantTokenNone>(UnwindDestToken))
720  continue;
721  } else {
722  // This catchswitch has no parent to inherit constraints from, and
723  // none of its descendants can have an unwind edge that exits it and
724  // targets another funclet in the inlinee. It may or may not have a
725  // descendant that definitively has an unwind to caller. In either
726  // case, we'll have to assume that any unwinds out of it may need to
727  // be routed to the caller, so treat it as though it has a definitive
728  // unwind to caller.
729  UnwindDestToken = ConstantTokenNone::get(Caller->getContext());
730  }
731  auto *NewCatchSwitch = CatchSwitchInst::Create(
732  CatchSwitch->getParentPad(), UnwindDest,
733  CatchSwitch->getNumHandlers(), CatchSwitch->getName(),
734  CatchSwitch);
735  for (BasicBlock *PadBB : CatchSwitch->handlers())
736  NewCatchSwitch->addHandler(PadBB);
737  // Propagate info for the old catchswitch over to the new one in
738  // the unwind map. This also serves to short-circuit any subsequent
739  // checks for the unwind dest of this catchswitch, which would get
740  // confused if they found the outer handler in the callee.
741  FuncletUnwindMap[NewCatchSwitch] = UnwindDestToken;
742  Replacement = NewCatchSwitch;
743  }
744  } else if (!isa<FuncletPadInst>(I)) {
745  llvm_unreachable("unexpected EHPad!");
746  }
747 
748  if (Replacement) {
749  Replacement->takeName(I);
750  I->replaceAllUsesWith(Replacement);
751  I->eraseFromParent();
752  UpdatePHINodes(&*BB);
753  }
754  }
755 
756  if (InlinedCodeInfo.ContainsCalls)
757  for (Function::iterator BB = FirstNewBlock->getIterator(),
758  E = Caller->end();
759  BB != E; ++BB)
761  &*BB, UnwindDest, &FuncletUnwindMap))
762  // Update any PHI nodes in the exceptional block to indicate that there
763  // is now a new entry in them.
764  UpdatePHINodes(NewBB);
765 
766  // Now that everything is happy, we have one final detail. The PHI nodes in
767  // the exception destination block still have entries due to the original
768  // invoke instruction. Eliminate these entries (which might even delete the
769  // PHI node) now.
770  UnwindDest->removePredecessor(InvokeBB);
771 }
772 
773 /// When inlining a call site that has !llvm.mem.parallel_loop_access or
774 /// llvm.access.group metadata, that metadata should be propagated to all
775 /// memory-accessing cloned instructions.
777  ValueToValueMapTy &VMap) {
778  MDNode *M =
780  MDNode *CallAccessGroup =
782  if (!M && !CallAccessGroup)
783  return;
784 
785  for (ValueToValueMapTy::iterator VMI = VMap.begin(), VMIE = VMap.end();
786  VMI != VMIE; ++VMI) {
787  if (!VMI->second)
788  continue;
789 
790  Instruction *NI = dyn_cast<Instruction>(VMI->second);
791  if (!NI)
792  continue;
793 
794  if (M) {
795  if (MDNode *PM =
797  M = MDNode::concatenate(PM, M);
799  } else if (NI->mayReadOrWriteMemory()) {
801  }
802  }
803 
804  if (NI->mayReadOrWriteMemory()) {
805  MDNode *UnitedAccGroups = uniteAccessGroups(
806  NI->getMetadata(LLVMContext::MD_access_group), CallAccessGroup);
807  NI->setMetadata(LLVMContext::MD_access_group, UnitedAccGroups);
808  }
809  }
810 }
811 
812 /// When inlining a function that contains noalias scope metadata,
813 /// this metadata needs to be cloned so that the inlined blocks
814 /// have different "unique scopes" at every call site. Were this not done, then
815 /// aliasing scopes from a function inlined into a caller multiple times could
816 /// not be differentiated (and this would lead to miscompiles because the
817 /// non-aliasing property communicated by the metadata could have
818 /// call-site-specific control dependencies).
820  const Function *CalledFunc = CS.getCalledFunction();
822 
823  // Note: We could only clone the metadata if it is already used in the
824  // caller. I'm omitting that check here because it might confuse
825  // inter-procedural alias analysis passes. We can revisit this if it becomes
826  // an efficiency or overhead problem.
827 
828  for (const BasicBlock &I : *CalledFunc)
829  for (const Instruction &J : I) {
830  if (const MDNode *M = J.getMetadata(LLVMContext::MD_alias_scope))
831  MD.insert(M);
832  if (const MDNode *M = J.getMetadata(LLVMContext::MD_noalias))
833  MD.insert(M);
834  }
835 
836  if (MD.empty())
837  return;
838 
839  // Walk the existing metadata, adding the complete (perhaps cyclic) chain to
840  // the set.
841  SmallVector<const Metadata *, 16> Queue(MD.begin(), MD.end());
842  while (!Queue.empty()) {
843  const MDNode *M = cast<MDNode>(Queue.pop_back_val());
844  for (unsigned i = 0, ie = M->getNumOperands(); i != ie; ++i)
845  if (const MDNode *M1 = dyn_cast<MDNode>(M->getOperand(i)))
846  if (MD.insert(M1))
847  Queue.push_back(M1);
848  }
849 
850  // Now we have a complete set of all metadata in the chains used to specify
851  // the noalias scopes and the lists of those scopes.
852  SmallVector<TempMDTuple, 16> DummyNodes;
854  for (const MDNode *I : MD) {
855  DummyNodes.push_back(MDTuple::getTemporary(CalledFunc->getContext(), None));
856  MDMap[I].reset(DummyNodes.back().get());
857  }
858 
859  // Create new metadata nodes to replace the dummy nodes, replacing old
860  // metadata references with either a dummy node or an already-created new
861  // node.
862  for (const MDNode *I : MD) {
864  for (unsigned i = 0, ie = I->getNumOperands(); i != ie; ++i) {
865  const Metadata *V = I->getOperand(i);
866  if (const MDNode *M = dyn_cast<MDNode>(V))
867  NewOps.push_back(MDMap[M]);
868  else
869  NewOps.push_back(const_cast<Metadata *>(V));
870  }
871 
872  MDNode *NewM = MDNode::get(CalledFunc->getContext(), NewOps);
873  MDTuple *TempM = cast<MDTuple>(MDMap[I]);
874  assert(TempM->isTemporary() && "Expected temporary node");
875 
876  TempM->replaceAllUsesWith(NewM);
877  }
878 
879  // Now replace the metadata in the new inlined instructions with the
880  // repacements from the map.
881  for (ValueToValueMapTy::iterator VMI = VMap.begin(), VMIE = VMap.end();
882  VMI != VMIE; ++VMI) {
883  if (!VMI->second)
884  continue;
885 
886  Instruction *NI = dyn_cast<Instruction>(VMI->second);
887  if (!NI)
888  continue;
889 
891  MDNode *NewMD = MDMap[M];
892  // If the call site also had alias scope metadata (a list of scopes to
893  // which instructions inside it might belong), propagate those scopes to
894  // the inlined instructions.
895  if (MDNode *CSM =
897  NewMD = MDNode::concatenate(NewMD, CSM);
899  } else if (NI->mayReadOrWriteMemory()) {
900  if (MDNode *M =
903  }
904 
906  MDNode *NewMD = MDMap[M];
907  // If the call site also had noalias metadata (a list of scopes with
908  // which instructions inside it don't alias), propagate those scopes to
909  // the inlined instructions.
910  if (MDNode *CSM =
912  NewMD = MDNode::concatenate(NewMD, CSM);
914  } else if (NI->mayReadOrWriteMemory()) {
917  }
918  }
919 }
920 
921 /// If the inlined function has noalias arguments,
922 /// then add new alias scopes for each noalias argument, tag the mapped noalias
923 /// parameters with noalias metadata specifying the new scope, and tag all
924 /// non-derived loads, stores and memory intrinsics with the new alias scopes.
926  const DataLayout &DL, AAResults *CalleeAAR) {
928  return;
929 
930  const Function *CalledFunc = CS.getCalledFunction();
932 
933  for (const Argument &Arg : CalledFunc->args())
934  if (Arg.hasNoAliasAttr() && !Arg.use_empty())
935  NoAliasArgs.push_back(&Arg);
936 
937  if (NoAliasArgs.empty())
938  return;
939 
940  // To do a good job, if a noalias variable is captured, we need to know if
941  // the capture point dominates the particular use we're considering.
942  DominatorTree DT;
943  DT.recalculate(const_cast<Function&>(*CalledFunc));
944 
945  // noalias indicates that pointer values based on the argument do not alias
946  // pointer values which are not based on it. So we add a new "scope" for each
947  // noalias function argument. Accesses using pointers based on that argument
948  // become part of that alias scope, accesses using pointers not based on that
949  // argument are tagged as noalias with that scope.
950 
952  MDBuilder MDB(CalledFunc->getContext());
953 
954  // Create a new scope domain for this function.
955  MDNode *NewDomain =
956  MDB.createAnonymousAliasScopeDomain(CalledFunc->getName());
957  for (unsigned i = 0, e = NoAliasArgs.size(); i != e; ++i) {
958  const Argument *A = NoAliasArgs[i];
959 
960  std::string Name = CalledFunc->getName();
961  if (A->hasName()) {
962  Name += ": %";
963  Name += A->getName();
964  } else {
965  Name += ": argument ";
966  Name += utostr(i);
967  }
968 
969  // Note: We always create a new anonymous root here. This is true regardless
970  // of the linkage of the callee because the aliasing "scope" is not just a
971  // property of the callee, but also all control dependencies in the caller.
972  MDNode *NewScope = MDB.createAnonymousAliasScope(NewDomain, Name);
973  NewScopes.insert(std::make_pair(A, NewScope));
974  }
975 
976  // Iterate over all new instructions in the map; for all memory-access
977  // instructions, add the alias scope metadata.
978  for (ValueToValueMapTy::iterator VMI = VMap.begin(), VMIE = VMap.end();
979  VMI != VMIE; ++VMI) {
980  if (const Instruction *I = dyn_cast<Instruction>(VMI->first)) {
981  if (!VMI->second)
982  continue;
983 
984  Instruction *NI = dyn_cast<Instruction>(VMI->second);
985  if (!NI)
986  continue;
987 
988  bool IsArgMemOnlyCall = false, IsFuncCall = false;
990 
991  if (const LoadInst *LI = dyn_cast<LoadInst>(I))
992  PtrArgs.push_back(LI->getPointerOperand());
993  else if (const StoreInst *SI = dyn_cast<StoreInst>(I))
994  PtrArgs.push_back(SI->getPointerOperand());
995  else if (const VAArgInst *VAAI = dyn_cast<VAArgInst>(I))
996  PtrArgs.push_back(VAAI->getPointerOperand());
997  else if (const AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(I))
998  PtrArgs.push_back(CXI->getPointerOperand());
999  else if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I))
1000  PtrArgs.push_back(RMWI->getPointerOperand());
1001  else if (const auto *Call = dyn_cast<CallBase>(I)) {
1002  // If we know that the call does not access memory, then we'll still
1003  // know that about the inlined clone of this call site, and we don't
1004  // need to add metadata.
1005  if (Call->doesNotAccessMemory())
1006  continue;
1007 
1008  IsFuncCall = true;
1009  if (CalleeAAR) {
1010  FunctionModRefBehavior MRB = CalleeAAR->getModRefBehavior(Call);
1011  if (MRB == FMRB_OnlyAccessesArgumentPointees ||
1013  IsArgMemOnlyCall = true;
1014  }
1015 
1016  for (Value *Arg : Call->args()) {
1017  // We need to check the underlying objects of all arguments, not just
1018  // the pointer arguments, because we might be passing pointers as
1019  // integers, etc.
1020  // However, if we know that the call only accesses pointer arguments,
1021  // then we only need to check the pointer arguments.
1022  if (IsArgMemOnlyCall && !Arg->getType()->isPointerTy())
1023  continue;
1024 
1025  PtrArgs.push_back(Arg);
1026  }
1027  }
1028 
1029  // If we found no pointers, then this instruction is not suitable for
1030  // pairing with an instruction to receive aliasing metadata.
1031  // However, if this is a call, this we might just alias with none of the
1032  // noalias arguments.
1033  if (PtrArgs.empty() && !IsFuncCall)
1034  continue;
1035 
1036  // It is possible that there is only one underlying object, but you
1037  // need to go through several PHIs to see it, and thus could be
1038  // repeated in the Objects list.
1041 
1043  for (const Value *V : PtrArgs) {
1045  GetUnderlyingObjects(V, Objects, DL, /* LI = */ nullptr);
1046 
1047  for (const Value *O : Objects)
1048  ObjSet.insert(O);
1049  }
1050 
1051  // Figure out if we're derived from anything that is not a noalias
1052  // argument.
1053  bool CanDeriveViaCapture = false, UsesAliasingPtr = false;
1054  for (const Value *V : ObjSet) {
1055  // Is this value a constant that cannot be derived from any pointer
1056  // value (we need to exclude constant expressions, for example, that
1057  // are formed from arithmetic on global symbols).
1058  bool IsNonPtrConst = isa<ConstantInt>(V) || isa<ConstantFP>(V) ||
1059  isa<ConstantPointerNull>(V) ||
1060  isa<ConstantDataVector>(V) || isa<UndefValue>(V);
1061  if (IsNonPtrConst)
1062  continue;
1063 
1064  // If this is anything other than a noalias argument, then we cannot
1065  // completely describe the aliasing properties using alias.scope
1066  // metadata (and, thus, won't add any).
1067  if (const Argument *A = dyn_cast<Argument>(V)) {
1068  if (!A->hasNoAliasAttr())
1069  UsesAliasingPtr = true;
1070  } else {
1071  UsesAliasingPtr = true;
1072  }
1073 
1074  // If this is not some identified function-local object (which cannot
1075  // directly alias a noalias argument), or some other argument (which,
1076  // by definition, also cannot alias a noalias argument), then we could
1077  // alias a noalias argument that has been captured).
1078  if (!isa<Argument>(V) &&
1079  !isIdentifiedFunctionLocal(const_cast<Value*>(V)))
1080  CanDeriveViaCapture = true;
1081  }
1082 
1083  // A function call can always get captured noalias pointers (via other
1084  // parameters, globals, etc.).
1085  if (IsFuncCall && !IsArgMemOnlyCall)
1086  CanDeriveViaCapture = true;
1087 
1088  // First, we want to figure out all of the sets with which we definitely
1089  // don't alias. Iterate over all noalias set, and add those for which:
1090  // 1. The noalias argument is not in the set of objects from which we
1091  // definitely derive.
1092  // 2. The noalias argument has not yet been captured.
1093  // An arbitrary function that might load pointers could see captured
1094  // noalias arguments via other noalias arguments or globals, and so we
1095  // must always check for prior capture.
1096  for (const Argument *A : NoAliasArgs) {
1097  if (!ObjSet.count(A) && (!CanDeriveViaCapture ||
1098  // It might be tempting to skip the
1099  // PointerMayBeCapturedBefore check if
1100  // A->hasNoCaptureAttr() is true, but this is
1101  // incorrect because nocapture only guarantees
1102  // that no copies outlive the function, not
1103  // that the value cannot be locally captured.
1105  /* ReturnCaptures */ false,
1106  /* StoreCaptures */ false, I, &DT)))
1107  NoAliases.push_back(NewScopes[A]);
1108  }
1109 
1110  if (!NoAliases.empty())
1114  MDNode::get(CalledFunc->getContext(), NoAliases)));
1115 
1116  // Next, we want to figure out all of the sets to which we might belong.
1117  // We might belong to a set if the noalias argument is in the set of
1118  // underlying objects. If there is some non-noalias argument in our list
1119  // of underlying objects, then we cannot add a scope because the fact
1120  // that some access does not alias with any set of our noalias arguments
1121  // cannot itself guarantee that it does not alias with this access
1122  // (because there is some pointer of unknown origin involved and the
1123  // other access might also depend on this pointer). We also cannot add
1124  // scopes to arbitrary functions unless we know they don't access any
1125  // non-parameter pointer-values.
1126  bool CanAddScopes = !UsesAliasingPtr;
1127  if (CanAddScopes && IsFuncCall)
1128  CanAddScopes = IsArgMemOnlyCall;
1129 
1130  if (CanAddScopes)
1131  for (const Argument *A : NoAliasArgs) {
1132  if (ObjSet.count(A))
1133  Scopes.push_back(NewScopes[A]);
1134  }
1135 
1136  if (!Scopes.empty())
1137  NI->setMetadata(
1140  MDNode::get(CalledFunc->getContext(), Scopes)));
1141  }
1142  }
1143 }
1144 
1145 /// If the inlined function has non-byval align arguments, then
1146 /// add @llvm.assume-based alignment assumptions to preserve this information.
1149  return;
1150 
1151  AssumptionCache *AC = &(*IFI.GetAssumptionCache)(*CS.getCaller());
1152  auto &DL = CS.getCaller()->getParent()->getDataLayout();
1153 
1154  // To avoid inserting redundant assumptions, we should check for assumptions
1155  // already in the caller. To do this, we might need a DT of the caller.
1156  DominatorTree DT;
1157  bool DTCalculated = false;
1158 
1159  Function *CalledFunc = CS.getCalledFunction();
1160  for (Argument &Arg : CalledFunc->args()) {
1161  unsigned Align = Arg.getType()->isPointerTy() ? Arg.getParamAlignment() : 0;
1162  if (Align && !Arg.hasByValOrInAllocaAttr() && !Arg.hasNUses(0)) {
1163  if (!DTCalculated) {
1164  DT.recalculate(*CS.getCaller());
1165  DTCalculated = true;
1166  }
1167 
1168  // If we can already prove the asserted alignment in the context of the
1169  // caller, then don't bother inserting the assumption.
1170  Value *ArgVal = CS.getArgument(Arg.getArgNo());
1171  if (getKnownAlignment(ArgVal, DL, CS.getInstruction(), AC, &DT) >= Align)
1172  continue;
1173 
1174  CallInst *NewAsmp = IRBuilder<>(CS.getInstruction())
1175  .CreateAlignmentAssumption(DL, ArgVal, Align);
1176  AC->registerAssumption(NewAsmp);
1177  }
1178  }
1179 }
1180 
1181 /// Once we have cloned code over from a callee into the caller,
1182 /// update the specified callgraph to reflect the changes we made.
1183 /// Note that it's possible that not all code was copied over, so only
1184 /// some edges of the callgraph may remain.
1186  Function::iterator FirstNewBlock,
1187  ValueToValueMapTy &VMap,
1188  InlineFunctionInfo &IFI) {
1189  CallGraph &CG = *IFI.CG;
1190  const Function *Caller = CS.getCaller();
1191  const Function *Callee = CS.getCalledFunction();
1192  CallGraphNode *CalleeNode = CG[Callee];
1193  CallGraphNode *CallerNode = CG[Caller];
1194 
1195  // Since we inlined some uninlined call sites in the callee into the caller,
1196  // add edges from the caller to all of the callees of the callee.
1197  CallGraphNode::iterator I = CalleeNode->begin(), E = CalleeNode->end();
1198 
1199  // Consider the case where CalleeNode == CallerNode.
1201  if (CalleeNode == CallerNode) {
1202  CallCache.assign(I, E);
1203  I = CallCache.begin();
1204  E = CallCache.end();
1205  }
1206 
1207  for (; I != E; ++I) {
1208  const Value *OrigCall = I->first;
1209 
1210  ValueToValueMapTy::iterator VMI = VMap.find(OrigCall);
1211  // Only copy the edge if the call was inlined!
1212  if (VMI == VMap.end() || VMI->second == nullptr)
1213  continue;
1214 
1215  // If the call was inlined, but then constant folded, there is no edge to
1216  // add. Check for this case.
1217  auto *NewCall = dyn_cast<CallBase>(VMI->second);
1218  if (!NewCall)
1219  continue;
1220 
1221  // We do not treat intrinsic calls like real function calls because we
1222  // expect them to become inline code; do not add an edge for an intrinsic.
1223  if (NewCall->getCalledFunction() &&
1224  NewCall->getCalledFunction()->isIntrinsic())
1225  continue;
1226 
1227  // Remember that this call site got inlined for the client of
1228  // InlineFunction.
1229  IFI.InlinedCalls.push_back(NewCall);
1230 
1231  // It's possible that inlining the callsite will cause it to go from an
1232  // indirect to a direct call by resolving a function pointer. If this
1233  // happens, set the callee of the new call site to a more precise
1234  // destination. This can also happen if the call graph node of the caller
1235  // was just unnecessarily imprecise.
1236  if (!I->second->getFunction())
1237  if (Function *F = NewCall->getCalledFunction()) {
1238  // Indirect call site resolved to direct call.
1239  CallerNode->addCalledFunction(NewCall, CG[F]);
1240 
1241  continue;
1242  }
1243 
1244  CallerNode->addCalledFunction(NewCall, I->second);
1245  }
1246 
1247  // Update the call graph by deleting the edge from Callee to Caller. We must
1248  // do this after the loop above in case Caller and Callee are the same.
1249  CallerNode->removeCallEdgeFor(*cast<CallBase>(CS.getInstruction()));
1250 }
1251 
1252 static void HandleByValArgumentInit(Value *Dst, Value *Src, Module *M,
1253  BasicBlock *InsertBlock,
1254  InlineFunctionInfo &IFI) {
1255  Type *AggTy = cast<PointerType>(Src->getType())->getElementType();
1256  IRBuilder<> Builder(InsertBlock, InsertBlock->begin());
1257 
1258  Value *Size = Builder.getInt64(M->getDataLayout().getTypeStoreSize(AggTy));
1259 
1260  // Always generate a memcpy of alignment 1 here because we don't know
1261  // the alignment of the src pointer. Other optimizations can infer
1262  // better alignment.
1263  Builder.CreateMemCpy(Dst, /*DstAlign*/1, Src, /*SrcAlign*/1, Size);
1264 }
1265 
1266 /// When inlining a call site that has a byval argument,
1267 /// we have to make the implicit memcpy explicit by adding it.
1269  const Function *CalledFunc,
1270  InlineFunctionInfo &IFI,
1271  unsigned ByValAlignment) {
1272  PointerType *ArgTy = cast<PointerType>(Arg->getType());
1273  Type *AggTy = ArgTy->getElementType();
1274 
1275  Function *Caller = TheCall->getFunction();
1276  const DataLayout &DL = Caller->getParent()->getDataLayout();
1277 
1278  // If the called function is readonly, then it could not mutate the caller's
1279  // copy of the byval'd memory. In this case, it is safe to elide the copy and
1280  // temporary.
1281  if (CalledFunc->onlyReadsMemory()) {
1282  // If the byval argument has a specified alignment that is greater than the
1283  // passed in pointer, then we either have to round up the input pointer or
1284  // give up on this transformation.
1285  if (ByValAlignment <= 1) // 0 = unspecified, 1 = no particular alignment.
1286  return Arg;
1287 
1288  AssumptionCache *AC =
1289  IFI.GetAssumptionCache ? &(*IFI.GetAssumptionCache)(*Caller) : nullptr;
1290 
1291  // If the pointer is already known to be sufficiently aligned, or if we can
1292  // round it up to a larger alignment, then we don't need a temporary.
1293  if (getOrEnforceKnownAlignment(Arg, ByValAlignment, DL, TheCall, AC) >=
1294  ByValAlignment)
1295  return Arg;
1296 
1297  // Otherwise, we have to make a memcpy to get a safe alignment. This is bad
1298  // for code quality, but rarely happens and is required for correctness.
1299  }
1300 
1301  // Create the alloca. If we have DataLayout, use nice alignment.
1302  unsigned Align = DL.getPrefTypeAlignment(AggTy);
1303 
1304  // If the byval had an alignment specified, we *must* use at least that
1305  // alignment, as it is required by the byval argument (and uses of the
1306  // pointer inside the callee).
1307  Align = std::max(Align, ByValAlignment);
1308 
1309  Value *NewAlloca = new AllocaInst(AggTy, DL.getAllocaAddrSpace(),
1310  nullptr, Align, Arg->getName(),
1311  &*Caller->begin()->begin());
1312  IFI.StaticAllocas.push_back(cast<AllocaInst>(NewAlloca));
1313 
1314  // Uses of the argument in the function should use our new alloca
1315  // instead.
1316  return NewAlloca;
1317 }
1318 
1319 // Check whether this Value is used by a lifetime intrinsic.
1321  for (User *U : V->users())
1322  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U))
1323  if (II->isLifetimeStartOrEnd())
1324  return true;
1325  return false;
1326 }
1327 
1328 // Check whether the given alloca already has
1329 // lifetime.start or lifetime.end intrinsics.
1330 static bool hasLifetimeMarkers(AllocaInst *AI) {
1331  Type *Ty = AI->getType();
1332  Type *Int8PtrTy = Type::getInt8PtrTy(Ty->getContext(),
1333  Ty->getPointerAddressSpace());
1334  if (Ty == Int8PtrTy)
1335  return isUsedByLifetimeMarker(AI);
1336 
1337  // Do a scan to find all the casts to i8*.
1338  for (User *U : AI->users()) {
1339  if (U->getType() != Int8PtrTy) continue;
1340  if (U->stripPointerCasts() != AI) continue;
1341  if (isUsedByLifetimeMarker(U))
1342  return true;
1343  }
1344  return false;
1345 }
1346 
1347 /// Return the result of AI->isStaticAlloca() if AI were moved to the entry
1348 /// block. Allocas used in inalloca calls and allocas of dynamic array size
1349 /// cannot be static.
1350 static bool allocaWouldBeStaticInEntry(const AllocaInst *AI ) {
1351  return isa<Constant>(AI->getArraySize()) && !AI->isUsedWithInAlloca();
1352 }
1353 
1354 /// Update inlined instructions' line numbers to
1355 /// to encode location where these instructions are inlined.
1357  Instruction *TheCall, bool CalleeHasDebugInfo) {
1358  const DebugLoc &TheCallDL = TheCall->getDebugLoc();
1359  if (!TheCallDL)
1360  return;
1361 
1362  auto &Ctx = Fn->getContext();
1363  DILocation *InlinedAtNode = TheCallDL;
1364 
1365  // Create a unique call site, not to be confused with any other call from the
1366  // same location.
1367  InlinedAtNode = DILocation::getDistinct(
1368  Ctx, InlinedAtNode->getLine(), InlinedAtNode->getColumn(),
1369  InlinedAtNode->getScope(), InlinedAtNode->getInlinedAt());
1370 
1371  // Cache the inlined-at nodes as they're built so they are reused, without
1372  // this every instruction's inlined-at chain would become distinct from each
1373  // other.
1375 
1376  for (; FI != Fn->end(); ++FI) {
1377  for (BasicBlock::iterator BI = FI->begin(), BE = FI->end();
1378  BI != BE; ++BI) {
1379  if (DebugLoc DL = BI->getDebugLoc()) {
1380  auto IA = DebugLoc::appendInlinedAt(DL, InlinedAtNode, BI->getContext(),
1381  IANodes);
1382  auto IDL = DebugLoc::get(DL.getLine(), DL.getCol(), DL.getScope(), IA);
1383  BI->setDebugLoc(IDL);
1384  continue;
1385  }
1386 
1387  if (CalleeHasDebugInfo)
1388  continue;
1389 
1390  // If the inlined instruction has no line number, make it look as if it
1391  // originates from the call location. This is important for
1392  // ((__always_inline__, __nodebug__)) functions which must use caller
1393  // location for all instructions in their function body.
1394 
1395  // Don't update static allocas, as they may get moved later.
1396  if (auto *AI = dyn_cast<AllocaInst>(BI))
1398  continue;
1399 
1400  BI->setDebugLoc(TheCallDL);
1401  }
1402  }
1403 }
1404 
1405 /// Update the block frequencies of the caller after a callee has been inlined.
1406 ///
1407 /// Each block cloned into the caller has its block frequency scaled by the
1408 /// ratio of CallSiteFreq/CalleeEntryFreq. This ensures that the cloned copy of
1409 /// callee's entry block gets the same frequency as the callsite block and the
1410 /// relative frequencies of all cloned blocks remain the same after cloning.
1411 static void updateCallerBFI(BasicBlock *CallSiteBlock,
1412  const ValueToValueMapTy &VMap,
1413  BlockFrequencyInfo *CallerBFI,
1414  BlockFrequencyInfo *CalleeBFI,
1415  const BasicBlock &CalleeEntryBlock) {
1417  for (auto const &Entry : VMap) {
1418  if (!isa<BasicBlock>(Entry.first) || !Entry.second)
1419  continue;
1420  auto *OrigBB = cast<BasicBlock>(Entry.first);
1421  auto *ClonedBB = cast<BasicBlock>(Entry.second);
1422  uint64_t Freq = CalleeBFI->getBlockFreq(OrigBB).getFrequency();
1423  if (!ClonedBBs.insert(ClonedBB).second) {
1424  // Multiple blocks in the callee might get mapped to one cloned block in
1425  // the caller since we prune the callee as we clone it. When that happens,
1426  // we want to use the maximum among the original blocks' frequencies.
1427  uint64_t NewFreq = CallerBFI->getBlockFreq(ClonedBB).getFrequency();
1428  if (NewFreq > Freq)
1429  Freq = NewFreq;
1430  }
1431  CallerBFI->setBlockFreq(ClonedBB, Freq);
1432  }
1433  BasicBlock *EntryClone = cast<BasicBlock>(VMap.lookup(&CalleeEntryBlock));
1434  CallerBFI->setBlockFreqAndScale(
1435  EntryClone, CallerBFI->getBlockFreq(CallSiteBlock).getFrequency(),
1436  ClonedBBs);
1437 }
1438 
1439 /// Update the branch metadata for cloned call instructions.
1440 static void updateCallProfile(Function *Callee, const ValueToValueMapTy &VMap,
1441  const ProfileCount &CalleeEntryCount,
1442  const Instruction *TheCall,
1443  ProfileSummaryInfo *PSI,
1444  BlockFrequencyInfo *CallerBFI) {
1445  if (!CalleeEntryCount.hasValue() || CalleeEntryCount.isSynthetic() ||
1446  CalleeEntryCount.getCount() < 1)
1447  return;
1448  auto CallSiteCount = PSI ? PSI->getProfileCount(TheCall, CallerBFI) : None;
1449  int64_t CallCount =
1450  std::min(CallSiteCount.hasValue() ? CallSiteCount.getValue() : 0,
1451  CalleeEntryCount.getCount());
1452  updateProfileCallee(Callee, -CallCount, &VMap);
1453 }
1454 
1456  Function *Callee, int64_t entryDelta,
1458  auto CalleeCount = Callee->getEntryCount();
1459  if (!CalleeCount.hasValue())
1460  return;
1461 
1462  uint64_t priorEntryCount = CalleeCount.getCount();
1463  uint64_t newEntryCount = priorEntryCount;
1464 
1465  // Since CallSiteCount is an estimate, it could exceed the original callee
1466  // count and has to be set to 0 so guard against underflow.
1467  if (entryDelta < 0 && static_cast<uint64_t>(-entryDelta) > priorEntryCount)
1468  newEntryCount = 0;
1469  else
1470  newEntryCount = priorEntryCount + entryDelta;
1471 
1472  Callee->setEntryCount(newEntryCount);
1473 
1474  // During inlining ?
1475  if (VMap) {
1476  uint64_t cloneEntryCount = priorEntryCount - newEntryCount;
1477  for (auto const &Entry : *VMap)
1478  if (isa<CallInst>(Entry.first))
1479  if (auto *CI = dyn_cast_or_null<CallInst>(Entry.second))
1480  CI->updateProfWeight(cloneEntryCount, priorEntryCount);
1481  }
1482  for (BasicBlock &BB : *Callee)
1483  // No need to update the callsite if it is pruned during inlining.
1484  if (!VMap || VMap->count(&BB))
1485  for (Instruction &I : BB)
1486  if (CallInst *CI = dyn_cast<CallInst>(&I))
1487  CI->updateProfWeight(newEntryCount, priorEntryCount);
1488 }
1489 
1490 /// This function inlines the called function into the basic block of the
1491 /// caller. This returns false if it is not possible to inline this call.
1492 /// The program is still in a well defined state if this occurs though.
1493 ///
1494 /// Note that this only does one level of inlining. For example, if the
1495 /// instruction 'call B' is inlined, and 'B' calls 'C', then the call to 'C' now
1496 /// exists in the instruction stream. Similarly this will inline a recursive
1497 /// function by one level.
1499  AAResults *CalleeAAR,
1500  bool InsertLifetime,
1501  Function *ForwardVarArgsTo) {
1502  Instruction *TheCall = CS.getInstruction();
1503  assert(TheCall->getParent() && TheCall->getFunction()
1504  && "Instruction not in function!");
1505 
1506  // FIXME: we don't inline callbr yet.
1507  if (isa<CallBrInst>(TheCall))
1508  return false;
1509 
1510  // If IFI has any state in it, zap it before we fill it in.
1511  IFI.reset();
1512 
1513  Function *CalledFunc = CS.getCalledFunction();
1514  if (!CalledFunc || // Can't inline external function or indirect
1515  CalledFunc->isDeclaration()) // call!
1516  return "external or indirect";
1517 
1518  // The inliner does not know how to inline through calls with operand bundles
1519  // in general ...
1520  if (CS.hasOperandBundles()) {
1521  for (int i = 0, e = CS.getNumOperandBundles(); i != e; ++i) {
1523  // ... but it knows how to inline through "deopt" operand bundles ...
1524  if (Tag == LLVMContext::OB_deopt)
1525  continue;
1526  // ... and "funclet" operand bundles.
1527  if (Tag == LLVMContext::OB_funclet)
1528  continue;
1529 
1530  return "unsupported operand bundle";
1531  }
1532  }
1533 
1534  // If the call to the callee cannot throw, set the 'nounwind' flag on any
1535  // calls that we inline.
1536  bool MarkNoUnwind = CS.doesNotThrow();
1537 
1538  BasicBlock *OrigBB = TheCall->getParent();
1539  Function *Caller = OrigBB->getParent();
1540 
1541  // GC poses two hazards to inlining, which only occur when the callee has GC:
1542  // 1. If the caller has no GC, then the callee's GC must be propagated to the
1543  // caller.
1544  // 2. If the caller has a differing GC, it is invalid to inline.
1545  if (CalledFunc->hasGC()) {
1546  if (!Caller->hasGC())
1547  Caller->setGC(CalledFunc->getGC());
1548  else if (CalledFunc->getGC() != Caller->getGC())
1549  return "incompatible GC";
1550  }
1551 
1552  // Get the personality function from the callee if it contains a landing pad.
1553  Constant *CalledPersonality =
1554  CalledFunc->hasPersonalityFn()
1555  ? CalledFunc->getPersonalityFn()->stripPointerCasts()
1556  : nullptr;
1557 
1558  // Find the personality function used by the landing pads of the caller. If it
1559  // exists, then check to see that it matches the personality function used in
1560  // the callee.
1561  Constant *CallerPersonality =
1562  Caller->hasPersonalityFn()
1563  ? Caller->getPersonalityFn()->stripPointerCasts()
1564  : nullptr;
1565  if (CalledPersonality) {
1566  if (!CallerPersonality)
1567  Caller->setPersonalityFn(CalledPersonality);
1568  // If the personality functions match, then we can perform the
1569  // inlining. Otherwise, we can't inline.
1570  // TODO: This isn't 100% true. Some personality functions are proper
1571  // supersets of others and can be used in place of the other.
1572  else if (CalledPersonality != CallerPersonality)
1573  return "incompatible personality";
1574  }
1575 
1576  // We need to figure out which funclet the callsite was in so that we may
1577  // properly nest the callee.
1578  Instruction *CallSiteEHPad = nullptr;
1579  if (CallerPersonality) {
1580  EHPersonality Personality = classifyEHPersonality(CallerPersonality);
1581  if (isScopedEHPersonality(Personality)) {
1582  Optional<OperandBundleUse> ParentFunclet =
1584  if (ParentFunclet)
1585  CallSiteEHPad = cast<FuncletPadInst>(ParentFunclet->Inputs.front());
1586 
1587  // OK, the inlining site is legal. What about the target function?
1588 
1589  if (CallSiteEHPad) {
1590  if (Personality == EHPersonality::MSVC_CXX) {
1591  // The MSVC personality cannot tolerate catches getting inlined into
1592  // cleanup funclets.
1593  if (isa<CleanupPadInst>(CallSiteEHPad)) {
1594  // Ok, the call site is within a cleanuppad. Let's check the callee
1595  // for catchpads.
1596  for (const BasicBlock &CalledBB : *CalledFunc) {
1597  if (isa<CatchSwitchInst>(CalledBB.getFirstNonPHI()))
1598  return "catch in cleanup funclet";
1599  }
1600  }
1601  } else if (isAsynchronousEHPersonality(Personality)) {
1602  // SEH is even less tolerant, there may not be any sort of exceptional
1603  // funclet in the callee.
1604  for (const BasicBlock &CalledBB : *CalledFunc) {
1605  if (CalledBB.isEHPad())
1606  return "SEH in cleanup funclet";
1607  }
1608  }
1609  }
1610  }
1611  }
1612 
1613  // Determine if we are dealing with a call in an EHPad which does not unwind
1614  // to caller.
1615  bool EHPadForCallUnwindsLocally = false;
1616  if (CallSiteEHPad && CS.isCall()) {
1617  UnwindDestMemoTy FuncletUnwindMap;
1618  Value *CallSiteUnwindDestToken =
1619  getUnwindDestToken(CallSiteEHPad, FuncletUnwindMap);
1620 
1621  EHPadForCallUnwindsLocally =
1622  CallSiteUnwindDestToken &&
1623  !isa<ConstantTokenNone>(CallSiteUnwindDestToken);
1624  }
1625 
1626  // Get an iterator to the last basic block in the function, which will have
1627  // the new function inlined after it.
1628  Function::iterator LastBlock = --Caller->end();
1629 
1630  // Make sure to capture all of the return instructions from the cloned
1631  // function.
1633  ClonedCodeInfo InlinedFunctionInfo;
1634  Function::iterator FirstNewBlock;
1635 
1636  { // Scope to destroy VMap after cloning.
1637  ValueToValueMapTy VMap;
1638  // Keep a list of pair (dst, src) to emit byval initializations.
1640 
1641  auto &DL = Caller->getParent()->getDataLayout();
1642 
1643  // Calculate the vector of arguments to pass into the function cloner, which
1644  // matches up the formal to the actual argument values.
1646  unsigned ArgNo = 0;
1647  for (Function::arg_iterator I = CalledFunc->arg_begin(),
1648  E = CalledFunc->arg_end(); I != E; ++I, ++AI, ++ArgNo) {
1649  Value *ActualArg = *AI;
1650 
1651  // When byval arguments actually inlined, we need to make the copy implied
1652  // by them explicit. However, we don't do this if the callee is readonly
1653  // or readnone, because the copy would be unneeded: the callee doesn't
1654  // modify the struct.
1655  if (CS.isByValArgument(ArgNo)) {
1656  ActualArg = HandleByValArgument(ActualArg, TheCall, CalledFunc, IFI,
1657  CalledFunc->getParamAlignment(ArgNo));
1658  if (ActualArg != *AI)
1659  ByValInit.push_back(std::make_pair(ActualArg, (Value*) *AI));
1660  }
1661 
1662  VMap[&*I] = ActualArg;
1663  }
1664 
1665  // Add alignment assumptions if necessary. We do this before the inlined
1666  // instructions are actually cloned into the caller so that we can easily
1667  // check what will be known at the start of the inlined code.
1668  AddAlignmentAssumptions(CS, IFI);
1669 
1670  // We want the inliner to prune the code as it copies. We would LOVE to
1671  // have no dead or constant instructions leftover after inlining occurs
1672  // (which can happen, e.g., because an argument was constant), but we'll be
1673  // happy with whatever the cloner can do.
1674  CloneAndPruneFunctionInto(Caller, CalledFunc, VMap,
1675  /*ModuleLevelChanges=*/false, Returns, ".i",
1676  &InlinedFunctionInfo, TheCall);
1677  // Remember the first block that is newly cloned over.
1678  FirstNewBlock = LastBlock; ++FirstNewBlock;
1679 
1680  if (IFI.CallerBFI != nullptr && IFI.CalleeBFI != nullptr)
1681  // Update the BFI of blocks cloned into the caller.
1682  updateCallerBFI(OrigBB, VMap, IFI.CallerBFI, IFI.CalleeBFI,
1683  CalledFunc->front());
1684 
1685  updateCallProfile(CalledFunc, VMap, CalledFunc->getEntryCount(), TheCall,
1686  IFI.PSI, IFI.CallerBFI);
1687 
1688  // Inject byval arguments initialization.
1689  for (std::pair<Value*, Value*> &Init : ByValInit)
1690  HandleByValArgumentInit(Init.first, Init.second, Caller->getParent(),
1691  &*FirstNewBlock, IFI);
1692 
1693  Optional<OperandBundleUse> ParentDeopt =
1695  if (ParentDeopt) {
1697 
1698  for (auto &VH : InlinedFunctionInfo.OperandBundleCallSites) {
1699  Instruction *I = dyn_cast_or_null<Instruction>(VH);
1700  if (!I) continue; // instruction was DCE'd or RAUW'ed to undef
1701 
1702  OpDefs.clear();
1703 
1704  CallSite ICS(I);
1705  OpDefs.reserve(ICS.getNumOperandBundles());
1706 
1707  for (unsigned i = 0, e = ICS.getNumOperandBundles(); i < e; ++i) {
1708  auto ChildOB = ICS.getOperandBundleAt(i);
1709  if (ChildOB.getTagID() != LLVMContext::OB_deopt) {
1710  // If the inlined call has other operand bundles, let them be
1711  OpDefs.emplace_back(ChildOB);
1712  continue;
1713  }
1714 
1715  // It may be useful to separate this logic (of handling operand
1716  // bundles) out to a separate "policy" component if this gets crowded.
1717  // Prepend the parent's deoptimization continuation to the newly
1718  // inlined call's deoptimization continuation.
1719  std::vector<Value *> MergedDeoptArgs;
1720  MergedDeoptArgs.reserve(ParentDeopt->Inputs.size() +
1721  ChildOB.Inputs.size());
1722 
1723  MergedDeoptArgs.insert(MergedDeoptArgs.end(),
1724  ParentDeopt->Inputs.begin(),
1725  ParentDeopt->Inputs.end());
1726  MergedDeoptArgs.insert(MergedDeoptArgs.end(), ChildOB.Inputs.begin(),
1727  ChildOB.Inputs.end());
1728 
1729  OpDefs.emplace_back("deopt", std::move(MergedDeoptArgs));
1730  }
1731 
1732  Instruction *NewI = nullptr;
1733  if (isa<CallInst>(I))
1734  NewI = CallInst::Create(cast<CallInst>(I), OpDefs, I);
1735  else if (isa<CallBrInst>(I))
1736  NewI = CallBrInst::Create(cast<CallBrInst>(I), OpDefs, I);
1737  else
1738  NewI = InvokeInst::Create(cast<InvokeInst>(I), OpDefs, I);
1739 
1740  // Note: the RAUW does the appropriate fixup in VMap, so we need to do
1741  // this even if the call returns void.
1742  I->replaceAllUsesWith(NewI);
1743 
1744  VH = nullptr;
1745  I->eraseFromParent();
1746  }
1747  }
1748 
1749  // Update the callgraph if requested.
1750  if (IFI.CG)
1751  UpdateCallGraphAfterInlining(CS, FirstNewBlock, VMap, IFI);
1752 
1753  // For 'nodebug' functions, the associated DISubprogram is always null.
1754  // Conservatively avoid propagating the callsite debug location to
1755  // instructions inlined from a function whose DISubprogram is not null.
1756  fixupLineNumbers(Caller, FirstNewBlock, TheCall,
1757  CalledFunc->getSubprogram() != nullptr);
1758 
1759  // Clone existing noalias metadata if necessary.
1760  CloneAliasScopeMetadata(CS, VMap);
1761 
1762  // Add noalias metadata if necessary.
1763  AddAliasScopeMetadata(CS, VMap, DL, CalleeAAR);
1764 
1765  // Propagate llvm.mem.parallel_loop_access if necessary.
1767 
1768  // Register any cloned assumptions.
1769  if (IFI.GetAssumptionCache)
1770  for (BasicBlock &NewBlock :
1771  make_range(FirstNewBlock->getIterator(), Caller->end()))
1772  for (Instruction &I : NewBlock) {
1773  if (auto *II = dyn_cast<IntrinsicInst>(&I))
1774  if (II->getIntrinsicID() == Intrinsic::assume)
1775  (*IFI.GetAssumptionCache)(*Caller).registerAssumption(II);
1776  }
1777  }
1778 
1779  // If there are any alloca instructions in the block that used to be the entry
1780  // block for the callee, move them to the entry block of the caller. First
1781  // calculate which instruction they should be inserted before. We insert the
1782  // instructions at the end of the current alloca list.
1783  {
1784  BasicBlock::iterator InsertPoint = Caller->begin()->begin();
1785  for (BasicBlock::iterator I = FirstNewBlock->begin(),
1786  E = FirstNewBlock->end(); I != E; ) {
1787  AllocaInst *AI = dyn_cast<AllocaInst>(I++);
1788  if (!AI) continue;
1789 
1790  // If the alloca is now dead, remove it. This often occurs due to code
1791  // specialization.
1792  if (AI->use_empty()) {
1793  AI->eraseFromParent();
1794  continue;
1795  }
1796 
1797  if (!allocaWouldBeStaticInEntry(AI))
1798  continue;
1799 
1800  // Keep track of the static allocas that we inline into the caller.
1801  IFI.StaticAllocas.push_back(AI);
1802 
1803  // Scan for the block of allocas that we can move over, and move them
1804  // all at once.
1805  while (isa<AllocaInst>(I) &&
1806  allocaWouldBeStaticInEntry(cast<AllocaInst>(I))) {
1807  IFI.StaticAllocas.push_back(cast<AllocaInst>(I));
1808  ++I;
1809  }
1810 
1811  // Transfer all of the allocas over in a block. Using splice means
1812  // that the instructions aren't removed from the symbol table, then
1813  // reinserted.
1814  Caller->getEntryBlock().getInstList().splice(
1815  InsertPoint, FirstNewBlock->getInstList(), AI->getIterator(), I);
1816  }
1817  // Move any dbg.declares describing the allocas into the entry basic block.
1818  DIBuilder DIB(*Caller->getParent());
1819  for (auto &AI : IFI.StaticAllocas)
1822  }
1823 
1824  SmallVector<Value*,4> VarArgsToForward;
1825  SmallVector<AttributeSet, 4> VarArgsAttrs;
1826  for (unsigned i = CalledFunc->getFunctionType()->getNumParams();
1827  i < CS.getNumArgOperands(); i++) {
1828  VarArgsToForward.push_back(CS.getArgOperand(i));
1829  VarArgsAttrs.push_back(CS.getAttributes().getParamAttributes(i));
1830  }
1831 
1832  bool InlinedMustTailCalls = false, InlinedDeoptimizeCalls = false;
1833  if (InlinedFunctionInfo.ContainsCalls) {
1834  CallInst::TailCallKind CallSiteTailKind = CallInst::TCK_None;
1835  if (CallInst *CI = dyn_cast<CallInst>(TheCall))
1836  CallSiteTailKind = CI->getTailCallKind();
1837 
1838  // For inlining purposes, the "notail" marker is the same as no marker.
1839  if (CallSiteTailKind == CallInst::TCK_NoTail)
1840  CallSiteTailKind = CallInst::TCK_None;
1841 
1842  for (Function::iterator BB = FirstNewBlock, E = Caller->end(); BB != E;
1843  ++BB) {
1844  for (auto II = BB->begin(); II != BB->end();) {
1845  Instruction &I = *II++;
1846  CallInst *CI = dyn_cast<CallInst>(&I);
1847  if (!CI)
1848  continue;
1849 
1850  // Forward varargs from inlined call site to calls to the
1851  // ForwardVarArgsTo function, if requested, and to musttail calls.
1852  if (!VarArgsToForward.empty() &&
1853  ((ForwardVarArgsTo &&
1854  CI->getCalledFunction() == ForwardVarArgsTo) ||
1855  CI->isMustTailCall())) {
1856  // Collect attributes for non-vararg parameters.
1859  if (!Attrs.isEmpty() || !VarArgsAttrs.empty()) {
1860  for (unsigned ArgNo = 0;
1861  ArgNo < CI->getFunctionType()->getNumParams(); ++ArgNo)
1862  ArgAttrs.push_back(Attrs.getParamAttributes(ArgNo));
1863  }
1864 
1865  // Add VarArg attributes.
1866  ArgAttrs.append(VarArgsAttrs.begin(), VarArgsAttrs.end());
1867  Attrs = AttributeList::get(CI->getContext(), Attrs.getFnAttributes(),
1868  Attrs.getRetAttributes(), ArgAttrs);
1869  // Add VarArgs to existing parameters.
1870  SmallVector<Value *, 6> Params(CI->arg_operands());
1871  Params.append(VarArgsToForward.begin(), VarArgsToForward.end());
1872  CallInst *NewCI = CallInst::Create(
1873  CI->getFunctionType(), CI->getCalledOperand(), Params, "", CI);
1874  NewCI->setDebugLoc(CI->getDebugLoc());
1875  NewCI->setAttributes(Attrs);
1876  NewCI->setCallingConv(CI->getCallingConv());
1877  CI->replaceAllUsesWith(NewCI);
1878  CI->eraseFromParent();
1879  CI = NewCI;
1880  }
1881 
1882  if (Function *F = CI->getCalledFunction())
1883  InlinedDeoptimizeCalls |=
1884  F->getIntrinsicID() == Intrinsic::experimental_deoptimize;
1885 
1886  // We need to reduce the strength of any inlined tail calls. For
1887  // musttail, we have to avoid introducing potential unbounded stack
1888  // growth. For example, if functions 'f' and 'g' are mutually recursive
1889  // with musttail, we can inline 'g' into 'f' so long as we preserve
1890  // musttail on the cloned call to 'f'. If either the inlined call site
1891  // or the cloned call site is *not* musttail, the program already has
1892  // one frame of stack growth, so it's safe to remove musttail. Here is
1893  // a table of example transformations:
1894  //
1895  // f -> musttail g -> musttail f ==> f -> musttail f
1896  // f -> musttail g -> tail f ==> f -> tail f
1897  // f -> g -> musttail f ==> f -> f
1898  // f -> g -> tail f ==> f -> f
1899  //
1900  // Inlined notail calls should remain notail calls.
1901  CallInst::TailCallKind ChildTCK = CI->getTailCallKind();
1902  if (ChildTCK != CallInst::TCK_NoTail)
1903  ChildTCK = std::min(CallSiteTailKind, ChildTCK);
1904  CI->setTailCallKind(ChildTCK);
1905  InlinedMustTailCalls |= CI->isMustTailCall();
1906 
1907  // Calls inlined through a 'nounwind' call site should be marked
1908  // 'nounwind'.
1909  if (MarkNoUnwind)
1910  CI->setDoesNotThrow();
1911  }
1912  }
1913  }
1914 
1915  // Leave lifetime markers for the static alloca's, scoping them to the
1916  // function we just inlined.
1917  if (InsertLifetime && !IFI.StaticAllocas.empty()) {
1918  IRBuilder<> builder(&FirstNewBlock->front());
1919  for (unsigned ai = 0, ae = IFI.StaticAllocas.size(); ai != ae; ++ai) {
1920  AllocaInst *AI = IFI.StaticAllocas[ai];
1921  // Don't mark swifterror allocas. They can't have bitcast uses.
1922  if (AI->isSwiftError())
1923  continue;
1924 
1925  // If the alloca is already scoped to something smaller than the whole
1926  // function then there's no need to add redundant, less accurate markers.
1927  if (hasLifetimeMarkers(AI))
1928  continue;
1929 
1930  // Try to determine the size of the allocation.
1931  ConstantInt *AllocaSize = nullptr;
1932  if (ConstantInt *AIArraySize =
1933  dyn_cast<ConstantInt>(AI->getArraySize())) {
1934  auto &DL = Caller->getParent()->getDataLayout();
1935  Type *AllocaType = AI->getAllocatedType();
1936  uint64_t AllocaTypeSize = DL.getTypeAllocSize(AllocaType);
1937  uint64_t AllocaArraySize = AIArraySize->getLimitedValue();
1938 
1939  // Don't add markers for zero-sized allocas.
1940  if (AllocaArraySize == 0)
1941  continue;
1942 
1943  // Check that array size doesn't saturate uint64_t and doesn't
1944  // overflow when it's multiplied by type size.
1945  if (AllocaArraySize != std::numeric_limits<uint64_t>::max() &&
1946  std::numeric_limits<uint64_t>::max() / AllocaArraySize >=
1947  AllocaTypeSize) {
1948  AllocaSize = ConstantInt::get(Type::getInt64Ty(AI->getContext()),
1949  AllocaArraySize * AllocaTypeSize);
1950  }
1951  }
1952 
1953  builder.CreateLifetimeStart(AI, AllocaSize);
1954  for (ReturnInst *RI : Returns) {
1955  // Don't insert llvm.lifetime.end calls between a musttail or deoptimize
1956  // call and a return. The return kills all local allocas.
1957  if (InlinedMustTailCalls &&
1959  continue;
1960  if (InlinedDeoptimizeCalls &&
1962  continue;
1963  IRBuilder<>(RI).CreateLifetimeEnd(AI, AllocaSize);
1964  }
1965  }
1966  }
1967 
1968  // If the inlined code contained dynamic alloca instructions, wrap the inlined
1969  // code with llvm.stacksave/llvm.stackrestore intrinsics.
1970  if (InlinedFunctionInfo.ContainsDynamicAllocas) {
1971  Module *M = Caller->getParent();
1972  // Get the two intrinsics we care about.
1973  Function *StackSave = Intrinsic::getDeclaration(M, Intrinsic::stacksave);
1974  Function *StackRestore=Intrinsic::getDeclaration(M,Intrinsic::stackrestore);
1975 
1976  // Insert the llvm.stacksave.
1977  CallInst *SavedPtr = IRBuilder<>(&*FirstNewBlock, FirstNewBlock->begin())
1978  .CreateCall(StackSave, {}, "savedstack");
1979 
1980  // Insert a call to llvm.stackrestore before any return instructions in the
1981  // inlined function.
1982  for (ReturnInst *RI : Returns) {
1983  // Don't insert llvm.stackrestore calls between a musttail or deoptimize
1984  // call and a return. The return will restore the stack pointer.
1985  if (InlinedMustTailCalls && RI->getParent()->getTerminatingMustTailCall())
1986  continue;
1987  if (InlinedDeoptimizeCalls && RI->getParent()->getTerminatingDeoptimizeCall())
1988  continue;
1989  IRBuilder<>(RI).CreateCall(StackRestore, SavedPtr);
1990  }
1991  }
1992 
1993  // If we are inlining for an invoke instruction, we must make sure to rewrite
1994  // any call instructions into invoke instructions. This is sensitive to which
1995  // funclet pads were top-level in the inlinee, so must be done before
1996  // rewriting the "parent pad" links.
1997  if (auto *II = dyn_cast<InvokeInst>(TheCall)) {
1998  BasicBlock *UnwindDest = II->getUnwindDest();
1999  Instruction *FirstNonPHI = UnwindDest->getFirstNonPHI();
2000  if (isa<LandingPadInst>(FirstNonPHI)) {
2001  HandleInlinedLandingPad(II, &*FirstNewBlock, InlinedFunctionInfo);
2002  } else {
2003  HandleInlinedEHPad(II, &*FirstNewBlock, InlinedFunctionInfo);
2004  }
2005  }
2006 
2007  // Update the lexical scopes of the new funclets and callsites.
2008  // Anything that had 'none' as its parent is now nested inside the callsite's
2009  // EHPad.
2010 
2011  if (CallSiteEHPad) {
2012  for (Function::iterator BB = FirstNewBlock->getIterator(),
2013  E = Caller->end();
2014  BB != E; ++BB) {
2015  // Add bundle operands to any top-level call sites.
2017  for (BasicBlock::iterator BBI = BB->begin(), E = BB->end(); BBI != E;) {
2018  Instruction *I = &*BBI++;
2019  CallSite CS(I);
2020  if (!CS)
2021  continue;
2022 
2023  // Skip call sites which are nounwind intrinsics.
2024  auto *CalledFn =
2026  if (CalledFn && CalledFn->isIntrinsic() && CS.doesNotThrow())
2027  continue;
2028 
2029  // Skip call sites which already have a "funclet" bundle.
2031  continue;
2032 
2033  CS.getOperandBundlesAsDefs(OpBundles);
2034  OpBundles.emplace_back("funclet", CallSiteEHPad);
2035 
2036  Instruction *NewInst;
2037  if (CS.isCall())
2038  NewInst = CallInst::Create(cast<CallInst>(I), OpBundles, I);
2039  else if (CS.isCallBr())
2040  NewInst = CallBrInst::Create(cast<CallBrInst>(I), OpBundles, I);
2041  else
2042  NewInst = InvokeInst::Create(cast<InvokeInst>(I), OpBundles, I);
2043  NewInst->takeName(I);
2044  I->replaceAllUsesWith(NewInst);
2045  I->eraseFromParent();
2046 
2047  OpBundles.clear();
2048  }
2049 
2050  // It is problematic if the inlinee has a cleanupret which unwinds to
2051  // caller and we inline it into a call site which doesn't unwind but into
2052  // an EH pad that does. Such an edge must be dynamically unreachable.
2053  // As such, we replace the cleanupret with unreachable.
2054  if (auto *CleanupRet = dyn_cast<CleanupReturnInst>(BB->getTerminator()))
2055  if (CleanupRet->unwindsToCaller() && EHPadForCallUnwindsLocally)
2056  changeToUnreachable(CleanupRet, /*UseLLVMTrap=*/false);
2057 
2058  Instruction *I = BB->getFirstNonPHI();
2059  if (!I->isEHPad())
2060  continue;
2061 
2062  if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(I)) {
2063  if (isa<ConstantTokenNone>(CatchSwitch->getParentPad()))
2064  CatchSwitch->setParentPad(CallSiteEHPad);
2065  } else {
2066  auto *FPI = cast<FuncletPadInst>(I);
2067  if (isa<ConstantTokenNone>(FPI->getParentPad()))
2068  FPI->setParentPad(CallSiteEHPad);
2069  }
2070  }
2071  }
2072 
2073  if (InlinedDeoptimizeCalls) {
2074  // We need to at least remove the deoptimizing returns from the Return set,
2075  // so that the control flow from those returns does not get merged into the
2076  // caller (but terminate it instead). If the caller's return type does not
2077  // match the callee's return type, we also need to change the return type of
2078  // the intrinsic.
2079  if (Caller->getReturnType() == TheCall->getType()) {
2080  auto NewEnd = llvm::remove_if(Returns, [](ReturnInst *RI) {
2081  return RI->getParent()->getTerminatingDeoptimizeCall() != nullptr;
2082  });
2083  Returns.erase(NewEnd, Returns.end());
2084  } else {
2085  SmallVector<ReturnInst *, 8> NormalReturns;
2086  Function *NewDeoptIntrinsic = Intrinsic::getDeclaration(
2087  Caller->getParent(), Intrinsic::experimental_deoptimize,
2088  {Caller->getReturnType()});
2089 
2090  for (ReturnInst *RI : Returns) {
2091  CallInst *DeoptCall = RI->getParent()->getTerminatingDeoptimizeCall();
2092  if (!DeoptCall) {
2093  NormalReturns.push_back(RI);
2094  continue;
2095  }
2096 
2097  // The calling convention on the deoptimize call itself may be bogus,
2098  // since the code we're inlining may have undefined behavior (and may
2099  // never actually execute at runtime); but all
2100  // @llvm.experimental.deoptimize declarations have to have the same
2101  // calling convention in a well-formed module.
2102  auto CallingConv = DeoptCall->getCalledFunction()->getCallingConv();
2103  NewDeoptIntrinsic->setCallingConv(CallingConv);
2104  auto *CurBB = RI->getParent();
2105  RI->eraseFromParent();
2106 
2107  SmallVector<Value *, 4> CallArgs(DeoptCall->arg_begin(),
2108  DeoptCall->arg_end());
2109 
2111  DeoptCall->getOperandBundlesAsDefs(OpBundles);
2112  DeoptCall->eraseFromParent();
2113  assert(!OpBundles.empty() &&
2114  "Expected at least the deopt operand bundle");
2115 
2116  IRBuilder<> Builder(CurBB);
2117  CallInst *NewDeoptCall =
2118  Builder.CreateCall(NewDeoptIntrinsic, CallArgs, OpBundles);
2119  NewDeoptCall->setCallingConv(CallingConv);
2120  if (NewDeoptCall->getType()->isVoidTy())
2121  Builder.CreateRetVoid();
2122  else
2123  Builder.CreateRet(NewDeoptCall);
2124  }
2125 
2126  // Leave behind the normal returns so we can merge control flow.
2127  std::swap(Returns, NormalReturns);
2128  }
2129  }
2130 
2131  // Handle any inlined musttail call sites. In order for a new call site to be
2132  // musttail, the source of the clone and the inlined call site must have been
2133  // musttail. Therefore it's safe to return without merging control into the
2134  // phi below.
2135  if (InlinedMustTailCalls) {
2136  // Check if we need to bitcast the result of any musttail calls.
2137  Type *NewRetTy = Caller->getReturnType();
2138  bool NeedBitCast = !TheCall->use_empty() && TheCall->getType() != NewRetTy;
2139 
2140  // Handle the returns preceded by musttail calls separately.
2141  SmallVector<ReturnInst *, 8> NormalReturns;
2142  for (ReturnInst *RI : Returns) {
2143  CallInst *ReturnedMustTail =
2145  if (!ReturnedMustTail) {
2146  NormalReturns.push_back(RI);
2147  continue;
2148  }
2149  if (!NeedBitCast)
2150  continue;
2151 
2152  // Delete the old return and any preceding bitcast.
2153  BasicBlock *CurBB = RI->getParent();
2154  auto *OldCast = dyn_cast_or_null<BitCastInst>(RI->getReturnValue());
2155  RI->eraseFromParent();
2156  if (OldCast)
2157  OldCast->eraseFromParent();
2158 
2159  // Insert a new bitcast and return with the right type.
2160  IRBuilder<> Builder(CurBB);
2161  Builder.CreateRet(Builder.CreateBitCast(ReturnedMustTail, NewRetTy));
2162  }
2163 
2164  // Leave behind the normal returns so we can merge control flow.
2165  std::swap(Returns, NormalReturns);
2166  }
2167 
2168  // Now that all of the transforms on the inlined code have taken place but
2169  // before we splice the inlined code into the CFG and lose track of which
2170  // blocks were actually inlined, collect the call sites. We only do this if
2171  // call graph updates weren't requested, as those provide value handle based
2172  // tracking of inlined call sites instead.
2173  if (InlinedFunctionInfo.ContainsCalls && !IFI.CG) {
2174  // Otherwise just collect the raw call sites that were inlined.
2175  for (BasicBlock &NewBB :
2176  make_range(FirstNewBlock->getIterator(), Caller->end()))
2177  for (Instruction &I : NewBB)
2178  if (auto CS = CallSite(&I))
2179  IFI.InlinedCallSites.push_back(CS);
2180  }
2181 
2182  // If we cloned in _exactly one_ basic block, and if that block ends in a
2183  // return instruction, we splice the body of the inlined callee directly into
2184  // the calling basic block.
2185  if (Returns.size() == 1 && std::distance(FirstNewBlock, Caller->end()) == 1) {
2186  // Move all of the instructions right before the call.
2187  OrigBB->getInstList().splice(TheCall->getIterator(),
2188  FirstNewBlock->getInstList(),
2189  FirstNewBlock->begin(), FirstNewBlock->end());
2190  // Remove the cloned basic block.
2191  Caller->getBasicBlockList().pop_back();
2192 
2193  // If the call site was an invoke instruction, add a branch to the normal
2194  // destination.
2195  if (InvokeInst *II = dyn_cast<InvokeInst>(TheCall)) {
2196  BranchInst *NewBr = BranchInst::Create(II->getNormalDest(), TheCall);
2197  NewBr->setDebugLoc(Returns[0]->getDebugLoc());
2198  }
2199 
2200  // If the return instruction returned a value, replace uses of the call with
2201  // uses of the returned value.
2202  if (!TheCall->use_empty()) {
2203  ReturnInst *R = Returns[0];
2204  if (TheCall == R->getReturnValue())
2205  TheCall->replaceAllUsesWith(UndefValue::get(TheCall->getType()));
2206  else
2207  TheCall->replaceAllUsesWith(R->getReturnValue());
2208  }
2209  // Since we are now done with the Call/Invoke, we can delete it.
2210  TheCall->eraseFromParent();
2211 
2212  // Since we are now done with the return instruction, delete it also.
2213  Returns[0]->eraseFromParent();
2214 
2215  // We are now done with the inlining.
2216  return true;
2217  }
2218 
2219  // Otherwise, we have the normal case, of more than one block to inline or
2220  // multiple return sites.
2221 
2222  // We want to clone the entire callee function into the hole between the
2223  // "starter" and "ender" blocks. How we accomplish this depends on whether
2224  // this is an invoke instruction or a call instruction.
2225  BasicBlock *AfterCallBB;
2226  BranchInst *CreatedBranchToNormalDest = nullptr;
2227  if (InvokeInst *II = dyn_cast<InvokeInst>(TheCall)) {
2228 
2229  // Add an unconditional branch to make this look like the CallInst case...
2230  CreatedBranchToNormalDest = BranchInst::Create(II->getNormalDest(), TheCall);
2231 
2232  // Split the basic block. This guarantees that no PHI nodes will have to be
2233  // updated due to new incoming edges, and make the invoke case more
2234  // symmetric to the call case.
2235  AfterCallBB =
2236  OrigBB->splitBasicBlock(CreatedBranchToNormalDest->getIterator(),
2237  CalledFunc->getName() + ".exit");
2238 
2239  } else { // It's a call
2240  // If this is a call instruction, we need to split the basic block that
2241  // the call lives in.
2242  //
2243  AfterCallBB = OrigBB->splitBasicBlock(TheCall->getIterator(),
2244  CalledFunc->getName() + ".exit");
2245  }
2246 
2247  if (IFI.CallerBFI) {
2248  // Copy original BB's block frequency to AfterCallBB
2249  IFI.CallerBFI->setBlockFreq(
2250  AfterCallBB, IFI.CallerBFI->getBlockFreq(OrigBB).getFrequency());
2251  }
2252 
2253  // Change the branch that used to go to AfterCallBB to branch to the first
2254  // basic block of the inlined function.
2255  //
2256  Instruction *Br = OrigBB->getTerminator();
2257  assert(Br && Br->getOpcode() == Instruction::Br &&
2258  "splitBasicBlock broken!");
2259  Br->setOperand(0, &*FirstNewBlock);
2260 
2261  // Now that the function is correct, make it a little bit nicer. In
2262  // particular, move the basic blocks inserted from the end of the function
2263  // into the space made by splitting the source basic block.
2264  Caller->getBasicBlockList().splice(AfterCallBB->getIterator(),
2265  Caller->getBasicBlockList(), FirstNewBlock,
2266  Caller->end());
2267 
2268  // Handle all of the return instructions that we just cloned in, and eliminate
2269  // any users of the original call/invoke instruction.
2270  Type *RTy = CalledFunc->getReturnType();
2271 
2272  PHINode *PHI = nullptr;
2273  if (Returns.size() > 1) {
2274  // The PHI node should go at the front of the new basic block to merge all
2275  // possible incoming values.
2276  if (!TheCall->use_empty()) {
2277  PHI = PHINode::Create(RTy, Returns.size(), TheCall->getName(),
2278  &AfterCallBB->front());
2279  // Anything that used the result of the function call should now use the
2280  // PHI node as their operand.
2281  TheCall->replaceAllUsesWith(PHI);
2282  }
2283 
2284  // Loop over all of the return instructions adding entries to the PHI node
2285  // as appropriate.
2286  if (PHI) {
2287  for (unsigned i = 0, e = Returns.size(); i != e; ++i) {
2288  ReturnInst *RI = Returns[i];
2289  assert(RI->getReturnValue()->getType() == PHI->getType() &&
2290  "Ret value not consistent in function!");
2291  PHI->addIncoming(RI->getReturnValue(), RI->getParent());
2292  }
2293  }
2294 
2295  // Add a branch to the merge points and remove return instructions.
2296  DebugLoc Loc;
2297  for (unsigned i = 0, e = Returns.size(); i != e; ++i) {
2298  ReturnInst *RI = Returns[i];
2299  BranchInst* BI = BranchInst::Create(AfterCallBB, RI);
2300  Loc = RI->getDebugLoc();
2301  BI->setDebugLoc(Loc);
2302  RI->eraseFromParent();
2303  }
2304  // We need to set the debug location to *somewhere* inside the
2305  // inlined function. The line number may be nonsensical, but the
2306  // instruction will at least be associated with the right
2307  // function.
2308  if (CreatedBranchToNormalDest)
2309  CreatedBranchToNormalDest->setDebugLoc(Loc);
2310  } else if (!Returns.empty()) {
2311  // Otherwise, if there is exactly one return value, just replace anything
2312  // using the return value of the call with the computed value.
2313  if (!TheCall->use_empty()) {
2314  if (TheCall == Returns[0]->getReturnValue())
2315  TheCall->replaceAllUsesWith(UndefValue::get(TheCall->getType()));
2316  else
2317  TheCall->replaceAllUsesWith(Returns[0]->getReturnValue());
2318  }
2319 
2320  // Update PHI nodes that use the ReturnBB to use the AfterCallBB.
2321  BasicBlock *ReturnBB = Returns[0]->getParent();
2322  ReturnBB->replaceAllUsesWith(AfterCallBB);
2323 
2324  // Splice the code from the return block into the block that it will return
2325  // to, which contains the code that was after the call.
2326  AfterCallBB->getInstList().splice(AfterCallBB->begin(),
2327  ReturnBB->getInstList());
2328 
2329  if (CreatedBranchToNormalDest)
2330  CreatedBranchToNormalDest->setDebugLoc(Returns[0]->getDebugLoc());
2331 
2332  // Delete the return instruction now and empty ReturnBB now.
2333  Returns[0]->eraseFromParent();
2334  ReturnBB->eraseFromParent();
2335  } else if (!TheCall->use_empty()) {
2336  // No returns, but something is using the return value of the call. Just
2337  // nuke the result.
2338  TheCall->replaceAllUsesWith(UndefValue::get(TheCall->getType()));
2339  }
2340 
2341  // Since we are now done with the Call/Invoke, we can delete it.
2342  TheCall->eraseFromParent();
2343 
2344  // If we inlined any musttail calls and the original return is now
2345  // unreachable, delete it. It can only contain a bitcast and ret.
2346  if (InlinedMustTailCalls && pred_begin(AfterCallBB) == pred_end(AfterCallBB))
2347  AfterCallBB->eraseFromParent();
2348 
2349  // We should always be able to fold the entry block of the function into the
2350  // single predecessor of the block...
2351  assert(cast<BranchInst>(Br)->isUnconditional() && "splitBasicBlock broken!");
2352  BasicBlock *CalleeEntry = cast<BranchInst>(Br)->getSuccessor(0);
2353 
2354  // Splice the code entry block into calling block, right before the
2355  // unconditional branch.
2356  CalleeEntry->replaceAllUsesWith(OrigBB); // Update PHI nodes
2357  OrigBB->getInstList().splice(Br->getIterator(), CalleeEntry->getInstList());
2358 
2359  // Remove the unconditional branch.
2360  OrigBB->getInstList().erase(Br);
2361 
2362  // Now we can remove the CalleeEntry block, which is now empty.
2363  Caller->getBasicBlockList().erase(CalleeEntry);
2364 
2365  // If we inserted a phi node, check to see if it has a single value (e.g. all
2366  // the entries are the same or undef). If so, remove the PHI so it doesn't
2367  // block other optimizations.
2368  if (PHI) {
2369  AssumptionCache *AC =
2370  IFI.GetAssumptionCache ? &(*IFI.GetAssumptionCache)(*Caller) : nullptr;
2371  auto &DL = Caller->getParent()->getDataLayout();
2372  if (Value *V = SimplifyInstruction(PHI, {DL, nullptr, nullptr, AC})) {
2373  PHI->replaceAllUsesWith(V);
2374  PHI->eraseFromParent();
2375  }
2376  }
2377 
2378  return true;
2379 }
bool onlyReadsMemory() const
Determine if the function does not access or only reads memory.
Definition: Function.h:470
Return a value (possibly void), from a function.
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks &#39;this&#39; from the containing basic block and deletes it.
Definition: Instruction.cpp:67
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:110
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:645
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
DILocation * get() const
Get the underlying DILocation.
Definition: DebugLoc.cpp:21
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static cl::opt< bool > EnableNoAliasConversion("enable-noalias-to-md-conversion", cl::init(true), cl::Hidden, cl::desc("Convert noalias attributes to metadata during inlining."))
unsigned getOrEnforceKnownAlignment(Value *V, unsigned PrefAlign, const DataLayout &DL, const Instruction *CxtI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
Try to ensure that the alignment of V is at least PrefAlign bytes.
Definition: Local.cpp:1200
static void updateCallProfile(Function *Callee, const ValueToValueMapTy &VMap, const ProfileCount &CalleeEntryCount, const Instruction *TheCall, ProfileSummaryInfo *PSI, BlockFrequencyInfo *CallerBFI)
Update the branch metadata for cloned call instructions.
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
This class represents an incoming formal argument to a Function.
Definition: Argument.h:29
bool replaceDbgDeclareForAlloca(AllocaInst *AI, Value *NewAllocaAddress, DIBuilder &Builder, bool DerefBefore, int Offset, bool DerefAfter)
Replaces llvm.dbg.declare instruction when the alloca it describes is replaced with a new value...
Definition: Local.cpp:1573
iterator erase(iterator where)
Definition: ilist.h:265
IterTy arg_begin() const
Definition: CallSite.h:579
This class represents lattice values for constants.
Definition: AllocatorList.h:23
CallGraph * CG
If non-null, InlineFunction will update the callgraph to reflect the changes it makes.
Definition: Cloning.h:186
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:65
iterator end()
Definition: Function.h:663
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
an instruction that atomically checks whether a specified value is in a memory location, and, if it is, stores a new value there.
Definition: Instructions.h:528
static DebugLoc appendInlinedAt(DebugLoc DL, DILocation *InlinedAt, LLVMContext &Ctx, DenseMap< const MDNode *, MDNode *> &Cache, bool ReplaceLast=false)
Rebuild the entire inlined-at chain for this instruction so that the top of the chain now is inlined-...
Definition: DebugLoc.cpp:82
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
std::function< AssumptionCache &(Function &)> * GetAssumptionCache
Definition: Cloning.h:187
void removePredecessor(BasicBlock *Pred, bool KeepOneInputPHIs=false)
Notify the BasicBlock that the predecessor Pred is no longer able to reach it.
Definition: BasicBlock.cpp:301
void push_back(const T &Elt)
Definition: SmallVector.h:211
void recalculate(ParentType &Func)
recalculate - compute a dominator tree for the given function
This provides a very simple, boring adaptor for a begin and end iterator into a range type...
unsigned getParamAlignment(unsigned ArgNo) const
Extract the alignment for a call or parameter (0=unknown).
Definition: Function.h:430
Analysis providing profile information.
const CallInst * getTerminatingMustTailCall() const
Returns the call instruction marked &#39;musttail&#39; prior to the terminating return instruction of this ba...
Definition: BasicBlock.cpp:143
This class represents a function call, abstracting a target machine&#39;s calling convention.
void setGC(std::string Str)
Definition: Function.cpp:473
This file contains the declarations for metadata subclasses.
A cache of @llvm.assume calls within a function.
bool isSwiftError() const
Return true if this alloca is used as a swifterror argument to a call.
Definition: Instructions.h:135
uint64_t getFrequency() const
Returns the frequency as a fixpoint number scaled by the entry frequency.
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:709
static void AddAlignmentAssumptions(CallSite CS, InlineFunctionInfo &IFI)
If the inlined function has non-byval align arguments, then add .assume-based alignment assumptions t...
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Definition: CallSite.h:590
arg_iterator arg_end()
Definition: Function.h:685
A debug info location.
Definition: DebugLoc.h:33
Metadata node.
Definition: Metadata.h:863
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1077
F(f)
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1189
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1068
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:534
An instruction for reading from memory.
Definition: Instructions.h:167
FunTy * getCalledFunction() const
Return the function being called if this is a direct call, otherwise return null (if it&#39;s an indirect...
Definition: CallSite.h:111
Optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Definition: CallSite.h:563
InlineResult InlineFunction(CallInst *C, InlineFunctionInfo &IFI, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true)
This function inlines the called function into the basic block of the caller.
static void AddAliasScopeMetadata(CallSite CS, ValueToValueMapTy &VMap, const DataLayout &DL, AAResults *CalleeAAR)
If the inlined function has noalias arguments, then add new alias scopes for each noalias argument...
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:176
an instruction that atomically reads a memory location, combines it with another value, and then stores the result back.
Definition: Instructions.h:691
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:137
Constant * getClause(unsigned Idx) const
Get the value of the clause at index Idx.
void reserve(size_type N)
Definition: SmallVector.h:369
bool isMustTailCall() const
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
Definition: Cloning.h:173
iterator end()
Get an iterator to the end of the SetVector.
Definition: SetVector.h:92
static CallBrInst * Create(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest, ArrayRef< BasicBlock *> IndirectDests, ArrayRef< Value *> Args, const Twine &NameStr, Instruction *InsertBefore=nullptr)
A node in the call graph for a module.
Definition: CallGraph.h:164
Tuple of metadata.
Definition: Metadata.h:1105
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
static void fixupLineNumbers(Function *Fn, Function::iterator FI, Instruction *TheCall, bool CalleeHasDebugInfo)
Update inlined instructions&#39; line numbers to to encode location where these instructions are inlined...
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:343
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:268
Function::ProfileCount ProfileCount
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:221
unsigned getAllocaAddrSpace() const
Definition: DataLayout.h:269
const CallInst * getTerminatingDeoptimizeCall() const
Returns the call instruction calling @llvm.experimental.deoptimize prior to the terminating return in...
Definition: BasicBlock.cpp:174
const DataLayout & getDataLayout() const
Get the data layout for the module&#39;s target platform.
Definition: Module.cpp:369
void setEntryCount(ProfileCount Count, const DenseSet< GlobalValue::GUID > *Imports=nullptr)
Set the entry count for this function.
Definition: Function.cpp:1370
iterator end()
Definition: CallGraph.h:190
static void HandleInlinedLandingPad(InvokeInst *II, BasicBlock *FirstNewBlock, ClonedCodeInfo &InlinedCodeInfo)
If we inlined an invoke site, we need to convert calls in the body of the inlined function into invok...
PointerType * getType() const
Overload to return most specific pointer type.
Definition: Instructions.h:96
TailCallKind getTailCallKind() const
A Use represents the edge between a Value definition and its users.
Definition: Use.h:55
ValTy * getCalledValue() const
Return the pointer to function that is being called.
Definition: CallSite.h:104
uint32_t getTagID() const
Return the tag of this operand bundle as an integer.
Definition: InstrTypes.h:1005
static void updateCallerBFI(BasicBlock *CallSiteBlock, const ValueToValueMapTy &VMap, BlockFrequencyInfo *CallerBFI, BlockFrequencyInfo *CalleeBFI, const BasicBlock &CalleeEntryBlock)
Update the block frequencies of the caller after a callee has been inlined.
ReturnInst * CreateRet(Value *V)
Create a &#39;ret <val>&#39; instruction.
Definition: IRBuilder.h:828
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:742
AttributeSet getRetAttributes() const
The attributes for the ret value are returned.
static void CloneAliasScopeMetadata(CallSite CS, ValueToValueMapTy &VMap)
When inlining a function that contains noalias scope metadata, this metadata needs to be cloned so th...
The only memory references in this function (if it has any) are non-volatile loads from objects point...
void setBlockFreqAndScale(const BasicBlock *ReferenceBB, uint64_t Freq, SmallPtrSetImpl< BasicBlock *> &BlocksToScale)
Set the frequency of ReferenceBB to Freq and scale the frequencies of the blocks in BlocksToScale suc...
unsigned getNumClauses() const
Get the number of clauses for this landing pad.
std::vector< CallRecord > CalledFunctionsVector
Definition: CallGraph.h:171
bool isScopedEHPersonality(EHPersonality Pers)
Returns true if this personality uses scope-style EH IR instructions: catchswitch, catchpad/ret, and cleanuppad/ret.
FunctionModRefBehavior getModRefBehavior(const CallBase *Call)
Return the behavior of the given call site.
void addHandler(BasicBlock *Dest)
Add an entry to the switch instruction...
InstrTy * getInstruction() const
Definition: CallSite.h:96
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
bool doesNotThrow() const
Determine if the call cannot unwind.
Definition: InstrTypes.h:1654
static void UpdateCallGraphAfterInlining(CallSite CS, Function::iterator FirstNewBlock, ValueToValueMapTy &VMap, InlineFunctionInfo &IFI)
Once we have cloned code over from a callee into the caller, update the specified callgraph to reflec...
void CloneAndPruneFunctionInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, bool ModuleLevelChanges, SmallVectorImpl< ReturnInst *> &Returns, const char *NameSuffix="", ClonedCodeInfo *CodeInfo=nullptr, Instruction *TheCall=nullptr)
This works exactly like CloneFunctionInto, except that it does some simple constant prop and DCE on t...
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1767
static void UpdatePHINodes(BasicBlock *OrigBB, BasicBlock *NewBB, ArrayRef< BasicBlock *> Preds, BranchInst *BI, bool HasLoopExit)
Update the PHI nodes in OrigBB to include the values coming from NewBB.
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:244
std::vector< WeakTrackingVH > OperandBundleCallSites
All cloned call sites that have operand bundles attached are appended to this vector.
Definition: Cloning.h:77
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:141
LandingPadInst * getLandingPadInst() const
Get the landingpad instruction from the landing pad block (the unwind destination).
Value * getCalledOperand() const
Definition: InstrTypes.h:1253
LLVMContext & getContext() const
Definition: Metadata.h:923
FunctionModRefBehavior
Summary of how a function affects memory in the program.
AttributeSet getParamAttributes(unsigned ArgNo) const
The attributes for the argument or parameter at the given index are returned.
bool isUsedWithInAlloca() const
Return true if this alloca is used as an inalloca argument to a call.
Definition: Instructions.h:124
iterator find(const KeyT &Val)
Definition: ValueMap.h:161
iterator begin()
Get an iterator to the beginning of the SetVector.
Definition: SetVector.h:82
BlockFrequencyInfo * CalleeBFI
Definition: Cloning.h:189
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:234
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:125
iterator_range< User::op_iterator > arg_operands()
Definition: InstrTypes.h:1210
const std::string & getGC() const
Definition: Function.cpp:468
An instruction for storing to memory.
Definition: Instructions.h:320
bool hasPersonalityFn() const
Check whether this function has a personality function.
Definition: Function.h:707
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:429
bool PointerMayBeCapturedBefore(const Value *V, bool ReturnCaptures, bool StoreCaptures, const Instruction *I, const DominatorTree *DT, bool IncludeI=false, OrderedBasicBlock *OBB=nullptr, unsigned MaxUsesToExplore=DefaultMaxUsesToExplore)
PointerMayBeCapturedBefore - Return true if this pointer value may be captured by the enclosing funct...
bool doesNotThrow() const
Determine if the call cannot unwind.
Definition: CallSite.h:513
Debug location.
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:291
iterator begin()
Definition: Function.h:661
InlineResult is basically true or false.
Definition: InlineCost.h:135
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Definition: Dominators.h:144
Function * getDeclaration(Module *M, ID id, ArrayRef< Type *> Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1022
Value * getOperand(unsigned i) const
Definition: User.h:169
Class to represent pointers.
Definition: DerivedTypes.h:498
Optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Definition: InstrTypes.h:1779
static BasicBlock * HandleCallsInBlockInlinedThroughInvoke(BasicBlock *BB, BasicBlock *UnwindEdge, UnwindDestMemoTy *FuncletUnwindMap=nullptr)
When we inline a basic block into an invoke, we have to turn all of the calls that can throw into inv...
static bool isUsedByLifetimeMarker(Value *V)
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:176
static cl::opt< bool > PreserveAlignmentAssumptions("preserve-alignment-assumptions-during-inlining", cl::init(true), cl::Hidden, cl::desc("Convert align attributes to assumptions during inlining."))
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:140
const BasicBlock & getEntryBlock() const
Definition: Function.h:645
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
static TempMDTuple getTemporary(LLVMContext &Context, ArrayRef< Metadata *> MDs)
Return a temporary node.
Definition: Metadata.h:1152
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata *> MDs)
Definition: Metadata.h:1165
void setCallingConv(CallingConv::ID CC)
Definition: Function.h:216
SmallVector< CallSite, 8 > InlinedCallSites
All of the new call sites inlined into the caller.
Definition: Cloning.h:204
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:168
The landingpad instruction holds all of the information necessary to generate correct exception handl...
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:189
void GetUnderlyingObjects(const Value *V, SmallVectorImpl< const Value *> &Objects, const DataLayout &DL, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to GetUnderlyingObject except that it can look through phi and select instruct...
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
Definition: Instruction.h:318
uint64_t getCount() const
Definition: Function.h:271
unsigned getNumArgOperands() const
Definition: CallSite.h:303
Value * getCalledValue() const
Definition: InstrTypes.h:1257
bool hasName() const
Definition: Value.h:250
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1504
Conditional or Unconditional Branch instruction.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Definition: Constant.h:41
ArrayRef< Use > Inputs
Definition: InstrTypes.h:978
Resume the propagation of an exception.
static MDTuple * getDistinct(LLVMContext &Context, ArrayRef< Metadata *> MDs)
Definition: Metadata.h:1173
Value * getIncomingValueForBlock(const BasicBlock *BB) const
This file contains the declarations for the subclasses of Constant, which represent the different fla...
const Instruction & front() const
Definition: BasicBlock.h:280
AttributeList getAttributes() const
Get the parameter attributes of the call.
Definition: CallSite.h:337
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1121
ValTy * getArgument(unsigned ArgNo) const
Definition: CallSite.h:193
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition: DerivedTypes.h:138
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:370
ProfileCount getEntryCount(bool AllowSynthetic=false) const
Get the entry count for this function.
Definition: Function.cpp:1388
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Definition: Interval.h:112
unsigned getPrefTypeAlignment(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:759
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
void splice(iterator where, iplist_impl &L2)
Definition: ilist.h:327
static Value * getUnwindDestTokenHelper(Instruction *EHPad, UnwindDestMemoTy &MemoMap)
Helper for getUnwindDestToken that does the descendant-ward part of the search.
ProfileSummaryInfo * PSI
Definition: Cloning.h:188
void removeCallEdgeFor(CallBase &Call)
Removes the edge in the node for the specified call site.
Definition: CallGraph.cpp:186
static bool hasLifetimeMarkers(AllocaInst *AI)
void setCallingConv(CallingConv::ID CC)
Definition: InstrTypes.h:1325
The only memory references in this function (if it has any) are non-volatile loads and stores from ob...
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Interval::pred_iterator pred_end(Interval *I)
Definition: Interval.h:115
arg_iterator arg_begin()
Definition: Function.h:676
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:381
self_iterator getIterator()
Definition: ilist_node.h:81
void setTailCallKind(TailCallKind TCK)
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:59
auto remove_if(R &&Range, UnaryPredicate P) -> decltype(adl_begin(Range))
Provide wrappers to std::remove_if which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1225
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:196
static UndefValue * get(Type *T)
Static factory methods - Return an &#39;undef&#39; object of the specified type.
Definition: Constants.cpp:1424
MDNode * uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2)
Compute the union of two access-group lists.
const Constant * stripPointerCasts() const
Definition: Constant.h:177
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs, and aliases.
Definition: Value.cpp:529
This class represents the va_arg llvm instruction, which returns an argument of the specified type gi...
iterator erase(const_iterator CI)
Definition: SmallVector.h:438
Class to represent profile counts.
Definition: Function.h:260
const Value * getArraySize() const
Get the number of elements allocated.
Definition: Instructions.h:92
size_t size() const
Definition: SmallVector.h:52
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:219
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Optional< uint64_t > getProfileCount(const Instruction *CallInst, BlockFrequencyInfo *BFI, bool AllowSynthetic=false)
Returns the profile count for CallInst.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1222
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:105
bool isIdentifiedFunctionLocal(const Value *V)
Return true if V is umabigously identified at the function-level.
iterator end()
Definition: ValueMap.h:141
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
static void HandleByValArgumentInit(Value *Dst, Value *Src, Module *M, BasicBlock *InsertBlock, InlineFunctionInfo &IFI)
const InstListType & getInstList() const
Return the underlying instruction list container.
Definition: BasicBlock.h:333
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:297
Iterator for intrusive lists based on ilist_node.
This is the shared class of boolean and integer constants.
Definition: Constants.h:83
static CatchSwitchInst * Create(Value *ParentPad, BasicBlock *UnwindDest, unsigned NumHandlers, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
iterator end()
Definition: BasicBlock.h:270
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:212
Module.h This file contains the declarations for the Module class.
unsigned getNumOperandBundles() const
Definition: CallSite.h:539
void setBlockFreq(const BasicBlock *BB, uint64_t Freq)
LLVM_NODISCARD T pop_back_val()
Definition: SmallVector.h:374
static Value * getParentPad(Value *EHPad)
Helper for getUnwindDestToken/getUnwindDestTokenHelper.
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition: InstrTypes.h:1814
static cl::opt< bool > NoAliases("riscv-no-aliases", cl::desc("Disable the emission of assembler pseudo instructions"), cl::init(false), cl::Hidden)
static Value * getUnwindDestToken(Instruction *EHPad, UnwindDestMemoTy &MemoMap)
Given an EH pad, find where it unwinds.
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:631
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
std::string utostr(uint64_t X, bool isNeg=false)
Definition: StringExtras.h:223
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1343
BlockFrequency getBlockFreq(const BasicBlock *BB) const
getblockFreq - Return block frequency.
void setOperand(unsigned i, Value *Val)
Definition: User.h:174
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:940
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:163
amdgpu Simplify well known AMD library false FunctionCallee Callee
bool isCleanup() const
Return &#39;true&#39; if this landingpad instruction is a cleanup.
iterator_range< user_iterator > users()
Definition: Value.h:399
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:387
bool ContainsCalls
This is set to true if the cloned code contains a normal call instruction.
Definition: Cloning.h:67
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1183
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
SmallVector< AllocaInst *, 4 > StaticAllocas
InlineFunction fills this in with all static allocas that get copied into the caller.
Definition: Cloning.h:193
The basic data container for the call graph of a Module of IR.
Definition: CallGraph.h:73
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
Definition: Function.h:351
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:321
void registerAssumption(CallInst *CI)
Add an @llvm.assume intrinsic to this function&#39;s cache.
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:55
SmallVector< WeakTrackingVH, 8 > InlinedCalls
InlineFunction fills this in with callsites that were inlined from the callee.
Definition: Cloning.h:197
static InvokeInst * Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef< Value *> Args, const Twine &NameStr, Instruction *InsertBefore=nullptr)
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1321
StringRef getName() const
Return a constant reference to the value&#39;s name.
Definition: Value.cpp:214
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
Definition: InstrTypes.h:1264
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:106
static MDNode * concatenate(MDNode *A, MDNode *B)
Methods for metadata merging.
Definition: Metadata.cpp:895
static void PropagateParallelLoopAccessMetadata(CallSite CS, ValueToValueMapTy &VMap)
When inlining a call site that has !llvm.mem.parallel_loop_access or llvm.access.group metadata...
SymbolTableList< BasicBlock >::iterator eraseFromParent()
Unlink &#39;this&#39; from the containing function and delete it.
Definition: BasicBlock.cpp:114
#define I(x, y, z)
Definition: MD5.cpp:58
bool empty() const
Determine if the SetVector is empty or not.
Definition: SetVector.h:72
static Value * HandleByValArgument(Value *Arg, Instruction *TheCall, const Function *CalledFunc, InlineFunctionInfo &IFI, unsigned ByValAlignment)
When inlining a call site that has a byval argument, we have to make the implicit memcpy explicit by ...
iterator end()
Definition: DenseMap.h:108
This struct can be used to capture information about code being cloned, while it is being cloned...
Definition: Cloning.h:65
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
Definition: InstrTypes.h:1347
unsigned getKnownAlignment(Value *V, const DataLayout &DL, const Instruction *CxtI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
Try to infer an alignment for the specified pointer.
Definition: Local.h:267
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:332
const BasicBlockListType & getBasicBlockList() const
Get the underlying elements of the Function...
Definition: Function.h:638
uint32_t Size
Definition: Profile.cpp:46
static ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition: Constants.cpp:1139
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="")
Split the basic block into two basic blocks at the specified instruction.
Definition: BasicBlock.cpp:407
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
std::vector< CallRecord >::iterator iterator
Definition: CallGraph.h:183
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:171
void updateProfileCallee(Function *Callee, int64_t entryDelta, const ValueMap< const Value *, WeakTrackingVH > *VMap=nullptr)
Updates profile information by adjusting the entry count by adding entryDelta then scaling callsite i...
bool isCallBr() const
Return true if a CallBrInst is enclosed.
Definition: CallSite.h:94
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:205
static bool allocaWouldBeStaticInEntry(const AllocaInst *AI)
Return the result of AI->isStaticAlloca() if AI were moved to the entry block.
BlockFrequencyInfo * CallerBFI
Definition: Cloning.h:189
bool isCall() const
Return true if a CallInst is enclosed.
Definition: CallSite.h:87
size_type count(const KeyT &Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: ValueMap.h:157
bool isByValArgument(unsigned ArgNo) const
Determine whether this argument is passed by value.
Definition: CallSite.h:602
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool ContainsDynamicAllocas
This is set to true if the cloned code contains a &#39;dynamic&#39; alloca.
Definition: Cloning.h:72
const BasicBlock & front() const
Definition: Function.h:668
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
BasicBlock * getUnwindDest() const
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:565
LLVM Value Representation.
Definition: Value.h:72
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1306
uint64_t getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type...
Definition: DataLayout.h:444
A vector that has set insertion semantics.
Definition: SetVector.h:40
static CleanupReturnInst * Create(Value *CleanupPad, BasicBlock *UnwindBB=nullptr, Instruction *InsertBefore=nullptr)
AttributeSet getFnAttributes() const
The function attributes are returned.
BasicBlock * changeToInvokeAndSplitBasicBlock(CallInst *CI, BasicBlock *UnwindEdge)
Convert the CallInst to InvokeInst with the specified unwind edge basic block.
Definition: Local.cpp:2006
Invoke instruction.
bool isEHPad() const
Return true if the instruction is a variety of EH-block.
Definition: Instruction.h:583
iterator begin()
Definition: CallGraph.h:189
OperandBundleUse getOperandBundleAt(unsigned Index) const
Definition: CallSite.h:559
void setPersonalityFn(Constant *Fn)
Definition: Function.cpp:1311
static void HandleInlinedEHPad(InvokeInst *II, BasicBlock *FirstNewBlock, ClonedCodeInfo &InlinedCodeInfo)
If we inlined an invoke site, we need to convert calls in the body of the inlined function into invok...
void pop_back()
Definition: ilist.h:316
unsigned getNumOperands() const
Return number of MDNode operands.
Definition: Metadata.h:1074
bool isEmpty() const
Return true if there are no attributes.
Definition: Attributes.h:655
Value * SimplifyInstruction(Instruction *I, const SimplifyQuery &Q, OptimizationRemarkEmitter *ORE=nullptr)
See if we can compute a simplified version of this instruction.
Root of the metadata hierarchy.
Definition: Metadata.h:57
unsigned changeToUnreachable(Instruction *I, bool UseLLVMTrap, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...
Definition: Local.cpp:1936
void setDoesNotThrow()
Definition: InstrTypes.h:1655
bool use_empty() const
Definition: Value.h:322
iterator begin()
Definition: ValueMap.h:140
Type * getElementType() const
Definition: DerivedTypes.h:517
ValTy * getArgOperand(unsigned i) const
Definition: CallSite.h:307
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute >> Attrs)
Create an AttributeList with the specified parameters in it.
Definition: Attributes.cpp:874
iterator_range< arg_iterator > args()
Definition: Function.h:694
bool hasOperandBundles() const
Definition: CallSite.h:543
void addCalledFunction(CallBase *Call, CallGraphNode *M)
Adds a function to the list of functions called by this one.
Definition: CallGraph.h:232
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:43
const BasicBlock * getParent() const
Definition: Instruction.h:66
an instruction to allocate memory on the stack
Definition: Instructions.h:59
bool mayReadOrWriteMemory() const
Return true if this instruction may read or write memory.
Definition: Instruction.h:532
FunTy * getCaller() const
Return the caller function for this call site.
Definition: CallSite.h:275