LLVM 17.0.0git
InlineFunction.cpp
Go to the documentation of this file.
1//===- InlineFunction.cpp - Code to perform function inlining -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements inlining of a function into a call site, resolving
10// parameters and the return value as appropriate.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ADT/DenseMap.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SetVector.h"
33#include "llvm/IR/Argument.h"
34#include "llvm/IR/BasicBlock.h"
35#include "llvm/IR/CFG.h"
36#include "llvm/IR/Constant.h"
37#include "llvm/IR/Constants.h"
38#include "llvm/IR/DataLayout.h"
39#include "llvm/IR/DebugInfo.h"
41#include "llvm/IR/DebugLoc.h"
43#include "llvm/IR/Dominators.h"
45#include "llvm/IR/Function.h"
46#include "llvm/IR/IRBuilder.h"
47#include "llvm/IR/InlineAsm.h"
48#include "llvm/IR/InstrTypes.h"
49#include "llvm/IR/Instruction.h"
52#include "llvm/IR/Intrinsics.h"
53#include "llvm/IR/LLVMContext.h"
54#include "llvm/IR/MDBuilder.h"
55#include "llvm/IR/Metadata.h"
56#include "llvm/IR/Module.h"
57#include "llvm/IR/Type.h"
58#include "llvm/IR/User.h"
59#include "llvm/IR/Value.h"
67#include <algorithm>
68#include <cassert>
69#include <cstdint>
70#include <iterator>
71#include <limits>
72#include <optional>
73#include <string>
74#include <utility>
75#include <vector>
76
77#define DEBUG_TYPE "inline-function"
78
79using namespace llvm;
80using namespace llvm::memprof;
82
83static cl::opt<bool>
84EnableNoAliasConversion("enable-noalias-to-md-conversion", cl::init(true),
86 cl::desc("Convert noalias attributes to metadata during inlining."));
87
88static cl::opt<bool>
89 UseNoAliasIntrinsic("use-noalias-intrinsic-during-inlining", cl::Hidden,
90 cl::init(true),
91 cl::desc("Use the llvm.experimental.noalias.scope.decl "
92 "intrinsic during inlining."));
93
94// Disabled by default, because the added alignment assumptions may increase
95// compile-time and block optimizations. This option is not suitable for use
96// with frontends that emit comprehensive parameter alignment annotations.
97static cl::opt<bool>
98PreserveAlignmentAssumptions("preserve-alignment-assumptions-during-inlining",
99 cl::init(false), cl::Hidden,
100 cl::desc("Convert align attributes to assumptions during inlining."));
101
103 "update-return-attrs", cl::init(true), cl::Hidden,
104 cl::desc("Update return attributes on calls within inlined body"));
105
107 "max-inst-checked-for-throw-during-inlining", cl::Hidden,
108 cl::desc("the maximum number of instructions analyzed for may throw during "
109 "attribute inference in inlined body"),
110 cl::init(4));
111
112namespace {
113
114 /// A class for recording information about inlining a landing pad.
115 class LandingPadInliningInfo {
116 /// Destination of the invoke's unwind.
117 BasicBlock *OuterResumeDest;
118
119 /// Destination for the callee's resume.
120 BasicBlock *InnerResumeDest = nullptr;
121
122 /// LandingPadInst associated with the invoke.
123 LandingPadInst *CallerLPad = nullptr;
124
125 /// PHI for EH values from landingpad insts.
126 PHINode *InnerEHValuesPHI = nullptr;
127
128 SmallVector<Value*, 8> UnwindDestPHIValues;
129
130 public:
131 LandingPadInliningInfo(InvokeInst *II)
132 : OuterResumeDest(II->getUnwindDest()) {
133 // If there are PHI nodes in the unwind destination block, we need to keep
134 // track of which values came into them from the invoke before removing
135 // the edge from this block.
136 BasicBlock *InvokeBB = II->getParent();
137 BasicBlock::iterator I = OuterResumeDest->begin();
138 for (; isa<PHINode>(I); ++I) {
139 // Save the value to use for this edge.
140 PHINode *PHI = cast<PHINode>(I);
141 UnwindDestPHIValues.push_back(PHI->getIncomingValueForBlock(InvokeBB));
142 }
143
144 CallerLPad = cast<LandingPadInst>(I);
145 }
146
147 /// The outer unwind destination is the target of
148 /// unwind edges introduced for calls within the inlined function.
149 BasicBlock *getOuterResumeDest() const {
150 return OuterResumeDest;
151 }
152
153 BasicBlock *getInnerResumeDest();
154
155 LandingPadInst *getLandingPadInst() const { return CallerLPad; }
156
157 /// Forward the 'resume' instruction to the caller's landing pad block.
158 /// When the landing pad block has only one predecessor, this is
159 /// a simple branch. When there is more than one predecessor, we need to
160 /// split the landing pad block after the landingpad instruction and jump
161 /// to there.
162 void forwardResume(ResumeInst *RI,
164
165 /// Add incoming-PHI values to the unwind destination block for the given
166 /// basic block, using the values for the original invoke's source block.
167 void addIncomingPHIValuesFor(BasicBlock *BB) const {
168 addIncomingPHIValuesForInto(BB, OuterResumeDest);
169 }
170
171 void addIncomingPHIValuesForInto(BasicBlock *src, BasicBlock *dest) const {
172 BasicBlock::iterator I = dest->begin();
173 for (unsigned i = 0, e = UnwindDestPHIValues.size(); i != e; ++i, ++I) {
174 PHINode *phi = cast<PHINode>(I);
175 phi->addIncoming(UnwindDestPHIValues[i], src);
176 }
177 }
178 };
179
180} // end anonymous namespace
181
182/// Get or create a target for the branch from ResumeInsts.
183BasicBlock *LandingPadInliningInfo::getInnerResumeDest() {
184 if (InnerResumeDest) return InnerResumeDest;
185
186 // Split the landing pad.
187 BasicBlock::iterator SplitPoint = ++CallerLPad->getIterator();
188 InnerResumeDest =
189 OuterResumeDest->splitBasicBlock(SplitPoint,
190 OuterResumeDest->getName() + ".body");
191
192 // The number of incoming edges we expect to the inner landing pad.
193 const unsigned PHICapacity = 2;
194
195 // Create corresponding new PHIs for all the PHIs in the outer landing pad.
196 Instruction *InsertPoint = &InnerResumeDest->front();
197 BasicBlock::iterator I = OuterResumeDest->begin();
198 for (unsigned i = 0, e = UnwindDestPHIValues.size(); i != e; ++i, ++I) {
199 PHINode *OuterPHI = cast<PHINode>(I);
200 PHINode *InnerPHI = PHINode::Create(OuterPHI->getType(), PHICapacity,
201 OuterPHI->getName() + ".lpad-body",
202 InsertPoint);
203 OuterPHI->replaceAllUsesWith(InnerPHI);
204 InnerPHI->addIncoming(OuterPHI, OuterResumeDest);
205 }
206
207 // Create a PHI for the exception values.
208 InnerEHValuesPHI = PHINode::Create(CallerLPad->getType(), PHICapacity,
209 "eh.lpad-body", InsertPoint);
210 CallerLPad->replaceAllUsesWith(InnerEHValuesPHI);
211 InnerEHValuesPHI->addIncoming(CallerLPad, OuterResumeDest);
212
213 // All done.
214 return InnerResumeDest;
215}
216
217/// Forward the 'resume' instruction to the caller's landing pad block.
218/// When the landing pad block has only one predecessor, this is a simple
219/// branch. When there is more than one predecessor, we need to split the
220/// landing pad block after the landingpad instruction and jump to there.
221void LandingPadInliningInfo::forwardResume(
223 BasicBlock *Dest = getInnerResumeDest();
224 BasicBlock *Src = RI->getParent();
225
226 BranchInst::Create(Dest, Src);
227
228 // Update the PHIs in the destination. They were inserted in an order which
229 // makes this work.
230 addIncomingPHIValuesForInto(Src, Dest);
231
232 InnerEHValuesPHI->addIncoming(RI->getOperand(0), Src);
233 RI->eraseFromParent();
234}
235
236/// Helper for getUnwindDestToken/getUnwindDestTokenHelper.
237static Value *getParentPad(Value *EHPad) {
238 if (auto *FPI = dyn_cast<FuncletPadInst>(EHPad))
239 return FPI->getParentPad();
240 return cast<CatchSwitchInst>(EHPad)->getParentPad();
241}
242
244
245/// Helper for getUnwindDestToken that does the descendant-ward part of
246/// the search.
248 UnwindDestMemoTy &MemoMap) {
249 SmallVector<Instruction *, 8> Worklist(1, EHPad);
250
251 while (!Worklist.empty()) {
252 Instruction *CurrentPad = Worklist.pop_back_val();
253 // We only put pads on the worklist that aren't in the MemoMap. When
254 // we find an unwind dest for a pad we may update its ancestors, but
255 // the queue only ever contains uncles/great-uncles/etc. of CurrentPad,
256 // so they should never get updated while queued on the worklist.
257 assert(!MemoMap.count(CurrentPad));
258 Value *UnwindDestToken = nullptr;
259 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(CurrentPad)) {
260 if (CatchSwitch->hasUnwindDest()) {
261 UnwindDestToken = CatchSwitch->getUnwindDest()->getFirstNonPHI();
262 } else {
263 // Catchswitch doesn't have a 'nounwind' variant, and one might be
264 // annotated as "unwinds to caller" when really it's nounwind (see
265 // e.g. SimplifyCFGOpt::SimplifyUnreachable), so we can't infer the
266 // parent's unwind dest from this. We can check its catchpads'
267 // descendants, since they might include a cleanuppad with an
268 // "unwinds to caller" cleanupret, which can be trusted.
269 for (auto HI = CatchSwitch->handler_begin(),
270 HE = CatchSwitch->handler_end();
271 HI != HE && !UnwindDestToken; ++HI) {
272 BasicBlock *HandlerBlock = *HI;
273 auto *CatchPad = cast<CatchPadInst>(HandlerBlock->getFirstNonPHI());
274 for (User *Child : CatchPad->users()) {
275 // Intentionally ignore invokes here -- since the catchswitch is
276 // marked "unwind to caller", it would be a verifier error if it
277 // contained an invoke which unwinds out of it, so any invoke we'd
278 // encounter must unwind to some child of the catch.
279 if (!isa<CleanupPadInst>(Child) && !isa<CatchSwitchInst>(Child))
280 continue;
281
282 Instruction *ChildPad = cast<Instruction>(Child);
283 auto Memo = MemoMap.find(ChildPad);
284 if (Memo == MemoMap.end()) {
285 // Haven't figured out this child pad yet; queue it.
286 Worklist.push_back(ChildPad);
287 continue;
288 }
289 // We've already checked this child, but might have found that
290 // it offers no proof either way.
291 Value *ChildUnwindDestToken = Memo->second;
292 if (!ChildUnwindDestToken)
293 continue;
294 // We already know the child's unwind dest, which can either
295 // be ConstantTokenNone to indicate unwind to caller, or can
296 // be another child of the catchpad. Only the former indicates
297 // the unwind dest of the catchswitch.
298 if (isa<ConstantTokenNone>(ChildUnwindDestToken)) {
299 UnwindDestToken = ChildUnwindDestToken;
300 break;
301 }
302 assert(getParentPad(ChildUnwindDestToken) == CatchPad);
303 }
304 }
305 }
306 } else {
307 auto *CleanupPad = cast<CleanupPadInst>(CurrentPad);
308 for (User *U : CleanupPad->users()) {
309 if (auto *CleanupRet = dyn_cast<CleanupReturnInst>(U)) {
310 if (BasicBlock *RetUnwindDest = CleanupRet->getUnwindDest())
311 UnwindDestToken = RetUnwindDest->getFirstNonPHI();
312 else
313 UnwindDestToken = ConstantTokenNone::get(CleanupPad->getContext());
314 break;
315 }
316 Value *ChildUnwindDestToken;
317 if (auto *Invoke = dyn_cast<InvokeInst>(U)) {
318 ChildUnwindDestToken = Invoke->getUnwindDest()->getFirstNonPHI();
319 } else if (isa<CleanupPadInst>(U) || isa<CatchSwitchInst>(U)) {
320 Instruction *ChildPad = cast<Instruction>(U);
321 auto Memo = MemoMap.find(ChildPad);
322 if (Memo == MemoMap.end()) {
323 // Haven't resolved this child yet; queue it and keep searching.
324 Worklist.push_back(ChildPad);
325 continue;
326 }
327 // We've checked this child, but still need to ignore it if it
328 // had no proof either way.
329 ChildUnwindDestToken = Memo->second;
330 if (!ChildUnwindDestToken)
331 continue;
332 } else {
333 // Not a relevant user of the cleanuppad
334 continue;
335 }
336 // In a well-formed program, the child/invoke must either unwind to
337 // an(other) child of the cleanup, or exit the cleanup. In the
338 // first case, continue searching.
339 if (isa<Instruction>(ChildUnwindDestToken) &&
340 getParentPad(ChildUnwindDestToken) == CleanupPad)
341 continue;
342 UnwindDestToken = ChildUnwindDestToken;
343 break;
344 }
345 }
346 // If we haven't found an unwind dest for CurrentPad, we may have queued its
347 // children, so move on to the next in the worklist.
348 if (!UnwindDestToken)
349 continue;
350
351 // Now we know that CurrentPad unwinds to UnwindDestToken. It also exits
352 // any ancestors of CurrentPad up to but not including UnwindDestToken's
353 // parent pad. Record this in the memo map, and check to see if the
354 // original EHPad being queried is one of the ones exited.
355 Value *UnwindParent;
356 if (auto *UnwindPad = dyn_cast<Instruction>(UnwindDestToken))
357 UnwindParent = getParentPad(UnwindPad);
358 else
359 UnwindParent = nullptr;
360 bool ExitedOriginalPad = false;
361 for (Instruction *ExitedPad = CurrentPad;
362 ExitedPad && ExitedPad != UnwindParent;
363 ExitedPad = dyn_cast<Instruction>(getParentPad(ExitedPad))) {
364 // Skip over catchpads since they just follow their catchswitches.
365 if (isa<CatchPadInst>(ExitedPad))
366 continue;
367 MemoMap[ExitedPad] = UnwindDestToken;
368 ExitedOriginalPad |= (ExitedPad == EHPad);
369 }
370
371 if (ExitedOriginalPad)
372 return UnwindDestToken;
373
374 // Continue the search.
375 }
376
377 // No definitive information is contained within this funclet.
378 return nullptr;
379}
380
381/// Given an EH pad, find where it unwinds. If it unwinds to an EH pad,
382/// return that pad instruction. If it unwinds to caller, return
383/// ConstantTokenNone. If it does not have a definitive unwind destination,
384/// return nullptr.
385///
386/// This routine gets invoked for calls in funclets in inlinees when inlining
387/// an invoke. Since many funclets don't have calls inside them, it's queried
388/// on-demand rather than building a map of pads to unwind dests up front.
389/// Determining a funclet's unwind dest may require recursively searching its
390/// descendants, and also ancestors and cousins if the descendants don't provide
391/// an answer. Since most funclets will have their unwind dest immediately
392/// available as the unwind dest of a catchswitch or cleanupret, this routine
393/// searches top-down from the given pad and then up. To avoid worst-case
394/// quadratic run-time given that approach, it uses a memo map to avoid
395/// re-processing funclet trees. The callers that rewrite the IR as they go
396/// take advantage of this, for correctness, by checking/forcing rewritten
397/// pads' entries to match the original callee view.
399 UnwindDestMemoTy &MemoMap) {
400 // Catchpads unwind to the same place as their catchswitch;
401 // redirct any queries on catchpads so the code below can
402 // deal with just catchswitches and cleanuppads.
403 if (auto *CPI = dyn_cast<CatchPadInst>(EHPad))
404 EHPad = CPI->getCatchSwitch();
405
406 // Check if we've already determined the unwind dest for this pad.
407 auto Memo = MemoMap.find(EHPad);
408 if (Memo != MemoMap.end())
409 return Memo->second;
410
411 // Search EHPad and, if necessary, its descendants.
412 Value *UnwindDestToken = getUnwindDestTokenHelper(EHPad, MemoMap);
413 assert((UnwindDestToken == nullptr) != (MemoMap.count(EHPad) != 0));
414 if (UnwindDestToken)
415 return UnwindDestToken;
416
417 // No information is available for this EHPad from itself or any of its
418 // descendants. An unwind all the way out to a pad in the caller would
419 // need also to agree with the unwind dest of the parent funclet, so
420 // search up the chain to try to find a funclet with information. Put
421 // null entries in the memo map to avoid re-processing as we go up.
422 MemoMap[EHPad] = nullptr;
423#ifndef NDEBUG
425 TempMemos.insert(EHPad);
426#endif
427 Instruction *LastUselessPad = EHPad;
428 Value *AncestorToken;
429 for (AncestorToken = getParentPad(EHPad);
430 auto *AncestorPad = dyn_cast<Instruction>(AncestorToken);
431 AncestorToken = getParentPad(AncestorToken)) {
432 // Skip over catchpads since they just follow their catchswitches.
433 if (isa<CatchPadInst>(AncestorPad))
434 continue;
435 // If the MemoMap had an entry mapping AncestorPad to nullptr, since we
436 // haven't yet called getUnwindDestTokenHelper for AncestorPad in this
437 // call to getUnwindDestToken, that would mean that AncestorPad had no
438 // information in itself, its descendants, or its ancestors. If that
439 // were the case, then we should also have recorded the lack of information
440 // for the descendant that we're coming from. So assert that we don't
441 // find a null entry in the MemoMap for AncestorPad.
442 assert(!MemoMap.count(AncestorPad) || MemoMap[AncestorPad]);
443 auto AncestorMemo = MemoMap.find(AncestorPad);
444 if (AncestorMemo == MemoMap.end()) {
445 UnwindDestToken = getUnwindDestTokenHelper(AncestorPad, MemoMap);
446 } else {
447 UnwindDestToken = AncestorMemo->second;
448 }
449 if (UnwindDestToken)
450 break;
451 LastUselessPad = AncestorPad;
452 MemoMap[LastUselessPad] = nullptr;
453#ifndef NDEBUG
454 TempMemos.insert(LastUselessPad);
455#endif
456 }
457
458 // We know that getUnwindDestTokenHelper was called on LastUselessPad and
459 // returned nullptr (and likewise for EHPad and any of its ancestors up to
460 // LastUselessPad), so LastUselessPad has no information from below. Since
461 // getUnwindDestTokenHelper must investigate all downward paths through
462 // no-information nodes to prove that a node has no information like this,
463 // and since any time it finds information it records it in the MemoMap for
464 // not just the immediately-containing funclet but also any ancestors also
465 // exited, it must be the case that, walking downward from LastUselessPad,
466 // visiting just those nodes which have not been mapped to an unwind dest
467 // by getUnwindDestTokenHelper (the nullptr TempMemos notwithstanding, since
468 // they are just used to keep getUnwindDestTokenHelper from repeating work),
469 // any node visited must have been exhaustively searched with no information
470 // for it found.
471 SmallVector<Instruction *, 8> Worklist(1, LastUselessPad);
472 while (!Worklist.empty()) {
473 Instruction *UselessPad = Worklist.pop_back_val();
474 auto Memo = MemoMap.find(UselessPad);
475 if (Memo != MemoMap.end() && Memo->second) {
476 // Here the name 'UselessPad' is a bit of a misnomer, because we've found
477 // that it is a funclet that does have information about unwinding to
478 // a particular destination; its parent was a useless pad.
479 // Since its parent has no information, the unwind edge must not escape
480 // the parent, and must target a sibling of this pad. This local unwind
481 // gives us no information about EHPad. Leave it and the subtree rooted
482 // at it alone.
483 assert(getParentPad(Memo->second) == getParentPad(UselessPad));
484 continue;
485 }
486 // We know we don't have information for UselesPad. If it has an entry in
487 // the MemoMap (mapping it to nullptr), it must be one of the TempMemos
488 // added on this invocation of getUnwindDestToken; if a previous invocation
489 // recorded nullptr, it would have had to prove that the ancestors of
490 // UselessPad, which include LastUselessPad, had no information, and that
491 // in turn would have required proving that the descendants of
492 // LastUselesPad, which include EHPad, have no information about
493 // LastUselessPad, which would imply that EHPad was mapped to nullptr in
494 // the MemoMap on that invocation, which isn't the case if we got here.
495 assert(!MemoMap.count(UselessPad) || TempMemos.count(UselessPad));
496 // Assert as we enumerate users that 'UselessPad' doesn't have any unwind
497 // information that we'd be contradicting by making a map entry for it
498 // (which is something that getUnwindDestTokenHelper must have proved for
499 // us to get here). Just assert on is direct users here; the checks in
500 // this downward walk at its descendants will verify that they don't have
501 // any unwind edges that exit 'UselessPad' either (i.e. they either have no
502 // unwind edges or unwind to a sibling).
503 MemoMap[UselessPad] = UnwindDestToken;
504 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(UselessPad)) {
505 assert(CatchSwitch->getUnwindDest() == nullptr && "Expected useless pad");
506 for (BasicBlock *HandlerBlock : CatchSwitch->handlers()) {
507 auto *CatchPad = HandlerBlock->getFirstNonPHI();
508 for (User *U : CatchPad->users()) {
509 assert(
510 (!isa<InvokeInst>(U) ||
512 cast<InvokeInst>(U)->getUnwindDest()->getFirstNonPHI()) ==
513 CatchPad)) &&
514 "Expected useless pad");
515 if (isa<CatchSwitchInst>(U) || isa<CleanupPadInst>(U))
516 Worklist.push_back(cast<Instruction>(U));
517 }
518 }
519 } else {
520 assert(isa<CleanupPadInst>(UselessPad));
521 for (User *U : UselessPad->users()) {
522 assert(!isa<CleanupReturnInst>(U) && "Expected useless pad");
523 assert((!isa<InvokeInst>(U) ||
525 cast<InvokeInst>(U)->getUnwindDest()->getFirstNonPHI()) ==
526 UselessPad)) &&
527 "Expected useless pad");
528 if (isa<CatchSwitchInst>(U) || isa<CleanupPadInst>(U))
529 Worklist.push_back(cast<Instruction>(U));
530 }
531 }
532 }
533
534 return UnwindDestToken;
535}
536
537/// When we inline a basic block into an invoke,
538/// we have to turn all of the calls that can throw into invokes.
539/// This function analyze BB to see if there are any calls, and if so,
540/// it rewrites them to be invokes that jump to InvokeDest and fills in the PHI
541/// nodes in that block with the values specified in InvokeDestPHIValues.
543 BasicBlock *BB, BasicBlock *UnwindEdge,
544 UnwindDestMemoTy *FuncletUnwindMap = nullptr) {
546 // We only need to check for function calls: inlined invoke
547 // instructions require no special handling.
548 CallInst *CI = dyn_cast<CallInst>(&I);
549
550 if (!CI || CI->doesNotThrow())
551 continue;
552
553 // We do not need to (and in fact, cannot) convert possibly throwing calls
554 // to @llvm.experimental_deoptimize (resp. @llvm.experimental.guard) into
555 // invokes. The caller's "segment" of the deoptimization continuation
556 // attached to the newly inlined @llvm.experimental_deoptimize
557 // (resp. @llvm.experimental.guard) call should contain the exception
558 // handling logic, if any.
559 if (auto *F = CI->getCalledFunction())
560 if (F->getIntrinsicID() == Intrinsic::experimental_deoptimize ||
561 F->getIntrinsicID() == Intrinsic::experimental_guard)
562 continue;
563
564 if (auto FuncletBundle = CI->getOperandBundle(LLVMContext::OB_funclet)) {
565 // This call is nested inside a funclet. If that funclet has an unwind
566 // destination within the inlinee, then unwinding out of this call would
567 // be UB. Rewriting this call to an invoke which targets the inlined
568 // invoke's unwind dest would give the call's parent funclet multiple
569 // unwind destinations, which is something that subsequent EH table
570 // generation can't handle and that the veirifer rejects. So when we
571 // see such a call, leave it as a call.
572 auto *FuncletPad = cast<Instruction>(FuncletBundle->Inputs[0]);
573 Value *UnwindDestToken =
574 getUnwindDestToken(FuncletPad, *FuncletUnwindMap);
575 if (UnwindDestToken && !isa<ConstantTokenNone>(UnwindDestToken))
576 continue;
577#ifndef NDEBUG
578 Instruction *MemoKey;
579 if (auto *CatchPad = dyn_cast<CatchPadInst>(FuncletPad))
580 MemoKey = CatchPad->getCatchSwitch();
581 else
582 MemoKey = FuncletPad;
583 assert(FuncletUnwindMap->count(MemoKey) &&
584 (*FuncletUnwindMap)[MemoKey] == UnwindDestToken &&
585 "must get memoized to avoid confusing later searches");
586#endif // NDEBUG
587 }
588
589 changeToInvokeAndSplitBasicBlock(CI, UnwindEdge);
590 return BB;
591 }
592 return nullptr;
593}
594
595/// If we inlined an invoke site, we need to convert calls
596/// in the body of the inlined function into invokes.
597///
598/// II is the invoke instruction being inlined. FirstNewBlock is the first
599/// block of the inlined code (the last block is the end of the function),
600/// and InlineCodeInfo is information about the code that got inlined.
601static void HandleInlinedLandingPad(InvokeInst *II, BasicBlock *FirstNewBlock,
602 ClonedCodeInfo &InlinedCodeInfo) {
603 BasicBlock *InvokeDest = II->getUnwindDest();
604
605 Function *Caller = FirstNewBlock->getParent();
606
607 // The inlined code is currently at the end of the function, scan from the
608 // start of the inlined code to its end, checking for stuff we need to
609 // rewrite.
610 LandingPadInliningInfo Invoke(II);
611
612 // Get all of the inlined landing pad instructions.
614 for (Function::iterator I = FirstNewBlock->getIterator(), E = Caller->end();
615 I != E; ++I)
616 if (InvokeInst *II = dyn_cast<InvokeInst>(I->getTerminator()))
617 InlinedLPads.insert(II->getLandingPadInst());
618
619 // Append the clauses from the outer landing pad instruction into the inlined
620 // landing pad instructions.
621 LandingPadInst *OuterLPad = Invoke.getLandingPadInst();
622 for (LandingPadInst *InlinedLPad : InlinedLPads) {
623 unsigned OuterNum = OuterLPad->getNumClauses();
624 InlinedLPad->reserveClauses(OuterNum);
625 for (unsigned OuterIdx = 0; OuterIdx != OuterNum; ++OuterIdx)
626 InlinedLPad->addClause(OuterLPad->getClause(OuterIdx));
627 if (OuterLPad->isCleanup())
628 InlinedLPad->setCleanup(true);
629 }
630
631 for (Function::iterator BB = FirstNewBlock->getIterator(), E = Caller->end();
632 BB != E; ++BB) {
633 if (InlinedCodeInfo.ContainsCalls)
635 &*BB, Invoke.getOuterResumeDest()))
636 // Update any PHI nodes in the exceptional block to indicate that there
637 // is now a new entry in them.
638 Invoke.addIncomingPHIValuesFor(NewBB);
639
640 // Forward any resumes that are remaining here.
641 if (ResumeInst *RI = dyn_cast<ResumeInst>(BB->getTerminator()))
642 Invoke.forwardResume(RI, InlinedLPads);
643 }
644
645 // Now that everything is happy, we have one final detail. The PHI nodes in
646 // the exception destination block still have entries due to the original
647 // invoke instruction. Eliminate these entries (which might even delete the
648 // PHI node) now.
649 InvokeDest->removePredecessor(II->getParent());
650}
651
652/// If we inlined an invoke site, we need to convert calls
653/// in the body of the inlined function into invokes.
654///
655/// II is the invoke instruction being inlined. FirstNewBlock is the first
656/// block of the inlined code (the last block is the end of the function),
657/// and InlineCodeInfo is information about the code that got inlined.
658static void HandleInlinedEHPad(InvokeInst *II, BasicBlock *FirstNewBlock,
659 ClonedCodeInfo &InlinedCodeInfo) {
660 BasicBlock *UnwindDest = II->getUnwindDest();
661 Function *Caller = FirstNewBlock->getParent();
662
663 assert(UnwindDest->getFirstNonPHI()->isEHPad() && "unexpected BasicBlock!");
664
665 // If there are PHI nodes in the unwind destination block, we need to keep
666 // track of which values came into them from the invoke before removing the
667 // edge from this block.
668 SmallVector<Value *, 8> UnwindDestPHIValues;
669 BasicBlock *InvokeBB = II->getParent();
670 for (PHINode &PHI : UnwindDest->phis()) {
671 // Save the value to use for this edge.
672 UnwindDestPHIValues.push_back(PHI.getIncomingValueForBlock(InvokeBB));
673 }
674
675 // Add incoming-PHI values to the unwind destination block for the given basic
676 // block, using the values for the original invoke's source block.
677 auto UpdatePHINodes = [&](BasicBlock *Src) {
678 BasicBlock::iterator I = UnwindDest->begin();
679 for (Value *V : UnwindDestPHIValues) {
680 PHINode *PHI = cast<PHINode>(I);
681 PHI->addIncoming(V, Src);
682 ++I;
683 }
684 };
685
686 // This connects all the instructions which 'unwind to caller' to the invoke
687 // destination.
688 UnwindDestMemoTy FuncletUnwindMap;
689 for (Function::iterator BB = FirstNewBlock->getIterator(), E = Caller->end();
690 BB != E; ++BB) {
691 if (auto *CRI = dyn_cast<CleanupReturnInst>(BB->getTerminator())) {
692 if (CRI->unwindsToCaller()) {
693 auto *CleanupPad = CRI->getCleanupPad();
694 CleanupReturnInst::Create(CleanupPad, UnwindDest, CRI);
695 CRI->eraseFromParent();
696 UpdatePHINodes(&*BB);
697 // Finding a cleanupret with an unwind destination would confuse
698 // subsequent calls to getUnwindDestToken, so map the cleanuppad
699 // to short-circuit any such calls and recognize this as an "unwind
700 // to caller" cleanup.
701 assert(!FuncletUnwindMap.count(CleanupPad) ||
702 isa<ConstantTokenNone>(FuncletUnwindMap[CleanupPad]));
703 FuncletUnwindMap[CleanupPad] =
704 ConstantTokenNone::get(Caller->getContext());
705 }
706 }
707
708 Instruction *I = BB->getFirstNonPHI();
709 if (!I->isEHPad())
710 continue;
711
712 Instruction *Replacement = nullptr;
713 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(I)) {
714 if (CatchSwitch->unwindsToCaller()) {
715 Value *UnwindDestToken;
716 if (auto *ParentPad =
717 dyn_cast<Instruction>(CatchSwitch->getParentPad())) {
718 // This catchswitch is nested inside another funclet. If that
719 // funclet has an unwind destination within the inlinee, then
720 // unwinding out of this catchswitch would be UB. Rewriting this
721 // catchswitch to unwind to the inlined invoke's unwind dest would
722 // give the parent funclet multiple unwind destinations, which is
723 // something that subsequent EH table generation can't handle and
724 // that the veirifer rejects. So when we see such a call, leave it
725 // as "unwind to caller".
726 UnwindDestToken = getUnwindDestToken(ParentPad, FuncletUnwindMap);
727 if (UnwindDestToken && !isa<ConstantTokenNone>(UnwindDestToken))
728 continue;
729 } else {
730 // This catchswitch has no parent to inherit constraints from, and
731 // none of its descendants can have an unwind edge that exits it and
732 // targets another funclet in the inlinee. It may or may not have a
733 // descendant that definitively has an unwind to caller. In either
734 // case, we'll have to assume that any unwinds out of it may need to
735 // be routed to the caller, so treat it as though it has a definitive
736 // unwind to caller.
737 UnwindDestToken = ConstantTokenNone::get(Caller->getContext());
738 }
739 auto *NewCatchSwitch = CatchSwitchInst::Create(
740 CatchSwitch->getParentPad(), UnwindDest,
741 CatchSwitch->getNumHandlers(), CatchSwitch->getName(),
742 CatchSwitch);
743 for (BasicBlock *PadBB : CatchSwitch->handlers())
744 NewCatchSwitch->addHandler(PadBB);
745 // Propagate info for the old catchswitch over to the new one in
746 // the unwind map. This also serves to short-circuit any subsequent
747 // checks for the unwind dest of this catchswitch, which would get
748 // confused if they found the outer handler in the callee.
749 FuncletUnwindMap[NewCatchSwitch] = UnwindDestToken;
750 Replacement = NewCatchSwitch;
751 }
752 } else if (!isa<FuncletPadInst>(I)) {
753 llvm_unreachable("unexpected EHPad!");
754 }
755
756 if (Replacement) {
757 Replacement->takeName(I);
758 I->replaceAllUsesWith(Replacement);
759 I->eraseFromParent();
760 UpdatePHINodes(&*BB);
761 }
762 }
763
764 if (InlinedCodeInfo.ContainsCalls)
765 for (Function::iterator BB = FirstNewBlock->getIterator(),
766 E = Caller->end();
767 BB != E; ++BB)
769 &*BB, UnwindDest, &FuncletUnwindMap))
770 // Update any PHI nodes in the exceptional block to indicate that there
771 // is now a new entry in them.
772 UpdatePHINodes(NewBB);
773
774 // Now that everything is happy, we have one final detail. The PHI nodes in
775 // the exception destination block still have entries due to the original
776 // invoke instruction. Eliminate these entries (which might even delete the
777 // PHI node) now.
778 UnwindDest->removePredecessor(InvokeBB);
779}
780
781static bool haveCommonPrefix(MDNode *MIBStackContext,
782 MDNode *CallsiteStackContext) {
783 assert(MIBStackContext->getNumOperands() > 0 &&
784 CallsiteStackContext->getNumOperands() > 0);
785 // Because of the context trimming performed during matching, the callsite
786 // context could have more stack ids than the MIB. We match up to the end of
787 // the shortest stack context.
788 for (auto MIBStackIter = MIBStackContext->op_begin(),
789 CallsiteStackIter = CallsiteStackContext->op_begin();
790 MIBStackIter != MIBStackContext->op_end() &&
791 CallsiteStackIter != CallsiteStackContext->op_end();
792 MIBStackIter++, CallsiteStackIter++) {
793 auto *Val1 = mdconst::dyn_extract<ConstantInt>(*MIBStackIter);
794 auto *Val2 = mdconst::dyn_extract<ConstantInt>(*CallsiteStackIter);
795 assert(Val1 && Val2);
796 if (Val1->getZExtValue() != Val2->getZExtValue())
797 return false;
798 }
799 return true;
800}
801
802static void removeMemProfMetadata(CallBase *Call) {
803 Call->setMetadata(LLVMContext::MD_memprof, nullptr);
804}
805
807 Call->setMetadata(LLVMContext::MD_callsite, nullptr);
808}
809
811 const std::vector<Metadata *> &MIBList) {
812 assert(!MIBList.empty());
813 // Remove existing memprof, which will either be replaced or may not be needed
814 // if we are able to use a single allocation type function attribute.
817 for (Metadata *MIB : MIBList)
818 CallStack.addCallStack(cast<MDNode>(MIB));
819 bool MemprofMDAttached = CallStack.buildAndAttachMIBMetadata(CI);
820 assert(MemprofMDAttached == CI->hasMetadata(LLVMContext::MD_memprof));
821 if (!MemprofMDAttached)
822 // If we used a function attribute remove the callsite metadata as well.
824}
825
826// Update the metadata on the inlined copy ClonedCall of a call OrigCall in the
827// inlined callee body, based on the callsite metadata InlinedCallsiteMD from
828// the call that was inlined.
829static void propagateMemProfHelper(const CallBase *OrigCall,
830 CallBase *ClonedCall,
831 MDNode *InlinedCallsiteMD) {
832 MDNode *OrigCallsiteMD = ClonedCall->getMetadata(LLVMContext::MD_callsite);
833 MDNode *ClonedCallsiteMD = nullptr;
834 // Check if the call originally had callsite metadata, and update it for the
835 // new call in the inlined body.
836 if (OrigCallsiteMD) {
837 // The cloned call's context is now the concatenation of the original call's
838 // callsite metadata and the callsite metadata on the call where it was
839 // inlined.
840 ClonedCallsiteMD = MDNode::concatenate(OrigCallsiteMD, InlinedCallsiteMD);
841 ClonedCall->setMetadata(LLVMContext::MD_callsite, ClonedCallsiteMD);
842 }
843
844 // Update any memprof metadata on the cloned call.
845 MDNode *OrigMemProfMD = ClonedCall->getMetadata(LLVMContext::MD_memprof);
846 if (!OrigMemProfMD)
847 return;
848 // We currently expect that allocations with memprof metadata also have
849 // callsite metadata for the allocation's part of the context.
850 assert(OrigCallsiteMD);
851
852 // New call's MIB list.
853 std::vector<Metadata *> NewMIBList;
854
855 // For each MIB metadata, check if its call stack context starts with the
856 // new clone's callsite metadata. If so, that MIB goes onto the cloned call in
857 // the inlined body. If not, it stays on the out-of-line original call.
858 for (auto &MIBOp : OrigMemProfMD->operands()) {
859 MDNode *MIB = dyn_cast<MDNode>(MIBOp);
860 // Stack is first operand of MIB.
861 MDNode *StackMD = getMIBStackNode(MIB);
862 assert(StackMD);
863 // See if the new cloned callsite context matches this profiled context.
864 if (haveCommonPrefix(StackMD, ClonedCallsiteMD))
865 // Add it to the cloned call's MIB list.
866 NewMIBList.push_back(MIB);
867 }
868 if (NewMIBList.empty()) {
869 removeMemProfMetadata(ClonedCall);
870 removeCallsiteMetadata(ClonedCall);
871 return;
872 }
873 if (NewMIBList.size() < OrigMemProfMD->getNumOperands())
874 updateMemprofMetadata(ClonedCall, NewMIBList);
875}
876
877// Update memprof related metadata (!memprof and !callsite) based on the
878// inlining of Callee into the callsite at CB. The updates include merging the
879// inlined callee's callsite metadata with that of the inlined call,
880// and moving the subset of any memprof contexts to the inlined callee
881// allocations if they match the new inlined call stack.
882// FIXME: Replace memprof metadata with function attribute if all MIB end up
883// having the same behavior. Do other context trimming/merging optimizations
884// too.
885static void
887 bool ContainsMemProfMetadata,
889 MDNode *CallsiteMD = CB.getMetadata(LLVMContext::MD_callsite);
890 // Only need to update if the inlined callsite had callsite metadata, or if
891 // there was any memprof metadata inlined.
892 if (!CallsiteMD && !ContainsMemProfMetadata)
893 return;
894
895 // Propagate metadata onto the cloned calls in the inlined callee.
896 for (const auto &Entry : VMap) {
897 // See if this is a call that has been inlined and remapped, and not
898 // simplified away in the process.
899 auto *OrigCall = dyn_cast_or_null<CallBase>(Entry.first);
900 auto *ClonedCall = dyn_cast_or_null<CallBase>(Entry.second);
901 if (!OrigCall || !ClonedCall)
902 continue;
903 // If the inlined callsite did not have any callsite metadata, then it isn't
904 // involved in any profiled call contexts, and we can remove any memprof
905 // metadata on the cloned call.
906 if (!CallsiteMD) {
907 removeMemProfMetadata(ClonedCall);
908 removeCallsiteMetadata(ClonedCall);
909 continue;
910 }
911 propagateMemProfHelper(OrigCall, ClonedCall, CallsiteMD);
912 }
913}
914
915/// When inlining a call site that has !llvm.mem.parallel_loop_access,
916/// !llvm.access.group, !alias.scope or !noalias metadata, that metadata should
917/// be propagated to all memory-accessing cloned instructions.
919 Function::iterator FEnd) {
920 MDNode *MemParallelLoopAccess =
921 CB.getMetadata(LLVMContext::MD_mem_parallel_loop_access);
922 MDNode *AccessGroup = CB.getMetadata(LLVMContext::MD_access_group);
923 MDNode *AliasScope = CB.getMetadata(LLVMContext::MD_alias_scope);
924 MDNode *NoAlias = CB.getMetadata(LLVMContext::MD_noalias);
925 if (!MemParallelLoopAccess && !AccessGroup && !AliasScope && !NoAlias)
926 return;
927
928 for (BasicBlock &BB : make_range(FStart, FEnd)) {
929 for (Instruction &I : BB) {
930 // This metadata is only relevant for instructions that access memory.
931 if (!I.mayReadOrWriteMemory())
932 continue;
933
934 if (MemParallelLoopAccess) {
935 // TODO: This probably should not overwrite MemParalleLoopAccess.
936 MemParallelLoopAccess = MDNode::concatenate(
937 I.getMetadata(LLVMContext::MD_mem_parallel_loop_access),
938 MemParallelLoopAccess);
939 I.setMetadata(LLVMContext::MD_mem_parallel_loop_access,
940 MemParallelLoopAccess);
941 }
942
943 if (AccessGroup)
944 I.setMetadata(LLVMContext::MD_access_group, uniteAccessGroups(
945 I.getMetadata(LLVMContext::MD_access_group), AccessGroup));
946
947 if (AliasScope)
948 I.setMetadata(LLVMContext::MD_alias_scope, MDNode::concatenate(
949 I.getMetadata(LLVMContext::MD_alias_scope), AliasScope));
950
951 if (NoAlias)
952 I.setMetadata(LLVMContext::MD_noalias, MDNode::concatenate(
953 I.getMetadata(LLVMContext::MD_noalias), NoAlias));
954 }
955 }
956}
957
958/// Bundle operands of the inlined function must be added to inlined call sites.
960 Instruction *CallSiteEHPad) {
961 for (Instruction &II : llvm::make_early_inc_range(*InlinedBB)) {
962 CallBase *I = dyn_cast<CallBase>(&II);
963 if (!I)
964 continue;
965 // Skip call sites which already have a "funclet" bundle.
966 if (I->getOperandBundle(LLVMContext::OB_funclet))
967 continue;
968 // Skip call sites which are nounwind intrinsics (as long as they don't
969 // lower into regular function calls in the course of IR transformations).
970 auto *CalledFn =
971 dyn_cast<Function>(I->getCalledOperand()->stripPointerCasts());
972 if (CalledFn && CalledFn->isIntrinsic() && I->doesNotThrow() &&
973 !IntrinsicInst::mayLowerToFunctionCall(CalledFn->getIntrinsicID()))
974 continue;
975
977 I->getOperandBundlesAsDefs(OpBundles);
978 OpBundles.emplace_back("funclet", CallSiteEHPad);
979
980 Instruction *NewInst = CallBase::Create(I, OpBundles, I);
981 NewInst->takeName(I);
982 I->replaceAllUsesWith(NewInst);
983 I->eraseFromParent();
984 }
985}
986
987namespace {
988/// Utility for cloning !noalias and !alias.scope metadata. When a code region
989/// using scoped alias metadata is inlined, the aliasing relationships may not
990/// hold between the two version. It is necessary to create a deep clone of the
991/// metadata, putting the two versions in separate scope domains.
992class ScopedAliasMetadataDeepCloner {
995 MetadataMap MDMap;
996 void addRecursiveMetadataUses();
997
998public:
999 ScopedAliasMetadataDeepCloner(const Function *F);
1000
1001 /// Create a new clone of the scoped alias metadata, which will be used by
1002 /// subsequent remap() calls.
1003 void clone();
1004
1005 /// Remap instructions in the given range from the original to the cloned
1006 /// metadata.
1007 void remap(Function::iterator FStart, Function::iterator FEnd);
1008};
1009} // namespace
1010
1011ScopedAliasMetadataDeepCloner::ScopedAliasMetadataDeepCloner(
1012 const Function *F) {
1013 for (const BasicBlock &BB : *F) {
1014 for (const Instruction &I : BB) {
1015 if (const MDNode *M = I.getMetadata(LLVMContext::MD_alias_scope))
1016 MD.insert(M);
1017 if (const MDNode *M = I.getMetadata(LLVMContext::MD_noalias))
1018 MD.insert(M);
1019
1020 // We also need to clone the metadata in noalias intrinsics.
1021 if (const auto *Decl = dyn_cast<NoAliasScopeDeclInst>(&I))
1022 MD.insert(Decl->getScopeList());
1023 }
1024 }
1025 addRecursiveMetadataUses();
1026}
1027
1028void ScopedAliasMetadataDeepCloner::addRecursiveMetadataUses() {
1029 SmallVector<const Metadata *, 16> Queue(MD.begin(), MD.end());
1030 while (!Queue.empty()) {
1031 const MDNode *M = cast<MDNode>(Queue.pop_back_val());
1032 for (const Metadata *Op : M->operands())
1033 if (const MDNode *OpMD = dyn_cast<MDNode>(Op))
1034 if (MD.insert(OpMD))
1035 Queue.push_back(OpMD);
1036 }
1037}
1038
1039void ScopedAliasMetadataDeepCloner::clone() {
1040 assert(MDMap.empty() && "clone() already called ?");
1041
1043 for (const MDNode *I : MD) {
1044 DummyNodes.push_back(MDTuple::getTemporary(I->getContext(), std::nullopt));
1045 MDMap[I].reset(DummyNodes.back().get());
1046 }
1047
1048 // Create new metadata nodes to replace the dummy nodes, replacing old
1049 // metadata references with either a dummy node or an already-created new
1050 // node.
1052 for (const MDNode *I : MD) {
1053 for (const Metadata *Op : I->operands()) {
1054 if (const MDNode *M = dyn_cast<MDNode>(Op))
1055 NewOps.push_back(MDMap[M]);
1056 else
1057 NewOps.push_back(const_cast<Metadata *>(Op));
1058 }
1059
1060 MDNode *NewM = MDNode::get(I->getContext(), NewOps);
1061 MDTuple *TempM = cast<MDTuple>(MDMap[I]);
1062 assert(TempM->isTemporary() && "Expected temporary node");
1063
1064 TempM->replaceAllUsesWith(NewM);
1065 NewOps.clear();
1066 }
1067}
1068
1069void ScopedAliasMetadataDeepCloner::remap(Function::iterator FStart,
1070 Function::iterator FEnd) {
1071 if (MDMap.empty())
1072 return; // Nothing to do.
1073
1074 for (BasicBlock &BB : make_range(FStart, FEnd)) {
1075 for (Instruction &I : BB) {
1076 // TODO: The null checks for the MDMap.lookup() results should no longer
1077 // be necessary.
1078 if (MDNode *M = I.getMetadata(LLVMContext::MD_alias_scope))
1079 if (MDNode *MNew = MDMap.lookup(M))
1080 I.setMetadata(LLVMContext::MD_alias_scope, MNew);
1081
1082 if (MDNode *M = I.getMetadata(LLVMContext::MD_noalias))
1083 if (MDNode *MNew = MDMap.lookup(M))
1084 I.setMetadata(LLVMContext::MD_noalias, MNew);
1085
1086 if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(&I))
1087 if (MDNode *MNew = MDMap.lookup(Decl->getScopeList()))
1088 Decl->setScopeList(MNew);
1089 }
1090 }
1091}
1092
1093/// If the inlined function has noalias arguments,
1094/// then add new alias scopes for each noalias argument, tag the mapped noalias
1095/// parameters with noalias metadata specifying the new scope, and tag all
1096/// non-derived loads, stores and memory intrinsics with the new alias scopes.
1098 const DataLayout &DL, AAResults *CalleeAAR,
1099 ClonedCodeInfo &InlinedFunctionInfo) {
1101 return;
1102
1103 const Function *CalledFunc = CB.getCalledFunction();
1105
1106 for (const Argument &Arg : CalledFunc->args())
1107 if (CB.paramHasAttr(Arg.getArgNo(), Attribute::NoAlias) && !Arg.use_empty())
1108 NoAliasArgs.push_back(&Arg);
1109
1110 if (NoAliasArgs.empty())
1111 return;
1112
1113 // To do a good job, if a noalias variable is captured, we need to know if
1114 // the capture point dominates the particular use we're considering.
1115 DominatorTree DT;
1116 DT.recalculate(const_cast<Function&>(*CalledFunc));
1117
1118 // noalias indicates that pointer values based on the argument do not alias
1119 // pointer values which are not based on it. So we add a new "scope" for each
1120 // noalias function argument. Accesses using pointers based on that argument
1121 // become part of that alias scope, accesses using pointers not based on that
1122 // argument are tagged as noalias with that scope.
1123
1125 MDBuilder MDB(CalledFunc->getContext());
1126
1127 // Create a new scope domain for this function.
1128 MDNode *NewDomain =
1129 MDB.createAnonymousAliasScopeDomain(CalledFunc->getName());
1130 for (unsigned i = 0, e = NoAliasArgs.size(); i != e; ++i) {
1131 const Argument *A = NoAliasArgs[i];
1132
1133 std::string Name = std::string(CalledFunc->getName());
1134 if (A->hasName()) {
1135 Name += ": %";
1136 Name += A->getName();
1137 } else {
1138 Name += ": argument ";
1139 Name += utostr(i);
1140 }
1141
1142 // Note: We always create a new anonymous root here. This is true regardless
1143 // of the linkage of the callee because the aliasing "scope" is not just a
1144 // property of the callee, but also all control dependencies in the caller.
1145 MDNode *NewScope = MDB.createAnonymousAliasScope(NewDomain, Name);
1146 NewScopes.insert(std::make_pair(A, NewScope));
1147
1148 if (UseNoAliasIntrinsic) {
1149 // Introduce a llvm.experimental.noalias.scope.decl for the noalias
1150 // argument.
1151 MDNode *AScopeList = MDNode::get(CalledFunc->getContext(), NewScope);
1152 auto *NoAliasDecl =
1154 // Ignore the result for now. The result will be used when the
1155 // llvm.noalias intrinsic is introduced.
1156 (void)NoAliasDecl;
1157 }
1158 }
1159
1160 // Iterate over all new instructions in the map; for all memory-access
1161 // instructions, add the alias scope metadata.
1162 for (ValueToValueMapTy::iterator VMI = VMap.begin(), VMIE = VMap.end();
1163 VMI != VMIE; ++VMI) {
1164 if (const Instruction *I = dyn_cast<Instruction>(VMI->first)) {
1165 if (!VMI->second)
1166 continue;
1167
1168 Instruction *NI = dyn_cast<Instruction>(VMI->second);
1169 if (!NI || InlinedFunctionInfo.isSimplified(I, NI))
1170 continue;
1171
1172 bool IsArgMemOnlyCall = false, IsFuncCall = false;
1174
1175 if (const LoadInst *LI = dyn_cast<LoadInst>(I))
1176 PtrArgs.push_back(LI->getPointerOperand());
1177 else if (const StoreInst *SI = dyn_cast<StoreInst>(I))
1178 PtrArgs.push_back(SI->getPointerOperand());
1179 else if (const VAArgInst *VAAI = dyn_cast<VAArgInst>(I))
1180 PtrArgs.push_back(VAAI->getPointerOperand());
1181 else if (const AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(I))
1182 PtrArgs.push_back(CXI->getPointerOperand());
1183 else if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I))
1184 PtrArgs.push_back(RMWI->getPointerOperand());
1185 else if (const auto *Call = dyn_cast<CallBase>(I)) {
1186 // If we know that the call does not access memory, then we'll still
1187 // know that about the inlined clone of this call site, and we don't
1188 // need to add metadata.
1189 if (Call->doesNotAccessMemory())
1190 continue;
1191
1192 IsFuncCall = true;
1193 if (CalleeAAR) {
1194 MemoryEffects ME = CalleeAAR->getMemoryEffects(Call);
1195
1196 // We'll retain this knowledge without additional metadata.
1198 continue;
1199
1200 if (ME.onlyAccessesArgPointees())
1201 IsArgMemOnlyCall = true;
1202 }
1203
1204 for (Value *Arg : Call->args()) {
1205 // Only care about pointer arguments. If a noalias argument is
1206 // accessed through a non-pointer argument, it must be captured
1207 // first (e.g. via ptrtoint), and we protect against captures below.
1208 if (!Arg->getType()->isPointerTy())
1209 continue;
1210
1211 PtrArgs.push_back(Arg);
1212 }
1213 }
1214
1215 // If we found no pointers, then this instruction is not suitable for
1216 // pairing with an instruction to receive aliasing metadata.
1217 // However, if this is a call, this we might just alias with none of the
1218 // noalias arguments.
1219 if (PtrArgs.empty() && !IsFuncCall)
1220 continue;
1221
1222 // It is possible that there is only one underlying object, but you
1223 // need to go through several PHIs to see it, and thus could be
1224 // repeated in the Objects list.
1227
1229 for (const Value *V : PtrArgs) {
1231 getUnderlyingObjects(V, Objects, /* LI = */ nullptr);
1232
1233 for (const Value *O : Objects)
1234 ObjSet.insert(O);
1235 }
1236
1237 // Figure out if we're derived from anything that is not a noalias
1238 // argument.
1239 bool RequiresNoCaptureBefore = false, UsesAliasingPtr = false,
1240 UsesUnknownObject = false;
1241 for (const Value *V : ObjSet) {
1242 // Is this value a constant that cannot be derived from any pointer
1243 // value (we need to exclude constant expressions, for example, that
1244 // are formed from arithmetic on global symbols).
1245 bool IsNonPtrConst = isa<ConstantInt>(V) || isa<ConstantFP>(V) ||
1246 isa<ConstantPointerNull>(V) ||
1247 isa<ConstantDataVector>(V) || isa<UndefValue>(V);
1248 if (IsNonPtrConst)
1249 continue;
1250
1251 // If this is anything other than a noalias argument, then we cannot
1252 // completely describe the aliasing properties using alias.scope
1253 // metadata (and, thus, won't add any).
1254 if (const Argument *A = dyn_cast<Argument>(V)) {
1255 if (!CB.paramHasAttr(A->getArgNo(), Attribute::NoAlias))
1256 UsesAliasingPtr = true;
1257 } else {
1258 UsesAliasingPtr = true;
1259 }
1260
1261 if (isEscapeSource(V)) {
1262 // An escape source can only alias with a noalias argument if it has
1263 // been captured beforehand.
1264 RequiresNoCaptureBefore = true;
1265 } else if (!isa<Argument>(V) && !isIdentifiedObject(V)) {
1266 // If this is neither an escape source, nor some identified object
1267 // (which cannot directly alias a noalias argument), nor some other
1268 // argument (which, by definition, also cannot alias a noalias
1269 // argument), conservatively do not make any assumptions.
1270 UsesUnknownObject = true;
1271 }
1272 }
1273
1274 // Nothing we can do if the used underlying object cannot be reliably
1275 // determined.
1276 if (UsesUnknownObject)
1277 continue;
1278
1279 // A function call can always get captured noalias pointers (via other
1280 // parameters, globals, etc.).
1281 if (IsFuncCall && !IsArgMemOnlyCall)
1282 RequiresNoCaptureBefore = true;
1283
1284 // First, we want to figure out all of the sets with which we definitely
1285 // don't alias. Iterate over all noalias set, and add those for which:
1286 // 1. The noalias argument is not in the set of objects from which we
1287 // definitely derive.
1288 // 2. The noalias argument has not yet been captured.
1289 // An arbitrary function that might load pointers could see captured
1290 // noalias arguments via other noalias arguments or globals, and so we
1291 // must always check for prior capture.
1292 for (const Argument *A : NoAliasArgs) {
1293 if (ObjSet.contains(A))
1294 continue; // May be based on a noalias argument.
1295
1296 // It might be tempting to skip the PointerMayBeCapturedBefore check if
1297 // A->hasNoCaptureAttr() is true, but this is incorrect because
1298 // nocapture only guarantees that no copies outlive the function, not
1299 // that the value cannot be locally captured.
1300 if (!RequiresNoCaptureBefore ||
1301 !PointerMayBeCapturedBefore(A, /* ReturnCaptures */ false,
1302 /* StoreCaptures */ false, I, &DT))
1303 NoAliases.push_back(NewScopes[A]);
1304 }
1305
1306 if (!NoAliases.empty())
1307 NI->setMetadata(LLVMContext::MD_noalias,
1309 NI->getMetadata(LLVMContext::MD_noalias),
1310 MDNode::get(CalledFunc->getContext(), NoAliases)));
1311
1312 // Next, we want to figure out all of the sets to which we might belong.
1313 // We might belong to a set if the noalias argument is in the set of
1314 // underlying objects. If there is some non-noalias argument in our list
1315 // of underlying objects, then we cannot add a scope because the fact
1316 // that some access does not alias with any set of our noalias arguments
1317 // cannot itself guarantee that it does not alias with this access
1318 // (because there is some pointer of unknown origin involved and the
1319 // other access might also depend on this pointer). We also cannot add
1320 // scopes to arbitrary functions unless we know they don't access any
1321 // non-parameter pointer-values.
1322 bool CanAddScopes = !UsesAliasingPtr;
1323 if (CanAddScopes && IsFuncCall)
1324 CanAddScopes = IsArgMemOnlyCall;
1325
1326 if (CanAddScopes)
1327 for (const Argument *A : NoAliasArgs) {
1328 if (ObjSet.count(A))
1329 Scopes.push_back(NewScopes[A]);
1330 }
1331
1332 if (!Scopes.empty())
1333 NI->setMetadata(
1334 LLVMContext::MD_alias_scope,
1335 MDNode::concatenate(NI->getMetadata(LLVMContext::MD_alias_scope),
1336 MDNode::get(CalledFunc->getContext(), Scopes)));
1337 }
1338 }
1339}
1340
1342 Instruction *End) {
1343
1344 assert(Begin->getParent() == End->getParent() &&
1345 "Expected to be in same basic block!");
1347 Begin->getIterator(), End->getIterator(), InlinerAttributeWindow + 1);
1348}
1349
1351
1353 if (!AB.hasAttributes())
1354 return AB;
1355 AttrBuilder Valid(CB.getContext());
1356 // Only allow these white listed attributes to be propagated back to the
1357 // callee. This is because other attributes may only be valid on the call
1358 // itself, i.e. attributes such as signext and zeroext.
1359 if (auto DerefBytes = AB.getDereferenceableBytes())
1360 Valid.addDereferenceableAttr(DerefBytes);
1361 if (auto DerefOrNullBytes = AB.getDereferenceableOrNullBytes())
1362 Valid.addDereferenceableOrNullAttr(DerefOrNullBytes);
1363 if (AB.contains(Attribute::NoAlias))
1364 Valid.addAttribute(Attribute::NoAlias);
1365 if (AB.contains(Attribute::NonNull))
1366 Valid.addAttribute(Attribute::NonNull);
1367 return Valid;
1368}
1369
1372 return;
1373
1375 if (!Valid.hasAttributes())
1376 return;
1377 auto *CalledFunction = CB.getCalledFunction();
1378 auto &Context = CalledFunction->getContext();
1379
1380 for (auto &BB : *CalledFunction) {
1381 auto *RI = dyn_cast<ReturnInst>(BB.getTerminator());
1382 if (!RI || !isa<CallBase>(RI->getOperand(0)))
1383 continue;
1384 auto *RetVal = cast<CallBase>(RI->getOperand(0));
1385 // Check that the cloned RetVal exists and is a call, otherwise we cannot
1386 // add the attributes on the cloned RetVal. Simplification during inlining
1387 // could have transformed the cloned instruction.
1388 auto *NewRetVal = dyn_cast_or_null<CallBase>(VMap.lookup(RetVal));
1389 if (!NewRetVal)
1390 continue;
1391 // Backward propagation of attributes to the returned value may be incorrect
1392 // if it is control flow dependent.
1393 // Consider:
1394 // @callee {
1395 // %rv = call @foo()
1396 // %rv2 = call @bar()
1397 // if (%rv2 != null)
1398 // return %rv2
1399 // if (%rv == null)
1400 // exit()
1401 // return %rv
1402 // }
1403 // caller() {
1404 // %val = call nonnull @callee()
1405 // }
1406 // Here we cannot add the nonnull attribute on either foo or bar. So, we
1407 // limit the check to both RetVal and RI are in the same basic block and
1408 // there are no throwing/exiting instructions between these instructions.
1409 if (RI->getParent() != RetVal->getParent() ||
1411 continue;
1412 // Add to the existing attributes of NewRetVal, i.e. the cloned call
1413 // instruction.
1414 // NB! When we have the same attribute already existing on NewRetVal, but
1415 // with a differing value, the AttributeList's merge API honours the already
1416 // existing attribute value (i.e. attributes such as dereferenceable,
1417 // dereferenceable_or_null etc). See AttrBuilder::merge for more details.
1418 AttributeList AL = NewRetVal->getAttributes();
1419 AttributeList NewAL = AL.addRetAttributes(Context, Valid);
1420 NewRetVal->setAttributes(NewAL);
1421 }
1422}
1423
1424/// If the inlined function has non-byval align arguments, then
1425/// add @llvm.assume-based alignment assumptions to preserve this information.
1428 return;
1429
1431 auto &DL = CB.getCaller()->getParent()->getDataLayout();
1432
1433 // To avoid inserting redundant assumptions, we should check for assumptions
1434 // already in the caller. To do this, we might need a DT of the caller.
1435 DominatorTree DT;
1436 bool DTCalculated = false;
1437
1438 Function *CalledFunc = CB.getCalledFunction();
1439 for (Argument &Arg : CalledFunc->args()) {
1440 if (!Arg.getType()->isPointerTy() || Arg.hasPassPointeeByValueCopyAttr() ||
1441 Arg.hasNUses(0))
1442 continue;
1443 MaybeAlign Alignment = Arg.getParamAlign();
1444 if (!Alignment)
1445 continue;
1446
1447 if (!DTCalculated) {
1448 DT.recalculate(*CB.getCaller());
1449 DTCalculated = true;
1450 }
1451 // If we can already prove the asserted alignment in the context of the
1452 // caller, then don't bother inserting the assumption.
1453 Value *ArgVal = CB.getArgOperand(Arg.getArgNo());
1454 if (getKnownAlignment(ArgVal, DL, &CB, AC, &DT) >= *Alignment)
1455 continue;
1456
1458 DL, ArgVal, Alignment->value());
1459 AC->registerAssumption(cast<AssumeInst>(NewAsmp));
1460 }
1461}
1462
1463static void HandleByValArgumentInit(Type *ByValType, Value *Dst, Value *Src,
1464 Module *M, BasicBlock *InsertBlock,
1465 InlineFunctionInfo &IFI,
1466 Function *CalledFunc) {
1467 IRBuilder<> Builder(InsertBlock, InsertBlock->begin());
1468
1469 Value *Size =
1470 Builder.getInt64(M->getDataLayout().getTypeStoreSize(ByValType));
1471
1472 // Always generate a memcpy of alignment 1 here because we don't know
1473 // the alignment of the src pointer. Other optimizations can infer
1474 // better alignment.
1475 CallInst *CI = Builder.CreateMemCpy(Dst, /*DstAlign*/ Align(1), Src,
1476 /*SrcAlign*/ Align(1), Size);
1477
1478 // The verifier requires that all calls of debug-info-bearing functions
1479 // from debug-info-bearing functions have a debug location (for inlining
1480 // purposes). Assign a dummy location to satisfy the constraint.
1481 if (!CI->getDebugLoc() && InsertBlock->getParent()->getSubprogram())
1482 if (DISubprogram *SP = CalledFunc->getSubprogram())
1483 CI->setDebugLoc(DILocation::get(SP->getContext(), 0, 0, SP));
1484}
1485
1486/// When inlining a call site that has a byval argument,
1487/// we have to make the implicit memcpy explicit by adding it.
1489 Instruction *TheCall,
1490 const Function *CalledFunc,
1491 InlineFunctionInfo &IFI,
1492 MaybeAlign ByValAlignment) {
1493 assert(cast<PointerType>(Arg->getType())
1494 ->isOpaqueOrPointeeTypeMatches(ByValType));
1495 Function *Caller = TheCall->getFunction();
1496 const DataLayout &DL = Caller->getParent()->getDataLayout();
1497
1498 // If the called function is readonly, then it could not mutate the caller's
1499 // copy of the byval'd memory. In this case, it is safe to elide the copy and
1500 // temporary.
1501 if (CalledFunc->onlyReadsMemory()) {
1502 // If the byval argument has a specified alignment that is greater than the
1503 // passed in pointer, then we either have to round up the input pointer or
1504 // give up on this transformation.
1505 if (ByValAlignment.valueOrOne() == 1)
1506 return Arg;
1507
1508 AssumptionCache *AC =
1509 IFI.GetAssumptionCache ? &IFI.GetAssumptionCache(*Caller) : nullptr;
1510
1511 // If the pointer is already known to be sufficiently aligned, or if we can
1512 // round it up to a larger alignment, then we don't need a temporary.
1513 if (getOrEnforceKnownAlignment(Arg, *ByValAlignment, DL, TheCall, AC) >=
1514 *ByValAlignment)
1515 return Arg;
1516
1517 // Otherwise, we have to make a memcpy to get a safe alignment. This is bad
1518 // for code quality, but rarely happens and is required for correctness.
1519 }
1520
1521 // Create the alloca. If we have DataLayout, use nice alignment.
1522 Align Alignment = DL.getPrefTypeAlign(ByValType);
1523
1524 // If the byval had an alignment specified, we *must* use at least that
1525 // alignment, as it is required by the byval argument (and uses of the
1526 // pointer inside the callee).
1527 if (ByValAlignment)
1528 Alignment = std::max(Alignment, *ByValAlignment);
1529
1530 Value *NewAlloca =
1531 new AllocaInst(ByValType, DL.getAllocaAddrSpace(), nullptr, Alignment,
1532 Arg->getName(), &*Caller->begin()->begin());
1533 IFI.StaticAllocas.push_back(cast<AllocaInst>(NewAlloca));
1534
1535 // Uses of the argument in the function should use our new alloca
1536 // instead.
1537 return NewAlloca;
1538}
1539
1540// Check whether this Value is used by a lifetime intrinsic.
1542 for (User *U : V->users())
1543 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U))
1544 if (II->isLifetimeStartOrEnd())
1545 return true;
1546 return false;
1547}
1548
1549// Check whether the given alloca already has
1550// lifetime.start or lifetime.end intrinsics.
1552 Type *Ty = AI->getType();
1553 Type *Int8PtrTy = Type::getInt8PtrTy(Ty->getContext(),
1555 if (Ty == Int8PtrTy)
1556 return isUsedByLifetimeMarker(AI);
1557
1558 // Do a scan to find all the casts to i8*.
1559 for (User *U : AI->users()) {
1560 if (U->getType() != Int8PtrTy) continue;
1561 if (U->stripPointerCasts() != AI) continue;
1563 return true;
1564 }
1565 return false;
1566}
1567
1568/// Return the result of AI->isStaticAlloca() if AI were moved to the entry
1569/// block. Allocas used in inalloca calls and allocas of dynamic array size
1570/// cannot be static.
1572 return isa<Constant>(AI->getArraySize()) && !AI->isUsedWithInAlloca();
1573}
1574
1575/// Returns a DebugLoc for a new DILocation which is a clone of \p OrigDL
1576/// inlined at \p InlinedAt. \p IANodes is an inlined-at cache.
1577static DebugLoc inlineDebugLoc(DebugLoc OrigDL, DILocation *InlinedAt,
1578 LLVMContext &Ctx,
1580 auto IA = DebugLoc::appendInlinedAt(OrigDL, InlinedAt, Ctx, IANodes);
1581 return DILocation::get(Ctx, OrigDL.getLine(), OrigDL.getCol(),
1582 OrigDL.getScope(), IA);
1583}
1584
1585/// Update inlined instructions' line numbers to
1586/// to encode location where these instructions are inlined.
1588 Instruction *TheCall, bool CalleeHasDebugInfo) {
1589 const DebugLoc &TheCallDL = TheCall->getDebugLoc();
1590 if (!TheCallDL)
1591 return;
1592
1593 auto &Ctx = Fn->getContext();
1594 DILocation *InlinedAtNode = TheCallDL;
1595
1596 // Create a unique call site, not to be confused with any other call from the
1597 // same location.
1598 InlinedAtNode = DILocation::getDistinct(
1599 Ctx, InlinedAtNode->getLine(), InlinedAtNode->getColumn(),
1600 InlinedAtNode->getScope(), InlinedAtNode->getInlinedAt());
1601
1602 // Cache the inlined-at nodes as they're built so they are reused, without
1603 // this every instruction's inlined-at chain would become distinct from each
1604 // other.
1606
1607 // Check if we are not generating inline line tables and want to use
1608 // the call site location instead.
1609 bool NoInlineLineTables = Fn->hasFnAttribute("no-inline-line-tables");
1610
1611 for (; FI != Fn->end(); ++FI) {
1612 for (BasicBlock::iterator BI = FI->begin(), BE = FI->end();
1613 BI != BE; ++BI) {
1614 // Loop metadata needs to be updated so that the start and end locs
1615 // reference inlined-at locations.
1616 auto updateLoopInfoLoc = [&Ctx, &InlinedAtNode,
1617 &IANodes](Metadata *MD) -> Metadata * {
1618 if (auto *Loc = dyn_cast_or_null<DILocation>(MD))
1619 return inlineDebugLoc(Loc, InlinedAtNode, Ctx, IANodes).get();
1620 return MD;
1621 };
1622 updateLoopMetadataDebugLocations(*BI, updateLoopInfoLoc);
1623
1624 if (!NoInlineLineTables)
1625 if (DebugLoc DL = BI->getDebugLoc()) {
1626 DebugLoc IDL =
1627 inlineDebugLoc(DL, InlinedAtNode, BI->getContext(), IANodes);
1628 BI->setDebugLoc(IDL);
1629 continue;
1630 }
1631
1632 if (CalleeHasDebugInfo && !NoInlineLineTables)
1633 continue;
1634
1635 // If the inlined instruction has no line number, or if inline info
1636 // is not being generated, make it look as if it originates from the call
1637 // location. This is important for ((__always_inline, __nodebug__))
1638 // functions which must use caller location for all instructions in their
1639 // function body.
1640
1641 // Don't update static allocas, as they may get moved later.
1642 if (auto *AI = dyn_cast<AllocaInst>(BI))
1644 continue;
1645
1646 BI->setDebugLoc(TheCallDL);
1647 }
1648
1649 // Remove debug info intrinsics if we're not keeping inline info.
1650 if (NoInlineLineTables) {
1651 BasicBlock::iterator BI = FI->begin();
1652 while (BI != FI->end()) {
1653 if (isa<DbgInfoIntrinsic>(BI)) {
1654 BI = BI->eraseFromParent();
1655 continue;
1656 }
1657 ++BI;
1658 }
1659 }
1660
1661 }
1662}
1663
1664#undef DEBUG_TYPE
1665#define DEBUG_TYPE "assignment-tracking"
1666/// Find Alloca and linked DbgAssignIntrinsic for locals escaped by \p CB.
1668 const CallBase &CB) {
1669 at::StorageToVarsMap EscapedLocals;
1671
1672 LLVM_DEBUG(
1673 errs() << "# Finding caller local variables escaped by callee\n");
1674 for (const Value *Arg : CB.args()) {
1675 LLVM_DEBUG(errs() << "INSPECT: " << *Arg << "\n");
1676 if (!Arg->getType()->isPointerTy()) {
1677 LLVM_DEBUG(errs() << " | SKIP: Not a pointer\n");
1678 continue;
1679 }
1680
1681 const Instruction *I = dyn_cast<Instruction>(Arg);
1682 if (!I) {
1683 LLVM_DEBUG(errs() << " | SKIP: Not result of instruction\n");
1684 continue;
1685 }
1686
1687 // Walk back to the base storage.
1688 assert(Arg->getType()->isPtrOrPtrVectorTy());
1689 APInt TmpOffset(DL.getIndexTypeSizeInBits(Arg->getType()), 0, false);
1690 const AllocaInst *Base = dyn_cast<AllocaInst>(
1691 Arg->stripAndAccumulateConstantOffsets(DL, TmpOffset, true));
1692 if (!Base) {
1693 LLVM_DEBUG(errs() << " | SKIP: Couldn't walk back to base storage\n");
1694 continue;
1695 }
1696
1697 assert(Base);
1698 LLVM_DEBUG(errs() << " | BASE: " << *Base << "\n");
1699 // We only need to process each base address once - skip any duplicates.
1700 if (!SeenBases.insert(Base).second)
1701 continue;
1702
1703 // Find all local variables associated with the backing storage.
1704 for (auto *DAI : at::getAssignmentMarkers(Base)) {
1705 // Skip variables from inlined functions - they are not local variables.
1706 if (DAI->getDebugLoc().getInlinedAt())
1707 continue;
1708 LLVM_DEBUG(errs() << " > DEF : " << *DAI << "\n");
1709 EscapedLocals[Base].insert(at::VarRecord(DAI));
1710 }
1711 }
1712 return EscapedLocals;
1713}
1714
1716 const CallBase &CB) {
1717 LLVM_DEBUG(errs() << "trackInlinedStores into "
1718 << Start->getParent()->getName() << " from "
1719 << CB.getCalledFunction()->getName() << "\n");
1720 std::unique_ptr<DataLayout> DL = std::make_unique<DataLayout>(CB.getModule());
1721 at::trackAssignments(Start, End, collectEscapedLocals(*DL, CB), *DL);
1722}
1723
1724/// Update inlined instructions' DIAssignID metadata. We need to do this
1725/// otherwise a function inlined more than once into the same function
1726/// will cause DIAssignID to be shared by many instructions.
1728 // Map {Old, New} metadata. Not used directly - use GetNewID.
1730 auto GetNewID = [&Map](Metadata *Old) {
1731 DIAssignID *OldID = cast<DIAssignID>(Old);
1732 if (DIAssignID *NewID = Map.lookup(OldID))
1733 return NewID;
1735 Map[OldID] = NewID;
1736 return NewID;
1737 };
1738 // Loop over all the inlined instructions. If we find a DIAssignID
1739 // attachment or use, replace it with a new version.
1740 for (auto BBI = Start; BBI != End; ++BBI) {
1741 for (Instruction &I : *BBI) {
1742 if (auto *ID = I.getMetadata(LLVMContext::MD_DIAssignID))
1743 I.setMetadata(LLVMContext::MD_DIAssignID, GetNewID(ID));
1744 else if (auto *DAI = dyn_cast<DbgAssignIntrinsic>(&I))
1745 DAI->setAssignId(GetNewID(DAI->getAssignID()));
1746 }
1747 }
1748}
1749#undef DEBUG_TYPE
1750#define DEBUG_TYPE "inline-function"
1751
1752/// Update the block frequencies of the caller after a callee has been inlined.
1753///
1754/// Each block cloned into the caller has its block frequency scaled by the
1755/// ratio of CallSiteFreq/CalleeEntryFreq. This ensures that the cloned copy of
1756/// callee's entry block gets the same frequency as the callsite block and the
1757/// relative frequencies of all cloned blocks remain the same after cloning.
1758static void updateCallerBFI(BasicBlock *CallSiteBlock,
1759 const ValueToValueMapTy &VMap,
1760 BlockFrequencyInfo *CallerBFI,
1761 BlockFrequencyInfo *CalleeBFI,
1762 const BasicBlock &CalleeEntryBlock) {
1764 for (auto Entry : VMap) {
1765 if (!isa<BasicBlock>(Entry.first) || !Entry.second)
1766 continue;
1767 auto *OrigBB = cast<BasicBlock>(Entry.first);
1768 auto *ClonedBB = cast<BasicBlock>(Entry.second);
1769 uint64_t Freq = CalleeBFI->getBlockFreq(OrigBB).getFrequency();
1770 if (!ClonedBBs.insert(ClonedBB).second) {
1771 // Multiple blocks in the callee might get mapped to one cloned block in
1772 // the caller since we prune the callee as we clone it. When that happens,
1773 // we want to use the maximum among the original blocks' frequencies.
1774 uint64_t NewFreq = CallerBFI->getBlockFreq(ClonedBB).getFrequency();
1775 if (NewFreq > Freq)
1776 Freq = NewFreq;
1777 }
1778 CallerBFI->setBlockFreq(ClonedBB, Freq);
1779 }
1780 BasicBlock *EntryClone = cast<BasicBlock>(VMap.lookup(&CalleeEntryBlock));
1781 CallerBFI->setBlockFreqAndScale(
1782 EntryClone, CallerBFI->getBlockFreq(CallSiteBlock).getFrequency(),
1783 ClonedBBs);
1784}
1785
1786/// Update the branch metadata for cloned call instructions.
1788 const ProfileCount &CalleeEntryCount,
1789 const CallBase &TheCall, ProfileSummaryInfo *PSI,
1790 BlockFrequencyInfo *CallerBFI) {
1791 if (CalleeEntryCount.isSynthetic() || CalleeEntryCount.getCount() < 1)
1792 return;
1793 auto CallSiteCount =
1794 PSI ? PSI->getProfileCount(TheCall, CallerBFI) : std::nullopt;
1795 int64_t CallCount =
1796 std::min(CallSiteCount.value_or(0), CalleeEntryCount.getCount());
1797 updateProfileCallee(Callee, -CallCount, &VMap);
1798}
1799
1801 Function *Callee, int64_t EntryDelta,
1803 auto CalleeCount = Callee->getEntryCount();
1804 if (!CalleeCount)
1805 return;
1806
1807 const uint64_t PriorEntryCount = CalleeCount->getCount();
1808
1809 // Since CallSiteCount is an estimate, it could exceed the original callee
1810 // count and has to be set to 0 so guard against underflow.
1811 const uint64_t NewEntryCount =
1812 (EntryDelta < 0 && static_cast<uint64_t>(-EntryDelta) > PriorEntryCount)
1813 ? 0
1814 : PriorEntryCount + EntryDelta;
1815
1816 // During inlining ?
1817 if (VMap) {
1818 uint64_t CloneEntryCount = PriorEntryCount - NewEntryCount;
1819 for (auto Entry : *VMap)
1820 if (isa<CallInst>(Entry.first))
1821 if (auto *CI = dyn_cast_or_null<CallInst>(Entry.second))
1822 CI->updateProfWeight(CloneEntryCount, PriorEntryCount);
1823 }
1824
1825 if (EntryDelta) {
1826 Callee->setEntryCount(NewEntryCount);
1827
1828 for (BasicBlock &BB : *Callee)
1829 // No need to update the callsite if it is pruned during inlining.
1830 if (!VMap || VMap->count(&BB))
1831 for (Instruction &I : BB)
1832 if (CallInst *CI = dyn_cast<CallInst>(&I))
1833 CI->updateProfWeight(NewEntryCount, PriorEntryCount);
1834 }
1835}
1836
1837/// An operand bundle "clang.arc.attachedcall" on a call indicates the call
1838/// result is implicitly consumed by a call to retainRV or claimRV immediately
1839/// after the call. This function inlines the retainRV/claimRV calls.
1840///
1841/// There are three cases to consider:
1842///
1843/// 1. If there is a call to autoreleaseRV that takes a pointer to the returned
1844/// object in the callee return block, the autoreleaseRV call and the
1845/// retainRV/claimRV call in the caller cancel out. If the call in the caller
1846/// is a claimRV call, a call to objc_release is emitted.
1847///
1848/// 2. If there is a call in the callee return block that doesn't have operand
1849/// bundle "clang.arc.attachedcall", the operand bundle on the original call
1850/// is transferred to the call in the callee.
1851///
1852/// 3. Otherwise, a call to objc_retain is inserted if the call in the caller is
1853/// a retainRV call.
1854static void
1856 const SmallVectorImpl<ReturnInst *> &Returns) {
1857 Module *Mod = CB.getModule();
1858 assert(objcarc::isRetainOrClaimRV(RVCallKind) && "unexpected ARC function");
1859 bool IsRetainRV = RVCallKind == objcarc::ARCInstKind::RetainRV,
1860 IsUnsafeClaimRV = !IsRetainRV;
1861
1862 for (auto *RI : Returns) {
1863 Value *RetOpnd = objcarc::GetRCIdentityRoot(RI->getOperand(0));
1864 bool InsertRetainCall = IsRetainRV;
1866
1867 // Walk backwards through the basic block looking for either a matching
1868 // autoreleaseRV call or an unannotated call.
1869 auto InstRange = llvm::make_range(++(RI->getIterator().getReverse()),
1870 RI->getParent()->rend());
1871 for (Instruction &I : llvm::make_early_inc_range(InstRange)) {
1872 // Ignore casts.
1873 if (isa<CastInst>(I))
1874 continue;
1875
1876 if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
1877 if (II->getIntrinsicID() != Intrinsic::objc_autoreleaseReturnValue ||
1878 !II->hasNUses(0) ||
1879 objcarc::GetRCIdentityRoot(II->getOperand(0)) != RetOpnd)
1880 break;
1881
1882 // If we've found a matching authoreleaseRV call:
1883 // - If claimRV is attached to the call, insert a call to objc_release
1884 // and erase the autoreleaseRV call.
1885 // - If retainRV is attached to the call, just erase the autoreleaseRV
1886 // call.
1887 if (IsUnsafeClaimRV) {
1888 Builder.SetInsertPoint(II);
1889 Function *IFn =
1890 Intrinsic::getDeclaration(Mod, Intrinsic::objc_release);
1891 Value *BC = Builder.CreateBitCast(RetOpnd, IFn->getArg(0)->getType());
1892 Builder.CreateCall(IFn, BC, "");
1893 }
1894 II->eraseFromParent();
1895 InsertRetainCall = false;
1896 break;
1897 }
1898
1899 auto *CI = dyn_cast<CallInst>(&I);
1900
1901 if (!CI)
1902 break;
1903
1904 if (objcarc::GetRCIdentityRoot(CI) != RetOpnd ||
1906 break;
1907
1908 // If we've found an unannotated call that defines RetOpnd, add a
1909 // "clang.arc.attachedcall" operand bundle.
1910 Value *BundleArgs[] = {*objcarc::getAttachedARCFunction(&CB)};
1911 OperandBundleDef OB("clang.arc.attachedcall", BundleArgs);
1912 auto *NewCall = CallBase::addOperandBundle(
1914 NewCall->copyMetadata(*CI);
1915 CI->replaceAllUsesWith(NewCall);
1916 CI->eraseFromParent();
1917 InsertRetainCall = false;
1918 break;
1919 }
1920
1921 if (InsertRetainCall) {
1922 // The retainRV is attached to the call and we've failed to find a
1923 // matching autoreleaseRV or an annotated call in the callee. Emit a call
1924 // to objc_retain.
1925 Builder.SetInsertPoint(RI);
1926 Function *IFn = Intrinsic::getDeclaration(Mod, Intrinsic::objc_retain);
1927 Value *BC = Builder.CreateBitCast(RetOpnd, IFn->getArg(0)->getType());
1928 Builder.CreateCall(IFn, BC, "");
1929 }
1930 }
1931}
1932
1933/// This function inlines the called function into the basic block of the
1934/// caller. This returns false if it is not possible to inline this call.
1935/// The program is still in a well defined state if this occurs though.
1936///
1937/// Note that this only does one level of inlining. For example, if the
1938/// instruction 'call B' is inlined, and 'B' calls 'C', then the call to 'C' now
1939/// exists in the instruction stream. Similarly this will inline a recursive
1940/// function by one level.
1942 bool MergeAttributes,
1943 AAResults *CalleeAAR,
1944 bool InsertLifetime,
1945 Function *ForwardVarArgsTo) {
1946 assert(CB.getParent() && CB.getFunction() && "Instruction not in function!");
1947
1948 // FIXME: we don't inline callbr yet.
1949 if (isa<CallBrInst>(CB))
1950 return InlineResult::failure("We don't inline callbr yet.");
1951
1952 // If IFI has any state in it, zap it before we fill it in.
1953 IFI.reset();
1954
1955 Function *CalledFunc = CB.getCalledFunction();
1956 if (!CalledFunc || // Can't inline external function or indirect
1957 CalledFunc->isDeclaration()) // call!
1958 return InlineResult::failure("external or indirect");
1959
1960 // The inliner does not know how to inline through calls with operand bundles
1961 // in general ...
1962 if (CB.hasOperandBundles()) {
1963 for (int i = 0, e = CB.getNumOperandBundles(); i != e; ++i) {
1965 // ... but it knows how to inline through "deopt" operand bundles ...
1967 continue;
1968 // ... and "funclet" operand bundles.
1970 continue;
1972 continue;
1974 continue;
1975
1976 return InlineResult::failure("unsupported operand bundle");
1977 }
1978 }
1979
1980 // If the call to the callee cannot throw, set the 'nounwind' flag on any
1981 // calls that we inline.
1982 bool MarkNoUnwind = CB.doesNotThrow();
1983
1984 BasicBlock *OrigBB = CB.getParent();
1985 Function *Caller = OrigBB->getParent();
1986
1987 // Do not inline strictfp function into non-strictfp one. It would require
1988 // conversion of all FP operations in host function to constrained intrinsics.
1989 if (CalledFunc->getAttributes().hasFnAttr(Attribute::StrictFP) &&
1990 !Caller->getAttributes().hasFnAttr(Attribute::StrictFP)) {
1991 return InlineResult::failure("incompatible strictfp attributes");
1992 }
1993
1994 // GC poses two hazards to inlining, which only occur when the callee has GC:
1995 // 1. If the caller has no GC, then the callee's GC must be propagated to the
1996 // caller.
1997 // 2. If the caller has a differing GC, it is invalid to inline.
1998 if (CalledFunc->hasGC()) {
1999 if (!Caller->hasGC())
2000 Caller->setGC(CalledFunc->getGC());
2001 else if (CalledFunc->getGC() != Caller->getGC())
2002 return InlineResult::failure("incompatible GC");
2003 }
2004
2005 // Get the personality function from the callee if it contains a landing pad.
2006 Constant *CalledPersonality =
2007 CalledFunc->hasPersonalityFn()
2008 ? CalledFunc->getPersonalityFn()->stripPointerCasts()
2009 : nullptr;
2010
2011 // Find the personality function used by the landing pads of the caller. If it
2012 // exists, then check to see that it matches the personality function used in
2013 // the callee.
2014 Constant *CallerPersonality =
2015 Caller->hasPersonalityFn()
2016 ? Caller->getPersonalityFn()->stripPointerCasts()
2017 : nullptr;
2018 if (CalledPersonality) {
2019 if (!CallerPersonality)
2020 Caller->setPersonalityFn(CalledPersonality);
2021 // If the personality functions match, then we can perform the
2022 // inlining. Otherwise, we can't inline.
2023 // TODO: This isn't 100% true. Some personality functions are proper
2024 // supersets of others and can be used in place of the other.
2025 else if (CalledPersonality != CallerPersonality)
2026 return InlineResult::failure("incompatible personality");
2027 }
2028
2029 // We need to figure out which funclet the callsite was in so that we may
2030 // properly nest the callee.
2031 Instruction *CallSiteEHPad = nullptr;
2032 if (CallerPersonality) {
2033 EHPersonality Personality = classifyEHPersonality(CallerPersonality);
2034 if (isScopedEHPersonality(Personality)) {
2035 std::optional<OperandBundleUse> ParentFunclet =
2037 if (ParentFunclet)
2038 CallSiteEHPad = cast<FuncletPadInst>(ParentFunclet->Inputs.front());
2039
2040 // OK, the inlining site is legal. What about the target function?
2041
2042 if (CallSiteEHPad) {
2043 if (Personality == EHPersonality::MSVC_CXX) {
2044 // The MSVC personality cannot tolerate catches getting inlined into
2045 // cleanup funclets.
2046 if (isa<CleanupPadInst>(CallSiteEHPad)) {
2047 // Ok, the call site is within a cleanuppad. Let's check the callee
2048 // for catchpads.
2049 for (const BasicBlock &CalledBB : *CalledFunc) {
2050 if (isa<CatchSwitchInst>(CalledBB.getFirstNonPHI()))
2051 return InlineResult::failure("catch in cleanup funclet");
2052 }
2053 }
2054 } else if (isAsynchronousEHPersonality(Personality)) {
2055 // SEH is even less tolerant, there may not be any sort of exceptional
2056 // funclet in the callee.
2057 for (const BasicBlock &CalledBB : *CalledFunc) {
2058 if (CalledBB.isEHPad())
2059 return InlineResult::failure("SEH in cleanup funclet");
2060 }
2061 }
2062 }
2063 }
2064 }
2065
2066 // Determine if we are dealing with a call in an EHPad which does not unwind
2067 // to caller.
2068 bool EHPadForCallUnwindsLocally = false;
2069 if (CallSiteEHPad && isa<CallInst>(CB)) {
2070 UnwindDestMemoTy FuncletUnwindMap;
2071 Value *CallSiteUnwindDestToken =
2072 getUnwindDestToken(CallSiteEHPad, FuncletUnwindMap);
2073
2074 EHPadForCallUnwindsLocally =
2075 CallSiteUnwindDestToken &&
2076 !isa<ConstantTokenNone>(CallSiteUnwindDestToken);
2077 }
2078
2079 // Get an iterator to the last basic block in the function, which will have
2080 // the new function inlined after it.
2081 Function::iterator LastBlock = --Caller->end();
2082
2083 // Make sure to capture all of the return instructions from the cloned
2084 // function.
2086 ClonedCodeInfo InlinedFunctionInfo;
2087 Function::iterator FirstNewBlock;
2088
2089 { // Scope to destroy VMap after cloning.
2090 ValueToValueMapTy VMap;
2091 struct ByValInit {
2092 Value *Dst;
2093 Value *Src;
2094 Type *Ty;
2095 };
2096 // Keep a list of pair (dst, src) to emit byval initializations.
2097 SmallVector<ByValInit, 4> ByValInits;
2098
2099 // When inlining a function that contains noalias scope metadata,
2100 // this metadata needs to be cloned so that the inlined blocks
2101 // have different "unique scopes" at every call site.
2102 // Track the metadata that must be cloned. Do this before other changes to
2103 // the function, so that we do not get in trouble when inlining caller ==
2104 // callee.
2105 ScopedAliasMetadataDeepCloner SAMetadataCloner(CB.getCalledFunction());
2106
2107 auto &DL = Caller->getParent()->getDataLayout();
2108
2109 // Calculate the vector of arguments to pass into the function cloner, which
2110 // matches up the formal to the actual argument values.
2111 auto AI = CB.arg_begin();
2112 unsigned ArgNo = 0;
2113 for (Function::arg_iterator I = CalledFunc->arg_begin(),
2114 E = CalledFunc->arg_end(); I != E; ++I, ++AI, ++ArgNo) {
2115 Value *ActualArg = *AI;
2116
2117 // When byval arguments actually inlined, we need to make the copy implied
2118 // by them explicit. However, we don't do this if the callee is readonly
2119 // or readnone, because the copy would be unneeded: the callee doesn't
2120 // modify the struct.
2121 if (CB.isByValArgument(ArgNo)) {
2122 ActualArg = HandleByValArgument(CB.getParamByValType(ArgNo), ActualArg,
2123 &CB, CalledFunc, IFI,
2124 CalledFunc->getParamAlign(ArgNo));
2125 if (ActualArg != *AI)
2126 ByValInits.push_back(
2127 {ActualArg, (Value *)*AI, CB.getParamByValType(ArgNo)});
2128 }
2129
2130 VMap[&*I] = ActualArg;
2131 }
2132
2133 // TODO: Remove this when users have been updated to the assume bundles.
2134 // Add alignment assumptions if necessary. We do this before the inlined
2135 // instructions are actually cloned into the caller so that we can easily
2136 // check what will be known at the start of the inlined code.
2137 AddAlignmentAssumptions(CB, IFI);
2138
2139 AssumptionCache *AC =
2140 IFI.GetAssumptionCache ? &IFI.GetAssumptionCache(*Caller) : nullptr;
2141
2142 /// Preserve all attributes on of the call and its parameters.
2143 salvageKnowledge(&CB, AC);
2144
2145 // We want the inliner to prune the code as it copies. We would LOVE to
2146 // have no dead or constant instructions leftover after inlining occurs
2147 // (which can happen, e.g., because an argument was constant), but we'll be
2148 // happy with whatever the cloner can do.
2149 CloneAndPruneFunctionInto(Caller, CalledFunc, VMap,
2150 /*ModuleLevelChanges=*/false, Returns, ".i",
2151 &InlinedFunctionInfo);
2152 // Remember the first block that is newly cloned over.
2153 FirstNewBlock = LastBlock; ++FirstNewBlock;
2154
2155 // Insert retainRV/clainRV runtime calls.
2157 if (RVCallKind != objcarc::ARCInstKind::None)
2158 inlineRetainOrClaimRVCalls(CB, RVCallKind, Returns);
2159
2160 // Updated caller/callee profiles only when requested. For sample loader
2161 // inlining, the context-sensitive inlinee profile doesn't need to be
2162 // subtracted from callee profile, and the inlined clone also doesn't need
2163 // to be scaled based on call site count.
2164 if (IFI.UpdateProfile) {
2165 if (IFI.CallerBFI != nullptr && IFI.CalleeBFI != nullptr)
2166 // Update the BFI of blocks cloned into the caller.
2167 updateCallerBFI(OrigBB, VMap, IFI.CallerBFI, IFI.CalleeBFI,
2168 CalledFunc->front());
2169
2170 if (auto Profile = CalledFunc->getEntryCount())
2171 updateCallProfile(CalledFunc, VMap, *Profile, CB, IFI.PSI,
2172 IFI.CallerBFI);
2173 }
2174
2175 // Inject byval arguments initialization.
2176 for (ByValInit &Init : ByValInits)
2177 HandleByValArgumentInit(Init.Ty, Init.Dst, Init.Src, Caller->getParent(),
2178 &*FirstNewBlock, IFI, CalledFunc);
2179
2180 std::optional<OperandBundleUse> ParentDeopt =
2182 if (ParentDeopt) {
2184
2185 for (auto &VH : InlinedFunctionInfo.OperandBundleCallSites) {
2186 CallBase *ICS = dyn_cast_or_null<CallBase>(VH);
2187 if (!ICS)
2188 continue; // instruction was DCE'd or RAUW'ed to undef
2189
2190 OpDefs.clear();
2191
2192 OpDefs.reserve(ICS->getNumOperandBundles());
2193
2194 for (unsigned COBi = 0, COBe = ICS->getNumOperandBundles(); COBi < COBe;
2195 ++COBi) {
2196 auto ChildOB = ICS->getOperandBundleAt(COBi);
2197 if (ChildOB.getTagID() != LLVMContext::OB_deopt) {
2198 // If the inlined call has other operand bundles, let them be
2199 OpDefs.emplace_back(ChildOB);
2200 continue;
2201 }
2202
2203 // It may be useful to separate this logic (of handling operand
2204 // bundles) out to a separate "policy" component if this gets crowded.
2205 // Prepend the parent's deoptimization continuation to the newly
2206 // inlined call's deoptimization continuation.
2207 std::vector<Value *> MergedDeoptArgs;
2208 MergedDeoptArgs.reserve(ParentDeopt->Inputs.size() +
2209 ChildOB.Inputs.size());
2210
2211 llvm::append_range(MergedDeoptArgs, ParentDeopt->Inputs);
2212 llvm::append_range(MergedDeoptArgs, ChildOB.Inputs);
2213
2214 OpDefs.emplace_back("deopt", std::move(MergedDeoptArgs));
2215 }
2216
2217 Instruction *NewI = CallBase::Create(ICS, OpDefs, ICS);
2218
2219 // Note: the RAUW does the appropriate fixup in VMap, so we need to do
2220 // this even if the call returns void.
2221 ICS->replaceAllUsesWith(NewI);
2222
2223 VH = nullptr;
2224 ICS->eraseFromParent();
2225 }
2226 }
2227
2228 // For 'nodebug' functions, the associated DISubprogram is always null.
2229 // Conservatively avoid propagating the callsite debug location to
2230 // instructions inlined from a function whose DISubprogram is not null.
2231 fixupLineNumbers(Caller, FirstNewBlock, &CB,
2232 CalledFunc->getSubprogram() != nullptr);
2233
2234 if (isAssignmentTrackingEnabled(*Caller->getParent())) {
2235 // Interpret inlined stores to caller-local variables as assignments.
2236 trackInlinedStores(FirstNewBlock, Caller->end(), CB);
2237
2238 // Update DIAssignID metadata attachments and uses so that they are
2239 // unique to this inlined instance.
2240 fixupAssignments(FirstNewBlock, Caller->end());
2241 }
2242
2243 // Now clone the inlined noalias scope metadata.
2244 SAMetadataCloner.clone();
2245 SAMetadataCloner.remap(FirstNewBlock, Caller->end());
2246
2247 // Add noalias metadata if necessary.
2248 AddAliasScopeMetadata(CB, VMap, DL, CalleeAAR, InlinedFunctionInfo);
2249
2250 // Clone return attributes on the callsite into the calls within the inlined
2251 // function which feed into its return value.
2252 AddReturnAttributes(CB, VMap);
2253
2254 propagateMemProfMetadata(CalledFunc, CB,
2255 InlinedFunctionInfo.ContainsMemProfMetadata, VMap);
2256
2257 // Propagate metadata on the callsite if necessary.
2258 PropagateCallSiteMetadata(CB, FirstNewBlock, Caller->end());
2259
2260 // Register any cloned assumptions.
2261 if (IFI.GetAssumptionCache)
2262 for (BasicBlock &NewBlock :
2263 make_range(FirstNewBlock->getIterator(), Caller->end()))
2264 for (Instruction &I : NewBlock)
2265 if (auto *II = dyn_cast<AssumeInst>(&I))
2266 IFI.GetAssumptionCache(*Caller).registerAssumption(II);
2267 }
2268
2269 // If there are any alloca instructions in the block that used to be the entry
2270 // block for the callee, move them to the entry block of the caller. First
2271 // calculate which instruction they should be inserted before. We insert the
2272 // instructions at the end of the current alloca list.
2273 {
2274 BasicBlock::iterator InsertPoint = Caller->begin()->begin();
2275 for (BasicBlock::iterator I = FirstNewBlock->begin(),
2276 E = FirstNewBlock->end(); I != E; ) {
2277 AllocaInst *AI = dyn_cast<AllocaInst>(I++);
2278 if (!AI) continue;
2279
2280 // If the alloca is now dead, remove it. This often occurs due to code
2281 // specialization.
2282 if (AI->use_empty()) {
2283 AI->eraseFromParent();
2284 continue;
2285 }
2286
2288 continue;
2289
2290 // Keep track of the static allocas that we inline into the caller.
2291 IFI.StaticAllocas.push_back(AI);
2292
2293 // Scan for the block of allocas that we can move over, and move them
2294 // all at once.
2295 while (isa<AllocaInst>(I) &&
2296 !cast<AllocaInst>(I)->use_empty() &&
2297 allocaWouldBeStaticInEntry(cast<AllocaInst>(I))) {
2298 IFI.StaticAllocas.push_back(cast<AllocaInst>(I));
2299 ++I;
2300 }
2301
2302 // Transfer all of the allocas over in a block. Using splice means
2303 // that the instructions aren't removed from the symbol table, then
2304 // reinserted.
2305 Caller->getEntryBlock().splice(InsertPoint, &*FirstNewBlock,
2306 AI->getIterator(), I);
2307 }
2308 }
2309
2310 SmallVector<Value*,4> VarArgsToForward;
2311 SmallVector<AttributeSet, 4> VarArgsAttrs;
2312 for (unsigned i = CalledFunc->getFunctionType()->getNumParams();
2313 i < CB.arg_size(); i++) {
2314 VarArgsToForward.push_back(CB.getArgOperand(i));
2315 VarArgsAttrs.push_back(CB.getAttributes().getParamAttrs(i));
2316 }
2317
2318 bool InlinedMustTailCalls = false, InlinedDeoptimizeCalls = false;
2319 if (InlinedFunctionInfo.ContainsCalls) {
2320 CallInst::TailCallKind CallSiteTailKind = CallInst::TCK_None;
2321 if (CallInst *CI = dyn_cast<CallInst>(&CB))
2322 CallSiteTailKind = CI->getTailCallKind();
2323
2324 // For inlining purposes, the "notail" marker is the same as no marker.
2325 if (CallSiteTailKind == CallInst::TCK_NoTail)
2326 CallSiteTailKind = CallInst::TCK_None;
2327
2328 for (Function::iterator BB = FirstNewBlock, E = Caller->end(); BB != E;
2329 ++BB) {
2331 CallInst *CI = dyn_cast<CallInst>(&I);
2332 if (!CI)
2333 continue;
2334
2335 // Forward varargs from inlined call site to calls to the
2336 // ForwardVarArgsTo function, if requested, and to musttail calls.
2337 if (!VarArgsToForward.empty() &&
2338 ((ForwardVarArgsTo &&
2339 CI->getCalledFunction() == ForwardVarArgsTo) ||
2340 CI->isMustTailCall())) {
2341 // Collect attributes for non-vararg parameters.
2342 AttributeList Attrs = CI->getAttributes();
2344 if (!Attrs.isEmpty() || !VarArgsAttrs.empty()) {
2345 for (unsigned ArgNo = 0;
2346 ArgNo < CI->getFunctionType()->getNumParams(); ++ArgNo)
2347 ArgAttrs.push_back(Attrs.getParamAttrs(ArgNo));
2348 }
2349
2350 // Add VarArg attributes.
2351 ArgAttrs.append(VarArgsAttrs.begin(), VarArgsAttrs.end());
2352 Attrs = AttributeList::get(CI->getContext(), Attrs.getFnAttrs(),
2353 Attrs.getRetAttrs(), ArgAttrs);
2354 // Add VarArgs to existing parameters.
2355 SmallVector<Value *, 6> Params(CI->args());
2356 Params.append(VarArgsToForward.begin(), VarArgsToForward.end());
2357 CallInst *NewCI = CallInst::Create(
2358 CI->getFunctionType(), CI->getCalledOperand(), Params, "", CI);
2359 NewCI->setDebugLoc(CI->getDebugLoc());
2360 NewCI->setAttributes(Attrs);
2361 NewCI->setCallingConv(CI->getCallingConv());
2362 CI->replaceAllUsesWith(NewCI);
2363 CI->eraseFromParent();
2364 CI = NewCI;
2365 }
2366
2367 if (Function *F = CI->getCalledFunction())
2368 InlinedDeoptimizeCalls |=
2369 F->getIntrinsicID() == Intrinsic::experimental_deoptimize;
2370
2371 // We need to reduce the strength of any inlined tail calls. For
2372 // musttail, we have to avoid introducing potential unbounded stack
2373 // growth. For example, if functions 'f' and 'g' are mutually recursive
2374 // with musttail, we can inline 'g' into 'f' so long as we preserve
2375 // musttail on the cloned call to 'f'. If either the inlined call site
2376 // or the cloned call site is *not* musttail, the program already has
2377 // one frame of stack growth, so it's safe to remove musttail. Here is
2378 // a table of example transformations:
2379 //
2380 // f -> musttail g -> musttail f ==> f -> musttail f
2381 // f -> musttail g -> tail f ==> f -> tail f
2382 // f -> g -> musttail f ==> f -> f
2383 // f -> g -> tail f ==> f -> f
2384 //
2385 // Inlined notail calls should remain notail calls.
2386 CallInst::TailCallKind ChildTCK = CI->getTailCallKind();
2387 if (ChildTCK != CallInst::TCK_NoTail)
2388 ChildTCK = std::min(CallSiteTailKind, ChildTCK);
2389 CI->setTailCallKind(ChildTCK);
2390 InlinedMustTailCalls |= CI->isMustTailCall();
2391
2392 // Call sites inlined through a 'nounwind' call site should be
2393 // 'nounwind' as well. However, avoid marking call sites explicitly
2394 // where possible. This helps expose more opportunities for CSE after
2395 // inlining, commonly when the callee is an intrinsic.
2396 if (MarkNoUnwind && !CI->doesNotThrow())
2397 CI->setDoesNotThrow();
2398 }
2399 }
2400 }
2401
2402 // Leave lifetime markers for the static alloca's, scoping them to the
2403 // function we just inlined.
2404 // We need to insert lifetime intrinsics even at O0 to avoid invalid
2405 // access caused by multithreaded coroutines. The check
2406 // `Caller->isPresplitCoroutine()` would affect AlwaysInliner at O0 only.
2407 if ((InsertLifetime || Caller->isPresplitCoroutine()) &&
2408 !IFI.StaticAllocas.empty()) {
2409 IRBuilder<> builder(&FirstNewBlock->front());
2410 for (unsigned ai = 0, ae = IFI.StaticAllocas.size(); ai != ae; ++ai) {
2411 AllocaInst *AI = IFI.StaticAllocas[ai];
2412 // Don't mark swifterror allocas. They can't have bitcast uses.
2413 if (AI->isSwiftError())
2414 continue;
2415
2416 // If the alloca is already scoped to something smaller than the whole
2417 // function then there's no need to add redundant, less accurate markers.
2418 if (hasLifetimeMarkers(AI))
2419 continue;
2420
2421 // Try to determine the size of the allocation.
2422 ConstantInt *AllocaSize = nullptr;
2423 if (ConstantInt *AIArraySize =
2424 dyn_cast<ConstantInt>(AI->getArraySize())) {
2425 auto &DL = Caller->getParent()->getDataLayout();
2426 Type *AllocaType = AI->getAllocatedType();
2427 TypeSize AllocaTypeSize = DL.getTypeAllocSize(AllocaType);
2428 uint64_t AllocaArraySize = AIArraySize->getLimitedValue();
2429
2430 // Don't add markers for zero-sized allocas.
2431 if (AllocaArraySize == 0)
2432 continue;
2433
2434 // Check that array size doesn't saturate uint64_t and doesn't
2435 // overflow when it's multiplied by type size.
2436 if (!AllocaTypeSize.isScalable() &&
2437 AllocaArraySize != std::numeric_limits<uint64_t>::max() &&
2438 std::numeric_limits<uint64_t>::max() / AllocaArraySize >=
2439 AllocaTypeSize.getFixedValue()) {
2440 AllocaSize = ConstantInt::get(Type::getInt64Ty(AI->getContext()),
2441 AllocaArraySize * AllocaTypeSize);
2442 }
2443 }
2444
2445 builder.CreateLifetimeStart(AI, AllocaSize);
2446 for (ReturnInst *RI : Returns) {
2447 // Don't insert llvm.lifetime.end calls between a musttail or deoptimize
2448 // call and a return. The return kills all local allocas.
2449 if (InlinedMustTailCalls &&
2451 continue;
2452 if (InlinedDeoptimizeCalls &&
2454 continue;
2455 IRBuilder<>(RI).CreateLifetimeEnd(AI, AllocaSize);
2456 }
2457 }
2458 }
2459
2460 // If the inlined code contained dynamic alloca instructions, wrap the inlined
2461 // code with llvm.stacksave/llvm.stackrestore intrinsics.
2462 if (InlinedFunctionInfo.ContainsDynamicAllocas) {
2463 Module *M = Caller->getParent();
2464 // Get the two intrinsics we care about.
2465 Function *StackSave = Intrinsic::getDeclaration(M, Intrinsic::stacksave);
2466 Function *StackRestore=Intrinsic::getDeclaration(M,Intrinsic::stackrestore);
2467
2468 // Insert the llvm.stacksave.
2469 CallInst *SavedPtr = IRBuilder<>(&*FirstNewBlock, FirstNewBlock->begin())
2470 .CreateCall(StackSave, {}, "savedstack");
2471
2472 // Insert a call to llvm.stackrestore before any return instructions in the
2473 // inlined function.
2474 for (ReturnInst *RI : Returns) {
2475 // Don't insert llvm.stackrestore calls between a musttail or deoptimize
2476 // call and a return. The return will restore the stack pointer.
2477 if (InlinedMustTailCalls && RI->getParent()->getTerminatingMustTailCall())
2478 continue;
2479 if (InlinedDeoptimizeCalls && RI->getParent()->getTerminatingDeoptimizeCall())
2480 continue;
2481 IRBuilder<>(RI).CreateCall(StackRestore, SavedPtr);
2482 }
2483 }
2484
2485 // If we are inlining for an invoke instruction, we must make sure to rewrite
2486 // any call instructions into invoke instructions. This is sensitive to which
2487 // funclet pads were top-level in the inlinee, so must be done before
2488 // rewriting the "parent pad" links.
2489 if (auto *II = dyn_cast<InvokeInst>(&CB)) {
2490 BasicBlock *UnwindDest = II->getUnwindDest();
2491 Instruction *FirstNonPHI = UnwindDest->getFirstNonPHI();
2492 if (isa<LandingPadInst>(FirstNonPHI)) {
2493 HandleInlinedLandingPad(II, &*FirstNewBlock, InlinedFunctionInfo);
2494 } else {
2495 HandleInlinedEHPad(II, &*FirstNewBlock, InlinedFunctionInfo);
2496 }
2497 }
2498
2499 // Update the lexical scopes of the new funclets and callsites.
2500 // Anything that had 'none' as its parent is now nested inside the callsite's
2501 // EHPad.
2502 if (CallSiteEHPad) {
2503 for (Function::iterator BB = FirstNewBlock->getIterator(),
2504 E = Caller->end();
2505 BB != E; ++BB) {
2506 // Add bundle operands to inlined call sites.
2507 PropagateOperandBundles(BB, CallSiteEHPad);
2508
2509 // It is problematic if the inlinee has a cleanupret which unwinds to
2510 // caller and we inline it into a call site which doesn't unwind but into
2511 // an EH pad that does. Such an edge must be dynamically unreachable.
2512 // As such, we replace the cleanupret with unreachable.
2513 if (auto *CleanupRet = dyn_cast<CleanupReturnInst>(BB->getTerminator()))
2514 if (CleanupRet->unwindsToCaller() && EHPadForCallUnwindsLocally)
2515 changeToUnreachable(CleanupRet);
2516
2517 Instruction *I = BB->getFirstNonPHI();
2518 if (!I->isEHPad())
2519 continue;
2520
2521 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(I)) {
2522 if (isa<ConstantTokenNone>(CatchSwitch->getParentPad()))
2523 CatchSwitch->setParentPad(CallSiteEHPad);
2524 } else {
2525 auto *FPI = cast<FuncletPadInst>(I);
2526 if (isa<ConstantTokenNone>(FPI->getParentPad()))
2527 FPI->setParentPad(CallSiteEHPad);
2528 }
2529 }
2530 }
2531
2532 if (InlinedDeoptimizeCalls) {
2533 // We need to at least remove the deoptimizing returns from the Return set,
2534 // so that the control flow from those returns does not get merged into the
2535 // caller (but terminate it instead). If the caller's return type does not
2536 // match the callee's return type, we also need to change the return type of
2537 // the intrinsic.
2538 if (Caller->getReturnType() == CB.getType()) {
2539 llvm::erase_if(Returns, [](ReturnInst *RI) {
2540 return RI->getParent()->getTerminatingDeoptimizeCall() != nullptr;
2541 });
2542 } else {
2543 SmallVector<ReturnInst *, 8> NormalReturns;
2544 Function *NewDeoptIntrinsic = Intrinsic::getDeclaration(
2545 Caller->getParent(), Intrinsic::experimental_deoptimize,
2546 {Caller->getReturnType()});
2547
2548 for (ReturnInst *RI : Returns) {
2549 CallInst *DeoptCall = RI->getParent()->getTerminatingDeoptimizeCall();
2550 if (!DeoptCall) {
2551 NormalReturns.push_back(RI);
2552 continue;
2553 }
2554
2555 // The calling convention on the deoptimize call itself may be bogus,
2556 // since the code we're inlining may have undefined behavior (and may
2557 // never actually execute at runtime); but all
2558 // @llvm.experimental.deoptimize declarations have to have the same
2559 // calling convention in a well-formed module.
2560 auto CallingConv = DeoptCall->getCalledFunction()->getCallingConv();
2561 NewDeoptIntrinsic->setCallingConv(CallingConv);
2562 auto *CurBB = RI->getParent();
2563 RI->eraseFromParent();
2564
2565 SmallVector<Value *, 4> CallArgs(DeoptCall->args());
2566
2568 DeoptCall->getOperandBundlesAsDefs(OpBundles);
2569 auto DeoptAttributes = DeoptCall->getAttributes();
2570 DeoptCall->eraseFromParent();
2571 assert(!OpBundles.empty() &&
2572 "Expected at least the deopt operand bundle");
2573
2574 IRBuilder<> Builder(CurBB);
2575 CallInst *NewDeoptCall =
2576 Builder.CreateCall(NewDeoptIntrinsic, CallArgs, OpBundles);
2577 NewDeoptCall->setCallingConv(CallingConv);
2578 NewDeoptCall->setAttributes(DeoptAttributes);
2579 if (NewDeoptCall->getType()->isVoidTy())
2580 Builder.CreateRetVoid();
2581 else
2582 Builder.CreateRet(NewDeoptCall);
2583 }
2584
2585 // Leave behind the normal returns so we can merge control flow.
2586 std::swap(Returns, NormalReturns);
2587 }
2588 }
2589
2590 // Handle any inlined musttail call sites. In order for a new call site to be
2591 // musttail, the source of the clone and the inlined call site must have been
2592 // musttail. Therefore it's safe to return without merging control into the
2593 // phi below.
2594 if (InlinedMustTailCalls) {
2595 // Check if we need to bitcast the result of any musttail calls.
2596 Type *NewRetTy = Caller->getReturnType();
2597 bool NeedBitCast = !CB.use_empty() && CB.getType() != NewRetTy;
2598
2599 // Handle the returns preceded by musttail calls separately.
2600 SmallVector<ReturnInst *, 8> NormalReturns;
2601 for (ReturnInst *RI : Returns) {
2602 CallInst *ReturnedMustTail =
2604 if (!ReturnedMustTail) {
2605 NormalReturns.push_back(RI);
2606 continue;
2607 }
2608 if (!NeedBitCast)
2609 continue;
2610
2611 // Delete the old return and any preceding bitcast.
2612 BasicBlock *CurBB = RI->getParent();
2613 auto *OldCast = dyn_cast_or_null<BitCastInst>(RI->getReturnValue());
2614 RI->eraseFromParent();
2615 if (OldCast)
2616 OldCast->eraseFromParent();
2617
2618 // Insert a new bitcast and return with the right type.
2619 IRBuilder<> Builder(CurBB);
2620 Builder.CreateRet(Builder.CreateBitCast(ReturnedMustTail, NewRetTy));
2621 }
2622
2623 // Leave behind the normal returns so we can merge control flow.
2624 std::swap(Returns, NormalReturns);
2625 }
2626
2627 // Now that all of the transforms on the inlined code have taken place but
2628 // before we splice the inlined code into the CFG and lose track of which
2629 // blocks were actually inlined, collect the call sites. We only do this if
2630 // call graph updates weren't requested, as those provide value handle based
2631 // tracking of inlined call sites instead. Calls to intrinsics are not
2632 // collected because they are not inlineable.
2633 if (InlinedFunctionInfo.ContainsCalls) {
2634 // Otherwise just collect the raw call sites that were inlined.
2635 for (BasicBlock &NewBB :
2636 make_range(FirstNewBlock->getIterator(), Caller->end()))
2637 for (Instruction &I : NewBB)
2638 if (auto *CB = dyn_cast<CallBase>(&I))
2639 if (!(CB->getCalledFunction() &&
2641 IFI.InlinedCallSites.push_back(CB);
2642 }
2643
2644 // If we cloned in _exactly one_ basic block, and if that block ends in a
2645 // return instruction, we splice the body of the inlined callee directly into
2646 // the calling basic block.
2647 if (Returns.size() == 1 && std::distance(FirstNewBlock, Caller->end()) == 1) {
2648 // Move all of the instructions right before the call.
2649 OrigBB->splice(CB.getIterator(), &*FirstNewBlock, FirstNewBlock->begin(),
2650 FirstNewBlock->end());
2651 // Remove the cloned basic block.
2652 Caller->back().eraseFromParent();
2653
2654 // If the call site was an invoke instruction, add a branch to the normal
2655 // destination.
2656 if (InvokeInst *II = dyn_cast<InvokeInst>(&CB)) {
2657 BranchInst *NewBr = BranchInst::Create(II->getNormalDest(), &CB);
2658 NewBr->setDebugLoc(Returns[0]->getDebugLoc());
2659 }
2660
2661 // If the return instruction returned a value, replace uses of the call with
2662 // uses of the returned value.
2663 if (!CB.use_empty()) {
2664 ReturnInst *R = Returns[0];
2665 if (&CB == R->getReturnValue())
2667 else
2668 CB.replaceAllUsesWith(R->getReturnValue());
2669 }
2670 // Since we are now done with the Call/Invoke, we can delete it.
2671 CB.eraseFromParent();
2672
2673 // Since we are now done with the return instruction, delete it also.
2674 Returns[0]->eraseFromParent();
2675
2676 if (MergeAttributes)
2677 AttributeFuncs::mergeAttributesForInlining(*Caller, *CalledFunc);
2678
2679 // We are now done with the inlining.
2680 return InlineResult::success();
2681 }
2682
2683 // Otherwise, we have the normal case, of more than one block to inline or
2684 // multiple return sites.
2685
2686 // We want to clone the entire callee function into the hole between the
2687 // "starter" and "ender" blocks. How we accomplish this depends on whether
2688 // this is an invoke instruction or a call instruction.
2689 BasicBlock *AfterCallBB;
2690 BranchInst *CreatedBranchToNormalDest = nullptr;
2691 if (InvokeInst *II = dyn_cast<InvokeInst>(&CB)) {
2692
2693 // Add an unconditional branch to make this look like the CallInst case...
2694 CreatedBranchToNormalDest = BranchInst::Create(II->getNormalDest(), &CB);
2695
2696 // Split the basic block. This guarantees that no PHI nodes will have to be
2697 // updated due to new incoming edges, and make the invoke case more
2698 // symmetric to the call case.
2699 AfterCallBB =
2700 OrigBB->splitBasicBlock(CreatedBranchToNormalDest->getIterator(),
2701 CalledFunc->getName() + ".exit");
2702
2703 } else { // It's a call
2704 // If this is a call instruction, we need to split the basic block that
2705 // the call lives in.
2706 //
2707 AfterCallBB = OrigBB->splitBasicBlock(CB.getIterator(),
2708 CalledFunc->getName() + ".exit");
2709 }
2710
2711 if (IFI.CallerBFI) {
2712 // Copy original BB's block frequency to AfterCallBB
2714 AfterCallBB, IFI.CallerBFI->getBlockFreq(OrigBB).getFrequency());
2715 }
2716
2717 // Change the branch that used to go to AfterCallBB to branch to the first
2718 // basic block of the inlined function.
2719 //
2720 Instruction *Br = OrigBB->getTerminator();
2721 assert(Br && Br->getOpcode() == Instruction::Br &&
2722 "splitBasicBlock broken!");
2723 Br->setOperand(0, &*FirstNewBlock);
2724
2725 // Now that the function is correct, make it a little bit nicer. In
2726 // particular, move the basic blocks inserted from the end of the function
2727 // into the space made by splitting the source basic block.
2728 Caller->splice(AfterCallBB->getIterator(), Caller, FirstNewBlock,
2729 Caller->end());
2730
2731 // Handle all of the return instructions that we just cloned in, and eliminate
2732 // any users of the original call/invoke instruction.
2733 Type *RTy = CalledFunc->getReturnType();
2734
2735 PHINode *PHI = nullptr;
2736 if (Returns.size() > 1) {
2737 // The PHI node should go at the front of the new basic block to merge all
2738 // possible incoming values.
2739 if (!CB.use_empty()) {
2740 PHI = PHINode::Create(RTy, Returns.size(), CB.getName(),
2741 &AfterCallBB->front());
2742 // Anything that used the result of the function call should now use the
2743 // PHI node as their operand.
2745 }
2746
2747 // Loop over all of the return instructions adding entries to the PHI node
2748 // as appropriate.
2749 if (PHI) {
2750 for (unsigned i = 0, e = Returns.size(); i != e; ++i) {
2751 ReturnInst *RI = Returns[i];
2752 assert(RI->getReturnValue()->getType() == PHI->getType() &&
2753 "Ret value not consistent in function!");
2754 PHI->addIncoming(RI->getReturnValue(), RI->getParent());
2755 }
2756 }
2757
2758 // Add a branch to the merge points and remove return instructions.
2759 DebugLoc Loc;
2760 for (unsigned i = 0, e = Returns.size(); i != e; ++i) {
2761 ReturnInst *RI = Returns[i];
2762 BranchInst* BI = BranchInst::Create(AfterCallBB, RI);
2763 Loc = RI->getDebugLoc();
2764 BI->setDebugLoc(Loc);
2765 RI->eraseFromParent();
2766 }
2767 // We need to set the debug location to *somewhere* inside the
2768 // inlined function. The line number may be nonsensical, but the
2769 // instruction will at least be associated with the right
2770 // function.
2771 if (CreatedBranchToNormalDest)
2772 CreatedBranchToNormalDest->setDebugLoc(Loc);
2773 } else if (!Returns.empty()) {
2774 // Otherwise, if there is exactly one return value, just replace anything
2775 // using the return value of the call with the computed value.
2776 if (!CB.use_empty()) {
2777 if (&CB == Returns[0]->getReturnValue())
2779 else
2780 CB.replaceAllUsesWith(Returns[0]->getReturnValue());
2781 }
2782
2783 // Update PHI nodes that use the ReturnBB to use the AfterCallBB.
2784 BasicBlock *ReturnBB = Returns[0]->getParent();
2785 ReturnBB->replaceAllUsesWith(AfterCallBB);
2786
2787 // Splice the code from the return block into the block that it will return
2788 // to, which contains the code that was after the call.
2789 AfterCallBB->splice(AfterCallBB->begin(), ReturnBB);
2790
2791 if (CreatedBranchToNormalDest)
2792 CreatedBranchToNormalDest->setDebugLoc(Returns[0]->getDebugLoc());
2793
2794 // Delete the return instruction now and empty ReturnBB now.
2795 Returns[0]->eraseFromParent();
2796 ReturnBB->eraseFromParent();
2797 } else if (!CB.use_empty()) {
2798 // No returns, but something is using the return value of the call. Just
2799 // nuke the result.
2801 }
2802
2803 // Since we are now done with the Call/Invoke, we can delete it.
2804 CB.eraseFromParent();
2805
2806 // If we inlined any musttail calls and the original return is now
2807 // unreachable, delete it. It can only contain a bitcast and ret.
2808 if (InlinedMustTailCalls && pred_empty(AfterCallBB))
2809 AfterCallBB->eraseFromParent();
2810
2811 // We should always be able to fold the entry block of the function into the
2812 // single predecessor of the block...
2813 assert(cast<BranchInst>(Br)->isUnconditional() && "splitBasicBlock broken!");
2814 BasicBlock *CalleeEntry = cast<BranchInst>(Br)->getSuccessor(0);
2815
2816 // Splice the code entry block into calling block, right before the
2817 // unconditional branch.
2818 CalleeEntry->replaceAllUsesWith(OrigBB); // Update PHI nodes
2819 OrigBB->splice(Br->getIterator(), CalleeEntry);
2820
2821 // Remove the unconditional branch.
2822 Br->eraseFromParent();
2823
2824 // Now we can remove the CalleeEntry block, which is now empty.
2825 CalleeEntry->eraseFromParent();
2826
2827 // If we inserted a phi node, check to see if it has a single value (e.g. all
2828 // the entries are the same or undef). If so, remove the PHI so it doesn't
2829 // block other optimizations.
2830 if (PHI) {
2831 AssumptionCache *AC =
2832 IFI.GetAssumptionCache ? &IFI.GetAssumptionCache(*Caller) : nullptr;
2833 auto &DL = Caller->getParent()->getDataLayout();
2834 if (Value *V = simplifyInstruction(PHI, {DL, nullptr, nullptr, AC})) {
2835 PHI->replaceAllUsesWith(V);
2836 PHI->eraseFromParent();
2837 }
2838 }
2839
2840 if (MergeAttributes)
2841 AttributeFuncs::mergeAttributesForInlining(*Caller, *CalledFunc);
2842
2843 return InlineResult::success();
2844}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Rewrite undef for PHI
assume builder
assume Assume Builder
static void UpdatePHINodes(BasicBlock *OrigBB, BasicBlock *NewBB, ArrayRef< BasicBlock * > Preds, BranchInst *BI, bool HasLoopExit)
Update the PHI nodes in OrigBB to include the values coming from NewBB.
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static cl::opt< bool > NoAliases("csky-no-aliases", cl::desc("Disable the emission of assembler pseudo instructions"), cl::init(false), cl::Hidden)
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the DenseMap class.
std::string Name
uint64_t Size
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static at::StorageToVarsMap collectEscapedLocals(const DataLayout &DL, const CallBase &CB)
Find Alloca and linked DbgAssignIntrinsic for locals escaped by CB.
static void fixupLineNumbers(Function *Fn, Function::iterator FI, Instruction *TheCall, bool CalleeHasDebugInfo)
Update inlined instructions' line numbers to to encode location where these instructions are inlined.
static void removeCallsiteMetadata(CallBase *Call)
static void propagateMemProfHelper(const CallBase *OrigCall, CallBase *ClonedCall, MDNode *InlinedCallsiteMD)
static Value * getUnwindDestToken(Instruction *EHPad, UnwindDestMemoTy &MemoMap)
Given an EH pad, find where it unwinds.
static cl::opt< bool > PreserveAlignmentAssumptions("preserve-alignment-assumptions-during-inlining", cl::init(false), cl::Hidden, cl::desc("Convert align attributes to assumptions during inlining."))
static void AddReturnAttributes(CallBase &CB, ValueToValueMapTy &VMap)
static void HandleInlinedLandingPad(InvokeInst *II, BasicBlock *FirstNewBlock, ClonedCodeInfo &InlinedCodeInfo)
If we inlined an invoke site, we need to convert calls in the body of the inlined function into invok...
static Value * getUnwindDestTokenHelper(Instruction *EHPad, UnwindDestMemoTy &MemoMap)
Helper for getUnwindDestToken that does the descendant-ward part of the search.
static BasicBlock * HandleCallsInBlockInlinedThroughInvoke(BasicBlock *BB, BasicBlock *UnwindEdge, UnwindDestMemoTy *FuncletUnwindMap=nullptr)
When we inline a basic block into an invoke, we have to turn all of the calls that can throw into inv...
static DebugLoc inlineDebugLoc(DebugLoc OrigDL, DILocation *InlinedAt, LLVMContext &Ctx, DenseMap< const MDNode *, MDNode * > &IANodes)
Returns a DebugLoc for a new DILocation which is a clone of OrigDL inlined at InlinedAt.
static cl::opt< bool > UpdateReturnAttributes("update-return-attrs", cl::init(true), cl::Hidden, cl::desc("Update return attributes on calls within inlined body"))
static cl::opt< bool > UseNoAliasIntrinsic("use-noalias-intrinsic-during-inlining", cl::Hidden, cl::init(true), cl::desc("Use the llvm.experimental.noalias.scope.decl " "intrinsic during inlining."))
static AttrBuilder IdentifyValidAttributes(CallBase &CB)
static void PropagateCallSiteMetadata(CallBase &CB, Function::iterator FStart, Function::iterator FEnd)
When inlining a call site that has !llvm.mem.parallel_loop_access, !llvm.access.group,...
static void propagateMemProfMetadata(Function *Callee, CallBase &CB, bool ContainsMemProfMetadata, const ValueMap< const Value *, WeakTrackingVH > &VMap)
static void updateCallProfile(Function *Callee, const ValueToValueMapTy &VMap, const ProfileCount &CalleeEntryCount, const CallBase &TheCall, ProfileSummaryInfo *PSI, BlockFrequencyInfo *CallerBFI)
Update the branch metadata for cloned call instructions.
static void updateCallerBFI(BasicBlock *CallSiteBlock, const ValueToValueMapTy &VMap, BlockFrequencyInfo *CallerBFI, BlockFrequencyInfo *CalleeBFI, const BasicBlock &CalleeEntryBlock)
Update the block frequencies of the caller after a callee has been inlined.
static void HandleByValArgumentInit(Type *ByValType, Value *Dst, Value *Src, Module *M, BasicBlock *InsertBlock, InlineFunctionInfo &IFI, Function *CalledFunc)
static cl::opt< bool > EnableNoAliasConversion("enable-noalias-to-md-conversion", cl::init(true), cl::Hidden, cl::desc("Convert noalias attributes to metadata during inlining."))
static void AddAliasScopeMetadata(CallBase &CB, ValueToValueMapTy &VMap, const DataLayout &DL, AAResults *CalleeAAR, ClonedCodeInfo &InlinedFunctionInfo)
If the inlined function has noalias arguments, then add new alias scopes for each noalias argument,...
static void HandleInlinedEHPad(InvokeInst *II, BasicBlock *FirstNewBlock, ClonedCodeInfo &InlinedCodeInfo)
If we inlined an invoke site, we need to convert calls in the body of the inlined function into invok...
static void inlineRetainOrClaimRVCalls(CallBase &CB, objcarc::ARCInstKind RVCallKind, const SmallVectorImpl< ReturnInst * > &Returns)
An operand bundle "clang.arc.attachedcall" on a call indicates the call result is implicitly consumed...
static Value * getParentPad(Value *EHPad)
Helper for getUnwindDestToken/getUnwindDestTokenHelper.
static void fixupAssignments(Function::iterator Start, Function::iterator End)
Update inlined instructions' DIAssignID metadata.
static bool allocaWouldBeStaticInEntry(const AllocaInst *AI)
Return the result of AI->isStaticAlloca() if AI were moved to the entry block.
static bool isUsedByLifetimeMarker(Value *V)
static void removeMemProfMetadata(CallBase *Call)
static Value * HandleByValArgument(Type *ByValType, Value *Arg, Instruction *TheCall, const Function *CalledFunc, InlineFunctionInfo &IFI, MaybeAlign ByValAlignment)
When inlining a call site that has a byval argument, we have to make the implicit memcpy explicit by ...
static bool MayContainThrowingOrExitingCall(Instruction *Begin, Instruction *End)
static void AddAlignmentAssumptions(CallBase &CB, InlineFunctionInfo &IFI)
If the inlined function has non-byval align arguments, then add @llvm.assume-based alignment assumpti...
static void trackInlinedStores(Function::iterator Start, Function::iterator End, const CallBase &CB)
static cl::opt< unsigned > InlinerAttributeWindow("max-inst-checked-for-throw-during-inlining", cl::Hidden, cl::desc("the maximum number of instructions analyzed for may throw during " "attribute inference in inlined body"), cl::init(4))
static bool haveCommonPrefix(MDNode *MIBStackContext, MDNode *CallsiteStackContext)
static void PropagateOperandBundles(Function::iterator InlinedBB, Instruction *CallSiteEHPad)
Bundle operands of the inlined function must be added to inlined call sites.
static bool hasLifetimeMarkers(AllocaInst *AI)
static void updateMemprofMetadata(CallBase *CI, const std::vector< Metadata * > &MIBList)
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Load MIR Sample Profile
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
This file contains the declarations for metadata subclasses.
Module.h This file contains the declarations for the Module class.
LLVMContext & Context
This file defines common analysis utilities used by the ObjC ARC Optimizer.
This file defines ARC utility functions which are used by various parts of the compiler.
Module * Mod
@ SI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file contains some functions that are useful when dealing with strings.
MemoryEffects getMemoryEffects(const CallBase *Call)
Return the behavior of the given call site.
Class for arbitrary precision integers.
Definition: APInt.h:75
an instruction to allocate memory on the stack
Definition: Instructions.h:58
bool isSwiftError() const
Return true if this alloca is used as a swifterror argument to a call.
Definition: Instructions.h:150
PointerType * getType() const
Overload to return most specific pointer type.
Definition: Instructions.h:100
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:118
bool isUsedWithInAlloca() const
Return true if this alloca is used as an inalloca argument to a call.
Definition: Instructions.h:140
const Value * getArraySize() const
Get the number of elements allocated.
Definition: Instructions.h:96
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
A cache of @llvm.assume calls within a function.
void registerAssumption(AssumeInst *CI)
Add an @llvm.assume intrinsic to this function's cache.
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:513
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:718
bool hasAttributes() const
Return true if the builder has IR-level attributes.
Definition: Attributes.h:1111
AttrBuilder & addAttribute(Attribute::AttrKind Val)
Add an attribute to the builder.
AttrBuilder & addDereferenceableAttr(uint64_t Bytes)
This turns the number of dereferenceable bytes into the form used internally in Attribute.
AttrBuilder & addDereferenceableOrNullAttr(uint64_t Bytes)
This turns the number of dereferenceable_or_null bytes into the form used internally in Attribute.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
AttributeSet getParamAttrs(unsigned ArgNo) const
The attributes for the argument or parameter at the given index are returned.
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:314
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:372
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:208
const Instruction & front() const
Definition: BasicBlock.h:326
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
Definition: BasicBlock.cpp:401
const CallInst * getTerminatingDeoptimizeCall() const
Returns the call instruction calling @llvm.experimental.deoptimize prior to the terminating return in...
Definition: BasicBlock.cpp:181
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:112
SymbolTableList< BasicBlock >::iterator eraseFromParent()
Unlink 'this' from the containing function and delete it.
Definition: BasicBlock.cpp:132
reverse_iterator rend()
Definition: BasicBlock.h:321
InstListType::iterator iterator
Instruction iterators...
Definition: BasicBlock.h:87
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:127
const CallInst * getTerminatingMustTailCall() const
Returns the call instruction marked 'musttail' prior to the terminating return instruction of this ba...
Definition: BasicBlock.cpp:150
void splice(BasicBlock::iterator ToIt, BasicBlock *FromBB)
Transfer all instructions from FromBB to this basic block at ToIt.
Definition: BasicBlock.h:468
void removePredecessor(BasicBlock *Pred, bool KeepOneInputPHIs=false)
Update PHI nodes in this BasicBlock before removal of predecessor Pred.
Definition: BasicBlock.cpp:341
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
void setBlockFreq(const BasicBlock *BB, uint64_t Freq)
void setBlockFreqAndScale(const BasicBlock *ReferenceBB, uint64_t Freq, SmallPtrSetImpl< BasicBlock * > &BlocksToScale)
Set the frequency of ReferenceBB to Freq and scale the frequencies of the blocks in BlocksToScale suc...
BlockFrequency getBlockFreq(const BasicBlock *BB) const
getblockFreq - Return block frequency.
uint64_t getFrequency() const
Returns the frequency as a fixpoint number scaled by the entry frequency.
Conditional or Unconditional Branch instruction.
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1186
void setCallingConv(CallingConv::ID CC)
Definition: InstrTypes.h:1471
void setDoesNotThrow()
Definition: InstrTypes.h:1919
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
OperandBundleUse getOperandBundleAt(unsigned Index) const
Return the operand bundle at a specific index.
Definition: InstrTypes.h:2015
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Definition: InstrTypes.h:2046
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1408
unsigned getNumOperandBundles() const
Return the number of operand bundles associated with this User.
Definition: InstrTypes.h:1959
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1467
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1328
bool isByValArgument(unsigned ArgNo) const
Determine whether this argument is passed by value.
Definition: InstrTypes.h:1690
static CallBase * Create(CallBase *CB, ArrayRef< OperandBundleDef > Bundles, Instruction *InsertPt=nullptr)
Create a clone of CB with a different set of operand bundles and insert it before InsertPt.
Type * getParamByValType(unsigned ArgNo) const
Extract the byval type for a call or parameter.
Definition: InstrTypes.h:1763
Value * getCalledOperand() const
Definition: InstrTypes.h:1401
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
Definition: InstrTypes.h:1490
bool doesNotThrow() const
Determine if the call cannot unwind.
Definition: InstrTypes.h:1918
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1353
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1266
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition: InstrTypes.h:1344
unsigned arg_size() const
Definition: InstrTypes.h:1351
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1486
static CallBase * addOperandBundle(CallBase *CB, uint32_t ID, OperandBundleDef OB, Instruction *InsertPt=nullptr)
Create a clone of CB with operand bundle OB added.
bool hasOperandBundles() const
Return true if this User has any operand bundles.
Definition: InstrTypes.h:1964
Function * getCaller()
Helper to get the caller (the parent function).
This class represents a function call, abstracting a target machine's calling convention.
void setTailCallKind(TailCallKind TCK)
TailCallKind getTailCallKind() const
bool isMustTailCall() const
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
static CatchSwitchInst * Create(Value *ParentPad, BasicBlock *UnwindDest, unsigned NumHandlers, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
static CleanupReturnInst * Create(Value *CleanupPad, BasicBlock *UnwindBB=nullptr, Instruction *InsertBefore=nullptr)
This is the shared class of boolean and integer constants.
Definition: Constants.h:78
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:888
static ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition: Constants.cpp:1422
This is an important base class in LLVM.
Definition: Constant.h:41
const Constant * stripPointerCasts() const
Definition: Constant.h:213
Assignment ID.
static DIAssignID * getDistinct(LLVMContext &Context)
Debug location.
Subprogram description.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
A debug info location.
Definition: DebugLoc.h:33
unsigned getLine() const
Definition: DebugLoc.cpp:24
DILocation * get() const
Get the underlying DILocation.
Definition: DebugLoc.cpp:20
MDNode * getScope() const
Definition: DebugLoc.cpp:34
static DebugLoc appendInlinedAt(const DebugLoc &DL, DILocation *InlinedAt, LLVMContext &Ctx, DenseMap< const MDNode *, MDNode * > &Cache)
Rebuild the entire inlined-at chain for this instruction so that the top of the chain now is inlined-...
Definition: DebugLoc.cpp:110
unsigned getCol() const
Definition: DebugLoc.cpp:29
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
iterator end()
Definition: DenseMap.h:84
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:220
void recalculate(ParentType &Func)
recalculate - compute a dominator tree for the given function
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:166
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition: DerivedTypes.h:139
Class to represent profile counts.
Definition: Function.h:252
uint64_t getCount() const
Definition: Function.h:260
BasicBlockListType::iterator iterator
Definition: Function.h:65
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:174
const BasicBlock & front() const
Definition: Function.h:763
iterator_range< arg_iterator > args()
Definition: Function.h:795
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1625
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
Definition: Function.h:305
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:237
bool hasPersonalityFn() const
Check whether this function has a personality function.
Definition: Function.h:808
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1999
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:313
arg_iterator arg_end()
Definition: Function.h:780
arg_iterator arg_begin()
Definition: Function.h:771
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:209
MaybeAlign getParamAlign(unsigned ArgNo) const
Definition: Function.h:440
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:319
const std::string & getGC() const
Definition: Function.cpp:718
std::optional< ProfileCount > getEntryCount(bool AllowSynthetic=false) const
Get the entry count for this function.
Definition: Function.cpp:2085
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:179
iterator end()
Definition: Function.h:758
void setCallingConv(CallingConv::ID CC)
Definition: Function.h:241
Argument * getArg(unsigned i) const
Definition: Function.h:789
bool onlyReadsMemory() const
Determine if the function does not access or only reads memory.
Definition: Function.cpp:775
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.cpp:644
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:275
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
CallInst * CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue, unsigned Alignment, Value *OffsetValue=nullptr)
Create an assume intrinsic call that represents an alignment assumption on the provided pointer.
Definition: IRBuilder.cpp:1391
CallInst * CreateLifetimeEnd(Value *Ptr, ConstantInt *Size=nullptr)
Create a lifetime.end intrinsic.
Definition: IRBuilder.cpp:501
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2301
Instruction * CreateNoAliasScopeDeclaration(Value *Scope)
Create a llvm.experimental.noalias.scope.decl intrinsic call.
Definition: IRBuilder.cpp:580
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2558
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
Definition: Cloning.h:203
ProfileSummaryInfo * PSI
Definition: Cloning.h:216
bool UpdateProfile
Update profile for callee as well as cloned version.
Definition: Cloning.h:236
function_ref< AssumptionCache &(Function &)> GetAssumptionCache
If non-null, InlineFunction will update the callgraph to reflect the changes it makes.
Definition: Cloning.h:215
BlockFrequencyInfo * CalleeBFI
Definition: Cloning.h:217
SmallVector< AllocaInst *, 4 > StaticAllocas
InlineFunction fills this in with all static allocas that get copied into the caller.
Definition: Cloning.h:221
BlockFrequencyInfo * CallerBFI
Definition: Cloning.h:217
SmallVector< CallBase *, 8 > InlinedCallSites
All of the new call sites inlined into the caller.
Definition: Cloning.h:232
InlineResult is basically true or false.
Definition: InlineCost.h:179
static InlineResult success()
Definition: InlineCost.h:184
static InlineResult failure(const char *Reason)
Definition: InlineCost.h:185
bool isLifetimeStartOrEnd() const LLVM_READONLY
Return true if the instruction is a llvm.lifetime.start or llvm.lifetime.end marker.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:358
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:70
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
Definition: Instruction.h:257
bool isEHPad() const
Return true if the instruction is a variety of EH-block.
Definition: Instruction.h:684
const BasicBlock * getParent() const
Definition: Instruction.h:90
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:74
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:275
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1455
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:168
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:82
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
Definition: Instruction.h:355
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:47
static bool mayLowerToFunctionCall(Intrinsic::ID IID)
Check if the intrinsic might lower into a regular function call in the course of IR transformations.
Invoke instruction.
BasicBlock * getUnwindDest() const
LandingPadInst * getLandingPadInst() const
Get the landingpad instruction from the landing pad block (the unwind destination).
BasicBlock * getNormalDest() const
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
The landingpad instruction holds all of the information necessary to generate correct exception handl...
bool isCleanup() const
Return 'true' if this landingpad instruction is a cleanup.
unsigned getNumClauses() const
Get the number of clauses for this landing pad.
Constant * getClause(unsigned Idx) const
Get the value of the clause at index Idx.
An instruction for reading from memory.
Definition: Instructions.h:177
MDNode * createAnonymousAliasScope(MDNode *Domain, StringRef Name=StringRef())
Return metadata appropriate for an alias scope root node.
Definition: MDBuilder.h:159
MDNode * createAnonymousAliasScopeDomain(StringRef Name=StringRef())
Return metadata appropriate for an alias scope domain node.
Definition: MDBuilder.h:152
Metadata node.
Definition: Metadata.h:943
void replaceAllUsesWith(Metadata *MD)
RAUW a temporary.
Definition: Metadata.h:1132
static MDNode * concatenate(MDNode *A, MDNode *B)
Methods for metadata merging.
Definition: Metadata.cpp:1005
bool isTemporary() const
Definition: Metadata.h:1127
ArrayRef< MDOperand > operands() const
Definition: Metadata.h:1289
op_iterator op_end() const
Definition: Metadata.h:1285
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1399
unsigned getNumOperands() const
Return number of MDNode operands.
Definition: Metadata.h:1297
op_iterator op_begin() const
Definition: Metadata.h:1281
LLVMContext & getContext() const
Definition: Metadata.h:1107
Tuple of metadata.
Definition: Metadata.h:1328
static TempMDTuple getTemporary(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Return a temporary node.
Definition: Metadata.h:1376
Summary of how a function affects memory in the program.
Definition: ModRef.h:63
bool onlyAccessesArgPointees() const
Whether this function only (at most) accesses argument memory.
Definition: ModRef.h:200
bool onlyAccessesInaccessibleMem() const
Whether this function only (at most) accesses inaccessible memory.
Definition: ModRef.h:210
Root of the metadata hierarchy.
Definition: Metadata.h:61
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:398
A container for an operand bundle being viewed as a set of values rather than a set of uses.
Definition: InstrTypes.h:1139
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1750
Analysis providing profile information.
std::optional< uint64_t > getProfileCount(const CallBase &CallInst, BlockFrequencyInfo *BFI, bool AllowSynthetic=false) const
Returns the profile count for CallInst.
Resume the propagation of an exception.
Return a value (possibly void), from a function.
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
A vector that has set insertion semantics.
Definition: SetVector.h:40
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:344
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:383
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:365
bool contains(ConstPtrType Ptr) const
Definition: SmallPtrSet.h:389
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:450
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:301
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:941
void reserve(size_type N)
Definition: SmallVector.h:667
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:687
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
An instruction for storing to memory.
Definition: Instructions.h:301
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:140
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1731
void setOperand(unsigned i, Value *Val)
Definition: User.h:174
Value * getOperand(unsigned i) const
Definition: User.h:169
This class represents the va_arg llvm instruction, which returns an argument of the specified type gi...
See the file comment.
Definition: ValueMap.h:84
ValueT lookup(const KeyT &Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition: ValueMap.h:164
size_type count(const KeyT &Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: ValueMap.h:151
iterator begin()
Definition: ValueMap.h:134
iterator end()
Definition: ValueMap.h:135
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:532
iterator_range< user_iterator > users()
Definition: Value.h:421
bool hasNUses(unsigned N) const
Return true if this Value has exactly N uses.
Definition: Value.cpp:148
bool use_empty() const
Definition: Value.h:344
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:994
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:308
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:381
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:182
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:166
ilist_iterator< OptionsT, !IsReverse, IsConst > getReverse() const
Get a reverse iterator to the same node.
self_iterator getIterator()
Definition: ilist_node.h:82
Class to build a trie of call stack contexts for a particular profiled allocation call,...
Helper class to iterate through stack ids in both metadata (memprof MIB and callsite) and the corresp...
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void mergeAttributesForInlining(Function &Caller, const Function &Callee)
Merge caller's and callee's attributes.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1506
AssignmentMarkerRange getAssignmentMarkers(DIAssignID *ID)
Return a range of dbg.assign intrinsics which use \ID as an operand.
Definition: DebugInfo.cpp:1741
void trackAssignments(Function::iterator Start, Function::iterator End, const StorageToVarsMap &Vars, const DataLayout &DL, bool DebugPrints=false)
Track assignments to Vars between Start and End.
Definition: DebugInfo.cpp:1862
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
MDNode * getMIBStackNode(const MDNode *MIB)
Returns the stack node from an MIB metadata node.
constexpr double phi
Definition: MathExtras.h:45
ARCInstKind getAttachedARCFunctionKind(const CallBase *CB)
This function returns the ARCInstKind of the function attached to operand bundle clang_arc_attachedca...
Definition: ObjCARCUtil.h:60
ARCInstKind
Equivalence classes of instructions in the ARC Model.
std::optional< Function * > getAttachedARCFunction(const CallBase *CB)
This function returns operand bundle clang_arc_attachedcall's argument, which is the address of the A...
Definition: ObjCARCUtil.h:43
bool isRetainOrClaimRV(ARCInstKind Kind)
Check whether the function is retainRV/unsafeClaimRV.
Definition: ObjCARCUtil.h:52
const Value * GetRCIdentityRoot(const Value *V)
The RCIdentity root of a value V is a dominating value U for which retaining or releasing U is equiva...
bool hasAttachedCallOpBundle(const CallBase *CB)
Definition: ObjCARCUtil.h:29
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
BasicBlock * changeToInvokeAndSplitBasicBlock(CallInst *CI, BasicBlock *UnwindEdge, DomTreeUpdater *DTU=nullptr)
Convert the CallInst to InvokeInst with the specified unwind edge basic block.
Definition: Local.cpp:2328
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
bool PointerMayBeCapturedBefore(const Value *V, bool ReturnCaptures, bool StoreCaptures, const Instruction *I, const DominatorTree *DT, bool IncludeI=false, unsigned MaxUsesToExplore=0, const LoopInfo *LI=nullptr)
PointerMayBeCapturedBefore - Return true if this pointer value may be captured by the enclosing funct...
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
Definition: STLExtras.h:2129
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:748
bool isScopedEHPersonality(EHPersonality Pers)
Returns true if this personality uses scope-style EH IR instructions: catchswitch,...
Align getKnownAlignment(Value *V, const DataLayout &DL, const Instruction *CxtI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
Try to infer an alignment for the specified pointer.
Definition: Local.h:222
Align getOrEnforceKnownAlignment(Value *V, MaybeAlign PrefAlign, const DataLayout &DL, const Instruction *CxtI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
Try to ensure that the alignment of V is at least PrefAlign bytes.
Definition: Local.cpp:1439
void CloneAndPruneFunctionInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, bool ModuleLevelChanges, SmallVectorImpl< ReturnInst * > &Returns, const char *NameSuffix="", ClonedCodeInfo *CodeInfo=nullptr)
This works exactly like CloneFunctionInto, except that it does some simple constant prop and DCE on t...
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
unsigned changeToUnreachable(Instruction *I, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...
Definition: Local.cpp:2243
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
bool salvageKnowledge(Instruction *I, AssumptionCache *AC=nullptr, DominatorTree *DT=nullptr)
Calls BuildAssumeFromInst and if the resulting llvm.assume is valid insert if before I.
void updateProfileCallee(Function *Callee, int64_t EntryDelta, const ValueMap< const Value *, WeakTrackingVH > *VMap=nullptr)
Updates profile information by adjusting the entry count by adding EntryDelta then scaling callsite i...
bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
Definition: DebugInfo.cpp:2027
Value * simplifyInstruction(Instruction *I, const SimplifyQuery &Q, OptimizationRemarkEmitter *ORE=nullptr)
See if we can compute a simplified version of this instruction.
MDNode * uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2)
Compute the union of two access-group lists.
InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)
This function inlines the called function into the basic block of the caller.
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
bool isGuaranteedToTransferExecutionToSuccessor(const Instruction *I)
Return true if this function can prove that the instruction I will always transfer execution to one o...
bool isEscapeSource(const Value *V)
Returns true if the pointer is one which would have been considered an escape by isNonEscapingLocalOb...
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition: STLExtras.h:2113
bool pred_empty(const BasicBlock *BB)
Definition: CFG.h:118
void updateLoopMetadataDebugLocations(Instruction &I, function_ref< Metadata *(Metadata *)> Updater)
Update the debug locations contained within the MD_loop metadata attached to the instruction I,...
Definition: DebugInfo.cpp:377
bool isIdentifiedObject(const Value *V)
Return true if this pointer refers to a distinct and identifiable object.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
This struct can be used to capture information about code being cloned, while it is being cloned.
Definition: Cloning.h:62
bool ContainsDynamicAllocas
This is set to true if the cloned code contains a 'dynamic' alloca.
Definition: Cloning.h:73
bool isSimplified(const Value *From, const Value *To) const
Definition: Cloning.h:87
bool ContainsCalls
This is set to true if the cloned code contains a normal call instruction.
Definition: Cloning.h:64
bool ContainsMemProfMetadata
This is set to true if there is memprof related metadata (memprof or callsite metadata) in the cloned...
Definition: Cloning.h:68
std::vector< WeakTrackingVH > OperandBundleCallSites
All cloned call sites that have operand bundles attached are appended to this vector.
Definition: Cloning.h:78
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
Definition: Alignment.h:141
uint32_t getTagID() const
Return the tag of this operand bundle as an integer.
Definition: InstrTypes.h:1109
Helper struct for trackAssignments, below.
Definition: DebugInfo.h:233