LLVM  16.0.0git
CoroFrame.cpp
Go to the documentation of this file.
1 //===- CoroFrame.cpp - Builds and manipulates coroutine frame -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 // This file contains classes used to discover if for a particular value
9 // there from sue to definition that crosses a suspend block.
10 //
11 // Using the information discovered we form a Coroutine Frame structure to
12 // contain those values. All uses of those values are replaced with appropriate
13 // GEP + load from the coroutine frame. At the point of the definition we spill
14 // the value into the coroutine frame.
15 //===----------------------------------------------------------------------===//
16 
17 #include "CoroInternal.h"
18 #include "llvm/ADT/BitVector.h"
19 #include "llvm/ADT/ScopeExit.h"
20 #include "llvm/ADT/SmallString.h"
23 #include "llvm/Config/llvm-config.h"
24 #include "llvm/IR/CFG.h"
25 #include "llvm/IR/DIBuilder.h"
26 #include "llvm/IR/DebugInfo.h"
27 #include "llvm/IR/Dominators.h"
28 #include "llvm/IR/IRBuilder.h"
29 #include "llvm/IR/InstIterator.h"
30 #include "llvm/IR/IntrinsicInst.h"
31 #include "llvm/Support/Debug.h"
39 #include <algorithm>
40 #include <optional>
41 
42 using namespace llvm;
43 
44 // The "coro-suspend-crossing" flag is very noisy. There is another debug type,
45 // "coro-frame", which results in leaner debug spew.
46 #define DEBUG_TYPE "coro-suspend-crossing"
47 
48 enum { SmallVectorThreshold = 32 };
49 
50 // Provides two way mapping between the blocks and numbers.
51 namespace {
52 class BlockToIndexMapping {
54 
55 public:
56  size_t size() const { return V.size(); }
57 
58  BlockToIndexMapping(Function &F) {
59  for (BasicBlock &BB : F)
60  V.push_back(&BB);
61  llvm::sort(V);
62  }
63 
64  size_t blockToIndex(BasicBlock *BB) const {
65  auto *I = llvm::lower_bound(V, BB);
66  assert(I != V.end() && *I == BB && "BasicBlockNumberng: Unknown block");
67  return I - V.begin();
68  }
69 
70  BasicBlock *indexToBlock(unsigned Index) const { return V[Index]; }
71 };
72 } // end anonymous namespace
73 
74 // The SuspendCrossingInfo maintains data that allows to answer a question
75 // whether given two BasicBlocks A and B there is a path from A to B that
76 // passes through a suspend point.
77 //
78 // For every basic block 'i' it maintains a BlockData that consists of:
79 // Consumes: a bit vector which contains a set of indices of blocks that can
80 // reach block 'i'
81 // Kills: a bit vector which contains a set of indices of blocks that can
82 // reach block 'i', but one of the path will cross a suspend point
83 // Suspend: a boolean indicating whether block 'i' contains a suspend point.
84 // End: a boolean indicating whether block 'i' contains a coro.end intrinsic.
85 //
86 namespace {
87 struct SuspendCrossingInfo {
88  BlockToIndexMapping Mapping;
89 
90  struct BlockData {
91  BitVector Consumes;
92  BitVector Kills;
93  bool Suspend = false;
94  bool End = false;
95  };
97 
98  iterator_range<succ_iterator> successors(BlockData const &BD) const {
99  BasicBlock *BB = Mapping.indexToBlock(&BD - &Block[0]);
100  return llvm::successors(BB);
101  }
102 
103  BlockData &getBlockData(BasicBlock *BB) {
104  return Block[Mapping.blockToIndex(BB)];
105  }
106 
107  void dump() const;
108  void dump(StringRef Label, BitVector const &BV) const;
109 
110  SuspendCrossingInfo(Function &F, coro::Shape &Shape);
111 
112  bool hasPathCrossingSuspendPoint(BasicBlock *DefBB, BasicBlock *UseBB) const {
113  size_t const DefIndex = Mapping.blockToIndex(DefBB);
114  size_t const UseIndex = Mapping.blockToIndex(UseBB);
115 
116  bool const Result = Block[UseIndex].Kills[DefIndex];
117  LLVM_DEBUG(dbgs() << UseBB->getName() << " => " << DefBB->getName()
118  << " answer is " << Result << "\n");
119  return Result;
120  }
121 
122  bool isDefinitionAcrossSuspend(BasicBlock *DefBB, User *U) const {
123  auto *I = cast<Instruction>(U);
124 
125  // We rewrote PHINodes, so that only the ones with exactly one incoming
126  // value need to be analyzed.
127  if (auto *PN = dyn_cast<PHINode>(I))
128  if (PN->getNumIncomingValues() > 1)
129  return false;
130 
131  BasicBlock *UseBB = I->getParent();
132 
133  // As a special case, treat uses by an llvm.coro.suspend.retcon or an
134  // llvm.coro.suspend.async as if they were uses in the suspend's single
135  // predecessor: the uses conceptually occur before the suspend.
136  if (isa<CoroSuspendRetconInst>(I) || isa<CoroSuspendAsyncInst>(I)) {
137  UseBB = UseBB->getSinglePredecessor();
138  assert(UseBB && "should have split coro.suspend into its own block");
139  }
140 
141  return hasPathCrossingSuspendPoint(DefBB, UseBB);
142  }
143 
144  bool isDefinitionAcrossSuspend(Argument &A, User *U) const {
145  return isDefinitionAcrossSuspend(&A.getParent()->getEntryBlock(), U);
146  }
147 
148  bool isDefinitionAcrossSuspend(Instruction &I, User *U) const {
149  auto *DefBB = I.getParent();
150 
151  // As a special case, treat values produced by an llvm.coro.suspend.*
152  // as if they were defined in the single successor: the uses
153  // conceptually occur after the suspend.
154  if (isa<AnyCoroSuspendInst>(I)) {
155  DefBB = DefBB->getSingleSuccessor();
156  assert(DefBB && "should have split coro.suspend into its own block");
157  }
158 
159  return isDefinitionAcrossSuspend(DefBB, U);
160  }
161 
162  bool isDefinitionAcrossSuspend(Value &V, User *U) const {
163  if (auto *Arg = dyn_cast<Argument>(&V))
164  return isDefinitionAcrossSuspend(*Arg, U);
165  if (auto *Inst = dyn_cast<Instruction>(&V))
166  return isDefinitionAcrossSuspend(*Inst, U);
167 
169  "Coroutine could only collect Argument and Instruction now.");
170  }
171 };
172 } // end anonymous namespace
173 
174 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
176  BitVector const &BV) const {
177  dbgs() << Label << ":";
178  for (size_t I = 0, N = BV.size(); I < N; ++I)
179  if (BV[I])
180  dbgs() << " " << Mapping.indexToBlock(I)->getName();
181  dbgs() << "\n";
182 }
183 
185  for (size_t I = 0, N = Block.size(); I < N; ++I) {
186  BasicBlock *const B = Mapping.indexToBlock(I);
187  dbgs() << B->getName() << ":\n";
188  dump(" Consumes", Block[I].Consumes);
189  dump(" Kills", Block[I].Kills);
190  }
191  dbgs() << "\n";
192 }
193 #endif
194 
195 SuspendCrossingInfo::SuspendCrossingInfo(Function &F, coro::Shape &Shape)
196  : Mapping(F) {
197  const size_t N = Mapping.size();
198  Block.resize(N);
199 
200  // Initialize every block so that it consumes itself
201  for (size_t I = 0; I < N; ++I) {
202  auto &B = Block[I];
203  B.Consumes.resize(N);
204  B.Kills.resize(N);
205  B.Consumes.set(I);
206  }
207 
208  // Mark all CoroEnd Blocks. We do not propagate Kills beyond coro.ends as
209  // the code beyond coro.end is reachable during initial invocation of the
210  // coroutine.
211  for (auto *CE : Shape.CoroEnds)
212  getBlockData(CE->getParent()).End = true;
213 
214  // Mark all suspend blocks and indicate that they kill everything they
215  // consume. Note, that crossing coro.save also requires a spill, as any code
216  // between coro.save and coro.suspend may resume the coroutine and all of the
217  // state needs to be saved by that time.
218  auto markSuspendBlock = [&](IntrinsicInst *BarrierInst) {
219  BasicBlock *SuspendBlock = BarrierInst->getParent();
220  auto &B = getBlockData(SuspendBlock);
221  B.Suspend = true;
222  B.Kills |= B.Consumes;
223  };
224  for (auto *CSI : Shape.CoroSuspends) {
225  markSuspendBlock(CSI);
226  if (auto *Save = CSI->getCoroSave())
227  markSuspendBlock(Save);
228  }
229 
230  // Iterate propagating consumes and kills until they stop changing.
231  int Iteration = 0;
232  (void)Iteration;
233 
234  bool Changed;
235  do {
236  LLVM_DEBUG(dbgs() << "iteration " << ++Iteration);
237  LLVM_DEBUG(dbgs() << "==============\n");
238 
239  Changed = false;
240  for (size_t I = 0; I < N; ++I) {
241  auto &B = Block[I];
242  for (BasicBlock *SI : successors(B)) {
243 
244  auto SuccNo = Mapping.blockToIndex(SI);
245 
246  // Saved Consumes and Kills bitsets so that it is easy to see
247  // if anything changed after propagation.
248  auto &S = Block[SuccNo];
249  auto SavedConsumes = S.Consumes;
250  auto SavedKills = S.Kills;
251 
252  // Propagate Kills and Consumes from block B into its successor S.
253  S.Consumes |= B.Consumes;
254  S.Kills |= B.Kills;
255 
256  // If block B is a suspend block, it should propagate kills into the
257  // its successor for every block B consumes.
258  if (B.Suspend) {
259  S.Kills |= B.Consumes;
260  }
261  if (S.Suspend) {
262  // If block S is a suspend block, it should kill all of the blocks it
263  // consumes.
264  S.Kills |= S.Consumes;
265  } else if (S.End) {
266  // If block S is an end block, it should not propagate kills as the
267  // blocks following coro.end() are reached during initial invocation
268  // of the coroutine while all the data are still available on the
269  // stack or in the registers.
270  S.Kills.reset();
271  } else {
272  // This is reached when S block it not Suspend nor coro.end and it
273  // need to make sure that it is not in the kill set.
274  S.Kills.reset(SuccNo);
275  }
276 
277  // See if anything changed.
278  Changed |= (S.Kills != SavedKills) || (S.Consumes != SavedConsumes);
279 
280  if (S.Kills != SavedKills) {
281  LLVM_DEBUG(dbgs() << "\nblock " << I << " follower " << SI->getName()
282  << "\n");
283  LLVM_DEBUG(dump("S.Kills", S.Kills));
284  LLVM_DEBUG(dump("SavedKills", SavedKills));
285  }
286  if (S.Consumes != SavedConsumes) {
287  LLVM_DEBUG(dbgs() << "\nblock " << I << " follower " << SI << "\n");
288  LLVM_DEBUG(dump("S.Consume", S.Consumes));
289  LLVM_DEBUG(dump("SavedCons", SavedConsumes));
290  }
291  }
292  }
293  } while (Changed);
294  LLVM_DEBUG(dump());
295 }
296 
297 #undef DEBUG_TYPE // "coro-suspend-crossing"
298 #define DEBUG_TYPE "coro-frame"
299 
300 namespace {
301 class FrameTypeBuilder;
302 // Mapping from the to-be-spilled value to all the users that need reload.
304 struct AllocaInfo {
305  AllocaInst *Alloca;
307  bool MayWriteBeforeCoroBegin;
308  AllocaInfo(AllocaInst *Alloca,
310  bool MayWriteBeforeCoroBegin)
311  : Alloca(Alloca), Aliases(std::move(Aliases)),
312  MayWriteBeforeCoroBegin(MayWriteBeforeCoroBegin) {}
313 };
314 struct FrameDataInfo {
315  // All the values (that are not allocas) that needs to be spilled to the
316  // frame.
317  SpillInfo Spills;
318  // Allocas contains all values defined as allocas that need to live in the
319  // frame.
321 
322  SmallVector<Value *, 8> getAllDefs() const {
324  for (const auto &P : Spills)
325  Defs.push_back(P.first);
326  for (const auto &A : Allocas)
327  Defs.push_back(A.Alloca);
328  return Defs;
329  }
330 
331  uint32_t getFieldIndex(Value *V) const {
332  auto Itr = FieldIndexMap.find(V);
333  assert(Itr != FieldIndexMap.end() &&
334  "Value does not have a frame field index");
335  return Itr->second;
336  }
337 
338  void setFieldIndex(Value *V, uint32_t Index) {
339  assert((LayoutIndexUpdateStarted || FieldIndexMap.count(V) == 0) &&
340  "Cannot set the index for the same field twice.");
341  FieldIndexMap[V] = Index;
342  }
343 
344  Align getAlign(Value *V) const {
345  auto Iter = FieldAlignMap.find(V);
346  assert(Iter != FieldAlignMap.end());
347  return Iter->second;
348  }
349 
350  void setAlign(Value *V, Align AL) {
351  assert(FieldAlignMap.count(V) == 0);
352  FieldAlignMap.insert({V, AL});
353  }
354 
355  uint64_t getDynamicAlign(Value *V) const {
356  auto Iter = FieldDynamicAlignMap.find(V);
357  assert(Iter != FieldDynamicAlignMap.end());
358  return Iter->second;
359  }
360 
361  void setDynamicAlign(Value *V, uint64_t Align) {
362  assert(FieldDynamicAlignMap.count(V) == 0);
363  FieldDynamicAlignMap.insert({V, Align});
364  }
365 
366  uint64_t getOffset(Value *V) const {
367  auto Iter = FieldOffsetMap.find(V);
368  assert(Iter != FieldOffsetMap.end());
369  return Iter->second;
370  }
371 
372  void setOffset(Value *V, uint64_t Offset) {
373  assert(FieldOffsetMap.count(V) == 0);
374  FieldOffsetMap.insert({V, Offset});
375  }
376 
377  // Remap the index of every field in the frame, using the final layout index.
378  void updateLayoutIndex(FrameTypeBuilder &B);
379 
380 private:
381  // LayoutIndexUpdateStarted is used to avoid updating the index of any field
382  // twice by mistake.
383  bool LayoutIndexUpdateStarted = false;
384  // Map from values to their slot indexes on the frame. They will be first set
385  // with their original insertion field index. After the frame is built, their
386  // indexes will be updated into the final layout index.
387  DenseMap<Value *, uint32_t> FieldIndexMap;
388  // Map from values to their alignment on the frame. They would be set after
389  // the frame is built.
390  DenseMap<Value *, Align> FieldAlignMap;
391  DenseMap<Value *, uint64_t> FieldDynamicAlignMap;
392  // Map from values to their offset on the frame. They would be set after
393  // the frame is built.
394  DenseMap<Value *, uint64_t> FieldOffsetMap;
395 };
396 } // namespace
397 
398 #ifndef NDEBUG
399 static void dumpSpills(StringRef Title, const SpillInfo &Spills) {
400  dbgs() << "------------- " << Title << "--------------\n";
401  for (const auto &E : Spills) {
402  E.first->dump();
403  dbgs() << " user: ";
404  for (auto *I : E.second)
405  I->dump();
406  }
407 }
408 
409 static void dumpAllocas(const SmallVectorImpl<AllocaInfo> &Allocas) {
410  dbgs() << "------------- Allocas --------------\n";
411  for (const auto &A : Allocas) {
412  A.Alloca->dump();
413  }
414 }
415 #endif
416 
417 namespace {
418 using FieldIDType = size_t;
419 // We cannot rely solely on natural alignment of a type when building a
420 // coroutine frame and if the alignment specified on the Alloca instruction
421 // differs from the natural alignment of the alloca type we will need to insert
422 // padding.
423 class FrameTypeBuilder {
424 private:
425  struct Field {
426  uint64_t Size;
428  Type *Ty;
429  FieldIDType LayoutFieldIndex;
430  Align Alignment;
431  Align TyAlignment;
432  uint64_t DynamicAlignBuffer;
433  };
434 
435  const DataLayout &DL;
437  uint64_t StructSize = 0;
438  Align StructAlign;
439  bool IsFinished = false;
440 
441  Optional<Align> MaxFrameAlignment;
442 
443  SmallVector<Field, 8> Fields;
444  DenseMap<Value*, unsigned> FieldIndexByKey;
445 
446 public:
447  FrameTypeBuilder(LLVMContext &Context, const DataLayout &DL,
448  Optional<Align> MaxFrameAlignment)
449  : DL(DL), Context(Context), MaxFrameAlignment(MaxFrameAlignment) {}
450 
451  /// Add a field to this structure for the storage of an `alloca`
452  /// instruction.
453  [[nodiscard]] FieldIDType addFieldForAlloca(AllocaInst *AI,
454  bool IsHeader = false) {
455  Type *Ty = AI->getAllocatedType();
456 
457  // Make an array type if this is a static array allocation.
458  if (AI->isArrayAllocation()) {
459  if (auto *CI = dyn_cast<ConstantInt>(AI->getArraySize()))
460  Ty = ArrayType::get(Ty, CI->getValue().getZExtValue());
461  else
462  report_fatal_error("Coroutines cannot handle non static allocas yet");
463  }
464 
465  return addField(Ty, AI->getAlign(), IsHeader);
466  }
467 
468  /// We want to put the allocas whose lifetime-ranges are not overlapped
469  /// into one slot of coroutine frame.
470  /// Consider the example at:https://bugs.llvm.org/show_bug.cgi?id=45566
471  ///
472  /// cppcoro::task<void> alternative_paths(bool cond) {
473  /// if (cond) {
474  /// big_structure a;
475  /// process(a);
476  /// co_await something();
477  /// } else {
478  /// big_structure b;
479  /// process2(b);
480  /// co_await something();
481  /// }
482  /// }
483  ///
484  /// We want to put variable a and variable b in the same slot to
485  /// reduce the size of coroutine frame.
486  ///
487  /// This function use StackLifetime algorithm to partition the AllocaInsts in
488  /// Spills to non-overlapped sets in order to put Alloca in the same
489  /// non-overlapped set into the same slot in the Coroutine Frame. Then add
490  /// field for the allocas in the same non-overlapped set by using the largest
491  /// type as the field type.
492  ///
493  /// Side Effects: Because We sort the allocas, the order of allocas in the
494  /// frame may be different with the order in the source code.
495  void addFieldForAllocas(const Function &F, FrameDataInfo &FrameData,
496  coro::Shape &Shape);
497 
498  /// Add a field to this structure.
499  [[nodiscard]] FieldIDType addField(Type *Ty, MaybeAlign MaybeFieldAlignment,
500  bool IsHeader = false,
501  bool IsSpillOfValue = false) {
502  assert(!IsFinished && "adding fields to a finished builder");
503  assert(Ty && "must provide a type for a field");
504 
505  // The field size is always the alloc size of the type.
506  uint64_t FieldSize = DL.getTypeAllocSize(Ty);
507 
508  // For an alloca with size=0, we don't need to add a field and they
509  // can just point to any index in the frame. Use index 0.
510  if (FieldSize == 0) {
511  return 0;
512  }
513 
514  // The field alignment might not be the type alignment, but we need
515  // to remember the type alignment anyway to build the type.
516  // If we are spilling values we don't need to worry about ABI alignment
517  // concerns.
518  Align ABIAlign = DL.getABITypeAlign(Ty);
519  Align TyAlignment = ABIAlign;
520  if (IsSpillOfValue && MaxFrameAlignment && *MaxFrameAlignment < ABIAlign)
521  TyAlignment = *MaxFrameAlignment;
522  Align FieldAlignment = MaybeFieldAlignment.value_or(TyAlignment);
523 
524  // The field alignment could be bigger than the max frame case, in that case
525  // we request additional storage to be able to dynamically align the
526  // pointer.
527  uint64_t DynamicAlignBuffer = 0;
528  if (MaxFrameAlignment && (FieldAlignment > *MaxFrameAlignment)) {
529  DynamicAlignBuffer =
530  offsetToAlignment(MaxFrameAlignment->value(), FieldAlignment);
531  FieldAlignment = *MaxFrameAlignment;
532  FieldSize = FieldSize + DynamicAlignBuffer;
533  }
534 
535  // Lay out header fields immediately.
537  if (IsHeader) {
538  Offset = alignTo(StructSize, FieldAlignment);
539  StructSize = Offset + FieldSize;
540 
541  // Everything else has a flexible offset.
542  } else {
543  Offset = OptimizedStructLayoutField::FlexibleOffset;
544  }
545 
546  Fields.push_back({FieldSize, Offset, Ty, 0, FieldAlignment, TyAlignment,
547  DynamicAlignBuffer});
548  return Fields.size() - 1;
549  }
550 
551  /// Finish the layout and set the body on the given type.
552  void finish(StructType *Ty);
553 
554  uint64_t getStructSize() const {
555  assert(IsFinished && "not yet finished!");
556  return StructSize;
557  }
558 
559  Align getStructAlign() const {
560  assert(IsFinished && "not yet finished!");
561  return StructAlign;
562  }
563 
564  FieldIDType getLayoutFieldIndex(FieldIDType Id) const {
565  assert(IsFinished && "not yet finished!");
566  return Fields[Id].LayoutFieldIndex;
567  }
568 
569  Field getLayoutField(FieldIDType Id) const {
570  assert(IsFinished && "not yet finished!");
571  return Fields[Id];
572  }
573 };
574 } // namespace
575 
576 void FrameDataInfo::updateLayoutIndex(FrameTypeBuilder &B) {
577  auto Updater = [&](Value *I) {
578  auto Field = B.getLayoutField(getFieldIndex(I));
579  setFieldIndex(I, Field.LayoutFieldIndex);
580  setAlign(I, Field.Alignment);
581  uint64_t dynamicAlign =
582  Field.DynamicAlignBuffer
583  ? Field.DynamicAlignBuffer + Field.Alignment.value()
584  : 0;
585  setDynamicAlign(I, dynamicAlign);
586  setOffset(I, Field.Offset);
587  };
588  LayoutIndexUpdateStarted = true;
589  for (auto &S : Spills)
590  Updater(S.first);
591  for (const auto &A : Allocas)
592  Updater(A.Alloca);
593  LayoutIndexUpdateStarted = false;
594 }
595 
596 void FrameTypeBuilder::addFieldForAllocas(const Function &F,
597  FrameDataInfo &FrameData,
598  coro::Shape &Shape) {
599  using AllocaSetType = SmallVector<AllocaInst *, 4>;
600  SmallVector<AllocaSetType, 4> NonOverlapedAllocas;
601 
602  // We need to add field for allocas at the end of this function.
603  auto AddFieldForAllocasAtExit = make_scope_exit([&]() {
604  for (auto AllocaList : NonOverlapedAllocas) {
605  auto *LargestAI = *AllocaList.begin();
606  FieldIDType Id = addFieldForAlloca(LargestAI);
607  for (auto *Alloca : AllocaList)
608  FrameData.setFieldIndex(Alloca, Id);
609  }
610  });
611 
612  if (!Shape.OptimizeFrame) {
613  for (const auto &A : FrameData.Allocas) {
614  AllocaInst *Alloca = A.Alloca;
615  NonOverlapedAllocas.emplace_back(AllocaSetType(1, Alloca));
616  }
617  return;
618  }
619 
620  // Because there are pathes from the lifetime.start to coro.end
621  // for each alloca, the liferanges for every alloca is overlaped
622  // in the blocks who contain coro.end and the successor blocks.
623  // So we choose to skip there blocks when we calculates the liferange
624  // for each alloca. It should be reasonable since there shouldn't be uses
625  // in these blocks and the coroutine frame shouldn't be used outside the
626  // coroutine body.
627  //
628  // Note that the user of coro.suspend may not be SwitchInst. However, this
629  // case seems too complex to handle. And it is harmless to skip these
630  // patterns since it just prevend putting the allocas to live in the same
631  // slot.
632  DenseMap<SwitchInst *, BasicBlock *> DefaultSuspendDest;
633  for (auto *CoroSuspendInst : Shape.CoroSuspends) {
634  for (auto *U : CoroSuspendInst->users()) {
635  if (auto *ConstSWI = dyn_cast<SwitchInst>(U)) {
636  auto *SWI = const_cast<SwitchInst *>(ConstSWI);
637  DefaultSuspendDest[SWI] = SWI->getDefaultDest();
638  SWI->setDefaultDest(SWI->getSuccessor(1));
639  }
640  }
641  }
642 
643  auto ExtractAllocas = [&]() {
644  AllocaSetType Allocas;
645  Allocas.reserve(FrameData.Allocas.size());
646  for (const auto &A : FrameData.Allocas)
647  Allocas.push_back(A.Alloca);
648  return Allocas;
649  };
650  StackLifetime StackLifetimeAnalyzer(F, ExtractAllocas(),
651  StackLifetime::LivenessType::May);
652  StackLifetimeAnalyzer.run();
653  auto IsAllocaInferenre = [&](const AllocaInst *AI1, const AllocaInst *AI2) {
654  return StackLifetimeAnalyzer.getLiveRange(AI1).overlaps(
655  StackLifetimeAnalyzer.getLiveRange(AI2));
656  };
657  auto GetAllocaSize = [&](const AllocaInfo &A) {
658  std::optional<TypeSize> RetSize = A.Alloca->getAllocationSizeInBits(DL);
659  assert(RetSize && "Variable Length Arrays (VLA) are not supported.\n");
660  assert(!RetSize->isScalable() && "Scalable vectors are not yet supported");
661  return RetSize->getFixedSize();
662  };
663  // Put larger allocas in the front. So the larger allocas have higher
664  // priority to merge, which can save more space potentially. Also each
665  // AllocaSet would be ordered. So we can get the largest Alloca in one
666  // AllocaSet easily.
667  sort(FrameData.Allocas, [&](const auto &Iter1, const auto &Iter2) {
668  return GetAllocaSize(Iter1) > GetAllocaSize(Iter2);
669  });
670  for (const auto &A : FrameData.Allocas) {
671  AllocaInst *Alloca = A.Alloca;
672  bool Merged = false;
673  // Try to find if the Alloca is not inferenced with any existing
674  // NonOverlappedAllocaSet. If it is true, insert the alloca to that
675  // NonOverlappedAllocaSet.
676  for (auto &AllocaSet : NonOverlapedAllocas) {
677  assert(!AllocaSet.empty() && "Processing Alloca Set is not empty.\n");
678  bool NoInference = none_of(AllocaSet, [&](auto Iter) {
679  return IsAllocaInferenre(Alloca, Iter);
680  });
681  // If the alignment of A is multiple of the alignment of B, the address
682  // of A should satisfy the requirement for aligning for B.
683  //
684  // There may be other more fine-grained strategies to handle the alignment
685  // infomation during the merging process. But it seems hard to handle
686  // these strategies and benefit little.
687  bool Alignable = [&]() -> bool {
688  auto *LargestAlloca = *AllocaSet.begin();
689  return LargestAlloca->getAlign().value() % Alloca->getAlign().value() ==
690  0;
691  }();
692  bool CouldMerge = NoInference && Alignable;
693  if (!CouldMerge)
694  continue;
695  AllocaSet.push_back(Alloca);
696  Merged = true;
697  break;
698  }
699  if (!Merged) {
700  NonOverlapedAllocas.emplace_back(AllocaSetType(1, Alloca));
701  }
702  }
703  // Recover the default target destination for each Switch statement
704  // reserved.
705  for (auto SwitchAndDefaultDest : DefaultSuspendDest) {
706  SwitchInst *SWI = SwitchAndDefaultDest.first;
707  BasicBlock *DestBB = SwitchAndDefaultDest.second;
708  SWI->setDefaultDest(DestBB);
709  }
710  // This Debug Info could tell us which allocas are merged into one slot.
711  LLVM_DEBUG(for (auto &AllocaSet
712  : NonOverlapedAllocas) {
713  if (AllocaSet.size() > 1) {
714  dbgs() << "In Function:" << F.getName() << "\n";
715  dbgs() << "Find Union Set "
716  << "\n";
717  dbgs() << "\tAllocas are \n";
718  for (auto Alloca : AllocaSet)
719  dbgs() << "\t\t" << *Alloca << "\n";
720  }
721  });
722 }
723 
724 void FrameTypeBuilder::finish(StructType *Ty) {
725  assert(!IsFinished && "already finished!");
726 
727  // Prepare the optimal-layout field array.
728  // The Id in the layout field is a pointer to our Field for it.
730  LayoutFields.reserve(Fields.size());
731  for (auto &Field : Fields) {
732  LayoutFields.emplace_back(&Field, Field.Size, Field.Alignment,
733  Field.Offset);
734  }
735 
736  // Perform layout.
737  auto SizeAndAlign = performOptimizedStructLayout(LayoutFields);
738  StructSize = SizeAndAlign.first;
739  StructAlign = SizeAndAlign.second;
740 
741  auto getField = [](const OptimizedStructLayoutField &LayoutField) -> Field & {
742  return *static_cast<Field *>(const_cast<void*>(LayoutField.Id));
743  };
744 
745  // We need to produce a packed struct type if there's a field whose
746  // assigned offset isn't a multiple of its natural type alignment.
747  bool Packed = [&] {
748  for (auto &LayoutField : LayoutFields) {
749  auto &F = getField(LayoutField);
750  if (!isAligned(F.TyAlignment, LayoutField.Offset))
751  return true;
752  }
753  return false;
754  }();
755 
756  // Build the struct body.
757  SmallVector<Type*, 16> FieldTypes;
758  FieldTypes.reserve(LayoutFields.size() * 3 / 2);
759  uint64_t LastOffset = 0;
760  for (auto &LayoutField : LayoutFields) {
761  auto &F = getField(LayoutField);
762 
763  auto Offset = LayoutField.Offset;
764 
765  // Add a padding field if there's a padding gap and we're either
766  // building a packed struct or the padding gap is more than we'd
767  // get from aligning to the field type's natural alignment.
768  assert(Offset >= LastOffset);
769  if (Offset != LastOffset) {
770  if (Packed || alignTo(LastOffset, F.TyAlignment) != Offset)
771  FieldTypes.push_back(ArrayType::get(Type::getInt8Ty(Context),
772  Offset - LastOffset));
773  }
774 
775  F.Offset = Offset;
776  F.LayoutFieldIndex = FieldTypes.size();
777 
778  FieldTypes.push_back(F.Ty);
779  if (F.DynamicAlignBuffer) {
780  FieldTypes.push_back(
781  ArrayType::get(Type::getInt8Ty(Context), F.DynamicAlignBuffer));
782  }
783  LastOffset = Offset + F.Size;
784  }
785 
786  Ty->setBody(FieldTypes, Packed);
787 
788 #ifndef NDEBUG
789  // Check that the IR layout matches the offsets we expect.
790  auto Layout = DL.getStructLayout(Ty);
791  for (auto &F : Fields) {
792  assert(Ty->getElementType(F.LayoutFieldIndex) == F.Ty);
793  assert(Layout->getElementOffset(F.LayoutFieldIndex) == F.Offset);
794  }
795 #endif
796 
797  IsFinished = true;
798 }
799 
800 static void cacheDIVar(FrameDataInfo &FrameData,
802  for (auto *V : FrameData.getAllDefs()) {
803  if (DIVarCache.find(V) != DIVarCache.end())
804  continue;
805 
806  auto DDIs = FindDbgDeclareUses(V);
807  auto *I = llvm::find_if(DDIs, [](DbgDeclareInst *DDI) {
808  return DDI->getExpression()->getNumElements() == 0;
809  });
810  if (I != DDIs.end())
811  DIVarCache.insert({V, (*I)->getVariable()});
812  }
813 }
814 
815 /// Create name for Type. It uses MDString to store new created string to
816 /// avoid memory leak.
818  if (Ty->isIntegerTy()) {
819  // The longest name in common may be '__int_128', which has 9 bits.
820  SmallString<16> Buffer;
821  raw_svector_ostream OS(Buffer);
822  OS << "__int_" << cast<IntegerType>(Ty)->getBitWidth();
823  auto *MDName = MDString::get(Ty->getContext(), OS.str());
824  return MDName->getString();
825  }
826 
827  if (Ty->isFloatingPointTy()) {
828  if (Ty->isFloatTy())
829  return "__float_";
830  if (Ty->isDoubleTy())
831  return "__double_";
832  return "__floating_type_";
833  }
834 
835  if (auto *PtrTy = dyn_cast<PointerType>(Ty)) {
836  if (PtrTy->isOpaque())
837  return "PointerType";
838  Type *PointeeTy = PtrTy->getNonOpaquePointerElementType();
839  auto Name = solveTypeName(PointeeTy);
840  if (Name == "UnknownType")
841  return "PointerType";
842  SmallString<16> Buffer;
843  Twine(Name + "_Ptr").toStringRef(Buffer);
844  auto *MDName = MDString::get(Ty->getContext(), Buffer.str());
845  return MDName->getString();
846  }
847 
848  if (Ty->isStructTy()) {
849  if (!cast<StructType>(Ty)->hasName())
850  return "__LiteralStructType_";
851 
852  auto Name = Ty->getStructName();
853 
854  SmallString<16> Buffer(Name);
855  for (auto &Iter : Buffer)
856  if (Iter == '.' || Iter == ':')
857  Iter = '_';
858  auto *MDName = MDString::get(Ty->getContext(), Buffer.str());
859  return MDName->getString();
860  }
861 
862  return "UnknownType";
863 }
864 
866  const DataLayout &Layout, DIScope *Scope,
867  unsigned LineNum,
868  DenseMap<Type *, DIType *> &DITypeCache) {
869  if (DIType *DT = DITypeCache.lookup(Ty))
870  return DT;
871 
873 
874  DIType *RetType = nullptr;
875 
876  if (Ty->isIntegerTy()) {
877  auto BitWidth = cast<IntegerType>(Ty)->getBitWidth();
878  RetType = Builder.createBasicType(Name, BitWidth, dwarf::DW_ATE_signed,
879  llvm::DINode::FlagArtificial);
880  } else if (Ty->isFloatingPointTy()) {
881  RetType = Builder.createBasicType(Name, Layout.getTypeSizeInBits(Ty),
882  dwarf::DW_ATE_float,
883  llvm::DINode::FlagArtificial);
884  } else if (Ty->isPointerTy()) {
885  // Construct PointerType points to null (aka void *) instead of exploring
886  // pointee type to avoid infinite search problem. For example, we would be
887  // in trouble if we traverse recursively:
888  //
889  // struct Node {
890  // Node* ptr;
891  // };
892  RetType =
893  Builder.createPointerType(nullptr, Layout.getTypeSizeInBits(Ty),
894  Layout.getABITypeAlignment(Ty) * CHAR_BIT,
895  /*DWARFAddressSpace=*/std::nullopt, Name);
896  } else if (Ty->isStructTy()) {
897  auto *DIStruct = Builder.createStructType(
898  Scope, Name, Scope->getFile(), LineNum, Layout.getTypeSizeInBits(Ty),
899  Layout.getPrefTypeAlignment(Ty) * CHAR_BIT,
900  llvm::DINode::FlagArtificial, nullptr, llvm::DINodeArray());
901 
902  auto *StructTy = cast<StructType>(Ty);
904  for (unsigned I = 0; I < StructTy->getNumElements(); I++) {
905  DIType *DITy = solveDIType(Builder, StructTy->getElementType(I), Layout,
906  Scope, LineNum, DITypeCache);
907  assert(DITy);
908  Elements.push_back(Builder.createMemberType(
909  Scope, DITy->getName(), Scope->getFile(), LineNum,
910  DITy->getSizeInBits(), DITy->getAlignInBits(),
911  Layout.getStructLayout(StructTy)->getElementOffsetInBits(I),
912  llvm::DINode::FlagArtificial, DITy));
913  }
914 
915  Builder.replaceArrays(DIStruct, Builder.getOrCreateArray(Elements));
916 
917  RetType = DIStruct;
918  } else {
919  LLVM_DEBUG(dbgs() << "Unresolved Type: " << *Ty << "\n");
920  TypeSize Size = Layout.getTypeSizeInBits(Ty);
921  auto *CharSizeType = Builder.createBasicType(
922  Name, 8, dwarf::DW_ATE_unsigned_char, llvm::DINode::FlagArtificial);
923 
924  if (Size <= 8)
925  RetType = CharSizeType;
926  else {
927  if (Size % 8 != 0)
928  Size = TypeSize::Fixed(Size + 8 - (Size % 8));
929 
930  RetType = Builder.createArrayType(
931  Size, Layout.getPrefTypeAlign(Ty).value(), CharSizeType,
932  Builder.getOrCreateArray(Builder.getOrCreateSubrange(0, Size / 8)));
933  }
934  }
935 
936  DITypeCache.insert({Ty, RetType});
937  return RetType;
938 }
939 
940 /// Build artificial debug info for C++ coroutine frames to allow users to
941 /// inspect the contents of the frame directly
942 ///
943 /// Create Debug information for coroutine frame with debug name "__coro_frame".
944 /// The debug information for the fields of coroutine frame is constructed from
945 /// the following way:
946 /// 1. For all the value in the Frame, we search the use of dbg.declare to find
947 /// the corresponding debug variables for the value. If we can find the
948 /// debug variable, we can get full and accurate debug information.
949 /// 2. If we can't get debug information in step 1 and 2, we could only try to
950 /// build the DIType by Type. We did this in solveDIType. We only handle
951 /// integer, float, double, integer type and struct type for now.
953  FrameDataInfo &FrameData) {
954  DISubprogram *DIS = F.getSubprogram();
955  // If there is no DISubprogram for F, it implies the Function are not compiled
956  // with debug info. So we also don't need to generate debug info for the frame
957  // neither.
958  if (!DIS || !DIS->getUnit() ||
960  (dwarf::SourceLanguage)DIS->getUnit()->getSourceLanguage()))
961  return;
962 
963  assert(Shape.ABI == coro::ABI::Switch &&
964  "We could only build debug infomation for C++ coroutine now.\n");
965 
966  DIBuilder DBuilder(*F.getParent(), /*AllowUnresolved*/ false);
967 
968  AllocaInst *PromiseAlloca = Shape.getPromiseAlloca();
969  assert(PromiseAlloca &&
970  "Coroutine with switch ABI should own Promise alloca");
971 
973  if (DIs.empty())
974  return;
975 
976  DbgDeclareInst *PromiseDDI = DIs.front();
977  DILocalVariable *PromiseDIVariable = PromiseDDI->getVariable();
978  DILocalScope *PromiseDIScope = PromiseDIVariable->getScope();
979  DIFile *DFile = PromiseDIScope->getFile();
980  DILocation *DILoc = PromiseDDI->getDebugLoc().get();
981  unsigned LineNum = PromiseDIVariable->getLine();
982 
983  DICompositeType *FrameDITy = DBuilder.createStructType(
984  DIS->getUnit(), Twine(F.getName() + ".coro_frame_ty").str(),
985  DFile, LineNum, Shape.FrameSize * 8,
986  Shape.FrameAlign.value() * 8, llvm::DINode::FlagArtificial, nullptr,
987  llvm::DINodeArray());
988  StructType *FrameTy = Shape.FrameTy;
990  DataLayout Layout = F.getParent()->getDataLayout();
991 
993  cacheDIVar(FrameData, DIVarCache);
994 
995  unsigned ResumeIndex = coro::Shape::SwitchFieldIndex::Resume;
996  unsigned DestroyIndex = coro::Shape::SwitchFieldIndex::Destroy;
997  unsigned IndexIndex = Shape.SwitchLowering.IndexField;
998 
1000  NameCache.insert({ResumeIndex, "__resume_fn"});
1001  NameCache.insert({DestroyIndex, "__destroy_fn"});
1002  NameCache.insert({IndexIndex, "__coro_index"});
1003 
1004  Type *ResumeFnTy = FrameTy->getElementType(ResumeIndex),
1005  *DestroyFnTy = FrameTy->getElementType(DestroyIndex),
1006  *IndexTy = FrameTy->getElementType(IndexIndex);
1007 
1009  TyCache.insert(
1010  {ResumeIndex, DBuilder.createPointerType(
1011  nullptr, Layout.getTypeSizeInBits(ResumeFnTy))});
1012  TyCache.insert(
1013  {DestroyIndex, DBuilder.createPointerType(
1014  nullptr, Layout.getTypeSizeInBits(DestroyFnTy))});
1015 
1016  /// FIXME: If we fill the field `SizeInBits` with the actual size of
1017  /// __coro_index in bits, then __coro_index wouldn't show in the debugger.
1018  TyCache.insert({IndexIndex, DBuilder.createBasicType(
1019  "__coro_index",
1020  (Layout.getTypeSizeInBits(IndexTy) < 8)
1021  ? 8
1022  : Layout.getTypeSizeInBits(IndexTy),
1023  dwarf::DW_ATE_unsigned_char)});
1024 
1025  for (auto *V : FrameData.getAllDefs()) {
1026  if (DIVarCache.find(V) == DIVarCache.end())
1027  continue;
1028 
1029  auto Index = FrameData.getFieldIndex(V);
1030 
1031  NameCache.insert({Index, DIVarCache[V]->getName()});
1032  TyCache.insert({Index, DIVarCache[V]->getType()});
1033  }
1034 
1035  // Cache from index to (Align, Offset Pair)
1037  // The Align and Offset of Resume function and Destroy function are fixed.
1038  OffsetCache.insert({ResumeIndex, {8, 0}});
1039  OffsetCache.insert({DestroyIndex, {8, 8}});
1040  OffsetCache.insert(
1041  {IndexIndex,
1043 
1044  for (auto *V : FrameData.getAllDefs()) {
1045  auto Index = FrameData.getFieldIndex(V);
1046 
1047  OffsetCache.insert(
1048  {Index, {FrameData.getAlign(V).value(), FrameData.getOffset(V)}});
1049  }
1050 
1051  DenseMap<Type *, DIType *> DITypeCache;
1052  // This counter is used to avoid same type names. e.g., there would be
1053  // many i32 and i64 types in one coroutine. And we would use i32_0 and
1054  // i32_1 to avoid the same type. Since it makes no sense the name of the
1055  // fields confilicts with each other.
1056  unsigned UnknownTypeNum = 0;
1057  for (unsigned Index = 0; Index < FrameTy->getNumElements(); Index++) {
1058  if (OffsetCache.find(Index) == OffsetCache.end())
1059  continue;
1060 
1061  std::string Name;
1062  uint64_t SizeInBits;
1063  uint32_t AlignInBits;
1064  uint64_t OffsetInBits;
1065  DIType *DITy = nullptr;
1066 
1067  Type *Ty = FrameTy->getElementType(Index);
1068  assert(Ty->isSized() && "We can't handle type which is not sized.\n");
1069  SizeInBits = Layout.getTypeSizeInBits(Ty).getFixedSize();
1070  AlignInBits = OffsetCache[Index].first * 8;
1071  OffsetInBits = OffsetCache[Index].second * 8;
1072 
1073  if (NameCache.find(Index) != NameCache.end()) {
1074  Name = NameCache[Index].str();
1075  DITy = TyCache[Index];
1076  } else {
1077  DITy = solveDIType(DBuilder, Ty, Layout, FrameDITy, LineNum, DITypeCache);
1078  assert(DITy && "SolveDIType shouldn't return nullptr.\n");
1079  Name = DITy->getName().str();
1080  Name += "_" + std::to_string(UnknownTypeNum);
1081  UnknownTypeNum++;
1082  }
1083 
1084  Elements.push_back(DBuilder.createMemberType(
1085  FrameDITy, Name, DFile, LineNum, SizeInBits, AlignInBits, OffsetInBits,
1086  llvm::DINode::FlagArtificial, DITy));
1087  }
1088 
1089  DBuilder.replaceArrays(FrameDITy, DBuilder.getOrCreateArray(Elements));
1090 
1091  auto *FrameDIVar = DBuilder.createAutoVariable(PromiseDIScope, "__coro_frame",
1092  DFile, LineNum, FrameDITy,
1093  true, DINode::FlagArtificial);
1094  assert(FrameDIVar->isValidLocationForIntrinsic(PromiseDDI->getDebugLoc()));
1095 
1096  // Subprogram would have ContainedNodes field which records the debug
1097  // variables it contained. So we need to add __coro_frame to the
1098  // ContainedNodes of it.
1099  //
1100  // If we don't add __coro_frame to the RetainedNodes, user may get
1101  // `no symbol __coro_frame in context` rather than `__coro_frame`
1102  // is optimized out, which is more precise.
1103  if (auto *SubProgram = dyn_cast<DISubprogram>(PromiseDIScope)) {
1104  auto RetainedNodes = SubProgram->getRetainedNodes();
1105  SmallVector<Metadata *, 32> RetainedNodesVec(RetainedNodes.begin(),
1106  RetainedNodes.end());
1107  RetainedNodesVec.push_back(FrameDIVar);
1108  SubProgram->replaceOperandWith(
1109  7, (MDTuple::get(F.getContext(), RetainedNodesVec)));
1110  }
1111 
1112  DBuilder.insertDeclare(Shape.FramePtr, FrameDIVar,
1113  DBuilder.createExpression(), DILoc,
1114  Shape.getInsertPtAfterFramePtr());
1115 }
1116 
1117 // Build a struct that will keep state for an active coroutine.
1118 // struct f.frame {
1119 // ResumeFnTy ResumeFnAddr;
1120 // ResumeFnTy DestroyFnAddr;
1121 // int ResumeIndex;
1122 // ... promise (if present) ...
1123 // ... spills ...
1124 // };
1126  FrameDataInfo &FrameData) {
1127  LLVMContext &C = F.getContext();
1128  const DataLayout &DL = F.getParent()->getDataLayout();
1129  StructType *FrameTy = [&] {
1130  SmallString<32> Name(F.getName());
1131  Name.append(".Frame");
1132  return StructType::create(C, Name);
1133  }();
1134 
1135  // We will use this value to cap the alignment of spilled values.
1136  Optional<Align> MaxFrameAlignment;
1137  if (Shape.ABI == coro::ABI::Async)
1138  MaxFrameAlignment = Shape.AsyncLowering.getContextAlignment();
1139  FrameTypeBuilder B(C, DL, MaxFrameAlignment);
1140 
1141  AllocaInst *PromiseAlloca = Shape.getPromiseAlloca();
1142  std::optional<FieldIDType> SwitchIndexFieldId;
1143 
1144  if (Shape.ABI == coro::ABI::Switch) {
1145  auto *FramePtrTy = FrameTy->getPointerTo();
1146  auto *FnTy = FunctionType::get(Type::getVoidTy(C), FramePtrTy,
1147  /*IsVarArg=*/false);
1148  auto *FnPtrTy = FnTy->getPointerTo();
1149 
1150  // Add header fields for the resume and destroy functions.
1151  // We can rely on these being perfectly packed.
1152  (void)B.addField(FnPtrTy, std::nullopt, /*header*/ true);
1153  (void)B.addField(FnPtrTy, std::nullopt, /*header*/ true);
1154 
1155  // PromiseAlloca field needs to be explicitly added here because it's
1156  // a header field with a fixed offset based on its alignment. Hence it
1157  // needs special handling and cannot be added to FrameData.Allocas.
1158  if (PromiseAlloca)
1159  FrameData.setFieldIndex(
1160  PromiseAlloca, B.addFieldForAlloca(PromiseAlloca, /*header*/ true));
1161 
1162  // Add a field to store the suspend index. This doesn't need to
1163  // be in the header.
1164  unsigned IndexBits = std::max(1U, Log2_64_Ceil(Shape.CoroSuspends.size()));
1165  Type *IndexType = Type::getIntNTy(C, IndexBits);
1166 
1167  SwitchIndexFieldId = B.addField(IndexType, std::nullopt);
1168  } else {
1169  assert(PromiseAlloca == nullptr && "lowering doesn't support promises");
1170  }
1171 
1172  // Because multiple allocas may own the same field slot,
1173  // we add allocas to field here.
1174  B.addFieldForAllocas(F, FrameData, Shape);
1175  // Add PromiseAlloca to Allocas list so that
1176  // 1. updateLayoutIndex could update its index after
1177  // `performOptimizedStructLayout`
1178  // 2. it is processed in insertSpills.
1179  if (Shape.ABI == coro::ABI::Switch && PromiseAlloca)
1180  // We assume that the promise alloca won't be modified before
1181  // CoroBegin and no alias will be create before CoroBegin.
1182  FrameData.Allocas.emplace_back(
1183  PromiseAlloca, DenseMap<Instruction *, llvm::Optional<APInt>>{}, false);
1184  // Create an entry for every spilled value.
1185  for (auto &S : FrameData.Spills) {
1186  Type *FieldType = S.first->getType();
1187  // For byval arguments, we need to store the pointed value in the frame,
1188  // instead of the pointer itself.
1189  if (const Argument *A = dyn_cast<Argument>(S.first))
1190  if (A->hasByValAttr())
1191  FieldType = A->getParamByValType();
1192  FieldIDType Id = B.addField(FieldType, std::nullopt, false /*header*/,
1193  true /*IsSpillOfValue*/);
1194  FrameData.setFieldIndex(S.first, Id);
1195  }
1196 
1197  B.finish(FrameTy);
1198  FrameData.updateLayoutIndex(B);
1199  Shape.FrameAlign = B.getStructAlign();
1200  Shape.FrameSize = B.getStructSize();
1201 
1202  switch (Shape.ABI) {
1203  case coro::ABI::Switch: {
1204  // In the switch ABI, remember the switch-index field.
1205  auto IndexField = B.getLayoutField(*SwitchIndexFieldId);
1206  Shape.SwitchLowering.IndexField = IndexField.LayoutFieldIndex;
1207  Shape.SwitchLowering.IndexAlign = IndexField.Alignment.value();
1208  Shape.SwitchLowering.IndexOffset = IndexField.Offset;
1209 
1210  // Also round the frame size up to a multiple of its alignment, as is
1211  // generally expected in C/C++.
1212  Shape.FrameSize = alignTo(Shape.FrameSize, Shape.FrameAlign);
1213  break;
1214  }
1215 
1216  // In the retcon ABI, remember whether the frame is inline in the storage.
1217  case coro::ABI::Retcon:
1218  case coro::ABI::RetconOnce: {
1219  auto Id = Shape.getRetconCoroId();
1221  = (B.getStructSize() <= Id->getStorageSize() &&
1222  B.getStructAlign() <= Id->getStorageAlignment());
1223  break;
1224  }
1225  case coro::ABI::Async: {
1226  Shape.AsyncLowering.FrameOffset =
1228  // Also make the final context size a multiple of the context alignment to
1229  // make allocation easier for allocators.
1230  Shape.AsyncLowering.ContextSize =
1233  if (Shape.AsyncLowering.getContextAlignment() < Shape.FrameAlign) {
1235  "The alignment requirment of frame variables cannot be higher than "
1236  "the alignment of the async function context");
1237  }
1238  break;
1239  }
1240  }
1241 
1242  return FrameTy;
1243 }
1244 
1245 // We use a pointer use visitor to track how an alloca is being used.
1246 // The goal is to be able to answer the following three questions:
1247 // 1. Should this alloca be allocated on the frame instead.
1248 // 2. Could the content of the alloca be modified prior to CoroBegn, which would
1249 // require copying the data from alloca to the frame after CoroBegin.
1250 // 3. Is there any alias created for this alloca prior to CoroBegin, but used
1251 // after CoroBegin. In that case, we will need to recreate the alias after
1252 // CoroBegin based off the frame. To answer question 1, we track two things:
1253 // a. List of all BasicBlocks that use this alloca or any of the aliases of
1254 // the alloca. In the end, we check if there exists any two basic blocks that
1255 // cross suspension points. If so, this alloca must be put on the frame. b.
1256 // Whether the alloca or any alias of the alloca is escaped at some point,
1257 // either by storing the address somewhere, or the address is used in a
1258 // function call that might capture. If it's ever escaped, this alloca must be
1259 // put on the frame conservatively.
1260 // To answer quetion 2, we track through the variable MayWriteBeforeCoroBegin.
1261 // Whenever a potential write happens, either through a store instruction, a
1262 // function call or any of the memory intrinsics, we check whether this
1263 // instruction is prior to CoroBegin. To answer question 3, we track the offsets
1264 // of all aliases created for the alloca prior to CoroBegin but used after
1265 // CoroBegin. llvm::Optional is used to be able to represent the case when the
1266 // offset is unknown (e.g. when you have a PHINode that takes in different
1267 // offset values). We cannot handle unknown offsets and will assert. This is the
1268 // potential issue left out. An ideal solution would likely require a
1269 // significant redesign.
1270 namespace {
1271 struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> {
1273  AllocaUseVisitor(const DataLayout &DL, const DominatorTree &DT,
1274  const CoroBeginInst &CB, const SuspendCrossingInfo &Checker,
1275  bool ShouldUseLifetimeStartInfo)
1276  : PtrUseVisitor(DL), DT(DT), CoroBegin(CB), Checker(Checker),
1277  ShouldUseLifetimeStartInfo(ShouldUseLifetimeStartInfo) {}
1278 
1279  void visit(Instruction &I) {
1280  Users.insert(&I);
1281  Base::visit(I);
1282  // If the pointer is escaped prior to CoroBegin, we have to assume it would
1283  // be written into before CoroBegin as well.
1284  if (PI.isEscaped() && !DT.dominates(&CoroBegin, PI.getEscapingInst())) {
1285  MayWriteBeforeCoroBegin = true;
1286  }
1287  }
1288  // We need to provide this overload as PtrUseVisitor uses a pointer based
1289  // visiting function.
1290  void visit(Instruction *I) { return visit(*I); }
1291 
1292  void visitPHINode(PHINode &I) {
1293  enqueueUsers(I);
1294  handleAlias(I);
1295  }
1296 
1297  void visitSelectInst(SelectInst &I) {
1298  enqueueUsers(I);
1299  handleAlias(I);
1300  }
1301 
1302  void visitStoreInst(StoreInst &SI) {
1303  // Regardless whether the alias of the alloca is the value operand or the
1304  // pointer operand, we need to assume the alloca is been written.
1305  handleMayWrite(SI);
1306 
1307  if (SI.getValueOperand() != U->get())
1308  return;
1309 
1310  // We are storing the pointer into a memory location, potentially escaping.
1311  // As an optimization, we try to detect simple cases where it doesn't
1312  // actually escape, for example:
1313  // %ptr = alloca ..
1314  // %addr = alloca ..
1315  // store %ptr, %addr
1316  // %x = load %addr
1317  // ..
1318  // If %addr is only used by loading from it, we could simply treat %x as
1319  // another alias of %ptr, and not considering %ptr being escaped.
1320  auto IsSimpleStoreThenLoad = [&]() {
1321  auto *AI = dyn_cast<AllocaInst>(SI.getPointerOperand());
1322  // If the memory location we are storing to is not an alloca, it
1323  // could be an alias of some other memory locations, which is difficult
1324  // to analyze.
1325  if (!AI)
1326  return false;
1327  // StoreAliases contains aliases of the memory location stored into.
1328  SmallVector<Instruction *, 4> StoreAliases = {AI};
1329  while (!StoreAliases.empty()) {
1330  Instruction *I = StoreAliases.pop_back_val();
1331  for (User *U : I->users()) {
1332  // If we are loading from the memory location, we are creating an
1333  // alias of the original pointer.
1334  if (auto *LI = dyn_cast<LoadInst>(U)) {
1335  enqueueUsers(*LI);
1336  handleAlias(*LI);
1337  continue;
1338  }
1339  // If we are overriding the memory location, the pointer certainly
1340  // won't escape.
1341  if (auto *S = dyn_cast<StoreInst>(U))
1342  if (S->getPointerOperand() == I)
1343  continue;
1344  if (auto *II = dyn_cast<IntrinsicInst>(U))
1345  if (II->isLifetimeStartOrEnd())
1346  continue;
1347  // BitCastInst creats aliases of the memory location being stored
1348  // into.
1349  if (auto *BI = dyn_cast<BitCastInst>(U)) {
1350  StoreAliases.push_back(BI);
1351  continue;
1352  }
1353  return false;
1354  }
1355  }
1356 
1357  return true;
1358  };
1359 
1360  if (!IsSimpleStoreThenLoad())
1361  PI.setEscaped(&SI);
1362  }
1363 
1364  // All mem intrinsics modify the data.
1365  void visitMemIntrinsic(MemIntrinsic &MI) { handleMayWrite(MI); }
1366 
1367  void visitBitCastInst(BitCastInst &BC) {
1369  handleAlias(BC);
1370  }
1371 
1374  handleAlias(ASC);
1375  }
1376 
1378  // The base visitor will adjust Offset accordingly.
1380  handleAlias(GEPI);
1381  }
1382 
1383  void visitIntrinsicInst(IntrinsicInst &II) {
1384  // When we found the lifetime markers refers to a
1385  // subrange of the original alloca, ignore the lifetime
1386  // markers to avoid misleading the analysis.
1387  if (II.getIntrinsicID() != Intrinsic::lifetime_start || !IsOffsetKnown ||
1388  !Offset.isZero())
1389  return Base::visitIntrinsicInst(II);
1390  LifetimeStarts.insert(&II);
1391  }
1392 
1393  void visitCallBase(CallBase &CB) {
1394  for (unsigned Op = 0, OpCount = CB.arg_size(); Op < OpCount; ++Op)
1395  if (U->get() == CB.getArgOperand(Op) && !CB.doesNotCapture(Op))
1396  PI.setEscaped(&CB);
1397  handleMayWrite(CB);
1398  }
1399 
1400  bool getShouldLiveOnFrame() const {
1401  if (!ShouldLiveOnFrame)
1402  ShouldLiveOnFrame = computeShouldLiveOnFrame();
1403  return *ShouldLiveOnFrame;
1404  }
1405 
1406  bool getMayWriteBeforeCoroBegin() const { return MayWriteBeforeCoroBegin; }
1407 
1408  DenseMap<Instruction *, llvm::Optional<APInt>> getAliasesCopy() const {
1409  assert(getShouldLiveOnFrame() && "This method should only be called if the "
1410  "alloca needs to live on the frame.");
1411  for (const auto &P : AliasOffetMap)
1412  if (!P.second)
1413  report_fatal_error("Unable to handle an alias with unknown offset "
1414  "created before CoroBegin.");
1415  return AliasOffetMap;
1416  }
1417 
1418 private:
1419  const DominatorTree &DT;
1420  const CoroBeginInst &CoroBegin;
1421  const SuspendCrossingInfo &Checker;
1422  // All alias to the original AllocaInst, created before CoroBegin and used
1423  // after CoroBegin. Each entry contains the instruction and the offset in the
1424  // original Alloca. They need to be recreated after CoroBegin off the frame.
1427  SmallPtrSet<IntrinsicInst *, 2> LifetimeStarts{};
1428  bool MayWriteBeforeCoroBegin{false};
1429  bool ShouldUseLifetimeStartInfo{true};
1430 
1431  mutable llvm::Optional<bool> ShouldLiveOnFrame{};
1432 
1433  bool computeShouldLiveOnFrame() const {
1434  // If lifetime information is available, we check it first since it's
1435  // more precise. We look at every pair of lifetime.start intrinsic and
1436  // every basic block that uses the pointer to see if they cross suspension
1437  // points. The uses cover both direct uses as well as indirect uses.
1438  if (ShouldUseLifetimeStartInfo && !LifetimeStarts.empty()) {
1439  for (auto *I : Users)
1440  for (auto *S : LifetimeStarts)
1441  if (Checker.isDefinitionAcrossSuspend(*S, I))
1442  return true;
1443  return false;
1444  }
1445  // FIXME: Ideally the isEscaped check should come at the beginning.
1446  // However there are a few loose ends that need to be fixed first before
1447  // we can do that. We need to make sure we are not over-conservative, so
1448  // that the data accessed in-between await_suspend and symmetric transfer
1449  // is always put on the stack, and also data accessed after coro.end is
1450  // always put on the stack (esp the return object). To fix that, we need
1451  // to:
1452  // 1) Potentially treat sret as nocapture in calls
1453  // 2) Special handle the return object and put it on the stack
1454  // 3) Utilize lifetime.end intrinsic
1455  if (PI.isEscaped())
1456  return true;
1457 
1458  for (auto *U1 : Users)
1459  for (auto *U2 : Users)
1460  if (Checker.isDefinitionAcrossSuspend(*U1, U2))
1461  return true;
1462 
1463  return false;
1464  }
1465 
1466  void handleMayWrite(const Instruction &I) {
1467  if (!DT.dominates(&CoroBegin, &I))
1468  MayWriteBeforeCoroBegin = true;
1469  }
1470 
1471  bool usedAfterCoroBegin(Instruction &I) {
1472  for (auto &U : I.uses())
1473  if (DT.dominates(&CoroBegin, U))
1474  return true;
1475  return false;
1476  }
1477 
1478  void handleAlias(Instruction &I) {
1479  // We track all aliases created prior to CoroBegin but used after.
1480  // These aliases may need to be recreated after CoroBegin if the alloca
1481  // need to live on the frame.
1482  if (DT.dominates(&CoroBegin, &I) || !usedAfterCoroBegin(I))
1483  return;
1484 
1485  if (!IsOffsetKnown) {
1486  AliasOffetMap[&I].reset();
1487  } else {
1488  auto Itr = AliasOffetMap.find(&I);
1489  if (Itr == AliasOffetMap.end()) {
1490  AliasOffetMap[&I] = Offset;
1491  } else if (Itr->second && *Itr->second != Offset) {
1492  // If we have seen two different possible values for this alias, we set
1493  // it to empty.
1494  AliasOffetMap[&I].reset();
1495  }
1496  }
1497  }
1498 };
1499 } // namespace
1500 
1501 // We need to make room to insert a spill after initial PHIs, but before
1502 // catchswitch instruction. Placing it before violates the requirement that
1503 // catchswitch, like all other EHPads must be the first nonPHI in a block.
1504 //
1505 // Split away catchswitch into a separate block and insert in its place:
1506 //
1507 // cleanuppad <InsertPt> cleanupret.
1508 //
1509 // cleanupret instruction will act as an insert point for the spill.
1511  BasicBlock *CurrentBlock = CatchSwitch->getParent();
1512  BasicBlock *NewBlock = CurrentBlock->splitBasicBlock(CatchSwitch);
1513  CurrentBlock->getTerminator()->eraseFromParent();
1514 
1515  auto *CleanupPad =
1516  CleanupPadInst::Create(CatchSwitch->getParentPad(), {}, "", CurrentBlock);
1517  auto *CleanupRet =
1518  CleanupReturnInst::Create(CleanupPad, NewBlock, CurrentBlock);
1519  return CleanupRet;
1520 }
1521 
1522 static void createFramePtr(coro::Shape &Shape) {
1523  auto *CB = Shape.CoroBegin;
1525  StructType *FrameTy = Shape.FrameTy;
1526  PointerType *FramePtrTy = FrameTy->getPointerTo();
1527  Shape.FramePtr =
1528  cast<Instruction>(Builder.CreateBitCast(CB, FramePtrTy, "FramePtr"));
1529 }
1530 
1531 // Replace all alloca and SSA values that are accessed across suspend points
1532 // with GetElementPointer from coroutine frame + loads and stores. Create an
1533 // AllocaSpillBB that will become the new entry block for the resume parts of
1534 // the coroutine:
1535 //
1536 // %hdl = coro.begin(...)
1537 // whatever
1538 //
1539 // becomes:
1540 //
1541 // %hdl = coro.begin(...)
1542 // %FramePtr = bitcast i8* hdl to %f.frame*
1543 // br label %AllocaSpillBB
1544 //
1545 // AllocaSpillBB:
1546 // ; geps corresponding to allocas that were moved to coroutine frame
1547 // br label PostSpill
1548 //
1549 // PostSpill:
1550 // whatever
1551 //
1552 //
1553 static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
1554  auto *CB = Shape.CoroBegin;
1555  LLVMContext &C = CB->getContext();
1557  StructType *FrameTy = Shape.FrameTy;
1558  Value *FramePtr = Shape.FramePtr;
1559  DominatorTree DT(*CB->getFunction());
1561 
1562  // Create a GEP with the given index into the coroutine frame for the original
1563  // value Orig. Appends an extra 0 index for array-allocas, preserving the
1564  // original type.
1565  auto GetFramePointer = [&](Value *Orig) -> Value * {
1566  FieldIDType Index = FrameData.getFieldIndex(Orig);
1567  SmallVector<Value *, 3> Indices = {
1568  ConstantInt::get(Type::getInt32Ty(C), 0),
1569  ConstantInt::get(Type::getInt32Ty(C), Index),
1570  };
1571 
1572  if (auto *AI = dyn_cast<AllocaInst>(Orig)) {
1573  if (auto *CI = dyn_cast<ConstantInt>(AI->getArraySize())) {
1574  auto Count = CI->getValue().getZExtValue();
1575  if (Count > 1) {
1576  Indices.push_back(ConstantInt::get(Type::getInt32Ty(C), 0));
1577  }
1578  } else {
1579  report_fatal_error("Coroutines cannot handle non static allocas yet");
1580  }
1581  }
1582 
1583  auto GEP = cast<GetElementPtrInst>(
1584  Builder.CreateInBoundsGEP(FrameTy, FramePtr, Indices));
1585  if (auto *AI = dyn_cast<AllocaInst>(Orig)) {
1586  if (FrameData.getDynamicAlign(Orig) != 0) {
1587  assert(FrameData.getDynamicAlign(Orig) == AI->getAlign().value());
1588  auto *M = AI->getModule();
1589  auto *IntPtrTy = M->getDataLayout().getIntPtrType(AI->getType());
1590  auto *PtrValue = Builder.CreatePtrToInt(GEP, IntPtrTy);
1591  auto *AlignMask =
1592  ConstantInt::get(IntPtrTy, AI->getAlign().value() - 1);
1593  PtrValue = Builder.CreateAdd(PtrValue, AlignMask);
1594  PtrValue = Builder.CreateAnd(PtrValue, Builder.CreateNot(AlignMask));
1595  return Builder.CreateIntToPtr(PtrValue, AI->getType());
1596  }
1597  // If the type of GEP is not equal to the type of AllocaInst, it implies
1598  // that the AllocaInst may be reused in the Frame slot of other
1599  // AllocaInst. So We cast GEP to the AllocaInst here to re-use
1600  // the Frame storage.
1601  //
1602  // Note: If we change the strategy dealing with alignment, we need to refine
1603  // this casting.
1604  if (GEP->getResultElementType() != Orig->getType())
1605  return Builder.CreateBitCast(GEP, Orig->getType(),
1606  Orig->getName() + Twine(".cast"));
1607  }
1608  return GEP;
1609  };
1610 
1611  for (auto const &E : FrameData.Spills) {
1612  Value *Def = E.first;
1613  auto SpillAlignment = Align(FrameData.getAlign(Def));
1614  // Create a store instruction storing the value into the
1615  // coroutine frame.
1616  Instruction *InsertPt = nullptr;
1617  Type *ByValTy = nullptr;
1618  if (auto *Arg = dyn_cast<Argument>(Def)) {
1619  // For arguments, we will place the store instruction right after
1620  // the coroutine frame pointer instruction, i.e. bitcast of
1621  // coro.begin from i8* to %f.frame*.
1622  InsertPt = Shape.getInsertPtAfterFramePtr();
1623 
1624  // If we're spilling an Argument, make sure we clear 'nocapture'
1625  // from the coroutine function.
1626  Arg->getParent()->removeParamAttr(Arg->getArgNo(), Attribute::NoCapture);
1627 
1628  if (Arg->hasByValAttr())
1629  ByValTy = Arg->getParamByValType();
1630  } else if (auto *CSI = dyn_cast<AnyCoroSuspendInst>(Def)) {
1631  // Don't spill immediately after a suspend; splitting assumes
1632  // that the suspend will be followed by a branch.
1633  InsertPt = CSI->getParent()->getSingleSuccessor()->getFirstNonPHI();
1634  } else {
1635  auto *I = cast<Instruction>(Def);
1636  if (!DT.dominates(CB, I)) {
1637  // If it is not dominated by CoroBegin, then spill should be
1638  // inserted immediately after CoroFrame is computed.
1639  InsertPt = Shape.getInsertPtAfterFramePtr();
1640  } else if (auto *II = dyn_cast<InvokeInst>(I)) {
1641  // If we are spilling the result of the invoke instruction, split
1642  // the normal edge and insert the spill in the new block.
1643  auto *NewBB = SplitEdge(II->getParent(), II->getNormalDest());
1644  InsertPt = NewBB->getTerminator();
1645  } else if (isa<PHINode>(I)) {
1646  // Skip the PHINodes and EH pads instructions.
1647  BasicBlock *DefBlock = I->getParent();
1648  if (auto *CSI = dyn_cast<CatchSwitchInst>(DefBlock->getTerminator()))
1649  InsertPt = splitBeforeCatchSwitch(CSI);
1650  else
1651  InsertPt = &*DefBlock->getFirstInsertionPt();
1652  } else {
1653  assert(!I->isTerminator() && "unexpected terminator");
1654  // For all other values, the spill is placed immediately after
1655  // the definition.
1656  InsertPt = I->getNextNode();
1657  }
1658  }
1659 
1660  auto Index = FrameData.getFieldIndex(Def);
1661  Builder.SetInsertPoint(InsertPt);
1662  auto *G = Builder.CreateConstInBoundsGEP2_32(
1663  FrameTy, FramePtr, 0, Index, Def->getName() + Twine(".spill.addr"));
1664  if (ByValTy) {
1665  // For byval arguments, we need to store the pointed value in the frame,
1666  // instead of the pointer itself.
1667  auto *Value = Builder.CreateLoad(ByValTy, Def);
1668  Builder.CreateAlignedStore(Value, G, SpillAlignment);
1669  } else {
1670  Builder.CreateAlignedStore(Def, G, SpillAlignment);
1671  }
1672 
1673  BasicBlock *CurrentBlock = nullptr;
1674  Value *CurrentReload = nullptr;
1675  for (auto *U : E.second) {
1676  // If we have not seen the use block, create a load instruction to reload
1677  // the spilled value from the coroutine frame. Populates the Value pointer
1678  // reference provided with the frame GEP.
1679  if (CurrentBlock != U->getParent()) {
1680  CurrentBlock = U->getParent();
1681  Builder.SetInsertPoint(&*CurrentBlock->getFirstInsertionPt());
1682 
1683  auto *GEP = GetFramePointer(E.first);
1684  GEP->setName(E.first->getName() + Twine(".reload.addr"));
1685  if (ByValTy)
1686  CurrentReload = GEP;
1687  else
1688  CurrentReload = Builder.CreateAlignedLoad(
1689  FrameTy->getElementType(FrameData.getFieldIndex(E.first)), GEP,
1690  SpillAlignment, E.first->getName() + Twine(".reload"));
1691 
1693  for (DbgDeclareInst *DDI : DIs) {
1694  bool AllowUnresolved = false;
1695  // This dbg.declare is preserved for all coro-split function
1696  // fragments. It will be unreachable in the main function, and
1697  // processed by coro::salvageDebugInfo() by CoroCloner.
1698  DIBuilder(*CurrentBlock->getParent()->getParent(), AllowUnresolved)
1699  .insertDeclare(CurrentReload, DDI->getVariable(),
1700  DDI->getExpression(), DDI->getDebugLoc(),
1701  &*Builder.GetInsertPoint());
1702  // This dbg.declare is for the main function entry point. It
1703  // will be deleted in all coro-split functions.
1704  coro::salvageDebugInfo(DbgPtrAllocaCache, DDI, Shape.OptimizeFrame);
1705  }
1706  }
1707 
1708  // Salvage debug info on any dbg.addr that we see. We do not insert them
1709  // into each block where we have a use though.
1710  if (auto *DI = dyn_cast<DbgAddrIntrinsic>(U)) {
1711  coro::salvageDebugInfo(DbgPtrAllocaCache, DI, Shape.OptimizeFrame);
1712  }
1713 
1714  // If we have a single edge PHINode, remove it and replace it with a
1715  // reload from the coroutine frame. (We already took care of multi edge
1716  // PHINodes by rewriting them in the rewritePHIs function).
1717  if (auto *PN = dyn_cast<PHINode>(U)) {
1718  assert(PN->getNumIncomingValues() == 1 &&
1719  "unexpected number of incoming "
1720  "values in the PHINode");
1721  PN->replaceAllUsesWith(CurrentReload);
1722  PN->eraseFromParent();
1723  continue;
1724  }
1725 
1726  // Replace all uses of CurrentValue in the current instruction with
1727  // reload.
1728  U->replaceUsesOfWith(Def, CurrentReload);
1729  }
1730  }
1731 
1732  BasicBlock *FramePtrBB = Shape.getInsertPtAfterFramePtr()->getParent();
1733 
1734  auto SpillBlock = FramePtrBB->splitBasicBlock(
1735  Shape.getInsertPtAfterFramePtr(), "AllocaSpillBB");
1736  SpillBlock->splitBasicBlock(&SpillBlock->front(), "PostSpill");
1737  Shape.AllocaSpillBlock = SpillBlock;
1738 
1739  // retcon and retcon.once lowering assumes all uses have been sunk.
1740  if (Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce ||
1741  Shape.ABI == coro::ABI::Async) {
1742  // If we found any allocas, replace all of their remaining uses with Geps.
1743  Builder.SetInsertPoint(&SpillBlock->front());
1744  for (const auto &P : FrameData.Allocas) {
1745  AllocaInst *Alloca = P.Alloca;
1746  auto *G = GetFramePointer(Alloca);
1747 
1748  // We are not using ReplaceInstWithInst(P.first, cast<Instruction>(G))
1749  // here, as we are changing location of the instruction.
1750  G->takeName(Alloca);
1751  Alloca->replaceAllUsesWith(G);
1752  Alloca->eraseFromParent();
1753  }
1754  return;
1755  }
1756 
1757  // If we found any alloca, replace all of their remaining uses with GEP
1758  // instructions. To remain debugbility, we replace the uses of allocas for
1759  // dbg.declares and dbg.values with the reload from the frame.
1760  // Note: We cannot replace the alloca with GEP instructions indiscriminately,
1761  // as some of the uses may not be dominated by CoroBegin.
1762  Builder.SetInsertPoint(&Shape.AllocaSpillBlock->front());
1763  SmallVector<Instruction *, 4> UsersToUpdate;
1764  for (const auto &A : FrameData.Allocas) {
1765  AllocaInst *Alloca = A.Alloca;
1766  UsersToUpdate.clear();
1767  for (User *U : Alloca->users()) {
1768  auto *I = cast<Instruction>(U);
1769  if (DT.dominates(CB, I))
1770  UsersToUpdate.push_back(I);
1771  }
1772  if (UsersToUpdate.empty())
1773  continue;
1774  auto *G = GetFramePointer(Alloca);
1775  G->setName(Alloca->getName() + Twine(".reload.addr"));
1776 
1778  findDbgUsers(DIs, Alloca);
1779  for (auto *DVI : DIs)
1780  DVI->replaceUsesOfWith(Alloca, G);
1781 
1782  for (Instruction *I : UsersToUpdate) {
1783  // It is meaningless to remain the lifetime intrinsics refer for the
1784  // member of coroutine frames and the meaningless lifetime intrinsics
1785  // are possible to block further optimizations.
1786  if (I->isLifetimeStartOrEnd())
1787  continue;
1788 
1789  I->replaceUsesOfWith(Alloca, G);
1790  }
1791  }
1792  Builder.SetInsertPoint(Shape.getInsertPtAfterFramePtr());
1793  for (const auto &A : FrameData.Allocas) {
1794  AllocaInst *Alloca = A.Alloca;
1795  if (A.MayWriteBeforeCoroBegin) {
1796  // isEscaped really means potentially modified before CoroBegin.
1797  if (Alloca->isArrayAllocation())
1799  "Coroutines cannot handle copying of array allocas yet");
1800 
1801  auto *G = GetFramePointer(Alloca);
1802  auto *Value = Builder.CreateLoad(Alloca->getAllocatedType(), Alloca);
1803  Builder.CreateStore(Value, G);
1804  }
1805  // For each alias to Alloca created before CoroBegin but used after
1806  // CoroBegin, we recreate them after CoroBegin by appplying the offset
1807  // to the pointer in the frame.
1808  for (const auto &Alias : A.Aliases) {
1809  auto *FramePtr = GetFramePointer(Alloca);
1810  auto *FramePtrRaw =
1811  Builder.CreateBitCast(FramePtr, Type::getInt8PtrTy(C));
1812  auto &Value = *Alias.second;
1813  auto ITy = IntegerType::get(C, Value.getBitWidth());
1814  auto *AliasPtr = Builder.CreateGEP(Type::getInt8Ty(C), FramePtrRaw,
1815  ConstantInt::get(ITy, Value));
1816  auto *AliasPtrTyped =
1817  Builder.CreateBitCast(AliasPtr, Alias.first->getType());
1818  Alias.first->replaceUsesWithIf(
1819  AliasPtrTyped, [&](Use &U) { return DT.dominates(CB, U); });
1820  }
1821  }
1822 
1823  // PromiseAlloca is not collected in FrameData.Allocas. So we don't handle
1824  // the case that the PromiseAlloca may have writes before CoroBegin in the
1825  // above codes. And it may be problematic in edge cases. See
1826  // https://github.com/llvm/llvm-project/issues/57861 for an example.
1827  if (Shape.ABI == coro::ABI::Switch && Shape.SwitchLowering.PromiseAlloca) {
1829  // If there is memory accessing to promise alloca before CoroBegin;
1830  bool HasAccessingPromiseBeforeCB = llvm::any_of(PA->uses(), [&](Use &U) {
1831  auto *Inst = dyn_cast<Instruction>(U.getUser());
1832  if (!Inst || DT.dominates(CB, Inst))
1833  return false;
1834 
1835  if (auto *CI = dyn_cast<CallInst>(Inst)) {
1836  // It is fine if the call wouldn't write to the Promise.
1837  // This is possible for @llvm.coro.id intrinsics, which
1838  // would take the promise as the second argument as a
1839  // marker.
1840  if (CI->onlyReadsMemory() ||
1841  CI->onlyReadsMemory(CI->getArgOperandNo(&U)))
1842  return false;
1843  return true;
1844  }
1845 
1846  return isa<StoreInst>(Inst) ||
1847  // It may take too much time to track the uses.
1848  // Be conservative about the case the use may escape.
1849  isa<GetElementPtrInst>(Inst) ||
1850  // There would always be a bitcast for the promise alloca
1851  // before we enabled Opaque pointers. And now given
1852  // opaque pointers are enabled by default. This should be
1853  // fine.
1854  isa<BitCastInst>(Inst);
1855  });
1856  if (HasAccessingPromiseBeforeCB) {
1857  Builder.SetInsertPoint(Shape.getInsertPtAfterFramePtr());
1858  auto *G = GetFramePointer(PA);
1859  auto *Value = Builder.CreateLoad(PA->getAllocatedType(), PA);
1860  Builder.CreateStore(Value, G);
1861  }
1862  }
1863 }
1864 
1865 // Moves the values in the PHIs in SuccBB that correspong to PredBB into a new
1866 // PHI in InsertedBB.
1868  BasicBlock *InsertedBB,
1869  BasicBlock *PredBB,
1870  PHINode *UntilPHI = nullptr) {
1871  auto *PN = cast<PHINode>(&SuccBB->front());
1872  do {
1873  int Index = PN->getBasicBlockIndex(InsertedBB);
1874  Value *V = PN->getIncomingValue(Index);
1875  PHINode *InputV = PHINode::Create(
1876  V->getType(), 1, V->getName() + Twine(".") + SuccBB->getName(),
1877  &InsertedBB->front());
1878  InputV->addIncoming(V, PredBB);
1879  PN->setIncomingValue(Index, InputV);
1880  PN = dyn_cast<PHINode>(PN->getNextNode());
1881  } while (PN != UntilPHI);
1882 }
1883 
1884 // Rewrites the PHI Nodes in a cleanuppad.
1885 static void rewritePHIsForCleanupPad(BasicBlock *CleanupPadBB,
1886  CleanupPadInst *CleanupPad) {
1887  // For every incoming edge to a CleanupPad we will create a new block holding
1888  // all incoming values in single-value PHI nodes. We will then create another
1889  // block to act as a dispather (as all unwind edges for related EH blocks
1890  // must be the same).
1891  //
1892  // cleanuppad:
1893  // %2 = phi i32[%0, %catchswitch], [%1, %catch.1]
1894  // %3 = cleanuppad within none []
1895  //
1896  // It will create:
1897  //
1898  // cleanuppad.corodispatch
1899  // %2 = phi i8[0, %catchswitch], [1, %catch.1]
1900  // %3 = cleanuppad within none []
1901  // switch i8 % 2, label %unreachable
1902  // [i8 0, label %cleanuppad.from.catchswitch
1903  // i8 1, label %cleanuppad.from.catch.1]
1904  // cleanuppad.from.catchswitch:
1905  // %4 = phi i32 [%0, %catchswitch]
1906  // br %label cleanuppad
1907  // cleanuppad.from.catch.1:
1908  // %6 = phi i32 [%1, %catch.1]
1909  // br %label cleanuppad
1910  // cleanuppad:
1911  // %8 = phi i32 [%4, %cleanuppad.from.catchswitch],
1912  // [%6, %cleanuppad.from.catch.1]
1913 
1914  // Unreachable BB, in case switching on an invalid value in the dispatcher.
1915  auto *UnreachBB = BasicBlock::Create(
1916  CleanupPadBB->getContext(), "unreachable", CleanupPadBB->getParent());
1917  IRBuilder<> Builder(UnreachBB);
1918  Builder.CreateUnreachable();
1919 
1920  // Create a new cleanuppad which will be the dispatcher.
1921  auto *NewCleanupPadBB =
1922  BasicBlock::Create(CleanupPadBB->getContext(),
1923  CleanupPadBB->getName() + Twine(".corodispatch"),
1924  CleanupPadBB->getParent(), CleanupPadBB);
1925  Builder.SetInsertPoint(NewCleanupPadBB);
1926  auto *SwitchType = Builder.getInt8Ty();
1927  auto *SetDispatchValuePN =
1928  Builder.CreatePHI(SwitchType, pred_size(CleanupPadBB));
1929  CleanupPad->removeFromParent();
1930  CleanupPad->insertAfter(SetDispatchValuePN);
1931  auto *SwitchOnDispatch = Builder.CreateSwitch(SetDispatchValuePN, UnreachBB,
1932  pred_size(CleanupPadBB));
1933 
1934  int SwitchIndex = 0;
1935  SmallVector<BasicBlock *, 8> Preds(predecessors(CleanupPadBB));
1936  for (BasicBlock *Pred : Preds) {
1937  // Create a new cleanuppad and move the PHI values to there.
1938  auto *CaseBB = BasicBlock::Create(CleanupPadBB->getContext(),
1939  CleanupPadBB->getName() +
1940  Twine(".from.") + Pred->getName(),
1941  CleanupPadBB->getParent(), CleanupPadBB);
1942  updatePhiNodes(CleanupPadBB, Pred, CaseBB);
1943  CaseBB->setName(CleanupPadBB->getName() + Twine(".from.") +
1944  Pred->getName());
1945  Builder.SetInsertPoint(CaseBB);
1946  Builder.CreateBr(CleanupPadBB);
1947  movePHIValuesToInsertedBlock(CleanupPadBB, CaseBB, NewCleanupPadBB);
1948 
1949  // Update this Pred to the new unwind point.
1950  setUnwindEdgeTo(Pred->getTerminator(), NewCleanupPadBB);
1951 
1952  // Setup the switch in the dispatcher.
1953  auto *SwitchConstant = ConstantInt::get(SwitchType, SwitchIndex);
1954  SetDispatchValuePN->addIncoming(SwitchConstant, Pred);
1955  SwitchOnDispatch->addCase(SwitchConstant, CaseBB);
1956  SwitchIndex++;
1957  }
1958 }
1959 
1961  SmallVector<PHINode *, 32> Worklist;
1962  for (auto &BB : F) {
1963  for (auto &Phi : BB.phis()) {
1964  if (Phi.getNumIncomingValues() == 1) {
1965  Worklist.push_back(&Phi);
1966  } else
1967  break;
1968  }
1969  }
1970  while (!Worklist.empty()) {
1971  auto *Phi = Worklist.pop_back_val();
1972  auto *OriginalValue = Phi->getIncomingValue(0);
1973  Phi->replaceAllUsesWith(OriginalValue);
1974  }
1975 }
1976 
1977 static void rewritePHIs(BasicBlock &BB) {
1978  // For every incoming edge we will create a block holding all
1979  // incoming values in a single PHI nodes.
1980  //
1981  // loop:
1982  // %n.val = phi i32[%n, %entry], [%inc, %loop]
1983  //
1984  // It will create:
1985  //
1986  // loop.from.entry:
1987  // %n.loop.pre = phi i32 [%n, %entry]
1988  // br %label loop
1989  // loop.from.loop:
1990  // %inc.loop.pre = phi i32 [%inc, %loop]
1991  // br %label loop
1992  //
1993  // After this rewrite, further analysis will ignore any phi nodes with more
1994  // than one incoming edge.
1995 
1996  // TODO: Simplify PHINodes in the basic block to remove duplicate
1997  // predecessors.
1998 
1999  // Special case for CleanupPad: all EH blocks must have the same unwind edge
2000  // so we need to create an additional "dispatcher" block.
2001  if (auto *CleanupPad =
2002  dyn_cast_or_null<CleanupPadInst>(BB.getFirstNonPHI())) {
2004  for (BasicBlock *Pred : Preds) {
2005  if (CatchSwitchInst *CS =
2006  dyn_cast<CatchSwitchInst>(Pred->getTerminator())) {
2007  // CleanupPad with a CatchSwitch predecessor: therefore this is an
2008  // unwind destination that needs to be handle specially.
2009  assert(CS->getUnwindDest() == &BB);
2010  (void)CS;
2011  rewritePHIsForCleanupPad(&BB, CleanupPad);
2012  return;
2013  }
2014  }
2015  }
2016 
2017  LandingPadInst *LandingPad = nullptr;
2018  PHINode *ReplPHI = nullptr;
2019  if ((LandingPad = dyn_cast_or_null<LandingPadInst>(BB.getFirstNonPHI()))) {
2020  // ehAwareSplitEdge will clone the LandingPad in all the edge blocks.
2021  // We replace the original landing pad with a PHINode that will collect the
2022  // results from all of them.
2023  ReplPHI = PHINode::Create(LandingPad->getType(), 1, "", LandingPad);
2024  ReplPHI->takeName(LandingPad);
2025  LandingPad->replaceAllUsesWith(ReplPHI);
2026  // We will erase the original landing pad at the end of this function after
2027  // ehAwareSplitEdge cloned it in the transition blocks.
2028  }
2029 
2031  for (BasicBlock *Pred : Preds) {
2032  auto *IncomingBB = ehAwareSplitEdge(Pred, &BB, LandingPad, ReplPHI);
2033  IncomingBB->setName(BB.getName() + Twine(".from.") + Pred->getName());
2034 
2035  // Stop the moving of values at ReplPHI, as this is either null or the PHI
2036  // that replaced the landing pad.
2037  movePHIValuesToInsertedBlock(&BB, IncomingBB, Pred, ReplPHI);
2038  }
2039 
2040  if (LandingPad) {
2041  // Calls to ehAwareSplitEdge function cloned the original lading pad.
2042  // No longer need it.
2043  LandingPad->eraseFromParent();
2044  }
2045 }
2046 
2047 static void rewritePHIs(Function &F) {
2049 
2050  for (BasicBlock &BB : F)
2051  if (auto *PN = dyn_cast<PHINode>(&BB.front()))
2052  if (PN->getNumIncomingValues() > 1)
2053  WorkList.push_back(&BB);
2054 
2055  for (BasicBlock *BB : WorkList)
2056  rewritePHIs(*BB);
2057 }
2058 
2059 // Check for instructions that we can recreate on resume as opposed to spill
2060 // the result into a coroutine frame.
2061 static bool materializable(Instruction &V) {
2062  return isa<CastInst>(&V) || isa<GetElementPtrInst>(&V) ||
2063  isa<BinaryOperator>(&V) || isa<CmpInst>(&V) || isa<SelectInst>(&V);
2064 }
2065 
2066 // Check for structural coroutine intrinsics that should not be spilled into
2067 // the coroutine frame.
2069  return isa<CoroIdInst>(&I) || isa<CoroSaveInst>(&I) ||
2070  isa<CoroSuspendInst>(&I);
2071 }
2072 
2073 // For every use of the value that is across suspend point, recreate that value
2074 // after a suspend point.
2076  const SpillInfo &Spills) {
2077  for (const auto &E : Spills) {
2078  Value *Def = E.first;
2079  BasicBlock *CurrentBlock = nullptr;
2080  Instruction *CurrentMaterialization = nullptr;
2081  for (Instruction *U : E.second) {
2082  // If we have not seen this block, materialize the value.
2083  if (CurrentBlock != U->getParent()) {
2084 
2085  bool IsInCoroSuspendBlock = isa<AnyCoroSuspendInst>(U);
2086  CurrentBlock = U->getParent();
2087  auto *InsertBlock = IsInCoroSuspendBlock
2088  ? CurrentBlock->getSinglePredecessor()
2089  : CurrentBlock;
2090  CurrentMaterialization = cast<Instruction>(Def)->clone();
2091  CurrentMaterialization->setName(Def->getName());
2092  CurrentMaterialization->insertBefore(
2093  IsInCoroSuspendBlock ? InsertBlock->getTerminator()
2094  : &*InsertBlock->getFirstInsertionPt());
2095  }
2096  if (auto *PN = dyn_cast<PHINode>(U)) {
2097  assert(PN->getNumIncomingValues() == 1 &&
2098  "unexpected number of incoming "
2099  "values in the PHINode");
2100  PN->replaceAllUsesWith(CurrentMaterialization);
2101  PN->eraseFromParent();
2102  continue;
2103  }
2104  // Replace all uses of Def in the current instruction with the
2105  // CurrentMaterialization for the block.
2106  U->replaceUsesOfWith(Def, CurrentMaterialization);
2107  }
2108  }
2109 }
2110 
2111 // Splits the block at a particular instruction unless it is the first
2112 // instruction in the block with a single predecessor.
2114  auto *BB = I->getParent();
2115  if (&BB->front() == I) {
2116  if (BB->getSinglePredecessor()) {
2117  BB->setName(Name);
2118  return BB;
2119  }
2120  }
2121  return BB->splitBasicBlock(I, Name);
2122 }
2123 
2124 // Split above and below a particular instruction so that it
2125 // will be all alone by itself in a block.
2126 static void splitAround(Instruction *I, const Twine &Name) {
2128  splitBlockIfNotFirst(I->getNextNode(), "After" + Name);
2129 }
2130 
2132  return isa<AnyCoroSuspendInst>(BB->front());
2133 }
2134 
2136 
2137 /// Does control flow starting at the given block ever reach a suspend
2138 /// instruction before reaching a block in VisitedOrFreeBBs?
2140  VisitedBlocksSet &VisitedOrFreeBBs) {
2141  // Eagerly try to add this block to the visited set. If it's already
2142  // there, stop recursing; this path doesn't reach a suspend before
2143  // either looping or reaching a freeing block.
2144  if (!VisitedOrFreeBBs.insert(From).second)
2145  return false;
2146 
2147  // We assume that we'll already have split suspends into their own blocks.
2148  if (isSuspendBlock(From))
2149  return true;
2150 
2151  // Recurse on the successors.
2152  for (auto *Succ : successors(From)) {
2153  if (isSuspendReachableFrom(Succ, VisitedOrFreeBBs))
2154  return true;
2155  }
2156 
2157  return false;
2158 }
2159 
2160 /// Is the given alloca "local", i.e. bounded in lifetime to not cross a
2161 /// suspend point?
2163  // Seed the visited set with all the basic blocks containing a free
2164  // so that we won't pass them up.
2165  VisitedBlocksSet VisitedOrFreeBBs;
2166  for (auto *User : AI->users()) {
2167  if (auto FI = dyn_cast<CoroAllocaFreeInst>(User))
2168  VisitedOrFreeBBs.insert(FI->getParent());
2169  }
2170 
2171  return !isSuspendReachableFrom(AI->getParent(), VisitedOrFreeBBs);
2172 }
2173 
2174 /// After we split the coroutine, will the given basic block be along
2175 /// an obvious exit path for the resumption function?
2177  unsigned depth = 3) {
2178  // If we've bottomed out our depth count, stop searching and assume
2179  // that the path might loop back.
2180  if (depth == 0) return false;
2181 
2182  // If this is a suspend block, we're about to exit the resumption function.
2183  if (isSuspendBlock(BB)) return true;
2184 
2185  // Recurse into the successors.
2186  for (auto *Succ : successors(BB)) {
2187  if (!willLeaveFunctionImmediatelyAfter(Succ, depth - 1))
2188  return false;
2189  }
2190 
2191  // If none of the successors leads back in a loop, we're on an exit/abort.
2192  return true;
2193 }
2194 
2196  // Look for a free that isn't sufficiently obviously followed by
2197  // either a suspend or a termination, i.e. something that will leave
2198  // the coro resumption frame.
2199  for (auto *U : AI->users()) {
2200  auto FI = dyn_cast<CoroAllocaFreeInst>(U);
2201  if (!FI) continue;
2202 
2203  if (!willLeaveFunctionImmediatelyAfter(FI->getParent()))
2204  return true;
2205  }
2206 
2207  // If we never found one, we don't need a stack save.
2208  return false;
2209 }
2210 
2211 /// Turn each of the given local allocas into a normal (dynamic) alloca
2212 /// instruction.
2214  SmallVectorImpl<Instruction*> &DeadInsts) {
2215  for (auto *AI : LocalAllocas) {
2216  auto M = AI->getModule();
2217  IRBuilder<> Builder(AI);
2218 
2219  // Save the stack depth. Try to avoid doing this if the stackrestore
2220  // is going to immediately precede a return or something.
2221  Value *StackSave = nullptr;
2222  if (localAllocaNeedsStackSave(AI))
2223  StackSave = Builder.CreateCall(
2224  Intrinsic::getDeclaration(M, Intrinsic::stacksave));
2225 
2226  // Allocate memory.
2227  auto Alloca = Builder.CreateAlloca(Builder.getInt8Ty(), AI->getSize());
2228  Alloca->setAlignment(AI->getAlignment());
2229 
2230  for (auto *U : AI->users()) {
2231  // Replace gets with the allocation.
2232  if (isa<CoroAllocaGetInst>(U)) {
2233  U->replaceAllUsesWith(Alloca);
2234 
2235  // Replace frees with stackrestores. This is safe because
2236  // alloca.alloc is required to obey a stack discipline, although we
2237  // don't enforce that structurally.
2238  } else {
2239  auto FI = cast<CoroAllocaFreeInst>(U);
2240  if (StackSave) {
2241  Builder.SetInsertPoint(FI);
2242  Builder.CreateCall(
2243  Intrinsic::getDeclaration(M, Intrinsic::stackrestore),
2244  StackSave);
2245  }
2246  }
2247  DeadInsts.push_back(cast<Instruction>(U));
2248  }
2249 
2250  DeadInsts.push_back(AI);
2251  }
2252 }
2253 
2254 /// Turn the given coro.alloca.alloc call into a dynamic allocation.
2255 /// This happens during the all-instructions iteration, so it must not
2256 /// delete the call.
2258  coro::Shape &Shape,
2259  SmallVectorImpl<Instruction*> &DeadInsts) {
2260  IRBuilder<> Builder(AI);
2261  auto Alloc = Shape.emitAlloc(Builder, AI->getSize(), nullptr);
2262 
2263  for (User *U : AI->users()) {
2264  if (isa<CoroAllocaGetInst>(U)) {
2265  U->replaceAllUsesWith(Alloc);
2266  } else {
2267  auto FI = cast<CoroAllocaFreeInst>(U);
2268  Builder.SetInsertPoint(FI);
2269  Shape.emitDealloc(Builder, Alloc, nullptr);
2270  }
2271  DeadInsts.push_back(cast<Instruction>(U));
2272  }
2273 
2274  // Push this on last so that it gets deleted after all the others.
2275  DeadInsts.push_back(AI);
2276 
2277  // Return the new allocation value so that we can check for needed spills.
2278  return cast<Instruction>(Alloc);
2279 }
2280 
2281 /// Get the current swifterror value.
2283  coro::Shape &Shape) {
2284  // Make a fake function pointer as a sort of intrinsic.
2285  auto FnTy = FunctionType::get(ValueTy, {}, false);
2286  auto Fn = ConstantPointerNull::get(FnTy->getPointerTo());
2287 
2288  auto Call = Builder.CreateCall(FnTy, Fn, {});
2289  Shape.SwiftErrorOps.push_back(Call);
2290 
2291  return Call;
2292 }
2293 
2294 /// Set the given value as the current swifterror value.
2295 ///
2296 /// Returns a slot that can be used as a swifterror slot.
2298  coro::Shape &Shape) {
2299  // Make a fake function pointer as a sort of intrinsic.
2300  auto FnTy = FunctionType::get(V->getType()->getPointerTo(),
2301  {V->getType()}, false);
2302  auto Fn = ConstantPointerNull::get(FnTy->getPointerTo());
2303 
2304  auto Call = Builder.CreateCall(FnTy, Fn, { V });
2305  Shape.SwiftErrorOps.push_back(Call);
2306 
2307  return Call;
2308 }
2309 
2310 /// Set the swifterror value from the given alloca before a call,
2311 /// then put in back in the alloca afterwards.
2312 ///
2313 /// Returns an address that will stand in for the swifterror slot
2314 /// until splitting.
2316  AllocaInst *Alloca,
2317  coro::Shape &Shape) {
2318  auto ValueTy = Alloca->getAllocatedType();
2320 
2321  // Load the current value from the alloca and set it as the
2322  // swifterror value.
2323  auto ValueBeforeCall = Builder.CreateLoad(ValueTy, Alloca);
2324  auto Addr = emitSetSwiftErrorValue(Builder, ValueBeforeCall, Shape);
2325 
2326  // Move to after the call. Since swifterror only has a guaranteed
2327  // value on normal exits, we can ignore implicit and explicit unwind
2328  // edges.
2329  if (isa<CallInst>(Call)) {
2330  Builder.SetInsertPoint(Call->getNextNode());
2331  } else {
2332  auto Invoke = cast<InvokeInst>(Call);
2333  Builder.SetInsertPoint(Invoke->getNormalDest()->getFirstNonPHIOrDbg());
2334  }
2335 
2336  // Get the current swifterror value and store it to the alloca.
2337  auto ValueAfterCall = emitGetSwiftErrorValue(Builder, ValueTy, Shape);
2338  Builder.CreateStore(ValueAfterCall, Alloca);
2339 
2340  return Addr;
2341 }
2342 
2343 /// Eliminate a formerly-swifterror alloca by inserting the get/set
2344 /// intrinsics and attempting to MemToReg the alloca away.
2346  coro::Shape &Shape) {
2347  for (Use &Use : llvm::make_early_inc_range(Alloca->uses())) {
2348  // swifterror values can only be used in very specific ways.
2349  // We take advantage of that here.
2350  auto User = Use.getUser();
2351  if (isa<LoadInst>(User) || isa<StoreInst>(User))
2352  continue;
2353 
2354  assert(isa<CallInst>(User) || isa<InvokeInst>(User));
2355  auto Call = cast<Instruction>(User);
2356 
2357  auto Addr = emitSetAndGetSwiftErrorValueAround(Call, Alloca, Shape);
2358 
2359  // Use the returned slot address as the call argument.
2360  Use.set(Addr);
2361  }
2362 
2363  // All the uses should be loads and stores now.
2364  assert(isAllocaPromotable(Alloca));
2365 }
2366 
2367 /// "Eliminate" a swifterror argument by reducing it to the alloca case
2368 /// and then loading and storing in the prologue and epilog.
2369 ///
2370 /// The argument keeps the swifterror flag.
2372  coro::Shape &Shape,
2373  SmallVectorImpl<AllocaInst*> &AllocasToPromote) {
2374  IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHIOrDbg());
2375 
2376  auto ArgTy = cast<PointerType>(Arg.getType());
2377  // swifterror arguments are required to have pointer-to-pointer type,
2378  // so create a pointer-typed alloca with opaque pointers.
2379  auto ValueTy = ArgTy->isOpaque() ? PointerType::getUnqual(F.getContext())
2380  : ArgTy->getNonOpaquePointerElementType();
2381 
2382  // Reduce to the alloca case:
2383 
2384  // Create an alloca and replace all uses of the arg with it.
2385  auto Alloca = Builder.CreateAlloca(ValueTy, ArgTy->getAddressSpace());
2386  Arg.replaceAllUsesWith(Alloca);
2387 
2388  // Set an initial value in the alloca. swifterror is always null on entry.
2389  auto InitialValue = Constant::getNullValue(ValueTy);
2390  Builder.CreateStore(InitialValue, Alloca);
2391 
2392  // Find all the suspends in the function and save and restore around them.
2393  for (auto *Suspend : Shape.CoroSuspends) {
2394  (void) emitSetAndGetSwiftErrorValueAround(Suspend, Alloca, Shape);
2395  }
2396 
2397  // Find all the coro.ends in the function and restore the error value.
2398  for (auto *End : Shape.CoroEnds) {
2399  Builder.SetInsertPoint(End);
2400  auto FinalValue = Builder.CreateLoad(ValueTy, Alloca);
2401  (void) emitSetSwiftErrorValue(Builder, FinalValue, Shape);
2402  }
2403 
2404  // Now we can use the alloca logic.
2405  AllocasToPromote.push_back(Alloca);
2406  eliminateSwiftErrorAlloca(F, Alloca, Shape);
2407 }
2408 
2409 /// Eliminate all problematic uses of swifterror arguments and allocas
2410 /// from the function. We'll fix them up later when splitting the function.
2412  SmallVector<AllocaInst*, 4> AllocasToPromote;
2413 
2414  // Look for a swifterror argument.
2415  for (auto &Arg : F.args()) {
2416  if (!Arg.hasSwiftErrorAttr()) continue;
2417 
2418  eliminateSwiftErrorArgument(F, Arg, Shape, AllocasToPromote);
2419  break;
2420  }
2421 
2422  // Look for swifterror allocas.
2423  for (auto &Inst : F.getEntryBlock()) {
2424  auto Alloca = dyn_cast<AllocaInst>(&Inst);
2425  if (!Alloca || !Alloca->isSwiftError()) continue;
2426 
2427  // Clear the swifterror flag.
2428  Alloca->setSwiftError(false);
2429 
2430  AllocasToPromote.push_back(Alloca);
2431  eliminateSwiftErrorAlloca(F, Alloca, Shape);
2432  }
2433 
2434  // If we have any allocas to promote, compute a dominator tree and
2435  // promote them en masse.
2436  if (!AllocasToPromote.empty()) {
2437  DominatorTree DT(F);
2438  PromoteMemToReg(AllocasToPromote, DT);
2439  }
2440 }
2441 
2442 /// retcon and retcon.once conventions assume that all spill uses can be sunk
2443 /// after the coro.begin intrinsic.
2445  const FrameDataInfo &FrameData,
2446  CoroBeginInst *CoroBegin) {
2447  DominatorTree Dom(F);
2448 
2451 
2452  // Collect all users that precede coro.begin.
2453  for (auto *Def : FrameData.getAllDefs()) {
2454  for (User *U : Def->users()) {
2455  auto Inst = cast<Instruction>(U);
2456  if (Inst->getParent() != CoroBegin->getParent() ||
2457  Dom.dominates(CoroBegin, Inst))
2458  continue;
2459  if (ToMove.insert(Inst))
2460  Worklist.push_back(Inst);
2461  }
2462  }
2463  // Recursively collect users before coro.begin.
2464  while (!Worklist.empty()) {
2465  auto *Def = Worklist.pop_back_val();
2466  for (User *U : Def->users()) {
2467  auto Inst = cast<Instruction>(U);
2468  if (Dom.dominates(CoroBegin, Inst))
2469  continue;
2470  if (ToMove.insert(Inst))
2471  Worklist.push_back(Inst);
2472  }
2473  }
2474 
2475  // Sort by dominance.
2476  SmallVector<Instruction *, 64> InsertionList(ToMove.begin(), ToMove.end());
2477  llvm::sort(InsertionList, [&Dom](Instruction *A, Instruction *B) -> bool {
2478  // If a dominates b it should preceed (<) b.
2479  return Dom.dominates(A, B);
2480  });
2481 
2482  Instruction *InsertPt = CoroBegin->getNextNode();
2483  for (Instruction *Inst : InsertionList)
2484  Inst->moveBefore(InsertPt);
2485 }
2486 
2487 /// For each local variable that all of its user are only used inside one of
2488 /// suspended region, we sink their lifetime.start markers to the place where
2489 /// after the suspend block. Doing so minimizes the lifetime of each variable,
2490 /// hence minimizing the amount of data we end up putting on the frame.
2492  SuspendCrossingInfo &Checker) {
2493  DominatorTree DT(F);
2494 
2495  // Collect all possible basic blocks which may dominate all uses of allocas.
2497  DomSet.insert(&F.getEntryBlock());
2498  for (auto *CSI : Shape.CoroSuspends) {
2499  BasicBlock *SuspendBlock = CSI->getParent();
2500  assert(isSuspendBlock(SuspendBlock) && SuspendBlock->getSingleSuccessor() &&
2501  "should have split coro.suspend into its own block");
2502  DomSet.insert(SuspendBlock->getSingleSuccessor());
2503  }
2504 
2505  for (Instruction &I : instructions(F)) {
2506  AllocaInst* AI = dyn_cast<AllocaInst>(&I);
2507  if (!AI)
2508  continue;
2509 
2510  for (BasicBlock *DomBB : DomSet) {
2511  bool Valid = true;
2513 
2514  auto isLifetimeStart = [](Instruction* I) {
2515  if (auto* II = dyn_cast<IntrinsicInst>(I))
2516  return II->getIntrinsicID() == Intrinsic::lifetime_start;
2517  return false;
2518  };
2519 
2520  auto collectLifetimeStart = [&](Instruction *U, AllocaInst *AI) {
2521  if (isLifetimeStart(U)) {
2522  Lifetimes.push_back(U);
2523  return true;
2524  }
2525  if (!U->hasOneUse() || U->stripPointerCasts() != AI)
2526  return false;
2527  if (isLifetimeStart(U->user_back())) {
2528  Lifetimes.push_back(U->user_back());
2529  return true;
2530  }
2531  return false;
2532  };
2533 
2534  for (User *U : AI->users()) {
2535  Instruction *UI = cast<Instruction>(U);
2536  // For all users except lifetime.start markers, if they are all
2537  // dominated by one of the basic blocks and do not cross
2538  // suspend points as well, then there is no need to spill the
2539  // instruction.
2540  if (!DT.dominates(DomBB, UI->getParent()) ||
2541  Checker.isDefinitionAcrossSuspend(DomBB, UI)) {
2542  // Skip lifetime.start, GEP and bitcast used by lifetime.start
2543  // markers.
2544  if (collectLifetimeStart(UI, AI))
2545  continue;
2546  Valid = false;
2547  break;
2548  }
2549  }
2550  // Sink lifetime.start markers to dominate block when they are
2551  // only used outside the region.
2552  if (Valid && Lifetimes.size() != 0) {
2553  // May be AI itself, when the type of AI is i8*
2554  auto *NewBitCast = [&](AllocaInst *AI) -> Value* {
2555  if (isa<AllocaInst>(Lifetimes[0]->getOperand(1)))
2556  return AI;
2557  auto *Int8PtrTy = Type::getInt8PtrTy(F.getContext());
2558  return CastInst::Create(Instruction::BitCast, AI, Int8PtrTy, "",
2559  DomBB->getTerminator());
2560  }(AI);
2561 
2562  auto *NewLifetime = Lifetimes[0]->clone();
2563  NewLifetime->replaceUsesOfWith(NewLifetime->getOperand(1), NewBitCast);
2564  NewLifetime->insertBefore(DomBB->getTerminator());
2565 
2566  // All the outsided lifetime.start markers are no longer necessary.
2567  for (Instruction *S : Lifetimes)
2568  S->eraseFromParent();
2569 
2570  break;
2571  }
2572  }
2573  }
2574 }
2575 
2577  const SuspendCrossingInfo &Checker,
2578  SmallVectorImpl<AllocaInfo> &Allocas) {
2579  for (Instruction &I : instructions(F)) {
2580  auto *AI = dyn_cast<AllocaInst>(&I);
2581  if (!AI)
2582  continue;
2583  // The PromiseAlloca will be specially handled since it needs to be in a
2584  // fixed position in the frame.
2585  if (AI == Shape.SwitchLowering.PromiseAlloca) {
2586  continue;
2587  }
2588  DominatorTree DT(F);
2589  // The code that uses lifetime.start intrinsic does not work for functions
2590  // with loops without exit. Disable it on ABIs we know to generate such
2591  // code.
2592  bool ShouldUseLifetimeStartInfo =
2593  (Shape.ABI != coro::ABI::Async && Shape.ABI != coro::ABI::Retcon &&
2594  Shape.ABI != coro::ABI::RetconOnce);
2595  AllocaUseVisitor Visitor{F.getParent()->getDataLayout(), DT,
2596  *Shape.CoroBegin, Checker,
2597  ShouldUseLifetimeStartInfo};
2598  Visitor.visitPtr(*AI);
2599  if (!Visitor.getShouldLiveOnFrame())
2600  continue;
2601  Allocas.emplace_back(AI, Visitor.getAliasesCopy(),
2602  Visitor.getMayWriteBeforeCoroBegin());
2603  }
2604 }
2605 
2608  DbgVariableIntrinsic *DVI, bool OptimizeFrame) {
2609  Function *F = DVI->getFunction();
2610  IRBuilder<> Builder(F->getContext());
2611  auto InsertPt = F->getEntryBlock().getFirstInsertionPt();
2612  while (isa<IntrinsicInst>(InsertPt))
2613  ++InsertPt;
2614  Builder.SetInsertPoint(&F->getEntryBlock(), InsertPt);
2615  DIExpression *Expr = DVI->getExpression();
2616  // Follow the pointer arithmetic all the way to the incoming
2617  // function argument and convert into a DIExpression.
2618  bool SkipOutermostLoad = !isa<DbgValueInst>(DVI);
2619  Value *Storage = DVI->getVariableLocationOp(0);
2620  Value *OriginalStorage = Storage;
2621 
2622  while (auto *Inst = dyn_cast_or_null<Instruction>(Storage)) {
2623  if (auto *LdInst = dyn_cast<LoadInst>(Inst)) {
2624  Storage = LdInst->getOperand(0);
2625  // FIXME: This is a heuristic that works around the fact that
2626  // LLVM IR debug intrinsics cannot yet distinguish between
2627  // memory and value locations: Because a dbg.declare(alloca) is
2628  // implicitly a memory location no DW_OP_deref operation for the
2629  // last direct load from an alloca is necessary. This condition
2630  // effectively drops the *last* DW_OP_deref in the expression.
2631  if (!SkipOutermostLoad)
2632  Expr = DIExpression::prepend(Expr, DIExpression::DerefBefore);
2633  } else if (auto *StInst = dyn_cast<StoreInst>(Inst)) {
2634  Storage = StInst->getOperand(0);
2635  } else {
2637  SmallVector<Value *, 0> AdditionalValues;
2639  *Inst, Expr ? Expr->getNumLocationOperands() : 0, Ops,
2640  AdditionalValues);
2641  if (!Op || !AdditionalValues.empty()) {
2642  // If salvaging failed or salvaging produced more than one location
2643  // operand, give up.
2644  break;
2645  }
2646  Storage = Op;
2647  Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, /*StackValue*/ false);
2648  }
2649  SkipOutermostLoad = false;
2650  }
2651  if (!Storage)
2652  return;
2653 
2654  // Store a pointer to the coroutine frame object in an alloca so it
2655  // is available throughout the function when producing unoptimized
2656  // code. Extending the lifetime this way is correct because the
2657  // variable has been declared by a dbg.declare intrinsic.
2658  //
2659  // Avoid to create the alloca would be eliminated by optimization
2660  // passes and the corresponding dbg.declares would be invalid.
2661  if (!OptimizeFrame)
2662  if (auto *Arg = dyn_cast<llvm::Argument>(Storage)) {
2663  auto &Cached = DbgPtrAllocaCache[Storage];
2664  if (!Cached) {
2665  Cached = Builder.CreateAlloca(Storage->getType(), 0, nullptr,
2666  Arg->getName() + ".debug");
2667  Builder.CreateStore(Storage, Cached);
2668  }
2669  Storage = Cached;
2670  // FIXME: LLVM lacks nuanced semantics to differentiate between
2671  // memory and direct locations at the IR level. The backend will
2672  // turn a dbg.declare(alloca, ..., DIExpression()) into a memory
2673  // location. Thus, if there are deref and offset operations in the
2674  // expression, we need to add a DW_OP_deref at the *start* of the
2675  // expression to first load the contents of the alloca before
2676  // adjusting it with the expression.
2677  Expr = DIExpression::prepend(Expr, DIExpression::DerefBefore);
2678  }
2679 
2680  DVI->replaceVariableLocationOp(OriginalStorage, Storage);
2681  DVI->setExpression(Expr);
2682  // We only hoist dbg.declare today since it doesn't make sense to hoist
2683  // dbg.value or dbg.addr since they do not have the same function wide
2684  // guarantees that dbg.declare does.
2685  if (!isa<DbgValueInst>(DVI) && !isa<DbgAddrIntrinsic>(DVI)) {
2686  Instruction *InsertPt = nullptr;
2687  if (auto *I = dyn_cast<Instruction>(Storage))
2688  InsertPt = I->getInsertionPointAfterDef();
2689  else if (isa<Argument>(Storage))
2690  InsertPt = &*F->getEntryBlock().begin();
2691  if (InsertPt)
2692  DVI->moveBefore(InsertPt);
2693  }
2694 }
2695 
2697  // Don't eliminate swifterror in async functions that won't be split.
2698  if (Shape.ABI != coro::ABI::Async || !Shape.CoroSuspends.empty())
2700 
2701  if (Shape.ABI == coro::ABI::Switch &&
2704  }
2705 
2706  // Make sure that all coro.save, coro.suspend and the fallthrough coro.end
2707  // intrinsics are in their own blocks to simplify the logic of building up
2708  // SuspendCrossing data.
2709  for (auto *CSI : Shape.CoroSuspends) {
2710  if (auto *Save = CSI->getCoroSave())
2711  splitAround(Save, "CoroSave");
2712  splitAround(CSI, "CoroSuspend");
2713  }
2714 
2715  // Put CoroEnds into their own blocks.
2716  for (AnyCoroEndInst *CE : Shape.CoroEnds) {
2717  splitAround(CE, "CoroEnd");
2718 
2719  // Emit the musttail call function in a new block before the CoroEnd.
2720  // We do this here so that the right suspend crossing info is computed for
2721  // the uses of the musttail call function call. (Arguments to the coro.end
2722  // instructions would be ignored)
2723  if (auto *AsyncEnd = dyn_cast<CoroAsyncEndInst>(CE)) {
2724  auto *MustTailCallFn = AsyncEnd->getMustTailCallFunction();
2725  if (!MustTailCallFn)
2726  continue;
2727  IRBuilder<> Builder(AsyncEnd);
2728  SmallVector<Value *, 8> Args(AsyncEnd->args());
2730  auto *Call = createMustTailCall(AsyncEnd->getDebugLoc(), MustTailCallFn,
2731  Arguments, Builder);
2732  splitAround(Call, "MustTailCall.Before.CoroEnd");
2733  }
2734  }
2735 
2736  // Later code makes structural assumptions about single predecessors phis e.g
2737  // that they are not live across a suspend point.
2739 
2740  // Transforms multi-edge PHI Nodes, so that any value feeding into a PHI will
2741  // never has its definition separated from the PHI by the suspend point.
2742  rewritePHIs(F);
2743 
2744  // Build suspend crossing info.
2745  SuspendCrossingInfo Checker(F, Shape);
2746 
2747  IRBuilder<> Builder(F.getContext());
2748  FrameDataInfo FrameData;
2750  SmallVector<Instruction*, 4> DeadInstructions;
2751 
2752  {
2753  SpillInfo Spills;
2754  for (int Repeat = 0; Repeat < 4; ++Repeat) {
2755  // See if there are materializable instructions across suspend points.
2756  for (Instruction &I : instructions(F))
2757  if (materializable(I)) {
2758  for (User *U : I.users())
2759  if (Checker.isDefinitionAcrossSuspend(I, U))
2760  Spills[&I].push_back(cast<Instruction>(U));
2761  }
2762 
2763  if (Spills.empty())
2764  break;
2765 
2766  // Rewrite materializable instructions to be materialized at the use
2767  // point.
2768  LLVM_DEBUG(dumpSpills("Materializations", Spills));
2770  Spills.clear();
2771  }
2772  }
2773 
2774  if (Shape.ABI != coro::ABI::Async && Shape.ABI != coro::ABI::Retcon &&
2775  Shape.ABI != coro::ABI::RetconOnce)
2776  sinkLifetimeStartMarkers(F, Shape, Checker);
2777 
2778  if (Shape.ABI == coro::ABI::Switch || !Shape.CoroSuspends.empty())
2779  collectFrameAllocas(F, Shape, Checker, FrameData.Allocas);
2780  LLVM_DEBUG(dumpAllocas(FrameData.Allocas));
2781 
2782  // Collect the spills for arguments and other not-materializable values.
2783  for (Argument &A : F.args())
2784  for (User *U : A.users())
2785  if (Checker.isDefinitionAcrossSuspend(A, U))
2786  FrameData.Spills[&A].push_back(cast<Instruction>(U));
2787 
2788  for (Instruction &I : instructions(F)) {
2789  // Values returned from coroutine structure intrinsics should not be part
2790  // of the Coroutine Frame.
2792  continue;
2793 
2794  // The Coroutine Promise always included into coroutine frame, no need to
2795  // check for suspend crossing.
2796  if (Shape.ABI == coro::ABI::Switch &&
2798  continue;
2799 
2800  // Handle alloca.alloc specially here.
2801  if (auto AI = dyn_cast<CoroAllocaAllocInst>(&I)) {
2802  // Check whether the alloca's lifetime is bounded by suspend points.
2803  if (isLocalAlloca(AI)) {
2804  LocalAllocas.push_back(AI);
2805  continue;
2806  }
2807 
2808  // If not, do a quick rewrite of the alloca and then add spills of
2809  // the rewritten value. The rewrite doesn't invalidate anything in
2810  // Spills because the other alloca intrinsics have no other operands
2811  // besides AI, and it doesn't invalidate the iteration because we delay
2812  // erasing AI.
2813  auto Alloc = lowerNonLocalAlloca(AI, Shape, DeadInstructions);
2814 
2815  for (User *U : Alloc->users()) {
2816  if (Checker.isDefinitionAcrossSuspend(*Alloc, U))
2817  FrameData.Spills[Alloc].push_back(cast<Instruction>(U));
2818  }
2819  continue;
2820  }
2821 
2822  // Ignore alloca.get; we process this as part of coro.alloca.alloc.
2823  if (isa<CoroAllocaGetInst>(I))
2824  continue;
2825 
2826  if (isa<AllocaInst>(I))
2827  continue;
2828 
2829  for (User *U : I.users())
2830  if (Checker.isDefinitionAcrossSuspend(I, U)) {
2831  // We cannot spill a token.
2832  if (I.getType()->isTokenTy())
2834  "token definition is separated from the use by a suspend point");
2835  FrameData.Spills[&I].push_back(cast<Instruction>(U));
2836  }
2837  }
2838 
2839  // We don't want the layout of coroutine frame to be affected
2840  // by debug information. So we only choose to salvage DbgValueInst for
2841  // whose value is already in the frame.
2842  // We would handle the dbg.values for allocas specially
2843  for (auto &Iter : FrameData.Spills) {
2844  auto *V = Iter.first;
2846  findDbgValues(DVIs, V);
2847  for (DbgValueInst *DVI : DVIs)
2848  if (Checker.isDefinitionAcrossSuspend(*V, DVI))
2849  FrameData.Spills[V].push_back(DVI);
2850  }
2851 
2852  LLVM_DEBUG(dumpSpills("Spills", FrameData.Spills));
2853  if (Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce ||
2854  Shape.ABI == coro::ABI::Async)
2856  Shape.FrameTy = buildFrameType(F, Shape, FrameData);
2858  // For now, this works for C++ programs only.
2859  buildFrameDebugInfo(F, Shape, FrameData);
2860  insertSpills(FrameData, Shape);
2861  lowerLocalAllocas(LocalAllocas, DeadInstructions);
2862 
2863  for (auto *I : DeadInstructions)
2864  I->eraseFromParent();
2865 }
llvm::Check::Size
@ Size
Definition: FileCheck.h:77
llvm::coro::Shape::getInsertPtAfterFramePtr
Instruction * getInsertPtAfterFramePtr() const
Definition: CoroInternal.h:240
PtrUseVisitor.h
alignTo
static int alignTo(int Num, int PowOf2)
Definition: AArch64LoadStoreOptimizer.cpp:1204
llvm::detail::PtrUseVisitorBase::enqueueUsers
void enqueueUsers(Instruction &I)
Enqueue the users of this instruction in the visit worklist.
Definition: PtrUseVisitor.cpp:20
splitBlockIfNotFirst
static BasicBlock * splitBlockIfNotFirst(Instruction *I, const Twine &Name)
Definition: CoroFrame.cpp:2113
llvm::DbgVariableIntrinsic::getExpression
DIExpression * getExpression() const
Definition: IntrinsicInst.h:292
llvm::Argument
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
llvm::coro::createMustTailCall
CallInst * createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, ArrayRef< Value * > Arguments, IRBuilder<> &)
Definition: CoroSplit.cpp:1675
llvm::Type::isSized
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:283
llvm::coro::Shape::AsyncLoweringStorage::FrameOffset
uint64_t FrameOffset
Definition: CoroInternal.h:133
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:109
LLVM_DUMP_METHOD
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Definition: Compiler.h:492
MathExtras.h
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
llvm::DIType
Base class for types.
Definition: DebugInfoMetadata.h:699
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::DIExpression::getNumLocationOperands
uint64_t getNumLocationOperands() const
Return the number of unique location operands referred to (via DW_OP_LLVM_arg) in this expression; th...
Definition: DebugInfoMetadata.cpp:1749
llvm::Instruction::getModule
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:69
llvm::AArch64CC::AL
@ AL
Definition: AArch64BaseInfo.h:269
llvm::none_of
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1748
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::Value::hasOneUse
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
llvm::OptimizedStructLayoutField::Offset
uint64_t Offset
The offset of this field in the final layout.
Definition: OptimizedStructLayout.h:59
eliminateSwiftError
static void eliminateSwiftError(Function &F, coro::Shape &Shape)
Eliminate all problematic uses of swifterror arguments and allocas from the function.
Definition: CoroFrame.cpp:2411
llvm::coro::Shape::SwitchLowering
SwitchLoweringStorage SwitchLowering
Definition: CoroInternal.h:141
llvm::DIType::getSizeInBits
uint64_t getSizeInBits() const
Definition: DebugInfoMetadata.h:738
llvm::AllocaInst::getAlign
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:122
llvm::BasicBlock::getParent
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:104
IntrinsicInst.h
llvm::Type::isPointerTy
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:237
llvm::DIBuilder
Definition: DIBuilder.h:42
dumpAllocas
static void dumpAllocas(const SmallVectorImpl< AllocaInfo > &Allocas)
Definition: CoroFrame.cpp:409
llvm::coro::Shape::FrameTy
StructType * FrameTy
Definition: CoroInternal.h:98
InstIterator.h
llvm::TypeSize::getFixedSize
ScalarTy getFixedSize() const
Definition: TypeSize.h:444
splitAround
static void splitAround(Instruction *I, const Twine &Name)
Definition: CoroFrame.cpp:2126
llvm::Function
Definition: Function.h:60
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::lookup
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition: DenseMap.h:197
llvm::lower_bound
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1922
P
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Definition: README-SSE.txt:411
llvm::IntrinsicInst::getIntrinsicID
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:54
emitSetSwiftErrorValue
static Value * emitSetSwiftErrorValue(IRBuilder<> &Builder, Value *V, coro::Shape &Shape)
Set the given value as the current swifterror value.
Definition: CoroFrame.cpp:2297
llvm::DataLayout::getTypeSizeInBits
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:673
llvm::StructType::setBody
void setBody(ArrayRef< Type * > Elements, bool isPacked=false)
Specify a body for an opaque identified type.
Definition: Type.cpp:445
llvm::BitCastInst
This class represents a no-op cast from one type to another.
Definition: Instructions.h:5253
llvm::AllocaInst::getType
PointerType * getType() const
Overload to return most specific pointer type.
Definition: Instructions.h:101
llvm::ilist_node_with_parent::getNextNode
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:289
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1199
llvm::pred_size
unsigned pred_size(MachineBasicBlock *BB)
Definition: MachineSSAContext.h:32
llvm::LandingPadInst
The landingpad instruction holds all of the information necessary to generate correct exception handl...
Definition: Instructions.h:2949
llvm::InstVisitor< DerivedT >::visitIntrinsicInst
void visitIntrinsicInst(IntrinsicInst &I)
Definition: InstVisitor.h:219
llvm::IRBuilder<>
llvm::SmallDenseMap
Definition: DenseMap.h:880
llvm::findDbgUsers
void findDbgUsers(SmallVectorImpl< DbgVariableIntrinsic * > &DbgInsts, Value *V)
Finds the debug info intrinsics describing a value.
Definition: DebugInfo.cpp:112
Local.h
llvm::Instruction::insertBefore
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
Definition: Instruction.cpp:87
llvm::DbgVariableIntrinsic::replaceVariableLocationOp
void replaceVariableLocationOp(Value *OldValue, Value *NewValue)
Definition: IntrinsicInst.cpp:114
llvm::DominatorTree
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:166
rewritePHIsForCleanupPad
static void rewritePHIsForCleanupPad(BasicBlock *CleanupPadBB, CleanupPadInst *CleanupPad)
Definition: CoroFrame.cpp:1885
llvm::DataLayout::getStructLayout
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:678
llvm::DILocation
Debug location.
Definition: DebugInfoMetadata.h:1600
llvm::CoroAllocaAllocInst
This represents the llvm.coro.alloca.alloc instruction.
Definition: CoroInstr.h:670
llvm::DIType::getAlignInBits
uint32_t getAlignInBits() const
Definition: DebugInfoMetadata.h:739
llvm::InstVisitor< DerivedT >::visitBitCastInst
void visitBitCastInst(BitCastInst &I)
Definition: InstVisitor.h:187
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
eliminateSwiftErrorArgument
static void eliminateSwiftErrorArgument(Function &F, Argument &Arg, coro::Shape &Shape, SmallVectorImpl< AllocaInst * > &AllocasToPromote)
"Eliminate" a swifterror argument by reducing it to the alloca case and then loading and storing in t...
Definition: CoroFrame.cpp:2371
llvm::coro::Shape::ABI
coro::ABI ABI
Definition: CoroInternal.h:96
BlockData::BlockData
BlockData()
Definition: SIModeRegister.cpp:105
llvm::BasicBlock::getSingleSuccessor
const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
Definition: BasicBlock.cpp:315
llvm::BasicBlock::splitBasicBlock
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
Definition: BasicBlock.cpp:402
llvm::TinyPtrVector::front
EltTy front() const
Definition: TinyPtrVector.h:230
llvm::DbgVariableIntrinsic::getVariable
DILocalVariable * getVariable() const
Definition: IntrinsicInst.h:288
llvm::MemIntrinsic
This is the common base class for memset/memcpy/memmove.
Definition: IntrinsicInst.h:1043
collectFrameAllocas
static void collectFrameAllocas(Function &F, coro::Shape &Shape, const SuspendCrossingInfo &Checker, SmallVectorImpl< AllocaInfo > &Allocas)
Definition: CoroFrame.cpp:2576
llvm::Optional
Definition: APInt.h:33
llvm::coro::Shape::getPromiseAlloca
AllocaInst * getPromiseAlloca() const
Definition: CoroInternal.h:234
Offset
uint64_t Offset
Definition: ELFObjHandler.cpp:80
llvm::SmallPtrSet< Instruction *, 4 >
solveTypeName
static StringRef solveTypeName(Type *Ty)
Create name for Type.
Definition: CoroFrame.cpp:817
llvm::PromoteMemToReg
void PromoteMemToReg(ArrayRef< AllocaInst * > Allocas, DominatorTree &DT, AssumptionCache *AC=nullptr)
Promote the specified list of alloca instructions into scalar registers, inserting PHI nodes as appro...
Definition: PromoteMemoryToRegister.cpp:1107
llvm::successors
auto successors(MachineBasicBlock *BB)
Definition: MachineSSAContext.h:29
llvm::max
Expected< ExpressionValue > max(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:337
llvm::BasicBlock::getSinglePredecessor
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:285
llvm::dump
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
Definition: SparseBitVector.h:877
llvm::DIBuilder::createMemberType
DIDerivedType * createMemberType(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNo, uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits, DINode::DIFlags Flags, DIType *Ty, DINodeArray Annotations=nullptr)
Create debugging information entry for a member.
Definition: DIBuilder.cpp:380
llvm::DIExpression
DWARF expression.
Definition: DebugInfoMetadata.h:2601
willLeaveFunctionImmediatelyAfter
static bool willLeaveFunctionImmediatelyAfter(BasicBlock *BB, unsigned depth=3)
After we split the coroutine, will the given basic block be along an obvious exit path for the resump...
Definition: CoroFrame.cpp:2176
llvm::coro::buildCoroutineFrame
void buildCoroutineFrame(Function &F, Shape &Shape)
Definition: CoroFrame.cpp:2696
llvm::Type::isFloatingPointTy
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition: Type.h:184
llvm::Type::getNonOpaquePointerElementType
Type * getNonOpaquePointerElementType() const
Only use this method in code that is not reachable with opaque pointers, or part of deprecated method...
Definition: Type.h:401
llvm::coro::Shape::SwitchLoweringStorage::PromiseAlloca
AllocaInst * PromiseAlloca
Definition: CoroInternal.h:109
llvm::getOffset
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
Definition: RuntimeDyld.cpp:172
llvm::DbgVariableIntrinsic::getVariableLocationOp
Value * getVariableLocationOp(unsigned OpIdx) const
Definition: IntrinsicInst.cpp:92
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::coro::Shape::AsyncLoweringStorage::ContextHeaderSize
uint64_t ContextHeaderSize
Definition: CoroInternal.h:131
F
#define F(x, y, z)
Definition: MD5.cpp:55
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:55
instructions
print must be executed print the must be executed context for all instructions
Definition: MustExecute.cpp:362
llvm::coro::Shape::emitDealloc
void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const
Deallocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:476
llvm::DIBuilder::createAutoVariable
DILocalVariable * createAutoVariable(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNo, DIType *Ty, bool AlwaysPreserve=false, DINode::DIFlags Flags=DINode::FlagZero, uint32_t AlignInBits=0)
Create a new descriptor for an auto variable.
Definition: DIBuilder.cpp:798
Context
LLVMContext & Context
Definition: NVVMIntrRange.cpp:66
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::DominatorTree::dominates
bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
Definition: Dominators.cpp:122
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:187
llvm::DIBuilder::createPointerType
DIDerivedType * createPointerType(DIType *PointeeTy, uint64_t SizeInBits, uint32_t AlignInBits=0, std::optional< unsigned > DWARFAddressSpace=std::nullopt, StringRef Name="", DINodeArray Annotations=nullptr)
Create debugging information entry for a pointer.
Definition: DIBuilder.cpp:319
llvm::Instruction::insertAfter
void insertAfter(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately after the specified instruction.
Definition: Instruction.cpp:93
llvm::coro::Shape::AllocaSpillBlock
BasicBlock * AllocaSpillBlock
Definition: CoroInternal.h:102
cleanupSinglePredPHIs
static void cleanupSinglePredPHIs(Function &F)
Definition: CoroFrame.cpp:1960
llvm::coro::Shape::RetconLoweringStorage::IsFrameInlineInStorage
bool IsFrameInlineInStorage
Definition: CoroInternal.h:123
rewriteMaterializableInstructions
static void rewriteMaterializableInstructions(IRBuilder<> &IRB, const SpillInfo &Spills)
Definition: CoroFrame.cpp:2075
StackLifetime.h
llvm::AddrSpaceCastInst
This class represents a conversion between pointers from one address space to another.
Definition: Instructions.h:5293
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::begin
iterator begin()
Get an iterator to the beginning of the SetVector.
Definition: SetVector.h:82
emitGetSwiftErrorValue
static Value * emitGetSwiftErrorValue(IRBuilder<> &Builder, Type *ValueTy, coro::Shape &Shape)
Get the current swifterror value.
Definition: CoroFrame.cpp:2282
llvm::PHINode::getIncomingValue
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
Definition: Instructions.h:2793
llvm::DbgValueInst
This represents the llvm.dbg.value instruction.
Definition: IntrinsicInst.h:388
llvm::AllocaInst::getAllocatedType
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:115
SmallString.h
llvm::DataLayout::getABITypeAlignment
uint64_t getABITypeAlignment(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:825
llvm::DILocalVariable
Local variable.
Definition: DebugInfoMetadata.h:3101
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::PtrUseVisitor::visitGetElementPtrInst
void visitGetElementPtrInst(GetElementPtrInst &GEPI)
Definition: PtrUseVisitor.h:259
llvm::User
Definition: User.h:44
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
localAllocaNeedsStackSave
static bool localAllocaNeedsStackSave(CoroAllocaAllocInst *AI)
Definition: CoroFrame.cpp:2195
llvm::ARM_PROC::A
@ A
Definition: ARMBaseInfo.h:34
llvm::BitVector::size
size_type size() const
size - Returns the number of bits in this bitvector.
Definition: BitVector.h:152
llvm::Type::getStructName
StringRef getStructName() const
Definition: DerivedTypes.h:344
lowerNonLocalAlloca
static Instruction * lowerNonLocalAlloca(CoroAllocaAllocInst *AI, coro::Shape &Shape, SmallVectorImpl< Instruction * > &DeadInsts)
Turn the given coro.alloca.alloc call into a dynamic allocation.
Definition: CoroFrame.cpp:2257
SI
@ SI
Definition: SIInstrInfo.cpp:7966
llvm::InstVisitor< DerivedT >::visitGetElementPtrInst
void visitGetElementPtrInst(GetElementPtrInst &I)
Definition: InstVisitor.h:174
llvm::CoroSuspendInst
This represents the llvm.coro.suspend instruction.
Definition: CoroInstr.h:493
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
llvm::StructLayout::getElementOffsetInBits
uint64_t getElementOffsetInBits(unsigned Idx) const
Definition: DataLayout.h:657
llvm::DIType::getName
StringRef getName() const
Definition: DebugInfoMetadata.h:745
llvm::Value::uses
iterator_range< use_iterator > uses()
Definition: Value.h:376
llvm::BasicBlock::getFirstInsertionPt
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:246
llvm::coro::Shape::CoroSuspends
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
Definition: CoroInternal.h:78
FramePtr
static const unsigned FramePtr
Definition: XCoreFrameLowering.cpp:34
eliminateSwiftErrorAlloca
static void eliminateSwiftErrorAlloca(Function &F, AllocaInst *Alloca, coro::Shape &Shape)
Eliminate a formerly-swifterror alloca by inserting the get/set intrinsics and attempting to MemToReg...
Definition: CoroFrame.cpp:2345
llvm::dwarf::Index
Index
Definition: Dwarf.h:472
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::DbgVariableIntrinsic::setExpression
void setExpression(DIExpression *NewExpr)
Definition: IntrinsicInst.h:250
llvm::coro::Shape::CoroEnds
SmallVector< AnyCoroEndInst *, 4 > CoroEnds
Definition: CoroInternal.h:75
llvm::Instruction
Definition: Instruction.h:42
llvm::FindDbgDeclareUses
TinyPtrVector< DbgDeclareInst * > FindDbgDeclareUses(Value *V)
Like FindDbgAddrUses, but only returns dbg.declare intrinsics, not dbg.addr.
Definition: DebugInfo.cpp:78
llvm::AllocaInst::getArraySize
const Value * getArraySize() const
Get the number of elements allocated.
Definition: Instructions.h:97
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
llvm::salvageDebugInfo
void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI)
Assuming the instruction MI is going to be deleted, attempt to salvage debug users of MI by writing t...
Definition: Utils.cpp:1361
llvm::predecessors
auto predecessors(MachineBasicBlock *BB)
Definition: MachineSSAContext.h:30
llvm::Value::setName
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:375
BitVector.h
llvm::Use::getUser
User * getUser() const
Returns the User that contains this Use.
Definition: Use.h:72
sinkSpillUsesAfterCoroBegin
static void sinkSpillUsesAfterCoroBegin(Function &F, const FrameDataInfo &FrameData, CoroBeginInst *CoroBegin)
retcon and retcon.once conventions assume that all spill uses can be sunk after the coro....
Definition: CoroFrame.cpp:2444
llvm::InstVisitor< DerivedT >::visitPHINode
void visitPHINode(PHINode &I)
Definition: InstVisitor.h:175
llvm::BitVector
Definition: BitVector.h:75
llvm::Instruction::removeFromParent
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
Definition: Instruction.cpp:77
llvm::CoroIdInst::clearPromise
void clearPromise()
Definition: CoroInstr.h:124
Align
uint64_t Align
Definition: ELFObjHandler.cpp:82
llvm::BasicBlock::getFirstNonPHI
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:209
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::MCID::Call
@ Call
Definition: MCInstrDesc.h:155
llvm::ehAwareSplitEdge
BasicBlock * ehAwareSplitEdge(BasicBlock *BB, BasicBlock *Succ, LandingPadInst *OriginalPad=nullptr, PHINode *LandingPadReplacement=nullptr, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions(), const Twine &BBName="")
Split the edge connect the specficed blocks in the case that Succ is an Exception Handling Block.
Definition: BasicBlockUtils.cpp:668
llvm::DIBuilder::replaceArrays
void replaceArrays(DICompositeType *&T, DINodeArray Elements, DINodeArray TParams=DINodeArray())
Replace arrays on a composite type.
Definition: DIBuilder.cpp:1168
sinkLifetimeStartMarkers
static void sinkLifetimeStartMarkers(Function &F, coro::Shape &Shape, SuspendCrossingInfo &Checker)
For each local variable that all of its user are only used inside one of suspended region,...
Definition: CoroFrame.cpp:2491
llvm::coro::Shape::FrameAlign
Align FrameAlign
Definition: CoroInternal.h:99
llvm::SmallString< 16 >
CFG.h
llvm::AllocaInst::isSwiftError
bool isSwiftError() const
Return true if this alloca is used as a swifterror argument to a call.
Definition: Instructions.h:147
llvm::Twine::str
std::string str() const
Return the twine contents as a std::string.
Definition: Twine.cpp:17
llvm::DILocalVariable::getScope
DILocalScope * getScope() const
Get the local scope for this variable.
Definition: DebugInfoMetadata.h:3159
llvm::coro::Shape
Definition: CoroInternal.h:73
llvm::coro::Shape::AsyncLowering
AsyncLoweringStorage AsyncLowering
Definition: CoroInternal.h:143
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:210
llvm::sort
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1682
llvm::PtrUseVisitor::visitMemIntrinsic
void visitMemIntrinsic(MemIntrinsic &I)
Definition: PtrUseVisitor.h:276
llvm::DataLayout::getPrefTypeAlign
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:838
isCoroutineStructureIntrinsic
static bool isCoroutineStructureIntrinsic(Instruction &I)
Definition: CoroFrame.cpp:2068
createFramePtr
static void createFramePtr(coro::Shape &Shape)
Definition: CoroFrame.cpp:1522
G
const DataFlowGraph & G
Definition: RDFGraph.cpp:200
llvm::SmallMapVector
A MapVector that performs no allocations if smaller than a certain size.
Definition: MapVector.h:233
CoroInternal.h
llvm::OptimizedStructLayoutField::Alignment
Align Alignment
The required alignment of this field.
Definition: OptimizedStructLayout.h:73
llvm::Use::set
void set(Value *Val)
Definition: Value.h:865
llvm::DbgVariableIntrinsic
This is the common base class for debug info intrinsics for variables.
Definition: IntrinsicInst.h:181
llvm::coro::Shape::SwiftErrorOps
SmallVector< CallInst *, 2 > SwiftErrorOps
Definition: CoroInternal.h:79
llvm::CleanupPadInst
Definition: Instructions.h:4486
llvm::InstVisitor< DerivedT >::visitSelectInst
void visitSelectInst(SelectInst &I)
Definition: InstVisitor.h:189
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:298
llvm::ArrayRef::drop_front
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
Definition: ArrayRef.h:203
llvm::Instruction::eraseFromParent
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:81
dumpSpills
static void dumpSpills(StringRef Title, const SpillInfo &Spills)
Definition: CoroFrame.cpp:399
buildFrameType
static StructType * buildFrameType(Function &F, coro::Shape &Shape, FrameDataInfo &FrameData)
Definition: CoroFrame.cpp:1125
Index
uint32_t Index
Definition: ELFObjHandler.cpp:83
llvm::SwitchInst::setDefaultDest
void setDefaultDest(BasicBlock *DefaultCase)
Definition: Instructions.h:3487
uint64_t
llvm::Optional::reset
void reset()
Definition: Optional.h:276
llvm::GlobalValue::getParent
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
llvm::DbgDeclareInst
This represents the llvm.dbg.declare instruction.
Definition: IntrinsicInst.h:349
Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:79
llvm::Instruction::user_back
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
Definition: Instruction.h:88
llvm::PHINode::addIncoming
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
Definition: Instructions.h:2847
move
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
Definition: README.txt:546
llvm::coro::Shape::SwitchLoweringStorage::IndexOffset
unsigned IndexOffset
Definition: CoroInternal.h:113
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
llvm::coro::Shape::SwitchLoweringStorage::IndexAlign
unsigned IndexAlign
Definition: CoroInternal.h:112
PromoteMemToReg.h
llvm::DenseMap
Definition: DenseMap.h:714
DebugInfo.h
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::DebugLoc::get
DILocation * get() const
Get the underlying DILocation.
Definition: DebugLoc.cpp:20
llvm::GetElementPtrInst
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:929
buildFrameDebugInfo
static void buildFrameDebugInfo(Function &F, coro::Shape &Shape, FrameDataInfo &FrameData)
Build artificial debug info for C++ coroutine frames to allow users to inspect the contents of the fr...
Definition: CoroFrame.cpp:952
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:716
DIBuilder.h
llvm::StackLifetime
Compute live ranges of allocas.
Definition: StackLifetime.h:37
llvm::PointerType
Class to represent pointers.
Definition: DerivedTypes.h:632
llvm::findDbgValues
void findDbgValues(SmallVectorImpl< DbgValueInst * > &DbgValues, Value *V)
Finds the llvm.dbg.value intrinsics describing a value.
Definition: DebugInfo.cpp:86
insertSpills
static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape)
Definition: CoroFrame.cpp:1553
llvm::updatePhiNodes
void updatePhiNodes(BasicBlock *DestBB, BasicBlock *OldPred, BasicBlock *NewPred, PHINode *Until=nullptr)
Replaces all uses of OldPred with the NewPred block in all PHINodes in a block.
Definition: BasicBlockUtils.cpp:646
llvm::InstVisitor< DerivedT >::visit
void visit(Iterator Start, Iterator End)
Definition: InstVisitor.h:87
llvm::Log2_64_Ceil
unsigned Log2_64_Ceil(uint64_t Value)
Return the ceil log base 2 of the specified value, 64 if the value is zero.
Definition: MathExtras.h:566
llvm::CallBase::doesNotCapture
bool doesNotCapture(unsigned OpNo) const
Determine whether this data operand is not captured.
Definition: InstrTypes.h:1673
materializable
static bool materializable(Instruction &V)
Definition: CoroFrame.cpp:2061
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::find
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:150
OptimizedStructLayout.h
IRBuilder.h
isSuspendReachableFrom
static bool isSuspendReachableFrom(BasicBlock *From, VisitedBlocksSet &VisitedOrFreeBBs)
Does control flow starting at the given block ever reach a suspend instruction before reaching a bloc...
Definition: CoroFrame.cpp:2139
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
solveDIType
static DIType * solveDIType(DIBuilder &Builder, Type *Ty, const DataLayout &Layout, DIScope *Scope, unsigned LineNum, DenseMap< Type *, DIType * > &DITypeCache)
Definition: CoroFrame.cpp:865
llvm::DataLayout::getPrefTypeAlignment
uint64_t getPrefTypeAlignment(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:834
llvm::InstVisitor< DerivedT >::visitAddrSpaceCastInst
void visitAddrSpaceCastInst(AddrSpaceCastInst &I)
Definition: InstVisitor.h:188
llvm::SelectInst
This class represents the LLVM 'select' instruction.
Definition: Instructions.h:1735
llvm::coro::Shape::OptimizeFrame
bool OptimizeFrame
This would only be true if optimization are enabled.
Definition: CoroInternal.h:105
llvm::WinEH::EncodingType::CE
@ CE
Windows NT (Windows on ARM)
emitSetAndGetSwiftErrorValueAround
static Value * emitSetAndGetSwiftErrorValueAround(Instruction *Call, AllocaInst *Alloca, coro::Shape &Shape)
Set the swifterror value from the given alloca before a call, then put in back in the alloca afterwar...
Definition: CoroFrame.cpp:2315
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::insert
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:141
llvm::coro::Shape::AsyncLoweringStorage::ContextSize
uint64_t ContextSize
Definition: CoroInternal.h:134
isAligned
static bool isAligned(const Value *Base, const APInt &Offset, Align Alignment, const DataLayout &DL)
Definition: Loads.cpp:29
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1715
llvm::dwarf::SourceLanguage
SourceLanguage
Definition: Dwarf.h:201
llvm::DICompositeType
Composite types.
Definition: DebugInfoMetadata.h:1085
llvm::Instruction::clone
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
Definition: Instruction.cpp:899
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1741
llvm::StructType
Class to represent struct types.
Definition: DerivedTypes.h:213
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
llvm::raw_svector_ostream::str
StringRef str() const
Return a StringRef for the vector contents.
Definition: raw_ostream.h:683
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::DIVariable::getLine
unsigned getLine() const
Definition: DebugInfoMetadata.h:2545
llvm::Instruction::getFunction
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:73
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:532
A
* A
Definition: README_ALTIVEC.txt:89
llvm::PtrUseVisitor
A base class for visitors over the uses of a pointer value.
Definition: PtrUseVisitor.h:198
uint32_t
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:994
llvm::AllocaInst::isArrayAllocation
bool isArrayAllocation() const
Return true if there is an allocation size parameter to the allocation instruction that is not 1.
Definition: Instructions.cpp:1500
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
movePHIValuesToInsertedBlock
static void movePHIValuesToInsertedBlock(BasicBlock *SuccBB, BasicBlock *InsertedBB, BasicBlock *PredBB, PHINode *UntilPHI=nullptr)
Definition: CoroFrame.cpp:1867
S
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
Definition: README.txt:210
llvm::PtrUseVisitor::visitCallBase
void visitCallBase(CallBase &CB)
Definition: PtrUseVisitor.h:290
llvm::coro::Shape::CoroBegin
CoroBeginInst * CoroBegin
Definition: CoroInternal.h:74
llvm::OptimizedStructLayoutField::Size
uint64_t Size
The required size of this field in bytes.
Definition: OptimizedStructLayout.h:63
llvm::AnyCoroEndInst
Definition: CoroInstr.h:614
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:308
llvm::Type::getContext
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:128
llvm::detail::PtrUseVisitorBase::Offset
APInt Offset
The constant offset of the use if that is known.
Definition: PtrUseVisitor.h:149
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::insert
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:207
llvm::coro::Shape::RetconLowering
RetconLoweringStorage RetconLowering
Definition: CoroInternal.h:142
llvm::User::replaceUsesOfWith
bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Definition: User.cpp:21
llvm::performOptimizedStructLayout
std::pair< uint64_t, Align > performOptimizedStructLayout(MutableArrayRef< OptimizedStructLayoutField > Fields)
Compute a layout for a struct containing the given fields, making a best-effort attempt to minimize t...
Definition: OptimizedStructLayout.cpp:42
getAlign
static MaybeAlign getAlign(Value *Ptr)
Definition: IRBuilder.cpp:530
llvm::StructType::getNumElements
unsigned getNumElements() const
Random access to the elements.
Definition: DerivedTypes.h:327
llvm::Intrinsic::getDeclaration
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1481
llvm::Value::stripPointerCasts
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:685
llvm::make_scope_exit
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:59
llvm::find_if
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1761
llvm::BasicBlock::front
const Instruction & front() const
Definition: BasicBlock.h:318
llvm::BasicBlock::getContext
LLVMContext & getContext() const
Get the context in which this basic block lives.
Definition: BasicBlock.cpp:35
llvm::DIBuilder::createExpression
DIExpression * createExpression(ArrayRef< uint64_t > Addr=std::nullopt)
Create a new descriptor for the specified variable which has a complex address expression for its add...
Definition: DIBuilder.cpp:836
llvm::codeview::ClassOptions::Packed
@ Packed
llvm::DIBuilder::createStructType
DICompositeType * createStructType(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber, uint64_t SizeInBits, uint32_t AlignInBits, DINode::DIFlags Flags, DIType *DerivedFrom, DINodeArray Elements, unsigned RunTimeLang=0, DIType *VTableHolder=nullptr, StringRef UniqueIdentifier="")
Create debugging information entry for a struct.
Definition: DIBuilder.cpp:508
llvm::SplitEdge
BasicBlock * SplitEdge(BasicBlock *From, BasicBlock *To, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="")
Split the edge connecting the specified blocks, and return the newly created basic block between From...
Definition: BasicBlockUtils.cpp:597
llvm::AArch64::Alias
StringRef Alias
Definition: AArch64TargetParser.h:140
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
llvm::pdb::PDB_SymType::Label
@ Label
splitBeforeCatchSwitch
static Instruction * splitBeforeCatchSwitch(CatchSwitchInst *CatchSwitch)
Definition: CoroFrame.cpp:1510
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:50
llvm::setUnwindEdgeTo
void setUnwindEdgeTo(Instruction *TI, BasicBlock *Succ)
Sets the unwind edge of an instruction to a particular successor.
Definition: BasicBlockUtils.cpp:635
llvm::Type::isFloatTy
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition: Type.h:153
llvm::DIScope
Base class for scope-like contexts.
Definition: DebugInfoMetadata.h:509
llvm::PtrUseVisitor::visitAddrSpaceCastInst
void visitAddrSpaceCastInst(AddrSpaceCastInst &ASC)
Definition: PtrUseVisitor.h:251
get
Should compile to something r4 addze r3 instead we get
Definition: README.txt:24
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::end
iterator end()
Definition: DenseMap.h:84
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:351
llvm::Align::value
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
llvm::CallBase::arg_size
unsigned arg_size() const
Definition: InstrTypes.h:1339
llvm::TypeSize
Definition: TypeSize.h:435
llvm::salvageDebugInfoImpl
Value * salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps, SmallVectorImpl< uint64_t > &Ops, SmallVectorImpl< Value * > &AdditionalValues)
Definition: Local.cpp:2006
circular_raw_ostream.h
llvm::BitWidth
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:147
Arguments
AMDGPU Lower Kernel Arguments
Definition: AMDGPULowerKernelArguments.cpp:242
llvm::DIBuilder::getOrCreateArray
DINodeArray getOrCreateArray(ArrayRef< Metadata * > Elements)
Get a DINodeArray, create one if required.
Definition: DIBuilder.cpp:679
llvm::Type::getPointerTo
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:774
llvm::Type::isDoubleTy
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
Definition: Type.h:156
llvm::coro::Shape::emitAlloc
Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const
Allocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:453
llvm::coro::Shape::getSwitchCoroId
CoroIdInst * getSwitchCoroId() const
Definition: CoroInternal.h:146
llvm::SmallVectorImpl::clear
void clear()
Definition: SmallVector.h:614
rewritePHIs
static void rewritePHIs(BasicBlock &BB)
Definition: CoroFrame.cpp:1977
llvm::dwarf::isCPlusPlus
bool isCPlusPlus(SourceLanguage S)
Definition: Dwarf.h:209
llvm::CoroAllocaAllocInst::getSize
Value * getSize() const
Definition: CoroInstr.h:673
llvm::SmallString::str
StringRef str() const
Explicit conversion to StringRef.
Definition: SmallString.h:260
llvm::CatchSwitchInst::getParentPad
Value * getParentPad() const
Definition: Instructions.h:4370
llvm::IntrinsicInst
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:47
isLifetimeStart
static bool isLifetimeStart(const Instruction *Inst)
Definition: GVN.cpp:1039
lowerLocalAllocas
static void lowerLocalAllocas(ArrayRef< CoroAllocaAllocInst * > LocalAllocas, SmallVectorImpl< Instruction * > &DeadInsts)
Turn each of the given local allocas into a normal (dynamic) alloca instruction.
Definition: CoroFrame.cpp:2213
isSuspendBlock
static bool isSuspendBlock(BasicBlock *BB)
Definition: CoroFrame.cpp:2131
llvm::AllocaInst::setAlignment
void setAlignment(Align Align)
Definition: Instructions.h:126
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::end
iterator end()
Get an iterator to the end of the SetVector.
Definition: SetVector.h:92
llvm::TinyPtrVector::empty
bool empty() const
Definition: TinyPtrVector.h:163
llvm::coro::Shape::getRetconCoroId
AnyCoroIdRetconInst * getRetconCoroId() const
Definition: CoroInternal.h:151
llvm::Type::isStructTy
bool isStructTy() const
True if this is an instance of StructType.
Definition: Type.h:231
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:359
isLocalAlloca
static bool isLocalAlloca(CoroAllocaAllocInst *AI)
Is the given alloca "local", i.e.
Definition: CoroFrame.cpp:2162
Dominators.h
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1341
N
#define N
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:91
llvm::coro::Shape::FrameSize
uint64_t FrameSize
Definition: CoroInternal.h:100
llvm::SmallVectorImpl::pop_back_val
T pop_back_val()
Definition: SmallVector.h:677
llvm::to_string
std::string to_string(const T &Value)
Definition: ScopedPrinter.h:85
Users
iv Induction Variable Users
Definition: IVUsers.cpp:48
llvm::iterator_range
A range adaptor for a pair of iterators.
Definition: iterator_range.h:30
llvm::raw_svector_ostream
A raw_ostream that writes to an SmallVector or SmallString.
Definition: raw_ostream.h:658
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::reserve
void reserve(size_type NumEntries)
Grow the densemap so that it can contain at least NumEntries items before resizing again.
Definition: DenseMap.h:103
llvm::PHINode
Definition: Instructions.h:2697
llvm::BasicBlock::getTerminator
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:119
ScopeExit.h
llvm::CoroBeginInst
This class represents the llvm.coro.begin instruction.
Definition: CoroInstr.h:420
llvm::DISubprogram
Subprogram description.
Definition: DebugInfoMetadata.h:1846
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
llvm::StructType::getElementType
Type * getElementType(unsigned N) const
Definition: DerivedTypes.h:328
llvm::coro::Shape::FramePtr
Value * FramePtr
Definition: CoroInternal.h:101
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1174
llvm::SmallSetVector
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:307
llvm::PtrUseVisitor::visitStoreInst
void visitStoreInst(StoreInst &SI)
Definition: PtrUseVisitor.h:242
llvm::StringRef::str
std::string str() const
str - Get the contents as an std::string.
Definition: StringRef.h:221
llvm::Twine::toStringRef
StringRef toStringRef(SmallVectorImpl< char > &Out) const
This returns the twine as a single StringRef if it can be represented as such.
Definition: Twine.h:473
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
GEP
Hexagon Common GEP
Definition: HexagonCommonGEP.cpp:171
llvm::DILocalScope
A scope for locals.
Definition: DebugInfoMetadata.h:1570
llvm::DIBuilder::createBasicType
DIBasicType * createBasicType(StringRef Name, uint64_t SizeInBits, unsigned Encoding, DINode::DIFlags Flags=DINode::FlagZero)
Create debugging information entry for a basic type.
Definition: DIBuilder.cpp:283
llvm::PtrUseVisitor::visitIntrinsicInst
void visitIntrinsicInst(IntrinsicInst &II)
Definition: PtrUseVisitor.h:277
InlinePriorityMode::Size
@ Size
From
BlockVerifier::State From
Definition: BlockVerifier.cpp:55
llvm::SwitchInst
Multiway switch.
Definition: Instructions.h:3276
llvm::OptimizedStructLayoutField
A field in a structure.
Definition: OptimizedStructLayout.h:45
llvm::Value::takeName
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:381
llvm::AMDGPU::HSAMD::Kernel::Key::Args
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Definition: AMDGPUMetadata.h:394
llvm::AllocaInst
an instruction to allocate memory on the stack
Definition: Instructions.h:59
VisitedBlocksSet
SmallPtrSet< BasicBlock *, 8 > VisitedBlocksSet
Definition: CoroFrame.cpp:2135
cacheDIVar
static void cacheDIVar(FrameDataInfo &FrameData, DenseMap< Value *, DILocalVariable * > &DIVarCache)
Definition: CoroFrame.cpp:800
llvm::coro::Shape::AsyncLoweringStorage::getContextAlignment
Align getContextAlignment() const
Definition: CoroInternal.h:137
raw_ostream.h
llvm::TinyPtrVector
TinyPtrVector - This class is specialized for cases where there are normally 0 or 1 element in a vect...
Definition: TinyPtrVector.h:30
llvm::AMDGPU::VGPRIndexMode::Id
Id
Definition: SIDefines.h:244
llvm::SmallVectorImpl::reserve
void reserve(size_type N)
Definition: SmallVector.h:667
SmallVectorThreshold
@ SmallVectorThreshold
Definition: CoroFrame.cpp:48
BasicBlockUtils.h
llvm::PtrUseVisitor::visitBitCastInst
void visitBitCastInst(BitCastInst &BC)
Definition: PtrUseVisitor.h:247
llvm::pdb::PDB_SymType::Block
@ Block
llvm::CatchSwitchInst
Definition: Instructions.h:4308
llvm::Optional::value_or
constexpr T value_or(U &&alt) const &
Definition: Optional.h:292
llvm::offsetToAlignment
uint64_t offsetToAlignment(uint64_t Value, Align Alignment)
Returns the offset to the next integer (mod 2**64) that is greater than or equal to Value and is a mu...
Definition: Alignment.h:198
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
Debug.h
llvm::Value::users
iterator_range< user_iterator > users()
Definition: Value.h:421
llvm::coro::Shape::SwitchLoweringStorage::IndexField
unsigned IndexField
Definition: CoroInternal.h:111
llvm::AllocaInst::setSwiftError
void setSwiftError(bool V)
Specify whether this alloca is used to represent a swifterror.
Definition: Instructions.h:149
llvm::codeview::DebugSubsectionKind::FrameData
@ FrameData
llvm::isAllocaPromotable
bool isAllocaPromotable(const AllocaInst *AI)
Return true if this alloca is legal for promotion.
Definition: PromoteMemoryToRegister.cpp:63
llvm::sampleprof::Base
@ Base
Definition: Discriminator.h:58
llvm::Instruction::moveBefore
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
Definition: Instruction.cpp:107
llvm::DIScope::getFile
DIFile * getFile() const
Definition: DebugInfoMetadata.h:517
llvm::DIFile
File.
Definition: DebugInfoMetadata.h:563
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:941
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:365