LLVM  9.0.0svn
StatepointLowering.cpp
Go to the documentation of this file.
1 //===- StatepointLowering.cpp - SDAGBuilder's statepoint code -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file includes support code use by SelectionDAGBuilder when lowering a
10 // statepoint sequence in SelectionDAG IR.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "StatepointLowering.h"
15 #include "SelectionDAGBuilder.h"
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/DenseMap.h"
18 #include "llvm/ADT/None.h"
19 #include "llvm/ADT/Optional.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallVector.h"
22 #include "llvm/ADT/Statistic.h"
33 #include "llvm/CodeGen/StackMaps.h"
36 #include "llvm/IR/CallingConv.h"
37 #include "llvm/IR/DerivedTypes.h"
38 #include "llvm/IR/Instruction.h"
39 #include "llvm/IR/Instructions.h"
40 #include "llvm/IR/LLVMContext.h"
41 #include "llvm/IR/Statepoint.h"
42 #include "llvm/IR/Type.h"
43 #include "llvm/Support/Casting.h"
47 #include <cassert>
48 #include <cstddef>
49 #include <cstdint>
50 #include <iterator>
51 #include <tuple>
52 #include <utility>
53 
54 using namespace llvm;
55 
56 #define DEBUG_TYPE "statepoint-lowering"
57 
58 STATISTIC(NumSlotsAllocatedForStatepoints,
59  "Number of stack slots allocated for statepoints");
60 STATISTIC(NumOfStatepoints, "Number of statepoint nodes encountered");
61 STATISTIC(StatepointMaxSlotsRequired,
62  "Maximum number of stack slots required for a singe statepoint");
63 
65  SelectionDAGBuilder &Builder, uint64_t Value) {
66  SDLoc L = Builder.getCurSDLoc();
67  Ops.push_back(Builder.DAG.getTargetConstant(StackMaps::ConstantOp, L,
68  MVT::i64));
69  Ops.push_back(Builder.DAG.getTargetConstant(Value, L, MVT::i64));
70 }
71 
73  // Consistency check
74  assert(PendingGCRelocateCalls.empty() &&
75  "Trying to visit statepoint before finished processing previous one");
76  Locations.clear();
77  NextSlotToAllocate = 0;
78  // Need to resize this on each safepoint - we need the two to stay in sync and
79  // the clear patterns of a SelectionDAGBuilder have no relation to
80  // FunctionLoweringInfo. Also need to ensure used bits get cleared.
81  AllocatedStackSlots.clear();
82  AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size());
83 }
84 
86  Locations.clear();
87  AllocatedStackSlots.clear();
88  assert(PendingGCRelocateCalls.empty() &&
89  "cleared before statepoint sequence completed");
90 }
91 
92 SDValue
94  SelectionDAGBuilder &Builder) {
95  NumSlotsAllocatedForStatepoints++;
97 
98  unsigned SpillSize = ValueType.getStoreSize();
99  assert((SpillSize * 8) == ValueType.getSizeInBits() && "Size not in bytes?");
100 
101  // First look for a previously created stack slot which is not in
102  // use (accounting for the fact arbitrary slots may already be
103  // reserved), or to create a new stack slot and use it.
104 
105  const size_t NumSlots = AllocatedStackSlots.size();
106  assert(NextSlotToAllocate <= NumSlots && "Broken invariant");
107 
108  assert(AllocatedStackSlots.size() ==
109  Builder.FuncInfo.StatepointStackSlots.size() &&
110  "Broken invariant");
111 
112  for (; NextSlotToAllocate < NumSlots; NextSlotToAllocate++) {
113  if (!AllocatedStackSlots.test(NextSlotToAllocate)) {
114  const int FI = Builder.FuncInfo.StatepointStackSlots[NextSlotToAllocate];
115  if (MFI.getObjectSize(FI) == SpillSize) {
116  AllocatedStackSlots.set(NextSlotToAllocate);
117  // TODO: Is ValueType the right thing to use here?
118  return Builder.DAG.getFrameIndex(FI, ValueType);
119  }
120  }
121  }
122 
123  // Couldn't find a free slot, so create a new one:
124 
125  SDValue SpillSlot = Builder.DAG.CreateStackTemporary(ValueType);
126  const unsigned FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
128 
130  AllocatedStackSlots.resize(AllocatedStackSlots.size()+1, true);
131  assert(AllocatedStackSlots.size() ==
132  Builder.FuncInfo.StatepointStackSlots.size() &&
133  "Broken invariant");
134 
135  StatepointMaxSlotsRequired.updateMax(
137 
138  return SpillSlot;
139 }
140 
141 /// Utility function for reservePreviousStackSlotForValue. Tries to find
142 /// stack slot index to which we have spilled value for previous statepoints.
143 /// LookUpDepth specifies maximum DFS depth this function is allowed to look.
145  SelectionDAGBuilder &Builder,
146  int LookUpDepth) {
147  // Can not look any further - give up now
148  if (LookUpDepth <= 0)
149  return None;
150 
151  // Spill location is known for gc relocates
152  if (const auto *Relocate = dyn_cast<GCRelocateInst>(Val)) {
153  const auto &SpillMap =
154  Builder.FuncInfo.StatepointSpillMaps[Relocate->getStatepoint()];
155 
156  auto It = SpillMap.find(Relocate->getDerivedPtr());
157  if (It == SpillMap.end())
158  return None;
159 
160  return It->second;
161  }
162 
163  // Look through bitcast instructions.
164  if (const BitCastInst *Cast = dyn_cast<BitCastInst>(Val))
165  return findPreviousSpillSlot(Cast->getOperand(0), Builder, LookUpDepth - 1);
166 
167  // Look through phi nodes
168  // All incoming values should have same known stack slot, otherwise result
169  // is unknown.
170  if (const PHINode *Phi = dyn_cast<PHINode>(Val)) {
171  Optional<int> MergedResult = None;
172 
173  for (auto &IncomingValue : Phi->incoming_values()) {
174  Optional<int> SpillSlot =
175  findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth - 1);
176  if (!SpillSlot.hasValue())
177  return None;
178 
179  if (MergedResult.hasValue() && *MergedResult != *SpillSlot)
180  return None;
181 
182  MergedResult = SpillSlot;
183  }
184  return MergedResult;
185  }
186 
187  // TODO: We can do better for PHI nodes. In cases like this:
188  // ptr = phi(relocated_pointer, not_relocated_pointer)
189  // statepoint(ptr)
190  // We will return that stack slot for ptr is unknown. And later we might
191  // assign different stack slots for ptr and relocated_pointer. This limits
192  // llvm's ability to remove redundant stores.
193  // Unfortunately it's hard to accomplish in current infrastructure.
194  // We use this function to eliminate spill store completely, while
195  // in example we still need to emit store, but instead of any location
196  // we need to use special "preferred" location.
197 
198  // TODO: handle simple updates. If a value is modified and the original
199  // value is no longer live, it would be nice to put the modified value in the
200  // same slot. This allows folding of the memory accesses for some
201  // instructions types (like an increment).
202  // statepoint (i)
203  // i1 = i+1
204  // statepoint (i1)
205  // However we need to be careful for cases like this:
206  // statepoint(i)
207  // i1 = i+1
208  // statepoint(i, i1)
209  // Here we want to reserve spill slot for 'i', but not for 'i+1'. If we just
210  // put handling of simple modifications in this function like it's done
211  // for bitcasts we might end up reserving i's slot for 'i+1' because order in
212  // which we visit values is unspecified.
213 
214  // Don't know any information about this instruction
215  return None;
216 }
217 
218 /// Try to find existing copies of the incoming values in stack slots used for
219 /// statepoint spilling. If we can find a spill slot for the incoming value,
220 /// mark that slot as allocated, and reuse the same slot for this safepoint.
221 /// This helps to avoid series of loads and stores that only serve to reshuffle
222 /// values on the stack between calls.
223 static void reservePreviousStackSlotForValue(const Value *IncomingValue,
224  SelectionDAGBuilder &Builder) {
225  SDValue Incoming = Builder.getValue(IncomingValue);
226 
227  if (isa<ConstantSDNode>(Incoming) || isa<FrameIndexSDNode>(Incoming)) {
228  // We won't need to spill this, so no need to check for previously
229  // allocated stack slots
230  return;
231  }
232 
233  SDValue OldLocation = Builder.StatepointLowering.getLocation(Incoming);
234  if (OldLocation.getNode())
235  // Duplicates in input
236  return;
237 
238  const int LookUpDepth = 6;
240  findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth);
241  if (!Index.hasValue())
242  return;
243 
244  const auto &StatepointSlots = Builder.FuncInfo.StatepointStackSlots;
245 
246  auto SlotIt = find(StatepointSlots, *Index);
247  assert(SlotIt != StatepointSlots.end() &&
248  "Value spilled to the unknown stack slot");
249 
250  // This is one of our dedicated lowering slots
251  const int Offset = std::distance(StatepointSlots.begin(), SlotIt);
252  if (Builder.StatepointLowering.isStackSlotAllocated(Offset)) {
253  // stack slot already assigned to someone else, can't use it!
254  // TODO: currently we reserve space for gc arguments after doing
255  // normal allocation for deopt arguments. We should reserve for
256  // _all_ deopt and gc arguments, then start allocating. This
257  // will prevent some moves being inserted when vm state changes,
258  // but gc state doesn't between two calls.
259  return;
260  }
261  // Reserve this stack slot
262  Builder.StatepointLowering.reserveStackSlot(Offset);
263 
264  // Cache this slot so we find it when going through the normal
265  // assignment loop.
266  SDValue Loc =
267  Builder.DAG.getTargetFrameIndex(*Index, Builder.getFrameIndexTy());
268  Builder.StatepointLowering.setLocation(Incoming, Loc);
269 }
270 
271 /// Remove any duplicate (as SDValues) from the derived pointer pairs. This
272 /// is not required for correctness. It's purpose is to reduce the size of
273 /// StackMap section. It has no effect on the number of spill slots required
274 /// or the actual lowering.
275 static void
279  SelectionDAGBuilder &Builder,
282 
283  SmallVector<const Value *, 64> NewBases, NewPtrs;
285  for (size_t i = 0, e = Ptrs.size(); i < e; i++) {
286  SDValue SD = Builder.getValue(Ptrs[i]);
287  auto SeenIt = Seen.find(SD);
288 
289  if (SeenIt == Seen.end()) {
290  // Only add non-duplicates
291  NewBases.push_back(Bases[i]);
292  NewPtrs.push_back(Ptrs[i]);
293  NewRelocs.push_back(Relocs[i]);
294  Seen[SD] = Ptrs[i];
295  } else {
296  // Duplicate pointer found, note in SSM and move on:
297  SSM.DuplicateMap[Ptrs[i]] = SeenIt->second;
298  }
299  }
300  assert(Bases.size() >= NewBases.size());
301  assert(Ptrs.size() >= NewPtrs.size());
302  assert(Relocs.size() >= NewRelocs.size());
303  Bases = NewBases;
304  Ptrs = NewPtrs;
305  Relocs = NewRelocs;
306  assert(Ptrs.size() == Bases.size());
307  assert(Ptrs.size() == Relocs.size());
308 }
309 
310 /// Extract call from statepoint, lower it and return pointer to the
311 /// call node. Also update NodeMap so that getValue(statepoint) will
312 /// reference lowered call result
313 static std::pair<SDValue, SDNode *> lowerCallFromStatepointLoweringInfo(
315  SelectionDAGBuilder &Builder, SmallVectorImpl<SDValue> &PendingExports) {
316  SDValue ReturnValue, CallEndVal;
317  std::tie(ReturnValue, CallEndVal) =
318  Builder.lowerInvokable(SI.CLI, SI.EHPadBB);
319  SDNode *CallEnd = CallEndVal.getNode();
320 
321  // Get a call instruction from the call sequence chain. Tail calls are not
322  // allowed. The following code is essentially reverse engineering X86's
323  // LowerCallTo.
324  //
325  // We are expecting DAG to have the following form:
326  //
327  // ch = eh_label (only in case of invoke statepoint)
328  // ch, glue = callseq_start ch
329  // ch, glue = X86::Call ch, glue
330  // ch, glue = callseq_end ch, glue
331  // get_return_value ch, glue
332  //
333  // get_return_value can either be a sequence of CopyFromReg instructions
334  // to grab the return value from the return register(s), or it can be a LOAD
335  // to load a value returned by reference via a stack slot.
336 
337  bool HasDef = !SI.CLI.RetTy->isVoidTy();
338  if (HasDef) {
339  if (CallEnd->getOpcode() == ISD::LOAD)
340  CallEnd = CallEnd->getOperand(0).getNode();
341  else
342  while (CallEnd->getOpcode() == ISD::CopyFromReg)
343  CallEnd = CallEnd->getOperand(0).getNode();
344  }
345 
346  assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!");
347  return std::make_pair(ReturnValue, CallEnd->getOperand(0).getNode());
348 }
349 
351  FrameIndexSDNode &FI) {
352  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI.getIndex());
353  auto MMOFlags = MachineMemOperand::MOStore |
355  auto &MFI = MF.getFrameInfo();
356  return MF.getMachineMemOperand(PtrInfo, MMOFlags,
357  MFI.getObjectSize(FI.getIndex()),
358  MFI.getObjectAlignment(FI.getIndex()));
359 }
360 
361 /// Spill a value incoming to the statepoint. It might be either part of
362 /// vmstate
363 /// or gcstate. In both cases unconditionally spill it on the stack unless it
364 /// is a null constant. Return pair with first element being frame index
365 /// containing saved value and second element with outgoing chain from the
366 /// emitted store
367 static std::tuple<SDValue, SDValue, MachineMemOperand*>
369  SelectionDAGBuilder &Builder) {
370  SDValue Loc = Builder.StatepointLowering.getLocation(Incoming);
371  MachineMemOperand* MMO = nullptr;
372 
373  // Emit new store if we didn't do it for this ptr before
374  if (!Loc.getNode()) {
375  Loc = Builder.StatepointLowering.allocateStackSlot(Incoming.getValueType(),
376  Builder);
377  int Index = cast<FrameIndexSDNode>(Loc)->getIndex();
378  // We use TargetFrameIndex so that isel will not select it into LEA
379  Loc = Builder.DAG.getTargetFrameIndex(Index, Builder.getFrameIndexTy());
380 
381 #ifndef NDEBUG
382  // Right now we always allocate spill slots that are of the same
383  // size as the value we're about to spill (the size of spillee can
384  // vary since we spill vectors of pointers too). At some point we
385  // can consider allowing spills of smaller values to larger slots
386  // (i.e. change the '==' in the assert below to a '>=').
388  assert((MFI.getObjectSize(Index) * 8) == Incoming.getValueSizeInBits() &&
389  "Bad spill: stack slot does not match!");
390 #endif
391 
392  auto &MF = Builder.DAG.getMachineFunction();
393  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, Index);
394  Chain = Builder.DAG.getStore(Chain, Builder.getCurSDLoc(), Incoming, Loc,
395  PtrInfo);
396 
397  MMO = getMachineMemOperand(MF, *cast<FrameIndexSDNode>(Loc));
398 
399  Builder.StatepointLowering.setLocation(Incoming, Loc);
400  }
401 
402  assert(Loc.getNode());
403  return std::make_tuple(Loc, Chain, MMO);
404 }
405 
406 /// Lower a single value incoming to a statepoint node. This value can be
407 /// either a deopt value or a gc value, the handling is the same. We special
408 /// case constants and allocas, then fall back to spilling if required.
409 static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly,
412  SelectionDAGBuilder &Builder) {
413  // Note: We know all of these spills are independent, but don't bother to
414  // exploit that chain wise. DAGCombine will happily do so as needed, so
415  // doing it here would be a small compile time win at most.
416  SDValue Chain = Builder.getRoot();
417 
418  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
419  // If the original value was a constant, make sure it gets recorded as
420  // such in the stackmap. This is required so that the consumer can
421  // parse any internal format to the deopt state. It also handles null
422  // pointers and other constant pointers in GC states. Note the constant
423  // vectors do not appear to actually hit this path and that anything larger
424  // than an i64 value (not type!) will fail asserts here.
425  pushStackMapConstant(Ops, Builder, C->getSExtValue());
426  } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
427  // This handles allocas as arguments to the statepoint (this is only
428  // really meaningful for a deopt value. For GC, we'd be trying to
429  // relocate the address of the alloca itself?)
430  assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
431  "Incoming value is a frame index!");
432  Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
433  Builder.getFrameIndexTy()));
434 
435  auto &MF = Builder.DAG.getMachineFunction();
436  auto *MMO = getMachineMemOperand(MF, *FI);
437  MemRefs.push_back(MMO);
438 
439  } else if (LiveInOnly) {
440  // If this value is live in (not live-on-return, or live-through), we can
441  // treat it the same way patchpoint treats it's "live in" values. We'll
442  // end up folding some of these into stack references, but they'll be
443  // handled by the register allocator. Note that we do not have the notion
444  // of a late use so these values might be placed in registers which are
445  // clobbered by the call. This is fine for live-in.
446  Ops.push_back(Incoming);
447  } else {
448  // Otherwise, locate a spill slot and explicitly spill it so it
449  // can be found by the runtime later. We currently do not support
450  // tracking values through callee saved registers to their eventual
451  // spill location. This would be a useful optimization, but would
452  // need to be optional since it requires a lot of complexity on the
453  // runtime side which not all would support.
454  auto Res = spillIncomingStatepointValue(Incoming, Chain, Builder);
455  Ops.push_back(std::get<0>(Res));
456  if (auto *MMO = std::get<2>(Res))
457  MemRefs.push_back(MMO);
458  Chain = std::get<1>(Res);;
459  }
460 
461  Builder.DAG.setRoot(Chain);
462 }
463 
464 /// Lower deopt state and gc pointer arguments of the statepoint. The actual
465 /// lowering is described in lowerIncomingStatepointValue. This function is
466 /// responsible for lowering everything in the right position and playing some
467 /// tricks to avoid redundant stack manipulation where possible. On
468 /// completion, 'Ops' will contain ready to use operands for machine code
469 /// statepoint. The chain nodes will have already been created and the DAG root
470 /// will be set to the last value spilled (if any were).
471 static void
474  SelectionDAGBuilder &Builder) {
475  // Lower the deopt and gc arguments for this statepoint. Layout will be:
476  // deopt argument length, deopt arguments.., gc arguments...
477 #ifndef NDEBUG
478  if (auto *GFI = Builder.GFI) {
479  // Check that each of the gc pointer and bases we've gotten out of the
480  // safepoint is something the strategy thinks might be a pointer (or vector
481  // of pointers) into the GC heap. This is basically just here to help catch
482  // errors during statepoint insertion. TODO: This should actually be in the
483  // Verifier, but we can't get to the GCStrategy from there (yet).
484  GCStrategy &S = GFI->getStrategy();
485  for (const Value *V : SI.Bases) {
486  auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
487  if (Opt.hasValue()) {
488  assert(Opt.getValue() &&
489  "non gc managed base pointer found in statepoint");
490  }
491  }
492  for (const Value *V : SI.Ptrs) {
493  auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
494  if (Opt.hasValue()) {
495  assert(Opt.getValue() &&
496  "non gc managed derived pointer found in statepoint");
497  }
498  }
499  assert(SI.Bases.size() == SI.Ptrs.size() && "Pointer without base!");
500  } else {
501  assert(SI.Bases.empty() && "No gc specified, so cannot relocate pointers!");
502  assert(SI.Ptrs.empty() && "No gc specified, so cannot relocate pointers!");
503  }
504 #endif
505 
506  // Figure out what lowering strategy we're going to use for each part
507  // Note: Is is conservatively correct to lower both "live-in" and "live-out"
508  // as "live-through". A "live-through" variable is one which is "live-in",
509  // "live-out", and live throughout the lifetime of the call (i.e. we can find
510  // it from any PC within the transitive callee of the statepoint). In
511  // particular, if the callee spills callee preserved registers we may not
512  // be able to find a value placed in that register during the call. This is
513  // fine for live-out, but not for live-through. If we were willing to make
514  // assumptions about the code generator producing the callee, we could
515  // potentially allow live-through values in callee saved registers.
516  const bool LiveInDeopt =
518 
519  auto isGCValue =[&](const Value *V) {
520  return is_contained(SI.Ptrs, V) || is_contained(SI.Bases, V);
521  };
522 
523  // Before we actually start lowering (and allocating spill slots for values),
524  // reserve any stack slots which we judge to be profitable to reuse for a
525  // particular value. This is purely an optimization over the code below and
526  // doesn't change semantics at all. It is important for performance that we
527  // reserve slots for both deopt and gc values before lowering either.
528  for (const Value *V : SI.DeoptState) {
529  if (!LiveInDeopt || isGCValue(V))
531  }
532  for (unsigned i = 0; i < SI.Bases.size(); ++i) {
533  reservePreviousStackSlotForValue(SI.Bases[i], Builder);
534  reservePreviousStackSlotForValue(SI.Ptrs[i], Builder);
535  }
536 
537  // First, prefix the list with the number of unique values to be
538  // lowered. Note that this is the number of *Values* not the
539  // number of SDValues required to lower them.
540  const int NumVMSArgs = SI.DeoptState.size();
541  pushStackMapConstant(Ops, Builder, NumVMSArgs);
542 
543  // The vm state arguments are lowered in an opaque manner. We do not know
544  // what type of values are contained within.
545  for (const Value *V : SI.DeoptState) {
546  SDValue Incoming;
547  // If this is a function argument at a static frame index, generate it as
548  // the frame index.
549  if (const Argument *Arg = dyn_cast<Argument>(V)) {
550  int FI = Builder.FuncInfo.getArgumentFrameIndex(Arg);
551  if (FI != INT_MAX)
552  Incoming = Builder.DAG.getFrameIndex(FI, Builder.getFrameIndexTy());
553  }
554  if (!Incoming.getNode())
555  Incoming = Builder.getValue(V);
556  const bool LiveInValue = LiveInDeopt && !isGCValue(V);
557  lowerIncomingStatepointValue(Incoming, LiveInValue, Ops, MemRefs, Builder);
558  }
559 
560  // Finally, go ahead and lower all the gc arguments. There's no prefixed
561  // length for this one. After lowering, we'll have the base and pointer
562  // arrays interwoven with each (lowered) base pointer immediately followed by
563  // it's (lowered) derived pointer. i.e
564  // (base[0], ptr[0], base[1], ptr[1], ...)
565  for (unsigned i = 0; i < SI.Bases.size(); ++i) {
566  const Value *Base = SI.Bases[i];
567  lowerIncomingStatepointValue(Builder.getValue(Base), /*LiveInOnly*/ false,
568  Ops, MemRefs, Builder);
569 
570  const Value *Ptr = SI.Ptrs[i];
571  lowerIncomingStatepointValue(Builder.getValue(Ptr), /*LiveInOnly*/ false,
572  Ops, MemRefs, Builder);
573  }
574 
575  // If there are any explicit spill slots passed to the statepoint, record
576  // them, but otherwise do not do anything special. These are user provided
577  // allocas and give control over placement to the consumer. In this case,
578  // it is the contents of the slot which may get updated, not the pointer to
579  // the alloca
580  for (Value *V : SI.GCArgs) {
581  SDValue Incoming = Builder.getValue(V);
582  if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
583  // This handles allocas as arguments to the statepoint
584  assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
585  "Incoming value is a frame index!");
586  Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
587  Builder.getFrameIndexTy()));
588 
589  auto &MF = Builder.DAG.getMachineFunction();
590  auto *MMO = getMachineMemOperand(MF, *FI);
591  MemRefs.push_back(MMO);
592  }
593  }
594 
595  // Record computed locations for all lowered values.
596  // This can not be embedded in lowering loops as we need to record *all*
597  // values, while previous loops account only values with unique SDValues.
598  const Instruction *StatepointInstr = SI.StatepointInstr;
599  auto &SpillMap = Builder.FuncInfo.StatepointSpillMaps[StatepointInstr];
600 
601  for (const GCRelocateInst *Relocate : SI.GCRelocates) {
602  const Value *V = Relocate->getDerivedPtr();
603  SDValue SDV = Builder.getValue(V);
604  SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
605 
606  if (Loc.getNode()) {
607  SpillMap.SlotMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
608  } else {
609  // Record value as visited, but not spilled. This is case for allocas
610  // and constants. For this values we can avoid emitting spill load while
611  // visiting corresponding gc_relocate.
612  // Actually we do not need to record them in this map at all.
613  // We do this only to check that we are not relocating any unvisited
614  // value.
615  SpillMap.SlotMap[V] = None;
616 
617  // Default llvm mechanisms for exporting values which are used in
618  // different basic blocks does not work for gc relocates.
619  // Note that it would be incorrect to teach llvm that all relocates are
620  // uses of the corresponding values so that it would automatically
621  // export them. Relocates of the spilled values does not use original
622  // value.
623  if (Relocate->getParent() != StatepointInstr->getParent())
624  Builder.ExportFromCurrentBlock(V);
625  }
626  }
627 }
628 
631  // The basic scheme here is that information about both the original call and
632  // the safepoint is encoded in the CallInst. We create a temporary call and
633  // lower it, then reverse engineer the calling sequence.
634 
635  NumOfStatepoints++;
636  // Clear state
637  StatepointLowering.startNewStatepoint(*this);
638 
639 #ifndef NDEBUG
640  // We schedule gc relocates before removeDuplicateGCPtrs since we _will_
641  // encounter the duplicate gc relocates we elide in removeDuplicateGCPtrs.
642  for (auto *Reloc : SI.GCRelocates)
643  if (Reloc->getParent() == SI.StatepointInstr->getParent())
644  StatepointLowering.scheduleRelocCall(*Reloc);
645 #endif
646 
647  // Remove any redundant llvm::Values which map to the same SDValue as another
648  // input. Also has the effect of removing duplicates in the original
649  // llvm::Value input list as well. This is a useful optimization for
650  // reducing the size of the StackMap section. It has no other impact.
651  removeDuplicateGCPtrs(SI.Bases, SI.Ptrs, SI.GCRelocates, *this,
652  FuncInfo.StatepointSpillMaps[SI.StatepointInstr]);
653  assert(SI.Bases.size() == SI.Ptrs.size() &&
654  SI.Ptrs.size() == SI.GCRelocates.size());
655 
656  // Lower statepoint vmstate and gcstate arguments
657  SmallVector<SDValue, 10> LoweredMetaArgs;
659  lowerStatepointMetaArgs(LoweredMetaArgs, MemRefs, SI, *this);
660 
661  // Now that we've emitted the spills, we need to update the root so that the
662  // call sequence is ordered correctly.
663  SI.CLI.setChain(getRoot());
664 
665  // Get call node, we will replace it later with statepoint
666  SDValue ReturnVal;
667  SDNode *CallNode;
668  std::tie(ReturnVal, CallNode) =
669  lowerCallFromStatepointLoweringInfo(SI, *this, PendingExports);
670 
671  // Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END
672  // nodes with all the appropriate arguments and return values.
673 
674  // Call Node: Chain, Target, {Args}, RegMask, [Glue]
675  SDValue Chain = CallNode->getOperand(0);
676 
677  SDValue Glue;
678  bool CallHasIncomingGlue = CallNode->getGluedNode();
679  if (CallHasIncomingGlue) {
680  // Glue is always last operand
681  Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
682  }
683 
684  // Build the GC_TRANSITION_START node if necessary.
685  //
686  // The operands to the GC_TRANSITION_{START,END} nodes are laid out in the
687  // order in which they appear in the call to the statepoint intrinsic. If
688  // any of the operands is a pointer-typed, that operand is immediately
689  // followed by a SRCVALUE for the pointer that may be used during lowering
690  // (e.g. to form MachinePointerInfo values for loads/stores).
691  const bool IsGCTransition =
694  if (IsGCTransition) {
696 
697  // Add chain
698  TSOps.push_back(Chain);
699 
700  // Add GC transition arguments
701  for (const Value *V : SI.GCTransitionArgs) {
702  TSOps.push_back(getValue(V));
703  if (V->getType()->isPointerTy())
704  TSOps.push_back(DAG.getSrcValue(V));
705  }
706 
707  // Add glue if necessary
708  if (CallHasIncomingGlue)
709  TSOps.push_back(Glue);
710 
711  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
712 
713  SDValue GCTransitionStart =
714  DAG.getNode(ISD::GC_TRANSITION_START, getCurSDLoc(), NodeTys, TSOps);
715 
716  Chain = GCTransitionStart.getValue(0);
717  Glue = GCTransitionStart.getValue(1);
718  }
719 
720  // TODO: Currently, all of these operands are being marked as read/write in
721  // PrologEpilougeInserter.cpp, we should special case the VMState arguments
722  // and flags to be read-only.
724 
725  // Add the <id> and <numBytes> constants.
726  Ops.push_back(DAG.getTargetConstant(SI.ID, getCurSDLoc(), MVT::i64));
727  Ops.push_back(
728  DAG.getTargetConstant(SI.NumPatchBytes, getCurSDLoc(), MVT::i32));
729 
730  // Calculate and push starting position of vmstate arguments
731  // Get number of arguments incoming directly into call node
732  unsigned NumCallRegArgs =
733  CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);
734  Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));
735 
736  // Add call target
737  SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
738  Ops.push_back(CallTarget);
739 
740  // Add call arguments
741  // Get position of register mask in the call
742  SDNode::op_iterator RegMaskIt;
743  if (CallHasIncomingGlue)
744  RegMaskIt = CallNode->op_end() - 2;
745  else
746  RegMaskIt = CallNode->op_end() - 1;
747  Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);
748 
749  // Add a constant argument for the calling convention
750  pushStackMapConstant(Ops, *this, SI.CLI.CallConv);
751 
752  // Add a constant argument for the flags
753  uint64_t Flags = SI.StatepointFlags;
754  assert(((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0) &&
755  "Unknown flag used");
756  pushStackMapConstant(Ops, *this, Flags);
757 
758  // Insert all vmstate and gcstate arguments
759  Ops.insert(Ops.end(), LoweredMetaArgs.begin(), LoweredMetaArgs.end());
760 
761  // Add register mask from call node
762  Ops.push_back(*RegMaskIt);
763 
764  // Add chain
765  Ops.push_back(Chain);
766 
767  // Same for the glue, but we add it only if original call had it
768  if (Glue.getNode())
769  Ops.push_back(Glue);
770 
771  // Compute return values. Provide a glue output since we consume one as
772  // input. This allows someone else to chain off us as needed.
773  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
774 
775  MachineSDNode *StatepointMCNode =
776  DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
777  DAG.setNodeMemRefs(StatepointMCNode, MemRefs);
778 
779  SDNode *SinkNode = StatepointMCNode;
780 
781  // Build the GC_TRANSITION_END node if necessary.
782  //
783  // See the comment above regarding GC_TRANSITION_START for the layout of
784  // the operands to the GC_TRANSITION_END node.
785  if (IsGCTransition) {
787 
788  // Add chain
789  TEOps.push_back(SDValue(StatepointMCNode, 0));
790 
791  // Add GC transition arguments
792  for (const Value *V : SI.GCTransitionArgs) {
793  TEOps.push_back(getValue(V));
794  if (V->getType()->isPointerTy())
795  TEOps.push_back(DAG.getSrcValue(V));
796  }
797 
798  // Add glue
799  TEOps.push_back(SDValue(StatepointMCNode, 1));
800 
801  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
802 
803  SDValue GCTransitionStart =
804  DAG.getNode(ISD::GC_TRANSITION_END, getCurSDLoc(), NodeTys, TEOps);
805 
806  SinkNode = GCTransitionStart.getNode();
807  }
808 
809  // Replace original call
810  DAG.ReplaceAllUsesWith(CallNode, SinkNode); // This may update Root
811  // Remove original call node
812  DAG.DeleteNode(CallNode);
813 
814  // DON'T set the root - under the assumption that it's already set past the
815  // inserted node we created.
816 
817  // TODO: A better future implementation would be to emit a single variable
818  // argument, variable return value STATEPOINT node here and then hookup the
819  // return value of each gc.relocate to the respective output of the
820  // previously emitted STATEPOINT value. Unfortunately, this doesn't appear
821  // to actually be possible today.
822 
823  return ReturnVal;
824 }
825 
826 void
828  const BasicBlock *EHPadBB /*= nullptr*/) {
830  "anyregcc is not supported on statepoints!");
831 
832 #ifndef NDEBUG
833  // If this is a malformed statepoint, report it early to simplify debugging.
834  // This should catch any IR level mistake that's made when constructing or
835  // transforming statepoints.
836  ISP.verify();
837 
838  // Check that the associated GCStrategy expects to encounter statepoints.
839  assert(GFI->getStrategy().useStatepoints() &&
840  "GCStrategy does not expect to encounter statepoints");
841 #endif
842 
843  SDValue ActualCallee;
844 
845  if (ISP.getNumPatchBytes() > 0) {
846  // If we've been asked to emit a nop sequence instead of a call instruction
847  // for this statepoint then don't lower the call target, but use a constant
848  // `null` instead. Not lowering the call target lets statepoint clients get
849  // away without providing a physical address for the symbolic call target at
850  // link time.
851 
852  const auto &TLI = DAG.getTargetLoweringInfo();
853  const auto &DL = DAG.getDataLayout();
854 
855  unsigned AS = ISP.getCalledValue()->getType()->getPointerAddressSpace();
856  ActualCallee = DAG.getConstant(0, getCurSDLoc(), TLI.getPointerTy(DL, AS));
857  } else {
858  ActualCallee = getValue(ISP.getCalledValue());
859  }
860 
862  populateCallLoweringInfo(SI.CLI, ISP.getCall(),
864  ISP.getNumCallArgs(), ActualCallee,
865  ISP.getActualReturnType(), false /* IsPatchPoint */);
866 
867  for (const GCRelocateInst *Relocate : ISP.getRelocates()) {
868  SI.GCRelocates.push_back(Relocate);
869  SI.Bases.push_back(Relocate->getBasePtr());
870  SI.Ptrs.push_back(Relocate->getDerivedPtr());
871  }
872 
873  SI.GCArgs = ArrayRef<const Use>(ISP.gc_args_begin(), ISP.gc_args_end());
874  SI.StatepointInstr = ISP.getInstruction();
875  SI.GCTransitionArgs =
877  SI.ID = ISP.getID();
878  SI.DeoptState = ArrayRef<const Use>(ISP.deopt_begin(), ISP.deopt_end());
879  SI.StatepointFlags = ISP.getFlags();
880  SI.NumPatchBytes = ISP.getNumPatchBytes();
881  SI.EHPadBB = EHPadBB;
882 
883  SDValue ReturnValue = LowerAsSTATEPOINT(SI);
884 
885  // Export the result value if needed
886  const GCResultInst *GCResult = ISP.getGCResult();
887  Type *RetTy = ISP.getActualReturnType();
888  if (!RetTy->isVoidTy() && GCResult) {
889  if (GCResult->getParent() != ISP.getCall()->getParent()) {
890  // Result value will be used in a different basic block so we need to
891  // export it now. Default exporting mechanism will not work here because
892  // statepoint call has a different type than the actual call. It means
893  // that by default llvm will create export register of the wrong type
894  // (always i32 in our case). So instead we need to create export register
895  // with correct type manually.
896  // TODO: To eliminate this problem we can remove gc.result intrinsics
897  // completely and make statepoint call to return a tuple.
898  unsigned Reg = FuncInfo.CreateRegs(RetTy);
899  RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
900  DAG.getDataLayout(), Reg, RetTy,
901  ISP.getCall()->getCallingConv());
902  SDValue Chain = DAG.getEntryNode();
903 
904  RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
905  PendingExports.push_back(Chain);
906  FuncInfo.ValueMap[ISP.getInstruction()] = Reg;
907  } else {
908  // Result value will be used in a same basic block. Don't export it or
909  // perform any explicit register copies.
910  // We'll replace the actuall call node shortly. gc_result will grab
911  // this value.
912  setValue(ISP.getInstruction(), ReturnValue);
913  }
914  } else {
915  // The token value is never used from here on, just generate a poison value
916  setValue(ISP.getInstruction(), DAG.getIntPtrConstant(-1, getCurSDLoc()));
917  }
918 }
919 
921  const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB,
922  bool VarArgDisallowed, bool ForceVoidReturnTy) {
924  unsigned ArgBeginIndex = Call->arg_begin() - Call->op_begin();
925  populateCallLoweringInfo(
926  SI.CLI, Call, ArgBeginIndex, Call->getNumArgOperands(), Callee,
927  ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : Call->getType(),
928  false);
929  if (!VarArgDisallowed)
930  SI.CLI.IsVarArg = Call->getFunctionType()->isVarArg();
931 
932  auto DeoptBundle = *Call->getOperandBundle(LLVMContext::OB_deopt);
933 
935 
937  SI.ID = SD.StatepointID.getValueOr(DefaultID);
938  SI.NumPatchBytes = SD.NumPatchBytes.getValueOr(0);
939 
940  SI.DeoptState =
941  ArrayRef<const Use>(DeoptBundle.Inputs.begin(), DeoptBundle.Inputs.end());
942  SI.StatepointFlags = static_cast<uint64_t>(StatepointFlags::None);
943  SI.EHPadBB = EHPadBB;
944 
945  // NB! The GC arguments are deliberately left empty.
946 
947  if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) {
948  ReturnVal = lowerRangeToAssertZExt(DAG, *Call, ReturnVal);
949  setValue(Call, ReturnVal);
950  }
951 }
952 
954  const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB) {
955  LowerCallSiteWithDeoptBundleImpl(Call, Callee, EHPadBB,
956  /* VarArgDisallowed = */ false,
957  /* ForceVoidReturnTy = */ false);
958 }
959 
960 void SelectionDAGBuilder::visitGCResult(const GCResultInst &CI) {
961  // The result value of the gc_result is simply the result of the actual
962  // call. We've already emitted this, so just grab the value.
963  const Instruction *I = CI.getStatepoint();
964 
965  if (I->getParent() != CI.getParent()) {
966  // Statepoint is in different basic block so we should have stored call
967  // result in a virtual register.
968  // We can not use default getValue() functionality to copy value from this
969  // register because statepoint and actual call return types can be
970  // different, and getValue() will use CopyFromReg of the wrong type,
971  // which is always i32 in our case.
972  PointerType *CalleeType = cast<PointerType>(
974  Type *RetTy =
975  cast<FunctionType>(CalleeType->getElementType())->getReturnType();
976  SDValue CopyFromReg = getCopyFromRegs(I, RetTy);
977 
978  assert(CopyFromReg.getNode());
979  setValue(&CI, CopyFromReg);
980  } else {
981  setValue(&CI, getValue(I));
982  }
983 }
984 
985 void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
986 #ifndef NDEBUG
987  // Consistency check
988  // We skip this check for relocates not in the same basic block as their
989  // statepoint. It would be too expensive to preserve validation info through
990  // different basic blocks.
991  if (Relocate.getStatepoint()->getParent() == Relocate.getParent())
992  StatepointLowering.relocCallVisited(Relocate);
993 
994  auto *Ty = Relocate.getType()->getScalarType();
995  if (auto IsManaged = GFI->getStrategy().isGCManagedPointer(Ty))
996  assert(*IsManaged && "Non gc managed pointer relocated!");
997 #endif
998 
999  const Value *DerivedPtr = Relocate.getDerivedPtr();
1000  SDValue SD = getValue(DerivedPtr);
1001 
1002  auto &SpillMap = FuncInfo.StatepointSpillMaps[Relocate.getStatepoint()];
1003  auto SlotIt = SpillMap.find(DerivedPtr);
1004  assert(SlotIt != SpillMap.end() && "Relocating not lowered gc value");
1005  Optional<int> DerivedPtrLocation = SlotIt->second;
1006 
1007  // We didn't need to spill these special cases (constants and allocas).
1008  // See the handling in spillIncomingValueForStatepoint for detail.
1009  if (!DerivedPtrLocation) {
1010  setValue(&Relocate, SD);
1011  return;
1012  }
1013 
1014  SDValue SpillSlot =
1015  DAG.getTargetFrameIndex(*DerivedPtrLocation, getFrameIndexTy());
1016 
1017  // Note: We know all of these reloads are independent, but don't bother to
1018  // exploit that chain wise. DAGCombine will happily do so as needed, so
1019  // doing it here would be a small compile time win at most.
1020  SDValue Chain = getRoot();
1021 
1022  SDValue SpillLoad =
1023  DAG.getLoad(DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
1024  Relocate.getType()),
1025  getCurSDLoc(), Chain, SpillSlot,
1026  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
1027  *DerivedPtrLocation));
1028 
1029  DAG.setRoot(SpillLoad.getValue(1));
1030 
1031  assert(SpillLoad.getNode());
1032  setValue(&Relocate, SpillLoad);
1033 }
1034 
1036  const auto &TLI = DAG.getTargetLoweringInfo();
1037  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE),
1038  TLI.getPointerTy(DAG.getDataLayout()));
1039 
1040  // We don't lower calls to __llvm_deoptimize as varargs, but as a regular
1041  // call. We also do not lower the return value to any virtual register, and
1042  // change the immediately following return to a trap instruction.
1043  LowerCallSiteWithDeoptBundleImpl(CI, Callee, /* EHPadBB = */ nullptr,
1044  /* VarArgDisallowed = */ true,
1045  /* ForceVoidReturnTy = */ true);
1046 }
1047 
1049  // We do not lower the return value from llvm.deoptimize to any virtual
1050  // register, and change the immediately following return to a trap
1051  // instruction.
1052  if (DAG.getTarget().Options.TrapUnreachable)
1053  DAG.setRoot(
1054  DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
1055 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
uint64_t CallInst * C
Represents calls to the gc.result intrinsic.
Definition: Statepoint.h:396
EVT getValueType() const
Return the ValueType of the referenced return value.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
This class represents an incoming formal argument to a Function.
Definition: Argument.h:29
This class represents lattice values for constants.
Definition: AllocatorList.h:23
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn&#39;t known to be exported from the current basic block...
void LowerDeoptimizeCall(const CallInst *CI)
TargetLowering::CallLoweringInfo CLI
Information regarding the underlying call instruction.
StatepointDirectives parseStatepointDirectivesFromAttrs(AttributeList AS)
Parse out statepoint directives from the function attributes present in AS.
Definition: Statepoint.cpp:60
SmallVector< const GCRelocateInst *, 16 > GCRelocates
The set of gc.relocate calls associated with this gc.statepoint.
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
A specialization of it&#39;s base class for read only access to a gc.statepoint.
Definition: Statepoint.h:295
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
Type * getActualReturnType() const
Return the type of the value returned by the call underlying the statepoint.
Definition: Statepoint.h:160
This class represents a function call, abstracting a target machine&#39;s calling convention.
unsigned Reg
static std::pair< SDValue, SDNode * > lowerCallFromStatepointLoweringInfo(SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder, SmallVectorImpl< SDValue > &PendingExports)
Extract call from statepoint, lower it and return pointer to the call node.
void resize(unsigned N, bool t=false)
Grow or shrink the bitvector.
STATISTIC(NumFunctions, "Total number of functions")
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1014
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:534
SDNode * getNode() const
get the SDNode which holds the desired result
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:465
CallBaseTy * getCall() const
Return the underlying call instruction.
Definition: Statepoint.h:110
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
op_iterator op_begin()
Definition: User.h:229
static void reservePreviousStackSlotForValue(const Value *IncomingValue, SelectionDAGBuilder &Builder)
Try to find existing copies of the incoming values in stack slots used for statepoint spilling...
Keep track of frame indices allocated for statepoints as they could be used across basic block bounda...
const CallBase * getStatepoint() const
The statepoint with which this gc.relocate is associated.
Definition: Statepoint.h:339
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
bool test(unsigned Idx) const
void verify()
Asserts if this statepoint is malformed.
Definition: Statepoint.h:276
unsigned ID
The ID that the resulting STATEPOINT instruction has to report.
SDValue getRoot()
Return the current virtual root of the Selection DAG, flushing any PendingLoad items.
InstructionTy * getInstruction() const
Definition: Statepoint.h:141
A description of a memory reference used in the backend.
uint64_t StatepointFlags
Flags associated with the meta arguments being lowered.
Value * getDerivedPtr() const
Definition: Statepoint.h:390
ArrayRef< const Use > GCTransitionArgs
The list of gc transition arguments present in the gc.statepoint being lowered.
CallLoweringInfo & setChain(SDValue InChain)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:41
op_iterator op_end() const
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:303
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
static void removeDuplicateGCPtrs(SmallVectorImpl< const Value *> &Bases, SmallVectorImpl< const Value *> &Ptrs, SmallVectorImpl< const GCRelocateInst *> &Relocs, SelectionDAGBuilder &Builder, FunctionLoweringInfo::StatepointSpillMap &SSM)
Remove any duplicate (as SDValues) from the derived pointer pairs.
const GCResultInst * getGCResult() const
Get the experimental_gc_result call tied to this statepoint.
Definition: Statepoint.h:265
arg_iterator gc_args_begin() const
Definition: Statepoint.h:244
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
SDValue LowerAsSTATEPOINT(StatepointLoweringInfo &SI)
Lower SLI into a STATEPOINT instruction.
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
int getArgumentFrameIndex(const Argument *A)
getArgumentFrameIndex - Get frame index for the byval argument.
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:291
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:244
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:397
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:630
virtual Optional< bool > isGCManagedPointer(const Type *Ty) const
If the type specified can be reliably distinguished, returns true for pointers to GC managed location...
Definition: GCStrategy.h:99
bool isVarArg() const
Definition: DerivedTypes.h:122
This class represents a no-op cast from one type to another.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
op_iterator op_begin() const
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:578
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
Class to represent pointers.
Definition: DerivedTypes.h:498
Optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Definition: InstrTypes.h:1716
static const uint64_t DeoptBundleStatepointID
Definition: Statepoint.h:445
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return &#39;this&#39;.
Definition: Type.h:303
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:140
The memory access is volatile.
uint64_t getID() const
Return the ID associated with this statepoint.
Definition: Statepoint.h:121
arg_iterator deopt_end() const
Definition: Statepoint.h:233
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
const Instruction * StatepointInstr
The gc.statepoint instruction.
GC_TRANSITION_START/GC_TRANSITION_END - These operators mark the beginning and end of GC transition s...
Definition: ISDOpcodes.h:858
ArrayRef< const Use > DeoptState
The deoptimization state associated with this gc.statepoint call, if any.
static std::tuple< SDValue, SDValue, MachineMemOperand * > spillIncomingStatepointValue(SDValue Incoming, SDValue Chain, SelectionDAGBuilder &Builder)
Spill a value incoming to the statepoint.
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
static void pushStackMapConstant(SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder, uint64_t Value)
const SDValue & getOperand(unsigned Num) const
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:223
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1058
void LowerCallSiteWithDeoptBundleImpl(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB, bool VarArgDisallowed, bool ForceVoidReturnTy)
SDValue getLocation(SDValue Val)
Returns the spill location of a value incoming to the current statepoint.
SDValue allocateStackSlot(EVT ValueType, SelectionDAGBuilder &Builder)
Get a stack slot we can use to store an value of type ValueType.
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:160
void clear()
Clear the memory usage of this object.
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:771
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
int getNumCallArgs() const
Number of arguments to be passed to the actual callee.
Definition: Statepoint.h:167
static Optional< int > findPreviousSpillSlot(const Value *Val, SelectionDAGBuilder &Builder, int LookUpDepth)
Utility function for reservePreviousStackSlotForValue.
DenseMap< const Instruction *, StatepointSpillMap > StatepointSpillMaps
Maps gc.statepoint instructions to their corresponding StatepointSpillMap instances.
Extended Value Type.
Definition: ValueTypes.h:33
size_t size() const
Definition: SmallVector.h:52
auto find(R &&Range, const T &Val) -> decltype(adl_begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1206
unsigned getNumOperands() const
Return the number of values used by this operation.
void setLocation(SDValue Val, SDValue Location)
SmallVector< unsigned, 50 > StatepointStackSlots
StatepointStackSlots - A list of temporary stack slots (frame indices) used to spill values at a stat...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
void markAsStatepointSpillSlotObjectIndex(int ObjectIdx)
SmallBitVector & set()
The memory access writes data.
void clear()
Clear all bits.
arg_iterator gc_args_end() const
Definition: Statepoint.h:245
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:841
DenseMap< const Value *, const Value * > DuplicateMap
Maps llvm IR values to the values they were de-duplicated to.
An SDNode that represents everything that will be needed to construct a MachineInstr.
Indicates that this statepoint is a transition from GC-aware code to code that is not GC-aware...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
size_t size() const
Returns the number of bits in this bitvector.
Represents one node in the SelectionDAG.
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1280
GCFunctionInfo * GFI
Garbage collection metadata for the function.
amdgpu Simplify well known AMD library false FunctionCallee Callee
static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< MachineMemOperand *> &MemRefs, SelectionDAGBuilder &Builder)
Lower a single value incoming to a statepoint node.
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:471
The memory access reads data.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1120
bool hasValue() const
Definition: Optional.h:259
Represents a use of a SDNode.
void LowerStatepoint(ImmutableStatepoint ISP, const BasicBlock *EHPadBB=nullptr)
GCStrategy describes a garbage collector algorithm&#39;s code generation requirements, and provides overridable hooks for those needs which cannot be abstractly described.
Definition: GCStrategy.h:66
unsigned getNumArgOperands() const
Definition: InstrTypes.h:1153
uint32_t getNumPatchBytes() const
Return the number of patchable bytes associated with this statepoint.
Definition: Statepoint.h:127
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1258
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:615
#define I(x, y, z)
Definition: MD5.cpp:58
void startNewStatepoint(SelectionDAGBuilder &Builder)
Reset all state tracking for a newly encountered safepoint.
SmallVector< const Value *, 16 > Bases
Bases[i] is the base pointer for Ptrs[i].
SDValue getValue(unsigned R) const
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:174
std::vector< const GCRelocateInst * > getRelocates() const
Get list of all gc reloactes linked to this statepoint May contain several relocations for the same b...
Definition: Statepoint.h:410
unsigned NumPatchBytes
The number of patchable bytes the call needs to get lowered into.
uint64_t getFlags() const
Definition: Statepoint.h:115
ValueTy * getCalledValue() const
Return the value actually being called or invoked.
Definition: Statepoint.h:136
FunctionLoweringInfo & FuncInfo
Information about the function as a whole.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
const BasicBlock * EHPadBB
The exception handling unwind destination, in case this represents an invoke of gc.statepoint.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
Represents calls to the gc.relocate intrinsic.
Definition: Statepoint.h:363
Mark the deopt arguments associated with the statepoint as only being "live-in".
LLVM Value Representation.
Definition: Value.h:72
static void lowerStatepointMetaArgs(SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< MachineMemOperand *> &MemRefs, SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder)
Lower deopt state and gc pointer arguments of the statepoint.
static MachineMemOperand * getMachineMemOperand(MachineFunction &MF, FrameIndexSDNode &FI)
ArrayRef< const Use > GCArgs
The full list of gc arguments to the gc.statepoint being lowered.
MVT getFrameIndexTy()
Returns the type of FrameIndex and TargetFrameIndex nodes.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
A bitmask that includes all valid flags.
Type * getElementType() const
Definition: DerivedTypes.h:517
arg_iterator deopt_begin() const
Definition: Statepoint.h:228
Describes a gc.statepoint or a gc.statepoint like thing for the purposes of lowering into a STATEPOIN...
This file describes how to lower LLVM code to machine code.
const BasicBlock * getParent() const
Definition: Instruction.h:66
A discriminated union of two pointer types, with the discriminator in the low bit of the pointer...
Definition: PointerUnion.h:86
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:1244