LLVM  10.0.0svn
StatepointLowering.cpp
Go to the documentation of this file.
1 //===- StatepointLowering.cpp - SDAGBuilder's statepoint code -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file includes support code use by SelectionDAGBuilder when lowering a
10 // statepoint sequence in SelectionDAG IR.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "StatepointLowering.h"
15 #include "SelectionDAGBuilder.h"
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/DenseMap.h"
18 #include "llvm/ADT/None.h"
19 #include "llvm/ADT/Optional.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallVector.h"
22 #include "llvm/ADT/Statistic.h"
33 #include "llvm/CodeGen/StackMaps.h"
36 #include "llvm/IR/CallingConv.h"
37 #include "llvm/IR/DerivedTypes.h"
38 #include "llvm/IR/Instruction.h"
39 #include "llvm/IR/Instructions.h"
40 #include "llvm/IR/LLVMContext.h"
41 #include "llvm/IR/Statepoint.h"
42 #include "llvm/IR/Type.h"
43 #include "llvm/Support/Casting.h"
47 #include <cassert>
48 #include <cstddef>
49 #include <cstdint>
50 #include <iterator>
51 #include <tuple>
52 #include <utility>
53 
54 using namespace llvm;
55 
56 #define DEBUG_TYPE "statepoint-lowering"
57 
58 STATISTIC(NumSlotsAllocatedForStatepoints,
59  "Number of stack slots allocated for statepoints");
60 STATISTIC(NumOfStatepoints, "Number of statepoint nodes encountered");
61 STATISTIC(StatepointMaxSlotsRequired,
62  "Maximum number of stack slots required for a singe statepoint");
63 
65  SelectionDAGBuilder &Builder, uint64_t Value) {
66  SDLoc L = Builder.getCurSDLoc();
67  Ops.push_back(Builder.DAG.getTargetConstant(StackMaps::ConstantOp, L,
68  MVT::i64));
69  Ops.push_back(Builder.DAG.getTargetConstant(Value, L, MVT::i64));
70 }
71 
73  // Consistency check
74  assert(PendingGCRelocateCalls.empty() &&
75  "Trying to visit statepoint before finished processing previous one");
76  Locations.clear();
77  NextSlotToAllocate = 0;
78  // Need to resize this on each safepoint - we need the two to stay in sync and
79  // the clear patterns of a SelectionDAGBuilder have no relation to
80  // FunctionLoweringInfo. Also need to ensure used bits get cleared.
81  AllocatedStackSlots.clear();
82  AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size());
83 }
84 
86  Locations.clear();
87  AllocatedStackSlots.clear();
88  assert(PendingGCRelocateCalls.empty() &&
89  "cleared before statepoint sequence completed");
90 }
91 
92 SDValue
94  SelectionDAGBuilder &Builder) {
95  NumSlotsAllocatedForStatepoints++;
97 
98  unsigned SpillSize = ValueType.getStoreSize();
99  assert((SpillSize * 8) == ValueType.getSizeInBits() && "Size not in bytes?");
100 
101  // First look for a previously created stack slot which is not in
102  // use (accounting for the fact arbitrary slots may already be
103  // reserved), or to create a new stack slot and use it.
104 
105  const size_t NumSlots = AllocatedStackSlots.size();
106  assert(NextSlotToAllocate <= NumSlots && "Broken invariant");
107 
108  assert(AllocatedStackSlots.size() ==
109  Builder.FuncInfo.StatepointStackSlots.size() &&
110  "Broken invariant");
111 
112  for (; NextSlotToAllocate < NumSlots; NextSlotToAllocate++) {
113  if (!AllocatedStackSlots.test(NextSlotToAllocate)) {
114  const int FI = Builder.FuncInfo.StatepointStackSlots[NextSlotToAllocate];
115  if (MFI.getObjectSize(FI) == SpillSize) {
116  AllocatedStackSlots.set(NextSlotToAllocate);
117  // TODO: Is ValueType the right thing to use here?
118  return Builder.DAG.getFrameIndex(FI, ValueType);
119  }
120  }
121  }
122 
123  // Couldn't find a free slot, so create a new one:
124 
125  SDValue SpillSlot = Builder.DAG.CreateStackTemporary(ValueType);
126  const unsigned FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
128 
130  AllocatedStackSlots.resize(AllocatedStackSlots.size()+1, true);
131  assert(AllocatedStackSlots.size() ==
132  Builder.FuncInfo.StatepointStackSlots.size() &&
133  "Broken invariant");
134 
135  StatepointMaxSlotsRequired.updateMax(
137 
138  return SpillSlot;
139 }
140 
141 /// Utility function for reservePreviousStackSlotForValue. Tries to find
142 /// stack slot index to which we have spilled value for previous statepoints.
143 /// LookUpDepth specifies maximum DFS depth this function is allowed to look.
145  SelectionDAGBuilder &Builder,
146  int LookUpDepth) {
147  // Can not look any further - give up now
148  if (LookUpDepth <= 0)
149  return None;
150 
151  // Spill location is known for gc relocates
152  if (const auto *Relocate = dyn_cast<GCRelocateInst>(Val)) {
153  const auto &SpillMap =
154  Builder.FuncInfo.StatepointSpillMaps[Relocate->getStatepoint()];
155 
156  auto It = SpillMap.find(Relocate->getDerivedPtr());
157  if (It == SpillMap.end())
158  return None;
159 
160  return It->second;
161  }
162 
163  // Look through bitcast instructions.
164  if (const BitCastInst *Cast = dyn_cast<BitCastInst>(Val))
165  return findPreviousSpillSlot(Cast->getOperand(0), Builder, LookUpDepth - 1);
166 
167  // Look through phi nodes
168  // All incoming values should have same known stack slot, otherwise result
169  // is unknown.
170  if (const PHINode *Phi = dyn_cast<PHINode>(Val)) {
171  Optional<int> MergedResult = None;
172 
173  for (auto &IncomingValue : Phi->incoming_values()) {
174  Optional<int> SpillSlot =
175  findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth - 1);
176  if (!SpillSlot.hasValue())
177  return None;
178 
179  if (MergedResult.hasValue() && *MergedResult != *SpillSlot)
180  return None;
181 
182  MergedResult = SpillSlot;
183  }
184  return MergedResult;
185  }
186 
187  // TODO: We can do better for PHI nodes. In cases like this:
188  // ptr = phi(relocated_pointer, not_relocated_pointer)
189  // statepoint(ptr)
190  // We will return that stack slot for ptr is unknown. And later we might
191  // assign different stack slots for ptr and relocated_pointer. This limits
192  // llvm's ability to remove redundant stores.
193  // Unfortunately it's hard to accomplish in current infrastructure.
194  // We use this function to eliminate spill store completely, while
195  // in example we still need to emit store, but instead of any location
196  // we need to use special "preferred" location.
197 
198  // TODO: handle simple updates. If a value is modified and the original
199  // value is no longer live, it would be nice to put the modified value in the
200  // same slot. This allows folding of the memory accesses for some
201  // instructions types (like an increment).
202  // statepoint (i)
203  // i1 = i+1
204  // statepoint (i1)
205  // However we need to be careful for cases like this:
206  // statepoint(i)
207  // i1 = i+1
208  // statepoint(i, i1)
209  // Here we want to reserve spill slot for 'i', but not for 'i+1'. If we just
210  // put handling of simple modifications in this function like it's done
211  // for bitcasts we might end up reserving i's slot for 'i+1' because order in
212  // which we visit values is unspecified.
213 
214  // Don't know any information about this instruction
215  return None;
216 }
217 
218 /// Try to find existing copies of the incoming values in stack slots used for
219 /// statepoint spilling. If we can find a spill slot for the incoming value,
220 /// mark that slot as allocated, and reuse the same slot for this safepoint.
221 /// This helps to avoid series of loads and stores that only serve to reshuffle
222 /// values on the stack between calls.
223 static void reservePreviousStackSlotForValue(const Value *IncomingValue,
224  SelectionDAGBuilder &Builder) {
225  SDValue Incoming = Builder.getValue(IncomingValue);
226 
227  if (isa<ConstantSDNode>(Incoming) || isa<FrameIndexSDNode>(Incoming)) {
228  // We won't need to spill this, so no need to check for previously
229  // allocated stack slots
230  return;
231  }
232 
233  SDValue OldLocation = Builder.StatepointLowering.getLocation(Incoming);
234  if (OldLocation.getNode())
235  // Duplicates in input
236  return;
237 
238  const int LookUpDepth = 6;
240  findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth);
241  if (!Index.hasValue())
242  return;
243 
244  const auto &StatepointSlots = Builder.FuncInfo.StatepointStackSlots;
245 
246  auto SlotIt = find(StatepointSlots, *Index);
247  assert(SlotIt != StatepointSlots.end() &&
248  "Value spilled to the unknown stack slot");
249 
250  // This is one of our dedicated lowering slots
251  const int Offset = std::distance(StatepointSlots.begin(), SlotIt);
252  if (Builder.StatepointLowering.isStackSlotAllocated(Offset)) {
253  // stack slot already assigned to someone else, can't use it!
254  // TODO: currently we reserve space for gc arguments after doing
255  // normal allocation for deopt arguments. We should reserve for
256  // _all_ deopt and gc arguments, then start allocating. This
257  // will prevent some moves being inserted when vm state changes,
258  // but gc state doesn't between two calls.
259  return;
260  }
261  // Reserve this stack slot
262  Builder.StatepointLowering.reserveStackSlot(Offset);
263 
264  // Cache this slot so we find it when going through the normal
265  // assignment loop.
266  SDValue Loc =
267  Builder.DAG.getTargetFrameIndex(*Index, Builder.getFrameIndexTy());
268  Builder.StatepointLowering.setLocation(Incoming, Loc);
269 }
270 
271 /// Remove any duplicate (as SDValues) from the derived pointer pairs. This
272 /// is not required for correctness. It's purpose is to reduce the size of
273 /// StackMap section. It has no effect on the number of spill slots required
274 /// or the actual lowering.
275 static void
279  SelectionDAGBuilder &Builder,
282 
283  SmallVector<const Value *, 64> NewBases, NewPtrs;
285  for (size_t i = 0, e = Ptrs.size(); i < e; i++) {
286  SDValue SD = Builder.getValue(Ptrs[i]);
287  auto SeenIt = Seen.find(SD);
288 
289  if (SeenIt == Seen.end()) {
290  // Only add non-duplicates
291  NewBases.push_back(Bases[i]);
292  NewPtrs.push_back(Ptrs[i]);
293  NewRelocs.push_back(Relocs[i]);
294  Seen[SD] = Ptrs[i];
295  } else {
296  // Duplicate pointer found, note in SSM and move on:
297  SSM.DuplicateMap[Ptrs[i]] = SeenIt->second;
298  }
299  }
300  assert(Bases.size() >= NewBases.size());
301  assert(Ptrs.size() >= NewPtrs.size());
302  assert(Relocs.size() >= NewRelocs.size());
303  Bases = NewBases;
304  Ptrs = NewPtrs;
305  Relocs = NewRelocs;
306  assert(Ptrs.size() == Bases.size());
307  assert(Ptrs.size() == Relocs.size());
308 }
309 
310 /// Extract call from statepoint, lower it and return pointer to the
311 /// call node. Also update NodeMap so that getValue(statepoint) will
312 /// reference lowered call result
313 static std::pair<SDValue, SDNode *> lowerCallFromStatepointLoweringInfo(
315  SelectionDAGBuilder &Builder, SmallVectorImpl<SDValue> &PendingExports) {
316  SDValue ReturnValue, CallEndVal;
317  std::tie(ReturnValue, CallEndVal) =
318  Builder.lowerInvokable(SI.CLI, SI.EHPadBB);
319  SDNode *CallEnd = CallEndVal.getNode();
320 
321  // Get a call instruction from the call sequence chain. Tail calls are not
322  // allowed. The following code is essentially reverse engineering X86's
323  // LowerCallTo.
324  //
325  // We are expecting DAG to have the following form:
326  //
327  // ch = eh_label (only in case of invoke statepoint)
328  // ch, glue = callseq_start ch
329  // ch, glue = X86::Call ch, glue
330  // ch, glue = callseq_end ch, glue
331  // get_return_value ch, glue
332  //
333  // get_return_value can either be a sequence of CopyFromReg instructions
334  // to grab the return value from the return register(s), or it can be a LOAD
335  // to load a value returned by reference via a stack slot.
336 
337  bool HasDef = !SI.CLI.RetTy->isVoidTy();
338  if (HasDef) {
339  if (CallEnd->getOpcode() == ISD::LOAD)
340  CallEnd = CallEnd->getOperand(0).getNode();
341  else
342  while (CallEnd->getOpcode() == ISD::CopyFromReg)
343  CallEnd = CallEnd->getOperand(0).getNode();
344  }
345 
346  assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!");
347  return std::make_pair(ReturnValue, CallEnd->getOperand(0).getNode());
348 }
349 
351  FrameIndexSDNode &FI) {
352  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI.getIndex());
353  auto MMOFlags = MachineMemOperand::MOStore |
355  auto &MFI = MF.getFrameInfo();
356  return MF.getMachineMemOperand(PtrInfo, MMOFlags,
357  MFI.getObjectSize(FI.getIndex()),
358  MFI.getObjectAlignment(FI.getIndex()));
359 }
360 
361 /// Spill a value incoming to the statepoint. It might be either part of
362 /// vmstate
363 /// or gcstate. In both cases unconditionally spill it on the stack unless it
364 /// is a null constant. Return pair with first element being frame index
365 /// containing saved value and second element with outgoing chain from the
366 /// emitted store
367 static std::tuple<SDValue, SDValue, MachineMemOperand*>
369  SelectionDAGBuilder &Builder) {
370  SDValue Loc = Builder.StatepointLowering.getLocation(Incoming);
371  MachineMemOperand* MMO = nullptr;
372 
373  // Emit new store if we didn't do it for this ptr before
374  if (!Loc.getNode()) {
375  Loc = Builder.StatepointLowering.allocateStackSlot(Incoming.getValueType(),
376  Builder);
377  int Index = cast<FrameIndexSDNode>(Loc)->getIndex();
378  // We use TargetFrameIndex so that isel will not select it into LEA
379  Loc = Builder.DAG.getTargetFrameIndex(Index, Builder.getFrameIndexTy());
380 
381  // Right now we always allocate spill slots that are of the same
382  // size as the value we're about to spill (the size of spillee can
383  // vary since we spill vectors of pointers too). At some point we
384  // can consider allowing spills of smaller values to larger slots
385  // (i.e. change the '==' in the assert below to a '>=').
387  assert((MFI.getObjectSize(Index) * 8) == Incoming.getValueSizeInBits() &&
388  "Bad spill: stack slot does not match!");
389 
390  // Note: Using the alignment of the spill slot (rather than the abi or
391  // preferred alignment) is required for correctness when dealing with spill
392  // slots with preferred alignments larger than frame alignment..
393  auto &MF = Builder.DAG.getMachineFunction();
394  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, Index);
395  auto *StoreMMO =
396  MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
397  MFI.getObjectSize(Index),
398  MFI.getObjectAlignment(Index));
399  Chain = Builder.DAG.getStore(Chain, Builder.getCurSDLoc(), Incoming, Loc,
400  StoreMMO);
401 
402  MMO = getMachineMemOperand(MF, *cast<FrameIndexSDNode>(Loc));
403 
404  Builder.StatepointLowering.setLocation(Incoming, Loc);
405  }
406 
407  assert(Loc.getNode());
408  return std::make_tuple(Loc, Chain, MMO);
409 }
410 
411 /// Lower a single value incoming to a statepoint node. This value can be
412 /// either a deopt value or a gc value, the handling is the same. We special
413 /// case constants and allocas, then fall back to spilling if required.
414 static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly,
417  SelectionDAGBuilder &Builder) {
418  // Note: We know all of these spills are independent, but don't bother to
419  // exploit that chain wise. DAGCombine will happily do so as needed, so
420  // doing it here would be a small compile time win at most.
421  SDValue Chain = Builder.getRoot();
422 
423  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
424  // If the original value was a constant, make sure it gets recorded as
425  // such in the stackmap. This is required so that the consumer can
426  // parse any internal format to the deopt state. It also handles null
427  // pointers and other constant pointers in GC states. Note the constant
428  // vectors do not appear to actually hit this path and that anything larger
429  // than an i64 value (not type!) will fail asserts here.
430  pushStackMapConstant(Ops, Builder, C->getSExtValue());
431  } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
432  // This handles allocas as arguments to the statepoint (this is only
433  // really meaningful for a deopt value. For GC, we'd be trying to
434  // relocate the address of the alloca itself?)
435  assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
436  "Incoming value is a frame index!");
437  Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
438  Builder.getFrameIndexTy()));
439 
440  auto &MF = Builder.DAG.getMachineFunction();
441  auto *MMO = getMachineMemOperand(MF, *FI);
442  MemRefs.push_back(MMO);
443 
444  } else if (LiveInOnly) {
445  // If this value is live in (not live-on-return, or live-through), we can
446  // treat it the same way patchpoint treats it's "live in" values. We'll
447  // end up folding some of these into stack references, but they'll be
448  // handled by the register allocator. Note that we do not have the notion
449  // of a late use so these values might be placed in registers which are
450  // clobbered by the call. This is fine for live-in.
451  Ops.push_back(Incoming);
452  } else {
453  // Otherwise, locate a spill slot and explicitly spill it so it
454  // can be found by the runtime later. We currently do not support
455  // tracking values through callee saved registers to their eventual
456  // spill location. This would be a useful optimization, but would
457  // need to be optional since it requires a lot of complexity on the
458  // runtime side which not all would support.
459  auto Res = spillIncomingStatepointValue(Incoming, Chain, Builder);
460  Ops.push_back(std::get<0>(Res));
461  if (auto *MMO = std::get<2>(Res))
462  MemRefs.push_back(MMO);
463  Chain = std::get<1>(Res);;
464  }
465 
466  Builder.DAG.setRoot(Chain);
467 }
468 
469 /// Lower deopt state and gc pointer arguments of the statepoint. The actual
470 /// lowering is described in lowerIncomingStatepointValue. This function is
471 /// responsible for lowering everything in the right position and playing some
472 /// tricks to avoid redundant stack manipulation where possible. On
473 /// completion, 'Ops' will contain ready to use operands for machine code
474 /// statepoint. The chain nodes will have already been created and the DAG root
475 /// will be set to the last value spilled (if any were).
476 static void
479  SelectionDAGBuilder &Builder) {
480  // Lower the deopt and gc arguments for this statepoint. Layout will be:
481  // deopt argument length, deopt arguments.., gc arguments...
482 #ifndef NDEBUG
483  if (auto *GFI = Builder.GFI) {
484  // Check that each of the gc pointer and bases we've gotten out of the
485  // safepoint is something the strategy thinks might be a pointer (or vector
486  // of pointers) into the GC heap. This is basically just here to help catch
487  // errors during statepoint insertion. TODO: This should actually be in the
488  // Verifier, but we can't get to the GCStrategy from there (yet).
489  GCStrategy &S = GFI->getStrategy();
490  for (const Value *V : SI.Bases) {
491  auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
492  if (Opt.hasValue()) {
493  assert(Opt.getValue() &&
494  "non gc managed base pointer found in statepoint");
495  }
496  }
497  for (const Value *V : SI.Ptrs) {
498  auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
499  if (Opt.hasValue()) {
500  assert(Opt.getValue() &&
501  "non gc managed derived pointer found in statepoint");
502  }
503  }
504  assert(SI.Bases.size() == SI.Ptrs.size() && "Pointer without base!");
505  } else {
506  assert(SI.Bases.empty() && "No gc specified, so cannot relocate pointers!");
507  assert(SI.Ptrs.empty() && "No gc specified, so cannot relocate pointers!");
508  }
509 #endif
510 
511  // Figure out what lowering strategy we're going to use for each part
512  // Note: Is is conservatively correct to lower both "live-in" and "live-out"
513  // as "live-through". A "live-through" variable is one which is "live-in",
514  // "live-out", and live throughout the lifetime of the call (i.e. we can find
515  // it from any PC within the transitive callee of the statepoint). In
516  // particular, if the callee spills callee preserved registers we may not
517  // be able to find a value placed in that register during the call. This is
518  // fine for live-out, but not for live-through. If we were willing to make
519  // assumptions about the code generator producing the callee, we could
520  // potentially allow live-through values in callee saved registers.
521  const bool LiveInDeopt =
523 
524  auto isGCValue =[&](const Value *V) {
525  return is_contained(SI.Ptrs, V) || is_contained(SI.Bases, V);
526  };
527 
528  // Before we actually start lowering (and allocating spill slots for values),
529  // reserve any stack slots which we judge to be profitable to reuse for a
530  // particular value. This is purely an optimization over the code below and
531  // doesn't change semantics at all. It is important for performance that we
532  // reserve slots for both deopt and gc values before lowering either.
533  for (const Value *V : SI.DeoptState) {
534  if (!LiveInDeopt || isGCValue(V))
536  }
537  for (unsigned i = 0; i < SI.Bases.size(); ++i) {
538  reservePreviousStackSlotForValue(SI.Bases[i], Builder);
539  reservePreviousStackSlotForValue(SI.Ptrs[i], Builder);
540  }
541 
542  // First, prefix the list with the number of unique values to be
543  // lowered. Note that this is the number of *Values* not the
544  // number of SDValues required to lower them.
545  const int NumVMSArgs = SI.DeoptState.size();
546  pushStackMapConstant(Ops, Builder, NumVMSArgs);
547 
548  // The vm state arguments are lowered in an opaque manner. We do not know
549  // what type of values are contained within.
550  for (const Value *V : SI.DeoptState) {
551  SDValue Incoming;
552  // If this is a function argument at a static frame index, generate it as
553  // the frame index.
554  if (const Argument *Arg = dyn_cast<Argument>(V)) {
555  int FI = Builder.FuncInfo.getArgumentFrameIndex(Arg);
556  if (FI != INT_MAX)
557  Incoming = Builder.DAG.getFrameIndex(FI, Builder.getFrameIndexTy());
558  }
559  if (!Incoming.getNode())
560  Incoming = Builder.getValue(V);
561  const bool LiveInValue = LiveInDeopt && !isGCValue(V);
562  lowerIncomingStatepointValue(Incoming, LiveInValue, Ops, MemRefs, Builder);
563  }
564 
565  // Finally, go ahead and lower all the gc arguments. There's no prefixed
566  // length for this one. After lowering, we'll have the base and pointer
567  // arrays interwoven with each (lowered) base pointer immediately followed by
568  // it's (lowered) derived pointer. i.e
569  // (base[0], ptr[0], base[1], ptr[1], ...)
570  for (unsigned i = 0; i < SI.Bases.size(); ++i) {
571  const Value *Base = SI.Bases[i];
572  lowerIncomingStatepointValue(Builder.getValue(Base), /*LiveInOnly*/ false,
573  Ops, MemRefs, Builder);
574 
575  const Value *Ptr = SI.Ptrs[i];
576  lowerIncomingStatepointValue(Builder.getValue(Ptr), /*LiveInOnly*/ false,
577  Ops, MemRefs, Builder);
578  }
579 
580  // If there are any explicit spill slots passed to the statepoint, record
581  // them, but otherwise do not do anything special. These are user provided
582  // allocas and give control over placement to the consumer. In this case,
583  // it is the contents of the slot which may get updated, not the pointer to
584  // the alloca
585  for (Value *V : SI.GCArgs) {
586  SDValue Incoming = Builder.getValue(V);
587  if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
588  // This handles allocas as arguments to the statepoint
589  assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
590  "Incoming value is a frame index!");
591  Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
592  Builder.getFrameIndexTy()));
593 
594  auto &MF = Builder.DAG.getMachineFunction();
595  auto *MMO = getMachineMemOperand(MF, *FI);
596  MemRefs.push_back(MMO);
597  }
598  }
599 
600  // Record computed locations for all lowered values.
601  // This can not be embedded in lowering loops as we need to record *all*
602  // values, while previous loops account only values with unique SDValues.
603  const Instruction *StatepointInstr = SI.StatepointInstr;
604  auto &SpillMap = Builder.FuncInfo.StatepointSpillMaps[StatepointInstr];
605 
606  for (const GCRelocateInst *Relocate : SI.GCRelocates) {
607  const Value *V = Relocate->getDerivedPtr();
608  SDValue SDV = Builder.getValue(V);
609  SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
610 
611  if (Loc.getNode()) {
612  SpillMap.SlotMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
613  } else {
614  // Record value as visited, but not spilled. This is case for allocas
615  // and constants. For this values we can avoid emitting spill load while
616  // visiting corresponding gc_relocate.
617  // Actually we do not need to record them in this map at all.
618  // We do this only to check that we are not relocating any unvisited
619  // value.
620  SpillMap.SlotMap[V] = None;
621 
622  // Default llvm mechanisms for exporting values which are used in
623  // different basic blocks does not work for gc relocates.
624  // Note that it would be incorrect to teach llvm that all relocates are
625  // uses of the corresponding values so that it would automatically
626  // export them. Relocates of the spilled values does not use original
627  // value.
628  if (Relocate->getParent() != StatepointInstr->getParent())
629  Builder.ExportFromCurrentBlock(V);
630  }
631  }
632 }
633 
636  // The basic scheme here is that information about both the original call and
637  // the safepoint is encoded in the CallInst. We create a temporary call and
638  // lower it, then reverse engineer the calling sequence.
639 
640  NumOfStatepoints++;
641  // Clear state
642  StatepointLowering.startNewStatepoint(*this);
643 
644 #ifndef NDEBUG
645  // We schedule gc relocates before removeDuplicateGCPtrs since we _will_
646  // encounter the duplicate gc relocates we elide in removeDuplicateGCPtrs.
647  for (auto *Reloc : SI.GCRelocates)
648  if (Reloc->getParent() == SI.StatepointInstr->getParent())
649  StatepointLowering.scheduleRelocCall(*Reloc);
650 #endif
651 
652  // Remove any redundant llvm::Values which map to the same SDValue as another
653  // input. Also has the effect of removing duplicates in the original
654  // llvm::Value input list as well. This is a useful optimization for
655  // reducing the size of the StackMap section. It has no other impact.
656  removeDuplicateGCPtrs(SI.Bases, SI.Ptrs, SI.GCRelocates, *this,
657  FuncInfo.StatepointSpillMaps[SI.StatepointInstr]);
658  assert(SI.Bases.size() == SI.Ptrs.size() &&
659  SI.Ptrs.size() == SI.GCRelocates.size());
660 
661  // Lower statepoint vmstate and gcstate arguments
662  SmallVector<SDValue, 10> LoweredMetaArgs;
664  lowerStatepointMetaArgs(LoweredMetaArgs, MemRefs, SI, *this);
665 
666  // Now that we've emitted the spills, we need to update the root so that the
667  // call sequence is ordered correctly.
668  SI.CLI.setChain(getRoot());
669 
670  // Get call node, we will replace it later with statepoint
671  SDValue ReturnVal;
672  SDNode *CallNode;
673  std::tie(ReturnVal, CallNode) =
674  lowerCallFromStatepointLoweringInfo(SI, *this, PendingExports);
675 
676  // Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END
677  // nodes with all the appropriate arguments and return values.
678 
679  // Call Node: Chain, Target, {Args}, RegMask, [Glue]
680  SDValue Chain = CallNode->getOperand(0);
681 
682  SDValue Glue;
683  bool CallHasIncomingGlue = CallNode->getGluedNode();
684  if (CallHasIncomingGlue) {
685  // Glue is always last operand
686  Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
687  }
688 
689  // Build the GC_TRANSITION_START node if necessary.
690  //
691  // The operands to the GC_TRANSITION_{START,END} nodes are laid out in the
692  // order in which they appear in the call to the statepoint intrinsic. If
693  // any of the operands is a pointer-typed, that operand is immediately
694  // followed by a SRCVALUE for the pointer that may be used during lowering
695  // (e.g. to form MachinePointerInfo values for loads/stores).
696  const bool IsGCTransition =
699  if (IsGCTransition) {
701 
702  // Add chain
703  TSOps.push_back(Chain);
704 
705  // Add GC transition arguments
706  for (const Value *V : SI.GCTransitionArgs) {
707  TSOps.push_back(getValue(V));
708  if (V->getType()->isPointerTy())
709  TSOps.push_back(DAG.getSrcValue(V));
710  }
711 
712  // Add glue if necessary
713  if (CallHasIncomingGlue)
714  TSOps.push_back(Glue);
715 
716  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
717 
718  SDValue GCTransitionStart =
719  DAG.getNode(ISD::GC_TRANSITION_START, getCurSDLoc(), NodeTys, TSOps);
720 
721  Chain = GCTransitionStart.getValue(0);
722  Glue = GCTransitionStart.getValue(1);
723  }
724 
725  // TODO: Currently, all of these operands are being marked as read/write in
726  // PrologEpilougeInserter.cpp, we should special case the VMState arguments
727  // and flags to be read-only.
729 
730  // Add the <id> and <numBytes> constants.
731  Ops.push_back(DAG.getTargetConstant(SI.ID, getCurSDLoc(), MVT::i64));
732  Ops.push_back(
733  DAG.getTargetConstant(SI.NumPatchBytes, getCurSDLoc(), MVT::i32));
734 
735  // Calculate and push starting position of vmstate arguments
736  // Get number of arguments incoming directly into call node
737  unsigned NumCallRegArgs =
738  CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);
739  Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));
740 
741  // Add call target
742  SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
743  Ops.push_back(CallTarget);
744 
745  // Add call arguments
746  // Get position of register mask in the call
747  SDNode::op_iterator RegMaskIt;
748  if (CallHasIncomingGlue)
749  RegMaskIt = CallNode->op_end() - 2;
750  else
751  RegMaskIt = CallNode->op_end() - 1;
752  Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);
753 
754  // Add a constant argument for the calling convention
755  pushStackMapConstant(Ops, *this, SI.CLI.CallConv);
756 
757  // Add a constant argument for the flags
758  uint64_t Flags = SI.StatepointFlags;
759  assert(((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0) &&
760  "Unknown flag used");
761  pushStackMapConstant(Ops, *this, Flags);
762 
763  // Insert all vmstate and gcstate arguments
764  Ops.insert(Ops.end(), LoweredMetaArgs.begin(), LoweredMetaArgs.end());
765 
766  // Add register mask from call node
767  Ops.push_back(*RegMaskIt);
768 
769  // Add chain
770  Ops.push_back(Chain);
771 
772  // Same for the glue, but we add it only if original call had it
773  if (Glue.getNode())
774  Ops.push_back(Glue);
775 
776  // Compute return values. Provide a glue output since we consume one as
777  // input. This allows someone else to chain off us as needed.
778  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
779 
780  MachineSDNode *StatepointMCNode =
781  DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
782  DAG.setNodeMemRefs(StatepointMCNode, MemRefs);
783 
784  SDNode *SinkNode = StatepointMCNode;
785 
786  // Build the GC_TRANSITION_END node if necessary.
787  //
788  // See the comment above regarding GC_TRANSITION_START for the layout of
789  // the operands to the GC_TRANSITION_END node.
790  if (IsGCTransition) {
792 
793  // Add chain
794  TEOps.push_back(SDValue(StatepointMCNode, 0));
795 
796  // Add GC transition arguments
797  for (const Value *V : SI.GCTransitionArgs) {
798  TEOps.push_back(getValue(V));
799  if (V->getType()->isPointerTy())
800  TEOps.push_back(DAG.getSrcValue(V));
801  }
802 
803  // Add glue
804  TEOps.push_back(SDValue(StatepointMCNode, 1));
805 
806  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
807 
808  SDValue GCTransitionStart =
809  DAG.getNode(ISD::GC_TRANSITION_END, getCurSDLoc(), NodeTys, TEOps);
810 
811  SinkNode = GCTransitionStart.getNode();
812  }
813 
814  // Replace original call
815  DAG.ReplaceAllUsesWith(CallNode, SinkNode); // This may update Root
816  // Remove original call node
817  DAG.DeleteNode(CallNode);
818 
819  // DON'T set the root - under the assumption that it's already set past the
820  // inserted node we created.
821 
822  // TODO: A better future implementation would be to emit a single variable
823  // argument, variable return value STATEPOINT node here and then hookup the
824  // return value of each gc.relocate to the respective output of the
825  // previously emitted STATEPOINT value. Unfortunately, this doesn't appear
826  // to actually be possible today.
827 
828  return ReturnVal;
829 }
830 
831 void
833  const BasicBlock *EHPadBB /*= nullptr*/) {
835  "anyregcc is not supported on statepoints!");
836 
837 #ifndef NDEBUG
838  // If this is a malformed statepoint, report it early to simplify debugging.
839  // This should catch any IR level mistake that's made when constructing or
840  // transforming statepoints.
841  ISP.verify();
842 
843  // Check that the associated GCStrategy expects to encounter statepoints.
844  assert(GFI->getStrategy().useStatepoints() &&
845  "GCStrategy does not expect to encounter statepoints");
846 #endif
847 
848  SDValue ActualCallee;
849 
850  if (ISP.getNumPatchBytes() > 0) {
851  // If we've been asked to emit a nop sequence instead of a call instruction
852  // for this statepoint then don't lower the call target, but use a constant
853  // `null` instead. Not lowering the call target lets statepoint clients get
854  // away without providing a physical address for the symbolic call target at
855  // link time.
856 
857  const auto &TLI = DAG.getTargetLoweringInfo();
858  const auto &DL = DAG.getDataLayout();
859 
860  unsigned AS = ISP.getCalledValue()->getType()->getPointerAddressSpace();
861  ActualCallee = DAG.getConstant(0, getCurSDLoc(), TLI.getPointerTy(DL, AS));
862  } else {
863  ActualCallee = getValue(ISP.getCalledValue());
864  }
865 
867  populateCallLoweringInfo(SI.CLI, ISP.getCall(),
869  ISP.getNumCallArgs(), ActualCallee,
870  ISP.getActualReturnType(), false /* IsPatchPoint */);
871 
872  for (const GCRelocateInst *Relocate : ISP.getRelocates()) {
873  SI.GCRelocates.push_back(Relocate);
874  SI.Bases.push_back(Relocate->getBasePtr());
875  SI.Ptrs.push_back(Relocate->getDerivedPtr());
876  }
877 
878  SI.GCArgs = ArrayRef<const Use>(ISP.gc_args_begin(), ISP.gc_args_end());
879  SI.StatepointInstr = ISP.getInstruction();
880  SI.GCTransitionArgs =
882  SI.ID = ISP.getID();
883  SI.DeoptState = ArrayRef<const Use>(ISP.deopt_begin(), ISP.deopt_end());
884  SI.StatepointFlags = ISP.getFlags();
885  SI.NumPatchBytes = ISP.getNumPatchBytes();
886  SI.EHPadBB = EHPadBB;
887 
888  SDValue ReturnValue = LowerAsSTATEPOINT(SI);
889 
890  // Export the result value if needed
891  const GCResultInst *GCResult = ISP.getGCResult();
892  Type *RetTy = ISP.getActualReturnType();
893  if (!RetTy->isVoidTy() && GCResult) {
894  if (GCResult->getParent() != ISP.getCall()->getParent()) {
895  // Result value will be used in a different basic block so we need to
896  // export it now. Default exporting mechanism will not work here because
897  // statepoint call has a different type than the actual call. It means
898  // that by default llvm will create export register of the wrong type
899  // (always i32 in our case). So instead we need to create export register
900  // with correct type manually.
901  // TODO: To eliminate this problem we can remove gc.result intrinsics
902  // completely and make statepoint call to return a tuple.
903  unsigned Reg = FuncInfo.CreateRegs(RetTy);
904  RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
905  DAG.getDataLayout(), Reg, RetTy,
906  ISP.getCall()->getCallingConv());
907  SDValue Chain = DAG.getEntryNode();
908 
909  RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
910  PendingExports.push_back(Chain);
911  FuncInfo.ValueMap[ISP.getInstruction()] = Reg;
912  } else {
913  // Result value will be used in a same basic block. Don't export it or
914  // perform any explicit register copies.
915  // We'll replace the actuall call node shortly. gc_result will grab
916  // this value.
917  setValue(ISP.getInstruction(), ReturnValue);
918  }
919  } else {
920  // The token value is never used from here on, just generate a poison value
921  setValue(ISP.getInstruction(), DAG.getIntPtrConstant(-1, getCurSDLoc()));
922  }
923 }
924 
926  const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB,
927  bool VarArgDisallowed, bool ForceVoidReturnTy) {
929  unsigned ArgBeginIndex = Call->arg_begin() - Call->op_begin();
930  populateCallLoweringInfo(
931  SI.CLI, Call, ArgBeginIndex, Call->getNumArgOperands(), Callee,
932  ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : Call->getType(),
933  false);
934  if (!VarArgDisallowed)
935  SI.CLI.IsVarArg = Call->getFunctionType()->isVarArg();
936 
937  auto DeoptBundle = *Call->getOperandBundle(LLVMContext::OB_deopt);
938 
940 
942  SI.ID = SD.StatepointID.getValueOr(DefaultID);
943  SI.NumPatchBytes = SD.NumPatchBytes.getValueOr(0);
944 
945  SI.DeoptState =
946  ArrayRef<const Use>(DeoptBundle.Inputs.begin(), DeoptBundle.Inputs.end());
947  SI.StatepointFlags = static_cast<uint64_t>(StatepointFlags::None);
948  SI.EHPadBB = EHPadBB;
949 
950  // NB! The GC arguments are deliberately left empty.
951 
952  if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) {
953  ReturnVal = lowerRangeToAssertZExt(DAG, *Call, ReturnVal);
954  setValue(Call, ReturnVal);
955  }
956 }
957 
959  const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB) {
960  LowerCallSiteWithDeoptBundleImpl(Call, Callee, EHPadBB,
961  /* VarArgDisallowed = */ false,
962  /* ForceVoidReturnTy = */ false);
963 }
964 
965 void SelectionDAGBuilder::visitGCResult(const GCResultInst &CI) {
966  // The result value of the gc_result is simply the result of the actual
967  // call. We've already emitted this, so just grab the value.
968  const Instruction *I = CI.getStatepoint();
969 
970  if (I->getParent() != CI.getParent()) {
971  // Statepoint is in different basic block so we should have stored call
972  // result in a virtual register.
973  // We can not use default getValue() functionality to copy value from this
974  // register because statepoint and actual call return types can be
975  // different, and getValue() will use CopyFromReg of the wrong type,
976  // which is always i32 in our case.
977  PointerType *CalleeType = cast<PointerType>(
979  Type *RetTy =
980  cast<FunctionType>(CalleeType->getElementType())->getReturnType();
981  SDValue CopyFromReg = getCopyFromRegs(I, RetTy);
982 
983  assert(CopyFromReg.getNode());
984  setValue(&CI, CopyFromReg);
985  } else {
986  setValue(&CI, getValue(I));
987  }
988 }
989 
990 void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
991 #ifndef NDEBUG
992  // Consistency check
993  // We skip this check for relocates not in the same basic block as their
994  // statepoint. It would be too expensive to preserve validation info through
995  // different basic blocks.
996  if (Relocate.getStatepoint()->getParent() == Relocate.getParent())
997  StatepointLowering.relocCallVisited(Relocate);
998 
999  auto *Ty = Relocate.getType()->getScalarType();
1000  if (auto IsManaged = GFI->getStrategy().isGCManagedPointer(Ty))
1001  assert(*IsManaged && "Non gc managed pointer relocated!");
1002 #endif
1003 
1004  const Value *DerivedPtr = Relocate.getDerivedPtr();
1005  SDValue SD = getValue(DerivedPtr);
1006 
1007  auto &SpillMap = FuncInfo.StatepointSpillMaps[Relocate.getStatepoint()];
1008  auto SlotIt = SpillMap.find(DerivedPtr);
1009  assert(SlotIt != SpillMap.end() && "Relocating not lowered gc value");
1010  Optional<int> DerivedPtrLocation = SlotIt->second;
1011 
1012  // We didn't need to spill these special cases (constants and allocas).
1013  // See the handling in spillIncomingValueForStatepoint for detail.
1014  if (!DerivedPtrLocation) {
1015  setValue(&Relocate, SD);
1016  return;
1017  }
1018 
1019  unsigned Index = *DerivedPtrLocation;
1020  SDValue SpillSlot = DAG.getTargetFrameIndex(Index, getFrameIndexTy());
1021 
1022  // Note: We know all of these reloads are independent, but don't bother to
1023  // exploit that chain wise. DAGCombine will happily do so as needed, so
1024  // doing it here would be a small compile time win at most.
1025  SDValue Chain = getRoot();
1026 
1027  auto &MF = DAG.getMachineFunction();
1028  auto &MFI = MF.getFrameInfo();
1029  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, Index);
1030  auto *LoadMMO =
1031  MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
1032  MFI.getObjectSize(Index),
1033  MFI.getObjectAlignment(Index));
1034 
1035  auto LoadVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
1036  Relocate.getType());
1037 
1038  SDValue SpillLoad = DAG.getLoad(LoadVT, getCurSDLoc(), Chain,
1039  SpillSlot, LoadMMO);
1040 
1041  DAG.setRoot(SpillLoad.getValue(1));
1042 
1043  assert(SpillLoad.getNode());
1044  setValue(&Relocate, SpillLoad);
1045 }
1046 
1048  const auto &TLI = DAG.getTargetLoweringInfo();
1049  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE),
1050  TLI.getPointerTy(DAG.getDataLayout()));
1051 
1052  // We don't lower calls to __llvm_deoptimize as varargs, but as a regular
1053  // call. We also do not lower the return value to any virtual register, and
1054  // change the immediately following return to a trap instruction.
1055  LowerCallSiteWithDeoptBundleImpl(CI, Callee, /* EHPadBB = */ nullptr,
1056  /* VarArgDisallowed = */ true,
1057  /* ForceVoidReturnTy = */ true);
1058 }
1059 
1061  // We do not lower the return value from llvm.deoptimize to any virtual
1062  // register, and change the immediately following return to a trap
1063  // instruction.
1064  if (DAG.getTarget().Options.TrapUnreachable)
1065  DAG.setRoot(
1066  DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
1067 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
uint64_t CallInst * C
Represents calls to the gc.result intrinsic.
Definition: Statepoint.h:393
EVT getValueType() const
Return the ValueType of the referenced return value.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
This class represents an incoming formal argument to a Function.
Definition: Argument.h:29
This class represents lattice values for constants.
Definition: AllocatorList.h:23
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn&#39;t known to be exported from the current basic block...
void LowerDeoptimizeCall(const CallInst *CI)
TargetLowering::CallLoweringInfo CLI
Information regarding the underlying call instruction.
StatepointDirectives parseStatepointDirectivesFromAttrs(AttributeList AS)
Parse out statepoint directives from the function attributes present in AS.
Definition: Statepoint.cpp:60
SmallVector< const GCRelocateInst *, 16 > GCRelocates
The set of gc.relocate calls associated with this gc.statepoint.
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
A specialization of it&#39;s base class for read only access to a gc.statepoint.
Definition: Statepoint.h:292
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
Type * getActualReturnType() const
Return the type of the value returned by the call underlying the statepoint.
Definition: Statepoint.h:157
This class represents a function call, abstracting a target machine&#39;s calling convention.
unsigned Reg
static std::pair< SDValue, SDNode * > lowerCallFromStatepointLoweringInfo(SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder, SmallVectorImpl< SDValue > &PendingExports)
Extract call from statepoint, lower it and return pointer to the call node.
void resize(unsigned N, bool t=false)
Grow or shrink the bitvector.
STATISTIC(NumFunctions, "Total number of functions")
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1100
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:635
SDNode * getNode() const
get the SDNode which holds the desired result
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:483
CallBaseTy * getCall() const
Return the underlying call instruction.
Definition: Statepoint.h:107
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
op_iterator op_begin()
Definition: User.h:229
static void reservePreviousStackSlotForValue(const Value *IncomingValue, SelectionDAGBuilder &Builder)
Try to find existing copies of the incoming values in stack slots used for statepoint spilling...
Keep track of frame indices allocated for statepoints as they could be used across basic block bounda...
const CallBase * getStatepoint() const
The statepoint with which this gc.relocate is associated.
Definition: Statepoint.h:336
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
bool test(unsigned Idx) const
void verify()
Asserts if this statepoint is malformed.
Definition: Statepoint.h:273
unsigned ID
The ID that the resulting STATEPOINT instruction has to report.
SDValue getRoot()
Return the current virtual root of the Selection DAG, flushing any PendingLoad items.
InstructionTy * getInstruction() const
Definition: Statepoint.h:138
A description of a memory reference used in the backend.
uint64_t StatepointFlags
Flags associated with the meta arguments being lowered.
Value * getDerivedPtr() const
Definition: Statepoint.h:387
ArrayRef< const Use > GCTransitionArgs
The list of gc transition arguments present in the gc.statepoint being lowered.
CallLoweringInfo & setChain(SDValue InChain)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:41
op_iterator op_end() const
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:303
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
static void removeDuplicateGCPtrs(SmallVectorImpl< const Value *> &Bases, SmallVectorImpl< const Value *> &Ptrs, SmallVectorImpl< const GCRelocateInst *> &Relocs, SelectionDAGBuilder &Builder, FunctionLoweringInfo::StatepointSpillMap &SSM)
Remove any duplicate (as SDValues) from the derived pointer pairs.
const GCResultInst * getGCResult() const
Get the experimental_gc_result call tied to this statepoint.
Definition: Statepoint.h:262
arg_iterator gc_args_begin() const
Definition: Statepoint.h:241
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
SDValue LowerAsSTATEPOINT(StatepointLoweringInfo &SI)
Lower SLI into a STATEPOINT instruction.
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
int getArgumentFrameIndex(const Argument *A)
getArgumentFrameIndex - Get frame index for the byval argument.
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:291
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:246
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:414
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:647
virtual Optional< bool > isGCManagedPointer(const Type *Ty) const
If the type specified can be reliably distinguished, returns true for pointers to GC managed location...
Definition: GCStrategy.h:99
bool isVarArg() const
Definition: DerivedTypes.h:128
This class represents a no-op cast from one type to another.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
op_iterator op_begin() const
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:596
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
Class to represent pointers.
Definition: DerivedTypes.h:579
Optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Definition: InstrTypes.h:1814
static const uint64_t DeoptBundleStatepointID
Definition: Statepoint.h:442
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return &#39;this&#39;.
Definition: Type.h:307
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:141
The memory access is volatile.
uint64_t getID() const
Return the ID associated with this statepoint.
Definition: Statepoint.h:118
unsigned getObjectAlignment(int ObjectIdx) const
Return the alignment of the specified stack object.
arg_iterator deopt_end() const
Definition: Statepoint.h:230
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
const Instruction * StatepointInstr
The gc.statepoint instruction.
GC_TRANSITION_START/GC_TRANSITION_END - These operators mark the beginning and end of GC transition s...
Definition: ISDOpcodes.h:893
ArrayRef< const Use > DeoptState
The deoptimization state associated with this gc.statepoint call, if any.
static std::tuple< SDValue, SDValue, MachineMemOperand * > spillIncomingStatepointValue(SDValue Incoming, SDValue Chain, SelectionDAGBuilder &Builder)
Spill a value incoming to the statepoint.
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
static void pushStackMapConstant(SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder, uint64_t Value)
const SDValue & getOperand(unsigned Num) const
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:224
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1144
void LowerCallSiteWithDeoptBundleImpl(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB, bool VarArgDisallowed, bool ForceVoidReturnTy)
SDValue getLocation(SDValue Val)
Returns the spill location of a value incoming to the current statepoint.
SDValue allocateStackSlot(EVT ValueType, SelectionDAGBuilder &Builder)
Get a stack slot we can use to store an value of type ValueType.
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:165
void clear()
Clear the memory usage of this object.
constexpr double e
Definition: MathExtras.h:57
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:806
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
int getNumCallArgs() const
Number of arguments to be passed to the actual callee.
Definition: Statepoint.h:164
static Optional< int > findPreviousSpillSlot(const Value *Val, SelectionDAGBuilder &Builder, int LookUpDepth)
Utility function for reservePreviousStackSlotForValue.
DenseMap< const Instruction *, StatepointSpillMap > StatepointSpillMaps
Maps gc.statepoint instructions to their corresponding StatepointSpillMap instances.
Extended Value Type.
Definition: ValueTypes.h:33
size_t size() const
Definition: SmallVector.h:52
auto find(R &&Range, const T &Val) -> decltype(adl_begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1186
unsigned getNumOperands() const
Return the number of values used by this operation.
void setLocation(SDValue Val, SDValue Location)
SmallVector< unsigned, 50 > StatepointStackSlots
StatepointStackSlots - A list of temporary stack slots (frame indices) used to spill values at a stat...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
void markAsStatepointSpillSlotObjectIndex(int ObjectIdx)
SmallBitVector & set()
The memory access writes data.
void clear()
Clear all bits.
arg_iterator gc_args_end() const
Definition: Statepoint.h:242
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
DenseMap< const Value *, const Value * > DuplicateMap
Maps llvm IR values to the values they were de-duplicated to.
An SDNode that represents everything that will be needed to construct a MachineInstr.
Indicates that this statepoint is a transition from GC-aware code to code that is not GC-aware...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
size_t size() const
Returns the number of bits in this bitvector.
Represents one node in the SelectionDAG.
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1366
GCFunctionInfo * GFI
Garbage collection metadata for the function.
amdgpu Simplify well known AMD library false FunctionCallee Callee
static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< MachineMemOperand *> &MemRefs, SelectionDAGBuilder &Builder)
Lower a single value incoming to a statepoint node.
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:467
The memory access reads data.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1206
bool hasValue() const
Definition: Optional.h:259
Represents a use of a SDNode.
void LowerStatepoint(ImmutableStatepoint ISP, const BasicBlock *EHPadBB=nullptr)
GCStrategy describes a garbage collector algorithm&#39;s code generation requirements, and provides overridable hooks for those needs which cannot be abstractly described.
Definition: GCStrategy.h:66
unsigned getNumArgOperands() const
Definition: InstrTypes.h:1239
uint32_t getNumPatchBytes() const
Return the number of patchable bytes associated with this statepoint.
Definition: Statepoint.h:124
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1344
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:650
#define I(x, y, z)
Definition: MD5.cpp:58
void startNewStatepoint(SelectionDAGBuilder &Builder)
Reset all state tracking for a newly encountered safepoint.
SmallVector< const Value *, 16 > Bases
Bases[i] is the base pointer for Ptrs[i].
SDValue getValue(unsigned R) const
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:174
std::vector< const GCRelocateInst * > getRelocates() const
Get list of all gc reloactes linked to this statepoint May contain several relocations for the same b...
Definition: Statepoint.h:407
unsigned NumPatchBytes
The number of patchable bytes the call needs to get lowered into.
uint64_t getFlags() const
Definition: Statepoint.h:112
ValueTy * getCalledValue() const
Return the value actually being called or invoked.
Definition: Statepoint.h:133
FunctionLoweringInfo & FuncInfo
Information about the function as a whole.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
const BasicBlock * EHPadBB
The exception handling unwind destination, in case this represents an invoke of gc.statepoint.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
Represents calls to the gc.relocate intrinsic.
Definition: Statepoint.h:360
Mark the deopt arguments associated with the statepoint as only being "live-in".
LLVM Value Representation.
Definition: Value.h:74
static void lowerStatepointMetaArgs(SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< MachineMemOperand *> &MemRefs, SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder)
Lower deopt state and gc pointer arguments of the statepoint.
static MachineMemOperand * getMachineMemOperand(MachineFunction &MF, FrameIndexSDNode &FI)
ArrayRef< const Use > GCArgs
The full list of gc arguments to the gc.statepoint being lowered.
MVT getFrameIndexTy()
Returns the type of FrameIndex and TargetFrameIndex nodes.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
A bitmask that includes all valid flags.
Type * getElementType() const
Definition: DerivedTypes.h:598
arg_iterator deopt_begin() const
Definition: Statepoint.h:225
Describes a gc.statepoint or a gc.statepoint like thing for the purposes of lowering into a STATEPOIN...
This file describes how to lower LLVM code to machine code.
const BasicBlock * getParent() const
Definition: Instruction.h:66
A discriminated union of two or more pointer types, with the discriminator in the low bit of the poin...
Definition: PointerUnion.h:156
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:1224