LLVM  14.0.0git
StatepointLowering.cpp
Go to the documentation of this file.
1 //===- StatepointLowering.cpp - SDAGBuilder's statepoint code -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file includes support code use by SelectionDAGBuilder when lowering a
10 // statepoint sequence in SelectionDAG IR.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "StatepointLowering.h"
15 #include "SelectionDAGBuilder.h"
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/None.h"
18 #include "llvm/ADT/Optional.h"
19 #include "llvm/ADT/STLExtras.h"
20 #include "llvm/ADT/SmallSet.h"
21 #include "llvm/ADT/Statistic.h"
30 #include "llvm/CodeGen/StackMaps.h"
33 #include "llvm/IR/CallingConv.h"
34 #include "llvm/IR/DerivedTypes.h"
35 #include "llvm/IR/GCStrategy.h"
36 #include "llvm/IR/Instruction.h"
37 #include "llvm/IR/Instructions.h"
38 #include "llvm/IR/LLVMContext.h"
39 #include "llvm/IR/Statepoint.h"
40 #include "llvm/IR/Type.h"
41 #include "llvm/Support/Casting.h"
46 #include <cassert>
47 #include <cstddef>
48 #include <cstdint>
49 #include <iterator>
50 #include <tuple>
51 #include <utility>
52 
53 using namespace llvm;
54 
55 #define DEBUG_TYPE "statepoint-lowering"
56 
57 STATISTIC(NumSlotsAllocatedForStatepoints,
58  "Number of stack slots allocated for statepoints");
59 STATISTIC(NumOfStatepoints, "Number of statepoint nodes encountered");
60 STATISTIC(StatepointMaxSlotsRequired,
61  "Maximum number of stack slots required for a singe statepoint");
62 
64  "use-registers-for-deopt-values", cl::Hidden, cl::init(false),
65  cl::desc("Allow using registers for non pointer deopt args"));
66 
68  "use-registers-for-gc-values-in-landing-pad", cl::Hidden, cl::init(false),
69  cl::desc("Allow using registers for gc pointer in landing pad"));
70 
72  "max-registers-for-gc-values", cl::Hidden, cl::init(0),
73  cl::desc("Max number of VRegs allowed to pass GC pointer meta args in"));
74 
76 
78  SelectionDAGBuilder &Builder, uint64_t Value) {
79  SDLoc L = Builder.getCurSDLoc();
80  Ops.push_back(Builder.DAG.getTargetConstant(StackMaps::ConstantOp, L,
81  MVT::i64));
82  Ops.push_back(Builder.DAG.getTargetConstant(Value, L, MVT::i64));
83 }
84 
86  // Consistency check
87  assert(PendingGCRelocateCalls.empty() &&
88  "Trying to visit statepoint before finished processing previous one");
89  Locations.clear();
90  NextSlotToAllocate = 0;
91  // Need to resize this on each safepoint - we need the two to stay in sync and
92  // the clear patterns of a SelectionDAGBuilder have no relation to
93  // FunctionLoweringInfo. Also need to ensure used bits get cleared.
94  AllocatedStackSlots.clear();
95  AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size());
96 }
97 
99  Locations.clear();
100  AllocatedStackSlots.clear();
101  assert(PendingGCRelocateCalls.empty() &&
102  "cleared before statepoint sequence completed");
103 }
104 
105 SDValue
108  NumSlotsAllocatedForStatepoints++;
109  MachineFrameInfo &MFI = Builder.DAG.getMachineFunction().getFrameInfo();
110 
111  unsigned SpillSize = ValueType.getStoreSize();
112  assert((SpillSize * 8) ==
113  (-8u & (7 + ValueType.getSizeInBits())) && // Round up modulo 8.
114  "Size not in bytes?");
115 
116  // First look for a previously created stack slot which is not in
117  // use (accounting for the fact arbitrary slots may already be
118  // reserved), or to create a new stack slot and use it.
119 
120  const size_t NumSlots = AllocatedStackSlots.size();
121  assert(NextSlotToAllocate <= NumSlots && "Broken invariant");
122 
123  assert(AllocatedStackSlots.size() ==
124  Builder.FuncInfo.StatepointStackSlots.size() &&
125  "Broken invariant");
126 
127  for (; NextSlotToAllocate < NumSlots; NextSlotToAllocate++) {
128  if (!AllocatedStackSlots.test(NextSlotToAllocate)) {
129  const int FI = Builder.FuncInfo.StatepointStackSlots[NextSlotToAllocate];
130  if (MFI.getObjectSize(FI) == SpillSize) {
131  AllocatedStackSlots.set(NextSlotToAllocate);
132  // TODO: Is ValueType the right thing to use here?
133  return Builder.DAG.getFrameIndex(FI, ValueType);
134  }
135  }
136  }
137 
138  // Couldn't find a free slot, so create a new one:
139 
140  SDValue SpillSlot = Builder.DAG.CreateStackTemporary(ValueType);
141  const unsigned FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
143 
144  Builder.FuncInfo.StatepointStackSlots.push_back(FI);
145  AllocatedStackSlots.resize(AllocatedStackSlots.size()+1, true);
146  assert(AllocatedStackSlots.size() ==
147  Builder.FuncInfo.StatepointStackSlots.size() &&
148  "Broken invariant");
149 
150  StatepointMaxSlotsRequired.updateMax(
151  Builder.FuncInfo.StatepointStackSlots.size());
152 
153  return SpillSlot;
154 }
155 
156 /// Utility function for reservePreviousStackSlotForValue. Tries to find
157 /// stack slot index to which we have spilled value for previous statepoints.
158 /// LookUpDepth specifies maximum DFS depth this function is allowed to look.
161  int LookUpDepth) {
162  // Can not look any further - give up now
163  if (LookUpDepth <= 0)
164  return None;
165 
166  // Spill location is known for gc relocates
167  if (const auto *Relocate = dyn_cast<GCRelocateInst>(Val)) {
168  const auto &RelocationMap =
169  Builder.FuncInfo.StatepointRelocationMaps[Relocate->getStatepoint()];
170 
171  auto It = RelocationMap.find(Relocate->getDerivedPtr());
172  if (It == RelocationMap.end())
173  return None;
174 
175  auto &Record = It->second;
176  if (Record.type != RecordType::Spill)
177  return None;
178 
179  return Record.payload.FI;
180  }
181 
182  // Look through bitcast instructions.
183  if (const BitCastInst *Cast = dyn_cast<BitCastInst>(Val))
184  return findPreviousSpillSlot(Cast->getOperand(0), Builder, LookUpDepth - 1);
185 
186  // Look through phi nodes
187  // All incoming values should have same known stack slot, otherwise result
188  // is unknown.
189  if (const PHINode *Phi = dyn_cast<PHINode>(Val)) {
190  Optional<int> MergedResult = None;
191 
192  for (auto &IncomingValue : Phi->incoming_values()) {
193  Optional<int> SpillSlot =
194  findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth - 1);
195  if (!SpillSlot.hasValue())
196  return None;
197 
198  if (MergedResult.hasValue() && *MergedResult != *SpillSlot)
199  return None;
200 
201  MergedResult = SpillSlot;
202  }
203  return MergedResult;
204  }
205 
206  // TODO: We can do better for PHI nodes. In cases like this:
207  // ptr = phi(relocated_pointer, not_relocated_pointer)
208  // statepoint(ptr)
209  // We will return that stack slot for ptr is unknown. And later we might
210  // assign different stack slots for ptr and relocated_pointer. This limits
211  // llvm's ability to remove redundant stores.
212  // Unfortunately it's hard to accomplish in current infrastructure.
213  // We use this function to eliminate spill store completely, while
214  // in example we still need to emit store, but instead of any location
215  // we need to use special "preferred" location.
216 
217  // TODO: handle simple updates. If a value is modified and the original
218  // value is no longer live, it would be nice to put the modified value in the
219  // same slot. This allows folding of the memory accesses for some
220  // instructions types (like an increment).
221  // statepoint (i)
222  // i1 = i+1
223  // statepoint (i1)
224  // However we need to be careful for cases like this:
225  // statepoint(i)
226  // i1 = i+1
227  // statepoint(i, i1)
228  // Here we want to reserve spill slot for 'i', but not for 'i+1'. If we just
229  // put handling of simple modifications in this function like it's done
230  // for bitcasts we might end up reserving i's slot for 'i+1' because order in
231  // which we visit values is unspecified.
232 
233  // Don't know any information about this instruction
234  return None;
235 }
236 
237 /// Return true if-and-only-if the given SDValue can be lowered as either a
238 /// constant argument or a stack reference. The key point is that the value
239 /// doesn't need to be spilled or tracked as a vreg use.
240 static bool willLowerDirectly(SDValue Incoming) {
241  // We are making an unchecked assumption that the frame size <= 2^16 as that
242  // is the largest offset which can be encoded in the stackmap format.
243  if (isa<FrameIndexSDNode>(Incoming))
244  return true;
245 
246  // The largest constant describeable in the StackMap format is 64 bits.
247  // Potential Optimization: Constants values are sign extended by consumer,
248  // and thus there are many constants of static type > 64 bits whose value
249  // happens to be sext(Con64) and could thus be lowered directly.
250  if (Incoming.getValueType().getSizeInBits() > 64)
251  return false;
252 
253  return (isa<ConstantSDNode>(Incoming) || isa<ConstantFPSDNode>(Incoming) ||
254  Incoming.isUndef());
255 }
256 
257 /// Try to find existing copies of the incoming values in stack slots used for
258 /// statepoint spilling. If we can find a spill slot for the incoming value,
259 /// mark that slot as allocated, and reuse the same slot for this safepoint.
260 /// This helps to avoid series of loads and stores that only serve to reshuffle
261 /// values on the stack between calls.
262 static void reservePreviousStackSlotForValue(const Value *IncomingValue,
264  SDValue Incoming = Builder.getValue(IncomingValue);
265 
266  // If we won't spill this, we don't need to check for previously allocated
267  // stack slots.
268  if (willLowerDirectly(Incoming))
269  return;
270 
271  SDValue OldLocation = Builder.StatepointLowering.getLocation(Incoming);
272  if (OldLocation.getNode())
273  // Duplicates in input
274  return;
275 
276  const int LookUpDepth = 6;
278  findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth);
279  if (!Index.hasValue())
280  return;
281 
282  const auto &StatepointSlots = Builder.FuncInfo.StatepointStackSlots;
283 
284  auto SlotIt = find(StatepointSlots, *Index);
285  assert(SlotIt != StatepointSlots.end() &&
286  "Value spilled to the unknown stack slot");
287 
288  // This is one of our dedicated lowering slots
289  const int Offset = std::distance(StatepointSlots.begin(), SlotIt);
290  if (Builder.StatepointLowering.isStackSlotAllocated(Offset)) {
291  // stack slot already assigned to someone else, can't use it!
292  // TODO: currently we reserve space for gc arguments after doing
293  // normal allocation for deopt arguments. We should reserve for
294  // _all_ deopt and gc arguments, then start allocating. This
295  // will prevent some moves being inserted when vm state changes,
296  // but gc state doesn't between two calls.
297  return;
298  }
299  // Reserve this stack slot
300  Builder.StatepointLowering.reserveStackSlot(Offset);
301 
302  // Cache this slot so we find it when going through the normal
303  // assignment loop.
304  SDValue Loc =
305  Builder.DAG.getTargetFrameIndex(*Index, Builder.getFrameIndexTy());
306  Builder.StatepointLowering.setLocation(Incoming, Loc);
307 }
308 
309 /// Extract call from statepoint, lower it and return pointer to the
310 /// call node. Also update NodeMap so that getValue(statepoint) will
311 /// reference lowered call result
312 static std::pair<SDValue, SDNode *> lowerCallFromStatepointLoweringInfo(
315  SDValue ReturnValue, CallEndVal;
316  std::tie(ReturnValue, CallEndVal) =
317  Builder.lowerInvokable(SI.CLI, SI.EHPadBB);
318  SDNode *CallEnd = CallEndVal.getNode();
319 
320  // Get a call instruction from the call sequence chain. Tail calls are not
321  // allowed. The following code is essentially reverse engineering X86's
322  // LowerCallTo.
323  //
324  // We are expecting DAG to have the following form:
325  //
326  // ch = eh_label (only in case of invoke statepoint)
327  // ch, glue = callseq_start ch
328  // ch, glue = X86::Call ch, glue
329  // ch, glue = callseq_end ch, glue
330  // get_return_value ch, glue
331  //
332  // get_return_value can either be a sequence of CopyFromReg instructions
333  // to grab the return value from the return register(s), or it can be a LOAD
334  // to load a value returned by reference via a stack slot.
335 
336  bool HasDef = !SI.CLI.RetTy->isVoidTy();
337  if (HasDef) {
338  if (CallEnd->getOpcode() == ISD::LOAD)
339  CallEnd = CallEnd->getOperand(0).getNode();
340  else
341  while (CallEnd->getOpcode() == ISD::CopyFromReg)
342  CallEnd = CallEnd->getOperand(0).getNode();
343  }
344 
345  assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!");
346  return std::make_pair(ReturnValue, CallEnd->getOperand(0).getNode());
347 }
348 
350  FrameIndexSDNode &FI) {
351  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI.getIndex());
352  auto MMOFlags = MachineMemOperand::MOStore |
354  auto &MFI = MF.getFrameInfo();
355  return MF.getMachineMemOperand(PtrInfo, MMOFlags,
356  MFI.getObjectSize(FI.getIndex()),
357  MFI.getObjectAlign(FI.getIndex()));
358 }
359 
360 /// Spill a value incoming to the statepoint. It might be either part of
361 /// vmstate
362 /// or gcstate. In both cases unconditionally spill it on the stack unless it
363 /// is a null constant. Return pair with first element being frame index
364 /// containing saved value and second element with outgoing chain from the
365 /// emitted store
366 static std::tuple<SDValue, SDValue, MachineMemOperand*>
369  SDValue Loc = Builder.StatepointLowering.getLocation(Incoming);
370  MachineMemOperand* MMO = nullptr;
371 
372  // Emit new store if we didn't do it for this ptr before
373  if (!Loc.getNode()) {
374  Loc = Builder.StatepointLowering.allocateStackSlot(Incoming.getValueType(),
375  Builder);
376  int Index = cast<FrameIndexSDNode>(Loc)->getIndex();
377  // We use TargetFrameIndex so that isel will not select it into LEA
378  Loc = Builder.DAG.getTargetFrameIndex(Index, Builder.getFrameIndexTy());
379 
380  // Right now we always allocate spill slots that are of the same
381  // size as the value we're about to spill (the size of spillee can
382  // vary since we spill vectors of pointers too). At some point we
383  // can consider allowing spills of smaller values to larger slots
384  // (i.e. change the '==' in the assert below to a '>=').
385  MachineFrameInfo &MFI = Builder.DAG.getMachineFunction().getFrameInfo();
386  assert((MFI.getObjectSize(Index) * 8) ==
387  (-8 & (7 + // Round up modulo 8.
388  (int64_t)Incoming.getValueSizeInBits())) &&
389  "Bad spill: stack slot does not match!");
390 
391  // Note: Using the alignment of the spill slot (rather than the abi or
392  // preferred alignment) is required for correctness when dealing with spill
393  // slots with preferred alignments larger than frame alignment..
394  auto &MF = Builder.DAG.getMachineFunction();
395  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, Index);
396  auto *StoreMMO = MF.getMachineMemOperand(
398  MFI.getObjectAlign(Index));
399  Chain = Builder.DAG.getStore(Chain, Builder.getCurSDLoc(), Incoming, Loc,
400  StoreMMO);
401 
402  MMO = getMachineMemOperand(MF, *cast<FrameIndexSDNode>(Loc));
403 
404  Builder.StatepointLowering.setLocation(Incoming, Loc);
405  }
406 
407  assert(Loc.getNode());
408  return std::make_tuple(Loc, Chain, MMO);
409 }
410 
411 /// Lower a single value incoming to a statepoint node. This value can be
412 /// either a deopt value or a gc value, the handling is the same. We special
413 /// case constants and allocas, then fall back to spilling if required.
414 static void
415 lowerIncomingStatepointValue(SDValue Incoming, bool RequireSpillSlot,
419 
420  if (willLowerDirectly(Incoming)) {
421  if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
422  // This handles allocas as arguments to the statepoint (this is only
423  // really meaningful for a deopt value. For GC, we'd be trying to
424  // relocate the address of the alloca itself?)
425  assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
426  "Incoming value is a frame index!");
427  Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
428  Builder.getFrameIndexTy()));
429 
430  auto &MF = Builder.DAG.getMachineFunction();
431  auto *MMO = getMachineMemOperand(MF, *FI);
432  MemRefs.push_back(MMO);
433  return;
434  }
435 
436  assert(Incoming.getValueType().getSizeInBits() <= 64);
437 
438  if (Incoming.isUndef()) {
439  // Put an easily recognized constant that's unlikely to be a valid
440  // value so that uses of undef by the consumer of the stackmap is
441  // easily recognized. This is legal since the compiler is always
442  // allowed to chose an arbitrary value for undef.
443  pushStackMapConstant(Ops, Builder, 0xFEFEFEFE);
444  return;
445  }
446 
447  // If the original value was a constant, make sure it gets recorded as
448  // such in the stackmap. This is required so that the consumer can
449  // parse any internal format to the deopt state. It also handles null
450  // pointers and other constant pointers in GC states.
451  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
452  pushStackMapConstant(Ops, Builder, C->getSExtValue());
453  return;
454  } else if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Incoming)) {
456  C->getValueAPF().bitcastToAPInt().getZExtValue());
457  return;
458  }
459 
460  llvm_unreachable("unhandled direct lowering case");
461  }
462 
463 
464 
465  if (!RequireSpillSlot) {
466  // If this value is live in (not live-on-return, or live-through), we can
467  // treat it the same way patchpoint treats it's "live in" values. We'll
468  // end up folding some of these into stack references, but they'll be
469  // handled by the register allocator. Note that we do not have the notion
470  // of a late use so these values might be placed in registers which are
471  // clobbered by the call. This is fine for live-in. For live-through
472  // fix-up pass should be executed to force spilling of such registers.
473  Ops.push_back(Incoming);
474  } else {
475  // Otherwise, locate a spill slot and explicitly spill it so it can be
476  // found by the runtime later. Note: We know all of these spills are
477  // independent, but don't bother to exploit that chain wise. DAGCombine
478  // will happily do so as needed, so doing it here would be a small compile
479  // time win at most.
480  SDValue Chain = Builder.getRoot();
481  auto Res = spillIncomingStatepointValue(Incoming, Chain, Builder);
482  Ops.push_back(std::get<0>(Res));
483  if (auto *MMO = std::get<2>(Res))
484  MemRefs.push_back(MMO);
485  Chain = std::get<1>(Res);;
486  Builder.DAG.setRoot(Chain);
487  }
488 
489 }
490 
491 /// Return true if value V represents the GC value. The behavior is conservative
492 /// in case it is not sure that value is not GC the function returns true.
493 static bool isGCValue(const Value *V, SelectionDAGBuilder &Builder) {
494  auto *Ty = V->getType();
495  if (!Ty->isPtrOrPtrVectorTy())
496  return false;
497  if (auto *GFI = Builder.GFI)
498  if (auto IsManaged = GFI->getStrategy().isGCManagedPointer(Ty))
499  return *IsManaged;
500  return true; // conservative
501 }
502 
503 /// Lower deopt state and gc pointer arguments of the statepoint. The actual
504 /// lowering is described in lowerIncomingStatepointValue. This function is
505 /// responsible for lowering everything in the right position and playing some
506 /// tricks to avoid redundant stack manipulation where possible. On
507 /// completion, 'Ops' will contain ready to use operands for machine code
508 /// statepoint. The chain nodes will have already been created and the DAG root
509 /// will be set to the last value spilled (if any were).
510 static void
513  SmallVectorImpl<SDValue> &GCPtrs,
514  DenseMap<SDValue, int> &LowerAsVReg,
517  // Lower the deopt and gc arguments for this statepoint. Layout will be:
518  // deopt argument length, deopt arguments.., gc arguments...
519 #ifndef NDEBUG
520  if (auto *GFI = Builder.GFI) {
521  // Check that each of the gc pointer and bases we've gotten out of the
522  // safepoint is something the strategy thinks might be a pointer (or vector
523  // of pointers) into the GC heap. This is basically just here to help catch
524  // errors during statepoint insertion. TODO: This should actually be in the
525  // Verifier, but we can't get to the GCStrategy from there (yet).
526  GCStrategy &S = GFI->getStrategy();
527  for (const Value *V : SI.Bases) {
528  auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
529  if (Opt.hasValue()) {
530  assert(Opt.getValue() &&
531  "non gc managed base pointer found in statepoint");
532  }
533  }
534  for (const Value *V : SI.Ptrs) {
535  auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
536  if (Opt.hasValue()) {
537  assert(Opt.getValue() &&
538  "non gc managed derived pointer found in statepoint");
539  }
540  }
541  assert(SI.Bases.size() == SI.Ptrs.size() && "Pointer without base!");
542  } else {
543  assert(SI.Bases.empty() && "No gc specified, so cannot relocate pointers!");
544  assert(SI.Ptrs.empty() && "No gc specified, so cannot relocate pointers!");
545  }
546 #endif
547 
548  // Figure out what lowering strategy we're going to use for each part
549  // Note: Is is conservatively correct to lower both "live-in" and "live-out"
550  // as "live-through". A "live-through" variable is one which is "live-in",
551  // "live-out", and live throughout the lifetime of the call (i.e. we can find
552  // it from any PC within the transitive callee of the statepoint). In
553  // particular, if the callee spills callee preserved registers we may not
554  // be able to find a value placed in that register during the call. This is
555  // fine for live-out, but not for live-through. If we were willing to make
556  // assumptions about the code generator producing the callee, we could
557  // potentially allow live-through values in callee saved registers.
558  const bool LiveInDeopt =
559  SI.StatepointFlags & (uint64_t)StatepointFlags::DeoptLiveIn;
560 
561  // Decide which deriver pointers will go on VRegs
562  unsigned MaxVRegPtrs = MaxRegistersForGCPointers.getValue();
563 
564  // Pointers used on exceptional path of invoke statepoint.
565  // We cannot assing them to VRegs.
566  SmallSet<SDValue, 8> LPadPointers;
568  if (auto *StInvoke = dyn_cast_or_null<InvokeInst>(SI.StatepointInstr)) {
569  LandingPadInst *LPI = StInvoke->getLandingPadInst();
570  for (auto *Relocate : SI.GCRelocates)
571  if (Relocate->getOperand(0) == LPI) {
572  LPadPointers.insert(Builder.getValue(Relocate->getBasePtr()));
573  LPadPointers.insert(Builder.getValue(Relocate->getDerivedPtr()));
574  }
575  }
576 
577  LLVM_DEBUG(dbgs() << "Deciding how to lower GC Pointers:\n");
578 
579  // List of unique lowered GC Pointer values.
580  SmallSetVector<SDValue, 16> LoweredGCPtrs;
581  // Map lowered GC Pointer value to the index in above vector
582  DenseMap<SDValue, unsigned> GCPtrIndexMap;
583 
584  unsigned CurNumVRegs = 0;
585 
586  auto canPassGCPtrOnVReg = [&](SDValue SD) {
587  if (SD.getValueType().isVector())
588  return false;
589  if (LPadPointers.count(SD))
590  return false;
591  return !willLowerDirectly(SD);
592  };
593 
594  auto processGCPtr = [&](const Value *V) {
595  SDValue PtrSD = Builder.getValue(V);
596  if (!LoweredGCPtrs.insert(PtrSD))
597  return; // skip duplicates
598  GCPtrIndexMap[PtrSD] = LoweredGCPtrs.size() - 1;
599 
600  assert(!LowerAsVReg.count(PtrSD) && "must not have been seen");
601  if (LowerAsVReg.size() == MaxVRegPtrs)
602  return;
603  assert(V->getType()->isVectorTy() == PtrSD.getValueType().isVector() &&
604  "IR and SD types disagree");
605  if (!canPassGCPtrOnVReg(PtrSD)) {
606  LLVM_DEBUG(dbgs() << "direct/spill "; PtrSD.dump(&Builder.DAG));
607  return;
608  }
609  LLVM_DEBUG(dbgs() << "vreg "; PtrSD.dump(&Builder.DAG));
610  LowerAsVReg[PtrSD] = CurNumVRegs++;
611  };
612 
613  // Process derived pointers first to give them more chance to go on VReg.
614  for (const Value *V : SI.Ptrs)
615  processGCPtr(V);
616  for (const Value *V : SI.Bases)
617  processGCPtr(V);
618 
619  LLVM_DEBUG(dbgs() << LowerAsVReg.size() << " pointers will go in vregs\n");
620 
621  auto requireSpillSlot = [&](const Value *V) {
622  if (!Builder.DAG.getTargetLoweringInfo().isTypeLegal(
623  Builder.getValue(V).getValueType()))
624  return true;
625  if (isGCValue(V, Builder))
626  return !LowerAsVReg.count(Builder.getValue(V));
627  return !(LiveInDeopt || UseRegistersForDeoptValues);
628  };
629 
630  // Before we actually start lowering (and allocating spill slots for values),
631  // reserve any stack slots which we judge to be profitable to reuse for a
632  // particular value. This is purely an optimization over the code below and
633  // doesn't change semantics at all. It is important for performance that we
634  // reserve slots for both deopt and gc values before lowering either.
635  for (const Value *V : SI.DeoptState) {
636  if (requireSpillSlot(V))
638  }
639 
640  for (const Value *V : SI.Ptrs) {
641  SDValue SDV = Builder.getValue(V);
642  if (!LowerAsVReg.count(SDV))
644  }
645 
646  for (const Value *V : SI.Bases) {
647  SDValue SDV = Builder.getValue(V);
648  if (!LowerAsVReg.count(SDV))
650  }
651 
652  // First, prefix the list with the number of unique values to be
653  // lowered. Note that this is the number of *Values* not the
654  // number of SDValues required to lower them.
655  const int NumVMSArgs = SI.DeoptState.size();
656  pushStackMapConstant(Ops, Builder, NumVMSArgs);
657 
658  // The vm state arguments are lowered in an opaque manner. We do not know
659  // what type of values are contained within.
660  LLVM_DEBUG(dbgs() << "Lowering deopt state\n");
661  for (const Value *V : SI.DeoptState) {
662  SDValue Incoming;
663  // If this is a function argument at a static frame index, generate it as
664  // the frame index.
665  if (const Argument *Arg = dyn_cast<Argument>(V)) {
666  int FI = Builder.FuncInfo.getArgumentFrameIndex(Arg);
667  if (FI != INT_MAX)
668  Incoming = Builder.DAG.getFrameIndex(FI, Builder.getFrameIndexTy());
669  }
670  if (!Incoming.getNode())
671  Incoming = Builder.getValue(V);
672  LLVM_DEBUG(dbgs() << "Value " << *V
673  << " requireSpillSlot = " << requireSpillSlot(V) << "\n");
674  lowerIncomingStatepointValue(Incoming, requireSpillSlot(V), Ops, MemRefs,
675  Builder);
676  }
677 
678  // Finally, go ahead and lower all the gc arguments.
679  pushStackMapConstant(Ops, Builder, LoweredGCPtrs.size());
680  for (SDValue SDV : LoweredGCPtrs)
681  lowerIncomingStatepointValue(SDV, !LowerAsVReg.count(SDV), Ops, MemRefs,
682  Builder);
683 
684  // Copy to out vector. LoweredGCPtrs will be empty after this point.
685  GCPtrs = LoweredGCPtrs.takeVector();
686 
687  // If there are any explicit spill slots passed to the statepoint, record
688  // them, but otherwise do not do anything special. These are user provided
689  // allocas and give control over placement to the consumer. In this case,
690  // it is the contents of the slot which may get updated, not the pointer to
691  // the alloca
692  SmallVector<SDValue, 4> Allocas;
693  for (Value *V : SI.GCArgs) {
694  SDValue Incoming = Builder.getValue(V);
695  if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
696  // This handles allocas as arguments to the statepoint
697  assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
698  "Incoming value is a frame index!");
699  Allocas.push_back(Builder.DAG.getTargetFrameIndex(
700  FI->getIndex(), Builder.getFrameIndexTy()));
701 
702  auto &MF = Builder.DAG.getMachineFunction();
703  auto *MMO = getMachineMemOperand(MF, *FI);
704  MemRefs.push_back(MMO);
705  }
706  }
707  pushStackMapConstant(Ops, Builder, Allocas.size());
708  Ops.append(Allocas.begin(), Allocas.end());
709 
710  // Now construct GC base/derived map;
711  pushStackMapConstant(Ops, Builder, SI.Ptrs.size());
712  SDLoc L = Builder.getCurSDLoc();
713  for (unsigned i = 0; i < SI.Ptrs.size(); ++i) {
714  SDValue Base = Builder.getValue(SI.Bases[i]);
715  assert(GCPtrIndexMap.count(Base) && "base not found in index map");
716  Ops.push_back(
717  Builder.DAG.getTargetConstant(GCPtrIndexMap[Base], L, MVT::i64));
718  SDValue Derived = Builder.getValue(SI.Ptrs[i]);
719  assert(GCPtrIndexMap.count(Derived) && "derived not found in index map");
720  Ops.push_back(
721  Builder.DAG.getTargetConstant(GCPtrIndexMap[Derived], L, MVT::i64));
722  }
723 }
724 
727  // The basic scheme here is that information about both the original call and
728  // the safepoint is encoded in the CallInst. We create a temporary call and
729  // lower it, then reverse engineer the calling sequence.
730 
731  NumOfStatepoints++;
732  // Clear state
734  assert(SI.Bases.size() == SI.Ptrs.size());
735 
736  LLVM_DEBUG(dbgs() << "Lowering statepoint " << *SI.StatepointInstr << "\n");
737 #ifndef NDEBUG
738  for (auto *Reloc : SI.GCRelocates)
739  if (Reloc->getParent() == SI.StatepointInstr->getParent())
741 #endif
742 
743  // Lower statepoint vmstate and gcstate arguments
744 
745  // All lowered meta args.
746  SmallVector<SDValue, 10> LoweredMetaArgs;
747  // Lowered GC pointers (subset of above).
748  SmallVector<SDValue, 16> LoweredGCArgs;
750  // Maps derived pointer SDValue to statepoint result of relocated pointer.
751  DenseMap<SDValue, int> LowerAsVReg;
752  lowerStatepointMetaArgs(LoweredMetaArgs, MemRefs, LoweredGCArgs, LowerAsVReg,
753  SI, *this);
754 
755  // Now that we've emitted the spills, we need to update the root so that the
756  // call sequence is ordered correctly.
757  SI.CLI.setChain(getRoot());
758 
759  // Get call node, we will replace it later with statepoint
760  SDValue ReturnVal;
761  SDNode *CallNode;
762  std::tie(ReturnVal, CallNode) =
763  lowerCallFromStatepointLoweringInfo(SI, *this, PendingExports);
764 
765  // Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END
766  // nodes with all the appropriate arguments and return values.
767 
768  // Call Node: Chain, Target, {Args}, RegMask, [Glue]
769  SDValue Chain = CallNode->getOperand(0);
770 
771  SDValue Glue;
772  bool CallHasIncomingGlue = CallNode->getGluedNode();
773  if (CallHasIncomingGlue) {
774  // Glue is always last operand
775  Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
776  }
777 
778  // Build the GC_TRANSITION_START node if necessary.
779  //
780  // The operands to the GC_TRANSITION_{START,END} nodes are laid out in the
781  // order in which they appear in the call to the statepoint intrinsic. If
782  // any of the operands is a pointer-typed, that operand is immediately
783  // followed by a SRCVALUE for the pointer that may be used during lowering
784  // (e.g. to form MachinePointerInfo values for loads/stores).
785  const bool IsGCTransition =
786  (SI.StatepointFlags & (uint64_t)StatepointFlags::GCTransition) ==
788  if (IsGCTransition) {
790 
791  // Add chain
792  TSOps.push_back(Chain);
793 
794  // Add GC transition arguments
795  for (const Value *V : SI.GCTransitionArgs) {
796  TSOps.push_back(getValue(V));
797  if (V->getType()->isPointerTy())
798  TSOps.push_back(DAG.getSrcValue(V));
799  }
800 
801  // Add glue if necessary
802  if (CallHasIncomingGlue)
803  TSOps.push_back(Glue);
804 
806 
807  SDValue GCTransitionStart =
808  DAG.getNode(ISD::GC_TRANSITION_START, getCurSDLoc(), NodeTys, TSOps);
809 
810  Chain = GCTransitionStart.getValue(0);
811  Glue = GCTransitionStart.getValue(1);
812  }
813 
814  // TODO: Currently, all of these operands are being marked as read/write in
815  // PrologEpilougeInserter.cpp, we should special case the VMState arguments
816  // and flags to be read-only.
818 
819  // Add the <id> and <numBytes> constants.
820  Ops.push_back(DAG.getTargetConstant(SI.ID, getCurSDLoc(), MVT::i64));
821  Ops.push_back(
822  DAG.getTargetConstant(SI.NumPatchBytes, getCurSDLoc(), MVT::i32));
823 
824  // Calculate and push starting position of vmstate arguments
825  // Get number of arguments incoming directly into call node
826  unsigned NumCallRegArgs =
827  CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);
828  Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));
829 
830  // Add call target
831  SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
832  Ops.push_back(CallTarget);
833 
834  // Add call arguments
835  // Get position of register mask in the call
836  SDNode::op_iterator RegMaskIt;
837  if (CallHasIncomingGlue)
838  RegMaskIt = CallNode->op_end() - 2;
839  else
840  RegMaskIt = CallNode->op_end() - 1;
841  Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);
842 
843  // Add a constant argument for the calling convention
844  pushStackMapConstant(Ops, *this, SI.CLI.CallConv);
845 
846  // Add a constant argument for the flags
847  uint64_t Flags = SI.StatepointFlags;
848  assert(((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0) &&
849  "Unknown flag used");
850  pushStackMapConstant(Ops, *this, Flags);
851 
852  // Insert all vmstate and gcstate arguments
853  llvm::append_range(Ops, LoweredMetaArgs);
854 
855  // Add register mask from call node
856  Ops.push_back(*RegMaskIt);
857 
858  // Add chain
859  Ops.push_back(Chain);
860 
861  // Same for the glue, but we add it only if original call had it
862  if (Glue.getNode())
863  Ops.push_back(Glue);
864 
865  // Compute return values. Provide a glue output since we consume one as
866  // input. This allows someone else to chain off us as needed.
867  SmallVector<EVT, 8> NodeTys;
868  for (auto SD : LoweredGCArgs) {
869  if (!LowerAsVReg.count(SD))
870  continue;
871  NodeTys.push_back(SD.getValueType());
872  }
873  LLVM_DEBUG(dbgs() << "Statepoint has " << NodeTys.size() << " results\n");
874  assert(NodeTys.size() == LowerAsVReg.size() && "Inconsistent GC Ptr lowering");
875  NodeTys.push_back(MVT::Other);
876  NodeTys.push_back(MVT::Glue);
877 
878  unsigned NumResults = NodeTys.size();
879  MachineSDNode *StatepointMCNode =
880  DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
881  DAG.setNodeMemRefs(StatepointMCNode, MemRefs);
882 
883  // For values lowered to tied-defs, create the virtual registers. Note that
884  // for simplicity, we *always* create a vreg even within a single block.
886  for (const auto *Relocate : SI.GCRelocates) {
887  Value *Derived = Relocate->getDerivedPtr();
888  SDValue SD = getValue(Derived);
889  if (!LowerAsVReg.count(SD))
890  continue;
891 
892  // Handle multiple gc.relocates of the same input efficiently.
893  if (VirtRegs.count(SD))
894  continue;
895 
896  SDValue Relocated = SDValue(StatepointMCNode, LowerAsVReg[SD]);
897 
898  auto *RetTy = Relocate->getType();
899  Register Reg = FuncInfo.CreateRegs(RetTy);
901  DAG.getDataLayout(), Reg, RetTy, None);
902  SDValue Chain = DAG.getRoot();
903  RFV.getCopyToRegs(Relocated, DAG, getCurSDLoc(), Chain, nullptr);
904  PendingExports.push_back(Chain);
905 
906  VirtRegs[SD] = Reg;
907  }
908 
909  // Record for later use how each relocation was lowered. This is needed to
910  // allow later gc.relocates to mirror the lowering chosen.
911  const Instruction *StatepointInstr = SI.StatepointInstr;
912  auto &RelocationMap = FuncInfo.StatepointRelocationMaps[StatepointInstr];
913  for (const GCRelocateInst *Relocate : SI.GCRelocates) {
914  const Value *V = Relocate->getDerivedPtr();
915  SDValue SDV = getValue(V);
917 
919  if (LowerAsVReg.count(SDV)) {
920  Record.type = RecordType::VReg;
921  assert(VirtRegs.count(SDV));
922  Record.payload.Reg = VirtRegs[SDV];
923  } else if (Loc.getNode()) {
924  Record.type = RecordType::Spill;
925  Record.payload.FI = cast<FrameIndexSDNode>(Loc)->getIndex();
926  } else {
928  // If we didn't relocate a value, we'll essentialy end up inserting an
929  // additional use of the original value when lowering the gc.relocate.
930  // We need to make sure the value is available at the new use, which
931  // might be in another block.
932  if (Relocate->getParent() != StatepointInstr->getParent())
934  }
935  RelocationMap[V] = Record;
936  }
937 
938 
939 
940  SDNode *SinkNode = StatepointMCNode;
941 
942  // Build the GC_TRANSITION_END node if necessary.
943  //
944  // See the comment above regarding GC_TRANSITION_START for the layout of
945  // the operands to the GC_TRANSITION_END node.
946  if (IsGCTransition) {
948 
949  // Add chain
950  TEOps.push_back(SDValue(StatepointMCNode, NumResults - 2));
951 
952  // Add GC transition arguments
953  for (const Value *V : SI.GCTransitionArgs) {
954  TEOps.push_back(getValue(V));
955  if (V->getType()->isPointerTy())
956  TEOps.push_back(DAG.getSrcValue(V));
957  }
958 
959  // Add glue
960  TEOps.push_back(SDValue(StatepointMCNode, NumResults - 1));
961 
963 
964  SDValue GCTransitionStart =
965  DAG.getNode(ISD::GC_TRANSITION_END, getCurSDLoc(), NodeTys, TEOps);
966 
967  SinkNode = GCTransitionStart.getNode();
968  }
969 
970  // Replace original call
971  // Call: ch,glue = CALL ...
972  // Statepoint: [gc relocates],ch,glue = STATEPOINT ...
973  unsigned NumSinkValues = SinkNode->getNumValues();
974  SDValue StatepointValues[2] = {SDValue(SinkNode, NumSinkValues - 2),
975  SDValue(SinkNode, NumSinkValues - 1)};
976  DAG.ReplaceAllUsesWith(CallNode, StatepointValues);
977  // Remove original call node
978  DAG.DeleteNode(CallNode);
979 
980  // Since we always emit CopyToRegs (even for local relocates), we must
981  // update root, so that they are emitted before any local uses.
982  (void)getControlRoot();
983 
984  // TODO: A better future implementation would be to emit a single variable
985  // argument, variable return value STATEPOINT node here and then hookup the
986  // return value of each gc.relocate to the respective output of the
987  // previously emitted STATEPOINT value. Unfortunately, this doesn't appear
988  // to actually be possible today.
989 
990  return ReturnVal;
991 }
992 
993 void
995  const BasicBlock *EHPadBB /*= nullptr*/) {
996  assert(I.getCallingConv() != CallingConv::AnyReg &&
997  "anyregcc is not supported on statepoints!");
998 
999 #ifndef NDEBUG
1000  // Check that the associated GCStrategy expects to encounter statepoints.
1002  "GCStrategy does not expect to encounter statepoints");
1003 #endif
1004 
1005  SDValue ActualCallee;
1006  SDValue Callee = getValue(I.getActualCalledOperand());
1007 
1008  if (I.getNumPatchBytes() > 0) {
1009  // If we've been asked to emit a nop sequence instead of a call instruction
1010  // for this statepoint then don't lower the call target, but use a constant
1011  // `undef` instead. Not lowering the call target lets statepoint clients
1012  // get away without providing a physical address for the symbolic call
1013  // target at link time.
1014  ActualCallee = DAG.getUNDEF(Callee.getValueType());
1015  } else {
1016  ActualCallee = Callee;
1017  }
1018 
1021  I.getNumCallArgs(), ActualCallee,
1022  I.getActualReturnType(), false /* IsPatchPoint */);
1023 
1024  // There may be duplication in the gc.relocate list; such as two copies of
1025  // each relocation on normal and exceptional path for an invoke. We only
1026  // need to spill once and record one copy in the stackmap, but we need to
1027  // reload once per gc.relocate. (Dedupping gc.relocates is trickier and best
1028  // handled as a CSE problem elsewhere.)
1029  // TODO: There a couple of major stackmap size optimizations we could do
1030  // here if we wished.
1031  // 1) If we've encountered a derived pair {B, D}, we don't need to actually
1032  // record {B,B} if it's seen later.
1033  // 2) Due to rematerialization, actual derived pointers are somewhat rare;
1034  // given that, we could change the format to record base pointer relocations
1035  // separately with half the space. This would require a format rev and a
1036  // fairly major rework of the STATEPOINT node though.
1037  SmallSet<SDValue, 8> Seen;
1038  for (const GCRelocateInst *Relocate : I.getGCRelocates()) {
1039  SI.GCRelocates.push_back(Relocate);
1040 
1041  SDValue DerivedSD = getValue(Relocate->getDerivedPtr());
1042  if (Seen.insert(DerivedSD).second) {
1043  SI.Bases.push_back(Relocate->getBasePtr());
1044  SI.Ptrs.push_back(Relocate->getDerivedPtr());
1045  }
1046  }
1047 
1048  // If we find a deopt value which isn't explicitly added, we need to
1049  // ensure it gets lowered such that gc cycles occurring before the
1050  // deoptimization event during the lifetime of the call don't invalidate
1051  // the pointer we're deopting with. Note that we assume that all
1052  // pointers passed to deopt are base pointers; relaxing that assumption
1053  // would require relatively large changes to how we represent relocations.
1054  for (Value *V : I.deopt_operands()) {
1055  if (!isGCValue(V, *this))
1056  continue;
1057  if (Seen.insert(getValue(V)).second) {
1058  SI.Bases.push_back(V);
1059  SI.Ptrs.push_back(V);
1060  }
1061  }
1062 
1063  SI.GCArgs = ArrayRef<const Use>(I.gc_args_begin(), I.gc_args_end());
1064  SI.StatepointInstr = &I;
1065  SI.ID = I.getID();
1066 
1067  SI.DeoptState = ArrayRef<const Use>(I.deopt_begin(), I.deopt_end());
1068  SI.GCTransitionArgs = ArrayRef<const Use>(I.gc_transition_args_begin(),
1069  I.gc_transition_args_end());
1070 
1071  SI.StatepointFlags = I.getFlags();
1072  SI.NumPatchBytes = I.getNumPatchBytes();
1073  SI.EHPadBB = EHPadBB;
1074 
1075  SDValue ReturnValue = LowerAsSTATEPOINT(SI);
1076 
1077  // Export the result value if needed
1078  const std::pair<bool, bool> GCResultLocality = I.getGCResultLocality();
1079  Type *RetTy = I.getActualReturnType();
1080 
1081  if (RetTy->isVoidTy() ||
1082  (!GCResultLocality.first && !GCResultLocality.second)) {
1083  // The return value is not needed, just generate a poison value.
1085  return;
1086  }
1087 
1088  if (GCResultLocality.first) {
1089  // Result value will be used in a same basic block. Don't export it or
1090  // perform any explicit register copies. The gc_result will simply grab
1091  // this value.
1092  setValue(&I, ReturnValue);
1093  }
1094 
1095  if (!GCResultLocality.second)
1096  return;
1097  // Result value will be used in a different basic block so we need to export
1098  // it now. Default exporting mechanism will not work here because statepoint
1099  // call has a different type than the actual call. It means that by default
1100  // llvm will create export register of the wrong type (always i32 in our
1101  // case). So instead we need to create export register with correct type
1102  // manually.
1103  // TODO: To eliminate this problem we can remove gc.result intrinsics
1104  // completely and make statepoint call to return a tuple.
1105  unsigned Reg = FuncInfo.CreateRegs(RetTy);
1107  DAG.getDataLayout(), Reg, RetTy,
1108  I.getCallingConv());
1109  SDValue Chain = DAG.getEntryNode();
1110 
1111  RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
1112  PendingExports.push_back(Chain);
1113  FuncInfo.ValueMap[&I] = Reg;
1114 }
1115 
1117  const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB,
1118  bool VarArgDisallowed, bool ForceVoidReturnTy) {
1120  unsigned ArgBeginIndex = Call->arg_begin() - Call->op_begin();
1122  SI.CLI, Call, ArgBeginIndex, Call->getNumArgOperands(), Callee,
1123  ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : Call->getType(),
1124  false);
1125  if (!VarArgDisallowed)
1126  SI.CLI.IsVarArg = Call->getFunctionType()->isVarArg();
1127 
1128  auto DeoptBundle = *Call->getOperandBundle(LLVMContext::OB_deopt);
1129 
1130  unsigned DefaultID = StatepointDirectives::DeoptBundleStatepointID;
1131 
1132  auto SD = parseStatepointDirectivesFromAttrs(Call->getAttributes());
1133  SI.ID = SD.StatepointID.getValueOr(DefaultID);
1134  SI.NumPatchBytes = SD.NumPatchBytes.getValueOr(0);
1135 
1136  SI.DeoptState =
1137  ArrayRef<const Use>(DeoptBundle.Inputs.begin(), DeoptBundle.Inputs.end());
1138  SI.StatepointFlags = static_cast<uint64_t>(StatepointFlags::None);
1139  SI.EHPadBB = EHPadBB;
1140 
1141  // NB! The GC arguments are deliberately left empty.
1142 
1143  if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) {
1144  ReturnVal = lowerRangeToAssertZExt(DAG, *Call, ReturnVal);
1145  setValue(Call, ReturnVal);
1146  }
1147 }
1148 
1150  const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB) {
1152  /* VarArgDisallowed = */ false,
1153  /* ForceVoidReturnTy = */ false);
1154 }
1155 
1156 void SelectionDAGBuilder::visitGCResult(const GCResultInst &CI) {
1157  // The result value of the gc_result is simply the result of the actual
1158  // call. We've already emitted this, so just grab the value.
1159  const GCStatepointInst *SI = CI.getStatepoint();
1160 
1161  if (SI->getParent() == CI.getParent()) {
1162  setValue(&CI, getValue(SI));
1163  return;
1164  }
1165  // Statepoint is in different basic block so we should have stored call
1166  // result in a virtual register.
1167  // We can not use default getValue() functionality to copy value from this
1168  // register because statepoint and actual call return types can be
1169  // different, and getValue() will use CopyFromReg of the wrong type,
1170  // which is always i32 in our case.
1171  Type *RetTy = SI->getActualReturnType();
1173 
1174  assert(CopyFromReg.getNode());
1175  setValue(&CI, CopyFromReg);
1176 }
1177 
1178 void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
1179 #ifndef NDEBUG
1180  // Consistency check
1181  // We skip this check for relocates not in the same basic block as their
1182  // statepoint. It would be too expensive to preserve validation info through
1183  // different basic blocks.
1184  if (Relocate.getStatepoint()->getParent() == Relocate.getParent())
1186 
1187  auto *Ty = Relocate.getType()->getScalarType();
1188  if (auto IsManaged = GFI->getStrategy().isGCManagedPointer(Ty))
1189  assert(*IsManaged && "Non gc managed pointer relocated!");
1190 #endif
1191 
1192  const Value *DerivedPtr = Relocate.getDerivedPtr();
1193  auto &RelocationMap =
1195  auto SlotIt = RelocationMap.find(DerivedPtr);
1196  assert(SlotIt != RelocationMap.end() && "Relocating not lowered gc value");
1197  const RecordType &Record = SlotIt->second;
1198 
1199  // If relocation was done via virtual register..
1200  if (Record.type == RecordType::VReg) {
1201  Register InReg = Record.payload.Reg;
1203  DAG.getDataLayout(), InReg, Relocate.getType(),
1204  None); // This is not an ABI copy.
1205  // We generate copy to/from regs even for local uses, hence we must
1206  // chain with current root to ensure proper ordering of copies w.r.t.
1207  // statepoint.
1208  SDValue Chain = DAG.getRoot();
1209  SDValue Relocation = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
1210  Chain, nullptr, nullptr);
1211  setValue(&Relocate, Relocation);
1212  return;
1213  }
1214 
1215  if (Record.type == RecordType::Spill) {
1216  unsigned Index = Record.payload.FI;
1218 
1219  // All the reloads are independent and are reading memory only modified by
1220  // statepoints (i.e. no other aliasing stores); informing SelectionDAG of
1221  // this this let's CSE kick in for free and allows reordering of
1222  // instructions if possible. The lowering for statepoint sets the root,
1223  // so this is ordering all reloads with the either
1224  // a) the statepoint node itself, or
1225  // b) the entry of the current block for an invoke statepoint.
1226  const SDValue Chain = DAG.getRoot(); // != Builder.getRoot()
1227 
1228  auto &MF = DAG.getMachineFunction();
1229  auto &MFI = MF.getFrameInfo();
1230  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, Index);
1231  auto *LoadMMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
1232  MFI.getObjectSize(Index),
1233  MFI.getObjectAlign(Index));
1234 
1236  Relocate.getType());
1237 
1238  SDValue SpillLoad =
1239  DAG.getLoad(LoadVT, getCurSDLoc(), Chain, SpillSlot, LoadMMO);
1240  PendingLoads.push_back(SpillLoad.getValue(1));
1241 
1242  assert(SpillLoad.getNode());
1243  setValue(&Relocate, SpillLoad);
1244  return;
1245  }
1246 
1248  SDValue SD = getValue(DerivedPtr);
1249 
1250  if (SD.isUndef() && SD.getValueType().getSizeInBits() <= 64) {
1251  // Lowering relocate(undef) as arbitrary constant. Current constant value
1252  // is chosen such that it's unlikely to be a valid pointer.
1253  setValue(&Relocate, DAG.getTargetConstant(0xFEFEFEFE, SDLoc(SD), MVT::i64));
1254  return;
1255  }
1256 
1257  // We didn't need to spill these special cases (constants and allocas).
1258  // See the handling in spillIncomingValueForStatepoint for detail.
1259  setValue(&Relocate, SD);
1260 }
1261 
1263  const auto &TLI = DAG.getTargetLoweringInfo();
1264  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE),
1265  TLI.getPointerTy(DAG.getDataLayout()));
1266 
1267  // We don't lower calls to __llvm_deoptimize as varargs, but as a regular
1268  // call. We also do not lower the return value to any virtual register, and
1269  // change the immediately following return to a trap instruction.
1270  LowerCallSiteWithDeoptBundleImpl(CI, Callee, /* EHPadBB = */ nullptr,
1271  /* VarArgDisallowed = */ true,
1272  /* ForceVoidReturnTy = */ true);
1273 }
1274 
1276  // We do not lower the return value from llvm.deoptimize to any virtual
1277  // register, and change the immediately following return to a trap
1278  // instruction.
1280  DAG.setRoot(
1282 }
llvm::SelectionDAGBuilder::LowerStatepoint
void LowerStatepoint(const GCStatepointInst &I, const BasicBlock *EHPadBB=nullptr)
Definition: StatepointLowering.cpp:994
i
i
Definition: README.txt:29
llvm::Argument
This class represents an incoming formal argument to a Function.
Definition: Argument.h:29
llvm::SmallBitVector::set
SmallBitVector & set()
Definition: SmallBitVector.h:363
pushStackMapConstant
static void pushStackMapConstant(SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder, uint64_t Value)
Definition: StatepointLowering.cpp:77
llvm::ConstantSDNode
Definition: SelectionDAGNodes.h:1542
llvm::SDValue::dump
void dump() const
Definition: SelectionDAGNodes.h:1165
llvm::SDUse
Represents a use of a SDNode.
Definition: SelectionDAGNodes.h:277
llvm
---------------------— PointerInfo ------------------------------------—
Definition: AllocatorList.h:23
Reg
unsigned Reg
Definition: MachineSink.cpp:1566
GCMetadata.h
Optional.h
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1078
llvm::GCStrategy
GCStrategy describes a garbage collector algorithm's code generation requirements,...
Definition: GCStrategy.h:66
llvm::StatepointLoweringState::startNewStatepoint
void startNewStatepoint(SelectionDAGBuilder &Builder)
Reset all state tracking for a newly encountered safepoint.
Definition: StatepointLowering.cpp:85
llvm::SelectionDAGBuilder::setValue
void setValue(const Value *V, SDValue NewN)
Definition: SelectionDAGBuilder.h:528
llvm::Type::isPointerTy
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:228
llvm::SelectionDAGBuilder::PendingLoads
SmallVector< SDValue, 8 > PendingLoads
Loads are not emitted to the program immediately.
Definition: SelectionDAGBuilder.h:130
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:152
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::size
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:77
llvm::BitCastInst
This class represents a no-op cast from one type to another.
Definition: Instructions.h:5194
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:319
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
Statistic.h
llvm::FunctionLoweringInfo::StatepointRelocationRecord
Helper object to track which of three possible relocation mechanisms are used for a particular value ...
Definition: FunctionLoweringInfo.h:96
llvm::LandingPadInst
The landingpad instruction holds all of the information necessary to generate correct exception handl...
Definition: Instructions.h:2879
llvm::SelectionDAG::getVTList
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
Definition: SelectionDAG.cpp:8157
llvm::MachineFunction::getMachineMemOperand
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Definition: MachineFunction.cpp:430
llvm::MachineSDNode
An SDNode that represents everything that will be needed to construct a MachineInstr.
Definition: SelectionDAGNodes.h:2530
llvm::SelectionDAGBuilder::getValue
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
Definition: SelectionDAGBuilder.cpp:1477
llvm::SelectionDAG::getRoot
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:513
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:455
llvm::SelectionDAG::ReplaceAllUsesWith
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
Definition: SelectionDAG.cpp:9029
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:143
llvm::MVT::Glue
@ Glue
Definition: MachineValueType.h:262
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
findPreviousSpillSlot
static Optional< int > findPreviousSpillSlot(const Value *Val, SelectionDAGBuilder &Builder, int LookUpDepth)
Utility function for reservePreviousStackSlotForValue.
Definition: StatepointLowering.cpp:159
llvm::SelectionDAGBuilder::LowerCallSiteWithDeoptBundle
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
Definition: StatepointLowering.cpp:1149
llvm::MachineMemOperand
A description of a memory reference used in the backend.
Definition: MachineMemOperand.h:128
llvm::TargetOptions::TrapUnreachable
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
Definition: TargetOptions.h:252
llvm::FunctionLoweringInfo::ValueMap
DenseMap< const Value *, Register > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
Definition: FunctionLoweringInfo.h:78
llvm::SmallSet
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:134
llvm::GCFunctionInfo::getStrategy
GCStrategy & getStrategy()
getStrategy - Return the GC strategy for the function.
Definition: GCMetadata.h:108
llvm::Optional< int >
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::count
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:145
Offset
uint64_t Offset
Definition: ELFObjHandler.cpp:81
STLExtras.h
llvm::SmallBitVector::test
bool test(unsigned Idx) const
Definition: SmallBitVector.h:463
llvm::StatepointLoweringState::allocateStackSlot
SDValue allocateStackSlot(EVT ValueType, SelectionDAGBuilder &Builder)
Get a stack slot we can use to store an value of type ValueType.
Definition: StatepointLowering.cpp:106
SelectionDAG.h
llvm::ISD::GC_TRANSITION_END
@ GC_TRANSITION_END
Definition: ISDOpcodes.h:1187
llvm::ISD::CALLSEQ_END
@ CALLSEQ_END
Definition: ISDOpcodes.h:1027
llvm::FunctionLoweringInfo::StatepointRelocationRecord::VReg
@ VReg
Definition: FunctionLoweringInfo.h:104
llvm::SelectionDAG::getContext
LLVMContext * getContext() const
Definition: SelectionDAG.h:447
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:122
llvm::RegsForValue::getCopyToRegs
void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl, SDValue &Chain, SDValue *Flag, const Value *V=nullptr, ISD::NodeType PreferredExtendType=ISD::ANY_EXTEND) const
Emit a series of CopyToReg nodes that copies the specified value into the registers specified by this...
Definition: SelectionDAGBuilder.cpp:899
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
MachineValueType.h
llvm::Optional::hasValue
constexpr bool hasValue() const
Definition: Optional.h:288
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::SDNode::op_end
op_iterator op_end() const
Definition: SelectionDAGNodes.h:904
llvm::GCStatepointInst
Represents a gc.statepoint intrinsic call.
Definition: Statepoint.h:63
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:206
llvm::ISD::GC_TRANSITION_START
@ GC_TRANSITION_START
GC_TRANSITION_START/GC_TRANSITION_END - These operators mark the beginning and end of GC transition s...
Definition: ISDOpcodes.h:1186
Instruction.h
CommandLine.h
TargetLowering.h
llvm::SelectionDAG::getLoad
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
Definition: SelectionDAG.cpp:7436
llvm::SDNode::getOpcode
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
Definition: SelectionDAGNodes.h:621
llvm::SelectionDAGBuilder::DAG
SelectionDAG & DAG
Definition: SelectionDAGBuilder.h:389
llvm::SelectionDAG::getTargetFrameIndex
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:688
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1113
TargetMachine.h
llvm::SelectionDAGBuilder::lowerRangeToAssertZExt
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)
Definition: SelectionDAGBuilder.cpp:8996
FunctionLoweringInfo.h
llvm::SmallVectorImpl::append
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:648
llvm::SelectionDAG::getUNDEF
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:951
llvm::SelectionDAG::getTargetLoweringInfo
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:443
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:35
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::SelectionDAG::getSrcValue
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
Definition: SelectionDAG.cpp:2033
llvm::StatepointDirectives::DeoptBundleStatepointID
static const uint64_t DeoptBundleStatepointID
Definition: Statepoint.h:260
UseRegistersForGCPointersInLandingPad
cl::opt< bool > UseRegistersForGCPointersInLandingPad("use-registers-for-gc-values-in-landing-pad", cl::Hidden, cl::init(false), cl::desc("Allow using registers for gc pointer in landing pad"))
reservePreviousStackSlotForValue
static void reservePreviousStackSlotForValue(const Value *IncomingValue, SelectionDAGBuilder &Builder)
Try to find existing copies of the incoming values in stack slots used for statepoint spilling.
Definition: StatepointLowering.cpp:262
TargetOpcodes.h
llvm::SelectionDAG::setRoot
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:522
getMachineMemOperand
static MachineMemOperand * getMachineMemOperand(MachineFunction &MF, FrameIndexSDNode &FI)
Definition: StatepointLowering.cpp:349
llvm::Instruction
Definition: Instruction.h:45
MaxRegistersForGCPointers
cl::opt< unsigned > MaxRegistersForGCPointers("max-registers-for-gc-values", cl::Hidden, cl::init(0), cl::desc("Max number of VRegs allowed to pass GC pointer meta args in"))
llvm::STATISTIC
STATISTIC(NumFunctions, "Total number of functions")
llvm::FrameIndexSDNode
Definition: SelectionDAGNodes.h:1726
llvm::SelectionDAGBuilder::LowerDeoptimizeCall
void LowerDeoptimizeCall(const CallInst *CI)
Definition: StatepointLowering.cpp:1262
llvm::MCID::Call
@ Call
Definition: MCInstrDesc.h:153
llvm::SDValue::getValueSizeInBits
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
Definition: SelectionDAGNodes.h:192
llvm::ISD::CopyFromReg
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:208
llvm::None
const NoneType None
Definition: None.h:23
Statepoint.h
Type.h
llvm::RegsForValue
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
Definition: SelectionDAGBuilder.h:836
llvm::StatepointLoweringState::relocCallVisited
void relocCallVisited(const GCRelocateInst &RelocCall)
Remove this gc_relocate from the list we're expecting to see before the next statepoint.
Definition: StatepointLowering.h:75
llvm::cl::opt< bool >
llvm::StatepointLoweringState::clear
void clear()
Clear the memory usage of this object.
Definition: StatepointLowering.cpp:98
llvm::SmallBitVector::clear
void clear()
Clear all bits.
Definition: SmallBitVector.h:322
llvm::SmallSet::count
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:164
llvm::GCRelocateInst
Represents calls to the gc.relocate intrinsic.
Definition: IntrinsicInst.h:1256
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:341
llvm::StatepointFlags::None
@ None
Index
uint32_t Index
Definition: ELFObjHandler.cpp:84
llvm::MachineFrameInfo::getObjectSize
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
Definition: MachineFrameInfo.h:451
RuntimeLibcalls.h
llvm::ConstantFPSDNode
Definition: SelectionDAGNodes.h:1587
llvm::find
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1567
llvm::ISD::LOAD
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:921
llvm::SelectionDAG::getIntPtrConstant
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
Definition: SelectionDAG.cpp:1471
willLowerDirectly
static bool willLowerDirectly(SDValue Incoming)
Return true if-and-only-if the given SDValue can be lowered as either a constant argument or a stack ...
Definition: StatepointLowering.cpp:240
llvm::GCRelocateInst::getDerivedPtr
Value * getDerivedPtr() const
Definition: IntrinsicInst.cpp:572
llvm::ISD::TRAP
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1088
llvm::DenseMap
Definition: DenseMap.h:714
llvm::SelectionDAGBuilder::getCopyFromRegs
SDValue getCopyFromRegs(const Value *V, Type *Ty)
If there was virtual register allocated for the value V emit CopyFromReg of the specified type Ty.
Definition: SelectionDAGBuilder.cpp:1457
llvm::SDNode::getOperand
const SDValue & getOperand(unsigned Num) const
Definition: SelectionDAGNodes.h:896
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::FrameIndexSDNode::getIndex
int getIndex() const
Definition: SelectionDAGNodes.h:1737
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:7922
llvm::MachineFrameInfo::getObjectAlign
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
Definition: MachineFrameInfo.h:465
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
spillIncomingStatepointValue
static std::tuple< SDValue, SDValue, MachineMemOperand * > spillIncomingStatepointValue(SDValue Incoming, SDValue Chain, SelectionDAGBuilder &Builder)
Spill a value incoming to the statepoint.
Definition: StatepointLowering.cpp:367
ArrayRef.h
llvm::SelectionDAGBuilder::LowerAsSTATEPOINT
SDValue LowerAsSTATEPOINT(StatepointLoweringInfo &SI)
Lower SLI into a STATEPOINT instruction.
Definition: StatepointLowering.cpp:725
llvm::SelectionDAGBuilder::StatepointLowering
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
Definition: SelectionDAGBuilder.h:134
llvm::SDValue::getValue
SDValue getValue(unsigned R) const
Definition: SelectionDAGNodes.h:172
llvm::TargetMachine::Options
TargetOptions Options
Definition: TargetMachine.h:115
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::MVT::Other
@ Other
Definition: MachineValueType.h:42
llvm::SelectionDAGBuilder::getFrameIndexTy
MVT getFrameIndexTy()
Returns the type of FrameIndex and TargetFrameIndex nodes.
Definition: SelectionDAGBuilder.h:640
llvm::MachineFunction::getFrameInfo
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Definition: MachineFunction.h:638
SI
StandardInstrumentations SI(Debug, VerifyEach)
llvm::FunctionLoweringInfo::CreateRegs
Register CreateRegs(const Value *V)
Definition: FunctionLoweringInfo.cpp:406
llvm::SelectionDAG::getMachineNode
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
Definition: SelectionDAG.cpp:8595
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:138
llvm::Record
Definition: Record.h:1472
llvm::SelectionDAG::setNodeMemRefs
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
Definition: SelectionDAG.cpp:8363
llvm::SmallBitVector::resize
void resize(unsigned N, bool t=false)
Grow or shrink the bitvector.
Definition: SmallBitVector.h:329
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::SelectionDAGBuilder::StatepointLoweringInfo
Describes a gc.statepoint or a gc.statepoint like thing for the purposes of lowering into a STATEPOIN...
Definition: SelectionDAGBuilder.h:578
llvm::MachineFunction
Definition: MachineFunction.h:230
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::insert
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:141
llvm::StatepointLoweringState::getLocation
SDValue getLocation(SDValue Val)
Returns the spill location of a value incoming to the current statepoint.
Definition: StatepointLowering.h:50
TargetOptions.h
StatepointLowering.h
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::EVT::isVector
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:155
None.h
llvm::LLVMContext::OB_deopt
@ OB_deopt
Definition: LLVMContext.h:90
llvm::MVT::i64
@ i64
Definition: MachineValueType.h:47
llvm::SelectionDAGBuilder
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
Definition: SelectionDAGBuilder.h:93
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:136
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:256
llvm::GCResultInst
Represents calls to the gc.result intrinsic.
Definition: IntrinsicInst.h:1284
llvm::append_range
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
Definition: STLExtras.h:1744
S
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
Definition: README.txt:210
llvm::SelectionDAGBuilder::getCurSDLoc
SDLoc getCurSDLoc() const
Definition: SelectionDAGBuilder.h:477
llvm::SmallSet::insert
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:180
llvm::FunctionLoweringInfo::StatepointRelocationMaps
DenseMap< const Instruction *, StatepointSpillMapTy > StatepointRelocationMaps
Definition: FunctionLoweringInfo.h:120
llvm::StatepointFlags::GCTransition
@ GCTransition
Indicates that this statepoint is a transition from GC-aware code to code that is not GC-aware.
llvm::SDVTList
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Definition: SelectionDAGNodes.h:79
llvm::MachineMemOperand::MOVolatile
@ MOVolatile
The memory access is volatile.
Definition: MachineMemOperand.h:139
lowerIncomingStatepointValue
static void lowerIncomingStatepointValue(SDValue Incoming, bool RequireSpillSlot, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< MachineMemOperand * > &MemRefs, SelectionDAGBuilder &Builder)
Lower a single value incoming to a statepoint node.
Definition: StatepointLowering.cpp:415
llvm::MachineMemOperand::MOLoad
@ MOLoad
The memory access reads data.
Definition: MachineMemOperand.h:135
llvm::PointerUnion< const Value *, const PseudoSourceValue * >
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
Callee
amdgpu Simplify well known AMD library false FunctionCallee Callee
Definition: AMDGPULibCalls.cpp:206
llvm::GCStrategy::isGCManagedPointer
virtual Optional< bool > isGCManagedPointer(const Type *Ty) const
If the type specified can be reliably distinguished, returns true for pointers to GC managed location...
Definition: GCStrategy.h:99
CallingConv.h
isGCValue
static bool isGCValue(const Value *V, SelectionDAGBuilder &Builder)
Return true if value V represents the GC value.
Definition: StatepointLowering.cpp:493
llvm::SDNode::getNumOperands
unsigned getNumOperands() const
Return the number of values used by this operation.
Definition: SelectionDAGNodes.h:883
llvm::FunctionLoweringInfo::StatepointRelocationRecord::Spill
@ Spill
Definition: FunctionLoweringInfo.h:101
MachineFrameInfo.h
llvm::SelectionDAG::getEntryNode
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:516
llvm::SelectionDAG::getDataLayout
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:440
ISDOpcodes.h
lowerCallFromStatepointLoweringInfo
static std::pair< SDValue, SDNode * > lowerCallFromStatepointLoweringInfo(SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder, SmallVectorImpl< SDValue > &PendingExports)
Extract call from statepoint, lower it and return pointer to the call node.
Definition: StatepointLowering.cpp:312
Casting.h
llvm::SelectionDAGBuilder::FuncInfo
FunctionLoweringInfo & FuncInfo
Information about the function as a whole.
Definition: SelectionDAGBuilder.h:422
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::size
unsigned size() const
Definition: DenseMap.h:100
llvm::MVT::i32
@ i32
Definition: MachineValueType.h:46
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:138
StackMaps.h
llvm::SDNode::getNumValues
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Definition: SelectionDAGNodes.h:955
llvm::SelectionDAGBuilder::LowerCallSiteWithDeoptBundleImpl
void LowerCallSiteWithDeoptBundleImpl(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB, bool VarArgDisallowed, bool ForceVoidReturnTy)
Definition: StatepointLowering.cpp:1116
llvm::MachineMemOperand::MOStore
@ MOStore
The memory access writes data.
Definition: MachineMemOperand.h:137
llvm::SDValue::isUndef
bool isUndef() const
Definition: SelectionDAGNodes.h:1149
SelectionDAGBuilder.h
llvm::FunctionLoweringInfo::StatepointRelocationRecord::NoRelocate
@ NoRelocate
Definition: FunctionLoweringInfo.h:99
llvm::Type::getVoidTy
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:186
llvm::SDNode::op_begin
op_iterator op_begin() const
Definition: SelectionDAGNodes.h:903
llvm::MachineFrameInfo
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
Definition: MachineFrameInfo.h:107
llvm::StatepointLoweringState::scheduleRelocCall
void scheduleRelocCall(const GCRelocateInst &RelocCall)
Record the fact that we expect to encounter a given gc_relocate before the next statepoint.
Definition: StatepointLowering.h:66
Instructions.h
llvm::GCStrategy::useStatepoints
bool useStatepoints() const
Returns true if this strategy is expecting the use of gc.statepoints, and false otherwise.
Definition: GCStrategy.h:90
llvm::parseStatepointDirectivesFromAttrs
StatepointDirectives parseStatepointDirectivesFromAttrs(AttributeList AS)
Parse out statepoint directives from the function attributes present in AS.
Definition: Statepoint.cpp:26
llvm::MachinePointerInfo::getFixedStack
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Definition: MachineOperand.cpp:1003
UseRegistersForDeoptValues
cl::opt< bool > UseRegistersForDeoptValues("use-registers-for-deopt-values", cl::Hidden, cl::init(false), cl::desc("Allow using registers for non pointer deopt args"))
llvm::GCStatepointInst::CallArgsBeginPos
@ CallArgsBeginPos
Definition: Statepoint.h:85
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:94
llvm::SelectionDAGBuilder::LowerDeoptimizingReturn
void LowerDeoptimizingReturn()
Definition: StatepointLowering.cpp:1275
llvm::CallingConv::AnyReg
@ AnyReg
Definition: CallingConv.h:62
llvm::MachineFrameInfo::markAsStatepointSpillSlotObjectIndex
void markAsStatepointSpillSlotObjectIndex(int ObjectIdx)
Definition: MachineFrameInfo.h:725
llvm::PHINode
Definition: Instructions.h:2627
MachineMemOperand.h
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:43
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1161
llvm::SelectionDAG::getTargetConstant
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:637
DerivedTypes.h
llvm::SmallSetVector
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:307
llvm::StatepointFlags::DeoptLiveIn
@ DeoptLiveIn
Mark the deopt arguments associated with the statepoint as only being "live-in".
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1475
llvm::SelectionDAG::DeleteNode
void DeleteNode(SDNode *N)
Remove the specified node from the system.
Definition: SelectionDAG.cpp:867
llvm::SelectionDAG::getMachineFunction
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:437
llvm::SelectionDAG::getExternalSymbol
SDValue getExternalSymbol(const char *Sym, EVT VT)
Definition: SelectionDAG.cpp:1711
LLVMContext.h
llvm::SmallBitVector::size
size_t size() const
Returns the number of bits in this bitvector.
Definition: SmallBitVector.h:192
llvm::cl::desc
Definition: CommandLine.h:414
llvm::StatepointFlags::MaskAll
@ MaskAll
A bitmask that includes all valid flags.
llvm::TargetLoweringBase::getValueType
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
Definition: TargetLowering.h:1408
MachineFunction.h
llvm::Value
LLVM Value Representation.
Definition: Value.h:75
llvm::SelectionDAG::getTarget
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:441
llvm::SelectionDAGBuilder::getRoot
SDValue getRoot()
Similar to getMemoryRoot, but also flushes PendingConstrainedFP(Strict) items.
Definition: SelectionDAGBuilder.cpp:1085
llvm::SelectionDAGBuilder::ExportFromCurrentBlock
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block,...
Definition: SelectionDAGBuilder.cpp:2064
llvm::SelectionDAGBuilder::populateCallLoweringInfo
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, bool IsPatchPoint)
Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.
Definition: SelectionDAGBuilder.cpp:9040
llvm::GCProjectionInst::getStatepoint
const GCStatepointInst * getStatepoint() const
The statepoint with which this gc.relocate is associated.
Definition: IntrinsicInst.cpp:547
llvm::SDNode::getGluedNode
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
Definition: SelectionDAGNodes.h:931
llvm::sampleprof::Base
@ Base
Definition: Discriminator.h:58
llvm::SelectionDAGBuilder::getControlRoot
SDValue getControlRoot()
Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports (...
Definition: SelectionDAGBuilder.cpp:1101
SmallSet.h
lowerStatepointMetaArgs
static void lowerStatepointMetaArgs(SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< MachineMemOperand * > &MemRefs, SmallVectorImpl< SDValue > &GCPtrs, DenseMap< SDValue, int > &LowerAsVReg, SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder)
Lower deopt state and gc pointer arguments of the statepoint.
Definition: StatepointLowering.cpp:511
GCStrategy.h
RecordType
FunctionLoweringInfo::StatepointRelocationRecord RecordType
Definition: StatepointLowering.cpp:75
llvm::SmallVectorImpl::insert
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:773
llvm::SelectionDAGBuilder::GFI
GCFunctionInfo * GFI
Garbage collection metadata for the function.
Definition: SelectionDAGBuilder.h:428