LLVM  8.0.0svn
FastISel.cpp
Go to the documentation of this file.
1 //===- FastISel.cpp - Implementation of the FastISel class ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains the implementation of the FastISel class.
11 //
12 // "Fast" instruction selection is designed to emit very poor code quickly.
13 // Also, it is not designed to be able to do much lowering, so most illegal
14 // types (e.g. i64 on 32-bit targets) and operations are not supported. It is
15 // also not intended to be able to do much optimization, except in a few cases
16 // where doing optimizations reduces overall compile time. For example, folding
17 // constants into immediate fields is often done, because it's cheap and it
18 // reduces the number of instructions later phases have to examine.
19 //
20 // "Fast" instruction selection is able to fail gracefully and transfer
21 // control to the SelectionDAG selector for operations that it doesn't
22 // support. In many cases, this allows us to avoid duplicating a lot of
23 // the complicated lowering logic that SelectionDAG currently has.
24 //
25 // The intended use for "fast" instruction selection is "-O0" mode
26 // compilation, where the quality of the generated code is irrelevant when
27 // weighed against the speed at which the code can be generated. Also,
28 // at -O0, the LLVM optimizers are not running, and this makes the
29 // compile time of codegen a much higher portion of the overall compile
30 // time. Despite its limitations, "fast" instruction selection is able to
31 // handle enough code on its own to provide noticeable overall speedups
32 // in -O0 compiles.
33 //
34 // Basic operations are supported in a target-independent way, by reading
35 // the same instruction descriptions that the SelectionDAG selector reads,
36 // and identifying simple arithmetic operations that can be directly selected
37 // from simple operators. More complicated operations currently require
38 // target-specific code.
39 //
40 //===----------------------------------------------------------------------===//
41 
42 #include "llvm/CodeGen/FastISel.h"
43 #include "llvm/ADT/APFloat.h"
44 #include "llvm/ADT/APSInt.h"
45 #include "llvm/ADT/DenseMap.h"
46 #include "llvm/ADT/Optional.h"
47 #include "llvm/ADT/SmallPtrSet.h"
48 #include "llvm/ADT/SmallString.h"
49 #include "llvm/ADT/SmallVector.h"
50 #include "llvm/ADT/Statistic.h"
53 #include "llvm/CodeGen/Analysis.h"
64 #include "llvm/CodeGen/StackMaps.h"
69 #include "llvm/IR/Argument.h"
70 #include "llvm/IR/Attributes.h"
71 #include "llvm/IR/BasicBlock.h"
72 #include "llvm/IR/CallSite.h"
73 #include "llvm/IR/CallingConv.h"
74 #include "llvm/IR/Constant.h"
75 #include "llvm/IR/Constants.h"
76 #include "llvm/IR/DataLayout.h"
77 #include "llvm/IR/DebugInfo.h"
78 #include "llvm/IR/DebugLoc.h"
79 #include "llvm/IR/DerivedTypes.h"
80 #include "llvm/IR/Function.h"
82 #include "llvm/IR/GlobalValue.h"
83 #include "llvm/IR/InlineAsm.h"
84 #include "llvm/IR/InstrTypes.h"
85 #include "llvm/IR/Instruction.h"
86 #include "llvm/IR/Instructions.h"
87 #include "llvm/IR/IntrinsicInst.h"
88 #include "llvm/IR/LLVMContext.h"
89 #include "llvm/IR/Mangler.h"
90 #include "llvm/IR/Metadata.h"
91 #include "llvm/IR/Operator.h"
92 #include "llvm/IR/Type.h"
93 #include "llvm/IR/User.h"
94 #include "llvm/IR/Value.h"
95 #include "llvm/MC/MCContext.h"
96 #include "llvm/MC/MCInstrDesc.h"
97 #include "llvm/MC/MCRegisterInfo.h"
98 #include "llvm/Support/Casting.h"
99 #include "llvm/Support/Debug.h"
102 #include "llvm/Support/MathExtras.h"
106 #include <algorithm>
107 #include <cassert>
108 #include <cstdint>
109 #include <iterator>
110 #include <utility>
111 
112 using namespace llvm;
113 
114 #define DEBUG_TYPE "isel"
115 
116 // FIXME: Remove this after the feature has proven reliable.
117 static cl::opt<bool> SinkLocalValues("fast-isel-sink-local-values",
118  cl::init(true), cl::Hidden,
119  cl::desc("Sink local values in FastISel"));
120 
121 STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by "
122  "target-independent selector");
123 STATISTIC(NumFastIselSuccessTarget, "Number of insts selected by "
124  "target-specific selector");
125 STATISTIC(NumFastIselDead, "Number of dead insts removed on failure");
126 
127 /// Set the current block to which generated machine instructions will be
128 /// appended.
131  "local values should be cleared after finishing a BB");
132 
133  // Instructions are appended to FuncInfo.MBB. If the basic block already
134  // contains labels or copies, use the last instruction as the last local
135  // value.
136  EmitStartPt = nullptr;
137  if (!FuncInfo.MBB->empty())
138  EmitStartPt = &FuncInfo.MBB->back();
140 }
141 
142 /// Flush the local CSE map and sink anything we can.
143 void FastISel::finishBasicBlock() { flushLocalValueMap(); }
144 
147  // Fallback to SDISel argument lowering code to deal with sret pointer
148  // parameter.
149  return false;
150 
151  if (!fastLowerArguments())
152  return false;
153 
154  // Enter arguments into ValueMap for uses in non-entry BBs.
156  E = FuncInfo.Fn->arg_end();
157  I != E; ++I) {
159  assert(VI != LocalValueMap.end() && "Missed an argument?");
160  FuncInfo.ValueMap[&*I] = VI->second;
161  }
162  return true;
163 }
164 
165 /// Return the defined register if this instruction defines exactly one
166 /// virtual register and uses no other virtual registers. Otherwise return 0.
168  unsigned RegDef = 0;
169  for (const MachineOperand &MO : MI.operands()) {
170  if (!MO.isReg())
171  continue;
172  if (MO.isDef()) {
173  if (RegDef)
174  return 0;
175  RegDef = MO.getReg();
176  } else if (TargetRegisterInfo::isVirtualRegister(MO.getReg())) {
177  // This is another use of a vreg. Don't try to sink it.
178  return 0;
179  }
180  }
181  return RegDef;
182 }
183 
184 void FastISel::flushLocalValueMap() {
185  // Try to sink local values down to their first use so that we can give them a
186  // better debug location. This has the side effect of shrinking local value
187  // live ranges, which helps out fast regalloc.
189  // Sink local value materialization instructions between EmitStartPt and
190  // LastLocalValue. Visit them bottom-up, starting from LastLocalValue, to
191  // avoid inserting into the range that we're iterating over.
194  : FuncInfo.MBB->rend();
195  MachineBasicBlock::reverse_iterator RI(LastLocalValue);
196 
197  InstOrderMap OrderMap;
198  for (; RI != RE;) {
199  MachineInstr &LocalMI = *RI;
200  ++RI;
201  bool Store = true;
202  if (!LocalMI.isSafeToMove(nullptr, Store))
203  continue;
204  unsigned DefReg = findSinkableLocalRegDef(LocalMI);
205  if (DefReg == 0)
206  continue;
207 
208  sinkLocalValueMaterialization(LocalMI, DefReg, OrderMap);
209  }
210  }
211 
215  SavedInsertPt = FuncInfo.InsertPt;
217 }
218 
219 static bool isRegUsedByPhiNodes(unsigned DefReg,
221  for (auto &P : FuncInfo.PHINodesToUpdate)
222  if (P.second == DefReg)
223  return true;
224  return false;
225 }
226 
227 /// Build a map of instruction orders. Return the first terminator and its
228 /// order. Consider EH_LABEL instructions to be terminators as well, since local
229 /// values for phis after invokes must be materialized before the call.
232  unsigned Order = 0;
233  for (MachineInstr &I : *MBB) {
234  if (!FirstTerminator &&
235  (I.isTerminator() || (I.isEHLabel() && &I != &MBB->front()))) {
236  FirstTerminator = &I;
237  FirstTerminatorOrder = Order;
238  }
239  Orders[&I] = Order++;
240 
241  // We don't need to order instructions past the last flush point.
242  if (I.getIterator() == LastFlushPoint)
243  break;
244  }
245 }
246 
247 void FastISel::sinkLocalValueMaterialization(MachineInstr &LocalMI,
248  unsigned DefReg,
249  InstOrderMap &OrderMap) {
250  // If this register is used by a register fixup, MRI will not contain all
251  // the uses until after register fixups, so don't attempt to sink or DCE
252  // this instruction. Register fixups typically come from no-op cast
253  // instructions, which replace the cast instruction vreg with the local
254  // value vreg.
255  if (FuncInfo.RegsWithFixups.count(DefReg))
256  return;
257 
258  // We can DCE this instruction if there are no uses and it wasn't a
259  // materialized for a successor PHI node.
260  bool UsedByPHI = isRegUsedByPhiNodes(DefReg, FuncInfo);
261  if (!UsedByPHI && MRI.use_nodbg_empty(DefReg)) {
262  if (EmitStartPt == &LocalMI)
264  LLVM_DEBUG(dbgs() << "removing dead local value materialization "
265  << LocalMI);
266  OrderMap.Orders.erase(&LocalMI);
267  LocalMI.eraseFromParent();
268  return;
269  }
270 
271  // Number the instructions if we haven't yet so we can efficiently find the
272  // earliest use.
273  if (OrderMap.Orders.empty())
274  OrderMap.initialize(FuncInfo.MBB, LastFlushPoint);
275 
276  // Find the first user in the BB.
277  MachineInstr *FirstUser = nullptr;
278  unsigned FirstOrder = std::numeric_limits<unsigned>::max();
279  for (MachineInstr &UseInst : MRI.use_nodbg_instructions(DefReg)) {
280  auto I = OrderMap.Orders.find(&UseInst);
281  assert(I != OrderMap.Orders.end() &&
282  "local value used by instruction outside local region");
283  unsigned UseOrder = I->second;
284  if (UseOrder < FirstOrder) {
285  FirstOrder = UseOrder;
286  FirstUser = &UseInst;
287  }
288  }
289 
290  // The insertion point will be the first terminator or the first user,
291  // whichever came first. If there was no terminator, this must be a
292  // fallthrough block and the insertion point is the end of the block.
294  if (UsedByPHI && OrderMap.FirstTerminatorOrder < FirstOrder) {
295  FirstOrder = OrderMap.FirstTerminatorOrder;
296  SinkPos = OrderMap.FirstTerminator->getIterator();
297  } else if (FirstUser) {
298  SinkPos = FirstUser->getIterator();
299  } else {
300  assert(UsedByPHI && "must be users if not used by a phi");
301  SinkPos = FuncInfo.MBB->instr_end();
302  }
303 
304  // Collect all DBG_VALUEs before the new insertion position so that we can
305  // sink them.
307  for (MachineInstr &DbgVal : MRI.use_instructions(DefReg)) {
308  if (!DbgVal.isDebugValue())
309  continue;
310  unsigned UseOrder = OrderMap.Orders[&DbgVal];
311  if (UseOrder < FirstOrder)
312  DbgValues.push_back(&DbgVal);
313  }
314 
315  // Sink LocalMI before SinkPos and assign it the same DebugLoc.
316  LLVM_DEBUG(dbgs() << "sinking local value to first use " << LocalMI);
317  FuncInfo.MBB->remove(&LocalMI);
318  FuncInfo.MBB->insert(SinkPos, &LocalMI);
319  if (SinkPos != FuncInfo.MBB->end())
320  LocalMI.setDebugLoc(SinkPos->getDebugLoc());
321 
322  // Sink any debug values that we've collected.
323  for (MachineInstr *DI : DbgValues) {
324  FuncInfo.MBB->remove(DI);
325  FuncInfo.MBB->insert(SinkPos, DI);
326  }
327 }
328 
330  // Don't consider constants or arguments to have trivial kills.
331  const Instruction *I = dyn_cast<Instruction>(V);
332  if (!I)
333  return false;
334 
335  // No-op casts are trivially coalesced by fast-isel.
336  if (const auto *Cast = dyn_cast<CastInst>(I))
337  if (Cast->isNoopCast(DL) && !hasTrivialKill(Cast->getOperand(0)))
338  return false;
339 
340  // Even the value might have only one use in the LLVM IR, it is possible that
341  // FastISel might fold the use into another instruction and now there is more
342  // than one use at the Machine Instruction level.
343  unsigned Reg = lookUpRegForValue(V);
344  if (Reg && !MRI.use_empty(Reg))
345  return false;
346 
347  // GEPs with all zero indices are trivially coalesced by fast-isel.
348  if (const auto *GEP = dyn_cast<GetElementPtrInst>(I))
349  if (GEP->hasAllZeroIndices() && !hasTrivialKill(GEP->getOperand(0)))
350  return false;
351 
352  // Only instructions with a single use in the same basic block are considered
353  // to have trivial kills.
354  return I->hasOneUse() &&
355  !(I->getOpcode() == Instruction::BitCast ||
356  I->getOpcode() == Instruction::PtrToInt ||
357  I->getOpcode() == Instruction::IntToPtr) &&
358  cast<Instruction>(*I->user_begin())->getParent() == I->getParent();
359 }
360 
361 unsigned FastISel::getRegForValue(const Value *V) {
362  EVT RealVT = TLI.getValueType(DL, V->getType(), /*AllowUnknown=*/true);
363  // Don't handle non-simple values in FastISel.
364  if (!RealVT.isSimple())
365  return 0;
366 
367  // Ignore illegal types. We must do this before looking up the value
368  // in ValueMap because Arguments are given virtual registers regardless
369  // of whether FastISel can handle them.
370  MVT VT = RealVT.getSimpleVT();
371  if (!TLI.isTypeLegal(VT)) {
372  // Handle integer promotions, though, because they're common and easy.
373  if (VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)
374  VT = TLI.getTypeToTransformTo(V->getContext(), VT).getSimpleVT();
375  else
376  return 0;
377  }
378 
379  // Look up the value to see if we already have a register for it.
380  unsigned Reg = lookUpRegForValue(V);
381  if (Reg)
382  return Reg;
383 
384  // In bottom-up mode, just create the virtual register which will be used
385  // to hold the value. It will be materialized later.
386  if (isa<Instruction>(V) &&
387  (!isa<AllocaInst>(V) ||
388  !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(V))))
390 
391  SavePoint SaveInsertPt = enterLocalValueArea();
392 
393  // Materialize the value in a register. Emit any instructions in the
394  // local value area.
395  Reg = materializeRegForValue(V, VT);
396 
397  leaveLocalValueArea(SaveInsertPt);
398 
399  return Reg;
400 }
401 
402 unsigned FastISel::materializeConstant(const Value *V, MVT VT) {
403  unsigned Reg = 0;
404  if (const auto *CI = dyn_cast<ConstantInt>(V)) {
405  if (CI->getValue().getActiveBits() <= 64)
406  Reg = fastEmit_i(VT, VT, ISD::Constant, CI->getZExtValue());
407  } else if (isa<AllocaInst>(V))
408  Reg = fastMaterializeAlloca(cast<AllocaInst>(V));
409  else if (isa<ConstantPointerNull>(V))
410  // Translate this as an integer zero so that it can be
411  // local-CSE'd with actual integer zeros.
412  Reg = getRegForValue(
414  else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
415  if (CF->isNullValue())
416  Reg = fastMaterializeFloatZero(CF);
417  else
418  // Try to emit the constant directly.
419  Reg = fastEmit_f(VT, VT, ISD::ConstantFP, CF);
420 
421  if (!Reg) {
422  // Try to emit the constant by using an integer constant with a cast.
423  const APFloat &Flt = CF->getValueAPF();
424  EVT IntVT = TLI.getPointerTy(DL);
425  uint32_t IntBitWidth = IntVT.getSizeInBits();
426  APSInt SIntVal(IntBitWidth, /*isUnsigned=*/false);
427  bool isExact;
428  (void)Flt.convertToInteger(SIntVal, APFloat::rmTowardZero, &isExact);
429  if (isExact) {
430  unsigned IntegerReg =
432  if (IntegerReg != 0)
433  Reg = fastEmit_r(IntVT.getSimpleVT(), VT, ISD::SINT_TO_FP, IntegerReg,
434  /*Kill=*/false);
435  }
436  }
437  } else if (const auto *Op = dyn_cast<Operator>(V)) {
438  if (!selectOperator(Op, Op->getOpcode()))
439  if (!isa<Instruction>(Op) ||
440  !fastSelectInstruction(cast<Instruction>(Op)))
441  return 0;
442  Reg = lookUpRegForValue(Op);
443  } else if (isa<UndefValue>(V)) {
446  TII.get(TargetOpcode::IMPLICIT_DEF), Reg);
447  }
448  return Reg;
449 }
450 
451 /// Helper for getRegForValue. This function is called when the value isn't
452 /// already available in a register and must be materialized with new
453 /// instructions.
454 unsigned FastISel::materializeRegForValue(const Value *V, MVT VT) {
455  unsigned Reg = 0;
456  // Give the target-specific code a try first.
457  if (isa<Constant>(V))
458  Reg = fastMaterializeConstant(cast<Constant>(V));
459 
460  // If target-specific code couldn't or didn't want to handle the value, then
461  // give target-independent code a try.
462  if (!Reg)
463  Reg = materializeConstant(V, VT);
464 
465  // Don't cache constant materializations in the general ValueMap.
466  // To do so would require tracking what uses they dominate.
467  if (Reg) {
468  LocalValueMap[V] = Reg;
470  }
471  return Reg;
472 }
473 
474 unsigned FastISel::lookUpRegForValue(const Value *V) {
475  // Look up the value to see if we already have a register for it. We
476  // cache values defined by Instructions across blocks, and other values
477  // only locally. This is because Instructions already have the SSA
478  // def-dominates-use requirement enforced.
480  if (I != FuncInfo.ValueMap.end())
481  return I->second;
482  return LocalValueMap[V];
483 }
484 
485 void FastISel::updateValueMap(const Value *I, unsigned Reg, unsigned NumRegs) {
486  if (!isa<Instruction>(I)) {
487  LocalValueMap[I] = Reg;
488  return;
489  }
490 
491  unsigned &AssignedReg = FuncInfo.ValueMap[I];
492  if (AssignedReg == 0)
493  // Use the new register.
494  AssignedReg = Reg;
495  else if (Reg != AssignedReg) {
496  // Arrange for uses of AssignedReg to be replaced by uses of Reg.
497  for (unsigned i = 0; i < NumRegs; i++) {
498  FuncInfo.RegFixups[AssignedReg + i] = Reg + i;
499  FuncInfo.RegsWithFixups.insert(Reg + i);
500  }
501 
502  AssignedReg = Reg;
503  }
504 }
505 
506 std::pair<unsigned, bool> FastISel::getRegForGEPIndex(const Value *Idx) {
507  unsigned IdxN = getRegForValue(Idx);
508  if (IdxN == 0)
509  // Unhandled operand. Halt "fast" selection and bail.
510  return std::pair<unsigned, bool>(0, false);
511 
512  bool IdxNIsKill = hasTrivialKill(Idx);
513 
514  // If the index is smaller or larger than intptr_t, truncate or extend it.
515  MVT PtrVT = TLI.getPointerTy(DL);
516  EVT IdxVT = EVT::getEVT(Idx->getType(), /*HandleUnknown=*/false);
517  if (IdxVT.bitsLT(PtrVT)) {
518  IdxN = fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::SIGN_EXTEND, IdxN,
519  IdxNIsKill);
520  IdxNIsKill = true;
521  } else if (IdxVT.bitsGT(PtrVT)) {
522  IdxN =
523  fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::TRUNCATE, IdxN, IdxNIsKill);
524  IdxNIsKill = true;
525  }
526  return std::pair<unsigned, bool>(IdxN, IdxNIsKill);
527 }
528 
530  if (getLastLocalValue()) {
532  FuncInfo.MBB = FuncInfo.InsertPt->getParent();
533  ++FuncInfo.InsertPt;
534  } else
536 
537  // Now skip past any EH_LABELs, which must remain at the beginning.
538  while (FuncInfo.InsertPt != FuncInfo.MBB->end() &&
539  FuncInfo.InsertPt->getOpcode() == TargetOpcode::EH_LABEL)
540  ++FuncInfo.InsertPt;
541 }
542 
545  assert(I.isValid() && E.isValid() && std::distance(I, E) > 0 &&
546  "Invalid iterator!");
547  while (I != E) {
548  MachineInstr *Dead = &*I;
549  ++I;
550  Dead->eraseFromParent();
551  ++NumFastIselDead;
552  }
554 }
555 
558  DebugLoc OldDL = DbgLoc;
560  DbgLoc = DebugLoc();
561  SavePoint SP = {OldInsertPt, OldDL};
562  return SP;
563 }
564 
566  if (FuncInfo.InsertPt != FuncInfo.MBB->begin())
567  LastLocalValue = &*std::prev(FuncInfo.InsertPt);
568 
569  // Restore the previous insert position.
570  FuncInfo.InsertPt = OldInsertPt.InsertPt;
571  DbgLoc = OldInsertPt.DL;
572 }
573 
574 bool FastISel::selectBinaryOp(const User *I, unsigned ISDOpcode) {
575  EVT VT = EVT::getEVT(I->getType(), /*HandleUnknown=*/true);
576  if (VT == MVT::Other || !VT.isSimple())
577  // Unhandled type. Halt "fast" selection and bail.
578  return false;
579 
580  // We only handle legal types. For example, on x86-32 the instruction
581  // selector contains all of the 64-bit instructions from x86-64,
582  // under the assumption that i64 won't be used if the target doesn't
583  // support it.
584  if (!TLI.isTypeLegal(VT)) {
585  // MVT::i1 is special. Allow AND, OR, or XOR because they
586  // don't require additional zeroing, which makes them easy.
587  if (VT == MVT::i1 && (ISDOpcode == ISD::AND || ISDOpcode == ISD::OR ||
588  ISDOpcode == ISD::XOR))
589  VT = TLI.getTypeToTransformTo(I->getContext(), VT);
590  else
591  return false;
592  }
593 
594  // Check if the first operand is a constant, and handle it as "ri". At -O0,
595  // we don't have anything that canonicalizes operand order.
596  if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(0)))
597  if (isa<Instruction>(I) && cast<Instruction>(I)->isCommutative()) {
598  unsigned Op1 = getRegForValue(I->getOperand(1));
599  if (!Op1)
600  return false;
601  bool Op1IsKill = hasTrivialKill(I->getOperand(1));
602 
603  unsigned ResultReg =
604  fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op1, Op1IsKill,
605  CI->getZExtValue(), VT.getSimpleVT());
606  if (!ResultReg)
607  return false;
608 
609  // We successfully emitted code for the given LLVM Instruction.
610  updateValueMap(I, ResultReg);
611  return true;
612  }
613 
614  unsigned Op0 = getRegForValue(I->getOperand(0));
615  if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
616  return false;
617  bool Op0IsKill = hasTrivialKill(I->getOperand(0));
618 
619  // Check if the second operand is a constant and handle it appropriately.
620  if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(1))) {
621  uint64_t Imm = CI->getSExtValue();
622 
623  // Transform "sdiv exact X, 8" -> "sra X, 3".
624  if (ISDOpcode == ISD::SDIV && isa<BinaryOperator>(I) &&
625  cast<BinaryOperator>(I)->isExact() && isPowerOf2_64(Imm)) {
626  Imm = Log2_64(Imm);
627  ISDOpcode = ISD::SRA;
628  }
629 
630  // Transform "urem x, pow2" -> "and x, pow2-1".
631  if (ISDOpcode == ISD::UREM && isa<BinaryOperator>(I) &&
632  isPowerOf2_64(Imm)) {
633  --Imm;
634  ISDOpcode = ISD::AND;
635  }
636 
637  unsigned ResultReg = fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op0,
638  Op0IsKill, Imm, VT.getSimpleVT());
639  if (!ResultReg)
640  return false;
641 
642  // We successfully emitted code for the given LLVM Instruction.
643  updateValueMap(I, ResultReg);
644  return true;
645  }
646 
647  unsigned Op1 = getRegForValue(I->getOperand(1));
648  if (!Op1) // Unhandled operand. Halt "fast" selection and bail.
649  return false;
650  bool Op1IsKill = hasTrivialKill(I->getOperand(1));
651 
652  // Now we have both operands in registers. Emit the instruction.
653  unsigned ResultReg = fastEmit_rr(VT.getSimpleVT(), VT.getSimpleVT(),
654  ISDOpcode, Op0, Op0IsKill, Op1, Op1IsKill);
655  if (!ResultReg)
656  // Target-specific code wasn't able to find a machine opcode for
657  // the given ISD opcode and type. Halt "fast" selection and bail.
658  return false;
659 
660  // We successfully emitted code for the given LLVM Instruction.
661  updateValueMap(I, ResultReg);
662  return true;
663 }
664 
666  unsigned N = getRegForValue(I->getOperand(0));
667  if (!N) // Unhandled operand. Halt "fast" selection and bail.
668  return false;
669  bool NIsKill = hasTrivialKill(I->getOperand(0));
670 
671  // Keep a running tab of the total offset to coalesce multiple N = N + Offset
672  // into a single N = N + TotalOffset.
673  uint64_t TotalOffs = 0;
674  // FIXME: What's a good SWAG number for MaxOffs?
675  uint64_t MaxOffs = 2048;
676  MVT VT = TLI.getPointerTy(DL);
677  for (gep_type_iterator GTI = gep_type_begin(I), E = gep_type_end(I);
678  GTI != E; ++GTI) {
679  const Value *Idx = GTI.getOperand();
680  if (StructType *StTy = GTI.getStructTypeOrNull()) {
681  uint64_t Field = cast<ConstantInt>(Idx)->getZExtValue();
682  if (Field) {
683  // N = N + Offset
684  TotalOffs += DL.getStructLayout(StTy)->getElementOffset(Field);
685  if (TotalOffs >= MaxOffs) {
686  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
687  if (!N) // Unhandled operand. Halt "fast" selection and bail.
688  return false;
689  NIsKill = true;
690  TotalOffs = 0;
691  }
692  }
693  } else {
694  Type *Ty = GTI.getIndexedType();
695 
696  // If this is a constant subscript, handle it quickly.
697  if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
698  if (CI->isZero())
699  continue;
700  // N = N + Offset
701  uint64_t IdxN = CI->getValue().sextOrTrunc(64).getSExtValue();
702  TotalOffs += DL.getTypeAllocSize(Ty) * IdxN;
703  if (TotalOffs >= MaxOffs) {
704  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
705  if (!N) // Unhandled operand. Halt "fast" selection and bail.
706  return false;
707  NIsKill = true;
708  TotalOffs = 0;
709  }
710  continue;
711  }
712  if (TotalOffs) {
713  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
714  if (!N) // Unhandled operand. Halt "fast" selection and bail.
715  return false;
716  NIsKill = true;
717  TotalOffs = 0;
718  }
719 
720  // N = N + Idx * ElementSize;
721  uint64_t ElementSize = DL.getTypeAllocSize(Ty);
722  std::pair<unsigned, bool> Pair = getRegForGEPIndex(Idx);
723  unsigned IdxN = Pair.first;
724  bool IdxNIsKill = Pair.second;
725  if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
726  return false;
727 
728  if (ElementSize != 1) {
729  IdxN = fastEmit_ri_(VT, ISD::MUL, IdxN, IdxNIsKill, ElementSize, VT);
730  if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
731  return false;
732  IdxNIsKill = true;
733  }
734  N = fastEmit_rr(VT, VT, ISD::ADD, N, NIsKill, IdxN, IdxNIsKill);
735  if (!N) // Unhandled operand. Halt "fast" selection and bail.
736  return false;
737  }
738  }
739  if (TotalOffs) {
740  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
741  if (!N) // Unhandled operand. Halt "fast" selection and bail.
742  return false;
743  }
744 
745  // We successfully emitted code for the given LLVM Instruction.
746  updateValueMap(I, N);
747  return true;
748 }
749 
750 bool FastISel::addStackMapLiveVars(SmallVectorImpl<MachineOperand> &Ops,
751  const CallInst *CI, unsigned StartIdx) {
752  for (unsigned i = StartIdx, e = CI->getNumArgOperands(); i != e; ++i) {
753  Value *Val = CI->getArgOperand(i);
754  // Check for constants and encode them with a StackMaps::ConstantOp prefix.
755  if (const auto *C = dyn_cast<ConstantInt>(Val)) {
756  Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
757  Ops.push_back(MachineOperand::CreateImm(C->getSExtValue()));
758  } else if (isa<ConstantPointerNull>(Val)) {
759  Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
761  } else if (auto *AI = dyn_cast<AllocaInst>(Val)) {
762  // Values coming from a stack location also require a special encoding,
763  // but that is added later on by the target specific frame index
764  // elimination implementation.
765  auto SI = FuncInfo.StaticAllocaMap.find(AI);
766  if (SI != FuncInfo.StaticAllocaMap.end())
767  Ops.push_back(MachineOperand::CreateFI(SI->second));
768  else
769  return false;
770  } else {
771  unsigned Reg = getRegForValue(Val);
772  if (!Reg)
773  return false;
774  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
775  }
776  }
777  return true;
778 }
779 
781  // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
782  // [live variables...])
784  "Stackmap cannot return a value.");
785 
786  // The stackmap intrinsic only records the live variables (the arguments
787  // passed to it) and emits NOPS (if requested). Unlike the patchpoint
788  // intrinsic, this won't be lowered to a function call. This means we don't
789  // have to worry about calling conventions and target-specific lowering code.
790  // Instead we perform the call lowering right here.
791  //
792  // CALLSEQ_START(0, 0...)
793  // STACKMAP(id, nbytes, ...)
794  // CALLSEQ_END(0, 0)
795  //
797 
798  // Add the <id> and <numBytes> constants.
799  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
800  "Expected a constant integer.");
801  const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
802  Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
803 
804  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
805  "Expected a constant integer.");
806  const auto *NumBytes =
807  cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
808  Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
809 
810  // Push live variables for the stack map (skipping the first two arguments
811  // <id> and <numBytes>).
812  if (!addStackMapLiveVars(Ops, I, 2))
813  return false;
814 
815  // We are not adding any register mask info here, because the stackmap doesn't
816  // clobber anything.
817 
818  // Add scratch registers as implicit def and early clobber.
820  const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
821  for (unsigned i = 0; ScratchRegs[i]; ++i)
822  Ops.push_back(MachineOperand::CreateReg(
823  ScratchRegs[i], /*IsDef=*/true, /*IsImp=*/true, /*IsKill=*/false,
824  /*IsDead=*/false, /*IsUndef=*/false, /*IsEarlyClobber=*/true));
825 
826  // Issue CALLSEQ_START
827  unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
828  auto Builder =
829  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackDown));
830  const MCInstrDesc &MCID = Builder.getInstr()->getDesc();
831  for (unsigned I = 0, E = MCID.getNumOperands(); I < E; ++I)
832  Builder.addImm(0);
833 
834  // Issue STACKMAP.
836  TII.get(TargetOpcode::STACKMAP));
837  for (auto const &MO : Ops)
838  MIB.add(MO);
839 
840  // Issue CALLSEQ_END
841  unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
842  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackUp))
843  .addImm(0)
844  .addImm(0);
845 
846  // Inform the Frame Information that we have a stackmap in this function.
848 
849  return true;
850 }
851 
852 /// Lower an argument list according to the target calling convention.
853 ///
854 /// This is a helper for lowering intrinsics that follow a target calling
855 /// convention or require stack pointer adjustment. Only a subset of the
856 /// intrinsic's operands need to participate in the calling convention.
857 bool FastISel::lowerCallOperands(const CallInst *CI, unsigned ArgIdx,
858  unsigned NumArgs, const Value *Callee,
859  bool ForceRetVoidTy, CallLoweringInfo &CLI) {
860  ArgListTy Args;
861  Args.reserve(NumArgs);
862 
863  // Populate the argument list.
864  ImmutableCallSite CS(CI);
865  for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI) {
866  Value *V = CI->getOperand(ArgI);
867 
868  assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
869 
870  ArgListEntry Entry;
871  Entry.Val = V;
872  Entry.Ty = V->getType();
873  Entry.setAttributes(&CS, ArgI);
874  Args.push_back(Entry);
875  }
876 
877  Type *RetTy = ForceRetVoidTy ? Type::getVoidTy(CI->getType()->getContext())
878  : CI->getType();
879  CLI.setCallee(CI->getCallingConv(), RetTy, Callee, std::move(Args), NumArgs);
880 
881  return lowerCallTo(CLI);
882 }
883 
885  const DataLayout &DL, MCContext &Ctx, CallingConv::ID CC, Type *ResultTy,
886  StringRef Target, ArgListTy &&ArgsList, unsigned FixedArgs) {
887  SmallString<32> MangledName;
888  Mangler::getNameWithPrefix(MangledName, Target, DL);
889  MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
890  return setCallee(CC, ResultTy, Sym, std::move(ArgsList), FixedArgs);
891 }
892 
894  // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
895  // i32 <numBytes>,
896  // i8* <target>,
897  // i32 <numArgs>,
898  // [Args...],
899  // [live variables...])
901  bool IsAnyRegCC = CC == CallingConv::AnyReg;
902  bool HasDef = !I->getType()->isVoidTy();
904 
905  // Get the real number of arguments participating in the call <numArgs>
906  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)) &&
907  "Expected a constant integer.");
908  const auto *NumArgsVal =
909  cast<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos));
910  unsigned NumArgs = NumArgsVal->getZExtValue();
911 
912  // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
913  // This includes all meta-operands up to but not including CC.
914  unsigned NumMetaOpers = PatchPointOpers::CCPos;
915  assert(I->getNumArgOperands() >= NumMetaOpers + NumArgs &&
916  "Not enough arguments provided to the patchpoint intrinsic");
917 
918  // For AnyRegCC the arguments are lowered later on manually.
919  unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
920  CallLoweringInfo CLI;
921  CLI.setIsPatchPoint();
922  if (!lowerCallOperands(I, NumMetaOpers, NumCallArgs, Callee, IsAnyRegCC, CLI))
923  return false;
924 
925  assert(CLI.Call && "No call instruction specified.");
926 
928 
929  // Add an explicit result reg if we use the anyreg calling convention.
930  if (IsAnyRegCC && HasDef) {
931  assert(CLI.NumResultRegs == 0 && "Unexpected result register.");
933  CLI.NumResultRegs = 1;
934  Ops.push_back(MachineOperand::CreateReg(CLI.ResultReg, /*IsDef=*/true));
935  }
936 
937  // Add the <id> and <numBytes> constants.
938  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
939  "Expected a constant integer.");
940  const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
941  Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
942 
943  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
944  "Expected a constant integer.");
945  const auto *NumBytes =
946  cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
947  Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
948 
949  // Add the call target.
950  if (const auto *C = dyn_cast<IntToPtrInst>(Callee)) {
951  uint64_t CalleeConstAddr =
952  cast<ConstantInt>(C->getOperand(0))->getZExtValue();
953  Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
954  } else if (const auto *C = dyn_cast<ConstantExpr>(Callee)) {
955  if (C->getOpcode() == Instruction::IntToPtr) {
956  uint64_t CalleeConstAddr =
957  cast<ConstantInt>(C->getOperand(0))->getZExtValue();
958  Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
959  } else
960  llvm_unreachable("Unsupported ConstantExpr.");
961  } else if (const auto *GV = dyn_cast<GlobalValue>(Callee)) {
963  } else if (isa<ConstantPointerNull>(Callee))
965  else
966  llvm_unreachable("Unsupported callee address.");
967 
968  // Adjust <numArgs> to account for any arguments that have been passed on
969  // the stack instead.
970  unsigned NumCallRegArgs = IsAnyRegCC ? NumArgs : CLI.OutRegs.size();
971  Ops.push_back(MachineOperand::CreateImm(NumCallRegArgs));
972 
973  // Add the calling convention
974  Ops.push_back(MachineOperand::CreateImm((unsigned)CC));
975 
976  // Add the arguments we omitted previously. The register allocator should
977  // place these in any free register.
978  if (IsAnyRegCC) {
979  for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) {
980  unsigned Reg = getRegForValue(I->getArgOperand(i));
981  if (!Reg)
982  return false;
983  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
984  }
985  }
986 
987  // Push the arguments from the call instruction.
988  for (auto Reg : CLI.OutRegs)
989  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
990 
991  // Push live variables for the stack map.
992  if (!addStackMapLiveVars(Ops, I, NumMetaOpers + NumArgs))
993  return false;
994 
995  // Push the register mask info.
998 
999  // Add scratch registers as implicit def and early clobber.
1000  const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
1001  for (unsigned i = 0; ScratchRegs[i]; ++i)
1003  ScratchRegs[i], /*IsDef=*/true, /*IsImp=*/true, /*IsKill=*/false,
1004  /*IsDead=*/false, /*IsUndef=*/false, /*IsEarlyClobber=*/true));
1005 
1006  // Add implicit defs (return values).
1007  for (auto Reg : CLI.InRegs)
1008  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/true,
1009  /*IsImpl=*/true));
1010 
1011  // Insert the patchpoint instruction before the call generated by the target.
1013  TII.get(TargetOpcode::PATCHPOINT));
1014 
1015  for (auto &MO : Ops)
1016  MIB.add(MO);
1017 
1018  MIB->setPhysRegsDeadExcept(CLI.InRegs, TRI);
1019 
1020  // Delete the original call instruction.
1021  CLI.Call->eraseFromParent();
1022 
1023  // Inform the Frame Information that we have a patchpoint in this function.
1025 
1026  if (CLI.NumResultRegs)
1027  updateValueMap(I, CLI.ResultReg, CLI.NumResultRegs);
1028  return true;
1029 }
1030 
1032  const auto &Triple = TM.getTargetTriple();
1033  if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
1034  return true; // don't do anything to this instruction.
1037  /*IsDef=*/false));
1039  /*IsDef=*/false));
1040  MachineInstrBuilder MIB =
1042  TII.get(TargetOpcode::PATCHABLE_EVENT_CALL));
1043  for (auto &MO : Ops)
1044  MIB.add(MO);
1045 
1046  // Insert the Patchable Event Call instruction, that gets lowered properly.
1047  return true;
1048 }
1049 
1051  const auto &Triple = TM.getTargetTriple();
1052  if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
1053  return true; // don't do anything to this instruction.
1056  /*IsDef=*/false));
1058  /*IsDef=*/false));
1060  /*IsDef=*/false));
1061  MachineInstrBuilder MIB =
1063  TII.get(TargetOpcode::PATCHABLE_TYPED_EVENT_CALL));
1064  for (auto &MO : Ops)
1065  MIB.add(MO);
1066 
1067  // Insert the Patchable Typed Event Call instruction, that gets lowered properly.
1068  return true;
1069 }
1070 
1071 /// Returns an AttributeList representing the attributes applied to the return
1072 /// value of the given call.
1075  if (CLI.RetSExt)
1076  Attrs.push_back(Attribute::SExt);
1077  if (CLI.RetZExt)
1078  Attrs.push_back(Attribute::ZExt);
1079  if (CLI.IsInReg)
1080  Attrs.push_back(Attribute::InReg);
1081 
1083  Attrs);
1084 }
1085 
1086 bool FastISel::lowerCallTo(const CallInst *CI, const char *SymName,
1087  unsigned NumArgs) {
1088  MCContext &Ctx = MF->getContext();
1089  SmallString<32> MangledName;
1090  Mangler::getNameWithPrefix(MangledName, SymName, DL);
1091  MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
1092  return lowerCallTo(CI, Sym, NumArgs);
1093 }
1094 
1096  unsigned NumArgs) {
1097  ImmutableCallSite CS(CI);
1098 
1099  FunctionType *FTy = CS.getFunctionType();
1100  Type *RetTy = CS.getType();
1101 
1102  ArgListTy Args;
1103  Args.reserve(NumArgs);
1104 
1105  // Populate the argument list.
1106  // Attributes for args start at offset 1, after the return attribute.
1107  for (unsigned ArgI = 0; ArgI != NumArgs; ++ArgI) {
1108  Value *V = CI->getOperand(ArgI);
1109 
1110  assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
1111 
1112  ArgListEntry Entry;
1113  Entry.Val = V;
1114  Entry.Ty = V->getType();
1115  Entry.setAttributes(&CS, ArgI);
1116  Args.push_back(Entry);
1117  }
1119 
1120  CallLoweringInfo CLI;
1121  CLI.setCallee(RetTy, FTy, Symbol, std::move(Args), CS, NumArgs);
1122 
1123  return lowerCallTo(CLI);
1124 }
1125 
1127  // Handle the incoming return values from the call.
1128  CLI.clearIns();
1129  SmallVector<EVT, 4> RetTys;
1130  ComputeValueVTs(TLI, DL, CLI.RetTy, RetTys);
1131 
1133  GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, TLI, DL);
1134 
1135  bool CanLowerReturn = TLI.CanLowerReturn(
1136  CLI.CallConv, *FuncInfo.MF, CLI.IsVarArg, Outs, CLI.RetTy->getContext());
1137 
1138  // FIXME: sret demotion isn't supported yet - bail out.
1139  if (!CanLowerReturn)
1140  return false;
1141 
1142  for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
1143  EVT VT = RetTys[I];
1144  MVT RegisterVT = TLI.getRegisterType(CLI.RetTy->getContext(), VT);
1145  unsigned NumRegs = TLI.getNumRegisters(CLI.RetTy->getContext(), VT);
1146  for (unsigned i = 0; i != NumRegs; ++i) {
1147  ISD::InputArg MyFlags;
1148  MyFlags.VT = RegisterVT;
1149  MyFlags.ArgVT = VT;
1150  MyFlags.Used = CLI.IsReturnValueUsed;
1151  if (CLI.RetSExt)
1152  MyFlags.Flags.setSExt();
1153  if (CLI.RetZExt)
1154  MyFlags.Flags.setZExt();
1155  if (CLI.IsInReg)
1156  MyFlags.Flags.setInReg();
1157  CLI.Ins.push_back(MyFlags);
1158  }
1159  }
1160 
1161  // Handle all of the outgoing arguments.
1162  CLI.clearOuts();
1163  for (auto &Arg : CLI.getArgs()) {
1164  Type *FinalType = Arg.Ty;
1165  if (Arg.IsByVal)
1166  FinalType = cast<PointerType>(Arg.Ty)->getElementType();
1167  bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1168  FinalType, CLI.CallConv, CLI.IsVarArg);
1169 
1170  ISD::ArgFlagsTy Flags;
1171  if (Arg.IsZExt)
1172  Flags.setZExt();
1173  if (Arg.IsSExt)
1174  Flags.setSExt();
1175  if (Arg.IsInReg)
1176  Flags.setInReg();
1177  if (Arg.IsSRet)
1178  Flags.setSRet();
1179  if (Arg.IsSwiftSelf)
1180  Flags.setSwiftSelf();
1181  if (Arg.IsSwiftError)
1182  Flags.setSwiftError();
1183  if (Arg.IsByVal)
1184  Flags.setByVal();
1185  if (Arg.IsInAlloca) {
1186  Flags.setInAlloca();
1187  // Set the byval flag for CCAssignFn callbacks that don't know about
1188  // inalloca. This way we can know how many bytes we should've allocated
1189  // and how many bytes a callee cleanup function will pop. If we port
1190  // inalloca to more targets, we'll have to add custom inalloca handling in
1191  // the various CC lowering callbacks.
1192  Flags.setByVal();
1193  }
1194  if (Arg.IsByVal || Arg.IsInAlloca) {
1195  PointerType *Ty = cast<PointerType>(Arg.Ty);
1196  Type *ElementTy = Ty->getElementType();
1197  unsigned FrameSize = DL.getTypeAllocSize(ElementTy);
1198  // For ByVal, alignment should come from FE. BE will guess if this info is
1199  // not there, but there are cases it cannot get right.
1200  unsigned FrameAlign = Arg.Alignment;
1201  if (!FrameAlign)
1202  FrameAlign = TLI.getByValTypeAlignment(ElementTy, DL);
1203  Flags.setByValSize(FrameSize);
1204  Flags.setByValAlign(FrameAlign);
1205  }
1206  if (Arg.IsNest)
1207  Flags.setNest();
1208  if (NeedsRegBlock)
1209  Flags.setInConsecutiveRegs();
1210  unsigned OriginalAlignment = DL.getABITypeAlignment(Arg.Ty);
1211  Flags.setOrigAlign(OriginalAlignment);
1212 
1213  CLI.OutVals.push_back(Arg.Val);
1214  CLI.OutFlags.push_back(Flags);
1215  }
1216 
1217  if (!fastLowerCall(CLI))
1218  return false;
1219 
1220  // Set all unused physreg defs as dead.
1221  assert(CLI.Call && "No call instruction specified.");
1222  CLI.Call->setPhysRegsDeadExcept(CLI.InRegs, TRI);
1223 
1224  if (CLI.NumResultRegs && CLI.CS)
1226 
1227  return true;
1228 }
1229 
1231  ImmutableCallSite CS(CI);
1232 
1233  FunctionType *FuncTy = CS.getFunctionType();
1234  Type *RetTy = CS.getType();
1235 
1236  ArgListTy Args;
1237  ArgListEntry Entry;
1238  Args.reserve(CS.arg_size());
1239 
1240  for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
1241  i != e; ++i) {
1242  Value *V = *i;
1243 
1244  // Skip empty types
1245  if (V->getType()->isEmptyTy())
1246  continue;
1247 
1248  Entry.Val = V;
1249  Entry.Ty = V->getType();
1250 
1251  // Skip the first return-type Attribute to get to params.
1252  Entry.setAttributes(&CS, i - CS.arg_begin());
1253  Args.push_back(Entry);
1254  }
1255 
1256  // Check if target-independent constraints permit a tail call here.
1257  // Target-dependent constraints are checked within fastLowerCall.
1258  bool IsTailCall = CI->isTailCall();
1259  if (IsTailCall && !isInTailCallPosition(CS, TM))
1260  IsTailCall = false;
1261 
1262  CallLoweringInfo CLI;
1263  CLI.setCallee(RetTy, FuncTy, CI->getCalledValue(), std::move(Args), CS)
1264  .setTailCall(IsTailCall);
1265 
1266  return lowerCallTo(CLI);
1267 }
1268 
1270  const CallInst *Call = cast<CallInst>(I);
1271 
1272  // Handle simple inline asms.
1273  if (const InlineAsm *IA = dyn_cast<InlineAsm>(Call->getCalledValue())) {
1274  // If the inline asm has side effects, then make sure that no local value
1275  // lives across by flushing the local value map.
1276  if (IA->hasSideEffects())
1277  flushLocalValueMap();
1278 
1279  // Don't attempt to handle constraints.
1280  if (!IA->getConstraintString().empty())
1281  return false;
1282 
1283  unsigned ExtraInfo = 0;
1284  if (IA->hasSideEffects())
1285  ExtraInfo |= InlineAsm::Extra_HasSideEffects;
1286  if (IA->isAlignStack())
1287  ExtraInfo |= InlineAsm::Extra_IsAlignStack;
1288 
1291  .addExternalSymbol(IA->getAsmString().c_str())
1292  .addImm(ExtraInfo);
1293  return true;
1294  }
1295 
1296  MachineModuleInfo &MMI = FuncInfo.MF->getMMI();
1297  computeUsesVAFloatArgument(*Call, MMI);
1298 
1299  // Handle intrinsic function calls.
1300  if (const auto *II = dyn_cast<IntrinsicInst>(Call))
1301  return selectIntrinsicCall(II);
1302 
1303  // Usually, it does not make sense to initialize a value,
1304  // make an unrelated function call and use the value, because
1305  // it tends to be spilled on the stack. So, we move the pointer
1306  // to the last local value to the beginning of the block, so that
1307  // all the values which have already been materialized,
1308  // appear after the call. It also makes sense to skip intrinsics
1309  // since they tend to be inlined.
1310  flushLocalValueMap();
1311 
1312  return lowerCall(Call);
1313 }
1314 
1316  switch (II->getIntrinsicID()) {
1317  default:
1318  break;
1319  // At -O0 we don't care about the lifetime intrinsics.
1320  case Intrinsic::lifetime_start:
1321  case Intrinsic::lifetime_end:
1322  // The donothing intrinsic does, well, nothing.
1323  case Intrinsic::donothing:
1324  // Neither does the sideeffect intrinsic.
1325  case Intrinsic::sideeffect:
1326  // Neither does the assume intrinsic; it's also OK not to codegen its operand.
1327  case Intrinsic::assume:
1328  return true;
1329  case Intrinsic::dbg_declare: {
1330  const DbgDeclareInst *DI = cast<DbgDeclareInst>(II);
1331  assert(DI->getVariable() && "Missing variable");
1332  if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1333  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1334  return true;
1335  }
1336 
1337  const Value *Address = DI->getAddress();
1338  if (!Address || isa<UndefValue>(Address)) {
1339  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1340  return true;
1341  }
1342 
1343  // Byval arguments with frame indices were already handled after argument
1344  // lowering and before isel.
1345  const auto *Arg =
1347  if (Arg && FuncInfo.getArgumentFrameIndex(Arg) != INT_MAX)
1348  return true;
1349 
1351  if (unsigned Reg = lookUpRegForValue(Address))
1352  Op = MachineOperand::CreateReg(Reg, false);
1353 
1354  // If we have a VLA that has a "use" in a metadata node that's then used
1355  // here but it has no other uses, then we have a problem. E.g.,
1356  //
1357  // int foo (const int *x) {
1358  // char a[*x];
1359  // return 0;
1360  // }
1361  //
1362  // If we assign 'a' a vreg and fast isel later on has to use the selection
1363  // DAG isel, it will want to copy the value to the vreg. However, there are
1364  // no uses, which goes counter to what selection DAG isel expects.
1365  if (!Op && !Address->use_empty() && isa<Instruction>(Address) &&
1366  (!isa<AllocaInst>(Address) ||
1367  !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(Address))))
1369  false);
1370 
1371  if (Op) {
1373  "Expected inlined-at fields to agree");
1374  // A dbg.declare describes the address of a source variable, so lower it
1375  // into an indirect DBG_VALUE.
1377  TII.get(TargetOpcode::DBG_VALUE), /*IsIndirect*/ true,
1378  *Op, DI->getVariable(), DI->getExpression());
1379  } else {
1380  // We can't yet handle anything else here because it would require
1381  // generating code, thus altering codegen because of debug info.
1382  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1383  }
1384  return true;
1385  }
1386  case Intrinsic::dbg_value: {
1387  // This form of DBG_VALUE is target-independent.
1388  const DbgValueInst *DI = cast<DbgValueInst>(II);
1389  const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE);
1390  const Value *V = DI->getValue();
1392  "Expected inlined-at fields to agree");
1393  if (!V) {
1394  // Currently the optimizer can produce this; insert an undef to
1395  // help debugging. Probably the optimizer should not do this.
1396  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, false, 0U,
1397  DI->getVariable(), DI->getExpression());
1398  } else if (const auto *CI = dyn_cast<ConstantInt>(V)) {
1399  if (CI->getBitWidth() > 64)
1401  .addCImm(CI)
1402  .addImm(0U)
1403  .addMetadata(DI->getVariable())
1404  .addMetadata(DI->getExpression());
1405  else
1407  .addImm(CI->getZExtValue())
1408  .addImm(0U)
1409  .addMetadata(DI->getVariable())
1410  .addMetadata(DI->getExpression());
1411  } else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
1413  .addFPImm(CF)
1414  .addImm(0U)
1415  .addMetadata(DI->getVariable())
1416  .addMetadata(DI->getExpression());
1417  } else if (unsigned Reg = lookUpRegForValue(V)) {
1418  // FIXME: This does not handle register-indirect values at offset 0.
1419  bool IsIndirect = false;
1420  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, IsIndirect, Reg,
1421  DI->getVariable(), DI->getExpression());
1422  } else {
1423  // We can't yet handle anything else here because it would require
1424  // generating code, thus altering codegen because of debug info.
1425  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1426  }
1427  return true;
1428  }
1429  case Intrinsic::dbg_label: {
1430  const DbgLabelInst *DI = cast<DbgLabelInst>(II);
1431  assert(DI->getLabel() && "Missing label");
1432  if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1433  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1434  return true;
1435  }
1436 
1438  TII.get(TargetOpcode::DBG_LABEL)).addMetadata(DI->getLabel());
1439  return true;
1440  }
1441  case Intrinsic::objectsize: {
1442  ConstantInt *CI = cast<ConstantInt>(II->getArgOperand(1));
1443  unsigned long long Res = CI->isZero() ? -1ULL : 0;
1444  Constant *ResCI = ConstantInt::get(II->getType(), Res);
1445  unsigned ResultReg = getRegForValue(ResCI);
1446  if (!ResultReg)
1447  return false;
1448  updateValueMap(II, ResultReg);
1449  return true;
1450  }
1451  case Intrinsic::launder_invariant_group:
1452  case Intrinsic::strip_invariant_group:
1453  case Intrinsic::expect: {
1454  unsigned ResultReg = getRegForValue(II->getArgOperand(0));
1455  if (!ResultReg)
1456  return false;
1457  updateValueMap(II, ResultReg);
1458  return true;
1459  }
1460  case Intrinsic::experimental_stackmap:
1461  return selectStackmap(II);
1462  case Intrinsic::experimental_patchpoint_void:
1463  case Intrinsic::experimental_patchpoint_i64:
1464  return selectPatchpoint(II);
1465 
1466  case Intrinsic::xray_customevent:
1467  return selectXRayCustomEvent(II);
1468  case Intrinsic::xray_typedevent:
1469  return selectXRayTypedEvent(II);
1470  }
1471 
1472  return fastLowerIntrinsicCall(II);
1473 }
1474 
1475 bool FastISel::selectCast(const User *I, unsigned Opcode) {
1476  EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1477  EVT DstVT = TLI.getValueType(DL, I->getType());
1478 
1479  if (SrcVT == MVT::Other || !SrcVT.isSimple() || DstVT == MVT::Other ||
1480  !DstVT.isSimple())
1481  // Unhandled type. Halt "fast" selection and bail.
1482  return false;
1483 
1484  // Check if the destination type is legal.
1485  if (!TLI.isTypeLegal(DstVT))
1486  return false;
1487 
1488  // Check if the source operand is legal.
1489  if (!TLI.isTypeLegal(SrcVT))
1490  return false;
1491 
1492  unsigned InputReg = getRegForValue(I->getOperand(0));
1493  if (!InputReg)
1494  // Unhandled operand. Halt "fast" selection and bail.
1495  return false;
1496 
1497  bool InputRegIsKill = hasTrivialKill(I->getOperand(0));
1498 
1499  unsigned ResultReg = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(),
1500  Opcode, InputReg, InputRegIsKill);
1501  if (!ResultReg)
1502  return false;
1503 
1504  updateValueMap(I, ResultReg);
1505  return true;
1506 }
1507 
1509  // If the bitcast doesn't change the type, just use the operand value.
1510  if (I->getType() == I->getOperand(0)->getType()) {
1511  unsigned Reg = getRegForValue(I->getOperand(0));
1512  if (!Reg)
1513  return false;
1514  updateValueMap(I, Reg);
1515  return true;
1516  }
1517 
1518  // Bitcasts of other values become reg-reg copies or BITCAST operators.
1519  EVT SrcEVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1520  EVT DstEVT = TLI.getValueType(DL, I->getType());
1521  if (SrcEVT == MVT::Other || DstEVT == MVT::Other ||
1522  !TLI.isTypeLegal(SrcEVT) || !TLI.isTypeLegal(DstEVT))
1523  // Unhandled type. Halt "fast" selection and bail.
1524  return false;
1525 
1526  MVT SrcVT = SrcEVT.getSimpleVT();
1527  MVT DstVT = DstEVT.getSimpleVT();
1528  unsigned Op0 = getRegForValue(I->getOperand(0));
1529  if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
1530  return false;
1531  bool Op0IsKill = hasTrivialKill(I->getOperand(0));
1532 
1533  // First, try to perform the bitcast by inserting a reg-reg copy.
1534  unsigned ResultReg = 0;
1535  if (SrcVT == DstVT) {
1536  const TargetRegisterClass *SrcClass = TLI.getRegClassFor(SrcVT);
1537  const TargetRegisterClass *DstClass = TLI.getRegClassFor(DstVT);
1538  // Don't attempt a cross-class copy. It will likely fail.
1539  if (SrcClass == DstClass) {
1540  ResultReg = createResultReg(DstClass);
1542  TII.get(TargetOpcode::COPY), ResultReg).addReg(Op0);
1543  }
1544  }
1545 
1546  // If the reg-reg copy failed, select a BITCAST opcode.
1547  if (!ResultReg)
1548  ResultReg = fastEmit_r(SrcVT, DstVT, ISD::BITCAST, Op0, Op0IsKill);
1549 
1550  if (!ResultReg)
1551  return false;
1552 
1553  updateValueMap(I, ResultReg);
1554  return true;
1555 }
1556 
1557 // Remove local value instructions starting from the instruction after
1558 // SavedLastLocalValue to the current function insert point.
1559 void FastISel::removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue)
1560 {
1561  MachineInstr *CurLastLocalValue = getLastLocalValue();
1562  if (CurLastLocalValue != SavedLastLocalValue) {
1563  // Find the first local value instruction to be deleted.
1564  // This is the instruction after SavedLastLocalValue if it is non-NULL.
1565  // Otherwise it's the first instruction in the block.
1566  MachineBasicBlock::iterator FirstDeadInst(SavedLastLocalValue);
1567  if (SavedLastLocalValue)
1568  ++FirstDeadInst;
1569  else
1570  FirstDeadInst = FuncInfo.MBB->getFirstNonPHI();
1571  setLastLocalValue(SavedLastLocalValue);
1572  removeDeadCode(FirstDeadInst, FuncInfo.InsertPt);
1573  }
1574 }
1575 
1577  MachineInstr *SavedLastLocalValue = getLastLocalValue();
1578  // Just before the terminator instruction, insert instructions to
1579  // feed PHI nodes in successor blocks.
1580  if (I->isTerminator()) {
1581  if (!handlePHINodesInSuccessorBlocks(I->getParent())) {
1582  // PHI node handling may have generated local value instructions,
1583  // even though it failed to handle all PHI nodes.
1584  // We remove these instructions because SelectionDAGISel will generate
1585  // them again.
1586  removeDeadLocalValueCode(SavedLastLocalValue);
1587  return false;
1588  }
1589  }
1590 
1591  // FastISel does not handle any operand bundles except OB_funclet.
1593  for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i)
1594  if (CS.getOperandBundleAt(i).getTagID() != LLVMContext::OB_funclet)
1595  return false;
1596 
1597  DbgLoc = I->getDebugLoc();
1598 
1599  SavedInsertPt = FuncInfo.InsertPt;
1600 
1601  if (const auto *Call = dyn_cast<CallInst>(I)) {
1602  const Function *F = Call->getCalledFunction();
1603  LibFunc Func;
1604 
1605  // As a special case, don't handle calls to builtin library functions that
1606  // may be translated directly to target instructions.
1607  if (F && !F->hasLocalLinkage() && F->hasName() &&
1608  LibInfo->getLibFunc(F->getName(), Func) &&
1610  return false;
1611 
1612  // Don't handle Intrinsic::trap if a trap function is specified.
1613  if (F && F->getIntrinsicID() == Intrinsic::trap &&
1614  Call->hasFnAttr("trap-func-name"))
1615  return false;
1616  }
1617 
1618  // First, try doing target-independent selection.
1620  if (selectOperator(I, I->getOpcode())) {
1621  ++NumFastIselSuccessIndependent;
1622  DbgLoc = DebugLoc();
1623  return true;
1624  }
1625  // Remove dead code.
1627  if (SavedInsertPt != FuncInfo.InsertPt)
1628  removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
1629  SavedInsertPt = FuncInfo.InsertPt;
1630  }
1631  // Next, try calling the target to attempt to handle the instruction.
1632  if (fastSelectInstruction(I)) {
1633  ++NumFastIselSuccessTarget;
1634  DbgLoc = DebugLoc();
1635  return true;
1636  }
1637  // Remove dead code.
1639  if (SavedInsertPt != FuncInfo.InsertPt)
1640  removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
1641 
1642  DbgLoc = DebugLoc();
1643  // Undo phi node updates, because they will be added again by SelectionDAG.
1644  if (I->isTerminator()) {
1645  // PHI node handling may have generated local value instructions.
1646  // We remove them because SelectionDAGISel will generate them again.
1647  removeDeadLocalValueCode(SavedLastLocalValue);
1649  }
1650  return false;
1651 }
1652 
1653 /// Emit an unconditional branch to the given block, unless it is the immediate
1654 /// (fall-through) successor, and update the CFG.
1656  const DebugLoc &DbgLoc) {
1657  if (FuncInfo.MBB->getBasicBlock()->size() > 1 &&
1658  FuncInfo.MBB->isLayoutSuccessor(MSucc)) {
1659  // For more accurate line information if this is the only instruction
1660  // in the block then emit it, otherwise we have the unconditional
1661  // fall-through case, which needs no instructions.
1662  } else {
1663  // The unconditional branch case.
1664  TII.insertBranch(*FuncInfo.MBB, MSucc, nullptr,
1665  SmallVector<MachineOperand, 0>(), DbgLoc);
1666  }
1667  if (FuncInfo.BPI) {
1669  FuncInfo.MBB->getBasicBlock(), MSucc->getBasicBlock());
1671  } else
1673 }
1674 
1676  MachineBasicBlock *TrueMBB,
1677  MachineBasicBlock *FalseMBB) {
1678  // Add TrueMBB as successor unless it is equal to the FalseMBB: This can
1679  // happen in degenerate IR and MachineIR forbids to have a block twice in the
1680  // successor/predecessor lists.
1681  if (TrueMBB != FalseMBB) {
1682  if (FuncInfo.BPI) {
1683  auto BranchProbability =
1684  FuncInfo.BPI->getEdgeProbability(BranchBB, TrueMBB->getBasicBlock());
1686  } else
1688  }
1689 
1690  fastEmitBranch(FalseMBB, DbgLoc);
1691 }
1692 
1693 /// Emit an FNeg operation.
1695  unsigned OpReg = getRegForValue(BinaryOperator::getFNegArgument(I));
1696  if (!OpReg)
1697  return false;
1698  bool OpRegIsKill = hasTrivialKill(I);
1699 
1700  // If the target has ISD::FNEG, use it.
1701  EVT VT = TLI.getValueType(DL, I->getType());
1702  unsigned ResultReg = fastEmit_r(VT.getSimpleVT(), VT.getSimpleVT(), ISD::FNEG,
1703  OpReg, OpRegIsKill);
1704  if (ResultReg) {
1705  updateValueMap(I, ResultReg);
1706  return true;
1707  }
1708 
1709  // Bitcast the value to integer, twiddle the sign bit with xor,
1710  // and then bitcast it back to floating-point.
1711  if (VT.getSizeInBits() > 64)
1712  return false;
1713  EVT IntVT = EVT::getIntegerVT(I->getContext(), VT.getSizeInBits());
1714  if (!TLI.isTypeLegal(IntVT))
1715  return false;
1716 
1717  unsigned IntReg = fastEmit_r(VT.getSimpleVT(), IntVT.getSimpleVT(),
1718  ISD::BITCAST, OpReg, OpRegIsKill);
1719  if (!IntReg)
1720  return false;
1721 
1722  unsigned IntResultReg = fastEmit_ri_(
1723  IntVT.getSimpleVT(), ISD::XOR, IntReg, /*IsKill=*/true,
1724  UINT64_C(1) << (VT.getSizeInBits() - 1), IntVT.getSimpleVT());
1725  if (!IntResultReg)
1726  return false;
1727 
1728  ResultReg = fastEmit_r(IntVT.getSimpleVT(), VT.getSimpleVT(), ISD::BITCAST,
1729  IntResultReg, /*IsKill=*/true);
1730  if (!ResultReg)
1731  return false;
1732 
1733  updateValueMap(I, ResultReg);
1734  return true;
1735 }
1736 
1738  const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(U);
1739  if (!EVI)
1740  return false;
1741 
1742  // Make sure we only try to handle extracts with a legal result. But also
1743  // allow i1 because it's easy.
1744  EVT RealVT = TLI.getValueType(DL, EVI->getType(), /*AllowUnknown=*/true);
1745  if (!RealVT.isSimple())
1746  return false;
1747  MVT VT = RealVT.getSimpleVT();
1748  if (!TLI.isTypeLegal(VT) && VT != MVT::i1)
1749  return false;
1750 
1751  const Value *Op0 = EVI->getOperand(0);
1752  Type *AggTy = Op0->getType();
1753 
1754  // Get the base result register.
1755  unsigned ResultReg;
1757  if (I != FuncInfo.ValueMap.end())
1758  ResultReg = I->second;
1759  else if (isa<Instruction>(Op0))
1760  ResultReg = FuncInfo.InitializeRegForValue(Op0);
1761  else
1762  return false; // fast-isel can't handle aggregate constants at the moment
1763 
1764  // Get the actual result register, which is an offset from the base register.
1765  unsigned VTIndex = ComputeLinearIndex(AggTy, EVI->getIndices());
1766 
1767  SmallVector<EVT, 4> AggValueVTs;
1768  ComputeValueVTs(TLI, DL, AggTy, AggValueVTs);
1769 
1770  for (unsigned i = 0; i < VTIndex; i++)
1771  ResultReg += TLI.getNumRegisters(FuncInfo.Fn->getContext(), AggValueVTs[i]);
1772 
1773  updateValueMap(EVI, ResultReg);
1774  return true;
1775 }
1776 
1777 bool FastISel::selectOperator(const User *I, unsigned Opcode) {
1778  switch (Opcode) {
1779  case Instruction::Add:
1780  return selectBinaryOp(I, ISD::ADD);
1781  case Instruction::FAdd:
1782  return selectBinaryOp(I, ISD::FADD);
1783  case Instruction::Sub:
1784  return selectBinaryOp(I, ISD::SUB);
1785  case Instruction::FSub:
1786  // FNeg is currently represented in LLVM IR as a special case of FSub.
1787  if (BinaryOperator::isFNeg(I))
1788  return selectFNeg(I);
1789  return selectBinaryOp(I, ISD::FSUB);
1790  case Instruction::Mul:
1791  return selectBinaryOp(I, ISD::MUL);
1792  case Instruction::FMul:
1793  return selectBinaryOp(I, ISD::FMUL);
1794  case Instruction::SDiv:
1795  return selectBinaryOp(I, ISD::SDIV);
1796  case Instruction::UDiv:
1797  return selectBinaryOp(I, ISD::UDIV);
1798  case Instruction::FDiv:
1799  return selectBinaryOp(I, ISD::FDIV);
1800  case Instruction::SRem:
1801  return selectBinaryOp(I, ISD::SREM);
1802  case Instruction::URem:
1803  return selectBinaryOp(I, ISD::UREM);
1804  case Instruction::FRem:
1805  return selectBinaryOp(I, ISD::FREM);
1806  case Instruction::Shl:
1807  return selectBinaryOp(I, ISD::SHL);
1808  case Instruction::LShr:
1809  return selectBinaryOp(I, ISD::SRL);
1810  case Instruction::AShr:
1811  return selectBinaryOp(I, ISD::SRA);
1812  case Instruction::And:
1813  return selectBinaryOp(I, ISD::AND);
1814  case Instruction::Or:
1815  return selectBinaryOp(I, ISD::OR);
1816  case Instruction::Xor:
1817  return selectBinaryOp(I, ISD::XOR);
1818 
1819  case Instruction::GetElementPtr:
1820  return selectGetElementPtr(I);
1821 
1822  case Instruction::Br: {
1823  const BranchInst *BI = cast<BranchInst>(I);
1824 
1825  if (BI->isUnconditional()) {
1826  const BasicBlock *LLVMSucc = BI->getSuccessor(0);
1827  MachineBasicBlock *MSucc = FuncInfo.MBBMap[LLVMSucc];
1828  fastEmitBranch(MSucc, BI->getDebugLoc());
1829  return true;
1830  }
1831 
1832  // Conditional branches are not handed yet.
1833  // Halt "fast" selection and bail.
1834  return false;
1835  }
1836 
1837  case Instruction::Unreachable:
1839  return fastEmit_(MVT::Other, MVT::Other, ISD::TRAP) != 0;
1840  else
1841  return true;
1842 
1843  case Instruction::Alloca:
1844  // FunctionLowering has the static-sized case covered.
1845  if (FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(I)))
1846  return true;
1847 
1848  // Dynamic-sized alloca is not handled yet.
1849  return false;
1850 
1851  case Instruction::Call:
1852  return selectCall(I);
1853 
1854  case Instruction::BitCast:
1855  return selectBitCast(I);
1856 
1857  case Instruction::FPToSI:
1858  return selectCast(I, ISD::FP_TO_SINT);
1859  case Instruction::ZExt:
1860  return selectCast(I, ISD::ZERO_EXTEND);
1861  case Instruction::SExt:
1862  return selectCast(I, ISD::SIGN_EXTEND);
1863  case Instruction::Trunc:
1864  return selectCast(I, ISD::TRUNCATE);
1865  case Instruction::SIToFP:
1866  return selectCast(I, ISD::SINT_TO_FP);
1867 
1868  case Instruction::IntToPtr: // Deliberate fall-through.
1869  case Instruction::PtrToInt: {
1870  EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1871  EVT DstVT = TLI.getValueType(DL, I->getType());
1872  if (DstVT.bitsGT(SrcVT))
1873  return selectCast(I, ISD::ZERO_EXTEND);
1874  if (DstVT.bitsLT(SrcVT))
1875  return selectCast(I, ISD::TRUNCATE);
1876  unsigned Reg = getRegForValue(I->getOperand(0));
1877  if (!Reg)
1878  return false;
1879  updateValueMap(I, Reg);
1880  return true;
1881  }
1882 
1883  case Instruction::ExtractValue:
1884  return selectExtractValue(I);
1885 
1886  case Instruction::PHI:
1887  llvm_unreachable("FastISel shouldn't visit PHI nodes!");
1888 
1889  default:
1890  // Unhandled instruction. Halt "fast" selection and bail.
1891  return false;
1892  }
1893 }
1894 
1896  const TargetLibraryInfo *LibInfo,
1898  : FuncInfo(FuncInfo), MF(FuncInfo.MF), MRI(FuncInfo.MF->getRegInfo()),
1899  MFI(FuncInfo.MF->getFrameInfo()), MCP(*FuncInfo.MF->getConstantPool()),
1900  TM(FuncInfo.MF->getTarget()), DL(MF->getDataLayout()),
1901  TII(*MF->getSubtarget().getInstrInfo()),
1902  TLI(*MF->getSubtarget().getTargetLowering()),
1903  TRI(*MF->getSubtarget().getRegisterInfo()), LibInfo(LibInfo),
1904  SkipTargetIndependentISel(SkipTargetIndependentISel) {}
1905 
1906 FastISel::~FastISel() = default;
1907 
1908 bool FastISel::fastLowerArguments() { return false; }
1909 
1910 bool FastISel::fastLowerCall(CallLoweringInfo & /*CLI*/) { return false; }
1911 
1913  return false;
1914 }
1915 
1916 unsigned FastISel::fastEmit_(MVT, MVT, unsigned) { return 0; }
1917 
1918 unsigned FastISel::fastEmit_r(MVT, MVT, unsigned, unsigned /*Op0*/,
1919  bool /*Op0IsKill*/) {
1920  return 0;
1921 }
1922 
1923 unsigned FastISel::fastEmit_rr(MVT, MVT, unsigned, unsigned /*Op0*/,
1924  bool /*Op0IsKill*/, unsigned /*Op1*/,
1925  bool /*Op1IsKill*/) {
1926  return 0;
1927 }
1928 
1929 unsigned FastISel::fastEmit_i(MVT, MVT, unsigned, uint64_t /*Imm*/) {
1930  return 0;
1931 }
1932 
1933 unsigned FastISel::fastEmit_f(MVT, MVT, unsigned,
1934  const ConstantFP * /*FPImm*/) {
1935  return 0;
1936 }
1937 
1938 unsigned FastISel::fastEmit_ri(MVT, MVT, unsigned, unsigned /*Op0*/,
1939  bool /*Op0IsKill*/, uint64_t /*Imm*/) {
1940  return 0;
1941 }
1942 
1943 /// This method is a wrapper of fastEmit_ri. It first tries to emit an
1944 /// instruction with an immediate operand using fastEmit_ri.
1945 /// If that fails, it materializes the immediate into a register and try
1946 /// fastEmit_rr instead.
1947 unsigned FastISel::fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0,
1948  bool Op0IsKill, uint64_t Imm, MVT ImmType) {
1949  // If this is a multiply by a power of two, emit this as a shift left.
1950  if (Opcode == ISD::MUL && isPowerOf2_64(Imm)) {
1951  Opcode = ISD::SHL;
1952  Imm = Log2_64(Imm);
1953  } else if (Opcode == ISD::UDIV && isPowerOf2_64(Imm)) {
1954  // div x, 8 -> srl x, 3
1955  Opcode = ISD::SRL;
1956  Imm = Log2_64(Imm);
1957  }
1958 
1959  // Horrible hack (to be removed), check to make sure shift amounts are
1960  // in-range.
1961  if ((Opcode == ISD::SHL || Opcode == ISD::SRA || Opcode == ISD::SRL) &&
1962  Imm >= VT.getSizeInBits())
1963  return 0;
1964 
1965  // First check if immediate type is legal. If not, we can't use the ri form.
1966  unsigned ResultReg = fastEmit_ri(VT, VT, Opcode, Op0, Op0IsKill, Imm);
1967  if (ResultReg)
1968  return ResultReg;
1969  unsigned MaterialReg = fastEmit_i(ImmType, ImmType, ISD::Constant, Imm);
1970  bool IsImmKill = true;
1971  if (!MaterialReg) {
1972  // This is a bit ugly/slow, but failing here means falling out of
1973  // fast-isel, which would be very slow.
1974  IntegerType *ITy =
1976  MaterialReg = getRegForValue(ConstantInt::get(ITy, Imm));
1977  if (!MaterialReg)
1978  return 0;
1979  // FIXME: If the materialized register here has no uses yet then this
1980  // will be the first use and we should be able to mark it as killed.
1981  // However, the local value area for materialising constant expressions
1982  // grows down, not up, which means that any constant expressions we generate
1983  // later which also use 'Imm' could be after this instruction and therefore
1984  // after this kill.
1985  IsImmKill = false;
1986  }
1987  return fastEmit_rr(VT, VT, Opcode, Op0, Op0IsKill, MaterialReg, IsImmKill);
1988 }
1989 
1991  return MRI.createVirtualRegister(RC);
1992 }
1993 
1995  unsigned OpNum) {
1997  const TargetRegisterClass *RegClass =
1998  TII.getRegClass(II, OpNum, &TRI, *FuncInfo.MF);
1999  if (!MRI.constrainRegClass(Op, RegClass)) {
2000  // If it's not legal to COPY between the register classes, something
2001  // has gone very wrong before we got here.
2002  unsigned NewOp = createResultReg(RegClass);
2004  TII.get(TargetOpcode::COPY), NewOp).addReg(Op);
2005  return NewOp;
2006  }
2007  }
2008  return Op;
2009 }
2010 
2011 unsigned FastISel::fastEmitInst_(unsigned MachineInstOpcode,
2012  const TargetRegisterClass *RC) {
2013  unsigned ResultReg = createResultReg(RC);
2014  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2015 
2016  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg);
2017  return ResultReg;
2018 }
2019 
2020 unsigned FastISel::fastEmitInst_r(unsigned MachineInstOpcode,
2021  const TargetRegisterClass *RC, unsigned Op0,
2022  bool Op0IsKill) {
2023  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2024 
2025  unsigned ResultReg = createResultReg(RC);
2026  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2027 
2028  if (II.getNumDefs() >= 1)
2029  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2030  .addReg(Op0, getKillRegState(Op0IsKill));
2031  else {
2033  .addReg(Op0, getKillRegState(Op0IsKill));
2035  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2036  }
2037 
2038  return ResultReg;
2039 }
2040 
2041 unsigned FastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
2042  const TargetRegisterClass *RC, unsigned Op0,
2043  bool Op0IsKill, unsigned Op1,
2044  bool Op1IsKill) {
2045  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2046 
2047  unsigned ResultReg = createResultReg(RC);
2048  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2049  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2050 
2051  if (II.getNumDefs() >= 1)
2052  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2053  .addReg(Op0, getKillRegState(Op0IsKill))
2054  .addReg(Op1, getKillRegState(Op1IsKill));
2055  else {
2057  .addReg(Op0, getKillRegState(Op0IsKill))
2058  .addReg(Op1, getKillRegState(Op1IsKill));
2060  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2061  }
2062  return ResultReg;
2063 }
2064 
2065 unsigned FastISel::fastEmitInst_rrr(unsigned MachineInstOpcode,
2066  const TargetRegisterClass *RC, unsigned Op0,
2067  bool Op0IsKill, unsigned Op1,
2068  bool Op1IsKill, unsigned Op2,
2069  bool Op2IsKill) {
2070  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2071 
2072  unsigned ResultReg = createResultReg(RC);
2073  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2074  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2075  Op2 = constrainOperandRegClass(II, Op2, II.getNumDefs() + 2);
2076 
2077  if (II.getNumDefs() >= 1)
2078  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2079  .addReg(Op0, getKillRegState(Op0IsKill))
2080  .addReg(Op1, getKillRegState(Op1IsKill))
2081  .addReg(Op2, getKillRegState(Op2IsKill));
2082  else {
2084  .addReg(Op0, getKillRegState(Op0IsKill))
2085  .addReg(Op1, getKillRegState(Op1IsKill))
2086  .addReg(Op2, getKillRegState(Op2IsKill));
2088  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2089  }
2090  return ResultReg;
2091 }
2092 
2093 unsigned FastISel::fastEmitInst_ri(unsigned MachineInstOpcode,
2094  const TargetRegisterClass *RC, unsigned Op0,
2095  bool Op0IsKill, uint64_t Imm) {
2096  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2097 
2098  unsigned ResultReg = createResultReg(RC);
2099  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2100 
2101  if (II.getNumDefs() >= 1)
2102  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2103  .addReg(Op0, getKillRegState(Op0IsKill))
2104  .addImm(Imm);
2105  else {
2107  .addReg(Op0, getKillRegState(Op0IsKill))
2108  .addImm(Imm);
2110  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2111  }
2112  return ResultReg;
2113 }
2114 
2115 unsigned FastISel::fastEmitInst_rii(unsigned MachineInstOpcode,
2116  const TargetRegisterClass *RC, unsigned Op0,
2117  bool Op0IsKill, uint64_t Imm1,
2118  uint64_t Imm2) {
2119  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2120 
2121  unsigned ResultReg = createResultReg(RC);
2122  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2123 
2124  if (II.getNumDefs() >= 1)
2125  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2126  .addReg(Op0, getKillRegState(Op0IsKill))
2127  .addImm(Imm1)
2128  .addImm(Imm2);
2129  else {
2131  .addReg(Op0, getKillRegState(Op0IsKill))
2132  .addImm(Imm1)
2133  .addImm(Imm2);
2135  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2136  }
2137  return ResultReg;
2138 }
2139 
2140 unsigned FastISel::fastEmitInst_f(unsigned MachineInstOpcode,
2141  const TargetRegisterClass *RC,
2142  const ConstantFP *FPImm) {
2143  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2144 
2145  unsigned ResultReg = createResultReg(RC);
2146 
2147  if (II.getNumDefs() >= 1)
2148  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2149  .addFPImm(FPImm);
2150  else {
2152  .addFPImm(FPImm);
2154  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2155  }
2156  return ResultReg;
2157 }
2158 
2159 unsigned FastISel::fastEmitInst_rri(unsigned MachineInstOpcode,
2160  const TargetRegisterClass *RC, unsigned Op0,
2161  bool Op0IsKill, unsigned Op1,
2162  bool Op1IsKill, uint64_t Imm) {
2163  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2164 
2165  unsigned ResultReg = createResultReg(RC);
2166  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2167  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2168 
2169  if (II.getNumDefs() >= 1)
2170  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2171  .addReg(Op0, getKillRegState(Op0IsKill))
2172  .addReg(Op1, getKillRegState(Op1IsKill))
2173  .addImm(Imm);
2174  else {
2176  .addReg(Op0, getKillRegState(Op0IsKill))
2177  .addReg(Op1, getKillRegState(Op1IsKill))
2178  .addImm(Imm);
2180  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2181  }
2182  return ResultReg;
2183 }
2184 
2185 unsigned FastISel::fastEmitInst_i(unsigned MachineInstOpcode,
2186  const TargetRegisterClass *RC, uint64_t Imm) {
2187  unsigned ResultReg = createResultReg(RC);
2188  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2189 
2190  if (II.getNumDefs() >= 1)
2191  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2192  .addImm(Imm);
2193  else {
2196  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2197  }
2198  return ResultReg;
2199 }
2200 
2201 unsigned FastISel::fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0,
2202  bool Op0IsKill, uint32_t Idx) {
2203  unsigned ResultReg = createResultReg(TLI.getRegClassFor(RetVT));
2205  "Cannot yet extract from physregs");
2206  const TargetRegisterClass *RC = MRI.getRegClass(Op0);
2208  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY),
2209  ResultReg).addReg(Op0, getKillRegState(Op0IsKill), Idx);
2210  return ResultReg;
2211 }
2212 
2213 /// Emit MachineInstrs to compute the value of Op with all but the least
2214 /// significant bit set to zero.
2215 unsigned FastISel::fastEmitZExtFromI1(MVT VT, unsigned Op0, bool Op0IsKill) {
2216  return fastEmit_ri(VT, VT, ISD::AND, Op0, Op0IsKill, 1);
2217 }
2218 
2219 /// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks.
2220 /// Emit code to ensure constants are copied into registers when needed.
2221 /// Remember the virtual registers that need to be added to the Machine PHI
2222 /// nodes as input. We cannot just directly add them, because expansion
2223 /// might result in multiple MBB's for one BB. As such, the start of the
2224 /// BB might correspond to a different MBB than the end.
2225 bool FastISel::handlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
2226  const TerminatorInst *TI = LLVMBB->getTerminator();
2227 
2230 
2231  // Check successor nodes' PHI nodes that expect a constant to be available
2232  // from this block.
2233  for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
2234  const BasicBlock *SuccBB = TI->getSuccessor(succ);
2235  if (!isa<PHINode>(SuccBB->begin()))
2236  continue;
2237  MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
2238 
2239  // If this terminator has multiple identical successors (common for
2240  // switches), only handle each succ once.
2241  if (!SuccsHandled.insert(SuccMBB).second)
2242  continue;
2243 
2244  MachineBasicBlock::iterator MBBI = SuccMBB->begin();
2245 
2246  // At this point we know that there is a 1-1 correspondence between LLVM PHI
2247  // nodes and Machine PHI nodes, but the incoming operands have not been
2248  // emitted yet.
2249  for (const PHINode &PN : SuccBB->phis()) {
2250  // Ignore dead phi's.
2251  if (PN.use_empty())
2252  continue;
2253 
2254  // Only handle legal types. Two interesting things to note here. First,
2255  // by bailing out early, we may leave behind some dead instructions,
2256  // since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its
2257  // own moves. Second, this check is necessary because FastISel doesn't
2258  // use CreateRegs to create registers, so it always creates
2259  // exactly one register for each non-void instruction.
2260  EVT VT = TLI.getValueType(DL, PN.getType(), /*AllowUnknown=*/true);
2261  if (VT == MVT::Other || !TLI.isTypeLegal(VT)) {
2262  // Handle integer promotions, though, because they're common and easy.
2263  if (!(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)) {
2265  return false;
2266  }
2267  }
2268 
2269  const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
2270 
2271  // Set the DebugLoc for the copy. Prefer the location of the operand
2272  // if there is one; use the location of the PHI otherwise.
2273  DbgLoc = PN.getDebugLoc();
2274  if (const auto *Inst = dyn_cast<Instruction>(PHIOp))
2275  DbgLoc = Inst->getDebugLoc();
2276 
2277  unsigned Reg = getRegForValue(PHIOp);
2278  if (!Reg) {
2280  return false;
2281  }
2282  FuncInfo.PHINodesToUpdate.push_back(std::make_pair(&*MBBI++, Reg));
2283  DbgLoc = DebugLoc();
2284  }
2285  }
2286 
2287  return true;
2288 }
2289 
2290 bool FastISel::tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst) {
2291  assert(LI->hasOneUse() &&
2292  "tryToFoldLoad expected a LoadInst with a single use");
2293  // We know that the load has a single use, but don't know what it is. If it
2294  // isn't one of the folded instructions, then we can't succeed here. Handle
2295  // this by scanning the single-use users of the load until we get to FoldInst.
2296  unsigned MaxUsers = 6; // Don't scan down huge single-use chains of instrs.
2297 
2298  const Instruction *TheUser = LI->user_back();
2299  while (TheUser != FoldInst && // Scan up until we find FoldInst.
2300  // Stay in the right block.
2301  TheUser->getParent() == FoldInst->getParent() &&
2302  --MaxUsers) { // Don't scan too far.
2303  // If there are multiple or no uses of this instruction, then bail out.
2304  if (!TheUser->hasOneUse())
2305  return false;
2306 
2307  TheUser = TheUser->user_back();
2308  }
2309 
2310  // If we didn't find the fold instruction, then we failed to collapse the
2311  // sequence.
2312  if (TheUser != FoldInst)
2313  return false;
2314 
2315  // Don't try to fold volatile loads. Target has to deal with alignment
2316  // constraints.
2317  if (LI->isVolatile())
2318  return false;
2319 
2320  // Figure out which vreg this is going into. If there is no assigned vreg yet
2321  // then there actually was no reference to it. Perhaps the load is referenced
2322  // by a dead instruction.
2323  unsigned LoadReg = getRegForValue(LI);
2324  if (!LoadReg)
2325  return false;
2326 
2327  // We can't fold if this vreg has no uses or more than one use. Multiple uses
2328  // may mean that the instruction got lowered to multiple MIs, or the use of
2329  // the loaded value ended up being multiple operands of the result.
2330  if (!MRI.hasOneUse(LoadReg))
2331  return false;
2332 
2334  MachineInstr *User = RI->getParent();
2335 
2336  // Set the insertion point properly. Folding the load can cause generation of
2337  // other random instructions (like sign extends) for addressing modes; make
2338  // sure they get inserted in a logical place before the new instruction.
2339  FuncInfo.InsertPt = User;
2340  FuncInfo.MBB = User->getParent();
2341 
2342  // Ask the target to try folding the load.
2343  return tryToFoldLoadIntoMI(User, RI.getOperandNo(), LI);
2344 }
2345 
2347  // Must be an add.
2348  if (!isa<AddOperator>(Add))
2349  return false;
2350  // Type size needs to match.
2351  if (DL.getTypeSizeInBits(GEP->getType()) !=
2352  DL.getTypeSizeInBits(Add->getType()))
2353  return false;
2354  // Must be in the same basic block.
2355  if (isa<Instruction>(Add) &&
2356  FuncInfo.MBBMap[cast<Instruction>(Add)->getParent()] != FuncInfo.MBB)
2357  return false;
2358  // Must have a constant operand.
2359  return isa<ConstantInt>(cast<AddOperator>(Add)->getOperand(1));
2360 }
2361 
2364  const Value *Ptr;
2365  Type *ValTy;
2366  unsigned Alignment;
2368  bool IsVolatile;
2369 
2370  if (const auto *LI = dyn_cast<LoadInst>(I)) {
2371  Alignment = LI->getAlignment();
2372  IsVolatile = LI->isVolatile();
2373  Flags = MachineMemOperand::MOLoad;
2374  Ptr = LI->getPointerOperand();
2375  ValTy = LI->getType();
2376  } else if (const auto *SI = dyn_cast<StoreInst>(I)) {
2377  Alignment = SI->getAlignment();
2378  IsVolatile = SI->isVolatile();
2380  Ptr = SI->getPointerOperand();
2381  ValTy = SI->getValueOperand()->getType();
2382  } else
2383  return nullptr;
2384 
2385  bool IsNonTemporal = I->getMetadata(LLVMContext::MD_nontemporal) != nullptr;
2386  bool IsInvariant = I->getMetadata(LLVMContext::MD_invariant_load) != nullptr;
2387  bool IsDereferenceable =
2389  const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range);
2390 
2391  AAMDNodes AAInfo;
2392  I->getAAMetadata(AAInfo);
2393 
2394  if (Alignment == 0) // Ensure that codegen never sees alignment 0.
2395  Alignment = DL.getABITypeAlignment(ValTy);
2396 
2397  unsigned Size = DL.getTypeStoreSize(ValTy);
2398 
2399  if (IsVolatile)
2401  if (IsNonTemporal)
2403  if (IsDereferenceable)
2405  if (IsInvariant)
2407 
2408  return FuncInfo.MF->getMachineMemOperand(MachinePointerInfo(Ptr), Flags, Size,
2409  Alignment, AAInfo, Ranges);
2410 }
2411 
2413  // If both operands are the same, then try to optimize or fold the cmp.
2415  if (CI->getOperand(0) != CI->getOperand(1))
2416  return Predicate;
2417 
2418  switch (Predicate) {
2419  default: llvm_unreachable("Invalid predicate!");
2420  case CmpInst::FCMP_FALSE: Predicate = CmpInst::FCMP_FALSE; break;
2421  case CmpInst::FCMP_OEQ: Predicate = CmpInst::FCMP_ORD; break;
2422  case CmpInst::FCMP_OGT: Predicate = CmpInst::FCMP_FALSE; break;
2423  case CmpInst::FCMP_OGE: Predicate = CmpInst::FCMP_ORD; break;
2424  case CmpInst::FCMP_OLT: Predicate = CmpInst::FCMP_FALSE; break;
2425  case CmpInst::FCMP_OLE: Predicate = CmpInst::FCMP_ORD; break;
2426  case CmpInst::FCMP_ONE: Predicate = CmpInst::FCMP_FALSE; break;
2427  case CmpInst::FCMP_ORD: Predicate = CmpInst::FCMP_ORD; break;
2428  case CmpInst::FCMP_UNO: Predicate = CmpInst::FCMP_UNO; break;
2429  case CmpInst::FCMP_UEQ: Predicate = CmpInst::FCMP_TRUE; break;
2430  case CmpInst::FCMP_UGT: Predicate = CmpInst::FCMP_UNO; break;
2431  case CmpInst::FCMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2432  case CmpInst::FCMP_ULT: Predicate = CmpInst::FCMP_UNO; break;
2433  case CmpInst::FCMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2434  case CmpInst::FCMP_UNE: Predicate = CmpInst::FCMP_UNO; break;
2435  case CmpInst::FCMP_TRUE: Predicate = CmpInst::FCMP_TRUE; break;
2436 
2437  case CmpInst::ICMP_EQ: Predicate = CmpInst::FCMP_TRUE; break;
2438  case CmpInst::ICMP_NE: Predicate = CmpInst::FCMP_FALSE; break;
2439  case CmpInst::ICMP_UGT: Predicate = CmpInst::FCMP_FALSE; break;
2440  case CmpInst::ICMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2441  case CmpInst::ICMP_ULT: Predicate = CmpInst::FCMP_FALSE; break;
2442  case CmpInst::ICMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2443  case CmpInst::ICMP_SGT: Predicate = CmpInst::FCMP_FALSE; break;
2444  case CmpInst::ICMP_SGE: Predicate = CmpInst::FCMP_TRUE; break;
2445  case CmpInst::ICMP_SLT: Predicate = CmpInst::FCMP_FALSE; break;
2446  case CmpInst::ICMP_SLE: Predicate = CmpInst::FCMP_TRUE; break;
2447  }
2448 
2449  return Predicate;
2450 }
void setHasStackMap(bool s=true)
uint64_t CallInst * C
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:541
unsigned fastEmitInst_rrr(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill, unsigned Op2, bool Op2IsKill)
Emit a MachineInstr with three register operands and a result register in the given register class...
Definition: FastISel.cpp:2065
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
void setByValAlign(unsigned A)
const MachineInstrBuilder & add(const MachineOperand &MO) const
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:111
Function * getCalledFunction() const
Return the function called, or null if this is an indirect function invocation.
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:675
unsigned fastEmitZExtFromI1(MVT VT, unsigned Op0, bool Op0IsKill)
Emit MachineInstrs to compute the value of Op with all but the least significant bit set to zero...
Definition: FastISel.cpp:2215
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
bool hasLocalLinkage() const
Definition: GlobalValue.h:436
This instruction extracts a struct member or array element value from an aggregate value...
bool use_nodbg_empty(unsigned RegNo) const
use_nodbg_empty - Return true if there are no non-Debug instructions using the specified register...
MachineConstantPool & MCP
Definition: FastISel.h:208
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
This class represents an incoming formal argument to a Function.
Definition: Argument.h:30
instr_iterator instr_end()
bool lowerCall(const CallInst *I)
Definition: FastISel.cpp:1230
unsigned arg_size() const
Definition: CallSite.h:219
static const Value * getFNegArgument(const Value *BinOp)
bool hasDebugInfo() const
Returns true if valid debug info is present.
CallingConv::ID getCallingConv() const
Get the calling convention of the call.
Definition: CallSite.h:312
const TargetRegisterClass * getRegClass(unsigned Reg) const
Return the register class of the specified virtual register.
This represents the llvm.dbg.label instruction.
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
virtual unsigned fastMaterializeConstant(const Constant *C)
Emit a constant in a register using target-specific logic, such as constant pool loads.
Definition: FastISel.h:482
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:42
BasicBlock * getSuccessor(unsigned Idx) const
Return the specified successor. This instruction must be a terminator.
constexpr char IsVolatile[]
Key for Kernel::Arg::Metadata::mIsVolatile.
bool selectXRayTypedEvent(const CallInst *II)
Definition: FastISel.cpp:1050
ImmutableCallSite * CS
Definition: FastISel.h:90
bool selectGetElementPtr(const User *I)
Definition: FastISel.cpp:665
void leaveLocalValueArea(SavePoint Old)
Reset InsertPt to the given old insert position.
Definition: FastISel.cpp:565
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:588
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
bool selectStackmap(const CallInst *I)
Definition: FastISel.cpp:780
This class represents a function call, abstracting a target machine&#39;s calling convention.
unsigned Reg
This file contains the declarations for metadata subclasses.
virtual bool tryToFoldLoadIntoMI(MachineInstr *, unsigned, const LoadInst *)
The specified machine instr operand is a vreg, and that vreg is being provided by the specified load ...
Definition: FastISel.h:305
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:253
gep_type_iterator gep_type_end(const User *GEP)
unsigned less or equal
Definition: InstrTypes.h:711
unsigned less than
Definition: InstrTypes.h:710
virtual unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const
Insert branch code into the end of the specified MachineBasicBlock.
0 1 0 0 True if ordered and less than
Definition: InstrTypes.h:691
MachineMemOperand * createMachineMemOperandFor(const Instruction *I) const
Create a machine mem operand from the given instruction.
Definition: FastISel.cpp:2363
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:714
virtual void markLibCallAttributes(MachineFunction *MF, unsigned CC, ArgListTy &Args) const
bool isTerminator() const
Definition: Instruction.h:129
1 1 1 0 True if unordered or not equal
Definition: InstrTypes.h:701
void addSuccessorWithoutProb(MachineBasicBlock *Succ)
Add Succ as a successor of this MachineBasicBlock.
virtual unsigned fastEmit_(MVT VT, MVT RetVT, unsigned Opcode)
This method is called by target-independent code to request that an instruction with the given type a...
Definition: FastISel.cpp:1916
BasicBlock * getSuccessor(unsigned i) const
arg_iterator arg_end()
Definition: Function.h:680
virtual const TargetRegisterClass * getRegClassFor(MVT VT) const
Return the register class that should be used for the specified value type.
STATISTIC(NumFunctions, "Total number of functions")
A debug info location.
Definition: DebugLoc.h:34
Metadata node.
Definition: Metadata.h:864
F(f)
MachineModuleInfo & getMMI() const
SmallVector< unsigned, 4 > InRegs
Definition: FastISel.h:99
unsigned getCallFrameDestroyOpcode() const
An instruction for reading from memory.
Definition: Instructions.h:168
Hexagon Common GEP
CallingConv::ID getCallingConv() const
getCallingConv/setCallingConv - Get or set the calling convention of this function call...
bool CanLowerReturn
CanLowerReturn - true iff the function&#39;s return value can be lowered to registers.
virtual unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t Imm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1929
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:459
virtual unsigned fastMaterializeFloatZero(const ConstantFP *CF)
Emit the floating-point constant +0.0 in a register using target- specific logic. ...
Definition: FastISel.h:489
void setPhysRegsDeadExcept(ArrayRef< unsigned > UsedRegs, const TargetRegisterInfo &TRI)
Mark every physreg used by this instruction as dead except those in the UsedRegs list.
virtual unsigned getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Return the desired alignment for ByVal or InAlloca aggregate function arguments in the caller paramet...
static MachineOperand CreateReg(unsigned Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
void computeUsesVAFloatArgument(const CallInst &I, MachineModuleInfo &MMI)
Determine if any floating-point values are being passed to this variadic function, and set the MachineModuleInfo&#39;s usesVAFloatArgument flag if so.
virtual bool fastLowerCall(CallLoweringInfo &CLI)
This method is called by target-independent code to do target- specific call lowering.
Definition: FastISel.cpp:1910
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:130
static Constant * getNullValue(Type *Ty)
Constructor to create a &#39;0&#39; constant of arbitrary type.
Definition: Constants.cpp:268
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:264
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
Definition: FastISel.cpp:1576
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
unsigned fastEmitInst_rii(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, uint64_t Imm1, uint64_t Imm2)
Emit a MachineInstr with one register operand and two immediate operands.
Definition: FastISel.cpp:2115
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Definition: APFloat.h:1069
1 0 0 1 True if unordered or equal
Definition: InstrTypes.h:696
MachineFunction * MF
Definition: FastISel.h:205
DenseMap< const Value *, unsigned > LocalValueMap
Definition: FastISel.h:203
unsigned fastEmitInst_ri(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, uint64_t Imm)
Emit a MachineInstr with a register operand, an immediate, and a result register in the given registe...
Definition: FastISel.cpp:2093
ArrayRef< unsigned > getIndices() const
void setLastLocalValue(MachineInstr *I)
Update the position of the last instruction emitted for materializing constants for use in the curren...
Definition: FastISel.h:242
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition: InstrTypes.h:695
unsigned fastEmitInst_rri(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill, uint64_t Imm)
Emit a MachineInstr with two register operands, an immediate, and a result register in the given regi...
Definition: FastISel.cpp:2159
CmpInst::Predicate optimizeCmpPredicate(const CmpInst *CI) const
Definition: FastISel.cpp:2412
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Definition: Instructions.h:221
A description of a memory reference used in the backend.
void setHasPatchPoint(bool s=true)
TargetLoweringBase::ArgListTy ArgListTy
Definition: FastISel.h:70
Shift and rotation operations.
Definition: ISDOpcodes.h:382
const TargetRegisterClass * getRegClass(const MCInstrDesc &MCID, unsigned OpNum, const TargetRegisterInfo *TRI, const MachineFunction &MF) const
Given a machine instruction descriptor, returns the register class constraint for OpNum...
Class to represent struct types.
Definition: DerivedTypes.h:201
A Use represents the edge between a Value definition and its users.
Definition: Use.h:56
DILabel * getLabel() const
bool canFoldAddIntoGEP(const User *GEP, const Value *Add)
Check if Add is an add that can be safely folded into GEP.
Definition: FastISel.cpp:2346
DenseMap< const Value *, unsigned > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
MachineBasicBlock iterator that automatically skips over MIs that are inside bundles (i...
IterTy arg_end() const
Definition: CallSite.h:575
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
unsigned fastEmitInst_r(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill)
Emit a MachineInstr with one register operand and a result register in the given register class...
Definition: FastISel.cpp:2020
MachineInstr * EmitStartPt
The top most instruction in the current block that is allowed for emitting local variables.
Definition: FastISel.h:227
0 1 0 1 True if ordered and less than or equal
Definition: InstrTypes.h:692
This file contains the simple types necessary to represent the attributes associated with functions a...
InstrTy * getInstruction() const
Definition: CallSite.h:92
The memory access is dereferenceable (i.e., doesn&#39;t trap).
static MachineOperand CreateRegMask(const uint32_t *Mask)
CreateRegMask - Creates a register mask operand referencing Mask.
virtual const TargetRegisterClass * getSubClassWithSubReg(const TargetRegisterClass *RC, unsigned Idx) const
Returns the largest legal sub-class of RC that supports the sub-register index Idx.
void setByValSize(unsigned S)
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineBasicBlock::iterator LastFlushPoint
Last local value flush point.
Definition: FastISel.h:231
const TargetMachine & TM
Definition: FastISel.h:210
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:628
bool selectIntrinsicCall(const IntrinsicInst *II)
Definition: FastISel.cpp:1315
bool selectCast(const User *I, unsigned Opcode)
Definition: FastISel.cpp:1475
unsigned getSizeInBits() const
MachineInstr * getVRegDef(unsigned Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
Context object for machine code objects.
Definition: MCContext.h:63
int getArgumentFrameIndex(const Argument *A)
getArgumentFrameIndex - Get frame index for the byval argument.
Class to represent function types.
Definition: DerivedTypes.h:103
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:292
SmallVector< ISD::InputArg, 4 > Ins
Definition: FastISel.h:98
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:245
unsigned constrainOperandRegClass(const MCInstrDesc &II, unsigned Op, unsigned OpNum)
Try to constrain Op so that it is usable by argument OpNum of the provided MCInstrDesc.
Definition: FastISel.cpp:1994
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:450
bool selectOperator(const User *I, unsigned Opcode)
Do "fast" instruction selection for the given LLVM IR operator (Instruction or ConstantExpr), and append generated machine instructions to the current block.
Definition: FastISel.cpp:1777
ArchType getArch() const
getArch - Get the parsed architecture type of this triple.
Definition: Triple.h:289
MachineInstrBundleIterator< MachineInstr, true > reverse_iterator
bool selectExtractValue(const User *U)
Definition: FastISel.cpp:1737
unsigned getRegForValue(const Value *V)
Create a virtual register and arrange for it to be assigned the value for the given LLVM value...
Definition: FastISel.cpp:361
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:201
DenseSet< unsigned > RegsWithFixups
unsigned fastEmitInst_(unsigned MachineInstOpcode, const TargetRegisterClass *RC)
Emit a MachineInstr with no operands and a result register in the given register class.
Definition: FastISel.cpp:2011
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:217
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:126
bool hasTrivialKill(const Value *V)
Test whether the given value has exactly one use.
Definition: FastISel.cpp:329
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
void setOrigAlign(unsigned A)
reverse_iterator rend()
amdgpu Simplify well known AMD library false Value * Callee
MachineInstr * getLastLocalValue()
Return the position of the last instruction emitted for materializing constants for use in the curren...
Definition: FastISel.h:238
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:84
const TargetRegisterClass * constrainRegClass(unsigned Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
unsigned getNumSuccessors() const
Return the number of successors that this instruction has.
Value * getOperand(unsigned i) const
Definition: User.h:170
Class to represent pointers.
Definition: DerivedTypes.h:467
unsigned getKillRegState(bool B)
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:494
unsigned lookUpRegForValue(const Value *V)
Look up the value to see if its value is already cached in a register.
Definition: FastISel.cpp:474
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
void finishBasicBlock()
Flush the local value map and sink local values if possible.
Definition: FastISel.cpp:143
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:147
Value * getOperand(unsigned i_nocapture) const
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:229
MCContext & getContext() const
void setAttributes(ImmutableCallSite *CS, unsigned ArgIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes...
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:141
The memory access is volatile.
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space...
Definition: DataLayout.cpp:742
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
void getAAMetadata(AAMDNodes &N, bool Merge=false) const
Fills the AAMDNodes structure with AA metadata from this instruction.
#define P(N)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:410
virtual const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const
Return a mask of call-preserved registers for the given calling convention on the current function...
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:169
virtual ~FastISel()
Subclasses of this class are all able to terminate a basic block.
Definition: InstrTypes.h:55
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
std::vector< std::pair< MachineInstr *, unsigned > > PHINodesToUpdate
PHINodesToUpdate - A list of phi instructions whose operand list will be updated after processing the...
CallLoweringInfo & setCallee(Type *ResultTy, FunctionType *FuncTy, const Value *Target, ArgListTy &&ArgsList, ImmutableCallSite &Call)
Definition: FastISel.h:105
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
Machine Value Type.
bool hasName() const
Definition: Value.h:251
LLVM Basic Block Representation.
Definition: BasicBlock.h:59
const MachineInstrBuilder & addCImm(const ConstantInt *Val) const
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
Simple binary floating point operators.
Definition: ISDOpcodes.h:260
Conditional or Unconditional Branch instruction.
Value * getAddress() const
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Definition: Constant.h:42
void removeDeadCode(MachineBasicBlock::iterator I, MachineBasicBlock::iterator E)
Remove all dead instructions between the I and E.
Definition: FastISel.cpp:543
Value * getValue() const
SmallVector< ISD::ArgFlagsTy, 16 > OutFlags
Definition: FastISel.h:96
This file contains the declarations for the subclasses of Constant, which represent the different fla...
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:187
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:264
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:371
const MCPhysReg * ImplicitDefs
Definition: MCInstrDesc.h:173
size_t size() const
Definition: BasicBlock.h:274
MachineFrameInfo & MFI
Definition: FastISel.h:207
DIExpression * getExpression() const
virtual unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1918
bool SkipTargetIndependentISel
Definition: FastISel.h:216
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
unsigned getCallFrameSetupOpcode() const
These methods return the opcode of the frame setup/destroy instructions if they exist (-1 otherwise)...
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:434
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:161
bool tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst)
We&#39;re checking to see if we can fold LI into FoldInst.
Definition: FastISel.cpp:2290
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
Definition: FastISel.cpp:145
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:685
std::pair< unsigned, bool > getRegForGEPIndex(const Value *Idx)
This is a wrapper around getRegForValue that also takes care of truncating or sign-extending the give...
Definition: FastISel.cpp:506
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, unsigned char TargetFlags=0)
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:728
const Triple & getTargetTriple() const
0 1 1 1 True if ordered (no nans)
Definition: InstrTypes.h:694
arg_iterator arg_begin()
Definition: Function.h:671
self_iterator getIterator()
Definition: ilist_node.h:82
The memory access is non-temporal.
Class to represent integer types.
Definition: DerivedTypes.h:40
bool selectXRayCustomEvent(const CallInst *II)
Definition: FastISel.cpp:1031
const TargetRegisterInfo & TRI
Definition: FastISel.h:214
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:702
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:194
static unsigned findSinkableLocalRegDef(MachineInstr &MI)
Return the defined register if this instruction defines exactly one virtual register and uses no othe...
Definition: FastISel.cpp:167
Extended Value Type.
Definition: ValueTypes.h:34
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg) const
For some targets, an LLVM struct type must be broken down into multiple simple types, but the calling convention specifies that the entire struct must be passed in a block of consecutive registers.
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs, and aliases.
Definition: Value.cpp:539
bool selectFNeg(const User *I)
Emit an FNeg operation.
Definition: FastISel.cpp:1694
size_t size() const
Definition: SmallVector.h:53
This class contains a discriminated union of information about pointers in memory operands...
1 1 0 1 True if unordered, less than, or equal
Definition: InstrTypes.h:700
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
SmallVector< Value *, 16 > OutVals
Definition: FastISel.h:95
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call...
Definition: FastISel.cpp:1073
const TargetInstrInfo & TII
Definition: FastISel.h:212
MachineBasicBlock * MBB
MBB - The current block.
bool isInTailCallPosition(ImmutableCallSite CS, const TargetMachine &TM)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition: Analysis.cpp:471
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
signed greater than
Definition: InstrTypes.h:712
MachineInstr * LastLocalValue
The position of the last instruction for materializing constants for use in the current block...
Definition: FastISel.h:222
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:633
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge&#39;s probability, relative to other out-edges of the Src.
virtual const MCPhysReg * getScratchRegisters(CallingConv::ID CC) const
Returns a 0 terminated array of registers that can be safely used as scratch registers.
void recomputeInsertPt()
Reset InsertPt to prepare for inserting instructions into the current block.
Definition: FastISel.cpp:529
The memory access writes data.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:51
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags...
0 0 1 0 True if ordered and greater than
Definition: InstrTypes.h:689
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:240
virtual unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1923
Iterator for intrusive lists based on ilist_node.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:418
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
This is the shared class of boolean and integer constants.
Definition: Constants.h:84
virtual unsigned fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t Imm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1938
DenseMap< unsigned, unsigned > RegFixups
RegFixups - Registers which need to be replaced after isel is done.
IterTy arg_begin() const
Definition: CallSite.h:571
MachineOperand class - Representation of each machine instruction operand.
1 1 0 0 True if unordered or less than
Definition: InstrTypes.h:699
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:847
bool selectCall(const User *I)
Definition: FastISel.cpp:1269
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
Definition: Instruction.h:64
Provides information about what library functions are available for the current target.
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
Definition: PPCPredicates.h:27
void finishCondBranch(const BasicBlock *BranchBB, MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB)
Emit an unconditional branch to FalseMBB, obtains the branch weight and adds TrueMBB and FalseMBB to ...
Definition: FastISel.cpp:1675
const TargetLibraryInfo * LibInfo
Definition: FastISel.h:215
unsigned getABITypeAlignment(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:722
bool isOSLinux() const
Tests whether the OS is Linux.
Definition: Triple.h:576
signed less than
Definition: InstrTypes.h:714
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:644
reg_iterator reg_begin(unsigned RegNo) const
unsigned fastEmitInst_rr(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill)
Emit a MachineInstr with two register operands and a result register in the given register class...
Definition: FastISel.cpp:2041
bool hasOptimizedCodeGen(LibFunc F) const
Tests if the function is both available and a candidate for optimized code generation.
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:621
void updateValueMap(const Value *I, unsigned Reg, unsigned NumRegs=1)
Update the value map to include the new mapping for this instruction, or insert an extra copy to get ...
Definition: FastISel.cpp:485
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
Definition: MCInstrDesc.h:225
MachineInstr * remove(MachineInstr *I)
Remove the unbundled instruction from the instruction list without deleting it.
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:194
void startNewBlock()
Set the current block to which generated machine instructions will be appended.
Definition: FastISel.cpp:129
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:133
signed less or equal
Definition: InstrTypes.h:715
bool selectBitCast(const User *I)
Definition: FastISel.cpp:1508
Target - Wrapper for Target specific information.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual unsigned fastEmit_f(MVT VT, MVT RetVT, unsigned Opcode, const ConstantFP *FPImm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1933
SmallVector< unsigned, 16 > OutRegs
Definition: FastISel.h:97
const DataLayout & DL
Definition: FastISel.h:211
bool selectBinaryOp(const User *I, unsigned ISDOpcode)
Select and emit code for a binary operator instruction, which has an opcode which directly correspond...
Definition: FastISel.cpp:574
void setDebugLoc(DebugLoc dl)
Replace current source information with new such.
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, ArrayRef< StringRef > StandardNames)
Initialize the set of available library functions based on the specified target triple.
BranchProbabilityInfo * BPI
This file defines the FastISel class.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:241
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:440
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
bool use_empty(unsigned RegNo) const
use_empty - Return true if there are no instructions using the specified register.
bool isTailCall() const
DebugLoc DbgLoc
Definition: FastISel.h:209
Flags
Flags values. These may be or&#39;d together.
amdgpu Simplify well known AMD library false Value Value * Arg
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:254
The memory access reads data.
uint64_t getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:560
SavePoint enterLocalValueArea()
Prepare InsertPt to begin inserting instructions into the local value area and return the old insert ...
Definition: FastISel.cpp:556
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:428
Representation of each machine instruction.
Definition: MachineInstr.h:64
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:759
virtual bool fastLowerIntrinsicCall(const IntrinsicInst *II)
This method is called by target-independent code to do target- specific intrinsic lowering...
Definition: FastISel.cpp:1912
unsigned getOperandNo() const
getOperandNo - Return the operand # of this MachineOperand in its MachineInstr.
bool selectPatchpoint(const CallInst *I)
Definition: FastISel.cpp:893
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:307
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:363
MachineRegisterInfo & MRI
Definition: FastISel.h:206
bool hasOneUse(unsigned RegNo) const
hasOneUse - Return true if there is exactly one instruction using the specified register.
uint64_t getElementOffset(unsigned Idx) const
Definition: DataLayout.h:543
MCSymbol * getOrCreateSymbol(const Twine &Name)
Lookup the symbol inside with the specified Name.
Definition: MCContext.cpp:123
unsigned getNumArgOperands() const
Return the number of call arguments.
unsigned greater or equal
Definition: InstrTypes.h:709
This represents the llvm.dbg.value instruction.
bool lowerCallTo(const CallInst *CI, MCSymbol *Symbol, unsigned NumArgs)
Definition: FastISel.cpp:1095
static bool isFNeg(const Value *V, bool IgnoreZeroSign=false)
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:45
StringRef getName() const
Return a constant reference to the value&#39;s name.
Definition: Value.cpp:224
TargetOptions Options
Definition: TargetMachine.h:97
Establish a view to a call site for examination.
Definition: CallSite.h:714
unsigned fastEmitInst_i(unsigned MachineInstOpcode, const TargetRegisterClass *RC, uint64_t Imm)
Emit a MachineInstr with a single immediate operand, and a result register in the given register clas...
Definition: FastISel.cpp:2185
static MachineOperand CreateImm(int64_t Val)
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
The memory access always returns the same value (or traps).
virtual unsigned fastMaterializeAlloca(const AllocaInst *C)
Emit an alloca address in a register using target-specific logic.
Definition: FastISel.h:485
iterator end()
Definition: DenseMap.h:80
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:193
static bool isRegUsedByPhiNodes(unsigned DefReg, FunctionLoweringInfo &FuncInfo)
Definition: FastISel.cpp:219
0 1 1 0 True if ordered and operands are unequal
Definition: InstrTypes.h:693
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
MachineBasicBlock::iterator InsertPt
MBB - The current insert position inside the current block.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:323
uint32_t Size
Definition: Profile.cpp:47
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:320
DILocalVariable * getVariable() const
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:91
bool isUnconditional() const
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block...
1 0 1 0 True if unordered or greater than
Definition: InstrTypes.h:697
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:309
unsigned getNumRegisters(LLVMContext &Context, EVT VT) const
Return the number of registers that this ValueType will eventually require.
LLVM_NODISCARD bool empty() const
Definition: DenseMap.h:94
Type * getType() const
Return the type of the instruction that generated this call site.
Definition: CallSite.h:264
const TargetLowering & TLI
Definition: FastISel.h:213
unsigned createResultReg(const TargetRegisterClass *RC)
Definition: FastISel.cpp:1990
CallLoweringInfo & setIsPatchPoint(bool Value=true)
Definition: FastISel.h:183
unsigned fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t Imm, MVT ImmType)
This method is a wrapper of fastEmit_ri.
Definition: FastISel.cpp:1947
iterator_range< use_instr_iterator > use_instructions(unsigned Reg) const
unsigned fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0, bool Op0IsKill, uint32_t Idx)
Emit a MachineInstr for an extract_subreg from a specified index of a superregister to a specified ty...
Definition: FastISel.cpp:2201
MachineBasicBlock::iterator InsertPt
Definition: FastISel.h:319
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
user_iterator user_begin()
Definition: Value.h:376
FastISel(FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo, bool SkipTargetIndependentISel=false)
Definition: FastISel.cpp:1895
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th call argument.
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
virtual bool fastLowerArguments()
This method is called by target-independent code to do target- specific argument lowering.
Definition: FastISel.cpp:1908
0 0 0 1 True if ordered and equal
Definition: InstrTypes.h:688
LLVM Value Representation.
Definition: Value.h:73
1 0 1 1 True if unordered, greater than, or equal
Definition: InstrTypes.h:698
uint64_t getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type...
Definition: DataLayout.h:411
FunctionType * getFunctionType() const
Definition: CallSite.h:320
static const Function * getParent(const Value *V)
const Value * getCalledValue() const
Get a pointer to the function that is invoked by this instruction.
void getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV, bool CannotUsePrivateLabel) const
Print the appropriate prefix and the specified global variable&#39;s name.
Definition: Mangler.cpp:112
DenseMap< const BasicBlock *, MachineBasicBlock * > MBBMap
MBBMap - A mapping from LLVM basic blocks to their machine code entry.
IRTranslator LLVM IR MI
bool hasOneUse() const
Return true if there is exactly one user of this value.
Definition: Value.h:413
void fastEmitBranch(MachineBasicBlock *MSucc, const DebugLoc &DbgLoc)
Emit an unconditional branch to the given block, unless it is the immediate (fall-through) successor...
Definition: FastISel.cpp:1655
unsigned greater than
Definition: InstrTypes.h:708
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:49
unsigned TrapUnreachable
Emit target-specific trap instruction for &#39;unreachable&#39; IR instructions.
virtual bool fastSelectInstruction(const Instruction *I)=0
This method is called by target-independent code when the normal FastISel process fails to select an ...
static cl::opt< bool > SinkLocalValues("fast-isel-sink-local-values", cl::init(true), cl::Hidden, cl::desc("Sink local values in FastISel"))
unsigned fastEmitInst_f(unsigned MachineInstOpcode, const TargetRegisterClass *RC, const ConstantFP *FPImm)
Emit a MachineInstr with a floating point immediate, and a result register in the given register clas...
Definition: FastISel.cpp:2140
bool isEmptyTy() const
Return true if this type is empty, that is, it has no elements or all of its elements are empty...
Definition: Type.cpp:98
Conversion operators.
Definition: ISDOpcodes.h:437
const TerminatorInst * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:138
FunctionLoweringInfo & FuncInfo
Definition: FastISel.h:204
const Value * stripInBoundsConstantOffsets() const
Strip off pointer casts and all-constant inbounds GEPs.
Definition: Value.cpp:547
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:446
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:126
#define LLVM_DEBUG(X)
Definition: Debug.h:123
iterator_range< use_instr_nodbg_iterator > use_nodbg_instructions(unsigned Reg) const
0 0 1 1 True if ordered and greater than or equal
Definition: InstrTypes.h:690
unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
Definition: Analysis.cpp:36
reg_begin/reg_end - Provide iteration support to walk over all definitions and uses of a register wit...
This represents the llvm.dbg.declare instruction.
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:554
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
static MachineOperand CreateFI(int Idx)
bool use_empty() const
Definition: Value.h:323
bool isSafeToMove(AliasAnalysis *AA, bool &SawStore) const
Return true if it is safe to move this instruction.
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:545
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Type * getElementType() const
Definition: DerivedTypes.h:486
unsigned createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute >> Attrs)
Create an AttributeList with the specified parameters in it.
Definition: Attributes.cpp:873
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:687
signed greater or equal
Definition: InstrTypes.h:713
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:44
This class contains meta information specific to a module.
This file describes how to lower LLVM code to machine code.
const BasicBlock * getParent() const
Definition: Instruction.h:67
unsigned InitializeRegForValue(const Value *V)
gep_type_iterator gep_type_begin(const User *GEP)