LLVM  9.0.0svn
FastISel.cpp
Go to the documentation of this file.
1 //===- FastISel.cpp - Implementation of the FastISel class ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the implementation of the FastISel class.
10 //
11 // "Fast" instruction selection is designed to emit very poor code quickly.
12 // Also, it is not designed to be able to do much lowering, so most illegal
13 // types (e.g. i64 on 32-bit targets) and operations are not supported. It is
14 // also not intended to be able to do much optimization, except in a few cases
15 // where doing optimizations reduces overall compile time. For example, folding
16 // constants into immediate fields is often done, because it's cheap and it
17 // reduces the number of instructions later phases have to examine.
18 //
19 // "Fast" instruction selection is able to fail gracefully and transfer
20 // control to the SelectionDAG selector for operations that it doesn't
21 // support. In many cases, this allows us to avoid duplicating a lot of
22 // the complicated lowering logic that SelectionDAG currently has.
23 //
24 // The intended use for "fast" instruction selection is "-O0" mode
25 // compilation, where the quality of the generated code is irrelevant when
26 // weighed against the speed at which the code can be generated. Also,
27 // at -O0, the LLVM optimizers are not running, and this makes the
28 // compile time of codegen a much higher portion of the overall compile
29 // time. Despite its limitations, "fast" instruction selection is able to
30 // handle enough code on its own to provide noticeable overall speedups
31 // in -O0 compiles.
32 //
33 // Basic operations are supported in a target-independent way, by reading
34 // the same instruction descriptions that the SelectionDAG selector reads,
35 // and identifying simple arithmetic operations that can be directly selected
36 // from simple operators. More complicated operations currently require
37 // target-specific code.
38 //
39 //===----------------------------------------------------------------------===//
40 
41 #include "llvm/CodeGen/FastISel.h"
42 #include "llvm/ADT/APFloat.h"
43 #include "llvm/ADT/APSInt.h"
44 #include "llvm/ADT/DenseMap.h"
45 #include "llvm/ADT/Optional.h"
46 #include "llvm/ADT/SmallPtrSet.h"
47 #include "llvm/ADT/SmallString.h"
48 #include "llvm/ADT/SmallVector.h"
49 #include "llvm/ADT/Statistic.h"
52 #include "llvm/CodeGen/Analysis.h"
63 #include "llvm/CodeGen/StackMaps.h"
68 #include "llvm/IR/Argument.h"
69 #include "llvm/IR/Attributes.h"
70 #include "llvm/IR/BasicBlock.h"
71 #include "llvm/IR/CallSite.h"
72 #include "llvm/IR/CallingConv.h"
73 #include "llvm/IR/Constant.h"
74 #include "llvm/IR/Constants.h"
75 #include "llvm/IR/DataLayout.h"
76 #include "llvm/IR/DebugInfo.h"
77 #include "llvm/IR/DebugLoc.h"
78 #include "llvm/IR/DerivedTypes.h"
79 #include "llvm/IR/Function.h"
81 #include "llvm/IR/GlobalValue.h"
82 #include "llvm/IR/InlineAsm.h"
83 #include "llvm/IR/InstrTypes.h"
84 #include "llvm/IR/Instruction.h"
85 #include "llvm/IR/Instructions.h"
86 #include "llvm/IR/IntrinsicInst.h"
87 #include "llvm/IR/LLVMContext.h"
88 #include "llvm/IR/Mangler.h"
89 #include "llvm/IR/Metadata.h"
90 #include "llvm/IR/Operator.h"
91 #include "llvm/IR/PatternMatch.h"
92 #include "llvm/IR/Type.h"
93 #include "llvm/IR/User.h"
94 #include "llvm/IR/Value.h"
95 #include "llvm/MC/MCContext.h"
96 #include "llvm/MC/MCInstrDesc.h"
97 #include "llvm/MC/MCRegisterInfo.h"
98 #include "llvm/Support/Casting.h"
99 #include "llvm/Support/Debug.h"
102 #include "llvm/Support/MathExtras.h"
106 #include <algorithm>
107 #include <cassert>
108 #include <cstdint>
109 #include <iterator>
110 #include <utility>
111 
112 using namespace llvm;
113 using namespace PatternMatch;
114 
115 #define DEBUG_TYPE "isel"
116 
117 // FIXME: Remove this after the feature has proven reliable.
118 static cl::opt<bool> SinkLocalValues("fast-isel-sink-local-values",
119  cl::init(true), cl::Hidden,
120  cl::desc("Sink local values in FastISel"));
121 
122 STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by "
123  "target-independent selector");
124 STATISTIC(NumFastIselSuccessTarget, "Number of insts selected by "
125  "target-specific selector");
126 STATISTIC(NumFastIselDead, "Number of dead insts removed on failure");
127 
128 /// Set the current block to which generated machine instructions will be
129 /// appended.
131  assert(LocalValueMap.empty() &&
132  "local values should be cleared after finishing a BB");
133 
134  // Instructions are appended to FuncInfo.MBB. If the basic block already
135  // contains labels or copies, use the last instruction as the last local
136  // value.
137  EmitStartPt = nullptr;
138  if (!FuncInfo.MBB->empty())
139  EmitStartPt = &FuncInfo.MBB->back();
140  LastLocalValue = EmitStartPt;
141 }
142 
143 /// Flush the local CSE map and sink anything we can.
144 void FastISel::finishBasicBlock() { flushLocalValueMap(); }
145 
147  if (!FuncInfo.CanLowerReturn)
148  // Fallback to SDISel argument lowering code to deal with sret pointer
149  // parameter.
150  return false;
151 
152  if (!fastLowerArguments())
153  return false;
154 
155  // Enter arguments into ValueMap for uses in non-entry BBs.
156  for (Function::const_arg_iterator I = FuncInfo.Fn->arg_begin(),
157  E = FuncInfo.Fn->arg_end();
158  I != E; ++I) {
159  DenseMap<const Value *, unsigned>::iterator VI = LocalValueMap.find(&*I);
160  assert(VI != LocalValueMap.end() && "Missed an argument?");
161  FuncInfo.ValueMap[&*I] = VI->second;
162  }
163  return true;
164 }
165 
166 /// Return the defined register if this instruction defines exactly one
167 /// virtual register and uses no other virtual registers. Otherwise return 0.
169  unsigned RegDef = 0;
170  for (const MachineOperand &MO : MI.operands()) {
171  if (!MO.isReg())
172  continue;
173  if (MO.isDef()) {
174  if (RegDef)
175  return 0;
176  RegDef = MO.getReg();
177  } else if (TargetRegisterInfo::isVirtualRegister(MO.getReg())) {
178  // This is another use of a vreg. Don't try to sink it.
179  return 0;
180  }
181  }
182  return RegDef;
183 }
184 
185 void FastISel::flushLocalValueMap() {
186  // Try to sink local values down to their first use so that we can give them a
187  // better debug location. This has the side effect of shrinking local value
188  // live ranges, which helps out fast regalloc.
189  if (SinkLocalValues && LastLocalValue != EmitStartPt) {
190  // Sink local value materialization instructions between EmitStartPt and
191  // LastLocalValue. Visit them bottom-up, starting from LastLocalValue, to
192  // avoid inserting into the range that we're iterating over.
194  EmitStartPt ? MachineBasicBlock::reverse_iterator(EmitStartPt)
195  : FuncInfo.MBB->rend();
196  MachineBasicBlock::reverse_iterator RI(LastLocalValue);
197 
198  InstOrderMap OrderMap;
199  for (; RI != RE;) {
200  MachineInstr &LocalMI = *RI;
201  ++RI;
202  bool Store = true;
203  if (!LocalMI.isSafeToMove(nullptr, Store))
204  continue;
205  unsigned DefReg = findSinkableLocalRegDef(LocalMI);
206  if (DefReg == 0)
207  continue;
208 
209  sinkLocalValueMaterialization(LocalMI, DefReg, OrderMap);
210  }
211  }
212 
213  LocalValueMap.clear();
214  LastLocalValue = EmitStartPt;
215  recomputeInsertPt();
216  SavedInsertPt = FuncInfo.InsertPt;
217  LastFlushPoint = FuncInfo.InsertPt;
218 }
219 
220 static bool isRegUsedByPhiNodes(unsigned DefReg,
221  FunctionLoweringInfo &FuncInfo) {
222  for (auto &P : FuncInfo.PHINodesToUpdate)
223  if (P.second == DefReg)
224  return true;
225  return false;
226 }
227 
228 /// Build a map of instruction orders. Return the first terminator and its
229 /// order. Consider EH_LABEL instructions to be terminators as well, since local
230 /// values for phis after invokes must be materialized before the call.
232  MachineBasicBlock *MBB, MachineBasicBlock::iterator LastFlushPoint) {
233  unsigned Order = 0;
234  for (MachineInstr &I : *MBB) {
235  if (!FirstTerminator &&
236  (I.isTerminator() || (I.isEHLabel() && &I != &MBB->front()))) {
237  FirstTerminator = &I;
238  FirstTerminatorOrder = Order;
239  }
240  Orders[&I] = Order++;
241 
242  // We don't need to order instructions past the last flush point.
243  if (I.getIterator() == LastFlushPoint)
244  break;
245  }
246 }
247 
248 void FastISel::sinkLocalValueMaterialization(MachineInstr &LocalMI,
249  unsigned DefReg,
250  InstOrderMap &OrderMap) {
251  // If this register is used by a register fixup, MRI will not contain all
252  // the uses until after register fixups, so don't attempt to sink or DCE
253  // this instruction. Register fixups typically come from no-op cast
254  // instructions, which replace the cast instruction vreg with the local
255  // value vreg.
256  if (FuncInfo.RegsWithFixups.count(DefReg))
257  return;
258 
259  // We can DCE this instruction if there are no uses and it wasn't a
260  // materialized for a successor PHI node.
261  bool UsedByPHI = isRegUsedByPhiNodes(DefReg, FuncInfo);
262  if (!UsedByPHI && MRI.use_nodbg_empty(DefReg)) {
263  if (EmitStartPt == &LocalMI)
264  EmitStartPt = EmitStartPt->getPrevNode();
265  LLVM_DEBUG(dbgs() << "removing dead local value materialization "
266  << LocalMI);
267  OrderMap.Orders.erase(&LocalMI);
268  LocalMI.eraseFromParent();
269  return;
270  }
271 
272  // Number the instructions if we haven't yet so we can efficiently find the
273  // earliest use.
274  if (OrderMap.Orders.empty())
275  OrderMap.initialize(FuncInfo.MBB, LastFlushPoint);
276 
277  // Find the first user in the BB.
278  MachineInstr *FirstUser = nullptr;
279  unsigned FirstOrder = std::numeric_limits<unsigned>::max();
280  for (MachineInstr &UseInst : MRI.use_nodbg_instructions(DefReg)) {
281  auto I = OrderMap.Orders.find(&UseInst);
282  assert(I != OrderMap.Orders.end() &&
283  "local value used by instruction outside local region");
284  unsigned UseOrder = I->second;
285  if (UseOrder < FirstOrder) {
286  FirstOrder = UseOrder;
287  FirstUser = &UseInst;
288  }
289  }
290 
291  // The insertion point will be the first terminator or the first user,
292  // whichever came first. If there was no terminator, this must be a
293  // fallthrough block and the insertion point is the end of the block.
295  if (UsedByPHI && OrderMap.FirstTerminatorOrder < FirstOrder) {
296  FirstOrder = OrderMap.FirstTerminatorOrder;
297  SinkPos = OrderMap.FirstTerminator->getIterator();
298  } else if (FirstUser) {
299  SinkPos = FirstUser->getIterator();
300  } else {
301  assert(UsedByPHI && "must be users if not used by a phi");
302  SinkPos = FuncInfo.MBB->instr_end();
303  }
304 
305  // Collect all DBG_VALUEs before the new insertion position so that we can
306  // sink them.
308  for (MachineInstr &DbgVal : MRI.use_instructions(DefReg)) {
309  if (!DbgVal.isDebugValue())
310  continue;
311  unsigned UseOrder = OrderMap.Orders[&DbgVal];
312  if (UseOrder < FirstOrder)
313  DbgValues.push_back(&DbgVal);
314  }
315 
316  // Sink LocalMI before SinkPos and assign it the same DebugLoc.
317  LLVM_DEBUG(dbgs() << "sinking local value to first use " << LocalMI);
318  FuncInfo.MBB->remove(&LocalMI);
319  FuncInfo.MBB->insert(SinkPos, &LocalMI);
320  if (SinkPos != FuncInfo.MBB->end())
321  LocalMI.setDebugLoc(SinkPos->getDebugLoc());
322 
323  // Sink any debug values that we've collected.
324  for (MachineInstr *DI : DbgValues) {
325  FuncInfo.MBB->remove(DI);
326  FuncInfo.MBB->insert(SinkPos, DI);
327  }
328 }
329 
331  // Don't consider constants or arguments to have trivial kills.
332  const Instruction *I = dyn_cast<Instruction>(V);
333  if (!I)
334  return false;
335 
336  // No-op casts are trivially coalesced by fast-isel.
337  if (const auto *Cast = dyn_cast<CastInst>(I))
338  if (Cast->isNoopCast(DL) && !hasTrivialKill(Cast->getOperand(0)))
339  return false;
340 
341  // Even the value might have only one use in the LLVM IR, it is possible that
342  // FastISel might fold the use into another instruction and now there is more
343  // than one use at the Machine Instruction level.
344  unsigned Reg = lookUpRegForValue(V);
345  if (Reg && !MRI.use_empty(Reg))
346  return false;
347 
348  // GEPs with all zero indices are trivially coalesced by fast-isel.
349  if (const auto *GEP = dyn_cast<GetElementPtrInst>(I))
350  if (GEP->hasAllZeroIndices() && !hasTrivialKill(GEP->getOperand(0)))
351  return false;
352 
353  // Only instructions with a single use in the same basic block are considered
354  // to have trivial kills.
355  return I->hasOneUse() &&
356  !(I->getOpcode() == Instruction::BitCast ||
357  I->getOpcode() == Instruction::PtrToInt ||
358  I->getOpcode() == Instruction::IntToPtr) &&
359  cast<Instruction>(*I->user_begin())->getParent() == I->getParent();
360 }
361 
362 unsigned FastISel::getRegForValue(const Value *V) {
363  EVT RealVT = TLI.getValueType(DL, V->getType(), /*AllowUnknown=*/true);
364  // Don't handle non-simple values in FastISel.
365  if (!RealVT.isSimple())
366  return 0;
367 
368  // Ignore illegal types. We must do this before looking up the value
369  // in ValueMap because Arguments are given virtual registers regardless
370  // of whether FastISel can handle them.
371  MVT VT = RealVT.getSimpleVT();
372  if (!TLI.isTypeLegal(VT)) {
373  // Handle integer promotions, though, because they're common and easy.
374  if (VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)
375  VT = TLI.getTypeToTransformTo(V->getContext(), VT).getSimpleVT();
376  else
377  return 0;
378  }
379 
380  // Look up the value to see if we already have a register for it.
381  unsigned Reg = lookUpRegForValue(V);
382  if (Reg)
383  return Reg;
384 
385  // In bottom-up mode, just create the virtual register which will be used
386  // to hold the value. It will be materialized later.
387  if (isa<Instruction>(V) &&
388  (!isa<AllocaInst>(V) ||
389  !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(V))))
390  return FuncInfo.InitializeRegForValue(V);
391 
392  SavePoint SaveInsertPt = enterLocalValueArea();
393 
394  // Materialize the value in a register. Emit any instructions in the
395  // local value area.
396  Reg = materializeRegForValue(V, VT);
397 
398  leaveLocalValueArea(SaveInsertPt);
399 
400  return Reg;
401 }
402 
403 unsigned FastISel::materializeConstant(const Value *V, MVT VT) {
404  unsigned Reg = 0;
405  if (const auto *CI = dyn_cast<ConstantInt>(V)) {
406  if (CI->getValue().getActiveBits() <= 64)
407  Reg = fastEmit_i(VT, VT, ISD::Constant, CI->getZExtValue());
408  } else if (isa<AllocaInst>(V))
409  Reg = fastMaterializeAlloca(cast<AllocaInst>(V));
410  else if (isa<ConstantPointerNull>(V))
411  // Translate this as an integer zero so that it can be
412  // local-CSE'd with actual integer zeros.
413  Reg = getRegForValue(
414  Constant::getNullValue(DL.getIntPtrType(V->getContext())));
415  else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
416  if (CF->isNullValue())
417  Reg = fastMaterializeFloatZero(CF);
418  else
419  // Try to emit the constant directly.
420  Reg = fastEmit_f(VT, VT, ISD::ConstantFP, CF);
421 
422  if (!Reg) {
423  // Try to emit the constant by using an integer constant with a cast.
424  const APFloat &Flt = CF->getValueAPF();
425  EVT IntVT = TLI.getPointerTy(DL);
426  uint32_t IntBitWidth = IntVT.getSizeInBits();
427  APSInt SIntVal(IntBitWidth, /*isUnsigned=*/false);
428  bool isExact;
429  (void)Flt.convertToInteger(SIntVal, APFloat::rmTowardZero, &isExact);
430  if (isExact) {
431  unsigned IntegerReg =
432  getRegForValue(ConstantInt::get(V->getContext(), SIntVal));
433  if (IntegerReg != 0)
434  Reg = fastEmit_r(IntVT.getSimpleVT(), VT, ISD::SINT_TO_FP, IntegerReg,
435  /*Kill=*/false);
436  }
437  }
438  } else if (const auto *Op = dyn_cast<Operator>(V)) {
439  if (!selectOperator(Op, Op->getOpcode()))
440  if (!isa<Instruction>(Op) ||
441  !fastSelectInstruction(cast<Instruction>(Op)))
442  return 0;
443  Reg = lookUpRegForValue(Op);
444  } else if (isa<UndefValue>(V)) {
445  Reg = createResultReg(TLI.getRegClassFor(VT));
446  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
447  TII.get(TargetOpcode::IMPLICIT_DEF), Reg);
448  }
449  return Reg;
450 }
451 
452 /// Helper for getRegForValue. This function is called when the value isn't
453 /// already available in a register and must be materialized with new
454 /// instructions.
455 unsigned FastISel::materializeRegForValue(const Value *V, MVT VT) {
456  unsigned Reg = 0;
457  // Give the target-specific code a try first.
458  if (isa<Constant>(V))
459  Reg = fastMaterializeConstant(cast<Constant>(V));
460 
461  // If target-specific code couldn't or didn't want to handle the value, then
462  // give target-independent code a try.
463  if (!Reg)
464  Reg = materializeConstant(V, VT);
465 
466  // Don't cache constant materializations in the general ValueMap.
467  // To do so would require tracking what uses they dominate.
468  if (Reg) {
469  LocalValueMap[V] = Reg;
470  LastLocalValue = MRI.getVRegDef(Reg);
471  }
472  return Reg;
473 }
474 
475 unsigned FastISel::lookUpRegForValue(const Value *V) {
476  // Look up the value to see if we already have a register for it. We
477  // cache values defined by Instructions across blocks, and other values
478  // only locally. This is because Instructions already have the SSA
479  // def-dominates-use requirement enforced.
480  DenseMap<const Value *, unsigned>::iterator I = FuncInfo.ValueMap.find(V);
481  if (I != FuncInfo.ValueMap.end())
482  return I->second;
483  return LocalValueMap[V];
484 }
485 
486 void FastISel::updateValueMap(const Value *I, unsigned Reg, unsigned NumRegs) {
487  if (!isa<Instruction>(I)) {
488  LocalValueMap[I] = Reg;
489  return;
490  }
491 
492  unsigned &AssignedReg = FuncInfo.ValueMap[I];
493  if (AssignedReg == 0)
494  // Use the new register.
495  AssignedReg = Reg;
496  else if (Reg != AssignedReg) {
497  // Arrange for uses of AssignedReg to be replaced by uses of Reg.
498  for (unsigned i = 0; i < NumRegs; i++) {
499  FuncInfo.RegFixups[AssignedReg + i] = Reg + i;
500  FuncInfo.RegsWithFixups.insert(Reg + i);
501  }
502 
503  AssignedReg = Reg;
504  }
505 }
506 
507 std::pair<unsigned, bool> FastISel::getRegForGEPIndex(const Value *Idx) {
508  unsigned IdxN = getRegForValue(Idx);
509  if (IdxN == 0)
510  // Unhandled operand. Halt "fast" selection and bail.
511  return std::pair<unsigned, bool>(0, false);
512 
513  bool IdxNIsKill = hasTrivialKill(Idx);
514 
515  // If the index is smaller or larger than intptr_t, truncate or extend it.
516  MVT PtrVT = TLI.getPointerTy(DL);
517  EVT IdxVT = EVT::getEVT(Idx->getType(), /*HandleUnknown=*/false);
518  if (IdxVT.bitsLT(PtrVT)) {
519  IdxN = fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::SIGN_EXTEND, IdxN,
520  IdxNIsKill);
521  IdxNIsKill = true;
522  } else if (IdxVT.bitsGT(PtrVT)) {
523  IdxN =
524  fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::TRUNCATE, IdxN, IdxNIsKill);
525  IdxNIsKill = true;
526  }
527  return std::pair<unsigned, bool>(IdxN, IdxNIsKill);
528 }
529 
531  if (getLastLocalValue()) {
532  FuncInfo.InsertPt = getLastLocalValue();
533  FuncInfo.MBB = FuncInfo.InsertPt->getParent();
534  ++FuncInfo.InsertPt;
535  } else
536  FuncInfo.InsertPt = FuncInfo.MBB->getFirstNonPHI();
537 
538  // Now skip past any EH_LABELs, which must remain at the beginning.
539  while (FuncInfo.InsertPt != FuncInfo.MBB->end() &&
540  FuncInfo.InsertPt->getOpcode() == TargetOpcode::EH_LABEL)
541  ++FuncInfo.InsertPt;
542 }
543 
546  assert(I.isValid() && E.isValid() && std::distance(I, E) > 0 &&
547  "Invalid iterator!");
548  while (I != E) {
549  if (LastFlushPoint == I)
550  LastFlushPoint = E;
551  if (SavedInsertPt == I)
552  SavedInsertPt = E;
553  if (EmitStartPt == I)
554  EmitStartPt = E.isValid() ? &*E : nullptr;
555  if (LastLocalValue == I)
556  LastLocalValue = E.isValid() ? &*E : nullptr;
557 
558  MachineInstr *Dead = &*I;
559  ++I;
560  Dead->eraseFromParent();
561  ++NumFastIselDead;
562  }
563  recomputeInsertPt();
564 }
565 
567  MachineBasicBlock::iterator OldInsertPt = FuncInfo.InsertPt;
568  DebugLoc OldDL = DbgLoc;
569  recomputeInsertPt();
570  DbgLoc = DebugLoc();
571  SavePoint SP = {OldInsertPt, OldDL};
572  return SP;
573 }
574 
576  if (FuncInfo.InsertPt != FuncInfo.MBB->begin())
577  LastLocalValue = &*std::prev(FuncInfo.InsertPt);
578 
579  // Restore the previous insert position.
580  FuncInfo.InsertPt = OldInsertPt.InsertPt;
581  DbgLoc = OldInsertPt.DL;
582 }
583 
584 bool FastISel::selectBinaryOp(const User *I, unsigned ISDOpcode) {
585  EVT VT = EVT::getEVT(I->getType(), /*HandleUnknown=*/true);
586  if (VT == MVT::Other || !VT.isSimple())
587  // Unhandled type. Halt "fast" selection and bail.
588  return false;
589 
590  // We only handle legal types. For example, on x86-32 the instruction
591  // selector contains all of the 64-bit instructions from x86-64,
592  // under the assumption that i64 won't be used if the target doesn't
593  // support it.
594  if (!TLI.isTypeLegal(VT)) {
595  // MVT::i1 is special. Allow AND, OR, or XOR because they
596  // don't require additional zeroing, which makes them easy.
597  if (VT == MVT::i1 && (ISDOpcode == ISD::AND || ISDOpcode == ISD::OR ||
598  ISDOpcode == ISD::XOR))
599  VT = TLI.getTypeToTransformTo(I->getContext(), VT);
600  else
601  return false;
602  }
603 
604  // Check if the first operand is a constant, and handle it as "ri". At -O0,
605  // we don't have anything that canonicalizes operand order.
606  if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(0)))
607  if (isa<Instruction>(I) && cast<Instruction>(I)->isCommutative()) {
608  unsigned Op1 = getRegForValue(I->getOperand(1));
609  if (!Op1)
610  return false;
611  bool Op1IsKill = hasTrivialKill(I->getOperand(1));
612 
613  unsigned ResultReg =
614  fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op1, Op1IsKill,
615  CI->getZExtValue(), VT.getSimpleVT());
616  if (!ResultReg)
617  return false;
618 
619  // We successfully emitted code for the given LLVM Instruction.
620  updateValueMap(I, ResultReg);
621  return true;
622  }
623 
624  unsigned Op0 = getRegForValue(I->getOperand(0));
625  if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
626  return false;
627  bool Op0IsKill = hasTrivialKill(I->getOperand(0));
628 
629  // Check if the second operand is a constant and handle it appropriately.
630  if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(1))) {
631  uint64_t Imm = CI->getSExtValue();
632 
633  // Transform "sdiv exact X, 8" -> "sra X, 3".
634  if (ISDOpcode == ISD::SDIV && isa<BinaryOperator>(I) &&
635  cast<BinaryOperator>(I)->isExact() && isPowerOf2_64(Imm)) {
636  Imm = Log2_64(Imm);
637  ISDOpcode = ISD::SRA;
638  }
639 
640  // Transform "urem x, pow2" -> "and x, pow2-1".
641  if (ISDOpcode == ISD::UREM && isa<BinaryOperator>(I) &&
642  isPowerOf2_64(Imm)) {
643  --Imm;
644  ISDOpcode = ISD::AND;
645  }
646 
647  unsigned ResultReg = fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op0,
648  Op0IsKill, Imm, VT.getSimpleVT());
649  if (!ResultReg)
650  return false;
651 
652  // We successfully emitted code for the given LLVM Instruction.
653  updateValueMap(I, ResultReg);
654  return true;
655  }
656 
657  unsigned Op1 = getRegForValue(I->getOperand(1));
658  if (!Op1) // Unhandled operand. Halt "fast" selection and bail.
659  return false;
660  bool Op1IsKill = hasTrivialKill(I->getOperand(1));
661 
662  // Now we have both operands in registers. Emit the instruction.
663  unsigned ResultReg = fastEmit_rr(VT.getSimpleVT(), VT.getSimpleVT(),
664  ISDOpcode, Op0, Op0IsKill, Op1, Op1IsKill);
665  if (!ResultReg)
666  // Target-specific code wasn't able to find a machine opcode for
667  // the given ISD opcode and type. Halt "fast" selection and bail.
668  return false;
669 
670  // We successfully emitted code for the given LLVM Instruction.
671  updateValueMap(I, ResultReg);
672  return true;
673 }
674 
676  unsigned N = getRegForValue(I->getOperand(0));
677  if (!N) // Unhandled operand. Halt "fast" selection and bail.
678  return false;
679  bool NIsKill = hasTrivialKill(I->getOperand(0));
680 
681  // Keep a running tab of the total offset to coalesce multiple N = N + Offset
682  // into a single N = N + TotalOffset.
683  uint64_t TotalOffs = 0;
684  // FIXME: What's a good SWAG number for MaxOffs?
685  uint64_t MaxOffs = 2048;
686  MVT VT = TLI.getPointerTy(DL);
687  for (gep_type_iterator GTI = gep_type_begin(I), E = gep_type_end(I);
688  GTI != E; ++GTI) {
689  const Value *Idx = GTI.getOperand();
690  if (StructType *StTy = GTI.getStructTypeOrNull()) {
691  uint64_t Field = cast<ConstantInt>(Idx)->getZExtValue();
692  if (Field) {
693  // N = N + Offset
694  TotalOffs += DL.getStructLayout(StTy)->getElementOffset(Field);
695  if (TotalOffs >= MaxOffs) {
696  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
697  if (!N) // Unhandled operand. Halt "fast" selection and bail.
698  return false;
699  NIsKill = true;
700  TotalOffs = 0;
701  }
702  }
703  } else {
704  Type *Ty = GTI.getIndexedType();
705 
706  // If this is a constant subscript, handle it quickly.
707  if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
708  if (CI->isZero())
709  continue;
710  // N = N + Offset
711  uint64_t IdxN = CI->getValue().sextOrTrunc(64).getSExtValue();
712  TotalOffs += DL.getTypeAllocSize(Ty) * IdxN;
713  if (TotalOffs >= MaxOffs) {
714  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
715  if (!N) // Unhandled operand. Halt "fast" selection and bail.
716  return false;
717  NIsKill = true;
718  TotalOffs = 0;
719  }
720  continue;
721  }
722  if (TotalOffs) {
723  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
724  if (!N) // Unhandled operand. Halt "fast" selection and bail.
725  return false;
726  NIsKill = true;
727  TotalOffs = 0;
728  }
729 
730  // N = N + Idx * ElementSize;
731  uint64_t ElementSize = DL.getTypeAllocSize(Ty);
732  std::pair<unsigned, bool> Pair = getRegForGEPIndex(Idx);
733  unsigned IdxN = Pair.first;
734  bool IdxNIsKill = Pair.second;
735  if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
736  return false;
737 
738  if (ElementSize != 1) {
739  IdxN = fastEmit_ri_(VT, ISD::MUL, IdxN, IdxNIsKill, ElementSize, VT);
740  if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
741  return false;
742  IdxNIsKill = true;
743  }
744  N = fastEmit_rr(VT, VT, ISD::ADD, N, NIsKill, IdxN, IdxNIsKill);
745  if (!N) // Unhandled operand. Halt "fast" selection and bail.
746  return false;
747  }
748  }
749  if (TotalOffs) {
750  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
751  if (!N) // Unhandled operand. Halt "fast" selection and bail.
752  return false;
753  }
754 
755  // We successfully emitted code for the given LLVM Instruction.
756  updateValueMap(I, N);
757  return true;
758 }
759 
760 bool FastISel::addStackMapLiveVars(SmallVectorImpl<MachineOperand> &Ops,
761  const CallInst *CI, unsigned StartIdx) {
762  for (unsigned i = StartIdx, e = CI->getNumArgOperands(); i != e; ++i) {
763  Value *Val = CI->getArgOperand(i);
764  // Check for constants and encode them with a StackMaps::ConstantOp prefix.
765  if (const auto *C = dyn_cast<ConstantInt>(Val)) {
766  Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
767  Ops.push_back(MachineOperand::CreateImm(C->getSExtValue()));
768  } else if (isa<ConstantPointerNull>(Val)) {
769  Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
771  } else if (auto *AI = dyn_cast<AllocaInst>(Val)) {
772  // Values coming from a stack location also require a special encoding,
773  // but that is added later on by the target specific frame index
774  // elimination implementation.
775  auto SI = FuncInfo.StaticAllocaMap.find(AI);
776  if (SI != FuncInfo.StaticAllocaMap.end())
777  Ops.push_back(MachineOperand::CreateFI(SI->second));
778  else
779  return false;
780  } else {
781  unsigned Reg = getRegForValue(Val);
782  if (!Reg)
783  return false;
784  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
785  }
786  }
787  return true;
788 }
789 
791  // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
792  // [live variables...])
794  "Stackmap cannot return a value.");
795 
796  // The stackmap intrinsic only records the live variables (the arguments
797  // passed to it) and emits NOPS (if requested). Unlike the patchpoint
798  // intrinsic, this won't be lowered to a function call. This means we don't
799  // have to worry about calling conventions and target-specific lowering code.
800  // Instead we perform the call lowering right here.
801  //
802  // CALLSEQ_START(0, 0...)
803  // STACKMAP(id, nbytes, ...)
804  // CALLSEQ_END(0, 0)
805  //
807 
808  // Add the <id> and <numBytes> constants.
809  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
810  "Expected a constant integer.");
811  const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
812  Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
813 
814  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
815  "Expected a constant integer.");
816  const auto *NumBytes =
817  cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
818  Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
819 
820  // Push live variables for the stack map (skipping the first two arguments
821  // <id> and <numBytes>).
822  if (!addStackMapLiveVars(Ops, I, 2))
823  return false;
824 
825  // We are not adding any register mask info here, because the stackmap doesn't
826  // clobber anything.
827 
828  // Add scratch registers as implicit def and early clobber.
830  const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
831  for (unsigned i = 0; ScratchRegs[i]; ++i)
832  Ops.push_back(MachineOperand::CreateReg(
833  ScratchRegs[i], /*IsDef=*/true, /*IsImp=*/true, /*IsKill=*/false,
834  /*IsDead=*/false, /*IsUndef=*/false, /*IsEarlyClobber=*/true));
835 
836  // Issue CALLSEQ_START
837  unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
838  auto Builder =
839  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackDown));
840  const MCInstrDesc &MCID = Builder.getInstr()->getDesc();
841  for (unsigned I = 0, E = MCID.getNumOperands(); I < E; ++I)
842  Builder.addImm(0);
843 
844  // Issue STACKMAP.
845  MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
846  TII.get(TargetOpcode::STACKMAP));
847  for (auto const &MO : Ops)
848  MIB.add(MO);
849 
850  // Issue CALLSEQ_END
851  unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
852  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackUp))
853  .addImm(0)
854  .addImm(0);
855 
856  // Inform the Frame Information that we have a stackmap in this function.
857  FuncInfo.MF->getFrameInfo().setHasStackMap();
858 
859  return true;
860 }
861 
862 /// Lower an argument list according to the target calling convention.
863 ///
864 /// This is a helper for lowering intrinsics that follow a target calling
865 /// convention or require stack pointer adjustment. Only a subset of the
866 /// intrinsic's operands need to participate in the calling convention.
867 bool FastISel::lowerCallOperands(const CallInst *CI, unsigned ArgIdx,
868  unsigned NumArgs, const Value *Callee,
869  bool ForceRetVoidTy, CallLoweringInfo &CLI) {
870  ArgListTy Args;
871  Args.reserve(NumArgs);
872 
873  // Populate the argument list.
874  ImmutableCallSite CS(CI);
875  for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI) {
876  Value *V = CI->getOperand(ArgI);
877 
878  assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
879 
881  Entry.Val = V;
882  Entry.Ty = V->getType();
883  Entry.setAttributes(&CS, ArgI);
884  Args.push_back(Entry);
885  }
886 
887  Type *RetTy = ForceRetVoidTy ? Type::getVoidTy(CI->getType()->getContext())
888  : CI->getType();
889  CLI.setCallee(CI->getCallingConv(), RetTy, Callee, std::move(Args), NumArgs);
890 
891  return lowerCallTo(CLI);
892 }
893 
895  const DataLayout &DL, MCContext &Ctx, CallingConv::ID CC, Type *ResultTy,
896  StringRef Target, ArgListTy &&ArgsList, unsigned FixedArgs) {
897  SmallString<32> MangledName;
898  Mangler::getNameWithPrefix(MangledName, Target, DL);
899  MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
900  return setCallee(CC, ResultTy, Sym, std::move(ArgsList), FixedArgs);
901 }
902 
904  // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
905  // i32 <numBytes>,
906  // i8* <target>,
907  // i32 <numArgs>,
908  // [Args...],
909  // [live variables...])
911  bool IsAnyRegCC = CC == CallingConv::AnyReg;
912  bool HasDef = !I->getType()->isVoidTy();
914 
915  // Get the real number of arguments participating in the call <numArgs>
916  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)) &&
917  "Expected a constant integer.");
918  const auto *NumArgsVal =
919  cast<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos));
920  unsigned NumArgs = NumArgsVal->getZExtValue();
921 
922  // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
923  // This includes all meta-operands up to but not including CC.
924  unsigned NumMetaOpers = PatchPointOpers::CCPos;
925  assert(I->getNumArgOperands() >= NumMetaOpers + NumArgs &&
926  "Not enough arguments provided to the patchpoint intrinsic");
927 
928  // For AnyRegCC the arguments are lowered later on manually.
929  unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
930  CallLoweringInfo CLI;
931  CLI.setIsPatchPoint();
932  if (!lowerCallOperands(I, NumMetaOpers, NumCallArgs, Callee, IsAnyRegCC, CLI))
933  return false;
934 
935  assert(CLI.Call && "No call instruction specified.");
936 
938 
939  // Add an explicit result reg if we use the anyreg calling convention.
940  if (IsAnyRegCC && HasDef) {
941  assert(CLI.NumResultRegs == 0 && "Unexpected result register.");
942  CLI.ResultReg = createResultReg(TLI.getRegClassFor(MVT::i64));
943  CLI.NumResultRegs = 1;
944  Ops.push_back(MachineOperand::CreateReg(CLI.ResultReg, /*IsDef=*/true));
945  }
946 
947  // Add the <id> and <numBytes> constants.
948  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
949  "Expected a constant integer.");
950  const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
951  Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
952 
953  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
954  "Expected a constant integer.");
955  const auto *NumBytes =
956  cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
957  Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
958 
959  // Add the call target.
960  if (const auto *C = dyn_cast<IntToPtrInst>(Callee)) {
961  uint64_t CalleeConstAddr =
962  cast<ConstantInt>(C->getOperand(0))->getZExtValue();
963  Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
964  } else if (const auto *C = dyn_cast<ConstantExpr>(Callee)) {
965  if (C->getOpcode() == Instruction::IntToPtr) {
966  uint64_t CalleeConstAddr =
967  cast<ConstantInt>(C->getOperand(0))->getZExtValue();
968  Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
969  } else
970  llvm_unreachable("Unsupported ConstantExpr.");
971  } else if (const auto *GV = dyn_cast<GlobalValue>(Callee)) {
973  } else if (isa<ConstantPointerNull>(Callee))
975  else
976  llvm_unreachable("Unsupported callee address.");
977 
978  // Adjust <numArgs> to account for any arguments that have been passed on
979  // the stack instead.
980  unsigned NumCallRegArgs = IsAnyRegCC ? NumArgs : CLI.OutRegs.size();
981  Ops.push_back(MachineOperand::CreateImm(NumCallRegArgs));
982 
983  // Add the calling convention
984  Ops.push_back(MachineOperand::CreateImm((unsigned)CC));
985 
986  // Add the arguments we omitted previously. The register allocator should
987  // place these in any free register.
988  if (IsAnyRegCC) {
989  for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) {
990  unsigned Reg = getRegForValue(I->getArgOperand(i));
991  if (!Reg)
992  return false;
993  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
994  }
995  }
996 
997  // Push the arguments from the call instruction.
998  for (auto Reg : CLI.OutRegs)
999  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
1000 
1001  // Push live variables for the stack map.
1002  if (!addStackMapLiveVars(Ops, I, NumMetaOpers + NumArgs))
1003  return false;
1004 
1005  // Push the register mask info.
1007  TRI.getCallPreservedMask(*FuncInfo.MF, CC)));
1008 
1009  // Add scratch registers as implicit def and early clobber.
1010  const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
1011  for (unsigned i = 0; ScratchRegs[i]; ++i)
1013  ScratchRegs[i], /*IsDef=*/true, /*IsImp=*/true, /*IsKill=*/false,
1014  /*IsDead=*/false, /*IsUndef=*/false, /*IsEarlyClobber=*/true));
1015 
1016  // Add implicit defs (return values).
1017  for (auto Reg : CLI.InRegs)
1018  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/true,
1019  /*IsImpl=*/true));
1020 
1021  // Insert the patchpoint instruction before the call generated by the target.
1022  MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, CLI.Call, DbgLoc,
1023  TII.get(TargetOpcode::PATCHPOINT));
1024 
1025  for (auto &MO : Ops)
1026  MIB.add(MO);
1027 
1028  MIB->setPhysRegsDeadExcept(CLI.InRegs, TRI);
1029 
1030  // Delete the original call instruction.
1031  CLI.Call->eraseFromParent();
1032 
1033  // Inform the Frame Information that we have a patchpoint in this function.
1034  FuncInfo.MF->getFrameInfo().setHasPatchPoint();
1035 
1036  if (CLI.NumResultRegs)
1037  updateValueMap(I, CLI.ResultReg, CLI.NumResultRegs);
1038  return true;
1039 }
1040 
1042  const auto &Triple = TM.getTargetTriple();
1043  if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
1044  return true; // don't do anything to this instruction.
1046  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(0)),
1047  /*IsDef=*/false));
1048  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(1)),
1049  /*IsDef=*/false));
1050  MachineInstrBuilder MIB =
1051  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1052  TII.get(TargetOpcode::PATCHABLE_EVENT_CALL));
1053  for (auto &MO : Ops)
1054  MIB.add(MO);
1055 
1056  // Insert the Patchable Event Call instruction, that gets lowered properly.
1057  return true;
1058 }
1059 
1061  const auto &Triple = TM.getTargetTriple();
1062  if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
1063  return true; // don't do anything to this instruction.
1065  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(0)),
1066  /*IsDef=*/false));
1067  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(1)),
1068  /*IsDef=*/false));
1069  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(2)),
1070  /*IsDef=*/false));
1071  MachineInstrBuilder MIB =
1072  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1073  TII.get(TargetOpcode::PATCHABLE_TYPED_EVENT_CALL));
1074  for (auto &MO : Ops)
1075  MIB.add(MO);
1076 
1077  // Insert the Patchable Typed Event Call instruction, that gets lowered properly.
1078  return true;
1079 }
1080 
1081 /// Returns an AttributeList representing the attributes applied to the return
1082 /// value of the given call.
1085  if (CLI.RetSExt)
1086  Attrs.push_back(Attribute::SExt);
1087  if (CLI.RetZExt)
1088  Attrs.push_back(Attribute::ZExt);
1089  if (CLI.IsInReg)
1090  Attrs.push_back(Attribute::InReg);
1091 
1093  Attrs);
1094 }
1095 
1096 bool FastISel::lowerCallTo(const CallInst *CI, const char *SymName,
1097  unsigned NumArgs) {
1098  MCContext &Ctx = MF->getContext();
1099  SmallString<32> MangledName;
1100  Mangler::getNameWithPrefix(MangledName, SymName, DL);
1101  MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
1102  return lowerCallTo(CI, Sym, NumArgs);
1103 }
1104 
1106  unsigned NumArgs) {
1107  ImmutableCallSite CS(CI);
1108 
1109  FunctionType *FTy = CS.getFunctionType();
1110  Type *RetTy = CS.getType();
1111 
1112  ArgListTy Args;
1113  Args.reserve(NumArgs);
1114 
1115  // Populate the argument list.
1116  // Attributes for args start at offset 1, after the return attribute.
1117  for (unsigned ArgI = 0; ArgI != NumArgs; ++ArgI) {
1118  Value *V = CI->getOperand(ArgI);
1119 
1120  assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
1121 
1123  Entry.Val = V;
1124  Entry.Ty = V->getType();
1125  Entry.setAttributes(&CS, ArgI);
1126  Args.push_back(Entry);
1127  }
1128  TLI.markLibCallAttributes(MF, CS.getCallingConv(), Args);
1129 
1130  CallLoweringInfo CLI;
1131  CLI.setCallee(RetTy, FTy, Symbol, std::move(Args), CS, NumArgs);
1132 
1133  return lowerCallTo(CLI);
1134 }
1135 
1137  // Handle the incoming return values from the call.
1138  CLI.clearIns();
1139  SmallVector<EVT, 4> RetTys;
1140  ComputeValueVTs(TLI, DL, CLI.RetTy, RetTys);
1141 
1143  GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, TLI, DL);
1144 
1145  bool CanLowerReturn = TLI.CanLowerReturn(
1146  CLI.CallConv, *FuncInfo.MF, CLI.IsVarArg, Outs, CLI.RetTy->getContext());
1147 
1148  // FIXME: sret demotion isn't supported yet - bail out.
1149  if (!CanLowerReturn)
1150  return false;
1151 
1152  for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
1153  EVT VT = RetTys[I];
1154  MVT RegisterVT = TLI.getRegisterType(CLI.RetTy->getContext(), VT);
1155  unsigned NumRegs = TLI.getNumRegisters(CLI.RetTy->getContext(), VT);
1156  for (unsigned i = 0; i != NumRegs; ++i) {
1157  ISD::InputArg MyFlags;
1158  MyFlags.VT = RegisterVT;
1159  MyFlags.ArgVT = VT;
1160  MyFlags.Used = CLI.IsReturnValueUsed;
1161  if (CLI.RetSExt)
1162  MyFlags.Flags.setSExt();
1163  if (CLI.RetZExt)
1164  MyFlags.Flags.setZExt();
1165  if (CLI.IsInReg)
1166  MyFlags.Flags.setInReg();
1167  CLI.Ins.push_back(MyFlags);
1168  }
1169  }
1170 
1171  // Handle all of the outgoing arguments.
1172  CLI.clearOuts();
1173  for (auto &Arg : CLI.getArgs()) {
1174  Type *FinalType = Arg.Ty;
1175  if (Arg.IsByVal)
1176  FinalType = cast<PointerType>(Arg.Ty)->getElementType();
1177  bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1178  FinalType, CLI.CallConv, CLI.IsVarArg);
1179 
1180  ISD::ArgFlagsTy Flags;
1181  if (Arg.IsZExt)
1182  Flags.setZExt();
1183  if (Arg.IsSExt)
1184  Flags.setSExt();
1185  if (Arg.IsInReg)
1186  Flags.setInReg();
1187  if (Arg.IsSRet)
1188  Flags.setSRet();
1189  if (Arg.IsSwiftSelf)
1190  Flags.setSwiftSelf();
1191  if (Arg.IsSwiftError)
1192  Flags.setSwiftError();
1193  if (Arg.IsByVal)
1194  Flags.setByVal();
1195  if (Arg.IsInAlloca) {
1196  Flags.setInAlloca();
1197  // Set the byval flag for CCAssignFn callbacks that don't know about
1198  // inalloca. This way we can know how many bytes we should've allocated
1199  // and how many bytes a callee cleanup function will pop. If we port
1200  // inalloca to more targets, we'll have to add custom inalloca handling in
1201  // the various CC lowering callbacks.
1202  Flags.setByVal();
1203  }
1204  if (Arg.IsByVal || Arg.IsInAlloca) {
1205  PointerType *Ty = cast<PointerType>(Arg.Ty);
1206  Type *ElementTy = Ty->getElementType();
1207  unsigned FrameSize = DL.getTypeAllocSize(ElementTy);
1208  // For ByVal, alignment should come from FE. BE will guess if this info is
1209  // not there, but there are cases it cannot get right.
1210  unsigned FrameAlign = Arg.Alignment;
1211  if (!FrameAlign)
1212  FrameAlign = TLI.getByValTypeAlignment(ElementTy, DL);
1213  Flags.setByValSize(FrameSize);
1214  Flags.setByValAlign(FrameAlign);
1215  }
1216  if (Arg.IsNest)
1217  Flags.setNest();
1218  if (NeedsRegBlock)
1219  Flags.setInConsecutiveRegs();
1220  unsigned OriginalAlignment = DL.getABITypeAlignment(Arg.Ty);
1221  Flags.setOrigAlign(OriginalAlignment);
1222 
1223  CLI.OutVals.push_back(Arg.Val);
1224  CLI.OutFlags.push_back(Flags);
1225  }
1226 
1227  if (!fastLowerCall(CLI))
1228  return false;
1229 
1230  // Set all unused physreg defs as dead.
1231  assert(CLI.Call && "No call instruction specified.");
1232  CLI.Call->setPhysRegsDeadExcept(CLI.InRegs, TRI);
1233 
1234  if (CLI.NumResultRegs && CLI.CS)
1235  updateValueMap(CLI.CS->getInstruction(), CLI.ResultReg, CLI.NumResultRegs);
1236 
1237  // Set labels for heapallocsite call.
1238  if (CLI.CS && CLI.CS->getInstruction()->getMetadata("heapallocsite")) {
1239  MDNode *MD = CLI.CS->getInstruction()->getMetadata("heapallocsite");
1240  MF->addCodeViewHeapAllocSite(CLI.Call, MD);
1241  }
1242 
1243  return true;
1244 }
1245 
1247  ImmutableCallSite CS(CI);
1248 
1249  FunctionType *FuncTy = CS.getFunctionType();
1250  Type *RetTy = CS.getType();
1251 
1252  ArgListTy Args;
1254  Args.reserve(CS.arg_size());
1255 
1256  for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
1257  i != e; ++i) {
1258  Value *V = *i;
1259 
1260  // Skip empty types
1261  if (V->getType()->isEmptyTy())
1262  continue;
1263 
1264  Entry.Val = V;
1265  Entry.Ty = V->getType();
1266 
1267  // Skip the first return-type Attribute to get to params.
1268  Entry.setAttributes(&CS, i - CS.arg_begin());
1269  Args.push_back(Entry);
1270  }
1271 
1272  // Check if target-independent constraints permit a tail call here.
1273  // Target-dependent constraints are checked within fastLowerCall.
1274  bool IsTailCall = CI->isTailCall();
1275  if (IsTailCall && !isInTailCallPosition(CS, TM))
1276  IsTailCall = false;
1277 
1278  CallLoweringInfo CLI;
1279  CLI.setCallee(RetTy, FuncTy, CI->getCalledValue(), std::move(Args), CS)
1280  .setTailCall(IsTailCall);
1281 
1282  return lowerCallTo(CLI);
1283 }
1284 
1286  const CallInst *Call = cast<CallInst>(I);
1287 
1288  // Handle simple inline asms.
1289  if (const InlineAsm *IA = dyn_cast<InlineAsm>(Call->getCalledValue())) {
1290  // If the inline asm has side effects, then make sure that no local value
1291  // lives across by flushing the local value map.
1292  if (IA->hasSideEffects())
1293  flushLocalValueMap();
1294 
1295  // Don't attempt to handle constraints.
1296  if (!IA->getConstraintString().empty())
1297  return false;
1298 
1299  unsigned ExtraInfo = 0;
1300  if (IA->hasSideEffects())
1301  ExtraInfo |= InlineAsm::Extra_HasSideEffects;
1302  if (IA->isAlignStack())
1303  ExtraInfo |= InlineAsm::Extra_IsAlignStack;
1304 
1305  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1307  .addExternalSymbol(IA->getAsmString().c_str())
1308  .addImm(ExtraInfo);
1309  return true;
1310  }
1311 
1312  // Handle intrinsic function calls.
1313  if (const auto *II = dyn_cast<IntrinsicInst>(Call))
1314  return selectIntrinsicCall(II);
1315 
1316  // Usually, it does not make sense to initialize a value,
1317  // make an unrelated function call and use the value, because
1318  // it tends to be spilled on the stack. So, we move the pointer
1319  // to the last local value to the beginning of the block, so that
1320  // all the values which have already been materialized,
1321  // appear after the call. It also makes sense to skip intrinsics
1322  // since they tend to be inlined.
1323  flushLocalValueMap();
1324 
1325  return lowerCall(Call);
1326 }
1327 
1329  switch (II->getIntrinsicID()) {
1330  default:
1331  break;
1332  // At -O0 we don't care about the lifetime intrinsics.
1333  case Intrinsic::lifetime_start:
1334  case Intrinsic::lifetime_end:
1335  // The donothing intrinsic does, well, nothing.
1336  case Intrinsic::donothing:
1337  // Neither does the sideeffect intrinsic.
1338  case Intrinsic::sideeffect:
1339  // Neither does the assume intrinsic; it's also OK not to codegen its operand.
1340  case Intrinsic::assume:
1341  return true;
1342  case Intrinsic::dbg_declare: {
1343  const DbgDeclareInst *DI = cast<DbgDeclareInst>(II);
1344  assert(DI->getVariable() && "Missing variable");
1345  if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1346  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1347  return true;
1348  }
1349 
1350  const Value *Address = DI->getAddress();
1351  if (!Address || isa<UndefValue>(Address)) {
1352  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1353  return true;
1354  }
1355 
1356  // Byval arguments with frame indices were already handled after argument
1357  // lowering and before isel.
1358  const auto *Arg =
1360  if (Arg && FuncInfo.getArgumentFrameIndex(Arg) != INT_MAX)
1361  return true;
1362 
1364  if (unsigned Reg = lookUpRegForValue(Address))
1365  Op = MachineOperand::CreateReg(Reg, false);
1366 
1367  // If we have a VLA that has a "use" in a metadata node that's then used
1368  // here but it has no other uses, then we have a problem. E.g.,
1369  //
1370  // int foo (const int *x) {
1371  // char a[*x];
1372  // return 0;
1373  // }
1374  //
1375  // If we assign 'a' a vreg and fast isel later on has to use the selection
1376  // DAG isel, it will want to copy the value to the vreg. However, there are
1377  // no uses, which goes counter to what selection DAG isel expects.
1378  if (!Op && !Address->use_empty() && isa<Instruction>(Address) &&
1379  (!isa<AllocaInst>(Address) ||
1380  !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(Address))))
1381  Op = MachineOperand::CreateReg(FuncInfo.InitializeRegForValue(Address),
1382  false);
1383 
1384  if (Op) {
1386  "Expected inlined-at fields to agree");
1387  // A dbg.declare describes the address of a source variable, so lower it
1388  // into an indirect DBG_VALUE.
1389  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1390  TII.get(TargetOpcode::DBG_VALUE), /*IsIndirect*/ true,
1391  *Op, DI->getVariable(), DI->getExpression());
1392  } else {
1393  // We can't yet handle anything else here because it would require
1394  // generating code, thus altering codegen because of debug info.
1395  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1396  }
1397  return true;
1398  }
1399  case Intrinsic::dbg_value: {
1400  // This form of DBG_VALUE is target-independent.
1401  const DbgValueInst *DI = cast<DbgValueInst>(II);
1402  const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE);
1403  const Value *V = DI->getValue();
1405  "Expected inlined-at fields to agree");
1406  if (!V) {
1407  // Currently the optimizer can produce this; insert an undef to
1408  // help debugging. Probably the optimizer should not do this.
1409  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, false, 0U,
1410  DI->getVariable(), DI->getExpression());
1411  } else if (const auto *CI = dyn_cast<ConstantInt>(V)) {
1412  if (CI->getBitWidth() > 64)
1413  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
1414  .addCImm(CI)
1415  .addImm(0U)
1416  .addMetadata(DI->getVariable())
1417  .addMetadata(DI->getExpression());
1418  else
1419  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
1420  .addImm(CI->getZExtValue())
1421  .addImm(0U)
1422  .addMetadata(DI->getVariable())
1423  .addMetadata(DI->getExpression());
1424  } else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
1425  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
1426  .addFPImm(CF)
1427  .addImm(0U)
1428  .addMetadata(DI->getVariable())
1429  .addMetadata(DI->getExpression());
1430  } else if (unsigned Reg = lookUpRegForValue(V)) {
1431  // FIXME: This does not handle register-indirect values at offset 0.
1432  bool IsIndirect = false;
1433  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, IsIndirect, Reg,
1434  DI->getVariable(), DI->getExpression());
1435  } else {
1436  // We can't yet handle anything else here because it would require
1437  // generating code, thus altering codegen because of debug info.
1438  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1439  }
1440  return true;
1441  }
1442  case Intrinsic::dbg_label: {
1443  const DbgLabelInst *DI = cast<DbgLabelInst>(II);
1444  assert(DI->getLabel() && "Missing label");
1445  if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1446  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1447  return true;
1448  }
1449 
1450  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1451  TII.get(TargetOpcode::DBG_LABEL)).addMetadata(DI->getLabel());
1452  return true;
1453  }
1454  case Intrinsic::objectsize: {
1455  ConstantInt *CI = cast<ConstantInt>(II->getArgOperand(1));
1456  unsigned long long Res = CI->isZero() ? -1ULL : 0;
1457  Constant *ResCI = ConstantInt::get(II->getType(), Res);
1458  unsigned ResultReg = getRegForValue(ResCI);
1459  if (!ResultReg)
1460  return false;
1461  updateValueMap(II, ResultReg);
1462  return true;
1463  }
1464  case Intrinsic::is_constant: {
1465  Constant *ResCI = ConstantInt::get(II->getType(), 0);
1466  unsigned ResultReg = getRegForValue(ResCI);
1467  if (!ResultReg)
1468  return false;
1469  updateValueMap(II, ResultReg);
1470  return true;
1471  }
1472  case Intrinsic::launder_invariant_group:
1473  case Intrinsic::strip_invariant_group:
1474  case Intrinsic::expect: {
1475  unsigned ResultReg = getRegForValue(II->getArgOperand(0));
1476  if (!ResultReg)
1477  return false;
1478  updateValueMap(II, ResultReg);
1479  return true;
1480  }
1481  case Intrinsic::experimental_stackmap:
1482  return selectStackmap(II);
1483  case Intrinsic::experimental_patchpoint_void:
1484  case Intrinsic::experimental_patchpoint_i64:
1485  return selectPatchpoint(II);
1486 
1487  case Intrinsic::xray_customevent:
1488  return selectXRayCustomEvent(II);
1489  case Intrinsic::xray_typedevent:
1490  return selectXRayTypedEvent(II);
1491  }
1492 
1493  return fastLowerIntrinsicCall(II);
1494 }
1495 
1496 bool FastISel::selectCast(const User *I, unsigned Opcode) {
1497  EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1498  EVT DstVT = TLI.getValueType(DL, I->getType());
1499 
1500  if (SrcVT == MVT::Other || !SrcVT.isSimple() || DstVT == MVT::Other ||
1501  !DstVT.isSimple())
1502  // Unhandled type. Halt "fast" selection and bail.
1503  return false;
1504 
1505  // Check if the destination type is legal.
1506  if (!TLI.isTypeLegal(DstVT))
1507  return false;
1508 
1509  // Check if the source operand is legal.
1510  if (!TLI.isTypeLegal(SrcVT))
1511  return false;
1512 
1513  unsigned InputReg = getRegForValue(I->getOperand(0));
1514  if (!InputReg)
1515  // Unhandled operand. Halt "fast" selection and bail.
1516  return false;
1517 
1518  bool InputRegIsKill = hasTrivialKill(I->getOperand(0));
1519 
1520  unsigned ResultReg = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(),
1521  Opcode, InputReg, InputRegIsKill);
1522  if (!ResultReg)
1523  return false;
1524 
1525  updateValueMap(I, ResultReg);
1526  return true;
1527 }
1528 
1530  // If the bitcast doesn't change the type, just use the operand value.
1531  if (I->getType() == I->getOperand(0)->getType()) {
1532  unsigned Reg = getRegForValue(I->getOperand(0));
1533  if (!Reg)
1534  return false;
1535  updateValueMap(I, Reg);
1536  return true;
1537  }
1538 
1539  // Bitcasts of other values become reg-reg copies or BITCAST operators.
1540  EVT SrcEVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1541  EVT DstEVT = TLI.getValueType(DL, I->getType());
1542  if (SrcEVT == MVT::Other || DstEVT == MVT::Other ||
1543  !TLI.isTypeLegal(SrcEVT) || !TLI.isTypeLegal(DstEVT))
1544  // Unhandled type. Halt "fast" selection and bail.
1545  return false;
1546 
1547  MVT SrcVT = SrcEVT.getSimpleVT();
1548  MVT DstVT = DstEVT.getSimpleVT();
1549  unsigned Op0 = getRegForValue(I->getOperand(0));
1550  if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
1551  return false;
1552  bool Op0IsKill = hasTrivialKill(I->getOperand(0));
1553 
1554  // First, try to perform the bitcast by inserting a reg-reg copy.
1555  unsigned ResultReg = 0;
1556  if (SrcVT == DstVT) {
1557  const TargetRegisterClass *SrcClass = TLI.getRegClassFor(SrcVT);
1558  const TargetRegisterClass *DstClass = TLI.getRegClassFor(DstVT);
1559  // Don't attempt a cross-class copy. It will likely fail.
1560  if (SrcClass == DstClass) {
1561  ResultReg = createResultReg(DstClass);
1562  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1563  TII.get(TargetOpcode::COPY), ResultReg).addReg(Op0);
1564  }
1565  }
1566 
1567  // If the reg-reg copy failed, select a BITCAST opcode.
1568  if (!ResultReg)
1569  ResultReg = fastEmit_r(SrcVT, DstVT, ISD::BITCAST, Op0, Op0IsKill);
1570 
1571  if (!ResultReg)
1572  return false;
1573 
1574  updateValueMap(I, ResultReg);
1575  return true;
1576 }
1577 
1578 // Remove local value instructions starting from the instruction after
1579 // SavedLastLocalValue to the current function insert point.
1580 void FastISel::removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue)
1581 {
1582  MachineInstr *CurLastLocalValue = getLastLocalValue();
1583  if (CurLastLocalValue != SavedLastLocalValue) {
1584  // Find the first local value instruction to be deleted.
1585  // This is the instruction after SavedLastLocalValue if it is non-NULL.
1586  // Otherwise it's the first instruction in the block.
1587  MachineBasicBlock::iterator FirstDeadInst(SavedLastLocalValue);
1588  if (SavedLastLocalValue)
1589  ++FirstDeadInst;
1590  else
1591  FirstDeadInst = FuncInfo.MBB->getFirstNonPHI();
1592  setLastLocalValue(SavedLastLocalValue);
1593  removeDeadCode(FirstDeadInst, FuncInfo.InsertPt);
1594  }
1595 }
1596 
1598  MachineInstr *SavedLastLocalValue = getLastLocalValue();
1599  // Just before the terminator instruction, insert instructions to
1600  // feed PHI nodes in successor blocks.
1601  if (I->isTerminator()) {
1602  if (!handlePHINodesInSuccessorBlocks(I->getParent())) {
1603  // PHI node handling may have generated local value instructions,
1604  // even though it failed to handle all PHI nodes.
1605  // We remove these instructions because SelectionDAGISel will generate
1606  // them again.
1607  removeDeadLocalValueCode(SavedLastLocalValue);
1608  return false;
1609  }
1610  }
1611 
1612  // FastISel does not handle any operand bundles except OB_funclet.
1614  for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i)
1615  if (CS.getOperandBundleAt(i).getTagID() != LLVMContext::OB_funclet)
1616  return false;
1617 
1618  DbgLoc = I->getDebugLoc();
1619 
1620  SavedInsertPt = FuncInfo.InsertPt;
1621 
1622  if (const auto *Call = dyn_cast<CallInst>(I)) {
1623  const Function *F = Call->getCalledFunction();
1624  LibFunc Func;
1625 
1626  // As a special case, don't handle calls to builtin library functions that
1627  // may be translated directly to target instructions.
1628  if (F && !F->hasLocalLinkage() && F->hasName() &&
1629  LibInfo->getLibFunc(F->getName(), Func) &&
1630  LibInfo->hasOptimizedCodeGen(Func))
1631  return false;
1632 
1633  // Don't handle Intrinsic::trap if a trap function is specified.
1634  if (F && F->getIntrinsicID() == Intrinsic::trap &&
1635  Call->hasFnAttr("trap-func-name"))
1636  return false;
1637  }
1638 
1639  // First, try doing target-independent selection.
1640  if (!SkipTargetIndependentISel) {
1641  if (selectOperator(I, I->getOpcode())) {
1642  ++NumFastIselSuccessIndependent;
1643  DbgLoc = DebugLoc();
1644  return true;
1645  }
1646  // Remove dead code.
1647  recomputeInsertPt();
1648  if (SavedInsertPt != FuncInfo.InsertPt)
1649  removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
1650  SavedInsertPt = FuncInfo.InsertPt;
1651  }
1652  // Next, try calling the target to attempt to handle the instruction.
1653  if (fastSelectInstruction(I)) {
1654  ++NumFastIselSuccessTarget;
1655  DbgLoc = DebugLoc();
1656  return true;
1657  }
1658  // Remove dead code.
1659  recomputeInsertPt();
1660  if (SavedInsertPt != FuncInfo.InsertPt)
1661  removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
1662 
1663  DbgLoc = DebugLoc();
1664  // Undo phi node updates, because they will be added again by SelectionDAG.
1665  if (I->isTerminator()) {
1666  // PHI node handling may have generated local value instructions.
1667  // We remove them because SelectionDAGISel will generate them again.
1668  removeDeadLocalValueCode(SavedLastLocalValue);
1669  FuncInfo.PHINodesToUpdate.resize(FuncInfo.OrigNumPHINodesToUpdate);
1670  }
1671  return false;
1672 }
1673 
1674 /// Emit an unconditional branch to the given block, unless it is the immediate
1675 /// (fall-through) successor, and update the CFG.
1677  const DebugLoc &DbgLoc) {
1678  if (FuncInfo.MBB->getBasicBlock()->size() > 1 &&
1679  FuncInfo.MBB->isLayoutSuccessor(MSucc)) {
1680  // For more accurate line information if this is the only instruction
1681  // in the block then emit it, otherwise we have the unconditional
1682  // fall-through case, which needs no instructions.
1683  } else {
1684  // The unconditional branch case.
1685  TII.insertBranch(*FuncInfo.MBB, MSucc, nullptr,
1686  SmallVector<MachineOperand, 0>(), DbgLoc);
1687  }
1688  if (FuncInfo.BPI) {
1689  auto BranchProbability = FuncInfo.BPI->getEdgeProbability(
1690  FuncInfo.MBB->getBasicBlock(), MSucc->getBasicBlock());
1691  FuncInfo.MBB->addSuccessor(MSucc, BranchProbability);
1692  } else
1693  FuncInfo.MBB->addSuccessorWithoutProb(MSucc);
1694 }
1695 
1697  MachineBasicBlock *TrueMBB,
1698  MachineBasicBlock *FalseMBB) {
1699  // Add TrueMBB as successor unless it is equal to the FalseMBB: This can
1700  // happen in degenerate IR and MachineIR forbids to have a block twice in the
1701  // successor/predecessor lists.
1702  if (TrueMBB != FalseMBB) {
1703  if (FuncInfo.BPI) {
1704  auto BranchProbability =
1705  FuncInfo.BPI->getEdgeProbability(BranchBB, TrueMBB->getBasicBlock());
1706  FuncInfo.MBB->addSuccessor(TrueMBB, BranchProbability);
1707  } else
1708  FuncInfo.MBB->addSuccessorWithoutProb(TrueMBB);
1709  }
1710 
1711  fastEmitBranch(FalseMBB, DbgLoc);
1712 }
1713 
1714 /// Emit an FNeg operation.
1715 bool FastISel::selectFNeg(const User *I, const Value *In) {
1716  unsigned OpReg = getRegForValue(In);
1717  if (!OpReg)
1718  return false;
1719  bool OpRegIsKill = hasTrivialKill(In);
1720 
1721  // If the target has ISD::FNEG, use it.
1722  EVT VT = TLI.getValueType(DL, I->getType());
1723  unsigned ResultReg = fastEmit_r(VT.getSimpleVT(), VT.getSimpleVT(), ISD::FNEG,
1724  OpReg, OpRegIsKill);
1725  if (ResultReg) {
1726  updateValueMap(I, ResultReg);
1727  return true;
1728  }
1729 
1730  // Bitcast the value to integer, twiddle the sign bit with xor,
1731  // and then bitcast it back to floating-point.
1732  if (VT.getSizeInBits() > 64)
1733  return false;
1734  EVT IntVT = EVT::getIntegerVT(I->getContext(), VT.getSizeInBits());
1735  if (!TLI.isTypeLegal(IntVT))
1736  return false;
1737 
1738  unsigned IntReg = fastEmit_r(VT.getSimpleVT(), IntVT.getSimpleVT(),
1739  ISD::BITCAST, OpReg, OpRegIsKill);
1740  if (!IntReg)
1741  return false;
1742 
1743  unsigned IntResultReg = fastEmit_ri_(
1744  IntVT.getSimpleVT(), ISD::XOR, IntReg, /*IsKill=*/true,
1745  UINT64_C(1) << (VT.getSizeInBits() - 1), IntVT.getSimpleVT());
1746  if (!IntResultReg)
1747  return false;
1748 
1749  ResultReg = fastEmit_r(IntVT.getSimpleVT(), VT.getSimpleVT(), ISD::BITCAST,
1750  IntResultReg, /*IsKill=*/true);
1751  if (!ResultReg)
1752  return false;
1753 
1754  updateValueMap(I, ResultReg);
1755  return true;
1756 }
1757 
1759  const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(U);
1760  if (!EVI)
1761  return false;
1762 
1763  // Make sure we only try to handle extracts with a legal result. But also
1764  // allow i1 because it's easy.
1765  EVT RealVT = TLI.getValueType(DL, EVI->getType(), /*AllowUnknown=*/true);
1766  if (!RealVT.isSimple())
1767  return false;
1768  MVT VT = RealVT.getSimpleVT();
1769  if (!TLI.isTypeLegal(VT) && VT != MVT::i1)
1770  return false;
1771 
1772  const Value *Op0 = EVI->getOperand(0);
1773  Type *AggTy = Op0->getType();
1774 
1775  // Get the base result register.
1776  unsigned ResultReg;
1777  DenseMap<const Value *, unsigned>::iterator I = FuncInfo.ValueMap.find(Op0);
1778  if (I != FuncInfo.ValueMap.end())
1779  ResultReg = I->second;
1780  else if (isa<Instruction>(Op0))
1781  ResultReg = FuncInfo.InitializeRegForValue(Op0);
1782  else
1783  return false; // fast-isel can't handle aggregate constants at the moment
1784 
1785  // Get the actual result register, which is an offset from the base register.
1786  unsigned VTIndex = ComputeLinearIndex(AggTy, EVI->getIndices());
1787 
1788  SmallVector<EVT, 4> AggValueVTs;
1789  ComputeValueVTs(TLI, DL, AggTy, AggValueVTs);
1790 
1791  for (unsigned i = 0; i < VTIndex; i++)
1792  ResultReg += TLI.getNumRegisters(FuncInfo.Fn->getContext(), AggValueVTs[i]);
1793 
1794  updateValueMap(EVI, ResultReg);
1795  return true;
1796 }
1797 
1798 bool FastISel::selectOperator(const User *I, unsigned Opcode) {
1799  switch (Opcode) {
1800  case Instruction::Add:
1801  return selectBinaryOp(I, ISD::ADD);
1802  case Instruction::FAdd:
1803  return selectBinaryOp(I, ISD::FADD);
1804  case Instruction::Sub:
1805  return selectBinaryOp(I, ISD::SUB);
1806  case Instruction::FSub: {
1807  // FNeg is currently represented in LLVM IR as a special case of FSub.
1808  Value *X;
1809  if (match(I, m_FNeg(m_Value(X))))
1810  return selectFNeg(I, X);
1811  return selectBinaryOp(I, ISD::FSUB);
1812  }
1813  case Instruction::Mul:
1814  return selectBinaryOp(I, ISD::MUL);
1815  case Instruction::FMul:
1816  return selectBinaryOp(I, ISD::FMUL);
1817  case Instruction::SDiv:
1818  return selectBinaryOp(I, ISD::SDIV);
1819  case Instruction::UDiv:
1820  return selectBinaryOp(I, ISD::UDIV);
1821  case Instruction::FDiv:
1822  return selectBinaryOp(I, ISD::FDIV);
1823  case Instruction::SRem:
1824  return selectBinaryOp(I, ISD::SREM);
1825  case Instruction::URem:
1826  return selectBinaryOp(I, ISD::UREM);
1827  case Instruction::FRem:
1828  return selectBinaryOp(I, ISD::FREM);
1829  case Instruction::Shl:
1830  return selectBinaryOp(I, ISD::SHL);
1831  case Instruction::LShr:
1832  return selectBinaryOp(I, ISD::SRL);
1833  case Instruction::AShr:
1834  return selectBinaryOp(I, ISD::SRA);
1835  case Instruction::And:
1836  return selectBinaryOp(I, ISD::AND);
1837  case Instruction::Or:
1838  return selectBinaryOp(I, ISD::OR);
1839  case Instruction::Xor:
1840  return selectBinaryOp(I, ISD::XOR);
1841 
1842  case Instruction::FNeg:
1843  return selectFNeg(I, I->getOperand(0));
1844 
1845  case Instruction::GetElementPtr:
1846  return selectGetElementPtr(I);
1847 
1848  case Instruction::Br: {
1849  const BranchInst *BI = cast<BranchInst>(I);
1850 
1851  if (BI->isUnconditional()) {
1852  const BasicBlock *LLVMSucc = BI->getSuccessor(0);
1853  MachineBasicBlock *MSucc = FuncInfo.MBBMap[LLVMSucc];
1854  fastEmitBranch(MSucc, BI->getDebugLoc());
1855  return true;
1856  }
1857 
1858  // Conditional branches are not handed yet.
1859  // Halt "fast" selection and bail.
1860  return false;
1861  }
1862 
1863  case Instruction::Unreachable:
1864  if (TM.Options.TrapUnreachable)
1865  return fastEmit_(MVT::Other, MVT::Other, ISD::TRAP) != 0;
1866  else
1867  return true;
1868 
1869  case Instruction::Alloca:
1870  // FunctionLowering has the static-sized case covered.
1871  if (FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(I)))
1872  return true;
1873 
1874  // Dynamic-sized alloca is not handled yet.
1875  return false;
1876 
1877  case Instruction::Call:
1878  return selectCall(I);
1879 
1880  case Instruction::BitCast:
1881  return selectBitCast(I);
1882 
1883  case Instruction::FPToSI:
1884  return selectCast(I, ISD::FP_TO_SINT);
1885  case Instruction::ZExt:
1886  return selectCast(I, ISD::ZERO_EXTEND);
1887  case Instruction::SExt:
1888  return selectCast(I, ISD::SIGN_EXTEND);
1889  case Instruction::Trunc:
1890  return selectCast(I, ISD::TRUNCATE);
1891  case Instruction::SIToFP:
1892  return selectCast(I, ISD::SINT_TO_FP);
1893 
1894  case Instruction::IntToPtr: // Deliberate fall-through.
1895  case Instruction::PtrToInt: {
1896  EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1897  EVT DstVT = TLI.getValueType(DL, I->getType());
1898  if (DstVT.bitsGT(SrcVT))
1899  return selectCast(I, ISD::ZERO_EXTEND);
1900  if (DstVT.bitsLT(SrcVT))
1901  return selectCast(I, ISD::TRUNCATE);
1902  unsigned Reg = getRegForValue(I->getOperand(0));
1903  if (!Reg)
1904  return false;
1905  updateValueMap(I, Reg);
1906  return true;
1907  }
1908 
1909  case Instruction::ExtractValue:
1910  return selectExtractValue(I);
1911 
1912  case Instruction::PHI:
1913  llvm_unreachable("FastISel shouldn't visit PHI nodes!");
1914 
1915  default:
1916  // Unhandled instruction. Halt "fast" selection and bail.
1917  return false;
1918  }
1919 }
1920 
1922  const TargetLibraryInfo *LibInfo,
1923  bool SkipTargetIndependentISel)
1924  : FuncInfo(FuncInfo), MF(FuncInfo.MF), MRI(FuncInfo.MF->getRegInfo()),
1925  MFI(FuncInfo.MF->getFrameInfo()), MCP(*FuncInfo.MF->getConstantPool()),
1926  TM(FuncInfo.MF->getTarget()), DL(MF->getDataLayout()),
1927  TII(*MF->getSubtarget().getInstrInfo()),
1928  TLI(*MF->getSubtarget().getTargetLowering()),
1929  TRI(*MF->getSubtarget().getRegisterInfo()), LibInfo(LibInfo),
1930  SkipTargetIndependentISel(SkipTargetIndependentISel) {}
1931 
1932 FastISel::~FastISel() = default;
1933 
1934 bool FastISel::fastLowerArguments() { return false; }
1935 
1936 bool FastISel::fastLowerCall(CallLoweringInfo & /*CLI*/) { return false; }
1937 
1939  return false;
1940 }
1941 
1942 unsigned FastISel::fastEmit_(MVT, MVT, unsigned) { return 0; }
1943 
1944 unsigned FastISel::fastEmit_r(MVT, MVT, unsigned, unsigned /*Op0*/,
1945  bool /*Op0IsKill*/) {
1946  return 0;
1947 }
1948 
1949 unsigned FastISel::fastEmit_rr(MVT, MVT, unsigned, unsigned /*Op0*/,
1950  bool /*Op0IsKill*/, unsigned /*Op1*/,
1951  bool /*Op1IsKill*/) {
1952  return 0;
1953 }
1954 
1955 unsigned FastISel::fastEmit_i(MVT, MVT, unsigned, uint64_t /*Imm*/) {
1956  return 0;
1957 }
1958 
1959 unsigned FastISel::fastEmit_f(MVT, MVT, unsigned,
1960  const ConstantFP * /*FPImm*/) {
1961  return 0;
1962 }
1963 
1964 unsigned FastISel::fastEmit_ri(MVT, MVT, unsigned, unsigned /*Op0*/,
1965  bool /*Op0IsKill*/, uint64_t /*Imm*/) {
1966  return 0;
1967 }
1968 
1969 /// This method is a wrapper of fastEmit_ri. It first tries to emit an
1970 /// instruction with an immediate operand using fastEmit_ri.
1971 /// If that fails, it materializes the immediate into a register and try
1972 /// fastEmit_rr instead.
1973 unsigned FastISel::fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0,
1974  bool Op0IsKill, uint64_t Imm, MVT ImmType) {
1975  // If this is a multiply by a power of two, emit this as a shift left.
1976  if (Opcode == ISD::MUL && isPowerOf2_64(Imm)) {
1977  Opcode = ISD::SHL;
1978  Imm = Log2_64(Imm);
1979  } else if (Opcode == ISD::UDIV && isPowerOf2_64(Imm)) {
1980  // div x, 8 -> srl x, 3
1981  Opcode = ISD::SRL;
1982  Imm = Log2_64(Imm);
1983  }
1984 
1985  // Horrible hack (to be removed), check to make sure shift amounts are
1986  // in-range.
1987  if ((Opcode == ISD::SHL || Opcode == ISD::SRA || Opcode == ISD::SRL) &&
1988  Imm >= VT.getSizeInBits())
1989  return 0;
1990 
1991  // First check if immediate type is legal. If not, we can't use the ri form.
1992  unsigned ResultReg = fastEmit_ri(VT, VT, Opcode, Op0, Op0IsKill, Imm);
1993  if (ResultReg)
1994  return ResultReg;
1995  unsigned MaterialReg = fastEmit_i(ImmType, ImmType, ISD::Constant, Imm);
1996  bool IsImmKill = true;
1997  if (!MaterialReg) {
1998  // This is a bit ugly/slow, but failing here means falling out of
1999  // fast-isel, which would be very slow.
2000  IntegerType *ITy =
2002  MaterialReg = getRegForValue(ConstantInt::get(ITy, Imm));
2003  if (!MaterialReg)
2004  return 0;
2005  // FIXME: If the materialized register here has no uses yet then this
2006  // will be the first use and we should be able to mark it as killed.
2007  // However, the local value area for materialising constant expressions
2008  // grows down, not up, which means that any constant expressions we generate
2009  // later which also use 'Imm' could be after this instruction and therefore
2010  // after this kill.
2011  IsImmKill = false;
2012  }
2013  return fastEmit_rr(VT, VT, Opcode, Op0, Op0IsKill, MaterialReg, IsImmKill);
2014 }
2015 
2017  return MRI.createVirtualRegister(RC);
2018 }
2019 
2021  unsigned OpNum) {
2023  const TargetRegisterClass *RegClass =
2024  TII.getRegClass(II, OpNum, &TRI, *FuncInfo.MF);
2025  if (!MRI.constrainRegClass(Op, RegClass)) {
2026  // If it's not legal to COPY between the register classes, something
2027  // has gone very wrong before we got here.
2028  unsigned NewOp = createResultReg(RegClass);
2030  TII.get(TargetOpcode::COPY), NewOp).addReg(Op);
2031  return NewOp;
2032  }
2033  }
2034  return Op;
2035 }
2036 
2037 unsigned FastISel::fastEmitInst_(unsigned MachineInstOpcode,
2038  const TargetRegisterClass *RC) {
2039  unsigned ResultReg = createResultReg(RC);
2040  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2041 
2042  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg);
2043  return ResultReg;
2044 }
2045 
2046 unsigned FastISel::fastEmitInst_r(unsigned MachineInstOpcode,
2047  const TargetRegisterClass *RC, unsigned Op0,
2048  bool Op0IsKill) {
2049  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2050 
2051  unsigned ResultReg = createResultReg(RC);
2052  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2053 
2054  if (II.getNumDefs() >= 1)
2055  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2056  .addReg(Op0, getKillRegState(Op0IsKill));
2057  else {
2059  .addReg(Op0, getKillRegState(Op0IsKill));
2061  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2062  }
2063 
2064  return ResultReg;
2065 }
2066 
2067 unsigned FastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
2068  const TargetRegisterClass *RC, unsigned Op0,
2069  bool Op0IsKill, unsigned Op1,
2070  bool Op1IsKill) {
2071  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2072 
2073  unsigned ResultReg = createResultReg(RC);
2074  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2075  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2076 
2077  if (II.getNumDefs() >= 1)
2078  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2079  .addReg(Op0, getKillRegState(Op0IsKill))
2080  .addReg(Op1, getKillRegState(Op1IsKill));
2081  else {
2083  .addReg(Op0, getKillRegState(Op0IsKill))
2084  .addReg(Op1, getKillRegState(Op1IsKill));
2086  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2087  }
2088  return ResultReg;
2089 }
2090 
2091 unsigned FastISel::fastEmitInst_rrr(unsigned MachineInstOpcode,
2092  const TargetRegisterClass *RC, unsigned Op0,
2093  bool Op0IsKill, unsigned Op1,
2094  bool Op1IsKill, unsigned Op2,
2095  bool Op2IsKill) {
2096  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2097 
2098  unsigned ResultReg = createResultReg(RC);
2099  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2100  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2101  Op2 = constrainOperandRegClass(II, Op2, II.getNumDefs() + 2);
2102 
2103  if (II.getNumDefs() >= 1)
2104  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2105  .addReg(Op0, getKillRegState(Op0IsKill))
2106  .addReg(Op1, getKillRegState(Op1IsKill))
2107  .addReg(Op2, getKillRegState(Op2IsKill));
2108  else {
2110  .addReg(Op0, getKillRegState(Op0IsKill))
2111  .addReg(Op1, getKillRegState(Op1IsKill))
2112  .addReg(Op2, getKillRegState(Op2IsKill));
2114  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2115  }
2116  return ResultReg;
2117 }
2118 
2119 unsigned FastISel::fastEmitInst_ri(unsigned MachineInstOpcode,
2120  const TargetRegisterClass *RC, unsigned Op0,
2121  bool Op0IsKill, uint64_t Imm) {
2122  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2123 
2124  unsigned ResultReg = createResultReg(RC);
2125  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2126 
2127  if (II.getNumDefs() >= 1)
2128  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2129  .addReg(Op0, getKillRegState(Op0IsKill))
2130  .addImm(Imm);
2131  else {
2133  .addReg(Op0, getKillRegState(Op0IsKill))
2134  .addImm(Imm);
2136  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2137  }
2138  return ResultReg;
2139 }
2140 
2141 unsigned FastISel::fastEmitInst_rii(unsigned MachineInstOpcode,
2142  const TargetRegisterClass *RC, unsigned Op0,
2143  bool Op0IsKill, uint64_t Imm1,
2144  uint64_t Imm2) {
2145  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2146 
2147  unsigned ResultReg = createResultReg(RC);
2148  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2149 
2150  if (II.getNumDefs() >= 1)
2151  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2152  .addReg(Op0, getKillRegState(Op0IsKill))
2153  .addImm(Imm1)
2154  .addImm(Imm2);
2155  else {
2157  .addReg(Op0, getKillRegState(Op0IsKill))
2158  .addImm(Imm1)
2159  .addImm(Imm2);
2161  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2162  }
2163  return ResultReg;
2164 }
2165 
2166 unsigned FastISel::fastEmitInst_f(unsigned MachineInstOpcode,
2167  const TargetRegisterClass *RC,
2168  const ConstantFP *FPImm) {
2169  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2170 
2171  unsigned ResultReg = createResultReg(RC);
2172 
2173  if (II.getNumDefs() >= 1)
2174  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2175  .addFPImm(FPImm);
2176  else {
2178  .addFPImm(FPImm);
2180  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2181  }
2182  return ResultReg;
2183 }
2184 
2185 unsigned FastISel::fastEmitInst_rri(unsigned MachineInstOpcode,
2186  const TargetRegisterClass *RC, unsigned Op0,
2187  bool Op0IsKill, unsigned Op1,
2188  bool Op1IsKill, uint64_t Imm) {
2189  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2190 
2191  unsigned ResultReg = createResultReg(RC);
2192  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2193  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2194 
2195  if (II.getNumDefs() >= 1)
2196  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2197  .addReg(Op0, getKillRegState(Op0IsKill))
2198  .addReg(Op1, getKillRegState(Op1IsKill))
2199  .addImm(Imm);
2200  else {
2202  .addReg(Op0, getKillRegState(Op0IsKill))
2203  .addReg(Op1, getKillRegState(Op1IsKill))
2204  .addImm(Imm);
2206  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2207  }
2208  return ResultReg;
2209 }
2210 
2211 unsigned FastISel::fastEmitInst_i(unsigned MachineInstOpcode,
2212  const TargetRegisterClass *RC, uint64_t Imm) {
2213  unsigned ResultReg = createResultReg(RC);
2214  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2215 
2216  if (II.getNumDefs() >= 1)
2217  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2218  .addImm(Imm);
2219  else {
2222  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2223  }
2224  return ResultReg;
2225 }
2226 
2227 unsigned FastISel::fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0,
2228  bool Op0IsKill, uint32_t Idx) {
2229  unsigned ResultReg = createResultReg(TLI.getRegClassFor(RetVT));
2231  "Cannot yet extract from physregs");
2232  const TargetRegisterClass *RC = MRI.getRegClass(Op0);
2234  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY),
2235  ResultReg).addReg(Op0, getKillRegState(Op0IsKill), Idx);
2236  return ResultReg;
2237 }
2238 
2239 /// Emit MachineInstrs to compute the value of Op with all but the least
2240 /// significant bit set to zero.
2241 unsigned FastISel::fastEmitZExtFromI1(MVT VT, unsigned Op0, bool Op0IsKill) {
2242  return fastEmit_ri(VT, VT, ISD::AND, Op0, Op0IsKill, 1);
2243 }
2244 
2245 /// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks.
2246 /// Emit code to ensure constants are copied into registers when needed.
2247 /// Remember the virtual registers that need to be added to the Machine PHI
2248 /// nodes as input. We cannot just directly add them, because expansion
2249 /// might result in multiple MBB's for one BB. As such, the start of the
2250 /// BB might correspond to a different MBB than the end.
2251 bool FastISel::handlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
2252  const Instruction *TI = LLVMBB->getTerminator();
2253 
2256 
2257  // Check successor nodes' PHI nodes that expect a constant to be available
2258  // from this block.
2259  for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
2260  const BasicBlock *SuccBB = TI->getSuccessor(succ);
2261  if (!isa<PHINode>(SuccBB->begin()))
2262  continue;
2263  MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
2264 
2265  // If this terminator has multiple identical successors (common for
2266  // switches), only handle each succ once.
2267  if (!SuccsHandled.insert(SuccMBB).second)
2268  continue;
2269 
2270  MachineBasicBlock::iterator MBBI = SuccMBB->begin();
2271 
2272  // At this point we know that there is a 1-1 correspondence between LLVM PHI
2273  // nodes and Machine PHI nodes, but the incoming operands have not been
2274  // emitted yet.
2275  for (const PHINode &PN : SuccBB->phis()) {
2276  // Ignore dead phi's.
2277  if (PN.use_empty())
2278  continue;
2279 
2280  // Only handle legal types. Two interesting things to note here. First,
2281  // by bailing out early, we may leave behind some dead instructions,
2282  // since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its
2283  // own moves. Second, this check is necessary because FastISel doesn't
2284  // use CreateRegs to create registers, so it always creates
2285  // exactly one register for each non-void instruction.
2286  EVT VT = TLI.getValueType(DL, PN.getType(), /*AllowUnknown=*/true);
2287  if (VT == MVT::Other || !TLI.isTypeLegal(VT)) {
2288  // Handle integer promotions, though, because they're common and easy.
2289  if (!(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)) {
2291  return false;
2292  }
2293  }
2294 
2295  const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
2296 
2297  // Set the DebugLoc for the copy. Prefer the location of the operand
2298  // if there is one; use the location of the PHI otherwise.
2299  DbgLoc = PN.getDebugLoc();
2300  if (const auto *Inst = dyn_cast<Instruction>(PHIOp))
2301  DbgLoc = Inst->getDebugLoc();
2302 
2303  unsigned Reg = getRegForValue(PHIOp);
2304  if (!Reg) {
2306  return false;
2307  }
2308  FuncInfo.PHINodesToUpdate.push_back(std::make_pair(&*MBBI++, Reg));
2309  DbgLoc = DebugLoc();
2310  }
2311  }
2312 
2313  return true;
2314 }
2315 
2316 bool FastISel::tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst) {
2317  assert(LI->hasOneUse() &&
2318  "tryToFoldLoad expected a LoadInst with a single use");
2319  // We know that the load has a single use, but don't know what it is. If it
2320  // isn't one of the folded instructions, then we can't succeed here. Handle
2321  // this by scanning the single-use users of the load until we get to FoldInst.
2322  unsigned MaxUsers = 6; // Don't scan down huge single-use chains of instrs.
2323 
2324  const Instruction *TheUser = LI->user_back();
2325  while (TheUser != FoldInst && // Scan up until we find FoldInst.
2326  // Stay in the right block.
2327  TheUser->getParent() == FoldInst->getParent() &&
2328  --MaxUsers) { // Don't scan too far.
2329  // If there are multiple or no uses of this instruction, then bail out.
2330  if (!TheUser->hasOneUse())
2331  return false;
2332 
2333  TheUser = TheUser->user_back();
2334  }
2335 
2336  // If we didn't find the fold instruction, then we failed to collapse the
2337  // sequence.
2338  if (TheUser != FoldInst)
2339  return false;
2340 
2341  // Don't try to fold volatile loads. Target has to deal with alignment
2342  // constraints.
2343  if (LI->isVolatile())
2344  return false;
2345 
2346  // Figure out which vreg this is going into. If there is no assigned vreg yet
2347  // then there actually was no reference to it. Perhaps the load is referenced
2348  // by a dead instruction.
2349  unsigned LoadReg = getRegForValue(LI);
2350  if (!LoadReg)
2351  return false;
2352 
2353  // We can't fold if this vreg has no uses or more than one use. Multiple uses
2354  // may mean that the instruction got lowered to multiple MIs, or the use of
2355  // the loaded value ended up being multiple operands of the result.
2356  if (!MRI.hasOneUse(LoadReg))
2357  return false;
2358 
2360  MachineInstr *User = RI->getParent();
2361 
2362  // Set the insertion point properly. Folding the load can cause generation of
2363  // other random instructions (like sign extends) for addressing modes; make
2364  // sure they get inserted in a logical place before the new instruction.
2365  FuncInfo.InsertPt = User;
2366  FuncInfo.MBB = User->getParent();
2367 
2368  // Ask the target to try folding the load.
2369  return tryToFoldLoadIntoMI(User, RI.getOperandNo(), LI);
2370 }
2371 
2373  // Must be an add.
2374  if (!isa<AddOperator>(Add))
2375  return false;
2376  // Type size needs to match.
2377  if (DL.getTypeSizeInBits(GEP->getType()) !=
2378  DL.getTypeSizeInBits(Add->getType()))
2379  return false;
2380  // Must be in the same basic block.
2381  if (isa<Instruction>(Add) &&
2382  FuncInfo.MBBMap[cast<Instruction>(Add)->getParent()] != FuncInfo.MBB)
2383  return false;
2384  // Must have a constant operand.
2385  return isa<ConstantInt>(cast<AddOperator>(Add)->getOperand(1));
2386 }
2387 
2390  const Value *Ptr;
2391  Type *ValTy;
2392  unsigned Alignment;
2394  bool IsVolatile;
2395 
2396  if (const auto *LI = dyn_cast<LoadInst>(I)) {
2397  Alignment = LI->getAlignment();
2398  IsVolatile = LI->isVolatile();
2399  Flags = MachineMemOperand::MOLoad;
2400  Ptr = LI->getPointerOperand();
2401  ValTy = LI->getType();
2402  } else if (const auto *SI = dyn_cast<StoreInst>(I)) {
2403  Alignment = SI->getAlignment();
2404  IsVolatile = SI->isVolatile();
2406  Ptr = SI->getPointerOperand();
2407  ValTy = SI->getValueOperand()->getType();
2408  } else
2409  return nullptr;
2410 
2411  bool IsNonTemporal = I->getMetadata(LLVMContext::MD_nontemporal) != nullptr;
2412  bool IsInvariant = I->getMetadata(LLVMContext::MD_invariant_load) != nullptr;
2413  bool IsDereferenceable =
2415  const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range);
2416 
2417  AAMDNodes AAInfo;
2418  I->getAAMetadata(AAInfo);
2419 
2420  if (Alignment == 0) // Ensure that codegen never sees alignment 0.
2421  Alignment = DL.getABITypeAlignment(ValTy);
2422 
2423  unsigned Size = DL.getTypeStoreSize(ValTy);
2424 
2425  if (IsVolatile)
2427  if (IsNonTemporal)
2429  if (IsDereferenceable)
2431  if (IsInvariant)
2433 
2434  return FuncInfo.MF->getMachineMemOperand(MachinePointerInfo(Ptr), Flags, Size,
2435  Alignment, AAInfo, Ranges);
2436 }
2437 
2439  // If both operands are the same, then try to optimize or fold the cmp.
2441  if (CI->getOperand(0) != CI->getOperand(1))
2442  return Predicate;
2443 
2444  switch (Predicate) {
2445  default: llvm_unreachable("Invalid predicate!");
2446  case CmpInst::FCMP_FALSE: Predicate = CmpInst::FCMP_FALSE; break;
2447  case CmpInst::FCMP_OEQ: Predicate = CmpInst::FCMP_ORD; break;
2448  case CmpInst::FCMP_OGT: Predicate = CmpInst::FCMP_FALSE; break;
2449  case CmpInst::FCMP_OGE: Predicate = CmpInst::FCMP_ORD; break;
2450  case CmpInst::FCMP_OLT: Predicate = CmpInst::FCMP_FALSE; break;
2451  case CmpInst::FCMP_OLE: Predicate = CmpInst::FCMP_ORD; break;
2452  case CmpInst::FCMP_ONE: Predicate = CmpInst::FCMP_FALSE; break;
2453  case CmpInst::FCMP_ORD: Predicate = CmpInst::FCMP_ORD; break;
2454  case CmpInst::FCMP_UNO: Predicate = CmpInst::FCMP_UNO; break;
2455  case CmpInst::FCMP_UEQ: Predicate = CmpInst::FCMP_TRUE; break;
2456  case CmpInst::FCMP_UGT: Predicate = CmpInst::FCMP_UNO; break;
2457  case CmpInst::FCMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2458  case CmpInst::FCMP_ULT: Predicate = CmpInst::FCMP_UNO; break;
2459  case CmpInst::FCMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2460  case CmpInst::FCMP_UNE: Predicate = CmpInst::FCMP_UNO; break;
2461  case CmpInst::FCMP_TRUE: Predicate = CmpInst::FCMP_TRUE; break;
2462 
2463  case CmpInst::ICMP_EQ: Predicate = CmpInst::FCMP_TRUE; break;
2464  case CmpInst::ICMP_NE: Predicate = CmpInst::FCMP_FALSE; break;
2465  case CmpInst::ICMP_UGT: Predicate = CmpInst::FCMP_FALSE; break;
2466  case CmpInst::ICMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2467  case CmpInst::ICMP_ULT: Predicate = CmpInst::FCMP_FALSE; break;
2468  case CmpInst::ICMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2469  case CmpInst::ICMP_SGT: Predicate = CmpInst::FCMP_FALSE; break;
2470  case CmpInst::ICMP_SGE: Predicate = CmpInst::FCMP_TRUE; break;
2471  case CmpInst::ICMP_SLT: Predicate = CmpInst::FCMP_FALSE; break;
2472  case CmpInst::ICMP_SLE: Predicate = CmpInst::FCMP_TRUE; break;
2473  }
2474 
2475  return Predicate;
2476 }
void setHasStackMap(bool s=true)
IterTy arg_end() const
Definition: CallSite.h:583
uint64_t CallInst * C
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:595
unsigned fastEmitInst_rrr(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill, unsigned Op2, bool Op2IsKill)
Emit a MachineInstr with three register operands and a result register in the given register class...
Definition: FastISel.cpp:2091
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
void setByValAlign(unsigned A)
const MachineInstrBuilder & add(const MachineOperand &MO) const
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:110
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:70
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:699
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
unsigned fastEmitZExtFromI1(MVT VT, unsigned Op0, bool Op0IsKill)
Emit MachineInstrs to compute the value of Op with all but the least significant bit set to zero...
Definition: FastISel.cpp:2241
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
bool hasLocalLinkage() const
Definition: GlobalValue.h:435
This instruction extracts a struct member or array element value from an aggregate value...
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
This class represents an incoming formal argument to a Function.
Definition: Argument.h:29
bool lowerCall(const CallInst *I)
Definition: FastISel.cpp:1246
const TargetRegisterClass * getRegClass(unsigned Reg) const
Return the register class of the specified virtual register.
This represents the llvm.dbg.label instruction.
IterTy arg_begin() const
Definition: CallSite.h:579
This class represents lattice values for constants.
Definition: AllocatorList.h:23
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
bool selectFNeg(const User *I, const Value *In)
Emit an FNeg operation.
Definition: FastISel.cpp:1715
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
BasicBlock * getSuccessor(unsigned Idx) const
Return the specified successor. This instruction must be a terminator.
constexpr char IsVolatile[]
Key for Kernel::Arg::Metadata::mIsVolatile.
bool selectXRayTypedEvent(const CallInst *II)
Definition: FastISel.cpp:1060
ImmutableCallSite * CS
Definition: FastISel.h:89
bool selectGetElementPtr(const User *I)
Definition: FastISel.cpp:675
void leaveLocalValueArea(SavePoint Old)
Reset InsertPt to the given old insert position.
Definition: FastISel.cpp:575
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
bool selectStackmap(const CallInst *I)
Definition: FastISel.cpp:790
This class represents a function call, abstracting a target machine&#39;s calling convention.
unsigned Reg
This file contains the declarations for metadata subclasses.
virtual bool tryToFoldLoadIntoMI(MachineInstr *, unsigned, const LoadInst *)
The specified machine instr operand is a vreg, and that vreg is being provided by the specified load ...
Definition: FastISel.h:304
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:252
gep_type_iterator gep_type_end(const User *GEP)
unsigned less or equal
Definition: InstrTypes.h:735
unsigned less than
Definition: InstrTypes.h:734
0 1 0 0 True if ordered and less than
Definition: InstrTypes.h:715
MachineMemOperand * createMachineMemOperandFor(const Instruction *I) const
Create a machine mem operand from the given instruction.
Definition: FastISel.cpp:2389
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:709
bool isTerminator() const
Definition: Instruction.h:128
1 1 1 0 True if unordered or not equal
Definition: InstrTypes.h:725
virtual unsigned fastEmit_(MVT VT, MVT RetVT, unsigned Opcode)
This method is called by target-independent code to request that an instruction with the given type a...
Definition: FastISel.cpp:1942
BasicBlock * getSuccessor(unsigned i) const
virtual const TargetRegisterClass * getRegClassFor(MVT VT) const
Return the register class that should be used for the specified value type.
STATISTIC(NumFunctions, "Total number of functions")
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:33
Metadata node.
Definition: Metadata.h:863
F(f)
SmallVector< unsigned, 4 > InRegs
Definition: FastISel.h:98
An instruction for reading from memory.
Definition: Instructions.h:167
Hexagon Common GEP
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:137
void setAttributes(const CallBase *Call, unsigned ArgIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes...
virtual unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t Imm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1955
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:458
void setPhysRegsDeadExcept(ArrayRef< unsigned > UsedRegs, const TargetRegisterInfo &TRI)
Mark every physreg used by this instruction as dead except those in the UsedRegs list.
FunctionType * getFunctionType() const
Definition: CallSite.h:328
static MachineOperand CreateReg(unsigned Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
virtual bool fastLowerCall(CallLoweringInfo &CLI)
This method is called by target-independent code to do target- specific call lowering.
Definition: FastISel.cpp:1936
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
static Constant * getNullValue(Type *Ty)
Constructor to create a &#39;0&#39; constant of arbitrary type.
Definition: Constants.cpp:274
CallingConv::ID getCallingConv() const
Get the calling convention of the call.
Definition: CallSite.h:320
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:268
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
Definition: FastISel.cpp:1597
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1218
unsigned fastEmitInst_rii(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, uint64_t Imm1, uint64_t Imm2)
Emit a MachineInstr with one register operand and two immediate operands.
Definition: FastISel.cpp:2141
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Definition: APFloat.h:1068
1 0 0 1 True if unordered or equal
Definition: InstrTypes.h:720
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:47
unsigned fastEmitInst_ri(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, uint64_t Imm)
Emit a MachineInstr with a register operand, an immediate, and a result register in the given registe...
Definition: FastISel.cpp:2119
ArrayRef< unsigned > getIndices() const
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition: InstrTypes.h:719
unsigned fastEmitInst_rri(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill, uint64_t Imm)
Emit a MachineInstr with two register operands, an immediate, and a result register in the given regi...
Definition: FastISel.cpp:2185
CmpInst::Predicate optimizeCmpPredicate(const CmpInst *CI) const
Definition: FastISel.cpp:2438
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Definition: Instructions.h:231
A description of a memory reference used in the backend.
const HexagonInstrInfo * TII
TargetLoweringBase::ArgListTy ArgListTy
Definition: FastISel.h:69
Shift and rotation operations.
Definition: ISDOpcodes.h:434
const TargetRegisterClass * getRegClass(const MCInstrDesc &MCID, unsigned OpNum, const TargetRegisterInfo *TRI, const MachineFunction &MF) const
Given a machine instruction descriptor, returns the register class constraint for OpNum...
Class to represent struct types.
Definition: DerivedTypes.h:232
A Use represents the edge between a Value definition and its users.
Definition: Use.h:55
DILabel * getLabel() const
bool canFoldAddIntoGEP(const User *GEP, const Value *Add)
Check if Add is an add that can be safely folded into GEP.
Definition: FastISel.cpp:2372
MachineBasicBlock iterator that automatically skips over MIs that are inside bundles (i...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:41
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
unsigned fastEmitInst_r(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill)
Emit a MachineInstr with one register operand and a result register in the given register class...
Definition: FastISel.cpp:2046
0 1 0 1 True if ordered and less than or equal
Definition: InstrTypes.h:716
This file contains the simple types necessary to represent the attributes associated with functions a...
The memory access is dereferenceable (i.e., doesn&#39;t trap).
static MachineOperand CreateRegMask(const uint32_t *Mask)
CreateRegMask - Creates a register mask operand referencing Mask.
virtual const TargetRegisterClass * getSubClassWithSubReg(const TargetRegisterClass *RC, unsigned Idx) const
Returns the largest legal sub-class of RC that supports the sub-register index Idx.
void setByValSize(unsigned S)
Type * getType() const
Return the type of the instruction that generated this call site.
Definition: CallSite.h:272
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:695
bool selectIntrinsicCall(const IntrinsicInst *II)
Definition: FastISel.cpp:1328
bool selectCast(const User *I, unsigned Opcode)
Definition: FastISel.cpp:1496
unsigned getSizeInBits() const
Context object for machine code objects.
Definition: MCContext.h:62
InstrTy * getInstruction() const
Definition: CallSite.h:96
Class to represent function types.
Definition: DerivedTypes.h:102
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:291
SmallVector< ISD::InputArg, 4 > Ins
Definition: FastISel.h:97
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:244
unsigned constrainOperandRegClass(const MCInstrDesc &II, unsigned Op, unsigned OpNum)
Try to constrain Op so that it is usable by argument OpNum of the provided MCInstrDesc.
Definition: FastISel.cpp:2020
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:502
bool selectOperator(const User *I, unsigned Opcode)
Do "fast" instruction selection for the given LLVM IR operator (Instruction or ConstantExpr), and append generated machine instructions to the current block.
Definition: FastISel.cpp:1798
ArchType getArch() const
getArch - Get the parsed architecture type of this triple.
Definition: Triple.h:294
MachineInstrBundleIterator< MachineInstr, true > reverse_iterator
bool selectExtractValue(const User *U)
Definition: FastISel.cpp:1758
unsigned getRegForValue(const Value *V)
Create a virtual register and arrange for it to be assigned the value for the given LLVM value...
Definition: FastISel.cpp:362
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
unsigned fastEmitInst_(unsigned MachineInstOpcode, const TargetRegisterClass *RC)
Emit a MachineInstr with no operands and a result register in the given register class.
Definition: FastISel.cpp:2037
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:234
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:125
bool hasTrivialKill(const Value *V)
Test whether the given value has exactly one use.
Definition: FastISel.cpp:330
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
void setOrigAlign(unsigned A)
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:119
const TargetRegisterClass * constrainRegClass(unsigned Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
unsigned getNumSuccessors() const
Return the number of successors that this instruction has.
Value * getOperand(unsigned i) const
Definition: User.h:169
Class to represent pointers.
Definition: DerivedTypes.h:498
unsigned getKillRegState(bool B)
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:548
unsigned lookUpRegForValue(const Value *V)
Look up the value to see if its value is already cached in a register.
Definition: FastISel.cpp:475
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
void finishBasicBlock()
Flush the local value map and sink local values if possible.
Definition: FastISel.cpp:144
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:228
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:140
The memory access is volatile.
static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID, unsigned OpSize)
Select the AArch64 opcode for the basic binary operation GenericOpc (such as G_OR or G_SDIV)...
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
void getAAMetadata(AAMDNodes &N, bool Merge=false) const
Fills the AAMDNodes structure with AA metadata from this instruction.
#define P(N)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:168
virtual ~FastISel()
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
std::vector< std::pair< MachineInstr *, unsigned > > PHINodesToUpdate
PHINodesToUpdate - A list of phi instructions whose operand list will be updated after processing the...
unsigned const MachineRegisterInfo * MRI
CallLoweringInfo & setCallee(Type *ResultTy, FunctionType *FuncTy, const Value *Target, ArgListTy &&ArgsList, ImmutableCallSite &Call)
Definition: FastISel.h:104
Machine Value Type.
Value * getCalledValue() const
Definition: InstrTypes.h:1257
bool hasName() const
Definition: Value.h:250
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
const MachineInstrBuilder & addCImm(const ConstantInt *Val) const
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
Simple binary floating point operators.
Definition: ISDOpcodes.h:287
Conditional or Unconditional Branch instruction.
Value * getAddress() const
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Definition: Constant.h:41
void removeDeadCode(MachineBasicBlock::iterator I, MachineBasicBlock::iterator E)
Remove all dead instructions between the I and E.
Definition: FastISel.cpp:544
Value * getValue() const
SmallVector< ISD::ArgFlagsTy, 16 > OutFlags
Definition: FastISel.h:95
This file contains the declarations for the subclasses of Constant, which represent the different fla...
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:263
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:370
const MCPhysReg * ImplicitDefs
Definition: MCInstrDesc.h:173
DIExpression * getExpression() const
virtual unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1944
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:433
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:160
bool tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst)
We&#39;re checking to see if we can fold LI into FoldInst.
Definition: FastISel.cpp:2316
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
Definition: FastISel.cpp:146
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:709
std::pair< unsigned, bool > getRegForGEPIndex(const Value *Idx)
This is a wrapper around getRegForValue that also takes care of truncating or sign-extending the give...
Definition: FastISel.cpp:507
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, unsigned char TargetFlags=0)
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:798
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
0 1 1 1 True if ordered (no nans)
Definition: InstrTypes.h:718
self_iterator getIterator()
Definition: ilist_node.h:81
The memory access is non-temporal.
Class to represent integer types.
Definition: DerivedTypes.h:39
bool selectXRayCustomEvent(const CallInst *II)
Definition: FastISel.cpp:1041
const TargetRegisterInfo & TRI
Definition: FastISel.h:213
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:726
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:196
static unsigned findSinkableLocalRegDef(MachineInstr &MI)
Return the defined register if this instruction defines exactly one virtual register and uses no othe...
Definition: FastISel.cpp:168
Extended Value Type.
Definition: ValueTypes.h:33
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs, and aliases.
Definition: Value.cpp:529
size_t size() const
Definition: SmallVector.h:52
This class contains a discriminated union of information about pointers in memory operands...
1 1 0 1 True if unordered, less than, or equal
Definition: InstrTypes.h:724
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
SmallVector< Value *, 16 > OutVals
Definition: FastISel.h:94
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call...
Definition: FastISel.cpp:1083
const TargetInstrInfo & TII
Definition: FastISel.h:211
MachineBasicBlock * MBB
MBB - The current block.
bool isInTailCallPosition(ImmutableCallSite CS, const TargetMachine &TM)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition: Analysis.cpp:511
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:43
signed greater than
Definition: InstrTypes.h:736
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:703
void recomputeInsertPt()
Reset InsertPt to prepare for inserting instructions into the current block.
Definition: FastISel.cpp:530
The memory access writes data.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:50
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags...
0 0 1 0 True if ordered and greater than
Definition: InstrTypes.h:713
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:239
virtual unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1949
Iterator for intrusive lists based on ilist_node.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:417
This is the shared class of boolean and integer constants.
Definition: Constants.h:83
virtual unsigned fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t Imm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1964
MachineOperand class - Representation of each machine instruction operand.
1 1 0 0 True if unordered or less than
Definition: InstrTypes.h:723
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:841
bool selectCall(const User *I)
Definition: FastISel.cpp:1285
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
Definition: Instruction.h:63
Provides information about what library functions are available for the current target.
unsigned arg_size() const
Definition: CallSite.h:226
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
Definition: PPCPredicates.h:26
void finishCondBranch(const BasicBlock *BranchBB, MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB)
Emit an unconditional branch to FalseMBB, obtains the branch weight and adds TrueMBB and FalseMBB to ...
Definition: FastISel.cpp:1696
unsigned getABITypeAlignment(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:749
bool isOSLinux() const
Tests whether the OS is Linux.
Definition: Triple.h:580
signed less than
Definition: InstrTypes.h:738
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:643
reg_iterator reg_begin(unsigned RegNo) const
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
Insert branch code into the end of the specified MachineBasicBlock.
unsigned fastEmitInst_rr(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill)
Emit a MachineInstr with two register operands and a result register in the given register class...
Definition: FastISel.cpp:2067
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:631
void updateValueMap(const Value *I, unsigned Reg, unsigned NumRegs=1)
Update the value map to include the new mapping for this instruction, or insert an extra copy to get ...
Definition: FastISel.cpp:486
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
Definition: MCInstrDesc.h:225
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:193
void startNewBlock()
Set the current block to which generated machine instructions will be appended.
Definition: FastISel.cpp:130
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
signed less or equal
Definition: InstrTypes.h:739
bool selectBitCast(const User *I)
Definition: FastISel.cpp:1529
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
Target - Wrapper for Target specific information.
virtual unsigned fastEmit_f(MVT VT, MVT RetVT, unsigned Opcode, const ConstantFP *FPImm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1959
SmallVector< unsigned, 16 > OutRegs
Definition: FastISel.h:96
const DataLayout & DL
Definition: FastISel.h:210
bool selectBinaryOp(const User *I, unsigned ISDOpcode)
Select and emit code for a binary operator instruction, which has an opcode which directly correspond...
Definition: FastISel.cpp:584
void setDebugLoc(DebugLoc dl)
Replace current source information with new such.
amdgpu Simplify well known AMD library false FunctionCallee Callee
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, ArrayRef< StringRef > StandardNames)
Initialize the set of available library functions based on the specified target triple.
This file defines the FastISel class.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:240
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:492
bool isTailCall() const
DebugLoc DbgLoc
Definition: FastISel.h:208
Flags
Flags values. These may be or&#39;d together.
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:253
The memory access reads data.
uint64_t getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:593
SavePoint enterLocalValueArea()
Prepare InsertPt to begin inserting instructions into the local value area and return the old insert ...
Definition: FastISel.cpp:566
Representation of each machine instruction.
Definition: MachineInstr.h:63
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:784
virtual bool fastLowerIntrinsicCall(const IntrinsicInst *II)
This method is called by target-independent code to do target- specific intrinsic lowering...
Definition: FastISel.cpp:1938
unsigned getOperandNo() const
getOperandNo - Return the operand # of this MachineOperand in its MachineInstr.
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match &#39;fneg X&#39; as &#39;fsub -0.0, X&#39;.
Definition: PatternMatch.h:696
bool selectPatchpoint(const CallInst *I)
Definition: FastISel.cpp:903
unsigned getNumArgOperands() const
Definition: InstrTypes.h:1216
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:321
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:411
MachineRegisterInfo & MRI
Definition: FastISel.h:205
bool hasOneUse(unsigned RegNo) const
hasOneUse - Return true if there is exactly one instruction using the specified register.
MCSymbol * getOrCreateSymbol(const Twine &Name)
Lookup the symbol inside with the specified Name.
Definition: MCContext.cpp:123
unsigned greater or equal
Definition: InstrTypes.h:733
This represents the llvm.dbg.value instruction.
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1321
bool lowerCallTo(const CallInst *CI, MCSymbol *Symbol, unsigned NumArgs)
Definition: FastISel.cpp:1105
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:44
StringRef getName() const
Return a constant reference to the value&#39;s name.
Definition: Value.cpp:214
Establish a view to a call site for examination.
Definition: CallSite.h:892
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
Definition: InstrTypes.h:1264
unsigned fastEmitInst_i(unsigned MachineInstOpcode, const TargetRegisterClass *RC, uint64_t Imm)
Emit a MachineInstr with a single immediate operand, and a result register in the given register clas...
Definition: FastISel.cpp:2211
static MachineOperand CreateImm(int64_t Val)
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
static void addStackMapLiveVars(ImmutableCallSite CS, unsigned StartIdx, const SDLoc &DL, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Add a stack map intrinsic call&#39;s live variable operands to a stackmap or patchpoint target node&#39;s ope...
The memory access always returns the same value (or traps).
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:192
static bool isRegUsedByPhiNodes(unsigned DefReg, FunctionLoweringInfo &FuncInfo)
Definition: FastISel.cpp:220
0 1 1 0 True if ordered and operands are unequal
Definition: InstrTypes.h:717
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
MachineBasicBlock::iterator InsertPt
MBB - The current insert position inside the current block.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:332
uint32_t Size
Definition: Profile.cpp:46
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:324
DILocalVariable * getVariable() const
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
bool isUnconditional() const
1 0 1 0 True if unordered or greater than
Definition: InstrTypes.h:721
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:316
const TargetLowering & TLI
Definition: FastISel.h:212
unsigned createResultReg(const TargetRegisterClass *RC)
Definition: FastISel.cpp:2016
CallLoweringInfo & setIsPatchPoint(bool Value=true)
Definition: FastISel.h:182
unsigned fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t Imm, MVT ImmType)
This method is a wrapper of fastEmit_ri.
Definition: FastISel.cpp:1973
unsigned fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0, bool Op0IsKill, uint32_t Idx)
Emit a MachineInstr for an extract_subreg from a specified index of a superregister to a specified ty...
Definition: FastISel.cpp:2227
MachineBasicBlock::iterator InsertPt
Definition: FastISel.h:318
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
user_iterator user_begin()
Definition: Value.h:375
FastISel(FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo, bool SkipTargetIndependentISel=false)
Definition: FastISel.cpp:1921
virtual bool fastLowerArguments()
This method is called by target-independent code to do target- specific argument lowering.
Definition: FastISel.cpp:1934
0 0 0 1 True if ordered and equal
Definition: InstrTypes.h:712
LLVM Value Representation.
Definition: Value.h:72
1 0 1 1 True if unordered, greater than, or equal
Definition: InstrTypes.h:722
uint64_t getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type...
Definition: DataLayout.h:444
static const Function * getParent(const Value *V)
static bool isCommutative(Instruction *I)
void getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV, bool CannotUsePrivateLabel) const
Print the appropriate prefix and the specified global variable&#39;s name.
Definition: Mangler.cpp:111
DenseMap< const BasicBlock *, MachineBasicBlock * > MBBMap
MBBMap - A mapping from LLVM basic blocks to their machine code entry.
IRTranslator LLVM IR MI
bool hasOneUse() const
Return true if there is exactly one user of this value.
Definition: Value.h:412
void fastEmitBranch(MachineBasicBlock *MSucc, const DebugLoc &DbgLoc)
Emit an unconditional branch to the given block, unless it is the immediate (fall-through) successor...
Definition: FastISel.cpp:1676
unsigned greater than
Definition: InstrTypes.h:732
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
static cl::opt< bool > SinkLocalValues("fast-isel-sink-local-values", cl::init(true), cl::Hidden, cl::desc("Sink local values in FastISel"))
unsigned fastEmitInst_f(unsigned MachineInstOpcode, const TargetRegisterClass *RC, const ConstantFP *FPImm)
Emit a MachineInstr with a floating point immediate, and a result register in the given register clas...
Definition: FastISel.cpp:2166
bool isEmptyTy() const
Return true if this type is empty, that is, it has no elements or all of its elements are empty...
Definition: Type.cpp:97
Conversion operators.
Definition: ISDOpcodes.h:489
FunctionLoweringInfo & FuncInfo
Definition: FastISel.h:203
const Value * stripInBoundsConstantOffsets() const
Strip off pointer casts and all-constant inbounds GEPs.
Definition: Value.cpp:537
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:498
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:125
#define LLVM_DEBUG(X)
Definition: Debug.h:122
0 0 1 1 True if ordered and greater than or equal
Definition: InstrTypes.h:714
unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
Definition: Analysis.cpp:35
reg_begin/reg_end - Provide iteration support to walk over all definitions and uses of a register wit...
This represents the llvm.dbg.declare instruction.
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:610
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:63
static MachineOperand CreateFI(int Idx)
bool use_empty() const
Definition: Value.h:322
bool isSafeToMove(AliasAnalysis *AA, bool &SawStore) const
Return true if it is safe to move this instruction.
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:544
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Type * getElementType() const
Definition: DerivedTypes.h:517
unsigned createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute >> Attrs)
Create an AttributeList with the specified parameters in it.
Definition: Attributes.cpp:874
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:711
signed greater or equal
Definition: InstrTypes.h:737
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:43
This file describes how to lower LLVM code to machine code.
const BasicBlock * getParent() const
Definition: Instruction.h:66
gep_type_iterator gep_type_begin(const User *GEP)