LLVM  10.0.0svn
IRTranslator.cpp
Go to the documentation of this file.
1 //===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the IRTranslator class.
10 //===----------------------------------------------------------------------===//
11 
14 #include "llvm/ADT/STLExtras.h"
15 #include "llvm/ADT/ScopeExit.h"
16 #include "llvm/ADT/SmallSet.h"
17 #include "llvm/ADT/SmallVector.h"
21 #include "llvm/CodeGen/Analysis.h"
39 #include "llvm/IR/BasicBlock.h"
40 #include "llvm/IR/CFG.h"
41 #include "llvm/IR/Constant.h"
42 #include "llvm/IR/Constants.h"
43 #include "llvm/IR/DataLayout.h"
44 #include "llvm/IR/DebugInfo.h"
45 #include "llvm/IR/DerivedTypes.h"
46 #include "llvm/IR/Function.h"
48 #include "llvm/IR/InlineAsm.h"
49 #include "llvm/IR/InstrTypes.h"
50 #include "llvm/IR/Instructions.h"
51 #include "llvm/IR/IntrinsicInst.h"
52 #include "llvm/IR/Intrinsics.h"
53 #include "llvm/IR/LLVMContext.h"
54 #include "llvm/IR/Metadata.h"
55 #include "llvm/IR/Type.h"
56 #include "llvm/IR/User.h"
57 #include "llvm/IR/Value.h"
58 #include "llvm/MC/MCContext.h"
59 #include "llvm/Pass.h"
60 #include "llvm/Support/Casting.h"
61 #include "llvm/Support/CodeGen.h"
62 #include "llvm/Support/Debug.h"
69 #include <algorithm>
70 #include <cassert>
71 #include <cstdint>
72 #include <iterator>
73 #include <string>
74 #include <utility>
75 #include <vector>
76 
77 #define DEBUG_TYPE "irtranslator"
78 
79 using namespace llvm;
80 
81 static cl::opt<bool>
82  EnableCSEInIRTranslator("enable-cse-in-irtranslator",
83  cl::desc("Should enable CSE in irtranslator"),
84  cl::Optional, cl::init(false));
85 char IRTranslator::ID = 0;
86 
87 INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
88  false, false)
91 INITIALIZE_PASS_END(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
92  false, false)
93 
99 
100  // Print the function name explicitly if we don't have a debug location (which
101  // makes the diagnostic less useful) or if we're going to emit a raw error.
102  if (!R.getLocation().isValid() || TPC.isGlobalISelAbortEnabled())
103  R << (" (in function: " + MF.getName() + ")").str();
104 
105  if (TPC.isGlobalISelAbortEnabled())
106  report_fatal_error(R.getMsg());
107  else
108  ORE.emit(R);
109 }
110 
112 
113 #ifndef NDEBUG
114 namespace {
115 /// Verify that every instruction created has the same DILocation as the
116 /// instruction being translated.
117 class DILocationVerifier : public GISelChangeObserver {
118  const Instruction *CurrInst = nullptr;
119 
120 public:
121  DILocationVerifier() = default;
122  ~DILocationVerifier() = default;
123 
124  const Instruction *getCurrentInst() const { return CurrInst; }
125  void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
126 
127  void erasingInstr(MachineInstr &MI) override {}
128  void changingInstr(MachineInstr &MI) override {}
129  void changedInstr(MachineInstr &MI) override {}
130 
131  void createdInstr(MachineInstr &MI) override {
132  assert(getCurrentInst() && "Inserted instruction without a current MI");
133 
134  // Only print the check message if we're actually checking it.
135 #ifndef NDEBUG
136  LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
137  << " was copied to " << MI);
138 #endif
139  // We allow insts in the entry block to have a debug loc line of 0 because
140  // they could have originated from constants, and we don't want a jumpy
141  // debug experience.
142  assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
143  MI.getDebugLoc().getLine() == 0) &&
144  "Line info was not transferred to all instructions");
145  }
146 };
147 } // namespace
148 #endif // ifndef NDEBUG
149 
150 
157 }
158 
160 IRTranslator::allocateVRegs(const Value &Val) {
161  assert(!VMap.contains(Val) && "Value already allocated in VMap");
162  auto *Regs = VMap.getVRegs(Val);
163  auto *Offsets = VMap.getOffsets(Val);
164  SmallVector<LLT, 4> SplitTys;
165  computeValueLLTs(*DL, *Val.getType(), SplitTys,
166  Offsets->empty() ? Offsets : nullptr);
167  for (unsigned i = 0; i < SplitTys.size(); ++i)
168  Regs->push_back(0);
169  return *Regs;
170 }
171 
172 ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
173  auto VRegsIt = VMap.findVRegs(Val);
174  if (VRegsIt != VMap.vregs_end())
175  return *VRegsIt->second;
176 
177  if (Val.getType()->isVoidTy())
178  return *VMap.getVRegs(Val);
179 
180  // Create entry for this type.
181  auto *VRegs = VMap.getVRegs(Val);
182  auto *Offsets = VMap.getOffsets(Val);
183 
184  assert(Val.getType()->isSized() &&
185  "Don't know how to create an empty vreg");
186 
187  SmallVector<LLT, 4> SplitTys;
188  computeValueLLTs(*DL, *Val.getType(), SplitTys,
189  Offsets->empty() ? Offsets : nullptr);
190 
191  if (!isa<Constant>(Val)) {
192  for (auto Ty : SplitTys)
193  VRegs->push_back(MRI->createGenericVirtualRegister(Ty));
194  return *VRegs;
195  }
196 
197  if (Val.getType()->isAggregateType()) {
198  // UndefValue, ConstantAggregateZero
199  auto &C = cast<Constant>(Val);
200  unsigned Idx = 0;
201  while (auto Elt = C.getAggregateElement(Idx++)) {
202  auto EltRegs = getOrCreateVRegs(*Elt);
203  llvm::copy(EltRegs, std::back_inserter(*VRegs));
204  }
205  } else {
206  assert(SplitTys.size() == 1 && "unexpectedly split LLT");
207  VRegs->push_back(MRI->createGenericVirtualRegister(SplitTys[0]));
208  bool Success = translate(cast<Constant>(Val), VRegs->front());
209  if (!Success) {
210  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
211  MF->getFunction().getSubprogram(),
212  &MF->getFunction().getEntryBlock());
213  R << "unable to translate constant: " << ore::NV("Type", Val.getType());
214  reportTranslationError(*MF, *TPC, *ORE, R);
215  return *VRegs;
216  }
217  }
218 
219  return *VRegs;
220 }
221 
222 int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
223  if (FrameIndices.find(&AI) != FrameIndices.end())
224  return FrameIndices[&AI];
225 
226  unsigned ElementSize = DL->getTypeAllocSize(AI.getAllocatedType());
227  unsigned Size =
228  ElementSize * cast<ConstantInt>(AI.getArraySize())->getZExtValue();
229 
230  // Always allocate at least one byte.
231  Size = std::max(Size, 1u);
232 
233  unsigned Alignment = AI.getAlignment();
234  if (!Alignment)
235  Alignment = DL->getABITypeAlignment(AI.getAllocatedType());
236 
237  int &FI = FrameIndices[&AI];
238  FI = MF->getFrameInfo().CreateStackObject(Size, Alignment, false, &AI);
239  return FI;
240 }
241 
242 unsigned IRTranslator::getMemOpAlignment(const Instruction &I) {
243  unsigned Alignment = 0;
244  Type *ValTy = nullptr;
245  if (const StoreInst *SI = dyn_cast<StoreInst>(&I)) {
246  Alignment = SI->getAlignment();
247  ValTy = SI->getValueOperand()->getType();
248  } else if (const LoadInst *LI = dyn_cast<LoadInst>(&I)) {
249  Alignment = LI->getAlignment();
250  ValTy = LI->getType();
251  } else if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I)) {
252  // TODO(PR27168): This instruction has no alignment attribute, but unlike
253  // the default alignment for load/store, the default here is to assume
254  // it has NATURAL alignment, not DataLayout-specified alignment.
255  const DataLayout &DL = AI->getModule()->getDataLayout();
256  Alignment = DL.getTypeStoreSize(AI->getCompareOperand()->getType());
257  ValTy = AI->getCompareOperand()->getType();
258  } else if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I)) {
259  // TODO(PR27168): This instruction has no alignment attribute, but unlike
260  // the default alignment for load/store, the default here is to assume
261  // it has NATURAL alignment, not DataLayout-specified alignment.
262  const DataLayout &DL = AI->getModule()->getDataLayout();
263  Alignment = DL.getTypeStoreSize(AI->getValOperand()->getType());
264  ValTy = AI->getType();
265  } else {
266  OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
267  R << "unable to translate memop: " << ore::NV("Opcode", &I);
268  reportTranslationError(*MF, *TPC, *ORE, R);
269  return 1;
270  }
271 
272  return Alignment ? Alignment : DL->getABITypeAlignment(ValTy);
273 }
274 
275 MachineBasicBlock &IRTranslator::getMBB(const BasicBlock &BB) {
276  MachineBasicBlock *&MBB = BBToMBB[&BB];
277  assert(MBB && "BasicBlock was not encountered before");
278  return *MBB;
279 }
280 
281 void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) {
282  assert(NewPred && "new predecessor must be a real MachineBasicBlock");
283  MachinePreds[Edge].push_back(NewPred);
284 }
285 
286 bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
287  MachineIRBuilder &MIRBuilder) {
288  // Get or create a virtual register for each value.
289  // Unless the value is a Constant => loadimm cst?
290  // or inline constant each time?
291  // Creation of a virtual register needs to have a size.
292  Register Op0 = getOrCreateVReg(*U.getOperand(0));
293  Register Op1 = getOrCreateVReg(*U.getOperand(1));
294  Register Res = getOrCreateVReg(U);
295  uint16_t Flags = 0;
296  if (isa<Instruction>(U)) {
297  const Instruction &I = cast<Instruction>(U);
299  }
300 
301  MIRBuilder.buildInstr(Opcode, {Res}, {Op0, Op1}, Flags);
302  return true;
303 }
304 
305 bool IRTranslator::translateFSub(const User &U, MachineIRBuilder &MIRBuilder) {
306  // -0.0 - X --> G_FNEG
307  if (isa<Constant>(U.getOperand(0)) &&
309  Register Op1 = getOrCreateVReg(*U.getOperand(1));
310  Register Res = getOrCreateVReg(U);
311  uint16_t Flags = 0;
312  if (isa<Instruction>(U)) {
313  const Instruction &I = cast<Instruction>(U);
315  }
316  // Negate the last operand of the FSUB
317  MIRBuilder.buildInstr(TargetOpcode::G_FNEG, {Res}, {Op1}, Flags);
318  return true;
319  }
320  return translateBinaryOp(TargetOpcode::G_FSUB, U, MIRBuilder);
321 }
322 
323 bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) {
324  Register Op0 = getOrCreateVReg(*U.getOperand(0));
325  Register Res = getOrCreateVReg(U);
326  uint16_t Flags = 0;
327  if (isa<Instruction>(U)) {
328  const Instruction &I = cast<Instruction>(U);
330  }
331  MIRBuilder.buildInstr(TargetOpcode::G_FNEG, {Res}, {Op0}, Flags);
332  return true;
333 }
334 
335 bool IRTranslator::translateCompare(const User &U,
336  MachineIRBuilder &MIRBuilder) {
337  const CmpInst *CI = dyn_cast<CmpInst>(&U);
338  Register Op0 = getOrCreateVReg(*U.getOperand(0));
339  Register Op1 = getOrCreateVReg(*U.getOperand(1));
340  Register Res = getOrCreateVReg(U);
341  CmpInst::Predicate Pred =
342  CI ? CI->getPredicate() : static_cast<CmpInst::Predicate>(
343  cast<ConstantExpr>(U).getPredicate());
344  if (CmpInst::isIntPredicate(Pred))
345  MIRBuilder.buildICmp(Pred, Res, Op0, Op1);
346  else if (Pred == CmpInst::FCMP_FALSE)
347  MIRBuilder.buildCopy(
348  Res, getOrCreateVReg(*Constant::getNullValue(CI->getType())));
349  else if (Pred == CmpInst::FCMP_TRUE)
350  MIRBuilder.buildCopy(
351  Res, getOrCreateVReg(*Constant::getAllOnesValue(CI->getType())));
352  else {
353  MIRBuilder.buildInstr(TargetOpcode::G_FCMP, {Res}, {Pred, Op0, Op1},
355  }
356 
357  return true;
358 }
359 
360 bool IRTranslator::translateRet(const User &U, MachineIRBuilder &MIRBuilder) {
361  const ReturnInst &RI = cast<ReturnInst>(U);
362  const Value *Ret = RI.getReturnValue();
363  if (Ret && DL->getTypeStoreSize(Ret->getType()) == 0)
364  Ret = nullptr;
365 
366  ArrayRef<Register> VRegs;
367  if (Ret)
368  VRegs = getOrCreateVRegs(*Ret);
369 
370  Register SwiftErrorVReg = 0;
371  if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {
372  SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
373  &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
374  }
375 
376  // The target may mess up with the insertion point, but
377  // this is not important as a return is the last instruction
378  // of the block anyway.
379  return CLI->lowerReturn(MIRBuilder, Ret, VRegs, SwiftErrorVReg);
380 }
381 
382 bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) {
383  const BranchInst &BrInst = cast<BranchInst>(U);
384  unsigned Succ = 0;
385  if (!BrInst.isUnconditional()) {
386  // We want a G_BRCOND to the true BB followed by an unconditional branch.
387  Register Tst = getOrCreateVReg(*BrInst.getCondition());
388  const BasicBlock &TrueTgt = *cast<BasicBlock>(BrInst.getSuccessor(Succ++));
389  MachineBasicBlock &TrueBB = getMBB(TrueTgt);
390  MIRBuilder.buildBrCond(Tst, TrueBB);
391  }
392 
393  const BasicBlock &BrTgt = *cast<BasicBlock>(BrInst.getSuccessor(Succ));
394  MachineBasicBlock &TgtBB = getMBB(BrTgt);
395  MachineBasicBlock &CurBB = MIRBuilder.getMBB();
396 
397  // If the unconditional target is the layout successor, fallthrough.
398  if (!CurBB.isLayoutSuccessor(&TgtBB))
399  MIRBuilder.buildBr(TgtBB);
400 
401  // Link successors.
402  for (const BasicBlock *Succ : successors(&BrInst))
403  CurBB.addSuccessor(&getMBB(*Succ));
404  return true;
405 }
406 
407 void IRTranslator::addSuccessorWithProb(MachineBasicBlock *Src,
408  MachineBasicBlock *Dst,
409  BranchProbability Prob) {
410  if (!FuncInfo.BPI) {
411  Src->addSuccessorWithoutProb(Dst);
412  return;
413  }
414  if (Prob.isUnknown())
415  Prob = getEdgeProbability(Src, Dst);
416  Src->addSuccessor(Dst, Prob);
417 }
418 
420 IRTranslator::getEdgeProbability(const MachineBasicBlock *Src,
421  const MachineBasicBlock *Dst) const {
422  const BasicBlock *SrcBB = Src->getBasicBlock();
423  const BasicBlock *DstBB = Dst->getBasicBlock();
424  if (!FuncInfo.BPI) {
425  // If BPI is not available, set the default probability as 1 / N, where N is
426  // the number of successors.
427  auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
428  return BranchProbability(1, SuccSize);
429  }
430  return FuncInfo.BPI->getEdgeProbability(SrcBB, DstBB);
431 }
432 
433 bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) {
434  using namespace SwitchCG;
435  // Extract cases from the switch.
436  const SwitchInst &SI = cast<SwitchInst>(U);
437  BranchProbabilityInfo *BPI = FuncInfo.BPI;
438  CaseClusterVector Clusters;
439  Clusters.reserve(SI.getNumCases());
440  for (auto &I : SI.cases()) {
441  MachineBasicBlock *Succ = &getMBB(*I.getCaseSuccessor());
442  assert(Succ && "Could not find successor mbb in mapping");
443  const ConstantInt *CaseVal = I.getCaseValue();
444  BranchProbability Prob =
445  BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
446  : BranchProbability(1, SI.getNumCases() + 1);
447  Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
448  }
449 
450  MachineBasicBlock *DefaultMBB = &getMBB(*SI.getDefaultDest());
451 
452  // Cluster adjacent cases with the same destination. We do this at all
453  // optimization levels because it's cheap to do and will make codegen faster
454  // if there are many clusters.
455  sortAndRangeify(Clusters);
456 
457  MachineBasicBlock *SwitchMBB = &getMBB(*SI.getParent());
458 
459  // If there is only the default destination, jump there directly.
460  if (Clusters.empty()) {
461  SwitchMBB->addSuccessor(DefaultMBB);
462  if (DefaultMBB != SwitchMBB->getNextNode())
463  MIB.buildBr(*DefaultMBB);
464  return true;
465  }
466 
467  SL->findJumpTables(Clusters, &SI, DefaultMBB);
468 
469  LLVM_DEBUG({
470  dbgs() << "Case clusters: ";
471  for (const CaseCluster &C : Clusters) {
472  if (C.Kind == CC_JumpTable)
473  dbgs() << "JT:";
474  if (C.Kind == CC_BitTests)
475  dbgs() << "BT:";
476 
477  C.Low->getValue().print(dbgs(), true);
478  if (C.Low != C.High) {
479  dbgs() << '-';
480  C.High->getValue().print(dbgs(), true);
481  }
482  dbgs() << ' ';
483  }
484  dbgs() << '\n';
485  });
486 
487  assert(!Clusters.empty());
488  SwitchWorkList WorkList;
489  CaseClusterIt First = Clusters.begin();
490  CaseClusterIt Last = Clusters.end() - 1;
491  auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB);
492  WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
493 
494  // FIXME: At the moment we don't do any splitting optimizations here like
495  // SelectionDAG does, so this worklist only has one entry.
496  while (!WorkList.empty()) {
497  SwitchWorkListItem W = WorkList.back();
498  WorkList.pop_back();
499  if (!lowerSwitchWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
500  return false;
501  }
502  return true;
503 }
504 
505 void IRTranslator::emitJumpTable(SwitchCG::JumpTable &JT,
506  MachineBasicBlock *MBB) {
507  // Emit the code for the jump table
508  assert(JT.Reg != -1U && "Should lower JT Header first!");
509  MachineIRBuilder MIB(*MBB->getParent());
510  MIB.setMBB(*MBB);
511  MIB.setDebugLoc(CurBuilder->getDebugLoc());
512 
513  Type *PtrIRTy = Type::getInt8PtrTy(MF->getFunction().getContext());
514  const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
515 
516  auto Table = MIB.buildJumpTable(PtrTy, JT.JTI);
517  MIB.buildBrJT(Table.getReg(0), JT.JTI, JT.Reg);
518 }
519 
520 bool IRTranslator::emitJumpTableHeader(SwitchCG::JumpTable &JT,
522  MachineBasicBlock *HeaderBB) {
523  MachineIRBuilder MIB(*HeaderBB->getParent());
524  MIB.setMBB(*HeaderBB);
525  MIB.setDebugLoc(CurBuilder->getDebugLoc());
526 
527  const Value &SValue = *JTH.SValue;
528  // Subtract the lowest switch case value from the value being switched on.
529  const LLT SwitchTy = getLLTForType(*SValue.getType(), *DL);
530  Register SwitchOpReg = getOrCreateVReg(SValue);
531  auto FirstCst = MIB.buildConstant(SwitchTy, JTH.First);
532  auto Sub = MIB.buildSub({SwitchTy}, SwitchOpReg, FirstCst);
533 
534  // This value may be smaller or larger than the target's pointer type, and
535  // therefore require extension or truncating.
536  Type *PtrIRTy = SValue.getType()->getPointerTo();
537  const LLT PtrScalarTy = LLT::scalar(DL->getTypeSizeInBits(PtrIRTy));
538  Sub = MIB.buildZExtOrTrunc(PtrScalarTy, Sub);
539 
540  JT.Reg = Sub.getReg(0);
541 
542  if (JTH.OmitRangeCheck) {
543  if (JT.MBB != HeaderBB->getNextNode())
544  MIB.buildBr(*JT.MBB);
545  return true;
546  }
547 
548  // Emit the range check for the jump table, and branch to the default block
549  // for the switch statement if the value being switched on exceeds the
550  // largest case in the switch.
551  auto Cst = getOrCreateVReg(
552  *ConstantInt::get(SValue.getType(), JTH.Last - JTH.First));
553  Cst = MIB.buildZExtOrTrunc(PtrScalarTy, Cst).getReg(0);
554  auto Cmp = MIB.buildICmp(CmpInst::ICMP_UGT, LLT::scalar(1), Sub, Cst);
555 
556  auto BrCond = MIB.buildBrCond(Cmp.getReg(0), *JT.Default);
557 
558  // Avoid emitting unnecessary branches to the next block.
559  if (JT.MBB != HeaderBB->getNextNode())
560  BrCond = MIB.buildBr(*JT.MBB);
561  return true;
562 }
563 
564 void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
565  MachineBasicBlock *SwitchBB,
566  MachineIRBuilder &MIB) {
567  Register CondLHS = getOrCreateVReg(*CB.CmpLHS);
568  Register Cond;
569  DebugLoc OldDbgLoc = MIB.getDebugLoc();
570  MIB.setDebugLoc(CB.DbgLoc);
571  MIB.setMBB(*CB.ThisBB);
572 
573  if (CB.PredInfo.NoCmp) {
574  // Branch or fall through to TrueBB.
575  addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
576  addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
577  CB.ThisBB);
579  if (CB.TrueBB != CB.ThisBB->getNextNode())
580  MIB.buildBr(*CB.TrueBB);
581  MIB.setDebugLoc(OldDbgLoc);
582  return;
583  }
584 
585  const LLT i1Ty = LLT::scalar(1);
586  // Build the compare.
587  if (!CB.CmpMHS) {
588  Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
589  Cond = MIB.buildICmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
590  } else {
592  "Can only handle ULE ranges");
593 
594  const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
595  const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
596 
597  Register CmpOpReg = getOrCreateVReg(*CB.CmpMHS);
598  if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
599  Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
600  Cond =
601  MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, CmpOpReg, CondRHS).getReg(0);
602  } else {
603  const LLT &CmpTy = MRI->getType(CmpOpReg);
604  auto Sub = MIB.buildSub({CmpTy}, CmpOpReg, CondLHS);
605  auto Diff = MIB.buildConstant(CmpTy, High - Low);
606  Cond = MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, Sub, Diff).getReg(0);
607  }
608  }
609 
610  // Update successor info
611  addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
612 
613  addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
614  CB.ThisBB);
615 
616  // TrueBB and FalseBB are always different unless the incoming IR is
617  // degenerate. This only happens when running llc on weird IR.
618  if (CB.TrueBB != CB.FalseBB)
619  addSuccessorWithProb(CB.ThisBB, CB.FalseBB, CB.FalseProb);
621 
622  // if (SwitchBB->getBasicBlock() != CB.FalseBB->getBasicBlock())
623  addMachineCFGPred({SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
624  CB.ThisBB);
625 
626  // If the lhs block is the next block, invert the condition so that we can
627  // fall through to the lhs instead of the rhs block.
628  if (CB.TrueBB == CB.ThisBB->getNextNode()) {
629  std::swap(CB.TrueBB, CB.FalseBB);
630  auto True = MIB.buildConstant(i1Ty, 1);
631  Cond = MIB.buildInstr(TargetOpcode::G_XOR, {i1Ty}, {Cond, True}, None)
632  .getReg(0);
633  }
634 
635  MIB.buildBrCond(Cond, *CB.TrueBB);
636  MIB.buildBr(*CB.FalseBB);
637  MIB.setDebugLoc(OldDbgLoc);
638 }
639 
640 bool IRTranslator::lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W,
641  MachineBasicBlock *SwitchMBB,
642  MachineBasicBlock *CurMBB,
643  MachineBasicBlock *DefaultMBB,
644  MachineIRBuilder &MIB,
646  BranchProbability UnhandledProbs,
648  MachineBasicBlock *Fallthrough,
649  bool FallthroughUnreachable) {
650  using namespace SwitchCG;
651  MachineFunction *CurMF = SwitchMBB->getParent();
652  // FIXME: Optimize away range check based on pivot comparisons.
653  JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
654  SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
655  BranchProbability DefaultProb = W.DefaultProb;
656 
657  // The jump block hasn't been inserted yet; insert it here.
658  MachineBasicBlock *JumpMBB = JT->MBB;
659  CurMF->insert(BBI, JumpMBB);
660 
661  // Since the jump table block is separate from the switch block, we need
662  // to keep track of it as a machine predecessor to the default block,
663  // otherwise we lose the phi edges.
664  addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
665  CurMBB);
666  addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
667  JumpMBB);
668 
669  auto JumpProb = I->Prob;
670  auto FallthroughProb = UnhandledProbs;
671 
672  // If the default statement is a target of the jump table, we evenly
673  // distribute the default probability to successors of CurMBB. Also
674  // update the probability on the edge from JumpMBB to Fallthrough.
676  SE = JumpMBB->succ_end();
677  SI != SE; ++SI) {
678  if (*SI == DefaultMBB) {
679  JumpProb += DefaultProb / 2;
680  FallthroughProb -= DefaultProb / 2;
681  JumpMBB->setSuccProbability(SI, DefaultProb / 2);
682  JumpMBB->normalizeSuccProbs();
683  } else {
684  // Also record edges from the jump table block to it's successors.
685  addMachineCFGPred({SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
686  JumpMBB);
687  }
688  }
689 
690  // Skip the range check if the fallthrough block is unreachable.
691  if (FallthroughUnreachable)
692  JTH->OmitRangeCheck = true;
693 
694  if (!JTH->OmitRangeCheck)
695  addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
696  addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
697  CurMBB->normalizeSuccProbs();
698 
699  // The jump table header will be inserted in our current block, do the
700  // range check, and fall through to our fallthrough block.
701  JTH->HeaderBB = CurMBB;
702  JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
703 
704  // If we're in the right place, emit the jump table header right now.
705  if (CurMBB == SwitchMBB) {
706  if (!emitJumpTableHeader(*JT, *JTH, CurMBB))
707  return false;
708  JTH->Emitted = true;
709  }
710  return true;
711 }
712 bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,
713  Value *Cond,
714  MachineBasicBlock *Fallthrough,
715  bool FallthroughUnreachable,
716  BranchProbability UnhandledProbs,
717  MachineBasicBlock *CurMBB,
718  MachineIRBuilder &MIB,
719  MachineBasicBlock *SwitchMBB) {
720  using namespace SwitchCG;
721  const Value *RHS, *LHS, *MHS;
722  CmpInst::Predicate Pred;
723  if (I->Low == I->High) {
724  // Check Cond == I->Low.
725  Pred = CmpInst::ICMP_EQ;
726  LHS = Cond;
727  RHS = I->Low;
728  MHS = nullptr;
729  } else {
730  // Check I->Low <= Cond <= I->High.
731  Pred = CmpInst::ICMP_ULE;
732  LHS = I->Low;
733  MHS = Cond;
734  RHS = I->High;
735  }
736 
737  // If Fallthrough is unreachable, fold away the comparison.
738  // The false probability is the sum of all unhandled cases.
739  CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
740  CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
741 
742  emitSwitchCase(CB, SwitchMBB, MIB);
743  return true;
744 }
745 
746 bool IRTranslator::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
747  Value *Cond,
748  MachineBasicBlock *SwitchMBB,
749  MachineBasicBlock *DefaultMBB,
750  MachineIRBuilder &MIB) {
751  using namespace SwitchCG;
752  MachineFunction *CurMF = FuncInfo.MF;
753  MachineBasicBlock *NextMBB = nullptr;
755  if (++BBI != FuncInfo.MF->end())
756  NextMBB = &*BBI;
757 
758  if (EnableOpts) {
759  // Here, we order cases by probability so the most likely case will be
760  // checked first. However, two clusters can have the same probability in
761  // which case their relative ordering is non-deterministic. So we use Low
762  // as a tie-breaker as clusters are guaranteed to never overlap.
764  [](const CaseCluster &a, const CaseCluster &b) {
765  return a.Prob != b.Prob
766  ? a.Prob > b.Prob
767  : a.Low->getValue().slt(b.Low->getValue());
768  });
769 
770  // Rearrange the case blocks so that the last one falls through if possible
771  // without changing the order of probabilities.
772  for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
773  --I;
774  if (I->Prob > W.LastCluster->Prob)
775  break;
776  if (I->Kind == CC_Range && I->MBB == NextMBB) {
777  std::swap(*I, *W.LastCluster);
778  break;
779  }
780  }
781  }
782 
783  // Compute total probability.
784  BranchProbability DefaultProb = W.DefaultProb;
785  BranchProbability UnhandledProbs = DefaultProb;
786  for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
787  UnhandledProbs += I->Prob;
788 
789  MachineBasicBlock *CurMBB = W.MBB;
790  for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
791  bool FallthroughUnreachable = false;
792  MachineBasicBlock *Fallthrough;
793  if (I == W.LastCluster) {
794  // For the last cluster, fall through to the default destination.
795  Fallthrough = DefaultMBB;
796  FallthroughUnreachable = isa<UnreachableInst>(
797  DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
798  } else {
799  Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
800  CurMF->insert(BBI, Fallthrough);
801  }
802  UnhandledProbs -= I->Prob;
803 
804  switch (I->Kind) {
805  case CC_BitTests: {
806  LLVM_DEBUG(dbgs() << "Switch to bit test optimization unimplemented");
807  return false; // Bit tests currently unimplemented.
808  }
809  case CC_JumpTable: {
810  if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
811  UnhandledProbs, I, Fallthrough,
812  FallthroughUnreachable)) {
813  LLVM_DEBUG(dbgs() << "Failed to lower jump table");
814  return false;
815  }
816  break;
817  }
818  case CC_Range: {
819  if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
820  FallthroughUnreachable, UnhandledProbs,
821  CurMBB, MIB, SwitchMBB)) {
822  LLVM_DEBUG(dbgs() << "Failed to lower switch range");
823  return false;
824  }
825  break;
826  }
827  }
828  CurMBB = Fallthrough;
829  }
830 
831  return true;
832 }
833 
834 bool IRTranslator::translateIndirectBr(const User &U,
835  MachineIRBuilder &MIRBuilder) {
836  const IndirectBrInst &BrInst = cast<IndirectBrInst>(U);
837 
838  const Register Tgt = getOrCreateVReg(*BrInst.getAddress());
839  MIRBuilder.buildBrIndirect(Tgt);
840 
841  // Link successors.
842  MachineBasicBlock &CurBB = MIRBuilder.getMBB();
843  for (const BasicBlock *Succ : successors(&BrInst))
844  CurBB.addSuccessor(&getMBB(*Succ));
845 
846  return true;
847 }
848 
849 static bool isSwiftError(const Value *V) {
850  if (auto Arg = dyn_cast<Argument>(V))
851  return Arg->hasSwiftErrorAttr();
852  if (auto AI = dyn_cast<AllocaInst>(V))
853  return AI->isSwiftError();
854  return false;
855 }
856 
857 bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
858  const LoadInst &LI = cast<LoadInst>(U);
859 
860  auto Flags = LI.isVolatile() ? MachineMemOperand::MOVolatile
862  Flags |= MachineMemOperand::MOLoad;
863 
864  if (DL->getTypeStoreSize(LI.getType()) == 0)
865  return true;
866 
867  ArrayRef<Register> Regs = getOrCreateVRegs(LI);
868  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(LI);
869  Register Base = getOrCreateVReg(*LI.getPointerOperand());
870 
871  Type *OffsetIRTy = DL->getIntPtrType(LI.getPointerOperandType());
872  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
873 
874  if (CLI->supportSwiftError() && isSwiftError(LI.getPointerOperand())) {
875  assert(Regs.size() == 1 && "swifterror should be single pointer");
876  Register VReg = SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(),
877  LI.getPointerOperand());
878  MIRBuilder.buildCopy(Regs[0], VReg);
879  return true;
880  }
881 
882  const MDNode *Ranges =
883  Regs.size() == 1 ? LI.getMetadata(LLVMContext::MD_range) : nullptr;
884  for (unsigned i = 0; i < Regs.size(); ++i) {
885  Register Addr;
886  MIRBuilder.materializeGEP(Addr, Base, OffsetTy, Offsets[i] / 8);
887 
888  MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i] / 8);
889  unsigned BaseAlign = getMemOpAlignment(LI);
890  auto MMO = MF->getMachineMemOperand(
891  Ptr, Flags, (MRI->getType(Regs[i]).getSizeInBits() + 7) / 8,
892  MinAlign(BaseAlign, Offsets[i] / 8), AAMDNodes(), Ranges,
893  LI.getSyncScopeID(), LI.getOrdering());
894  MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
895  }
896 
897  return true;
898 }
899 
900 bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
901  const StoreInst &SI = cast<StoreInst>(U);
902  auto Flags = SI.isVolatile() ? MachineMemOperand::MOVolatile
905 
906  if (DL->getTypeStoreSize(SI.getValueOperand()->getType()) == 0)
907  return true;
908 
909  ArrayRef<Register> Vals = getOrCreateVRegs(*SI.getValueOperand());
910  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*SI.getValueOperand());
911  Register Base = getOrCreateVReg(*SI.getPointerOperand());
912 
913  Type *OffsetIRTy = DL->getIntPtrType(SI.getPointerOperandType());
914  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
915 
916  if (CLI->supportSwiftError() && isSwiftError(SI.getPointerOperand())) {
917  assert(Vals.size() == 1 && "swifterror should be single pointer");
918 
919  Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
920  SI.getPointerOperand());
921  MIRBuilder.buildCopy(VReg, Vals[0]);
922  return true;
923  }
924 
925  for (unsigned i = 0; i < Vals.size(); ++i) {
926  Register Addr;
927  MIRBuilder.materializeGEP(Addr, Base, OffsetTy, Offsets[i] / 8);
928 
929  MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i] / 8);
930  unsigned BaseAlign = getMemOpAlignment(SI);
931  auto MMO = MF->getMachineMemOperand(
932  Ptr, Flags, (MRI->getType(Vals[i]).getSizeInBits() + 7) / 8,
933  MinAlign(BaseAlign, Offsets[i] / 8), AAMDNodes(), nullptr,
934  SI.getSyncScopeID(), SI.getOrdering());
935  MIRBuilder.buildStore(Vals[i], Addr, *MMO);
936  }
937  return true;
938 }
939 
940 static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL) {
941  const Value *Src = U.getOperand(0);
943 
944  // getIndexedOffsetInType is designed for GEPs, so the first index is the
945  // usual array element rather than looking into the actual aggregate.
946  SmallVector<Value *, 1> Indices;
947  Indices.push_back(ConstantInt::get(Int32Ty, 0));
948 
949  if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(&U)) {
950  for (auto Idx : EVI->indices())
951  Indices.push_back(ConstantInt::get(Int32Ty, Idx));
952  } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(&U)) {
953  for (auto Idx : IVI->indices())
954  Indices.push_back(ConstantInt::get(Int32Ty, Idx));
955  } else {
956  for (unsigned i = 1; i < U.getNumOperands(); ++i)
957  Indices.push_back(U.getOperand(i));
958  }
959 
960  return 8 * static_cast<uint64_t>(
961  DL.getIndexedOffsetInType(Src->getType(), Indices));
962 }
963 
964 bool IRTranslator::translateExtractValue(const User &U,
965  MachineIRBuilder &MIRBuilder) {
966  const Value *Src = U.getOperand(0);
967  uint64_t Offset = getOffsetFromIndices(U, *DL);
968  ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
969  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*Src);
970  unsigned Idx = llvm::lower_bound(Offsets, Offset) - Offsets.begin();
971  auto &DstRegs = allocateVRegs(U);
972 
973  for (unsigned i = 0; i < DstRegs.size(); ++i)
974  DstRegs[i] = SrcRegs[Idx++];
975 
976  return true;
977 }
978 
979 bool IRTranslator::translateInsertValue(const User &U,
980  MachineIRBuilder &MIRBuilder) {
981  const Value *Src = U.getOperand(0);
982  uint64_t Offset = getOffsetFromIndices(U, *DL);
983  auto &DstRegs = allocateVRegs(U);
984  ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(U);
985  ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
986  ArrayRef<Register> InsertedRegs = getOrCreateVRegs(*U.getOperand(1));
987  auto InsertedIt = InsertedRegs.begin();
988 
989  for (unsigned i = 0; i < DstRegs.size(); ++i) {
990  if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
991  DstRegs[i] = *InsertedIt++;
992  else
993  DstRegs[i] = SrcRegs[i];
994  }
995 
996  return true;
997 }
998 
999 bool IRTranslator::translateSelect(const User &U,
1000  MachineIRBuilder &MIRBuilder) {
1001  Register Tst = getOrCreateVReg(*U.getOperand(0));
1002  ArrayRef<Register> ResRegs = getOrCreateVRegs(U);
1003  ArrayRef<Register> Op0Regs = getOrCreateVRegs(*U.getOperand(1));
1004  ArrayRef<Register> Op1Regs = getOrCreateVRegs(*U.getOperand(2));
1005 
1006  const SelectInst &SI = cast<SelectInst>(U);
1007  uint16_t Flags = 0;
1008  if (const CmpInst *Cmp = dyn_cast<CmpInst>(SI.getCondition()))
1010 
1011  for (unsigned i = 0; i < ResRegs.size(); ++i) {
1012  MIRBuilder.buildInstr(TargetOpcode::G_SELECT, {ResRegs[i]},
1013  {Tst, Op0Regs[i], Op1Regs[i]}, Flags);
1014  }
1015 
1016  return true;
1017 }
1018 
1019 bool IRTranslator::translateBitCast(const User &U,
1020  MachineIRBuilder &MIRBuilder) {
1021  // If we're bitcasting to the source type, we can reuse the source vreg.
1022  if (getLLTForType(*U.getOperand(0)->getType(), *DL) ==
1023  getLLTForType(*U.getType(), *DL)) {
1024  Register SrcReg = getOrCreateVReg(*U.getOperand(0));
1025  auto &Regs = *VMap.getVRegs(U);
1026  // If we already assigned a vreg for this bitcast, we can't change that.
1027  // Emit a copy to satisfy the users we already emitted.
1028  if (!Regs.empty())
1029  MIRBuilder.buildCopy(Regs[0], SrcReg);
1030  else {
1031  Regs.push_back(SrcReg);
1032  VMap.getOffsets(U)->push_back(0);
1033  }
1034  return true;
1035  }
1036  return translateCast(TargetOpcode::G_BITCAST, U, MIRBuilder);
1037 }
1038 
1039 bool IRTranslator::translateCast(unsigned Opcode, const User &U,
1040  MachineIRBuilder &MIRBuilder) {
1041  Register Op = getOrCreateVReg(*U.getOperand(0));
1042  Register Res = getOrCreateVReg(U);
1043  MIRBuilder.buildInstr(Opcode, {Res}, {Op});
1044  return true;
1045 }
1046 
1047 bool IRTranslator::translateGetElementPtr(const User &U,
1048  MachineIRBuilder &MIRBuilder) {
1049  // FIXME: support vector GEPs.
1050  if (U.getType()->isVectorTy())
1051  return false;
1052 
1053  Value &Op0 = *U.getOperand(0);
1054  Register BaseReg = getOrCreateVReg(Op0);
1055  Type *PtrIRTy = Op0.getType();
1056  LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
1057  Type *OffsetIRTy = DL->getIntPtrType(PtrIRTy);
1058  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1059 
1060  int64_t Offset = 0;
1061  for (gep_type_iterator GTI = gep_type_begin(&U), E = gep_type_end(&U);
1062  GTI != E; ++GTI) {
1063  const Value *Idx = GTI.getOperand();
1064  if (StructType *StTy = GTI.getStructTypeOrNull()) {
1065  unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
1066  Offset += DL->getStructLayout(StTy)->getElementOffset(Field);
1067  continue;
1068  } else {
1069  uint64_t ElementSize = DL->getTypeAllocSize(GTI.getIndexedType());
1070 
1071  // If this is a scalar constant or a splat vector of constants,
1072  // handle it quickly.
1073  if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
1074  Offset += ElementSize * CI->getSExtValue();
1075  continue;
1076  }
1077 
1078  if (Offset != 0) {
1079  Register NewBaseReg = MRI->createGenericVirtualRegister(PtrTy);
1080  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1081  auto OffsetMIB = MIRBuilder.buildConstant({OffsetTy}, Offset);
1082  MIRBuilder.buildGEP(NewBaseReg, BaseReg, OffsetMIB.getReg(0));
1083 
1084  BaseReg = NewBaseReg;
1085  Offset = 0;
1086  }
1087 
1088  Register IdxReg = getOrCreateVReg(*Idx);
1089  if (MRI->getType(IdxReg) != OffsetTy) {
1090  Register NewIdxReg = MRI->createGenericVirtualRegister(OffsetTy);
1091  MIRBuilder.buildSExtOrTrunc(NewIdxReg, IdxReg);
1092  IdxReg = NewIdxReg;
1093  }
1094 
1095  // N = N + Idx * ElementSize;
1096  // Avoid doing it for ElementSize of 1.
1097  Register GepOffsetReg;
1098  if (ElementSize != 1) {
1099  GepOffsetReg = MRI->createGenericVirtualRegister(OffsetTy);
1100  auto ElementSizeMIB = MIRBuilder.buildConstant(
1101  getLLTForType(*OffsetIRTy, *DL), ElementSize);
1102  MIRBuilder.buildMul(GepOffsetReg, ElementSizeMIB.getReg(0), IdxReg);
1103  } else
1104  GepOffsetReg = IdxReg;
1105 
1106  Register NewBaseReg = MRI->createGenericVirtualRegister(PtrTy);
1107  MIRBuilder.buildGEP(NewBaseReg, BaseReg, GepOffsetReg);
1108  BaseReg = NewBaseReg;
1109  }
1110  }
1111 
1112  if (Offset != 0) {
1113  auto OffsetMIB =
1114  MIRBuilder.buildConstant(getLLTForType(*OffsetIRTy, *DL), Offset);
1115  MIRBuilder.buildGEP(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0));
1116  return true;
1117  }
1118 
1119  MIRBuilder.buildCopy(getOrCreateVReg(U), BaseReg);
1120  return true;
1121 }
1122 
1123 bool IRTranslator::translateMemFunc(const CallInst &CI,
1124  MachineIRBuilder &MIRBuilder,
1125  Intrinsic::ID ID) {
1126 
1127  // If the source is undef, then just emit a nop.
1128  if (isa<UndefValue>(CI.getArgOperand(1)))
1129  return true;
1130 
1131  ArrayRef<Register> Res;
1132  auto ICall = MIRBuilder.buildIntrinsic(ID, Res, true);
1133  for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(AI) != AE; ++AI)
1134  ICall.addUse(getOrCreateVReg(**AI));
1135 
1136  unsigned DstAlign = 0, SrcAlign = 0;
1137  unsigned IsVol =
1138  cast<ConstantInt>(CI.getArgOperand(CI.getNumArgOperands() - 1))
1139  ->getZExtValue();
1140 
1141  if (auto *MCI = dyn_cast<MemCpyInst>(&CI)) {
1142  DstAlign = std::max<unsigned>(MCI->getDestAlignment(), 1);
1143  SrcAlign = std::max<unsigned>(MCI->getSourceAlignment(), 1);
1144  } else if (auto *MMI = dyn_cast<MemMoveInst>(&CI)) {
1145  DstAlign = std::max<unsigned>(MMI->getDestAlignment(), 1);
1146  SrcAlign = std::max<unsigned>(MMI->getSourceAlignment(), 1);
1147  } else {
1148  auto *MSI = cast<MemSetInst>(&CI);
1149  DstAlign = std::max<unsigned>(MSI->getDestAlignment(), 1);
1150  }
1151 
1152  // Create mem operands to store the alignment and volatile info.
1154  ICall.addMemOperand(MF->getMachineMemOperand(
1156  MachineMemOperand::MOStore | VolFlag, 1, DstAlign));
1157  if (ID != Intrinsic::memset)
1158  ICall.addMemOperand(MF->getMachineMemOperand(
1160  MachineMemOperand::MOLoad | VolFlag, 1, SrcAlign));
1161 
1162  return true;
1163 }
1164 
1165 void IRTranslator::getStackGuard(Register DstReg,
1166  MachineIRBuilder &MIRBuilder) {
1167  const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1168  MRI->setRegClass(DstReg, TRI->getPointerRegClass(*MF));
1169  auto MIB = MIRBuilder.buildInstr(TargetOpcode::LOAD_STACK_GUARD);
1170  MIB.addDef(DstReg);
1171 
1172  auto &TLI = *MF->getSubtarget().getTargetLowering();
1173  Value *Global = TLI.getSDagStackGuard(*MF->getFunction().getParent());
1174  if (!Global)
1175  return;
1176 
1177  MachinePointerInfo MPInfo(Global);
1181  MF->getMachineMemOperand(MPInfo, Flags, DL->getPointerSizeInBits() / 8,
1182  DL->getPointerABIAlignment(0));
1183  MIB.setMemRefs({MemRef});
1184 }
1185 
1186 bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
1187  MachineIRBuilder &MIRBuilder) {
1188  ArrayRef<Register> ResRegs = getOrCreateVRegs(CI);
1189  MIRBuilder.buildInstr(Op)
1190  .addDef(ResRegs[0])
1191  .addDef(ResRegs[1])
1192  .addUse(getOrCreateVReg(*CI.getOperand(0)))
1193  .addUse(getOrCreateVReg(*CI.getOperand(1)));
1194 
1195  return true;
1196 }
1197 
1198 unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
1199  switch (ID) {
1200  default:
1201  break;
1202  case Intrinsic::bswap:
1203  return TargetOpcode::G_BSWAP;
1204  case Intrinsic::ceil:
1205  return TargetOpcode::G_FCEIL;
1206  case Intrinsic::cos:
1207  return TargetOpcode::G_FCOS;
1208  case Intrinsic::ctpop:
1209  return TargetOpcode::G_CTPOP;
1210  case Intrinsic::exp:
1211  return TargetOpcode::G_FEXP;
1212  case Intrinsic::exp2:
1213  return TargetOpcode::G_FEXP2;
1214  case Intrinsic::fabs:
1215  return TargetOpcode::G_FABS;
1216  case Intrinsic::copysign:
1217  return TargetOpcode::G_FCOPYSIGN;
1218  case Intrinsic::minnum:
1219  return TargetOpcode::G_FMINNUM;
1220  case Intrinsic::maxnum:
1221  return TargetOpcode::G_FMAXNUM;
1222  case Intrinsic::minimum:
1223  return TargetOpcode::G_FMINIMUM;
1224  case Intrinsic::maximum:
1225  return TargetOpcode::G_FMAXIMUM;
1226  case Intrinsic::canonicalize:
1227  return TargetOpcode::G_FCANONICALIZE;
1228  case Intrinsic::floor:
1229  return TargetOpcode::G_FFLOOR;
1230  case Intrinsic::fma:
1231  return TargetOpcode::G_FMA;
1232  case Intrinsic::log:
1233  return TargetOpcode::G_FLOG;
1234  case Intrinsic::log2:
1235  return TargetOpcode::G_FLOG2;
1236  case Intrinsic::log10:
1237  return TargetOpcode::G_FLOG10;
1238  case Intrinsic::nearbyint:
1239  return TargetOpcode::G_FNEARBYINT;
1240  case Intrinsic::pow:
1241  return TargetOpcode::G_FPOW;
1242  case Intrinsic::rint:
1243  return TargetOpcode::G_FRINT;
1244  case Intrinsic::round:
1245  return TargetOpcode::G_INTRINSIC_ROUND;
1246  case Intrinsic::sin:
1247  return TargetOpcode::G_FSIN;
1248  case Intrinsic::sqrt:
1249  return TargetOpcode::G_FSQRT;
1250  case Intrinsic::trunc:
1251  return TargetOpcode::G_INTRINSIC_TRUNC;
1252  }
1253  return Intrinsic::not_intrinsic;
1254 }
1255 
1256 bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
1257  Intrinsic::ID ID,
1258  MachineIRBuilder &MIRBuilder) {
1259 
1260  unsigned Op = getSimpleIntrinsicOpcode(ID);
1261 
1262  // Is this a simple intrinsic?
1263  if (Op == Intrinsic::not_intrinsic)
1264  return false;
1265 
1266  // Yes. Let's translate it.
1268  for (auto &Arg : CI.arg_operands())
1269  VRegs.push_back(getOrCreateVReg(*Arg));
1270 
1271  MIRBuilder.buildInstr(Op, {getOrCreateVReg(CI)}, VRegs,
1273  return true;
1274 }
1275 
1276 bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
1277  MachineIRBuilder &MIRBuilder) {
1278 
1279  // If this is a simple intrinsic (that is, we just need to add a def of
1280  // a vreg, and uses for each arg operand, then translate it.
1281  if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
1282  return true;
1283 
1284  switch (ID) {
1285  default:
1286  break;
1287  case Intrinsic::lifetime_start:
1288  case Intrinsic::lifetime_end: {
1289  // No stack colouring in O0, discard region information.
1290  if (MF->getTarget().getOptLevel() == CodeGenOpt::None)
1291  return true;
1292 
1293  unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
1295 
1296  // Get the underlying objects for the location passed on the lifetime
1297  // marker.
1299  GetUnderlyingObjects(CI.getArgOperand(1), Allocas, *DL);
1300 
1301  // Iterate over each underlying object, creating lifetime markers for each
1302  // static alloca. Quit if we find a non-static alloca.
1303  for (const Value *V : Allocas) {
1304  const AllocaInst *AI = dyn_cast<AllocaInst>(V);
1305  if (!AI)
1306  continue;
1307 
1308  if (!AI->isStaticAlloca())
1309  return true;
1310 
1311  MIRBuilder.buildInstr(Op).addFrameIndex(getOrCreateFrameIndex(*AI));
1312  }
1313  return true;
1314  }
1315  case Intrinsic::dbg_declare: {
1316  const DbgDeclareInst &DI = cast<DbgDeclareInst>(CI);
1317  assert(DI.getVariable() && "Missing variable");
1318 
1319  const Value *Address = DI.getAddress();
1320  if (!Address || isa<UndefValue>(Address)) {
1321  LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
1322  return true;
1323  }
1324 
1326  MIRBuilder.getDebugLoc()) &&
1327  "Expected inlined-at fields to agree");
1328  auto AI = dyn_cast<AllocaInst>(Address);
1329  if (AI && AI->isStaticAlloca()) {
1330  // Static allocas are tracked at the MF level, no need for DBG_VALUE
1331  // instructions (in fact, they get ignored if they *do* exist).
1332  MF->setVariableDbgInfo(DI.getVariable(), DI.getExpression(),
1333  getOrCreateFrameIndex(*AI), DI.getDebugLoc());
1334  } else {
1335  // A dbg.declare describes the address of a source variable, so lower it
1336  // into an indirect DBG_VALUE.
1337  MIRBuilder.buildIndirectDbgValue(getOrCreateVReg(*Address),
1338  DI.getVariable(), DI.getExpression());
1339  }
1340  return true;
1341  }
1342  case Intrinsic::dbg_label: {
1343  const DbgLabelInst &DI = cast<DbgLabelInst>(CI);
1344  assert(DI.getLabel() && "Missing label");
1345 
1347  MIRBuilder.getDebugLoc()) &&
1348  "Expected inlined-at fields to agree");
1349 
1350  MIRBuilder.buildDbgLabel(DI.getLabel());
1351  return true;
1352  }
1353  case Intrinsic::vaend:
1354  // No target I know of cares about va_end. Certainly no in-tree target
1355  // does. Simplest intrinsic ever!
1356  return true;
1357  case Intrinsic::vastart: {
1358  auto &TLI = *MF->getSubtarget().getTargetLowering();
1359  Value *Ptr = CI.getArgOperand(0);
1360  unsigned ListSize = TLI.getVaListSizeInBits(*DL) / 8;
1361 
1362  // FIXME: Get alignment
1363  MIRBuilder.buildInstr(TargetOpcode::G_VASTART)
1364  .addUse(getOrCreateVReg(*Ptr))
1365  .addMemOperand(MF->getMachineMemOperand(
1366  MachinePointerInfo(Ptr), MachineMemOperand::MOStore, ListSize, 1));
1367  return true;
1368  }
1369  case Intrinsic::dbg_value: {
1370  // This form of DBG_VALUE is target-independent.
1371  const DbgValueInst &DI = cast<DbgValueInst>(CI);
1372  const Value *V = DI.getValue();
1374  MIRBuilder.getDebugLoc()) &&
1375  "Expected inlined-at fields to agree");
1376  if (!V) {
1377  // Currently the optimizer can produce this; insert an undef to
1378  // help debugging. Probably the optimizer should not do this.
1379  MIRBuilder.buildIndirectDbgValue(0, DI.getVariable(), DI.getExpression());
1380  } else if (const auto *CI = dyn_cast<Constant>(V)) {
1381  MIRBuilder.buildConstDbgValue(*CI, DI.getVariable(), DI.getExpression());
1382  } else {
1383  Register Reg = getOrCreateVReg(*V);
1384  // FIXME: This does not handle register-indirect values at offset 0. The
1385  // direct/indirect thing shouldn't really be handled by something as
1386  // implicit as reg+noreg vs reg+imm in the first palce, but it seems
1387  // pretty baked in right now.
1388  MIRBuilder.buildDirectDbgValue(Reg, DI.getVariable(), DI.getExpression());
1389  }
1390  return true;
1391  }
1392  case Intrinsic::uadd_with_overflow:
1393  return translateOverflowIntrinsic(CI, TargetOpcode::G_UADDO, MIRBuilder);
1394  case Intrinsic::sadd_with_overflow:
1395  return translateOverflowIntrinsic(CI, TargetOpcode::G_SADDO, MIRBuilder);
1396  case Intrinsic::usub_with_overflow:
1397  return translateOverflowIntrinsic(CI, TargetOpcode::G_USUBO, MIRBuilder);
1398  case Intrinsic::ssub_with_overflow:
1399  return translateOverflowIntrinsic(CI, TargetOpcode::G_SSUBO, MIRBuilder);
1400  case Intrinsic::umul_with_overflow:
1401  return translateOverflowIntrinsic(CI, TargetOpcode::G_UMULO, MIRBuilder);
1402  case Intrinsic::smul_with_overflow:
1403  return translateOverflowIntrinsic(CI, TargetOpcode::G_SMULO, MIRBuilder);
1404  case Intrinsic::fmuladd: {
1405  const TargetMachine &TM = MF->getTarget();
1406  const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
1407  Register Dst = getOrCreateVReg(CI);
1408  Register Op0 = getOrCreateVReg(*CI.getArgOperand(0));
1409  Register Op1 = getOrCreateVReg(*CI.getArgOperand(1));
1410  Register Op2 = getOrCreateVReg(*CI.getArgOperand(2));
1412  TLI.isFMAFasterThanFMulAndFAdd(TLI.getValueType(*DL, CI.getType()))) {
1413  // TODO: Revisit this to see if we should move this part of the
1414  // lowering to the combiner.
1415  MIRBuilder.buildInstr(TargetOpcode::G_FMA, {Dst}, {Op0, Op1, Op2},
1417  } else {
1418  LLT Ty = getLLTForType(*CI.getType(), *DL);
1419  auto FMul = MIRBuilder.buildInstr(TargetOpcode::G_FMUL, {Ty}, {Op0, Op1},
1421  MIRBuilder.buildInstr(TargetOpcode::G_FADD, {Dst}, {FMul, Op2},
1423  }
1424  return true;
1425  }
1426  case Intrinsic::memcpy:
1427  case Intrinsic::memmove:
1428  case Intrinsic::memset:
1429  return translateMemFunc(CI, MIRBuilder, ID);
1430  case Intrinsic::eh_typeid_for: {
1432  Register Reg = getOrCreateVReg(CI);
1433  unsigned TypeID = MF->getTypeIDFor(GV);
1434  MIRBuilder.buildConstant(Reg, TypeID);
1435  return true;
1436  }
1437  case Intrinsic::objectsize: {
1438  // If we don't know by now, we're never going to know.
1439  const ConstantInt *Min = cast<ConstantInt>(CI.getArgOperand(1));
1440 
1441  MIRBuilder.buildConstant(getOrCreateVReg(CI), Min->isZero() ? -1ULL : 0);
1442  return true;
1443  }
1444  case Intrinsic::is_constant:
1445  // If this wasn't constant-folded away by now, then it's not a
1446  // constant.
1447  MIRBuilder.buildConstant(getOrCreateVReg(CI), 0);
1448  return true;
1449  case Intrinsic::stackguard:
1450  getStackGuard(getOrCreateVReg(CI), MIRBuilder);
1451  return true;
1452  case Intrinsic::stackprotector: {
1453  LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
1454  Register GuardVal = MRI->createGenericVirtualRegister(PtrTy);
1455  getStackGuard(GuardVal, MIRBuilder);
1456 
1457  AllocaInst *Slot = cast<AllocaInst>(CI.getArgOperand(1));
1458  int FI = getOrCreateFrameIndex(*Slot);
1459  MF->getFrameInfo().setStackProtectorIndex(FI);
1460 
1461  MIRBuilder.buildStore(
1462  GuardVal, getOrCreateVReg(*Slot),
1463  *MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(*MF, FI),
1466  PtrTy.getSizeInBits() / 8, 8));
1467  return true;
1468  }
1469  case Intrinsic::stacksave: {
1470  // Save the stack pointer to the location provided by the intrinsic.
1471  Register Reg = getOrCreateVReg(CI);
1472  Register StackPtr = MF->getSubtarget()
1473  .getTargetLowering()
1474  ->getStackPointerRegisterToSaveRestore();
1475 
1476  // If the target doesn't specify a stack pointer, then fall back.
1477  if (!StackPtr)
1478  return false;
1479 
1480  MIRBuilder.buildCopy(Reg, StackPtr);
1481  return true;
1482  }
1483  case Intrinsic::stackrestore: {
1484  // Restore the stack pointer from the location provided by the intrinsic.
1485  Register Reg = getOrCreateVReg(*CI.getArgOperand(0));
1486  Register StackPtr = MF->getSubtarget()
1487  .getTargetLowering()
1488  ->getStackPointerRegisterToSaveRestore();
1489 
1490  // If the target doesn't specify a stack pointer, then fall back.
1491  if (!StackPtr)
1492  return false;
1493 
1494  MIRBuilder.buildCopy(StackPtr, Reg);
1495  return true;
1496  }
1497  case Intrinsic::cttz:
1498  case Intrinsic::ctlz: {
1499  ConstantInt *Cst = cast<ConstantInt>(CI.getArgOperand(1));
1500  bool isTrailing = ID == Intrinsic::cttz;
1501  unsigned Opcode = isTrailing
1502  ? Cst->isZero() ? TargetOpcode::G_CTTZ
1503  : TargetOpcode::G_CTTZ_ZERO_UNDEF
1504  : Cst->isZero() ? TargetOpcode::G_CTLZ
1505  : TargetOpcode::G_CTLZ_ZERO_UNDEF;
1506  MIRBuilder.buildInstr(Opcode)
1507  .addDef(getOrCreateVReg(CI))
1508  .addUse(getOrCreateVReg(*CI.getArgOperand(0)));
1509  return true;
1510  }
1511  case Intrinsic::invariant_start: {
1512  LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
1513  Register Undef = MRI->createGenericVirtualRegister(PtrTy);
1514  MIRBuilder.buildUndef(Undef);
1515  return true;
1516  }
1517  case Intrinsic::invariant_end:
1518  return true;
1519  case Intrinsic::assume:
1520  case Intrinsic::var_annotation:
1521  case Intrinsic::sideeffect:
1522  // Discard annotate attributes, assumptions, and artificial side-effects.
1523  return true;
1524  }
1525  return false;
1526 }
1527 
1528 bool IRTranslator::translateInlineAsm(const CallInst &CI,
1529  MachineIRBuilder &MIRBuilder) {
1530  const InlineAsm &IA = cast<InlineAsm>(*CI.getCalledValue());
1531  if (!IA.getConstraintString().empty())
1532  return false;
1533 
1534  unsigned ExtraInfo = 0;
1535  if (IA.hasSideEffects())
1536  ExtraInfo |= InlineAsm::Extra_HasSideEffects;
1537  if (IA.getDialect() == InlineAsm::AD_Intel)
1538  ExtraInfo |= InlineAsm::Extra_AsmDialect;
1539 
1541  .addExternalSymbol(IA.getAsmString().c_str())
1542  .addImm(ExtraInfo);
1543 
1544  return true;
1545 }
1546 
1547 bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
1548  const CallInst &CI = cast<CallInst>(U);
1549  auto TII = MF->getTarget().getIntrinsicInfo();
1550  const Function *F = CI.getCalledFunction();
1551 
1552  // FIXME: support Windows dllimport function calls.
1553  if (F && F->hasDLLImportStorageClass())
1554  return false;
1555 
1556  if (CI.isInlineAsm())
1557  return translateInlineAsm(CI, MIRBuilder);
1558 
1560  if (F && F->isIntrinsic()) {
1561  ID = F->getIntrinsicID();
1562  if (TII && ID == Intrinsic::not_intrinsic)
1563  ID = static_cast<Intrinsic::ID>(TII->getIntrinsicID(F));
1564  }
1565 
1566  if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic) {
1567  ArrayRef<Register> Res = getOrCreateVRegs(CI);
1568 
1570  Register SwiftInVReg = 0;
1571  Register SwiftErrorVReg = 0;
1572  for (auto &Arg: CI.arg_operands()) {
1573  if (CLI->supportSwiftError() && isSwiftError(Arg)) {
1574  assert(SwiftInVReg == 0 && "Expected only one swift error argument");
1575  LLT Ty = getLLTForType(*Arg->getType(), *DL);
1576  SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
1577  MIRBuilder.buildCopy(SwiftInVReg, SwiftError.getOrCreateVRegUseAt(
1578  &CI, &MIRBuilder.getMBB(), Arg));
1579  Args.emplace_back(makeArrayRef(SwiftInVReg));
1580  SwiftErrorVReg =
1581  SwiftError.getOrCreateVRegDefAt(&CI, &MIRBuilder.getMBB(), Arg);
1582  continue;
1583  }
1584  Args.push_back(getOrCreateVRegs(*Arg));
1585  }
1586 
1587  MF->getFrameInfo().setHasCalls(true);
1588  bool Success =
1589  CLI->lowerCall(MIRBuilder, &CI, Res, Args, SwiftErrorVReg,
1590  [&]() { return getOrCreateVReg(*CI.getCalledValue()); });
1591 
1592  return Success;
1593  }
1594 
1595  assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
1596 
1597  if (translateKnownIntrinsic(CI, ID, MIRBuilder))
1598  return true;
1599 
1600  ArrayRef<Register> ResultRegs;
1601  if (!CI.getType()->isVoidTy())
1602  ResultRegs = getOrCreateVRegs(CI);
1603 
1604  // Ignore the callsite attributes. Backend code is most likely not expecting
1605  // an intrinsic to sometimes have side effects and sometimes not.
1606  MachineInstrBuilder MIB =
1607  MIRBuilder.buildIntrinsic(ID, ResultRegs, !F->doesNotAccessMemory());
1608  if (isa<FPMathOperator>(CI))
1609  MIB->copyIRFlags(CI);
1610 
1611  for (auto &Arg : CI.arg_operands()) {
1612  // Some intrinsics take metadata parameters. Reject them.
1613  if (isa<MetadataAsValue>(Arg))
1614  return false;
1615  ArrayRef<Register> VRegs = getOrCreateVRegs(*Arg);
1616  if (VRegs.size() > 1)
1617  return false;
1618  MIB.addUse(VRegs[0]);
1619  }
1620 
1621  // Add a MachineMemOperand if it is a target mem intrinsic.
1622  const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
1623  TargetLowering::IntrinsicInfo Info;
1624  // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
1625  if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) {
1626  unsigned Align = Info.align;
1627  if (Align == 0)
1628  Align = DL->getABITypeAlignment(Info.memVT.getTypeForEVT(F->getContext()));
1629 
1630  uint64_t Size = Info.memVT.getStoreSize();
1631  MIB.addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Info.ptrVal),
1632  Info.flags, Size, Align));
1633  }
1634 
1635  return true;
1636 }
1637 
1638 bool IRTranslator::translateInvoke(const User &U,
1639  MachineIRBuilder &MIRBuilder) {
1640  const InvokeInst &I = cast<InvokeInst>(U);
1641  MCContext &Context = MF->getContext();
1642 
1643  const BasicBlock *ReturnBB = I.getSuccessor(0);
1644  const BasicBlock *EHPadBB = I.getSuccessor(1);
1645 
1646  const Value *Callee = I.getCalledValue();
1647  const Function *Fn = dyn_cast<Function>(Callee);
1648  if (isa<InlineAsm>(Callee))
1649  return false;
1650 
1651  // FIXME: support invoking patchpoint and statepoint intrinsics.
1652  if (Fn && Fn->isIntrinsic())
1653  return false;
1654 
1655  // FIXME: support whatever these are.
1657  return false;
1658 
1659  // FIXME: support Windows exception handling.
1660  if (!isa<LandingPadInst>(EHPadBB->front()))
1661  return false;
1662 
1663  // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
1664  // the region covered by the try.
1665  MCSymbol *BeginSymbol = Context.createTempSymbol();
1666  MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol);
1667 
1668  ArrayRef<Register> Res;
1669  if (!I.getType()->isVoidTy())
1670  Res = getOrCreateVRegs(I);
1672  Register SwiftErrorVReg = 0;
1673  Register SwiftInVReg = 0;
1674  for (auto &Arg : I.arg_operands()) {
1675  if (CLI->supportSwiftError() && isSwiftError(Arg)) {
1676  assert(SwiftInVReg == 0 && "Expected only one swift error argument");
1677  LLT Ty = getLLTForType(*Arg->getType(), *DL);
1678  SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
1679  MIRBuilder.buildCopy(SwiftInVReg, SwiftError.getOrCreateVRegUseAt(
1680  &I, &MIRBuilder.getMBB(), Arg));
1681  Args.push_back(makeArrayRef(SwiftInVReg));
1682  SwiftErrorVReg =
1683  SwiftError.getOrCreateVRegDefAt(&I, &MIRBuilder.getMBB(), Arg);
1684  continue;
1685  }
1686 
1687  Args.push_back(getOrCreateVRegs(*Arg));
1688  }
1689 
1690  if (!CLI->lowerCall(MIRBuilder, &I, Res, Args, SwiftErrorVReg,
1691  [&]() { return getOrCreateVReg(*I.getCalledValue()); }))
1692  return false;
1693 
1694  MCSymbol *EndSymbol = Context.createTempSymbol();
1695  MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(EndSymbol);
1696 
1697  // FIXME: track probabilities.
1698  MachineBasicBlock &EHPadMBB = getMBB(*EHPadBB),
1699  &ReturnMBB = getMBB(*ReturnBB);
1700  MF->addInvoke(&EHPadMBB, BeginSymbol, EndSymbol);
1701  MIRBuilder.getMBB().addSuccessor(&ReturnMBB);
1702  MIRBuilder.getMBB().addSuccessor(&EHPadMBB);
1703  MIRBuilder.buildBr(ReturnMBB);
1704 
1705  return true;
1706 }
1707 
1708 bool IRTranslator::translateCallBr(const User &U,
1709  MachineIRBuilder &MIRBuilder) {
1710  // FIXME: Implement this.
1711  return false;
1712 }
1713 
1714 bool IRTranslator::translateLandingPad(const User &U,
1715  MachineIRBuilder &MIRBuilder) {
1716  const LandingPadInst &LP = cast<LandingPadInst>(U);
1717 
1718  MachineBasicBlock &MBB = MIRBuilder.getMBB();
1719 
1720  MBB.setIsEHPad();
1721 
1722  // If there aren't registers to copy the values into (e.g., during SjLj
1723  // exceptions), then don't bother.
1724  auto &TLI = *MF->getSubtarget().getTargetLowering();
1725  const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
1726  if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
1727  TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
1728  return true;
1729 
1730  // If landingpad's return type is token type, we don't create DAG nodes
1731  // for its exception pointer and selector value. The extraction of exception
1732  // pointer or selector value from token type landingpads is not currently
1733  // supported.
1734  if (LP.getType()->isTokenTy())
1735  return true;
1736 
1737  // Add a label to mark the beginning of the landing pad. Deletion of the
1738  // landing pad can thus be detected via the MachineModuleInfo.
1740  .addSym(MF->addLandingPad(&MBB));
1741 
1742  LLT Ty = getLLTForType(*LP.getType(), *DL);
1743  Register Undef = MRI->createGenericVirtualRegister(Ty);
1744  MIRBuilder.buildUndef(Undef);
1745 
1746  SmallVector<LLT, 2> Tys;
1747  for (Type *Ty : cast<StructType>(LP.getType())->elements())
1748  Tys.push_back(getLLTForType(*Ty, *DL));
1749  assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
1750 
1751  // Mark exception register as live in.
1752  Register ExceptionReg = TLI.getExceptionPointerRegister(PersonalityFn);
1753  if (!ExceptionReg)
1754  return false;
1755 
1756  MBB.addLiveIn(ExceptionReg);
1757  ArrayRef<Register> ResRegs = getOrCreateVRegs(LP);
1758  MIRBuilder.buildCopy(ResRegs[0], ExceptionReg);
1759 
1760  Register SelectorReg = TLI.getExceptionSelectorRegister(PersonalityFn);
1761  if (!SelectorReg)
1762  return false;
1763 
1764  MBB.addLiveIn(SelectorReg);
1765  Register PtrVReg = MRI->createGenericVirtualRegister(Tys[0]);
1766  MIRBuilder.buildCopy(PtrVReg, SelectorReg);
1767  MIRBuilder.buildCast(ResRegs[1], PtrVReg);
1768 
1769  return true;
1770 }
1771 
1772 bool IRTranslator::translateAlloca(const User &U,
1773  MachineIRBuilder &MIRBuilder) {
1774  auto &AI = cast<AllocaInst>(U);
1775 
1776  if (AI.isSwiftError())
1777  return true;
1778 
1779  if (AI.isStaticAlloca()) {
1780  Register Res = getOrCreateVReg(AI);
1781  int FI = getOrCreateFrameIndex(AI);
1782  MIRBuilder.buildFrameIndex(Res, FI);
1783  return true;
1784  }
1785 
1786  // FIXME: support stack probing for Windows.
1787  if (MF->getTarget().getTargetTriple().isOSWindows())
1788  return false;
1789 
1790  // Now we're in the harder dynamic case.
1791  Type *Ty = AI.getAllocatedType();
1792  unsigned Align =
1793  std::max((unsigned)DL->getPrefTypeAlignment(Ty), AI.getAlignment());
1794 
1795  Register NumElts = getOrCreateVReg(*AI.getArraySize());
1796 
1797  Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
1798  LLT IntPtrTy = getLLTForType(*IntPtrIRTy, *DL);
1799  if (MRI->getType(NumElts) != IntPtrTy) {
1800  Register ExtElts = MRI->createGenericVirtualRegister(IntPtrTy);
1801  MIRBuilder.buildZExtOrTrunc(ExtElts, NumElts);
1802  NumElts = ExtElts;
1803  }
1804 
1805  Register AllocSize = MRI->createGenericVirtualRegister(IntPtrTy);
1806  Register TySize =
1807  getOrCreateVReg(*ConstantInt::get(IntPtrIRTy, -DL->getTypeAllocSize(Ty)));
1808  MIRBuilder.buildMul(AllocSize, NumElts, TySize);
1809 
1810  LLT PtrTy = getLLTForType(*AI.getType(), *DL);
1811  auto &TLI = *MF->getSubtarget().getTargetLowering();
1812  Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1813 
1814  Register SPTmp = MRI->createGenericVirtualRegister(PtrTy);
1815  MIRBuilder.buildCopy(SPTmp, SPReg);
1816 
1817  Register AllocTmp = MRI->createGenericVirtualRegister(PtrTy);
1818  MIRBuilder.buildGEP(AllocTmp, SPTmp, AllocSize);
1819 
1820  // Handle alignment. We have to realign if the allocation granule was smaller
1821  // than stack alignment, or the specific alloca requires more than stack
1822  // alignment.
1823  unsigned StackAlign =
1824  MF->getSubtarget().getFrameLowering()->getStackAlignment();
1825  Align = std::max(Align, StackAlign);
1826  if (Align > StackAlign || DL->getTypeAllocSize(Ty) % StackAlign != 0) {
1827  // Round the size of the allocation up to the stack alignment size
1828  // by add SA-1 to the size. This doesn't overflow because we're computing
1829  // an address inside an alloca.
1830  Register AlignedAlloc = MRI->createGenericVirtualRegister(PtrTy);
1831  MIRBuilder.buildPtrMask(AlignedAlloc, AllocTmp, Log2_32(Align));
1832  AllocTmp = AlignedAlloc;
1833  }
1834 
1835  MIRBuilder.buildCopy(SPReg, AllocTmp);
1836  MIRBuilder.buildCopy(getOrCreateVReg(AI), AllocTmp);
1837 
1838  MF->getFrameInfo().CreateVariableSizedObject(Align ? Align : 1, &AI);
1839  assert(MF->getFrameInfo().hasVarSizedObjects());
1840  return true;
1841 }
1842 
1843 bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {
1844  // FIXME: We may need more info about the type. Because of how LLT works,
1845  // we're completely discarding the i64/double distinction here (amongst
1846  // others). Fortunately the ABIs I know of where that matters don't use va_arg
1847  // anyway but that's not guaranteed.
1848  MIRBuilder.buildInstr(TargetOpcode::G_VAARG)
1849  .addDef(getOrCreateVReg(U))
1850  .addUse(getOrCreateVReg(*U.getOperand(0)))
1851  .addImm(DL->getABITypeAlignment(U.getType()));
1852  return true;
1853 }
1854 
1855 bool IRTranslator::translateInsertElement(const User &U,
1856  MachineIRBuilder &MIRBuilder) {
1857  // If it is a <1 x Ty> vector, use the scalar as it is
1858  // not a legal vector type in LLT.
1859  if (U.getType()->getVectorNumElements() == 1) {
1860  Register Elt = getOrCreateVReg(*U.getOperand(1));
1861  auto &Regs = *VMap.getVRegs(U);
1862  if (Regs.empty()) {
1863  Regs.push_back(Elt);
1864  VMap.getOffsets(U)->push_back(0);
1865  } else {
1866  MIRBuilder.buildCopy(Regs[0], Elt);
1867  }
1868  return true;
1869  }
1870 
1871  Register Res = getOrCreateVReg(U);
1872  Register Val = getOrCreateVReg(*U.getOperand(0));
1873  Register Elt = getOrCreateVReg(*U.getOperand(1));
1874  Register Idx = getOrCreateVReg(*U.getOperand(2));
1875  MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
1876  return true;
1877 }
1878 
1879 bool IRTranslator::translateExtractElement(const User &U,
1880  MachineIRBuilder &MIRBuilder) {
1881  // If it is a <1 x Ty> vector, use the scalar as it is
1882  // not a legal vector type in LLT.
1883  if (U.getOperand(0)->getType()->getVectorNumElements() == 1) {
1884  Register Elt = getOrCreateVReg(*U.getOperand(0));
1885  auto &Regs = *VMap.getVRegs(U);
1886  if (Regs.empty()) {
1887  Regs.push_back(Elt);
1888  VMap.getOffsets(U)->push_back(0);
1889  } else {
1890  MIRBuilder.buildCopy(Regs[0], Elt);
1891  }
1892  return true;
1893  }
1894  Register Res = getOrCreateVReg(U);
1895  Register Val = getOrCreateVReg(*U.getOperand(0));
1896  const auto &TLI = *MF->getSubtarget().getTargetLowering();
1897  unsigned PreferredVecIdxWidth = TLI.getVectorIdxTy(*DL).getSizeInBits();
1898  Register Idx;
1899  if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(1))) {
1900  if (CI->getBitWidth() != PreferredVecIdxWidth) {
1901  APInt NewIdx = CI->getValue().sextOrTrunc(PreferredVecIdxWidth);
1902  auto *NewIdxCI = ConstantInt::get(CI->getContext(), NewIdx);
1903  Idx = getOrCreateVReg(*NewIdxCI);
1904  }
1905  }
1906  if (!Idx)
1907  Idx = getOrCreateVReg(*U.getOperand(1));
1908  if (MRI->getType(Idx).getSizeInBits() != PreferredVecIdxWidth) {
1909  const LLT &VecIdxTy = LLT::scalar(PreferredVecIdxWidth);
1910  Idx = MIRBuilder.buildSExtOrTrunc(VecIdxTy, Idx)->getOperand(0).getReg();
1911  }
1912  MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
1913  return true;
1914 }
1915 
1916 bool IRTranslator::translateShuffleVector(const User &U,
1917  MachineIRBuilder &MIRBuilder) {
1918  MIRBuilder.buildInstr(TargetOpcode::G_SHUFFLE_VECTOR)
1919  .addDef(getOrCreateVReg(U))
1920  .addUse(getOrCreateVReg(*U.getOperand(0)))
1921  .addUse(getOrCreateVReg(*U.getOperand(1)))
1922  .addUse(getOrCreateVReg(*U.getOperand(2)));
1923  return true;
1924 }
1925 
1926 bool IRTranslator::translatePHI(const User &U, MachineIRBuilder &MIRBuilder) {
1927  const PHINode &PI = cast<PHINode>(U);
1928 
1930  for (auto Reg : getOrCreateVRegs(PI)) {
1931  auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_PHI, {Reg}, {});
1932  Insts.push_back(MIB.getInstr());
1933  }
1934 
1935  PendingPHIs.emplace_back(&PI, std::move(Insts));
1936  return true;
1937 }
1938 
1939 bool IRTranslator::translateAtomicCmpXchg(const User &U,
1940  MachineIRBuilder &MIRBuilder) {
1941  const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(U);
1942 
1943  if (I.isWeak())
1944  return false;
1945 
1946  auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile
1949 
1950  Type *ResType = I.getType();
1951  Type *ValType = ResType->Type::getStructElementType(0);
1952 
1953  auto Res = getOrCreateVRegs(I);
1954  Register OldValRes = Res[0];
1955  Register SuccessRes = Res[1];
1956  Register Addr = getOrCreateVReg(*I.getPointerOperand());
1957  Register Cmp = getOrCreateVReg(*I.getCompareOperand());
1958  Register NewVal = getOrCreateVReg(*I.getNewValOperand());
1959 
1960  MIRBuilder.buildAtomicCmpXchgWithSuccess(
1961  OldValRes, SuccessRes, Addr, Cmp, NewVal,
1962  *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
1963  Flags, DL->getTypeStoreSize(ValType),
1964  getMemOpAlignment(I), AAMDNodes(), nullptr,
1966  I.getFailureOrdering()));
1967  return true;
1968 }
1969 
1970 bool IRTranslator::translateAtomicRMW(const User &U,
1971  MachineIRBuilder &MIRBuilder) {
1972  const AtomicRMWInst &I = cast<AtomicRMWInst>(U);
1973 
1974  auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile
1977 
1978  Type *ResType = I.getType();
1979 
1980  Register Res = getOrCreateVReg(I);
1981  Register Addr = getOrCreateVReg(*I.getPointerOperand());
1982  Register Val = getOrCreateVReg(*I.getValOperand());
1983 
1984  unsigned Opcode = 0;
1985  switch (I.getOperation()) {
1986  default:
1987  llvm_unreachable("Unknown atomicrmw op");
1988  return false;
1989  case AtomicRMWInst::Xchg:
1990  Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
1991  break;
1992  case AtomicRMWInst::Add:
1993  Opcode = TargetOpcode::G_ATOMICRMW_ADD;
1994  break;
1995  case AtomicRMWInst::Sub:
1996  Opcode = TargetOpcode::G_ATOMICRMW_SUB;
1997  break;
1998  case AtomicRMWInst::And:
1999  Opcode = TargetOpcode::G_ATOMICRMW_AND;
2000  break;
2001  case AtomicRMWInst::Nand:
2002  Opcode = TargetOpcode::G_ATOMICRMW_NAND;
2003  break;
2004  case AtomicRMWInst::Or:
2005  Opcode = TargetOpcode::G_ATOMICRMW_OR;
2006  break;
2007  case AtomicRMWInst::Xor:
2008  Opcode = TargetOpcode::G_ATOMICRMW_XOR;
2009  break;
2010  case AtomicRMWInst::Max:
2011  Opcode = TargetOpcode::G_ATOMICRMW_MAX;
2012  break;
2013  case AtomicRMWInst::Min:
2014  Opcode = TargetOpcode::G_ATOMICRMW_MIN;
2015  break;
2016  case AtomicRMWInst::UMax:
2017  Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
2018  break;
2019  case AtomicRMWInst::UMin:
2020  Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
2021  break;
2022  }
2023 
2024  MIRBuilder.buildAtomicRMW(
2025  Opcode, Res, Addr, Val,
2026  *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
2027  Flags, DL->getTypeStoreSize(ResType),
2028  getMemOpAlignment(I), AAMDNodes(), nullptr,
2029  I.getSyncScopeID(), I.getOrdering()));
2030  return true;
2031 }
2032 
2033 bool IRTranslator::translateFence(const User &U,
2034  MachineIRBuilder &MIRBuilder) {
2035  const FenceInst &Fence = cast<FenceInst>(U);
2036  MIRBuilder.buildFence(static_cast<unsigned>(Fence.getOrdering()),
2037  Fence.getSyncScopeID());
2038  return true;
2039 }
2040 
2041 void IRTranslator::finishPendingPhis() {
2042 #ifndef NDEBUG
2043  DILocationVerifier Verifier;
2044  GISelObserverWrapper WrapperObserver(&Verifier);
2045  RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
2046 #endif // ifndef NDEBUG
2047  for (auto &Phi : PendingPHIs) {
2048  const PHINode *PI = Phi.first;
2049  ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
2050  MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
2051  EntryBuilder->setDebugLoc(PI->getDebugLoc());
2052 #ifndef NDEBUG
2053  Verifier.setCurrentInst(PI);
2054 #endif // ifndef NDEBUG
2055 
2057  for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
2058  auto IRPred = PI->getIncomingBlock(i);
2059  ArrayRef<Register> ValRegs = getOrCreateVRegs(*PI->getIncomingValue(i));
2060  for (auto Pred : getMachinePredBBs({IRPred, PI->getParent()})) {
2061  if (SeenPreds.count(Pred) || !PhiMBB->isPredecessor(Pred))
2062  continue;
2063  SeenPreds.insert(Pred);
2064  for (unsigned j = 0; j < ValRegs.size(); ++j) {
2065  MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
2066  MIB.addUse(ValRegs[j]);
2067  MIB.addMBB(Pred);
2068  }
2069  }
2070  }
2071  }
2072 }
2073 
2074 bool IRTranslator::valueIsSplit(const Value &V,
2076  SmallVector<LLT, 4> SplitTys;
2077  if (Offsets && !Offsets->empty())
2078  Offsets->clear();
2079  computeValueLLTs(*DL, *V.getType(), SplitTys, Offsets);
2080  return SplitTys.size() > 1;
2081 }
2082 
2083 bool IRTranslator::translate(const Instruction &Inst) {
2084  CurBuilder->setDebugLoc(Inst.getDebugLoc());
2085  // We only emit constants into the entry block from here. To prevent jumpy
2086  // debug behaviour set the line to 0.
2087  if (const DebugLoc &DL = Inst.getDebugLoc())
2088  EntryBuilder->setDebugLoc(
2089  DebugLoc::get(0, 0, DL.getScope(), DL.getInlinedAt()));
2090  else
2091  EntryBuilder->setDebugLoc(DebugLoc());
2092 
2093  switch (Inst.getOpcode()) {
2094 #define HANDLE_INST(NUM, OPCODE, CLASS) \
2095  case Instruction::OPCODE: \
2096  return translate##OPCODE(Inst, *CurBuilder.get());
2097 #include "llvm/IR/Instruction.def"
2098  default:
2099  return false;
2100  }
2101 }
2102 
2103 bool IRTranslator::translate(const Constant &C, Register Reg) {
2104  if (auto CI = dyn_cast<ConstantInt>(&C))
2105  EntryBuilder->buildConstant(Reg, *CI);
2106  else if (auto CF = dyn_cast<ConstantFP>(&C))
2107  EntryBuilder->buildFConstant(Reg, *CF);
2108  else if (isa<UndefValue>(C))
2109  EntryBuilder->buildUndef(Reg);
2110  else if (isa<ConstantPointerNull>(C)) {
2111  // As we are trying to build a constant val of 0 into a pointer,
2112  // insert a cast to make them correct with respect to types.
2113  unsigned NullSize = DL->getTypeSizeInBits(C.getType());
2114  auto *ZeroTy = Type::getIntNTy(C.getContext(), NullSize);
2115  auto *ZeroVal = ConstantInt::get(ZeroTy, 0);
2116  Register ZeroReg = getOrCreateVReg(*ZeroVal);
2117  EntryBuilder->buildCast(Reg, ZeroReg);
2118  } else if (auto GV = dyn_cast<GlobalValue>(&C))
2119  EntryBuilder->buildGlobalValue(Reg, GV);
2120  else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
2121  if (!CAZ->getType()->isVectorTy())
2122  return false;
2123  // Return the scalar if it is a <1 x Ty> vector.
2124  if (CAZ->getNumElements() == 1)
2125  return translate(*CAZ->getElementValue(0u), Reg);
2127  for (unsigned i = 0; i < CAZ->getNumElements(); ++i) {
2128  Constant &Elt = *CAZ->getElementValue(i);
2129  Ops.push_back(getOrCreateVReg(Elt));
2130  }
2131  EntryBuilder->buildBuildVector(Reg, Ops);
2132  } else if (auto CV = dyn_cast<ConstantDataVector>(&C)) {
2133  // Return the scalar if it is a <1 x Ty> vector.
2134  if (CV->getNumElements() == 1)
2135  return translate(*CV->getElementAsConstant(0), Reg);
2137  for (unsigned i = 0; i < CV->getNumElements(); ++i) {
2138  Constant &Elt = *CV->getElementAsConstant(i);
2139  Ops.push_back(getOrCreateVReg(Elt));
2140  }
2141  EntryBuilder->buildBuildVector(Reg, Ops);
2142  } else if (auto CE = dyn_cast<ConstantExpr>(&C)) {
2143  switch(CE->getOpcode()) {
2144 #define HANDLE_INST(NUM, OPCODE, CLASS) \
2145  case Instruction::OPCODE: \
2146  return translate##OPCODE(*CE, *EntryBuilder.get());
2147 #include "llvm/IR/Instruction.def"
2148  default:
2149  return false;
2150  }
2151  } else if (auto CV = dyn_cast<ConstantVector>(&C)) {
2152  if (CV->getNumOperands() == 1)
2153  return translate(*CV->getOperand(0), Reg);
2155  for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
2156  Ops.push_back(getOrCreateVReg(*CV->getOperand(i)));
2157  }
2158  EntryBuilder->buildBuildVector(Reg, Ops);
2159  } else if (auto *BA = dyn_cast<BlockAddress>(&C)) {
2160  EntryBuilder->buildBlockAddress(Reg, BA);
2161  } else
2162  return false;
2163 
2164  return true;
2165 }
2166 
2167 void IRTranslator::finalizeBasicBlock() {
2168  for (auto &JTCase : SL->JTCases) {
2169  // Emit header first, if it wasn't already emitted.
2170  if (!JTCase.first.Emitted)
2171  emitJumpTableHeader(JTCase.second, JTCase.first, JTCase.first.HeaderBB);
2172 
2173  emitJumpTable(JTCase.second, JTCase.second.MBB);
2174  }
2175  SL->JTCases.clear();
2176 }
2177 
2178 void IRTranslator::finalizeFunction() {
2179  // Release the memory used by the different maps we
2180  // needed during the translation.
2181  PendingPHIs.clear();
2182  VMap.reset();
2183  FrameIndices.clear();
2184  MachinePreds.clear();
2185  // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
2186  // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
2187  // destroying it twice (in ~IRTranslator() and ~LLVMContext())
2188  EntryBuilder.reset();
2189  CurBuilder.reset();
2190  FuncInfo.clear();
2191 }
2192 
2194  MF = &CurMF;
2195  const Function &F = MF->getFunction();
2196  if (F.empty())
2197  return false;
2199  getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
2200  // Set the CSEConfig and run the analysis.
2201  GISelCSEInfo *CSEInfo = nullptr;
2202  TPC = &getAnalysis<TargetPassConfig>();
2203  bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
2205  : TPC->isGISelCSEEnabled();
2206 
2207  if (EnableCSE) {
2208  EntryBuilder = make_unique<CSEMIRBuilder>(CurMF);
2209  CSEInfo = &Wrapper.get(TPC->getCSEConfig());
2210  EntryBuilder->setCSEInfo(CSEInfo);
2211  CurBuilder = make_unique<CSEMIRBuilder>(CurMF);
2212  CurBuilder->setCSEInfo(CSEInfo);
2213  } else {
2214  EntryBuilder = make_unique<MachineIRBuilder>();
2215  CurBuilder = make_unique<MachineIRBuilder>();
2216  }
2217  CLI = MF->getSubtarget().getCallLowering();
2218  CurBuilder->setMF(*MF);
2219  EntryBuilder->setMF(*MF);
2220  MRI = &MF->getRegInfo();
2221  DL = &F.getParent()->getDataLayout();
2222  ORE = llvm::make_unique<OptimizationRemarkEmitter>(&F);
2223  FuncInfo.MF = MF;
2224  FuncInfo.BPI = nullptr;
2225  const auto &TLI = *MF->getSubtarget().getTargetLowering();
2226  const TargetMachine &TM = MF->getTarget();
2227  SL = make_unique<GISelSwitchLowering>(this, FuncInfo);
2228  SL->init(TLI, TM, *DL);
2229 
2230  EnableOpts = TM.getOptLevel() != CodeGenOpt::None && !skipFunction(F);
2231 
2232  assert(PendingPHIs.empty() && "stale PHIs");
2233 
2234  if (!DL->isLittleEndian()) {
2235  // Currently we don't properly handle big endian code.
2236  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
2237  F.getSubprogram(), &F.getEntryBlock());
2238  R << "unable to translate in big endian mode";
2239  reportTranslationError(*MF, *TPC, *ORE, R);
2240  }
2241 
2242  // Release the per-function state when we return, whether we succeeded or not.
2243  auto FinalizeOnReturn = make_scope_exit([this]() { finalizeFunction(); });
2244 
2245  // Setup a separate basic-block for the arguments and constants
2246  MachineBasicBlock *EntryBB = MF->CreateMachineBasicBlock();
2247  MF->push_back(EntryBB);
2248  EntryBuilder->setMBB(*EntryBB);
2249 
2251  SwiftError.setFunction(CurMF);
2252  SwiftError.createEntriesInEntryBlock(DbgLoc);
2253 
2254  // Create all blocks, in IR order, to preserve the layout.
2255  for (const BasicBlock &BB: F) {
2256  auto *&MBB = BBToMBB[&BB];
2257 
2258  MBB = MF->CreateMachineBasicBlock(&BB);
2259  MF->push_back(MBB);
2260 
2261  if (BB.hasAddressTaken())
2262  MBB->setHasAddressTaken();
2263  }
2264 
2265  // Make our arguments/constants entry block fallthrough to the IR entry block.
2266  EntryBB->addSuccessor(&getMBB(F.front()));
2267 
2268  // Lower the actual args into this basic block.
2269  SmallVector<ArrayRef<Register>, 8> VRegArgs;
2270  for (const Argument &Arg: F.args()) {
2271  if (DL->getTypeStoreSize(Arg.getType()) == 0)
2272  continue; // Don't handle zero sized types.
2273  ArrayRef<Register> VRegs = getOrCreateVRegs(Arg);
2274  VRegArgs.push_back(VRegs);
2275 
2276  if (Arg.hasSwiftErrorAttr()) {
2277  assert(VRegs.size() == 1 && "Too many vregs for Swift error");
2278  SwiftError.setCurrentVReg(EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
2279  }
2280  }
2281 
2282  // We don't currently support translating swifterror or swiftself functions.
2283  for (auto &Arg : F.args()) {
2284  if (Arg.hasSwiftSelfAttr()) {
2285  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
2286  F.getSubprogram(), &F.getEntryBlock());
2287  R << "unable to lower arguments due to swiftself: "
2288  << ore::NV("Prototype", F.getType());
2289  reportTranslationError(*MF, *TPC, *ORE, R);
2290  return false;
2291  }
2292  }
2293 
2294  if (!CLI->lowerFormalArguments(*EntryBuilder.get(), F, VRegArgs)) {
2295  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
2296  F.getSubprogram(), &F.getEntryBlock());
2297  R << "unable to lower arguments: " << ore::NV("Prototype", F.getType());
2298  reportTranslationError(*MF, *TPC, *ORE, R);
2299  return false;
2300  }
2301 
2302  // Need to visit defs before uses when translating instructions.
2303  GISelObserverWrapper WrapperObserver;
2304  if (EnableCSE && CSEInfo)
2305  WrapperObserver.addObserver(CSEInfo);
2306  {
2308 #ifndef NDEBUG
2309  DILocationVerifier Verifier;
2310  WrapperObserver.addObserver(&Verifier);
2311 #endif // ifndef NDEBUG
2312  RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
2313  for (const BasicBlock *BB : RPOT) {
2314  MachineBasicBlock &MBB = getMBB(*BB);
2315  // Set the insertion point of all the following translations to
2316  // the end of this basic block.
2317  CurBuilder->setMBB(MBB);
2318 
2319  for (const Instruction &Inst : *BB) {
2320 #ifndef NDEBUG
2321  Verifier.setCurrentInst(&Inst);
2322 #endif // ifndef NDEBUG
2323  if (translate(Inst))
2324  continue;
2325 
2326  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
2327  Inst.getDebugLoc(), BB);
2328  R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
2329 
2330  if (ORE->allowExtraAnalysis("gisel-irtranslator")) {
2331  std::string InstStrStorage;
2332  raw_string_ostream InstStr(InstStrStorage);
2333  InstStr << Inst;
2334 
2335  R << ": '" << InstStr.str() << "'";
2336  }
2337 
2338  reportTranslationError(*MF, *TPC, *ORE, R);
2339  return false;
2340  }
2341 
2342  finalizeBasicBlock();
2343  }
2344 #ifndef NDEBUG
2345  WrapperObserver.removeObserver(&Verifier);
2346 #endif
2347  }
2348 
2349  finishPendingPhis();
2350 
2351  SwiftError.propagateVRegs();
2352 
2353  // Merge the argument lowering and constants block with its single
2354  // successor, the LLVM-IR entry block. We want the basic block to
2355  // be maximal.
2356  assert(EntryBB->succ_size() == 1 &&
2357  "Custom BB used for lowering should have only one successor");
2358  // Get the successor of the current entry block.
2359  MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
2360  assert(NewEntryBB.pred_size() == 1 &&
2361  "LLVM-IR entry block has a predecessor!?");
2362  // Move all the instruction from the current entry block to the
2363  // new entry block.
2364  NewEntryBB.splice(NewEntryBB.begin(), EntryBB, EntryBB->begin(),
2365  EntryBB->end());
2366 
2367  // Update the live-in information for the new entry block.
2368  for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
2369  NewEntryBB.addLiveIn(LiveIn);
2370  NewEntryBB.sortUniqueLiveIns();
2371 
2372  // Get rid of the now empty basic block.
2373  EntryBB->removeSuccessor(&NewEntryBB);
2374  MF->remove(EntryBB);
2375  MF->DeleteMachineBasicBlock(EntryBB);
2376 
2377  assert(&MF->front() == &NewEntryBB &&
2378  "New entry wasn't next in the list of basic block!");
2379 
2380  // Initialize stack protector information.
2381  StackProtector &SP = getAnalysis<StackProtector>();
2382  SP.copyToMachineFrameInfo(MF->getFrameInfo());
2383 
2384  return false;
2385 }
auto lower_bound(R &&Range, T &&Value) -> decltype(adl_begin(Range))
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1288
MachineInstrBuilder buildDirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in Re...
uint64_t CallInst * C
Return a value (possibly void), from a function.
Value * getValueOperand()
Definition: Instructions.h:409
unsigned getNumCases() const
Return the number of &#39;cases&#39; in this switch instruction, excluding the default case.
bool isIntrinsic() const
isIntrinsic - Returns true if the function&#39;s name starts with "llvm.".
Definition: Function.h:198
A simple RAII based CSEInfo installer.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:110
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:641
MachineInstrBuilder buildJumpTable(const LLT PtrTy, unsigned JTI)
Build and insert Res = G_JUMP_TABLE JTI.
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:722
bool empty() const
Definition: Function.h:686
The CSE Analysis object.
Definition: CSEInfo.h:71
Diagnostic information for missed-optimization remarks.
DILocation * get() const
Get the underlying DILocation.
Definition: DebugLoc.cpp:21
This instruction extracts a struct member or array element value from an aggregate value...
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
*p = old <signed v ? old : v
Definition: Instructions.h:723
iterator_range< CaseIt > cases()
Iteration adapter for range-for loops.
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
This class represents an incoming formal argument to a Function.
Definition: Argument.h:29
LLVMContext & Context
DiagnosticInfoOptimizationBase::Argument NV
CaseClusterVector::iterator CaseClusterIt
This represents the llvm.dbg.label instruction.
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:288
MachineInstrBuilder buildBrCond(Register Tst, MachineBasicBlock &Dest)
Build and insert G_BRCOND Tst, Dest.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
This class represents lattice values for constants.
Definition: AllocatorList.h:23
void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU)
Modify analysis usage so it preserves passes required for the SelectionDAG fallback.
Definition: Utils.cpp:398
MachineInstrBuilder buildSExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
bool isSized(SmallPtrSetImpl< Type *> *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:264
iterator begin() const
Definition: ArrayRef.h:136
Register getReg(unsigned Idx) const
Get the register for the operand index.
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
An instruction for ordering other memory operations.
Definition: Instructions.h:454
An instruction that atomically checks whether a specified value is in a memory location, and, if it is, stores a new value there.
Definition: Instructions.h:530
void push_back(const T &Elt)
Definition: SmallVector.h:211
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:385
MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src)
Build and insert an appropriate cast between two registers of equal size.
IRTranslator LLVM IR static false void reportTranslationError(MachineFunction &MF, const TargetPassConfig &TPC, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R)
This class represents a function call, abstracting a target machine&#39;s calling convention.
unsigned Reg
This file contains the declarations for metadata subclasses.
Value * getCondition() const
static uint64_t round(uint64_t Acc, uint64_t Input)
Definition: xxhash.cpp:57
void setDebugLoc(const DebugLoc &DL)
Set the debug location to DL for all the next build instructions.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this store instruction.
Definition: Instructions.h:384
gep_type_iterator gep_type_end(const User *GEP)
unsigned less or equal
Definition: InstrTypes.h:758
const std::string & getAsmString() const
Definition: InlineAsm.h:80
*p = old <unsigned v ? old : v
Definition: Instructions.h:727
bool isSwiftError() const
Return true if this alloca is used as a swifterror argument to a call.
Definition: Instructions.h:135
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
Definition: Instructions.h:247
Offsets
Offsets in bytes from the start of the input buffer.
Definition: SIInstrInfo.h:1100
*p = old >unsigned v ? old : v
Definition: Instructions.h:725
LLVM_NODISCARD detail::scope_exit< typename std::decay< Callable >::type > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:58
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:732
void addSuccessorWithoutProb(MachineBasicBlock *Succ)
Add Succ as a successor of this MachineBasicBlock.
Type * getPointerOperandType() const
Definition: Instructions.h:415
unsigned getLine() const
Definition: DebugLoc.cpp:25
bool hasDLLImportStorageClass() const
Definition: GlobalValue.h:265
BasicBlock * getSuccessor(unsigned i) const
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:33
Metadata node.
Definition: Metadata.h:863
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
F(f)
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1212
The actual analysis pass wrapper.
Definition: CSEInfo.h:218
An instruction for reading from memory.
Definition: Instructions.h:167
void setMF(MachineFunction &MF)
Definition: CSEInfo.cpp:77
an instruction that atomically reads a memory location, combines it with another value, and then stores the result back.
Definition: Instructions.h:693
Value * getCondition() const
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:229
LLVM_READONLY APFloat maximum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2018 maximum semantics.
Definition: APFloat.h:1272
GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
Definition: Analysis.cpp:158
*p = old >signed v ? old : v
Definition: Instructions.h:721
uint64_t High
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
Optional< MachineInstrBuilder > materializeGEP(Register &Res, Register Op0, const LLT &ValueTy, uint64_t Value)
Materialize and insert Res = G_GEP Op0, (G_CONSTANT Value)
MachineInstrBuilder buildBrJT(Register TablePtr, unsigned JTI, Register IndexReg)
Build and insert G_BRJT TablePtr, JTI, IndexReg.
static Constant * getNullValue(Type *Ty)
Constructor to create a &#39;0&#39; constant of arbitrary type.
Definition: Constants.cpp:274
AtomicOrdering getFailureOrdering() const
Returns the failure ordering constraint of this cmpxchg instruction.
Definition: Instructions.h:597
bool hasSideEffects() const
Definition: InlineAsm.h:66
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1241
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:50
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Definition: Instructions.h:231
A description of a memory reference used in the backend.
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Definition: InstrTypes.h:1784
This class represents the LLVM &#39;select&#39; instruction.
const DataLayout & getDataLayout() const
Get the data layout for the module&#39;s target platform.
Definition: Module.cpp:369
#define DEBUG_TYPE
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
unsigned getAlignment() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:112
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:450
PointerType * getType() const
Overload to return most specific pointer type.
Definition: Instructions.h:96
Class to represent struct types.
Definition: DerivedTypes.h:233
DILabel * getLabel() const
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:654
BinOp getOperation() const
Definition: Instructions.h:752
const MachineInstrBuilder & addUse(unsigned RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
bool isWeak() const
Return true if this cmpxchg may spuriously fail.
Definition: Instructions.h:571
A cluster of cases suitable for jump table lowering.
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
LLVM_READONLY APFloat minimum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2018 minimum semantics.
Definition: APFloat.h:1259
The memory access is dereferenceable (i.e., doesn&#39;t trap).
std::vector< CaseCluster > CaseClusterVector
bool isVolatile() const
Return true if this is a cmpxchg from a volatile memory location.
Definition: Instructions.h:559
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Target-Independent Code Generator Pass Configuration Options.
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:695
Context object for machine code objects.
Definition: MCContext.h:64
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:244
MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope)
Build and insert G_FENCE Ordering, Scope.
MachineInstrBuilder buildBrIndirect(Register Tgt)
Build and insert G_BRINDIRECT Tgt.
Definition: Lint.cpp:83
AtomicOrdering getSuccessOrdering() const
Returns the success ordering constraint of this cmpxchg instruction.
Definition: Instructions.h:584
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:234
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:125
iterator_range< User::op_iterator > arg_operands()
Definition: InstrTypes.h:1233
An instruction for storing to memory.
Definition: Instructions.h:320
static LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_SUB Op0, Op1.
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
MachineInstrBuilder buildExtractVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
Value * getOperand(unsigned i) const
Definition: User.h:169
Analysis containing CSE Info
Definition: CSEInfo.cpp:20
BasicBlockListType::iterator iterator
unsigned JTI
The JumpTableIndex for this jump table in the function.
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
This corresponds to the llvm.lifetime.
Definition: ISDOpcodes.h:877
bool doesNotAccessMemory() const
Determine if the function does not access memory.
Definition: Function.h:473
MachineInstrBuilder buildDbgLabel(const MDNode *Label)
Build and insert a DBG_LABEL instructions specifying that Label is given.
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:140
The memory access is volatile.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this label.
const BasicBlock & getEntryBlock() const
Definition: Function.h:664
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
Definition: MathExtras.h:614
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
Abstract class that contains various methods for clients to notify about changes. ...
FPOpFusion::FPOpFusionMode AllowFPOpFusion
AllowFPOpFusion - This flag is set by the -fuse-fp-ops=xxx option.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:189
void GetUnderlyingObjects(const Value *V, SmallVectorImpl< const Value *> &Objects, const DataLayout &DL, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to GetUnderlyingObject except that it can look through phi and select instruct...
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this rmw instruction.
Definition: Instructions.h:804
CodeGenOpt::Level getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
unsigned const MachineRegisterInfo * MRI
Value * getCalledValue() const
Definition: InstrTypes.h:1280
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1504
Conditional or Unconditional Branch instruction.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
Value * getAddress() const
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:148
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Definition: Constant.h:41
bool isInlineAsm() const
Check if this call is an inline asm statement.
Definition: InstrTypes.h:1356
MachineBasicBlock * MBB
The MBB into which to emit the code for the indirect jump.
Value * getValue() const
void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:134
This file contains the declarations for the subclasses of Constant, which represent the different fla...
const Instruction & front() const
Definition: BasicBlock.h:280
void computeValueLLTs(const DataLayout &DL, Type &Ty, SmallVectorImpl< LLT > &ValueTys, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
computeValueLLTs - Given an LLVM IR type, compute a sequence of LLTs that represent all the individua...
Definition: Analysis.cpp:127
Indirect Branch Instruction.
SmallVector< SwitchWorkListItem, 4 > SwitchWorkList
Helper class to build MachineInstr.
BasicBlock * getDefaultDest() const
DIExpression * getExpression() const
bool isPredecessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a predecessor of this block.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
Represent the analysis usage information of a pass.
GISelCSEInfo & get(std::unique_ptr< CSEConfigBase > CSEOpt, bool ReCompute=false)
Takes a CSEConfigBase object that defines what opcodes get CSEd.
Definition: CSEInfo.cpp:363
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:732
void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
Definition: AsmWriter.cpp:4274
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Value * getPointerOperand()
Definition: Instructions.h:284
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn&#39;t already there.
Definition: SmallSet.h:180
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
MachineInstrBuilder buildGEP(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert Res = G_GEP Op0, Op1.
const MachineInstrBuilder & addFrameIndex(int Idx) const
static double log2(double V)
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:328
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:749
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:205
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this cmpxchg instruction.
Definition: Instructions.h:610
BasicBlock * getSuccessor(unsigned i) const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
const Value * getArraySize() const
Get the number of elements allocated.
Definition: Instructions.h:92
size_t size() const
Definition: SmallVector.h:52
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
static uint16_t copyFlagsFromInstruction(const Instruction &I)
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:219
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
Definition: Instructions.h:791
Simple wrapper that does the following.
Definition: CSEInfo.h:200
This class contains a discriminated union of information about pointers in memory operands...
std::string & str()
Flushes the stream contents to the target string and returns the string&#39;s reference.
Definition: raw_ostream.h:498
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE maxNum semantics.
Definition: APFloat.h:1248
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
const std::string & getConstraintString() const
Definition: InlineAsm.h:81
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:105
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:703
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects)
Build and insert either a G_INTRINSIC (if HasSideEffects is false) or G_INTRINSIC_W_SIDE_EFFECTS inst...
void copyIRFlags(const Instruction &I)
Copy all flags to MachineInst MIFlags.
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1122
LLT getLLTForType(Type &Ty, const DataLayout &DL)
Construct a low-level type based on an LLVM type.
The memory access writes data.
MachineInstrBuilder buildBr(MachineBasicBlock &Dest)
Build and insert G_BR Dest.
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches, switches, etc.
Definition: BasicBlock.h:391
MachineInstrBuilder buildConstDbgValue(const Constant &C, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instructions specifying that Variable is given by C (suitably modified b...
struct PredInfoPair PredInfo
Value * getValOperand()
Definition: Instructions.h:817
Predicate getPredicate(unsigned Condition, unsigned Hint)
Return predicate consisting of specified condition and hint bits.
Definition: PPCPredicates.h:87
unsigned getNumOperands() const
Definition: User.h:191
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_MUL Op0, Op1.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
This is the shared class of boolean and integer constants.
Definition: Constants.h:83
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
iterator end() const
Definition: ArrayRef.h:137
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition: Type.h:257
unsigned getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL)
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:643
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition: Type.cpp:179
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:643
DebugLoc getDebugLoc()
Get the current instruction&#39;s debug location.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
MachineInstrBuilder buildPtrMask(const DstOp &Res, const SrcOp &Op0, uint32_t NumBits)
Build and insert Res = G_PTR_MASK Op0, NumBits.
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:193
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:538
MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Elt, const SrcOp &Idx)
Build and insert Res = G_INSERT_VECTOR_ELT Val, Elt, Idx.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:940
unsigned getVectorNumElements() const
Definition: DerivedTypes.h:535
bool isIntPredicate() const
Definition: InstrTypes.h:825
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:55
Class for arbitrary precision integers.
Definition: APInt.h:69
amdgpu Simplify well known AMD library false FunctionCallee Callee
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
unsigned Reg
The virtual register containing the index of the jump table entry to jump to.
void setHasAddressTaken()
Set this block to reflect that it potentially is the target of an indirect branch.
static char ID
Definition: IRTranslator.h:63
MachineInstrBuilder buildAtomicCmpXchgWithSuccess(Register OldValRes, Register SuccessRes, Register Addr, Register CmpVal, Register NewVal, MachineMemOperand &MMO)
Build and insert OldValRes<def>, SuccessRes<def> = G_ATOMIC_CMPXCHG_WITH_SUCCESS Addr, CmpVal, NewVal, MMO.
virtual bool isFMAFasterThanFMulAndFAdd(EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
The memory access reads data.
#define Success
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1206
Representation of each machine instruction.
Definition: MachineInstr.h:64
unsigned succ_size(const Instruction *I)
Definition: CFG.h:256
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:807
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void addObserver(GISelChangeObserver *O)
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this fence instruction.
Definition: Instructions.h:490
bool isVolatile() const
Return true if this is a store to a volatile memory location.
Definition: Instructions.h:353
Analysis providing branch probability information.
unsigned getNumArgOperands() const
Definition: InstrTypes.h:1239
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:321
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Type * getPointerOperandType() const
Definition: Instructions.h:287
MachineBasicBlock * Default
The MBB of the default bb, which is a successor of the range check MBB.
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:175
void removeObserver(GISelChangeObserver *O)
A cluster of adjacent case labels with the same destination, or just one case.
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:55
AtomicOrdering getOrdering() const
Returns the ordering constraint of this store instruction.
Definition: Instructions.h:372
This represents the llvm.dbg.value instruction.
bool isTokenTy() const
Return true if this is &#39;token&#39;.
Definition: Type.h:193
verify safepoint Safepoint IR Verifier
Value * getPointerOperand()
Definition: Instructions.h:813
MachineInstrBuilder buildIndirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in me...
TargetOptions Options
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
Definition: InstrTypes.h:1287
static cl::opt< bool > EnableCSEInIRTranslator("enable-cse-in-irtranslator", cl::desc("Should enable CSE in irtranslator"), cl::Optional, cl::init(false))
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this load instruction.
Definition: Instructions.h:259
void setMBB(MachineBasicBlock &MBB)
Set the insertion point to the end of MBB.
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned char TargetFlags=0) const
void push_back(MachineInstr *MI)
#define I(x, y, z)
Definition: MD5.cpp:58
static Constant * getZeroValueForNegation(Type *Ty)
Floating point negation must be implemented with f(x) = -0.0 - x.
Definition: Constants.cpp:792
Pair of physical register and lane mask.
The memory access always returns the same value (or traps).
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:192
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:332
uint32_t Size
Definition: Profile.cpp:46
DILocalVariable * getVariable() const
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
bool isUnconditional() const
AsmDialect getDialect() const
Definition: InlineAsm.h:68
Multiway switch.
This file declares the IRTranslator pass.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
void insert(iterator MBBI, MachineBasicBlock *MBB)
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:482
aarch64 promote const
MachineInstrBuilder buildAtomicRMW(unsigned Opcode, Register OldValRes, Register Addr, Register Val, MachineMemOperand &MMO)
Build and insert OldValRes<def> = G_ATOMICRMW_<Opcode> Addr, Val, MMO.
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:575
LLVM Value Representation.
Definition: Value.h:72
static bool isSwiftError(const Value *V)
uint64_t getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type...
Definition: DataLayout.h:444
succ_range successors(Instruction *I)
Definition: CFG.h:259
This file describes how to lower LLVM calls to machine code calls.
INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI", false, false) INITIALIZE_PASS_END(IRTranslator
Invoke instruction.
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:65
IRTranslator LLVM IR MI
unsigned greater than
Definition: InstrTypes.h:755
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Definition: Pass.cpp:166
const Instruction * getFirstNonPHIOrDbg() const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic...
Definition: BasicBlock.cpp:196
AtomicOrdering getOrdering() const
Returns the ordering constraint of this fence instruction.
Definition: Instructions.h:478
const MachineInstrBuilder & addDef(unsigned RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
Simple wrapper observer that takes several observers, and calls each one for each event...
Register getReg() const
getReg - Returns the register number.
bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size...
A cluster of cases suitable for bit test lowering.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned char TargetFlags=0) const
#define LLVM_DEBUG(X)
Definition: Debug.h:122
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:416
OutputIt copy(R &&Range, OutputIt Out)
Definition: STLExtras.h:1244
This represents the llvm.dbg.declare instruction.
Value * getPointerOperand()
Definition: Instructions.h:412
The optimization diagnostic interface.
Statically lint checks LLVM IR
Definition: Lint.cpp:192
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE minNum semantics.
Definition: APFloat.h:1237
std::vector< MachineBasicBlock * >::iterator succ_iterator
int64_t getIndexedOffsetInType(Type *ElemTy, ArrayRef< Value *> Indices) const
Returns the offset from the beginning of the type for the specified indices.
Definition: DataLayout.cpp:803
bool isVolatile() const
Return true if this is a RMW on a volatile memory location.
Definition: Instructions.h:776
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:734
Wrapper class representing virtual and physical registers.
Definition: Register.h:18
IntegerType * Int32Ty
This file describes how to lower LLVM code to machine code.
const BasicBlock * getParent() const
Definition: Instruction.h:66
virtual const TargetRegisterClass * getPointerRegClass(const MachineFunction &MF, unsigned Kind=0) const
Returns a TargetRegisterClass used for pointer values.
an instruction to allocate memory on the stack
Definition: Instructions.h:59
This instruction inserts a struct field of array element value into an aggregate value.
gep_type_iterator gep_type_begin(const User *GEP)
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:164