LLVM  10.0.0svn
IRTranslator.cpp
Go to the documentation of this file.
1 //===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the IRTranslator class.
10 //===----------------------------------------------------------------------===//
11 
14 #include "llvm/ADT/STLExtras.h"
15 #include "llvm/ADT/ScopeExit.h"
16 #include "llvm/ADT/SmallSet.h"
17 #include "llvm/ADT/SmallVector.h"
21 #include "llvm/CodeGen/Analysis.h"
40 #include "llvm/IR/BasicBlock.h"
41 #include "llvm/IR/CFG.h"
42 #include "llvm/IR/Constant.h"
43 #include "llvm/IR/Constants.h"
44 #include "llvm/IR/DataLayout.h"
45 #include "llvm/IR/DebugInfo.h"
46 #include "llvm/IR/DerivedTypes.h"
47 #include "llvm/IR/Function.h"
49 #include "llvm/IR/InlineAsm.h"
50 #include "llvm/IR/InstrTypes.h"
51 #include "llvm/IR/Instructions.h"
52 #include "llvm/IR/IntrinsicInst.h"
53 #include "llvm/IR/Intrinsics.h"
54 #include "llvm/IR/LLVMContext.h"
55 #include "llvm/IR/Metadata.h"
56 #include "llvm/IR/Type.h"
57 #include "llvm/IR/User.h"
58 #include "llvm/IR/Value.h"
59 #include "llvm/MC/MCContext.h"
60 #include "llvm/Pass.h"
61 #include "llvm/Support/Casting.h"
62 #include "llvm/Support/CodeGen.h"
63 #include "llvm/Support/Debug.h"
70 #include <algorithm>
71 #include <cassert>
72 #include <cstdint>
73 #include <iterator>
74 #include <string>
75 #include <utility>
76 #include <vector>
77 
78 #define DEBUG_TYPE "irtranslator"
79 
80 using namespace llvm;
81 
82 static cl::opt<bool>
83  EnableCSEInIRTranslator("enable-cse-in-irtranslator",
84  cl::desc("Should enable CSE in irtranslator"),
85  cl::Optional, cl::init(false));
86 char IRTranslator::ID = 0;
87 
88 INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
89  false, false)
92 INITIALIZE_PASS_END(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
93  false, false)
94 
100 
101  // Print the function name explicitly if we don't have a debug location (which
102  // makes the diagnostic less useful) or if we're going to emit a raw error.
103  if (!R.getLocation().isValid() || TPC.isGlobalISelAbortEnabled())
104  R << (" (in function: " + MF.getName() + ")").str();
105 
106  if (TPC.isGlobalISelAbortEnabled())
107  report_fatal_error(R.getMsg());
108  else
109  ORE.emit(R);
110 }
111 
113 
114 #ifndef NDEBUG
115 namespace {
116 /// Verify that every instruction created has the same DILocation as the
117 /// instruction being translated.
118 class DILocationVerifier : public GISelChangeObserver {
119  const Instruction *CurrInst = nullptr;
120 
121 public:
122  DILocationVerifier() = default;
123  ~DILocationVerifier() = default;
124 
125  const Instruction *getCurrentInst() const { return CurrInst; }
126  void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
127 
128  void erasingInstr(MachineInstr &MI) override {}
129  void changingInstr(MachineInstr &MI) override {}
130  void changedInstr(MachineInstr &MI) override {}
131 
132  void createdInstr(MachineInstr &MI) override {
133  assert(getCurrentInst() && "Inserted instruction without a current MI");
134 
135  // Only print the check message if we're actually checking it.
136 #ifndef NDEBUG
137  LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
138  << " was copied to " << MI);
139 #endif
140  // We allow insts in the entry block to have a debug loc line of 0 because
141  // they could have originated from constants, and we don't want a jumpy
142  // debug experience.
143  assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
144  MI.getDebugLoc().getLine() == 0) &&
145  "Line info was not transferred to all instructions");
146  }
147 };
148 } // namespace
149 #endif // ifndef NDEBUG
150 
151 
158 }
159 
161 IRTranslator::allocateVRegs(const Value &Val) {
162  assert(!VMap.contains(Val) && "Value already allocated in VMap");
163  auto *Regs = VMap.getVRegs(Val);
164  auto *Offsets = VMap.getOffsets(Val);
165  SmallVector<LLT, 4> SplitTys;
166  computeValueLLTs(*DL, *Val.getType(), SplitTys,
167  Offsets->empty() ? Offsets : nullptr);
168  for (unsigned i = 0; i < SplitTys.size(); ++i)
169  Regs->push_back(0);
170  return *Regs;
171 }
172 
173 ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
174  auto VRegsIt = VMap.findVRegs(Val);
175  if (VRegsIt != VMap.vregs_end())
176  return *VRegsIt->second;
177 
178  if (Val.getType()->isVoidTy())
179  return *VMap.getVRegs(Val);
180 
181  // Create entry for this type.
182  auto *VRegs = VMap.getVRegs(Val);
183  auto *Offsets = VMap.getOffsets(Val);
184 
185  assert(Val.getType()->isSized() &&
186  "Don't know how to create an empty vreg");
187 
188  SmallVector<LLT, 4> SplitTys;
189  computeValueLLTs(*DL, *Val.getType(), SplitTys,
190  Offsets->empty() ? Offsets : nullptr);
191 
192  if (!isa<Constant>(Val)) {
193  for (auto Ty : SplitTys)
194  VRegs->push_back(MRI->createGenericVirtualRegister(Ty));
195  return *VRegs;
196  }
197 
198  if (Val.getType()->isAggregateType()) {
199  // UndefValue, ConstantAggregateZero
200  auto &C = cast<Constant>(Val);
201  unsigned Idx = 0;
202  while (auto Elt = C.getAggregateElement(Idx++)) {
203  auto EltRegs = getOrCreateVRegs(*Elt);
204  llvm::copy(EltRegs, std::back_inserter(*VRegs));
205  }
206  } else {
207  assert(SplitTys.size() == 1 && "unexpectedly split LLT");
208  VRegs->push_back(MRI->createGenericVirtualRegister(SplitTys[0]));
209  bool Success = translate(cast<Constant>(Val), VRegs->front());
210  if (!Success) {
211  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
212  MF->getFunction().getSubprogram(),
213  &MF->getFunction().getEntryBlock());
214  R << "unable to translate constant: " << ore::NV("Type", Val.getType());
215  reportTranslationError(*MF, *TPC, *ORE, R);
216  return *VRegs;
217  }
218  }
219 
220  return *VRegs;
221 }
222 
223 int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
224  if (FrameIndices.find(&AI) != FrameIndices.end())
225  return FrameIndices[&AI];
226 
227  unsigned ElementSize = DL->getTypeAllocSize(AI.getAllocatedType());
228  unsigned Size =
229  ElementSize * cast<ConstantInt>(AI.getArraySize())->getZExtValue();
230 
231  // Always allocate at least one byte.
232  Size = std::max(Size, 1u);
233 
234  unsigned Alignment = AI.getAlignment();
235  if (!Alignment)
236  Alignment = DL->getABITypeAlignment(AI.getAllocatedType());
237 
238  int &FI = FrameIndices[&AI];
239  FI = MF->getFrameInfo().CreateStackObject(Size, Alignment, false, &AI);
240  return FI;
241 }
242 
243 unsigned IRTranslator::getMemOpAlignment(const Instruction &I) {
244  unsigned Alignment = 0;
245  Type *ValTy = nullptr;
246  if (const StoreInst *SI = dyn_cast<StoreInst>(&I)) {
247  Alignment = SI->getAlignment();
248  ValTy = SI->getValueOperand()->getType();
249  } else if (const LoadInst *LI = dyn_cast<LoadInst>(&I)) {
250  Alignment = LI->getAlignment();
251  ValTy = LI->getType();
252  } else if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I)) {
253  // TODO(PR27168): This instruction has no alignment attribute, but unlike
254  // the default alignment for load/store, the default here is to assume
255  // it has NATURAL alignment, not DataLayout-specified alignment.
256  const DataLayout &DL = AI->getModule()->getDataLayout();
257  Alignment = DL.getTypeStoreSize(AI->getCompareOperand()->getType());
258  ValTy = AI->getCompareOperand()->getType();
259  } else if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I)) {
260  // TODO(PR27168): This instruction has no alignment attribute, but unlike
261  // the default alignment for load/store, the default here is to assume
262  // it has NATURAL alignment, not DataLayout-specified alignment.
263  const DataLayout &DL = AI->getModule()->getDataLayout();
264  Alignment = DL.getTypeStoreSize(AI->getValOperand()->getType());
265  ValTy = AI->getType();
266  } else {
267  OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
268  R << "unable to translate memop: " << ore::NV("Opcode", &I);
269  reportTranslationError(*MF, *TPC, *ORE, R);
270  return 1;
271  }
272 
273  return Alignment ? Alignment : DL->getABITypeAlignment(ValTy);
274 }
275 
276 MachineBasicBlock &IRTranslator::getMBB(const BasicBlock &BB) {
277  MachineBasicBlock *&MBB = BBToMBB[&BB];
278  assert(MBB && "BasicBlock was not encountered before");
279  return *MBB;
280 }
281 
282 void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) {
283  assert(NewPred && "new predecessor must be a real MachineBasicBlock");
284  MachinePreds[Edge].push_back(NewPred);
285 }
286 
287 bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
288  MachineIRBuilder &MIRBuilder) {
289  // Get or create a virtual register for each value.
290  // Unless the value is a Constant => loadimm cst?
291  // or inline constant each time?
292  // Creation of a virtual register needs to have a size.
293  Register Op0 = getOrCreateVReg(*U.getOperand(0));
294  Register Op1 = getOrCreateVReg(*U.getOperand(1));
295  Register Res = getOrCreateVReg(U);
296  uint16_t Flags = 0;
297  if (isa<Instruction>(U)) {
298  const Instruction &I = cast<Instruction>(U);
300  }
301 
302  MIRBuilder.buildInstr(Opcode, {Res}, {Op0, Op1}, Flags);
303  return true;
304 }
305 
306 bool IRTranslator::translateFSub(const User &U, MachineIRBuilder &MIRBuilder) {
307  // -0.0 - X --> G_FNEG
308  if (isa<Constant>(U.getOperand(0)) &&
310  Register Op1 = getOrCreateVReg(*U.getOperand(1));
311  Register Res = getOrCreateVReg(U);
312  uint16_t Flags = 0;
313  if (isa<Instruction>(U)) {
314  const Instruction &I = cast<Instruction>(U);
316  }
317  // Negate the last operand of the FSUB
318  MIRBuilder.buildInstr(TargetOpcode::G_FNEG, {Res}, {Op1}, Flags);
319  return true;
320  }
321  return translateBinaryOp(TargetOpcode::G_FSUB, U, MIRBuilder);
322 }
323 
324 bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) {
325  Register Op0 = getOrCreateVReg(*U.getOperand(0));
326  Register Res = getOrCreateVReg(U);
327  uint16_t Flags = 0;
328  if (isa<Instruction>(U)) {
329  const Instruction &I = cast<Instruction>(U);
331  }
332  MIRBuilder.buildInstr(TargetOpcode::G_FNEG, {Res}, {Op0}, Flags);
333  return true;
334 }
335 
336 bool IRTranslator::translateCompare(const User &U,
337  MachineIRBuilder &MIRBuilder) {
338  const CmpInst *CI = dyn_cast<CmpInst>(&U);
339  Register Op0 = getOrCreateVReg(*U.getOperand(0));
340  Register Op1 = getOrCreateVReg(*U.getOperand(1));
341  Register Res = getOrCreateVReg(U);
342  CmpInst::Predicate Pred =
343  CI ? CI->getPredicate() : static_cast<CmpInst::Predicate>(
344  cast<ConstantExpr>(U).getPredicate());
345  if (CmpInst::isIntPredicate(Pred))
346  MIRBuilder.buildICmp(Pred, Res, Op0, Op1);
347  else if (Pred == CmpInst::FCMP_FALSE)
348  MIRBuilder.buildCopy(
349  Res, getOrCreateVReg(*Constant::getNullValue(CI->getType())));
350  else if (Pred == CmpInst::FCMP_TRUE)
351  MIRBuilder.buildCopy(
352  Res, getOrCreateVReg(*Constant::getAllOnesValue(CI->getType())));
353  else {
354  MIRBuilder.buildInstr(TargetOpcode::G_FCMP, {Res}, {Pred, Op0, Op1},
356  }
357 
358  return true;
359 }
360 
361 bool IRTranslator::translateRet(const User &U, MachineIRBuilder &MIRBuilder) {
362  const ReturnInst &RI = cast<ReturnInst>(U);
363  const Value *Ret = RI.getReturnValue();
364  if (Ret && DL->getTypeStoreSize(Ret->getType()) == 0)
365  Ret = nullptr;
366 
367  ArrayRef<Register> VRegs;
368  if (Ret)
369  VRegs = getOrCreateVRegs(*Ret);
370 
371  Register SwiftErrorVReg = 0;
372  if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {
373  SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
374  &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
375  }
376 
377  // The target may mess up with the insertion point, but
378  // this is not important as a return is the last instruction
379  // of the block anyway.
380  return CLI->lowerReturn(MIRBuilder, Ret, VRegs, SwiftErrorVReg);
381 }
382 
383 bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) {
384  const BranchInst &BrInst = cast<BranchInst>(U);
385  unsigned Succ = 0;
386  if (!BrInst.isUnconditional()) {
387  // We want a G_BRCOND to the true BB followed by an unconditional branch.
388  Register Tst = getOrCreateVReg(*BrInst.getCondition());
389  const BasicBlock &TrueTgt = *cast<BasicBlock>(BrInst.getSuccessor(Succ++));
390  MachineBasicBlock &TrueBB = getMBB(TrueTgt);
391  MIRBuilder.buildBrCond(Tst, TrueBB);
392  }
393 
394  const BasicBlock &BrTgt = *cast<BasicBlock>(BrInst.getSuccessor(Succ));
395  MachineBasicBlock &TgtBB = getMBB(BrTgt);
396  MachineBasicBlock &CurBB = MIRBuilder.getMBB();
397 
398  // If the unconditional target is the layout successor, fallthrough.
399  if (!CurBB.isLayoutSuccessor(&TgtBB))
400  MIRBuilder.buildBr(TgtBB);
401 
402  // Link successors.
403  for (const BasicBlock *Succ : successors(&BrInst))
404  CurBB.addSuccessor(&getMBB(*Succ));
405  return true;
406 }
407 
408 void IRTranslator::addSuccessorWithProb(MachineBasicBlock *Src,
409  MachineBasicBlock *Dst,
410  BranchProbability Prob) {
411  if (!FuncInfo.BPI) {
412  Src->addSuccessorWithoutProb(Dst);
413  return;
414  }
415  if (Prob.isUnknown())
416  Prob = getEdgeProbability(Src, Dst);
417  Src->addSuccessor(Dst, Prob);
418 }
419 
421 IRTranslator::getEdgeProbability(const MachineBasicBlock *Src,
422  const MachineBasicBlock *Dst) const {
423  const BasicBlock *SrcBB = Src->getBasicBlock();
424  const BasicBlock *DstBB = Dst->getBasicBlock();
425  if (!FuncInfo.BPI) {
426  // If BPI is not available, set the default probability as 1 / N, where N is
427  // the number of successors.
428  auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
429  return BranchProbability(1, SuccSize);
430  }
431  return FuncInfo.BPI->getEdgeProbability(SrcBB, DstBB);
432 }
433 
434 bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) {
435  using namespace SwitchCG;
436  // Extract cases from the switch.
437  const SwitchInst &SI = cast<SwitchInst>(U);
438  BranchProbabilityInfo *BPI = FuncInfo.BPI;
439  CaseClusterVector Clusters;
440  Clusters.reserve(SI.getNumCases());
441  for (auto &I : SI.cases()) {
442  MachineBasicBlock *Succ = &getMBB(*I.getCaseSuccessor());
443  assert(Succ && "Could not find successor mbb in mapping");
444  const ConstantInt *CaseVal = I.getCaseValue();
445  BranchProbability Prob =
446  BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
447  : BranchProbability(1, SI.getNumCases() + 1);
448  Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
449  }
450 
451  MachineBasicBlock *DefaultMBB = &getMBB(*SI.getDefaultDest());
452 
453  // Cluster adjacent cases with the same destination. We do this at all
454  // optimization levels because it's cheap to do and will make codegen faster
455  // if there are many clusters.
456  sortAndRangeify(Clusters);
457 
458  MachineBasicBlock *SwitchMBB = &getMBB(*SI.getParent());
459 
460  // If there is only the default destination, jump there directly.
461  if (Clusters.empty()) {
462  SwitchMBB->addSuccessor(DefaultMBB);
463  if (DefaultMBB != SwitchMBB->getNextNode())
464  MIB.buildBr(*DefaultMBB);
465  return true;
466  }
467 
468  SL->findJumpTables(Clusters, &SI, DefaultMBB);
469 
470  LLVM_DEBUG({
471  dbgs() << "Case clusters: ";
472  for (const CaseCluster &C : Clusters) {
473  if (C.Kind == CC_JumpTable)
474  dbgs() << "JT:";
475  if (C.Kind == CC_BitTests)
476  dbgs() << "BT:";
477 
478  C.Low->getValue().print(dbgs(), true);
479  if (C.Low != C.High) {
480  dbgs() << '-';
481  C.High->getValue().print(dbgs(), true);
482  }
483  dbgs() << ' ';
484  }
485  dbgs() << '\n';
486  });
487 
488  assert(!Clusters.empty());
489  SwitchWorkList WorkList;
490  CaseClusterIt First = Clusters.begin();
491  CaseClusterIt Last = Clusters.end() - 1;
492  auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB);
493  WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
494 
495  // FIXME: At the moment we don't do any splitting optimizations here like
496  // SelectionDAG does, so this worklist only has one entry.
497  while (!WorkList.empty()) {
498  SwitchWorkListItem W = WorkList.back();
499  WorkList.pop_back();
500  if (!lowerSwitchWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
501  return false;
502  }
503  return true;
504 }
505 
506 void IRTranslator::emitJumpTable(SwitchCG::JumpTable &JT,
507  MachineBasicBlock *MBB) {
508  // Emit the code for the jump table
509  assert(JT.Reg != -1U && "Should lower JT Header first!");
510  MachineIRBuilder MIB(*MBB->getParent());
511  MIB.setMBB(*MBB);
512  MIB.setDebugLoc(CurBuilder->getDebugLoc());
513 
514  Type *PtrIRTy = Type::getInt8PtrTy(MF->getFunction().getContext());
515  const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
516 
517  auto Table = MIB.buildJumpTable(PtrTy, JT.JTI);
518  MIB.buildBrJT(Table.getReg(0), JT.JTI, JT.Reg);
519 }
520 
521 bool IRTranslator::emitJumpTableHeader(SwitchCG::JumpTable &JT,
523  MachineBasicBlock *HeaderBB) {
524  MachineIRBuilder MIB(*HeaderBB->getParent());
525  MIB.setMBB(*HeaderBB);
526  MIB.setDebugLoc(CurBuilder->getDebugLoc());
527 
528  const Value &SValue = *JTH.SValue;
529  // Subtract the lowest switch case value from the value being switched on.
530  const LLT SwitchTy = getLLTForType(*SValue.getType(), *DL);
531  Register SwitchOpReg = getOrCreateVReg(SValue);
532  auto FirstCst = MIB.buildConstant(SwitchTy, JTH.First);
533  auto Sub = MIB.buildSub({SwitchTy}, SwitchOpReg, FirstCst);
534 
535  // This value may be smaller or larger than the target's pointer type, and
536  // therefore require extension or truncating.
537  Type *PtrIRTy = SValue.getType()->getPointerTo();
538  const LLT PtrScalarTy = LLT::scalar(DL->getTypeSizeInBits(PtrIRTy));
539  Sub = MIB.buildZExtOrTrunc(PtrScalarTy, Sub);
540 
541  JT.Reg = Sub.getReg(0);
542 
543  if (JTH.OmitRangeCheck) {
544  if (JT.MBB != HeaderBB->getNextNode())
545  MIB.buildBr(*JT.MBB);
546  return true;
547  }
548 
549  // Emit the range check for the jump table, and branch to the default block
550  // for the switch statement if the value being switched on exceeds the
551  // largest case in the switch.
552  auto Cst = getOrCreateVReg(
553  *ConstantInt::get(SValue.getType(), JTH.Last - JTH.First));
554  Cst = MIB.buildZExtOrTrunc(PtrScalarTy, Cst).getReg(0);
555  auto Cmp = MIB.buildICmp(CmpInst::ICMP_UGT, LLT::scalar(1), Sub, Cst);
556 
557  auto BrCond = MIB.buildBrCond(Cmp.getReg(0), *JT.Default);
558 
559  // Avoid emitting unnecessary branches to the next block.
560  if (JT.MBB != HeaderBB->getNextNode())
561  BrCond = MIB.buildBr(*JT.MBB);
562  return true;
563 }
564 
565 void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
566  MachineBasicBlock *SwitchBB,
567  MachineIRBuilder &MIB) {
568  Register CondLHS = getOrCreateVReg(*CB.CmpLHS);
569  Register Cond;
570  DebugLoc OldDbgLoc = MIB.getDebugLoc();
571  MIB.setDebugLoc(CB.DbgLoc);
572  MIB.setMBB(*CB.ThisBB);
573 
574  if (CB.PredInfo.NoCmp) {
575  // Branch or fall through to TrueBB.
576  addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
577  addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
578  CB.ThisBB);
580  if (CB.TrueBB != CB.ThisBB->getNextNode())
581  MIB.buildBr(*CB.TrueBB);
582  MIB.setDebugLoc(OldDbgLoc);
583  return;
584  }
585 
586  const LLT i1Ty = LLT::scalar(1);
587  // Build the compare.
588  if (!CB.CmpMHS) {
589  Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
590  Cond = MIB.buildICmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
591  } else {
593  "Can only handle ULE ranges");
594 
595  const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
596  const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
597 
598  Register CmpOpReg = getOrCreateVReg(*CB.CmpMHS);
599  if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
600  Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
601  Cond =
602  MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, CmpOpReg, CondRHS).getReg(0);
603  } else {
604  const LLT &CmpTy = MRI->getType(CmpOpReg);
605  auto Sub = MIB.buildSub({CmpTy}, CmpOpReg, CondLHS);
606  auto Diff = MIB.buildConstant(CmpTy, High - Low);
607  Cond = MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, Sub, Diff).getReg(0);
608  }
609  }
610 
611  // Update successor info
612  addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
613 
614  addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
615  CB.ThisBB);
616 
617  // TrueBB and FalseBB are always different unless the incoming IR is
618  // degenerate. This only happens when running llc on weird IR.
619  if (CB.TrueBB != CB.FalseBB)
620  addSuccessorWithProb(CB.ThisBB, CB.FalseBB, CB.FalseProb);
622 
623  // if (SwitchBB->getBasicBlock() != CB.FalseBB->getBasicBlock())
624  addMachineCFGPred({SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
625  CB.ThisBB);
626 
627  // If the lhs block is the next block, invert the condition so that we can
628  // fall through to the lhs instead of the rhs block.
629  if (CB.TrueBB == CB.ThisBB->getNextNode()) {
630  std::swap(CB.TrueBB, CB.FalseBB);
631  auto True = MIB.buildConstant(i1Ty, 1);
632  Cond = MIB.buildInstr(TargetOpcode::G_XOR, {i1Ty}, {Cond, True}, None)
633  .getReg(0);
634  }
635 
636  MIB.buildBrCond(Cond, *CB.TrueBB);
637  MIB.buildBr(*CB.FalseBB);
638  MIB.setDebugLoc(OldDbgLoc);
639 }
640 
641 bool IRTranslator::lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W,
642  MachineBasicBlock *SwitchMBB,
643  MachineBasicBlock *CurMBB,
644  MachineBasicBlock *DefaultMBB,
645  MachineIRBuilder &MIB,
647  BranchProbability UnhandledProbs,
649  MachineBasicBlock *Fallthrough,
650  bool FallthroughUnreachable) {
651  using namespace SwitchCG;
652  MachineFunction *CurMF = SwitchMBB->getParent();
653  // FIXME: Optimize away range check based on pivot comparisons.
654  JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
655  SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
656  BranchProbability DefaultProb = W.DefaultProb;
657 
658  // The jump block hasn't been inserted yet; insert it here.
659  MachineBasicBlock *JumpMBB = JT->MBB;
660  CurMF->insert(BBI, JumpMBB);
661 
662  // Since the jump table block is separate from the switch block, we need
663  // to keep track of it as a machine predecessor to the default block,
664  // otherwise we lose the phi edges.
665  addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
666  CurMBB);
667  addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
668  JumpMBB);
669 
670  auto JumpProb = I->Prob;
671  auto FallthroughProb = UnhandledProbs;
672 
673  // If the default statement is a target of the jump table, we evenly
674  // distribute the default probability to successors of CurMBB. Also
675  // update the probability on the edge from JumpMBB to Fallthrough.
677  SE = JumpMBB->succ_end();
678  SI != SE; ++SI) {
679  if (*SI == DefaultMBB) {
680  JumpProb += DefaultProb / 2;
681  FallthroughProb -= DefaultProb / 2;
682  JumpMBB->setSuccProbability(SI, DefaultProb / 2);
683  JumpMBB->normalizeSuccProbs();
684  } else {
685  // Also record edges from the jump table block to it's successors.
686  addMachineCFGPred({SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
687  JumpMBB);
688  }
689  }
690 
691  // Skip the range check if the fallthrough block is unreachable.
692  if (FallthroughUnreachable)
693  JTH->OmitRangeCheck = true;
694 
695  if (!JTH->OmitRangeCheck)
696  addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
697  addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
698  CurMBB->normalizeSuccProbs();
699 
700  // The jump table header will be inserted in our current block, do the
701  // range check, and fall through to our fallthrough block.
702  JTH->HeaderBB = CurMBB;
703  JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
704 
705  // If we're in the right place, emit the jump table header right now.
706  if (CurMBB == SwitchMBB) {
707  if (!emitJumpTableHeader(*JT, *JTH, CurMBB))
708  return false;
709  JTH->Emitted = true;
710  }
711  return true;
712 }
713 bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,
714  Value *Cond,
715  MachineBasicBlock *Fallthrough,
716  bool FallthroughUnreachable,
717  BranchProbability UnhandledProbs,
718  MachineBasicBlock *CurMBB,
719  MachineIRBuilder &MIB,
720  MachineBasicBlock *SwitchMBB) {
721  using namespace SwitchCG;
722  const Value *RHS, *LHS, *MHS;
723  CmpInst::Predicate Pred;
724  if (I->Low == I->High) {
725  // Check Cond == I->Low.
726  Pred = CmpInst::ICMP_EQ;
727  LHS = Cond;
728  RHS = I->Low;
729  MHS = nullptr;
730  } else {
731  // Check I->Low <= Cond <= I->High.
732  Pred = CmpInst::ICMP_ULE;
733  LHS = I->Low;
734  MHS = Cond;
735  RHS = I->High;
736  }
737 
738  // If Fallthrough is unreachable, fold away the comparison.
739  // The false probability is the sum of all unhandled cases.
740  CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
741  CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
742 
743  emitSwitchCase(CB, SwitchMBB, MIB);
744  return true;
745 }
746 
747 bool IRTranslator::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
748  Value *Cond,
749  MachineBasicBlock *SwitchMBB,
750  MachineBasicBlock *DefaultMBB,
751  MachineIRBuilder &MIB) {
752  using namespace SwitchCG;
753  MachineFunction *CurMF = FuncInfo.MF;
754  MachineBasicBlock *NextMBB = nullptr;
756  if (++BBI != FuncInfo.MF->end())
757  NextMBB = &*BBI;
758 
759  if (EnableOpts) {
760  // Here, we order cases by probability so the most likely case will be
761  // checked first. However, two clusters can have the same probability in
762  // which case their relative ordering is non-deterministic. So we use Low
763  // as a tie-breaker as clusters are guaranteed to never overlap.
765  [](const CaseCluster &a, const CaseCluster &b) {
766  return a.Prob != b.Prob
767  ? a.Prob > b.Prob
768  : a.Low->getValue().slt(b.Low->getValue());
769  });
770 
771  // Rearrange the case blocks so that the last one falls through if possible
772  // without changing the order of probabilities.
773  for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
774  --I;
775  if (I->Prob > W.LastCluster->Prob)
776  break;
777  if (I->Kind == CC_Range && I->MBB == NextMBB) {
778  std::swap(*I, *W.LastCluster);
779  break;
780  }
781  }
782  }
783 
784  // Compute total probability.
785  BranchProbability DefaultProb = W.DefaultProb;
786  BranchProbability UnhandledProbs = DefaultProb;
787  for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
788  UnhandledProbs += I->Prob;
789 
790  MachineBasicBlock *CurMBB = W.MBB;
791  for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
792  bool FallthroughUnreachable = false;
793  MachineBasicBlock *Fallthrough;
794  if (I == W.LastCluster) {
795  // For the last cluster, fall through to the default destination.
796  Fallthrough = DefaultMBB;
797  FallthroughUnreachable = isa<UnreachableInst>(
798  DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
799  } else {
800  Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
801  CurMF->insert(BBI, Fallthrough);
802  }
803  UnhandledProbs -= I->Prob;
804 
805  switch (I->Kind) {
806  case CC_BitTests: {
807  LLVM_DEBUG(dbgs() << "Switch to bit test optimization unimplemented");
808  return false; // Bit tests currently unimplemented.
809  }
810  case CC_JumpTable: {
811  if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
812  UnhandledProbs, I, Fallthrough,
813  FallthroughUnreachable)) {
814  LLVM_DEBUG(dbgs() << "Failed to lower jump table");
815  return false;
816  }
817  break;
818  }
819  case CC_Range: {
820  if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
821  FallthroughUnreachable, UnhandledProbs,
822  CurMBB, MIB, SwitchMBB)) {
823  LLVM_DEBUG(dbgs() << "Failed to lower switch range");
824  return false;
825  }
826  break;
827  }
828  }
829  CurMBB = Fallthrough;
830  }
831 
832  return true;
833 }
834 
835 bool IRTranslator::translateIndirectBr(const User &U,
836  MachineIRBuilder &MIRBuilder) {
837  const IndirectBrInst &BrInst = cast<IndirectBrInst>(U);
838 
839  const Register Tgt = getOrCreateVReg(*BrInst.getAddress());
840  MIRBuilder.buildBrIndirect(Tgt);
841 
842  // Link successors.
843  MachineBasicBlock &CurBB = MIRBuilder.getMBB();
844  for (const BasicBlock *Succ : successors(&BrInst))
845  CurBB.addSuccessor(&getMBB(*Succ));
846 
847  return true;
848 }
849 
850 static bool isSwiftError(const Value *V) {
851  if (auto Arg = dyn_cast<Argument>(V))
852  return Arg->hasSwiftErrorAttr();
853  if (auto AI = dyn_cast<AllocaInst>(V))
854  return AI->isSwiftError();
855  return false;
856 }
857 
858 bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
859  const LoadInst &LI = cast<LoadInst>(U);
860 
861  auto Flags = LI.isVolatile() ? MachineMemOperand::MOVolatile
863  Flags |= MachineMemOperand::MOLoad;
864 
865  if (DL->getTypeStoreSize(LI.getType()) == 0)
866  return true;
867 
868  ArrayRef<Register> Regs = getOrCreateVRegs(LI);
869  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(LI);
870  Register Base = getOrCreateVReg(*LI.getPointerOperand());
871 
872  Type *OffsetIRTy = DL->getIntPtrType(LI.getPointerOperandType());
873  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
874 
875  if (CLI->supportSwiftError() && isSwiftError(LI.getPointerOperand())) {
876  assert(Regs.size() == 1 && "swifterror should be single pointer");
877  Register VReg = SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(),
878  LI.getPointerOperand());
879  MIRBuilder.buildCopy(Regs[0], VReg);
880  return true;
881  }
882 
883  const MDNode *Ranges =
884  Regs.size() == 1 ? LI.getMetadata(LLVMContext::MD_range) : nullptr;
885  for (unsigned i = 0; i < Regs.size(); ++i) {
886  Register Addr;
887  MIRBuilder.materializeGEP(Addr, Base, OffsetTy, Offsets[i] / 8);
888 
889  MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i] / 8);
890  unsigned BaseAlign = getMemOpAlignment(LI);
891  auto MMO = MF->getMachineMemOperand(
892  Ptr, Flags, (MRI->getType(Regs[i]).getSizeInBits() + 7) / 8,
893  MinAlign(BaseAlign, Offsets[i] / 8), AAMDNodes(), Ranges,
894  LI.getSyncScopeID(), LI.getOrdering());
895  MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
896  }
897 
898  return true;
899 }
900 
901 bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
902  const StoreInst &SI = cast<StoreInst>(U);
903  auto Flags = SI.isVolatile() ? MachineMemOperand::MOVolatile
906 
907  if (DL->getTypeStoreSize(SI.getValueOperand()->getType()) == 0)
908  return true;
909 
910  ArrayRef<Register> Vals = getOrCreateVRegs(*SI.getValueOperand());
911  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*SI.getValueOperand());
912  Register Base = getOrCreateVReg(*SI.getPointerOperand());
913 
914  Type *OffsetIRTy = DL->getIntPtrType(SI.getPointerOperandType());
915  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
916 
917  if (CLI->supportSwiftError() && isSwiftError(SI.getPointerOperand())) {
918  assert(Vals.size() == 1 && "swifterror should be single pointer");
919 
920  Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
921  SI.getPointerOperand());
922  MIRBuilder.buildCopy(VReg, Vals[0]);
923  return true;
924  }
925 
926  for (unsigned i = 0; i < Vals.size(); ++i) {
927  Register Addr;
928  MIRBuilder.materializeGEP(Addr, Base, OffsetTy, Offsets[i] / 8);
929 
930  MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i] / 8);
931  unsigned BaseAlign = getMemOpAlignment(SI);
932  auto MMO = MF->getMachineMemOperand(
933  Ptr, Flags, (MRI->getType(Vals[i]).getSizeInBits() + 7) / 8,
934  MinAlign(BaseAlign, Offsets[i] / 8), AAMDNodes(), nullptr,
935  SI.getSyncScopeID(), SI.getOrdering());
936  MIRBuilder.buildStore(Vals[i], Addr, *MMO);
937  }
938  return true;
939 }
940 
941 static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL) {
942  const Value *Src = U.getOperand(0);
944 
945  // getIndexedOffsetInType is designed for GEPs, so the first index is the
946  // usual array element rather than looking into the actual aggregate.
947  SmallVector<Value *, 1> Indices;
948  Indices.push_back(ConstantInt::get(Int32Ty, 0));
949 
950  if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(&U)) {
951  for (auto Idx : EVI->indices())
952  Indices.push_back(ConstantInt::get(Int32Ty, Idx));
953  } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(&U)) {
954  for (auto Idx : IVI->indices())
955  Indices.push_back(ConstantInt::get(Int32Ty, Idx));
956  } else {
957  for (unsigned i = 1; i < U.getNumOperands(); ++i)
958  Indices.push_back(U.getOperand(i));
959  }
960 
961  return 8 * static_cast<uint64_t>(
962  DL.getIndexedOffsetInType(Src->getType(), Indices));
963 }
964 
965 bool IRTranslator::translateExtractValue(const User &U,
966  MachineIRBuilder &MIRBuilder) {
967  const Value *Src = U.getOperand(0);
968  uint64_t Offset = getOffsetFromIndices(U, *DL);
969  ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
970  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*Src);
971  unsigned Idx = llvm::lower_bound(Offsets, Offset) - Offsets.begin();
972  auto &DstRegs = allocateVRegs(U);
973 
974  for (unsigned i = 0; i < DstRegs.size(); ++i)
975  DstRegs[i] = SrcRegs[Idx++];
976 
977  return true;
978 }
979 
980 bool IRTranslator::translateInsertValue(const User &U,
981  MachineIRBuilder &MIRBuilder) {
982  const Value *Src = U.getOperand(0);
983  uint64_t Offset = getOffsetFromIndices(U, *DL);
984  auto &DstRegs = allocateVRegs(U);
985  ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(U);
986  ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
987  ArrayRef<Register> InsertedRegs = getOrCreateVRegs(*U.getOperand(1));
988  auto InsertedIt = InsertedRegs.begin();
989 
990  for (unsigned i = 0; i < DstRegs.size(); ++i) {
991  if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
992  DstRegs[i] = *InsertedIt++;
993  else
994  DstRegs[i] = SrcRegs[i];
995  }
996 
997  return true;
998 }
999 
1000 bool IRTranslator::translateSelect(const User &U,
1001  MachineIRBuilder &MIRBuilder) {
1002  Register Tst = getOrCreateVReg(*U.getOperand(0));
1003  ArrayRef<Register> ResRegs = getOrCreateVRegs(U);
1004  ArrayRef<Register> Op0Regs = getOrCreateVRegs(*U.getOperand(1));
1005  ArrayRef<Register> Op1Regs = getOrCreateVRegs(*U.getOperand(2));
1006 
1007  const SelectInst &SI = cast<SelectInst>(U);
1008  uint16_t Flags = 0;
1009  if (const CmpInst *Cmp = dyn_cast<CmpInst>(SI.getCondition()))
1011 
1012  for (unsigned i = 0; i < ResRegs.size(); ++i) {
1013  MIRBuilder.buildInstr(TargetOpcode::G_SELECT, {ResRegs[i]},
1014  {Tst, Op0Regs[i], Op1Regs[i]}, Flags);
1015  }
1016 
1017  return true;
1018 }
1019 
1020 bool IRTranslator::translateBitCast(const User &U,
1021  MachineIRBuilder &MIRBuilder) {
1022  // If we're bitcasting to the source type, we can reuse the source vreg.
1023  if (getLLTForType(*U.getOperand(0)->getType(), *DL) ==
1024  getLLTForType(*U.getType(), *DL)) {
1025  Register SrcReg = getOrCreateVReg(*U.getOperand(0));
1026  auto &Regs = *VMap.getVRegs(U);
1027  // If we already assigned a vreg for this bitcast, we can't change that.
1028  // Emit a copy to satisfy the users we already emitted.
1029  if (!Regs.empty())
1030  MIRBuilder.buildCopy(Regs[0], SrcReg);
1031  else {
1032  Regs.push_back(SrcReg);
1033  VMap.getOffsets(U)->push_back(0);
1034  }
1035  return true;
1036  }
1037  return translateCast(TargetOpcode::G_BITCAST, U, MIRBuilder);
1038 }
1039 
1040 bool IRTranslator::translateCast(unsigned Opcode, const User &U,
1041  MachineIRBuilder &MIRBuilder) {
1042  Register Op = getOrCreateVReg(*U.getOperand(0));
1043  Register Res = getOrCreateVReg(U);
1044  MIRBuilder.buildInstr(Opcode, {Res}, {Op});
1045  return true;
1046 }
1047 
1048 bool IRTranslator::translateGetElementPtr(const User &U,
1049  MachineIRBuilder &MIRBuilder) {
1050  // FIXME: support vector GEPs.
1051  if (U.getType()->isVectorTy())
1052  return false;
1053 
1054  Value &Op0 = *U.getOperand(0);
1055  Register BaseReg = getOrCreateVReg(Op0);
1056  Type *PtrIRTy = Op0.getType();
1057  LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
1058  Type *OffsetIRTy = DL->getIntPtrType(PtrIRTy);
1059  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1060 
1061  int64_t Offset = 0;
1062  for (gep_type_iterator GTI = gep_type_begin(&U), E = gep_type_end(&U);
1063  GTI != E; ++GTI) {
1064  const Value *Idx = GTI.getOperand();
1065  if (StructType *StTy = GTI.getStructTypeOrNull()) {
1066  unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
1067  Offset += DL->getStructLayout(StTy)->getElementOffset(Field);
1068  continue;
1069  } else {
1070  uint64_t ElementSize = DL->getTypeAllocSize(GTI.getIndexedType());
1071 
1072  // If this is a scalar constant or a splat vector of constants,
1073  // handle it quickly.
1074  if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
1075  Offset += ElementSize * CI->getSExtValue();
1076  continue;
1077  }
1078 
1079  if (Offset != 0) {
1080  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1081  auto OffsetMIB = MIRBuilder.buildConstant({OffsetTy}, Offset);
1082  BaseReg =
1083  MIRBuilder.buildGEP(PtrTy, BaseReg, OffsetMIB.getReg(0)).getReg(0);
1084  Offset = 0;
1085  }
1086 
1087  Register IdxReg = getOrCreateVReg(*Idx);
1088  if (MRI->getType(IdxReg) != OffsetTy)
1089  IdxReg = MIRBuilder.buildSExtOrTrunc(OffsetTy, IdxReg).getReg(0);
1090 
1091  // N = N + Idx * ElementSize;
1092  // Avoid doing it for ElementSize of 1.
1093  Register GepOffsetReg;
1094  if (ElementSize != 1) {
1095  auto ElementSizeMIB = MIRBuilder.buildConstant(
1096  getLLTForType(*OffsetIRTy, *DL), ElementSize);
1097  GepOffsetReg =
1098  MIRBuilder.buildMul(OffsetTy, ElementSizeMIB, IdxReg).getReg(0);
1099  } else
1100  GepOffsetReg = IdxReg;
1101 
1102  BaseReg = MIRBuilder.buildGEP(PtrTy, BaseReg, GepOffsetReg).getReg(0);
1103  }
1104  }
1105 
1106  if (Offset != 0) {
1107  auto OffsetMIB =
1108  MIRBuilder.buildConstant(getLLTForType(*OffsetIRTy, *DL), Offset);
1109  MIRBuilder.buildGEP(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0));
1110  return true;
1111  }
1112 
1113  MIRBuilder.buildCopy(getOrCreateVReg(U), BaseReg);
1114  return true;
1115 }
1116 
1117 bool IRTranslator::translateMemFunc(const CallInst &CI,
1118  MachineIRBuilder &MIRBuilder,
1119  Intrinsic::ID ID) {
1120 
1121  // If the source is undef, then just emit a nop.
1122  if (isa<UndefValue>(CI.getArgOperand(1)))
1123  return true;
1124 
1125  ArrayRef<Register> Res;
1126  auto ICall = MIRBuilder.buildIntrinsic(ID, Res, true);
1127  for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(AI) != AE; ++AI)
1128  ICall.addUse(getOrCreateVReg(**AI));
1129 
1130  unsigned DstAlign = 0, SrcAlign = 0;
1131  unsigned IsVol =
1132  cast<ConstantInt>(CI.getArgOperand(CI.getNumArgOperands() - 1))
1133  ->getZExtValue();
1134 
1135  if (auto *MCI = dyn_cast<MemCpyInst>(&CI)) {
1136  DstAlign = std::max<unsigned>(MCI->getDestAlignment(), 1);
1137  SrcAlign = std::max<unsigned>(MCI->getSourceAlignment(), 1);
1138  } else if (auto *MMI = dyn_cast<MemMoveInst>(&CI)) {
1139  DstAlign = std::max<unsigned>(MMI->getDestAlignment(), 1);
1140  SrcAlign = std::max<unsigned>(MMI->getSourceAlignment(), 1);
1141  } else {
1142  auto *MSI = cast<MemSetInst>(&CI);
1143  DstAlign = std::max<unsigned>(MSI->getDestAlignment(), 1);
1144  }
1145 
1146  // Create mem operands to store the alignment and volatile info.
1148  ICall.addMemOperand(MF->getMachineMemOperand(
1150  MachineMemOperand::MOStore | VolFlag, 1, DstAlign));
1151  if (ID != Intrinsic::memset)
1152  ICall.addMemOperand(MF->getMachineMemOperand(
1154  MachineMemOperand::MOLoad | VolFlag, 1, SrcAlign));
1155 
1156  return true;
1157 }
1158 
1159 void IRTranslator::getStackGuard(Register DstReg,
1160  MachineIRBuilder &MIRBuilder) {
1161  const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1162  MRI->setRegClass(DstReg, TRI->getPointerRegClass(*MF));
1163  auto MIB = MIRBuilder.buildInstr(TargetOpcode::LOAD_STACK_GUARD);
1164  MIB.addDef(DstReg);
1165 
1166  auto &TLI = *MF->getSubtarget().getTargetLowering();
1167  Value *Global = TLI.getSDagStackGuard(*MF->getFunction().getParent());
1168  if (!Global)
1169  return;
1170 
1171  MachinePointerInfo MPInfo(Global);
1175  MF->getMachineMemOperand(MPInfo, Flags, DL->getPointerSizeInBits() / 8,
1176  DL->getPointerABIAlignment(0));
1177  MIB.setMemRefs({MemRef});
1178 }
1179 
1180 bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
1181  MachineIRBuilder &MIRBuilder) {
1182  ArrayRef<Register> ResRegs = getOrCreateVRegs(CI);
1183  MIRBuilder.buildInstr(Op)
1184  .addDef(ResRegs[0])
1185  .addDef(ResRegs[1])
1186  .addUse(getOrCreateVReg(*CI.getOperand(0)))
1187  .addUse(getOrCreateVReg(*CI.getOperand(1)));
1188 
1189  return true;
1190 }
1191 
1192 unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
1193  switch (ID) {
1194  default:
1195  break;
1196  case Intrinsic::bswap:
1197  return TargetOpcode::G_BSWAP;
1198  case Intrinsic::bitreverse:
1199  return TargetOpcode::G_BITREVERSE;
1200  case Intrinsic::ceil:
1201  return TargetOpcode::G_FCEIL;
1202  case Intrinsic::cos:
1203  return TargetOpcode::G_FCOS;
1204  case Intrinsic::ctpop:
1205  return TargetOpcode::G_CTPOP;
1206  case Intrinsic::exp:
1207  return TargetOpcode::G_FEXP;
1208  case Intrinsic::exp2:
1209  return TargetOpcode::G_FEXP2;
1210  case Intrinsic::fabs:
1211  return TargetOpcode::G_FABS;
1212  case Intrinsic::copysign:
1213  return TargetOpcode::G_FCOPYSIGN;
1214  case Intrinsic::minnum:
1215  return TargetOpcode::G_FMINNUM;
1216  case Intrinsic::maxnum:
1217  return TargetOpcode::G_FMAXNUM;
1218  case Intrinsic::minimum:
1219  return TargetOpcode::G_FMINIMUM;
1220  case Intrinsic::maximum:
1221  return TargetOpcode::G_FMAXIMUM;
1222  case Intrinsic::canonicalize:
1223  return TargetOpcode::G_FCANONICALIZE;
1224  case Intrinsic::floor:
1225  return TargetOpcode::G_FFLOOR;
1226  case Intrinsic::fma:
1227  return TargetOpcode::G_FMA;
1228  case Intrinsic::log:
1229  return TargetOpcode::G_FLOG;
1230  case Intrinsic::log2:
1231  return TargetOpcode::G_FLOG2;
1232  case Intrinsic::log10:
1233  return TargetOpcode::G_FLOG10;
1234  case Intrinsic::nearbyint:
1235  return TargetOpcode::G_FNEARBYINT;
1236  case Intrinsic::pow:
1237  return TargetOpcode::G_FPOW;
1238  case Intrinsic::rint:
1239  return TargetOpcode::G_FRINT;
1240  case Intrinsic::round:
1241  return TargetOpcode::G_INTRINSIC_ROUND;
1242  case Intrinsic::sin:
1243  return TargetOpcode::G_FSIN;
1244  case Intrinsic::sqrt:
1245  return TargetOpcode::G_FSQRT;
1246  case Intrinsic::trunc:
1247  return TargetOpcode::G_INTRINSIC_TRUNC;
1248  }
1249  return Intrinsic::not_intrinsic;
1250 }
1251 
1252 bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
1253  Intrinsic::ID ID,
1254  MachineIRBuilder &MIRBuilder) {
1255 
1256  unsigned Op = getSimpleIntrinsicOpcode(ID);
1257 
1258  // Is this a simple intrinsic?
1259  if (Op == Intrinsic::not_intrinsic)
1260  return false;
1261 
1262  // Yes. Let's translate it.
1264  for (auto &Arg : CI.arg_operands())
1265  VRegs.push_back(getOrCreateVReg(*Arg));
1266 
1267  MIRBuilder.buildInstr(Op, {getOrCreateVReg(CI)}, VRegs,
1269  return true;
1270 }
1271 
1272 bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
1273  MachineIRBuilder &MIRBuilder) {
1274 
1275  // If this is a simple intrinsic (that is, we just need to add a def of
1276  // a vreg, and uses for each arg operand, then translate it.
1277  if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
1278  return true;
1279 
1280  switch (ID) {
1281  default:
1282  break;
1283  case Intrinsic::lifetime_start:
1284  case Intrinsic::lifetime_end: {
1285  // No stack colouring in O0, discard region information.
1286  if (MF->getTarget().getOptLevel() == CodeGenOpt::None)
1287  return true;
1288 
1289  unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
1291 
1292  // Get the underlying objects for the location passed on the lifetime
1293  // marker.
1295  GetUnderlyingObjects(CI.getArgOperand(1), Allocas, *DL);
1296 
1297  // Iterate over each underlying object, creating lifetime markers for each
1298  // static alloca. Quit if we find a non-static alloca.
1299  for (const Value *V : Allocas) {
1300  const AllocaInst *AI = dyn_cast<AllocaInst>(V);
1301  if (!AI)
1302  continue;
1303 
1304  if (!AI->isStaticAlloca())
1305  return true;
1306 
1307  MIRBuilder.buildInstr(Op).addFrameIndex(getOrCreateFrameIndex(*AI));
1308  }
1309  return true;
1310  }
1311  case Intrinsic::dbg_declare: {
1312  const DbgDeclareInst &DI = cast<DbgDeclareInst>(CI);
1313  assert(DI.getVariable() && "Missing variable");
1314 
1315  const Value *Address = DI.getAddress();
1316  if (!Address || isa<UndefValue>(Address)) {
1317  LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
1318  return true;
1319  }
1320 
1322  MIRBuilder.getDebugLoc()) &&
1323  "Expected inlined-at fields to agree");
1324  auto AI = dyn_cast<AllocaInst>(Address);
1325  if (AI && AI->isStaticAlloca()) {
1326  // Static allocas are tracked at the MF level, no need for DBG_VALUE
1327  // instructions (in fact, they get ignored if they *do* exist).
1328  MF->setVariableDbgInfo(DI.getVariable(), DI.getExpression(),
1329  getOrCreateFrameIndex(*AI), DI.getDebugLoc());
1330  } else {
1331  // A dbg.declare describes the address of a source variable, so lower it
1332  // into an indirect DBG_VALUE.
1333  MIRBuilder.buildIndirectDbgValue(getOrCreateVReg(*Address),
1334  DI.getVariable(), DI.getExpression());
1335  }
1336  return true;
1337  }
1338  case Intrinsic::dbg_label: {
1339  const DbgLabelInst &DI = cast<DbgLabelInst>(CI);
1340  assert(DI.getLabel() && "Missing label");
1341 
1343  MIRBuilder.getDebugLoc()) &&
1344  "Expected inlined-at fields to agree");
1345 
1346  MIRBuilder.buildDbgLabel(DI.getLabel());
1347  return true;
1348  }
1349  case Intrinsic::vaend:
1350  // No target I know of cares about va_end. Certainly no in-tree target
1351  // does. Simplest intrinsic ever!
1352  return true;
1353  case Intrinsic::vastart: {
1354  auto &TLI = *MF->getSubtarget().getTargetLowering();
1355  Value *Ptr = CI.getArgOperand(0);
1356  unsigned ListSize = TLI.getVaListSizeInBits(*DL) / 8;
1357 
1358  // FIXME: Get alignment
1359  MIRBuilder.buildInstr(TargetOpcode::G_VASTART)
1360  .addUse(getOrCreateVReg(*Ptr))
1361  .addMemOperand(MF->getMachineMemOperand(
1362  MachinePointerInfo(Ptr), MachineMemOperand::MOStore, ListSize, 1));
1363  return true;
1364  }
1365  case Intrinsic::dbg_value: {
1366  // This form of DBG_VALUE is target-independent.
1367  const DbgValueInst &DI = cast<DbgValueInst>(CI);
1368  const Value *V = DI.getValue();
1370  MIRBuilder.getDebugLoc()) &&
1371  "Expected inlined-at fields to agree");
1372  if (!V) {
1373  // Currently the optimizer can produce this; insert an undef to
1374  // help debugging. Probably the optimizer should not do this.
1375  MIRBuilder.buildIndirectDbgValue(0, DI.getVariable(), DI.getExpression());
1376  } else if (const auto *CI = dyn_cast<Constant>(V)) {
1377  MIRBuilder.buildConstDbgValue(*CI, DI.getVariable(), DI.getExpression());
1378  } else {
1379  for (Register Reg : getOrCreateVRegs(*V)) {
1380  // FIXME: This does not handle register-indirect values at offset 0. The
1381  // direct/indirect thing shouldn't really be handled by something as
1382  // implicit as reg+noreg vs reg+imm in the first place, but it seems
1383  // pretty baked in right now.
1384  MIRBuilder.buildDirectDbgValue(Reg, DI.getVariable(), DI.getExpression());
1385  }
1386  }
1387  return true;
1388  }
1389  case Intrinsic::uadd_with_overflow:
1390  return translateOverflowIntrinsic(CI, TargetOpcode::G_UADDO, MIRBuilder);
1391  case Intrinsic::sadd_with_overflow:
1392  return translateOverflowIntrinsic(CI, TargetOpcode::G_SADDO, MIRBuilder);
1393  case Intrinsic::usub_with_overflow:
1394  return translateOverflowIntrinsic(CI, TargetOpcode::G_USUBO, MIRBuilder);
1395  case Intrinsic::ssub_with_overflow:
1396  return translateOverflowIntrinsic(CI, TargetOpcode::G_SSUBO, MIRBuilder);
1397  case Intrinsic::umul_with_overflow:
1398  return translateOverflowIntrinsic(CI, TargetOpcode::G_UMULO, MIRBuilder);
1399  case Intrinsic::smul_with_overflow:
1400  return translateOverflowIntrinsic(CI, TargetOpcode::G_SMULO, MIRBuilder);
1401  case Intrinsic::fmuladd: {
1402  const TargetMachine &TM = MF->getTarget();
1403  const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
1404  Register Dst = getOrCreateVReg(CI);
1405  Register Op0 = getOrCreateVReg(*CI.getArgOperand(0));
1406  Register Op1 = getOrCreateVReg(*CI.getArgOperand(1));
1407  Register Op2 = getOrCreateVReg(*CI.getArgOperand(2));
1409  TLI.isFMAFasterThanFMulAndFAdd(TLI.getValueType(*DL, CI.getType()))) {
1410  // TODO: Revisit this to see if we should move this part of the
1411  // lowering to the combiner.
1412  MIRBuilder.buildInstr(TargetOpcode::G_FMA, {Dst}, {Op0, Op1, Op2},
1414  } else {
1415  LLT Ty = getLLTForType(*CI.getType(), *DL);
1416  auto FMul = MIRBuilder.buildInstr(TargetOpcode::G_FMUL, {Ty}, {Op0, Op1},
1418  MIRBuilder.buildInstr(TargetOpcode::G_FADD, {Dst}, {FMul, Op2},
1420  }
1421  return true;
1422  }
1423  case Intrinsic::memcpy:
1424  case Intrinsic::memmove:
1425  case Intrinsic::memset:
1426  return translateMemFunc(CI, MIRBuilder, ID);
1427  case Intrinsic::eh_typeid_for: {
1429  Register Reg = getOrCreateVReg(CI);
1430  unsigned TypeID = MF->getTypeIDFor(GV);
1431  MIRBuilder.buildConstant(Reg, TypeID);
1432  return true;
1433  }
1434  case Intrinsic::objectsize: {
1435  // If we don't know by now, we're never going to know.
1436  const ConstantInt *Min = cast<ConstantInt>(CI.getArgOperand(1));
1437 
1438  MIRBuilder.buildConstant(getOrCreateVReg(CI), Min->isZero() ? -1ULL : 0);
1439  return true;
1440  }
1441  case Intrinsic::is_constant:
1442  // If this wasn't constant-folded away by now, then it's not a
1443  // constant.
1444  MIRBuilder.buildConstant(getOrCreateVReg(CI), 0);
1445  return true;
1446  case Intrinsic::stackguard:
1447  getStackGuard(getOrCreateVReg(CI), MIRBuilder);
1448  return true;
1449  case Intrinsic::stackprotector: {
1450  LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
1451  Register GuardVal = MRI->createGenericVirtualRegister(PtrTy);
1452  getStackGuard(GuardVal, MIRBuilder);
1453 
1454  AllocaInst *Slot = cast<AllocaInst>(CI.getArgOperand(1));
1455  int FI = getOrCreateFrameIndex(*Slot);
1456  MF->getFrameInfo().setStackProtectorIndex(FI);
1457 
1458  MIRBuilder.buildStore(
1459  GuardVal, getOrCreateVReg(*Slot),
1460  *MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(*MF, FI),
1463  PtrTy.getSizeInBits() / 8, 8));
1464  return true;
1465  }
1466  case Intrinsic::stacksave: {
1467  // Save the stack pointer to the location provided by the intrinsic.
1468  Register Reg = getOrCreateVReg(CI);
1469  Register StackPtr = MF->getSubtarget()
1470  .getTargetLowering()
1471  ->getStackPointerRegisterToSaveRestore();
1472 
1473  // If the target doesn't specify a stack pointer, then fall back.
1474  if (!StackPtr)
1475  return false;
1476 
1477  MIRBuilder.buildCopy(Reg, StackPtr);
1478  return true;
1479  }
1480  case Intrinsic::stackrestore: {
1481  // Restore the stack pointer from the location provided by the intrinsic.
1482  Register Reg = getOrCreateVReg(*CI.getArgOperand(0));
1483  Register StackPtr = MF->getSubtarget()
1484  .getTargetLowering()
1485  ->getStackPointerRegisterToSaveRestore();
1486 
1487  // If the target doesn't specify a stack pointer, then fall back.
1488  if (!StackPtr)
1489  return false;
1490 
1491  MIRBuilder.buildCopy(StackPtr, Reg);
1492  return true;
1493  }
1494  case Intrinsic::cttz:
1495  case Intrinsic::ctlz: {
1496  ConstantInt *Cst = cast<ConstantInt>(CI.getArgOperand(1));
1497  bool isTrailing = ID == Intrinsic::cttz;
1498  unsigned Opcode = isTrailing
1499  ? Cst->isZero() ? TargetOpcode::G_CTTZ
1500  : TargetOpcode::G_CTTZ_ZERO_UNDEF
1501  : Cst->isZero() ? TargetOpcode::G_CTLZ
1502  : TargetOpcode::G_CTLZ_ZERO_UNDEF;
1503  MIRBuilder.buildInstr(Opcode)
1504  .addDef(getOrCreateVReg(CI))
1505  .addUse(getOrCreateVReg(*CI.getArgOperand(0)));
1506  return true;
1507  }
1508  case Intrinsic::invariant_start: {
1509  LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
1510  Register Undef = MRI->createGenericVirtualRegister(PtrTy);
1511  MIRBuilder.buildUndef(Undef);
1512  return true;
1513  }
1514  case Intrinsic::invariant_end:
1515  return true;
1516  case Intrinsic::assume:
1517  case Intrinsic::var_annotation:
1518  case Intrinsic::sideeffect:
1519  // Discard annotate attributes, assumptions, and artificial side-effects.
1520  return true;
1521  }
1522  return false;
1523 }
1524 
1525 bool IRTranslator::translateInlineAsm(const CallInst &CI,
1526  MachineIRBuilder &MIRBuilder) {
1527  const InlineAsm &IA = cast<InlineAsm>(*CI.getCalledValue());
1528  if (!IA.getConstraintString().empty())
1529  return false;
1530 
1531  unsigned ExtraInfo = 0;
1532  if (IA.hasSideEffects())
1533  ExtraInfo |= InlineAsm::Extra_HasSideEffects;
1534  if (IA.getDialect() == InlineAsm::AD_Intel)
1535  ExtraInfo |= InlineAsm::Extra_AsmDialect;
1536 
1538  .addExternalSymbol(IA.getAsmString().c_str())
1539  .addImm(ExtraInfo);
1540 
1541  return true;
1542 }
1543 
1544 bool IRTranslator::translateCallSite(const ImmutableCallSite &CS,
1545  MachineIRBuilder &MIRBuilder) {
1546  const Instruction &I = *CS.getInstruction();
1547  ArrayRef<Register> Res = getOrCreateVRegs(I);
1548 
1550  Register SwiftInVReg = 0;
1551  Register SwiftErrorVReg = 0;
1552  for (auto &Arg : CS.args()) {
1553  if (CLI->supportSwiftError() && isSwiftError(Arg)) {
1554  assert(SwiftInVReg == 0 && "Expected only one swift error argument");
1555  LLT Ty = getLLTForType(*Arg->getType(), *DL);
1556  SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
1557  MIRBuilder.buildCopy(SwiftInVReg, SwiftError.getOrCreateVRegUseAt(
1558  &I, &MIRBuilder.getMBB(), Arg));
1559  Args.emplace_back(makeArrayRef(SwiftInVReg));
1560  SwiftErrorVReg =
1561  SwiftError.getOrCreateVRegDefAt(&I, &MIRBuilder.getMBB(), Arg);
1562  continue;
1563  }
1564  Args.push_back(getOrCreateVRegs(*Arg));
1565  }
1566 
1567  MF->getFrameInfo().setHasCalls(true);
1568  bool Success =
1569  CLI->lowerCall(MIRBuilder, CS, Res, Args, SwiftErrorVReg,
1570  [&]() { return getOrCreateVReg(*CS.getCalledValue()); });
1571 
1572  // Check if we just inserted a tail call.
1573  if (Success) {
1574  assert(!HasTailCall && "Can't tail call return twice from block?");
1575  const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
1576  HasTailCall = TII->isTailCall(*std::prev(MIRBuilder.getInsertPt()));
1577  }
1578 
1579  return Success;
1580 }
1581 
1582 bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
1583  const CallInst &CI = cast<CallInst>(U);
1584  auto TII = MF->getTarget().getIntrinsicInfo();
1585  const Function *F = CI.getCalledFunction();
1586 
1587  // FIXME: support Windows dllimport function calls.
1588  if (F && F->hasDLLImportStorageClass())
1589  return false;
1590 
1591  if (CI.isInlineAsm())
1592  return translateInlineAsm(CI, MIRBuilder);
1593 
1595  if (F && F->isIntrinsic()) {
1596  ID = F->getIntrinsicID();
1597  if (TII && ID == Intrinsic::not_intrinsic)
1598  ID = static_cast<Intrinsic::ID>(TII->getIntrinsicID(F));
1599  }
1600 
1601  if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic)
1602  return translateCallSite(&CI, MIRBuilder);
1603 
1604  assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
1605 
1606  if (translateKnownIntrinsic(CI, ID, MIRBuilder))
1607  return true;
1608 
1609  ArrayRef<Register> ResultRegs;
1610  if (!CI.getType()->isVoidTy())
1611  ResultRegs = getOrCreateVRegs(CI);
1612 
1613  // Ignore the callsite attributes. Backend code is most likely not expecting
1614  // an intrinsic to sometimes have side effects and sometimes not.
1615  MachineInstrBuilder MIB =
1616  MIRBuilder.buildIntrinsic(ID, ResultRegs, !F->doesNotAccessMemory());
1617  if (isa<FPMathOperator>(CI))
1618  MIB->copyIRFlags(CI);
1619 
1620  for (auto &Arg : CI.arg_operands()) {
1621  // Some intrinsics take metadata parameters. Reject them.
1622  if (isa<MetadataAsValue>(Arg))
1623  return false;
1624  ArrayRef<Register> VRegs = getOrCreateVRegs(*Arg);
1625  if (VRegs.size() > 1)
1626  return false;
1627  MIB.addUse(VRegs[0]);
1628  }
1629 
1630  // Add a MachineMemOperand if it is a target mem intrinsic.
1631  const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
1632  TargetLowering::IntrinsicInfo Info;
1633  // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
1634  if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) {
1635  MaybeAlign Align = Info.align;
1636  if (!Align)
1637  Align = MaybeAlign(
1638  DL->getABITypeAlignment(Info.memVT.getTypeForEVT(F->getContext())));
1639 
1640  uint64_t Size = Info.memVT.getStoreSize();
1641  MIB.addMemOperand(MF->getMachineMemOperand(
1642  MachinePointerInfo(Info.ptrVal), Info.flags, Size, Align->value()));
1643  }
1644 
1645  return true;
1646 }
1647 
1648 bool IRTranslator::translateInvoke(const User &U,
1649  MachineIRBuilder &MIRBuilder) {
1650  const InvokeInst &I = cast<InvokeInst>(U);
1651  MCContext &Context = MF->getContext();
1652 
1653  const BasicBlock *ReturnBB = I.getSuccessor(0);
1654  const BasicBlock *EHPadBB = I.getSuccessor(1);
1655 
1656  const Value *Callee = I.getCalledValue();
1657  const Function *Fn = dyn_cast<Function>(Callee);
1658  if (isa<InlineAsm>(Callee))
1659  return false;
1660 
1661  // FIXME: support invoking patchpoint and statepoint intrinsics.
1662  if (Fn && Fn->isIntrinsic())
1663  return false;
1664 
1665  // FIXME: support whatever these are.
1667  return false;
1668 
1669  // FIXME: support Windows exception handling.
1670  if (!isa<LandingPadInst>(EHPadBB->front()))
1671  return false;
1672 
1673  // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
1674  // the region covered by the try.
1675  MCSymbol *BeginSymbol = Context.createTempSymbol();
1676  MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol);
1677 
1678  if (!translateCallSite(&I, MIRBuilder))
1679  return false;
1680 
1681  MCSymbol *EndSymbol = Context.createTempSymbol();
1682  MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(EndSymbol);
1683 
1684  // FIXME: track probabilities.
1685  MachineBasicBlock &EHPadMBB = getMBB(*EHPadBB),
1686  &ReturnMBB = getMBB(*ReturnBB);
1687  MF->addInvoke(&EHPadMBB, BeginSymbol, EndSymbol);
1688  MIRBuilder.getMBB().addSuccessor(&ReturnMBB);
1689  MIRBuilder.getMBB().addSuccessor(&EHPadMBB);
1690  MIRBuilder.buildBr(ReturnMBB);
1691 
1692  return true;
1693 }
1694 
1695 bool IRTranslator::translateCallBr(const User &U,
1696  MachineIRBuilder &MIRBuilder) {
1697  // FIXME: Implement this.
1698  return false;
1699 }
1700 
1701 bool IRTranslator::translateLandingPad(const User &U,
1702  MachineIRBuilder &MIRBuilder) {
1703  const LandingPadInst &LP = cast<LandingPadInst>(U);
1704 
1705  MachineBasicBlock &MBB = MIRBuilder.getMBB();
1706 
1707  MBB.setIsEHPad();
1708 
1709  // If there aren't registers to copy the values into (e.g., during SjLj
1710  // exceptions), then don't bother.
1711  auto &TLI = *MF->getSubtarget().getTargetLowering();
1712  const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
1713  if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
1714  TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
1715  return true;
1716 
1717  // If landingpad's return type is token type, we don't create DAG nodes
1718  // for its exception pointer and selector value. The extraction of exception
1719  // pointer or selector value from token type landingpads is not currently
1720  // supported.
1721  if (LP.getType()->isTokenTy())
1722  return true;
1723 
1724  // Add a label to mark the beginning of the landing pad. Deletion of the
1725  // landing pad can thus be detected via the MachineModuleInfo.
1727  .addSym(MF->addLandingPad(&MBB));
1728 
1729  LLT Ty = getLLTForType(*LP.getType(), *DL);
1730  Register Undef = MRI->createGenericVirtualRegister(Ty);
1731  MIRBuilder.buildUndef(Undef);
1732 
1733  SmallVector<LLT, 2> Tys;
1734  for (Type *Ty : cast<StructType>(LP.getType())->elements())
1735  Tys.push_back(getLLTForType(*Ty, *DL));
1736  assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
1737 
1738  // Mark exception register as live in.
1739  Register ExceptionReg = TLI.getExceptionPointerRegister(PersonalityFn);
1740  if (!ExceptionReg)
1741  return false;
1742 
1743  MBB.addLiveIn(ExceptionReg);
1744  ArrayRef<Register> ResRegs = getOrCreateVRegs(LP);
1745  MIRBuilder.buildCopy(ResRegs[0], ExceptionReg);
1746 
1747  Register SelectorReg = TLI.getExceptionSelectorRegister(PersonalityFn);
1748  if (!SelectorReg)
1749  return false;
1750 
1751  MBB.addLiveIn(SelectorReg);
1752  Register PtrVReg = MRI->createGenericVirtualRegister(Tys[0]);
1753  MIRBuilder.buildCopy(PtrVReg, SelectorReg);
1754  MIRBuilder.buildCast(ResRegs[1], PtrVReg);
1755 
1756  return true;
1757 }
1758 
1759 bool IRTranslator::translateAlloca(const User &U,
1760  MachineIRBuilder &MIRBuilder) {
1761  auto &AI = cast<AllocaInst>(U);
1762 
1763  if (AI.isSwiftError())
1764  return true;
1765 
1766  if (AI.isStaticAlloca()) {
1767  Register Res = getOrCreateVReg(AI);
1768  int FI = getOrCreateFrameIndex(AI);
1769  MIRBuilder.buildFrameIndex(Res, FI);
1770  return true;
1771  }
1772 
1773  // FIXME: support stack probing for Windows.
1774  if (MF->getTarget().getTargetTriple().isOSWindows())
1775  return false;
1776 
1777  // Now we're in the harder dynamic case.
1778  Type *Ty = AI.getAllocatedType();
1779  unsigned Align =
1780  std::max((unsigned)DL->getPrefTypeAlignment(Ty), AI.getAlignment());
1781 
1782  Register NumElts = getOrCreateVReg(*AI.getArraySize());
1783 
1784  Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
1785  LLT IntPtrTy = getLLTForType(*IntPtrIRTy, *DL);
1786  if (MRI->getType(NumElts) != IntPtrTy) {
1787  Register ExtElts = MRI->createGenericVirtualRegister(IntPtrTy);
1788  MIRBuilder.buildZExtOrTrunc(ExtElts, NumElts);
1789  NumElts = ExtElts;
1790  }
1791 
1792  Register AllocSize = MRI->createGenericVirtualRegister(IntPtrTy);
1793  Register TySize =
1794  getOrCreateVReg(*ConstantInt::get(IntPtrIRTy, DL->getTypeAllocSize(Ty)));
1795  MIRBuilder.buildMul(AllocSize, NumElts, TySize);
1796 
1797  unsigned StackAlign =
1798  MF->getSubtarget().getFrameLowering()->getStackAlignment();
1799  if (Align <= StackAlign)
1800  Align = 0;
1801 
1802  // Round the size of the allocation up to the stack alignment size
1803  // by add SA-1 to the size. This doesn't overflow because we're computing
1804  // an address inside an alloca.
1805  auto SAMinusOne = MIRBuilder.buildConstant(IntPtrTy, StackAlign - 1);
1806  auto AllocAdd = MIRBuilder.buildAdd(IntPtrTy, AllocSize, SAMinusOne,
1808  auto AlignCst =
1809  MIRBuilder.buildConstant(IntPtrTy, ~(uint64_t)(StackAlign - 1));
1810  auto AlignedAlloc = MIRBuilder.buildAnd(IntPtrTy, AllocAdd, AlignCst);
1811 
1812  MIRBuilder.buildDynStackAlloc(getOrCreateVReg(AI), AlignedAlloc, Align);
1813 
1814  MF->getFrameInfo().CreateVariableSizedObject(Align ? Align : 1, &AI);
1815  assert(MF->getFrameInfo().hasVarSizedObjects());
1816  return true;
1817 }
1818 
1819 bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {
1820  // FIXME: We may need more info about the type. Because of how LLT works,
1821  // we're completely discarding the i64/double distinction here (amongst
1822  // others). Fortunately the ABIs I know of where that matters don't use va_arg
1823  // anyway but that's not guaranteed.
1824  MIRBuilder.buildInstr(TargetOpcode::G_VAARG)
1825  .addDef(getOrCreateVReg(U))
1826  .addUse(getOrCreateVReg(*U.getOperand(0)))
1827  .addImm(DL->getABITypeAlignment(U.getType()));
1828  return true;
1829 }
1830 
1831 bool IRTranslator::translateInsertElement(const User &U,
1832  MachineIRBuilder &MIRBuilder) {
1833  // If it is a <1 x Ty> vector, use the scalar as it is
1834  // not a legal vector type in LLT.
1835  if (U.getType()->getVectorNumElements() == 1) {
1836  Register Elt = getOrCreateVReg(*U.getOperand(1));
1837  auto &Regs = *VMap.getVRegs(U);
1838  if (Regs.empty()) {
1839  Regs.push_back(Elt);
1840  VMap.getOffsets(U)->push_back(0);
1841  } else {
1842  MIRBuilder.buildCopy(Regs[0], Elt);
1843  }
1844  return true;
1845  }
1846 
1847  Register Res = getOrCreateVReg(U);
1848  Register Val = getOrCreateVReg(*U.getOperand(0));
1849  Register Elt = getOrCreateVReg(*U.getOperand(1));
1850  Register Idx = getOrCreateVReg(*U.getOperand(2));
1851  MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
1852  return true;
1853 }
1854 
1855 bool IRTranslator::translateExtractElement(const User &U,
1856  MachineIRBuilder &MIRBuilder) {
1857  // If it is a <1 x Ty> vector, use the scalar as it is
1858  // not a legal vector type in LLT.
1859  if (U.getOperand(0)->getType()->getVectorNumElements() == 1) {
1860  Register Elt = getOrCreateVReg(*U.getOperand(0));
1861  auto &Regs = *VMap.getVRegs(U);
1862  if (Regs.empty()) {
1863  Regs.push_back(Elt);
1864  VMap.getOffsets(U)->push_back(0);
1865  } else {
1866  MIRBuilder.buildCopy(Regs[0], Elt);
1867  }
1868  return true;
1869  }
1870  Register Res = getOrCreateVReg(U);
1871  Register Val = getOrCreateVReg(*U.getOperand(0));
1872  const auto &TLI = *MF->getSubtarget().getTargetLowering();
1873  unsigned PreferredVecIdxWidth = TLI.getVectorIdxTy(*DL).getSizeInBits();
1874  Register Idx;
1875  if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(1))) {
1876  if (CI->getBitWidth() != PreferredVecIdxWidth) {
1877  APInt NewIdx = CI->getValue().sextOrTrunc(PreferredVecIdxWidth);
1878  auto *NewIdxCI = ConstantInt::get(CI->getContext(), NewIdx);
1879  Idx = getOrCreateVReg(*NewIdxCI);
1880  }
1881  }
1882  if (!Idx)
1883  Idx = getOrCreateVReg(*U.getOperand(1));
1884  if (MRI->getType(Idx).getSizeInBits() != PreferredVecIdxWidth) {
1885  const LLT &VecIdxTy = LLT::scalar(PreferredVecIdxWidth);
1886  Idx = MIRBuilder.buildSExtOrTrunc(VecIdxTy, Idx)->getOperand(0).getReg();
1887  }
1888  MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
1889  return true;
1890 }
1891 
1892 bool IRTranslator::translateShuffleVector(const User &U,
1893  MachineIRBuilder &MIRBuilder) {
1894  MIRBuilder.buildInstr(TargetOpcode::G_SHUFFLE_VECTOR)
1895  .addDef(getOrCreateVReg(U))
1896  .addUse(getOrCreateVReg(*U.getOperand(0)))
1897  .addUse(getOrCreateVReg(*U.getOperand(1)))
1898  .addShuffleMask(cast<Constant>(U.getOperand(2)));
1899  return true;
1900 }
1901 
1902 bool IRTranslator::translatePHI(const User &U, MachineIRBuilder &MIRBuilder) {
1903  const PHINode &PI = cast<PHINode>(U);
1904 
1906  for (auto Reg : getOrCreateVRegs(PI)) {
1907  auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_PHI, {Reg}, {});
1908  Insts.push_back(MIB.getInstr());
1909  }
1910 
1911  PendingPHIs.emplace_back(&PI, std::move(Insts));
1912  return true;
1913 }
1914 
1915 bool IRTranslator::translateAtomicCmpXchg(const User &U,
1916  MachineIRBuilder &MIRBuilder) {
1917  const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(U);
1918 
1919  if (I.isWeak())
1920  return false;
1921 
1922  auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile
1925 
1926  Type *ResType = I.getType();
1927  Type *ValType = ResType->Type::getStructElementType(0);
1928 
1929  auto Res = getOrCreateVRegs(I);
1930  Register OldValRes = Res[0];
1931  Register SuccessRes = Res[1];
1932  Register Addr = getOrCreateVReg(*I.getPointerOperand());
1933  Register Cmp = getOrCreateVReg(*I.getCompareOperand());
1934  Register NewVal = getOrCreateVReg(*I.getNewValOperand());
1935 
1936  MIRBuilder.buildAtomicCmpXchgWithSuccess(
1937  OldValRes, SuccessRes, Addr, Cmp, NewVal,
1938  *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
1939  Flags, DL->getTypeStoreSize(ValType),
1940  getMemOpAlignment(I), AAMDNodes(), nullptr,
1942  I.getFailureOrdering()));
1943  return true;
1944 }
1945 
1946 bool IRTranslator::translateAtomicRMW(const User &U,
1947  MachineIRBuilder &MIRBuilder) {
1948  const AtomicRMWInst &I = cast<AtomicRMWInst>(U);
1949 
1950  auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile
1953 
1954  Type *ResType = I.getType();
1955 
1956  Register Res = getOrCreateVReg(I);
1957  Register Addr = getOrCreateVReg(*I.getPointerOperand());
1958  Register Val = getOrCreateVReg(*I.getValOperand());
1959 
1960  unsigned Opcode = 0;
1961  switch (I.getOperation()) {
1962  default:
1963  return false;
1964  case AtomicRMWInst::Xchg:
1965  Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
1966  break;
1967  case AtomicRMWInst::Add:
1968  Opcode = TargetOpcode::G_ATOMICRMW_ADD;
1969  break;
1970  case AtomicRMWInst::Sub:
1971  Opcode = TargetOpcode::G_ATOMICRMW_SUB;
1972  break;
1973  case AtomicRMWInst::And:
1974  Opcode = TargetOpcode::G_ATOMICRMW_AND;
1975  break;
1976  case AtomicRMWInst::Nand:
1977  Opcode = TargetOpcode::G_ATOMICRMW_NAND;
1978  break;
1979  case AtomicRMWInst::Or:
1980  Opcode = TargetOpcode::G_ATOMICRMW_OR;
1981  break;
1982  case AtomicRMWInst::Xor:
1983  Opcode = TargetOpcode::G_ATOMICRMW_XOR;
1984  break;
1985  case AtomicRMWInst::Max:
1986  Opcode = TargetOpcode::G_ATOMICRMW_MAX;
1987  break;
1988  case AtomicRMWInst::Min:
1989  Opcode = TargetOpcode::G_ATOMICRMW_MIN;
1990  break;
1991  case AtomicRMWInst::UMax:
1992  Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
1993  break;
1994  case AtomicRMWInst::UMin:
1995  Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
1996  break;
1997  case AtomicRMWInst::FAdd:
1998  Opcode = TargetOpcode::G_ATOMICRMW_FADD;
1999  break;
2000  case AtomicRMWInst::FSub:
2001  Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
2002  break;
2003  }
2004 
2005  MIRBuilder.buildAtomicRMW(
2006  Opcode, Res, Addr, Val,
2007  *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
2008  Flags, DL->getTypeStoreSize(ResType),
2009  getMemOpAlignment(I), AAMDNodes(), nullptr,
2010  I.getSyncScopeID(), I.getOrdering()));
2011  return true;
2012 }
2013 
2014 bool IRTranslator::translateFence(const User &U,
2015  MachineIRBuilder &MIRBuilder) {
2016  const FenceInst &Fence = cast<FenceInst>(U);
2017  MIRBuilder.buildFence(static_cast<unsigned>(Fence.getOrdering()),
2018  Fence.getSyncScopeID());
2019  return true;
2020 }
2021 
2022 void IRTranslator::finishPendingPhis() {
2023 #ifndef NDEBUG
2024  DILocationVerifier Verifier;
2025  GISelObserverWrapper WrapperObserver(&Verifier);
2026  RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
2027 #endif // ifndef NDEBUG
2028  for (auto &Phi : PendingPHIs) {
2029  const PHINode *PI = Phi.first;
2030  ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
2031  MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
2032  EntryBuilder->setDebugLoc(PI->getDebugLoc());
2033 #ifndef NDEBUG
2034  Verifier.setCurrentInst(PI);
2035 #endif // ifndef NDEBUG
2036 
2038  for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
2039  auto IRPred = PI->getIncomingBlock(i);
2040  ArrayRef<Register> ValRegs = getOrCreateVRegs(*PI->getIncomingValue(i));
2041  for (auto Pred : getMachinePredBBs({IRPred, PI->getParent()})) {
2042  if (SeenPreds.count(Pred) || !PhiMBB->isPredecessor(Pred))
2043  continue;
2044  SeenPreds.insert(Pred);
2045  for (unsigned j = 0; j < ValRegs.size(); ++j) {
2046  MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
2047  MIB.addUse(ValRegs[j]);
2048  MIB.addMBB(Pred);
2049  }
2050  }
2051  }
2052  }
2053 }
2054 
2055 bool IRTranslator::valueIsSplit(const Value &V,
2057  SmallVector<LLT, 4> SplitTys;
2058  if (Offsets && !Offsets->empty())
2059  Offsets->clear();
2060  computeValueLLTs(*DL, *V.getType(), SplitTys, Offsets);
2061  return SplitTys.size() > 1;
2062 }
2063 
2064 bool IRTranslator::translate(const Instruction &Inst) {
2065  CurBuilder->setDebugLoc(Inst.getDebugLoc());
2066  // We only emit constants into the entry block from here. To prevent jumpy
2067  // debug behaviour set the line to 0.
2068  if (const DebugLoc &DL = Inst.getDebugLoc())
2069  EntryBuilder->setDebugLoc(
2070  DebugLoc::get(0, 0, DL.getScope(), DL.getInlinedAt()));
2071  else
2072  EntryBuilder->setDebugLoc(DebugLoc());
2073 
2074  switch (Inst.getOpcode()) {
2075 #define HANDLE_INST(NUM, OPCODE, CLASS) \
2076  case Instruction::OPCODE: \
2077  return translate##OPCODE(Inst, *CurBuilder.get());
2078 #include "llvm/IR/Instruction.def"
2079  default:
2080  return false;
2081  }
2082 }
2083 
2084 bool IRTranslator::translate(const Constant &C, Register Reg) {
2085  if (auto CI = dyn_cast<ConstantInt>(&C))
2086  EntryBuilder->buildConstant(Reg, *CI);
2087  else if (auto CF = dyn_cast<ConstantFP>(&C))
2088  EntryBuilder->buildFConstant(Reg, *CF);
2089  else if (isa<UndefValue>(C))
2090  EntryBuilder->buildUndef(Reg);
2091  else if (isa<ConstantPointerNull>(C)) {
2092  // As we are trying to build a constant val of 0 into a pointer,
2093  // insert a cast to make them correct with respect to types.
2094  unsigned NullSize = DL->getTypeSizeInBits(C.getType());
2095  auto *ZeroTy = Type::getIntNTy(C.getContext(), NullSize);
2096  auto *ZeroVal = ConstantInt::get(ZeroTy, 0);
2097  Register ZeroReg = getOrCreateVReg(*ZeroVal);
2098  EntryBuilder->buildCast(Reg, ZeroReg);
2099  } else if (auto GV = dyn_cast<GlobalValue>(&C))
2100  EntryBuilder->buildGlobalValue(Reg, GV);
2101  else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
2102  if (!CAZ->getType()->isVectorTy())
2103  return false;
2104  // Return the scalar if it is a <1 x Ty> vector.
2105  if (CAZ->getNumElements() == 1)
2106  return translate(*CAZ->getElementValue(0u), Reg);
2108  for (unsigned i = 0; i < CAZ->getNumElements(); ++i) {
2109  Constant &Elt = *CAZ->getElementValue(i);
2110  Ops.push_back(getOrCreateVReg(Elt));
2111  }
2112  EntryBuilder->buildBuildVector(Reg, Ops);
2113  } else if (auto CV = dyn_cast<ConstantDataVector>(&C)) {
2114  // Return the scalar if it is a <1 x Ty> vector.
2115  if (CV->getNumElements() == 1)
2116  return translate(*CV->getElementAsConstant(0), Reg);
2118  for (unsigned i = 0; i < CV->getNumElements(); ++i) {
2119  Constant &Elt = *CV->getElementAsConstant(i);
2120  Ops.push_back(getOrCreateVReg(Elt));
2121  }
2122  EntryBuilder->buildBuildVector(Reg, Ops);
2123  } else if (auto CE = dyn_cast<ConstantExpr>(&C)) {
2124  switch(CE->getOpcode()) {
2125 #define HANDLE_INST(NUM, OPCODE, CLASS) \
2126  case Instruction::OPCODE: \
2127  return translate##OPCODE(*CE, *EntryBuilder.get());
2128 #include "llvm/IR/Instruction.def"
2129  default:
2130  return false;
2131  }
2132  } else if (auto CV = dyn_cast<ConstantVector>(&C)) {
2133  if (CV->getNumOperands() == 1)
2134  return translate(*CV->getOperand(0), Reg);
2136  for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
2137  Ops.push_back(getOrCreateVReg(*CV->getOperand(i)));
2138  }
2139  EntryBuilder->buildBuildVector(Reg, Ops);
2140  } else if (auto *BA = dyn_cast<BlockAddress>(&C)) {
2141  EntryBuilder->buildBlockAddress(Reg, BA);
2142  } else
2143  return false;
2144 
2145  return true;
2146 }
2147 
2148 void IRTranslator::finalizeBasicBlock() {
2149  for (auto &JTCase : SL->JTCases) {
2150  // Emit header first, if it wasn't already emitted.
2151  if (!JTCase.first.Emitted)
2152  emitJumpTableHeader(JTCase.second, JTCase.first, JTCase.first.HeaderBB);
2153 
2154  emitJumpTable(JTCase.second, JTCase.second.MBB);
2155  }
2156  SL->JTCases.clear();
2157 }
2158 
2159 void IRTranslator::finalizeFunction() {
2160  // Release the memory used by the different maps we
2161  // needed during the translation.
2162  PendingPHIs.clear();
2163  VMap.reset();
2164  FrameIndices.clear();
2165  MachinePreds.clear();
2166  // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
2167  // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
2168  // destroying it twice (in ~IRTranslator() and ~LLVMContext())
2169  EntryBuilder.reset();
2170  CurBuilder.reset();
2171  FuncInfo.clear();
2172 }
2173 
2175  MF = &CurMF;
2176  const Function &F = MF->getFunction();
2177  if (F.empty())
2178  return false;
2180  getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
2181  // Set the CSEConfig and run the analysis.
2182  GISelCSEInfo *CSEInfo = nullptr;
2183  TPC = &getAnalysis<TargetPassConfig>();
2184  bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
2186  : TPC->isGISelCSEEnabled();
2187 
2188  if (EnableCSE) {
2189  EntryBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
2190  CSEInfo = &Wrapper.get(TPC->getCSEConfig());
2191  EntryBuilder->setCSEInfo(CSEInfo);
2192  CurBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
2193  CurBuilder->setCSEInfo(CSEInfo);
2194  } else {
2195  EntryBuilder = std::make_unique<MachineIRBuilder>();
2196  CurBuilder = std::make_unique<MachineIRBuilder>();
2197  }
2198  CLI = MF->getSubtarget().getCallLowering();
2199  CurBuilder->setMF(*MF);
2200  EntryBuilder->setMF(*MF);
2201  MRI = &MF->getRegInfo();
2202  DL = &F.getParent()->getDataLayout();
2203  ORE = std::make_unique<OptimizationRemarkEmitter>(&F);
2204  FuncInfo.MF = MF;
2205  FuncInfo.BPI = nullptr;
2206  const auto &TLI = *MF->getSubtarget().getTargetLowering();
2207  const TargetMachine &TM = MF->getTarget();
2208  SL = std::make_unique<GISelSwitchLowering>(this, FuncInfo);
2209  SL->init(TLI, TM, *DL);
2210 
2211  EnableOpts = TM.getOptLevel() != CodeGenOpt::None && !skipFunction(F);
2212 
2213  assert(PendingPHIs.empty() && "stale PHIs");
2214 
2215  if (!DL->isLittleEndian()) {
2216  // Currently we don't properly handle big endian code.
2217  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
2218  F.getSubprogram(), &F.getEntryBlock());
2219  R << "unable to translate in big endian mode";
2220  reportTranslationError(*MF, *TPC, *ORE, R);
2221  }
2222 
2223  // Release the per-function state when we return, whether we succeeded or not.
2224  auto FinalizeOnReturn = make_scope_exit([this]() { finalizeFunction(); });
2225 
2226  // Setup a separate basic-block for the arguments and constants
2227  MachineBasicBlock *EntryBB = MF->CreateMachineBasicBlock();
2228  MF->push_back(EntryBB);
2229  EntryBuilder->setMBB(*EntryBB);
2230 
2232  SwiftError.setFunction(CurMF);
2233  SwiftError.createEntriesInEntryBlock(DbgLoc);
2234 
2235  // Create all blocks, in IR order, to preserve the layout.
2236  for (const BasicBlock &BB: F) {
2237  auto *&MBB = BBToMBB[&BB];
2238 
2239  MBB = MF->CreateMachineBasicBlock(&BB);
2240  MF->push_back(MBB);
2241 
2242  if (BB.hasAddressTaken())
2243  MBB->setHasAddressTaken();
2244  }
2245 
2246  // Make our arguments/constants entry block fallthrough to the IR entry block.
2247  EntryBB->addSuccessor(&getMBB(F.front()));
2248 
2249  // Lower the actual args into this basic block.
2250  SmallVector<ArrayRef<Register>, 8> VRegArgs;
2251  for (const Argument &Arg: F.args()) {
2252  if (DL->getTypeStoreSize(Arg.getType()) == 0)
2253  continue; // Don't handle zero sized types.
2254  ArrayRef<Register> VRegs = getOrCreateVRegs(Arg);
2255  VRegArgs.push_back(VRegs);
2256 
2257  if (Arg.hasSwiftErrorAttr()) {
2258  assert(VRegs.size() == 1 && "Too many vregs for Swift error");
2259  SwiftError.setCurrentVReg(EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
2260  }
2261  }
2262 
2263  if (!CLI->lowerFormalArguments(*EntryBuilder.get(), F, VRegArgs)) {
2264  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
2265  F.getSubprogram(), &F.getEntryBlock());
2266  R << "unable to lower arguments: " << ore::NV("Prototype", F.getType());
2267  reportTranslationError(*MF, *TPC, *ORE, R);
2268  return false;
2269  }
2270 
2271  // Need to visit defs before uses when translating instructions.
2272  GISelObserverWrapper WrapperObserver;
2273  if (EnableCSE && CSEInfo)
2274  WrapperObserver.addObserver(CSEInfo);
2275  {
2277 #ifndef NDEBUG
2278  DILocationVerifier Verifier;
2279  WrapperObserver.addObserver(&Verifier);
2280 #endif // ifndef NDEBUG
2281  RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
2282  for (const BasicBlock *BB : RPOT) {
2283  MachineBasicBlock &MBB = getMBB(*BB);
2284  // Set the insertion point of all the following translations to
2285  // the end of this basic block.
2286  CurBuilder->setMBB(MBB);
2287  HasTailCall = false;
2288  for (const Instruction &Inst : *BB) {
2289  // If we translated a tail call in the last step, then we know
2290  // everything after the call is either a return, or something that is
2291  // handled by the call itself. (E.g. a lifetime marker or assume
2292  // intrinsic.) In this case, we should stop translating the block and
2293  // move on.
2294  if (HasTailCall)
2295  break;
2296 #ifndef NDEBUG
2297  Verifier.setCurrentInst(&Inst);
2298 #endif // ifndef NDEBUG
2299  if (translate(Inst))
2300  continue;
2301 
2302  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
2303  Inst.getDebugLoc(), BB);
2304  R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
2305 
2306  if (ORE->allowExtraAnalysis("gisel-irtranslator")) {
2307  std::string InstStrStorage;
2308  raw_string_ostream InstStr(InstStrStorage);
2309  InstStr << Inst;
2310 
2311  R << ": '" << InstStr.str() << "'";
2312  }
2313 
2314  reportTranslationError(*MF, *TPC, *ORE, R);
2315  return false;
2316  }
2317 
2318  finalizeBasicBlock();
2319  }
2320 #ifndef NDEBUG
2321  WrapperObserver.removeObserver(&Verifier);
2322 #endif
2323  }
2324 
2325  finishPendingPhis();
2326 
2327  SwiftError.propagateVRegs();
2328 
2329  // Merge the argument lowering and constants block with its single
2330  // successor, the LLVM-IR entry block. We want the basic block to
2331  // be maximal.
2332  assert(EntryBB->succ_size() == 1 &&
2333  "Custom BB used for lowering should have only one successor");
2334  // Get the successor of the current entry block.
2335  MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
2336  assert(NewEntryBB.pred_size() == 1 &&
2337  "LLVM-IR entry block has a predecessor!?");
2338  // Move all the instruction from the current entry block to the
2339  // new entry block.
2340  NewEntryBB.splice(NewEntryBB.begin(), EntryBB, EntryBB->begin(),
2341  EntryBB->end());
2342 
2343  // Update the live-in information for the new entry block.
2344  for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
2345  NewEntryBB.addLiveIn(LiveIn);
2346  NewEntryBB.sortUniqueLiveIns();
2347 
2348  // Get rid of the now empty basic block.
2349  EntryBB->removeSuccessor(&NewEntryBB);
2350  MF->remove(EntryBB);
2351  MF->DeleteMachineBasicBlock(EntryBB);
2352 
2353  assert(&MF->front() == &NewEntryBB &&
2354  "New entry wasn't next in the list of basic block!");
2355 
2356  // Initialize stack protector information.
2357  StackProtector &SP = getAnalysis<StackProtector>();
2358  SP.copyToMachineFrameInfo(MF->getFrameInfo());
2359 
2360  return false;
2361 }
auto lower_bound(R &&Range, T &&Value) -> decltype(adl_begin(Range))
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1261
MachineInstrBuilder buildDirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in Re...
uint64_t CallInst * C
Return a value (possibly void), from a function.
Value * getValueOperand()
Definition: Instructions.h:409
unsigned getNumCases() const
Return the number of &#39;cases&#39; in this switch instruction, excluding the default case.
bool isIntrinsic() const
isIntrinsic - Returns true if the function&#39;s name starts with "llvm.".
Definition: Function.h:198
A simple RAII based CSEInfo installer.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:111
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:641
MachineInstrBuilder buildJumpTable(const LLT PtrTy, unsigned JTI)
Build and insert Res = G_JUMP_TABLE JTI.
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:722
bool empty() const
Definition: Function.h:686
The CSE Analysis object.
Definition: CSEInfo.h:71
Diagnostic information for missed-optimization remarks.
DILocation * get() const
Get the underlying DILocation.
Definition: DebugLoc.cpp:21
This instruction extracts a struct member or array element value from an aggregate value...
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
*p = old <signed v ? old : v
Definition: Instructions.h:723
iterator_range< CaseIt > cases()
Iteration adapter for range-for loops.
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
This class represents an incoming formal argument to a Function.
Definition: Argument.h:29
LLVMContext & Context
DiagnosticInfoOptimizationBase::Argument NV
CaseClusterVector::iterator CaseClusterIt
This represents the llvm.dbg.label instruction.
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:288
MachineInstrBuilder buildBrCond(Register Tst, MachineBasicBlock &Dest)
Build and insert G_BRCOND Tst, Dest.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
This class represents lattice values for constants.
Definition: AllocatorList.h:23
void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU)
Modify analysis usage so it preserves passes required for the SelectionDAG fallback.
Definition: Utils.cpp:412
MachineInstrBuilder buildSExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
bool isSized(SmallPtrSetImpl< Type *> *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:264
iterator begin() const
Definition: ArrayRef.h:136
Register getReg(unsigned Idx) const
Get the register for the operand index.
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
An instruction for ordering other memory operations.
Definition: Instructions.h:454
An instruction that atomically checks whether a specified value is in a memory location, and, if it is, stores a new value there.
Definition: Instructions.h:530
void push_back(const T &Elt)
Definition: SmallVector.h:211
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:385
const MachineInstrBuilder & addShuffleMask(const Constant *Val) const
MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src)
Build and insert an appropriate cast between two registers of equal size.
IRTranslator LLVM IR static false void reportTranslationError(MachineFunction &MF, const TargetPassConfig &TPC, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R)
This class represents a function call, abstracting a target machine&#39;s calling convention.
unsigned Reg
This file contains the declarations for metadata subclasses.
Value * getCondition() const
static uint64_t round(uint64_t Acc, uint64_t Input)
Definition: xxhash.cpp:57
void setDebugLoc(const DebugLoc &DL)
Set the debug location to DL for all the next build instructions.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this store instruction.
Definition: Instructions.h:384
gep_type_iterator gep_type_end(const User *GEP)
unsigned less or equal
Definition: InstrTypes.h:758
const std::string & getAsmString() const
Definition: InlineAsm.h:80
*p = old <unsigned v ? old : v
Definition: Instructions.h:727
bool isSwiftError() const
Return true if this alloca is used as a swifterror argument to a call.
Definition: Instructions.h:135
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
Definition: Instructions.h:247
Offsets
Offsets in bytes from the start of the input buffer.
Definition: SIInstrInfo.h:1127
*p = old >unsigned v ? old : v
Definition: Instructions.h:725
LLVM_NODISCARD detail::scope_exit< typename std::decay< Callable >::type > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:58
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:733
void addSuccessorWithoutProb(MachineBasicBlock *Succ)
Add Succ as a successor of this MachineBasicBlock.
Type * getPointerOperandType() const
Definition: Instructions.h:415
unsigned getLine() const
Definition: DebugLoc.cpp:25
bool hasDLLImportStorageClass() const
Definition: GlobalValue.h:265
BasicBlock * getSuccessor(unsigned i) const
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:33
Metadata node.
Definition: Metadata.h:863
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
F(f)
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1212
The actual analysis pass wrapper.
Definition: CSEInfo.h:218
An instruction for reading from memory.
Definition: Instructions.h:167
void setMF(MachineFunction &MF)
Definition: CSEInfo.cpp:78
MachineInstrBuilder buildAtomicRMW(unsigned Opcode, const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val, MachineMemOperand &MMO)
Build and insert OldValRes<def> = G_ATOMICRMW_<Opcode> Addr, Val, MMO.
an instruction that atomically reads a memory location, combines it with another value, and then stores the result back.
Definition: Instructions.h:693
Value * getCondition() const
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:229
LLVM_READONLY APFloat maximum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2018 maximum semantics.
Definition: APFloat.h:1277
MachineInstrBuilder buildDynStackAlloc(const DstOp &Res, const SrcOp &Size, unsigned Align)
Build and insert Res = G_DYN_STACKALLOC Size, Align.
GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
Definition: Analysis.cpp:158
*p = old >signed v ? old : v
Definition: Instructions.h:721
uint64_t High
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
Optional< MachineInstrBuilder > materializeGEP(Register &Res, Register Op0, const LLT &ValueTy, uint64_t Value)
Materialize and insert Res = G_GEP Op0, (G_CONSTANT Value)
MachineInstrBuilder buildBrJT(Register TablePtr, unsigned JTI, Register IndexReg)
Build and insert G_BRJT TablePtr, JTI, IndexReg.
static Constant * getNullValue(Type *Ty)
Constructor to create a &#39;0&#39; constant of arbitrary type.
Definition: Constants.cpp:289
AtomicOrdering getFailureOrdering() const
Returns the failure ordering constraint of this cmpxchg instruction.
Definition: Instructions.h:597
bool hasSideEffects() const
Definition: InlineAsm.h:66
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1241
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:50
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Definition: Instructions.h:231
A description of a memory reference used in the backend.
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Definition: InstrTypes.h:1784
This class represents the LLVM &#39;select&#39; instruction.
const DataLayout & getDataLayout() const
Get the data layout for the module&#39;s target platform.
Definition: Module.cpp:369
#define DEBUG_TYPE
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
unsigned getAlignment() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:112
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:450
PointerType * getType() const
Overload to return most specific pointer type.
Definition: Instructions.h:96
Class to represent struct types.
Definition: DerivedTypes.h:233
ValTy * getCalledValue() const
Return the pointer to function that is being called.
Definition: CallSite.h:104
DILabel * getLabel() const
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:654
BinOp getOperation() const
Definition: Instructions.h:752
bool isWeak() const
Return true if this cmpxchg may spuriously fail.
Definition: Instructions.h:571
A cluster of cases suitable for jump table lowering.
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
LLVM_READONLY APFloat minimum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2018 minimum semantics.
Definition: APFloat.h:1264
The memory access is dereferenceable (i.e., doesn&#39;t trap).
std::vector< CaseCluster > CaseClusterVector
bool isVolatile() const
Return true if this is a cmpxchg from a volatile memory location.
Definition: Instructions.h:559
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Target-Independent Code Generator Pass Configuration Options.
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:695
Context object for machine code objects.
Definition: MCContext.h:65
InstrTy * getInstruction() const
Definition: CallSite.h:96
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:245
MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope)
Build and insert G_FENCE Ordering, Scope.
MachineInstrBuilder buildBrIndirect(Register Tgt)
Build and insert G_BRINDIRECT Tgt.
Definition: Lint.cpp:83
AtomicOrdering getSuccessOrdering() const
Returns the success ordering constraint of this cmpxchg instruction.
Definition: Instructions.h:584
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:244
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:125
iterator_range< User::op_iterator > arg_operands()
Definition: InstrTypes.h:1233
An instruction for storing to memory.
Definition: Instructions.h:320
static LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_SUB Op0, Op1.
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
MachineInstrBuilder buildExtractVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
Value * getOperand(unsigned i) const
Definition: User.h:169
Analysis containing CSE Info
Definition: CSEInfo.cpp:20
BasicBlockListType::iterator iterator
unsigned JTI
The JumpTableIndex for this jump table in the function.
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
TargetInstrInfo - Interface to description of machine instruction set.
This corresponds to the llvm.lifetime.
Definition: ISDOpcodes.h:877
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
bool doesNotAccessMemory() const
Determine if the function does not access memory.
Definition: Function.h:473
MachineInstrBuilder buildDbgLabel(const MDNode *Label)
Build and insert a DBG_LABEL instructions specifying that Label is given.
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:140
The memory access is volatile.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this label.
const BasicBlock & getEntryBlock() const
Definition: Function.h:664
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
Definition: MathExtras.h:614
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
Abstract class that contains various methods for clients to notify about changes. ...
FPOpFusion::FPOpFusionMode AllowFPOpFusion
AllowFPOpFusion - This flag is set by the -fuse-fp-ops=xxx option.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:189
void GetUnderlyingObjects(const Value *V, SmallVectorImpl< const Value *> &Objects, const DataLayout &DL, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to GetUnderlyingObject except that it can look through phi and select instruct...
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this rmw instruction.
Definition: Instructions.h:804
CodeGenOpt::Level getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
unsigned const MachineRegisterInfo * MRI
Value * getCalledValue() const
Definition: InstrTypes.h:1280
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1504
Conditional or Unconditional Branch instruction.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
Value * getAddress() const
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:148
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Definition: Constant.h:41
bool isInlineAsm() const
Check if this call is an inline asm statement.
Definition: InstrTypes.h:1356
MachineBasicBlock * MBB
The MBB into which to emit the code for the indirect jump.
Value * getValue() const
void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:134
This file contains the declarations for the subclasses of Constant, which represent the different fla...
const Instruction & front() const
Definition: BasicBlock.h:280
void computeValueLLTs(const DataLayout &DL, Type &Ty, SmallVectorImpl< LLT > &ValueTys, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
computeValueLLTs - Given an LLVM IR type, compute a sequence of LLTs that represent all the individua...
Definition: Analysis.cpp:127
Indirect Branch Instruction.
SmallVector< SwitchWorkListItem, 4 > SwitchWorkList
Helper class to build MachineInstr.
BasicBlock * getDefaultDest() const
DIExpression * getExpression() const
bool isPredecessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a predecessor of this block.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
Represent the analysis usage information of a pass.
GISelCSEInfo & get(std::unique_ptr< CSEConfigBase > CSEOpt, bool ReCompute=false)
Takes a CSEConfigBase object that defines what opcodes get CSEd.
Definition: CSEInfo.cpp:364
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:732
void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
Definition: AsmWriter.cpp:4278
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Value * getPointerOperand()
Definition: Instructions.h:284
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn&#39;t already there.
Definition: SmallSet.h:180
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
MachineInstrBuilder buildGEP(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert Res = G_GEP Op0, Op1.
const MachineInstrBuilder & addFrameIndex(int Idx) const
static double log2(double V)
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:343
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:749
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:205
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this cmpxchg instruction.
Definition: Instructions.h:610
BasicBlock * getSuccessor(unsigned i) const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
const Value * getArraySize() const
Get the number of elements allocated.
Definition: Instructions.h:92
size_t size() const
Definition: SmallVector.h:52
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
static uint16_t copyFlagsFromInstruction(const Instruction &I)
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:219
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
Definition: Instructions.h:791
Simple wrapper that does the following.
Definition: CSEInfo.h:200
This class contains a discriminated union of information about pointers in memory operands...
std::string & str()
Flushes the stream contents to the target string and returns the string&#39;s reference.
Definition: raw_ostream.h:519
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE maxNum semantics.
Definition: APFloat.h:1253
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
const std::string & getConstraintString() const
Definition: InlineAsm.h:81
iterator_range< IterTy > args() const
Definition: CallSite.h:222
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:40
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:105
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:703
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects)
Build and insert either a G_INTRINSIC (if HasSideEffects is false) or G_INTRINSIC_W_SIDE_EFFECTS inst...
void copyIRFlags(const Instruction &I)
Copy all flags to MachineInst MIFlags.
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1095
LLT getLLTForType(Type &Ty, const DataLayout &DL)
Construct a low-level type based on an LLVM type.
The memory access writes data.
MachineInstrBuilder buildBr(MachineBasicBlock &Dest)
Build and insert G_BR Dest.
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches, switches, etc.
Definition: BasicBlock.h:391
MachineInstrBuilder buildConstDbgValue(const Constant &C, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instructions specifying that Variable is given by C (suitably modified b...
struct PredInfoPair PredInfo
Value * getValOperand()
Definition: Instructions.h:817
Predicate getPredicate(unsigned Condition, unsigned Hint)
Return predicate consisting of specified condition and hint bits.
Definition: PPCPredicates.h:87
unsigned getNumOperands() const
Definition: User.h:191
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_MUL Op0, Op1.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
This is the shared class of boolean and integer constants.
Definition: Constants.h:83
This struct is a compact representation of a valid (power of two) or undefined (0) alignment...
Definition: Alignment.h:85
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
iterator end() const
Definition: ArrayRef.h:137
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition: Type.h:257
unsigned getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL)
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:643
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition: Type.cpp:179
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:653
DebugLoc getDebugLoc()
Get the current instruction&#39;s debug location.
virtual bool isTailCall(const MachineInstr &Inst) const
Determines whether Inst is a tail call instruction.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:193
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Elt, const SrcOp &Idx)
Build and insert Res = G_INSERT_VECTOR_ELT Val, Elt, Idx.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:940
unsigned getVectorNumElements() const
Definition: DerivedTypes.h:535
bool isIntPredicate() const
Definition: InstrTypes.h:825
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:55
Class for arbitrary precision integers.
Definition: APInt.h:69
amdgpu Simplify well known AMD library false FunctionCallee Callee
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
unsigned Reg
The virtual register containing the index of the jump table entry to jump to.
void setHasAddressTaken()
Set this block to reflect that it potentially is the target of an indirect branch.
static char ID
Definition: IRTranslator.h:63
MachineInstrBuilder buildAtomicCmpXchgWithSuccess(Register OldValRes, Register SuccessRes, Register Addr, Register CmpVal, Register NewVal, MachineMemOperand &MMO)
Build and insert OldValRes<def>, SuccessRes<def> = G_ATOMIC_CMPXCHG_WITH_SUCCESS Addr, CmpVal, NewVal, MMO.
virtual bool isFMAFasterThanFMulAndFAdd(EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
The memory access reads data.
#define Success
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1206
Representation of each machine instruction.
Definition: MachineInstr.h:64
unsigned succ_size(const Instruction *I)
Definition: CFG.h:256
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:807
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void addObserver(GISelChangeObserver *O)
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this fence instruction.
Definition: Instructions.h:490
bool isVolatile() const
Return true if this is a store to a volatile memory location.
Definition: Instructions.h:353
Analysis providing branch probability information.
unsigned getNumArgOperands() const
Definition: InstrTypes.h:1239
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:331
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Type * getPointerOperandType() const
Definition: Instructions.h:287
MachineBasicBlock * Default
The MBB of the default bb, which is a successor of the range check MBB.
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:175
void removeObserver(GISelChangeObserver *O)
A cluster of adjacent case labels with the same destination, or just one case.
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_ADD Op0, Op1.
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:55
AtomicOrdering getOrdering() const
Returns the ordering constraint of this store instruction.
Definition: Instructions.h:372
This represents the llvm.dbg.value instruction.
bool isTokenTy() const
Return true if this is &#39;token&#39;.
Definition: Type.h:193
verify safepoint Safepoint IR Verifier
Value * getPointerOperand()
Definition: Instructions.h:813
MachineInstrBuilder buildIndirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in me...
TargetOptions Options
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
Establish a view to a call site for examination.
Definition: CallSite.h:897
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
Definition: InstrTypes.h:1287
static cl::opt< bool > EnableCSEInIRTranslator("enable-cse-in-irtranslator", cl::desc("Should enable CSE in irtranslator"), cl::Optional, cl::init(false))
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this load instruction.
Definition: Instructions.h:259
void setMBB(MachineBasicBlock &MBB)
Set the insertion point to the end of MBB.
void push_back(MachineInstr *MI)
#define I(x, y, z)
Definition: MD5.cpp:58
static Constant * getZeroValueForNegation(Type *Ty)
Floating point negation must be implemented with f(x) = -0.0 - x.
Definition: Constants.cpp:802
Pair of physical register and lane mask.
The memory access always returns the same value (or traps).
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:192
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:332
uint32_t Size
Definition: Profile.cpp:46
DILocalVariable * getVariable() const
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
bool isUnconditional() const
AsmDialect getDialect() const
Definition: InlineAsm.h:68
Multiway switch.
This file declares the IRTranslator pass.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
void insert(iterator MBBI, MachineBasicBlock *MBB)
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:503
aarch64 promote const
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:575
LLVM Value Representation.
Definition: Value.h:73
static bool isSwiftError(const Value *V)
uint64_t getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type...
Definition: DataLayout.h:445
succ_range successors(Instruction *I)
Definition: CFG.h:259
This file describes how to lower LLVM calls to machine code calls.
INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI", false, false) INITIALIZE_PASS_END(IRTranslator
Invoke instruction.
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:65
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
IRTranslator LLVM IR MI
unsigned greater than
Definition: InstrTypes.h:755
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Definition: Pass.cpp:166
const Instruction * getFirstNonPHIOrDbg() const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic...
Definition: BasicBlock.cpp:196
AtomicOrdering getOrdering() const
Returns the ordering constraint of this fence instruction.
Definition: Instructions.h:478
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
Simple wrapper observer that takes several observers, and calls each one for each event...
Register getReg() const
getReg - Returns the register number.
bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size...
A cluster of cases suitable for bit test lowering.
#define LLVM_DEBUG(X)
Definition: Debug.h:122
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:416
OutputIt copy(R &&Range, OutputIt Out)
Definition: STLExtras.h:1217
This represents the llvm.dbg.declare instruction.
Value * getPointerOperand()
Definition: Instructions.h:412
The optimization diagnostic interface.
Statically lint checks LLVM IR
Definition: Lint.cpp:192
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE minNum semantics.
Definition: APFloat.h:1242
std::vector< MachineBasicBlock * >::iterator succ_iterator
int64_t getIndexedOffsetInType(Type *ElemTy, ArrayRef< Value *> Indices) const
Returns the offset from the beginning of the type for the specified indices.
Definition: DataLayout.cpp:803
bool isVolatile() const
Return true if this is a RMW on a volatile memory location.
Definition: Instructions.h:776
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:734
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
IntegerType * Int32Ty
This file describes how to lower LLVM code to machine code.
const BasicBlock * getParent() const
Definition: Instruction.h:66
virtual const TargetRegisterClass * getPointerRegClass(const MachineFunction &MF, unsigned Kind=0) const
Returns a TargetRegisterClass used for pointer values.
an instruction to allocate memory on the stack
Definition: Instructions.h:59
This instruction inserts a struct field of array element value into an aggregate value.
gep_type_iterator gep_type_begin(const User *GEP)
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:164