LLVM  13.0.0git
IRTranslator.cpp
Go to the documentation of this file.
1 //===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the IRTranslator class.
10 //===----------------------------------------------------------------------===//
11 
14 #include "llvm/ADT/STLExtras.h"
15 #include "llvm/ADT/ScopeExit.h"
16 #include "llvm/ADT/SmallSet.h"
17 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/Analysis/Loads.h"
22 #include "llvm/CodeGen/Analysis.h"
43 #include "llvm/IR/BasicBlock.h"
44 #include "llvm/IR/CFG.h"
45 #include "llvm/IR/Constant.h"
46 #include "llvm/IR/Constants.h"
47 #include "llvm/IR/DataLayout.h"
48 #include "llvm/IR/DebugInfo.h"
49 #include "llvm/IR/DerivedTypes.h"
50 #include "llvm/IR/Function.h"
52 #include "llvm/IR/InlineAsm.h"
53 #include "llvm/IR/InstrTypes.h"
54 #include "llvm/IR/Instructions.h"
55 #include "llvm/IR/IntrinsicInst.h"
56 #include "llvm/IR/Intrinsics.h"
57 #include "llvm/IR/LLVMContext.h"
58 #include "llvm/IR/Metadata.h"
59 #include "llvm/IR/PatternMatch.h"
60 #include "llvm/IR/Type.h"
61 #include "llvm/IR/User.h"
62 #include "llvm/IR/Value.h"
63 #include "llvm/InitializePasses.h"
64 #include "llvm/MC/MCContext.h"
65 #include "llvm/Pass.h"
66 #include "llvm/Support/Casting.h"
67 #include "llvm/Support/CodeGen.h"
68 #include "llvm/Support/Debug.h"
75 #include <algorithm>
76 #include <cassert>
77 #include <cstddef>
78 #include <cstdint>
79 #include <iterator>
80 #include <string>
81 #include <utility>
82 #include <vector>
83 
84 #define DEBUG_TYPE "irtranslator"
85 
86 using namespace llvm;
87 
88 static cl::opt<bool>
89  EnableCSEInIRTranslator("enable-cse-in-irtranslator",
90  cl::desc("Should enable CSE in irtranslator"),
91  cl::Optional, cl::init(false));
92 char IRTranslator::ID = 0;
93 
94 INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
95  false, false)
102 
104  const TargetPassConfig &TPC,
107  MF.getProperties().set(MachineFunctionProperties::Property::FailedISel);
108 
109  // Print the function name explicitly if we don't have a debug location (which
110  // makes the diagnostic less useful) or if we're going to emit a raw error.
111  if (!R.getLocation().isValid() || TPC.isGlobalISelAbortEnabled())
112  R << (" (in function: " + MF.getName() + ")").str();
113 
114  if (TPC.isGlobalISelAbortEnabled())
115  report_fatal_error(R.getMsg());
116  else
117  ORE.emit(R);
118 }
119 
121  : MachineFunctionPass(ID), OptLevel(optlevel) {}
122 
123 #ifndef NDEBUG
124 namespace {
125 /// Verify that every instruction created has the same DILocation as the
126 /// instruction being translated.
127 class DILocationVerifier : public GISelChangeObserver {
128  const Instruction *CurrInst = nullptr;
129 
130 public:
131  DILocationVerifier() = default;
132  ~DILocationVerifier() = default;
133 
134  const Instruction *getCurrentInst() const { return CurrInst; }
135  void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
136 
137  void erasingInstr(MachineInstr &MI) override {}
138  void changingInstr(MachineInstr &MI) override {}
139  void changedInstr(MachineInstr &MI) override {}
140 
141  void createdInstr(MachineInstr &MI) override {
142  assert(getCurrentInst() && "Inserted instruction without a current MI");
143 
144  // Only print the check message if we're actually checking it.
145 #ifndef NDEBUG
146  LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
147  << " was copied to " << MI);
148 #endif
149  // We allow insts in the entry block to have a debug loc line of 0 because
150  // they could have originated from constants, and we don't want a jumpy
151  // debug experience.
152  assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
153  MI.getDebugLoc().getLine() == 0) &&
154  "Line info was not transferred to all instructions");
155  }
156 };
157 } // namespace
158 #endif // ifndef NDEBUG
159 
160 
165  if (OptLevel != CodeGenOpt::None)
169 }
170 
172 IRTranslator::allocateVRegs(const Value &Val) {
173  auto VRegsIt = VMap.findVRegs(Val);
174  if (VRegsIt != VMap.vregs_end())
175  return *VRegsIt->second;
176  auto *Regs = VMap.getVRegs(Val);
177  auto *Offsets = VMap.getOffsets(Val);
178  SmallVector<LLT, 4> SplitTys;
179  computeValueLLTs(*DL, *Val.getType(), SplitTys,
180  Offsets->empty() ? Offsets : nullptr);
181  for (unsigned i = 0; i < SplitTys.size(); ++i)
182  Regs->push_back(0);
183  return *Regs;
184 }
185 
186 ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
187  auto VRegsIt = VMap.findVRegs(Val);
188  if (VRegsIt != VMap.vregs_end())
189  return *VRegsIt->second;
190 
191  if (Val.getType()->isVoidTy())
192  return *VMap.getVRegs(Val);
193 
194  // Create entry for this type.
195  auto *VRegs = VMap.getVRegs(Val);
196  auto *Offsets = VMap.getOffsets(Val);
197 
198  assert(Val.getType()->isSized() &&
199  "Don't know how to create an empty vreg");
200 
201  SmallVector<LLT, 4> SplitTys;
202  computeValueLLTs(*DL, *Val.getType(), SplitTys,
203  Offsets->empty() ? Offsets : nullptr);
204 
205  if (!isa<Constant>(Val)) {
206  for (auto Ty : SplitTys)
207  VRegs->push_back(MRI->createGenericVirtualRegister(Ty));
208  return *VRegs;
209  }
210 
211  if (Val.getType()->isAggregateType()) {
212  // UndefValue, ConstantAggregateZero
213  auto &C = cast<Constant>(Val);
214  unsigned Idx = 0;
215  while (auto Elt = C.getAggregateElement(Idx++)) {
216  auto EltRegs = getOrCreateVRegs(*Elt);
217  llvm::copy(EltRegs, std::back_inserter(*VRegs));
218  }
219  } else {
220  assert(SplitTys.size() == 1 && "unexpectedly split LLT");
221  VRegs->push_back(MRI->createGenericVirtualRegister(SplitTys[0]));
222  bool Success = translate(cast<Constant>(Val), VRegs->front());
223  if (!Success) {
224  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
225  MF->getFunction().getSubprogram(),
226  &MF->getFunction().getEntryBlock());
227  R << "unable to translate constant: " << ore::NV("Type", Val.getType());
228  reportTranslationError(*MF, *TPC, *ORE, R);
229  return *VRegs;
230  }
231  }
232 
233  return *VRegs;
234 }
235 
236 int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
237  auto MapEntry = FrameIndices.find(&AI);
238  if (MapEntry != FrameIndices.end())
239  return MapEntry->second;
240 
241  uint64_t ElementSize = DL->getTypeAllocSize(AI.getAllocatedType());
242  uint64_t Size =
243  ElementSize * cast<ConstantInt>(AI.getArraySize())->getZExtValue();
244 
245  // Always allocate at least one byte.
246  Size = std::max<uint64_t>(Size, 1u);
247 
248  int &FI = FrameIndices[&AI];
249  FI = MF->getFrameInfo().CreateStackObject(Size, AI.getAlign(), false, &AI);
250  return FI;
251 }
252 
253 Align IRTranslator::getMemOpAlign(const Instruction &I) {
254  if (const StoreInst *SI = dyn_cast<StoreInst>(&I))
255  return SI->getAlign();
256  if (const LoadInst *LI = dyn_cast<LoadInst>(&I))
257  return LI->getAlign();
258  if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I))
259  return AI->getAlign();
260  if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I))
261  return AI->getAlign();
262 
263  OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
264  R << "unable to translate memop: " << ore::NV("Opcode", &I);
265  reportTranslationError(*MF, *TPC, *ORE, R);
266  return Align(1);
267 }
268 
269 MachineBasicBlock &IRTranslator::getMBB(const BasicBlock &BB) {
270  MachineBasicBlock *&MBB = BBToMBB[&BB];
271  assert(MBB && "BasicBlock was not encountered before");
272  return *MBB;
273 }
274 
275 void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) {
276  assert(NewPred && "new predecessor must be a real MachineBasicBlock");
277  MachinePreds[Edge].push_back(NewPred);
278 }
279 
280 bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
281  MachineIRBuilder &MIRBuilder) {
282  // Get or create a virtual register for each value.
283  // Unless the value is a Constant => loadimm cst?
284  // or inline constant each time?
285  // Creation of a virtual register needs to have a size.
286  Register Op0 = getOrCreateVReg(*U.getOperand(0));
287  Register Op1 = getOrCreateVReg(*U.getOperand(1));
288  Register Res = getOrCreateVReg(U);
289  uint16_t Flags = 0;
290  if (isa<Instruction>(U)) {
291  const Instruction &I = cast<Instruction>(U);
293  }
294 
295  MIRBuilder.buildInstr(Opcode, {Res}, {Op0, Op1}, Flags);
296  return true;
297 }
298 
299 bool IRTranslator::translateUnaryOp(unsigned Opcode, const User &U,
300  MachineIRBuilder &MIRBuilder) {
301  Register Op0 = getOrCreateVReg(*U.getOperand(0));
302  Register Res = getOrCreateVReg(U);
303  uint16_t Flags = 0;
304  if (isa<Instruction>(U)) {
305  const Instruction &I = cast<Instruction>(U);
307  }
308  MIRBuilder.buildInstr(Opcode, {Res}, {Op0}, Flags);
309  return true;
310 }
311 
312 bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) {
313  return translateUnaryOp(TargetOpcode::G_FNEG, U, MIRBuilder);
314 }
315 
316 bool IRTranslator::translateCompare(const User &U,
317  MachineIRBuilder &MIRBuilder) {
318  auto *CI = dyn_cast<CmpInst>(&U);
319  Register Op0 = getOrCreateVReg(*U.getOperand(0));
320  Register Op1 = getOrCreateVReg(*U.getOperand(1));
321  Register Res = getOrCreateVReg(U);
322  CmpInst::Predicate Pred =
323  CI ? CI->getPredicate() : static_cast<CmpInst::Predicate>(
324  cast<ConstantExpr>(U).getPredicate());
325  if (CmpInst::isIntPredicate(Pred))
326  MIRBuilder.buildICmp(Pred, Res, Op0, Op1);
327  else if (Pred == CmpInst::FCMP_FALSE)
328  MIRBuilder.buildCopy(
329  Res, getOrCreateVReg(*Constant::getNullValue(U.getType())));
330  else if (Pred == CmpInst::FCMP_TRUE)
331  MIRBuilder.buildCopy(
332  Res, getOrCreateVReg(*Constant::getAllOnesValue(U.getType())));
333  else {
334  assert(CI && "Instruction should be CmpInst");
335  MIRBuilder.buildFCmp(Pred, Res, Op0, Op1,
337  }
338 
339  return true;
340 }
341 
342 bool IRTranslator::translateRet(const User &U, MachineIRBuilder &MIRBuilder) {
343  const ReturnInst &RI = cast<ReturnInst>(U);
344  const Value *Ret = RI.getReturnValue();
345  if (Ret && DL->getTypeStoreSize(Ret->getType()) == 0)
346  Ret = nullptr;
347 
348  ArrayRef<Register> VRegs;
349  if (Ret)
350  VRegs = getOrCreateVRegs(*Ret);
351 
352  Register SwiftErrorVReg = 0;
353  if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {
354  SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
355  &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
356  }
357 
358  // The target may mess up with the insertion point, but
359  // this is not important as a return is the last instruction
360  // of the block anyway.
361  return CLI->lowerReturn(MIRBuilder, Ret, VRegs, FuncInfo, SwiftErrorVReg);
362 }
363 
364 void IRTranslator::emitBranchForMergedCondition(
365  const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
366  MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
367  BranchProbability TProb, BranchProbability FProb, bool InvertCond) {
368  // If the leaf of the tree is a comparison, merge the condition into
369  // the caseblock.
370  if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
371  CmpInst::Predicate Condition;
372  if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
373  Condition = InvertCond ? IC->getInversePredicate() : IC->getPredicate();
374  } else {
375  const FCmpInst *FC = cast<FCmpInst>(Cond);
376  Condition = InvertCond ? FC->getInversePredicate() : FC->getPredicate();
377  }
378 
379  SwitchCG::CaseBlock CB(Condition, false, BOp->getOperand(0),
380  BOp->getOperand(1), nullptr, TBB, FBB, CurBB,
381  CurBuilder->getDebugLoc(), TProb, FProb);
382  SL->SwitchCases.push_back(CB);
383  return;
384  }
385 
386  // Create a CaseBlock record representing this branch.
389  Pred, false, Cond, ConstantInt::getTrue(MF->getFunction().getContext()),
390  nullptr, TBB, FBB, CurBB, CurBuilder->getDebugLoc(), TProb, FProb);
391  SL->SwitchCases.push_back(CB);
392 }
393 
394 static bool isValInBlock(const Value *V, const BasicBlock *BB) {
395  if (const Instruction *I = dyn_cast<Instruction>(V))
396  return I->getParent() == BB;
397  return true;
398 }
399 
400 void IRTranslator::findMergedConditions(
401  const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
402  MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
404  BranchProbability FProb, bool InvertCond) {
405  using namespace PatternMatch;
406  assert((Opc == Instruction::And || Opc == Instruction::Or) &&
407  "Expected Opc to be AND/OR");
408  // Skip over not part of the tree and remember to invert op and operands at
409  // next level.
410  Value *NotCond;
411  if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
412  isValInBlock(NotCond, CurBB->getBasicBlock())) {
413  findMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
414  !InvertCond);
415  return;
416  }
417 
418  const Instruction *BOp = dyn_cast<Instruction>(Cond);
419  const Value *BOpOp0, *BOpOp1;
420  // Compute the effective opcode for Cond, taking into account whether it needs
421  // to be inverted, e.g.
422  // and (not (or A, B)), C
423  // gets lowered as
424  // and (and (not A, not B), C)
426  if (BOp) {
427  BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
428  ? Instruction::And
429  : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
430  ? Instruction::Or
432  if (InvertCond) {
433  if (BOpc == Instruction::And)
434  BOpc = Instruction::Or;
435  else if (BOpc == Instruction::Or)
436  BOpc = Instruction::And;
437  }
438  }
439 
440  // If this node is not part of the or/and tree, emit it as a branch.
441  // Note that all nodes in the tree should have same opcode.
442  bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
443  if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
444  !isValInBlock(BOpOp0, CurBB->getBasicBlock()) ||
445  !isValInBlock(BOpOp1, CurBB->getBasicBlock())) {
446  emitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb,
447  InvertCond);
448  return;
449  }
450 
451  // Create TmpBB after CurBB.
452  MachineFunction::iterator BBI(CurBB);
453  MachineBasicBlock *TmpBB =
454  MF->CreateMachineBasicBlock(CurBB->getBasicBlock());
455  CurBB->getParent()->insert(++BBI, TmpBB);
456 
457  if (Opc == Instruction::Or) {
458  // Codegen X | Y as:
459  // BB1:
460  // jmp_if_X TBB
461  // jmp TmpBB
462  // TmpBB:
463  // jmp_if_Y TBB
464  // jmp FBB
465  //
466 
467  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
468  // The requirement is that
469  // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
470  // = TrueProb for original BB.
471  // Assuming the original probabilities are A and B, one choice is to set
472  // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
473  // A/(1+B) and 2B/(1+B). This choice assumes that
474  // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
475  // Another choice is to assume TrueProb for BB1 equals to TrueProb for
476  // TmpBB, but the math is more complicated.
477 
478  auto NewTrueProb = TProb / 2;
479  auto NewFalseProb = TProb / 2 + FProb;
480  // Emit the LHS condition.
481  findMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
482  NewFalseProb, InvertCond);
483 
484  // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
485  SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
486  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
487  // Emit the RHS condition into TmpBB.
488  findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
489  Probs[1], InvertCond);
490  } else {
491  assert(Opc == Instruction::And && "Unknown merge op!");
492  // Codegen X & Y as:
493  // BB1:
494  // jmp_if_X TmpBB
495  // jmp FBB
496  // TmpBB:
497  // jmp_if_Y TBB
498  // jmp FBB
499  //
500  // This requires creation of TmpBB after CurBB.
501 
502  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
503  // The requirement is that
504  // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
505  // = FalseProb for original BB.
506  // Assuming the original probabilities are A and B, one choice is to set
507  // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
508  // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
509  // TrueProb for BB1 * FalseProb for TmpBB.
510 
511  auto NewTrueProb = TProb + FProb / 2;
512  auto NewFalseProb = FProb / 2;
513  // Emit the LHS condition.
514  findMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
515  NewFalseProb, InvertCond);
516 
517  // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
518  SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
519  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
520  // Emit the RHS condition into TmpBB.
521  findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
522  Probs[1], InvertCond);
523  }
524 }
525 
526 bool IRTranslator::shouldEmitAsBranches(
527  const std::vector<SwitchCG::CaseBlock> &Cases) {
528  // For multiple cases, it's better to emit as branches.
529  if (Cases.size() != 2)
530  return true;
531 
532  // If this is two comparisons of the same values or'd or and'd together, they
533  // will get folded into a single comparison, so don't emit two blocks.
534  if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
535  Cases[0].CmpRHS == Cases[1].CmpRHS) ||
536  (Cases[0].CmpRHS == Cases[1].CmpLHS &&
537  Cases[0].CmpLHS == Cases[1].CmpRHS)) {
538  return false;
539  }
540 
541  // Handle: (X != null) | (Y != null) --> (X|Y) != 0
542  // Handle: (X == null) & (Y == null) --> (X|Y) == 0
543  if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
544  Cases[0].PredInfo.Pred == Cases[1].PredInfo.Pred &&
545  isa<Constant>(Cases[0].CmpRHS) &&
546  cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
547  if (Cases[0].PredInfo.Pred == CmpInst::ICMP_EQ &&
548  Cases[0].TrueBB == Cases[1].ThisBB)
549  return false;
550  if (Cases[0].PredInfo.Pred == CmpInst::ICMP_NE &&
551  Cases[0].FalseBB == Cases[1].ThisBB)
552  return false;
553  }
554 
555  return true;
556 }
557 
558 bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) {
559  const BranchInst &BrInst = cast<BranchInst>(U);
560  auto &CurMBB = MIRBuilder.getMBB();
561  auto *Succ0MBB = &getMBB(*BrInst.getSuccessor(0));
562 
563  if (BrInst.isUnconditional()) {
564  // If the unconditional target is the layout successor, fallthrough.
565  if (!CurMBB.isLayoutSuccessor(Succ0MBB))
566  MIRBuilder.buildBr(*Succ0MBB);
567 
568  // Link successors.
569  for (const BasicBlock *Succ : successors(&BrInst))
570  CurMBB.addSuccessor(&getMBB(*Succ));
571  return true;
572  }
573 
574  // If this condition is one of the special cases we handle, do special stuff
575  // now.
576  const Value *CondVal = BrInst.getCondition();
577  MachineBasicBlock *Succ1MBB = &getMBB(*BrInst.getSuccessor(1));
578 
579  const auto &TLI = *MF->getSubtarget().getTargetLowering();
580 
581  // If this is a series of conditions that are or'd or and'd together, emit
582  // this as a sequence of branches instead of setcc's with and/or operations.
583  // As long as jumps are not expensive (exceptions for multi-use logic ops,
584  // unpredictable branches, and vector extracts because those jumps are likely
585  // expensive for any target), this should improve performance.
586  // For example, instead of something like:
587  // cmp A, B
588  // C = seteq
589  // cmp D, E
590  // F = setle
591  // or C, F
592  // jnz foo
593  // Emit:
594  // cmp A, B
595  // je foo
596  // cmp D, E
597  // jle foo
598  using namespace PatternMatch;
599  const Instruction *CondI = dyn_cast<Instruction>(CondVal);
600  if (!TLI.isJumpExpensive() && CondI && CondI->hasOneUse() &&
601  !BrInst.hasMetadata(LLVMContext::MD_unpredictable)) {
603  Value *Vec;
604  const Value *BOp0, *BOp1;
605  if (match(CondI, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
606  Opcode = Instruction::And;
607  else if (match(CondI, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
608  Opcode = Instruction::Or;
609 
610  if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
611  match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {
612  findMergedConditions(CondI, Succ0MBB, Succ1MBB, &CurMBB, &CurMBB, Opcode,
613  getEdgeProbability(&CurMBB, Succ0MBB),
614  getEdgeProbability(&CurMBB, Succ1MBB),
615  /*InvertCond=*/false);
616  assert(SL->SwitchCases[0].ThisBB == &CurMBB && "Unexpected lowering!");
617 
618  // Allow some cases to be rejected.
619  if (shouldEmitAsBranches(SL->SwitchCases)) {
620  // Emit the branch for this block.
621  emitSwitchCase(SL->SwitchCases[0], &CurMBB, *CurBuilder);
622  SL->SwitchCases.erase(SL->SwitchCases.begin());
623  return true;
624  }
625 
626  // Okay, we decided not to do this, remove any inserted MBB's and clear
627  // SwitchCases.
628  for (unsigned I = 1, E = SL->SwitchCases.size(); I != E; ++I)
629  MF->erase(SL->SwitchCases[I].ThisBB);
630 
631  SL->SwitchCases.clear();
632  }
633  }
634 
635  // Create a CaseBlock record representing this branch.
636  SwitchCG::CaseBlock CB(CmpInst::ICMP_EQ, false, CondVal,
637  ConstantInt::getTrue(MF->getFunction().getContext()),
638  nullptr, Succ0MBB, Succ1MBB, &CurMBB,
639  CurBuilder->getDebugLoc());
640 
641  // Use emitSwitchCase to actually insert the fast branch sequence for this
642  // cond branch.
643  emitSwitchCase(CB, &CurMBB, *CurBuilder);
644  return true;
645 }
646 
647 void IRTranslator::addSuccessorWithProb(MachineBasicBlock *Src,
648  MachineBasicBlock *Dst,
649  BranchProbability Prob) {
650  if (!FuncInfo.BPI) {
651  Src->addSuccessorWithoutProb(Dst);
652  return;
653  }
654  if (Prob.isUnknown())
655  Prob = getEdgeProbability(Src, Dst);
656  Src->addSuccessor(Dst, Prob);
657 }
658 
660 IRTranslator::getEdgeProbability(const MachineBasicBlock *Src,
661  const MachineBasicBlock *Dst) const {
662  const BasicBlock *SrcBB = Src->getBasicBlock();
663  const BasicBlock *DstBB = Dst->getBasicBlock();
664  if (!FuncInfo.BPI) {
665  // If BPI is not available, set the default probability as 1 / N, where N is
666  // the number of successors.
667  auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
668  return BranchProbability(1, SuccSize);
669  }
670  return FuncInfo.BPI->getEdgeProbability(SrcBB, DstBB);
671 }
672 
673 bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) {
674  using namespace SwitchCG;
675  // Extract cases from the switch.
676  const SwitchInst &SI = cast<SwitchInst>(U);
677  BranchProbabilityInfo *BPI = FuncInfo.BPI;
678  CaseClusterVector Clusters;
679  Clusters.reserve(SI.getNumCases());
680  for (auto &I : SI.cases()) {
681  MachineBasicBlock *Succ = &getMBB(*I.getCaseSuccessor());
682  assert(Succ && "Could not find successor mbb in mapping");
683  const ConstantInt *CaseVal = I.getCaseValue();
684  BranchProbability Prob =
685  BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
686  : BranchProbability(1, SI.getNumCases() + 1);
687  Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
688  }
689 
690  MachineBasicBlock *DefaultMBB = &getMBB(*SI.getDefaultDest());
691 
692  // Cluster adjacent cases with the same destination. We do this at all
693  // optimization levels because it's cheap to do and will make codegen faster
694  // if there are many clusters.
695  sortAndRangeify(Clusters);
696 
697  MachineBasicBlock *SwitchMBB = &getMBB(*SI.getParent());
698 
699  // If there is only the default destination, jump there directly.
700  if (Clusters.empty()) {
701  SwitchMBB->addSuccessor(DefaultMBB);
702  if (DefaultMBB != SwitchMBB->getNextNode())
703  MIB.buildBr(*DefaultMBB);
704  return true;
705  }
706 
707  SL->findJumpTables(Clusters, &SI, DefaultMBB, nullptr, nullptr);
708  SL->findBitTestClusters(Clusters, &SI);
709 
710  LLVM_DEBUG({
711  dbgs() << "Case clusters: ";
712  for (const CaseCluster &C : Clusters) {
713  if (C.Kind == CC_JumpTable)
714  dbgs() << "JT:";
715  if (C.Kind == CC_BitTests)
716  dbgs() << "BT:";
717 
718  C.Low->getValue().print(dbgs(), true);
719  if (C.Low != C.High) {
720  dbgs() << '-';
721  C.High->getValue().print(dbgs(), true);
722  }
723  dbgs() << ' ';
724  }
725  dbgs() << '\n';
726  });
727 
728  assert(!Clusters.empty());
729  SwitchWorkList WorkList;
730  CaseClusterIt First = Clusters.begin();
731  CaseClusterIt Last = Clusters.end() - 1;
732  auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB);
733  WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
734 
735  // FIXME: At the moment we don't do any splitting optimizations here like
736  // SelectionDAG does, so this worklist only has one entry.
737  while (!WorkList.empty()) {
738  SwitchWorkListItem W = WorkList.back();
739  WorkList.pop_back();
740  if (!lowerSwitchWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
741  return false;
742  }
743  return true;
744 }
745 
746 void IRTranslator::emitJumpTable(SwitchCG::JumpTable &JT,
748  // Emit the code for the jump table
749  assert(JT.Reg != -1U && "Should lower JT Header first!");
750  MachineIRBuilder MIB(*MBB->getParent());
751  MIB.setMBB(*MBB);
752  MIB.setDebugLoc(CurBuilder->getDebugLoc());
753 
754  Type *PtrIRTy = Type::getInt8PtrTy(MF->getFunction().getContext());
755  const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
756 
757  auto Table = MIB.buildJumpTable(PtrTy, JT.JTI);
758  MIB.buildBrJT(Table.getReg(0), JT.JTI, JT.Reg);
759 }
760 
761 bool IRTranslator::emitJumpTableHeader(SwitchCG::JumpTable &JT,
763  MachineBasicBlock *HeaderBB) {
764  MachineIRBuilder MIB(*HeaderBB->getParent());
765  MIB.setMBB(*HeaderBB);
766  MIB.setDebugLoc(CurBuilder->getDebugLoc());
767 
768  const Value &SValue = *JTH.SValue;
769  // Subtract the lowest switch case value from the value being switched on.
770  const LLT SwitchTy = getLLTForType(*SValue.getType(), *DL);
771  Register SwitchOpReg = getOrCreateVReg(SValue);
772  auto FirstCst = MIB.buildConstant(SwitchTy, JTH.First);
773  auto Sub = MIB.buildSub({SwitchTy}, SwitchOpReg, FirstCst);
774 
775  // This value may be smaller or larger than the target's pointer type, and
776  // therefore require extension or truncating.
777  Type *PtrIRTy = SValue.getType()->getPointerTo();
778  const LLT PtrScalarTy = LLT::scalar(DL->getTypeSizeInBits(PtrIRTy));
779  Sub = MIB.buildZExtOrTrunc(PtrScalarTy, Sub);
780 
781  JT.Reg = Sub.getReg(0);
782 
783  if (JTH.OmitRangeCheck) {
784  if (JT.MBB != HeaderBB->getNextNode())
785  MIB.buildBr(*JT.MBB);
786  return true;
787  }
788 
789  // Emit the range check for the jump table, and branch to the default block
790  // for the switch statement if the value being switched on exceeds the
791  // largest case in the switch.
792  auto Cst = getOrCreateVReg(
793  *ConstantInt::get(SValue.getType(), JTH.Last - JTH.First));
794  Cst = MIB.buildZExtOrTrunc(PtrScalarTy, Cst).getReg(0);
795  auto Cmp = MIB.buildICmp(CmpInst::ICMP_UGT, LLT::scalar(1), Sub, Cst);
796 
797  auto BrCond = MIB.buildBrCond(Cmp.getReg(0), *JT.Default);
798 
799  // Avoid emitting unnecessary branches to the next block.
800  if (JT.MBB != HeaderBB->getNextNode())
801  BrCond = MIB.buildBr(*JT.MBB);
802  return true;
803 }
804 
805 void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
806  MachineBasicBlock *SwitchBB,
807  MachineIRBuilder &MIB) {
808  Register CondLHS = getOrCreateVReg(*CB.CmpLHS);
809  Register Cond;
810  DebugLoc OldDbgLoc = MIB.getDebugLoc();
811  MIB.setDebugLoc(CB.DbgLoc);
812  MIB.setMBB(*CB.ThisBB);
813 
814  if (CB.PredInfo.NoCmp) {
815  // Branch or fall through to TrueBB.
816  addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
817  addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
818  CB.ThisBB);
820  if (CB.TrueBB != CB.ThisBB->getNextNode())
821  MIB.buildBr(*CB.TrueBB);
822  MIB.setDebugLoc(OldDbgLoc);
823  return;
824  }
825 
826  const LLT i1Ty = LLT::scalar(1);
827  // Build the compare.
828  if (!CB.CmpMHS) {
829  const auto *CI = dyn_cast<ConstantInt>(CB.CmpRHS);
830  // For conditional branch lowering, we might try to do something silly like
831  // emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so,
832  // just re-use the existing condition vreg.
833  if (MRI->getType(CondLHS).getSizeInBits() == 1 && CI &&
834  CI->getZExtValue() == 1 && CB.PredInfo.Pred == CmpInst::ICMP_EQ) {
835  Cond = CondLHS;
836  } else {
837  Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
839  Cond =
840  MIB.buildFCmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
841  else
842  Cond =
843  MIB.buildICmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
844  }
845  } else {
847  "Can only handle SLE ranges");
848 
849  const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
850  const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
851 
852  Register CmpOpReg = getOrCreateVReg(*CB.CmpMHS);
853  if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
854  Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
855  Cond =
856  MIB.buildICmp(CmpInst::ICMP_SLE, i1Ty, CmpOpReg, CondRHS).getReg(0);
857  } else {
858  const LLT CmpTy = MRI->getType(CmpOpReg);
859  auto Sub = MIB.buildSub({CmpTy}, CmpOpReg, CondLHS);
860  auto Diff = MIB.buildConstant(CmpTy, High - Low);
861  Cond = MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, Sub, Diff).getReg(0);
862  }
863  }
864 
865  // Update successor info
866  addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
867 
868  addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
869  CB.ThisBB);
870 
871  // TrueBB and FalseBB are always different unless the incoming IR is
872  // degenerate. This only happens when running llc on weird IR.
873  if (CB.TrueBB != CB.FalseBB)
874  addSuccessorWithProb(CB.ThisBB, CB.FalseBB, CB.FalseProb);
876 
877  addMachineCFGPred({SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
878  CB.ThisBB);
879 
880  MIB.buildBrCond(Cond, *CB.TrueBB);
881  MIB.buildBr(*CB.FalseBB);
882  MIB.setDebugLoc(OldDbgLoc);
883 }
884 
885 bool IRTranslator::lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W,
886  MachineBasicBlock *SwitchMBB,
887  MachineBasicBlock *CurMBB,
888  MachineBasicBlock *DefaultMBB,
889  MachineIRBuilder &MIB,
891  BranchProbability UnhandledProbs,
893  MachineBasicBlock *Fallthrough,
894  bool FallthroughUnreachable) {
895  using namespace SwitchCG;
896  MachineFunction *CurMF = SwitchMBB->getParent();
897  // FIXME: Optimize away range check based on pivot comparisons.
898  JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
899  SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
900  BranchProbability DefaultProb = W.DefaultProb;
901 
902  // The jump block hasn't been inserted yet; insert it here.
903  MachineBasicBlock *JumpMBB = JT->MBB;
904  CurMF->insert(BBI, JumpMBB);
905 
906  // Since the jump table block is separate from the switch block, we need
907  // to keep track of it as a machine predecessor to the default block,
908  // otherwise we lose the phi edges.
909  addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
910  CurMBB);
911  addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
912  JumpMBB);
913 
914  auto JumpProb = I->Prob;
915  auto FallthroughProb = UnhandledProbs;
916 
917  // If the default statement is a target of the jump table, we evenly
918  // distribute the default probability to successors of CurMBB. Also
919  // update the probability on the edge from JumpMBB to Fallthrough.
921  SE = JumpMBB->succ_end();
922  SI != SE; ++SI) {
923  if (*SI == DefaultMBB) {
924  JumpProb += DefaultProb / 2;
925  FallthroughProb -= DefaultProb / 2;
926  JumpMBB->setSuccProbability(SI, DefaultProb / 2);
927  JumpMBB->normalizeSuccProbs();
928  } else {
929  // Also record edges from the jump table block to it's successors.
930  addMachineCFGPred({SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
931  JumpMBB);
932  }
933  }
934 
935  // Skip the range check if the fallthrough block is unreachable.
936  if (FallthroughUnreachable)
937  JTH->OmitRangeCheck = true;
938 
939  if (!JTH->OmitRangeCheck)
940  addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
941  addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
942  CurMBB->normalizeSuccProbs();
943 
944  // The jump table header will be inserted in our current block, do the
945  // range check, and fall through to our fallthrough block.
946  JTH->HeaderBB = CurMBB;
947  JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
948 
949  // If we're in the right place, emit the jump table header right now.
950  if (CurMBB == SwitchMBB) {
951  if (!emitJumpTableHeader(*JT, *JTH, CurMBB))
952  return false;
953  JTH->Emitted = true;
954  }
955  return true;
956 }
957 bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,
958  Value *Cond,
959  MachineBasicBlock *Fallthrough,
960  bool FallthroughUnreachable,
961  BranchProbability UnhandledProbs,
962  MachineBasicBlock *CurMBB,
963  MachineIRBuilder &MIB,
964  MachineBasicBlock *SwitchMBB) {
965  using namespace SwitchCG;
966  const Value *RHS, *LHS, *MHS;
967  CmpInst::Predicate Pred;
968  if (I->Low == I->High) {
969  // Check Cond == I->Low.
970  Pred = CmpInst::ICMP_EQ;
971  LHS = Cond;
972  RHS = I->Low;
973  MHS = nullptr;
974  } else {
975  // Check I->Low <= Cond <= I->High.
976  Pred = CmpInst::ICMP_SLE;
977  LHS = I->Low;
978  MHS = Cond;
979  RHS = I->High;
980  }
981 
982  // If Fallthrough is unreachable, fold away the comparison.
983  // The false probability is the sum of all unhandled cases.
984  CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
985  CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
986 
987  emitSwitchCase(CB, SwitchMBB, MIB);
988  return true;
989 }
990 
991 void IRTranslator::emitBitTestHeader(SwitchCG::BitTestBlock &B,
992  MachineBasicBlock *SwitchBB) {
993  MachineIRBuilder &MIB = *CurBuilder;
994  MIB.setMBB(*SwitchBB);
995 
996  // Subtract the minimum value.
997  Register SwitchOpReg = getOrCreateVReg(*B.SValue);
998 
999  LLT SwitchOpTy = MRI->getType(SwitchOpReg);
1000  Register MinValReg = MIB.buildConstant(SwitchOpTy, B.First).getReg(0);
1001  auto RangeSub = MIB.buildSub(SwitchOpTy, SwitchOpReg, MinValReg);
1002 
1003  // Ensure that the type will fit the mask value.
1004  LLT MaskTy = SwitchOpTy;
1005  for (unsigned I = 0, E = B.Cases.size(); I != E; ++I) {
1006  if (!isUIntN(SwitchOpTy.getSizeInBits(), B.Cases[I].Mask)) {
1007  // Switch table case range are encoded into series of masks.
1008  // Just use pointer type, it's guaranteed to fit.
1009  MaskTy = LLT::scalar(64);
1010  break;
1011  }
1012  }
1013  Register SubReg = RangeSub.getReg(0);
1014  if (SwitchOpTy != MaskTy)
1015  SubReg = MIB.buildZExtOrTrunc(MaskTy, SubReg).getReg(0);
1016 
1017  B.RegVT = getMVTForLLT(MaskTy);
1018  B.Reg = SubReg;
1019 
1020  MachineBasicBlock *MBB = B.Cases[0].ThisBB;
1021 
1022  if (!B.OmitRangeCheck)
1023  addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
1024  addSuccessorWithProb(SwitchBB, MBB, B.Prob);
1025 
1026  SwitchBB->normalizeSuccProbs();
1027 
1028  if (!B.OmitRangeCheck) {
1029  // Conditional branch to the default block.
1030  auto RangeCst = MIB.buildConstant(SwitchOpTy, B.Range);
1031  auto RangeCmp = MIB.buildICmp(CmpInst::Predicate::ICMP_UGT, LLT::scalar(1),
1032  RangeSub, RangeCst);
1033  MIB.buildBrCond(RangeCmp, *B.Default);
1034  }
1035 
1036  // Avoid emitting unnecessary branches to the next block.
1037  if (MBB != SwitchBB->getNextNode())
1038  MIB.buildBr(*MBB);
1039 }
1040 
1041 void IRTranslator::emitBitTestCase(SwitchCG::BitTestBlock &BB,
1042  MachineBasicBlock *NextMBB,
1043  BranchProbability BranchProbToNext,
1045  MachineBasicBlock *SwitchBB) {
1046  MachineIRBuilder &MIB = *CurBuilder;
1047  MIB.setMBB(*SwitchBB);
1048 
1049  LLT SwitchTy = getLLTForMVT(BB.RegVT);
1050  Register Cmp;
1051  unsigned PopCount = countPopulation(B.Mask);
1052  if (PopCount == 1) {
1053  // Testing for a single bit; just compare the shift count with what it
1054  // would need to be to shift a 1 bit in that position.
1055  auto MaskTrailingZeros =
1056  MIB.buildConstant(SwitchTy, countTrailingZeros(B.Mask));
1057  Cmp =
1058  MIB.buildICmp(ICmpInst::ICMP_EQ, LLT::scalar(1), Reg, MaskTrailingZeros)
1059  .getReg(0);
1060  } else if (PopCount == BB.Range) {
1061  // There is only one zero bit in the range, test for it directly.
1062  auto MaskTrailingOnes =
1063  MIB.buildConstant(SwitchTy, countTrailingOnes(B.Mask));
1064  Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Reg, MaskTrailingOnes)
1065  .getReg(0);
1066  } else {
1067  // Make desired shift.
1068  auto CstOne = MIB.buildConstant(SwitchTy, 1);
1069  auto SwitchVal = MIB.buildShl(SwitchTy, CstOne, Reg);
1070 
1071  // Emit bit tests and jumps.
1072  auto CstMask = MIB.buildConstant(SwitchTy, B.Mask);
1073  auto AndOp = MIB.buildAnd(SwitchTy, SwitchVal, CstMask);
1074  auto CstZero = MIB.buildConstant(SwitchTy, 0);
1075  Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), AndOp, CstZero)
1076  .getReg(0);
1077  }
1078 
1079  // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
1080  addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
1081  // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
1082  addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
1083  // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
1084  // one as they are relative probabilities (and thus work more like weights),
1085  // and hence we need to normalize them to let the sum of them become one.
1086  SwitchBB->normalizeSuccProbs();
1087 
1088  // Record the fact that the IR edge from the header to the bit test target
1089  // will go through our new block. Neeeded for PHIs to have nodes added.
1090  addMachineCFGPred({BB.Parent->getBasicBlock(), B.TargetBB->getBasicBlock()},
1091  SwitchBB);
1092 
1093  MIB.buildBrCond(Cmp, *B.TargetBB);
1094 
1095  // Avoid emitting unnecessary branches to the next block.
1096  if (NextMBB != SwitchBB->getNextNode())
1097  MIB.buildBr(*NextMBB);
1098 }
1099 
1100 bool IRTranslator::lowerBitTestWorkItem(
1102  MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
1104  BranchProbability DefaultProb, BranchProbability UnhandledProbs,
1106  bool FallthroughUnreachable) {
1107  using namespace SwitchCG;
1108  MachineFunction *CurMF = SwitchMBB->getParent();
1109  // FIXME: Optimize away range check based on pivot comparisons.
1110  BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
1111  // The bit test blocks haven't been inserted yet; insert them here.
1112  for (BitTestCase &BTC : BTB->Cases)
1113  CurMF->insert(BBI, BTC.ThisBB);
1114 
1115  // Fill in fields of the BitTestBlock.
1116  BTB->Parent = CurMBB;
1117  BTB->Default = Fallthrough;
1118 
1119  BTB->DefaultProb = UnhandledProbs;
1120  // If the cases in bit test don't form a contiguous range, we evenly
1121  // distribute the probability on the edge to Fallthrough to two
1122  // successors of CurMBB.
1123  if (!BTB->ContiguousRange) {
1124  BTB->Prob += DefaultProb / 2;
1125  BTB->DefaultProb -= DefaultProb / 2;
1126  }
1127 
1128  if (FallthroughUnreachable) {
1129  // Skip the range check if the fallthrough block is unreachable.
1130  BTB->OmitRangeCheck = true;
1131  }
1132 
1133  // If we're in the right place, emit the bit test header right now.
1134  if (CurMBB == SwitchMBB) {
1135  emitBitTestHeader(*BTB, SwitchMBB);
1136  BTB->Emitted = true;
1137  }
1138  return true;
1139 }
1140 
1141 bool IRTranslator::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
1142  Value *Cond,
1143  MachineBasicBlock *SwitchMBB,
1144  MachineBasicBlock *DefaultMBB,
1145  MachineIRBuilder &MIB) {
1146  using namespace SwitchCG;
1147  MachineFunction *CurMF = FuncInfo.MF;
1148  MachineBasicBlock *NextMBB = nullptr;
1149  MachineFunction::iterator BBI(W.MBB);
1150  if (++BBI != FuncInfo.MF->end())
1151  NextMBB = &*BBI;
1152 
1153  if (EnableOpts) {
1154  // Here, we order cases by probability so the most likely case will be
1155  // checked first. However, two clusters can have the same probability in
1156  // which case their relative ordering is non-deterministic. So we use Low
1157  // as a tie-breaker as clusters are guaranteed to never overlap.
1158  llvm::sort(W.FirstCluster, W.LastCluster + 1,
1159  [](const CaseCluster &a, const CaseCluster &b) {
1160  return a.Prob != b.Prob
1161  ? a.Prob > b.Prob
1162  : a.Low->getValue().slt(b.Low->getValue());
1163  });
1164 
1165  // Rearrange the case blocks so that the last one falls through if possible
1166  // without changing the order of probabilities.
1167  for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
1168  --I;
1169  if (I->Prob > W.LastCluster->Prob)
1170  break;
1171  if (I->Kind == CC_Range && I->MBB == NextMBB) {
1172  std::swap(*I, *W.LastCluster);
1173  break;
1174  }
1175  }
1176  }
1177 
1178  // Compute total probability.
1179  BranchProbability DefaultProb = W.DefaultProb;
1180  BranchProbability UnhandledProbs = DefaultProb;
1181  for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
1182  UnhandledProbs += I->Prob;
1183 
1184  MachineBasicBlock *CurMBB = W.MBB;
1185  for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
1186  bool FallthroughUnreachable = false;
1187  MachineBasicBlock *Fallthrough;
1188  if (I == W.LastCluster) {
1189  // For the last cluster, fall through to the default destination.
1190  Fallthrough = DefaultMBB;
1191  FallthroughUnreachable = isa<UnreachableInst>(
1192  DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
1193  } else {
1194  Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
1195  CurMF->insert(BBI, Fallthrough);
1196  }
1197  UnhandledProbs -= I->Prob;
1198 
1199  switch (I->Kind) {
1200  case CC_BitTests: {
1201  if (!lowerBitTestWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1202  DefaultProb, UnhandledProbs, I, Fallthrough,
1203  FallthroughUnreachable)) {
1204  LLVM_DEBUG(dbgs() << "Failed to lower bit test for switch");
1205  return false;
1206  }
1207  break;
1208  }
1209 
1210  case CC_JumpTable: {
1211  if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1212  UnhandledProbs, I, Fallthrough,
1213  FallthroughUnreachable)) {
1214  LLVM_DEBUG(dbgs() << "Failed to lower jump table");
1215  return false;
1216  }
1217  break;
1218  }
1219  case CC_Range: {
1220  if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
1221  FallthroughUnreachable, UnhandledProbs,
1222  CurMBB, MIB, SwitchMBB)) {
1223  LLVM_DEBUG(dbgs() << "Failed to lower switch range");
1224  return false;
1225  }
1226  break;
1227  }
1228  }
1229  CurMBB = Fallthrough;
1230  }
1231 
1232  return true;
1233 }
1234 
1235 bool IRTranslator::translateIndirectBr(const User &U,
1236  MachineIRBuilder &MIRBuilder) {
1237  const IndirectBrInst &BrInst = cast<IndirectBrInst>(U);
1238 
1239  const Register Tgt = getOrCreateVReg(*BrInst.getAddress());
1240  MIRBuilder.buildBrIndirect(Tgt);
1241 
1242  // Link successors.
1243  SmallPtrSet<const BasicBlock *, 32> AddedSuccessors;
1244  MachineBasicBlock &CurBB = MIRBuilder.getMBB();
1245  for (const BasicBlock *Succ : successors(&BrInst)) {
1246  // It's legal for indirectbr instructions to have duplicate blocks in the
1247  // destination list. We don't allow this in MIR. Skip anything that's
1248  // already a successor.
1249  if (!AddedSuccessors.insert(Succ).second)
1250  continue;
1251  CurBB.addSuccessor(&getMBB(*Succ));
1252  }
1253 
1254  return true;
1255 }
1256 
1257 static bool isSwiftError(const Value *V) {
1258  if (auto Arg = dyn_cast<Argument>(V))
1259  return Arg->hasSwiftErrorAttr();
1260  if (auto AI = dyn_cast<AllocaInst>(V))
1261  return AI->isSwiftError();
1262  return false;
1263 }
1264 
1265 bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
1266  const LoadInst &LI = cast<LoadInst>(U);
1267  if (DL->getTypeStoreSize(LI.getType()) == 0)
1268  return true;
1269 
1270  ArrayRef<Register> Regs = getOrCreateVRegs(LI);
1271  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(LI);
1272  Register Base = getOrCreateVReg(*LI.getPointerOperand());
1273 
1274  Type *OffsetIRTy = DL->getIntPtrType(LI.getPointerOperandType());
1275  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1276 
1277  if (CLI->supportSwiftError() && isSwiftError(LI.getPointerOperand())) {
1278  assert(Regs.size() == 1 && "swifterror should be single pointer");
1279  Register VReg = SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(),
1280  LI.getPointerOperand());
1281  MIRBuilder.buildCopy(Regs[0], VReg);
1282  return true;
1283  }
1284 
1285  auto &TLI = *MF->getSubtarget().getTargetLowering();
1286  MachineMemOperand::Flags Flags = TLI.getLoadMemOperandFlags(LI, *DL);
1287 
1288  const MDNode *Ranges =
1289  Regs.size() == 1 ? LI.getMetadata(LLVMContext::MD_range) : nullptr;
1290  for (unsigned i = 0; i < Regs.size(); ++i) {
1291  Register Addr;
1292  MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
1293 
1295  Align BaseAlign = getMemOpAlign(LI);
1296  AAMDNodes AAMetadata;
1297  LI.getAAMetadata(AAMetadata);
1298  auto MMO = MF->getMachineMemOperand(
1299  Ptr, Flags, MRI->getType(Regs[i]).getSizeInBytes(),
1300  commonAlignment(BaseAlign, Offsets[i] / 8), AAMetadata, Ranges,
1301  LI.getSyncScopeID(), LI.getOrdering());
1302  MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
1303  }
1304 
1305  return true;
1306 }
1307 
1308 bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
1309  const StoreInst &SI = cast<StoreInst>(U);
1310  if (DL->getTypeStoreSize(SI.getValueOperand()->getType()) == 0)
1311  return true;
1312 
1313  ArrayRef<Register> Vals = getOrCreateVRegs(*SI.getValueOperand());
1314  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*SI.getValueOperand());
1315  Register Base = getOrCreateVReg(*SI.getPointerOperand());
1316 
1317  Type *OffsetIRTy = DL->getIntPtrType(SI.getPointerOperandType());
1318  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1319 
1320  if (CLI->supportSwiftError() && isSwiftError(SI.getPointerOperand())) {
1321  assert(Vals.size() == 1 && "swifterror should be single pointer");
1322 
1323  Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
1324  SI.getPointerOperand());
1325  MIRBuilder.buildCopy(VReg, Vals[0]);
1326  return true;
1327  }
1328 
1329  auto &TLI = *MF->getSubtarget().getTargetLowering();
1331 
1332  for (unsigned i = 0; i < Vals.size(); ++i) {
1333  Register Addr;
1334  MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
1335 
1336  MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i] / 8);
1337  Align BaseAlign = getMemOpAlign(SI);
1338  AAMDNodes AAMetadata;
1339  SI.getAAMetadata(AAMetadata);
1340  auto MMO = MF->getMachineMemOperand(
1341  Ptr, Flags, MRI->getType(Vals[i]).getSizeInBytes(),
1342  commonAlignment(BaseAlign, Offsets[i] / 8), AAMetadata, nullptr,
1343  SI.getSyncScopeID(), SI.getOrdering());
1344  MIRBuilder.buildStore(Vals[i], Addr, *MMO);
1345  }
1346  return true;
1347 }
1348 
1349 static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL) {
1350  const Value *Src = U.getOperand(0);
1352 
1353  // getIndexedOffsetInType is designed for GEPs, so the first index is the
1354  // usual array element rather than looking into the actual aggregate.
1355  SmallVector<Value *, 1> Indices;
1356  Indices.push_back(ConstantInt::get(Int32Ty, 0));
1357 
1358  if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(&U)) {
1359  for (auto Idx : EVI->indices())
1360  Indices.push_back(ConstantInt::get(Int32Ty, Idx));
1361  } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(&U)) {
1362  for (auto Idx : IVI->indices())
1363  Indices.push_back(ConstantInt::get(Int32Ty, Idx));
1364  } else {
1365  for (unsigned i = 1; i < U.getNumOperands(); ++i)
1366  Indices.push_back(U.getOperand(i));
1367  }
1368 
1369  return 8 * static_cast<uint64_t>(
1370  DL.getIndexedOffsetInType(Src->getType(), Indices));
1371 }
1372 
1373 bool IRTranslator::translateExtractValue(const User &U,
1374  MachineIRBuilder &MIRBuilder) {
1375  const Value *Src = U.getOperand(0);
1376  uint64_t Offset = getOffsetFromIndices(U, *DL);
1377  ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
1378  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*Src);
1379  unsigned Idx = llvm::lower_bound(Offsets, Offset) - Offsets.begin();
1380  auto &DstRegs = allocateVRegs(U);
1381 
1382  for (unsigned i = 0; i < DstRegs.size(); ++i)
1383  DstRegs[i] = SrcRegs[Idx++];
1384 
1385  return true;
1386 }
1387 
1388 bool IRTranslator::translateInsertValue(const User &U,
1389  MachineIRBuilder &MIRBuilder) {
1390  const Value *Src = U.getOperand(0);
1391  uint64_t Offset = getOffsetFromIndices(U, *DL);
1392  auto &DstRegs = allocateVRegs(U);
1393  ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(U);
1394  ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
1395  ArrayRef<Register> InsertedRegs = getOrCreateVRegs(*U.getOperand(1));
1396  auto InsertedIt = InsertedRegs.begin();
1397 
1398  for (unsigned i = 0; i < DstRegs.size(); ++i) {
1399  if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
1400  DstRegs[i] = *InsertedIt++;
1401  else
1402  DstRegs[i] = SrcRegs[i];
1403  }
1404 
1405  return true;
1406 }
1407 
1408 bool IRTranslator::translateSelect(const User &U,
1409  MachineIRBuilder &MIRBuilder) {
1410  Register Tst = getOrCreateVReg(*U.getOperand(0));
1411  ArrayRef<Register> ResRegs = getOrCreateVRegs(U);
1412  ArrayRef<Register> Op0Regs = getOrCreateVRegs(*U.getOperand(1));
1413  ArrayRef<Register> Op1Regs = getOrCreateVRegs(*U.getOperand(2));
1414 
1415  uint16_t Flags = 0;
1416  if (const SelectInst *SI = dyn_cast<SelectInst>(&U))
1418 
1419  for (unsigned i = 0; i < ResRegs.size(); ++i) {
1420  MIRBuilder.buildSelect(ResRegs[i], Tst, Op0Regs[i], Op1Regs[i], Flags);
1421  }
1422 
1423  return true;
1424 }
1425 
1426 bool IRTranslator::translateCopy(const User &U, const Value &V,
1427  MachineIRBuilder &MIRBuilder) {
1428  Register Src = getOrCreateVReg(V);
1429  auto &Regs = *VMap.getVRegs(U);
1430  if (Regs.empty()) {
1431  Regs.push_back(Src);
1432  VMap.getOffsets(U)->push_back(0);
1433  } else {
1434  // If we already assigned a vreg for this instruction, we can't change that.
1435  // Emit a copy to satisfy the users we already emitted.
1436  MIRBuilder.buildCopy(Regs[0], Src);
1437  }
1438  return true;
1439 }
1440 
1441 bool IRTranslator::translateBitCast(const User &U,
1442  MachineIRBuilder &MIRBuilder) {
1443  // If we're bitcasting to the source type, we can reuse the source vreg.
1444  if (getLLTForType(*U.getOperand(0)->getType(), *DL) ==
1445  getLLTForType(*U.getType(), *DL))
1446  return translateCopy(U, *U.getOperand(0), MIRBuilder);
1447 
1448  return translateCast(TargetOpcode::G_BITCAST, U, MIRBuilder);
1449 }
1450 
1451 bool IRTranslator::translateCast(unsigned Opcode, const User &U,
1452  MachineIRBuilder &MIRBuilder) {
1453  Register Op = getOrCreateVReg(*U.getOperand(0));
1454  Register Res = getOrCreateVReg(U);
1455  MIRBuilder.buildInstr(Opcode, {Res}, {Op});
1456  return true;
1457 }
1458 
1459 bool IRTranslator::translateGetElementPtr(const User &U,
1460  MachineIRBuilder &MIRBuilder) {
1461  Value &Op0 = *U.getOperand(0);
1462  Register BaseReg = getOrCreateVReg(Op0);
1463  Type *PtrIRTy = Op0.getType();
1464  LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
1465  Type *OffsetIRTy = DL->getIntPtrType(PtrIRTy);
1466  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1467 
1468  // Normalize Vector GEP - all scalar operands should be converted to the
1469  // splat vector.
1470  unsigned VectorWidth = 0;
1471  if (auto *VT = dyn_cast<VectorType>(U.getType()))
1472  VectorWidth = cast<FixedVectorType>(VT)->getNumElements();
1473 
1474  // We might need to splat the base pointer into a vector if the offsets
1475  // are vectors.
1476  if (VectorWidth && !PtrTy.isVector()) {
1477  BaseReg =
1478  MIRBuilder.buildSplatVector(LLT::vector(VectorWidth, PtrTy), BaseReg)
1479  .getReg(0);
1480  PtrIRTy = FixedVectorType::get(PtrIRTy, VectorWidth);
1481  PtrTy = getLLTForType(*PtrIRTy, *DL);
1482  OffsetIRTy = DL->getIntPtrType(PtrIRTy);
1483  OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1484  }
1485 
1486  int64_t Offset = 0;
1487  for (gep_type_iterator GTI = gep_type_begin(&U), E = gep_type_end(&U);
1488  GTI != E; ++GTI) {
1489  const Value *Idx = GTI.getOperand();
1490  if (StructType *StTy = GTI.getStructTypeOrNull()) {
1491  unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
1493  continue;
1494  } else {
1495  uint64_t ElementSize = DL->getTypeAllocSize(GTI.getIndexedType());
1496 
1497  // If this is a scalar constant or a splat vector of constants,
1498  // handle it quickly.
1499  if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
1500  Offset += ElementSize * CI->getSExtValue();
1501  continue;
1502  }
1503 
1504  if (Offset != 0) {
1505  auto OffsetMIB = MIRBuilder.buildConstant({OffsetTy}, Offset);
1506  BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, OffsetMIB.getReg(0))
1507  .getReg(0);
1508  Offset = 0;
1509  }
1510 
1511  Register IdxReg = getOrCreateVReg(*Idx);
1512  LLT IdxTy = MRI->getType(IdxReg);
1513  if (IdxTy != OffsetTy) {
1514  if (!IdxTy.isVector() && VectorWidth) {
1515  IdxReg = MIRBuilder.buildSplatVector(
1516  OffsetTy.changeElementType(IdxTy), IdxReg).getReg(0);
1517  }
1518 
1519  IdxReg = MIRBuilder.buildSExtOrTrunc(OffsetTy, IdxReg).getReg(0);
1520  }
1521 
1522  // N = N + Idx * ElementSize;
1523  // Avoid doing it for ElementSize of 1.
1524  Register GepOffsetReg;
1525  if (ElementSize != 1) {
1526  auto ElementSizeMIB = MIRBuilder.buildConstant(
1527  getLLTForType(*OffsetIRTy, *DL), ElementSize);
1528  GepOffsetReg =
1529  MIRBuilder.buildMul(OffsetTy, IdxReg, ElementSizeMIB).getReg(0);
1530  } else
1531  GepOffsetReg = IdxReg;
1532 
1533  BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, GepOffsetReg).getReg(0);
1534  }
1535  }
1536 
1537  if (Offset != 0) {
1538  auto OffsetMIB =
1539  MIRBuilder.buildConstant(OffsetTy, Offset);
1540  MIRBuilder.buildPtrAdd(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0));
1541  return true;
1542  }
1543 
1544  MIRBuilder.buildCopy(getOrCreateVReg(U), BaseReg);
1545  return true;
1546 }
1547 
1548 bool IRTranslator::translateMemFunc(const CallInst &CI,
1549  MachineIRBuilder &MIRBuilder,
1550  unsigned Opcode) {
1551 
1552  // If the source is undef, then just emit a nop.
1553  if (isa<UndefValue>(CI.getArgOperand(1)))
1554  return true;
1555 
1556  SmallVector<Register, 3> SrcRegs;
1557 
1558  unsigned MinPtrSize = UINT_MAX;
1559  for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(AI) != AE; ++AI) {
1560  Register SrcReg = getOrCreateVReg(**AI);
1561  LLT SrcTy = MRI->getType(SrcReg);
1562  if (SrcTy.isPointer())
1563  MinPtrSize = std::min(SrcTy.getSizeInBits(), MinPtrSize);
1564  SrcRegs.push_back(SrcReg);
1565  }
1566 
1567  LLT SizeTy = LLT::scalar(MinPtrSize);
1568 
1569  // The size operand should be the minimum of the pointer sizes.
1570  Register &SizeOpReg = SrcRegs[SrcRegs.size() - 1];
1571  if (MRI->getType(SizeOpReg) != SizeTy)
1572  SizeOpReg = MIRBuilder.buildZExtOrTrunc(SizeTy, SizeOpReg).getReg(0);
1573 
1574  auto ICall = MIRBuilder.buildInstr(Opcode);
1575  for (Register SrcReg : SrcRegs)
1576  ICall.addUse(SrcReg);
1577 
1578  Align DstAlign;
1579  Align SrcAlign;
1580  unsigned IsVol =
1581  cast<ConstantInt>(CI.getArgOperand(CI.getNumArgOperands() - 1))
1582  ->getZExtValue();
1583 
1584  if (auto *MCI = dyn_cast<MemCpyInst>(&CI)) {
1585  DstAlign = MCI->getDestAlign().valueOrOne();
1586  SrcAlign = MCI->getSourceAlign().valueOrOne();
1587  } else if (auto *MMI = dyn_cast<MemMoveInst>(&CI)) {
1588  DstAlign = MMI->getDestAlign().valueOrOne();
1589  SrcAlign = MMI->getSourceAlign().valueOrOne();
1590  } else {
1591  auto *MSI = cast<MemSetInst>(&CI);
1592  DstAlign = MSI->getDestAlign().valueOrOne();
1593  }
1594 
1595  // We need to propagate the tail call flag from the IR inst as an argument.
1596  // Otherwise, we have to pessimize and assume later that we cannot tail call
1597  // any memory intrinsics.
1598  ICall.addImm(CI.isTailCall() ? 1 : 0);
1599 
1600  // Create mem operands to store the alignment and volatile info.
1602  ICall.addMemOperand(MF->getMachineMemOperand(
1604  MachineMemOperand::MOStore | VolFlag, 1, DstAlign));
1605  if (Opcode != TargetOpcode::G_MEMSET)
1606  ICall.addMemOperand(MF->getMachineMemOperand(
1608  MachineMemOperand::MOLoad | VolFlag, 1, SrcAlign));
1609 
1610  return true;
1611 }
1612 
1613 void IRTranslator::getStackGuard(Register DstReg,
1614  MachineIRBuilder &MIRBuilder) {
1616  MRI->setRegClass(DstReg, TRI->getPointerRegClass(*MF));
1617  auto MIB =
1618  MIRBuilder.buildInstr(TargetOpcode::LOAD_STACK_GUARD, {DstReg}, {});
1619 
1620  auto &TLI = *MF->getSubtarget().getTargetLowering();
1621  Value *Global = TLI.getSDagStackGuard(*MF->getFunction().getParent());
1622  if (!Global)
1623  return;
1624 
1625  MachinePointerInfo MPInfo(Global);
1629  MF->getMachineMemOperand(MPInfo, Flags, DL->getPointerSizeInBits() / 8,
1630  DL->getPointerABIAlignment(0));
1631  MIB.setMemRefs({MemRef});
1632 }
1633 
1634 bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
1635  MachineIRBuilder &MIRBuilder) {
1636  ArrayRef<Register> ResRegs = getOrCreateVRegs(CI);
1637  MIRBuilder.buildInstr(
1638  Op, {ResRegs[0], ResRegs[1]},
1639  {getOrCreateVReg(*CI.getOperand(0)), getOrCreateVReg(*CI.getOperand(1))});
1640 
1641  return true;
1642 }
1643 
1644 bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
1645  MachineIRBuilder &MIRBuilder) {
1646  Register Dst = getOrCreateVReg(CI);
1647  Register Src0 = getOrCreateVReg(*CI.getOperand(0));
1648  Register Src1 = getOrCreateVReg(*CI.getOperand(1));
1649  uint64_t Scale = cast<ConstantInt>(CI.getOperand(2))->getZExtValue();
1650  MIRBuilder.buildInstr(Op, {Dst}, { Src0, Src1, Scale });
1651  return true;
1652 }
1653 
1654 unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
1655  switch (ID) {
1656  default:
1657  break;
1658  case Intrinsic::bswap:
1659  return TargetOpcode::G_BSWAP;
1660  case Intrinsic::bitreverse:
1661  return TargetOpcode::G_BITREVERSE;
1662  case Intrinsic::fshl:
1663  return TargetOpcode::G_FSHL;
1664  case Intrinsic::fshr:
1665  return TargetOpcode::G_FSHR;
1666  case Intrinsic::ceil:
1667  return TargetOpcode::G_FCEIL;
1668  case Intrinsic::cos:
1669  return TargetOpcode::G_FCOS;
1670  case Intrinsic::ctpop:
1671  return TargetOpcode::G_CTPOP;
1672  case Intrinsic::exp:
1673  return TargetOpcode::G_FEXP;
1674  case Intrinsic::exp2:
1675  return TargetOpcode::G_FEXP2;
1676  case Intrinsic::fabs:
1677  return TargetOpcode::G_FABS;
1678  case Intrinsic::copysign:
1679  return TargetOpcode::G_FCOPYSIGN;
1680  case Intrinsic::minnum:
1681  return TargetOpcode::G_FMINNUM;
1682  case Intrinsic::maxnum:
1683  return TargetOpcode::G_FMAXNUM;
1684  case Intrinsic::minimum:
1685  return TargetOpcode::G_FMINIMUM;
1686  case Intrinsic::maximum:
1687  return TargetOpcode::G_FMAXIMUM;
1688  case Intrinsic::canonicalize:
1689  return TargetOpcode::G_FCANONICALIZE;
1690  case Intrinsic::floor:
1691  return TargetOpcode::G_FFLOOR;
1692  case Intrinsic::fma:
1693  return TargetOpcode::G_FMA;
1694  case Intrinsic::log:
1695  return TargetOpcode::G_FLOG;
1696  case Intrinsic::log2:
1697  return TargetOpcode::G_FLOG2;
1698  case Intrinsic::log10:
1699  return TargetOpcode::G_FLOG10;
1700  case Intrinsic::nearbyint:
1701  return TargetOpcode::G_FNEARBYINT;
1702  case Intrinsic::pow:
1703  return TargetOpcode::G_FPOW;
1704  case Intrinsic::powi:
1705  return TargetOpcode::G_FPOWI;
1706  case Intrinsic::rint:
1707  return TargetOpcode::G_FRINT;
1708  case Intrinsic::round:
1709  return TargetOpcode::G_INTRINSIC_ROUND;
1710  case Intrinsic::roundeven:
1711  return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
1712  case Intrinsic::sin:
1713  return TargetOpcode::G_FSIN;
1714  case Intrinsic::sqrt:
1715  return TargetOpcode::G_FSQRT;
1716  case Intrinsic::trunc:
1717  return TargetOpcode::G_INTRINSIC_TRUNC;
1718  case Intrinsic::readcyclecounter:
1719  return TargetOpcode::G_READCYCLECOUNTER;
1720  case Intrinsic::ptrmask:
1721  return TargetOpcode::G_PTRMASK;
1722  case Intrinsic::lrint:
1723  return TargetOpcode::G_INTRINSIC_LRINT;
1724  // FADD/FMUL require checking the FMF, so are handled elsewhere.
1725  case Intrinsic::vector_reduce_fmin:
1726  return TargetOpcode::G_VECREDUCE_FMIN;
1727  case Intrinsic::vector_reduce_fmax:
1728  return TargetOpcode::G_VECREDUCE_FMAX;
1729  case Intrinsic::vector_reduce_add:
1730  return TargetOpcode::G_VECREDUCE_ADD;
1731  case Intrinsic::vector_reduce_mul:
1732  return TargetOpcode::G_VECREDUCE_MUL;
1733  case Intrinsic::vector_reduce_and:
1734  return TargetOpcode::G_VECREDUCE_AND;
1735  case Intrinsic::vector_reduce_or:
1736  return TargetOpcode::G_VECREDUCE_OR;
1737  case Intrinsic::vector_reduce_xor:
1738  return TargetOpcode::G_VECREDUCE_XOR;
1739  case Intrinsic::vector_reduce_smax:
1740  return TargetOpcode::G_VECREDUCE_SMAX;
1741  case Intrinsic::vector_reduce_smin:
1742  return TargetOpcode::G_VECREDUCE_SMIN;
1743  case Intrinsic::vector_reduce_umax:
1744  return TargetOpcode::G_VECREDUCE_UMAX;
1745  case Intrinsic::vector_reduce_umin:
1746  return TargetOpcode::G_VECREDUCE_UMIN;
1747  }
1748  return Intrinsic::not_intrinsic;
1749 }
1750 
1751 bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
1752  Intrinsic::ID ID,
1753  MachineIRBuilder &MIRBuilder) {
1754 
1755  unsigned Op = getSimpleIntrinsicOpcode(ID);
1756 
1757  // Is this a simple intrinsic?
1759  return false;
1760 
1761  // Yes. Let's translate it.
1763  for (auto &Arg : CI.arg_operands())
1764  VRegs.push_back(getOrCreateVReg(*Arg));
1765 
1766  MIRBuilder.buildInstr(Op, {getOrCreateVReg(CI)}, VRegs,
1768  return true;
1769 }
1770 
1771 // TODO: Include ConstainedOps.def when all strict instructions are defined.
1773  switch (ID) {
1774  case Intrinsic::experimental_constrained_fadd:
1775  return TargetOpcode::G_STRICT_FADD;
1776  case Intrinsic::experimental_constrained_fsub:
1777  return TargetOpcode::G_STRICT_FSUB;
1778  case Intrinsic::experimental_constrained_fmul:
1779  return TargetOpcode::G_STRICT_FMUL;
1780  case Intrinsic::experimental_constrained_fdiv:
1781  return TargetOpcode::G_STRICT_FDIV;
1782  case Intrinsic::experimental_constrained_frem:
1783  return TargetOpcode::G_STRICT_FREM;
1784  case Intrinsic::experimental_constrained_fma:
1785  return TargetOpcode::G_STRICT_FMA;
1786  case Intrinsic::experimental_constrained_sqrt:
1787  return TargetOpcode::G_STRICT_FSQRT;
1788  default:
1789  return 0;
1790  }
1791 }
1792 
1793 bool IRTranslator::translateConstrainedFPIntrinsic(
1794  const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) {
1795  fp::ExceptionBehavior EB = FPI.getExceptionBehavior().getValue();
1796 
1797  unsigned Opcode = getConstrainedOpcode(FPI.getIntrinsicID());
1798  if (!Opcode)
1799  return false;
1800 
1801  unsigned Flags = MachineInstr::copyFlagsFromInstruction(FPI);
1803  Flags |= MachineInstr::NoFPExcept;
1804 
1806  VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(0)));
1807  if (!FPI.isUnaryOp())
1808  VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(1)));
1809  if (FPI.isTernaryOp())
1810  VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(2)));
1811 
1812  MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(FPI)}, VRegs, Flags);
1813  return true;
1814 }
1815 
1816 bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
1817  MachineIRBuilder &MIRBuilder) {
1818 
1819  // If this is a simple intrinsic (that is, we just need to add a def of
1820  // a vreg, and uses for each arg operand, then translate it.
1821  if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
1822  return true;
1823 
1824  switch (ID) {
1825  default:
1826  break;
1827  case Intrinsic::lifetime_start:
1828  case Intrinsic::lifetime_end: {
1829  // No stack colouring in O0, discard region information.
1830  if (MF->getTarget().getOptLevel() == CodeGenOpt::None)
1831  return true;
1832 
1833  unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
1835 
1836  // Get the underlying objects for the location passed on the lifetime
1837  // marker.
1839  getUnderlyingObjects(CI.getArgOperand(1), Allocas);
1840 
1841  // Iterate over each underlying object, creating lifetime markers for each
1842  // static alloca. Quit if we find a non-static alloca.
1843  for (const Value *V : Allocas) {
1844  const AllocaInst *AI = dyn_cast<AllocaInst>(V);
1845  if (!AI)
1846  continue;
1847 
1848  if (!AI->isStaticAlloca())
1849  return true;
1850 
1851  MIRBuilder.buildInstr(Op).addFrameIndex(getOrCreateFrameIndex(*AI));
1852  }
1853  return true;
1854  }
1855  case Intrinsic::dbg_declare: {
1856  const DbgDeclareInst &DI = cast<DbgDeclareInst>(CI);
1857  assert(DI.getVariable() && "Missing variable");
1858 
1859  const Value *Address = DI.getAddress();
1860  if (!Address || isa<UndefValue>(Address)) {
1861  LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
1862  return true;
1863  }
1864 
1866  MIRBuilder.getDebugLoc()) &&
1867  "Expected inlined-at fields to agree");
1868  auto AI = dyn_cast<AllocaInst>(Address);
1869  if (AI && AI->isStaticAlloca()) {
1870  // Static allocas are tracked at the MF level, no need for DBG_VALUE
1871  // instructions (in fact, they get ignored if they *do* exist).
1873  getOrCreateFrameIndex(*AI), DI.getDebugLoc());
1874  } else {
1875  // A dbg.declare describes the address of a source variable, so lower it
1876  // into an indirect DBG_VALUE.
1877  MIRBuilder.buildIndirectDbgValue(getOrCreateVReg(*Address),
1878  DI.getVariable(), DI.getExpression());
1879  }
1880  return true;
1881  }
1882  case Intrinsic::dbg_label: {
1883  const DbgLabelInst &DI = cast<DbgLabelInst>(CI);
1884  assert(DI.getLabel() && "Missing label");
1885 
1887  MIRBuilder.getDebugLoc()) &&
1888  "Expected inlined-at fields to agree");
1889 
1890  MIRBuilder.buildDbgLabel(DI.getLabel());
1891  return true;
1892  }
1893  case Intrinsic::vaend:
1894  // No target I know of cares about va_end. Certainly no in-tree target
1895  // does. Simplest intrinsic ever!
1896  return true;
1897  case Intrinsic::vastart: {
1898  auto &TLI = *MF->getSubtarget().getTargetLowering();
1899  Value *Ptr = CI.getArgOperand(0);
1900  unsigned ListSize = TLI.getVaListSizeInBits(*DL) / 8;
1901 
1902  // FIXME: Get alignment
1903  MIRBuilder.buildInstr(TargetOpcode::G_VASTART, {}, {getOrCreateVReg(*Ptr)})
1904  .addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Ptr),
1906  ListSize, Align(1)));
1907  return true;
1908  }
1909  case Intrinsic::dbg_value: {
1910  // This form of DBG_VALUE is target-independent.
1911  const DbgValueInst &DI = cast<DbgValueInst>(CI);
1912  const Value *V = DI.getValue();
1914  MIRBuilder.getDebugLoc()) &&
1915  "Expected inlined-at fields to agree");
1916  if (!V) {
1917  // Currently the optimizer can produce this; insert an undef to
1918  // help debugging. Probably the optimizer should not do this.
1919  MIRBuilder.buildIndirectDbgValue(0, DI.getVariable(), DI.getExpression());
1920  } else if (const auto *CI = dyn_cast<Constant>(V)) {
1921  MIRBuilder.buildConstDbgValue(*CI, DI.getVariable(), DI.getExpression());
1922  } else {
1923  for (Register Reg : getOrCreateVRegs(*V)) {
1924  // FIXME: This does not handle register-indirect values at offset 0. The
1925  // direct/indirect thing shouldn't really be handled by something as
1926  // implicit as reg+noreg vs reg+imm in the first place, but it seems
1927  // pretty baked in right now.
1928  MIRBuilder.buildDirectDbgValue(Reg, DI.getVariable(), DI.getExpression());
1929  }
1930  }
1931  return true;
1932  }
1933  case Intrinsic::uadd_with_overflow:
1934  return translateOverflowIntrinsic(CI, TargetOpcode::G_UADDO, MIRBuilder);
1935  case Intrinsic::sadd_with_overflow:
1936  return translateOverflowIntrinsic(CI, TargetOpcode::G_SADDO, MIRBuilder);
1937  case Intrinsic::usub_with_overflow:
1938  return translateOverflowIntrinsic(CI, TargetOpcode::G_USUBO, MIRBuilder);
1939  case Intrinsic::ssub_with_overflow:
1940  return translateOverflowIntrinsic(CI, TargetOpcode::G_SSUBO, MIRBuilder);
1941  case Intrinsic::umul_with_overflow:
1942  return translateOverflowIntrinsic(CI, TargetOpcode::G_UMULO, MIRBuilder);
1943  case Intrinsic::smul_with_overflow:
1944  return translateOverflowIntrinsic(CI, TargetOpcode::G_SMULO, MIRBuilder);
1945  case Intrinsic::uadd_sat:
1946  return translateBinaryOp(TargetOpcode::G_UADDSAT, CI, MIRBuilder);
1947  case Intrinsic::sadd_sat:
1948  return translateBinaryOp(TargetOpcode::G_SADDSAT, CI, MIRBuilder);
1949  case Intrinsic::usub_sat:
1950  return translateBinaryOp(TargetOpcode::G_USUBSAT, CI, MIRBuilder);
1951  case Intrinsic::ssub_sat:
1952  return translateBinaryOp(TargetOpcode::G_SSUBSAT, CI, MIRBuilder);
1953  case Intrinsic::ushl_sat:
1954  return translateBinaryOp(TargetOpcode::G_USHLSAT, CI, MIRBuilder);
1955  case Intrinsic::sshl_sat:
1956  return translateBinaryOp(TargetOpcode::G_SSHLSAT, CI, MIRBuilder);
1957  case Intrinsic::umin:
1958  return translateBinaryOp(TargetOpcode::G_UMIN, CI, MIRBuilder);
1959  case Intrinsic::umax:
1960  return translateBinaryOp(TargetOpcode::G_UMAX, CI, MIRBuilder);
1961  case Intrinsic::smin:
1962  return translateBinaryOp(TargetOpcode::G_SMIN, CI, MIRBuilder);
1963  case Intrinsic::smax:
1964  return translateBinaryOp(TargetOpcode::G_SMAX, CI, MIRBuilder);
1965  case Intrinsic::abs:
1966  // TODO: Preserve "int min is poison" arg in GMIR?
1967  return translateUnaryOp(TargetOpcode::G_ABS, CI, MIRBuilder);
1968  case Intrinsic::smul_fix:
1969  return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIX, CI, MIRBuilder);
1970  case Intrinsic::umul_fix:
1971  return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIX, CI, MIRBuilder);
1972  case Intrinsic::smul_fix_sat:
1973  return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
1974  case Intrinsic::umul_fix_sat:
1975  return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
1976  case Intrinsic::sdiv_fix:
1977  return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
1978  case Intrinsic::udiv_fix:
1979  return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
1980  case Intrinsic::sdiv_fix_sat:
1981  return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
1982  case Intrinsic::udiv_fix_sat:
1983  return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
1984  case Intrinsic::fmuladd: {
1985  const TargetMachine &TM = MF->getTarget();
1986  const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
1987  Register Dst = getOrCreateVReg(CI);
1988  Register Op0 = getOrCreateVReg(*CI.getArgOperand(0));
1989  Register Op1 = getOrCreateVReg(*CI.getArgOperand(1));
1990  Register Op2 = getOrCreateVReg(*CI.getArgOperand(2));
1991  if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
1993  TLI.getValueType(*DL, CI.getType()))) {
1994  // TODO: Revisit this to see if we should move this part of the
1995  // lowering to the combiner.
1996  MIRBuilder.buildFMA(Dst, Op0, Op1, Op2,
1998  } else {
1999  LLT Ty = getLLTForType(*CI.getType(), *DL);
2000  auto FMul = MIRBuilder.buildFMul(
2001  Ty, Op0, Op1, MachineInstr::copyFlagsFromInstruction(CI));
2002  MIRBuilder.buildFAdd(Dst, FMul, Op2,
2004  }
2005  return true;
2006  }
2007  case Intrinsic::convert_from_fp16:
2008  // FIXME: This intrinsic should probably be removed from the IR.
2009  MIRBuilder.buildFPExt(getOrCreateVReg(CI),
2010  getOrCreateVReg(*CI.getArgOperand(0)),
2012  return true;
2013  case Intrinsic::convert_to_fp16:
2014  // FIXME: This intrinsic should probably be removed from the IR.
2015  MIRBuilder.buildFPTrunc(getOrCreateVReg(CI),
2016  getOrCreateVReg(*CI.getArgOperand(0)),
2018  return true;
2019  case Intrinsic::memcpy:
2020  return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY);
2021  case Intrinsic::memmove:
2022  return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMMOVE);
2023  case Intrinsic::memset:
2024  return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMSET);
2025  case Intrinsic::eh_typeid_for: {
2027  Register Reg = getOrCreateVReg(CI);
2028  unsigned TypeID = MF->getTypeIDFor(GV);
2029  MIRBuilder.buildConstant(Reg, TypeID);
2030  return true;
2031  }
2032  case Intrinsic::objectsize:
2033  llvm_unreachable("llvm.objectsize.* should have been lowered already");
2034 
2035  case Intrinsic::is_constant:
2036  llvm_unreachable("llvm.is.constant.* should have been lowered already");
2037 
2038  case Intrinsic::stackguard:
2039  getStackGuard(getOrCreateVReg(CI), MIRBuilder);
2040  return true;
2041  case Intrinsic::stackprotector: {
2042  LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
2043  Register GuardVal = MRI->createGenericVirtualRegister(PtrTy);
2044  getStackGuard(GuardVal, MIRBuilder);
2045 
2046  AllocaInst *Slot = cast<AllocaInst>(CI.getArgOperand(1));
2047  int FI = getOrCreateFrameIndex(*Slot);
2049 
2050  MIRBuilder.buildStore(
2051  GuardVal, getOrCreateVReg(*Slot),
2055  PtrTy.getSizeInBits() / 8, Align(8)));
2056  return true;
2057  }
2058  case Intrinsic::stacksave: {
2059  // Save the stack pointer to the location provided by the intrinsic.
2060  Register Reg = getOrCreateVReg(CI);
2064 
2065  // If the target doesn't specify a stack pointer, then fall back.
2066  if (!StackPtr)
2067  return false;
2068 
2069  MIRBuilder.buildCopy(Reg, StackPtr);
2070  return true;
2071  }
2072  case Intrinsic::stackrestore: {
2073  // Restore the stack pointer from the location provided by the intrinsic.
2074  Register Reg = getOrCreateVReg(*CI.getArgOperand(0));
2078 
2079  // If the target doesn't specify a stack pointer, then fall back.
2080  if (!StackPtr)
2081  return false;
2082 
2083  MIRBuilder.buildCopy(StackPtr, Reg);
2084  return true;
2085  }
2086  case Intrinsic::cttz:
2087  case Intrinsic::ctlz: {
2088  ConstantInt *Cst = cast<ConstantInt>(CI.getArgOperand(1));
2089  bool isTrailing = ID == Intrinsic::cttz;
2090  unsigned Opcode = isTrailing
2091  ? Cst->isZero() ? TargetOpcode::G_CTTZ
2092  : TargetOpcode::G_CTTZ_ZERO_UNDEF
2093  : Cst->isZero() ? TargetOpcode::G_CTLZ
2094  : TargetOpcode::G_CTLZ_ZERO_UNDEF;
2095  MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(CI)},
2096  {getOrCreateVReg(*CI.getArgOperand(0))});
2097  return true;
2098  }
2099  case Intrinsic::invariant_start: {
2100  LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
2102  MIRBuilder.buildUndef(Undef);
2103  return true;
2104  }
2105  case Intrinsic::invariant_end:
2106  return true;
2107  case Intrinsic::expect:
2108  case Intrinsic::annotation:
2109  case Intrinsic::ptr_annotation:
2110  case Intrinsic::launder_invariant_group:
2111  case Intrinsic::strip_invariant_group: {
2112  // Drop the intrinsic, but forward the value.
2113  MIRBuilder.buildCopy(getOrCreateVReg(CI),
2114  getOrCreateVReg(*CI.getArgOperand(0)));
2115  return true;
2116  }
2117  case Intrinsic::assume:
2118  case Intrinsic::experimental_noalias_scope_decl:
2119  case Intrinsic::var_annotation:
2120  case Intrinsic::sideeffect:
2121  // Discard annotate attributes, assumptions, and artificial side-effects.
2122  return true;
2123  case Intrinsic::read_volatile_register:
2124  case Intrinsic::read_register: {
2125  Value *Arg = CI.getArgOperand(0);
2126  MIRBuilder
2127  .buildInstr(TargetOpcode::G_READ_REGISTER, {getOrCreateVReg(CI)}, {})
2128  .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()));
2129  return true;
2130  }
2131  case Intrinsic::write_register: {
2132  Value *Arg = CI.getArgOperand(0);
2133  MIRBuilder.buildInstr(TargetOpcode::G_WRITE_REGISTER)
2134  .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()))
2135  .addUse(getOrCreateVReg(*CI.getArgOperand(1)));
2136  return true;
2137  }
2138  case Intrinsic::localescape: {
2139  MachineBasicBlock &EntryMBB = MF->front();
2141 
2142  // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
2143  // is the same on all targets.
2144  for (unsigned Idx = 0, E = CI.getNumArgOperands(); Idx < E; ++Idx) {
2145  Value *Arg = CI.getArgOperand(Idx)->stripPointerCasts();
2146  if (isa<ConstantPointerNull>(Arg))
2147  continue; // Skip null pointers. They represent a hole in index space.
2148 
2149  int FI = getOrCreateFrameIndex(*cast<AllocaInst>(Arg));
2150  MCSymbol *FrameAllocSym =
2151  MF->getMMI().getContext().getOrCreateFrameAllocSymbol(EscapedName,
2152  Idx);
2153 
2154  // This should be inserted at the start of the entry block.
2155  auto LocalEscape =
2156  MIRBuilder.buildInstrNoInsert(TargetOpcode::LOCAL_ESCAPE)
2157  .addSym(FrameAllocSym)
2158  .addFrameIndex(FI);
2159 
2160  EntryMBB.insert(EntryMBB.begin(), LocalEscape);
2161  }
2162 
2163  return true;
2164  }
2165  case Intrinsic::vector_reduce_fadd:
2166  case Intrinsic::vector_reduce_fmul: {
2167  // Need to check for the reassoc flag to decide whether we want a
2168  // sequential reduction opcode or not.
2169  Register Dst = getOrCreateVReg(CI);
2170  Register ScalarSrc = getOrCreateVReg(*CI.getArgOperand(0));
2171  Register VecSrc = getOrCreateVReg(*CI.getArgOperand(1));
2172  unsigned Opc = 0;
2173  if (!CI.hasAllowReassoc()) {
2174  // The sequential ordering case.
2175  Opc = ID == Intrinsic::vector_reduce_fadd
2176  ? TargetOpcode::G_VECREDUCE_SEQ_FADD
2177  : TargetOpcode::G_VECREDUCE_SEQ_FMUL;
2178  MIRBuilder.buildInstr(Opc, {Dst}, {ScalarSrc, VecSrc},
2180  return true;
2181  }
2182  // We split the operation into a separate G_FADD/G_FMUL + the reduce,
2183  // since the associativity doesn't matter.
2184  unsigned ScalarOpc;
2185  if (ID == Intrinsic::vector_reduce_fadd) {
2186  Opc = TargetOpcode::G_VECREDUCE_FADD;
2187  ScalarOpc = TargetOpcode::G_FADD;
2188  } else {
2189  Opc = TargetOpcode::G_VECREDUCE_FMUL;
2190  ScalarOpc = TargetOpcode::G_FMUL;
2191  }
2192  LLT DstTy = MRI->getType(Dst);
2193  auto Rdx = MIRBuilder.buildInstr(
2194  Opc, {DstTy}, {VecSrc}, MachineInstr::copyFlagsFromInstruction(CI));
2195  MIRBuilder.buildInstr(ScalarOpc, {Dst}, {ScalarSrc, Rdx},
2197 
2198  return true;
2199  }
2200 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
2201  case Intrinsic::INTRINSIC:
2202 #include "llvm/IR/ConstrainedOps.def"
2203  return translateConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(CI),
2204  MIRBuilder);
2205 
2206  }
2207  return false;
2208 }
2209 
2210 bool IRTranslator::translateInlineAsm(const CallBase &CB,
2211  MachineIRBuilder &MIRBuilder) {
2212 
2214 
2215  if (!ALI) {
2216  LLVM_DEBUG(
2217  dbgs() << "Inline asm lowering is not supported for this target yet\n");
2218  return false;
2219  }
2220 
2221  return ALI->lowerInlineAsm(
2222  MIRBuilder, CB, [&](const Value &Val) { return getOrCreateVRegs(Val); });
2223 }
2224 
2225 bool IRTranslator::translateCallBase(const CallBase &CB,
2226  MachineIRBuilder &MIRBuilder) {
2227  ArrayRef<Register> Res = getOrCreateVRegs(CB);
2228 
2230  Register SwiftInVReg = 0;
2231  Register SwiftErrorVReg = 0;
2232  for (auto &Arg : CB.args()) {
2233  if (CLI->supportSwiftError() && isSwiftError(Arg)) {
2234  assert(SwiftInVReg == 0 && "Expected only one swift error argument");
2235  LLT Ty = getLLTForType(*Arg->getType(), *DL);
2236  SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
2237  MIRBuilder.buildCopy(SwiftInVReg, SwiftError.getOrCreateVRegUseAt(
2238  &CB, &MIRBuilder.getMBB(), Arg));
2239  Args.emplace_back(makeArrayRef(SwiftInVReg));
2240  SwiftErrorVReg =
2241  SwiftError.getOrCreateVRegDefAt(&CB, &MIRBuilder.getMBB(), Arg);
2242  continue;
2243  }
2244  Args.push_back(getOrCreateVRegs(*Arg));
2245  }
2246 
2247  // We don't set HasCalls on MFI here yet because call lowering may decide to
2248  // optimize into tail calls. Instead, we defer that to selection where a final
2249  // scan is done to check if any instructions are calls.
2250  bool Success =
2251  CLI->lowerCall(MIRBuilder, CB, Res, Args, SwiftErrorVReg,
2252  [&]() { return getOrCreateVReg(*CB.getCalledOperand()); });
2253 
2254  // Check if we just inserted a tail call.
2255  if (Success) {
2256  assert(!HasTailCall && "Can't tail call return twice from block?");
2257  const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
2258  HasTailCall = TII->isTailCall(*std::prev(MIRBuilder.getInsertPt()));
2259  }
2260 
2261  return Success;
2262 }
2263 
2264 bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
2265  const CallInst &CI = cast<CallInst>(U);
2266  auto TII = MF->getTarget().getIntrinsicInfo();
2267  const Function *F = CI.getCalledFunction();
2268 
2269  // FIXME: support Windows dllimport function calls.
2270  if (F && (F->hasDLLImportStorageClass() ||
2271  (MF->getTarget().getTargetTriple().isOSWindows() &&
2272  F->hasExternalWeakLinkage())))
2273  return false;
2274 
2275  // FIXME: support control flow guard targets.
2277  return false;
2278 
2279  if (CI.isInlineAsm())
2280  return translateInlineAsm(CI, MIRBuilder);
2281 
2283  if (F && F->isIntrinsic()) {
2284  ID = F->getIntrinsicID();
2285  if (TII && ID == Intrinsic::not_intrinsic)
2286  ID = static_cast<Intrinsic::ID>(TII->getIntrinsicID(F));
2287  }
2288 
2289  if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic)
2290  return translateCallBase(CI, MIRBuilder);
2291 
2292  assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
2293 
2294  if (translateKnownIntrinsic(CI, ID, MIRBuilder))
2295  return true;
2296 
2297  ArrayRef<Register> ResultRegs;
2298  if (!CI.getType()->isVoidTy())
2299  ResultRegs = getOrCreateVRegs(CI);
2300 
2301  // Ignore the callsite attributes. Backend code is most likely not expecting
2302  // an intrinsic to sometimes have side effects and sometimes not.
2303  MachineInstrBuilder MIB =
2304  MIRBuilder.buildIntrinsic(ID, ResultRegs, !F->doesNotAccessMemory());
2305  if (isa<FPMathOperator>(CI))
2306  MIB->copyIRFlags(CI);
2307 
2308  for (auto &Arg : enumerate(CI.arg_operands())) {
2309  // If this is required to be an immediate, don't materialize it in a
2310  // register.
2311  if (CI.paramHasAttr(Arg.index(), Attribute::ImmArg)) {
2312  if (ConstantInt *CI = dyn_cast<ConstantInt>(Arg.value())) {
2313  // imm arguments are more convenient than cimm (and realistically
2314  // probably sufficient), so use them.
2315  assert(CI->getBitWidth() <= 64 &&
2316  "large intrinsic immediates not handled");
2317  MIB.addImm(CI->getSExtValue());
2318  } else {
2319  MIB.addFPImm(cast<ConstantFP>(Arg.value()));
2320  }
2321  } else if (auto MD = dyn_cast<MetadataAsValue>(Arg.value())) {
2322  auto *MDN = dyn_cast<MDNode>(MD->getMetadata());
2323  if (!MDN) // This was probably an MDString.
2324  return false;
2325  MIB.addMetadata(MDN);
2326  } else {
2327  ArrayRef<Register> VRegs = getOrCreateVRegs(*Arg.value());
2328  if (VRegs.size() > 1)
2329  return false;
2330  MIB.addUse(VRegs[0]);
2331  }
2332  }
2333 
2334  // Add a MachineMemOperand if it is a target mem intrinsic.
2335  const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
2336  TargetLowering::IntrinsicInfo Info;
2337  // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
2338  if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) {
2339  Align Alignment = Info.align.getValueOr(
2340  DL->getABITypeAlign(Info.memVT.getTypeForEVT(F->getContext())));
2341 
2342  uint64_t Size = Info.memVT.getStoreSize();
2344  Info.flags, Size, Alignment));
2345  }
2346 
2347  return true;
2348 }
2349 
2350 bool IRTranslator::findUnwindDestinations(
2351  const BasicBlock *EHPadBB,
2352  BranchProbability Prob,
2353  SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2354  &UnwindDests) {
2355  EHPersonality Personality = classifyEHPersonality(
2356  EHPadBB->getParent()->getFunction().getPersonalityFn());
2357  bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2358  bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2359  bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2360  bool IsSEH = isAsynchronousEHPersonality(Personality);
2361 
2362  if (IsWasmCXX) {
2363  // Ignore this for now.
2364  return false;
2365  }
2366 
2367  while (EHPadBB) {
2368  const Instruction *Pad = EHPadBB->getFirstNonPHI();
2369  BasicBlock *NewEHPadBB = nullptr;
2370  if (isa<LandingPadInst>(Pad)) {
2371  // Stop on landingpads. They are not funclets.
2372  UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
2373  break;
2374  }
2375  if (isa<CleanupPadInst>(Pad)) {
2376  // Stop on cleanup pads. Cleanups are always funclet entries for all known
2377  // personalities.
2378  UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
2379  UnwindDests.back().first->setIsEHScopeEntry();
2380  UnwindDests.back().first->setIsEHFuncletEntry();
2381  break;
2382  }
2383  if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2384  // Add the catchpad handlers to the possible destinations.
2385  for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2386  UnwindDests.emplace_back(&getMBB(*CatchPadBB), Prob);
2387  // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2388  if (IsMSVCCXX || IsCoreCLR)
2389  UnwindDests.back().first->setIsEHFuncletEntry();
2390  if (!IsSEH)
2391  UnwindDests.back().first->setIsEHScopeEntry();
2392  }
2393  NewEHPadBB = CatchSwitch->getUnwindDest();
2394  } else {
2395  continue;
2396  }
2397 
2398  BranchProbabilityInfo *BPI = FuncInfo.BPI;
2399  if (BPI && NewEHPadBB)
2400  Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
2401  EHPadBB = NewEHPadBB;
2402  }
2403  return true;
2404 }
2405 
2406 bool IRTranslator::translateInvoke(const User &U,
2407  MachineIRBuilder &MIRBuilder) {
2408  const InvokeInst &I = cast<InvokeInst>(U);
2409  MCContext &Context = MF->getContext();
2410 
2411  const BasicBlock *ReturnBB = I.getSuccessor(0);
2412  const BasicBlock *EHPadBB = I.getSuccessor(1);
2413 
2414  const Function *Fn = I.getCalledFunction();
2415  if (I.isInlineAsm())
2416  return false;
2417 
2418  // FIXME: support invoking patchpoint and statepoint intrinsics.
2419  if (Fn && Fn->isIntrinsic())
2420  return false;
2421 
2422  // FIXME: support whatever these are.
2423  if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
2424  return false;
2425 
2426  // FIXME: support control flow guard targets.
2427  if (I.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget))
2428  return false;
2429 
2430  // FIXME: support Windows exception handling.
2431  if (!isa<LandingPadInst>(EHPadBB->getFirstNonPHI()))
2432  return false;
2433 
2434  // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
2435  // the region covered by the try.
2436  MCSymbol *BeginSymbol = Context.createTempSymbol();
2437  MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol);
2438 
2439  if (!translateCallBase(I, MIRBuilder))
2440  return false;
2441 
2442  MCSymbol *EndSymbol = Context.createTempSymbol();
2443  MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(EndSymbol);
2444 
2446  BranchProbabilityInfo *BPI = FuncInfo.BPI;
2447  MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB();
2448  BranchProbability EHPadBBProb =
2449  BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2451 
2452  if (!findUnwindDestinations(EHPadBB, EHPadBBProb, UnwindDests))
2453  return false;
2454 
2455  MachineBasicBlock &EHPadMBB = getMBB(*EHPadBB),
2456  &ReturnMBB = getMBB(*ReturnBB);
2457  // Update successor info.
2458  addSuccessorWithProb(InvokeMBB, &ReturnMBB);
2459  for (auto &UnwindDest : UnwindDests) {
2460  UnwindDest.first->setIsEHPad();
2461  addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2462  }
2463  InvokeMBB->normalizeSuccProbs();
2464 
2465  MF->addInvoke(&EHPadMBB, BeginSymbol, EndSymbol);
2466  MIRBuilder.buildBr(ReturnMBB);
2467  return true;
2468 }
2469 
2470 bool IRTranslator::translateCallBr(const User &U,
2471  MachineIRBuilder &MIRBuilder) {
2472  // FIXME: Implement this.
2473  return false;
2474 }
2475 
2476 bool IRTranslator::translateLandingPad(const User &U,
2477  MachineIRBuilder &MIRBuilder) {
2478  const LandingPadInst &LP = cast<LandingPadInst>(U);
2479 
2480  MachineBasicBlock &MBB = MIRBuilder.getMBB();
2481 
2482  MBB.setIsEHPad();
2483 
2484  // If there aren't registers to copy the values into (e.g., during SjLj
2485  // exceptions), then don't bother.
2486  auto &TLI = *MF->getSubtarget().getTargetLowering();
2487  const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
2488  if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2489  TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2490  return true;
2491 
2492  // If landingpad's return type is token type, we don't create DAG nodes
2493  // for its exception pointer and selector value. The extraction of exception
2494  // pointer or selector value from token type landingpads is not currently
2495  // supported.
2496  if (LP.getType()->isTokenTy())
2497  return true;
2498 
2499  // Add a label to mark the beginning of the landing pad. Deletion of the
2500  // landing pad can thus be detected via the MachineModuleInfo.
2502  .addSym(MF->addLandingPad(&MBB));
2503 
2504  // If the unwinder does not preserve all registers, ensure that the
2505  // function marks the clobbered registers as used.
2507  if (auto *RegMask = TRI.getCustomEHPadPreservedMask(*MF))
2508  MF->getRegInfo().addPhysRegsUsedFromRegMask(RegMask);
2509 
2510  LLT Ty = getLLTForType(*LP.getType(), *DL);
2512  MIRBuilder.buildUndef(Undef);
2513 
2514  SmallVector<LLT, 2> Tys;
2515  for (Type *Ty : cast<StructType>(LP.getType())->elements())
2516  Tys.push_back(getLLTForType(*Ty, *DL));
2517  assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
2518 
2519  // Mark exception register as live in.
2520  Register ExceptionReg = TLI.getExceptionPointerRegister(PersonalityFn);
2521  if (!ExceptionReg)
2522  return false;
2523 
2524  MBB.addLiveIn(ExceptionReg);
2525  ArrayRef<Register> ResRegs = getOrCreateVRegs(LP);
2526  MIRBuilder.buildCopy(ResRegs[0], ExceptionReg);
2527 
2528  Register SelectorReg = TLI.getExceptionSelectorRegister(PersonalityFn);
2529  if (!SelectorReg)
2530  return false;
2531 
2532  MBB.addLiveIn(SelectorReg);
2533  Register PtrVReg = MRI->createGenericVirtualRegister(Tys[0]);
2534  MIRBuilder.buildCopy(PtrVReg, SelectorReg);
2535  MIRBuilder.buildCast(ResRegs[1], PtrVReg);
2536 
2537  return true;
2538 }
2539 
2540 bool IRTranslator::translateAlloca(const User &U,
2541  MachineIRBuilder &MIRBuilder) {
2542  auto &AI = cast<AllocaInst>(U);
2543 
2544  if (AI.isSwiftError())
2545  return true;
2546 
2547  if (AI.isStaticAlloca()) {
2548  Register Res = getOrCreateVReg(AI);
2549  int FI = getOrCreateFrameIndex(AI);
2550  MIRBuilder.buildFrameIndex(Res, FI);
2551  return true;
2552  }
2553 
2554  // FIXME: support stack probing for Windows.
2555  if (MF->getTarget().getTargetTriple().isOSWindows())
2556  return false;
2557 
2558  // Now we're in the harder dynamic case.
2559  Register NumElts = getOrCreateVReg(*AI.getArraySize());
2560  Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
2561  LLT IntPtrTy = getLLTForType(*IntPtrIRTy, *DL);
2562  if (MRI->getType(NumElts) != IntPtrTy) {
2563  Register ExtElts = MRI->createGenericVirtualRegister(IntPtrTy);
2564  MIRBuilder.buildZExtOrTrunc(ExtElts, NumElts);
2565  NumElts = ExtElts;
2566  }
2567 
2568  Type *Ty = AI.getAllocatedType();
2569 
2570  Register AllocSize = MRI->createGenericVirtualRegister(IntPtrTy);
2571  Register TySize =
2572  getOrCreateVReg(*ConstantInt::get(IntPtrIRTy, DL->getTypeAllocSize(Ty)));
2573  MIRBuilder.buildMul(AllocSize, NumElts, TySize);
2574 
2575  // Round the size of the allocation up to the stack alignment size
2576  // by add SA-1 to the size. This doesn't overflow because we're computing
2577  // an address inside an alloca.
2579  auto SAMinusOne = MIRBuilder.buildConstant(IntPtrTy, StackAlign.value() - 1);
2580  auto AllocAdd = MIRBuilder.buildAdd(IntPtrTy, AllocSize, SAMinusOne,
2582  auto AlignCst =
2583  MIRBuilder.buildConstant(IntPtrTy, ~(uint64_t)(StackAlign.value() - 1));
2584  auto AlignedAlloc = MIRBuilder.buildAnd(IntPtrTy, AllocAdd, AlignCst);
2585 
2586  Align Alignment = std::max(AI.getAlign(), DL->getPrefTypeAlign(Ty));
2587  if (Alignment <= StackAlign)
2588  Alignment = Align(1);
2589  MIRBuilder.buildDynStackAlloc(getOrCreateVReg(AI), AlignedAlloc, Alignment);
2590 
2591  MF->getFrameInfo().CreateVariableSizedObject(Alignment, &AI);
2593  return true;
2594 }
2595 
2596 bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {
2597  // FIXME: We may need more info about the type. Because of how LLT works,
2598  // we're completely discarding the i64/double distinction here (amongst
2599  // others). Fortunately the ABIs I know of where that matters don't use va_arg
2600  // anyway but that's not guaranteed.
2601  MIRBuilder.buildInstr(TargetOpcode::G_VAARG, {getOrCreateVReg(U)},
2602  {getOrCreateVReg(*U.getOperand(0)),
2603  DL->getABITypeAlign(U.getType()).value()});
2604  return true;
2605 }
2606 
2607 bool IRTranslator::translateInsertElement(const User &U,
2608  MachineIRBuilder &MIRBuilder) {
2609  // If it is a <1 x Ty> vector, use the scalar as it is
2610  // not a legal vector type in LLT.
2611  if (cast<FixedVectorType>(U.getType())->getNumElements() == 1)
2612  return translateCopy(U, *U.getOperand(1), MIRBuilder);
2613 
2614  Register Res = getOrCreateVReg(U);
2615  Register Val = getOrCreateVReg(*U.getOperand(0));
2616  Register Elt = getOrCreateVReg(*U.getOperand(1));
2617  Register Idx = getOrCreateVReg(*U.getOperand(2));
2618  MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
2619  return true;
2620 }
2621 
2622 bool IRTranslator::translateExtractElement(const User &U,
2623  MachineIRBuilder &MIRBuilder) {
2624  // If it is a <1 x Ty> vector, use the scalar as it is
2625  // not a legal vector type in LLT.
2626  if (cast<FixedVectorType>(U.getOperand(0)->getType())->getNumElements() == 1)
2627  return translateCopy(U, *U.getOperand(0), MIRBuilder);
2628 
2629  Register Res = getOrCreateVReg(U);
2630  Register Val = getOrCreateVReg(*U.getOperand(0));
2631  const auto &TLI = *MF->getSubtarget().getTargetLowering();
2632  unsigned PreferredVecIdxWidth = TLI.getVectorIdxTy(*DL).getSizeInBits();
2633  Register Idx;
2634  if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(1))) {
2635  if (CI->getBitWidth() != PreferredVecIdxWidth) {
2636  APInt NewIdx = CI->getValue().sextOrTrunc(PreferredVecIdxWidth);
2637  auto *NewIdxCI = ConstantInt::get(CI->getContext(), NewIdx);
2638  Idx = getOrCreateVReg(*NewIdxCI);
2639  }
2640  }
2641  if (!Idx)
2642  Idx = getOrCreateVReg(*U.getOperand(1));
2643  if (MRI->getType(Idx).getSizeInBits() != PreferredVecIdxWidth) {
2644  const LLT VecIdxTy = LLT::scalar(PreferredVecIdxWidth);
2645  Idx = MIRBuilder.buildSExtOrTrunc(VecIdxTy, Idx).getReg(0);
2646  }
2647  MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
2648  return true;
2649 }
2650 
2651 bool IRTranslator::translateShuffleVector(const User &U,
2652  MachineIRBuilder &MIRBuilder) {
2654  if (auto *SVI = dyn_cast<ShuffleVectorInst>(&U))
2655  Mask = SVI->getShuffleMask();
2656  else
2657  Mask = cast<ConstantExpr>(U).getShuffleMask();
2658  ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
2659  MIRBuilder
2660  .buildInstr(TargetOpcode::G_SHUFFLE_VECTOR, {getOrCreateVReg(U)},
2661  {getOrCreateVReg(*U.getOperand(0)),
2662  getOrCreateVReg(*U.getOperand(1))})
2663  .addShuffleMask(MaskAlloc);
2664  return true;
2665 }
2666 
2667 bool IRTranslator::translatePHI(const User &U, MachineIRBuilder &MIRBuilder) {
2668  const PHINode &PI = cast<PHINode>(U);
2669 
2671  for (auto Reg : getOrCreateVRegs(PI)) {
2672  auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_PHI, {Reg}, {});
2673  Insts.push_back(MIB.getInstr());
2674  }
2675 
2676  PendingPHIs.emplace_back(&PI, std::move(Insts));
2677  return true;
2678 }
2679 
2680 bool IRTranslator::translateAtomicCmpXchg(const User &U,
2681  MachineIRBuilder &MIRBuilder) {
2682  const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(U);
2683 
2684  auto &TLI = *MF->getSubtarget().getTargetLowering();
2685  auto Flags = TLI.getAtomicMemOperandFlags(I, *DL);
2686 
2687  Type *ResType = I.getType();
2688  Type *ValType = ResType->Type::getStructElementType(0);
2689 
2690  auto Res = getOrCreateVRegs(I);
2691  Register OldValRes = Res[0];
2692  Register SuccessRes = Res[1];
2693  Register Addr = getOrCreateVReg(*I.getPointerOperand());
2694  Register Cmp = getOrCreateVReg(*I.getCompareOperand());
2695  Register NewVal = getOrCreateVReg(*I.getNewValOperand());
2696 
2697  AAMDNodes AAMetadata;
2698  I.getAAMetadata(AAMetadata);
2699 
2700  MIRBuilder.buildAtomicCmpXchgWithSuccess(
2701  OldValRes, SuccessRes, Addr, Cmp, NewVal,
2702  *MF->getMachineMemOperand(
2703  MachinePointerInfo(I.getPointerOperand()), Flags,
2704  DL->getTypeStoreSize(ValType), getMemOpAlign(I), AAMetadata, nullptr,
2705  I.getSyncScopeID(), I.getSuccessOrdering(), I.getFailureOrdering()));
2706  return true;
2707 }
2708 
2709 bool IRTranslator::translateAtomicRMW(const User &U,
2710  MachineIRBuilder &MIRBuilder) {
2711  const AtomicRMWInst &I = cast<AtomicRMWInst>(U);
2712  auto &TLI = *MF->getSubtarget().getTargetLowering();
2713  auto Flags = TLI.getAtomicMemOperandFlags(I, *DL);
2714 
2715  Type *ResType = I.getType();
2716 
2717  Register Res = getOrCreateVReg(I);
2718  Register Addr = getOrCreateVReg(*I.getPointerOperand());
2719  Register Val = getOrCreateVReg(*I.getValOperand());
2720 
2721  unsigned Opcode = 0;
2722  switch (I.getOperation()) {
2723  default:
2724  return false;
2725  case AtomicRMWInst::Xchg:
2726  Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
2727  break;
2728  case AtomicRMWInst::Add:
2729  Opcode = TargetOpcode::G_ATOMICRMW_ADD;
2730  break;
2731  case AtomicRMWInst::Sub:
2732  Opcode = TargetOpcode::G_ATOMICRMW_SUB;
2733  break;
2734  case AtomicRMWInst::And:
2735  Opcode = TargetOpcode::G_ATOMICRMW_AND;
2736  break;
2737  case AtomicRMWInst::Nand:
2738  Opcode = TargetOpcode::G_ATOMICRMW_NAND;
2739  break;
2740  case AtomicRMWInst::Or:
2741  Opcode = TargetOpcode::G_ATOMICRMW_OR;
2742  break;
2743  case AtomicRMWInst::Xor:
2744  Opcode = TargetOpcode::G_ATOMICRMW_XOR;
2745  break;
2746  case AtomicRMWInst::Max:
2747  Opcode = TargetOpcode::G_ATOMICRMW_MAX;
2748  break;
2749  case AtomicRMWInst::Min:
2750  Opcode = TargetOpcode::G_ATOMICRMW_MIN;
2751  break;
2752  case AtomicRMWInst::UMax:
2753  Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
2754  break;
2755  case AtomicRMWInst::UMin:
2756  Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
2757  break;
2758  case AtomicRMWInst::FAdd:
2759  Opcode = TargetOpcode::G_ATOMICRMW_FADD;
2760  break;
2761  case AtomicRMWInst::FSub:
2762  Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
2763  break;
2764  }
2765 
2766  AAMDNodes AAMetadata;
2767  I.getAAMetadata(AAMetadata);
2768 
2769  MIRBuilder.buildAtomicRMW(
2770  Opcode, Res, Addr, Val,
2771  *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
2772  Flags, DL->getTypeStoreSize(ResType),
2773  getMemOpAlign(I), AAMetadata, nullptr,
2774  I.getSyncScopeID(), I.getOrdering()));
2775  return true;
2776 }
2777 
2778 bool IRTranslator::translateFence(const User &U,
2779  MachineIRBuilder &MIRBuilder) {
2780  const FenceInst &Fence = cast<FenceInst>(U);
2781  MIRBuilder.buildFence(static_cast<unsigned>(Fence.getOrdering()),
2782  Fence.getSyncScopeID());
2783  return true;
2784 }
2785 
2786 bool IRTranslator::translateFreeze(const User &U,
2787  MachineIRBuilder &MIRBuilder) {
2788  const ArrayRef<Register> DstRegs = getOrCreateVRegs(U);
2789  const ArrayRef<Register> SrcRegs = getOrCreateVRegs(*U.getOperand(0));
2790 
2791  assert(DstRegs.size() == SrcRegs.size() &&
2792  "Freeze with different source and destination type?");
2793 
2794  for (unsigned I = 0; I < DstRegs.size(); ++I) {
2795  MIRBuilder.buildFreeze(DstRegs[I], SrcRegs[I]);
2796  }
2797 
2798  return true;
2799 }
2800 
2801 void IRTranslator::finishPendingPhis() {
2802 #ifndef NDEBUG
2803  DILocationVerifier Verifier;
2804  GISelObserverWrapper WrapperObserver(&Verifier);
2805  RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
2806 #endif // ifndef NDEBUG
2807  for (auto &Phi : PendingPHIs) {
2808  const PHINode *PI = Phi.first;
2809  ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
2810  MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
2811  EntryBuilder->setDebugLoc(PI->getDebugLoc());
2812 #ifndef NDEBUG
2813  Verifier.setCurrentInst(PI);
2814 #endif // ifndef NDEBUG
2815 
2817  for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
2818  auto IRPred = PI->getIncomingBlock(i);
2819  ArrayRef<Register> ValRegs = getOrCreateVRegs(*PI->getIncomingValue(i));
2820  for (auto Pred : getMachinePredBBs({IRPred, PI->getParent()})) {
2821  if (SeenPreds.count(Pred) || !PhiMBB->isPredecessor(Pred))
2822  continue;
2823  SeenPreds.insert(Pred);
2824  for (unsigned j = 0; j < ValRegs.size(); ++j) {
2825  MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
2826  MIB.addUse(ValRegs[j]);
2827  MIB.addMBB(Pred);
2828  }
2829  }
2830  }
2831  }
2832 }
2833 
2834 bool IRTranslator::valueIsSplit(const Value &V,
2836  SmallVector<LLT, 4> SplitTys;
2837  if (Offsets && !Offsets->empty())
2838  Offsets->clear();
2839  computeValueLLTs(*DL, *V.getType(), SplitTys, Offsets);
2840  return SplitTys.size() > 1;
2841 }
2842 
2843 bool IRTranslator::translate(const Instruction &Inst) {
2844  CurBuilder->setDebugLoc(Inst.getDebugLoc());
2845  // We only emit constants into the entry block from here. To prevent jumpy
2846  // debug behaviour set the line to 0.
2847  if (const DebugLoc &DL = Inst.getDebugLoc())
2848  EntryBuilder->setDebugLoc(DILocation::get(
2849  Inst.getContext(), 0, 0, DL.getScope(), DL.getInlinedAt()));
2850  else
2851  EntryBuilder->setDebugLoc(DebugLoc());
2852 
2853  auto &TLI = *MF->getSubtarget().getTargetLowering();
2854  if (TLI.fallBackToDAGISel(Inst))
2855  return false;
2856 
2857  switch (Inst.getOpcode()) {
2858 #define HANDLE_INST(NUM, OPCODE, CLASS) \
2859  case Instruction::OPCODE: \
2860  return translate##OPCODE(Inst, *CurBuilder.get());
2861 #include "llvm/IR/Instruction.def"
2862  default:
2863  return false;
2864  }
2865 }
2866 
2867 bool IRTranslator::translate(const Constant &C, Register Reg) {
2868  if (auto CI = dyn_cast<ConstantInt>(&C))
2869  EntryBuilder->buildConstant(Reg, *CI);
2870  else if (auto CF = dyn_cast<ConstantFP>(&C))
2871  EntryBuilder->buildFConstant(Reg, *CF);
2872  else if (isa<UndefValue>(C))
2873  EntryBuilder->buildUndef(Reg);
2874  else if (isa<ConstantPointerNull>(C))
2875  EntryBuilder->buildConstant(Reg, 0);
2876  else if (auto GV = dyn_cast<GlobalValue>(&C))
2877  EntryBuilder->buildGlobalValue(Reg, GV);
2878  else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
2879  if (!CAZ->getType()->isVectorTy())
2880  return false;
2881  // Return the scalar if it is a <1 x Ty> vector.
2882  if (CAZ->getNumElements() == 1)
2883  return translateCopy(C, *CAZ->getElementValue(0u), *EntryBuilder.get());
2885  for (unsigned i = 0; i < CAZ->getNumElements(); ++i) {
2886  Constant &Elt = *CAZ->getElementValue(i);
2887  Ops.push_back(getOrCreateVReg(Elt));
2888  }
2889  EntryBuilder->buildBuildVector(Reg, Ops);
2890  } else if (auto CV = dyn_cast<ConstantDataVector>(&C)) {
2891  // Return the scalar if it is a <1 x Ty> vector.
2892  if (CV->getNumElements() == 1)
2893  return translateCopy(C, *CV->getElementAsConstant(0),
2894  *EntryBuilder.get());
2896  for (unsigned i = 0; i < CV->getNumElements(); ++i) {
2897  Constant &Elt = *CV->getElementAsConstant(i);
2898  Ops.push_back(getOrCreateVReg(Elt));
2899  }
2900  EntryBuilder->buildBuildVector(Reg, Ops);
2901  } else if (auto CE = dyn_cast<ConstantExpr>(&C)) {
2902  switch(CE->getOpcode()) {
2903 #define HANDLE_INST(NUM, OPCODE, CLASS) \
2904  case Instruction::OPCODE: \
2905  return translate##OPCODE(*CE, *EntryBuilder.get());
2906 #include "llvm/IR/Instruction.def"
2907  default:
2908  return false;
2909  }
2910  } else if (auto CV = dyn_cast<ConstantVector>(&C)) {
2911  if (CV->getNumOperands() == 1)
2912  return translateCopy(C, *CV->getOperand(0), *EntryBuilder.get());
2914  for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
2915  Ops.push_back(getOrCreateVReg(*CV->getOperand(i)));
2916  }
2917  EntryBuilder->buildBuildVector(Reg, Ops);
2918  } else if (auto *BA = dyn_cast<BlockAddress>(&C)) {
2919  EntryBuilder->buildBlockAddress(Reg, BA);
2920  } else
2921  return false;
2922 
2923  return true;
2924 }
2925 
2926 void IRTranslator::finalizeBasicBlock() {
2927  for (auto &BTB : SL->BitTestCases) {
2928  // Emit header first, if it wasn't already emitted.
2929  if (!BTB.Emitted)
2930  emitBitTestHeader(BTB, BTB.Parent);
2931 
2932  BranchProbability UnhandledProb = BTB.Prob;
2933  for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
2934  UnhandledProb -= BTB.Cases[j].ExtraProb;
2935  // Set the current basic block to the mbb we wish to insert the code into
2936  MachineBasicBlock *MBB = BTB.Cases[j].ThisBB;
2937  // If all cases cover a contiguous range, it is not necessary to jump to
2938  // the default block after the last bit test fails. This is because the
2939  // range check during bit test header creation has guaranteed that every
2940  // case here doesn't go outside the range. In this case, there is no need
2941  // to perform the last bit test, as it will always be true. Instead, make
2942  // the second-to-last bit-test fall through to the target of the last bit
2943  // test, and delete the last bit test.
2944 
2945  MachineBasicBlock *NextMBB;
2946  if (BTB.ContiguousRange && j + 2 == ej) {
2947  // Second-to-last bit-test with contiguous range: fall through to the
2948  // target of the final bit test.
2949  NextMBB = BTB.Cases[j + 1].TargetBB;
2950  } else if (j + 1 == ej) {
2951  // For the last bit test, fall through to Default.
2952  NextMBB = BTB.Default;
2953  } else {
2954  // Otherwise, fall through to the next bit test.
2955  NextMBB = BTB.Cases[j + 1].ThisBB;
2956  }
2957 
2958  emitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j], MBB);
2959 
2960  // FIXME delete this block below?
2961  if (BTB.ContiguousRange && j + 2 == ej) {
2962  // Since we're not going to use the final bit test, remove it.
2963  BTB.Cases.pop_back();
2964  break;
2965  }
2966  }
2967  // This is "default" BB. We have two jumps to it. From "header" BB and from
2968  // last "case" BB, unless the latter was skipped.
2969  CFGEdge HeaderToDefaultEdge = {BTB.Parent->getBasicBlock(),
2970  BTB.Default->getBasicBlock()};
2971  addMachineCFGPred(HeaderToDefaultEdge, BTB.Parent);
2972  if (!BTB.ContiguousRange) {
2973  addMachineCFGPred(HeaderToDefaultEdge, BTB.Cases.back().ThisBB);
2974  }
2975  }
2976  SL->BitTestCases.clear();
2977 
2978  for (auto &JTCase : SL->JTCases) {
2979  // Emit header first, if it wasn't already emitted.
2980  if (!JTCase.first.Emitted)
2981  emitJumpTableHeader(JTCase.second, JTCase.first, JTCase.first.HeaderBB);
2982 
2983  emitJumpTable(JTCase.second, JTCase.second.MBB);
2984  }
2985  SL->JTCases.clear();
2986 
2987  for (auto &SwCase : SL->SwitchCases)
2988  emitSwitchCase(SwCase, &CurBuilder->getMBB(), *CurBuilder);
2989  SL->SwitchCases.clear();
2990 }
2991 
2992 void IRTranslator::finalizeFunction() {
2993  // Release the memory used by the different maps we
2994  // needed during the translation.
2995  PendingPHIs.clear();
2996  VMap.reset();
2997  FrameIndices.clear();
2998  MachinePreds.clear();
2999  // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
3000  // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
3001  // destroying it twice (in ~IRTranslator() and ~LLVMContext())
3002  EntryBuilder.reset();
3003  CurBuilder.reset();
3004  FuncInfo.clear();
3005 }
3006 
3007 /// Returns true if a BasicBlock \p BB within a variadic function contains a
3008 /// variadic musttail call.
3009 static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB) {
3010  if (!IsVarArg)
3011  return false;
3012 
3013  // Walk the block backwards, because tail calls usually only appear at the end
3014  // of a block.
3015  return std::any_of(BB.rbegin(), BB.rend(), [](const Instruction &I) {
3016  const auto *CI = dyn_cast<CallInst>(&I);
3017  return CI && CI->isMustTailCall();
3018  });
3019 }
3020 
3022  MF = &CurMF;
3023  const Function &F = MF->getFunction();
3024  if (F.empty())
3025  return false;
3027  getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
3028  // Set the CSEConfig and run the analysis.
3029  GISelCSEInfo *CSEInfo = nullptr;
3030  TPC = &getAnalysis<TargetPassConfig>();
3031  bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
3033  : TPC->isGISelCSEEnabled();
3034 
3035  if (EnableCSE) {
3036  EntryBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
3037  CSEInfo = &Wrapper.get(TPC->getCSEConfig());
3038  EntryBuilder->setCSEInfo(CSEInfo);
3039  CurBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
3040  CurBuilder->setCSEInfo(CSEInfo);
3041  } else {
3042  EntryBuilder = std::make_unique<MachineIRBuilder>();
3043  CurBuilder = std::make_unique<MachineIRBuilder>();
3044  }
3045  CLI = MF->getSubtarget().getCallLowering();
3046  CurBuilder->setMF(*MF);
3047  EntryBuilder->setMF(*MF);
3048  MRI = &MF->getRegInfo();
3049  DL = &F.getParent()->getDataLayout();
3050  ORE = std::make_unique<OptimizationRemarkEmitter>(&F);
3051  const TargetMachine &TM = MF->getTarget();
3052  TM.resetTargetOptions(F);
3053  EnableOpts = OptLevel != CodeGenOpt::None && !skipFunction(F);
3054  FuncInfo.MF = MF;
3055  if (EnableOpts)
3056  FuncInfo.BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
3057  else
3058  FuncInfo.BPI = nullptr;
3059 
3060  FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(*MF);
3061 
3062  const auto &TLI = *MF->getSubtarget().getTargetLowering();
3063 
3064  SL = std::make_unique<GISelSwitchLowering>(this, FuncInfo);
3065  SL->init(TLI, TM, *DL);
3066 
3067 
3068 
3069  assert(PendingPHIs.empty() && "stale PHIs");
3070 
3071  if (!DL->isLittleEndian()) {
3072  // Currently we don't properly handle big endian code.
3073  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3074  F.getSubprogram(), &F.getEntryBlock());
3075  R << "unable to translate in big endian mode";
3076  reportTranslationError(*MF, *TPC, *ORE, R);
3077  }
3078 
3079  // Release the per-function state when we return, whether we succeeded or not.
3080  auto FinalizeOnReturn = make_scope_exit([this]() { finalizeFunction(); });
3081 
3082  // Setup a separate basic-block for the arguments and constants
3083  MachineBasicBlock *EntryBB = MF->CreateMachineBasicBlock();
3084  MF->push_back(EntryBB);
3085  EntryBuilder->setMBB(*EntryBB);
3086 
3087  DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHI()->getDebugLoc();
3088  SwiftError.setFunction(CurMF);
3089  SwiftError.createEntriesInEntryBlock(DbgLoc);
3090 
3091  bool IsVarArg = F.isVarArg();
3092  bool HasMustTailInVarArgFn = false;
3093 
3094  // Create all blocks, in IR order, to preserve the layout.
3095  for (const BasicBlock &BB: F) {
3096  auto *&MBB = BBToMBB[&BB];
3097 
3098  MBB = MF->CreateMachineBasicBlock(&BB);
3099  MF->push_back(MBB);
3100 
3101  if (BB.hasAddressTaken())
3103 
3104  if (!HasMustTailInVarArgFn)
3105  HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB);
3106  }
3107 
3108  MF->getFrameInfo().setHasMustTailInVarArgFunc(HasMustTailInVarArgFn);
3109 
3110  // Make our arguments/constants entry block fallthrough to the IR entry block.
3111  EntryBB->addSuccessor(&getMBB(F.front()));
3112 
3113  if (CLI->fallBackToDAGISel(F)) {
3114  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3115  F.getSubprogram(), &F.getEntryBlock());
3116  R << "unable to lower function: " << ore::NV("Prototype", F.getType());
3117  reportTranslationError(*MF, *TPC, *ORE, R);
3118  return false;
3119  }
3120 
3121  // Lower the actual args into this basic block.
3122  SmallVector<ArrayRef<Register>, 8> VRegArgs;
3123  for (const Argument &Arg: F.args()) {
3124  if (DL->getTypeStoreSize(Arg.getType()).isZero())
3125  continue; // Don't handle zero sized types.
3126  ArrayRef<Register> VRegs = getOrCreateVRegs(Arg);
3127  VRegArgs.push_back(VRegs);
3128 
3129  if (Arg.hasSwiftErrorAttr()) {
3130  assert(VRegs.size() == 1 && "Too many vregs for Swift error");
3131  SwiftError.setCurrentVReg(EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
3132  }
3133  }
3134 
3135  if (!CLI->lowerFormalArguments(*EntryBuilder.get(), F, VRegArgs, FuncInfo)) {
3136  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3137  F.getSubprogram(), &F.getEntryBlock());
3138  R << "unable to lower arguments: " << ore::NV("Prototype", F.getType());
3139  reportTranslationError(*MF, *TPC, *ORE, R);
3140  return false;
3141  }
3142 
3143  // Need to visit defs before uses when translating instructions.
3144  GISelObserverWrapper WrapperObserver;
3145  if (EnableCSE && CSEInfo)
3146  WrapperObserver.addObserver(CSEInfo);
3147  {
3149 #ifndef NDEBUG
3150  DILocationVerifier Verifier;
3151  WrapperObserver.addObserver(&Verifier);
3152 #endif // ifndef NDEBUG
3153  RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
3154  RAIIMFObserverInstaller ObsInstall(*MF, WrapperObserver);
3155  for (const BasicBlock *BB : RPOT) {
3156  MachineBasicBlock &MBB = getMBB(*BB);
3157  // Set the insertion point of all the following translations to
3158  // the end of this basic block.
3159  CurBuilder->setMBB(MBB);
3160  HasTailCall = false;
3161  for (const Instruction &Inst : *BB) {
3162  // If we translated a tail call in the last step, then we know
3163  // everything after the call is either a return, or something that is
3164  // handled by the call itself. (E.g. a lifetime marker or assume
3165  // intrinsic.) In this case, we should stop translating the block and
3166  // move on.
3167  if (HasTailCall)
3168  break;
3169 #ifndef NDEBUG
3170  Verifier.setCurrentInst(&Inst);
3171 #endif // ifndef NDEBUG
3172  if (translate(Inst))
3173  continue;
3174 
3175  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3176  Inst.getDebugLoc(), BB);
3177  R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
3178 
3179  if (ORE->allowExtraAnalysis("gisel-irtranslator")) {
3180  std::string InstStrStorage;
3181  raw_string_ostream InstStr(InstStrStorage);
3182  InstStr << Inst;
3183 
3184  R << ": '" << InstStr.str() << "'";
3185  }
3186 
3187  reportTranslationError(*MF, *TPC, *ORE, R);
3188  return false;
3189  }
3190 
3191  finalizeBasicBlock();
3192  }
3193 #ifndef NDEBUG
3194  WrapperObserver.removeObserver(&Verifier);
3195 #endif
3196  }
3197 
3198  finishPendingPhis();
3199 
3200  SwiftError.propagateVRegs();
3201 
3202  // Merge the argument lowering and constants block with its single
3203  // successor, the LLVM-IR entry block. We want the basic block to
3204  // be maximal.
3205  assert(EntryBB->succ_size() == 1 &&
3206  "Custom BB used for lowering should have only one successor");
3207  // Get the successor of the current entry block.
3208  MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
3209  assert(NewEntryBB.pred_size() == 1 &&
3210  "LLVM-IR entry block has a predecessor!?");
3211  // Move all the instruction from the current entry block to the
3212  // new entry block.
3213  NewEntryBB.splice(NewEntryBB.begin(), EntryBB, EntryBB->begin(),
3214  EntryBB->end());
3215 
3216  // Update the live-in information for the new entry block.
3217  for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
3218  NewEntryBB.addLiveIn(LiveIn);
3219  NewEntryBB.sortUniqueLiveIns();
3220 
3221  // Get rid of the now empty basic block.
3222  EntryBB->removeSuccessor(&NewEntryBB);
3223  MF->remove(EntryBB);
3224  MF->DeleteMachineBasicBlock(EntryBB);
3225 
3226  assert(&MF->front() == &NewEntryBB &&
3227  "New entry wasn't next in the list of basic block!");
3228 
3229  // Initialize stack protector information.
3230  StackProtector &SP = getAnalysis<StackProtector>();
3232 
3233  return false;
3234 }
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
llvm::EHPersonality::MSVC_CXX
@ MSVC_CXX
i
i
Definition: README.txt:29
llvm::HexagonInstrInfo::isTailCall
bool isTailCall(const MachineInstr &MI) const override
Definition: HexagonInstrInfo.cpp:2611
llvm::MachineFunction::remove
void remove(iterator MBBI)
Definition: MachineFunction.h:769
llvm::isAsynchronousEHPersonality
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
Definition: EHPersonalities.h:50
llvm::TargetMachine::getOptLevel
CodeGenOpt::Level getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
Definition: TargetMachine.cpp:198
Int32Ty
IntegerType * Int32Ty
Definition: NVVMIntrRange.cpp:67
llvm::DbgVariableIntrinsic::getExpression
DIExpression * getExpression() const
Definition: IntrinsicInst.h:164
llvm::MachineFrameInfo::hasVarSizedObjects
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
Definition: MachineFrameInfo.h:351
llvm::Argument
This class represents an incoming formal argument to a Function.
Definition: Argument.h:29
llvm::MachineIRBuilder::buildInsertVectorElement
MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Elt, const SrcOp &Idx)
Build and insert Res = G_INSERT_VECTOR_ELT Val, Elt, Idx.
Definition: MachineIRBuilder.cpp:765
llvm::RISCVAttrs::StackAlign
StackAlign
Definition: RISCVAttributes.h:37
StackProtector.h
llvm::Type::isSized
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:272
llvm::OptimizationRemarkMissed
Diagnostic information for missed-optimization remarks.
Definition: DiagnosticInfo.h:728
llvm::Function::isIntrinsic
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:200
llvm::TargetLoweringBase::getStackPointerRegisterToSaveRestore
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
Definition: TargetLowering.h:1710
LowLevelType.h
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:100
MathExtras.h
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:132
llvm::SwitchCG::sortAndRangeify
void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
Definition: SwitchLoweringUtils.cpp:464
llvm
This class represents lattice values for constants.
Definition: AllocatorList.h:23
llvm::DbgDeclareInst::getAddress
Value * getAddress() const
Definition: IntrinsicInst.h:201
Reg
unsigned Reg
Definition: MachineSink.cpp:1540
llvm::MCSymbol
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
llvm::MachineIRBuilder::buildFMul
MachineInstrBuilder buildFMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Definition: MachineIRBuilder.h:1441
llvm::MachineIRBuilder::buildAtomicRMW
MachineInstrBuilder buildAtomicRMW(unsigned Opcode, const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val, MachineMemOperand &MMO)
Build and insert OldValRes<def> = G_ATOMICRMW_<Opcode> Addr, Val, MMO.
Definition: MachineIRBuilder.cpp:828
llvm::MachineIRBuilder::setDebugLoc
void setDebugLoc(const DebugLoc &DL)
Set the debug location to DL for all the next build instructions.
Definition: MachineIRBuilder.h:363
llvm::CmpInst::ICMP_EQ
@ ICMP_EQ
equal
Definition: InstrTypes.h:743
llvm::MachineInstrBuilder::addFPImm
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
Definition: MachineInstrBuilder.h:142
llvm::SystemZISD::TM
@ TM
Definition: SystemZISelLowering.h:65
llvm::ReturnInst
Return a value (possibly void), from a function.
Definition: Instructions.h:2933
TargetFrameLowering.h
llvm::MachineIRBuilder::buildICmp
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
Definition: MachineIRBuilder.cpp:739
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:111
llvm::BasicBlock::getFirstNonPHIOrDbg
const Instruction * getFirstNonPHIOrDbg(bool SkipPseudoOp=false) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
Definition: BasicBlock.cpp:219
llvm::Value::hasOneUse
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:437
llvm::GISelCSEAnalysisWrapperPass
The actual analysis pass wrapper.
Definition: CSEInfo.h:220
llvm::MachineBasicBlock::getBasicBlock
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
Definition: MachineBasicBlock.h:202
llvm::generic_gep_type_iterator
Definition: GetElementPtrTypeIterator.h:31
llvm::RecurKind::FMul
@ FMul
Product of floats.
Metadata.h
llvm::ISD::LIFETIME_END
@ LIFETIME_END
Definition: ISDOpcodes.h:1128
llvm::Type::getInt8PtrTy
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:249
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:722
llvm::MachineModuleInfo::getContext
const MCContext & getContext() const
Definition: MachineModuleInfo.h:167
llvm::AllocaInst::getAlign
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:119
CallLowering.h
llvm::BasicBlock::getParent
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:107
llvm::InlineAsmLowering
Definition: InlineAsmLowering.h:28
IntrinsicInst.h
ceil
We have fiadd patterns now but the followings have the same cost and complexity We need a way to specify the later is more profitable def def The FP stackifier should handle simple permutates to reduce number of shuffle e g ceil
Definition: README-FPStack.txt:54
llvm::GlobalValue::dropLLVMManglingEscape
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition: GlobalValue.h:482
llvm::HexagonISD::JT
@ JT
Definition: HexagonISelLowering.h:52
Loads.h
llvm::SwitchCG::CaseBlock
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
Definition: SwitchLoweringUtils.h:109
llvm::SwitchCG::CaseBlock::DbgLoc
DebugLoc DbgLoc
Definition: SwitchLoweringUtils.h:138
llvm::MCContext
Context object for machine code objects.
Definition: MCContext.h:68
llvm::Function
Definition: Function.h:61
llvm::SwitchCG::SwitchWorkList
SmallVector< SwitchWorkListItem, 4 > SwitchWorkList
Definition: SwitchLoweringUtils.h:247
llvm::MachineFunction::getContext
MCContext & getContext() const
Definition: MachineFunction.h:507
llvm::lower_bound
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1608
llvm::ISD::LIFETIME_START
@ LIFETIME_START
This corresponds to the llvm.lifetime.
Definition: ISDOpcodes.h:1127
llvm::MachineInstr::copyFlagsFromInstruction
static uint16_t copyFlagsFromInstruction(const Instruction &I)
Definition: MachineInstr.cpp:534
Pass.h
llvm::TargetSubtargetInfo::getInstrInfo
virtual const TargetInstrInfo * getInstrInfo() const
Definition: TargetSubtargetInfo.h:92
llvm::IntrinsicInst::getIntrinsicID
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:51
llvm::BlockFrequencyInfoWrapperPass
Legacy analysis pass which computes BlockFrequencyInfo.
Definition: BlockFrequencyInfo.h:138
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition: Instructions.h:728
llvm::StackProtector::copyToMachineFrameInfo
void copyToMachineFrameInfo(MachineFrameInfo &MFI) const
Definition: StackProtector.cpp:583
llvm::DataLayout::getTypeSizeInBits
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:661
llvm::raw_string_ostream
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:614
minimum
Should compile r2 movcc movcs str strb mov lr r1 movcs movcc mov lr r1 str mov mov cmp r1 movlo r2 str bx lr r0 mov mov cmp r0 movhs r2 mov r1 bx lr Some of the NEON intrinsics may be appropriate for more general either as target independent intrinsics or perhaps elsewhere in the ARM backend Some of them may also be lowered to target independent and perhaps some new SDNodes could be added For minimum
Definition: README.txt:489
GetElementPtrTypeIterator.h
llvm::MachineIRBuilder::buildSplatVector
MachineInstrBuilder buildSplatVector(const DstOp &Res, const SrcOp &Src)
Build and insert Res = G_BUILD_VECTOR with Src replicated to fill the number of elements.
Definition: MachineIRBuilder.cpp:634
High
uint64_t High
Definition: NVVMIntrRange.cpp:61
llvm::ReturnInst::getReturnValue
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
Definition: Instructions.h:2978
llvm::AllocaInst::getType
PointerType * getType() const
Overload to return most specific pointer type.
Definition: Instructions.h:103
llvm::ilist_node_with_parent::getNextNode
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:288
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
Wrapper
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
Definition: AMDGPUAliasAnalysis.cpp:30
llvm::RISCVFenceField::W
@ W
Definition: RISCVBaseInfo.h:130
InlineAsm.h
llvm::LandingPadInst
The landingpad instruction holds all of the information necessary to generate correct exception handl...
Definition: Instructions.h:2832
llvm::MachineFunction::getMachineMemOperand
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Definition: MachineFunction.cpp:430
llvm::enumerate
detail::enumerator< R > enumerate(R &&TheRange)
Given an input range, returns a new range whose values are are pair (A,B) such that A is the 0-based ...
Definition: STLExtras.h:1916
llvm::CallBase::isInlineAsm
bool isInlineAsm() const
Check if this call is an inline asm statement.
Definition: InstrTypes.h:1453
llvm::DataLayout::getTypeStoreSize
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Definition: DataLayout.h:465
llvm::FunctionPass::skipFunction
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Definition: Pass.cpp:163
llvm::Function::getSubprogram
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1513
llvm::InlineAsmLowering::lowerInlineAsm
bool lowerInlineAsm(MachineIRBuilder &MIRBuilder, const CallBase &CB, std::function< ArrayRef< Register >(const Value &Val)> GetOrCreateVRegs) const
Lower the given inline asm call instruction GetOrCreateVRegs is a callback to materialize a register ...
Definition: InlineAsmLowering.cpp:273
llvm::GISelCSEInfo
The CSE Analysis object.
Definition: CSEInfo.h:69
ErrorHandling.h
llvm::Function::getEntryBlock
const BasicBlock & getEntryBlock() const
Definition: Function.h:731
llvm::IRTranslator::runOnMachineFunction
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
Definition: IRTranslator.cpp:3021
llvm::SwiftErrorValueTracking::createEntriesInEntryBlock
bool createEntriesInEntryBlock(DebugLoc DbgLoc)
Create initial definitions of swifterror values in the entry block of the current function.
Definition: SwiftErrorValueTracking.cpp:115
llvm::MachineMemOperand::MOInvariant
@ MOInvariant
The memory access always returns the same value (or traps).
Definition: MachineMemOperand.h:144
llvm::SwiftErrorValueTracking::propagateVRegs
void propagateVRegs()
Propagate assigned swifterror vregs through a function, synthesizing PHI nodes when needed to maintai...
Definition: SwiftErrorValueTracking.cpp:147
llvm::ISD::EH_LABEL
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:942
llvm::CmpInst::ICMP_NE
@ ICMP_NE
not equal
Definition: InstrTypes.h:744
R600_InstFlag::FC
@ FC
Definition: R600Defines.h:32
llvm::TargetLoweringBase::getTgtMemIntrinsic
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
Definition: TargetLowering.h:998
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
llvm::MachineBasicBlock::setSuccProbability
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
Definition: MachineBasicBlock.cpp:1426
llvm::MachineIRBuilder::buildZExtOrTrunc
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
Definition: MachineIRBuilder.cpp:479
ValueTracking.h
llvm::SwitchCG::CC_JumpTable
@ CC_JumpTable
A cluster of cases suitable for jump table lowering.
Definition: SwitchLoweringUtils.h:36
OptimizationRemarkEmitter.h
MachineBasicBlock.h
llvm::SwitchCG::SwitchWorkListItem
Definition: SwitchLoweringUtils.h:239
llvm::TargetSubtargetInfo::getRegisterInfo
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Definition: TargetSubtargetInfo.h:124
llvm::LoadInst::getPointerOperandType
Type * getPointerOperandType() const
Definition: Instructions.h:269
llvm::DataLayout::getStructLayout
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:671
llvm::GISelObserverWrapper::removeObserver
void removeObserver(GISelChangeObserver *O)
Definition: GISelChangeObserver.h:77
reportTranslationError
IRTranslator LLVM IR static false void reportTranslationError(MachineFunction &MF, const TargetPassConfig &TPC, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R)
Definition: IRTranslator.cpp:103
llvm::TargetRegisterInfo
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Definition: TargetRegisterInfo.h:231
llvm::AAMDNodes
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:640
llvm::Function::getContext
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:300
llvm::MachineIRBuilder::buildDbgLabel
MachineInstrBuilder buildDbgLabel(const MDNode *Label)
Build and insert a DBG_LABEL instructions specifying that Label is given.
Definition: MachineIRBuilder.cpp:117
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:46
llvm::BranchProbability::getZero
static BranchProbability getZero()
Definition: BranchProbability.h:49
TargetInstrInfo.h
llvm::MachineMemOperand
A description of a memory reference used in the backend.
Definition: MachineMemOperand.h:127
llvm::MachineMemOperand::MODereferenceable
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
Definition: MachineMemOperand.h:142
llvm::TargetLoweringBase::getExceptionPointerRegister
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
Definition: TargetLowering.h:1717
llvm::MachineIRBuilder::buildFPTrunc
MachineInstrBuilder buildFPTrunc(const DstOp &Res, const SrcOp &Op, Optional< unsigned > Flags=None)
Build and insert Res = G_FPTRUNC Op.
Definition: MachineIRBuilder.cpp:733
llvm::MachineFunction::insert
void insert(iterator MBBI, MachineBasicBlock *MBB)
Definition: MachineFunction.h:756
llvm::copy
OutputIt copy(R &&Range, OutputIt Out)
Definition: STLExtras.h:1549
llvm::SmallSet
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:134
llvm::CallLowering::supportSwiftError
virtual bool supportSwiftError() const
Definition: CallLowering.h:321
llvm::SwitchCG::CaseClusterIt
CaseClusterVector::iterator CaseClusterIt
Definition: SwitchLoweringUtils.h:87
llvm::DbgVariableIntrinsic::getVariable
DILocalVariable * getVariable() const
Definition: IntrinsicInst.h:160
llvm::FenceInst
An instruction for ordering other memory operations.
Definition: Instructions.h:444
Offset
uint64_t Offset
Definition: ELFObjHandler.cpp:81
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:449
llvm::ore::NV
DiagnosticInfoOptimizationBase::Argument NV
Definition: OptimizationRemarkEmitter.h:128
llvm::getSelectionDAGFallbackAnalysisUsage
void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU)
Modify analysis usage so it preserves passes required for the SelectionDAG fallback.
Definition: Utils.cpp:656
llvm::TargetLoweringBase::getVectorIdxTy
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
Definition: TargetLowering.h:388
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition: InstrTypes.h:752
llvm::codeview::EncodedFramePtrReg::StackPtr
@ StackPtr
llvm::MachineIRBuilder::buildInstrNoInsert
MachineInstrBuilder buildInstrNoInsert(unsigned Opcode)
Build but don't insert <empty> = Opcode <empty>.
Definition: MachineIRBuilder.cpp:40
llvm::MipsISD::Ret
@ Ret
Definition: MipsISelLowering.h:116
llvm::Intrinsic::not_intrinsic
@ not_intrinsic
Definition: Intrinsics.h:45
llvm::CallBase::getNumArgOperands
unsigned getNumArgOperands() const
Definition: InstrTypes.h:1330
STLExtras.h
llvm::CallBase::arg_begin
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1297
llvm::MachineBasicBlock::addSuccessorWithoutProb
void addSuccessorWithoutProb(MachineBasicBlock *Succ)
Add Succ as a successor of this MachineBasicBlock.
Definition: MachineBasicBlock.cpp:749
llvm::LoadInst::getPointerOperand
Value * getPointerOperand()
Definition: Instructions.h:266
llvm::SwitchCG::JumpTableHeader::Last
APInt Last
Definition: SwitchLoweringUtils.h:182
llvm::gep_type_begin
gep_type_iterator gep_type_begin(const User *GEP)
Definition: GetElementPtrTypeIterator.h:133
llvm::successors
succ_range successors(Instruction *I)
Definition: CFG.h:260
llvm::MachineIRBuilder::getDebugLoc
DebugLoc getDebugLoc()
Get the current instruction's debug location.
Definition: MachineIRBuilder.h:366
llvm::MachineFrameInfo::CreateVariableSizedObject
int CreateVariableSizedObject(Align Alignment, const AllocaInst *Alloca)
Notify the MachineFrameInfo object that a variable sized object has been created.
Definition: MachineFrameInfo.cpp:74
llvm::RAIIMFObserverInstaller
A simple RAII based Observer installer.
Definition: GISelChangeObserver.h:119
llvm::MachineIRBuilder::buildFence
MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope)
Build and insert G_FENCE Ordering, Scope.
Definition: MachineIRBuilder.cpp:936
llvm::BitmaskEnumDetail::Mask
std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1541
llvm::MachineIRBuilder::buildDynStackAlloc
MachineInstrBuilder buildDynStackAlloc(const DstOp &Res, const SrcOp &Size, Align Alignment)
Build and insert Res = G_DYN_STACKALLOC Size, Align.
Definition: MachineIRBuilder.cpp:126
INITIALIZE_PASS_END
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
Definition: RegBankSelect.cpp:69
llvm::FunctionLoweringInfo::CanLowerReturn
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
Definition: FunctionLoweringInfo.h:63
llvm::Type::getInt32Ty
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:197
llvm::SwitchCG::CaseBlock::CmpMHS
const Value * CmpMHS
Definition: SwitchLoweringUtils.h:127
llvm::APIntOps::umin
const APInt & umin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be signed.
Definition: APInt.h:2183
llvm::ArrayRef::empty
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:158
llvm::MachineFunctionPass::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition: MachineFunctionPass.cpp:102
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:122
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1187
llvm::MachineIRBuilder::buildConstant
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
Definition: MachineIRBuilder.cpp:255
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::TargetPassConfig::isGISelCSEEnabled
virtual bool isGISelCSEEnabled() const
Check whether continuous CSE should be enabled in GISel passes.
Definition: TargetPassConfig.cpp:1461
llvm::RISCVFenceField::R
@ R
Definition: RISCVBaseInfo.h:129
MachineRegisterInfo.h
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
a
=0.0 ? 0.0 :(a > 0.0 ? 1.0 :-1.0) a
Definition: README.txt:489
llvm::gep_type_end
gep_type_iterator gep_type_end(const User *GEP)
Definition: GetElementPtrTypeIterator.h:140
floor
We have fiadd patterns now but the followings have the same cost and complexity We need a way to specify the later is more profitable def def The FP stackifier should handle simple permutates to reduce number of shuffle e g floor
Definition: README-FPStack.txt:54
llvm::DILabel::isValidLocationForIntrinsic
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this label.
Definition: DebugInfoMetadata.h:3157
llvm::getLLTForType
LLT getLLTForType(Type &Ty, const DataLayout &DL)
Construct a low-level type based on an LLVM type.
Definition: LowLevelType.cpp:21
Context
LLVMContext & Context
Definition: NVVMIntrRange.cpp:66
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
llvm::classifyEHPersonality
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
Definition: EHPersonalities.cpp:21
EnableCSEInIRTranslator
static cl::opt< bool > EnableCSEInIRTranslator("enable-cse-in-irtranslator", cl::desc("Should enable CSE in irtranslator"), cl::Optional, cl::init(false))
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:205
llvm::MachineBasicBlock::addSuccessor
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Definition: MachineBasicBlock.cpp:739
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:876
llvm::DbgValueInst::getValue
Value * getValue() const
Definition: IntrinsicInst.h:232
llvm::MachineBasicBlock::pred_size
unsigned pred_size() const
Definition: MachineBasicBlock.h:328
TargetLowering.h
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:77
llvm::AllocaInst::isStaticAlloca
bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size.
Definition: Instructions.cpp:1373
llvm::Instruction::getOpcode
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:160
llvm::FenceInst::getSyncScopeID
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this fence instruction.
Definition: Instructions.h:481
llvm::MachineFunction::front
const MachineBasicBlock & front() const
Definition: MachineFunction.h:749
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:565
llvm::RAIIDelegateInstaller
A simple RAII based Delegate installer.
Definition: GISelChangeObserver.h:107
llvm::MachineIRBuilder::buildAtomicCmpXchgWithSuccess
MachineInstrBuilder buildAtomicCmpXchgWithSuccess(Register OldValRes, Register SuccessRes, Register Addr, Register CmpVal, Register NewVal, MachineMemOperand &MMO)
Build and insert OldValRes<def>, SuccessRes<def> = G_ATOMIC_CMPXCHG_WITH_SUCCESS Addr,...
Definition: MachineIRBuilder.cpp:776
llvm::TargetInstrInfo
TargetInstrInfo - Interface to description of machine instruction set.
Definition: TargetInstrInfo.h:97
TargetMachine.h
llvm::SwitchCG::CaseBlock::FalseBB
MachineBasicBlock * FalseBB
Definition: SwitchLoweringUtils.h:130
llvm::MachineInstrBuilder::addMBB
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:147
llvm::MachineIRBuilder::buildExtractVectorElement
MachineInstrBuilder buildExtractVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
Definition: MachineIRBuilder.cpp:771
llvm::MachineIRBuilder::buildLoad
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
Definition: MachineIRBuilder.h:834
InlineAsmLowering.h
llvm::CmpInst::isFPPredicate
bool isFPPredicate() const
Definition: InstrTypes.h:816
Constants.h
llvm::PatternMatch::match
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
llvm::CallLowering::checkReturnTypeForCallConv
bool checkReturnTypeForCallConv(MachineFunction &MF) const
Toplevel function to check the return type based on the target calling convention.
Definition: CallLowering.cpp:723
llvm::MachineIRBuilder::buildShl
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Definition: MachineIRBuilder.h:1471
llvm::PHINode::getIncomingValue
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
Definition: Instructions.h:2676
llvm::DbgValueInst
This represents the llvm.dbg.value instruction.
Definition: IntrinsicInst.h:230
llvm::Instruction::getAAMetadata
void getAAMetadata(AAMDNodes &N, bool Merge=false) const
Fills the AAMDNodes structure with AA metadata from this instruction.
Definition: TypeBasedAliasAnalysis.cpp:524
llvm::AllocaInst::getAllocatedType
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:112
llvm::MachineIRBuilder::setMBB
void setMBB(MachineBasicBlock &MBB)
Set the insertion point to the end of MBB.
Definition: MachineIRBuilder.h:332
llvm::BranchProbabilityInfoWrapperPass
Legacy analysis pass which computes BranchProbabilityInfo.
Definition: BranchProbabilityInfo.h:446
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::MachineBasicBlock::succ_end
succ_iterator succ_end()
Definition: MachineBasicBlock.h:334
llvm::User
Definition: User.h:44
llvm::LLT::getSizeInBits
unsigned getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelTypeImpl.h:109
llvm::succ_size
unsigned succ_size(const Instruction *I)
Definition: CFG.h:257
llvm::TargetFrameLowering::getStackAlign
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
Definition: TargetFrameLowering.h:99
Intrinsics.h
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::ConstrainedFPIntrinsic
This is the common base class for constrained floating point intrinsics.
Definition: IntrinsicInst.h:310
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:748
llvm::getLLTForMVT
LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
Definition: LowLevelType.cpp:55
round
static uint64_t round(uint64_t Acc, uint64_t Input)
Definition: xxhash.cpp:57
InstrTypes.h
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
Definition: InstrTypes.h:1387
llvm::TargetMachine::getIntrinsicInfo
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
Definition: TargetMachine.h:209
llvm::TargetLowering
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Definition: TargetLowering.h:3143
llvm::CallLowering::lowerFormalArguments
virtual bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F, ArrayRef< ArrayRef< Register >> VRegs, FunctionLoweringInfo &FLI) const
This hook must be implemented to lower the incoming (formal) arguments, described by VRegs,...
Definition: CallLowering.h:416
MCContext.h
llvm::CallLowering::lowerReturn
virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, ArrayRef< Register > VRegs, FunctionLoweringInfo &FLI, Register SwiftErrorVReg) const
This hook must be implemented to lower outgoing return values, described by Val, into the specified v...
Definition: CallLowering.h:386
llvm::FCmpInst
This instruction compares its operands according to the predicate given to the constructor.
Definition: Instructions.h:1344
llvm::BranchProbabilityInfo
Analysis providing branch probability information.
Definition: BranchProbabilityInfo.h:115
llvm::StackProtector
Definition: StackProtector.h:37
llvm::LLT::getSizeInBytes
unsigned getSizeInBytes() const
Returns the total size of the type in bytes, i.e.
Definition: LowLevelTypeImpl.h:117
SI
@ SI
Definition: SIInstrInfo.cpp:7366
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
b
the resulting code requires compare and branches when and if the revised code is with conditional branches instead of More there is a byte word extend before each where there should be only and the condition codes are not remembered when the same two values are compared twice More LSR enhancements i8 and i32 load store addressing modes are identical int b
Definition: README.txt:418
llvm::RegState::Undef
@ Undef
Value of the register doesn't matter.
Definition: MachineInstrBuilder.h:53
llvm::MachineIRBuilder::buildFCmp
MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, Optional< unsigned > Flags=None)
Build and insert a Res = G_FCMP PredOp0, Op1.
Definition: MachineIRBuilder.cpp:746
false
Definition: StackSlotColoring.cpp:142
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:129
llvm::MachineFunction::getTypeIDFor
unsigned getTypeIDFor(const GlobalValue *TI)
Return the type id for the specified typeinfo. This is function wide.
Definition: MachineFunction.cpp:841
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
First
into llvm powi allowing the code generator to produce balanced multiplication trees First
Definition: README.txt:54
llvm::isUIntN
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:456
llvm::make_scope_exit
LLVM_NODISCARD detail::scope_exit< typename std::decay< Callable >::type > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:58
llvm::Constant::getAllOnesValue
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:405
llvm::Type::isTokenTy
bool isTokenTy() const
Return true if this is 'token'.
Definition: Type.h:199
llvm::MachineBasicBlock::RegisterMaskPair
Pair of physical register and lane mask.
Definition: MachineBasicBlock.h:101
llvm::Instruction
Definition: Instruction.h:45
llvm::SwiftErrorValueTracking::getOrCreateVRegUseAt
Register getOrCreateVRegUseAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a use of a swifterror by an instruction.
Definition: SwiftErrorValueTracking.cpp:65
llvm::DataLayout::getABITypeAlign
Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:826
llvm::SwitchCG::CaseBlock::PredInfoPair::NoCmp
bool NoCmp
Definition: SwitchLoweringUtils.h:114
llvm::AllocaInst::getArraySize
const Value * getArraySize() const
Get the number of elements allocated.
Definition: Instructions.h:99
llvm::report_fatal_error
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:140
llvm::AtomicRMWInst::Nand
@ Nand
*p = ~(old & v)
Definition: Instructions.h:724
SwitchLoweringUtils.h
llvm::cl::Option::getNumOccurrences
int getNumOccurrences() const
Definition: CommandLine.h:401
llvm::DILocalVariable::isValidLocationForIntrinsic
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
Definition: DebugInfoMetadata.h:3087
llvm::wasm::ValType
ValType
Definition: Wasm.h:378
llvm::FunctionLoweringInfo::BPI
BranchProbabilityInfo * BPI
Definition: FunctionLoweringInfo.h:59
llvm::MachineIRBuilder::buildAnd
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
Definition: MachineIRBuilder.h:1500
llvm::LLT::vector
static LLT vector(uint16_t NumElements, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
Definition: LowLevelTypeImpl.h:58
IR
Statically lint checks LLVM IR
Definition: Lint.cpp:742
llvm::SwitchCG::CaseBlock::FalseProb
BranchProbability FalseProb
Definition: SwitchLoweringUtils.h:141
Info
Analysis containing CSE Info
Definition: CSEInfo.cpp:26
getOffsetFromIndices
static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL)
Definition: IRTranslator.cpp:1349
Align
uint64_t Align
Definition: ELFObjHandler.cpp:83
PatternMatch.h
llvm::BasicBlock::getFirstNonPHI
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:212
llvm::MachineIRBuilder::buildIndirectDbgValue
MachineInstrBuilder buildIndirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in me...
Definition: MachineIRBuilder.cpp:65
llvm::FixedVectorType::get
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:643
llvm::CallInst::isTailCall
bool isTailCall() const
Definition: Instructions.h:1654
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::MachineIRBuilder::buildBrCond
MachineInstrBuilder buildBrCond(const SrcOp &Tst, MachineBasicBlock &Dest)
Build and insert G_BRCOND Tst, Dest.
Definition: MachineIRBuilder.cpp:332
llvm::AtomicRMWInst::Xchg
@ Xchg
*p = v
Definition: Instructions.h:716
llvm::PHINode::getNumIncomingValues
unsigned getNumIncomingValues() const
Return the number of incoming edges.
Definition: Instructions.h:2672
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition: Instructions.h:718
llvm::EHPersonality::Wasm_CXX
@ Wasm_CXX
llvm::LoadInst::getSyncScopeID
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this load instruction.
Definition: Instructions.h:241
llvm::MachineFunction::addInvoke
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
Definition: MachineFunction.cpp:696
llvm::MachineInstr::NoFPExcept
@ NoFPExcept
Definition: MachineInstr.h:106
Type.h
llvm::MachineInstrBuilder::getReg
Register getReg(unsigned Idx) const
Get the register for the operand index.
Definition: MachineInstrBuilder.h:95
llvm::SwitchCG::JumpTableHeader::First
APInt First
Definition: SwitchLoweringUtils.h:181
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:95
llvm::MachineIRBuilder::buildBrIndirect
MachineInstrBuilder buildBrIndirect(Register Tgt)
Build and insert G_BRINDIRECT Tgt.
Definition: MachineIRBuilder.cpp:222
LowLevelTypeImpl.h
llvm::Instruction::getMetadata
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:277
CFG.h
llvm::maxnum
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE maxNum semantics.
Definition: APFloat.h:1297
llvm::AllocaInst::isSwiftError
bool isSwiftError() const
Return true if this alloca is used as a swifterror argument to a call.
Definition: Instructions.h:147
llvm::BranchInst::getCondition
Value * getCondition() const
Definition: Instructions.h:3096
llvm::PatternMatch::m_ExtractElt
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
Definition: PatternMatch.h:1453
llvm::TargetRegisterInfo::getPointerRegClass
virtual const TargetRegisterClass * getPointerRegClass(const MachineFunction &MF, unsigned Kind=0) const
Returns a TargetRegisterClass used for pointer values.
Definition: TargetRegisterInfo.h:743
llvm::CmpInst::FCMP_FALSE
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:724
llvm::InvokeInst
Invoke instruction.
Definition: Instructions.h:3696
llvm::CmpInst
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:712
llvm::TargetPassConfig
Target-Independent Code Generator Pass Configuration Options.
Definition: TargetPassConfig.h:84
llvm::MachineInstr::copyIRFlags
void copyIRFlags(const Instruction &I)
Copy all flags to MachineInst MIFlags.
Definition: MachineInstr.cpp:572
llvm::MachineIRBuilder::buildBr
MachineInstrBuilder buildBr(MachineBasicBlock &Dest)
Build and insert G_BR Dest.
Definition: MachineIRBuilder.cpp:218
llvm::DataLayout::getPrefTypeAlign
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:835
llvm::MachineFunction::getMMI
MachineModuleInfo & getMMI() const
Definition: MachineFunction.h:506
llvm::MachineBasicBlock::succ_iterator
std::vector< MachineBasicBlock * >::iterator succ_iterator
Definition: MachineBasicBlock.h:306
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:555
llvm::MachineInstrBuilder::addFrameIndex
const MachineInstrBuilder & addFrameIndex(int Idx) const
Definition: MachineInstrBuilder.h:153
Verifier
verify safepoint Safepoint IR Verifier
Definition: SafepointIRVerifier.cpp:253
llvm::AtomicRMWInst::UMin
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:736
BasicBlock.h
llvm::cl::opt< bool >
llvm::CallLowering::fallBackToDAGISel
virtual bool fallBackToDAGISel(const Function &F) const
Definition: CallLowering.h:404
llvm::MachineFunction::push_back
void push_back(MachineBasicBlock *MBB)
Definition: MachineFunction.h:754
llvm::MachineIRBuilder::buildFAdd
MachineInstrBuilder buildFAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_FADD Op0, Op1.
Definition: MachineIRBuilder.h:1566
llvm::MachineFrameInfo::setHasMustTailInVarArgFunc
void setHasMustTailInVarArgFunc(bool B)
Definition: MachineFrameInfo.h:624
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:303
llvm::GlobalValue
Definition: GlobalValue.h:44
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::countPopulation
unsigned countPopulation(T Value)
Count the number of set bits in a value.
Definition: MathExtras.h:568
llvm::MachineIRBuilder::buildBrJT
MachineInstrBuilder buildBrJT(Register TablePtr, unsigned JTI, Register IndexReg)
Build and insert G_BRJT TablePtr, JTI, IndexReg.
Definition: MachineIRBuilder.cpp:227
llvm::SmallSet::count
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:164
llvm::getUnderlyingObjects
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
Definition: ValueTracking.cpp:4210
llvm::MachineIRBuilder::buildSExtOrTrunc
MachineInstrBuilder buildSExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
Definition: MachineIRBuilder.cpp:474
BranchProbabilityInfo.h
llvm::MachineIRBuilder
Helper class to build MachineInstr.
Definition: MachineIRBuilder.h:220
llvm::AtomicRMWInst::Sub
@ Sub
*p = old - v
Definition: Instructions.h:720
llvm::CallBase::countOperandBundlesOfType
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Definition: InstrTypes.h:1936
llvm::ICmpInst
This instruction compares its operands according to the predicate given to the constructor.
Definition: Instructions.h:1178
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:62
llvm::MachineInstrBuilder
Definition: MachineInstrBuilder.h:70
llvm::PatternMatch::m_LogicalOr
LogicalOp_match< LHS, RHS, Instruction::Or > m_LogicalOr(const LHS &L, const RHS &R)
Matches L || R either in the form of L | R or L ? true : R.
Definition: PatternMatch.h:2453
llvm::SwitchCG::CaseBlock::PredInfo
struct PredInfoPair PredInfo
Definition: SwitchLoweringUtils.h:121
llvm::ExtractTypeInfo
GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
Definition: Analysis.cpp:168
llvm::GlobalValue::getParent
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:572
llvm::DbgDeclareInst
This represents the llvm.dbg.declare instruction.
Definition: IntrinsicInst.h:199
const
aarch64 promote const
Definition: AArch64PromoteConstant.cpp:232
llvm::DbgLabelInst
This represents the llvm.dbg.label instruction.
Definition: IntrinsicInst.h:248
Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:80
llvm::SwitchCG::CaseBlock::TrueProb
BranchProbability TrueProb
Definition: SwitchLoweringUtils.h:141
llvm::MachineBasicBlock::setIsEHPad
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
Definition: MachineBasicBlock.h:439
INITIALIZE_PASS_DEPENDENCY
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
llvm::DbgLabelInst::getLabel
DILabel * getLabel() const
Definition: IntrinsicInst.h:250
llvm::MachineIRBuilder::buildMul
MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_MUL Op0, Op1.
Definition: MachineIRBuilder.h:1423
llvm::MachineIRBuilder::getInsertPt
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
Definition: MachineIRBuilder.h:309
llvm::DataLayout::isLittleEndian
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:240
llvm::MachinePointerInfo
This class contains a discriminated union of information about pointers in memory operands,...
Definition: MachineMemOperand.h:37
move
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
Definition: README.txt:546
llvm::MachineIRBuilder::buildPtrAdd
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert Res = G_PTR_ADD Op0, Op1.
Definition: MachineIRBuilder.cpp:182
DebugInfo.h
llvm::TargetLoweringBase::getAtomicMemOperandFlags
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
Definition: TargetLoweringBase.cpp:2227
I
#define I(x, y, z)
Definition: MD5.cpp:59
Analysis.h
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:732
llvm::countTrailingOnes
unsigned countTrailingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the least significant bit to the first zero bit.
Definition: MathExtras.h:526
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:440
llvm::LLVMContext::OB_cfguardtarget
@ OB_cfguardtarget
Definition: LLVMContext.h:93
llvm::LLT::isVector
bool isVector() const
Definition: LowLevelTypeImpl.h:96
llvm::SwitchCG::CaseBlock::CmpRHS
const Value * CmpRHS
Definition: SwitchLoweringUtils.h:127
llvm::AtomicRMWInst::Or
@ Or
*p = old | v
Definition: Instructions.h:726
llvm::MachineFunction::CreateMachineBasicBlock
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
Definition: MachineFunction.cpp:414
llvm::TargetLoweringBase::isFMAFasterThanFMulAndFAdd
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
Definition: TargetLowering.h:2694
TargetPassConfig.h
llvm::HighlightColor::Address
@ Address
llvm::MachineIRBuilder::buildSelect
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1, Optional< unsigned > Flags=None)
Build and insert a Res = G_SELECT Tst, Op0, Op1.
Definition: MachineIRBuilder.cpp:755
llvm::SwitchCG::CaseBlock::ThisBB
MachineBasicBlock * ThisBB
Definition: SwitchLoweringUtils.h:133
llvm::MachineMemOperand::Flags
Flags
Flags values. These may be or'd together.
Definition: MachineMemOperand.h:130
llvm::MachineFunction::getName
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Definition: MachineFunction.cpp:522
llvm::BranchProbability::isUnknown
bool isUnknown() const
Definition: BranchProbability.h:47
maximum
Should compile r2 movcc movcs str strb mov lr r1 movcs movcc mov lr r1 str mov mov cmp r1 movlo r2 str bx lr r0 mov mov cmp r0 movhs r2 mov r1 bx lr Some of the NEON intrinsics may be appropriate for more general either as target independent intrinsics or perhaps elsewhere in the ARM backend Some of them may also be lowered to target independent and perhaps some new SDNodes could be added For maximum
Definition: README.txt:489
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::TargetMachine
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:77
llvm::MachineIRBuilder::buildConstDbgValue
MachineInstrBuilder buildConstDbgValue(const Constant &C, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instructions specifying that Variable is given by C (suitably modified b...
Definition: MachineIRBuilder.cpp:92
llvm::LLT::isPointer
bool isPointer() const
Definition: LowLevelTypeImpl.h:94
llvm::SwitchCG::JumpTableHeader::OmitRangeCheck
bool OmitRangeCheck
Definition: SwitchLoweringUtils.h:186
llvm::Instruction::hasMetadata
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
Definition: Instruction.h:259
llvm::MVT::getSizeInBits
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
Definition: MachineValueType.h:806
llvm::MachineIRBuilder::getMBB
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
Definition: MachineIRBuilder.h:295
llvm::MachineBasicBlock::succ_begin
succ_iterator succ_begin()
Definition: MachineBasicBlock.h:332
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:944
llvm::MachineBasicBlock::setHasAddressTaken
void setHasAddressTaken()
Set this block to reflect that it potentially is the target of an indirect branch.
Definition: MachineBasicBlock.h:215
llvm::MachineFunction::getFrameInfo
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Definition: MachineFunction.h:571
llvm::DataLayout::getPointerABIAlignment
Align getPointerABIAlignment(unsigned AS) const
Layout pointer alignment.
Definition: DataLayout.cpp:694
memcpy
<%struct.s * > cast struct s *S to sbyte *< sbyte * > sbyte uint cast struct s *agg result to sbyte *< sbyte * > sbyte uint cast struct s *memtmp to sbyte *< sbyte * > sbyte uint ret void llc ends up issuing two memcpy or custom lower memcpy(of small size) to be ldmia/stmia. I think option 2 is better but the current register allocator cannot allocate a chunk of registers at a time. A feasible temporary solution is to use specific physical registers at the lowering time for small(<
llvm::IndexedInstrProf::HashT::Last
@ Last
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition: MachineBasicBlock.h:225
llvm::TargetLoweringBase::getStoreMemOperandFlags
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
Definition: TargetLoweringBase.cpp:2211
llvm::FenceInst::getOrdering
AtomicOrdering getOrdering() const
Returns the ordering constraint of this fence instruction.
Definition: Instructions.h:470
llvm::MachineInstrBuilder::addMemOperand
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
Definition: MachineInstrBuilder.h:203
llvm::SelectInst
This class represents the LLVM 'select' instruction.
Definition: Instructions.h:1725
llvm::SwitchCG::JumpTableHeader
Definition: SwitchLoweringUtils.h:180
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
MachineModuleInfo.h
llvm::WinEH::EncodingType::CE
@ CE
Windows NT (Windows on ARM)
llvm::MachineRegisterInfo::createGenericVirtualRegister
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
Definition: MachineRegisterInfo.cpp:188
llvm::SwiftErrorValueTracking::setFunction
void setFunction(MachineFunction &MF)
Initialize data structures for specified new function.
Definition: SwiftErrorValueTracking.cpp:79
llvm::MachineInstrBuilder::addUse
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
Definition: MachineInstrBuilder.h:124
llvm::MDNode
Metadata node.
Definition: Metadata.h:886
llvm::MachineIRBuilder::buildFMA
MachineInstrBuilder buildFMA(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, const SrcOp &Src2, Optional< unsigned > Flags=None)
Build and insert Res = G_FMA Op0, Op1, Op2.
Definition: MachineIRBuilder.h:1587
llvm::SwitchCG::CC_Range
@ CC_Range
A cluster of adjacent case labels with the same destination, or just one case.
Definition: SwitchLoweringUtils.h:34
llvm::MCContext::getOrCreateFrameAllocSymbol
MCSymbol * getOrCreateFrameAllocSymbol(StringRef FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
Definition: MCContext.cpp:155
llvm::EHPersonality::CoreCLR
@ CoreCLR
llvm::CallBase::arg_end
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1303
llvm::Function::getFunction
const Function & getFunction() const
Definition: Function.h:135
llvm::BranchInst::isUnconditional
bool isUnconditional() const
Definition: Instructions.h:3093
llvm::PatternMatch::m_Value
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:76
llvm::BranchProbabilityInfo::getEdgeProbability
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
Definition: BranchProbabilityInfo.cpp:1133
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:70
llvm::MachineFunction
Definition: MachineFunction.h:227
llvm::TargetSubtargetInfo::getCallLowering
virtual const CallLowering * getCallLowering() const
Definition: TargetSubtargetInfo.h:100
llvm::CmpInst::isIntPredicate
bool isIntPredicate() const
Definition: InstrTypes.h:817
llvm::CodeGenOpt::None
@ None
Definition: CodeGen.h:53
llvm::MachineIRBuilder::buildInstr
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
Definition: MachineIRBuilder.h:375
llvm::APIntOps::smin
const APInt & smin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be signed.
Definition: APInt.h:2173
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::ConstrainedFPIntrinsic::getExceptionBehavior
Optional< fp::ExceptionBehavior > getExceptionBehavior() const
Definition: IntrinsicInst.cpp:117
llvm::OptimizationRemarkEmitter
The optimization diagnostic interface.
Definition: OptimizationRemarkEmitter.h:33
llvm::min
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:339
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1505
DataLayout.h
llvm::MachineFunction::DeleteMachineBasicBlock
void DeleteMachineBasicBlock(MachineBasicBlock *MBB)
DeleteMachineBasicBlock - Delete the given MachineBasicBlock.
Definition: MachineFunction.cpp:421
llvm::MachineFunction::allocateShuffleMask
ArrayRef< int > allocateShuffleMask(ArrayRef< int > Mask)
Definition: MachineFunction.cpp:510
llvm::StructType
Class to represent struct types.
Definition: DerivedTypes.h:212
llvm::MachineFrameInfo::CreateStackObject
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
Definition: MachineFrameInfo.cpp:51
llvm::countTrailingZeros
unsigned countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: MathExtras.h:157
Cond
SmallVector< MachineOperand, 4 > Cond
Definition: BasicBlockSections.cpp:167
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:57
llvm::MachineBasicBlock::splice
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Definition: MachineBasicBlock.h:829
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI", false, false) INITIALIZE_PASS_END(IRTranslator
llvm::SwiftErrorValueTracking::getOrCreateVRegDefAt
Register getOrCreateVRegDefAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a def of a swifterror by an instruction.
Definition: SwiftErrorValueTracking.cpp:50
llvm::SwitchCG::CaseClusterVector
std::vector< CaseCluster > CaseClusterVector
Definition: SwitchLoweringUtils.h:86
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:136
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:246
llvm::ConstantInt::isZero
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:202
llvm::MachineIRBuilder::buildCopy
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
Definition: MachineIRBuilder.cpp:238
llvm::MachineIRBuilder::buildFPExt
MachineInstrBuilder buildFPExt(const DstOp &Res, const SrcOp &Op, Optional< unsigned > Flags=None)
Build and insert Res = G_FPEXT Op.
Definition: MachineIRBuilder.h:624
llvm::BranchProbability
Definition: BranchProbability.h:30
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:871
IRTranslator.h
TargetSubtargetInfo.h
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
isSwiftError
static bool isSwiftError(const Value *V)
Definition: IRTranslator.cpp:1257
trunc
We have fiadd patterns now but the followings have the same cost and complexity We need a way to specify the later is more profitable def def The FP stackifier should handle simple permutates to reduce number of shuffle e g trunc
Definition: README-FPStack.txt:63
llvm::SwitchCG::BitTestCase
Definition: SwitchLoweringUtils.h:195
llvm::CodeGenOpt::Level
Level
Definition: CodeGen.h:52
llvm::SmallSet::insert
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:180
llvm::LoadInst::getOrdering
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
Definition: Instructions.h:231
llvm::GISelChangeObserver
Abstract class that contains various methods for clients to notify about changes.
Definition: GISelChangeObserver.h:28
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:174
llvm::EHPersonality
EHPersonality
Definition: EHPersonalities.h:22
llvm::MachineMemOperand::MOVolatile
@ MOVolatile
The memory access is volatile.
Definition: MachineMemOperand.h:138
llvm::CallBase::paramHasAttr
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
Definition: Instructions.cpp:326
llvm::LLVMContext::OB_deopt
@ OB_deopt
Definition: LLVMContext.h:90
llvm::SwitchCG::BitTestBlock
Definition: SwitchLoweringUtils.h:208
llvm::MachineInstrBuilder::getInstr
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Definition: MachineInstrBuilder.h:90
llvm::LLT::changeElementType
LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
Definition: LowLevelTypeImpl.h:127
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:702
llvm::MachineMemOperand::MOLoad
@ MOLoad
The memory access reads data.
Definition: MachineMemOperand.h:134
llvm::Value::stripPointerCasts
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:633
llvm::MachineIRBuilder::materializePtrAdd
Optional< MachineInstrBuilder > materializePtrAdd(Register &Res, Register Op0, const LLT ValueTy, uint64_t Value)
Materialize and insert Res = G_PTR_ADD Op0, (G_CONSTANT Value)
Definition: MachineIRBuilder.cpp:193
llvm::MachineIRBuilder::buildJumpTable
MachineInstrBuilder buildJumpTable(const LLT PtrTy, unsigned JTI)
Build and insert Res = G_JUMP_TABLE JTI.
Definition: MachineIRBuilder.cpp:159
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::MachineBasicBlock::addLiveIn
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
Definition: MachineBasicBlock.h:367
llvm::MachineBasicBlock::isPredecessor
bool isPredecessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a predecessor of this block.
Definition: MachineBasicBlock.cpp:897
llvm::SwitchCG::JumpTable
Definition: SwitchLoweringUtils.h:165
llvm::cl::Optional
@ Optional
Definition: CommandLine.h:116
llvm::MachineIRBuilder::buildFrameIndex
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
Definition: MachineIRBuilder.cpp:137
llvm::Triple::isOSWindows
bool isOSWindows() const
Tests whether the OS is Windows.
Definition: Triple.h:545
MBB
MachineBasicBlock & MBB
Definition: AArch64SLSHardening.cpp:74
llvm::TargetPassConfig::getCSEConfig
virtual std::unique_ptr< CSEConfigBase > getCSEConfig() const
Returns the CSEConfig object to use for the current optimization level.
Definition: TargetPassConfig.cpp:1465
llvm::computeValueLLTs
void computeValueLLTs(const DataLayout &DL, Type &Ty, SmallVectorImpl< LLT > &ValueTys, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
computeValueLLTs - Given an LLVM IR type, compute a sequence of LLTs that represent all the individua...
Definition: Analysis.cpp:132
j
return j(j<< 16)
llvm::MachineFunctionProperties::Property::FailedISel
@ FailedISel
llvm::APIntOps::umax
const APInt & umax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be unsigned.
Definition: APInt.h:2188
llvm::DataLayout::getIntPtrType
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
Definition: DataLayout.cpp:839
Constant.h
DEBUG_TYPE
#define DEBUG_TYPE
Definition: IRTranslator.cpp:84
llvm::minnum
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE minNum semantics.
Definition: APFloat.h:1286
llvm::commonAlignment
Align commonAlignment(Align A, Align B)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:221
llvm::TargetLoweringBase::getExceptionSelectorRegister
virtual Register getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
Definition: TargetLowering.h:1724
llvm::ConstantInt::getTrue
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:824
llvm::Constant::getNullValue
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:347
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition: MachineFunction.h:521
llvm::IndirectBrInst::getAddress
Value * getAddress()
Definition: Instructions.h:3636
llvm::TargetSubtargetInfo::getFrameLowering
virtual const TargetFrameLowering * getFrameLowering() const
Definition: TargetSubtargetInfo.h:93
llvm::IRTranslator::ID
static char ID
Definition: IRTranslator.h:64
llvm::MachineBasicBlock::sortUniqueLiveIns
void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
Definition: MachineBasicBlock.cpp:574
uint16_t
llvm::ConstrainedFPIntrinsic::isUnaryOp
bool isUnaryOp() const
Definition: IntrinsicInst.cpp:148
llvm::MachineFunction::getTarget
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Definition: MachineFunction.h:551
llvm::AtomicRMWInst::FSub
@ FSub
*p = old - v
Definition: Instructions.h:742
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:302
llvm::StructLayout::getElementOffset
uint64_t getElementOffset(unsigned Idx) const
Definition: DataLayout.h:644
llvm::ArrayRef::begin
iterator begin() const
Definition: ArrayRef.h:151
llvm::X86::FirstMacroFusionInstKind::Cmp
@ Cmp
llvm::MachineBasicBlock::insert
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
Definition: MachineBasicBlock.cpp:1305
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
MachineFrameInfo.h
llvm::Align::value
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
llvm::FPOpFusion::Strict
@ Strict
Definition: TargetOptions.h:39
llvm::MachineIRBuilder::buildUndef
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
Definition: MachineIRBuilder.cpp:575
llvm::MachineIRBuilder::buildDirectDbgValue
MachineInstrBuilder buildDirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in Re...
Definition: MachineIRBuilder.cpp:52
llvm::ExtractValueInst
This instruction extracts a struct member or array element value from an aggregate value.
Definition: Instructions.h:2328
Success
#define Success
Definition: AArch64Disassembler.cpp:248
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition: Instructions.h:722
llvm::TargetRegisterInfo::getCustomEHPadPreservedMask
virtual const uint32_t * getCustomEHPadPreservedMask(const MachineFunction &MF) const
Return a register mask for the registers preserved by the unwinder, or nullptr if no custom mask is n...
Definition: TargetRegisterInfo.h:480
llvm::FunctionLoweringInfo::clear
void clear()
clear - Clear out all the function-specific state.
Definition: FunctionLoweringInfo.cpp:359
Casting.h
llvm::pdb::PDB_LocType::Slot
@ Slot
Function.h
llvm::sort
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1439
llvm::Type::getPointerTo
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:708
llvm::MachineRegisterInfo::addPhysRegsUsedFromRegMask
void addPhysRegsUsedFromRegMask(const uint32_t *RegMask)
addPhysRegsUsedFromRegMask - Mark any registers not in RegMask as used.
Definition: MachineRegisterInfo.h:851
powi
This is blocked on not handling X *X *X powi(X, 3)(see note above). The issue is that we end up getting t
llvm::SwitchCG::CaseBlock::CmpLHS
const Value * CmpLHS
Definition: SwitchLoweringUtils.h:127
llvm::log2
static double log2(double V)
Definition: AMDGPULibCalls.cpp:841
llvm::fp::ebIgnore
@ ebIgnore
This corresponds to "fpexcept.ignore".
Definition: FPEnv.h:32
llvm::TargetSubtargetInfo::getInlineAsmLowering
virtual const InlineAsmLowering * getInlineAsmLowering() const
Definition: TargetSubtargetInfo.h:102
llvm::ConstrainedFPIntrinsic::isTernaryOp
bool isTernaryOp() const
Definition: IntrinsicInst.cpp:159
MemRef
Definition: Lint.cpp:84
llvm::MachineFrameInfo::setStackProtectorIndex
void setStackProtectorIndex(int I)
Definition: MachineFrameInfo.h:355
llvm::GISelCSEAnalysisWrapper
Simple wrapper that does the following.
Definition: CSEInfo.h:202
llvm::Instruction::hasAllowReassoc
bool hasAllowReassoc() const
Determine whether the allow-reassociation flag is set.
Definition: Instruction.cpp:223
llvm::MachineBasicBlock::normalizeSuccProbs
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
Definition: MachineBasicBlock.h:563
llvm::UnivariateLinearPolyBase::isZero
bool isZero() const
Definition: TypeSize.h:225
llvm::CallBase::getCalledOperand
Value * getCalledOperand() const
Definition: InstrTypes.h:1380
llvm::Function::getPersonalityFn
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1665
llvm::MachineMemOperand::MOStore
@ MOStore
The memory access writes data.
Definition: MachineMemOperand.h:136
llvm::ReversePostOrderTraversal
Definition: PostOrderIterator.h:288
llvm::SwitchCG::CaseBlock::PredInfoPair::Pred
CmpInst::Predicate Pred
Definition: SwitchLoweringUtils.h:112
llvm::IndirectBrInst
Indirect Branch Instruction.
Definition: Instructions.h:3561
llvm::TargetLoweringBase::fallBackToDAGISel
virtual bool fallBackToDAGISel(const Instruction &Inst) const
Definition: TargetLowering.h:527
llvm::makeArrayRef
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:474
llvm::IRTranslator::IRTranslator
IRTranslator(CodeGenOpt::Level OptLevel=CodeGenOpt::None)
Definition: IRTranslator.cpp:120
CodeGen.h
llvm::Instruction::BinaryOps
BinaryOps
Definition: Instruction.h:761
llvm::BasicBlock::back
const Instruction & back() const
Definition: BasicBlock.h:310
Instructions.h
PostOrderIterator.h
llvm::FunctionLoweringInfo::MF
MachineFunction * MF
Definition: FunctionLoweringInfo.h:56
llvm::MachineInstrBuilder::addSym
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
Definition: MachineInstrBuilder.h:268
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition: User.h:191
llvm::TargetSubtargetInfo::getTargetLowering
virtual const TargetLowering * getTargetLowering() const
Definition: TargetSubtargetInfo.h:96
llvm::MachineIRBuilder::buildIntrinsic
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects)
Build and insert either a G_INTRINSIC (if HasSideEffects is false) or G_INTRINSIC_W_SIDE_EFFECTS inst...
Definition: MachineIRBuilder.cpp:704
llvm::BranchProbability::normalizeProbabilities
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Definition: BranchProbability.h:205
SmallVector.h
llvm::PatternMatch::m_Specific
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
Definition: PatternMatch.h:758
llvm::MachinePointerInfo::getFixedStack
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Definition: MachineOperand.cpp:995
llvm::MachineBasicBlock::begin
iterator begin()
Definition: MachineBasicBlock.h:268
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:365
llvm::MachineRegisterInfo::getType
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
Definition: MachineRegisterInfo.h:732
User.h
MachineInstrBuilder.h
llvm::IRTranslator::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
Definition: IRTranslator.cpp:161
llvm::GISelObserverWrapper::addObserver
void addObserver(GISelChangeObserver *O)
Definition: GISelChangeObserver.h:74
llvm::MachineInstr::NoUWrap
@ NoUWrap
Definition: MachineInstr.h:100
llvm::SwitchCG::JumpTableHeader::SValue
const Value * SValue
Definition: SwitchLoweringUtils.h:183
llvm::getMVTForLLT
MVT getMVTForLLT(LLT Ty)
Get a rough equivalent of an MVT for a given LLT.
Definition: LowLevelType.cpp:46
TargetIntrinsicInfo.h
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:745
llvm::fp::ExceptionBehavior
ExceptionBehavior
Exception behavior used for floating point operations.
Definition: FPEnv.h:31
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1332
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:94
llvm::SwitchCG::CaseBlock::TrueBB
MachineBasicBlock * TrueBB
Definition: SwitchLoweringUtils.h:130
llvm::TargetMachine::getTargetTriple
const Triple & getTargetTriple() const
Definition: TargetMachine.h:123
llvm::MachineIRBuilder::buildSub
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_SUB Op0, Op1.
Definition: MachineIRBuilder.h:1407
llvm::PHINode::getIncomingBlock
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Definition: Instructions.h:2696
llvm::ArrayRef::size
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:163
llvm::max
Align max(MaybeAlign Lhs, Align Rhs)
Definition: Alignment.h:350
llvm::AtomicRMWInst::FAdd
@ FAdd
*p = old + v
Definition: Instructions.h:739
getConstrainedOpcode
static unsigned getConstrainedOpcode(Intrinsic::ID ID)
Definition: IRTranslator.cpp:1772
llvm::pdb::PDB_DataKind::Global
@ Global
llvm::PHINode
Definition: Instructions.h:2582
ScopeExit.h
llvm::IRTranslator
Definition: IRTranslator.h:62
llvm::MachineInstrBuilder::addMetadata
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
Definition: MachineInstrBuilder.h:237
MachineMemOperand.h
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:43
MachineOperand.h
llvm::SwitchCG::CC_BitTests
@ CC_BitTests
A cluster of cases suitable for bit test lowering.
Definition: SwitchLoweringUtils.h:38
llvm::GISelObserverWrapper
Simple wrapper observer that takes several observers, and calls each one for each event.
Definition: GISelChangeObserver.h:65
isValInBlock
static bool isValInBlock(const Value *V, const BasicBlock *BB)
Definition: IRTranslator.cpp:394
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1164
DerivedTypes.h
llvm::DataLayout::getPointerSizeInBits
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Definition: DataLayout.h:403
checkForMustTailInVarArgFn
static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB)
Returns true if a BasicBlock BB within a variadic function contains a variadic musttail call.
Definition: IRTranslator.cpp:3009
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1450
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::MIPatternMatch::m_Not
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
Definition: MIPatternMatch.h:502
llvm::Type::isAggregateType
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition: Type.h:265
llvm::MachineIRBuilder::buildFreeze
MachineInstrBuilder buildFreeze(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_FREEZE Src.
Definition: MachineIRBuilder.h:1365
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition: PassAnalysisSupport.h:75
LLVMContext.h
llvm::SwitchInst
Multiway switch.
Definition: Instructions.h:3159
llvm::OptimizedStructLayoutField
A field in a structure.
Definition: OptimizedStructLayout.h:44
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::AMDGPU::HSAMD::Kernel::Key::Args
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Definition: AMDGPUMetadata.h:379
llvm::Type::TypeID
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:55
llvm::AllocaInst
an instruction to allocate memory on the stack
Definition: Instructions.h:61
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::MachineFunction::addLandingPad
MCSymbol * addLandingPad(MachineBasicBlock *LandingPad)
Add a new panding pad, and extract the exception handling information from the landingpad instruction...
Definition: MachineFunction.cpp:703
GISelChangeObserver.h
llvm::cl::desc
Definition: CommandLine.h:411
llvm::MachineIRBuilder::buildStore
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
Definition: MachineIRBuilder.cpp:388
llvm::TargetLoweringBase::getValueType
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
Definition: TargetLowering.h:1386
llvm::BranchInst
Conditional or Unconditional Branch instruction.
Definition: Instructions.h:3015
llvm::AtomicRMWInst::UMax
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:734
raw_ostream.h
llvm::SI::KernelInputOffsets::Offsets
Offsets
Offsets in bytes from the start of the input buffer.
Definition: SIInstrInfo.h:1224
llvm::MachineIRBuilder::buildAdd
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_ADD Op0, Op1.
Definition: MachineIRBuilder.h:1390
MachineFunction.h
llvm::CallBase::arg_operands
iterator_range< User::op_iterator > arg_operands()
Definition: InstrTypes.h:1324
llvm::InsertValueInst
This instruction inserts a struct field of array element value into an aggregate value.
Definition: Instructions.h:2439
llvm::CmpInst::FCMP_TRUE
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:739
llvm::LLT::scalar
static LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition: LowLevelTypeImpl.h:43
Value.h
llvm::abs
APFloat abs(APFloat X)
Returns the absolute value of the argument.
Definition: APFloat.h:1272
llvm::raw_string_ostream::str
std::string & str()
Flushes the stream contents to the target string and returns the string's reference.
Definition: raw_ostream.h:632
llvm::MachineMemOperand::MONone
@ MONone
Definition: MachineMemOperand.h:132
llvm::MIPatternMatch::m_OneUse
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
Definition: MIPatternMatch.h:38
InitializePasses.h
llvm::Value
LLVM Value Representation.
Definition: Value.h:75
llvm::AtomicCmpXchgInst
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:522
TargetRegisterInfo.h
Debug.h
llvm::MachineFunction::setVariableDbgInfo
void setVariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr, int Slot, const DILocation *Loc)
Collect information used to emit debugging information of a variable.
Definition: MachineFunction.h:1084
SubReg
unsigned SubReg
Definition: AArch64AdvSIMDScalarPass.cpp:104
llvm::ArrayRef::end
iterator end() const
Definition: ArrayRef.h:152
llvm::APIntOps::smax
const APInt & smax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be signed.
Definition: APInt.h:2178
llvm::PatternMatch::m_LogicalAnd
LogicalOp_match< LHS, RHS, Instruction::And > m_LogicalAnd(const LHS &L, const RHS &R)
Matches L && R either in the form of L & R or L ? R : false.
Definition: PatternMatch.h:2445
llvm::SwiftErrorValueTracking::getFunctionArg
const Value * getFunctionArg() const
Get the (unique) function argument that was marked swifterror, or nullptr if this function has no swi...
Definition: SwiftErrorValueTracking.h:74
llvm::MachineRegisterInfo::setRegClass
void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
Definition: MachineRegisterInfo.cpp:58
llvm::BranchInst::getSuccessor
BasicBlock * getSuccessor(unsigned i) const
Definition: Instructions.h:3108
llvm::CallBase::args
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition: InstrTypes.h:1313
llvm::SwiftErrorValueTracking::setCurrentVReg
void setCurrentVReg(const MachineBasicBlock *MBB, const Value *, Register)
Set the swifterror virtual register in the VRegDefMap for this basic block.
Definition: SwiftErrorValueTracking.cpp:45
llvm::MachineIRBuilder::buildCast
MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src)
Build and insert an appropriate cast between two registers of equal size.
Definition: MachineIRBuilder.cpp:489
SmallSet.h
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:364
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:37
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:730
llvm::DataLayout::getTypeAllocSize
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:497
llvm::LLT
Definition: LowLevelTypeImpl.h:40
llvm::CallLowering::lowerCall
virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info) const
This hook must be implemented to lower the given call instruction, including argument and return valu...
Definition: CallLowering.h:428