LLVM  14.0.0git
IRTranslator.cpp
Go to the documentation of this file.
1 //===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the IRTranslator class.
10 //===----------------------------------------------------------------------===//
11 
14 #include "llvm/ADT/STLExtras.h"
15 #include "llvm/ADT/ScopeExit.h"
16 #include "llvm/ADT/SmallSet.h"
17 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/Analysis/Loads.h"
22 #include "llvm/CodeGen/Analysis.h"
45 #include "llvm/IR/BasicBlock.h"
46 #include "llvm/IR/CFG.h"
47 #include "llvm/IR/Constant.h"
48 #include "llvm/IR/Constants.h"
49 #include "llvm/IR/DataLayout.h"
50 #include "llvm/IR/DebugInfo.h"
51 #include "llvm/IR/DerivedTypes.h"
52 #include "llvm/IR/DiagnosticInfo.h"
53 #include "llvm/IR/Function.h"
55 #include "llvm/IR/InlineAsm.h"
56 #include "llvm/IR/InstrTypes.h"
57 #include "llvm/IR/Instructions.h"
58 #include "llvm/IR/IntrinsicInst.h"
59 #include "llvm/IR/Intrinsics.h"
60 #include "llvm/IR/LLVMContext.h"
61 #include "llvm/IR/Metadata.h"
62 #include "llvm/IR/PatternMatch.h"
63 #include "llvm/IR/Type.h"
64 #include "llvm/IR/User.h"
65 #include "llvm/IR/Value.h"
66 #include "llvm/InitializePasses.h"
67 #include "llvm/MC/MCContext.h"
68 #include "llvm/Pass.h"
69 #include "llvm/Support/Casting.h"
70 #include "llvm/Support/CodeGen.h"
71 #include "llvm/Support/Debug.h"
79 #include <algorithm>
80 #include <cassert>
81 #include <cstddef>
82 #include <cstdint>
83 #include <iterator>
84 #include <string>
85 #include <utility>
86 #include <vector>
87 
88 #define DEBUG_TYPE "irtranslator"
89 
90 using namespace llvm;
91 
92 static cl::opt<bool>
93  EnableCSEInIRTranslator("enable-cse-in-irtranslator",
94  cl::desc("Should enable CSE in irtranslator"),
95  cl::Optional, cl::init(false));
96 char IRTranslator::ID = 0;
97 
98 INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
99  false, false)
107 
109  const TargetPassConfig &TPC,
112  MF.getProperties().set(MachineFunctionProperties::Property::FailedISel);
113 
114  // Print the function name explicitly if we don't have a debug location (which
115  // makes the diagnostic less useful) or if we're going to emit a raw error.
116  if (!R.getLocation().isValid() || TPC.isGlobalISelAbortEnabled())
117  R << (" (in function: " + MF.getName() + ")").str();
118 
119  if (TPC.isGlobalISelAbortEnabled())
120  report_fatal_error(Twine(R.getMsg()));
121  else
122  ORE.emit(R);
123 }
124 
126  : MachineFunctionPass(ID), OptLevel(optlevel) {}
127 
128 #ifndef NDEBUG
129 namespace {
130 /// Verify that every instruction created has the same DILocation as the
131 /// instruction being translated.
132 class DILocationVerifier : public GISelChangeObserver {
133  const Instruction *CurrInst = nullptr;
134 
135 public:
136  DILocationVerifier() = default;
137  ~DILocationVerifier() = default;
138 
139  const Instruction *getCurrentInst() const { return CurrInst; }
140  void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
141 
142  void erasingInstr(MachineInstr &MI) override {}
143  void changingInstr(MachineInstr &MI) override {}
144  void changedInstr(MachineInstr &MI) override {}
145 
146  void createdInstr(MachineInstr &MI) override {
147  assert(getCurrentInst() && "Inserted instruction without a current MI");
148 
149  // Only print the check message if we're actually checking it.
150 #ifndef NDEBUG
151  LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
152  << " was copied to " << MI);
153 #endif
154  // We allow insts in the entry block to have a debug loc line of 0 because
155  // they could have originated from constants, and we don't want a jumpy
156  // debug experience.
157  assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
158  MI.getDebugLoc().getLine() == 0) &&
159  "Line info was not transferred to all instructions");
160  }
161 };
162 } // namespace
163 #endif // ifndef NDEBUG
164 
165 
170  if (OptLevel != CodeGenOpt::None)
176 }
177 
179 IRTranslator::allocateVRegs(const Value &Val) {
180  auto VRegsIt = VMap.findVRegs(Val);
181  if (VRegsIt != VMap.vregs_end())
182  return *VRegsIt->second;
183  auto *Regs = VMap.getVRegs(Val);
184  auto *Offsets = VMap.getOffsets(Val);
185  SmallVector<LLT, 4> SplitTys;
186  computeValueLLTs(*DL, *Val.getType(), SplitTys,
187  Offsets->empty() ? Offsets : nullptr);
188  for (unsigned i = 0; i < SplitTys.size(); ++i)
189  Regs->push_back(0);
190  return *Regs;
191 }
192 
193 ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
194  auto VRegsIt = VMap.findVRegs(Val);
195  if (VRegsIt != VMap.vregs_end())
196  return *VRegsIt->second;
197 
198  if (Val.getType()->isVoidTy())
199  return *VMap.getVRegs(Val);
200 
201  // Create entry for this type.
202  auto *VRegs = VMap.getVRegs(Val);
203  auto *Offsets = VMap.getOffsets(Val);
204 
205  assert(Val.getType()->isSized() &&
206  "Don't know how to create an empty vreg");
207 
208  SmallVector<LLT, 4> SplitTys;
209  computeValueLLTs(*DL, *Val.getType(), SplitTys,
210  Offsets->empty() ? Offsets : nullptr);
211 
212  if (!isa<Constant>(Val)) {
213  for (auto Ty : SplitTys)
214  VRegs->push_back(MRI->createGenericVirtualRegister(Ty));
215  return *VRegs;
216  }
217 
218  if (Val.getType()->isAggregateType()) {
219  // UndefValue, ConstantAggregateZero
220  auto &C = cast<Constant>(Val);
221  unsigned Idx = 0;
222  while (auto Elt = C.getAggregateElement(Idx++)) {
223  auto EltRegs = getOrCreateVRegs(*Elt);
224  llvm::copy(EltRegs, std::back_inserter(*VRegs));
225  }
226  } else {
227  assert(SplitTys.size() == 1 && "unexpectedly split LLT");
228  VRegs->push_back(MRI->createGenericVirtualRegister(SplitTys[0]));
229  bool Success = translate(cast<Constant>(Val), VRegs->front());
230  if (!Success) {
231  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
232  MF->getFunction().getSubprogram(),
233  &MF->getFunction().getEntryBlock());
234  R << "unable to translate constant: " << ore::NV("Type", Val.getType());
235  reportTranslationError(*MF, *TPC, *ORE, R);
236  return *VRegs;
237  }
238  }
239 
240  return *VRegs;
241 }
242 
243 int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
244  auto MapEntry = FrameIndices.find(&AI);
245  if (MapEntry != FrameIndices.end())
246  return MapEntry->second;
247 
248  uint64_t ElementSize = DL->getTypeAllocSize(AI.getAllocatedType());
249  uint64_t Size =
250  ElementSize * cast<ConstantInt>(AI.getArraySize())->getZExtValue();
251 
252  // Always allocate at least one byte.
253  Size = std::max<uint64_t>(Size, 1u);
254 
255  int &FI = FrameIndices[&AI];
256  FI = MF->getFrameInfo().CreateStackObject(Size, AI.getAlign(), false, &AI);
257  return FI;
258 }
259 
260 Align IRTranslator::getMemOpAlign(const Instruction &I) {
261  if (const StoreInst *SI = dyn_cast<StoreInst>(&I))
262  return SI->getAlign();
263  if (const LoadInst *LI = dyn_cast<LoadInst>(&I))
264  return LI->getAlign();
265  if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I))
266  return AI->getAlign();
267  if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I))
268  return AI->getAlign();
269 
270  OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
271  R << "unable to translate memop: " << ore::NV("Opcode", &I);
272  reportTranslationError(*MF, *TPC, *ORE, R);
273  return Align(1);
274 }
275 
276 MachineBasicBlock &IRTranslator::getMBB(const BasicBlock &BB) {
277  MachineBasicBlock *&MBB = BBToMBB[&BB];
278  assert(MBB && "BasicBlock was not encountered before");
279  return *MBB;
280 }
281 
282 void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) {
283  assert(NewPred && "new predecessor must be a real MachineBasicBlock");
284  MachinePreds[Edge].push_back(NewPred);
285 }
286 
287 bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
288  MachineIRBuilder &MIRBuilder) {
289  // Get or create a virtual register for each value.
290  // Unless the value is a Constant => loadimm cst?
291  // or inline constant each time?
292  // Creation of a virtual register needs to have a size.
293  Register Op0 = getOrCreateVReg(*U.getOperand(0));
294  Register Op1 = getOrCreateVReg(*U.getOperand(1));
295  Register Res = getOrCreateVReg(U);
296  uint16_t Flags = 0;
297  if (isa<Instruction>(U)) {
298  const Instruction &I = cast<Instruction>(U);
300  }
301 
302  MIRBuilder.buildInstr(Opcode, {Res}, {Op0, Op1}, Flags);
303  return true;
304 }
305 
306 bool IRTranslator::translateUnaryOp(unsigned Opcode, const User &U,
307  MachineIRBuilder &MIRBuilder) {
308  Register Op0 = getOrCreateVReg(*U.getOperand(0));
309  Register Res = getOrCreateVReg(U);
310  uint16_t Flags = 0;
311  if (isa<Instruction>(U)) {
312  const Instruction &I = cast<Instruction>(U);
314  }
315  MIRBuilder.buildInstr(Opcode, {Res}, {Op0}, Flags);
316  return true;
317 }
318 
319 bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) {
320  return translateUnaryOp(TargetOpcode::G_FNEG, U, MIRBuilder);
321 }
322 
323 bool IRTranslator::translateCompare(const User &U,
324  MachineIRBuilder &MIRBuilder) {
325  auto *CI = dyn_cast<CmpInst>(&U);
326  Register Op0 = getOrCreateVReg(*U.getOperand(0));
327  Register Op1 = getOrCreateVReg(*U.getOperand(1));
328  Register Res = getOrCreateVReg(U);
329  CmpInst::Predicate Pred =
330  CI ? CI->getPredicate() : static_cast<CmpInst::Predicate>(
331  cast<ConstantExpr>(U).getPredicate());
332  if (CmpInst::isIntPredicate(Pred))
333  MIRBuilder.buildICmp(Pred, Res, Op0, Op1);
334  else if (Pred == CmpInst::FCMP_FALSE)
335  MIRBuilder.buildCopy(
336  Res, getOrCreateVReg(*Constant::getNullValue(U.getType())));
337  else if (Pred == CmpInst::FCMP_TRUE)
338  MIRBuilder.buildCopy(
339  Res, getOrCreateVReg(*Constant::getAllOnesValue(U.getType())));
340  else {
341  assert(CI && "Instruction should be CmpInst");
342  MIRBuilder.buildFCmp(Pred, Res, Op0, Op1,
344  }
345 
346  return true;
347 }
348 
349 bool IRTranslator::translateRet(const User &U, MachineIRBuilder &MIRBuilder) {
350  const ReturnInst &RI = cast<ReturnInst>(U);
351  const Value *Ret = RI.getReturnValue();
352  if (Ret && DL->getTypeStoreSize(Ret->getType()) == 0)
353  Ret = nullptr;
354 
355  ArrayRef<Register> VRegs;
356  if (Ret)
357  VRegs = getOrCreateVRegs(*Ret);
358 
359  Register SwiftErrorVReg = 0;
360  if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {
361  SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
362  &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
363  }
364 
365  // The target may mess up with the insertion point, but
366  // this is not important as a return is the last instruction
367  // of the block anyway.
368  return CLI->lowerReturn(MIRBuilder, Ret, VRegs, FuncInfo, SwiftErrorVReg);
369 }
370 
371 void IRTranslator::emitBranchForMergedCondition(
372  const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
373  MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
374  BranchProbability TProb, BranchProbability FProb, bool InvertCond) {
375  // If the leaf of the tree is a comparison, merge the condition into
376  // the caseblock.
377  if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
378  CmpInst::Predicate Condition;
379  if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
380  Condition = InvertCond ? IC->getInversePredicate() : IC->getPredicate();
381  } else {
382  const FCmpInst *FC = cast<FCmpInst>(Cond);
383  Condition = InvertCond ? FC->getInversePredicate() : FC->getPredicate();
384  }
385 
386  SwitchCG::CaseBlock CB(Condition, false, BOp->getOperand(0),
387  BOp->getOperand(1), nullptr, TBB, FBB, CurBB,
388  CurBuilder->getDebugLoc(), TProb, FProb);
389  SL->SwitchCases.push_back(CB);
390  return;
391  }
392 
393  // Create a CaseBlock record representing this branch.
396  Pred, false, Cond, ConstantInt::getTrue(MF->getFunction().getContext()),
397  nullptr, TBB, FBB, CurBB, CurBuilder->getDebugLoc(), TProb, FProb);
398  SL->SwitchCases.push_back(CB);
399 }
400 
401 static bool isValInBlock(const Value *V, const BasicBlock *BB) {
402  if (const Instruction *I = dyn_cast<Instruction>(V))
403  return I->getParent() == BB;
404  return true;
405 }
406 
407 void IRTranslator::findMergedConditions(
408  const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
409  MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
411  BranchProbability FProb, bool InvertCond) {
412  using namespace PatternMatch;
413  assert((Opc == Instruction::And || Opc == Instruction::Or) &&
414  "Expected Opc to be AND/OR");
415  // Skip over not part of the tree and remember to invert op and operands at
416  // next level.
417  Value *NotCond;
418  if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
419  isValInBlock(NotCond, CurBB->getBasicBlock())) {
420  findMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
421  !InvertCond);
422  return;
423  }
424 
425  const Instruction *BOp = dyn_cast<Instruction>(Cond);
426  const Value *BOpOp0, *BOpOp1;
427  // Compute the effective opcode for Cond, taking into account whether it needs
428  // to be inverted, e.g.
429  // and (not (or A, B)), C
430  // gets lowered as
431  // and (and (not A, not B), C)
433  if (BOp) {
434  BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
435  ? Instruction::And
436  : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
437  ? Instruction::Or
439  if (InvertCond) {
440  if (BOpc == Instruction::And)
441  BOpc = Instruction::Or;
442  else if (BOpc == Instruction::Or)
443  BOpc = Instruction::And;
444  }
445  }
446 
447  // If this node is not part of the or/and tree, emit it as a branch.
448  // Note that all nodes in the tree should have same opcode.
449  bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
450  if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
451  !isValInBlock(BOpOp0, CurBB->getBasicBlock()) ||
452  !isValInBlock(BOpOp1, CurBB->getBasicBlock())) {
453  emitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb,
454  InvertCond);
455  return;
456  }
457 
458  // Create TmpBB after CurBB.
459  MachineFunction::iterator BBI(CurBB);
460  MachineBasicBlock *TmpBB =
461  MF->CreateMachineBasicBlock(CurBB->getBasicBlock());
462  CurBB->getParent()->insert(++BBI, TmpBB);
463 
464  if (Opc == Instruction::Or) {
465  // Codegen X | Y as:
466  // BB1:
467  // jmp_if_X TBB
468  // jmp TmpBB
469  // TmpBB:
470  // jmp_if_Y TBB
471  // jmp FBB
472  //
473 
474  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
475  // The requirement is that
476  // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
477  // = TrueProb for original BB.
478  // Assuming the original probabilities are A and B, one choice is to set
479  // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
480  // A/(1+B) and 2B/(1+B). This choice assumes that
481  // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
482  // Another choice is to assume TrueProb for BB1 equals to TrueProb for
483  // TmpBB, but the math is more complicated.
484 
485  auto NewTrueProb = TProb / 2;
486  auto NewFalseProb = TProb / 2 + FProb;
487  // Emit the LHS condition.
488  findMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
489  NewFalseProb, InvertCond);
490 
491  // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
492  SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
493  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
494  // Emit the RHS condition into TmpBB.
495  findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
496  Probs[1], InvertCond);
497  } else {
498  assert(Opc == Instruction::And && "Unknown merge op!");
499  // Codegen X & Y as:
500  // BB1:
501  // jmp_if_X TmpBB
502  // jmp FBB
503  // TmpBB:
504  // jmp_if_Y TBB
505  // jmp FBB
506  //
507  // This requires creation of TmpBB after CurBB.
508 
509  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
510  // The requirement is that
511  // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
512  // = FalseProb for original BB.
513  // Assuming the original probabilities are A and B, one choice is to set
514  // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
515  // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
516  // TrueProb for BB1 * FalseProb for TmpBB.
517 
518  auto NewTrueProb = TProb + FProb / 2;
519  auto NewFalseProb = FProb / 2;
520  // Emit the LHS condition.
521  findMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
522  NewFalseProb, InvertCond);
523 
524  // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
525  SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
526  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
527  // Emit the RHS condition into TmpBB.
528  findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
529  Probs[1], InvertCond);
530  }
531 }
532 
533 bool IRTranslator::shouldEmitAsBranches(
534  const std::vector<SwitchCG::CaseBlock> &Cases) {
535  // For multiple cases, it's better to emit as branches.
536  if (Cases.size() != 2)
537  return true;
538 
539  // If this is two comparisons of the same values or'd or and'd together, they
540  // will get folded into a single comparison, so don't emit two blocks.
541  if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
542  Cases[0].CmpRHS == Cases[1].CmpRHS) ||
543  (Cases[0].CmpRHS == Cases[1].CmpLHS &&
544  Cases[0].CmpLHS == Cases[1].CmpRHS)) {
545  return false;
546  }
547 
548  // Handle: (X != null) | (Y != null) --> (X|Y) != 0
549  // Handle: (X == null) & (Y == null) --> (X|Y) == 0
550  if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
551  Cases[0].PredInfo.Pred == Cases[1].PredInfo.Pred &&
552  isa<Constant>(Cases[0].CmpRHS) &&
553  cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
554  if (Cases[0].PredInfo.Pred == CmpInst::ICMP_EQ &&
555  Cases[0].TrueBB == Cases[1].ThisBB)
556  return false;
557  if (Cases[0].PredInfo.Pred == CmpInst::ICMP_NE &&
558  Cases[0].FalseBB == Cases[1].ThisBB)
559  return false;
560  }
561 
562  return true;
563 }
564 
565 bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) {
566  const BranchInst &BrInst = cast<BranchInst>(U);
567  auto &CurMBB = MIRBuilder.getMBB();
568  auto *Succ0MBB = &getMBB(*BrInst.getSuccessor(0));
569 
570  if (BrInst.isUnconditional()) {
571  // If the unconditional target is the layout successor, fallthrough.
572  if (OptLevel == CodeGenOpt::None || !CurMBB.isLayoutSuccessor(Succ0MBB))
573  MIRBuilder.buildBr(*Succ0MBB);
574 
575  // Link successors.
576  for (const BasicBlock *Succ : successors(&BrInst))
577  CurMBB.addSuccessor(&getMBB(*Succ));
578  return true;
579  }
580 
581  // If this condition is one of the special cases we handle, do special stuff
582  // now.
583  const Value *CondVal = BrInst.getCondition();
584  MachineBasicBlock *Succ1MBB = &getMBB(*BrInst.getSuccessor(1));
585 
586  const auto &TLI = *MF->getSubtarget().getTargetLowering();
587 
588  // If this is a series of conditions that are or'd or and'd together, emit
589  // this as a sequence of branches instead of setcc's with and/or operations.
590  // As long as jumps are not expensive (exceptions for multi-use logic ops,
591  // unpredictable branches, and vector extracts because those jumps are likely
592  // expensive for any target), this should improve performance.
593  // For example, instead of something like:
594  // cmp A, B
595  // C = seteq
596  // cmp D, E
597  // F = setle
598  // or C, F
599  // jnz foo
600  // Emit:
601  // cmp A, B
602  // je foo
603  // cmp D, E
604  // jle foo
605  using namespace PatternMatch;
606  const Instruction *CondI = dyn_cast<Instruction>(CondVal);
607  if (!TLI.isJumpExpensive() && CondI && CondI->hasOneUse() &&
608  !BrInst.hasMetadata(LLVMContext::MD_unpredictable)) {
610  Value *Vec;
611  const Value *BOp0, *BOp1;
612  if (match(CondI, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
613  Opcode = Instruction::And;
614  else if (match(CondI, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
615  Opcode = Instruction::Or;
616 
617  if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
618  match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {
619  findMergedConditions(CondI, Succ0MBB, Succ1MBB, &CurMBB, &CurMBB, Opcode,
620  getEdgeProbability(&CurMBB, Succ0MBB),
621  getEdgeProbability(&CurMBB, Succ1MBB),
622  /*InvertCond=*/false);
623  assert(SL->SwitchCases[0].ThisBB == &CurMBB && "Unexpected lowering!");
624 
625  // Allow some cases to be rejected.
626  if (shouldEmitAsBranches(SL->SwitchCases)) {
627  // Emit the branch for this block.
628  emitSwitchCase(SL->SwitchCases[0], &CurMBB, *CurBuilder);
629  SL->SwitchCases.erase(SL->SwitchCases.begin());
630  return true;
631  }
632 
633  // Okay, we decided not to do this, remove any inserted MBB's and clear
634  // SwitchCases.
635  for (unsigned I = 1, E = SL->SwitchCases.size(); I != E; ++I)
636  MF->erase(SL->SwitchCases[I].ThisBB);
637 
638  SL->SwitchCases.clear();
639  }
640  }
641 
642  // Create a CaseBlock record representing this branch.
643  SwitchCG::CaseBlock CB(CmpInst::ICMP_EQ, false, CondVal,
644  ConstantInt::getTrue(MF->getFunction().getContext()),
645  nullptr, Succ0MBB, Succ1MBB, &CurMBB,
646  CurBuilder->getDebugLoc());
647 
648  // Use emitSwitchCase to actually insert the fast branch sequence for this
649  // cond branch.
650  emitSwitchCase(CB, &CurMBB, *CurBuilder);
651  return true;
652 }
653 
654 void IRTranslator::addSuccessorWithProb(MachineBasicBlock *Src,
655  MachineBasicBlock *Dst,
656  BranchProbability Prob) {
657  if (!FuncInfo.BPI) {
658  Src->addSuccessorWithoutProb(Dst);
659  return;
660  }
661  if (Prob.isUnknown())
662  Prob = getEdgeProbability(Src, Dst);
663  Src->addSuccessor(Dst, Prob);
664 }
665 
667 IRTranslator::getEdgeProbability(const MachineBasicBlock *Src,
668  const MachineBasicBlock *Dst) const {
669  const BasicBlock *SrcBB = Src->getBasicBlock();
670  const BasicBlock *DstBB = Dst->getBasicBlock();
671  if (!FuncInfo.BPI) {
672  // If BPI is not available, set the default probability as 1 / N, where N is
673  // the number of successors.
674  auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
675  return BranchProbability(1, SuccSize);
676  }
677  return FuncInfo.BPI->getEdgeProbability(SrcBB, DstBB);
678 }
679 
680 bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) {
681  using namespace SwitchCG;
682  // Extract cases from the switch.
683  const SwitchInst &SI = cast<SwitchInst>(U);
684  BranchProbabilityInfo *BPI = FuncInfo.BPI;
685  CaseClusterVector Clusters;
686  Clusters.reserve(SI.getNumCases());
687  for (auto &I : SI.cases()) {
688  MachineBasicBlock *Succ = &getMBB(*I.getCaseSuccessor());
689  assert(Succ && "Could not find successor mbb in mapping");
690  const ConstantInt *CaseVal = I.getCaseValue();
691  BranchProbability Prob =
692  BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
693  : BranchProbability(1, SI.getNumCases() + 1);
694  Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
695  }
696 
697  MachineBasicBlock *DefaultMBB = &getMBB(*SI.getDefaultDest());
698 
699  // Cluster adjacent cases with the same destination. We do this at all
700  // optimization levels because it's cheap to do and will make codegen faster
701  // if there are many clusters.
702  sortAndRangeify(Clusters);
703 
704  MachineBasicBlock *SwitchMBB = &getMBB(*SI.getParent());
705 
706  // If there is only the default destination, jump there directly.
707  if (Clusters.empty()) {
708  SwitchMBB->addSuccessor(DefaultMBB);
709  if (DefaultMBB != SwitchMBB->getNextNode())
710  MIB.buildBr(*DefaultMBB);
711  return true;
712  }
713 
714  SL->findJumpTables(Clusters, &SI, DefaultMBB, nullptr, nullptr);
715  SL->findBitTestClusters(Clusters, &SI);
716 
717  LLVM_DEBUG({
718  dbgs() << "Case clusters: ";
719  for (const CaseCluster &C : Clusters) {
720  if (C.Kind == CC_JumpTable)
721  dbgs() << "JT:";
722  if (C.Kind == CC_BitTests)
723  dbgs() << "BT:";
724 
725  C.Low->getValue().print(dbgs(), true);
726  if (C.Low != C.High) {
727  dbgs() << '-';
728  C.High->getValue().print(dbgs(), true);
729  }
730  dbgs() << ' ';
731  }
732  dbgs() << '\n';
733  });
734 
735  assert(!Clusters.empty());
736  SwitchWorkList WorkList;
737  CaseClusterIt First = Clusters.begin();
738  CaseClusterIt Last = Clusters.end() - 1;
739  auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB);
740  WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
741 
742  // FIXME: At the moment we don't do any splitting optimizations here like
743  // SelectionDAG does, so this worklist only has one entry.
744  while (!WorkList.empty()) {
745  SwitchWorkListItem W = WorkList.pop_back_val();
746  if (!lowerSwitchWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
747  return false;
748  }
749  return true;
750 }
751 
752 void IRTranslator::emitJumpTable(SwitchCG::JumpTable &JT,
754  // Emit the code for the jump table
755  assert(JT.Reg != -1U && "Should lower JT Header first!");
756  MachineIRBuilder MIB(*MBB->getParent());
757  MIB.setMBB(*MBB);
758  MIB.setDebugLoc(CurBuilder->getDebugLoc());
759 
760  Type *PtrIRTy = Type::getInt8PtrTy(MF->getFunction().getContext());
761  const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
762 
763  auto Table = MIB.buildJumpTable(PtrTy, JT.JTI);
764  MIB.buildBrJT(Table.getReg(0), JT.JTI, JT.Reg);
765 }
766 
767 bool IRTranslator::emitJumpTableHeader(SwitchCG::JumpTable &JT,
769  MachineBasicBlock *HeaderBB) {
770  MachineIRBuilder MIB(*HeaderBB->getParent());
771  MIB.setMBB(*HeaderBB);
772  MIB.setDebugLoc(CurBuilder->getDebugLoc());
773 
774  const Value &SValue = *JTH.SValue;
775  // Subtract the lowest switch case value from the value being switched on.
776  const LLT SwitchTy = getLLTForType(*SValue.getType(), *DL);
777  Register SwitchOpReg = getOrCreateVReg(SValue);
778  auto FirstCst = MIB.buildConstant(SwitchTy, JTH.First);
779  auto Sub = MIB.buildSub({SwitchTy}, SwitchOpReg, FirstCst);
780 
781  // This value may be smaller or larger than the target's pointer type, and
782  // therefore require extension or truncating.
783  Type *PtrIRTy = SValue.getType()->getPointerTo();
784  const LLT PtrScalarTy = LLT::scalar(DL->getTypeSizeInBits(PtrIRTy));
785  Sub = MIB.buildZExtOrTrunc(PtrScalarTy, Sub);
786 
787  JT.Reg = Sub.getReg(0);
788 
789  if (JTH.FallthroughUnreachable) {
790  if (JT.MBB != HeaderBB->getNextNode())
791  MIB.buildBr(*JT.MBB);
792  return true;
793  }
794 
795  // Emit the range check for the jump table, and branch to the default block
796  // for the switch statement if the value being switched on exceeds the
797  // largest case in the switch.
798  auto Cst = getOrCreateVReg(
799  *ConstantInt::get(SValue.getType(), JTH.Last - JTH.First));
800  Cst = MIB.buildZExtOrTrunc(PtrScalarTy, Cst).getReg(0);
801  auto Cmp = MIB.buildICmp(CmpInst::ICMP_UGT, LLT::scalar(1), Sub, Cst);
802 
803  auto BrCond = MIB.buildBrCond(Cmp.getReg(0), *JT.Default);
804 
805  // Avoid emitting unnecessary branches to the next block.
806  if (JT.MBB != HeaderBB->getNextNode())
807  BrCond = MIB.buildBr(*JT.MBB);
808  return true;
809 }
810 
811 void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
812  MachineBasicBlock *SwitchBB,
813  MachineIRBuilder &MIB) {
814  Register CondLHS = getOrCreateVReg(*CB.CmpLHS);
815  Register Cond;
816  DebugLoc OldDbgLoc = MIB.getDebugLoc();
817  MIB.setDebugLoc(CB.DbgLoc);
818  MIB.setMBB(*CB.ThisBB);
819 
820  if (CB.PredInfo.NoCmp) {
821  // Branch or fall through to TrueBB.
822  addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
823  addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
824  CB.ThisBB);
826  if (CB.TrueBB != CB.ThisBB->getNextNode())
827  MIB.buildBr(*CB.TrueBB);
828  MIB.setDebugLoc(OldDbgLoc);
829  return;
830  }
831 
832  const LLT i1Ty = LLT::scalar(1);
833  // Build the compare.
834  if (!CB.CmpMHS) {
835  const auto *CI = dyn_cast<ConstantInt>(CB.CmpRHS);
836  // For conditional branch lowering, we might try to do something silly like
837  // emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so,
838  // just re-use the existing condition vreg.
839  if (MRI->getType(CondLHS).getSizeInBits() == 1 && CI &&
840  CI->getZExtValue() == 1 && CB.PredInfo.Pred == CmpInst::ICMP_EQ) {
841  Cond = CondLHS;
842  } else {
843  Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
845  Cond =
846  MIB.buildFCmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
847  else
848  Cond =
849  MIB.buildICmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
850  }
851  } else {
853  "Can only handle SLE ranges");
854 
855  const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
856  const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
857 
858  Register CmpOpReg = getOrCreateVReg(*CB.CmpMHS);
859  if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
860  Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
861  Cond =
862  MIB.buildICmp(CmpInst::ICMP_SLE, i1Ty, CmpOpReg, CondRHS).getReg(0);
863  } else {
864  const LLT CmpTy = MRI->getType(CmpOpReg);
865  auto Sub = MIB.buildSub({CmpTy}, CmpOpReg, CondLHS);
866  auto Diff = MIB.buildConstant(CmpTy, High - Low);
867  Cond = MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, Sub, Diff).getReg(0);
868  }
869  }
870 
871  // Update successor info
872  addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
873 
874  addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
875  CB.ThisBB);
876 
877  // TrueBB and FalseBB are always different unless the incoming IR is
878  // degenerate. This only happens when running llc on weird IR.
879  if (CB.TrueBB != CB.FalseBB)
880  addSuccessorWithProb(CB.ThisBB, CB.FalseBB, CB.FalseProb);
882 
883  addMachineCFGPred({SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
884  CB.ThisBB);
885 
886  MIB.buildBrCond(Cond, *CB.TrueBB);
887  MIB.buildBr(*CB.FalseBB);
888  MIB.setDebugLoc(OldDbgLoc);
889 }
890 
891 bool IRTranslator::lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W,
892  MachineBasicBlock *SwitchMBB,
893  MachineBasicBlock *CurMBB,
894  MachineBasicBlock *DefaultMBB,
895  MachineIRBuilder &MIB,
897  BranchProbability UnhandledProbs,
899  MachineBasicBlock *Fallthrough,
900  bool FallthroughUnreachable) {
901  using namespace SwitchCG;
902  MachineFunction *CurMF = SwitchMBB->getParent();
903  // FIXME: Optimize away range check based on pivot comparisons.
904  JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
905  SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
906  BranchProbability DefaultProb = W.DefaultProb;
907 
908  // The jump block hasn't been inserted yet; insert it here.
909  MachineBasicBlock *JumpMBB = JT->MBB;
910  CurMF->insert(BBI, JumpMBB);
911 
912  // Since the jump table block is separate from the switch block, we need
913  // to keep track of it as a machine predecessor to the default block,
914  // otherwise we lose the phi edges.
915  addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
916  CurMBB);
917  addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
918  JumpMBB);
919 
920  auto JumpProb = I->Prob;
921  auto FallthroughProb = UnhandledProbs;
922 
923  // If the default statement is a target of the jump table, we evenly
924  // distribute the default probability to successors of CurMBB. Also
925  // update the probability on the edge from JumpMBB to Fallthrough.
927  SE = JumpMBB->succ_end();
928  SI != SE; ++SI) {
929  if (*SI == DefaultMBB) {
930  JumpProb += DefaultProb / 2;
931  FallthroughProb -= DefaultProb / 2;
932  JumpMBB->setSuccProbability(SI, DefaultProb / 2);
933  JumpMBB->normalizeSuccProbs();
934  } else {
935  // Also record edges from the jump table block to it's successors.
936  addMachineCFGPred({SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
937  JumpMBB);
938  }
939  }
940 
941  if (FallthroughUnreachable)
942  JTH->FallthroughUnreachable = true;
943 
944  if (!JTH->FallthroughUnreachable)
945  addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
946  addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
947  CurMBB->normalizeSuccProbs();
948 
949  // The jump table header will be inserted in our current block, do the
950  // range check, and fall through to our fallthrough block.
951  JTH->HeaderBB = CurMBB;
952  JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
953 
954  // If we're in the right place, emit the jump table header right now.
955  if (CurMBB == SwitchMBB) {
956  if (!emitJumpTableHeader(*JT, *JTH, CurMBB))
957  return false;
958  JTH->Emitted = true;
959  }
960  return true;
961 }
962 bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,
963  Value *Cond,
964  MachineBasicBlock *Fallthrough,
965  bool FallthroughUnreachable,
966  BranchProbability UnhandledProbs,
967  MachineBasicBlock *CurMBB,
968  MachineIRBuilder &MIB,
969  MachineBasicBlock *SwitchMBB) {
970  using namespace SwitchCG;
971  const Value *RHS, *LHS, *MHS;
972  CmpInst::Predicate Pred;
973  if (I->Low == I->High) {
974  // Check Cond == I->Low.
975  Pred = CmpInst::ICMP_EQ;
976  LHS = Cond;
977  RHS = I->Low;
978  MHS = nullptr;
979  } else {
980  // Check I->Low <= Cond <= I->High.
981  Pred = CmpInst::ICMP_SLE;
982  LHS = I->Low;
983  MHS = Cond;
984  RHS = I->High;
985  }
986 
987  // If Fallthrough is unreachable, fold away the comparison.
988  // The false probability is the sum of all unhandled cases.
989  CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
990  CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
991 
992  emitSwitchCase(CB, SwitchMBB, MIB);
993  return true;
994 }
995 
996 void IRTranslator::emitBitTestHeader(SwitchCG::BitTestBlock &B,
997  MachineBasicBlock *SwitchBB) {
998  MachineIRBuilder &MIB = *CurBuilder;
999  MIB.setMBB(*SwitchBB);
1000 
1001  // Subtract the minimum value.
1002  Register SwitchOpReg = getOrCreateVReg(*B.SValue);
1003 
1004  LLT SwitchOpTy = MRI->getType(SwitchOpReg);
1005  Register MinValReg = MIB.buildConstant(SwitchOpTy, B.First).getReg(0);
1006  auto RangeSub = MIB.buildSub(SwitchOpTy, SwitchOpReg, MinValReg);
1007 
1008  Type *PtrIRTy = Type::getInt8PtrTy(MF->getFunction().getContext());
1009  const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
1010 
1011  LLT MaskTy = SwitchOpTy;
1012  if (MaskTy.getSizeInBits() > PtrTy.getSizeInBits() ||
1013  !isPowerOf2_32(MaskTy.getSizeInBits()))
1014  MaskTy = LLT::scalar(PtrTy.getSizeInBits());
1015  else {
1016  // Ensure that the type will fit the mask value.
1017  for (unsigned I = 0, E = B.Cases.size(); I != E; ++I) {
1018  if (!isUIntN(SwitchOpTy.getSizeInBits(), B.Cases[I].Mask)) {
1019  // Switch table case range are encoded into series of masks.
1020  // Just use pointer type, it's guaranteed to fit.
1021  MaskTy = LLT::scalar(PtrTy.getSizeInBits());
1022  break;
1023  }
1024  }
1025  }
1026  Register SubReg = RangeSub.getReg(0);
1027  if (SwitchOpTy != MaskTy)
1028  SubReg = MIB.buildZExtOrTrunc(MaskTy, SubReg).getReg(0);
1029 
1030  B.RegVT = getMVTForLLT(MaskTy);
1031  B.Reg = SubReg;
1032 
1033  MachineBasicBlock *MBB = B.Cases[0].ThisBB;
1034 
1035  if (!B.FallthroughUnreachable)
1036  addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
1037  addSuccessorWithProb(SwitchBB, MBB, B.Prob);
1038 
1039  SwitchBB->normalizeSuccProbs();
1040 
1041  if (!B.FallthroughUnreachable) {
1042  // Conditional branch to the default block.
1043  auto RangeCst = MIB.buildConstant(SwitchOpTy, B.Range);
1044  auto RangeCmp = MIB.buildICmp(CmpInst::Predicate::ICMP_UGT, LLT::scalar(1),
1045  RangeSub, RangeCst);
1046  MIB.buildBrCond(RangeCmp, *B.Default);
1047  }
1048 
1049  // Avoid emitting unnecessary branches to the next block.
1050  if (MBB != SwitchBB->getNextNode())
1051  MIB.buildBr(*MBB);
1052 }
1053 
1054 void IRTranslator::emitBitTestCase(SwitchCG::BitTestBlock &BB,
1055  MachineBasicBlock *NextMBB,
1056  BranchProbability BranchProbToNext,
1058  MachineBasicBlock *SwitchBB) {
1059  MachineIRBuilder &MIB = *CurBuilder;
1060  MIB.setMBB(*SwitchBB);
1061 
1062  LLT SwitchTy = getLLTForMVT(BB.RegVT);
1063  Register Cmp;
1064  unsigned PopCount = countPopulation(B.Mask);
1065  if (PopCount == 1) {
1066  // Testing for a single bit; just compare the shift count with what it
1067  // would need to be to shift a 1 bit in that position.
1068  auto MaskTrailingZeros =
1069  MIB.buildConstant(SwitchTy, countTrailingZeros(B.Mask));
1070  Cmp =
1071  MIB.buildICmp(ICmpInst::ICMP_EQ, LLT::scalar(1), Reg, MaskTrailingZeros)
1072  .getReg(0);
1073  } else if (PopCount == BB.Range) {
1074  // There is only one zero bit in the range, test for it directly.
1075  auto MaskTrailingOnes =
1076  MIB.buildConstant(SwitchTy, countTrailingOnes(B.Mask));
1077  Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Reg, MaskTrailingOnes)
1078  .getReg(0);
1079  } else {
1080  // Make desired shift.
1081  auto CstOne = MIB.buildConstant(SwitchTy, 1);
1082  auto SwitchVal = MIB.buildShl(SwitchTy, CstOne, Reg);
1083 
1084  // Emit bit tests and jumps.
1085  auto CstMask = MIB.buildConstant(SwitchTy, B.Mask);
1086  auto AndOp = MIB.buildAnd(SwitchTy, SwitchVal, CstMask);
1087  auto CstZero = MIB.buildConstant(SwitchTy, 0);
1088  Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), AndOp, CstZero)
1089  .getReg(0);
1090  }
1091 
1092  // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
1093  addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
1094  // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
1095  addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
1096  // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
1097  // one as they are relative probabilities (and thus work more like weights),
1098  // and hence we need to normalize them to let the sum of them become one.
1099  SwitchBB->normalizeSuccProbs();
1100 
1101  // Record the fact that the IR edge from the header to the bit test target
1102  // will go through our new block. Neeeded for PHIs to have nodes added.
1103  addMachineCFGPred({BB.Parent->getBasicBlock(), B.TargetBB->getBasicBlock()},
1104  SwitchBB);
1105 
1106  MIB.buildBrCond(Cmp, *B.TargetBB);
1107 
1108  // Avoid emitting unnecessary branches to the next block.
1109  if (NextMBB != SwitchBB->getNextNode())
1110  MIB.buildBr(*NextMBB);
1111 }
1112 
1113 bool IRTranslator::lowerBitTestWorkItem(
1115  MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
1117  BranchProbability DefaultProb, BranchProbability UnhandledProbs,
1119  bool FallthroughUnreachable) {
1120  using namespace SwitchCG;
1121  MachineFunction *CurMF = SwitchMBB->getParent();
1122  // FIXME: Optimize away range check based on pivot comparisons.
1123  BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
1124  // The bit test blocks haven't been inserted yet; insert them here.
1125  for (BitTestCase &BTC : BTB->Cases)
1126  CurMF->insert(BBI, BTC.ThisBB);
1127 
1128  // Fill in fields of the BitTestBlock.
1129  BTB->Parent = CurMBB;
1130  BTB->Default = Fallthrough;
1131 
1132  BTB->DefaultProb = UnhandledProbs;
1133  // If the cases in bit test don't form a contiguous range, we evenly
1134  // distribute the probability on the edge to Fallthrough to two
1135  // successors of CurMBB.
1136  if (!BTB->ContiguousRange) {
1137  BTB->Prob += DefaultProb / 2;
1138  BTB->DefaultProb -= DefaultProb / 2;
1139  }
1140 
1141  if (FallthroughUnreachable)
1142  BTB->FallthroughUnreachable = true;
1143 
1144  // If we're in the right place, emit the bit test header right now.
1145  if (CurMBB == SwitchMBB) {
1146  emitBitTestHeader(*BTB, SwitchMBB);
1147  BTB->Emitted = true;
1148  }
1149  return true;
1150 }
1151 
1152 bool IRTranslator::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
1153  Value *Cond,
1154  MachineBasicBlock *SwitchMBB,
1155  MachineBasicBlock *DefaultMBB,
1156  MachineIRBuilder &MIB) {
1157  using namespace SwitchCG;
1158  MachineFunction *CurMF = FuncInfo.MF;
1159  MachineBasicBlock *NextMBB = nullptr;
1160  MachineFunction::iterator BBI(W.MBB);
1161  if (++BBI != FuncInfo.MF->end())
1162  NextMBB = &*BBI;
1163 
1164  if (EnableOpts) {
1165  // Here, we order cases by probability so the most likely case will be
1166  // checked first. However, two clusters can have the same probability in
1167  // which case their relative ordering is non-deterministic. So we use Low
1168  // as a tie-breaker as clusters are guaranteed to never overlap.
1169  llvm::sort(W.FirstCluster, W.LastCluster + 1,
1170  [](const CaseCluster &a, const CaseCluster &b) {
1171  return a.Prob != b.Prob
1172  ? a.Prob > b.Prob
1173  : a.Low->getValue().slt(b.Low->getValue());
1174  });
1175 
1176  // Rearrange the case blocks so that the last one falls through if possible
1177  // without changing the order of probabilities.
1178  for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
1179  --I;
1180  if (I->Prob > W.LastCluster->Prob)
1181  break;
1182  if (I->Kind == CC_Range && I->MBB == NextMBB) {
1183  std::swap(*I, *W.LastCluster);
1184  break;
1185  }
1186  }
1187  }
1188 
1189  // Compute total probability.
1190  BranchProbability DefaultProb = W.DefaultProb;
1191  BranchProbability UnhandledProbs = DefaultProb;
1192  for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
1193  UnhandledProbs += I->Prob;
1194 
1195  MachineBasicBlock *CurMBB = W.MBB;
1196  for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
1197  bool FallthroughUnreachable = false;
1198  MachineBasicBlock *Fallthrough;
1199  if (I == W.LastCluster) {
1200  // For the last cluster, fall through to the default destination.
1201  Fallthrough = DefaultMBB;
1202  FallthroughUnreachable = isa<UnreachableInst>(
1203  DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
1204  } else {
1205  Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
1206  CurMF->insert(BBI, Fallthrough);
1207  }
1208  UnhandledProbs -= I->Prob;
1209 
1210  switch (I->Kind) {
1211  case CC_BitTests: {
1212  if (!lowerBitTestWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1213  DefaultProb, UnhandledProbs, I, Fallthrough,
1214  FallthroughUnreachable)) {
1215  LLVM_DEBUG(dbgs() << "Failed to lower bit test for switch");
1216  return false;
1217  }
1218  break;
1219  }
1220 
1221  case CC_JumpTable: {
1222  if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1223  UnhandledProbs, I, Fallthrough,
1224  FallthroughUnreachable)) {
1225  LLVM_DEBUG(dbgs() << "Failed to lower jump table");
1226  return false;
1227  }
1228  break;
1229  }
1230  case CC_Range: {
1231  if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
1232  FallthroughUnreachable, UnhandledProbs,
1233  CurMBB, MIB, SwitchMBB)) {
1234  LLVM_DEBUG(dbgs() << "Failed to lower switch range");
1235  return false;
1236  }
1237  break;
1238  }
1239  }
1240  CurMBB = Fallthrough;
1241  }
1242 
1243  return true;
1244 }
1245 
1246 bool IRTranslator::translateIndirectBr(const User &U,
1247  MachineIRBuilder &MIRBuilder) {
1248  const IndirectBrInst &BrInst = cast<IndirectBrInst>(U);
1249 
1250  const Register Tgt = getOrCreateVReg(*BrInst.getAddress());
1251  MIRBuilder.buildBrIndirect(Tgt);
1252 
1253  // Link successors.
1254  SmallPtrSet<const BasicBlock *, 32> AddedSuccessors;
1255  MachineBasicBlock &CurBB = MIRBuilder.getMBB();
1256  for (const BasicBlock *Succ : successors(&BrInst)) {
1257  // It's legal for indirectbr instructions to have duplicate blocks in the
1258  // destination list. We don't allow this in MIR. Skip anything that's
1259  // already a successor.
1260  if (!AddedSuccessors.insert(Succ).second)
1261  continue;
1262  CurBB.addSuccessor(&getMBB(*Succ));
1263  }
1264 
1265  return true;
1266 }
1267 
1268 static bool isSwiftError(const Value *V) {
1269  if (auto Arg = dyn_cast<Argument>(V))
1270  return Arg->hasSwiftErrorAttr();
1271  if (auto AI = dyn_cast<AllocaInst>(V))
1272  return AI->isSwiftError();
1273  return false;
1274 }
1275 
1276 bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
1277  const LoadInst &LI = cast<LoadInst>(U);
1278  if (DL->getTypeStoreSize(LI.getType()) == 0)
1279  return true;
1280 
1281  ArrayRef<Register> Regs = getOrCreateVRegs(LI);
1282  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(LI);
1283  Register Base = getOrCreateVReg(*LI.getPointerOperand());
1284 
1285  Type *OffsetIRTy = DL->getIntPtrType(LI.getPointerOperandType());
1286  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1287 
1288  if (CLI->supportSwiftError() && isSwiftError(LI.getPointerOperand())) {
1289  assert(Regs.size() == 1 && "swifterror should be single pointer");
1290  Register VReg = SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(),
1291  LI.getPointerOperand());
1292  MIRBuilder.buildCopy(Regs[0], VReg);
1293  return true;
1294  }
1295 
1296  auto &TLI = *MF->getSubtarget().getTargetLowering();
1297  MachineMemOperand::Flags Flags = TLI.getLoadMemOperandFlags(LI, *DL);
1298 
1299  const MDNode *Ranges =
1300  Regs.size() == 1 ? LI.getMetadata(LLVMContext::MD_range) : nullptr;
1301  for (unsigned i = 0; i < Regs.size(); ++i) {
1302  Register Addr;
1303  MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
1304 
1306  Align BaseAlign = getMemOpAlign(LI);
1307  auto MMO = MF->getMachineMemOperand(
1308  Ptr, Flags, MRI->getType(Regs[i]),
1309  commonAlignment(BaseAlign, Offsets[i] / 8), LI.getAAMetadata(), Ranges,
1310  LI.getSyncScopeID(), LI.getOrdering());
1311  MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
1312  }
1313 
1314  return true;
1315 }
1316 
1317 bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
1318  const StoreInst &SI = cast<StoreInst>(U);
1319  if (DL->getTypeStoreSize(SI.getValueOperand()->getType()) == 0)
1320  return true;
1321 
1322  ArrayRef<Register> Vals = getOrCreateVRegs(*SI.getValueOperand());
1323  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*SI.getValueOperand());
1324  Register Base = getOrCreateVReg(*SI.getPointerOperand());
1325 
1326  Type *OffsetIRTy = DL->getIntPtrType(SI.getPointerOperandType());
1327  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1328 
1329  if (CLI->supportSwiftError() && isSwiftError(SI.getPointerOperand())) {
1330  assert(Vals.size() == 1 && "swifterror should be single pointer");
1331 
1332  Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
1333  SI.getPointerOperand());
1334  MIRBuilder.buildCopy(VReg, Vals[0]);
1335  return true;
1336  }
1337 
1338  auto &TLI = *MF->getSubtarget().getTargetLowering();
1340 
1341  for (unsigned i = 0; i < Vals.size(); ++i) {
1342  Register Addr;
1343  MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
1344 
1345  MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i] / 8);
1346  Align BaseAlign = getMemOpAlign(SI);
1347  auto MMO = MF->getMachineMemOperand(
1348  Ptr, Flags, MRI->getType(Vals[i]),
1349  commonAlignment(BaseAlign, Offsets[i] / 8), SI.getAAMetadata(), nullptr,
1350  SI.getSyncScopeID(), SI.getOrdering());
1351  MIRBuilder.buildStore(Vals[i], Addr, *MMO);
1352  }
1353  return true;
1354 }
1355 
1356 static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL) {
1357  const Value *Src = U.getOperand(0);
1359 
1360  // getIndexedOffsetInType is designed for GEPs, so the first index is the
1361  // usual array element rather than looking into the actual aggregate.
1362  SmallVector<Value *, 1> Indices;
1363  Indices.push_back(ConstantInt::get(Int32Ty, 0));
1364 
1365  if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(&U)) {
1366  for (auto Idx : EVI->indices())
1367  Indices.push_back(ConstantInt::get(Int32Ty, Idx));
1368  } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(&U)) {
1369  for (auto Idx : IVI->indices())
1370  Indices.push_back(ConstantInt::get(Int32Ty, Idx));
1371  } else {
1372  for (unsigned i = 1; i < U.getNumOperands(); ++i)
1373  Indices.push_back(U.getOperand(i));
1374  }
1375 
1376  return 8 * static_cast<uint64_t>(
1377  DL.getIndexedOffsetInType(Src->getType(), Indices));
1378 }
1379 
1380 bool IRTranslator::translateExtractValue(const User &U,
1381  MachineIRBuilder &MIRBuilder) {
1382  const Value *Src = U.getOperand(0);
1384  ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
1385  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*Src);
1386  unsigned Idx = llvm::lower_bound(Offsets, Offset) - Offsets.begin();
1387  auto &DstRegs = allocateVRegs(U);
1388 
1389  for (unsigned i = 0; i < DstRegs.size(); ++i)
1390  DstRegs[i] = SrcRegs[Idx++];
1391 
1392  return true;
1393 }
1394 
1395 bool IRTranslator::translateInsertValue(const User &U,
1396  MachineIRBuilder &MIRBuilder) {
1397  const Value *Src = U.getOperand(0);
1399  auto &DstRegs = allocateVRegs(U);
1400  ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(U);
1401  ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
1402  ArrayRef<Register> InsertedRegs = getOrCreateVRegs(*U.getOperand(1));
1403  auto InsertedIt = InsertedRegs.begin();
1404 
1405  for (unsigned i = 0; i < DstRegs.size(); ++i) {
1406  if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
1407  DstRegs[i] = *InsertedIt++;
1408  else
1409  DstRegs[i] = SrcRegs[i];
1410  }
1411 
1412  return true;
1413 }
1414 
1415 bool IRTranslator::translateSelect(const User &U,
1416  MachineIRBuilder &MIRBuilder) {
1417  Register Tst = getOrCreateVReg(*U.getOperand(0));
1418  ArrayRef<Register> ResRegs = getOrCreateVRegs(U);
1419  ArrayRef<Register> Op0Regs = getOrCreateVRegs(*U.getOperand(1));
1420  ArrayRef<Register> Op1Regs = getOrCreateVRegs(*U.getOperand(2));
1421 
1422  uint16_t Flags = 0;
1423  if (const SelectInst *SI = dyn_cast<SelectInst>(&U))
1425 
1426  for (unsigned i = 0; i < ResRegs.size(); ++i) {
1427  MIRBuilder.buildSelect(ResRegs[i], Tst, Op0Regs[i], Op1Regs[i], Flags);
1428  }
1429 
1430  return true;
1431 }
1432 
1433 bool IRTranslator::translateCopy(const User &U, const Value &V,
1434  MachineIRBuilder &MIRBuilder) {
1435  Register Src = getOrCreateVReg(V);
1436  auto &Regs = *VMap.getVRegs(U);
1437  if (Regs.empty()) {
1438  Regs.push_back(Src);
1439  VMap.getOffsets(U)->push_back(0);
1440  } else {
1441  // If we already assigned a vreg for this instruction, we can't change that.
1442  // Emit a copy to satisfy the users we already emitted.
1443  MIRBuilder.buildCopy(Regs[0], Src);
1444  }
1445  return true;
1446 }
1447 
1448 bool IRTranslator::translateBitCast(const User &U,
1449  MachineIRBuilder &MIRBuilder) {
1450  // If we're bitcasting to the source type, we can reuse the source vreg.
1451  if (getLLTForType(*U.getOperand(0)->getType(), *DL) ==
1452  getLLTForType(*U.getType(), *DL))
1453  return translateCopy(U, *U.getOperand(0), MIRBuilder);
1454 
1455  return translateCast(TargetOpcode::G_BITCAST, U, MIRBuilder);
1456 }
1457 
1458 bool IRTranslator::translateCast(unsigned Opcode, const User &U,
1459  MachineIRBuilder &MIRBuilder) {
1460  Register Op = getOrCreateVReg(*U.getOperand(0));
1461  Register Res = getOrCreateVReg(U);
1462  MIRBuilder.buildInstr(Opcode, {Res}, {Op});
1463  return true;
1464 }
1465 
1466 bool IRTranslator::translateGetElementPtr(const User &U,
1467  MachineIRBuilder &MIRBuilder) {
1468  Value &Op0 = *U.getOperand(0);
1469  Register BaseReg = getOrCreateVReg(Op0);
1470  Type *PtrIRTy = Op0.getType();
1471  LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
1472  Type *OffsetIRTy = DL->getIntPtrType(PtrIRTy);
1473  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1474 
1475  // Normalize Vector GEP - all scalar operands should be converted to the
1476  // splat vector.
1477  unsigned VectorWidth = 0;
1478 
1479  // True if we should use a splat vector; using VectorWidth alone is not
1480  // sufficient.
1481  bool WantSplatVector = false;
1482  if (auto *VT = dyn_cast<VectorType>(U.getType())) {
1483  VectorWidth = cast<FixedVectorType>(VT)->getNumElements();
1484  // We don't produce 1 x N vectors; those are treated as scalars.
1485  WantSplatVector = VectorWidth > 1;
1486  }
1487 
1488  // We might need to splat the base pointer into a vector if the offsets
1489  // are vectors.
1490  if (WantSplatVector && !PtrTy.isVector()) {
1491  BaseReg =
1492  MIRBuilder
1493  .buildSplatVector(LLT::fixed_vector(VectorWidth, PtrTy), BaseReg)
1494  .getReg(0);
1495  PtrIRTy = FixedVectorType::get(PtrIRTy, VectorWidth);
1496  PtrTy = getLLTForType(*PtrIRTy, *DL);
1497  OffsetIRTy = DL->getIntPtrType(PtrIRTy);
1498  OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1499  }
1500 
1501  int64_t Offset = 0;
1502  for (gep_type_iterator GTI = gep_type_begin(&U), E = gep_type_end(&U);
1503  GTI != E; ++GTI) {
1504  const Value *Idx = GTI.getOperand();
1505  if (StructType *StTy = GTI.getStructTypeOrNull()) {
1506  unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
1508  continue;
1509  } else {
1510  uint64_t ElementSize = DL->getTypeAllocSize(GTI.getIndexedType());
1511 
1512  // If this is a scalar constant or a splat vector of constants,
1513  // handle it quickly.
1514  if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
1515  Offset += ElementSize * CI->getSExtValue();
1516  continue;
1517  }
1518 
1519  if (Offset != 0) {
1520  auto OffsetMIB = MIRBuilder.buildConstant({OffsetTy}, Offset);
1521  BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, OffsetMIB.getReg(0))
1522  .getReg(0);
1523  Offset = 0;
1524  }
1525 
1526  Register IdxReg = getOrCreateVReg(*Idx);
1527  LLT IdxTy = MRI->getType(IdxReg);
1528  if (IdxTy != OffsetTy) {
1529  if (!IdxTy.isVector() && WantSplatVector) {
1530  IdxReg = MIRBuilder.buildSplatVector(
1531  OffsetTy.changeElementType(IdxTy), IdxReg).getReg(0);
1532  }
1533 
1534  IdxReg = MIRBuilder.buildSExtOrTrunc(OffsetTy, IdxReg).getReg(0);
1535  }
1536 
1537  // N = N + Idx * ElementSize;
1538  // Avoid doing it for ElementSize of 1.
1539  Register GepOffsetReg;
1540  if (ElementSize != 1) {
1541  auto ElementSizeMIB = MIRBuilder.buildConstant(
1542  getLLTForType(*OffsetIRTy, *DL), ElementSize);
1543  GepOffsetReg =
1544  MIRBuilder.buildMul(OffsetTy, IdxReg, ElementSizeMIB).getReg(0);
1545  } else
1546  GepOffsetReg = IdxReg;
1547 
1548  BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, GepOffsetReg).getReg(0);
1549  }
1550  }
1551 
1552  if (Offset != 0) {
1553  auto OffsetMIB =
1554  MIRBuilder.buildConstant(OffsetTy, Offset);
1555  MIRBuilder.buildPtrAdd(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0));
1556  return true;
1557  }
1558 
1559  MIRBuilder.buildCopy(getOrCreateVReg(U), BaseReg);
1560  return true;
1561 }
1562 
1563 bool IRTranslator::translateMemFunc(const CallInst &CI,
1564  MachineIRBuilder &MIRBuilder,
1565  unsigned Opcode) {
1566 
1567  // If the source is undef, then just emit a nop.
1568  if (isa<UndefValue>(CI.getArgOperand(1)))
1569  return true;
1570 
1571  SmallVector<Register, 3> SrcRegs;
1572 
1573  unsigned MinPtrSize = UINT_MAX;
1574  for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(AI) != AE; ++AI) {
1575  Register SrcReg = getOrCreateVReg(**AI);
1576  LLT SrcTy = MRI->getType(SrcReg);
1577  if (SrcTy.isPointer())
1578  MinPtrSize = std::min<unsigned>(SrcTy.getSizeInBits(), MinPtrSize);
1579  SrcRegs.push_back(SrcReg);
1580  }
1581 
1582  LLT SizeTy = LLT::scalar(MinPtrSize);
1583 
1584  // The size operand should be the minimum of the pointer sizes.
1585  Register &SizeOpReg = SrcRegs[SrcRegs.size() - 1];
1586  if (MRI->getType(SizeOpReg) != SizeTy)
1587  SizeOpReg = MIRBuilder.buildZExtOrTrunc(SizeTy, SizeOpReg).getReg(0);
1588 
1589  auto ICall = MIRBuilder.buildInstr(Opcode);
1590  for (Register SrcReg : SrcRegs)
1591  ICall.addUse(SrcReg);
1592 
1593  Align DstAlign;
1594  Align SrcAlign;
1595  unsigned IsVol =
1596  cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1))->getZExtValue();
1597 
1598  if (auto *MCI = dyn_cast<MemCpyInst>(&CI)) {
1599  DstAlign = MCI->getDestAlign().valueOrOne();
1600  SrcAlign = MCI->getSourceAlign().valueOrOne();
1601  } else if (auto *MCI = dyn_cast<MemCpyInlineInst>(&CI)) {
1602  DstAlign = MCI->getDestAlign().valueOrOne();
1603  SrcAlign = MCI->getSourceAlign().valueOrOne();
1604  } else if (auto *MMI = dyn_cast<MemMoveInst>(&CI)) {
1605  DstAlign = MMI->getDestAlign().valueOrOne();
1606  SrcAlign = MMI->getSourceAlign().valueOrOne();
1607  } else {
1608  auto *MSI = cast<MemSetInst>(&CI);
1609  DstAlign = MSI->getDestAlign().valueOrOne();
1610  }
1611 
1612  if (Opcode != TargetOpcode::G_MEMCPY_INLINE) {
1613  // We need to propagate the tail call flag from the IR inst as an argument.
1614  // Otherwise, we have to pessimize and assume later that we cannot tail call
1615  // any memory intrinsics.
1616  ICall.addImm(CI.isTailCall() ? 1 : 0);
1617  }
1618 
1619  // Create mem operands to store the alignment and volatile info.
1621  ICall.addMemOperand(MF->getMachineMemOperand(
1623  MachineMemOperand::MOStore | VolFlag, 1, DstAlign));
1624  if (Opcode != TargetOpcode::G_MEMSET)
1625  ICall.addMemOperand(MF->getMachineMemOperand(
1627  MachineMemOperand::MOLoad | VolFlag, 1, SrcAlign));
1628 
1629  return true;
1630 }
1631 
1632 void IRTranslator::getStackGuard(Register DstReg,
1633  MachineIRBuilder &MIRBuilder) {
1635  MRI->setRegClass(DstReg, TRI->getPointerRegClass(*MF));
1636  auto MIB =
1637  MIRBuilder.buildInstr(TargetOpcode::LOAD_STACK_GUARD, {DstReg}, {});
1638 
1639  auto &TLI = *MF->getSubtarget().getTargetLowering();
1640  Value *Global = TLI.getSDagStackGuard(*MF->getFunction().getParent());
1641  if (!Global)
1642  return;
1643 
1644  unsigned AddrSpace = Global->getType()->getPointerAddressSpace();
1645  LLT PtrTy = LLT::pointer(AddrSpace, DL->getPointerSizeInBits(AddrSpace));
1646 
1647  MachinePointerInfo MPInfo(Global);
1651  MPInfo, Flags, PtrTy, DL->getPointerABIAlignment(AddrSpace));
1652  MIB.setMemRefs({MemRef});
1653 }
1654 
1655 bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
1656  MachineIRBuilder &MIRBuilder) {
1657  ArrayRef<Register> ResRegs = getOrCreateVRegs(CI);
1658  MIRBuilder.buildInstr(
1659  Op, {ResRegs[0], ResRegs[1]},
1660  {getOrCreateVReg(*CI.getOperand(0)), getOrCreateVReg(*CI.getOperand(1))});
1661 
1662  return true;
1663 }
1664 
1665 bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
1666  MachineIRBuilder &MIRBuilder) {
1667  Register Dst = getOrCreateVReg(CI);
1668  Register Src0 = getOrCreateVReg(*CI.getOperand(0));
1669  Register Src1 = getOrCreateVReg(*CI.getOperand(1));
1670  uint64_t Scale = cast<ConstantInt>(CI.getOperand(2))->getZExtValue();
1671  MIRBuilder.buildInstr(Op, {Dst}, { Src0, Src1, Scale });
1672  return true;
1673 }
1674 
1675 unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
1676  switch (ID) {
1677  default:
1678  break;
1679  case Intrinsic::bswap:
1680  return TargetOpcode::G_BSWAP;
1681  case Intrinsic::bitreverse:
1682  return TargetOpcode::G_BITREVERSE;
1683  case Intrinsic::fshl:
1684  return TargetOpcode::G_FSHL;
1685  case Intrinsic::fshr:
1686  return TargetOpcode::G_FSHR;
1687  case Intrinsic::ceil:
1688  return TargetOpcode::G_FCEIL;
1689  case Intrinsic::cos:
1690  return TargetOpcode::G_FCOS;
1691  case Intrinsic::ctpop:
1692  return TargetOpcode::G_CTPOP;
1693  case Intrinsic::exp:
1694  return TargetOpcode::G_FEXP;
1695  case Intrinsic::exp2:
1696  return TargetOpcode::G_FEXP2;
1697  case Intrinsic::fabs:
1698  return TargetOpcode::G_FABS;
1699  case Intrinsic::copysign:
1700  return TargetOpcode::G_FCOPYSIGN;
1701  case Intrinsic::minnum:
1702  return TargetOpcode::G_FMINNUM;
1703  case Intrinsic::maxnum:
1704  return TargetOpcode::G_FMAXNUM;
1705  case Intrinsic::minimum:
1706  return TargetOpcode::G_FMINIMUM;
1707  case Intrinsic::maximum:
1708  return TargetOpcode::G_FMAXIMUM;
1709  case Intrinsic::canonicalize:
1710  return TargetOpcode::G_FCANONICALIZE;
1711  case Intrinsic::floor:
1712  return TargetOpcode::G_FFLOOR;
1713  case Intrinsic::fma:
1714  return TargetOpcode::G_FMA;
1715  case Intrinsic::log:
1716  return TargetOpcode::G_FLOG;
1717  case Intrinsic::log2:
1718  return TargetOpcode::G_FLOG2;
1719  case Intrinsic::log10:
1720  return TargetOpcode::G_FLOG10;
1721  case Intrinsic::nearbyint:
1722  return TargetOpcode::G_FNEARBYINT;
1723  case Intrinsic::pow:
1724  return TargetOpcode::G_FPOW;
1725  case Intrinsic::powi:
1726  return TargetOpcode::G_FPOWI;
1727  case Intrinsic::rint:
1728  return TargetOpcode::G_FRINT;
1729  case Intrinsic::round:
1730  return TargetOpcode::G_INTRINSIC_ROUND;
1731  case Intrinsic::roundeven:
1732  return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
1733  case Intrinsic::sin:
1734  return TargetOpcode::G_FSIN;
1735  case Intrinsic::sqrt:
1736  return TargetOpcode::G_FSQRT;
1737  case Intrinsic::trunc:
1738  return TargetOpcode::G_INTRINSIC_TRUNC;
1739  case Intrinsic::readcyclecounter:
1740  return TargetOpcode::G_READCYCLECOUNTER;
1741  case Intrinsic::ptrmask:
1742  return TargetOpcode::G_PTRMASK;
1743  case Intrinsic::lrint:
1744  return TargetOpcode::G_INTRINSIC_LRINT;
1745  // FADD/FMUL require checking the FMF, so are handled elsewhere.
1746  case Intrinsic::vector_reduce_fmin:
1747  return TargetOpcode::G_VECREDUCE_FMIN;
1748  case Intrinsic::vector_reduce_fmax:
1749  return TargetOpcode::G_VECREDUCE_FMAX;
1750  case Intrinsic::vector_reduce_add:
1751  return TargetOpcode::G_VECREDUCE_ADD;
1752  case Intrinsic::vector_reduce_mul:
1753  return TargetOpcode::G_VECREDUCE_MUL;
1754  case Intrinsic::vector_reduce_and:
1755  return TargetOpcode::G_VECREDUCE_AND;
1756  case Intrinsic::vector_reduce_or:
1757  return TargetOpcode::G_VECREDUCE_OR;
1758  case Intrinsic::vector_reduce_xor:
1759  return TargetOpcode::G_VECREDUCE_XOR;
1760  case Intrinsic::vector_reduce_smax:
1761  return TargetOpcode::G_VECREDUCE_SMAX;
1762  case Intrinsic::vector_reduce_smin:
1763  return TargetOpcode::G_VECREDUCE_SMIN;
1764  case Intrinsic::vector_reduce_umax:
1765  return TargetOpcode::G_VECREDUCE_UMAX;
1766  case Intrinsic::vector_reduce_umin:
1767  return TargetOpcode::G_VECREDUCE_UMIN;
1768  case Intrinsic::lround:
1769  return TargetOpcode::G_LROUND;
1770  case Intrinsic::llround:
1771  return TargetOpcode::G_LLROUND;
1772  }
1773  return Intrinsic::not_intrinsic;
1774 }
1775 
1776 bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
1777  Intrinsic::ID ID,
1778  MachineIRBuilder &MIRBuilder) {
1779 
1780  unsigned Op = getSimpleIntrinsicOpcode(ID);
1781 
1782  // Is this a simple intrinsic?
1784  return false;
1785 
1786  // Yes. Let's translate it.
1788  for (auto &Arg : CI.args())
1789  VRegs.push_back(getOrCreateVReg(*Arg));
1790 
1791  MIRBuilder.buildInstr(Op, {getOrCreateVReg(CI)}, VRegs,
1793  return true;
1794 }
1795 
1796 // TODO: Include ConstainedOps.def when all strict instructions are defined.
1798  switch (ID) {
1799  case Intrinsic::experimental_constrained_fadd:
1800  return TargetOpcode::G_STRICT_FADD;
1801  case Intrinsic::experimental_constrained_fsub:
1802  return TargetOpcode::G_STRICT_FSUB;
1803  case Intrinsic::experimental_constrained_fmul:
1804  return TargetOpcode::G_STRICT_FMUL;
1805  case Intrinsic::experimental_constrained_fdiv:
1806  return TargetOpcode::G_STRICT_FDIV;
1807  case Intrinsic::experimental_constrained_frem:
1808  return TargetOpcode::G_STRICT_FREM;
1809  case Intrinsic::experimental_constrained_fma:
1810  return TargetOpcode::G_STRICT_FMA;
1811  case Intrinsic::experimental_constrained_sqrt:
1812  return TargetOpcode::G_STRICT_FSQRT;
1813  default:
1814  return 0;
1815  }
1816 }
1817 
1818 bool IRTranslator::translateConstrainedFPIntrinsic(
1819  const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) {
1820  fp::ExceptionBehavior EB = FPI.getExceptionBehavior().getValue();
1821 
1822  unsigned Opcode = getConstrainedOpcode(FPI.getIntrinsicID());
1823  if (!Opcode)
1824  return false;
1825 
1826  unsigned Flags = MachineInstr::copyFlagsFromInstruction(FPI);
1828  Flags |= MachineInstr::NoFPExcept;
1829 
1831  VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(0)));
1832  if (!FPI.isUnaryOp())
1833  VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(1)));
1834  if (FPI.isTernaryOp())
1835  VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(2)));
1836 
1837  MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(FPI)}, VRegs, Flags);
1838  return true;
1839 }
1840 
1841 bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
1842  MachineIRBuilder &MIRBuilder) {
1843  if (auto *MI = dyn_cast<AnyMemIntrinsic>(&CI)) {
1844  if (ORE->enabled()) {
1845  const Function &F = *MI->getParent()->getParent();
1846  auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
1847  if (MemoryOpRemark::canHandle(MI, TLI)) {
1848  MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, TLI);
1849  R.visit(MI);
1850  }
1851  }
1852  }
1853 
1854  // If this is a simple intrinsic (that is, we just need to add a def of
1855  // a vreg, and uses for each arg operand, then translate it.
1856  if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
1857  return true;
1858 
1859  switch (ID) {
1860  default:
1861  break;
1862  case Intrinsic::lifetime_start:
1863  case Intrinsic::lifetime_end: {
1864  // No stack colouring in O0, discard region information.
1865  if (MF->getTarget().getOptLevel() == CodeGenOpt::None)
1866  return true;
1867 
1868  unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
1870 
1871  // Get the underlying objects for the location passed on the lifetime
1872  // marker.
1874  getUnderlyingObjects(CI.getArgOperand(1), Allocas);
1875 
1876  // Iterate over each underlying object, creating lifetime markers for each
1877  // static alloca. Quit if we find a non-static alloca.
1878  for (const Value *V : Allocas) {
1879  const AllocaInst *AI = dyn_cast<AllocaInst>(V);
1880  if (!AI)
1881  continue;
1882 
1883  if (!AI->isStaticAlloca())
1884  return true;
1885 
1886  MIRBuilder.buildInstr(Op).addFrameIndex(getOrCreateFrameIndex(*AI));
1887  }
1888  return true;
1889  }
1890  case Intrinsic::dbg_declare: {
1891  const DbgDeclareInst &DI = cast<DbgDeclareInst>(CI);
1892  assert(DI.getVariable() && "Missing variable");
1893 
1894  const Value *Address = DI.getAddress();
1895  if (!Address || isa<UndefValue>(Address)) {
1896  LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
1897  return true;
1898  }
1899 
1901  MIRBuilder.getDebugLoc()) &&
1902  "Expected inlined-at fields to agree");
1903  auto AI = dyn_cast<AllocaInst>(Address);
1904  if (AI && AI->isStaticAlloca()) {
1905  // Static allocas are tracked at the MF level, no need for DBG_VALUE
1906  // instructions (in fact, they get ignored if they *do* exist).
1908  getOrCreateFrameIndex(*AI), DI.getDebugLoc());
1909  } else {
1910  // A dbg.declare describes the address of a source variable, so lower it
1911  // into an indirect DBG_VALUE.
1912  MIRBuilder.buildIndirectDbgValue(getOrCreateVReg(*Address),
1913  DI.getVariable(), DI.getExpression());
1914  }
1915  return true;
1916  }
1917  case Intrinsic::dbg_label: {
1918  const DbgLabelInst &DI = cast<DbgLabelInst>(CI);
1919  assert(DI.getLabel() && "Missing label");
1920 
1922  MIRBuilder.getDebugLoc()) &&
1923  "Expected inlined-at fields to agree");
1924 
1925  MIRBuilder.buildDbgLabel(DI.getLabel());
1926  return true;
1927  }
1928  case Intrinsic::vaend:
1929  // No target I know of cares about va_end. Certainly no in-tree target
1930  // does. Simplest intrinsic ever!
1931  return true;
1932  case Intrinsic::vastart: {
1933  auto &TLI = *MF->getSubtarget().getTargetLowering();
1934  Value *Ptr = CI.getArgOperand(0);
1935  unsigned ListSize = TLI.getVaListSizeInBits(*DL) / 8;
1936 
1937  // FIXME: Get alignment
1938  MIRBuilder.buildInstr(TargetOpcode::G_VASTART, {}, {getOrCreateVReg(*Ptr)})
1941  ListSize, Align(1)));
1942  return true;
1943  }
1944  case Intrinsic::dbg_value: {
1945  // This form of DBG_VALUE is target-independent.
1946  const DbgValueInst &DI = cast<DbgValueInst>(CI);
1947  const Value *V = DI.getValue();
1949  MIRBuilder.getDebugLoc()) &&
1950  "Expected inlined-at fields to agree");
1951  if (!V || DI.hasArgList()) {
1952  // DI cannot produce a valid DBG_VALUE, so produce an undef DBG_VALUE to
1953  // terminate any prior location.
1954  MIRBuilder.buildIndirectDbgValue(0, DI.getVariable(), DI.getExpression());
1955  } else if (const auto *CI = dyn_cast<Constant>(V)) {
1956  MIRBuilder.buildConstDbgValue(*CI, DI.getVariable(), DI.getExpression());
1957  } else {
1958  for (Register Reg : getOrCreateVRegs(*V)) {
1959  // FIXME: This does not handle register-indirect values at offset 0. The
1960  // direct/indirect thing shouldn't really be handled by something as
1961  // implicit as reg+noreg vs reg+imm in the first place, but it seems
1962  // pretty baked in right now.
1963  MIRBuilder.buildDirectDbgValue(Reg, DI.getVariable(), DI.getExpression());
1964  }
1965  }
1966  return true;
1967  }
1968  case Intrinsic::uadd_with_overflow:
1969  return translateOverflowIntrinsic(CI, TargetOpcode::G_UADDO, MIRBuilder);
1970  case Intrinsic::sadd_with_overflow:
1971  return translateOverflowIntrinsic(CI, TargetOpcode::G_SADDO, MIRBuilder);
1972  case Intrinsic::usub_with_overflow:
1973  return translateOverflowIntrinsic(CI, TargetOpcode::G_USUBO, MIRBuilder);
1974  case Intrinsic::ssub_with_overflow:
1975  return translateOverflowIntrinsic(CI, TargetOpcode::G_SSUBO, MIRBuilder);
1976  case Intrinsic::umul_with_overflow:
1977  return translateOverflowIntrinsic(CI, TargetOpcode::G_UMULO, MIRBuilder);
1978  case Intrinsic::smul_with_overflow:
1979  return translateOverflowIntrinsic(CI, TargetOpcode::G_SMULO, MIRBuilder);
1980  case Intrinsic::uadd_sat:
1981  return translateBinaryOp(TargetOpcode::G_UADDSAT, CI, MIRBuilder);
1982  case Intrinsic::sadd_sat:
1983  return translateBinaryOp(TargetOpcode::G_SADDSAT, CI, MIRBuilder);
1984  case Intrinsic::usub_sat:
1985  return translateBinaryOp(TargetOpcode::G_USUBSAT, CI, MIRBuilder);
1986  case Intrinsic::ssub_sat:
1987  return translateBinaryOp(TargetOpcode::G_SSUBSAT, CI, MIRBuilder);
1988  case Intrinsic::ushl_sat:
1989  return translateBinaryOp(TargetOpcode::G_USHLSAT, CI, MIRBuilder);
1990  case Intrinsic::sshl_sat:
1991  return translateBinaryOp(TargetOpcode::G_SSHLSAT, CI, MIRBuilder);
1992  case Intrinsic::umin:
1993  return translateBinaryOp(TargetOpcode::G_UMIN, CI, MIRBuilder);
1994  case Intrinsic::umax:
1995  return translateBinaryOp(TargetOpcode::G_UMAX, CI, MIRBuilder);
1996  case Intrinsic::smin:
1997  return translateBinaryOp(TargetOpcode::G_SMIN, CI, MIRBuilder);
1998  case Intrinsic::smax:
1999  return translateBinaryOp(TargetOpcode::G_SMAX, CI, MIRBuilder);
2000  case Intrinsic::abs:
2001  // TODO: Preserve "int min is poison" arg in GMIR?
2002  return translateUnaryOp(TargetOpcode::G_ABS, CI, MIRBuilder);
2003  case Intrinsic::smul_fix:
2004  return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIX, CI, MIRBuilder);
2005  case Intrinsic::umul_fix:
2006  return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIX, CI, MIRBuilder);
2007  case Intrinsic::smul_fix_sat:
2008  return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
2009  case Intrinsic::umul_fix_sat:
2010  return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
2011  case Intrinsic::sdiv_fix:
2012  return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
2013  case Intrinsic::udiv_fix:
2014  return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
2015  case Intrinsic::sdiv_fix_sat:
2016  return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
2017  case Intrinsic::udiv_fix_sat:
2018  return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
2019  case Intrinsic::fmuladd: {
2020  const TargetMachine &TM = MF->getTarget();
2021  const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
2022  Register Dst = getOrCreateVReg(CI);
2023  Register Op0 = getOrCreateVReg(*CI.getArgOperand(0));
2024  Register Op1 = getOrCreateVReg(*CI.getArgOperand(1));
2025  Register Op2 = getOrCreateVReg(*CI.getArgOperand(2));
2026  if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
2028  TLI.getValueType(*DL, CI.getType()))) {
2029  // TODO: Revisit this to see if we should move this part of the
2030  // lowering to the combiner.
2031  MIRBuilder.buildFMA(Dst, Op0, Op1, Op2,
2033  } else {
2034  LLT Ty = getLLTForType(*CI.getType(), *DL);
2035  auto FMul = MIRBuilder.buildFMul(
2036  Ty, Op0, Op1, MachineInstr::copyFlagsFromInstruction(CI));
2037  MIRBuilder.buildFAdd(Dst, FMul, Op2,
2039  }
2040  return true;
2041  }
2042  case Intrinsic::convert_from_fp16:
2043  // FIXME: This intrinsic should probably be removed from the IR.
2044  MIRBuilder.buildFPExt(getOrCreateVReg(CI),
2045  getOrCreateVReg(*CI.getArgOperand(0)),
2047  return true;
2048  case Intrinsic::convert_to_fp16:
2049  // FIXME: This intrinsic should probably be removed from the IR.
2050  MIRBuilder.buildFPTrunc(getOrCreateVReg(CI),
2051  getOrCreateVReg(*CI.getArgOperand(0)),
2053  return true;
2054  case Intrinsic::memcpy_inline:
2055  return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY_INLINE);
2056  case Intrinsic::memcpy:
2057  return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY);
2058  case Intrinsic::memmove:
2059  return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMMOVE);
2060  case Intrinsic::memset:
2061  return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMSET);
2062  case Intrinsic::eh_typeid_for: {
2064  Register Reg = getOrCreateVReg(CI);
2065  unsigned TypeID = MF->getTypeIDFor(GV);
2066  MIRBuilder.buildConstant(Reg, TypeID);
2067  return true;
2068  }
2069  case Intrinsic::objectsize:
2070  llvm_unreachable("llvm.objectsize.* should have been lowered already");
2071 
2072  case Intrinsic::is_constant:
2073  llvm_unreachable("llvm.is.constant.* should have been lowered already");
2074 
2075  case Intrinsic::stackguard:
2076  getStackGuard(getOrCreateVReg(CI), MIRBuilder);
2077  return true;
2078  case Intrinsic::stackprotector: {
2079  LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
2080  Register GuardVal = MRI->createGenericVirtualRegister(PtrTy);
2081  getStackGuard(GuardVal, MIRBuilder);
2082 
2083  AllocaInst *Slot = cast<AllocaInst>(CI.getArgOperand(1));
2084  int FI = getOrCreateFrameIndex(*Slot);
2086 
2087  MIRBuilder.buildStore(
2088  GuardVal, getOrCreateVReg(*Slot),
2092  PtrTy, Align(8)));
2093  return true;
2094  }
2095  case Intrinsic::stacksave: {
2096  // Save the stack pointer to the location provided by the intrinsic.
2097  Register Reg = getOrCreateVReg(CI);
2101 
2102  // If the target doesn't specify a stack pointer, then fall back.
2103  if (!StackPtr)
2104  return false;
2105 
2106  MIRBuilder.buildCopy(Reg, StackPtr);
2107  return true;
2108  }
2109  case Intrinsic::stackrestore: {
2110  // Restore the stack pointer from the location provided by the intrinsic.
2111  Register Reg = getOrCreateVReg(*CI.getArgOperand(0));
2115 
2116  // If the target doesn't specify a stack pointer, then fall back.
2117  if (!StackPtr)
2118  return false;
2119 
2120  MIRBuilder.buildCopy(StackPtr, Reg);
2121  return true;
2122  }
2123  case Intrinsic::cttz:
2124  case Intrinsic::ctlz: {
2125  ConstantInt *Cst = cast<ConstantInt>(CI.getArgOperand(1));
2126  bool isTrailing = ID == Intrinsic::cttz;
2127  unsigned Opcode = isTrailing
2128  ? Cst->isZero() ? TargetOpcode::G_CTTZ
2129  : TargetOpcode::G_CTTZ_ZERO_UNDEF
2130  : Cst->isZero() ? TargetOpcode::G_CTLZ
2131  : TargetOpcode::G_CTLZ_ZERO_UNDEF;
2132  MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(CI)},
2133  {getOrCreateVReg(*CI.getArgOperand(0))});
2134  return true;
2135  }
2136  case Intrinsic::invariant_start: {
2137  LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
2139  MIRBuilder.buildUndef(Undef);
2140  return true;
2141  }
2142  case Intrinsic::invariant_end:
2143  return true;
2144  case Intrinsic::expect:
2145  case Intrinsic::annotation:
2146  case Intrinsic::ptr_annotation:
2147  case Intrinsic::launder_invariant_group:
2148  case Intrinsic::strip_invariant_group: {
2149  // Drop the intrinsic, but forward the value.
2150  MIRBuilder.buildCopy(getOrCreateVReg(CI),
2151  getOrCreateVReg(*CI.getArgOperand(0)));
2152  return true;
2153  }
2154  case Intrinsic::assume:
2155  case Intrinsic::experimental_noalias_scope_decl:
2156  case Intrinsic::var_annotation:
2157  case Intrinsic::sideeffect:
2158  // Discard annotate attributes, assumptions, and artificial side-effects.
2159  return true;
2160  case Intrinsic::read_volatile_register:
2161  case Intrinsic::read_register: {
2162  Value *Arg = CI.getArgOperand(0);
2163  MIRBuilder
2164  .buildInstr(TargetOpcode::G_READ_REGISTER, {getOrCreateVReg(CI)}, {})
2165  .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()));
2166  return true;
2167  }
2168  case Intrinsic::write_register: {
2169  Value *Arg = CI.getArgOperand(0);
2170  MIRBuilder.buildInstr(TargetOpcode::G_WRITE_REGISTER)
2171  .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()))
2172  .addUse(getOrCreateVReg(*CI.getArgOperand(1)));
2173  return true;
2174  }
2175  case Intrinsic::localescape: {
2176  MachineBasicBlock &EntryMBB = MF->front();
2178 
2179  // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
2180  // is the same on all targets.
2181  for (unsigned Idx = 0, E = CI.arg_size(); Idx < E; ++Idx) {
2182  Value *Arg = CI.getArgOperand(Idx)->stripPointerCasts();
2183  if (isa<ConstantPointerNull>(Arg))
2184  continue; // Skip null pointers. They represent a hole in index space.
2185 
2186  int FI = getOrCreateFrameIndex(*cast<AllocaInst>(Arg));
2187  MCSymbol *FrameAllocSym =
2188  MF->getMMI().getContext().getOrCreateFrameAllocSymbol(EscapedName,
2189  Idx);
2190 
2191  // This should be inserted at the start of the entry block.
2192  auto LocalEscape =
2193  MIRBuilder.buildInstrNoInsert(TargetOpcode::LOCAL_ESCAPE)
2194  .addSym(FrameAllocSym)
2195  .addFrameIndex(FI);
2196 
2197  EntryMBB.insert(EntryMBB.begin(), LocalEscape);
2198  }
2199 
2200  return true;
2201  }
2202  case Intrinsic::vector_reduce_fadd:
2203  case Intrinsic::vector_reduce_fmul: {
2204  // Need to check for the reassoc flag to decide whether we want a
2205  // sequential reduction opcode or not.
2206  Register Dst = getOrCreateVReg(CI);
2207  Register ScalarSrc = getOrCreateVReg(*CI.getArgOperand(0));
2208  Register VecSrc = getOrCreateVReg(*CI.getArgOperand(1));
2209  unsigned Opc = 0;
2210  if (!CI.hasAllowReassoc()) {
2211  // The sequential ordering case.
2212  Opc = ID == Intrinsic::vector_reduce_fadd
2213  ? TargetOpcode::G_VECREDUCE_SEQ_FADD
2214  : TargetOpcode::G_VECREDUCE_SEQ_FMUL;
2215  MIRBuilder.buildInstr(Opc, {Dst}, {ScalarSrc, VecSrc},
2217  return true;
2218  }
2219  // We split the operation into a separate G_FADD/G_FMUL + the reduce,
2220  // since the associativity doesn't matter.
2221  unsigned ScalarOpc;
2222  if (ID == Intrinsic::vector_reduce_fadd) {
2223  Opc = TargetOpcode::G_VECREDUCE_FADD;
2224  ScalarOpc = TargetOpcode::G_FADD;
2225  } else {
2226  Opc = TargetOpcode::G_VECREDUCE_FMUL;
2227  ScalarOpc = TargetOpcode::G_FMUL;
2228  }
2229  LLT DstTy = MRI->getType(Dst);
2230  auto Rdx = MIRBuilder.buildInstr(
2231  Opc, {DstTy}, {VecSrc}, MachineInstr::copyFlagsFromInstruction(CI));
2232  MIRBuilder.buildInstr(ScalarOpc, {Dst}, {ScalarSrc, Rdx},
2234 
2235  return true;
2236  }
2237  case Intrinsic::trap:
2238  case Intrinsic::debugtrap:
2239  case Intrinsic::ubsantrap: {
2240  StringRef TrapFuncName =
2241  CI.getAttributes().getFnAttr("trap-func-name").getValueAsString();
2242  if (TrapFuncName.empty())
2243  break; // Use the default handling.
2245  if (ID == Intrinsic::ubsantrap) {
2246  Info.OrigArgs.push_back({getOrCreateVRegs(*CI.getArgOperand(0)),
2247  CI.getArgOperand(0)->getType(), 0});
2248  }
2249  Info.Callee = MachineOperand::CreateES(TrapFuncName.data());
2250  Info.CB = &CI;
2251  Info.OrigRet = {Register(), Type::getVoidTy(CI.getContext()), 0};
2252  return CLI->lowerCall(MIRBuilder, Info);
2253  }
2254 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
2255  case Intrinsic::INTRINSIC:
2256 #include "llvm/IR/ConstrainedOps.def"
2257  return translateConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(CI),
2258  MIRBuilder);
2259 
2260  }
2261  return false;
2262 }
2263 
2264 bool IRTranslator::translateInlineAsm(const CallBase &CB,
2265  MachineIRBuilder &MIRBuilder) {
2266 
2268 
2269  if (!ALI) {
2270  LLVM_DEBUG(
2271  dbgs() << "Inline asm lowering is not supported for this target yet\n");
2272  return false;
2273  }
2274 
2275  return ALI->lowerInlineAsm(
2276  MIRBuilder, CB, [&](const Value &Val) { return getOrCreateVRegs(Val); });
2277 }
2278 
2279 bool IRTranslator::translateCallBase(const CallBase &CB,
2280  MachineIRBuilder &MIRBuilder) {
2281  ArrayRef<Register> Res = getOrCreateVRegs(CB);
2282 
2284  Register SwiftInVReg = 0;
2285  Register SwiftErrorVReg = 0;
2286  for (auto &Arg : CB.args()) {
2287  if (CLI->supportSwiftError() && isSwiftError(Arg)) {
2288  assert(SwiftInVReg == 0 && "Expected only one swift error argument");
2289  LLT Ty = getLLTForType(*Arg->getType(), *DL);
2290  SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
2291  MIRBuilder.buildCopy(SwiftInVReg, SwiftError.getOrCreateVRegUseAt(
2292  &CB, &MIRBuilder.getMBB(), Arg));
2293  Args.emplace_back(makeArrayRef(SwiftInVReg));
2294  SwiftErrorVReg =
2295  SwiftError.getOrCreateVRegDefAt(&CB, &MIRBuilder.getMBB(), Arg);
2296  continue;
2297  }
2298  Args.push_back(getOrCreateVRegs(*Arg));
2299  }
2300 
2301  if (auto *CI = dyn_cast<CallInst>(&CB)) {
2302  if (ORE->enabled()) {
2303  const Function &F = *CI->getParent()->getParent();
2304  auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
2305  if (MemoryOpRemark::canHandle(CI, TLI)) {
2306  MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, TLI);
2307  R.visit(CI);
2308  }
2309  }
2310  }
2311 
2312  // We don't set HasCalls on MFI here yet because call lowering may decide to
2313  // optimize into tail calls. Instead, we defer that to selection where a final
2314  // scan is done to check if any instructions are calls.
2315  bool Success =
2316  CLI->lowerCall(MIRBuilder, CB, Res, Args, SwiftErrorVReg,
2317  [&]() { return getOrCreateVReg(*CB.getCalledOperand()); });
2318 
2319  // Check if we just inserted a tail call.
2320  if (Success) {
2321  assert(!HasTailCall && "Can't tail call return twice from block?");
2322  const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
2323  HasTailCall = TII->isTailCall(*std::prev(MIRBuilder.getInsertPt()));
2324  }
2325 
2326  return Success;
2327 }
2328 
2329 bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
2330  const CallInst &CI = cast<CallInst>(U);
2331  auto TII = MF->getTarget().getIntrinsicInfo();
2332  const Function *F = CI.getCalledFunction();
2333 
2334  // FIXME: support Windows dllimport function calls.
2335  if (F && (F->hasDLLImportStorageClass() ||
2336  (MF->getTarget().getTargetTriple().isOSWindows() &&
2337  F->hasExternalWeakLinkage())))
2338  return false;
2339 
2340  // FIXME: support control flow guard targets.
2342  return false;
2343 
2344  if (CI.isInlineAsm())
2345  return translateInlineAsm(CI, MIRBuilder);
2346 
2347  diagnoseDontCall(CI);
2348 
2350  if (F && F->isIntrinsic()) {
2351  ID = F->getIntrinsicID();
2352  if (TII && ID == Intrinsic::not_intrinsic)
2353  ID = static_cast<Intrinsic::ID>(TII->getIntrinsicID(F));
2354  }
2355 
2356  if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic)
2357  return translateCallBase(CI, MIRBuilder);
2358 
2359  assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
2360 
2361  if (translateKnownIntrinsic(CI, ID, MIRBuilder))
2362  return true;
2363 
2364  ArrayRef<Register> ResultRegs;
2365  if (!CI.getType()->isVoidTy())
2366  ResultRegs = getOrCreateVRegs(CI);
2367 
2368  // Ignore the callsite attributes. Backend code is most likely not expecting
2369  // an intrinsic to sometimes have side effects and sometimes not.
2370  MachineInstrBuilder MIB =
2371  MIRBuilder.buildIntrinsic(ID, ResultRegs, !F->doesNotAccessMemory());
2372  if (isa<FPMathOperator>(CI))
2373  MIB->copyIRFlags(CI);
2374 
2375  for (auto &Arg : enumerate(CI.args())) {
2376  // If this is required to be an immediate, don't materialize it in a
2377  // register.
2378  if (CI.paramHasAttr(Arg.index(), Attribute::ImmArg)) {
2379  if (ConstantInt *CI = dyn_cast<ConstantInt>(Arg.value())) {
2380  // imm arguments are more convenient than cimm (and realistically
2381  // probably sufficient), so use them.
2382  assert(CI->getBitWidth() <= 64 &&
2383  "large intrinsic immediates not handled");
2384  MIB.addImm(CI->getSExtValue());
2385  } else {
2386  MIB.addFPImm(cast<ConstantFP>(Arg.value()));
2387  }
2388  } else if (auto *MDVal = dyn_cast<MetadataAsValue>(Arg.value())) {
2389  auto *MD = MDVal->getMetadata();
2390  auto *MDN = dyn_cast<MDNode>(MD);
2391  if (!MDN) {
2392  if (auto *ConstMD = dyn_cast<ConstantAsMetadata>(MD))
2393  MDN = MDNode::get(MF->getFunction().getContext(), ConstMD);
2394  else // This was probably an MDString.
2395  return false;
2396  }
2397  MIB.addMetadata(MDN);
2398  } else {
2399  ArrayRef<Register> VRegs = getOrCreateVRegs(*Arg.value());
2400  if (VRegs.size() > 1)
2401  return false;
2402  MIB.addUse(VRegs[0]);
2403  }
2404  }
2405 
2406  // Add a MachineMemOperand if it is a target mem intrinsic.
2407  const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
2408  TargetLowering::IntrinsicInfo Info;
2409  // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
2410  if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) {
2411  Align Alignment = Info.align.getValueOr(
2412  DL->getABITypeAlign(Info.memVT.getTypeForEVT(F->getContext())));
2413  LLT MemTy = Info.memVT.isSimple()
2414  ? getLLTForMVT(Info.memVT.getSimpleVT())
2415  : LLT::scalar(Info.memVT.getStoreSizeInBits());
2417  Info.flags, MemTy, Alignment));
2418  }
2419 
2420  return true;
2421 }
2422 
2423 bool IRTranslator::findUnwindDestinations(
2424  const BasicBlock *EHPadBB,
2425  BranchProbability Prob,
2426  SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2427  &UnwindDests) {
2428  EHPersonality Personality = classifyEHPersonality(
2429  EHPadBB->getParent()->getFunction().getPersonalityFn());
2430  bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2431  bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2432  bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2433  bool IsSEH = isAsynchronousEHPersonality(Personality);
2434 
2435  if (IsWasmCXX) {
2436  // Ignore this for now.
2437  return false;
2438  }
2439 
2440  while (EHPadBB) {
2441  const Instruction *Pad = EHPadBB->getFirstNonPHI();
2442  BasicBlock *NewEHPadBB = nullptr;
2443  if (isa<LandingPadInst>(Pad)) {
2444  // Stop on landingpads. They are not funclets.
2445  UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
2446  break;
2447  }
2448  if (isa<CleanupPadInst>(Pad)) {
2449  // Stop on cleanup pads. Cleanups are always funclet entries for all known
2450  // personalities.
2451  UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
2452  UnwindDests.back().first->setIsEHScopeEntry();
2453  UnwindDests.back().first->setIsEHFuncletEntry();
2454  break;
2455  }
2456  if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2457  // Add the catchpad handlers to the possible destinations.
2458  for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2459  UnwindDests.emplace_back(&getMBB(*CatchPadBB), Prob);
2460  // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2461  if (IsMSVCCXX || IsCoreCLR)
2462  UnwindDests.back().first->setIsEHFuncletEntry();
2463  if (!IsSEH)
2464  UnwindDests.back().first->setIsEHScopeEntry();
2465  }
2466  NewEHPadBB = CatchSwitch->getUnwindDest();
2467  } else {
2468  continue;
2469  }
2470 
2471  BranchProbabilityInfo *BPI = FuncInfo.BPI;
2472  if (BPI && NewEHPadBB)
2473  Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
2474  EHPadBB = NewEHPadBB;
2475  }
2476  return true;
2477 }
2478 
2479 bool IRTranslator::translateInvoke(const User &U,
2480  MachineIRBuilder &MIRBuilder) {
2481  const InvokeInst &I = cast<InvokeInst>(U);
2482  MCContext &Context = MF->getContext();
2483 
2484  const BasicBlock *ReturnBB = I.getSuccessor(0);
2485  const BasicBlock *EHPadBB = I.getSuccessor(1);
2486 
2487  const Function *Fn = I.getCalledFunction();
2488 
2489  // FIXME: support invoking patchpoint and statepoint intrinsics.
2490  if (Fn && Fn->isIntrinsic())
2491  return false;
2492 
2493  // FIXME: support whatever these are.
2494  if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
2495  return false;
2496 
2497  // FIXME: support control flow guard targets.
2498  if (I.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget))
2499  return false;
2500 
2501  // FIXME: support Windows exception handling.
2502  if (!isa<LandingPadInst>(EHPadBB->getFirstNonPHI()))
2503  return false;
2504 
2505  bool LowerInlineAsm = false;
2506  if (I.isInlineAsm()) {
2507  const InlineAsm *IA = cast<InlineAsm>(I.getCalledOperand());
2508  if (!IA->canThrow()) {
2509  // Fast path without emitting EH_LABELs.
2510 
2511  if (!translateInlineAsm(I, MIRBuilder))
2512  return false;
2513 
2514  MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB(),
2515  *ReturnMBB = &getMBB(*ReturnBB);
2516 
2517  // Update successor info.
2518  addSuccessorWithProb(InvokeMBB, ReturnMBB, BranchProbability::getOne());
2519 
2520  MIRBuilder.buildBr(*ReturnMBB);
2521  return true;
2522  } else {
2523  LowerInlineAsm = true;
2524  }
2525  }
2526 
2527  // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
2528  // the region covered by the try.
2529  MCSymbol *BeginSymbol = Context.createTempSymbol();
2530  MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol);
2531 
2532  if (LowerInlineAsm) {
2533  if (!translateInlineAsm(I, MIRBuilder))
2534  return false;
2535  } else if (!translateCallBase(I, MIRBuilder))
2536  return false;
2537 
2538  MCSymbol *EndSymbol = Context.createTempSymbol();
2539  MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(EndSymbol);
2540 
2542  BranchProbabilityInfo *BPI = FuncInfo.BPI;
2543  MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB();
2544  BranchProbability EHPadBBProb =
2545  BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2547 
2548  if (!findUnwindDestinations(EHPadBB, EHPadBBProb, UnwindDests))
2549  return false;
2550 
2551  MachineBasicBlock &EHPadMBB = getMBB(*EHPadBB),
2552  &ReturnMBB = getMBB(*ReturnBB);
2553  // Update successor info.
2554  addSuccessorWithProb(InvokeMBB, &ReturnMBB);
2555  for (auto &UnwindDest : UnwindDests) {
2556  UnwindDest.first->setIsEHPad();
2557  addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2558  }
2559  InvokeMBB->normalizeSuccProbs();
2560 
2561  MF->addInvoke(&EHPadMBB, BeginSymbol, EndSymbol);
2562  MIRBuilder.buildBr(ReturnMBB);
2563  return true;
2564 }
2565 
2566 bool IRTranslator::translateCallBr(const User &U,
2567  MachineIRBuilder &MIRBuilder) {
2568  // FIXME: Implement this.
2569  return false;
2570 }
2571 
2572 bool IRTranslator::translateLandingPad(const User &U,
2573  MachineIRBuilder &MIRBuilder) {
2574  const LandingPadInst &LP = cast<LandingPadInst>(U);
2575 
2576  MachineBasicBlock &MBB = MIRBuilder.getMBB();
2577 
2578  MBB.setIsEHPad();
2579 
2580  // If there aren't registers to copy the values into (e.g., during SjLj
2581  // exceptions), then don't bother.
2582  auto &TLI = *MF->getSubtarget().getTargetLowering();
2583  const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
2584  if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2585  TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2586  return true;
2587 
2588  // If landingpad's return type is token type, we don't create DAG nodes
2589  // for its exception pointer and selector value. The extraction of exception
2590  // pointer or selector value from token type landingpads is not currently
2591  // supported.
2592  if (LP.getType()->isTokenTy())
2593  return true;
2594 
2595  // Add a label to mark the beginning of the landing pad. Deletion of the
2596  // landing pad can thus be detected via the MachineModuleInfo.
2598  .addSym(MF->addLandingPad(&MBB));
2599 
2600  // If the unwinder does not preserve all registers, ensure that the
2601  // function marks the clobbered registers as used.
2603  if (auto *RegMask = TRI.getCustomEHPadPreservedMask(*MF))
2604  MF->getRegInfo().addPhysRegsUsedFromRegMask(RegMask);
2605 
2606  LLT Ty = getLLTForType(*LP.getType(), *DL);
2608  MIRBuilder.buildUndef(Undef);
2609 
2610  SmallVector<LLT, 2> Tys;
2611  for (Type *Ty : cast<StructType>(LP.getType())->elements())
2612  Tys.push_back(getLLTForType(*Ty, *DL));
2613  assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
2614 
2615  // Mark exception register as live in.
2616  Register ExceptionReg = TLI.getExceptionPointerRegister(PersonalityFn);
2617  if (!ExceptionReg)
2618  return false;
2619 
2620  MBB.addLiveIn(ExceptionReg);
2621  ArrayRef<Register> ResRegs = getOrCreateVRegs(LP);
2622  MIRBuilder.buildCopy(ResRegs[0], ExceptionReg);
2623 
2624  Register SelectorReg = TLI.getExceptionSelectorRegister(PersonalityFn);
2625  if (!SelectorReg)
2626  return false;
2627 
2628  MBB.addLiveIn(SelectorReg);
2629  Register PtrVReg = MRI->createGenericVirtualRegister(Tys[0]);
2630  MIRBuilder.buildCopy(PtrVReg, SelectorReg);
2631  MIRBuilder.buildCast(ResRegs[1], PtrVReg);
2632 
2633  return true;
2634 }
2635 
2636 bool IRTranslator::translateAlloca(const User &U,
2637  MachineIRBuilder &MIRBuilder) {
2638  auto &AI = cast<AllocaInst>(U);
2639 
2640  if (AI.isSwiftError())
2641  return true;
2642 
2643  if (AI.isStaticAlloca()) {
2644  Register Res = getOrCreateVReg(AI);
2645  int FI = getOrCreateFrameIndex(AI);
2646  MIRBuilder.buildFrameIndex(Res, FI);
2647  return true;
2648  }
2649 
2650  // FIXME: support stack probing for Windows.
2651  if (MF->getTarget().getTargetTriple().isOSWindows())
2652  return false;
2653 
2654  // Now we're in the harder dynamic case.
2655  Register NumElts = getOrCreateVReg(*AI.getArraySize());
2656  Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
2657  LLT IntPtrTy = getLLTForType(*IntPtrIRTy, *DL);
2658  if (MRI->getType(NumElts) != IntPtrTy) {
2659  Register ExtElts = MRI->createGenericVirtualRegister(IntPtrTy);
2660  MIRBuilder.buildZExtOrTrunc(ExtElts, NumElts);
2661  NumElts = ExtElts;
2662  }
2663 
2664  Type *Ty = AI.getAllocatedType();
2665 
2666  Register AllocSize = MRI->createGenericVirtualRegister(IntPtrTy);
2667  Register TySize =
2668  getOrCreateVReg(*ConstantInt::get(IntPtrIRTy, DL->getTypeAllocSize(Ty)));
2669  MIRBuilder.buildMul(AllocSize, NumElts, TySize);
2670 
2671  // Round the size of the allocation up to the stack alignment size
2672  // by add SA-1 to the size. This doesn't overflow because we're computing
2673  // an address inside an alloca.
2675  auto SAMinusOne = MIRBuilder.buildConstant(IntPtrTy, StackAlign.value() - 1);
2676  auto AllocAdd = MIRBuilder.buildAdd(IntPtrTy, AllocSize, SAMinusOne,
2678  auto AlignCst =
2679  MIRBuilder.buildConstant(IntPtrTy, ~(uint64_t)(StackAlign.value() - 1));
2680  auto AlignedAlloc = MIRBuilder.buildAnd(IntPtrTy, AllocAdd, AlignCst);
2681 
2682  Align Alignment = std::max(AI.getAlign(), DL->getPrefTypeAlign(Ty));
2683  if (Alignment <= StackAlign)
2684  Alignment = Align(1);
2685  MIRBuilder.buildDynStackAlloc(getOrCreateVReg(AI), AlignedAlloc, Alignment);
2686 
2687  MF->getFrameInfo().CreateVariableSizedObject(Alignment, &AI);
2689  return true;
2690 }
2691 
2692 bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {
2693  // FIXME: We may need more info about the type. Because of how LLT works,
2694  // we're completely discarding the i64/double distinction here (amongst
2695  // others). Fortunately the ABIs I know of where that matters don't use va_arg
2696  // anyway but that's not guaranteed.
2697  MIRBuilder.buildInstr(TargetOpcode::G_VAARG, {getOrCreateVReg(U)},
2698  {getOrCreateVReg(*U.getOperand(0)),
2699  DL->getABITypeAlign(U.getType()).value()});
2700  return true;
2701 }
2702 
2703 bool IRTranslator::translateUnreachable(const User &U, MachineIRBuilder &MIRBuilder) {
2704  if (!MF->getTarget().Options.TrapUnreachable)
2705  return true;
2706 
2707  auto &UI = cast<UnreachableInst>(U);
2708  // We may be able to ignore unreachable behind a noreturn call.
2710  const BasicBlock &BB = *UI.getParent();
2711  if (&UI != &BB.front()) {
2713  std::prev(BasicBlock::const_iterator(UI));
2714  if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
2715  if (Call->doesNotReturn())
2716  return true;
2717  }
2718  }
2719  }
2720 
2721  MIRBuilder.buildIntrinsic(Intrinsic::trap, ArrayRef<Register>(), true);
2722  return true;
2723 }
2724 
2725 bool IRTranslator::translateInsertElement(const User &U,
2726  MachineIRBuilder &MIRBuilder) {
2727  // If it is a <1 x Ty> vector, use the scalar as it is
2728  // not a legal vector type in LLT.
2729  if (cast<FixedVectorType>(U.getType())->getNumElements() == 1)
2730  return translateCopy(U, *U.getOperand(1), MIRBuilder);
2731 
2732  Register Res = getOrCreateVReg(U);
2733  Register Val = getOrCreateVReg(*U.getOperand(0));
2734  Register Elt = getOrCreateVReg(*U.getOperand(1));
2735  Register Idx = getOrCreateVReg(*U.getOperand(2));
2736  MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
2737  return true;
2738 }
2739 
2740 bool IRTranslator::translateExtractElement(const User &U,
2741  MachineIRBuilder &MIRBuilder) {
2742  // If it is a <1 x Ty> vector, use the scalar as it is
2743  // not a legal vector type in LLT.
2744  if (cast<FixedVectorType>(U.getOperand(0)->getType())->getNumElements() == 1)
2745  return translateCopy(U, *U.getOperand(0), MIRBuilder);
2746 
2747  Register Res = getOrCreateVReg(U);
2748  Register Val = getOrCreateVReg(*U.getOperand(0));
2749  const auto &TLI = *MF->getSubtarget().getTargetLowering();
2750  unsigned PreferredVecIdxWidth = TLI.getVectorIdxTy(*DL).getSizeInBits();
2751  Register Idx;
2752  if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(1))) {
2753  if (CI->getBitWidth() != PreferredVecIdxWidth) {
2754  APInt NewIdx = CI->getValue().sextOrTrunc(PreferredVecIdxWidth);
2755  auto *NewIdxCI = ConstantInt::get(CI->getContext(), NewIdx);
2756  Idx = getOrCreateVReg(*NewIdxCI);
2757  }
2758  }
2759  if (!Idx)
2760  Idx = getOrCreateVReg(*U.getOperand(1));
2761  if (MRI->getType(Idx).getSizeInBits() != PreferredVecIdxWidth) {
2762  const LLT VecIdxTy = LLT::scalar(PreferredVecIdxWidth);
2763  Idx = MIRBuilder.buildSExtOrTrunc(VecIdxTy, Idx).getReg(0);
2764  }
2765  MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
2766  return true;
2767 }
2768 
2769 bool IRTranslator::translateShuffleVector(const User &U,
2770  MachineIRBuilder &MIRBuilder) {
2772  if (auto *SVI = dyn_cast<ShuffleVectorInst>(&U))
2773  Mask = SVI->getShuffleMask();
2774  else
2775  Mask = cast<ConstantExpr>(U).getShuffleMask();
2776  ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
2777  MIRBuilder
2778  .buildInstr(TargetOpcode::G_SHUFFLE_VECTOR, {getOrCreateVReg(U)},
2779  {getOrCreateVReg(*U.getOperand(0)),
2780  getOrCreateVReg(*U.getOperand(1))})
2781  .addShuffleMask(MaskAlloc);
2782  return true;
2783 }
2784 
2785 bool IRTranslator::translatePHI(const User &U, MachineIRBuilder &MIRBuilder) {
2786  const PHINode &PI = cast<PHINode>(U);
2787 
2789  for (auto Reg : getOrCreateVRegs(PI)) {
2790  auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_PHI, {Reg}, {});
2791  Insts.push_back(MIB.getInstr());
2792  }
2793 
2794  PendingPHIs.emplace_back(&PI, std::move(Insts));
2795  return true;
2796 }
2797 
2798 bool IRTranslator::translateAtomicCmpXchg(const User &U,
2799  MachineIRBuilder &MIRBuilder) {
2800  const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(U);
2801 
2802  auto &TLI = *MF->getSubtarget().getTargetLowering();
2803  auto Flags = TLI.getAtomicMemOperandFlags(I, *DL);
2804 
2805  auto Res = getOrCreateVRegs(I);
2806  Register OldValRes = Res[0];
2807  Register SuccessRes = Res[1];
2808  Register Addr = getOrCreateVReg(*I.getPointerOperand());
2809  Register Cmp = getOrCreateVReg(*I.getCompareOperand());
2810  Register NewVal = getOrCreateVReg(*I.getNewValOperand());
2811 
2812  MIRBuilder.buildAtomicCmpXchgWithSuccess(
2813  OldValRes, SuccessRes, Addr, Cmp, NewVal,
2814  *MF->getMachineMemOperand(
2815  MachinePointerInfo(I.getPointerOperand()), Flags, MRI->getType(Cmp),
2816  getMemOpAlign(I), I.getAAMetadata(), nullptr, I.getSyncScopeID(),
2817  I.getSuccessOrdering(), I.getFailureOrdering()));
2818  return true;
2819 }
2820 
2821 bool IRTranslator::translateAtomicRMW(const User &U,
2822  MachineIRBuilder &MIRBuilder) {
2823  const AtomicRMWInst &I = cast<AtomicRMWInst>(U);
2824  auto &TLI = *MF->getSubtarget().getTargetLowering();
2825  auto Flags = TLI.getAtomicMemOperandFlags(I, *DL);
2826 
2827  Register Res = getOrCreateVReg(I);
2828  Register Addr = getOrCreateVReg(*I.getPointerOperand());
2829  Register Val = getOrCreateVReg(*I.getValOperand());
2830 
2831  unsigned Opcode = 0;
2832  switch (I.getOperation()) {
2833  default:
2834  return false;
2835  case AtomicRMWInst::Xchg:
2836  Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
2837  break;
2838  case AtomicRMWInst::Add:
2839  Opcode = TargetOpcode::G_ATOMICRMW_ADD;
2840  break;
2841  case AtomicRMWInst::Sub:
2842  Opcode = TargetOpcode::G_ATOMICRMW_SUB;
2843  break;
2844  case AtomicRMWInst::And:
2845  Opcode = TargetOpcode::G_ATOMICRMW_AND;
2846  break;
2847  case AtomicRMWInst::Nand:
2848  Opcode = TargetOpcode::G_ATOMICRMW_NAND;
2849  break;
2850  case AtomicRMWInst::Or:
2851  Opcode = TargetOpcode::G_ATOMICRMW_OR;
2852  break;
2853  case AtomicRMWInst::Xor:
2854  Opcode = TargetOpcode::G_ATOMICRMW_XOR;
2855  break;
2856  case AtomicRMWInst::Max:
2857  Opcode = TargetOpcode::G_ATOMICRMW_MAX;
2858  break;
2859  case AtomicRMWInst::Min:
2860  Opcode = TargetOpcode::G_ATOMICRMW_MIN;
2861  break;
2862  case AtomicRMWInst::UMax:
2863  Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
2864  break;
2865  case AtomicRMWInst::UMin:
2866  Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
2867  break;
2868  case AtomicRMWInst::FAdd:
2869  Opcode = TargetOpcode::G_ATOMICRMW_FADD;
2870  break;
2871  case AtomicRMWInst::FSub:
2872  Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
2873  break;
2874  }
2875 
2876  MIRBuilder.buildAtomicRMW(
2877  Opcode, Res, Addr, Val,
2878  *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
2879  Flags, MRI->getType(Val), getMemOpAlign(I),
2880  I.getAAMetadata(), nullptr, I.getSyncScopeID(),
2881  I.getOrdering()));
2882  return true;
2883 }
2884 
2885 bool IRTranslator::translateFence(const User &U,
2886  MachineIRBuilder &MIRBuilder) {
2887  const FenceInst &Fence = cast<FenceInst>(U);
2888  MIRBuilder.buildFence(static_cast<unsigned>(Fence.getOrdering()),
2889  Fence.getSyncScopeID());
2890  return true;
2891 }
2892 
2893 bool IRTranslator::translateFreeze(const User &U,
2894  MachineIRBuilder &MIRBuilder) {
2895  const ArrayRef<Register> DstRegs = getOrCreateVRegs(U);
2896  const ArrayRef<Register> SrcRegs = getOrCreateVRegs(*U.getOperand(0));
2897 
2898  assert(DstRegs.size() == SrcRegs.size() &&
2899  "Freeze with different source and destination type?");
2900 
2901  for (unsigned I = 0; I < DstRegs.size(); ++I) {
2902  MIRBuilder.buildFreeze(DstRegs[I], SrcRegs[I]);
2903  }
2904 
2905  return true;
2906 }
2907 
2908 void IRTranslator::finishPendingPhis() {
2909 #ifndef NDEBUG
2910  DILocationVerifier Verifier;
2911  GISelObserverWrapper WrapperObserver(&Verifier);
2912  RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
2913 #endif // ifndef NDEBUG
2914  for (auto &Phi : PendingPHIs) {
2915  const PHINode *PI = Phi.first;
2916  ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
2917  MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
2918  EntryBuilder->setDebugLoc(PI->getDebugLoc());
2919 #ifndef NDEBUG
2920  Verifier.setCurrentInst(PI);
2921 #endif // ifndef NDEBUG
2922 
2924  for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
2925  auto IRPred = PI->getIncomingBlock(i);
2926  ArrayRef<Register> ValRegs = getOrCreateVRegs(*PI->getIncomingValue(i));
2927  for (auto Pred : getMachinePredBBs({IRPred, PI->getParent()})) {
2928  if (SeenPreds.count(Pred) || !PhiMBB->isPredecessor(Pred))
2929  continue;
2930  SeenPreds.insert(Pred);
2931  for (unsigned j = 0; j < ValRegs.size(); ++j) {
2932  MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
2933  MIB.addUse(ValRegs[j]);
2934  MIB.addMBB(Pred);
2935  }
2936  }
2937  }
2938  }
2939 }
2940 
2941 bool IRTranslator::valueIsSplit(const Value &V,
2943  SmallVector<LLT, 4> SplitTys;
2944  if (Offsets && !Offsets->empty())
2945  Offsets->clear();
2946  computeValueLLTs(*DL, *V.getType(), SplitTys, Offsets);
2947  return SplitTys.size() > 1;
2948 }
2949 
2950 bool IRTranslator::translate(const Instruction &Inst) {
2951  CurBuilder->setDebugLoc(Inst.getDebugLoc());
2952 
2953  auto &TLI = *MF->getSubtarget().getTargetLowering();
2954  if (TLI.fallBackToDAGISel(Inst))
2955  return false;
2956 
2957  switch (Inst.getOpcode()) {
2958 #define HANDLE_INST(NUM, OPCODE, CLASS) \
2959  case Instruction::OPCODE: \
2960  return translate##OPCODE(Inst, *CurBuilder.get());
2961 #include "llvm/IR/Instruction.def"
2962  default:
2963  return false;
2964  }
2965 }
2966 
2967 bool IRTranslator::translate(const Constant &C, Register Reg) {
2968  // We only emit constants into the entry block from here. To prevent jumpy
2969  // debug behaviour set the line to 0.
2970  if (auto CurrInstDL = CurBuilder->getDL())
2971  EntryBuilder->setDebugLoc(DILocation::get(C.getContext(), 0, 0,
2972  CurrInstDL.getScope(),
2973  CurrInstDL.getInlinedAt()));
2974 
2975  if (auto CI = dyn_cast<ConstantInt>(&C))
2976  EntryBuilder->buildConstant(Reg, *CI);
2977  else if (auto CF = dyn_cast<ConstantFP>(&C))
2978  EntryBuilder->buildFConstant(Reg, *CF);
2979  else if (isa<UndefValue>(C))
2980  EntryBuilder->buildUndef(Reg);
2981  else if (isa<ConstantPointerNull>(C))
2982  EntryBuilder->buildConstant(Reg, 0);
2983  else if (auto GV = dyn_cast<GlobalValue>(&C))
2984  EntryBuilder->buildGlobalValue(Reg, GV);
2985  else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
2986  if (!isa<FixedVectorType>(CAZ->getType()))
2987  return false;
2988  // Return the scalar if it is a <1 x Ty> vector.
2989  unsigned NumElts = CAZ->getElementCount().getFixedValue();
2990  if (NumElts == 1)
2991  return translateCopy(C, *CAZ->getElementValue(0u), *EntryBuilder.get());
2993  for (unsigned I = 0; I < NumElts; ++I) {
2994  Constant &Elt = *CAZ->getElementValue(I);
2995  Ops.push_back(getOrCreateVReg(Elt));
2996  }
2997  EntryBuilder->buildBuildVector(Reg, Ops);
2998  } else if (auto CV = dyn_cast<ConstantDataVector>(&C)) {
2999  // Return the scalar if it is a <1 x Ty> vector.
3000  if (CV->getNumElements() == 1)
3001  return translateCopy(C, *CV->getElementAsConstant(0),
3002  *EntryBuilder.get());
3004  for (unsigned i = 0; i < CV->getNumElements(); ++i) {
3005  Constant &Elt = *CV->getElementAsConstant(i);
3006  Ops.push_back(getOrCreateVReg(Elt));
3007  }
3008  EntryBuilder->buildBuildVector(Reg, Ops);
3009  } else if (auto CE = dyn_cast<ConstantExpr>(&C)) {
3010  switch(CE->getOpcode()) {
3011 #define HANDLE_INST(NUM, OPCODE, CLASS) \
3012  case Instruction::OPCODE: \
3013  return translate##OPCODE(*CE, *EntryBuilder.get());
3014 #include "llvm/IR/Instruction.def"
3015  default:
3016  return false;
3017  }
3018  } else if (auto CV = dyn_cast<ConstantVector>(&C)) {
3019  if (CV->getNumOperands() == 1)
3020  return translateCopy(C, *CV->getOperand(0), *EntryBuilder.get());
3022  for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
3023  Ops.push_back(getOrCreateVReg(*CV->getOperand(i)));
3024  }
3025  EntryBuilder->buildBuildVector(Reg, Ops);
3026  } else if (auto *BA = dyn_cast<BlockAddress>(&C)) {
3027  EntryBuilder->buildBlockAddress(Reg, BA);
3028  } else
3029  return false;
3030 
3031  return true;
3032 }
3033 
3034 bool IRTranslator::finalizeBasicBlock(const BasicBlock &BB,
3036  for (auto &BTB : SL->BitTestCases) {
3037  // Emit header first, if it wasn't already emitted.
3038  if (!BTB.Emitted)
3039  emitBitTestHeader(BTB, BTB.Parent);
3040 
3041  BranchProbability UnhandledProb = BTB.Prob;
3042  for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
3043  UnhandledProb -= BTB.Cases[j].ExtraProb;
3044  // Set the current basic block to the mbb we wish to insert the code into
3045  MachineBasicBlock *MBB = BTB.Cases[j].ThisBB;
3046  // If all cases cover a contiguous range, it is not necessary to jump to
3047  // the default block after the last bit test fails. This is because the
3048  // range check during bit test header creation has guaranteed that every
3049  // case here doesn't go outside the range. In this case, there is no need
3050  // to perform the last bit test, as it will always be true. Instead, make
3051  // the second-to-last bit-test fall through to the target of the last bit
3052  // test, and delete the last bit test.
3053 
3054  MachineBasicBlock *NextMBB;
3055  if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3056  // Second-to-last bit-test with contiguous range: fall through to the
3057  // target of the final bit test.
3058  NextMBB = BTB.Cases[j + 1].TargetBB;
3059  } else if (j + 1 == ej) {
3060  // For the last bit test, fall through to Default.
3061  NextMBB = BTB.Default;
3062  } else {
3063  // Otherwise, fall through to the next bit test.
3064  NextMBB = BTB.Cases[j + 1].ThisBB;
3065  }
3066 
3067  emitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j], MBB);
3068 
3069  if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3070  // We need to record the replacement phi edge here that normally
3071  // happens in emitBitTestCase before we delete the case, otherwise the
3072  // phi edge will be lost.
3073  addMachineCFGPred({BTB.Parent->getBasicBlock(),
3074  BTB.Cases[ej - 1].TargetBB->getBasicBlock()},
3075  MBB);
3076  // Since we're not going to use the final bit test, remove it.
3077  BTB.Cases.pop_back();
3078  break;
3079  }
3080  }
3081  // This is "default" BB. We have two jumps to it. From "header" BB and from
3082  // last "case" BB, unless the latter was skipped.
3083  CFGEdge HeaderToDefaultEdge = {BTB.Parent->getBasicBlock(),
3084  BTB.Default->getBasicBlock()};
3085  addMachineCFGPred(HeaderToDefaultEdge, BTB.Parent);
3086  if (!BTB.ContiguousRange) {
3087  addMachineCFGPred(HeaderToDefaultEdge, BTB.Cases.back().ThisBB);
3088  }
3089  }
3090  SL->BitTestCases.clear();
3091 
3092  for (auto &JTCase : SL->JTCases) {
3093  // Emit header first, if it wasn't already emitted.
3094  if (!JTCase.first.Emitted)
3095  emitJumpTableHeader(JTCase.second, JTCase.first, JTCase.first.HeaderBB);
3096 
3097  emitJumpTable(JTCase.second, JTCase.second.MBB);
3098  }
3099  SL->JTCases.clear();
3100 
3101  for (auto &SwCase : SL->SwitchCases)
3102  emitSwitchCase(SwCase, &CurBuilder->getMBB(), *CurBuilder);
3103  SL->SwitchCases.clear();
3104 
3105  // Check if we need to generate stack-protector guard checks.
3106  StackProtector &SP = getAnalysis<StackProtector>();
3107  if (SP.shouldEmitSDCheck(BB)) {
3108  const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
3109  bool FunctionBasedInstrumentation =
3111  SPDescriptor.initialize(&BB, &MBB, FunctionBasedInstrumentation);
3112  }
3113  // Handle stack protector.
3114  if (SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
3115  LLVM_DEBUG(dbgs() << "Unimplemented stack protector case\n");
3116  return false;
3117  } else if (SPDescriptor.shouldEmitStackProtector()) {
3118  MachineBasicBlock *ParentMBB = SPDescriptor.getParentMBB();
3119  MachineBasicBlock *SuccessMBB = SPDescriptor.getSuccessMBB();
3120 
3121  // Find the split point to split the parent mbb. At the same time copy all
3122  // physical registers used in the tail of parent mbb into virtual registers
3123  // before the split point and back into physical registers after the split
3124  // point. This prevents us needing to deal with Live-ins and many other
3125  // register allocation issues caused by us splitting the parent mbb. The
3126  // register allocator will clean up said virtual copies later on.
3128  ParentMBB, *MF->getSubtarget().getInstrInfo());
3129 
3130  // Splice the terminator of ParentMBB into SuccessMBB.
3131  SuccessMBB->splice(SuccessMBB->end(), ParentMBB, SplitPoint,
3132  ParentMBB->end());
3133 
3134  // Add compare/jump on neq/jump to the parent BB.
3135  if (!emitSPDescriptorParent(SPDescriptor, ParentMBB))
3136  return false;
3137 
3138  // CodeGen Failure MBB if we have not codegened it yet.
3139  MachineBasicBlock *FailureMBB = SPDescriptor.getFailureMBB();
3140  if (FailureMBB->empty()) {
3141  if (!emitSPDescriptorFailure(SPDescriptor, FailureMBB))
3142  return false;
3143  }
3144 
3145  // Clear the Per-BB State.
3146  SPDescriptor.resetPerBBState();
3147  }
3148  return true;
3149 }
3150 
3151 bool IRTranslator::emitSPDescriptorParent(StackProtectorDescriptor &SPD,
3152  MachineBasicBlock *ParentBB) {
3153  CurBuilder->setInsertPt(*ParentBB, ParentBB->end());
3154  // First create the loads to the guard/stack slot for the comparison.
3155  const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
3156  Type *PtrIRTy = Type::getInt8PtrTy(MF->getFunction().getContext());
3157  const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
3158  LLT PtrMemTy = getLLTForMVT(TLI.getPointerMemTy(*DL));
3159 
3160  MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3161  int FI = MFI.getStackProtectorIndex();
3162 
3163  Register Guard;
3164  Register StackSlotPtr = CurBuilder->buildFrameIndex(PtrTy, FI).getReg(0);
3165  const Module &M = *ParentBB->getParent()->getFunction().getParent();
3166  Align Align = DL->getPrefTypeAlign(Type::getInt8PtrTy(M.getContext()));
3167 
3168  // Generate code to load the content of the guard slot.
3169  Register GuardVal =
3170  CurBuilder
3171  ->buildLoad(PtrMemTy, StackSlotPtr,
3174  .getReg(0);
3175 
3176  if (TLI.useStackGuardXorFP()) {
3177  LLVM_DEBUG(dbgs() << "Stack protector xor'ing with FP not yet implemented");
3178  return false;
3179  }
3180 
3181  // Retrieve guard check function, nullptr if instrumentation is inlined.
3182  if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
3183  // This path is currently untestable on GlobalISel, since the only platform
3184  // that needs this seems to be Windows, and we fall back on that currently.
3185  // The code still lives here in case that changes.
3186  // Silence warning about unused variable until the code below that uses
3187  // 'GuardCheckFn' is enabled.
3188  (void)GuardCheckFn;
3189  return false;
3190 #if 0
3191  // The target provides a guard check function to validate the guard value.
3192  // Generate a call to that function with the content of the guard slot as
3193  // argument.
3194  FunctionType *FnTy = GuardCheckFn->getFunctionType();
3195  assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3196  ISD::ArgFlagsTy Flags;
3197  if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
3198  Flags.setInReg();
3199  CallLowering::ArgInfo GuardArgInfo(
3200  {GuardVal, FnTy->getParamType(0), {Flags}});
3201 
3203  Info.OrigArgs.push_back(GuardArgInfo);
3204  Info.CallConv = GuardCheckFn->getCallingConv();
3205  Info.Callee = MachineOperand::CreateGA(GuardCheckFn, 0);
3206  Info.OrigRet = {Register(), FnTy->getReturnType()};
3207  if (!CLI->lowerCall(MIRBuilder, Info)) {
3208  LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector check\n");
3209  return false;
3210  }
3211  return true;
3212 #endif
3213  }
3214 
3215  // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
3216  // Otherwise, emit a volatile load to retrieve the stack guard value.
3217  if (TLI.useLoadStackGuardNode()) {
3218  Guard =
3220  getStackGuard(Guard, *CurBuilder);
3221  } else {
3222  // TODO: test using android subtarget when we support @llvm.thread.pointer.
3223  const Value *IRGuard = TLI.getSDagStackGuard(M);
3224  Register GuardPtr = getOrCreateVReg(*IRGuard);
3225 
3226  Guard = CurBuilder
3227  ->buildLoad(PtrMemTy, GuardPtr,
3231  .getReg(0);
3232  }
3233 
3234  // Perform the comparison.
3235  auto Cmp =
3236  CurBuilder->buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Guard, GuardVal);
3237  // If the guard/stackslot do not equal, branch to failure MBB.
3238  CurBuilder->buildBrCond(Cmp, *SPD.getFailureMBB());
3239  // Otherwise branch to success MBB.
3240  CurBuilder->buildBr(*SPD.getSuccessMBB());
3241  return true;
3242 }
3243 
3244 bool IRTranslator::emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
3245  MachineBasicBlock *FailureBB) {
3246  CurBuilder->setInsertPt(*FailureBB, FailureBB->end());
3247  const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
3248 
3249  const RTLIB::Libcall Libcall = RTLIB::STACKPROTECTOR_CHECK_FAIL;
3250  const char *Name = TLI.getLibcallName(Libcall);
3251 
3253  Info.CallConv = TLI.getLibcallCallingConv(Libcall);
3255  Info.OrigRet = {Register(), Type::getVoidTy(MF->getFunction().getContext()),
3256  0};
3257  if (!CLI->lowerCall(*CurBuilder, Info)) {
3258  LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector fail\n");
3259  return false;
3260  }
3261 
3262  // On PS4, the "return address" must still be within the calling function,
3263  // even if it's at the very end, so emit an explicit TRAP here.
3264  // Passing 'true' for doesNotReturn above won't generate the trap for us.
3265  // WebAssembly needs an unreachable instruction after a non-returning call,
3266  // because the function return type can be different from __stack_chk_fail's
3267  // return type (void).
3268  const TargetMachine &TM = MF->getTarget();
3269  if (TM.getTargetTriple().isPS4CPU() || TM.getTargetTriple().isWasm()) {
3270  LLVM_DEBUG(dbgs() << "Unhandled trap emission for stack protector fail\n");
3271  return false;
3272  }
3273  return true;
3274 }
3275 
3276 void IRTranslator::finalizeFunction() {
3277  // Release the memory used by the different maps we
3278  // needed during the translation.
3279  PendingPHIs.clear();
3280  VMap.reset();
3281  FrameIndices.clear();
3282  MachinePreds.clear();
3283  // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
3284  // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
3285  // destroying it twice (in ~IRTranslator() and ~LLVMContext())
3286  EntryBuilder.reset();
3287  CurBuilder.reset();
3288  FuncInfo.clear();
3289  SPDescriptor.resetPerFunctionState();
3290 }
3291 
3292 /// Returns true if a BasicBlock \p BB within a variadic function contains a
3293 /// variadic musttail call.
3294 static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB) {
3295  if (!IsVarArg)
3296  return false;
3297 
3298  // Walk the block backwards, because tail calls usually only appear at the end
3299  // of a block.
3300  return llvm::any_of(llvm::reverse(BB), [](const Instruction &I) {
3301  const auto *CI = dyn_cast<CallInst>(&I);
3302  return CI && CI->isMustTailCall();
3303  });
3304 }
3305 
3307  MF = &CurMF;
3308  const Function &F = MF->getFunction();
3310  getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
3311  // Set the CSEConfig and run the analysis.
3312  GISelCSEInfo *CSEInfo = nullptr;
3313  TPC = &getAnalysis<TargetPassConfig>();
3314  bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
3316  : TPC->isGISelCSEEnabled();
3317 
3318  if (EnableCSE) {
3319  EntryBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
3320  CSEInfo = &Wrapper.get(TPC->getCSEConfig());
3321  EntryBuilder->setCSEInfo(CSEInfo);
3322  CurBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
3323  CurBuilder->setCSEInfo(CSEInfo);
3324  } else {
3325  EntryBuilder = std::make_unique<MachineIRBuilder>();
3326  CurBuilder = std::make_unique<MachineIRBuilder>();
3327  }
3328  CLI = MF->getSubtarget().getCallLowering();
3329  CurBuilder->setMF(*MF);
3330  EntryBuilder->setMF(*MF);
3331  MRI = &MF->getRegInfo();
3332  DL = &F.getParent()->getDataLayout();
3333  ORE = std::make_unique<OptimizationRemarkEmitter>(&F);
3334  const TargetMachine &TM = MF->getTarget();
3335  TM.resetTargetOptions(F);
3336  EnableOpts = OptLevel != CodeGenOpt::None && !skipFunction(F);
3337  FuncInfo.MF = MF;
3338  if (EnableOpts)
3339  FuncInfo.BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
3340  else
3341  FuncInfo.BPI = nullptr;
3342 
3343  FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(*MF);
3344 
3345  const auto &TLI = *MF->getSubtarget().getTargetLowering();
3346 
3347  SL = std::make_unique<GISelSwitchLowering>(this, FuncInfo);
3348  SL->init(TLI, TM, *DL);
3349 
3350 
3351 
3352  assert(PendingPHIs.empty() && "stale PHIs");
3353 
3354  // Targets which want to use big endian can enable it using
3355  // enableBigEndian()
3356  if (!DL->isLittleEndian() && !CLI->enableBigEndian()) {
3357  // Currently we don't properly handle big endian code.
3358  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3359  F.getSubprogram(), &F.getEntryBlock());
3360  R << "unable to translate in big endian mode";
3361  reportTranslationError(*MF, *TPC, *ORE, R);
3362  }
3363 
3364  // Release the per-function state when we return, whether we succeeded or not.
3365  auto FinalizeOnReturn = make_scope_exit([this]() { finalizeFunction(); });
3366 
3367  // Setup a separate basic-block for the arguments and constants
3368  MachineBasicBlock *EntryBB = MF->CreateMachineBasicBlock();
3369  MF->push_back(EntryBB);
3370  EntryBuilder->setMBB(*EntryBB);
3371 
3372  DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHI()->getDebugLoc();
3373  SwiftError.setFunction(CurMF);
3374  SwiftError.createEntriesInEntryBlock(DbgLoc);
3375 
3376  bool IsVarArg = F.isVarArg();
3377  bool HasMustTailInVarArgFn = false;
3378 
3379  // Create all blocks, in IR order, to preserve the layout.
3380  for (const BasicBlock &BB: F) {
3381  auto *&MBB = BBToMBB[&BB];
3382 
3383  MBB = MF->CreateMachineBasicBlock(&BB);
3384  MF->push_back(MBB);
3385 
3386  if (BB.hasAddressTaken())
3388 
3389  if (!HasMustTailInVarArgFn)
3390  HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB);
3391  }
3392 
3393  MF->getFrameInfo().setHasMustTailInVarArgFunc(HasMustTailInVarArgFn);
3394 
3395  // Make our arguments/constants entry block fallthrough to the IR entry block.
3396  EntryBB->addSuccessor(&getMBB(F.front()));
3397 
3398  if (CLI->fallBackToDAGISel(*MF)) {
3399  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3400  F.getSubprogram(), &F.getEntryBlock());
3401  R << "unable to lower function: " << ore::NV("Prototype", F.getType());
3402  reportTranslationError(*MF, *TPC, *ORE, R);
3403  return false;
3404  }
3405 
3406  // Lower the actual args into this basic block.
3407  SmallVector<ArrayRef<Register>, 8> VRegArgs;
3408  for (const Argument &Arg: F.args()) {
3409  if (DL->getTypeStoreSize(Arg.getType()).isZero())
3410  continue; // Don't handle zero sized types.
3411  ArrayRef<Register> VRegs = getOrCreateVRegs(Arg);
3412  VRegArgs.push_back(VRegs);
3413 
3414  if (Arg.hasSwiftErrorAttr()) {
3415  assert(VRegs.size() == 1 && "Too many vregs for Swift error");
3416  SwiftError.setCurrentVReg(EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
3417  }
3418  }
3419 
3420  if (!CLI->lowerFormalArguments(*EntryBuilder.get(), F, VRegArgs, FuncInfo)) {
3421  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3422  F.getSubprogram(), &F.getEntryBlock());
3423  R << "unable to lower arguments: " << ore::NV("Prototype", F.getType());
3424  reportTranslationError(*MF, *TPC, *ORE, R);
3425  return false;
3426  }
3427 
3428  // Need to visit defs before uses when translating instructions.
3429  GISelObserverWrapper WrapperObserver;
3430  if (EnableCSE && CSEInfo)
3431  WrapperObserver.addObserver(CSEInfo);
3432  {
3434 #ifndef NDEBUG
3435  DILocationVerifier Verifier;
3436  WrapperObserver.addObserver(&Verifier);
3437 #endif // ifndef NDEBUG
3438  RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
3439  RAIIMFObserverInstaller ObsInstall(*MF, WrapperObserver);
3440  for (const BasicBlock *BB : RPOT) {
3441  MachineBasicBlock &MBB = getMBB(*BB);
3442  // Set the insertion point of all the following translations to
3443  // the end of this basic block.
3444  CurBuilder->setMBB(MBB);
3445  HasTailCall = false;
3446  for (const Instruction &Inst : *BB) {
3447  // If we translated a tail call in the last step, then we know
3448  // everything after the call is either a return, or something that is
3449  // handled by the call itself. (E.g. a lifetime marker or assume
3450  // intrinsic.) In this case, we should stop translating the block and
3451  // move on.
3452  if (HasTailCall)
3453  break;
3454 #ifndef NDEBUG
3455  Verifier.setCurrentInst(&Inst);
3456 #endif // ifndef NDEBUG
3457  if (translate(Inst))
3458  continue;
3459 
3460  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3461  Inst.getDebugLoc(), BB);
3462  R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
3463 
3464  if (ORE->allowExtraAnalysis("gisel-irtranslator")) {
3465  std::string InstStrStorage;
3466  raw_string_ostream InstStr(InstStrStorage);
3467  InstStr << Inst;
3468 
3469  R << ": '" << InstStr.str() << "'";
3470  }
3471 
3472  reportTranslationError(*MF, *TPC, *ORE, R);
3473  return false;
3474  }
3475 
3476  if (!finalizeBasicBlock(*BB, MBB))
3477  return false;
3478  }
3479 #ifndef NDEBUG
3480  WrapperObserver.removeObserver(&Verifier);
3481 #endif
3482  }
3483 
3484  finishPendingPhis();
3485 
3486  SwiftError.propagateVRegs();
3487 
3488  // Merge the argument lowering and constants block with its single
3489  // successor, the LLVM-IR entry block. We want the basic block to
3490  // be maximal.
3491  assert(EntryBB->succ_size() == 1 &&
3492  "Custom BB used for lowering should have only one successor");
3493  // Get the successor of the current entry block.
3494  MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
3495  assert(NewEntryBB.pred_size() == 1 &&
3496  "LLVM-IR entry block has a predecessor!?");
3497  // Move all the instruction from the current entry block to the
3498  // new entry block.
3499  NewEntryBB.splice(NewEntryBB.begin(), EntryBB, EntryBB->begin(),
3500  EntryBB->end());
3501 
3502  // Update the live-in information for the new entry block.
3503  for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
3504  NewEntryBB.addLiveIn(LiveIn);
3505  NewEntryBB.sortUniqueLiveIns();
3506 
3507  // Get rid of the now empty basic block.
3508  EntryBB->removeSuccessor(&NewEntryBB);
3509  MF->remove(EntryBB);
3510  MF->DeleteMachineBasicBlock(EntryBB);
3511 
3512  assert(&MF->front() == &NewEntryBB &&
3513  "New entry wasn't next in the list of basic block!");
3514 
3515  // Initialize stack protector information.
3516  StackProtector &SP = getAnalysis<StackProtector>();
3518 
3519  return false;
3520 }
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
llvm::EHPersonality::MSVC_CXX
@ MSVC_CXX
i
i
Definition: README.txt:29
llvm::HexagonInstrInfo::isTailCall
bool isTailCall(const MachineInstr &MI) const override
Definition: HexagonInstrInfo.cpp:2611
llvm::MachineFunction::remove
void remove(iterator MBBI)
Definition: MachineFunction.h:848
llvm::isAsynchronousEHPersonality
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
Definition: EHPersonalities.h:50
llvm::TargetMachine::getOptLevel
CodeGenOpt::Level getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
Definition: TargetMachine.cpp:188
Int32Ty
IntegerType * Int32Ty
Definition: NVVMIntrRange.cpp:67
llvm::DbgVariableIntrinsic::getExpression
DIExpression * getExpression() const
Definition: IntrinsicInst.h:257
llvm::TargetLoweringBase::getPointerMemTy
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
Definition: TargetLowering.h:353
llvm::MachineFrameInfo::hasVarSizedObjects
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
Definition: MachineFrameInfo.h:353
llvm::Argument
This class represents an incoming formal argument to a Function.
Definition: Argument.h:29
llvm::MachineIRBuilder::buildInsertVectorElement
MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Elt, const SrcOp &Idx)
Build and insert Res = G_INSERT_VECTOR_ELT Val, Elt, Idx.
Definition: MachineIRBuilder.cpp:774
llvm::RISCVAttrs::StackAlign
StackAlign
Definition: RISCVAttributes.h:37
StackProtector.h
llvm::Type::isSized
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:263
llvm::OptimizationRemarkMissed
Diagnostic information for missed-optimization remarks.
Definition: DiagnosticInfo.h:730
llvm::Function::isIntrinsic
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:212
llvm::TargetLoweringBase::getStackPointerRegisterToSaveRestore
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
Definition: TargetLowering.h:1746
LowLevelType.h
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:105
MathExtras.h
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:131
llvm::SwitchCG::sortAndRangeify
void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
Definition: SwitchLoweringUtils.cpp:464
llvm
This file implements support for optimizing divisions by a constant.
Definition: AllocatorList.h:23
llvm::DbgDeclareInst::getAddress
Value * getAddress() const
Definition: IntrinsicInst.h:310
Reg
unsigned Reg
Definition: MachineSink.cpp:1566
llvm::MCSymbol
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::MachineIRBuilder::buildFMul
MachineInstrBuilder buildFMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Definition: MachineIRBuilder.h:1446
llvm::MachineIRBuilder::buildAtomicRMW
MachineInstrBuilder buildAtomicRMW(unsigned Opcode, const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val, MachineMemOperand &MMO)
Build and insert OldValRes<def> = G_ATOMICRMW_<Opcode> Addr, Val, MMO.
Definition: MachineIRBuilder.cpp:837
llvm::MachineIRBuilder::setDebugLoc
void setDebugLoc(const DebugLoc &DL)
Set the debug location to DL for all the next build instructions.
Definition: MachineIRBuilder.h:355
llvm::StringRef::empty
LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:153
llvm::CmpInst::ICMP_EQ
@ ICMP_EQ
equal
Definition: InstrTypes.h:741
llvm::MachineInstrBuilder::addFPImm
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
Definition: MachineInstrBuilder.h:141
llvm::ReturnInst
Return a value (possibly void), from a function.
Definition: Instructions.h:2986
TargetFrameLowering.h
llvm::MachineIRBuilder::buildICmp
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
Definition: MachineIRBuilder.cpp:748
llvm::MemoryOpRemark
Definition: MemoryOpRemark.h:36
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::Value::hasOneUse
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
llvm::GISelCSEAnalysisWrapperPass
The actual analysis pass wrapper.
Definition: CSEInfo.h:220
llvm::MachineBasicBlock::getBasicBlock
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
Definition: MachineBasicBlock.h:202
llvm::generic_gep_type_iterator
Definition: GetElementPtrTypeIterator.h:31
llvm::RecurKind::FMul
@ FMul
Product of floats.
Metadata.h
llvm::ISD::LIFETIME_END
@ LIFETIME_END
Definition: ISDOpcodes.h:1178
llvm::Type::getInt8PtrTy
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:293
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:720
llvm::MachineModuleInfo::getContext
const MCContext & getContext() const
Definition: MachineModuleInfo.h:167
llvm::AllocaInst::getAlign
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:120
CallLowering.h
llvm::BasicBlock::getParent
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:107
llvm::InlineAsmLowering
Definition: InlineAsmLowering.h:28
IntrinsicInst.h
ceil
We have fiadd patterns now but the followings have the same cost and complexity We need a way to specify the later is more profitable def def The FP stackifier should handle simple permutates to reduce number of shuffle e g ceil
Definition: README-FPStack.txt:54
llvm::GlobalValue::dropLLVMManglingEscape
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition: GlobalValue.h:488
llvm::HexagonISD::JT
@ JT
Definition: HexagonISelLowering.h:52
Loads.h
llvm::SwitchCG::CaseBlock
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
Definition: SwitchLoweringUtils.h:109
llvm::SwitchCG::CaseBlock::DbgLoc
DebugLoc DbgLoc
Definition: SwitchLoweringUtils.h:138
llvm::MCContext
Context object for machine code objects.
Definition: MCContext.h:72
llvm::Function
Definition: Function.h:62
llvm::SwitchCG::SwitchWorkList
SmallVector< SwitchWorkListItem, 4 > SwitchWorkList
Definition: SwitchLoweringUtils.h:247
llvm::MachineFunction::getContext
MCContext & getContext() const
Definition: MachineFunction.h:586
llvm::lower_bound
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1661
llvm::ISD::LIFETIME_START
@ LIFETIME_START
This corresponds to the llvm.lifetime.
Definition: ISDOpcodes.h:1177
llvm::MachineInstr::copyFlagsFromInstruction
static uint16_t copyFlagsFromInstruction(const Instruction &I)
Definition: MachineInstr.cpp:537
Pass.h
llvm::TargetSubtargetInfo::getInstrInfo
virtual const TargetInstrInfo * getInstrInfo() const
Definition: TargetSubtargetInfo.h:92
llvm::IntrinsicInst::getIntrinsicID
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:52
llvm::BlockFrequencyInfoWrapperPass
Legacy analysis pass which computes BlockFrequencyInfo.
Definition: BlockFrequencyInfo.h:138
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition: Instructions.h:752
llvm::StackProtector::copyToMachineFrameInfo
void copyToMachineFrameInfo(MachineFrameInfo &MFI) const
Definition: StackProtector.cpp:596
llvm::DataLayout::getTypeSizeInBits
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:664
llvm::M68k::addMemOperand
static const MachineInstrBuilder & addMemOperand(const MachineInstrBuilder &MIB, int FI, int Offset=0)
Definition: M68kInstrBuilder.h:76
llvm::raw_string_ostream
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:625
minimum
Should compile r2 movcc movcs str strb mov lr r1 movcs movcc mov lr r1 str mov mov cmp r1 movlo r2 str bx lr r0 mov mov cmp r0 movhs r2 mov r1 bx lr Some of the NEON intrinsics may be appropriate for more general either as target independent intrinsics or perhaps elsewhere in the ARM backend Some of them may also be lowered to target independent and perhaps some new SDNodes could be added For minimum
Definition: README.txt:489
GetElementPtrTypeIterator.h
llvm::MachineIRBuilder::buildSplatVector
MachineInstrBuilder buildSplatVector(const DstOp &Res, const SrcOp &Src)
Build and insert Res = G_BUILD_VECTOR with Src replicated to fill the number of elements.
Definition: MachineIRBuilder.cpp:641
High
uint64_t High
Definition: NVVMIntrRange.cpp:61
llvm::ReturnInst::getReturnValue
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
Definition: Instructions.h:3031
llvm::AllocaInst::getType
PointerType * getType() const
Overload to return most specific pointer type.
Definition: Instructions.h:104
llvm::ilist_node_with_parent::getNextNode
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:288
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
Wrapper
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
Definition: AMDGPUAliasAnalysis.cpp:31
llvm::RISCVFenceField::W
@ W
Definition: RISCVBaseInfo.h:199
InlineAsm.h
llvm::LandingPadInst
The landingpad instruction holds all of the information necessary to generate correct exception handl...
Definition: Instructions.h:2885
llvm::StackProtectorDescriptor::getSuccessMBB
MachineBasicBlock * getSuccessMBB()
Definition: CodeGenCommonISel.h:170
llvm::MachineFunction::getMachineMemOperand
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Definition: MachineFunction.cpp:431
llvm::enumerate
detail::enumerator< R > enumerate(R &&TheRange)
Given an input range, returns a new range whose values are are pair (A,B) such that A is the 0-based ...
Definition: STLExtras.h:1981
llvm::SwitchCG::JumpTableHeader::FallthroughUnreachable
bool FallthroughUnreachable
Definition: SwitchLoweringUtils.h:186
llvm::CallBase::isInlineAsm
bool isInlineAsm() const
Check if this call is an inline asm statement.
Definition: InstrTypes.h:1448
llvm::DataLayout::getTypeStoreSize
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Definition: DataLayout.h:471
llvm::FunctionPass::skipFunction
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Definition: Pass.cpp:163
llvm::Function::getSubprogram
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1541
llvm::InlineAsmLowering::lowerInlineAsm
bool lowerInlineAsm(MachineIRBuilder &MIRBuilder, const CallBase &CB, std::function< ArrayRef< Register >(const Value &Val)> GetOrCreateVRegs) const
Lower the given inline asm call instruction GetOrCreateVRegs is a callback to materialize a register ...
Definition: InlineAsmLowering.cpp:273
llvm::GISelCSEInfo
The CSE Analysis object.
Definition: CSEInfo.h:69
ErrorHandling.h
llvm::Function::getEntryBlock
const BasicBlock & getEntryBlock() const
Definition: Function.h:718
llvm::IRTranslator::runOnMachineFunction
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
Definition: IRTranslator.cpp:3306
llvm::SwiftErrorValueTracking::createEntriesInEntryBlock
bool createEntriesInEntryBlock(DebugLoc DbgLoc)
Create initial definitions of swifterror values in the entry block of the current function.
Definition: SwiftErrorValueTracking.cpp:115
llvm::MachineMemOperand::MOInvariant
@ MOInvariant
The memory access always returns the same value (or traps).
Definition: MachineMemOperand.h:145
llvm::SwiftErrorValueTracking::propagateVRegs
void propagateVRegs()
Propagate assigned swifterror vregs through a function, synthesizing PHI nodes when needed to maintai...
Definition: SwiftErrorValueTracking.cpp:147
llvm::ISD::EH_LABEL
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:988
llvm::CmpInst::ICMP_NE
@ ICMP_NE
not equal
Definition: InstrTypes.h:742
R600_InstFlag::FC
@ FC
Definition: R600Defines.h:32
llvm::TargetLoweringBase::getTgtMemIntrinsic
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
Definition: TargetLowering.h:1016
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
llvm::MachineBasicBlock::setSuccProbability
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
Definition: MachineBasicBlock.cpp:1456
llvm::MachineIRBuilder::buildZExtOrTrunc
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
Definition: MachineIRBuilder.cpp:477
ValueTracking.h
llvm::SwitchCG::CC_JumpTable
@ CC_JumpTable
A cluster of cases suitable for jump table lowering.
Definition: SwitchLoweringUtils.h:36
OptimizationRemarkEmitter.h
llvm::TargetLoweringBase::getLibcallName
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
Definition: TargetLowering.h:2865
MachineBasicBlock.h
llvm::SwitchCG::SwitchWorkListItem
Definition: SwitchLoweringUtils.h:239
llvm::TargetSubtargetInfo::getRegisterInfo
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Definition: TargetSubtargetInfo.h:124
llvm::LoadInst::getPointerOperandType
Type * getPointerOperandType() const
Definition: Instructions.h:270
llvm::DataLayout::getStructLayout
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:676
llvm::GISelObserverWrapper::removeObserver
void removeObserver(GISelChangeObserver *O)
Definition: GISelChangeObserver.h:78
reportTranslationError
IRTranslator LLVM IR static false void reportTranslationError(MachineFunction &MF, const TargetPassConfig &TPC, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R)
Definition: IRTranslator.cpp:108
llvm::TargetRegisterInfo
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Definition: TargetRegisterInfo.h:233
llvm::Function::getContext
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:321
llvm::MachineIRBuilder::buildDbgLabel
MachineInstrBuilder buildDbgLabel(const MDNode *Label)
Build and insert a DBG_LABEL instructions specifying that Label is given.
Definition: MachineIRBuilder.cpp:117
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
llvm::RTLIB::Libcall
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Definition: RuntimeLibcalls.h:30
llvm::BranchProbability::getZero
static BranchProbability getZero()
Definition: BranchProbability.h:49
llvm::reverse
auto reverse(ContainerTy &&C, std::enable_if_t< has_rbegin< ContainerTy >::value > *=nullptr)
Definition: STLExtras.h:333
llvm::diagnoseDontCall
void diagnoseDontCall(const CallInst &CI)
Definition: DiagnosticInfo.cpp:403
TargetInstrInfo.h
llvm::CallBase::getAttributes
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1458
llvm::MachineMemOperand
A description of a memory reference used in the backend.
Definition: MachineMemOperand.h:128
llvm::TargetOptions::TrapUnreachable
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
Definition: TargetOptions.h:274
llvm::MachineMemOperand::MODereferenceable
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
Definition: MachineMemOperand.h:143
llvm::TargetLoweringBase::getExceptionPointerRegister
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
Definition: TargetLowering.h:1753
llvm::MachineIRBuilder::buildFPTrunc
MachineInstrBuilder buildFPTrunc(const DstOp &Res, const SrcOp &Op, Optional< unsigned > Flags=None)
Build and insert Res = G_FPTRUNC Op.
Definition: MachineIRBuilder.cpp:742
llvm::MachineFunction::insert
void insert(iterator MBBI, MachineBasicBlock *MBB)
Definition: MachineFunction.h:835
llvm::copy
OutputIt copy(R &&Range, OutputIt Out)
Definition: STLExtras.h:1602
llvm::SmallSet
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:134
llvm::CallLowering::supportSwiftError
virtual bool supportSwiftError() const
Definition: CallLowering.h:437
llvm::SwitchCG::CaseClusterIt
CaseClusterVector::iterator CaseClusterIt
Definition: SwitchLoweringUtils.h:87
llvm::DbgVariableIntrinsic::getVariable
DILocalVariable * getVariable() const
Definition: IntrinsicInst.h:253
llvm::FenceInst
An instruction for ordering other memory operations.
Definition: Instructions.h:444
Offset
uint64_t Offset
Definition: ELFObjHandler.cpp:81
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:449
llvm::ore::NV
DiagnosticInfoOptimizationBase::Argument NV
Definition: OptimizationRemarkEmitter.h:136
llvm::getSelectionDAGFallbackAnalysisUsage
void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU)
Modify analysis usage so it preserves passes required for the SelectionDAG fallback.
Definition: Utils.cpp:863
llvm::TargetLoweringBase::getVectorIdxTy
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
Definition: TargetLowering.h:400
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition: InstrTypes.h:750
llvm::codeview::EncodedFramePtrReg::StackPtr
@ StackPtr
llvm::FunctionType::getNumParams
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition: DerivedTypes.h:139
llvm::MachineIRBuilder::buildInstrNoInsert
MachineInstrBuilder buildInstrNoInsert(unsigned Opcode)
Build but don't insert <empty> = Opcode <empty>.
Definition: MachineIRBuilder.cpp:40
llvm::MipsISD::Ret
@ Ret
Definition: MipsISelLowering.h:116
llvm::Intrinsic::not_intrinsic
@ not_intrinsic
Definition: Intrinsics.h:45
MachineIRBuilder.h
STLExtras.h
llvm::CallBase::arg_begin
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1303
llvm::LoadInst::getPointerOperand
Value * getPointerOperand()
Definition: Instructions.h:267
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:491
llvm::SwitchCG::JumpTableHeader::Last
APInt Last
Definition: SwitchLoweringUtils.h:182
llvm::gep_type_begin
gep_type_iterator gep_type_begin(const User *GEP)
Definition: GetElementPtrTypeIterator.h:139
llvm::successors
succ_range successors(Instruction *I)
Definition: CFG.h:262
llvm::MachineFrameInfo::CreateVariableSizedObject
int CreateVariableSizedObject(Align Alignment, const AllocaInst *Alloca)
Notify the MachineFrameInfo object that a variable sized object has been created.
Definition: MachineFrameInfo.cpp:74
llvm::RAIIMFObserverInstaller
A simple RAII based Observer installer.
Definition: GISelChangeObserver.h:120
llvm::MachineIRBuilder::buildFence
MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope)
Build and insert G_FENCE Ordering, Scope.
Definition: MachineIRBuilder.cpp:945
llvm::BitmaskEnumDetail::Mask
std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1567
llvm::MachineIRBuilder::buildDynStackAlloc
MachineInstrBuilder buildDynStackAlloc(const DstOp &Res, const SrcOp &Size, Align Alignment)
Build and insert Res = G_DYN_STACKALLOC Size, Align.
Definition: MachineIRBuilder.cpp:126
llvm::FunctionLoweringInfo::CanLowerReturn
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
Definition: FunctionLoweringInfo.h:62
llvm::Type::getInt32Ty
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:241
llvm::MachineIRBuilder::getDebugLoc
const DebugLoc & getDebugLoc()
Get the current instruction's debug location.
Definition: MachineIRBuilder.h:358
llvm::SwitchCG::CaseBlock::CmpMHS
const Value * CmpMHS
Definition: SwitchLoweringUtils.h:127
llvm::APIntOps::umin
const APInt & umin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be unsigned.
Definition: APInt.h:2128
llvm::ArrayRef::empty
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
llvm::MachineFunctionPass::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition: MachineFunctionPass.cpp:102
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1233
llvm::MachineIRBuilder::buildConstant
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
Definition: MachineIRBuilder.cpp:255
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::TargetPassConfig::isGISelCSEEnabled
virtual bool isGISelCSEEnabled() const
Check whether continuous CSE should be enabled in GISel passes.
Definition: TargetPassConfig.cpp:1534
llvm::RISCVFenceField::R
@ R
Definition: RISCVBaseInfo.h:198
MachineRegisterInfo.h
llvm::MachineOperand::CreateES
static MachineOperand CreateES(const char *SymName, unsigned TargetFlags=0)
Definition: MachineOperand.h:858
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
a
=0.0 ? 0.0 :(a > 0.0 ? 1.0 :-1.0) a
Definition: README.txt:489
llvm::gep_type_end
gep_type_iterator gep_type_end(const User *GEP)
Definition: GetElementPtrTypeIterator.h:146
floor
We have fiadd patterns now but the followings have the same cost and complexity We need a way to specify the later is more profitable def def The FP stackifier should handle simple permutates to reduce number of shuffle e g floor
Definition: README-FPStack.txt:54
llvm::DILabel::isValidLocationForIntrinsic
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this label.
Definition: DebugInfoMetadata.h:3244
llvm::getLLTForType
LLT getLLTForType(Type &Ty, const DataLayout &DL)
Construct a low-level type based on an LLVM type.
Definition: LowLevelType.cpp:21
Context
LLVMContext & Context
Definition: NVVMIntrRange.cpp:66
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::classifyEHPersonality
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
Definition: EHPersonalities.cpp:21
llvm::LLT::fixed_vector
static LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
Definition: LowLevelTypeImpl.h:75
EnableCSEInIRTranslator
static cl::opt< bool > EnableCSEInIRTranslator("enable-cse-in-irtranslator", cl::desc("Should enable CSE in irtranslator"), cl::Optional, cl::init(false))
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:206
llvm::MachineBasicBlock::addSuccessor
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Definition: MachineBasicBlock.cpp:751
llvm::MachineBasicBlock::pred_size
unsigned pred_size() const
Definition: MachineBasicBlock.h:328
TargetLowering.h
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
llvm::AllocaInst::isStaticAlloca
bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size.
Definition: Instructions.cpp:1397
llvm::Instruction::getOpcode
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:160
llvm::FenceInst::getSyncScopeID
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this fence instruction.
Definition: Instructions.h:480
llvm::MachineFunction::front
const MachineBasicBlock & front() const
Definition: MachineFunction.h:828
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:644
llvm::RAIIDelegateInstaller
A simple RAII based Delegate installer.
Definition: GISelChangeObserver.h:108
llvm::MachineIRBuilder::buildAtomicCmpXchgWithSuccess
MachineInstrBuilder buildAtomicCmpXchgWithSuccess(Register OldValRes, Register SuccessRes, Register Addr, Register CmpVal, Register NewVal, MachineMemOperand &MMO)
Build and insert OldValRes<def>, SuccessRes<def> = G_ATOMIC_CMPXCHG_WITH_SUCCESS Addr,...
Definition: MachineIRBuilder.cpp:785
llvm::TargetInstrInfo
TargetInstrInfo - Interface to description of machine instruction set.
Definition: TargetInstrInfo.h:97
TargetMachine.h
llvm::SwitchCG::CaseBlock::FalseBB
MachineBasicBlock * FalseBB
Definition: SwitchLoweringUtils.h:130
llvm::MachineInstrBuilder::addMBB
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:146
llvm::MachineIRBuilder::buildExtractVectorElement
MachineInstrBuilder buildExtractVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
Definition: MachineIRBuilder.cpp:780
llvm::MachineIRBuilder::buildLoad
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
Definition: MachineIRBuilder.h:832
InlineAsmLowering.h
llvm::CmpInst::isFPPredicate
bool isFPPredicate() const
Definition: InstrTypes.h:813
Constants.h
llvm::PatternMatch::match
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
llvm::CallLowering::checkReturnTypeForCallConv
bool checkReturnTypeForCallConv(MachineFunction &MF) const
Toplevel function to check the return type based on the target calling convention.
Definition: CallLowering.cpp:929
llvm::RegState::Undef
@ Undef
Value of the register doesn't matter.
Definition: MachineInstrBuilder.h:52
llvm::MachineIRBuilder::buildShl
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Definition: MachineIRBuilder.h:1476
llvm::PHINode::getIncomingValue
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
Definition: Instructions.h:2729
llvm::DbgValueInst
This represents the llvm.dbg.value instruction.
Definition: IntrinsicInst.h:347
llvm::AllocaInst::getAllocatedType
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:113
llvm::MachineIRBuilder::setMBB
void setMBB(MachineBasicBlock &MBB)
Set the insertion point to the end of MBB.
Definition: MachineIRBuilder.h:324
llvm::BranchProbabilityInfoWrapperPass
Legacy analysis pass which computes BranchProbabilityInfo.
Definition: BranchProbabilityInfo.h:440
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::MachineBasicBlock::succ_end
succ_iterator succ_end()
Definition: MachineBasicBlock.h:334
llvm::User
Definition: User.h:44
llvm::succ_size
unsigned succ_size(const Instruction *I)
Definition: CFG.h:259
llvm::CallLowering::ArgInfo
Definition: CallLowering.h:61
llvm::TargetFrameLowering::getStackAlign
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
Definition: TargetFrameLowering.h:100
Intrinsics.h
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::ConstrainedFPIntrinsic
This is the common base class for constrained floating point intrinsics.
Definition: IntrinsicInst.h:474
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:746
llvm::getLLTForMVT
LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
Definition: LowLevelType.cpp:65
llvm::StackProtectorDescriptor
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
Definition: CodeGenCommonISel.h:115
round
static uint64_t round(uint64_t Acc, uint64_t Input)
Definition: xxhash.cpp:57
InstrTypes.h
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
Definition: InstrTypes.h:1383
llvm::TargetMachine::getIntrinsicInfo
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
Definition: TargetMachine.h:214
llvm::TargetLowering
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Definition: TargetLowering.h:3195
llvm::CallLowering::lowerFormalArguments
virtual bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F, ArrayRef< ArrayRef< Register >> VRegs, FunctionLoweringInfo &FLI) const
This hook must be implemented to lower the incoming (formal) arguments, described by VRegs,...
Definition: CallLowering.h:534
MCContext.h
llvm::LLT::getSizeInBits
TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelTypeImpl.h:153
llvm::CallLowering::lowerReturn
virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, ArrayRef< Register > VRegs, FunctionLoweringInfo &FLI, Register SwiftErrorVReg) const
This hook must be implemented to lower outgoing return values, described by Val, into the specified v...
Definition: CallLowering.h:502
llvm::FCmpInst
This instruction compares its operands according to the predicate given to the constructor.
Definition: Instructions.h:1369
llvm::BranchProbabilityInfo
Analysis providing branch probability information.
Definition: BranchProbabilityInfo.h:115
llvm::StackProtector
Definition: StackProtector.h:37
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
b
the resulting code requires compare and branches when and if the revised code is with conditional branches instead of More there is a byte word extend before each where there should be only and the condition codes are not remembered when the same two values are compared twice More LSR enhancements i8 and i32 load store addressing modes are identical int b
Definition: README.txt:418
llvm::MachineIRBuilder::buildFCmp
MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, Optional< unsigned > Flags=None)
Build and insert a Res = G_FCMP PredOp0, Op1.
Definition: MachineIRBuilder.cpp:755
false
Definition: StackSlotColoring.cpp:142
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:129
llvm::MachineFunction::getTypeIDFor
unsigned getTypeIDFor(const GlobalValue *TI)
Return the type id for the specified typeinfo. This is function wide.
Definition: MachineFunction.cpp:861
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
First
into llvm powi allowing the code generator to produce balanced multiplication trees First
Definition: README.txt:54
llvm::isUIntN
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:455
llvm::make_scope_exit
LLVM_NODISCARD detail::scope_exit< typename std::decay< Callable >::type > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:58
llvm::Constant::getAllOnesValue
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:405
llvm::Type::isTokenTy
bool isTokenTy() const
Return true if this is 'token'.
Definition: Type.h:187
llvm::MachineBasicBlock::RegisterMaskPair
Pair of physical register and lane mask.
Definition: MachineBasicBlock.h:101
llvm::Instruction
Definition: Instruction.h:45
llvm::SwiftErrorValueTracking::getOrCreateVRegUseAt
Register getOrCreateVRegUseAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a use of a swifterror by an instruction.
Definition: SwiftErrorValueTracking.cpp:65
llvm::DataLayout::getABITypeAlign
Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:826
llvm::LLVMContext::OB_cfguardtarget
@ OB_cfguardtarget
Definition: LLVMContext.h:93
llvm::SwitchCG::CaseBlock::PredInfoPair::NoCmp
bool NoCmp
Definition: SwitchLoweringUtils.h:114
llvm::AllocaInst::getArraySize
const Value * getArraySize() const
Get the number of elements allocated.
Definition: Instructions.h:100
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:140
llvm::AtomicRMWInst::Nand
@ Nand
*p = ~(old & v)
Definition: Instructions.h:748
llvm::TargetLoweringBase::getSDagStackGuard
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
Definition: TargetLoweringBase.cpp:1998
SwitchLoweringUtils.h
llvm::cl::Option::getNumOccurrences
int getNumOccurrences() const
Definition: CommandLine.h:402
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:925
llvm::DILocalVariable::isValidLocationForIntrinsic
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
Definition: DebugInfoMetadata.h:3174
llvm::FunctionLoweringInfo::BPI
BranchProbabilityInfo * BPI
Definition: FunctionLoweringInfo.h:58
llvm::MachineIRBuilder::buildAnd
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
Definition: MachineIRBuilder.h:1505
IR
Statically lint checks LLVM IR
Definition: Lint.cpp:744
llvm::Instruction::getAAMetadata
AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
Definition: Metadata.cpp:1370
llvm::DbgVariableIntrinsic::hasArgList
bool hasArgList() const
Definition: IntrinsicInst.h:227
llvm::SwitchCG::CaseBlock::FalseProb
BranchProbability FalseProb
Definition: SwitchLoweringUtils.h:141
Info
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
getOffsetFromIndices
static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL)
Definition: IRTranslator.cpp:1356
llvm::ISD::ArgFlagsTy::setInReg
void setInReg()
Definition: TargetCallingConv.h:80
Align
uint64_t Align
Definition: ELFObjHandler.cpp:83
PatternMatch.h
llvm::BasicBlock::getFirstNonPHI
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:216
llvm::MachineIRBuilder::buildIndirectDbgValue
MachineInstrBuilder buildIndirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in me...
Definition: MachineIRBuilder.cpp:65
llvm::FixedVectorType::get
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:686
llvm::CallInst::isTailCall
bool isTailCall() const
Definition: Instructions.h:1669
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::MCID::Call
@ Call
Definition: MCInstrDesc.h:153
llvm::MachineIRBuilder::buildBrCond
MachineInstrBuilder buildBrCond(const SrcOp &Tst, MachineBasicBlock &Dest)
Build and insert G_BRCOND Tst, Dest.
Definition: MachineIRBuilder.cpp:332
llvm::AtomicRMWInst::Xchg
@ Xchg
*p = v
Definition: Instructions.h:740
llvm::Attribute::getValueAsString
StringRef getValueAsString() const
Return the attribute's value as a string.
Definition: Attributes.cpp:301
llvm::PHINode::getNumIncomingValues
unsigned getNumIncomingValues() const
Return the number of incoming edges.
Definition: Instructions.h:2725
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition: Instructions.h:742
llvm::EHPersonality::Wasm_CXX
@ Wasm_CXX
llvm::LoadInst::getSyncScopeID
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this load instruction.
Definition: Instructions.h:242
llvm::MachineFunction::addInvoke
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
Definition: MachineFunction.cpp:716
llvm::MachineInstr::NoFPExcept
@ NoFPExcept
Definition: MachineInstr.h:108
Type.h
llvm::MachineInstrBuilder::getReg
Register getReg(unsigned Idx) const
Get the register for the operand index.
Definition: MachineInstrBuilder.h:94
llvm::SwitchCG::JumpTableHeader::First
APInt First
Definition: SwitchLoweringUtils.h:181
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:95
llvm::MachineIRBuilder::buildBrIndirect
MachineInstrBuilder buildBrIndirect(Register Tgt)
Build and insert G_BRINDIRECT Tgt.
Definition: MachineIRBuilder.cpp:222
LowLevelTypeImpl.h
llvm::Instruction::getMetadata
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:282
INITIALIZE_PASS_END
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:58
CFG.h
llvm::maxnum
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE maxNum semantics.
Definition: APFloat.h:1307
llvm::AllocaInst::isSwiftError
bool isSwiftError() const
Return true if this alloca is used as a swifterror argument to a call.
Definition: Instructions.h:148
llvm::BranchInst::getCondition
Value * getCondition() const
Definition: Instructions.h:3149
llvm::PatternMatch::m_ExtractElt
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
Definition: PatternMatch.h:1502
llvm::TargetRegisterInfo::getPointerRegClass
virtual const TargetRegisterClass * getPointerRegClass(const MachineFunction &MF, unsigned Kind=0) const
Returns a TargetRegisterClass used for pointer values.
Definition: TargetRegisterInfo.h:759
llvm::CmpInst::FCMP_FALSE
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:722
llvm::InvokeInst
Invoke instruction.
Definition: Instructions.h:3749
llvm::CmpInst
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:710
llvm::TargetPassConfig
Target-Independent Code Generator Pass Configuration Options.
Definition: TargetPassConfig.h:84
llvm::LLT::pointer
static LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
Definition: LowLevelTypeImpl.h:50
llvm::MachineInstr::copyIRFlags
void copyIRFlags(const Instruction &I)
Copy all flags to MachineInst MIFlags.
Definition: MachineInstr.cpp:575
llvm::InlineAsm
Definition: InlineAsm.h:31
llvm::MachineIRBuilder::buildBr
MachineInstrBuilder buildBr(MachineBasicBlock &Dest)
Build and insert G_BR Dest.
Definition: MachineIRBuilder.cpp:218
llvm::DataLayout::getPrefTypeAlign
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:835
llvm::MachineFunction::getMMI
MachineModuleInfo & getMMI() const
Definition: MachineFunction.h:585
llvm::StackProtectorDescriptor::resetPerBBState
void resetPerBBState()
Reset state that changes when we handle different basic blocks.
Definition: CodeGenCommonISel.h:153
llvm::MachineBasicBlock::succ_iterator
std::vector< MachineBasicBlock * >::iterator succ_iterator
Definition: MachineBasicBlock.h:306
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:634
llvm::MachineInstrBuilder::addFrameIndex
const MachineInstrBuilder & addFrameIndex(int Idx) const
Definition: MachineInstrBuilder.h:152
Verifier
verify safepoint Safepoint IR Verifier
Definition: SafepointIRVerifier.cpp:253
llvm::AtomicRMWInst::UMin
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:760
BasicBlock.h
llvm::cl::opt< bool >
llvm::MachineFunction::push_back
void push_back(MachineBasicBlock *MBB)
Definition: MachineFunction.h:833
llvm::StackProtectorDescriptor::getFailureMBB
MachineBasicBlock * getFailureMBB()
Definition: CodeGenCommonISel.h:171
llvm::MachineIRBuilder::buildFAdd
MachineInstrBuilder buildFAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_FADD Op0, Op1.
Definition: MachineIRBuilder.h:1579
llvm::MachineFrameInfo::setHasMustTailInVarArgFunc
void setHasMustTailInVarArgFunc(bool B)
Definition: MachineFrameInfo.h:603
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:304
llvm::GlobalValue
Definition: GlobalValue.h:44
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::countPopulation
unsigned countPopulation(T Value)
Count the number of set bits in a value.
Definition: MathExtras.h:567
llvm::MachineIRBuilder::buildBrJT
MachineInstrBuilder buildBrJT(Register TablePtr, unsigned JTI, Register IndexReg)
Build and insert G_BRJT TablePtr, JTI, IndexReg.
Definition: MachineIRBuilder.cpp:227
llvm::SmallSet::count
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:164
llvm::getUnderlyingObjects
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
Definition: ValueTracking.cpp:4434
llvm::MachineIRBuilder::buildSExtOrTrunc
MachineInstrBuilder buildSExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
Definition: MachineIRBuilder.cpp:472
BranchProbabilityInfo.h
llvm::MachineIRBuilder
Helper class to build MachineInstr.
Definition: MachineIRBuilder.h:212
llvm::StackProtectorDescriptor::shouldEmitStackProtector
bool shouldEmitStackProtector() const
Returns true if all fields of the stack protector descriptor are initialized implying that we should/...
Definition: CodeGenCommonISel.h:121
llvm::InlineAsm::canThrow
bool canThrow() const
Definition: InlineAsm.h:71
llvm::AtomicRMWInst::Sub
@ Sub
*p = old - v
Definition: Instructions.h:744
llvm::CallBase::countOperandBundlesOfType
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Definition: InstrTypes.h:1953
llvm::ICmpInst
This instruction compares its operands according to the predicate given to the constructor.
Definition: Instructions.h:1203
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:64
llvm::MachineInstrBuilder
Definition: MachineInstrBuilder.h:69
llvm::TargetLibraryInfoWrapperPass
Definition: TargetLibraryInfo.h:465
uint64_t
RuntimeLibcalls.h
llvm::PatternMatch::m_LogicalOr
LogicalOp_match< LHS, RHS, Instruction::Or > m_LogicalOr(const LHS &L, const RHS &R)
Matches L || R either in the form of L | R or L ? true : R.
Definition: PatternMatch.h:2522
llvm::SwitchCG::CaseBlock::PredInfo
struct PredInfoPair PredInfo
Definition: SwitchLoweringUtils.h:121
llvm::ExtractTypeInfo
GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
Definition: Analysis.cpp:168
llvm::GlobalValue::getParent
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:578
llvm::DbgDeclareInst
This represents the llvm.dbg.declare instruction.
Definition: IntrinsicInst.h:308
const
aarch64 promote const
Definition: AArch64PromoteConstant.cpp:232
llvm::DbgLabelInst
This represents the llvm.dbg.label instruction.
Definition: IntrinsicInst.h:370
Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:80
llvm::SwitchCG::CaseBlock::TrueProb
BranchProbability TrueProb
Definition: SwitchLoweringUtils.h:141
llvm::MachineBasicBlock::setIsEHPad
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
Definition: MachineBasicBlock.h:530
INITIALIZE_PASS_DEPENDENCY
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
llvm::DbgLabelInst::getLabel
DILabel * getLabel() const
Definition: IntrinsicInst.h:372
llvm::MachineIRBuilder::buildMul
MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_MUL Op0, Op1.
Definition: MachineIRBuilder.h:1421
llvm::MachineIRBuilder::getInsertPt
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
Definition: MachineIRBuilder.h:301
llvm::DataLayout::isLittleEndian
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:243
llvm::MachinePointerInfo
This class contains a discriminated union of information about pointers in memory operands,...
Definition: MachineMemOperand.h:38
move
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
Definition: README.txt:546
llvm::MachineIRBuilder::buildPtrAdd
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert Res = G_PTR_ADD Op0, Op1.
Definition: MachineIRBuilder.cpp:182
DebugInfo.h
llvm::TargetLoweringBase::getAtomicMemOperandFlags
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
Definition: TargetLoweringBase.cpp:2261
llvm::TargetOptions::NoTrapAfterNoreturn
unsigned NoTrapAfterNoreturn
Do not emit a trap instruction for 'unreachable' IR instructions behind noreturn calls,...
Definition: TargetOptions.h:278
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::FunctionType::getParamType
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition: DerivedTypes.h:135
Analysis.h
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:756
llvm::countTrailingOnes
unsigned countTrailingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the least significant bit to the first zero bit.
Definition: MathExtras.h:525
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:441
llvm::LLT::isVector
bool isVector() const
Definition: LowLevelTypeImpl.h:123
llvm::SwitchCG::CaseBlock::CmpRHS
const Value * CmpRHS
Definition: SwitchLoweringUtils.h:127
llvm::AtomicRMWInst::Or
@ Or
*p = old | v
Definition: Instructions.h:750
llvm::MachineFunction::CreateMachineBasicBlock
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
Definition: MachineFunction.cpp:415
llvm::TargetLoweringBase::isFMAFasterThanFMulAndFAdd
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
Definition: TargetLowering.h:2746
TargetPassConfig.h
llvm::HighlightColor::Address
@ Address
MemoryOpRemark.h
llvm::MachineIRBuilder::buildSelect
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1, Optional< unsigned > Flags=None)
Build and insert a Res = G_SELECT Tst, Op0, Op1.
Definition: MachineIRBuilder.cpp:764
llvm::SwitchCG::CaseBlock::ThisBB
MachineBasicBlock * ThisBB
Definition: SwitchLoweringUtils.h:133
llvm::MachineMemOperand::Flags
Flags
Flags values. These may be or'd together.
Definition: MachineMemOperand.h:131
llvm::MachineFunction::getName
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Definition: MachineFunction.cpp:542
llvm::BranchProbability::isUnknown
bool isUnknown() const
Definition: BranchProbability.h:47
llvm::TargetMachine::Options
TargetOptions Options
Definition: TargetMachine.h:120
maximum
Should compile r2 movcc movcs str strb mov lr r1 movcs movcc mov lr r1 str mov mov cmp r1 movlo r2 str bx lr r0 mov mov cmp r0 movhs r2 mov r1 bx lr Some of the NEON intrinsics may be appropriate for more general either as target independent intrinsics or perhaps elsewhere in the ARM backend Some of them may also be lowered to target independent and perhaps some new SDNodes could be added For maximum
Definition: README.txt:489
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::TargetMachine
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:79
llvm::MachineIRBuilder::buildConstDbgValue
MachineInstrBuilder buildConstDbgValue(const Constant &C, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instructions specifying that Variable is given by C (suitably modified b...
Definition: MachineIRBuilder.cpp:92
llvm::LLT::isPointer
bool isPointer() const
Definition: LowLevelTypeImpl.h:121
llvm::Instruction::hasMetadata
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
Definition: Instruction.h:264
llvm::MVT::getSizeInBits
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
Definition: MachineValueType.h:860
llvm::MachineIRBuilder::getMBB
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
Definition: MachineIRBuilder.h:287
llvm::MachineBasicBlock::succ_begin
succ_iterator succ_begin()
Definition: MachineBasicBlock.h:332
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:840
llvm::MachineBasicBlock::setHasAddressTaken
void setHasAddressTaken()
Set this block to reflect that it potentially is the target of an indirect branch.
Definition: MachineBasicBlock.h:215
llvm::MachineFunction::getFrameInfo
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Definition: MachineFunction.h:650
llvm::DataLayout::getPointerABIAlignment
Align getPointerABIAlignment(unsigned AS) const
Layout pointer alignment.
Definition: DataLayout.cpp:698
memcpy
<%struct.s * > cast struct s *S to sbyte *< sbyte * > sbyte uint cast struct s *agg result to sbyte *< sbyte * > sbyte uint cast struct s *memtmp to sbyte *< sbyte * > sbyte uint ret void llc ends up issuing two memcpy or custom lower memcpy(of small size) to be ldmia/stmia. I think option 2 is better but the current register allocator cannot allocate a chunk of registers at a time. A feasible temporary solution is to use specific physical registers at the lowering time for small(<
SI
StandardInstrumentations SI(Debug, VerifyEach)
llvm::IndexedInstrProf::HashT::Last
@ Last
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition: MachineBasicBlock.h:225
llvm::TargetLoweringBase::getStoreMemOperandFlags
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
Definition: TargetLoweringBase.cpp:2245
llvm::FenceInst::getOrdering
AtomicOrdering getOrdering() const
Returns the ordering constraint of this fence instruction.
Definition: Instructions.h:469
llvm::MachineInstrBuilder::addMemOperand
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
Definition: MachineInstrBuilder.h:202
llvm::SelectInst
This class represents the LLVM 'select' instruction.
Definition: Instructions.h:1738
llvm::SwitchCG::JumpTableHeader
Definition: SwitchLoweringUtils.h:180
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:138
MachineModuleInfo.h
llvm::WinEH::EncodingType::CE
@ CE
Windows NT (Windows on ARM)
llvm::MachineRegisterInfo::createGenericVirtualRegister
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
Definition: MachineRegisterInfo.cpp:188
llvm::SwiftErrorValueTracking::setFunction
void setFunction(MachineFunction &MF)
Initialize data structures for specified new function.
Definition: SwiftErrorValueTracking.cpp:79
llvm::MachineInstrBuilder::addUse
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
Definition: MachineInstrBuilder.h:123
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:67
llvm::MDNode
Metadata node.
Definition: Metadata.h:906
llvm::MachineIRBuilder::buildFMA
MachineInstrBuilder buildFMA(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, const SrcOp &Src2, Optional< unsigned > Flags=None)
Build and insert Res = G_FMA Op0, Op1, Op2.
Definition: MachineIRBuilder.h:1600
llvm::SwitchCG::CC_Range
@ CC_Range
A cluster of adjacent case labels with the same destination, or just one case.
Definition: SwitchLoweringUtils.h:34
llvm::MCContext::getOrCreateFrameAllocSymbol
MCSymbol * getOrCreateFrameAllocSymbol(StringRef FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
Definition: MCContext.cpp:204
llvm::EHPersonality::CoreCLR
@ CoreCLR
llvm::CallBase::arg_end
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1309
llvm::Function::getFunction
const Function & getFunction() const
Definition: Function.h:137
llvm::BranchInst::isUnconditional
bool isUnconditional() const
Definition: Instructions.h:3146
llvm::TargetLoweringBase::useStackGuardXorFP
virtual bool useStackGuardXorFP() const
If this function returns true, stack protection checks should XOR the frame pointer (or whichever poi...
Definition: TargetLowering.h:1807
llvm::PatternMatch::m_Value
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:76
llvm::BranchProbabilityInfo::getEdgeProbability
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
Definition: BranchProbabilityInfo.cpp:1113
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:75
llvm::MachineFunction
Definition: MachineFunction.h:234
llvm::BranchProbability::getOne
static BranchProbability getOne()
Definition: BranchProbability.h:50
llvm::TargetSubtargetInfo::getCallLowering
virtual const CallLowering * getCallLowering() const
Definition: TargetSubtargetInfo.h:100
llvm::CmpInst::isIntPredicate
bool isIntPredicate() const
Definition: InstrTypes.h:814
llvm::CodeGenOpt::None
@ None
Definition: CodeGen.h:53
llvm::MachineIRBuilder::buildInstr
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
Definition: MachineIRBuilder.h:367
llvm::APIntOps::smin
const APInt & smin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be signed.
Definition: APInt.h:2118
llvm::CallLowering::enableBigEndian
virtual bool enableBigEndian() const
For targets which want to use big-endian can enable it with enableBigEndian() hook.
Definition: CallLowering.h:582
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::ConstrainedFPIntrinsic::getExceptionBehavior
Optional< fp::ExceptionBehavior > getExceptionBehavior() const
Definition: IntrinsicInst.cpp:202
llvm::StackProtector::shouldEmitSDCheck
bool shouldEmitSDCheck(const BasicBlock &BB) const
Definition: StackProtector.cpp:592
llvm::OptimizationRemarkEmitter
The optimization diagnostic interface.
Definition: OptimizationRemarkEmitter.h:33
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1558
llvm::MachineOperand::CreateGA
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, unsigned TargetFlags=0)
Definition: MachineOperand.h:850
DataLayout.h
llvm::MachineFunction::DeleteMachineBasicBlock
void DeleteMachineBasicBlock(MachineBasicBlock *MBB)
DeleteMachineBasicBlock - Delete the given MachineBasicBlock.
Definition: MachineFunction.cpp:422
llvm::MachineFunction::allocateShuffleMask
ArrayRef< int > allocateShuffleMask(ArrayRef< int > Mask)
Definition: MachineFunction.cpp:530
llvm::StructType
Class to represent struct types.
Definition: DerivedTypes.h:213
llvm::MachineFrameInfo::CreateStackObject
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
Definition: MachineFrameInfo.cpp:51
llvm::countTrailingZeros
unsigned countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: MathExtras.h:156
Cond
SmallVector< MachineOperand, 4 > Cond
Definition: BasicBlockSections.cpp:179
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm::MachineBasicBlock::splice
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Definition: MachineBasicBlock.h:950
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI", false, false) INITIALIZE_PASS_END(IRTranslator
llvm::SwiftErrorValueTracking::getOrCreateVRegDefAt
Register getOrCreateVRegDefAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a def of a swifterror by an instruction.
Definition: SwiftErrorValueTracking.cpp:50
llvm::SwitchCG::CaseClusterVector
std::vector< CaseCluster > CaseClusterVector
Definition: SwitchLoweringUtils.h:86
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:134
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::ConstantInt::isZero
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:194
llvm::MachineIRBuilder::buildCopy
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
Definition: MachineIRBuilder.cpp:238
llvm::AnalysisUsage::addPreserved
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
Definition: PassAnalysisSupport.h:98
llvm::MachineIRBuilder::buildFPExt
MachineInstrBuilder buildFPExt(const DstOp &Res, const SrcOp &Op, Optional< unsigned > Flags=None)
Build and insert Res = G_FPEXT Op.
Definition: MachineIRBuilder.h:616
llvm::MemoryOpRemark::canHandle
static bool canHandle(const Instruction *I, const TargetLibraryInfo &TLI)
Definition: MemoryOpRemark.cpp:25
llvm::BranchProbability
Definition: BranchProbability.h:30
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:993
llvm::ISD::ArgFlagsTy
Definition: TargetCallingConv.h:27
IRTranslator.h
TargetSubtargetInfo.h
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
isSwiftError
static bool isSwiftError(const Value *V)
Definition: IRTranslator.cpp:1268
trunc
We have fiadd patterns now but the followings have the same cost and complexity We need a way to specify the later is more profitable def def The FP stackifier should handle simple permutates to reduce number of shuffle e g trunc
Definition: README-FPStack.txt:63
llvm::CallInst::isMustTailCall
bool isMustTailCall() const
Definition: Instructions.h:1674
llvm::SwitchCG::BitTestCase
Definition: SwitchLoweringUtils.h:195
llvm::CodeGenOpt::Level
Level
Definition: CodeGen.h:52
llvm::SmallSet::insert
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:180
llvm::LoadInst::getOrdering
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
Definition: Instructions.h:232
llvm::GISelChangeObserver
Abstract class that contains various methods for clients to notify about changes.
Definition: GISelChangeObserver.h:29
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:175
llvm::EHPersonality
EHPersonality
Definition: EHPersonalities.h:22
llvm::MachineMemOperand::MOVolatile
@ MOVolatile
The memory access is volatile.
Definition: MachineMemOperand.h:139
llvm::CallBase::paramHasAttr
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
Definition: Instructions.cpp:341
llvm::SwitchCG::BitTestBlock
Definition: SwitchLoweringUtils.h:208
llvm::MachineInstrBuilder::getInstr
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Definition: MachineInstrBuilder.h:89
llvm::StackProtectorDescriptor::resetPerFunctionState
void resetPerFunctionState()
Reset state that only changes when we switch functions.
Definition: CodeGenCommonISel.h:167
llvm::LLT::changeElementType
LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
Definition: LowLevelTypeImpl.h:174
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:726
llvm::MachineMemOperand::MOLoad
@ MOLoad
The memory access reads data.
Definition: MachineMemOperand.h:135
llvm::Value::stripPointerCasts
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:687
llvm::MachineIRBuilder::materializePtrAdd
Optional< MachineInstrBuilder > materializePtrAdd(Register &Res, Register Op0, const LLT ValueTy, uint64_t Value)
Materialize and insert Res = G_PTR_ADD Op0, (G_CONSTANT Value)
Definition: MachineIRBuilder.cpp:193
llvm::MachineIRBuilder::buildJumpTable
MachineInstrBuilder buildJumpTable(const LLT PtrTy, unsigned JTI)
Build and insert Res = G_JUMP_TABLE JTI.
Definition: MachineIRBuilder.cpp:159
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::MachineBasicBlock::addLiveIn
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
Definition: MachineBasicBlock.h:367
llvm::MachineBasicBlock::isPredecessor
bool isPredecessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a predecessor of this block.
Definition: MachineBasicBlock.cpp:909
llvm::SwitchCG::JumpTable
Definition: SwitchLoweringUtils.h:165
llvm::cl::Optional
@ Optional
Definition: CommandLine.h:119
llvm::StackProtectorDescriptor::getParentMBB
MachineBasicBlock * getParentMBB()
Definition: CodeGenCommonISel.h:169
llvm::MachineIRBuilder::buildFrameIndex
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
Definition: MachineIRBuilder.cpp:137
llvm::Triple::isOSWindows
bool isOSWindows() const
Tests whether the OS is Windows.
Definition: Triple.h:548
MBB
MachineBasicBlock & MBB
Definition: AArch64SLSHardening.cpp:74
llvm::TargetPassConfig::getCSEConfig
virtual std::unique_ptr< CSEConfigBase > getCSEConfig() const
Returns the CSEConfig object to use for the current optimization level.
Definition: TargetPassConfig.cpp:1538
llvm::computeValueLLTs
void computeValueLLTs(const DataLayout &DL, Type &Ty, SmallVectorImpl< LLT > &ValueTys, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
computeValueLLTs - Given an LLVM IR type, compute a sequence of LLTs that represent all the individua...
Definition: Analysis.cpp:132
llvm::TargetLoweringBase::getSSPStackGuardCheck
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
Definition: TargetLoweringBase.cpp:2002
j
return j(j<< 16)
llvm::MachineFunctionProperties::Property::FailedISel
@ FailedISel
llvm::APIntOps::umax
const APInt & umax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be unsigned.
Definition: APInt.h:2133
llvm::DataLayout::getIntPtrType
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
Definition: DataLayout.cpp:839
Constant.h
DEBUG_TYPE
#define DEBUG_TYPE
Definition: IRTranslator.cpp:88
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:83
llvm::minnum
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE minNum semantics.
Definition: APFloat.h:1296
llvm::commonAlignment
Align commonAlignment(Align A, Align B)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:211
llvm::TargetLoweringBase::getExceptionSelectorRegister
virtual Register getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
Definition: TargetLowering.h:1760
llvm::ConstantInt::getTrue
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:873
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:52
llvm::Constant::getNullValue
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:348
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition: MachineFunction.h:600
llvm::IndirectBrInst::getAddress
Value * getAddress()
Definition: Instructions.h:3689
llvm::TargetSubtargetInfo::getFrameLowering
virtual const TargetFrameLowering * getFrameLowering() const
Definition: TargetSubtargetInfo.h:93
llvm::IRTranslator::ID
static char ID
Definition: IRTranslator.h:65
llvm::MachineBasicBlock::sortUniqueLiveIns
void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
Definition: MachineBasicBlock.cpp:586
uint16_t
llvm::ConstrainedFPIntrinsic::isUnaryOp
bool isUnaryOp() const
Definition: IntrinsicInst.cpp:251
llvm::MachineFunction::getTarget
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Definition: MachineFunction.h:630
llvm::AtomicRMWInst::FSub
@ FSub
*p = old - v
Definition: Instructions.h:766
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:324
llvm::StructLayout::getElementOffset
uint64_t getElementOffset(unsigned Idx) const
Definition: DataLayout.h:643
llvm::ArrayRef::begin
iterator begin() const
Definition: ArrayRef.h:153
llvm::X86::FirstMacroFusionInstKind::Cmp
@ Cmp
llvm::MachineBasicBlock::insert
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
Definition: MachineBasicBlock.cpp:1317
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
MachineFrameInfo.h
llvm::Align::value
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
llvm::FPOpFusion::Strict
@ Strict
Definition: TargetOptions.h:39
llvm::MachineIRBuilder::buildUndef
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
Definition: MachineIRBuilder.cpp:582
llvm::LLVMContext::OB_deopt
@ OB_deopt
Definition: LLVMContext.h:90
llvm::CallBase::arg_size
unsigned arg_size() const
Definition: InstrTypes.h:1326
llvm::MachineIRBuilder::buildDirectDbgValue
MachineInstrBuilder buildDirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in Re...
Definition: MachineIRBuilder.cpp:52
llvm::ExtractValueInst
This instruction extracts a struct member or array element value from an aggregate value.
Definition: Instructions.h:2380
Success
#define Success
Definition: AArch64Disassembler.cpp:260
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition: Instructions.h:746
llvm::TargetRegisterInfo::getCustomEHPadPreservedMask
virtual const uint32_t * getCustomEHPadPreservedMask(const MachineFunction &MF) const
Return a register mask for the registers preserved by the unwinder, or nullptr if no custom mask is n...
Definition: TargetRegisterInfo.h:496
llvm::FunctionLoweringInfo::clear
void clear()
clear - Clear out all the function-specific state.
Definition: FunctionLoweringInfo.cpp:356
Casting.h
llvm::pdb::PDB_LocType::Slot
@ Slot
DiagnosticInfo.h
Function.h
llvm::sort
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1492
llvm::CallLowering::CallLoweringInfo
Definition: CallLowering.h:101
llvm::Type::getPointerTo
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:776
llvm::MachineRegisterInfo::addPhysRegsUsedFromRegMask
void addPhysRegsUsedFromRegMask(const uint32_t *RegMask)
addPhysRegsUsedFromRegMask - Mark any registers not in RegMask as used.
Definition: MachineRegisterInfo.h:879
powi
This is blocked on not handling X *X *X powi(X, 3)(see note above). The issue is that we end up getting t
llvm::SwitchCG::CaseBlock::CmpLHS
const Value * CmpLHS
Definition: SwitchLoweringUtils.h:127
llvm::log2
static double log2(double V)
Definition: AMDGPULibCalls.cpp:842
llvm::CallLowering::fallBackToDAGISel
virtual bool fallBackToDAGISel(const MachineFunction &MF) const
Definition: CallLowering.h:520
llvm::fp::ebIgnore
@ ebIgnore
This corresponds to "fpexcept.ignore".
Definition: FPEnv.h:32
llvm::TargetSubtargetInfo::getInlineAsmLowering
virtual const InlineAsmLowering * getInlineAsmLowering() const
Definition: TargetSubtargetInfo.h:102
llvm::ConstrainedFPIntrinsic::isTernaryOp
bool isTernaryOp() const
Definition: IntrinsicInst.cpp:262
MemRef
Definition: Lint.cpp:84
llvm::MachineFrameInfo::setStackProtectorIndex
void setStackProtectorIndex(int I)
Definition: MachineFrameInfo.h:357
llvm::GISelCSEAnalysisWrapper
Simple wrapper that does the following.
Definition: CSEInfo.h:202
llvm::Instruction::hasAllowReassoc
bool hasAllowReassoc() const
Determine whether the allow-reassociation flag is set.
Definition: Instruction.cpp:245
llvm::MachineBasicBlock::normalizeSuccProbs
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
Definition: MachineBasicBlock.h:654
llvm::UnivariateLinearPolyBase::isZero
bool isZero() const
Definition: TypeSize.h:229
llvm::CallBase::getCalledOperand
Value * getCalledOperand() const
Definition: InstrTypes.h:1376
llvm::Function::getPersonalityFn
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1823
llvm::MachineMemOperand::MOStore
@ MOStore
The memory access writes data.
Definition: MachineMemOperand.h:137
llvm::ReversePostOrderTraversal
Definition: PostOrderIterator.h:290
llvm::SwitchCG::CaseBlock::PredInfoPair::Pred
CmpInst::Predicate Pred
Definition: SwitchLoweringUtils.h:112
llvm::IndirectBrInst
Indirect Branch Instruction.
Definition: Instructions.h:3614
llvm::TargetLoweringBase::fallBackToDAGISel
virtual bool fallBackToDAGISel(const Instruction &Inst) const
Definition: TargetLowering.h:545
llvm::makeArrayRef
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:476
llvm::IRTranslator::IRTranslator
IRTranslator(CodeGenOpt::Level OptLevel=CodeGenOpt::None)
Definition: IRTranslator.cpp:125
CodeGen.h
llvm::Instruction::BinaryOps
BinaryOps
Definition: Instruction.h:785
llvm::BasicBlock::back
const Instruction & back() const
Definition: BasicBlock.h:310
llvm::Type::getVoidTy
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:224
llvm::StackProtectorDescriptor::shouldEmitFunctionBasedCheckStackProtector
bool shouldEmitFunctionBasedCheckStackProtector() const
Definition: CodeGenCommonISel.h:125
llvm::MachineFrameInfo
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
Definition: MachineFrameInfo.h:107
Instructions.h
PostOrderIterator.h
llvm::FunctionLoweringInfo::MF
MachineFunction * MF
Definition: FunctionLoweringInfo.h:55
llvm::MachineInstrBuilder::addSym
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
Definition: MachineInstrBuilder.h:267
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition: User.h:191
llvm::TargetSubtargetInfo::getTargetLowering
virtual const TargetLowering * getTargetLowering() const
Definition: TargetSubtargetInfo.h:96
llvm::MachineIRBuilder::buildIntrinsic
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects)
Build and insert either a G_INTRINSIC (if HasSideEffects is false) or G_INTRINSIC_W_SIDE_EFFECTS inst...
Definition: MachineIRBuilder.cpp:713
llvm::BranchProbability::normalizeProbabilities
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Definition: BranchProbability.h:205
SmallVector.h
llvm::PatternMatch::m_Specific
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
Definition: PatternMatch.h:802
llvm::MachinePointerInfo::getFixedStack
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Definition: MachineOperand.cpp:1008
llvm::MachineBasicBlock::begin
iterator begin()
Definition: MachineBasicBlock.h:268
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:370
llvm::MachineRegisterInfo::getType
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
Definition: MachineRegisterInfo.h:732
User.h
MachineInstrBuilder.h
llvm::IRTranslator::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
Definition: IRTranslator.cpp:166
llvm::GISelObserverWrapper::addObserver
void addObserver(GISelChangeObserver *O)
Definition: GISelChangeObserver.h:75
llvm::MachineInstr::NoUWrap
@ NoUWrap
Definition: MachineInstr.h:102
llvm::SwitchCG::JumpTableHeader::SValue
const Value * SValue
Definition: SwitchLoweringUtils.h:183
llvm::getMVTForLLT
MVT getMVTForLLT(LLT Ty)
Get a rough equivalent of an MVT for a given LLT.
Definition: LowLevelType.cpp:46
TargetIntrinsicInfo.h
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:743
llvm::DbgValueInst::getValue
Value * getValue(unsigned OpIdx=0) const
Definition: IntrinsicInst.h:351
llvm::fp::ExceptionBehavior
ExceptionBehavior
Exception behavior used for floating point operations.
Definition: FPEnv.h:31
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1328
llvm::TargetLowering::useLoadStackGuardNode
virtual bool useLoadStackGuardNode() const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
Definition: TargetLowering.h:4640
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:94
llvm::SwitchCG::CaseBlock::TrueBB
MachineBasicBlock * TrueBB
Definition: SwitchLoweringUtils.h:130
llvm::TargetMachine::getTargetTriple
const Triple & getTargetTriple() const
Definition: TargetMachine.h:128
llvm::MachineIRBuilder::buildSub
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_SUB Op0, Op1.
Definition: MachineIRBuilder.h:1405
llvm::PHINode::getIncomingBlock
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Definition: Instructions.h:2749
llvm::ArrayRef::size
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
llvm::max
Align max(MaybeAlign Lhs, Align Rhs)
Definition: Alignment.h:340
llvm::AtomicRMWInst::FAdd
@ FAdd
*p = old + v
Definition: Instructions.h:763
llvm::MachineBasicBlock::empty
bool empty() const
Definition: MachineBasicBlock.h:240
llvm::BasicBlock::getFirstNonPHIOrDbg
const Instruction * getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
Definition: BasicBlock.cpp:223
getConstrainedOpcode
static unsigned getConstrainedOpcode(Intrinsic::ID ID)
Definition: IRTranslator.cpp:1797
llvm::pdb::PDB_DataKind::Global
@ Global
llvm::PHINode
Definition: Instructions.h:2633
ScopeExit.h
llvm::StackProtectorDescriptor::initialize
void initialize(const BasicBlock *BB, MachineBasicBlock *MBB, bool FunctionBasedInstrumentation)
Initialize the stack protector descriptor structure for a new basic block.
Definition: CodeGenCommonISel.h:131
llvm::StringRef::data
const LLVM_NODISCARD char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:149
llvm::IRTranslator
Definition: IRTranslator.h:63
llvm::MachineInstrBuilder::addMetadata
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
Definition: MachineInstrBuilder.h:236
MachineMemOperand.h
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:43
MachineOperand.h
llvm::SwitchCG::CC_BitTests
@ CC_BitTests
A cluster of cases suitable for bit test lowering.
Definition: SwitchLoweringUtils.h:38
llvm::GISelObserverWrapper
Simple wrapper observer that takes several observers, and calls each one for each event.
Definition: GISelChangeObserver.h:66
isValInBlock
static bool isValInBlock(const Value *V, const BasicBlock *BB)
Definition: IRTranslator.cpp:401
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1161
DerivedTypes.h
TM
const char LLVMTargetMachineRef TM
Definition: PassBuilderBindings.cpp:47
llvm::DataLayout::getPointerSizeInBits
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Definition: DataLayout.h:409
checkForMustTailInVarArgFn
static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB)
Returns true if a BasicBlock BB within a variadic function contains a variadic musttail call.
Definition: IRTranslator.cpp:3294
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1475
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::MachineFrameInfo::getStackProtectorIndex
int getStackProtectorIndex() const
Return the index for the stack protector object.
Definition: MachineFrameInfo.h:356
llvm::MIPatternMatch::m_Not
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
Definition: MIPatternMatch.h:591
llvm::Type::isAggregateType
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition: Type.h:256
llvm::MachineIRBuilder::buildFreeze
MachineInstrBuilder buildFreeze(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_FREEZE Src.
Definition: MachineIRBuilder.h:1363
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition: PassAnalysisSupport.h:75
LLVMContext.h
llvm::SwitchInst
Multiway switch.
Definition: Instructions.h:3212
llvm::OptimizedStructLayoutField
A field in a structure.
Definition: OptimizedStructLayout.h:45
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::AMDGPU::HSAMD::Kernel::Key::Args
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Definition: AMDGPUMetadata.h:389
llvm::Type::TypeID
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
llvm::AllocaInst
an instruction to allocate memory on the stack
Definition: Instructions.h:62
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::MachineFunction::addLandingPad
MCSymbol * addLandingPad(MachineBasicBlock *LandingPad)
Add a new panding pad, and extract the exception handling information from the landingpad instruction...
Definition: MachineFunction.cpp:723
GISelChangeObserver.h
llvm::cl::desc
Definition: CommandLine.h:412
llvm::MachineIRBuilder::buildStore
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
Definition: MachineIRBuilder.cpp:387
llvm::TargetLoweringBase::getValueType
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
Definition: TargetLowering.h:1417
llvm::BranchInst
Conditional or Unconditional Branch instruction.
Definition: Instructions.h:3068
llvm::AtomicRMWInst::UMax
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:758
raw_ostream.h
llvm::SI::KernelInputOffsets::Offsets
Offsets
Offsets in bytes from the start of the input buffer.
Definition: SIInstrInfo.h:1264
llvm::MachineIRBuilder::buildAdd
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_ADD Op0, Op1.
Definition: MachineIRBuilder.h:1388
MachineFunction.h
llvm::InsertValueInst
This instruction inserts a struct field of array element value into an aggregate value.
Definition: Instructions.h:2491
llvm::CmpInst::FCMP_TRUE
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:737
llvm::MachineInstrBundleIterator< MachineInstr >
llvm::AttributeList::getFnAttr
Attribute getFnAttr(Attribute::AttrKind Kind) const
Return the attribute object that exists for the function.
Definition: Attributes.h:781
llvm::LLT::scalar
static LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition: LowLevelTypeImpl.h:43
Value.h
llvm::abs
APFloat abs(APFloat X)
Returns the absolute value of the argument.
Definition: APFloat.h:1282
llvm::findSplitPointForStackProtector
MachineBasicBlock::iterator findSplitPointForStackProtector(MachineBasicBlock *BB, const TargetInstrInfo &TII)
Find the split point at which to splice the end of BB into its success stack protector check machine ...
Definition: CodeGenCommonISel.cpp:124
llvm::raw_string_ostream::str
std::string & str()
Flushes the stream contents to the target string and returns the string's reference.
Definition: raw_ostream.h:643
llvm::MachineMemOperand::MONone
@ MONone
Definition: MachineMemOperand.h:133
llvm::MIPatternMatch::m_OneUse
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
Definition: MIPatternMatch.h:44
llvm::TargetLoweringBase::getLibcallCallingConv
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
Definition: TargetLowering.h:2887
InitializePasses.h
llvm::FunctionType::getReturnType
Type * getReturnType() const
Definition: DerivedTypes.h:124
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::AtomicCmpXchgInst
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:521
TargetRegisterInfo.h
Debug.h
llvm::MachineFunction::setVariableDbgInfo
void setVariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr, int Slot, const DILocation *Loc)
Collect information used to emit debugging information of a variable.
Definition: MachineFunction.h:1177
llvm::MachineBasicBlock::end
iterator end()
Definition: MachineBasicBlock.h:270
SubReg
unsigned SubReg
Definition: AArch64AdvSIMDScalarPass.cpp:104
llvm::ArrayRef::end
iterator end() const
Definition: ArrayRef.h:154
llvm::APIntOps::smax
const APInt & smax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be signed.
Definition: APInt.h:2123
llvm::PatternMatch::m_LogicalAnd
LogicalOp_match< LHS, RHS, Instruction::And > m_LogicalAnd(const LHS &L, const RHS &R)
Matches L && R either in the form of L & R or L ? R : false.
Definition: PatternMatch.h:2504
llvm::SwiftErrorValueTracking::getFunctionArg
const Value * getFunctionArg() const
Get the (unique) function argument that was marked swifterror, or nullptr if this function has no swi...
Definition: SwiftErrorValueTracking.h:74
llvm::MachineRegisterInfo::setRegClass
void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
Definition: MachineRegisterInfo.cpp:58
llvm::BranchInst::getSuccessor
BasicBlock * getSuccessor(unsigned i) const
Definition: Instructions.h:3161
llvm::CallBase::args
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition: InstrTypes.h:1319
llvm::BasicBlock::const_iterator
InstListType::const_iterator const_iterator
Definition: BasicBlock.h:91
llvm::SwiftErrorValueTracking::setCurrentVReg
void setCurrentVReg(const MachineBasicBlock *MBB, const Value *, Register)
Set the swifterror virtual register in the VRegDefMap for this basic block.
Definition: SwiftErrorValueTracking.cpp:45
llvm::sampleprof::Base
@ Base
Definition: Discriminator.h:58
llvm::MachineIRBuilder::buildCast
MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src)
Build and insert an appropriate cast between two registers of equal size.
Definition: MachineIRBuilder.cpp:496
llvm::FunctionType
Class to represent function types.
Definition: DerivedTypes.h:103
SmallSet.h
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:364
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:37
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:754
llvm::DataLayout::getTypeAllocSize
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:503
llvm::LLT
Definition: LowLevelTypeImpl.h:40
llvm::CallLowering::lowerCall
virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info) const
This hook must be implemented to lower the given call instruction, including argument and return valu...
Definition: CallLowering.h:546