LLVM  14.0.0git
SelectionDAGISel.cpp
Go to the documentation of this file.
1 //===- SelectionDAGISel.cpp - Implement the SelectionDAGISel class --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements the SelectionDAGISel class.
10 //
11 //===----------------------------------------------------------------------===//
12 
14 #include "ScheduleDAGSDNodes.h"
15 #include "SelectionDAGBuilder.h"
16 #include "llvm/ADT/APInt.h"
17 #include "llvm/ADT/DenseMap.h"
18 #include "llvm/ADT/None.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallPtrSet.h"
22 #include "llvm/ADT/SmallSet.h"
23 #include "llvm/ADT/SmallVector.h"
24 #include "llvm/ADT/Statistic.h"
25 #include "llvm/ADT/StringRef.h"
28 #include "llvm/Analysis/CFG.h"
37 #include "llvm/CodeGen/FastISel.h"
62 #include "llvm/IR/BasicBlock.h"
63 #include "llvm/IR/Constants.h"
64 #include "llvm/IR/DataLayout.h"
66 #include "llvm/IR/DebugLoc.h"
67 #include "llvm/IR/DiagnosticInfo.h"
68 #include "llvm/IR/Dominators.h"
69 #include "llvm/IR/Function.h"
70 #include "llvm/IR/InlineAsm.h"
71 #include "llvm/IR/InstIterator.h"
72 #include "llvm/IR/InstrTypes.h"
73 #include "llvm/IR/Instruction.h"
74 #include "llvm/IR/Instructions.h"
75 #include "llvm/IR/IntrinsicInst.h"
76 #include "llvm/IR/Intrinsics.h"
77 #include "llvm/IR/IntrinsicsWebAssembly.h"
78 #include "llvm/IR/Metadata.h"
79 #include "llvm/IR/Statepoint.h"
80 #include "llvm/IR/Type.h"
81 #include "llvm/IR/User.h"
82 #include "llvm/IR/Value.h"
83 #include "llvm/InitializePasses.h"
84 #include "llvm/MC/MCInstrDesc.h"
85 #include "llvm/MC/MCRegisterInfo.h"
86 #include "llvm/Pass.h"
88 #include "llvm/Support/Casting.h"
89 #include "llvm/Support/CodeGen.h"
91 #include "llvm/Support/Compiler.h"
92 #include "llvm/Support/Debug.h"
94 #include "llvm/Support/KnownBits.h"
96 #include "llvm/Support/Timer.h"
102 #include <algorithm>
103 #include <cassert>
104 #include <cstdint>
105 #include <iterator>
106 #include <limits>
107 #include <memory>
108 #include <string>
109 #include <utility>
110 #include <vector>
111 
112 using namespace llvm;
113 
114 #define DEBUG_TYPE "isel"
115 
116 STATISTIC(NumFastIselFailures, "Number of instructions fast isel failed on");
117 STATISTIC(NumFastIselSuccess, "Number of instructions fast isel selected");
118 STATISTIC(NumFastIselBlocks, "Number of blocks selected entirely by fast isel");
119 STATISTIC(NumDAGBlocks, "Number of blocks selected using DAG");
120 STATISTIC(NumDAGIselRetries,"Number of times dag isel has to try another path");
121 STATISTIC(NumEntryBlocks, "Number of entry blocks encountered");
122 STATISTIC(NumFastIselFailLowerArguments,
123  "Number of entry blocks where fast isel failed to lower arguments");
124 
126  "fast-isel-abort", cl::Hidden,
127  cl::desc("Enable abort calls when \"fast\" instruction selection "
128  "fails to lower an instruction: 0 disable the abort, 1 will "
129  "abort but for args, calls and terminators, 2 will also "
130  "abort for argument lowering, and 3 will never fallback "
131  "to SelectionDAG."));
132 
134  "fast-isel-report-on-fallback", cl::Hidden,
135  cl::desc("Emit a diagnostic when \"fast\" instruction selection "
136  "falls back to SelectionDAG."));
137 
138 static cl::opt<bool>
139 UseMBPI("use-mbpi",
140  cl::desc("use Machine Branch Probability Info"),
141  cl::init(true), cl::Hidden);
142 
143 #ifndef NDEBUG
145 FilterDAGBasicBlockName("filter-view-dags", cl::Hidden,
146  cl::desc("Only display the basic block whose name "
147  "matches this for all view-*-dags options"));
148 static cl::opt<bool>
149 ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden,
150  cl::desc("Pop up a window to show dags before the first "
151  "dag combine pass"));
152 static cl::opt<bool>
153 ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden,
154  cl::desc("Pop up a window to show dags before legalize types"));
155 static cl::opt<bool>
156  ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden,
157  cl::desc("Pop up a window to show dags before the post "
158  "legalize types dag combine pass"));
159 static cl::opt<bool>
160  ViewLegalizeDAGs("view-legalize-dags", cl::Hidden,
161  cl::desc("Pop up a window to show dags before legalize"));
162 static cl::opt<bool>
163 ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden,
164  cl::desc("Pop up a window to show dags before the second "
165  "dag combine pass"));
166 static cl::opt<bool>
167 ViewISelDAGs("view-isel-dags", cl::Hidden,
168  cl::desc("Pop up a window to show isel dags as they are selected"));
169 static cl::opt<bool>
170 ViewSchedDAGs("view-sched-dags", cl::Hidden,
171  cl::desc("Pop up a window to show sched dags as they are processed"));
172 static cl::opt<bool>
173 ViewSUnitDAGs("view-sunit-dags", cl::Hidden,
174  cl::desc("Pop up a window to show SUnit dags after they are processed"));
175 #else
176 static const bool ViewDAGCombine1 = false, ViewLegalizeTypesDAGs = false,
177  ViewDAGCombineLT = false, ViewLegalizeDAGs = false,
178  ViewDAGCombine2 = false, ViewISelDAGs = false,
179  ViewSchedDAGs = false, ViewSUnitDAGs = false;
180 #endif
181 
182 //===---------------------------------------------------------------------===//
183 ///
184 /// RegisterScheduler class - Track the registration of instruction schedulers.
185 ///
186 //===---------------------------------------------------------------------===//
189 
190 //===---------------------------------------------------------------------===//
191 ///
192 /// ISHeuristic command line option for instruction schedulers.
193 ///
194 //===---------------------------------------------------------------------===//
197 ISHeuristic("pre-RA-sched",
199  cl::desc("Instruction schedulers available (before register"
200  " allocation):"));
201 
202 static RegisterScheduler
203 defaultListDAGScheduler("default", "Best scheduler for the target",
205 
206 namespace llvm {
207 
208  //===--------------------------------------------------------------------===//
209  /// This class is used by SelectionDAGISel to temporarily override
210  /// the optimization level on a per-function basis.
212  SelectionDAGISel &IS;
213  CodeGenOpt::Level SavedOptLevel;
214  bool SavedFastISel;
215 
216  public:
218  CodeGenOpt::Level NewOptLevel) : IS(ISel) {
219  SavedOptLevel = IS.OptLevel;
220  SavedFastISel = IS.TM.Options.EnableFastISel;
221  if (NewOptLevel == SavedOptLevel)
222  return;
223  IS.OptLevel = NewOptLevel;
224  IS.TM.setOptLevel(NewOptLevel);
225  LLVM_DEBUG(dbgs() << "\nChanging optimization level for Function "
226  << IS.MF->getFunction().getName() << "\n");
227  LLVM_DEBUG(dbgs() << "\tBefore: -O" << SavedOptLevel << " ; After: -O"
228  << NewOptLevel << "\n");
229  if (NewOptLevel == CodeGenOpt::None) {
231  LLVM_DEBUG(
232  dbgs() << "\tFastISel is "
233  << (IS.TM.Options.EnableFastISel ? "enabled" : "disabled")
234  << "\n");
235  }
236  }
237 
239  if (IS.OptLevel == SavedOptLevel)
240  return;
241  LLVM_DEBUG(dbgs() << "\nRestoring optimization level for Function "
242  << IS.MF->getFunction().getName() << "\n");
243  LLVM_DEBUG(dbgs() << "\tBefore: -O" << IS.OptLevel << " ; After: -O"
244  << SavedOptLevel << "\n");
245  IS.OptLevel = SavedOptLevel;
246  IS.TM.setOptLevel(SavedOptLevel);
247  IS.TM.setFastISel(SavedFastISel);
248  }
249  };
250 
251  //===--------------------------------------------------------------------===//
252  /// createDefaultScheduler - This creates an instruction scheduler appropriate
253  /// for the target.
255  CodeGenOpt::Level OptLevel) {
256  const TargetLowering *TLI = IS->TLI;
257  const TargetSubtargetInfo &ST = IS->MF->getSubtarget();
258 
259  // Try first to see if the Target has its own way of selecting a scheduler
260  if (auto *SchedulerCtor = ST.getDAGScheduler(OptLevel)) {
261  return SchedulerCtor(IS, OptLevel);
262  }
263 
264  if (OptLevel == CodeGenOpt::None ||
265  (ST.enableMachineScheduler() && ST.enableMachineSchedDefaultSched()) ||
267  return createSourceListDAGScheduler(IS, OptLevel);
269  return createBURRListDAGScheduler(IS, OptLevel);
271  return createHybridListDAGScheduler(IS, OptLevel);
272  if (TLI->getSchedulingPreference() == Sched::VLIW)
273  return createVLIWDAGScheduler(IS, OptLevel);
274  if (TLI->getSchedulingPreference() == Sched::Fast)
275  return createFastDAGScheduler(IS, OptLevel);
277  return createDAGLinearizer(IS, OptLevel);
279  "Unknown sched type!");
280  return createILPListDAGScheduler(IS, OptLevel);
281  }
282 
283 } // end namespace llvm
284 
285 // EmitInstrWithCustomInserter - This method should be implemented by targets
286 // that mark instructions with the 'usesCustomInserter' flag. These
287 // instructions are special in various ways, which require special support to
288 // insert. The specified MachineInstr is created but not inserted into any
289 // basic blocks, and this method is called to expand it into a sequence of
290 // instructions, potentially also creating new basic blocks and control flow.
291 // When new basic blocks are inserted and the edges from MBB to its successors
292 // are modified, the method should insert pairs of <OldSucc, NewSucc> into the
293 // DenseMap.
296  MachineBasicBlock *MBB) const {
297 #ifndef NDEBUG
298  dbgs() << "If a target marks an instruction with "
299  "'usesCustomInserter', it must implement "
300  "TargetLowering::EmitInstrWithCustomInserter!";
301 #endif
302  llvm_unreachable(nullptr);
303 }
304 
306  SDNode *Node) const {
307  assert(!MI.hasPostISelHook() &&
308  "If a target marks an instruction with 'hasPostISelHook', "
309  "it must implement TargetLowering::AdjustInstrPostInstrSelection!");
310 }
311 
312 //===----------------------------------------------------------------------===//
313 // SelectionDAGISel code
314 //===----------------------------------------------------------------------===//
315 
317  : MachineFunctionPass(ID), TM(tm), FuncInfo(new FunctionLoweringInfo()),
318  SwiftError(new SwiftErrorValueTracking()),
319  CurDAG(new SelectionDAG(tm, OL)),
320  SDB(std::make_unique<SelectionDAGBuilder>(*CurDAG, *FuncInfo, *SwiftError,
321  OL)),
322  AA(), GFI(), OptLevel(OL), DAGSize(0) {
328 }
329 
331  delete CurDAG;
332  delete SwiftError;
333 }
334 
336  if (OptLevel != CodeGenOpt::None)
346  if (OptLevel != CodeGenOpt::None)
349 }
350 
351 /// SplitCriticalSideEffectEdges - Look for critical edges with a PHI value that
352 /// may trap on it. In this case we have to split the edge so that the path
353 /// through the predecessor block that doesn't go to the phi block doesn't
354 /// execute the possibly trapping instruction. If available, we pass domtree
355 /// and loop info to be updated when we split critical edges. This is because
356 /// SelectionDAGISel preserves these analyses.
357 /// This is required for correctness, so it must be done at -O0.
358 ///
360  LoopInfo *LI) {
361  // Loop for blocks with phi nodes.
362  for (BasicBlock &BB : Fn) {
363  PHINode *PN = dyn_cast<PHINode>(BB.begin());
364  if (!PN) continue;
365 
366  ReprocessBlock:
367  // For each block with a PHI node, check to see if any of the input values
368  // are potentially trapping constant expressions. Constant expressions are
369  // the only potentially trapping value that can occur as the argument to a
370  // PHI.
371  for (BasicBlock::iterator I = BB.begin(); (PN = dyn_cast<PHINode>(I)); ++I)
372  for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
373  ConstantExpr *CE = dyn_cast<ConstantExpr>(PN->getIncomingValue(i));
374  if (!CE || !CE->canTrap()) continue;
375 
376  // The only case we have to worry about is when the edge is critical.
377  // Since this block has a PHI Node, we assume it has multiple input
378  // edges: check to see if the pred has multiple successors.
379  BasicBlock *Pred = PN->getIncomingBlock(i);
380  if (Pred->getTerminator()->getNumSuccessors() == 1)
381  continue;
382 
383  // Okay, we have to split this edge.
385  Pred->getTerminator(), GetSuccessorNumber(Pred, &BB),
386  CriticalEdgeSplittingOptions(DT, LI).setMergeIdenticalEdges());
387  goto ReprocessBlock;
388  }
389  }
390 }
391 
392 static void computeUsesMSVCFloatingPoint(const Triple &TT, const Function &F,
393  MachineModuleInfo &MMI) {
394  // Only needed for MSVC
395  if (!TT.isWindowsMSVCEnvironment())
396  return;
397 
398  // If it's already set, nothing to do.
399  if (MMI.usesMSVCFloatingPoint())
400  return;
401 
402  for (const Instruction &I : instructions(F)) {
403  if (I.getType()->isFPOrFPVectorTy()) {
404  MMI.setUsesMSVCFloatingPoint(true);
405  return;
406  }
407  for (const auto &Op : I.operands()) {
408  if (Op->getType()->isFPOrFPVectorTy()) {
409  MMI.setUsesMSVCFloatingPoint(true);
410  return;
411  }
412  }
413  }
414 }
415 
417  // If we already selected that function, we do not need to run SDISel.
418  if (mf.getProperties().hasProperty(
420  return false;
421  // Do some sanity-checking on the command-line options.
423  "-fast-isel-abort > 0 requires -fast-isel");
424 
425  const Function &Fn = mf.getFunction();
426  MF = &mf;
427 
428  // Reset the target options before resetting the optimization
429  // level below.
430  // FIXME: This is a horrible hack and should be processed via
431  // codegen looking at the optimization level explicitly when
432  // it wants to look at it.
434  // Reset OptLevel to None for optnone functions.
435  CodeGenOpt::Level NewOptLevel = OptLevel;
436  if (OptLevel != CodeGenOpt::None && skipFunction(Fn))
437  NewOptLevel = CodeGenOpt::None;
438  OptLevelChanger OLC(*this, NewOptLevel);
439 
442  RegInfo = &MF->getRegInfo();
443  LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(Fn);
444  GFI = Fn.hasGC() ? &getAnalysis<GCModuleInfo>().getFunctionInfo(Fn) : nullptr;
445  ORE = std::make_unique<OptimizationRemarkEmitter>(&Fn);
446  auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
447  DominatorTree *DT = DTWP ? &DTWP->getDomTree() : nullptr;
448  auto *LIWP = getAnalysisIfAvailable<LoopInfoWrapperPass>();
449  LoopInfo *LI = LIWP ? &LIWP->getLoopInfo() : nullptr;
450  auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
451  BlockFrequencyInfo *BFI = nullptr;
452  if (PSI && PSI->hasProfileSummary() && OptLevel != CodeGenOpt::None)
453  BFI = &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI();
454 
455  LLVM_DEBUG(dbgs() << "\n\n\n=== " << Fn.getName() << "\n");
456 
457  SplitCriticalSideEffectEdges(const_cast<Function &>(Fn), DT, LI);
458 
459  CurDAG->init(*MF, *ORE, this, LibInfo,
460  getAnalysisIfAvailable<LegacyDivergenceAnalysis>(), PSI, BFI);
461  FuncInfo->set(Fn, *MF, CurDAG);
463 
464  // Now get the optional analyzes if we want to.
465  // This is based on the possibly changed OptLevel (after optnone is taken
466  // into account). That's unfortunate but OK because it just means we won't
467  // ask for passes that have been required anyway.
468 
470  FuncInfo->BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
471  else
472  FuncInfo->BPI = nullptr;
473 
474  if (OptLevel != CodeGenOpt::None)
475  AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
476  else
477  AA = nullptr;
478 
479  SDB->init(GFI, AA, LibInfo);
480 
481  MF->setHasInlineAsm(false);
482 
483  FuncInfo->SplitCSR = false;
484 
485  // We split CSR if the target supports it for the given function
486  // and the function has only return exits.
488  FuncInfo->SplitCSR = true;
489 
490  // Collect all the return blocks.
491  for (const BasicBlock &BB : Fn) {
492  if (!succ_empty(&BB))
493  continue;
494 
495  const Instruction *Term = BB.getTerminator();
496  if (isa<UnreachableInst>(Term) || isa<ReturnInst>(Term))
497  continue;
498 
499  // Bail out if the exit block is not Return nor Unreachable.
500  FuncInfo->SplitCSR = false;
501  break;
502  }
503  }
504 
505  MachineBasicBlock *EntryMBB = &MF->front();
506  if (FuncInfo->SplitCSR)
507  // This performs initialization so lowering for SplitCSR will be correct.
508  TLI->initializeSplitCSR(EntryMBB);
509 
510  SelectAllBasicBlocks(Fn);
512  DiagnosticInfoISelFallback DiagFallback(Fn);
513  Fn.getContext().diagnose(DiagFallback);
514  }
515 
516  // Replace forward-declared registers with the registers containing
517  // the desired value.
518  // Note: it is important that this happens **before** the call to
519  // EmitLiveInCopies, since implementations can skip copies of unused
520  // registers. If we don't apply the reg fixups before, some registers may
521  // appear as unused and will be skipped, resulting in bad MI.
523  for (DenseMap<Register, Register>::iterator I = FuncInfo->RegFixups.begin(),
524  E = FuncInfo->RegFixups.end();
525  I != E; ++I) {
526  Register From = I->first;
527  Register To = I->second;
528  // If To is also scheduled to be replaced, find what its ultimate
529  // replacement is.
530  while (true) {
531  DenseMap<Register, Register>::iterator J = FuncInfo->RegFixups.find(To);
532  if (J == E)
533  break;
534  To = J->second;
535  }
536  // Make sure the new register has a sufficiently constrained register class.
539  // Replace it.
540 
541  // Replacing one register with another won't touch the kill flags.
542  // We need to conservatively clear the kill flags as a kill on the old
543  // register might dominate existing uses of the new register.
544  if (!MRI.use_empty(To))
546  MRI.replaceRegWith(From, To);
547  }
548 
549  // If the first basic block in the function has live ins that need to be
550  // copied into vregs, emit the copies into the top of the block before
551  // emitting the code for the block.
553  RegInfo->EmitLiveInCopies(EntryMBB, TRI, *TII);
554 
555  // Insert copies in the entry block and the return blocks.
556  if (FuncInfo->SplitCSR) {
558  // Collect all the return blocks.
559  for (MachineBasicBlock &MBB : mf) {
560  if (!MBB.succ_empty())
561  continue;
562 
564  if (Term != MBB.end() && Term->isReturn()) {
565  Returns.push_back(&MBB);
566  continue;
567  }
568  }
569  TLI->insertCopiesSplitCSR(EntryMBB, Returns);
570  }
571 
573  if (!FuncInfo->ArgDbgValues.empty())
574  for (std::pair<unsigned, unsigned> LI : RegInfo->liveins())
575  if (LI.second)
576  LiveInMap.insert(LI);
577 
578  // Insert DBG_VALUE instructions for function arguments to the entry block.
579  bool InstrRef = MF->useDebugInstrRef();
580  for (unsigned i = 0, e = FuncInfo->ArgDbgValues.size(); i != e; ++i) {
581  MachineInstr *MI = FuncInfo->ArgDbgValues[e - i - 1];
582  assert(MI->getOpcode() != TargetOpcode::DBG_VALUE_LIST &&
583  "Function parameters should not be described by DBG_VALUE_LIST.");
584  bool hasFI = MI->getOperand(0).isFI();
585  Register Reg =
586  hasFI ? TRI.getFrameRegister(*MF) : MI->getOperand(0).getReg();
588  EntryMBB->insert(EntryMBB->begin(), MI);
589  else {
591  if (Def) {
592  MachineBasicBlock::iterator InsertPos = Def;
593  // FIXME: VR def may not be in entry block.
594  Def->getParent()->insert(std::next(InsertPos), MI);
595  } else
596  LLVM_DEBUG(dbgs() << "Dropping debug info for dead vreg"
597  << Register::virtReg2Index(Reg) << "\n");
598  }
599 
600  // Don't try and extend through copies in instruction referencing mode.
601  if (InstrRef)
602  continue;
603 
604  // If Reg is live-in then update debug info to track its copy in a vreg.
606  if (LDI != LiveInMap.end()) {
607  assert(!hasFI && "There's no handling of frame pointer updating here yet "
608  "- add if needed");
609  MachineInstr *Def = RegInfo->getVRegDef(LDI->second);
610  MachineBasicBlock::iterator InsertPos = Def;
611  const MDNode *Variable = MI->getDebugVariable();
612  const MDNode *Expr = MI->getDebugExpression();
613  DebugLoc DL = MI->getDebugLoc();
614  bool IsIndirect = MI->isIndirectDebugValue();
615  if (IsIndirect)
616  assert(MI->getOperand(1).getImm() == 0 &&
617  "DBG_VALUE with nonzero offset");
618  assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) &&
619  "Expected inlined-at fields to agree");
620  assert(MI->getOpcode() != TargetOpcode::DBG_VALUE_LIST &&
621  "Didn't expect to see a DBG_VALUE_LIST here");
622  // Def is never a terminator here, so it is ok to increment InsertPos.
623  BuildMI(*EntryMBB, ++InsertPos, DL, TII->get(TargetOpcode::DBG_VALUE),
624  IsIndirect, LDI->second, Variable, Expr);
625 
626  // If this vreg is directly copied into an exported register then
627  // that COPY instructions also need DBG_VALUE, if it is the only
628  // user of LDI->second.
629  MachineInstr *CopyUseMI = nullptr;
631  UI = RegInfo->use_instr_begin(LDI->second),
632  E = RegInfo->use_instr_end(); UI != E; ) {
633  MachineInstr *UseMI = &*(UI++);
634  if (UseMI->isDebugValue()) continue;
635  if (UseMI->isCopy() && !CopyUseMI && UseMI->getParent() == EntryMBB) {
636  CopyUseMI = UseMI; continue;
637  }
638  // Otherwise this is another use or second copy use.
639  CopyUseMI = nullptr; break;
640  }
641  if (CopyUseMI &&
642  TRI.getRegSizeInBits(LDI->second, MRI) ==
643  TRI.getRegSizeInBits(CopyUseMI->getOperand(0).getReg(), MRI)) {
644  // Use MI's debug location, which describes where Variable was
645  // declared, rather than whatever is attached to CopyUseMI.
646  MachineInstr *NewMI =
647  BuildMI(*MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
648  CopyUseMI->getOperand(0).getReg(), Variable, Expr);
649  MachineBasicBlock::iterator Pos = CopyUseMI;
650  EntryMBB->insertAfter(Pos, NewMI);
651  }
652  }
653  }
654 
655  // For debug-info, in instruction referencing mode, we need to perform some
656  // post-isel maintenence.
658 
659  // Determine if there are any calls in this machine function.
660  MachineFrameInfo &MFI = MF->getFrameInfo();
661  for (const auto &MBB : *MF) {
662  if (MFI.hasCalls() && MF->hasInlineAsm())
663  break;
664 
665  for (const auto &MI : MBB) {
666  const MCInstrDesc &MCID = TII->get(MI.getOpcode());
667  if ((MCID.isCall() && !MCID.isReturn()) ||
668  MI.isStackAligningInlineAsm()) {
669  MFI.setHasCalls(true);
670  }
671  if (MI.isInlineAsm()) {
672  MF->setHasInlineAsm(true);
673  }
674  }
675  }
676 
677  // Determine if there is a call to setjmp in the machine function.
679 
680  // Determine if floating point is used for msvc
682 
683  // Release function-specific state. SDB and CurDAG are already cleared
684  // at this point.
685  FuncInfo->clear();
686 
687  LLVM_DEBUG(dbgs() << "*** MachineFunction at end of ISel ***\n");
688  LLVM_DEBUG(MF->print(dbgs()));
689 
690  return true;
691 }
692 
696  bool ShouldAbort) {
697  // Print the function name explicitly if we don't have a debug location (which
698  // makes the diagnostic less useful) or if we're going to emit a raw error.
699  if (!R.getLocation().isValid() || ShouldAbort)
700  R << (" (in function: " + MF.getName() + ")").str();
701 
702  if (ShouldAbort)
703  report_fatal_error(Twine(R.getMsg()));
704 
705  ORE.emit(R);
706 }
707 
708 void SelectionDAGISel::SelectBasicBlock(BasicBlock::const_iterator Begin,
710  bool &HadTailCall) {
711  // Allow creating illegal types during DAG building for the basic block.
713 
714  // Lower the instructions. If a call is emitted as a tail call, cease emitting
715  // nodes for this block.
716  for (BasicBlock::const_iterator I = Begin; I != End && !SDB->HasTailCall; ++I) {
717  if (!ElidedArgCopyInstrs.count(&*I))
718  SDB->visit(*I);
719  }
720 
721  // Make sure the root of the DAG is up-to-date.
722  CurDAG->setRoot(SDB->getControlRoot());
723  HadTailCall = SDB->HasTailCall;
724  SDB->resolveOrClearDbgInfo();
725  SDB->clear();
726 
727  // Final step, emit the lowered DAG as machine code.
728  CodeGenAndEmitDAG();
729 }
730 
731 void SelectionDAGISel::ComputeLiveOutVRegInfo() {
733  SmallVector<SDNode*, 128> Worklist;
734 
735  Worklist.push_back(CurDAG->getRoot().getNode());
736  Added.insert(CurDAG->getRoot().getNode());
737 
738  KnownBits Known;
739 
740  do {
741  SDNode *N = Worklist.pop_back_val();
742 
743  // Otherwise, add all chain operands to the worklist.
744  for (const SDValue &Op : N->op_values())
745  if (Op.getValueType() == MVT::Other && Added.insert(Op.getNode()).second)
746  Worklist.push_back(Op.getNode());
747 
748  // If this is a CopyToReg with a vreg dest, process it.
749  if (N->getOpcode() != ISD::CopyToReg)
750  continue;
751 
752  unsigned DestReg = cast<RegisterSDNode>(N->getOperand(1))->getReg();
753  if (!Register::isVirtualRegister(DestReg))
754  continue;
755 
756  // Ignore non-integer values.
757  SDValue Src = N->getOperand(2);
758  EVT SrcVT = Src.getValueType();
759  if (!SrcVT.isInteger())
760  continue;
761 
762  unsigned NumSignBits = CurDAG->ComputeNumSignBits(Src);
763  Known = CurDAG->computeKnownBits(Src);
764  FuncInfo->AddLiveOutRegInfo(DestReg, NumSignBits, Known);
765  } while (!Worklist.empty());
766 }
767 
768 void SelectionDAGISel::CodeGenAndEmitDAG() {
769  StringRef GroupName = "sdag";
770  StringRef GroupDescription = "Instruction Selection and Scheduling";
771  std::string BlockName;
772  bool MatchFilterBB = false; (void)MatchFilterBB;
773 #ifndef NDEBUG
775  getAnalysis<TargetTransformInfoWrapperPass>().getTTI(*FuncInfo->Fn);
776 #endif
777 
778  // Pre-type legalization allow creation of any node types.
780 
781 #ifndef NDEBUG
782  MatchFilterBB = (FilterDAGBasicBlockName.empty() ||
784  FuncInfo->MBB->getBasicBlock()->getName());
785 #endif
786 #ifdef NDEBUG
790 #endif
791  {
792  BlockName =
793  (MF->getName() + ":" + FuncInfo->MBB->getBasicBlock()->getName()).str();
794  }
795  LLVM_DEBUG(dbgs() << "Initial selection DAG: "
796  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
797  << "'\n";
798  CurDAG->dump());
799 
800 #ifndef NDEBUG
801  if (TTI.hasBranchDivergence())
803 #endif
804 
805  if (ViewDAGCombine1 && MatchFilterBB)
806  CurDAG->viewGraph("dag-combine1 input for " + BlockName);
807 
808  // Run the DAG combiner in pre-legalize mode.
809  {
810  NamedRegionTimer T("combine1", "DAG Combining 1", GroupName,
811  GroupDescription, TimePassesIsEnabled);
813  }
814 
815  LLVM_DEBUG(dbgs() << "Optimized lowered selection DAG: "
816  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
817  << "'\n";
818  CurDAG->dump());
819 
820 #ifndef NDEBUG
821  if (TTI.hasBranchDivergence())
823 #endif
824 
825  // Second step, hack on the DAG until it only uses operations and types that
826  // the target supports.
827  if (ViewLegalizeTypesDAGs && MatchFilterBB)
828  CurDAG->viewGraph("legalize-types input for " + BlockName);
829 
830  bool Changed;
831  {
832  NamedRegionTimer T("legalize_types", "Type Legalization", GroupName,
833  GroupDescription, TimePassesIsEnabled);
834  Changed = CurDAG->LegalizeTypes();
835  }
836 
837  LLVM_DEBUG(dbgs() << "Type-legalized selection DAG: "
838  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
839  << "'\n";
840  CurDAG->dump());
841 
842 #ifndef NDEBUG
843  if (TTI.hasBranchDivergence())
845 #endif
846 
847  // Only allow creation of legal node types.
849 
850  if (Changed) {
851  if (ViewDAGCombineLT && MatchFilterBB)
852  CurDAG->viewGraph("dag-combine-lt input for " + BlockName);
853 
854  // Run the DAG combiner in post-type-legalize mode.
855  {
856  NamedRegionTimer T("combine_lt", "DAG Combining after legalize types",
857  GroupName, GroupDescription, TimePassesIsEnabled);
859  }
860 
861  LLVM_DEBUG(dbgs() << "Optimized type-legalized selection DAG: "
862  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
863  << "'\n";
864  CurDAG->dump());
865 
866 #ifndef NDEBUG
867  if (TTI.hasBranchDivergence())
869 #endif
870  }
871 
872  {
873  NamedRegionTimer T("legalize_vec", "Vector Legalization", GroupName,
874  GroupDescription, TimePassesIsEnabled);
875  Changed = CurDAG->LegalizeVectors();
876  }
877 
878  if (Changed) {
879  LLVM_DEBUG(dbgs() << "Vector-legalized selection DAG: "
880  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
881  << "'\n";
882  CurDAG->dump());
883 
884 #ifndef NDEBUG
885  if (TTI.hasBranchDivergence())
887 #endif
888 
889  {
890  NamedRegionTimer T("legalize_types2", "Type Legalization 2", GroupName,
891  GroupDescription, TimePassesIsEnabled);
893  }
894 
895  LLVM_DEBUG(dbgs() << "Vector/type-legalized selection DAG: "
896  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
897  << "'\n";
898  CurDAG->dump());
899 
900 #ifndef NDEBUG
901  if (TTI.hasBranchDivergence())
903 #endif
904 
905  if (ViewDAGCombineLT && MatchFilterBB)
906  CurDAG->viewGraph("dag-combine-lv input for " + BlockName);
907 
908  // Run the DAG combiner in post-type-legalize mode.
909  {
910  NamedRegionTimer T("combine_lv", "DAG Combining after legalize vectors",
911  GroupName, GroupDescription, TimePassesIsEnabled);
913  }
914 
915  LLVM_DEBUG(dbgs() << "Optimized vector-legalized selection DAG: "
916  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
917  << "'\n";
918  CurDAG->dump());
919 
920 #ifndef NDEBUG
921  if (TTI.hasBranchDivergence())
923 #endif
924  }
925 
926  if (ViewLegalizeDAGs && MatchFilterBB)
927  CurDAG->viewGraph("legalize input for " + BlockName);
928 
929  {
930  NamedRegionTimer T("legalize", "DAG Legalization", GroupName,
931  GroupDescription, TimePassesIsEnabled);
932  CurDAG->Legalize();
933  }
934 
935  LLVM_DEBUG(dbgs() << "Legalized selection DAG: "
936  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
937  << "'\n";
938  CurDAG->dump());
939 
940 #ifndef NDEBUG
941  if (TTI.hasBranchDivergence())
943 #endif
944 
945  if (ViewDAGCombine2 && MatchFilterBB)
946  CurDAG->viewGraph("dag-combine2 input for " + BlockName);
947 
948  // Run the DAG combiner in post-legalize mode.
949  {
950  NamedRegionTimer T("combine2", "DAG Combining 2", GroupName,
951  GroupDescription, TimePassesIsEnabled);
953  }
954 
955  LLVM_DEBUG(dbgs() << "Optimized legalized selection DAG: "
956  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
957  << "'\n";
958  CurDAG->dump());
959 
960 #ifndef NDEBUG
961  if (TTI.hasBranchDivergence())
963 #endif
964 
965  if (OptLevel != CodeGenOpt::None)
966  ComputeLiveOutVRegInfo();
967 
968  if (ViewISelDAGs && MatchFilterBB)
969  CurDAG->viewGraph("isel input for " + BlockName);
970 
971  // Third, instruction select all of the operations to machine code, adding the
972  // code to the MachineBasicBlock.
973  {
974  NamedRegionTimer T("isel", "Instruction Selection", GroupName,
975  GroupDescription, TimePassesIsEnabled);
976  DoInstructionSelection();
977  }
978 
979  LLVM_DEBUG(dbgs() << "Selected selection DAG: "
980  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
981  << "'\n";
982  CurDAG->dump());
983 
984  if (ViewSchedDAGs && MatchFilterBB)
985  CurDAG->viewGraph("scheduler input for " + BlockName);
986 
987  // Schedule machine code.
988  ScheduleDAGSDNodes *Scheduler = CreateScheduler();
989  {
990  NamedRegionTimer T("sched", "Instruction Scheduling", GroupName,
991  GroupDescription, TimePassesIsEnabled);
992  Scheduler->Run(CurDAG, FuncInfo->MBB);
993  }
994 
995  if (ViewSUnitDAGs && MatchFilterBB)
996  Scheduler->viewGraph();
997 
998  // Emit machine code to BB. This can change 'BB' to the last block being
999  // inserted into.
1000  MachineBasicBlock *FirstMBB = FuncInfo->MBB, *LastMBB;
1001  {
1002  NamedRegionTimer T("emit", "Instruction Creation", GroupName,
1003  GroupDescription, TimePassesIsEnabled);
1004 
1005  // FuncInfo->InsertPt is passed by reference and set to the end of the
1006  // scheduled instructions.
1007  LastMBB = FuncInfo->MBB = Scheduler->EmitSchedule(FuncInfo->InsertPt);
1008  }
1009 
1010  // If the block was split, make sure we update any references that are used to
1011  // update PHI nodes later on.
1012  if (FirstMBB != LastMBB)
1013  SDB->UpdateSplitBlock(FirstMBB, LastMBB);
1014 
1015  // Free the scheduler state.
1016  {
1017  NamedRegionTimer T("cleanup", "Instruction Scheduling Cleanup", GroupName,
1018  GroupDescription, TimePassesIsEnabled);
1019  delete Scheduler;
1020  }
1021 
1022  // Free the SelectionDAG state, now that we're finished with it.
1023  CurDAG->clear();
1024 }
1025 
1026 namespace {
1027 
1028 /// ISelUpdater - helper class to handle updates of the instruction selection
1029 /// graph.
1030 class ISelUpdater : public SelectionDAG::DAGUpdateListener {
1031  SelectionDAG::allnodes_iterator &ISelPosition;
1032 
1033 public:
1034  ISelUpdater(SelectionDAG &DAG, SelectionDAG::allnodes_iterator &isp)
1035  : SelectionDAG::DAGUpdateListener(DAG), ISelPosition(isp) {}
1036 
1037  /// NodeDeleted - Handle nodes deleted from the graph. If the node being
1038  /// deleted is the current ISelPosition node, update ISelPosition.
1039  ///
1040  void NodeDeleted(SDNode *N, SDNode *E) override {
1041  if (ISelPosition == SelectionDAG::allnodes_iterator(N))
1042  ++ISelPosition;
1043  }
1044 };
1045 
1046 } // end anonymous namespace
1047 
1048 // This function is used to enforce the topological node id property
1049 // leveraged during instruction selection. Before the selection process all
1050 // nodes are given a non-negative id such that all nodes have a greater id than
1051 // their operands. As this holds transitively we can prune checks that a node N
1052 // is a predecessor of M another by not recursively checking through M's
1053 // operands if N's ID is larger than M's ID. This significantly improves
1054 // performance of various legality checks (e.g. IsLegalToFold / UpdateChains).
1055 
1056 // However, when we fuse multiple nodes into a single node during the
1057 // selection we may induce a predecessor relationship between inputs and
1058 // outputs of distinct nodes being merged, violating the topological property.
1059 // Should a fused node have a successor which has yet to be selected,
1060 // our legality checks would be incorrect. To avoid this we mark all unselected
1061 // successor nodes, i.e. id != -1, as invalid for pruning by bit-negating (x =>
1062 // (-(x+1))) the ids and modify our pruning check to ignore negative Ids of M.
1063 // We use bit-negation to more clearly enforce that node id -1 can only be
1064 // achieved by selected nodes. As the conversion is reversable to the original
1065 // Id, topological pruning can still be leveraged when looking for unselected
1066 // nodes. This method is called internally in all ISel replacement related
1067 // functions.
1070  Nodes.push_back(Node);
1071 
1072  while (!Nodes.empty()) {
1073  SDNode *N = Nodes.pop_back_val();
1074  for (auto *U : N->uses()) {
1075  auto UId = U->getNodeId();
1076  if (UId > 0) {
1077  InvalidateNodeId(U);
1078  Nodes.push_back(U);
1079  }
1080  }
1081  }
1082 }
1083 
1084 // InvalidateNodeId - As explained in EnforceNodeIdInvariant, mark a
1085 // NodeId with the equivalent node id which is invalid for topological
1086 // pruning.
1088  int InvalidId = -(N->getNodeId() + 1);
1089  N->setNodeId(InvalidId);
1090 }
1091 
1092 // getUninvalidatedNodeId - get original uninvalidated node id.
1094  int Id = N->getNodeId();
1095  if (Id < -1)
1096  return -(Id + 1);
1097  return Id;
1098 }
1099 
1100 void SelectionDAGISel::DoInstructionSelection() {
1101  LLVM_DEBUG(dbgs() << "===== Instruction selection begins: "
1102  << printMBBReference(*FuncInfo->MBB) << " '"
1103  << FuncInfo->MBB->getName() << "'\n");
1104 
1106 
1107  // Select target instructions for the DAG.
1108  {
1109  // Number all nodes with a topological order and set DAGSize.
1111 
1112  // Create a dummy node (which is not added to allnodes), that adds
1113  // a reference to the root node, preventing it from being deleted,
1114  // and tracking any changes of the root.
1117  ++ISelPosition;
1118 
1119  // Make sure that ISelPosition gets properly updated when nodes are deleted
1120  // in calls made from this function.
1121  ISelUpdater ISU(*CurDAG, ISelPosition);
1122 
1123  // The AllNodes list is now topological-sorted. Visit the
1124  // nodes by starting at the end of the list (the root of the
1125  // graph) and preceding back toward the beginning (the entry
1126  // node).
1127  while (ISelPosition != CurDAG->allnodes_begin()) {
1128  SDNode *Node = &*--ISelPosition;
1129  // Skip dead nodes. DAGCombiner is expected to eliminate all dead nodes,
1130  // but there are currently some corner cases that it misses. Also, this
1131  // makes it theoretically possible to disable the DAGCombiner.
1132  if (Node->use_empty())
1133  continue;
1134 
1135 #ifndef NDEBUG
1137  Nodes.push_back(Node);
1138 
1139  while (!Nodes.empty()) {
1140  auto N = Nodes.pop_back_val();
1141  if (N->getOpcode() == ISD::TokenFactor || N->getNodeId() < 0)
1142  continue;
1143  for (const SDValue &Op : N->op_values()) {
1144  if (Op->getOpcode() == ISD::TokenFactor)
1145  Nodes.push_back(Op.getNode());
1146  else {
1147  // We rely on topological ordering of node ids for checking for
1148  // cycles when fusing nodes during selection. All unselected nodes
1149  // successors of an already selected node should have a negative id.
1150  // This assertion will catch such cases. If this assertion triggers
1151  // it is likely you using DAG-level Value/Node replacement functions
1152  // (versus equivalent ISEL replacement) in backend-specific
1153  // selections. See comment in EnforceNodeIdInvariant for more
1154  // details.
1155  assert(Op->getNodeId() != -1 &&
1156  "Node has already selected predecessor node");
1157  }
1158  }
1159  }
1160 #endif
1161 
1162  // When we are using non-default rounding modes or FP exception behavior
1163  // FP operations are represented by StrictFP pseudo-operations. For
1164  // targets that do not (yet) understand strict FP operations directly,
1165  // we convert them to normal FP opcodes instead at this point. This
1166  // will allow them to be handled by existing target-specific instruction
1167  // selectors.
1168  if (!TLI->isStrictFPEnabled() && Node->isStrictFPOpcode()) {
1169  // For some opcodes, we need to call TLI->getOperationAction using
1170  // the first operand type instead of the result type. Note that this
1171  // must match what SelectionDAGLegalize::LegalizeOp is doing.
1172  EVT ActionVT;
1173  switch (Node->getOpcode()) {
1176  case ISD::STRICT_LRINT:
1177  case ISD::STRICT_LLRINT:
1178  case ISD::STRICT_LROUND:
1179  case ISD::STRICT_LLROUND:
1180  case ISD::STRICT_FSETCC:
1181  case ISD::STRICT_FSETCCS:
1182  ActionVT = Node->getOperand(1).getValueType();
1183  break;
1184  default:
1185  ActionVT = Node->getValueType(0);
1186  break;
1187  }
1188  if (TLI->getOperationAction(Node->getOpcode(), ActionVT)
1190  Node = CurDAG->mutateStrictFPToFP(Node);
1191  }
1192 
1193  LLVM_DEBUG(dbgs() << "\nISEL: Starting selection on root node: ";
1194  Node->dump(CurDAG));
1195 
1196  Select(Node);
1197  }
1198 
1199  CurDAG->setRoot(Dummy.getValue());
1200  }
1201 
1202  LLVM_DEBUG(dbgs() << "\n===== Instruction selection ends:\n");
1203 
1205 }
1206 
1208  for (const User *U : CPI->users()) {
1209  if (const IntrinsicInst *EHPtrCall = dyn_cast<IntrinsicInst>(U)) {
1210  Intrinsic::ID IID = EHPtrCall->getIntrinsicID();
1211  if (IID == Intrinsic::eh_exceptionpointer ||
1212  IID == Intrinsic::eh_exceptioncode)
1213  return true;
1214  }
1215  }
1216  return false;
1217 }
1218 
1219 // wasm.landingpad.index intrinsic is for associating a landing pad index number
1220 // with a catchpad instruction. Retrieve the landing pad index in the intrinsic
1221 // and store the mapping in the function.
1223  const CatchPadInst *CPI) {
1224  MachineFunction *MF = MBB->getParent();
1225  // In case of single catch (...), we don't emit LSDA, so we don't need
1226  // this information.
1227  bool IsSingleCatchAllClause =
1228  CPI->getNumArgOperands() == 1 &&
1229  cast<Constant>(CPI->getArgOperand(0))->isNullValue();
1230  // cathchpads for longjmp use an empty type list, e.g. catchpad within %0 []
1231  // and they don't need LSDA info
1232  bool IsCatchLongjmp = CPI->getNumArgOperands() == 0;
1233  if (!IsSingleCatchAllClause && !IsCatchLongjmp) {
1234  // Create a mapping from landing pad label to landing pad index.
1235  bool IntrFound = false;
1236  for (const User *U : CPI->users()) {
1237  if (const auto *Call = dyn_cast<IntrinsicInst>(U)) {
1238  Intrinsic::ID IID = Call->getIntrinsicID();
1239  if (IID == Intrinsic::wasm_landingpad_index) {
1240  Value *IndexArg = Call->getArgOperand(1);
1241  int Index = cast<ConstantInt>(IndexArg)->getZExtValue();
1243  IntrFound = true;
1244  break;
1245  }
1246  }
1247  }
1248  assert(IntrFound && "wasm.landingpad.index intrinsic not found!");
1249  (void)IntrFound;
1250  }
1251 }
1252 
1253 /// PrepareEHLandingPad - Emit an EH_LABEL, set up live-in registers, and
1254 /// do other setup for EH landing-pad blocks.
1255 bool SelectionDAGISel::PrepareEHLandingPad() {
1256  MachineBasicBlock *MBB = FuncInfo->MBB;
1257  const Constant *PersonalityFn = FuncInfo->Fn->getPersonalityFn();
1258  const BasicBlock *LLVMBB = MBB->getBasicBlock();
1259  const TargetRegisterClass *PtrRC =
1261 
1262  auto Pers = classifyEHPersonality(PersonalityFn);
1263 
1264  // Catchpads have one live-in register, which typically holds the exception
1265  // pointer or code.
1266  if (isFuncletEHPersonality(Pers)) {
1267  if (const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->getFirstNonPHI())) {
1268  if (hasExceptionPointerOrCodeUser(CPI)) {
1269  // Get or create the virtual register to hold the pointer or code. Mark
1270  // the live in physreg and copy into the vreg.
1271  MCPhysReg EHPhysReg = TLI->getExceptionPointerRegister(PersonalityFn);
1272  assert(EHPhysReg && "target lacks exception pointer register");
1273  MBB->addLiveIn(EHPhysReg);
1274  unsigned VReg = FuncInfo->getCatchPadExceptionPointerVReg(CPI, PtrRC);
1275  BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(),
1276  TII->get(TargetOpcode::COPY), VReg)
1277  .addReg(EHPhysReg, RegState::Kill);
1278  }
1279  }
1280  return true;
1281  }
1282 
1283  // Add a label to mark the beginning of the landing pad. Deletion of the
1284  // landing pad can thus be detected via the MachineModuleInfo.
1286 
1287  const MCInstrDesc &II = TII->get(TargetOpcode::EH_LABEL);
1288  BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(), II)
1289  .addSym(Label);
1290 
1291  // If the unwinder does not preserve all registers, ensure that the
1292  // function marks the clobbered registers as used.
1294  if (auto *RegMask = TRI.getCustomEHPadPreservedMask(*MF))
1296 
1297  if (Pers == EHPersonality::Wasm_CXX) {
1298  if (const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->getFirstNonPHI()))
1300  } else {
1301  // Assign the call site to the landing pad's begin label.
1302  MF->setCallSiteLandingPad(Label, SDB->LPadToCallSiteMap[MBB]);
1303  // Mark exception register as live in.
1304  if (unsigned Reg = TLI->getExceptionPointerRegister(PersonalityFn))
1305  FuncInfo->ExceptionPointerVirtReg = MBB->addLiveIn(Reg, PtrRC);
1306  // Mark exception selector register as live in.
1307  if (unsigned Reg = TLI->getExceptionSelectorRegister(PersonalityFn))
1308  FuncInfo->ExceptionSelectorVirtReg = MBB->addLiveIn(Reg, PtrRC);
1309  }
1310 
1311  return true;
1312 }
1313 
1314 /// isFoldedOrDeadInstruction - Return true if the specified instruction is
1315 /// side-effect free and is either dead or folded into a generated instruction.
1316 /// Return false if it needs to be emitted.
1318  const FunctionLoweringInfo &FuncInfo) {
1319  return !I->mayWriteToMemory() && // Side-effecting instructions aren't folded.
1320  !I->isTerminator() && // Terminators aren't folded.
1321  !isa<DbgInfoIntrinsic>(I) && // Debug instructions aren't folded.
1322  !I->isEHPad() && // EH pad instructions aren't folded.
1323  !FuncInfo.isExportedInst(I); // Exported instrs must be computed.
1324 }
1325 
1326 /// Collect llvm.dbg.declare information. This is done after argument lowering
1327 /// in case the declarations refer to arguments.
1329  MachineFunction *MF = FuncInfo.MF;
1330  const DataLayout &DL = MF->getDataLayout();
1331  for (const BasicBlock &BB : *FuncInfo.Fn) {
1332  for (const Instruction &I : BB) {
1333  const DbgDeclareInst *DI = dyn_cast<DbgDeclareInst>(&I);
1334  if (!DI)
1335  continue;
1336 
1337  assert(DI->getVariable() && "Missing variable");
1338  assert(DI->getDebugLoc() && "Missing location");
1339  const Value *Address = DI->getAddress();
1340  if (!Address) {
1341  LLVM_DEBUG(dbgs() << "processDbgDeclares skipping " << *DI
1342  << " (bad address)\n");
1343  continue;
1344  }
1345 
1346  // Look through casts and constant offset GEPs. These mostly come from
1347  // inalloca.
1348  APInt Offset(DL.getTypeSizeInBits(Address->getType()), 0);
1349  Address = Address->stripAndAccumulateInBoundsConstantOffsets(DL, Offset);
1350 
1351  // Check if the variable is a static alloca or a byval or inalloca
1352  // argument passed in memory. If it is not, then we will ignore this
1353  // intrinsic and handle this during isel like dbg.value.
1354  int FI = std::numeric_limits<int>::max();
1355  if (const auto *AI = dyn_cast<AllocaInst>(Address)) {
1356  auto SI = FuncInfo.StaticAllocaMap.find(AI);
1357  if (SI != FuncInfo.StaticAllocaMap.end())
1358  FI = SI->second;
1359  } else if (const auto *Arg = dyn_cast<Argument>(Address))
1360  FI = FuncInfo.getArgumentFrameIndex(Arg);
1361 
1362  if (FI == std::numeric_limits<int>::max())
1363  continue;
1364 
1365  DIExpression *Expr = DI->getExpression();
1366  if (Offset.getBoolValue())
1368  Offset.getZExtValue());
1369  LLVM_DEBUG(dbgs() << "processDbgDeclares: setVariableDbgInfo FI=" << FI
1370  << ", " << *DI << "\n");
1371  MF->setVariableDbgInfo(DI->getVariable(), Expr, FI, DI->getDebugLoc());
1372  }
1373  }
1374 }
1375 
1376 void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) {
1377  FastISelFailed = false;
1378  // Initialize the Fast-ISel state, if needed.
1379  FastISel *FastIS = nullptr;
1380  if (TM.Options.EnableFastISel) {
1381  LLVM_DEBUG(dbgs() << "Enabling fast-isel\n");
1382  FastIS = TLI->createFastISel(*FuncInfo, LibInfo);
1383  }
1384 
1386 
1387  // Lower arguments up front. An RPO iteration always visits the entry block
1388  // first.
1389  assert(*RPOT.begin() == &Fn.getEntryBlock());
1390  ++NumEntryBlocks;
1391 
1392  // Set up FuncInfo for ISel. Entry blocks never have PHIs.
1393  FuncInfo->MBB = FuncInfo->MBBMap[&Fn.getEntryBlock()];
1394  FuncInfo->InsertPt = FuncInfo->MBB->begin();
1395 
1397 
1398  if (!FastIS) {
1399  LowerArguments(Fn);
1400  } else {
1401  // See if fast isel can lower the arguments.
1402  FastIS->startNewBlock();
1403  if (!FastIS->lowerArguments()) {
1404  FastISelFailed = true;
1405  // Fast isel failed to lower these arguments
1406  ++NumFastIselFailLowerArguments;
1407 
1408  OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1409  Fn.getSubprogram(),
1410  &Fn.getEntryBlock());
1411  R << "FastISel didn't lower all arguments: "
1412  << ore::NV("Prototype", Fn.getType());
1414 
1415  // Use SelectionDAG argument lowering
1416  LowerArguments(Fn);
1417  CurDAG->setRoot(SDB->getControlRoot());
1418  SDB->clear();
1419  CodeGenAndEmitDAG();
1420  }
1421 
1422  // If we inserted any instructions at the beginning, make a note of
1423  // where they are, so we can be sure to emit subsequent instructions
1424  // after them.
1425  if (FuncInfo->InsertPt != FuncInfo->MBB->begin())
1426  FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt));
1427  else
1428  FastIS->setLastLocalValue(nullptr);
1429  }
1430 
1431  bool Inserted = SwiftError->createEntriesInEntryBlock(SDB->getCurDebugLoc());
1432 
1433  if (FastIS && Inserted)
1434  FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt));
1435 
1437 
1438  // Iterate over all basic blocks in the function.
1439  StackProtector &SP = getAnalysis<StackProtector>();
1440  for (const BasicBlock *LLVMBB : RPOT) {
1441  if (OptLevel != CodeGenOpt::None) {
1442  bool AllPredsVisited = true;
1443  for (const BasicBlock *Pred : predecessors(LLVMBB)) {
1444  if (!FuncInfo->VisitedBBs.count(Pred)) {
1445  AllPredsVisited = false;
1446  break;
1447  }
1448  }
1449 
1450  if (AllPredsVisited) {
1451  for (const PHINode &PN : LLVMBB->phis())
1452  FuncInfo->ComputePHILiveOutRegInfo(&PN);
1453  } else {
1454  for (const PHINode &PN : LLVMBB->phis())
1455  FuncInfo->InvalidatePHILiveOutRegInfo(&PN);
1456  }
1457 
1458  FuncInfo->VisitedBBs.insert(LLVMBB);
1459  }
1460 
1461  BasicBlock::const_iterator const Begin =
1462  LLVMBB->getFirstNonPHI()->getIterator();
1463  BasicBlock::const_iterator const End = LLVMBB->end();
1464  BasicBlock::const_iterator BI = End;
1465 
1466  FuncInfo->MBB = FuncInfo->MBBMap[LLVMBB];
1467  if (!FuncInfo->MBB)
1468  continue; // Some blocks like catchpads have no code or MBB.
1469 
1470  // Insert new instructions after any phi or argument setup code.
1471  FuncInfo->InsertPt = FuncInfo->MBB->end();
1472 
1473  // Setup an EH landing-pad block.
1474  FuncInfo->ExceptionPointerVirtReg = 0;
1475  FuncInfo->ExceptionSelectorVirtReg = 0;
1476  if (LLVMBB->isEHPad())
1477  if (!PrepareEHLandingPad())
1478  continue;
1479 
1480  // Before doing SelectionDAG ISel, see if FastISel has been requested.
1481  if (FastIS) {
1482  if (LLVMBB != &Fn.getEntryBlock())
1483  FastIS->startNewBlock();
1484 
1485  unsigned NumFastIselRemaining = std::distance(Begin, End);
1486 
1487  // Pre-assign swifterror vregs.
1488  SwiftError->preassignVRegs(FuncInfo->MBB, Begin, End);
1489 
1490  // Do FastISel on as many instructions as possible.
1491  for (; BI != Begin; --BI) {
1492  const Instruction *Inst = &*std::prev(BI);
1493 
1494  // If we no longer require this instruction, skip it.
1495  if (isFoldedOrDeadInstruction(Inst, *FuncInfo) ||
1496  ElidedArgCopyInstrs.count(Inst)) {
1497  --NumFastIselRemaining;
1498  continue;
1499  }
1500 
1501  // Bottom-up: reset the insert pos at the top, after any local-value
1502  // instructions.
1503  FastIS->recomputeInsertPt();
1504 
1505  // Try to select the instruction with FastISel.
1506  if (FastIS->selectInstruction(Inst)) {
1507  --NumFastIselRemaining;
1508  ++NumFastIselSuccess;
1509  // If fast isel succeeded, skip over all the folded instructions, and
1510  // then see if there is a load right before the selected instructions.
1511  // Try to fold the load if so.
1512  const Instruction *BeforeInst = Inst;
1513  while (BeforeInst != &*Begin) {
1514  BeforeInst = &*std::prev(BasicBlock::const_iterator(BeforeInst));
1515  if (!isFoldedOrDeadInstruction(BeforeInst, *FuncInfo))
1516  break;
1517  }
1518  if (BeforeInst != Inst && isa<LoadInst>(BeforeInst) &&
1519  BeforeInst->hasOneUse() &&
1520  FastIS->tryToFoldLoad(cast<LoadInst>(BeforeInst), Inst)) {
1521  // If we succeeded, don't re-select the load.
1522  BI = std::next(BasicBlock::const_iterator(BeforeInst));
1523  --NumFastIselRemaining;
1524  ++NumFastIselSuccess;
1525  }
1526  continue;
1527  }
1528 
1529  FastISelFailed = true;
1530 
1531  // Then handle certain instructions as single-LLVM-Instruction blocks.
1532  // We cannot separate out GCrelocates to their own blocks since we need
1533  // to keep track of gc-relocates for a particular gc-statepoint. This is
1534  // done by SelectionDAGBuilder::LowerAsSTATEPOINT, called before
1535  // visitGCRelocate.
1536  if (isa<CallInst>(Inst) && !isa<GCStatepointInst>(Inst) &&
1537  !isa<GCRelocateInst>(Inst) && !isa<GCResultInst>(Inst)) {
1538  OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1539  Inst->getDebugLoc(), LLVMBB);
1540 
1541  R << "FastISel missed call";
1542 
1543  if (R.isEnabled() || EnableFastISelAbort) {
1544  std::string InstStrStorage;
1545  raw_string_ostream InstStr(InstStrStorage);
1546  InstStr << *Inst;
1547 
1548  R << ": " << InstStr.str();
1549  }
1550 
1552 
1553  if (!Inst->getType()->isVoidTy() && !Inst->getType()->isTokenTy() &&
1554  !Inst->use_empty()) {
1555  Register &R = FuncInfo->ValueMap[Inst];
1556  if (!R)
1557  R = FuncInfo->CreateRegs(Inst);
1558  }
1559 
1560  bool HadTailCall = false;
1561  MachineBasicBlock::iterator SavedInsertPt = FuncInfo->InsertPt;
1562  SelectBasicBlock(Inst->getIterator(), BI, HadTailCall);
1563 
1564  // If the call was emitted as a tail call, we're done with the block.
1565  // We also need to delete any previously emitted instructions.
1566  if (HadTailCall) {
1567  FastIS->removeDeadCode(SavedInsertPt, FuncInfo->MBB->end());
1568  --BI;
1569  break;
1570  }
1571 
1572  // Recompute NumFastIselRemaining as Selection DAG instruction
1573  // selection may have handled the call, input args, etc.
1574  unsigned RemainingNow = std::distance(Begin, BI);
1575  NumFastIselFailures += NumFastIselRemaining - RemainingNow;
1576  NumFastIselRemaining = RemainingNow;
1577  continue;
1578  }
1579 
1580  OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1581  Inst->getDebugLoc(), LLVMBB);
1582 
1583  bool ShouldAbort = EnableFastISelAbort;
1584  if (Inst->isTerminator()) {
1585  // Use a different message for terminator misses.
1586  R << "FastISel missed terminator";
1587  // Don't abort for terminator unless the level is really high
1588  ShouldAbort = (EnableFastISelAbort > 2);
1589  } else {
1590  R << "FastISel missed";
1591  }
1592 
1593  if (R.isEnabled() || EnableFastISelAbort) {
1594  std::string InstStrStorage;
1595  raw_string_ostream InstStr(InstStrStorage);
1596  InstStr << *Inst;
1597  R << ": " << InstStr.str();
1598  }
1599 
1600  reportFastISelFailure(*MF, *ORE, R, ShouldAbort);
1601 
1602  NumFastIselFailures += NumFastIselRemaining;
1603  break;
1604  }
1605 
1606  FastIS->recomputeInsertPt();
1607  }
1608 
1609  if (SP.shouldEmitSDCheck(*LLVMBB)) {
1610  bool FunctionBasedInstrumentation =
1612  SDB->SPDescriptor.initialize(LLVMBB, FuncInfo->MBBMap[LLVMBB],
1613  FunctionBasedInstrumentation);
1614  }
1615 
1616  if (Begin != BI)
1617  ++NumDAGBlocks;
1618  else
1619  ++NumFastIselBlocks;
1620 
1621  if (Begin != BI) {
1622  // Run SelectionDAG instruction selection on the remainder of the block
1623  // not handled by FastISel. If FastISel is not run, this is the entire
1624  // block.
1625  bool HadTailCall;
1626  SelectBasicBlock(Begin, BI, HadTailCall);
1627 
1628  // But if FastISel was run, we already selected some of the block.
1629  // If we emitted a tail-call, we need to delete any previously emitted
1630  // instruction that follows it.
1631  if (FastIS && HadTailCall && FuncInfo->InsertPt != FuncInfo->MBB->end())
1632  FastIS->removeDeadCode(FuncInfo->InsertPt, FuncInfo->MBB->end());
1633  }
1634 
1635  if (FastIS)
1636  FastIS->finishBasicBlock();
1637  FinishBasicBlock();
1638  FuncInfo->PHINodesToUpdate.clear();
1639  ElidedArgCopyInstrs.clear();
1640  }
1641 
1643 
1645 
1646  delete FastIS;
1647  SDB->clearDanglingDebugInfo();
1648  SDB->SPDescriptor.resetPerFunctionState();
1649 }
1650 
1651 void
1652 SelectionDAGISel::FinishBasicBlock() {
1653  LLVM_DEBUG(dbgs() << "Total amount of phi nodes to update: "
1654  << FuncInfo->PHINodesToUpdate.size() << "\n";
1655  for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e;
1656  ++i) dbgs()
1657  << "Node " << i << " : (" << FuncInfo->PHINodesToUpdate[i].first
1658  << ", " << FuncInfo->PHINodesToUpdate[i].second << ")\n");
1659 
1660  // Next, now that we know what the last MBB the LLVM BB expanded is, update
1661  // PHI nodes in successors.
1662  for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) {
1663  MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[i].first);
1664  assert(PHI->isPHI() &&
1665  "This is not a machine PHI node that we are updating!");
1666  if (!FuncInfo->MBB->isSuccessor(PHI->getParent()))
1667  continue;
1668  PHI.addReg(FuncInfo->PHINodesToUpdate[i].second).addMBB(FuncInfo->MBB);
1669  }
1670 
1671  // Handle stack protector.
1672  if (SDB->SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
1673  // The target provides a guard check function. There is no need to
1674  // generate error handling code or to split current basic block.
1675  MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB();
1676 
1677  // Add load and check to the basicblock.
1678  FuncInfo->MBB = ParentMBB;
1679  FuncInfo->InsertPt =
1680  findSplitPointForStackProtector(ParentMBB, *TII);
1681  SDB->visitSPDescriptorParent(SDB->SPDescriptor, ParentMBB);
1682  CurDAG->setRoot(SDB->getRoot());
1683  SDB->clear();
1684  CodeGenAndEmitDAG();
1685 
1686  // Clear the Per-BB State.
1687  SDB->SPDescriptor.resetPerBBState();
1688  } else if (SDB->SPDescriptor.shouldEmitStackProtector()) {
1689  MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB();
1690  MachineBasicBlock *SuccessMBB = SDB->SPDescriptor.getSuccessMBB();
1691 
1692  // Find the split point to split the parent mbb. At the same time copy all
1693  // physical registers used in the tail of parent mbb into virtual registers
1694  // before the split point and back into physical registers after the split
1695  // point. This prevents us needing to deal with Live-ins and many other
1696  // register allocation issues caused by us splitting the parent mbb. The
1697  // register allocator will clean up said virtual copies later on.
1698  MachineBasicBlock::iterator SplitPoint =
1699  findSplitPointForStackProtector(ParentMBB, *TII);
1700 
1701  // Splice the terminator of ParentMBB into SuccessMBB.
1702  SuccessMBB->splice(SuccessMBB->end(), ParentMBB,
1703  SplitPoint,
1704  ParentMBB->end());
1705 
1706  // Add compare/jump on neq/jump to the parent BB.
1707  FuncInfo->MBB = ParentMBB;
1708  FuncInfo->InsertPt = ParentMBB->end();
1709  SDB->visitSPDescriptorParent(SDB->SPDescriptor, ParentMBB);
1710  CurDAG->setRoot(SDB->getRoot());
1711  SDB->clear();
1712  CodeGenAndEmitDAG();
1713 
1714  // CodeGen Failure MBB if we have not codegened it yet.
1715  MachineBasicBlock *FailureMBB = SDB->SPDescriptor.getFailureMBB();
1716  if (FailureMBB->empty()) {
1717  FuncInfo->MBB = FailureMBB;
1718  FuncInfo->InsertPt = FailureMBB->end();
1719  SDB->visitSPDescriptorFailure(SDB->SPDescriptor);
1720  CurDAG->setRoot(SDB->getRoot());
1721  SDB->clear();
1722  CodeGenAndEmitDAG();
1723  }
1724 
1725  // Clear the Per-BB State.
1726  SDB->SPDescriptor.resetPerBBState();
1727  }
1728 
1729  // Lower each BitTestBlock.
1730  for (auto &BTB : SDB->SL->BitTestCases) {
1731  // Lower header first, if it wasn't already lowered
1732  if (!BTB.Emitted) {
1733  // Set the current basic block to the mbb we wish to insert the code into
1734  FuncInfo->MBB = BTB.Parent;
1735  FuncInfo->InsertPt = FuncInfo->MBB->end();
1736  // Emit the code
1737  SDB->visitBitTestHeader(BTB, FuncInfo->MBB);
1738  CurDAG->setRoot(SDB->getRoot());
1739  SDB->clear();
1740  CodeGenAndEmitDAG();
1741  }
1742 
1743  BranchProbability UnhandledProb = BTB.Prob;
1744  for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
1745  UnhandledProb -= BTB.Cases[j].ExtraProb;
1746  // Set the current basic block to the mbb we wish to insert the code into
1747  FuncInfo->MBB = BTB.Cases[j].ThisBB;
1748  FuncInfo->InsertPt = FuncInfo->MBB->end();
1749  // Emit the code
1750 
1751  // If all cases cover a contiguous range, it is not necessary to jump to
1752  // the default block after the last bit test fails. This is because the
1753  // range check during bit test header creation has guaranteed that every
1754  // case here doesn't go outside the range. In this case, there is no need
1755  // to perform the last bit test, as it will always be true. Instead, make
1756  // the second-to-last bit-test fall through to the target of the last bit
1757  // test, and delete the last bit test.
1758 
1759  MachineBasicBlock *NextMBB;
1760  if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
1761  // Second-to-last bit-test with contiguous range or omitted range
1762  // check: fall through to the target of the final bit test.
1763  NextMBB = BTB.Cases[j + 1].TargetBB;
1764  } else if (j + 1 == ej) {
1765  // For the last bit test, fall through to Default.
1766  NextMBB = BTB.Default;
1767  } else {
1768  // Otherwise, fall through to the next bit test.
1769  NextMBB = BTB.Cases[j + 1].ThisBB;
1770  }
1771 
1772  SDB->visitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j],
1773  FuncInfo->MBB);
1774 
1775  CurDAG->setRoot(SDB->getRoot());
1776  SDB->clear();
1777  CodeGenAndEmitDAG();
1778 
1779  if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
1780  // Since we're not going to use the final bit test, remove it.
1781  BTB.Cases.pop_back();
1782  break;
1783  }
1784  }
1785 
1786  // Update PHI Nodes
1787  for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size();
1788  pi != pe; ++pi) {
1789  MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first);
1790  MachineBasicBlock *PHIBB = PHI->getParent();
1791  assert(PHI->isPHI() &&
1792  "This is not a machine PHI node that we are updating!");
1793  // This is "default" BB. We have two jumps to it. From "header" BB and
1794  // from last "case" BB, unless the latter was skipped.
1795  if (PHIBB == BTB.Default) {
1796  PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(BTB.Parent);
1797  if (!BTB.ContiguousRange) {
1798  PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second)
1799  .addMBB(BTB.Cases.back().ThisBB);
1800  }
1801  }
1802  // One of "cases" BB.
1803  for (unsigned j = 0, ej = BTB.Cases.size();
1804  j != ej; ++j) {
1805  MachineBasicBlock* cBB = BTB.Cases[j].ThisBB;
1806  if (cBB->isSuccessor(PHIBB))
1807  PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(cBB);
1808  }
1809  }
1810  }
1811  SDB->SL->BitTestCases.clear();
1812 
1813  // If the JumpTable record is filled in, then we need to emit a jump table.
1814  // Updating the PHI nodes is tricky in this case, since we need to determine
1815  // whether the PHI is a successor of the range check MBB or the jump table MBB
1816  for (unsigned i = 0, e = SDB->SL->JTCases.size(); i != e; ++i) {
1817  // Lower header first, if it wasn't already lowered
1818  if (!SDB->SL->JTCases[i].first.Emitted) {
1819  // Set the current basic block to the mbb we wish to insert the code into
1820  FuncInfo->MBB = SDB->SL->JTCases[i].first.HeaderBB;
1821  FuncInfo->InsertPt = FuncInfo->MBB->end();
1822  // Emit the code
1823  SDB->visitJumpTableHeader(SDB->SL->JTCases[i].second,
1824  SDB->SL->JTCases[i].first, FuncInfo->MBB);
1825  CurDAG->setRoot(SDB->getRoot());
1826  SDB->clear();
1827  CodeGenAndEmitDAG();
1828  }
1829 
1830  // Set the current basic block to the mbb we wish to insert the code into
1831  FuncInfo->MBB = SDB->SL->JTCases[i].second.MBB;
1832  FuncInfo->InsertPt = FuncInfo->MBB->end();
1833  // Emit the code
1834  SDB->visitJumpTable(SDB->SL->JTCases[i].second);
1835  CurDAG->setRoot(SDB->getRoot());
1836  SDB->clear();
1837  CodeGenAndEmitDAG();
1838 
1839  // Update PHI Nodes
1840  for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size();
1841  pi != pe; ++pi) {
1842  MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first);
1843  MachineBasicBlock *PHIBB = PHI->getParent();
1844  assert(PHI->isPHI() &&
1845  "This is not a machine PHI node that we are updating!");
1846  // "default" BB. We can go there only from header BB.
1847  if (PHIBB == SDB->SL->JTCases[i].second.Default)
1848  PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second)
1849  .addMBB(SDB->SL->JTCases[i].first.HeaderBB);
1850  // JT BB. Just iterate over successors here
1851  if (FuncInfo->MBB->isSuccessor(PHIBB))
1852  PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(FuncInfo->MBB);
1853  }
1854  }
1855  SDB->SL->JTCases.clear();
1856 
1857  // If we generated any switch lowering information, build and codegen any
1858  // additional DAGs necessary.
1859  for (unsigned i = 0, e = SDB->SL->SwitchCases.size(); i != e; ++i) {
1860  // Set the current basic block to the mbb we wish to insert the code into
1861  FuncInfo->MBB = SDB->SL->SwitchCases[i].ThisBB;
1862  FuncInfo->InsertPt = FuncInfo->MBB->end();
1863 
1864  // Determine the unique successors.
1866  Succs.push_back(SDB->SL->SwitchCases[i].TrueBB);
1867  if (SDB->SL->SwitchCases[i].TrueBB != SDB->SL->SwitchCases[i].FalseBB)
1868  Succs.push_back(SDB->SL->SwitchCases[i].FalseBB);
1869 
1870  // Emit the code. Note that this could result in FuncInfo->MBB being split.
1871  SDB->visitSwitchCase(SDB->SL->SwitchCases[i], FuncInfo->MBB);
1872  CurDAG->setRoot(SDB->getRoot());
1873  SDB->clear();
1874  CodeGenAndEmitDAG();
1875 
1876  // Remember the last block, now that any splitting is done, for use in
1877  // populating PHI nodes in successors.
1878  MachineBasicBlock *ThisBB = FuncInfo->MBB;
1879 
1880  // Handle any PHI nodes in successors of this chunk, as if we were coming
1881  // from the original BB before switch expansion. Note that PHI nodes can
1882  // occur multiple times in PHINodesToUpdate. We have to be very careful to
1883  // handle them the right number of times.
1884  for (unsigned i = 0, e = Succs.size(); i != e; ++i) {
1885  FuncInfo->MBB = Succs[i];
1886  FuncInfo->InsertPt = FuncInfo->MBB->end();
1887  // FuncInfo->MBB may have been removed from the CFG if a branch was
1888  // constant folded.
1889  if (ThisBB->isSuccessor(FuncInfo->MBB)) {
1891  MBBI = FuncInfo->MBB->begin(), MBBE = FuncInfo->MBB->end();
1892  MBBI != MBBE && MBBI->isPHI(); ++MBBI) {
1893  MachineInstrBuilder PHI(*MF, MBBI);
1894  // This value for this PHI node is recorded in PHINodesToUpdate.
1895  for (unsigned pn = 0; ; ++pn) {
1896  assert(pn != FuncInfo->PHINodesToUpdate.size() &&
1897  "Didn't find PHI entry!");
1898  if (FuncInfo->PHINodesToUpdate[pn].first == PHI) {
1899  PHI.addReg(FuncInfo->PHINodesToUpdate[pn].second).addMBB(ThisBB);
1900  break;
1901  }
1902  }
1903  }
1904  }
1905  }
1906  }
1907  SDB->SL->SwitchCases.clear();
1908 }
1909 
1910 /// Create the scheduler. If a specific scheduler was specified
1911 /// via the SchedulerRegistry, use it, otherwise select the
1912 /// one preferred by the target.
1913 ///
1914 ScheduleDAGSDNodes *SelectionDAGISel::CreateScheduler() {
1915  return ISHeuristic(this, OptLevel);
1916 }
1917 
1918 //===----------------------------------------------------------------------===//
1919 // Helper functions used by the generated instruction selector.
1920 //===----------------------------------------------------------------------===//
1921 // Calls to these methods are generated by tblgen.
1922 
1923 /// CheckAndMask - The isel is trying to match something like (and X, 255). If
1924 /// the dag combiner simplified the 255, we still want to match. RHS is the
1925 /// actual value in the DAG on the RHS of an AND, and DesiredMaskS is the value
1926 /// specified in the .td file (e.g. 255).
1928  int64_t DesiredMaskS) const {
1929  const APInt &ActualMask = RHS->getAPIntValue();
1930  const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS);
1931 
1932  // If the actual mask exactly matches, success!
1933  if (ActualMask == DesiredMask)
1934  return true;
1935 
1936  // If the actual AND mask is allowing unallowed bits, this doesn't match.
1937  if (!ActualMask.isSubsetOf(DesiredMask))
1938  return false;
1939 
1940  // Otherwise, the DAG Combiner may have proven that the value coming in is
1941  // either already zero or is not demanded. Check for known zero input bits.
1942  APInt NeededMask = DesiredMask & ~ActualMask;
1943  if (CurDAG->MaskedValueIsZero(LHS, NeededMask))
1944  return true;
1945 
1946  // TODO: check to see if missing bits are just not demanded.
1947 
1948  // Otherwise, this pattern doesn't match.
1949  return false;
1950 }
1951 
1952 /// CheckOrMask - The isel is trying to match something like (or X, 255). If
1953 /// the dag combiner simplified the 255, we still want to match. RHS is the
1954 /// actual value in the DAG on the RHS of an OR, and DesiredMaskS is the value
1955 /// specified in the .td file (e.g. 255).
1957  int64_t DesiredMaskS) const {
1958  const APInt &ActualMask = RHS->getAPIntValue();
1959  const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS);
1960 
1961  // If the actual mask exactly matches, success!
1962  if (ActualMask == DesiredMask)
1963  return true;
1964 
1965  // If the actual AND mask is allowing unallowed bits, this doesn't match.
1966  if (!ActualMask.isSubsetOf(DesiredMask))
1967  return false;
1968 
1969  // Otherwise, the DAG Combiner may have proven that the value coming in is
1970  // either already zero or is not demanded. Check for known zero input bits.
1971  APInt NeededMask = DesiredMask & ~ActualMask;
1972  KnownBits Known = CurDAG->computeKnownBits(LHS);
1973 
1974  // If all the missing bits in the or are already known to be set, match!
1975  if (NeededMask.isSubsetOf(Known.One))
1976  return true;
1977 
1978  // TODO: check to see if missing bits are just not demanded.
1979 
1980  // Otherwise, this pattern doesn't match.
1981  return false;
1982 }
1983 
1984 /// SelectInlineAsmMemoryOperands - Calls to this are automatically generated
1985 /// by tblgen. Others should not call it.
1987  const SDLoc &DL) {
1988  std::vector<SDValue> InOps;
1989  std::swap(InOps, Ops);
1990 
1991  Ops.push_back(InOps[InlineAsm::Op_InputChain]); // 0
1992  Ops.push_back(InOps[InlineAsm::Op_AsmString]); // 1
1993  Ops.push_back(InOps[InlineAsm::Op_MDNode]); // 2, !srcloc
1994  Ops.push_back(InOps[InlineAsm::Op_ExtraInfo]); // 3 (SideEffect, AlignStack)
1995 
1996  unsigned i = InlineAsm::Op_FirstOperand, e = InOps.size();
1997  if (InOps[e-1].getValueType() == MVT::Glue)
1998  --e; // Don't process a glue operand if it is here.
1999 
2000  while (i != e) {
2001  unsigned Flags = cast<ConstantSDNode>(InOps[i])->getZExtValue();
2002  if (!InlineAsm::isMemKind(Flags)) {
2003  // Just skip over this operand, copying the operands verbatim.
2004  Ops.insert(Ops.end(), InOps.begin()+i,
2005  InOps.begin()+i+InlineAsm::getNumOperandRegisters(Flags) + 1);
2006  i += InlineAsm::getNumOperandRegisters(Flags) + 1;
2007  } else {
2009  "Memory operand with multiple values?");
2010 
2011  unsigned TiedToOperand;
2012  if (InlineAsm::isUseOperandTiedToDef(Flags, TiedToOperand)) {
2013  // We need the constraint ID from the operand this is tied to.
2014  unsigned CurOp = InlineAsm::Op_FirstOperand;
2015  Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
2016  for (; TiedToOperand; --TiedToOperand) {
2017  CurOp += InlineAsm::getNumOperandRegisters(Flags)+1;
2018  Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
2019  }
2020  }
2021 
2022  // Otherwise, this is a memory operand. Ask the target to select it.
2023  std::vector<SDValue> SelOps;
2024  unsigned ConstraintID = InlineAsm::getMemoryConstraintID(Flags);
2025  if (SelectInlineAsmMemoryOperand(InOps[i+1], ConstraintID, SelOps))
2026  report_fatal_error("Could not match memory address. Inline asm"
2027  " failure!");
2028 
2029  // Add this to the output node.
2030  unsigned NewFlags =
2032  NewFlags = InlineAsm::getFlagWordForMem(NewFlags, ConstraintID);
2033  Ops.push_back(CurDAG->getTargetConstant(NewFlags, DL, MVT::i32));
2034  llvm::append_range(Ops, SelOps);
2035  i += 2;
2036  }
2037  }
2038 
2039  // Add the glue input back if present.
2040  if (e != InOps.size())
2041  Ops.push_back(InOps.back());
2042 }
2043 
2044 /// findGlueUse - Return use of MVT::Glue value produced by the specified
2045 /// SDNode.
2046 ///
2048  unsigned FlagResNo = N->getNumValues()-1;
2049  for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
2050  SDUse &Use = I.getUse();
2051  if (Use.getResNo() == FlagResNo)
2052  return Use.getUser();
2053  }
2054  return nullptr;
2055 }
2056 
2057 /// findNonImmUse - Return true if "Def" is a predecessor of "Root" via a path
2058 /// beyond "ImmedUse". We may ignore chains as they are checked separately.
2059 static bool findNonImmUse(SDNode *Root, SDNode *Def, SDNode *ImmedUse,
2060  bool IgnoreChains) {
2063  // Only check if we have non-immediate uses of Def.
2064  if (ImmedUse->isOnlyUserOf(Def))
2065  return false;
2066 
2067  // We don't care about paths to Def that go through ImmedUse so mark it
2068  // visited and mark non-def operands as used.
2069  Visited.insert(ImmedUse);
2070  for (const SDValue &Op : ImmedUse->op_values()) {
2071  SDNode *N = Op.getNode();
2072  // Ignore chain deps (they are validated by
2073  // HandleMergeInputChains) and immediate uses
2074  if ((Op.getValueType() == MVT::Other && IgnoreChains) || N == Def)
2075  continue;
2076  if (!Visited.insert(N).second)
2077  continue;
2078  WorkList.push_back(N);
2079  }
2080 
2081  // Initialize worklist to operands of Root.
2082  if (Root != ImmedUse) {
2083  for (const SDValue &Op : Root->op_values()) {
2084  SDNode *N = Op.getNode();
2085  // Ignore chains (they are validated by HandleMergeInputChains)
2086  if ((Op.getValueType() == MVT::Other && IgnoreChains) || N == Def)
2087  continue;
2088  if (!Visited.insert(N).second)
2089  continue;
2090  WorkList.push_back(N);
2091  }
2092  }
2093 
2094  return SDNode::hasPredecessorHelper(Def, Visited, WorkList, 0, true);
2095 }
2096 
2097 /// IsProfitableToFold - Returns true if it's profitable to fold the specific
2098 /// operand node N of U during instruction selection that starts at Root.
2100  SDNode *Root) const {
2101  if (OptLevel == CodeGenOpt::None) return false;
2102  return N.hasOneUse();
2103 }
2104 
2105 /// IsLegalToFold - Returns true if the specific operand node N of
2106 /// U can be folded during instruction selection that starts at Root.
2108  CodeGenOpt::Level OptLevel,
2109  bool IgnoreChains) {
2110  if (OptLevel == CodeGenOpt::None) return false;
2111 
2112  // If Root use can somehow reach N through a path that that doesn't contain
2113  // U then folding N would create a cycle. e.g. In the following
2114  // diagram, Root can reach N through X. If N is folded into Root, then
2115  // X is both a predecessor and a successor of U.
2116  //
2117  // [N*] //
2118  // ^ ^ //
2119  // / \ //
2120  // [U*] [X]? //
2121  // ^ ^ //
2122  // \ / //
2123  // \ / //
2124  // [Root*] //
2125  //
2126  // * indicates nodes to be folded together.
2127  //
2128  // If Root produces glue, then it gets (even more) interesting. Since it
2129  // will be "glued" together with its glue use in the scheduler, we need to
2130  // check if it might reach N.
2131  //
2132  // [N*] //
2133  // ^ ^ //
2134  // / \ //
2135  // [U*] [X]? //
2136  // ^ ^ //
2137  // \ \ //
2138  // \ | //
2139  // [Root*] | //
2140  // ^ | //
2141  // f | //
2142  // | / //
2143  // [Y] / //
2144  // ^ / //
2145  // f / //
2146  // | / //
2147  // [GU] //
2148  //
2149  // If GU (glue use) indirectly reaches N (the load), and Root folds N
2150  // (call it Fold), then X is a predecessor of GU and a successor of
2151  // Fold. But since Fold and GU are glued together, this will create
2152  // a cycle in the scheduling graph.
2153 
2154  // If the node has glue, walk down the graph to the "lowest" node in the
2155  // glueged set.
2156  EVT VT = Root->getValueType(Root->getNumValues()-1);
2157  while (VT == MVT::Glue) {
2158  SDNode *GU = findGlueUse(Root);
2159  if (!GU)
2160  break;
2161  Root = GU;
2162  VT = Root->getValueType(Root->getNumValues()-1);
2163 
2164  // If our query node has a glue result with a use, we've walked up it. If
2165  // the user (which has already been selected) has a chain or indirectly uses
2166  // the chain, HandleMergeInputChains will not consider it. Because of
2167  // this, we cannot ignore chains in this predicate.
2168  IgnoreChains = false;
2169  }
2170 
2171  return !findNonImmUse(Root, N.getNode(), U, IgnoreChains);
2172 }
2173 
2174 void SelectionDAGISel::Select_INLINEASM(SDNode *N) {
2175  SDLoc DL(N);
2176 
2177  std::vector<SDValue> Ops(N->op_begin(), N->op_end());
2179 
2180  const EVT VTs[] = {MVT::Other, MVT::Glue};
2181  SDValue New = CurDAG->getNode(N->getOpcode(), DL, VTs, Ops);
2182  New->setNodeId(-1);
2183  ReplaceUses(N, New.getNode());
2185 }
2186 
2187 void SelectionDAGISel::Select_READ_REGISTER(SDNode *Op) {
2188  SDLoc dl(Op);
2189  MDNodeSDNode *MD = cast<MDNodeSDNode>(Op->getOperand(1));
2190  const MDString *RegStr = cast<MDString>(MD->getMD()->getOperand(0));
2191 
2192  EVT VT = Op->getValueType(0);
2193  LLT Ty = VT.isSimple() ? getLLTForMVT(VT.getSimpleVT()) : LLT();
2194  Register Reg =
2195  TLI->getRegisterByName(RegStr->getString().data(), Ty,
2198  Op->getOperand(0), dl, Reg, Op->getValueType(0));
2199  New->setNodeId(-1);
2200  ReplaceUses(Op, New.getNode());
2202 }
2203 
2204 void SelectionDAGISel::Select_WRITE_REGISTER(SDNode *Op) {
2205  SDLoc dl(Op);
2206  MDNodeSDNode *MD = cast<MDNodeSDNode>(Op->getOperand(1));
2207  const MDString *RegStr = cast<MDString>(MD->getMD()->getOperand(0));
2208 
2209  EVT VT = Op->getOperand(2).getValueType();
2210  LLT Ty = VT.isSimple() ? getLLTForMVT(VT.getSimpleVT()) : LLT();
2211 
2212  Register Reg = TLI->getRegisterByName(RegStr->getString().data(), Ty,
2215  Op->getOperand(0), dl, Reg, Op->getOperand(2));
2216  New->setNodeId(-1);
2217  ReplaceUses(Op, New.getNode());
2219 }
2220 
2221 void SelectionDAGISel::Select_UNDEF(SDNode *N) {
2222  CurDAG->SelectNodeTo(N, TargetOpcode::IMPLICIT_DEF, N->getValueType(0));
2223 }
2224 
2225 void SelectionDAGISel::Select_FREEZE(SDNode *N) {
2226  // TODO: We don't have FREEZE pseudo-instruction in MachineInstr-level now.
2227  // If FREEZE instruction is added later, the code below must be changed as
2228  // well.
2229  CurDAG->SelectNodeTo(N, TargetOpcode::COPY, N->getValueType(0),
2230  N->getOperand(0));
2231 }
2232 
2233 void SelectionDAGISel::Select_ARITH_FENCE(SDNode *N) {
2234  CurDAG->SelectNodeTo(N, TargetOpcode::ARITH_FENCE, N->getValueType(0),
2235  N->getOperand(0));
2236 }
2237 
2238 /// GetVBR - decode a vbr encoding whose top bit is set.
2240 GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) {
2241  assert(Val >= 128 && "Not a VBR");
2242  Val &= 127; // Remove first vbr bit.
2243 
2244  unsigned Shift = 7;
2245  uint64_t NextBits;
2246  do {
2247  NextBits = MatcherTable[Idx++];
2248  Val |= (NextBits&127) << Shift;
2249  Shift += 7;
2250  } while (NextBits & 128);
2251 
2252  return Val;
2253 }
2254 
2255 /// When a match is complete, this method updates uses of interior chain results
2256 /// to use the new results.
2257 void SelectionDAGISel::UpdateChains(
2258  SDNode *NodeToMatch, SDValue InputChain,
2259  SmallVectorImpl<SDNode *> &ChainNodesMatched, bool isMorphNodeTo) {
2260  SmallVector<SDNode*, 4> NowDeadNodes;
2261 
2262  // Now that all the normal results are replaced, we replace the chain and
2263  // glue results if present.
2264  if (!ChainNodesMatched.empty()) {
2265  assert(InputChain.getNode() &&
2266  "Matched input chains but didn't produce a chain");
2267  // Loop over all of the nodes we matched that produced a chain result.
2268  // Replace all the chain results with the final chain we ended up with.
2269  for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) {
2270  SDNode *ChainNode = ChainNodesMatched[i];
2271  // If ChainNode is null, it's because we replaced it on a previous
2272  // iteration and we cleared it out of the map. Just skip it.
2273  if (!ChainNode)
2274  continue;
2275 
2276  assert(ChainNode->getOpcode() != ISD::DELETED_NODE &&
2277  "Deleted node left in chain");
2278 
2279  // Don't replace the results of the root node if we're doing a
2280  // MorphNodeTo.
2281  if (ChainNode == NodeToMatch && isMorphNodeTo)
2282  continue;
2283 
2284  SDValue ChainVal = SDValue(ChainNode, ChainNode->getNumValues()-1);
2285  if (ChainVal.getValueType() == MVT::Glue)
2286  ChainVal = ChainVal.getValue(ChainVal->getNumValues()-2);
2287  assert(ChainVal.getValueType() == MVT::Other && "Not a chain?");
2289  *CurDAG, [&](SDNode *N, SDNode *E) {
2290  std::replace(ChainNodesMatched.begin(), ChainNodesMatched.end(), N,
2291  static_cast<SDNode *>(nullptr));
2292  });
2293  if (ChainNode->getOpcode() != ISD::TokenFactor)
2294  ReplaceUses(ChainVal, InputChain);
2295 
2296  // If the node became dead and we haven't already seen it, delete it.
2297  if (ChainNode != NodeToMatch && ChainNode->use_empty() &&
2298  !llvm::is_contained(NowDeadNodes, ChainNode))
2299  NowDeadNodes.push_back(ChainNode);
2300  }
2301  }
2302 
2303  if (!NowDeadNodes.empty())
2304  CurDAG->RemoveDeadNodes(NowDeadNodes);
2305 
2306  LLVM_DEBUG(dbgs() << "ISEL: Match complete!\n");
2307 }
2308 
2309 /// HandleMergeInputChains - This implements the OPC_EmitMergeInputChains
2310 /// operation for when the pattern matched at least one node with a chains. The
2311 /// input vector contains a list of all of the chained nodes that we match. We
2312 /// must determine if this is a valid thing to cover (i.e. matching it won't
2313 /// induce cycles in the DAG) and if so, creating a TokenFactor node. that will
2314 /// be used as the input node chain for the generated nodes.
2315 static SDValue
2317  SelectionDAG *CurDAG) {
2318 
2321  SmallVector<SDValue, 3> InputChains;
2322  unsigned int Max = 8192;
2323 
2324  // Quick exit on trivial merge.
2325  if (ChainNodesMatched.size() == 1)
2326  return ChainNodesMatched[0]->getOperand(0);
2327 
2328  // Add chains that aren't already added (internal). Peek through
2329  // token factors.
2330  std::function<void(const SDValue)> AddChains = [&](const SDValue V) {
2331  if (V.getValueType() != MVT::Other)
2332  return;
2333  if (V->getOpcode() == ISD::EntryToken)
2334  return;
2335  if (!Visited.insert(V.getNode()).second)
2336  return;
2337  if (V->getOpcode() == ISD::TokenFactor) {
2338  for (const SDValue &Op : V->op_values())
2339  AddChains(Op);
2340  } else
2341  InputChains.push_back(V);
2342  };
2343 
2344  for (auto *N : ChainNodesMatched) {
2345  Worklist.push_back(N);
2346  Visited.insert(N);
2347  }
2348 
2349  while (!Worklist.empty())
2350  AddChains(Worklist.pop_back_val()->getOperand(0));
2351 
2352  // Skip the search if there are no chain dependencies.
2353  if (InputChains.size() == 0)
2354  return CurDAG->getEntryNode();
2355 
2356  // If one of these chains is a successor of input, we must have a
2357  // node that is both the predecessor and successor of the
2358  // to-be-merged nodes. Fail.
2359  Visited.clear();
2360  for (SDValue V : InputChains)
2361  Worklist.push_back(V.getNode());
2362 
2363  for (auto *N : ChainNodesMatched)
2364  if (SDNode::hasPredecessorHelper(N, Visited, Worklist, Max, true))
2365  return SDValue();
2366 
2367  // Return merged chain.
2368  if (InputChains.size() == 1)
2369  return InputChains[0];
2370  return CurDAG->getNode(ISD::TokenFactor, SDLoc(ChainNodesMatched[0]),
2371  MVT::Other, InputChains);
2372 }
2373 
2374 /// MorphNode - Handle morphing a node in place for the selector.
2375 SDNode *SelectionDAGISel::
2376 MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList,
2377  ArrayRef<SDValue> Ops, unsigned EmitNodeInfo) {
2378  // It is possible we're using MorphNodeTo to replace a node with no
2379  // normal results with one that has a normal result (or we could be
2380  // adding a chain) and the input could have glue and chains as well.
2381  // In this case we need to shift the operands down.
2382  // FIXME: This is a horrible hack and broken in obscure cases, no worse
2383  // than the old isel though.
2384  int OldGlueResultNo = -1, OldChainResultNo = -1;
2385 
2386  unsigned NTMNumResults = Node->getNumValues();
2387  if (Node->getValueType(NTMNumResults-1) == MVT::Glue) {
2388  OldGlueResultNo = NTMNumResults-1;
2389  if (NTMNumResults != 1 &&
2390  Node->getValueType(NTMNumResults-2) == MVT::Other)
2391  OldChainResultNo = NTMNumResults-2;
2392  } else if (Node->getValueType(NTMNumResults-1) == MVT::Other)
2393  OldChainResultNo = NTMNumResults-1;
2394 
2395  // Call the underlying SelectionDAG routine to do the transmogrification. Note
2396  // that this deletes operands of the old node that become dead.
2397  SDNode *Res = CurDAG->MorphNodeTo(Node, ~TargetOpc, VTList, Ops);
2398 
2399  // MorphNodeTo can operate in two ways: if an existing node with the
2400  // specified operands exists, it can just return it. Otherwise, it
2401  // updates the node in place to have the requested operands.
2402  if (Res == Node) {
2403  // If we updated the node in place, reset the node ID. To the isel,
2404  // this should be just like a newly allocated machine node.
2405  Res->setNodeId(-1);
2406  }
2407 
2408  unsigned ResNumResults = Res->getNumValues();
2409  // Move the glue if needed.
2410  if ((EmitNodeInfo & OPFL_GlueOutput) && OldGlueResultNo != -1 &&
2411  (unsigned)OldGlueResultNo != ResNumResults-1)
2412  ReplaceUses(SDValue(Node, OldGlueResultNo),
2413  SDValue(Res, ResNumResults - 1));
2414 
2415  if ((EmitNodeInfo & OPFL_GlueOutput) != 0)
2416  --ResNumResults;
2417 
2418  // Move the chain reference if needed.
2419  if ((EmitNodeInfo & OPFL_Chain) && OldChainResultNo != -1 &&
2420  (unsigned)OldChainResultNo != ResNumResults-1)
2421  ReplaceUses(SDValue(Node, OldChainResultNo),
2422  SDValue(Res, ResNumResults - 1));
2423 
2424  // Otherwise, no replacement happened because the node already exists. Replace
2425  // Uses of the old node with the new one.
2426  if (Res != Node) {
2427  ReplaceNode(Node, Res);
2428  } else {
2430  }
2431 
2432  return Res;
2433 }
2434 
2435 /// CheckSame - Implements OP_CheckSame.
2436 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2437 CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2438  const SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes) {
2439  // Accept if it is exactly the same as a previously recorded node.
2440  unsigned RecNo = MatcherTable[MatcherIndex++];
2441  assert(RecNo < RecordedNodes.size() && "Invalid CheckSame");
2442  return N == RecordedNodes[RecNo].first;
2443 }
2444 
2445 /// CheckChildSame - Implements OP_CheckChildXSame.
2447  const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2448  const SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes,
2449  unsigned ChildNo) {
2450  if (ChildNo >= N.getNumOperands())
2451  return false; // Match fails if out of range child #.
2452  return ::CheckSame(MatcherTable, MatcherIndex, N.getOperand(ChildNo),
2453  RecordedNodes);
2454 }
2455 
2456 /// CheckPatternPredicate - Implements OP_CheckPatternPredicate.
2457 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2458 CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2459  const SelectionDAGISel &SDISel) {
2460  return SDISel.CheckPatternPredicate(MatcherTable[MatcherIndex++]);
2461 }
2462 
2463 /// CheckNodePredicate - Implements OP_CheckNodePredicate.
2464 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2465 CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2466  const SelectionDAGISel &SDISel, SDNode *N) {
2467  return SDISel.CheckNodePredicate(N, MatcherTable[MatcherIndex++]);
2468 }
2469 
2470 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2471 CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2472  SDNode *N) {
2473  uint16_t Opc = MatcherTable[MatcherIndex++];
2474  Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
2475  return N->getOpcode() == Opc;
2476 }
2477 
2478 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2479 CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2480  const TargetLowering *TLI, const DataLayout &DL) {
2481  MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
2482  if (N.getValueType() == VT) return true;
2483 
2484  // Handle the case when VT is iPTR.
2485  return VT == MVT::iPTR && N.getValueType() == TLI->getPointerTy(DL);
2486 }
2487 
2488 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2489 CheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2490  SDValue N, const TargetLowering *TLI, const DataLayout &DL,
2491  unsigned ChildNo) {
2492  if (ChildNo >= N.getNumOperands())
2493  return false; // Match fails if out of range child #.
2494  return ::CheckType(MatcherTable, MatcherIndex, N.getOperand(ChildNo), TLI,
2495  DL);
2496 }
2497 
2498 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2499 CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2500  SDValue N) {
2501  return cast<CondCodeSDNode>(N)->get() ==
2502  (ISD::CondCode)MatcherTable[MatcherIndex++];
2503 }
2504 
2505 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2506 CheckChild2CondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2507  SDValue N) {
2508  if (2 >= N.getNumOperands())
2509  return false;
2510  return ::CheckCondCode(MatcherTable, MatcherIndex, N.getOperand(2));
2511 }
2512 
2513 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2514 CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2515  SDValue N, const TargetLowering *TLI, const DataLayout &DL) {
2516  MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
2517  if (cast<VTSDNode>(N)->getVT() == VT)
2518  return true;
2519 
2520  // Handle the case when VT is iPTR.
2521  return VT == MVT::iPTR && cast<VTSDNode>(N)->getVT() == TLI->getPointerTy(DL);
2522 }
2523 
2524 // Bit 0 stores the sign of the immediate. The upper bits contain the magnitude
2525 // shifted left by 1.
2527  if ((V & 1) == 0)
2528  return V >> 1;
2529  if (V != 1)
2530  return -(V >> 1);
2531  // There is no such thing as -0 with integers. "-0" really means MININT.
2532  return 1ULL << 63;
2533 }
2534 
2535 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2536 CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2537  SDValue N) {
2538  int64_t Val = MatcherTable[MatcherIndex++];
2539  if (Val & 128)
2540  Val = GetVBR(Val, MatcherTable, MatcherIndex);
2541 
2542  Val = decodeSignRotatedValue(Val);
2543 
2544  ConstantSDNode *C = dyn_cast<ConstantSDNode>(N);
2545  return C && C->getSExtValue() == Val;
2546 }
2547 
2548 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2549 CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2550  SDValue N, unsigned ChildNo) {
2551  if (ChildNo >= N.getNumOperands())
2552  return false; // Match fails if out of range child #.
2553  return ::CheckInteger(MatcherTable, MatcherIndex, N.getOperand(ChildNo));
2554 }
2555 
2556 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2557 CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2558  SDValue N, const SelectionDAGISel &SDISel) {
2559  int64_t Val = MatcherTable[MatcherIndex++];
2560  if (Val & 128)
2561  Val = GetVBR(Val, MatcherTable, MatcherIndex);
2562 
2563  if (N->getOpcode() != ISD::AND) return false;
2564 
2565  ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2566  return C && SDISel.CheckAndMask(N.getOperand(0), C, Val);
2567 }
2568 
2569 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2570 CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2571  const SelectionDAGISel &SDISel) {
2572  int64_t Val = MatcherTable[MatcherIndex++];
2573  if (Val & 128)
2574  Val = GetVBR(Val, MatcherTable, MatcherIndex);
2575 
2576  if (N->getOpcode() != ISD::OR) return false;
2577 
2578  ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2579  return C && SDISel.CheckOrMask(N.getOperand(0), C, Val);
2580 }
2581 
2582 /// IsPredicateKnownToFail - If we know how and can do so without pushing a
2583 /// scope, evaluate the current node. If the current predicate is known to
2584 /// fail, set Result=true and return anything. If the current predicate is
2585 /// known to pass, set Result=false and return the MatcherIndex to continue
2586 /// with. If the current predicate is unknown, set Result=false and return the
2587 /// MatcherIndex to continue with.
2588 static unsigned IsPredicateKnownToFail(const unsigned char *Table,
2589  unsigned Index, SDValue N,
2590  bool &Result,
2591  const SelectionDAGISel &SDISel,
2592  SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes) {
2593  switch (Table[Index++]) {
2594  default:
2595  Result = false;
2596  return Index-1; // Could not evaluate this predicate.
2598  Result = !::CheckSame(Table, Index, N, RecordedNodes);
2599  return Index;
2604  Result = !::CheckChildSame(Table, Index, N, RecordedNodes,
2606  return Index;
2608  Result = !::CheckPatternPredicate(Table, Index, SDISel);
2609  return Index;
2611  Result = !::CheckNodePredicate(Table, Index, SDISel, N.getNode());
2612  return Index;
2614  Result = !::CheckOpcode(Table, Index, N.getNode());
2615  return Index;
2617  Result = !::CheckType(Table, Index, N, SDISel.TLI,
2618  SDISel.CurDAG->getDataLayout());
2619  return Index;
2621  unsigned Res = Table[Index++];
2622  Result = !::CheckType(Table, Index, N.getValue(Res), SDISel.TLI,
2623  SDISel.CurDAG->getDataLayout());
2624  return Index;
2625  }
2634  Result = !::CheckChildType(
2635  Table, Index, N, SDISel.TLI, SDISel.CurDAG->getDataLayout(),
2637  return Index;
2639  Result = !::CheckCondCode(Table, Index, N);
2640  return Index;
2642  Result = !::CheckChild2CondCode(Table, Index, N);
2643  return Index;
2645  Result = !::CheckValueType(Table, Index, N, SDISel.TLI,
2646  SDISel.CurDAG->getDataLayout());
2647  return Index;
2649  Result = !::CheckInteger(Table, Index, N);
2650  return Index;
2656  Result = !::CheckChildInteger(Table, Index, N,
2658  return Index;
2660  Result = !::CheckAndImm(Table, Index, N, SDISel);
2661  return Index;
2663  Result = !::CheckOrImm(Table, Index, N, SDISel);
2664  return Index;
2665  }
2666 }
2667 
2668 namespace {
2669 
2670 struct MatchScope {
2671  /// FailIndex - If this match fails, this is the index to continue with.
2672  unsigned FailIndex;
2673 
2674  /// NodeStack - The node stack when the scope was formed.
2675  SmallVector<SDValue, 4> NodeStack;
2676 
2677  /// NumRecordedNodes - The number of recorded nodes when the scope was formed.
2678  unsigned NumRecordedNodes;
2679 
2680  /// NumMatchedMemRefs - The number of matched memref entries.
2681  unsigned NumMatchedMemRefs;
2682 
2683  /// InputChain/InputGlue - The current chain/glue
2684  SDValue InputChain, InputGlue;
2685 
2686  /// HasChainNodesMatched - True if the ChainNodesMatched list is non-empty.
2687  bool HasChainNodesMatched;
2688 };
2689 
2690 /// \A DAG update listener to keep the matching state
2691 /// (i.e. RecordedNodes and MatchScope) uptodate if the target is allowed to
2692 /// change the DAG while matching. X86 addressing mode matcher is an example
2693 /// for this.
2694 class MatchStateUpdater : public SelectionDAG::DAGUpdateListener
2695 {
2696  SDNode **NodeToMatch;
2698  SmallVectorImpl<MatchScope> &MatchScopes;
2699 
2700 public:
2701  MatchStateUpdater(SelectionDAG &DAG, SDNode **NodeToMatch,
2702  SmallVectorImpl<std::pair<SDValue, SDNode *>> &RN,
2704  : SelectionDAG::DAGUpdateListener(DAG), NodeToMatch(NodeToMatch),
2705  RecordedNodes(RN), MatchScopes(MS) {}
2706 
2707  void NodeDeleted(SDNode *N, SDNode *E) override {
2708  // Some early-returns here to avoid the search if we deleted the node or
2709  // if the update comes from MorphNodeTo (MorphNodeTo is the last thing we
2710  // do, so it's unnecessary to update matching state at that point).
2711  // Neither of these can occur currently because we only install this
2712  // update listener during matching a complex patterns.
2713  if (!E || E->isMachineOpcode())
2714  return;
2715  // Check if NodeToMatch was updated.
2716  if (N == *NodeToMatch)
2717  *NodeToMatch = E;
2718  // Performing linear search here does not matter because we almost never
2719  // run this code. You'd have to have a CSE during complex pattern
2720  // matching.
2721  for (auto &I : RecordedNodes)
2722  if (I.first.getNode() == N)
2723  I.first.setNode(E);
2724 
2725  for (auto &I : MatchScopes)
2726  for (auto &J : I.NodeStack)
2727  if (J.getNode() == N)
2728  J.setNode(E);
2729  }
2730 };
2731 
2732 } // end anonymous namespace
2733 
2735  const unsigned char *MatcherTable,
2736  unsigned TableSize) {
2737  // FIXME: Should these even be selected? Handle these cases in the caller?
2738  switch (NodeToMatch->getOpcode()) {
2739  default:
2740  break;
2741  case ISD::EntryToken: // These nodes remain the same.
2742  case ISD::BasicBlock:
2743  case ISD::Register:
2744  case ISD::RegisterMask:
2745  case ISD::HANDLENODE:
2746  case ISD::MDNODE_SDNODE:
2747  case ISD::TargetConstant:
2748  case ISD::TargetConstantFP:
2750  case ISD::TargetFrameIndex:
2752  case ISD::MCSymbol:
2754  case ISD::TargetJumpTable:
2757  case ISD::TokenFactor:
2758  case ISD::CopyFromReg:
2759  case ISD::CopyToReg:
2760  case ISD::EH_LABEL:
2761  case ISD::ANNOTATION_LABEL:
2762  case ISD::LIFETIME_START:
2763  case ISD::LIFETIME_END:
2764  case ISD::PSEUDO_PROBE:
2765  NodeToMatch->setNodeId(-1); // Mark selected.
2766  return;
2767  case ISD::AssertSext:
2768  case ISD::AssertZext:
2769  case ISD::AssertAlign:
2770  ReplaceUses(SDValue(NodeToMatch, 0), NodeToMatch->getOperand(0));
2771  CurDAG->RemoveDeadNode(NodeToMatch);
2772  return;
2773  case ISD::INLINEASM:
2774  case ISD::INLINEASM_BR:
2775  Select_INLINEASM(NodeToMatch);
2776  return;
2777  case ISD::READ_REGISTER:
2778  Select_READ_REGISTER(NodeToMatch);
2779  return;
2780  case ISD::WRITE_REGISTER:
2781  Select_WRITE_REGISTER(NodeToMatch);
2782  return;
2783  case ISD::UNDEF:
2784  Select_UNDEF(NodeToMatch);
2785  return;
2786  case ISD::FREEZE:
2787  Select_FREEZE(NodeToMatch);
2788  return;
2789  case ISD::ARITH_FENCE:
2790  Select_ARITH_FENCE(NodeToMatch);
2791  return;
2792  }
2793 
2794  assert(!NodeToMatch->isMachineOpcode() && "Node already selected!");
2795 
2796  // Set up the node stack with NodeToMatch as the only node on the stack.
2797  SmallVector<SDValue, 8> NodeStack;
2798  SDValue N = SDValue(NodeToMatch, 0);
2799  NodeStack.push_back(N);
2800 
2801  // MatchScopes - Scopes used when matching, if a match failure happens, this
2802  // indicates where to continue checking.
2803  SmallVector<MatchScope, 8> MatchScopes;
2804 
2805  // RecordedNodes - This is the set of nodes that have been recorded by the
2806  // state machine. The second value is the parent of the node, or null if the
2807  // root is recorded.
2808  SmallVector<std::pair<SDValue, SDNode*>, 8> RecordedNodes;
2809 
2810  // MatchedMemRefs - This is the set of MemRef's we've seen in the input
2811  // pattern.
2812  SmallVector<MachineMemOperand*, 2> MatchedMemRefs;
2813 
2814  // These are the current input chain and glue for use when generating nodes.
2815  // Various Emit operations change these. For example, emitting a copytoreg
2816  // uses and updates these.
2817  SDValue InputChain, InputGlue;
2818 
2819  // ChainNodesMatched - If a pattern matches nodes that have input/output
2820  // chains, the OPC_EmitMergeInputChains operation is emitted which indicates
2821  // which ones they are. The result is captured into this list so that we can
2822  // update the chain results when the pattern is complete.
2823  SmallVector<SDNode*, 3> ChainNodesMatched;
2824 
2825  LLVM_DEBUG(dbgs() << "ISEL: Starting pattern match\n");
2826 
2827  // Determine where to start the interpreter. Normally we start at opcode #0,
2828  // but if the state machine starts with an OPC_SwitchOpcode, then we
2829  // accelerate the first lookup (which is guaranteed to be hot) with the
2830  // OpcodeOffset table.
2831  unsigned MatcherIndex = 0;
2832 
2833  if (!OpcodeOffset.empty()) {
2834  // Already computed the OpcodeOffset table, just index into it.
2835  if (N.getOpcode() < OpcodeOffset.size())
2836  MatcherIndex = OpcodeOffset[N.getOpcode()];
2837  LLVM_DEBUG(dbgs() << " Initial Opcode index to " << MatcherIndex << "\n");
2838 
2839  } else if (MatcherTable[0] == OPC_SwitchOpcode) {
2840  // Otherwise, the table isn't computed, but the state machine does start
2841  // with an OPC_SwitchOpcode instruction. Populate the table now, since this
2842  // is the first time we're selecting an instruction.
2843  unsigned Idx = 1;
2844  while (true) {
2845  // Get the size of this case.
2846  unsigned CaseSize = MatcherTable[Idx++];
2847  if (CaseSize & 128)
2848  CaseSize = GetVBR(CaseSize, MatcherTable, Idx);
2849  if (CaseSize == 0) break;
2850 
2851  // Get the opcode, add the index to the table.
2852  uint16_t Opc = MatcherTable[Idx++];
2853  Opc |= (unsigned short)MatcherTable[Idx++] << 8;
2854  if (Opc >= OpcodeOffset.size())
2855  OpcodeOffset.resize((Opc+1)*2);
2856  OpcodeOffset[Opc] = Idx;
2857  Idx += CaseSize;
2858  }
2859 
2860  // Okay, do the lookup for the first opcode.
2861  if (N.getOpcode() < OpcodeOffset.size())
2862  MatcherIndex = OpcodeOffset[N.getOpcode()];
2863  }
2864 
2865  while (true) {
2866  assert(MatcherIndex < TableSize && "Invalid index");
2867 #ifndef NDEBUG
2868  unsigned CurrentOpcodeIndex = MatcherIndex;
2869 #endif
2870  BuiltinOpcodes Opcode = (BuiltinOpcodes)MatcherTable[MatcherIndex++];
2871  switch (Opcode) {
2872  case OPC_Scope: {
2873  // Okay, the semantics of this operation are that we should push a scope
2874  // then evaluate the first child. However, pushing a scope only to have
2875  // the first check fail (which then pops it) is inefficient. If we can
2876  // determine immediately that the first check (or first several) will
2877  // immediately fail, don't even bother pushing a scope for them.
2878  unsigned FailIndex;
2879 
2880  while (true) {
2881  unsigned NumToSkip = MatcherTable[MatcherIndex++];
2882  if (NumToSkip & 128)
2883  NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex);
2884  // Found the end of the scope with no match.
2885  if (NumToSkip == 0) {
2886  FailIndex = 0;
2887  break;
2888  }
2889 
2890  FailIndex = MatcherIndex+NumToSkip;
2891 
2892  unsigned MatcherIndexOfPredicate = MatcherIndex;
2893  (void)MatcherIndexOfPredicate; // silence warning.
2894 
2895  // If we can't evaluate this predicate without pushing a scope (e.g. if
2896  // it is a 'MoveParent') or if the predicate succeeds on this node, we
2897  // push the scope and evaluate the full predicate chain.
2898  bool Result;
2899  MatcherIndex = IsPredicateKnownToFail(MatcherTable, MatcherIndex, N,
2900  Result, *this, RecordedNodes);
2901  if (!Result)
2902  break;
2903 
2904  LLVM_DEBUG(
2905  dbgs() << " Skipped scope entry (due to false predicate) at "
2906  << "index " << MatcherIndexOfPredicate << ", continuing at "
2907  << FailIndex << "\n");
2908  ++NumDAGIselRetries;
2909 
2910  // Otherwise, we know that this case of the Scope is guaranteed to fail,
2911  // move to the next case.
2912  MatcherIndex = FailIndex;
2913  }
2914 
2915  // If the whole scope failed to match, bail.
2916  if (FailIndex == 0) break;
2917 
2918  // Push a MatchScope which indicates where to go if the first child fails
2919  // to match.
2920  MatchScope NewEntry;
2921  NewEntry.FailIndex = FailIndex;
2922  NewEntry.NodeStack.append(NodeStack.begin(), NodeStack.end());
2923  NewEntry.NumRecordedNodes = RecordedNodes.size();
2924  NewEntry.NumMatchedMemRefs = MatchedMemRefs.size();
2925  NewEntry.InputChain = InputChain;
2926  NewEntry.InputGlue = InputGlue;
2927  NewEntry.HasChainNodesMatched = !ChainNodesMatched.empty();
2928  MatchScopes.push_back(NewEntry);
2929  continue;
2930  }
2931  case OPC_RecordNode: {
2932  // Remember this node, it may end up being an operand in the pattern.
2933  SDNode *Parent = nullptr;
2934  if (NodeStack.size() > 1)
2935  Parent = NodeStack[NodeStack.size()-2].getNode();
2936  RecordedNodes.push_back(std::make_pair(N, Parent));
2937  continue;
2938  }
2939 
2943  case OPC_RecordChild6: case OPC_RecordChild7: {
2944  unsigned ChildNo = Opcode-OPC_RecordChild0;
2945  if (ChildNo >= N.getNumOperands())
2946  break; // Match fails if out of range child #.
2947 
2948  RecordedNodes.push_back(std::make_pair(N->getOperand(ChildNo),
2949  N.getNode()));
2950  continue;
2951  }
2952  case OPC_RecordMemRef:
2953  if (auto *MN = dyn_cast<MemSDNode>(N))
2954  MatchedMemRefs.push_back(MN->getMemOperand());
2955  else {
2956  LLVM_DEBUG(dbgs() << "Expected MemSDNode "; N->dump(CurDAG);
2957  dbgs() << '\n');
2958  }
2959 
2960  continue;
2961 
2962  case OPC_CaptureGlueInput:
2963  // If the current node has an input glue, capture it in InputGlue.
2964  if (N->getNumOperands() != 0 &&
2965  N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Glue)
2966  InputGlue = N->getOperand(N->getNumOperands()-1);
2967  continue;
2968 
2969  case OPC_MoveChild: {
2970  unsigned ChildNo = MatcherTable[MatcherIndex++];
2971  if (ChildNo >= N.getNumOperands())
2972  break; // Match fails if out of range child #.
2973  N = N.getOperand(ChildNo);
2974  NodeStack.push_back(N);
2975  continue;
2976  }
2977 
2978  case OPC_MoveChild0: case OPC_MoveChild1:
2979  case OPC_MoveChild2: case OPC_MoveChild3:
2980  case OPC_MoveChild4: case OPC_MoveChild5:
2981  case OPC_MoveChild6: case OPC_MoveChild7: {
2982  unsigned ChildNo = Opcode-OPC_MoveChild0;
2983  if (ChildNo >= N.getNumOperands())
2984  break; // Match fails if out of range child #.
2985  N = N.getOperand(ChildNo);
2986  NodeStack.push_back(N);
2987  continue;
2988  }
2989 
2990  case OPC_MoveParent:
2991  // Pop the current node off the NodeStack.
2992  NodeStack.pop_back();
2993  assert(!NodeStack.empty() && "Node stack imbalance!");
2994  N = NodeStack.back();
2995  continue;
2996 
2997  case OPC_CheckSame:
2998  if (!::CheckSame(MatcherTable, MatcherIndex, N, RecordedNodes)) break;
2999  continue;
3000 
3003  if (!::CheckChildSame(MatcherTable, MatcherIndex, N, RecordedNodes,
3004  Opcode-OPC_CheckChild0Same))
3005  break;
3006  continue;
3007 
3009  if (!::CheckPatternPredicate(MatcherTable, MatcherIndex, *this)) break;
3010  continue;
3011  case OPC_CheckPredicate:
3012  if (!::CheckNodePredicate(MatcherTable, MatcherIndex, *this,
3013  N.getNode()))
3014  break;
3015  continue;
3017  unsigned OpNum = MatcherTable[MatcherIndex++];
3019 
3020  for (unsigned i = 0; i < OpNum; ++i)
3021  Operands.push_back(RecordedNodes[MatcherTable[MatcherIndex++]].first);
3022 
3023  unsigned PredNo = MatcherTable[MatcherIndex++];
3024  if (!CheckNodePredicateWithOperands(N.getNode(), PredNo, Operands))
3025  break;
3026  continue;
3027  }
3028  case OPC_CheckComplexPat: {
3029  unsigned CPNum = MatcherTable[MatcherIndex++];
3030  unsigned RecNo = MatcherTable[MatcherIndex++];
3031  assert(RecNo < RecordedNodes.size() && "Invalid CheckComplexPat");
3032 
3033  // If target can modify DAG during matching, keep the matching state
3034  // consistent.
3035  std::unique_ptr<MatchStateUpdater> MSU;
3037  MSU.reset(new MatchStateUpdater(*CurDAG, &NodeToMatch, RecordedNodes,
3038  MatchScopes));
3039 
3040  if (!CheckComplexPattern(NodeToMatch, RecordedNodes[RecNo].second,
3041  RecordedNodes[RecNo].first, CPNum,
3042  RecordedNodes))
3043  break;
3044  continue;
3045  }
3046  case OPC_CheckOpcode:
3047  if (!::CheckOpcode(MatcherTable, MatcherIndex, N.getNode())) break;
3048  continue;
3049 
3050  case OPC_CheckType:
3051  if (!::CheckType(MatcherTable, MatcherIndex, N, TLI,
3052  CurDAG->getDataLayout()))
3053  break;
3054  continue;
3055 
3056  case OPC_CheckTypeRes: {
3057  unsigned Res = MatcherTable[MatcherIndex++];
3058  if (!::CheckType(MatcherTable, MatcherIndex, N.getValue(Res), TLI,
3059  CurDAG->getDataLayout()))
3060  break;
3061  continue;
3062  }
3063 
3064  case OPC_SwitchOpcode: {
3065  unsigned CurNodeOpcode = N.getOpcode();
3066  unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3067  unsigned CaseSize;
3068  while (true) {
3069  // Get the size of this case.
3070  CaseSize = MatcherTable[MatcherIndex++];
3071  if (CaseSize & 128)
3072  CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);
3073  if (CaseSize == 0) break;
3074 
3075  uint16_t Opc = MatcherTable[MatcherIndex++];
3076  Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
3077 
3078  // If the opcode matches, then we will execute this case.
3079  if (CurNodeOpcode == Opc)
3080  break;
3081 
3082  // Otherwise, skip over this case.
3083  MatcherIndex += CaseSize;
3084  }
3085 
3086  // If no cases matched, bail out.
3087  if (CaseSize == 0) break;
3088 
3089  // Otherwise, execute the case we found.
3090  LLVM_DEBUG(dbgs() << " OpcodeSwitch from " << SwitchStart << " to "
3091  << MatcherIndex << "\n");
3092  continue;
3093  }
3094 
3095  case OPC_SwitchType: {
3096  MVT CurNodeVT = N.getSimpleValueType();
3097  unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3098  unsigned CaseSize;
3099  while (true) {
3100  // Get the size of this case.
3101  CaseSize = MatcherTable[MatcherIndex++];
3102  if (CaseSize & 128)
3103  CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);
3104  if (CaseSize == 0) break;
3105 
3106  MVT CaseVT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3107  if (CaseVT == MVT::iPTR)
3108  CaseVT = TLI->getPointerTy(CurDAG->getDataLayout());
3109 
3110  // If the VT matches, then we will execute this case.
3111  if (CurNodeVT == CaseVT)
3112  break;
3113 
3114  // Otherwise, skip over this case.
3115  MatcherIndex += CaseSize;
3116  }
3117 
3118  // If no cases matched, bail out.
3119  if (CaseSize == 0) break;
3120 
3121  // Otherwise, execute the case we found.
3122  LLVM_DEBUG(dbgs() << " TypeSwitch[" << EVT(CurNodeVT).getEVTString()
3123  << "] from " << SwitchStart << " to " << MatcherIndex
3124  << '\n');
3125  continue;
3126  }
3131  if (!::CheckChildType(MatcherTable, MatcherIndex, N, TLI,
3132  CurDAG->getDataLayout(),
3133  Opcode - OPC_CheckChild0Type))
3134  break;
3135  continue;
3136  case OPC_CheckCondCode:
3137  if (!::CheckCondCode(MatcherTable, MatcherIndex, N)) break;
3138  continue;
3140  if (!::CheckChild2CondCode(MatcherTable, MatcherIndex, N)) break;
3141  continue;
3142  case OPC_CheckValueType:
3143  if (!::CheckValueType(MatcherTable, MatcherIndex, N, TLI,
3144  CurDAG->getDataLayout()))
3145  break;
3146  continue;
3147  case OPC_CheckInteger:
3148  if (!::CheckInteger(MatcherTable, MatcherIndex, N)) break;
3149  continue;
3153  if (!::CheckChildInteger(MatcherTable, MatcherIndex, N,
3154  Opcode-OPC_CheckChild0Integer)) break;
3155  continue;
3156  case OPC_CheckAndImm:
3157  if (!::CheckAndImm(MatcherTable, MatcherIndex, N, *this)) break;
3158  continue;
3159  case OPC_CheckOrImm:
3160  if (!::CheckOrImm(MatcherTable, MatcherIndex, N, *this)) break;
3161  continue;
3162  case OPC_CheckImmAllOnesV:
3163  if (!ISD::isConstantSplatVectorAllOnes(N.getNode()))
3164  break;
3165  continue;
3166  case OPC_CheckImmAllZerosV:
3167  if (!ISD::isConstantSplatVectorAllZeros(N.getNode()))
3168  break;
3169  continue;
3170 
3172  assert(NodeStack.size() != 1 && "No parent node");
3173  // Verify that all intermediate nodes between the root and this one have
3174  // a single use (ignoring chains, which are handled in UpdateChains).
3175  bool HasMultipleUses = false;
3176  for (unsigned i = 1, e = NodeStack.size()-1; i != e; ++i) {
3177  unsigned NNonChainUses = 0;
3178  SDNode *NS = NodeStack[i].getNode();
3179  for (auto UI = NS->use_begin(), UE = NS->use_end(); UI != UE; ++UI)
3180  if (UI.getUse().getValueType() != MVT::Other)
3181  if (++NNonChainUses > 1) {
3182  HasMultipleUses = true;
3183  break;
3184  }
3185  if (HasMultipleUses) break;
3186  }
3187  if (HasMultipleUses) break;
3188 
3189  // Check to see that the target thinks this is profitable to fold and that
3190  // we can fold it without inducing cycles in the graph.
3191  if (!IsProfitableToFold(N, NodeStack[NodeStack.size()-2].getNode(),
3192  NodeToMatch) ||
3193  !IsLegalToFold(N, NodeStack[NodeStack.size()-2].getNode(),
3194  NodeToMatch, OptLevel,
3195  true/*We validate our own chains*/))
3196  break;
3197 
3198  continue;
3199  }
3200  case OPC_EmitInteger:
3201  case OPC_EmitStringInteger: {
3203  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3204  int64_t Val = MatcherTable[MatcherIndex++];
3205  if (Val & 128)
3206  Val = GetVBR(Val, MatcherTable, MatcherIndex);
3207  if (Opcode == OPC_EmitInteger)
3208  Val = decodeSignRotatedValue(Val);
3209  RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3210  CurDAG->getTargetConstant(Val, SDLoc(NodeToMatch),
3211  VT), nullptr));
3212  continue;
3213  }
3214  case OPC_EmitRegister: {
3216  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3217  unsigned RegNo = MatcherTable[MatcherIndex++];
3218  RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3219  CurDAG->getRegister(RegNo, VT), nullptr));
3220  continue;
3221  }
3222  case OPC_EmitRegister2: {
3223  // For targets w/ more than 256 register names, the register enum
3224  // values are stored in two bytes in the matcher table (just like
3225  // opcodes).
3227  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3228  unsigned RegNo = MatcherTable[MatcherIndex++];
3229  RegNo |= MatcherTable[MatcherIndex++] << 8;
3230  RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3231  CurDAG->getRegister(RegNo, VT), nullptr));
3232  continue;
3233  }
3234 
3235  case OPC_EmitConvertToTarget: {
3236  // Convert from IMM/FPIMM to target version.
3237  unsigned RecNo = MatcherTable[MatcherIndex++];
3238  assert(RecNo < RecordedNodes.size() && "Invalid EmitConvertToTarget");
3239  SDValue Imm = RecordedNodes[RecNo].first;
3240 
3241  if (Imm->getOpcode() == ISD::Constant) {
3242  const ConstantInt *Val=cast<ConstantSDNode>(Imm)->getConstantIntValue();
3243  Imm = CurDAG->getTargetConstant(*Val, SDLoc(NodeToMatch),
3244  Imm.getValueType());
3245  } else if (Imm->getOpcode() == ISD::ConstantFP) {
3246  const ConstantFP *Val=cast<ConstantFPSDNode>(Imm)->getConstantFPValue();
3247  Imm = CurDAG->getTargetConstantFP(*Val, SDLoc(NodeToMatch),
3248  Imm.getValueType());
3249  }
3250 
3251  RecordedNodes.push_back(std::make_pair(Imm, RecordedNodes[RecNo].second));
3252  continue;
3253  }
3254 
3255  case OPC_EmitMergeInputChains1_0: // OPC_EmitMergeInputChains, 1, 0
3256  case OPC_EmitMergeInputChains1_1: // OPC_EmitMergeInputChains, 1, 1
3257  case OPC_EmitMergeInputChains1_2: { // OPC_EmitMergeInputChains, 1, 2
3258  // These are space-optimized forms of OPC_EmitMergeInputChains.
3259  assert(!InputChain.getNode() &&
3260  "EmitMergeInputChains should be the first chain producing node");
3261  assert(ChainNodesMatched.empty() &&
3262  "Should only have one EmitMergeInputChains per match");
3263 
3264  // Read all of the chained nodes.
3265  unsigned RecNo = Opcode - OPC_EmitMergeInputChains1_0;
3266  assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains");
3267  ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());
3268 
3269  // FIXME: What if other value results of the node have uses not matched
3270  // by this pattern?
3271  if (ChainNodesMatched.back() != NodeToMatch &&
3272  !RecordedNodes[RecNo].first.hasOneUse()) {
3273  ChainNodesMatched.clear();
3274  break;
3275  }
3276 
3277  // Merge the input chains if they are not intra-pattern references.
3278  InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG);
3279 
3280  if (!InputChain.getNode())
3281  break; // Failed to merge.
3282  continue;
3283  }
3284 
3285  case OPC_EmitMergeInputChains: {
3286  assert(!InputChain.getNode() &&
3287  "EmitMergeInputChains should be the first chain producing node");
3288  // This node gets a list of nodes we matched in the input that have
3289  // chains. We want to token factor all of the input chains to these nodes
3290  // together. However, if any of the input chains is actually one of the
3291  // nodes matched in this pattern, then we have an intra-match reference.
3292  // Ignore these because the newly token factored chain should not refer to
3293  // the old nodes.
3294  unsigned NumChains = MatcherTable[MatcherIndex++];
3295  assert(NumChains != 0 && "Can't TF zero chains");
3296 
3297  assert(ChainNodesMatched.empty() &&
3298  "Should only have one EmitMergeInputChains per match");
3299 
3300  // Read all of the chained nodes.
3301  for (unsigned i = 0; i != NumChains; ++i) {
3302  unsigned RecNo = MatcherTable[MatcherIndex++];
3303  assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains");
3304  ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());
3305 
3306  // FIXME: What if other value results of the node have uses not matched
3307  // by this pattern?
3308  if (ChainNodesMatched.back() != NodeToMatch &&
3309  !RecordedNodes[RecNo].first.hasOneUse()) {
3310  ChainNodesMatched.clear();
3311  break;
3312  }
3313  }
3314 
3315  // If the inner loop broke out, the match fails.
3316  if (ChainNodesMatched.empty())
3317  break;
3318 
3319  // Merge the input chains if they are not intra-pattern references.
3320  InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG);
3321 
3322  if (!InputChain.getNode())
3323  break; // Failed to merge.
3324 
3325  continue;
3326  }
3327 
3328  case OPC_EmitCopyToReg:
3329  case OPC_EmitCopyToReg2: {
3330  unsigned RecNo = MatcherTable[MatcherIndex++];
3331  assert(RecNo < RecordedNodes.size() && "Invalid EmitCopyToReg");
3332  unsigned DestPhysReg = MatcherTable[MatcherIndex++];
3333  if (Opcode == OPC_EmitCopyToReg2)
3334  DestPhysReg |= MatcherTable[MatcherIndex++] << 8;
3335 
3336  if (!InputChain.getNode())
3337  InputChain = CurDAG->getEntryNode();
3338 
3339  InputChain = CurDAG->getCopyToReg(InputChain, SDLoc(NodeToMatch),
3340  DestPhysReg, RecordedNodes[RecNo].first,
3341  InputGlue);
3342 
3343  InputGlue = InputChain.getValue(1);
3344  continue;
3345  }
3346 
3347  case OPC_EmitNodeXForm: {
3348  unsigned XFormNo = MatcherTable[MatcherIndex++];
3349  unsigned RecNo = MatcherTable[MatcherIndex++];
3350  assert(RecNo < RecordedNodes.size() && "Invalid EmitNodeXForm");
3351  SDValue Res = RunSDNodeXForm(RecordedNodes[RecNo].first, XFormNo);
3352  RecordedNodes.push_back(std::pair<SDValue,SDNode*>(Res, nullptr));
3353  continue;
3354  }
3355  case OPC_Coverage: {
3356  // This is emitted right before MorphNode/EmitNode.
3357  // So it should be safe to assume that this node has been selected
3358  unsigned index = MatcherTable[MatcherIndex++];
3359  index |= (MatcherTable[MatcherIndex++] << 8);
3360  dbgs() << "COVERED: " << getPatternForIndex(index) << "\n";
3361  dbgs() << "INCLUDED: " << getIncludePathForIndex(index) << "\n";
3362  continue;
3363  }
3364 
3365  case OPC_EmitNode: case OPC_MorphNodeTo:
3366  case OPC_EmitNode0: case OPC_EmitNode1: case OPC_EmitNode2:
3368  uint16_t TargetOpc = MatcherTable[MatcherIndex++];
3369  TargetOpc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
3370  unsigned EmitNodeInfo = MatcherTable[MatcherIndex++];
3371  // Get the result VT list.
3372  unsigned NumVTs;
3373  // If this is one of the compressed forms, get the number of VTs based
3374  // on the Opcode. Otherwise read the next byte from the table.
3375  if (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2)
3376  NumVTs = Opcode - OPC_MorphNodeTo0;
3377  else if (Opcode >= OPC_EmitNode0 && Opcode <= OPC_EmitNode2)
3378  NumVTs = Opcode - OPC_EmitNode0;
3379  else
3380  NumVTs = MatcherTable[MatcherIndex++];
3381  SmallVector<EVT, 4> VTs;
3382  for (unsigned i = 0; i != NumVTs; ++i) {
3384  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3385  if (VT == MVT::iPTR)
3386  VT = TLI->getPointerTy(CurDAG->getDataLayout()).SimpleTy;
3387  VTs.push_back(VT);
3388  }
3389 
3390  if (EmitNodeInfo & OPFL_Chain)
3391  VTs.push_back(MVT::Other);
3392  if (EmitNodeInfo & OPFL_GlueOutput)
3393  VTs.push_back(MVT::Glue);
3394 
3395  // This is hot code, so optimize the two most common cases of 1 and 2
3396  // results.
3397  SDVTList VTList;
3398  if (VTs.size() == 1)
3399  VTList = CurDAG->getVTList(VTs[0]);
3400  else if (VTs.size() == 2)
3401  VTList = CurDAG->getVTList(VTs[0], VTs[1]);
3402  else
3403  VTList = CurDAG->getVTList(VTs);
3404 
3405  // Get the operand list.
3406  unsigned NumOps = MatcherTable[MatcherIndex++];
3408  for (unsigned i = 0; i != NumOps; ++i) {
3409  unsigned RecNo = MatcherTable[MatcherIndex++];
3410  if (RecNo & 128)
3411  RecNo = GetVBR(RecNo, MatcherTable, MatcherIndex);
3412 
3413  assert(RecNo < RecordedNodes.size() && "Invalid EmitNode");
3414  Ops.push_back(RecordedNodes[RecNo].first);
3415  }
3416 
3417  // If there are variadic operands to add, handle them now.
3418  if (EmitNodeInfo & OPFL_VariadicInfo) {
3419  // Determine the start index to copy from.
3420  unsigned FirstOpToCopy = getNumFixedFromVariadicInfo(EmitNodeInfo);
3421  FirstOpToCopy += (EmitNodeInfo & OPFL_Chain) ? 1 : 0;
3422  assert(NodeToMatch->getNumOperands() >= FirstOpToCopy &&
3423  "Invalid variadic node");
3424  // Copy all of the variadic operands, not including a potential glue
3425  // input.
3426  for (unsigned i = FirstOpToCopy, e = NodeToMatch->getNumOperands();
3427  i != e; ++i) {
3428  SDValue V = NodeToMatch->getOperand(i);
3429  if (V.getValueType() == MVT::Glue) break;
3430  Ops.push_back(V);
3431  }
3432  }
3433 
3434  // If this has chain/glue inputs, add them.
3435  if (EmitNodeInfo & OPFL_Chain)
3436  Ops.push_back(InputChain);
3437  if ((EmitNodeInfo & OPFL_GlueInput) && InputGlue.getNode() != nullptr)
3438  Ops.push_back(InputGlue);
3439 
3440  // Check whether any matched node could raise an FP exception. Since all
3441  // such nodes must have a chain, it suffices to check ChainNodesMatched.
3442  // We need to perform this check before potentially modifying one of the
3443  // nodes via MorphNode.
3444  bool MayRaiseFPException = false;
3445  for (auto *N : ChainNodesMatched)
3446  if (mayRaiseFPException(N) && !N->getFlags().hasNoFPExcept()) {
3447  MayRaiseFPException = true;
3448  break;
3449  }
3450 
3451  // Create the node.
3452  MachineSDNode *Res = nullptr;
3453  bool IsMorphNodeTo = Opcode == OPC_MorphNodeTo ||
3454  (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2);
3455  if (!IsMorphNodeTo) {
3456  // If this is a normal EmitNode command, just create the new node and
3457  // add the results to the RecordedNodes list.
3458  Res = CurDAG->getMachineNode(TargetOpc, SDLoc(NodeToMatch),
3459  VTList, Ops);
3460 
3461  // Add all the non-glue/non-chain results to the RecordedNodes list.
3462  for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
3463  if (VTs[i] == MVT::Other || VTs[i] == MVT::Glue) break;
3464  RecordedNodes.push_back(std::pair<SDValue,SDNode*>(SDValue(Res, i),
3465  nullptr));
3466  }
3467  } else {
3468  assert(NodeToMatch->getOpcode() != ISD::DELETED_NODE &&
3469  "NodeToMatch was removed partway through selection");
3471  SDNode *E) {
3473  auto &Chain = ChainNodesMatched;
3474  assert((!E || !is_contained(Chain, N)) &&
3475  "Chain node replaced during MorphNode");
3476  llvm::erase_value(Chain, N);
3477  });
3478  Res = cast<MachineSDNode>(MorphNode(NodeToMatch, TargetOpc, VTList,
3479  Ops, EmitNodeInfo));
3480  }
3481 
3482  // Set the NoFPExcept flag when no original matched node could
3483  // raise an FP exception, but the new node potentially might.
3485  SDNodeFlags Flags = Res->getFlags();
3486  Flags.setNoFPExcept(true);
3487  Res->setFlags(Flags);
3488  }
3489 
3490  // If the node had chain/glue results, update our notion of the current
3491  // chain and glue.
3492  if (EmitNodeInfo & OPFL_GlueOutput) {
3493  InputGlue = SDValue(Res, VTs.size()-1);
3494  if (EmitNodeInfo & OPFL_Chain)
3495  InputChain = SDValue(Res, VTs.size()-2);
3496  } else if (EmitNodeInfo & OPFL_Chain)
3497  InputChain = SDValue(Res, VTs.size()-1);
3498 
3499  // If the OPFL_MemRefs glue is set on this node, slap all of the
3500  // accumulated memrefs onto it.
3501  //
3502  // FIXME: This is vastly incorrect for patterns with multiple outputs
3503  // instructions that access memory and for ComplexPatterns that match
3504  // loads.
3505  if (EmitNodeInfo & OPFL_MemRefs) {
3506  // Only attach load or store memory operands if the generated
3507  // instruction may load or store.
3508  const MCInstrDesc &MCID = TII->get(TargetOpc);
3509  bool mayLoad = MCID.mayLoad();
3510  bool mayStore = MCID.mayStore();
3511 
3512  // We expect to have relatively few of these so just filter them into a
3513  // temporary buffer so that we can easily add them to the instruction.
3514  SmallVector<MachineMemOperand *, 4> FilteredMemRefs;
3515  for (MachineMemOperand *MMO : MatchedMemRefs) {
3516  if (MMO->isLoad()) {
3517  if (mayLoad)
3518  FilteredMemRefs.push_back(MMO);
3519  } else if (MMO->isStore()) {
3520  if (mayStore)
3521  FilteredMemRefs.push_back(MMO);
3522  } else {
3523  FilteredMemRefs.push_back(MMO);
3524  }
3525  }
3526 
3527  CurDAG->setNodeMemRefs(Res, FilteredMemRefs);
3528  }
3529 
3530  LLVM_DEBUG(if (!MatchedMemRefs.empty() && Res->memoperands_empty()) dbgs()
3531  << " Dropping mem operands\n";
3532  dbgs() << " " << (IsMorphNodeTo ? "Morphed" : "Created")
3533  << " node: ";
3534  Res->dump(CurDAG););
3535 
3536  // If this was a MorphNodeTo then we're completely done!
3537  if (IsMorphNodeTo) {
3538  // Update chain uses.
3539  UpdateChains(Res, InputChain, ChainNodesMatched, true);
3540  return;
3541  }
3542  continue;
3543  }
3544 
3545  case OPC_CompleteMatch: {
3546  // The match has been completed, and any new nodes (if any) have been
3547  // created. Patch up references to the matched dag to use the newly
3548  // created nodes.
3549  unsigned NumResults = MatcherTable[MatcherIndex++];
3550 
3551  for (unsigned i = 0; i != NumResults; ++i) {
3552  unsigned ResSlot = MatcherTable[MatcherIndex++];
3553  if (ResSlot & 128)
3554  ResSlot = GetVBR(ResSlot, MatcherTable, MatcherIndex);
3555 
3556  assert(ResSlot < RecordedNodes.size() && "Invalid CompleteMatch");
3557  SDValue Res = RecordedNodes[ResSlot].first;
3558 
3559  assert(i < NodeToMatch->getNumValues() &&
3560  NodeToMatch->getValueType(i) != MVT::Other &&
3561  NodeToMatch->getValueType(i) != MVT::Glue &&
3562  "Invalid number of results to complete!");
3563  assert((NodeToMatch->getValueType(i) == Res.getValueType() ||
3564  NodeToMatch->getValueType(i) == MVT::iPTR ||
3565  Res.getValueType() == MVT::iPTR ||
3566  NodeToMatch->getValueType(i).getSizeInBits() ==
3567  Res.getValueSizeInBits()) &&
3568  "invalid replacement");
3569  ReplaceUses(SDValue(NodeToMatch, i), Res);
3570  }
3571 
3572  // Update chain uses.
3573  UpdateChains(NodeToMatch, InputChain, ChainNodesMatched, false);
3574 
3575  // If the root node defines glue, we need to update it to the glue result.
3576  // TODO: This never happens in our tests and I think it can be removed /
3577  // replaced with an assert, but if we do it this the way the change is
3578  // NFC.
3579  if (NodeToMatch->getValueType(NodeToMatch->getNumValues() - 1) ==
3580  MVT::Glue &&
3581  InputGlue.getNode())
3582  ReplaceUses(SDValue(NodeToMatch, NodeToMatch->getNumValues() - 1),
3583  InputGlue);
3584 
3585  assert(NodeToMatch->use_empty() &&
3586  "Didn't replace all uses of the node?");
3587  CurDAG->RemoveDeadNode(NodeToMatch);
3588 
3589  return;
3590  }
3591  }
3592 
3593  // If the code reached this point, then the match failed. See if there is
3594  // another child to try in the current 'Scope', otherwise pop it until we
3595  // find a case to check.
3596  LLVM_DEBUG(dbgs() << " Match failed at index " << CurrentOpcodeIndex
3597  << "\n");
3598  ++NumDAGIselRetries;
3599  while (true) {
3600  if (MatchScopes.empty()) {
3601  CannotYetSelect(NodeToMatch);
3602  return;
3603  }
3604 
3605  // Restore the interpreter state back to the point where the scope was
3606  // formed.
3607  MatchScope &LastScope = MatchScopes.back();
3608  RecordedNodes.resize(LastScope.NumRecordedNodes);
3609  NodeStack.clear();
3610  NodeStack.append(LastScope.NodeStack.begin(), LastScope.NodeStack.end());
3611  N = NodeStack.back();
3612 
3613  if (LastScope.NumMatchedMemRefs != MatchedMemRefs.size())
3614  MatchedMemRefs.resize(LastScope.NumMatchedMemRefs);
3615  MatcherIndex = LastScope.FailIndex;
3616 
3617  LLVM_DEBUG(dbgs() << " Continuing at " << MatcherIndex << "\n");
3618 
3619  InputChain = LastScope.InputChain;
3620  InputGlue = LastScope.InputGlue;
3621  if (!LastScope.HasChainNodesMatched)
3622  ChainNodesMatched.clear();
3623 
3624  // Check to see what the offset is at the new MatcherIndex. If it is zero
3625  // we have reached the end of this scope, otherwise we have another child
3626  // in the current scope to try.
3627  unsigned NumToSkip = MatcherTable[MatcherIndex++];
3628  if (NumToSkip & 128)
3629  NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex);
3630 
3631  // If we have another child in this scope to match, update FailIndex and
3632  // try it.
3633  if (NumToSkip != 0) {
3634  LastScope.FailIndex = MatcherIndex+NumToSkip;
3635  break;
3636  }
3637 
3638  // End of this scope, pop it and try the next child in the containing
3639  // scope.
3640  MatchScopes.pop_back();
3641  }
3642  }
3643 }
3644 
3645 /// Return whether the node may raise an FP exception.
3647  // For machine opcodes, consult the MCID flag.
3648  if (N->isMachineOpcode()) {
3649  const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
3650  return MCID.mayRaiseFPException();
3651  }
3652 
3653  // For ISD opcodes, only StrictFP opcodes may raise an FP
3654  // exception.
3655  if (N->isTargetOpcode())
3656  return N->isTargetStrictFPOpcode();
3657  return N->isStrictFPOpcode();
3658 }
3659 
3661  assert(N->getOpcode() == ISD::OR && "Unexpected opcode");
3662  auto *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
3663  if (!C)
3664  return false;
3665 
3666  // Detect when "or" is used to add an offset to a stack object.
3667  if (auto *FN = dyn_cast<FrameIndexSDNode>(N->getOperand(0))) {
3668  MachineFrameInfo &MFI = MF->getFrameInfo();
3669  Align A = MFI.getObjectAlign(FN->getIndex());
3670  int32_t Off = C->getSExtValue();
3671  // If the alleged offset fits in the zero bits guaranteed by
3672  // the alignment, then this or is really an add.
3673  return (Off >= 0) && (((A.value() - 1) & Off) == unsigned(Off));
3674  }
3675  return false;
3676 }
3677 
3678 void SelectionDAGISel::CannotYetSelect(SDNode *N) {
3679  std::string msg;
3680  raw_string_ostream Msg(msg);
3681  Msg << "Cannot select: ";
3682 
3683  if (N->getOpcode() != ISD::INTRINSIC_W_CHAIN &&
3684  N->getOpcode() != ISD::INTRINSIC_WO_CHAIN &&
3685  N->getOpcode() != ISD::INTRINSIC_VOID) {
3686  N->printrFull(Msg, CurDAG);
3687  Msg << "\nIn function: " << MF->getName();
3688  } else {
3689  bool HasInputChain = N->getOperand(0).getValueType() == MVT::Other;
3690  unsigned iid =
3691  cast<ConstantSDNode>(N->getOperand(HasInputChain))->getZExtValue();
3692  if (iid < Intrinsic::num_intrinsics)
3693  Msg << "intrinsic %" << Intrinsic::getBaseName((Intrinsic::ID)iid);
3694  else if (const TargetIntrinsicInfo *TII = TM.getIntrinsicInfo())
3695  Msg << "target intrinsic %" << TII->getName(iid);
3696  else
3697  Msg << "unknown intrinsic #" << iid;
3698  }
3699  report_fatal_error(Twine(Msg.str()));
3700 }
3701 
3702 char SelectionDAGISel::ID = 0;
llvm::DIExpression::prepend
static DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
Definition: DebugInfoMetadata.cpp:1296
llvm::FunctionLoweringInfo::Fn
const Function * Fn
Definition: FunctionLoweringInfo.h:54
llvm::Intrinsic::getBaseName
StringRef getBaseName(ID id)
Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....
Definition: Function.cpp:874
llvm::MachineInstr::isDebugValue
bool isDebugValue() const
Definition: MachineInstr.h:1216
llvm::SelectionDAGISel::OPC_MoveChild2
@ OPC_MoveChild2
Definition: SelectionDAGISel.h:123
i
i
Definition: README.txt:29
llvm::MachineFunction::hasInlineAsm
bool hasInlineAsm() const
Returns true if the function contains any inline assembly.
Definition: MachineFunction.h:703
ValueTypes.h
CheckSame
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * >> &RecordedNodes)
CheckSame - Implements OP_CheckSame.
Definition: SelectionDAGISel.cpp:2437
llvm::SelectionDAGISel::FastISelFailed
bool FastISelFailed
Definition: SelectionDAGISel.h:54
llvm::DbgVariableIntrinsic::getExpression
DIExpression * getExpression() const
Definition: IntrinsicInst.h:257
llvm::ConstantSDNode
Definition: SelectionDAGNodes.h:1556
llvm::SelectionDAGISel::OPC_CheckComplexPat
@ OPC_CheckComplexPat
Definition: SelectionDAGISel.h:145
llvm::BasicBlock::end
iterator end()
Definition: BasicBlock.h:298
llvm::SelectionDAGISel::OPFL_GlueOutput
@ OPFL_GlueOutput
Definition: SelectionDAGISel.h:178
llvm::ISD::STRICT_FSETCC
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:462
llvm::FunctionLoweringInfo::StaticAllocaMap
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
Definition: FunctionLoweringInfo.h:124
StackProtector.h
llvm::SelectionDAGISel::OPC_EmitInteger
@ OPC_EmitInteger
Definition: SelectionDAGISel.h:151
llvm::OptimizationRemarkMissed
Diagnostic information for missed-optimization remarks.
Definition: DiagnosticInfo.h:730
llvm::createDAGLinearizer
ScheduleDAGSDNodes * createDAGLinearizer(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createDAGLinearizer - This creates a "no-scheduling" scheduler which linearize the DAG using topologi...
Definition: ScheduleDAGFast.cpp:800
llvm::predecessors
pred_range predecessors(BasicBlock *BB)
Definition: CFG.h:127
isFoldedOrDeadInstruction
static bool isFoldedOrDeadInstruction(const Instruction *I, const FunctionLoweringInfo &FuncInfo)
isFoldedOrDeadInstruction - Return true if the specified instruction is side-effect free and is eithe...
Definition: SelectionDAGISel.cpp:1317
llvm::MachineFunctionProperties::hasProperty
bool hasProperty(Property P) const
Definition: MachineFunction.h:169
llvm::ISD::INTRINSIC_VOID
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:199
llvm::SDUse
Represents a use of a SDNode.
Definition: SelectionDAGNodes.h:277
llvm::SelectionDAGISel::OPC_CheckChild2Integer
@ OPC_CheckChild2Integer
Definition: SelectionDAGISel.h:141
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:105
MachineInstr.h
llvm::SelectionDAGISel::TLI
const TargetLowering * TLI
Definition: SelectionDAGISel.h:53
llvm
This file implements support for optimizing divisions by a constant.
Definition: AllocatorList.h:23
llvm::SelectionDAGISel::getIncludePathForIndex
virtual StringRef getIncludePathForIndex(unsigned index)
getIncludePathForIndex - get the td source location of pattern instantiation
Definition: SelectionDAGISel.h:244
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
llvm::DbgDeclareInst::getAddress
Value * getAddress() const
Definition: IntrinsicInst.h:310
llvm::DiagnosticInfoISelFallback
Diagnostic information for ISel fallback path.
Definition: DiagnosticInfo.h:950
Reg
unsigned Reg
Definition: MachineSink.cpp:1566
llvm::SelectionDAG::AssignTopologicalOrder
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
Definition: SelectionDAG.cpp:9805
GCMetadata.h
llvm::SelectionDAGISel::TM
TargetMachine & TM
Definition: SelectionDAGISel.h:41
llvm::MCSymbol
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
llvm::SDNode::getValueType
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Definition: SelectionDAGNodes.h:966
UseMI
MachineInstrBuilder & UseMI
Definition: AArch64ExpandPseudoInsts.cpp:102
llvm::SelectionDAGISel::isOrEquivalentToAdd
bool isOrEquivalentToAdd(const SDNode *N) const
Definition: SelectionDAGISel.cpp:3660
llvm::ScheduleDAGSDNodes
ScheduleDAGSDNodes - A ScheduleDAG for scheduling SDNode-based DAGs.
Definition: ScheduleDAGSDNodes.h:46
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1086
llvm::SelectionDAGISel::OPC_EmitMergeInputChains1_2
@ OPC_EmitMergeInputChains1_2
Definition: SelectionDAGISel.h:159
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::ISD::OR
@ OR
Definition: ISDOpcodes.h:633
llvm::SelectionDAGISel::EnforceNodeIdInvariant
static void EnforceNodeIdInvariant(SDNode *N)
Definition: SelectionDAGISel.cpp:1068
llvm::TargetLowering::EmitInstrWithCustomInserter
virtual MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Definition: SelectionDAGISel.cpp:295
CheckChildSame
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * >> &RecordedNodes, unsigned ChildNo)
CheckChildSame - Implements OP_CheckChildXSame.
Definition: SelectionDAGISel.cpp:2446
llvm::Value::hasOneUse
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
llvm::MachineBasicBlock::getBasicBlock
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
Definition: MachineBasicBlock.h:202
llvm::SelectionDAGISel::CheckComplexPattern
virtual bool CheckComplexPattern(SDNode *Root, SDNode *Parent, SDValue N, unsigned PatternNo, SmallVectorImpl< std::pair< SDValue, SDNode * > > &Result)
Definition: SelectionDAGISel.h:287
Metadata.h
llvm::ISD::LIFETIME_END
@ LIFETIME_END
Definition: ISDOpcodes.h:1178
llvm::SelectionDAGISel::OPC_RecordChild7
@ OPC_RecordChild7
Definition: SelectionDAGISel.h:119
llvm::SelectionDAGISel::PostprocessISelDAG
virtual void PostprocessISelDAG()
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
Definition: SelectionDAGISel.h:81
llvm::BasicBlock::iterator
InstListType::iterator iterator
Instruction iterators...
Definition: BasicBlock.h:90
llvm::TargetLoweringBase::isStrictFPEnabled
bool isStrictFPEnabled() const
Return true if the target support strict float operation.
Definition: TargetLowering.h:330
llvm::ConstantSDNode::getAPIntValue
const APInt & getAPIntValue() const
Definition: SelectionDAGNodes.h:1570
IntrinsicInst.h
computeUsesMSVCFloatingPoint
static void computeUsesMSVCFloatingPoint(const Triple &TT, const Function &F, MachineModuleInfo &MMI)
Definition: SelectionDAGISel.cpp:392
hasExceptionPointerOrCodeUser
static bool hasExceptionPointerOrCodeUser(const CatchPadInst *CPI)
Definition: SelectionDAGISel.cpp:1207
DebugInfoMetadata.h
llvm::ISD::AssertSext
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
llvm::SelectionDAG::getCopyToReg
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:735
MCInstrDesc.h
llvm::SelectionDAGISel::DAGSize
unsigned DAGSize
DAGSize - Size of DAG being instruction selected.
Definition: SelectionDAGISel.h:202
EnableFastISelAbort
static cl::opt< int > EnableFastISelAbort("fast-isel-abort", cl::Hidden, cl::desc("Enable abort calls when \"fast\" instruction selection " "fails to lower an instruction: 0 disable the abort, 1 will " "abort but for args, calls and terminators, 2 will also " "abort for argument lowering, and 3 will never fallback " "to SelectionDAG."))
InstIterator.h
MachinePassRegistry.h
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:52
T
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:152
llvm::SelectionDAG::VerifyDAGDivergence
void VerifyDAGDivergence()
Definition: SelectionDAG.cpp:9732
llvm::SelectionDAGISel::OPC_EmitNode
@ OPC_EmitNode
Definition: SelectionDAGISel.h:163
llvm::Function
Definition: Function.h:62
llvm::ISD::ConstantFP
@ ConstantFP
Definition: ISDOpcodes.h:77
StringRef.h
llvm::ISD::LIFETIME_START
@ LIFETIME_START
This corresponds to the llvm.lifetime.
Definition: ISDOpcodes.h:1177
Pass.h
SchedulerRegistry.h
llvm::TargetSubtargetInfo::getInstrInfo
virtual const TargetInstrInfo * getInstrInfo() const
Definition: TargetSubtargetInfo.h:92
llvm::ISD::STRICT_UINT_TO_FP
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:436
llvm::SelectionDAGISel::SelectInlineAsmMemoryOperands
void SelectInlineAsmMemoryOperands(std::vector< SDValue > &Ops, const SDLoc &DL)
SelectInlineAsmMemoryOperands - Calls to this are automatically generated by tblgen.
Definition: SelectionDAGISel.cpp:1986
llvm::StackProtector::copyToMachineFrameInfo
void copyToMachineFrameInfo(MachineFrameInfo &MFI) const
Definition: StackProtector.cpp:596
llvm::raw_string_ostream
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:625
llvm::SDNode::op_values
iterator_range< value_op_iterator > op_values() const
Definition: SelectionDAGNodes.h:927
llvm::InlineAsm::Op_FirstOperand
@ Op_FirstOperand
Definition: InlineAsm.h:215
llvm::FuncletPadInst::getNumArgOperands
unsigned getNumArgOperands() const
getNumArgOperands - Return the number of funcletpad arguments.
Definition: InstrTypes.h:2321
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
llvm::TargetMachine::getO0WantsFastISel
bool getO0WantsFastISel()
Definition: TargetMachine.h:246
Statistic.h
llvm::SelectionDAG::init
void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, LegacyDivergenceAnalysis *Divergence, ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin)
Prepare this SelectionDAG to process code in the given MachineFunction.
Definition: SelectionDAG.cpp:1191
llvm::MachineFunction::useDebugInstrRef
bool useDebugInstrRef() const
Returns true if the function's variable locations should be tracked with instruction referencing.
Definition: MachineFunction.cpp:1224
InlineAsm.h
CheckType
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
Definition: SelectionDAGISel.cpp:2479
llvm::SelectionDAG::getVTList
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
Definition: SelectionDAG.cpp:8580
llvm::FastISel::finishBasicBlock
void finishBasicBlock()
Flush the local value map.
Definition: FastISel.cpp:138
llvm::printMBBReference
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Definition: MachineBasicBlock.cpp:119
llvm::FunctionPass::skipFunction
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Definition: Pass.cpp:163
llvm::Function::getSubprogram
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1541
llvm::MachineSDNode
An SDNode that represents everything that will be needed to construct a MachineInstr.
Definition: SelectionDAGNodes.h:2746
llvm::createDefaultScheduler
ScheduleDAGSDNodes * createDefaultScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createDefaultScheduler - This creates an instruction scheduler appropriate for the target.
Definition: SelectionDAGISel.cpp:254
ErrorHandling.h
llvm::FastISel::lowerArguments
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
Definition: FastISel.cpp:140
llvm::TargetTransformInfo
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Definition: TargetTransformInfo.h:168
LazyBlockFrequencyInfo.h
llvm::SelectionDAGISel::OPC_CheckOrImm
@ OPC_CheckOrImm
Definition: SelectionDAGISel.h:146
llvm::SelectionDAGISel::OPC_RecordChild3
@ OPC_RecordChild3
Definition: SelectionDAGISel.h:118
llvm::Function::getEntryBlock
const BasicBlock & getEntryBlock() const
Definition: Function.h:718
llvm::Sched::ILP
@ ILP
Definition: TargetLowering.h:103
llvm::SelectionDAG::allnodes_begin
allnodes_const_iterator allnodes_begin() const
Definition: SelectionDAG.h:493
llvm::SelectionDAG::getRoot
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:513
llvm::SelectionDAGISel::OPC_EmitNode1
@ OPC_EmitNode1
Definition: SelectionDAGISel.h:165
llvm::SwiftErrorValueTracking::createEntriesInEntryBlock
bool createEntriesInEntryBlock(DebugLoc DbgLoc)
Create initial definitions of swifterror values in the entry block of the current function.
Definition: SwiftErrorValueTracking.cpp:115
llvm::SwiftErrorValueTracking::propagateVRegs
void propagateVRegs()
Propagate assigned swifterror vregs through a function, synthesizing PHI nodes when needed to maintai...
Definition: SwiftErrorValueTracking.cpp:147
llvm::ISD::EH_LABEL
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:988
llvm::SelectionDAGISel::OPC_CheckChild1Integer
@ OPC_CheckChild1Integer
Definition: SelectionDAGISel.h:141
llvm::SelectionDAGISel::OPC_CheckFoldableChainNode
@ OPC_CheckFoldableChainNode
Definition: SelectionDAGISel.h:149
llvm::MachineRegisterInfo::defusechain_instr_iterator
defusechain_iterator - This class provides iterator support for machine operands in the function that...
Definition: MachineRegisterInfo.h:269
llvm::HandleSDNode
This class is used to form a handle around another node that is persistent and is updated across invo...
Definition: SelectionDAGNodes.h:1212
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
llvm::createFastDAGScheduler
ScheduleDAGSDNodes * createFastDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createFastDAGScheduler - This creates a "fast" scheduler.
Definition: ScheduleDAGFast.cpp:795
llvm::SelectionDAGISel::OPC_CheckChild1Same
@ OPC_CheckChild1Same
Definition: SelectionDAGISel.h:127
OptimizationRemarkEmitter.h
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:455
llvm::DominatorTree
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:151
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:45
MachineBasicBlock.h
llvm::TargetSubtargetInfo::getRegisterInfo
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Definition: TargetSubtargetInfo.h:124
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:143
llvm::MVT::Glue
@ Glue
Definition: MachineValueType.h:262
llvm::SelectionDAGISel::OPC_MorphNodeTo2
@ OPC_MorphNodeTo2
Definition: SelectionDAGISel.h:168
llvm::SelectionDAGISel::OPC_MoveChild7
@ OPC_MoveChild7
Definition: SelectionDAGISel.h:124
llvm::SDNode::use_iterator
This class provides iterator support for SDUse operands that use a specific SDNode.
Definition: SelectionDAGNodes.h:727
APInt.h
llvm::TargetRegisterInfo
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Definition: TargetRegisterInfo.h:233
llvm::MachineSDNode::memoperands_empty
bool memoperands_empty() const
Definition: SelectionDAGNodes.h:2790
llvm::SelectionDAGISel::CheckPatternPredicate
virtual bool CheckPatternPredicate(unsigned PredNo) const
CheckPatternPredicate - This function is generated by tblgen in the target.
Definition: SelectionDAGISel.h:264
llvm::LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage
static void getLazyBFIAnalysisUsage(AnalysisUsage &AU)
Helper for client passes to set up the analysis usage on behalf of this pass.
Definition: LazyBlockFrequencyInfo.cpp:62
llvm::Function::getContext
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:321
llvm::DenseMapIterator
Definition: DenseMap.h:56
Shift
bool Shift
Definition: README.txt:468
llvm::InlineAsm::isUseOperandTiedToDef
static bool isUseOperandTiedToDef(unsigned Flag, unsigned &Idx)
isUseOperandTiedToDef - Return true if the flag of the inline asm operand indicates it is an use oper...
Definition: InlineAsm.h:345
ViewSchedDAGs
static cl::opt< bool > ViewSchedDAGs("view-sched-dags", cl::Hidden, cl::desc("Pop up a window to show sched dags as they are processed"))
DenseMap.h
llvm::SplitCriticalEdge
BasicBlock * SplitCriticalEdge(Instruction *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions(), const Twine &BBName="")
If this edge is a critical edge, insert a new node to split the critical edge.
Definition: BreakCriticalEdges.cpp:103
llvm::TargetLowering::initializeSplitCSR
virtual void initializeSplitCSR(MachineBasicBlock *Entry) const
Perform necessary initialization to handle a subset of CSRs explicitly via copies.
Definition: TargetLowering.h:3661
llvm::TimePassesIsEnabled
bool TimePassesIsEnabled
If the user specifies the -time-passes argument on an LLVM tool command line then the value of this b...
Definition: PassTimingInfo.cpp:37
CheckChildInteger
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, unsigned ChildNo)
Definition: SelectionDAGISel.cpp:2549
llvm::SelectionDAGISel::OPC_CaptureGlueInput
@ OPC_CaptureGlueInput
Definition: SelectionDAGISel.h:121
llvm::SelectionDAGISel::OPC_EmitNodeXForm
@ OPC_EmitNodeXForm
Definition: SelectionDAGISel.h:162
TargetInstrInfo.h
llvm::MachineModuleInfo::setUsesMSVCFloatingPoint
void setUsesMSVCFloatingPoint(bool b)
Definition: MachineModuleInfo.h:208
llvm::MachineMemOperand
A description of a memory reference used in the backend.
Definition: MachineMemOperand.h:128
llvm::FunctionLoweringInfo::isExportedInst
bool isExportedInst(const Value *V) const
isExportedInst - Return true if the specified value is an instruction exported from its block.
Definition: FunctionLoweringInfo.h:194
EHPersonalities.h
llvm::TargetLoweringBase::getExceptionPointerRegister
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
Definition: TargetLowering.h:1745
llvm::RegisterScheduler::Registry
static MachinePassRegistry< FunctionPassCtor > Registry
RegisterScheduler class - Track the registration of instruction schedulers.
Definition: SchedulerRegistry.h:38
ViewDAGCombine1
static cl::opt< bool > ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the first " "dag combine pass"))
llvm::MachineFunction::setCallSiteLandingPad
void setCallSiteLandingPad(MCSymbol *Sym, ArrayRef< unsigned > Sites)
Map the landing pad's EH symbol to the call site indexes.
Definition: MachineFunction.cpp:856
llvm::SelectionDAGISel::OPC_CheckChild0Same
@ OPC_CheckChild0Same
Definition: SelectionDAGISel.h:127
llvm::DbgVariableIntrinsic::getVariable
DILocalVariable * getVariable() const
Definition: IntrinsicInst.h:253
LLVM_ATTRIBUTE_ALWAYS_INLINE
#define LLVM_ATTRIBUTE_ALWAYS_INLINE
LLVM_ATTRIBUTE_ALWAYS_INLINE - On compilers where we have a directive to do so, mark a method "always...
Definition: Compiler.h:246
llvm::MachineInstr::isCopy
bool isCopy() const
Definition: MachineInstr.h:1291
llvm::SelectionDAGISel::OPC_CheckPredicate
@ OPC_CheckPredicate
Definition: SelectionDAGISel.h:130
Offset
uint64_t Offset
Definition: ELFObjHandler.cpp:81
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:449
llvm::ore::NV
DiagnosticInfoOptimizationBase::Argument NV
Definition: OptimizationRemarkEmitter.h:136
llvm::SelectionDAGISel::OPC_Coverage
@ OPC_Coverage
Definition: SelectionDAGISel.h:171
llvm::ISD::isConstantSplatVectorAllZeros
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
Definition: SelectionDAG.cpp:220
llvm::TargetLowering::supportSplitCSR
virtual bool supportSplitCSR(MachineFunction *MF) const
Return true if the target supports that a subset of CSRs for the given machine function is handled ex...
Definition: TargetLowering.h:3654
llvm::SelectionDAG::RemoveDeadNodes
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
Definition: SelectionDAG.cpp:852
STLExtras.h
llvm::SmallVectorImpl::pop_back_val
LLVM_NODISCARD T pop_back_val()
Definition: SmallVector.h:635
llvm::DIExpression
DWARF expression.
Definition: DebugInfoMetadata.h:2586
llvm::AfterLegalizeTypes
@ AfterLegalizeTypes
Definition: DAGCombine.h:17
llvm::SelectionDAGISel::OPC_CheckAndImm
@ OPC_CheckAndImm
Definition: SelectionDAGISel.h:146
llvm::SelectionDAGISel::OPC_EmitCopyToReg
@ OPC_EmitCopyToReg
Definition: SelectionDAGISel.h:160
llvm::SelectionDAGISel::OPC_SwitchType
@ OPC_SwitchType
Definition: SelectionDAGISel.h:136
llvm::Sched::Fast
@ Fast
Definition: TargetLowering.h:105
llvm::succ_empty
bool succ_empty(const Instruction *I)
Definition: CFG.h:256
llvm::SelectionDAGISel::OPC_CheckChild0Type
@ OPC_CheckChild0Type
Definition: SelectionDAGISel.h:137
replace
static void replace(Module &M, GlobalVariable *Old, GlobalVariable *New)
Definition: ConstantMerge.cpp:116
llvm::Sched::Linearize
@ Linearize
Definition: TargetLowering.h:106
llvm::SelectionDAGISel::OPC_CheckCondCode
@ OPC_CheckCondCode
Definition: SelectionDAGISel.h:143
SelectionDAG.h
llvm::SelectionDAGISel::OPC_CheckChild7Type
@ OPC_CheckChild7Type
Definition: SelectionDAGISel.h:139
llvm::initializeGCModuleInfoPass
void initializeGCModuleInfoPass(PassRegistry &)
processDbgDeclares
static void processDbgDeclares(FunctionLoweringInfo &FuncInfo)
Collect llvm.dbg.declare information.
Definition: SelectionDAGISel.cpp:1328
llvm::ISD::FREEZE
@ FREEZE
Definition: ISDOpcodes.h:216
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1567
llvm::SelectionDAG::clear
void clear()
Clear state and free memory necessary to make this SelectionDAG ready to process a new block.
Definition: SelectionDAG.cpp:1271
FilterDAGBasicBlockName
static cl::opt< std::string > FilterDAGBasicBlockName("filter-view-dags", cl::Hidden, cl::desc("Only display the basic block whose name " "matches this for all view-*-dags options"))
llvm::TargetLoweringBase::getOperationAction
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
Definition: TargetLowering.h:1047
llvm::initializeTargetLibraryInfoWrapperPassPass
void initializeTargetLibraryInfoWrapperPassPass(PassRegistry &)
llvm::SelectionDAGISel::OPC_CheckChild1Type
@ OPC_CheckChild1Type
Definition: SelectionDAGISel.h:137
llvm::SelectionDAGISel::OPC_EmitStringInteger
@ OPC_EmitStringInteger
Definition: SelectionDAGISel.h:152
llvm::MachineFunctionPass::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition: MachineFunctionPass.cpp:102
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
new
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM ID Predecessors according to mbb< bb27, 0x8b0a7c0 > Note ADDri is not a two address instruction its result reg1037 is an operand of the PHI node in bb76 and its operand reg1039 is the result of the PHI node We should treat it as a two address code and make sure the ADDri is scheduled after any node that reads reg1039 Use info(i.e. register scavenger) to assign it a free register to allow reuse the collector could move the objects and invalidate the derived pointer This is bad enough in the first but safe points can crop up unpredictably **array_addr i32 n y store obj * new
Definition: README.txt:125
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::RISCVFenceField::R
@ R
Definition: RISCVBaseInfo.h:198
MachineRegisterInfo.h
KnownBits.h
llvm::SelectionDAGISel::PreprocessISelDAG
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
Definition: SelectionDAGISel.h:77
llvm::InlineAsm::Kind_Mem
@ Kind_Mem
Definition: InlineAsm.h:238
llvm::ISD::ARITH_FENCE
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1105
llvm::SelectionDAG::getRegister
SDValue getRegister(unsigned Reg, EVT VT)
Definition: SelectionDAG.cpp:2007
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
llvm::RegState::Kill
@ Kill
The last use of a register.
Definition: MachineInstrBuilder.h:48
llvm::ISD::INLINEASM
@ INLINEASM
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:980
llvm::EVT::isSimple
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:130
llvm::ISD::PSEUDO_PROBE
@ PSEUDO_PROBE
Pseudo probe for AutoFDO, as a place holder in a basic block to improve the sample counts quality.
Definition: ISDOpcodes.h:1197
llvm::TargetIntrinsicInfo
TargetIntrinsicInfo - Interface to description of machine instruction set.
Definition: TargetIntrinsicInfo.h:29
mapWasmLandingPadIndex
static void mapWasmLandingPadIndex(MachineBasicBlock *MBB, const CatchPadInst *CPI)
Definition: SelectionDAGISel.cpp:1222
CheckAndImm
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
Definition: SelectionDAGISel.cpp:2557
AliasAnalysis.h
UseMBPI
static cl::opt< bool > UseMBPI("use-mbpi", cl::desc("use Machine Branch Probability Info"), cl::init(true), cl::Hidden)
MachineValueType.h
llvm::MVT::SimpleValueType
SimpleValueType
Definition: MachineValueType.h:33
llvm::MachineFunction::setExposesReturnsTwice
void setExposesReturnsTwice(bool B)
setCallsSetJmp - Set a flag that indicates if there's a call to a "returns twice" function.
Definition: MachineFunction.h:698
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::classifyEHPersonality
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
Definition: EHPersonalities.cpp:21
llvm::MachineFunction::setWasmLandingPadIndex
void setWasmLandingPadIndex(const MachineBasicBlock *LPad, unsigned Index)
Map the landing pad to its index. Used for Wasm exception handling.
Definition: MachineFunction.h:1106
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:206
llvm::TargetLowering::AdjustInstrPostInstrSelection
virtual void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
Definition: SelectionDAGISel.cpp:305
llvm::SelectionDAGISel::OPC_EmitMergeInputChains1_0
@ OPC_EmitMergeInputChains1_0
Definition: SelectionDAGISel.h:157
llvm::SelectionDAGISel::ElidedArgCopyInstrs
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
Definition: SelectionDAGISel.h:55
llvm::SelectionDAGISel::OPC_MoveChild5
@ OPC_MoveChild5
Definition: SelectionDAGISel.h:124
Instruction.h
CommandLine.h
llvm::BeforeLegalizeTypes
@ BeforeLegalizeTypes
Definition: DAGCombine.h:16
llvm::InlineAsm::Op_ExtraInfo
@ Op_ExtraInfo
Definition: InlineAsm.h:214
TargetLowering.h
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
llvm::Instruction::getNumSuccessors
unsigned getNumSuccessors() const
Return the number of successors that this instruction has.
Definition: Instruction.cpp:765
llvm::FuncletPadInst::getArgOperand
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th funcletpad argument.
Definition: InstrTypes.h:2337
llvm::SDNode::getOpcode
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
Definition: SelectionDAGNodes.h:629
llvm::MachineFunction::front
const MachineBasicBlock & front() const
Definition: MachineFunction.h:824
llvm::BlockFrequencyInfo
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Definition: BlockFrequencyInfo.h:37
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:640
reportFastISelFailure
static void reportFastISelFailure(MachineFunction &MF, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R, bool ShouldAbort)
Definition: SelectionDAGISel.cpp:693
llvm::SelectionDAGISel::OPC_RecordChild6
@ OPC_RecordChild6
Definition: SelectionDAGISel.h:119
llvm::MachineBasicBlock::insertAfter
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
Definition: MachineBasicBlock.h:871
llvm::SelectionDAGISel::OPC_RecordChild1
@ OPC_RecordChild1
Definition: SelectionDAGISel.h:118
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1121
ViewDAGCombineLT
static cl::opt< bool > ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the post " "legalize types dag combine pass"))
CheckValueType
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
Definition: SelectionDAGISel.cpp:2514
llvm::MachineFunctionProperties::Property::Selected
@ Selected
TargetMachine.h
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:31
CheckOrImm
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
Definition: SelectionDAGISel.cpp:2570
llvm::SelectionDAGISel::OPC_EmitNode0
@ OPC_EmitNode0
Definition: SelectionDAGISel.h:165
llvm::SelectionDAG
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:216
SelectionDAGNodes.h
Constants.h
llvm::SelectionDAGISel::RunSDNodeXForm
virtual SDValue RunSDNodeXForm(SDValue V, unsigned XFormNo)
Definition: SelectionDAGISel.h:293
llvm::ISD::Constant
@ Constant
Definition: ISDOpcodes.h:76
llvm::SelectionDAGISel::SDB
std::unique_ptr< SelectionDAGBuilder > SDB
Definition: SelectionDAGISel.h:48
llvm::PHINode::getIncomingValue
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
Definition: Instructions.h:2729
llvm::SelectionDAGISel::OPC_CheckOpcode
@ OPC_CheckOpcode
Definition: SelectionDAGISel.h:132
llvm::SelectionDAGISel::OPC_CheckPatternPredicate
@ OPC_CheckPatternPredicate
Definition: SelectionDAGISel.h:129
llvm::BranchProbabilityInfoWrapperPass
Legacy analysis pass which computes BranchProbabilityInfo.
Definition: BranchProbabilityInfo.h:440
FunctionLoweringInfo.h
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
SplitCriticalSideEffectEdges
static void SplitCriticalSideEffectEdges(Function &Fn, DominatorTree *DT, LoopInfo *LI)
SplitCriticalSideEffectEdges - Look for critical edges with a PHI value that may trap on it.
Definition: SelectionDAGISel.cpp:359
llvm::SmallVectorImpl::append
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:648
llvm::SelectionDAG::LegalizeTypes
bool LegalizeTypes()
This transforms the SelectionDAG into a SelectionDAG that only uses types natively supported by the t...
Definition: LegalizeTypes.cpp:1054
llvm::DIExpression::ApplyOffset
@ ApplyOffset
Definition: DebugInfoMetadata.h:2793
llvm::User
Definition: User.h:44
llvm::SelectionDAGISel::OPC_MoveChild
@ OPC_MoveChild
Definition: SelectionDAGISel.h:122
llvm::ISD::CopyToReg
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition: ISDOpcodes.h:203
llvm::SDNode::setFlags
void setFlags(SDNodeFlags NewFlags)
Definition: SelectionDAGNodes.h:956
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:35
llvm::SelectionDAGISel::RegInfo
MachineRegisterInfo * RegInfo
Definition: SelectionDAGISel.h:46
Intrinsics.h
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::MachineInstr::getOperand
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:499
llvm::SelectionDAGISel::OptLevel
CodeGenOpt::Level OptLevel
Definition: SelectionDAGISel.h:51
llvm::SelectionDAGISel::CheckNodePredicate
virtual bool CheckNodePredicate(SDNode *N, unsigned PredNo) const
CheckNodePredicate - This function is generated by tblgen in the target.
Definition: SelectionDAGISel.h:272
llvm::getLLTForMVT
LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
Definition: LowLevelType.cpp:65
InstrTypes.h
llvm::TargetMachine::getIntrinsicInfo
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
Definition: TargetMachine.h:214
llvm::TargetLowering
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Definition: TargetLowering.h:3187
ViewLegalizeTypesDAGs
static cl::opt< bool > ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize types"))
llvm::KnownBits::One
APInt One
Definition: KnownBits.h:25
llvm::SelectionDAGISel::ComplexPatternFuncMutatesDAG
virtual bool ComplexPatternFuncMutatesDAG() const
Return true if complex patterns for this target can mutate the DAG.
Definition: SelectionDAGISel.h:302
llvm::TargetTransformInfo::hasBranchDivergence
bool hasBranchDivergence() const
Return true if branch divergence exists.
Definition: TargetTransformInfo.cpp:232
llvm::MachineBasicBlock::isSuccessor
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
Definition: MachineBasicBlock.cpp:913
llvm::StackProtector
Definition: StackProtector.h:37
llvm::SelectionDAG::MaskedValueIsZero
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
Definition: SelectionDAG.cpp:2485
llvm::createILPListDAGScheduler
ScheduleDAGSDNodes * createILPListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level)
createILPListDAGScheduler - This creates a bottom up register pressure aware list scheduler that trie...
Definition: ScheduleDAGRRList.cpp:3179
llvm::Register::isPhysicalRegister
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:65
llvm::TargetRegisterClass
Definition: TargetRegisterInfo.h:46
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
llvm::SelectionDAG::Legalize
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
Definition: LegalizeDAG.cpp:4956
ViewLegalizeDAGs
static cl::opt< bool > ViewLegalizeDAGs("view-legalize-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize"))
CheckInteger
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
Definition: SelectionDAGISel.cpp:2536
llvm::MachineFunction::getProperties
const MachineFunctionProperties & getProperties() const
Get the function properties.
Definition: MachineFunction.h:721
TargetLibraryInfo.h
llvm::CriticalEdgeSplittingOptions
Option class for critical edge splitting.
Definition: BasicBlockUtils.h:136
llvm::SelectionDAGISel::OPFL_MemRefs
@ OPFL_MemRefs
Definition: SelectionDAGISel.h:179
llvm::SelectionDAGISel::ReplaceNode
void ReplaceNode(SDNode *F, SDNode *T)
Replace all uses of F with T, then remove F from the DAG.
Definition: SelectionDAGISel.h:227
llvm::SelectionDAGISel::ORE
std::unique_ptr< OptimizationRemarkEmitter > ORE
Current optimization remark emitter.
Definition: SelectionDAGISel.h:59
llvm::SelectionDAGISel::OPC_MorphNodeTo1
@ OPC_MorphNodeTo1
Definition: SelectionDAGISel.h:168
llvm::ISD::DELETED_NODE
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:44
llvm::MCInstrDesc
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:195
CheckCondCode
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
Definition: SelectionDAGISel.cpp:2499
llvm::SelectionDAG::setRoot
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:522
llvm::SelectionDAGISel::OPFL_VariadicInfo
@ OPFL_VariadicInfo
Definition: SelectionDAGISel.h:188
llvm::EVT::isInteger
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:145
llvm::Instruction
Definition: Instruction.h:45
llvm::MachineModuleInfo
This class contains meta information specific to a module.
Definition: MachineModuleInfo.h:78
llvm::ISD::MDNODE_SDNODE
@ MDNODE_SDNODE
MDNODE_SDNODE - This is a node that holdes an MDNode*, which is used to reference metadata in the IR.
Definition: ISDOpcodes.h:1057
llvm::createSourceListDAGScheduler
ScheduleDAGSDNodes * createSourceListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createBURRListDAGScheduler - This creates a bottom up list scheduler that schedules nodes in source c...
Definition: ScheduleDAGRRList.cpp:3149
llvm::SDNode::use_empty
bool use_empty() const
Return true if there are no uses of this node.
Definition: SelectionDAGNodes.h:697
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:140
llvm::BasicBlock::phis
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:354
llvm::STATISTIC
STATISTIC(NumFunctions, "Total number of functions")
decodeSignRotatedValue
static uint64_t decodeSignRotatedValue(uint64_t V)
Decode a signed value stored with the sign bit in the LSB for dense VBR encoding.
Definition: SelectionDAGISel.cpp:2526
CheckPatternPredicate
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel)
CheckPatternPredicate - Implements OP_CheckPatternPredicate.
Definition: SelectionDAGISel.cpp:2458
llvm::SelectionDAG::DAGNodeDeletedListener
Definition: SelectionDAG.h:314
ScheduleDAGSDNodes.h
llvm::ConstantFP
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:257
llvm::SmallVectorImpl::resize
void resize(size_type N)
Definition: SmallVector.h:606
llvm::Use::getUser
User * getUser() const
Returns the User that contains this Use.
Definition: Use.h:73
llvm::SelectionDAG::dump
void dump() const
Definition: SelectionDAGDumper.cpp:913
llvm::SelectionDAGISel::ID
static char ID
Definition: SelectionDAGISel.h:61
llvm::ISD::AssertAlign
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
Definition: ISDOpcodes.h:68
DebugLoc.h
SmallPtrSet.h
llvm::SwiftErrorValueTracking::preassignVRegs
void preassignVRegs(MachineBasicBlock *MBB, BasicBlock::const_iterator Begin, BasicBlock::const_iterator End)
Definition: SwiftErrorValueTracking.cpp:259
llvm::RegisterScheduler::FunctionPassCtor
ScheduleDAGSDNodes *(*)(SelectionDAGISel *, CodeGenOpt::Level) FunctionPassCtor
Definition: SchedulerRegistry.h:36
llvm::ISD::AND
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:632
llvm::ISD::TargetGlobalAddress
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
Definition: ISDOpcodes.h:164
llvm::BasicBlock::getFirstNonPHI
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:216
llvm::ISD::EntryToken
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:47
CheckNodePredicate
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel, SDNode *N)
CheckNodePredicate - Implements OP_CheckNodePredicate.
Definition: SelectionDAGISel.cpp:2465
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
findNonImmUse
static bool findNonImmUse(SDNode *Root, SDNode *Def, SDNode *ImmedUse, bool IgnoreChains)
findNonImmUse - Return true if "Def" is a predecessor of "Root" via a path beyond "ImmedUse".
Definition: SelectionDAGISel.cpp:2059
llvm::SelectionDAGISel::OPC_CheckSame
@ OPC_CheckSame
Definition: SelectionDAGISel.h:126
llvm::MachineRegisterInfo::getVRegDef
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
Definition: MachineRegisterInfo.cpp:400
ViewISelDAGs
static cl::opt< bool > ViewISelDAGs("view-isel-dags", cl::Hidden, cl::desc("Pop up a window to show isel dags as they are selected"))
llvm::SDValue::getValueSizeInBits
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
Definition: SelectionDAGNodes.h:192
llvm::ISD::CopyFromReg
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:208
llvm::PHINode::getNumIncomingValues
unsigned getNumIncomingValues() const
Return the number of incoming edges.
Definition: Instructions.h:2725
llvm::RegisterPassParser
RegisterPassParser class - Handle the addition of new machine passes.
Definition: MachinePassRegistry.h:135
llvm::EHPersonality::Wasm_CXX
@ Wasm_CXX
llvm::MachineRegisterInfo::use_empty
bool use_empty(Register RegNo) const
use_empty - Return true if there are no instructions using the specified register.
Definition: MachineRegisterInfo.h:506
Statepoint.h
llvm::erase_value
void erase_value(Container &C, ValueType V)
Wrapper function to remove a value from a container:
Definition: STLExtras.h:1740
Type.h
BranchProbability.h
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:95
llvm::TargetMachine::resetTargetOptions
void resetTargetOptions(const Function &F) const
Reset the target options based on the function's attributes.
Definition: TargetMachine.cpp:56
llvm::Sched::RegPressure
@ RegPressure
Definition: TargetLowering.h:101
ViewSUnitDAGs
static cl::opt< bool > ViewSUnitDAGs("view-sunit-dags", cl::Hidden, cl::desc("Pop up a window to show SUnit dags after they are processed"))
llvm::InlineAsm::isMemKind
static bool isMemKind(unsigned Flag)
Definition: InlineAsm.h:280
llvm::OptLevelChanger::~OptLevelChanger
~OptLevelChanger()
Definition: SelectionDAGISel.cpp:238
llvm::TargetLowering::createFastISel
virtual FastISel * createFastISel(FunctionLoweringInfo &, const TargetLibraryInfo *) const
This method returns a target specific FastISel object, or null if the target does not support "fast" ...
Definition: TargetLowering.h:4135
Operands
mir Rename Register Operands
Definition: MIRNamerPass.cpp:78
llvm::MachineRegisterInfo::getRegClass
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
Definition: MachineRegisterInfo.h:634
llvm::MCID::MayRaiseFPException
@ MayRaiseFPException
Definition: MCInstrDesc.h:167
llvm::ISD::TargetConstantFP
@ TargetConstantFP
Definition: ISDOpcodes.h:159
llvm::SelectionDAGISel::InvalidateNodeId
static void InvalidateNodeId(SDNode *N)
Definition: SelectionDAGISel.cpp:1087
llvm::APInt::isSubsetOf
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1188
llvm::MDNode::getOperand
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1137
llvm::TargetLowering::insertCopiesSplitCSR
virtual void insertCopiesSplitCSR(MachineBasicBlock *Entry, const SmallVectorImpl< MachineBasicBlock * > &Exits) const
Insert explicit copies in entry and exit blocks.
Definition: TargetLowering.h:3669
llvm::MachineFunction::getMMI
MachineModuleInfo & getMMI() const
Definition: MachineFunction.h:581
llvm::ISD::STRICT_FSETCCS
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:463
llvm::MachineRegisterInfo::clearKillFlags
void clearKillFlags(Register Reg) const
clearKillFlags - Iterate over all the uses of the given register and clear the kill flag from the Mac...
Definition: MachineRegisterInfo.cpp:431
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:630
llvm::MCInstrDesc::mayLoad
bool mayLoad() const
Return true if this instruction could possibly read memory.
Definition: MCInstrDesc.h:429
llvm::ISD::WRITE_REGISTER
@ WRITE_REGISTER
Definition: ISDOpcodes.h:119
Timer.h
BasicBlock.h
llvm::cl::opt
Definition: CommandLine.h:1432
llvm::MachineFunction::setHasInlineAsm
void setHasInlineAsm(bool B)
Set a flag that indicates that the function contains inline assembly.
Definition: MachineFunction.h:708
llvm::SDNode::use_begin
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
Definition: SelectionDAGNodes.h:783
llvm::ISD::Register
@ Register
Definition: ISDOpcodes.h:74
llvm::SelectionDAGISel::OPC_CheckChild4Integer
@ OPC_CheckChild4Integer
Definition: SelectionDAGISel.h:142
llvm::instructions
inst_range instructions(Function *F)
Definition: InstIterator.h:133
llvm::FastISel::selectInstruction
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
Definition: FastISel.cpp:1500
llvm::SelectionDAGISel::OPC_CheckValueType
@ OPC_CheckValueType
Definition: SelectionDAGISel.h:144
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::SelectionDAG::RemoveDeadNode
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
Definition: SelectionDAG.cpp:906
llvm::SelectionDAGISel::OPC_CheckImmAllZerosV
@ OPC_CheckImmAllZerosV
Definition: SelectionDAGISel.h:148
BranchProbabilityInfo.h
llvm::SelectionDAGISel::OPC_CheckInteger
@ OPC_CheckInteger
Definition: SelectionDAGISel.h:140
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:341
llvm::SelectionDAGISel::IsProfitableToFold
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
Definition: SelectionDAGISel.cpp:2099
llvm::AfterLegalizeDAG
@ AfterLegalizeDAG
Definition: DAGCombine.h:19
CheckChild2CondCode
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChild2CondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
Definition: SelectionDAGISel.cpp:2506
Index
uint32_t Index
Definition: ELFObjHandler.cpp:84
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:64
index
splat index
Definition: README_ALTIVEC.txt:181
llvm::MachineInstrBuilder
Definition: MachineInstrBuilder.h:69
llvm::SelectionDAGISel::OPC_RecordMemRef
@ OPC_RecordMemRef
Definition: SelectionDAGISel.h:120
llvm::TargetLibraryInfoWrapperPass
Definition: TargetLibraryInfo.h:465
uint64_t
ProfileSummaryInfo.h
llvm::SelectionDAGISel::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition: SelectionDAGISel.cpp:335
llvm::TargetTransformInfoWrapperPass
Wrapper pass for TargetTransformInfo.
Definition: TargetTransformInfo.h:2387
llvm::SelectionDAGISel::TII
const TargetInstrInfo * TII
Definition: SelectionDAGISel.h:52
llvm::InlineAsm::Op_InputChain
@ Op_InputChain
Definition: InlineAsm.h:211
llvm::GlobalValue::getParent
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:578
llvm::MCInstrDesc::isCall
bool isCall() const
Return true if the instruction is a call.
Definition: MCInstrDesc.h:279
llvm::DbgDeclareInst
This represents the llvm.dbg.declare instruction.
Definition: IntrinsicInst.h:308
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::MachineRegisterInfo::use_instr_begin
use_instr_iterator use_instr_begin(Register RegNo) const
Definition: MachineRegisterInfo.h:477
llvm::SelectionDAGISel::OPC_RecordChild0
@ OPC_RecordChild0
Definition: SelectionDAGISel.h:118
HandleMergeInputChains
static SDValue HandleMergeInputChains(SmallVectorImpl< SDNode * > &ChainNodesMatched, SelectionDAG *CurDAG)
HandleMergeInputChains - This implements the OPC_EmitMergeInputChains operation for when the pattern ...
Definition: SelectionDAGISel.cpp:2316
llvm::ISD::AssertZext
@ AssertZext
Definition: ISDOpcodes.h:62
llvm::Function::hasGC
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
Definition: Function.h:319
llvm::ISD::STRICT_LRINT
@ STRICT_LRINT
Definition: ISDOpcodes.h:419
llvm::SelectionDAG::mutateStrictFPToFP
SDNode * mutateStrictFPToFP(SDNode *Node)
Mutate the specified strict FP node to its non-strict equivalent, unlinking the node from its chain a...
Definition: SelectionDAG.cpp:8970
llvm::SelectionDAGISel::FuncInfo
std::unique_ptr< FunctionLoweringInfo > FuncInfo
Definition: SelectionDAGISel.h:43
llvm::TargetRegisterInfo::getFrameRegister
virtual Register getFrameRegister(const MachineFunction &MF) const =0
Debug information queries.
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:57
llvm::SDNode::setNodeId
void setNodeId(int Id)
Set unique node id.
Definition: SelectionDAGNodes.h:710
llvm::SelectionDAGISel::OPFL_GlueInput
@ OPFL_GlueInput
Definition: SelectionDAGISel.h:177
llvm::SelectionDAG::getCopyFromReg
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:761
llvm::DenseMap< unsigned, unsigned >
llvm::SelectionDAGISel::IsLegalToFold
static bool IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, CodeGenOpt::Level OptLevel, bool IgnoreChains=false)
IsLegalToFold - Returns true if the specific operand node N of U can be folded during instruction sel...
Definition: SelectionDAGISel.cpp:2107
llvm::SelectionDAGISel::mayRaiseFPException
bool mayRaiseFPException(SDNode *Node) const
Return whether the node may raise an FP exception.
Definition: SelectionDAGISel.cpp:3646
llvm::SelectionDAGISel::OPC_CheckChild2Same
@ OPC_CheckChild2Same
Definition: SelectionDAGISel.h:128
llvm::SDNode::getOperand
const SDValue & getOperand(unsigned Num) const
Definition: SelectionDAGNodes.h:904
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::SwiftErrorValueTracking
Definition: SwiftErrorValueTracking.h:36
llvm::MCPhysReg
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition: MCRegister.h:21
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:8345
CheckOpcode
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDNode *N)
Definition: SelectionDAGISel.cpp:2471
llvm::MachineFrameInfo::getObjectAlign
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
Definition: MachineFrameInfo.h:467
llvm::SelectionDAGISel::OPC_CheckChild4Type
@ OPC_CheckChild4Type
Definition: SelectionDAGISel.h:138
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:441
MCRegisterInfo.h
llvm::ProfileSummaryInfoWrapperPass
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
Definition: ProfileSummaryInfo.h:193
llvm::is_contained
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:1616
llvm::SDNode::dump
void dump() const
Dump this node, for debugging.
Definition: SelectionDAGDumper.cpp:539
llvm::MachineRegisterInfo::liveins
ArrayRef< std::pair< MCRegister, Register > > liveins() const
Definition: MachineRegisterInfo.h:969
llvm::MachineFunction::finalizeDebugInstrRefs
void finalizeDebugInstrRefs()
Finalise any partially emitted debug instructions.
Definition: MachineFunction.cpp:1168
llvm::SelectionDAGISel::getPatternForIndex
virtual StringRef getPatternForIndex(unsigned index)
getPatternForIndex - Patterns selected by tablegen during ISEL
Definition: SelectionDAGISel.h:239
MachineFunctionPass.h
llvm::HighlightColor::Address
@ Address
llvm::SelectionDAG::MorphNodeTo
SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
Definition: SelectionDAG.cpp:8917
llvm::SelectionDAG::NewNodesMustHaveLegalTypes
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
Definition: SelectionDAG.h:355
llvm::Register::isVirtualRegister
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:71
llvm::SelectionDAGISel::OPC_CheckImmAllOnesV
@ OPC_CheckImmAllOnesV
Definition: SelectionDAGISel.h:147
llvm::SelectionDAGISel::OPC_SwitchOpcode
@ OPC_SwitchOpcode
Definition: SelectionDAGISel.h:133
llvm::DenseMapBase::find
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:150
llvm::MachineFunction::getName
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Definition: MachineFunction.cpp:542
llvm::SelectionDAGISel::OPC_CheckChild3Integer
@ OPC_CheckChild3Integer
Definition: SelectionDAGISel.h:142
llvm::ISD::STRICT_LROUND
@ STRICT_LROUND
Definition: ISDOpcodes.h:417
llvm::SDValue::getValue
SDValue getValue(unsigned R) const
Definition: SelectionDAGNodes.h:172
llvm::TargetMachine::Options
TargetOptions Options
Definition: TargetMachine.h:120
llvm::SelectionDAGISel::CheckOrMask
bool CheckOrMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const
CheckOrMask - The isel is trying to match something like (or X, 255).
Definition: SelectionDAGISel.cpp:1956
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::TargetMachine
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:79
llvm::SelectionDAGISel::~SelectionDAGISel
~SelectionDAGISel() override
Definition: SelectionDAGISel.cpp:330
llvm::FunctionLoweringInfo
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Definition: FunctionLoweringInfo.h:52
llvm::MVT::Other
@ Other
Definition: MachineValueType.h:42
llvm::SDNode::getNodeId
int getNodeId() const
Return the unique node id.
Definition: SelectionDAGNodes.h:707
llvm::createVLIWDAGScheduler
ScheduleDAGSDNodes * createVLIWDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createVLIWDAGScheduler - Scheduler for VLIW targets.
Definition: ScheduleDAGVLIW.cpp:273
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:840
llvm::MachineFunction::getFrameInfo
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Definition: MachineFunction.h:646
SI
StandardInstrumentations SI(Debug, VerifyEach)
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition: MachineBasicBlock.h:225
llvm::OptimizationRemarkEmitter::emit
void emit(DiagnosticInfoOptimizationBase &OptDiag)
Output the remark via the diagnostic handler and to the optimization record file.
Definition: OptimizationRemarkEmitter.cpp:77
llvm::SelectionDAGISel::OPC_EmitMergeInputChains
@ OPC_EmitMergeInputChains
Definition: SelectionDAGISel.h:156
llvm::InlineAsm::getMemoryConstraintID
static unsigned getMemoryConstraintID(unsigned Flag)
Definition: InlineAsm.h:332
llvm::ISD::CondCode
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1361
llvm::ISD::BasicBlock
@ BasicBlock
Various leaf nodes.
Definition: ISDOpcodes.h:71
llvm::SelectionDAGISel::CurDAG
SelectionDAG * CurDAG
Definition: SelectionDAGISel.h:47
function
print Print MemDeps of function
Definition: MemDepPrinter.cpp:83
llvm::SelectionDAG::getMachineNode
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
Definition: SelectionDAG.cpp:9018
CodeGenCommonISel.h
MachineModuleInfo.h
llvm::ISD::TargetGlobalTLSAddress
@ TargetGlobalTLSAddress
Definition: ISDOpcodes.h:165
llvm::Sched::VLIW
@ VLIW
Definition: TargetLowering.h:104
llvm::MVT
Machine Value Type.
Definition: MachineValueType.h:31
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:97
llvm::SwiftErrorValueTracking::setFunction
void setFunction(MachineFunction &MF)
Initialize data structures for specified new function.
Definition: SwiftErrorValueTracking.cpp:79
llvm::FastISel
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:65
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:360
llvm::SelectionDAGISel::getUninvalidatedNodeId
static int getUninvalidatedNodeId(SDNode *N)
Definition: SelectionDAGISel.cpp:1093
llvm::SelectionDAGISel::SelectionDAGISel
SelectionDAGISel(TargetMachine &tm, CodeGenOpt::Level OL=CodeGenOpt::Default)
Definition: SelectionDAGISel.cpp:316
llvm::SelectionDAG::setNodeMemRefs
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
Definition: SelectionDAG.cpp:8786
SwiftErrorValueTracking.h
llvm::MDNode
Metadata node.
Definition: Metadata.h:906
llvm::ISD::STRICT_LLRINT
@ STRICT_LLRINT
Definition: ISDOpcodes.h:420
llvm::AfterLegalizeVectorOps
@ AfterLegalizeVectorOps
Definition: DAGCombine.h:18
llvm::MCInstrInfo::getName
StringRef getName(unsigned Opcode) const
Returns the name for the instructions with the given opcode.
Definition: MCInstrInfo.h:68
llvm::SelectionDAG::SelectNodeTo
SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
Definition: SelectionDAG.cpp:8810
llvm::TargetOptions::EnableFastISel
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
Definition: TargetOptions.h:230
llvm::SDNode::isMachineOpcode
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
Definition: SelectionDAGNodes.h:681
llvm::ISD::TargetBlockAddress
@ TargetBlockAddress
Definition: ISDOpcodes.h:170
llvm::SelectionDAGISel::OPC_CheckPredicateWithOperands
@ OPC_CheckPredicateWithOperands
Definition: SelectionDAGISel.h:131
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:75
llvm::MachineFunction
Definition: MachineFunction.h:234
llvm::MCInstrDesc::mayStore
bool mayStore() const
Return true if this instruction could possibly modify memory.
Definition: MCInstrDesc.h:435
llvm::ISD::ANNOTATION_LABEL
@ ANNOTATION_LABEL
ANNOTATION_LABEL - Represents a mid basic block label used by annotations.
Definition: ISDOpcodes.h:994
llvm::SelectionDAGISel::CheckNodePredicateWithOperands
virtual bool CheckNodePredicateWithOperands(SDNode *N, unsigned PredNo, const SmallVectorImpl< SDValue > &Operands) const
CheckNodePredicateWithOperands - This function is generated by tblgen in the target.
Definition: SelectionDAGISel.h:281
llvm::MDNodeSDNode
Definition: SelectionDAGNodes.h:2081
llvm::createHybridListDAGScheduler
ScheduleDAGSDNodes * createHybridListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level)
createHybridListDAGScheduler - This creates a bottom up register pressure aware list scheduler that m...
Definition: ScheduleDAGRRList.cpp:3163
llvm::GetSuccessorNumber
unsigned GetSuccessorNumber(const BasicBlock *BB, const BasicBlock *Succ)
Search for the specified successor of basic block BB and return its position in the terminator instru...
Definition: CFG.cpp:79
llvm::CodeGenOpt::None
@ None
Definition: CodeGen.h:53
CFG.h
llvm::ISD::HANDLENODE
@ HANDLENODE
HANDLENODE node - Used as a handle for various purposes.
Definition: ISDOpcodes.h:1071
llvm::MachineBasicBlock::succ_empty
bool succ_empty() const
Definition: MachineBasicBlock.h:347
TargetOptions.h
SelectionDAGISel.h
ViewDAGCombine2
static cl::opt< bool > ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the second " "dag combine pass"))
llvm::ISD::TargetConstantPool
@ TargetConstantPool
Definition: ISDOpcodes.h:168
Scheduler
Machine Instruction Scheduler
Definition: MachineScheduler.cpp:220
llvm::NVPTXISD::Dummy
@ Dummy
Definition: NVPTXISelLowering.h:60
llvm::MachinePassRegistry
MachinePassRegistry - Track the registration of machine passes.
Definition: MachinePassRegistry.h:73
llvm::MachineBasicBlock::getFirstTerminator
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
Definition: MachineBasicBlock.cpp:242
llvm::SmallPtrSetImplBase::clear
void clear()
Definition: SmallPtrSet.h:94
llvm::Sched::Source
@ Source
Definition: TargetLowering.h:100
llvm::initializeAAResultsWrapperPassPass
void initializeAAResultsWrapperPassPass(PassRegistry &)
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::LoopInfo
Definition: LoopInfo.h:1083
llvm::StackProtector::shouldEmitSDCheck
bool shouldEmitSDCheck(const BasicBlock &BB) const
Definition: StackProtector.cpp:592
None.h
llvm::OptimizationRemarkEmitter
The optimization diagnostic interface.
Definition: OptimizationRemarkEmitter.h:33
llvm::SDNode::use_end
static use_iterator use_end()
Definition: SelectionDAGNodes.h:787
llvm::MachineFrameInfo::setHasCalls
void setHasCalls(bool V)
Definition: MachineFrameInfo.h:582
DataLayout.h
llvm::SelectionDAGISel::OPC_CheckChild3Type
@ OPC_CheckChild3Type
Definition: SelectionDAGISel.h:138
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm::MachineBasicBlock::splice
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Definition: MachineBasicBlock.h:950
llvm::SelectionDAGISel::SelectInlineAsmMemoryOperand
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
Definition: SelectionDAGISel.h:91
MBBI
MachineBasicBlock MachineBasicBlock::iterator MBBI
Definition: AArch64SLSHardening.cpp:75
llvm::SDNodeFlags::setNoFPExcept
void setNoFPExcept(bool b)
Definition: SelectionDAGNodes.h:421
llvm::SelectionDAGBuilder
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
Definition: SelectionDAGBuilder.h:94
llvm::InlineAsm::getFlagWord
static unsigned getFlagWord(unsigned Kind, unsigned NumOps)
Definition: InlineAsm.h:272
llvm::Sched::Hybrid
@ Hybrid
Definition: TargetLowering.h:102
llvm::ISD::STRICT_SINT_TO_FP
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:435
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:134
llvm::AnalysisUsage::addPreserved
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
Definition: PassAnalysisSupport.h:98
llvm::ms_demangle::IntrinsicFunctionKind::New
@ New
llvm::BranchProbability
Definition: BranchProbability.h:30
Compiler.h
llvm::append_range
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
Definition: STLExtras.h:1748
llvm::TargetLowering::getRegisterByName
virtual Register getRegisterByName(const char *RegName, LLT Ty, const MachineFunction &MF) const
Return the register ID of the name passed in.
Definition: TargetLowering.h:4018
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:81
llvm::SelectionDAG::setFunctionLoweringInfo
void setFunctionLoweringInfo(FunctionLoweringInfo *FuncInfo)
Definition: SelectionDAG.h:429
TargetSubtargetInfo.h
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::ISD::UNDEF
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:211
llvm::SelectionDAGISel::OPC_MoveChild3
@ OPC_MoveChild3
Definition: SelectionDAGISel.h:123
llvm::SelectionDAGISel::OPC_MoveChild6
@ OPC_MoveChild6
Definition: SelectionDAGISel.h:124
llvm::AMDGPUISD::BFI
@ BFI
Definition: AMDGPUISelLowering.h:421
llvm::CodeGenOpt::Level
Level
Definition: CodeGen.h:52
llvm::MachineInstr::getParent
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:286
llvm::MCInstrDesc::mayRaiseFPException
bool mayRaiseFPException() const
Return true if this instruction may raise a floating-point exception.
Definition: MCInstrDesc.h:438
llvm::TargetMachine::setFastISel
void setFastISel(bool Enable)
Definition: TargetMachine.h:245
llvm::Function::callsFunctionThatReturnsTwice
bool callsFunctionThatReturnsTwice() const
callsFunctionThatReturnsTwice - Return true if the function has a call to setjmp or other function th...
Definition: Function.cpp:1814
llvm::MVT::iPTR
@ iPTR
Definition: MachineValueType.h:312
llvm::SelectionDAGISel::OPC_EmitNode2
@ OPC_EmitNode2
Definition: SelectionDAGISel.h:165
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
llvm::SDVTList
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Definition: SelectionDAGNodes.h:79
for
this could be done in SelectionDAGISel along with other special for
Definition: README.txt:104
llvm::DenseMapBase::insert
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:207
llvm::SelectionDAGISel::SwiftError
SwiftErrorValueTracking * SwiftError
Definition: SelectionDAGISel.h:44
llvm::BasicBlock::getTerminator
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:152
llvm::TargetSubtargetInfo
TargetSubtargetInfo - Generic base class for all target subtargets.
Definition: TargetSubtargetInfo.h:59
llvm::OptLevelChanger
This class is used by SelectionDAGISel to temporarily override the optimization level on a per-functi...
Definition: SelectionDAGISel.cpp:211
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::ISD::INTRINSIC_WO_CHAIN
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:184
llvm::SelectionDAGISel::OPC_CheckChild3Same
@ OPC_CheckChild3Same
Definition: SelectionDAGISel.h:128
llvm::MachineBasicBlock::addLiveIn
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
Definition: MachineBasicBlock.h:367
llvm::InlineAsm::getFlagWordForMem
static unsigned getFlagWordForMem(unsigned InputFlag, unsigned Constraint)
Augment an existing flag word returned by getFlagWord with the constraint code for a memory constrain...
Definition: InlineAsm.h:315
ISHeuristic
static cl::opt< RegisterScheduler::FunctionPassCtor, false, RegisterPassParser< RegisterScheduler > > ISHeuristic("pre-RA-sched", cl::init(&createDefaultScheduler), cl::Hidden, cl::desc("Instruction schedulers available (before register" " allocation):"))
ISHeuristic command line option for instruction schedulers.
llvm::MachineFrameInfo::hasCalls
bool hasCalls() const
Return true if the current function has any function calls.
Definition: MachineFrameInfo.h:581
llvm::MachineRegisterInfo::replaceRegWith
void replaceRegWith(Register FromReg, Register ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
Definition: MachineRegisterInfo.cpp:380
MBB
MachineBasicBlock & MBB
Definition: AArch64SLSHardening.cpp:74
llvm::ISD::READ_REGISTER
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition: ISDOpcodes.h:118
llvm::SelectionDAGISel::BuiltinOpcodes
BuiltinOpcodes
Definition: SelectionDAGISel.h:115
llvm::TargetLoweringBase::getSSPStackGuardCheck
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
Definition: TargetLoweringBase.cpp:1995
llvm::SDNode::isOnlyUserOf
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
Definition: SelectionDAG.cpp:10221
llvm::SelectionDAGISel::MF
MachineFunction * MF
Definition: SelectionDAGISel.h:45