LLVM  17.0.0git
SelectionDAGISel.cpp
Go to the documentation of this file.
1 //===- SelectionDAGISel.cpp - Implement the SelectionDAGISel class --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements the SelectionDAGISel class.
10 //
11 //===----------------------------------------------------------------------===//
12 
14 #include "ScheduleDAGSDNodes.h"
15 #include "SelectionDAGBuilder.h"
16 #include "llvm/ADT/APInt.h"
17 #include "llvm/ADT/DenseMap.h"
19 #include "llvm/ADT/STLExtras.h"
20 #include "llvm/ADT/SmallPtrSet.h"
21 #include "llvm/ADT/SmallVector.h"
22 #include "llvm/ADT/Statistic.h"
23 #include "llvm/ADT/StringRef.h"
27 #include "llvm/Analysis/CFG.h"
36 #include "llvm/CodeGen/FastISel.h"
54 #include "llvm/CodeGen/StackMaps.h"
62 #include "llvm/IR/BasicBlock.h"
63 #include "llvm/IR/Constants.h"
64 #include "llvm/IR/DataLayout.h"
65 #include "llvm/IR/DebugInfo.h"
67 #include "llvm/IR/DebugLoc.h"
68 #include "llvm/IR/DiagnosticInfo.h"
70 #include "llvm/IR/Function.h"
71 #include "llvm/IR/InlineAsm.h"
72 #include "llvm/IR/InstIterator.h"
73 #include "llvm/IR/Instruction.h"
74 #include "llvm/IR/Instructions.h"
75 #include "llvm/IR/IntrinsicInst.h"
76 #include "llvm/IR/Intrinsics.h"
77 #include "llvm/IR/IntrinsicsWebAssembly.h"
78 #include "llvm/IR/Metadata.h"
79 #include "llvm/IR/Statepoint.h"
80 #include "llvm/IR/Type.h"
81 #include "llvm/IR/User.h"
82 #include "llvm/IR/Value.h"
83 #include "llvm/InitializePasses.h"
84 #include "llvm/MC/MCInstrDesc.h"
85 #include "llvm/Pass.h"
87 #include "llvm/Support/Casting.h"
88 #include "llvm/Support/CodeGen.h"
90 #include "llvm/Support/Compiler.h"
91 #include "llvm/Support/Debug.h"
93 #include "llvm/Support/KnownBits.h"
95 #include "llvm/Support/Timer.h"
101 #include <algorithm>
102 #include <cassert>
103 #include <cstdint>
104 #include <iterator>
105 #include <limits>
106 #include <memory>
107 #include <optional>
108 #include <string>
109 #include <utility>
110 #include <vector>
111 
112 using namespace llvm;
113 
114 #define DEBUG_TYPE "isel"
115 
116 STATISTIC(NumFastIselFailures, "Number of instructions fast isel failed on");
117 STATISTIC(NumFastIselSuccess, "Number of instructions fast isel selected");
118 STATISTIC(NumFastIselBlocks, "Number of blocks selected entirely by fast isel");
119 STATISTIC(NumDAGBlocks, "Number of blocks selected using DAG");
120 STATISTIC(NumDAGIselRetries,"Number of times dag isel has to try another path");
121 STATISTIC(NumEntryBlocks, "Number of entry blocks encountered");
122 STATISTIC(NumFastIselFailLowerArguments,
123  "Number of entry blocks where fast isel failed to lower arguments");
124 
126  "fast-isel-abort", cl::Hidden,
127  cl::desc("Enable abort calls when \"fast\" instruction selection "
128  "fails to lower an instruction: 0 disable the abort, 1 will "
129  "abort but for args, calls and terminators, 2 will also "
130  "abort for argument lowering, and 3 will never fallback "
131  "to SelectionDAG."));
132 
134  "fast-isel-report-on-fallback", cl::Hidden,
135  cl::desc("Emit a diagnostic when \"fast\" instruction selection "
136  "falls back to SelectionDAG."));
137 
138 static cl::opt<bool>
139 UseMBPI("use-mbpi",
140  cl::desc("use Machine Branch Probability Info"),
141  cl::init(true), cl::Hidden);
142 
143 #ifndef NDEBUG
145 FilterDAGBasicBlockName("filter-view-dags", cl::Hidden,
146  cl::desc("Only display the basic block whose name "
147  "matches this for all view-*-dags options"));
148 static cl::opt<bool>
149 ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden,
150  cl::desc("Pop up a window to show dags before the first "
151  "dag combine pass"));
152 static cl::opt<bool>
153 ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden,
154  cl::desc("Pop up a window to show dags before legalize types"));
155 static cl::opt<bool>
156  ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden,
157  cl::desc("Pop up a window to show dags before the post "
158  "legalize types dag combine pass"));
159 static cl::opt<bool>
160  ViewLegalizeDAGs("view-legalize-dags", cl::Hidden,
161  cl::desc("Pop up a window to show dags before legalize"));
162 static cl::opt<bool>
163 ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden,
164  cl::desc("Pop up a window to show dags before the second "
165  "dag combine pass"));
166 static cl::opt<bool>
167 ViewISelDAGs("view-isel-dags", cl::Hidden,
168  cl::desc("Pop up a window to show isel dags as they are selected"));
169 static cl::opt<bool>
170 ViewSchedDAGs("view-sched-dags", cl::Hidden,
171  cl::desc("Pop up a window to show sched dags as they are processed"));
172 static cl::opt<bool>
173 ViewSUnitDAGs("view-sunit-dags", cl::Hidden,
174  cl::desc("Pop up a window to show SUnit dags after they are processed"));
175 #else
176 static const bool ViewDAGCombine1 = false, ViewLegalizeTypesDAGs = false,
177  ViewDAGCombineLT = false, ViewLegalizeDAGs = false,
178  ViewDAGCombine2 = false, ViewISelDAGs = false,
179  ViewSchedDAGs = false, ViewSUnitDAGs = false;
180 #endif
181 
182 //===---------------------------------------------------------------------===//
183 ///
184 /// RegisterScheduler class - Track the registration of instruction schedulers.
185 ///
186 //===---------------------------------------------------------------------===//
189 
190 //===---------------------------------------------------------------------===//
191 ///
192 /// ISHeuristic command line option for instruction schedulers.
193 ///
194 //===---------------------------------------------------------------------===//
197 ISHeuristic("pre-RA-sched",
199  cl::desc("Instruction schedulers available (before register"
200  " allocation):"));
201 
202 static RegisterScheduler
203 defaultListDAGScheduler("default", "Best scheduler for the target",
205 
206 namespace llvm {
207 
208  //===--------------------------------------------------------------------===//
209  /// This class is used by SelectionDAGISel to temporarily override
210  /// the optimization level on a per-function basis.
212  SelectionDAGISel &IS;
213  CodeGenOpt::Level SavedOptLevel;
214  bool SavedFastISel;
215 
216  public:
218  CodeGenOpt::Level NewOptLevel) : IS(ISel) {
219  SavedOptLevel = IS.OptLevel;
220  SavedFastISel = IS.TM.Options.EnableFastISel;
221  if (NewOptLevel == SavedOptLevel)
222  return;
223  IS.OptLevel = NewOptLevel;
224  IS.TM.setOptLevel(NewOptLevel);
225  LLVM_DEBUG(dbgs() << "\nChanging optimization level for Function "
226  << IS.MF->getFunction().getName() << "\n");
227  LLVM_DEBUG(dbgs() << "\tBefore: -O" << SavedOptLevel << " ; After: -O"
228  << NewOptLevel << "\n");
229  if (NewOptLevel == CodeGenOpt::None) {
231  LLVM_DEBUG(
232  dbgs() << "\tFastISel is "
233  << (IS.TM.Options.EnableFastISel ? "enabled" : "disabled")
234  << "\n");
235  }
236  }
237 
239  if (IS.OptLevel == SavedOptLevel)
240  return;
241  LLVM_DEBUG(dbgs() << "\nRestoring optimization level for Function "
242  << IS.MF->getFunction().getName() << "\n");
243  LLVM_DEBUG(dbgs() << "\tBefore: -O" << IS.OptLevel << " ; After: -O"
244  << SavedOptLevel << "\n");
245  IS.OptLevel = SavedOptLevel;
246  IS.TM.setOptLevel(SavedOptLevel);
247  IS.TM.setFastISel(SavedFastISel);
248  }
249  };
250 
251  //===--------------------------------------------------------------------===//
252  /// createDefaultScheduler - This creates an instruction scheduler appropriate
253  /// for the target.
255  CodeGenOpt::Level OptLevel) {
256  const TargetLowering *TLI = IS->TLI;
257  const TargetSubtargetInfo &ST = IS->MF->getSubtarget();
258 
259  // Try first to see if the Target has its own way of selecting a scheduler
260  if (auto *SchedulerCtor = ST.getDAGScheduler(OptLevel)) {
261  return SchedulerCtor(IS, OptLevel);
262  }
263 
264  if (OptLevel == CodeGenOpt::None ||
265  (ST.enableMachineScheduler() && ST.enableMachineSchedDefaultSched()) ||
267  return createSourceListDAGScheduler(IS, OptLevel);
269  return createBURRListDAGScheduler(IS, OptLevel);
271  return createHybridListDAGScheduler(IS, OptLevel);
272  if (TLI->getSchedulingPreference() == Sched::VLIW)
273  return createVLIWDAGScheduler(IS, OptLevel);
274  if (TLI->getSchedulingPreference() == Sched::Fast)
275  return createFastDAGScheduler(IS, OptLevel);
277  return createDAGLinearizer(IS, OptLevel);
279  "Unknown sched type!");
280  return createILPListDAGScheduler(IS, OptLevel);
281  }
282 
283 } // end namespace llvm
284 
285 // EmitInstrWithCustomInserter - This method should be implemented by targets
286 // that mark instructions with the 'usesCustomInserter' flag. These
287 // instructions are special in various ways, which require special support to
288 // insert. The specified MachineInstr is created but not inserted into any
289 // basic blocks, and this method is called to expand it into a sequence of
290 // instructions, potentially also creating new basic blocks and control flow.
291 // When new basic blocks are inserted and the edges from MBB to its successors
292 // are modified, the method should insert pairs of <OldSucc, NewSucc> into the
293 // DenseMap.
296  MachineBasicBlock *MBB) const {
297 #ifndef NDEBUG
298  dbgs() << "If a target marks an instruction with "
299  "'usesCustomInserter', it must implement "
300  "TargetLowering::EmitInstrWithCustomInserter!\n";
301 #endif
302  llvm_unreachable(nullptr);
303 }
304 
306  SDNode *Node) const {
307  assert(!MI.hasPostISelHook() &&
308  "If a target marks an instruction with 'hasPostISelHook', "
309  "it must implement TargetLowering::AdjustInstrPostInstrSelection!");
310 }
311 
312 //===----------------------------------------------------------------------===//
313 // SelectionDAGISel code
314 //===----------------------------------------------------------------------===//
315 
318  : MachineFunctionPass(ID), TM(tm), FuncInfo(new FunctionLoweringInfo()),
319  SwiftError(new SwiftErrorValueTracking()),
320  CurDAG(new SelectionDAG(tm, OL)),
321  SDB(std::make_unique<SelectionDAGBuilder>(*CurDAG, *FuncInfo, *SwiftError,
322  OL)),
323  OptLevel(OL) {
329 }
330 
332  delete CurDAG;
333  delete SwiftError;
334 }
335 
337  if (OptLevel != CodeGenOpt::None)
348  // AssignmentTrackingAnalysis only runs if assignment tracking is enabled for
349  // the module.
352  if (OptLevel != CodeGenOpt::None)
355 }
356 
357 static void computeUsesMSVCFloatingPoint(const Triple &TT, const Function &F,
358  MachineModuleInfo &MMI) {
359  // Only needed for MSVC
360  if (!TT.isWindowsMSVCEnvironment())
361  return;
362 
363  // If it's already set, nothing to do.
364  if (MMI.usesMSVCFloatingPoint())
365  return;
366 
367  for (const Instruction &I : instructions(F)) {
368  if (I.getType()->isFPOrFPVectorTy()) {
369  MMI.setUsesMSVCFloatingPoint(true);
370  return;
371  }
372  for (const auto &Op : I.operands()) {
373  if (Op->getType()->isFPOrFPVectorTy()) {
374  MMI.setUsesMSVCFloatingPoint(true);
375  return;
376  }
377  }
378  }
379 }
380 
382  // If we already selected that function, we do not need to run SDISel.
383  if (mf.getProperties().hasProperty(
385  return false;
386  // Do some sanity-checking on the command-line options.
388  "-fast-isel-abort > 0 requires -fast-isel");
389 
390  const Function &Fn = mf.getFunction();
391  MF = &mf;
392 
393  // Decide what flavour of variable location debug-info will be used, before
394  // we change the optimisation level.
395  bool InstrRef = mf.shouldUseDebugInstrRef();
396  mf.setUseDebugInstrRef(InstrRef);
397 
398  // Reset the target options before resetting the optimization
399  // level below.
400  // FIXME: This is a horrible hack and should be processed via
401  // codegen looking at the optimization level explicitly when
402  // it wants to look at it.
404  // Reset OptLevel to None for optnone functions.
405  CodeGenOpt::Level NewOptLevel = OptLevel;
406  if (OptLevel != CodeGenOpt::None && skipFunction(Fn))
407  NewOptLevel = CodeGenOpt::None;
408  OptLevelChanger OLC(*this, NewOptLevel);
409 
412  RegInfo = &MF->getRegInfo();
413  LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(Fn);
414  GFI = Fn.hasGC() ? &getAnalysis<GCModuleInfo>().getFunctionInfo(Fn) : nullptr;
415  ORE = std::make_unique<OptimizationRemarkEmitter>(&Fn);
416  AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(mf.getFunction());
417  auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
418  BlockFrequencyInfo *BFI = nullptr;
419  if (PSI && PSI->hasProfileSummary() && OptLevel != CodeGenOpt::None)
420  BFI = &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI();
421 
422  FunctionVarLocs const *FnVarLocs = nullptr;
424  FnVarLocs = getAnalysis<AssignmentTrackingAnalysis>().getResults();
425 
426  LLVM_DEBUG(dbgs() << "\n\n\n=== " << Fn.getName() << "\n");
427 
428  CurDAG->init(*MF, *ORE, this, LibInfo,
429  getAnalysisIfAvailable<LegacyDivergenceAnalysis>(), PSI, BFI,
430  FnVarLocs);
431  FuncInfo->set(Fn, *MF, CurDAG);
433 
434  // Now get the optional analyzes if we want to.
435  // This is based on the possibly changed OptLevel (after optnone is taken
436  // into account). That's unfortunate but OK because it just means we won't
437  // ask for passes that have been required anyway.
438 
440  FuncInfo->BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
441  else
442  FuncInfo->BPI = nullptr;
443 
444  if (OptLevel != CodeGenOpt::None)
445  AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
446  else
447  AA = nullptr;
448 
449  SDB->init(GFI, AA, AC, LibInfo);
450 
451  MF->setHasInlineAsm(false);
452 
453  FuncInfo->SplitCSR = false;
454 
455  // We split CSR if the target supports it for the given function
456  // and the function has only return exits.
458  FuncInfo->SplitCSR = true;
459 
460  // Collect all the return blocks.
461  for (const BasicBlock &BB : Fn) {
462  if (!succ_empty(&BB))
463  continue;
464 
465  const Instruction *Term = BB.getTerminator();
466  if (isa<UnreachableInst>(Term) || isa<ReturnInst>(Term))
467  continue;
468 
469  // Bail out if the exit block is not Return nor Unreachable.
470  FuncInfo->SplitCSR = false;
471  break;
472  }
473  }
474 
475  MachineBasicBlock *EntryMBB = &MF->front();
476  if (FuncInfo->SplitCSR)
477  // This performs initialization so lowering for SplitCSR will be correct.
478  TLI->initializeSplitCSR(EntryMBB);
479 
480  SelectAllBasicBlocks(Fn);
482  DiagnosticInfoISelFallback DiagFallback(Fn);
483  Fn.getContext().diagnose(DiagFallback);
484  }
485 
486  // Replace forward-declared registers with the registers containing
487  // the desired value.
488  // Note: it is important that this happens **before** the call to
489  // EmitLiveInCopies, since implementations can skip copies of unused
490  // registers. If we don't apply the reg fixups before, some registers may
491  // appear as unused and will be skipped, resulting in bad MI.
493  for (DenseMap<Register, Register>::iterator I = FuncInfo->RegFixups.begin(),
494  E = FuncInfo->RegFixups.end();
495  I != E; ++I) {
496  Register From = I->first;
497  Register To = I->second;
498  // If To is also scheduled to be replaced, find what its ultimate
499  // replacement is.
500  while (true) {
501  DenseMap<Register, Register>::iterator J = FuncInfo->RegFixups.find(To);
502  if (J == E)
503  break;
504  To = J->second;
505  }
506  // Make sure the new register has a sufficiently constrained register class.
507  if (From.isVirtual() && To.isVirtual())
509  // Replace it.
510 
511  // Replacing one register with another won't touch the kill flags.
512  // We need to conservatively clear the kill flags as a kill on the old
513  // register might dominate existing uses of the new register.
514  if (!MRI.use_empty(To))
516  MRI.replaceRegWith(From, To);
517  }
518 
519  // If the first basic block in the function has live ins that need to be
520  // copied into vregs, emit the copies into the top of the block before
521  // emitting the code for the block.
523  RegInfo->EmitLiveInCopies(EntryMBB, TRI, *TII);
524 
525  // Insert copies in the entry block and the return blocks.
526  if (FuncInfo->SplitCSR) {
528  // Collect all the return blocks.
529  for (MachineBasicBlock &MBB : mf) {
530  if (!MBB.succ_empty())
531  continue;
532 
534  if (Term != MBB.end() && Term->isReturn()) {
535  Returns.push_back(&MBB);
536  continue;
537  }
538  }
539  TLI->insertCopiesSplitCSR(EntryMBB, Returns);
540  }
541 
543  if (!FuncInfo->ArgDbgValues.empty())
544  for (std::pair<unsigned, unsigned> LI : RegInfo->liveins())
545  if (LI.second)
546  LiveInMap.insert(LI);
547 
548  // Insert DBG_VALUE instructions for function arguments to the entry block.
549  for (unsigned i = 0, e = FuncInfo->ArgDbgValues.size(); i != e; ++i) {
550  MachineInstr *MI = FuncInfo->ArgDbgValues[e - i - 1];
551  assert(MI->getOpcode() != TargetOpcode::DBG_VALUE_LIST &&
552  "Function parameters should not be described by DBG_VALUE_LIST.");
553  bool hasFI = MI->getDebugOperand(0).isFI();
554  Register Reg =
555  hasFI ? TRI.getFrameRegister(*MF) : MI->getDebugOperand(0).getReg();
556  if (Reg.isPhysical())
557  EntryMBB->insert(EntryMBB->begin(), MI);
558  else {
560  if (Def) {
561  MachineBasicBlock::iterator InsertPos = Def;
562  // FIXME: VR def may not be in entry block.
563  Def->getParent()->insert(std::next(InsertPos), MI);
564  } else
565  LLVM_DEBUG(dbgs() << "Dropping debug info for dead vreg"
566  << Register::virtReg2Index(Reg) << "\n");
567  }
568 
569  // Don't try and extend through copies in instruction referencing mode.
570  if (InstrRef)
571  continue;
572 
573  // If Reg is live-in then update debug info to track its copy in a vreg.
575  if (LDI != LiveInMap.end()) {
576  assert(!hasFI && "There's no handling of frame pointer updating here yet "
577  "- add if needed");
578  MachineInstr *Def = RegInfo->getVRegDef(LDI->second);
579  MachineBasicBlock::iterator InsertPos = Def;
580  const MDNode *Variable = MI->getDebugVariable();
581  const MDNode *Expr = MI->getDebugExpression();
582  DebugLoc DL = MI->getDebugLoc();
583  bool IsIndirect = MI->isIndirectDebugValue();
584  if (IsIndirect)
585  assert(MI->getDebugOffset().getImm() == 0 &&
586  "DBG_VALUE with nonzero offset");
587  assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) &&
588  "Expected inlined-at fields to agree");
589  assert(MI->getOpcode() != TargetOpcode::DBG_VALUE_LIST &&
590  "Didn't expect to see a DBG_VALUE_LIST here");
591  // Def is never a terminator here, so it is ok to increment InsertPos.
592  BuildMI(*EntryMBB, ++InsertPos, DL, TII->get(TargetOpcode::DBG_VALUE),
593  IsIndirect, LDI->second, Variable, Expr);
594 
595  // If this vreg is directly copied into an exported register then
596  // that COPY instructions also need DBG_VALUE, if it is the only
597  // user of LDI->second.
598  MachineInstr *CopyUseMI = nullptr;
600  UI = RegInfo->use_instr_begin(LDI->second),
601  E = RegInfo->use_instr_end(); UI != E; ) {
602  MachineInstr *UseMI = &*(UI++);
603  if (UseMI->isDebugValue()) continue;
604  if (UseMI->isCopy() && !CopyUseMI && UseMI->getParent() == EntryMBB) {
605  CopyUseMI = UseMI; continue;
606  }
607  // Otherwise this is another use or second copy use.
608  CopyUseMI = nullptr; break;
609  }
610  if (CopyUseMI &&
611  TRI.getRegSizeInBits(LDI->second, MRI) ==
612  TRI.getRegSizeInBits(CopyUseMI->getOperand(0).getReg(), MRI)) {
613  // Use MI's debug location, which describes where Variable was
614  // declared, rather than whatever is attached to CopyUseMI.
615  MachineInstr *NewMI =
616  BuildMI(*MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
617  CopyUseMI->getOperand(0).getReg(), Variable, Expr);
618  MachineBasicBlock::iterator Pos = CopyUseMI;
619  EntryMBB->insertAfter(Pos, NewMI);
620  }
621  }
622  }
623 
624  // For debug-info, in instruction referencing mode, we need to perform some
625  // post-isel maintenence.
626  if (MF->useDebugInstrRef())
628 
629  // Determine if there are any calls in this machine function.
630  MachineFrameInfo &MFI = MF->getFrameInfo();
631  for (const auto &MBB : *MF) {
632  if (MFI.hasCalls() && MF->hasInlineAsm())
633  break;
634 
635  for (const auto &MI : MBB) {
636  const MCInstrDesc &MCID = TII->get(MI.getOpcode());
637  if ((MCID.isCall() && !MCID.isReturn()) ||
638  MI.isStackAligningInlineAsm()) {
639  MFI.setHasCalls(true);
640  }
641  if (MI.isInlineAsm()) {
642  MF->setHasInlineAsm(true);
643  }
644  }
645  }
646 
647  // Determine if there is a call to setjmp in the machine function.
649 
650  // Determine if floating point is used for msvc
652 
653  // Release function-specific state. SDB and CurDAG are already cleared
654  // at this point.
655  FuncInfo->clear();
656 
657  LLVM_DEBUG(dbgs() << "*** MachineFunction at end of ISel ***\n");
658  LLVM_DEBUG(MF->print(dbgs()));
659 
660  return true;
661 }
662 
666  bool ShouldAbort) {
667  // Print the function name explicitly if we don't have a debug location (which
668  // makes the diagnostic less useful) or if we're going to emit a raw error.
669  if (!R.getLocation().isValid() || ShouldAbort)
670  R << (" (in function: " + MF.getName() + ")").str();
671 
672  if (ShouldAbort)
673  report_fatal_error(Twine(R.getMsg()));
674 
675  ORE.emit(R);
676  LLVM_DEBUG(dbgs() << R.getMsg() << "\n");
677 }
678 
679 void SelectionDAGISel::SelectBasicBlock(BasicBlock::const_iterator Begin,
681  bool &HadTailCall) {
682  // Allow creating illegal types during DAG building for the basic block.
684 
685  // Lower the instructions. If a call is emitted as a tail call, cease emitting
686  // nodes for this block.
687  for (BasicBlock::const_iterator I = Begin; I != End && !SDB->HasTailCall; ++I) {
688  if (!ElidedArgCopyInstrs.count(&*I))
689  SDB->visit(*I);
690  }
691 
692  // Make sure the root of the DAG is up-to-date.
693  CurDAG->setRoot(SDB->getControlRoot());
694  HadTailCall = SDB->HasTailCall;
695  SDB->resolveOrClearDbgInfo();
696  SDB->clear();
697 
698  // Final step, emit the lowered DAG as machine code.
699  CodeGenAndEmitDAG();
700 }
701 
702 void SelectionDAGISel::ComputeLiveOutVRegInfo() {
704  SmallVector<SDNode*, 128> Worklist;
705 
706  Worklist.push_back(CurDAG->getRoot().getNode());
707  Added.insert(CurDAG->getRoot().getNode());
708 
709  KnownBits Known;
710 
711  do {
712  SDNode *N = Worklist.pop_back_val();
713 
714  // Otherwise, add all chain operands to the worklist.
715  for (const SDValue &Op : N->op_values())
716  if (Op.getValueType() == MVT::Other && Added.insert(Op.getNode()).second)
717  Worklist.push_back(Op.getNode());
718 
719  // If this is a CopyToReg with a vreg dest, process it.
720  if (N->getOpcode() != ISD::CopyToReg)
721  continue;
722 
723  unsigned DestReg = cast<RegisterSDNode>(N->getOperand(1))->getReg();
724  if (!Register::isVirtualRegister(DestReg))
725  continue;
726 
727  // Ignore non-integer values.
728  SDValue Src = N->getOperand(2);
729  EVT SrcVT = Src.getValueType();
730  if (!SrcVT.isInteger())
731  continue;
732 
733  unsigned NumSignBits = CurDAG->ComputeNumSignBits(Src);
734  Known = CurDAG->computeKnownBits(Src);
735  FuncInfo->AddLiveOutRegInfo(DestReg, NumSignBits, Known);
736  } while (!Worklist.empty());
737 }
738 
739 void SelectionDAGISel::CodeGenAndEmitDAG() {
740  StringRef GroupName = "sdag";
741  StringRef GroupDescription = "Instruction Selection and Scheduling";
742  std::string BlockName;
743  bool MatchFilterBB = false; (void)MatchFilterBB;
744 #ifndef NDEBUG
746  getAnalysis<TargetTransformInfoWrapperPass>().getTTI(*FuncInfo->Fn);
747 #endif
748 
749  // Pre-type legalization allow creation of any node types.
751 
752 #ifndef NDEBUG
753  MatchFilterBB = (FilterDAGBasicBlockName.empty() ||
755  FuncInfo->MBB->getBasicBlock()->getName());
756 #endif
757 #ifdef NDEBUG
761 #endif
762  {
763  BlockName =
764  (MF->getName() + ":" + FuncInfo->MBB->getBasicBlock()->getName()).str();
765  }
766  LLVM_DEBUG(dbgs() << "Initial selection DAG: "
767  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
768  << "'\n";
769  CurDAG->dump());
770 
771 #ifndef NDEBUG
772  if (TTI.hasBranchDivergence())
774 #endif
775 
776  if (ViewDAGCombine1 && MatchFilterBB)
777  CurDAG->viewGraph("dag-combine1 input for " + BlockName);
778 
779  // Run the DAG combiner in pre-legalize mode.
780  {
781  NamedRegionTimer T("combine1", "DAG Combining 1", GroupName,
782  GroupDescription, TimePassesIsEnabled);
784  }
785 
786  LLVM_DEBUG(dbgs() << "Optimized lowered selection DAG: "
787  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
788  << "'\n";
789  CurDAG->dump());
790 
791 #ifndef NDEBUG
792  if (TTI.hasBranchDivergence())
794 #endif
795 
796  // Second step, hack on the DAG until it only uses operations and types that
797  // the target supports.
798  if (ViewLegalizeTypesDAGs && MatchFilterBB)
799  CurDAG->viewGraph("legalize-types input for " + BlockName);
800 
801  bool Changed;
802  {
803  NamedRegionTimer T("legalize_types", "Type Legalization", GroupName,
804  GroupDescription, TimePassesIsEnabled);
805  Changed = CurDAG->LegalizeTypes();
806  }
807 
808  LLVM_DEBUG(dbgs() << "Type-legalized selection DAG: "
809  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
810  << "'\n";
811  CurDAG->dump());
812 
813 #ifndef NDEBUG
814  if (TTI.hasBranchDivergence())
816 #endif
817 
818  // Only allow creation of legal node types.
820 
821  if (Changed) {
822  if (ViewDAGCombineLT && MatchFilterBB)
823  CurDAG->viewGraph("dag-combine-lt input for " + BlockName);
824 
825  // Run the DAG combiner in post-type-legalize mode.
826  {
827  NamedRegionTimer T("combine_lt", "DAG Combining after legalize types",
828  GroupName, GroupDescription, TimePassesIsEnabled);
830  }
831 
832  LLVM_DEBUG(dbgs() << "Optimized type-legalized selection DAG: "
833  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
834  << "'\n";
835  CurDAG->dump());
836 
837 #ifndef NDEBUG
838  if (TTI.hasBranchDivergence())
840 #endif
841  }
842 
843  {
844  NamedRegionTimer T("legalize_vec", "Vector Legalization", GroupName,
845  GroupDescription, TimePassesIsEnabled);
846  Changed = CurDAG->LegalizeVectors();
847  }
848 
849  if (Changed) {
850  LLVM_DEBUG(dbgs() << "Vector-legalized selection DAG: "
851  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
852  << "'\n";
853  CurDAG->dump());
854 
855 #ifndef NDEBUG
856  if (TTI.hasBranchDivergence())
858 #endif
859 
860  {
861  NamedRegionTimer T("legalize_types2", "Type Legalization 2", GroupName,
862  GroupDescription, TimePassesIsEnabled);
864  }
865 
866  LLVM_DEBUG(dbgs() << "Vector/type-legalized selection DAG: "
867  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
868  << "'\n";
869  CurDAG->dump());
870 
871 #ifndef NDEBUG
872  if (TTI.hasBranchDivergence())
874 #endif
875 
876  if (ViewDAGCombineLT && MatchFilterBB)
877  CurDAG->viewGraph("dag-combine-lv input for " + BlockName);
878 
879  // Run the DAG combiner in post-type-legalize mode.
880  {
881  NamedRegionTimer T("combine_lv", "DAG Combining after legalize vectors",
882  GroupName, GroupDescription, TimePassesIsEnabled);
884  }
885 
886  LLVM_DEBUG(dbgs() << "Optimized vector-legalized selection DAG: "
887  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
888  << "'\n";
889  CurDAG->dump());
890 
891 #ifndef NDEBUG
892  if (TTI.hasBranchDivergence())
894 #endif
895  }
896 
897  if (ViewLegalizeDAGs && MatchFilterBB)
898  CurDAG->viewGraph("legalize input for " + BlockName);
899 
900  {
901  NamedRegionTimer T("legalize", "DAG Legalization", GroupName,
902  GroupDescription, TimePassesIsEnabled);
903  CurDAG->Legalize();
904  }
905 
906  LLVM_DEBUG(dbgs() << "Legalized selection DAG: "
907  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
908  << "'\n";
909  CurDAG->dump());
910 
911 #ifndef NDEBUG
912  if (TTI.hasBranchDivergence())
914 #endif
915 
916  if (ViewDAGCombine2 && MatchFilterBB)
917  CurDAG->viewGraph("dag-combine2 input for " + BlockName);
918 
919  // Run the DAG combiner in post-legalize mode.
920  {
921  NamedRegionTimer T("combine2", "DAG Combining 2", GroupName,
922  GroupDescription, TimePassesIsEnabled);
924  }
925 
926  LLVM_DEBUG(dbgs() << "Optimized legalized selection DAG: "
927  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
928  << "'\n";
929  CurDAG->dump());
930 
931 #ifndef NDEBUG
932  if (TTI.hasBranchDivergence())
934 #endif
935 
936  if (OptLevel != CodeGenOpt::None)
937  ComputeLiveOutVRegInfo();
938 
939  if (ViewISelDAGs && MatchFilterBB)
940  CurDAG->viewGraph("isel input for " + BlockName);
941 
942  // Third, instruction select all of the operations to machine code, adding the
943  // code to the MachineBasicBlock.
944  {
945  NamedRegionTimer T("isel", "Instruction Selection", GroupName,
946  GroupDescription, TimePassesIsEnabled);
947  DoInstructionSelection();
948  }
949 
950  LLVM_DEBUG(dbgs() << "Selected selection DAG: "
951  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
952  << "'\n";
953  CurDAG->dump());
954 
955  if (ViewSchedDAGs && MatchFilterBB)
956  CurDAG->viewGraph("scheduler input for " + BlockName);
957 
958  // Schedule machine code.
959  ScheduleDAGSDNodes *Scheduler = CreateScheduler();
960  {
961  NamedRegionTimer T("sched", "Instruction Scheduling", GroupName,
962  GroupDescription, TimePassesIsEnabled);
963  Scheduler->Run(CurDAG, FuncInfo->MBB);
964  }
965 
966  if (ViewSUnitDAGs && MatchFilterBB)
967  Scheduler->viewGraph();
968 
969  // Emit machine code to BB. This can change 'BB' to the last block being
970  // inserted into.
971  MachineBasicBlock *FirstMBB = FuncInfo->MBB, *LastMBB;
972  {
973  NamedRegionTimer T("emit", "Instruction Creation", GroupName,
974  GroupDescription, TimePassesIsEnabled);
975 
976  // FuncInfo->InsertPt is passed by reference and set to the end of the
977  // scheduled instructions.
978  LastMBB = FuncInfo->MBB = Scheduler->EmitSchedule(FuncInfo->InsertPt);
979  }
980 
981  // If the block was split, make sure we update any references that are used to
982  // update PHI nodes later on.
983  if (FirstMBB != LastMBB)
984  SDB->UpdateSplitBlock(FirstMBB, LastMBB);
985 
986  // Free the scheduler state.
987  {
988  NamedRegionTimer T("cleanup", "Instruction Scheduling Cleanup", GroupName,
989  GroupDescription, TimePassesIsEnabled);
990  delete Scheduler;
991  }
992 
993  // Free the SelectionDAG state, now that we're finished with it.
994  CurDAG->clear();
995 }
996 
997 namespace {
998 
999 /// ISelUpdater - helper class to handle updates of the instruction selection
1000 /// graph.
1001 class ISelUpdater : public SelectionDAG::DAGUpdateListener {
1002  SelectionDAG::allnodes_iterator &ISelPosition;
1003 
1004 public:
1005  ISelUpdater(SelectionDAG &DAG, SelectionDAG::allnodes_iterator &isp)
1006  : SelectionDAG::DAGUpdateListener(DAG), ISelPosition(isp) {}
1007 
1008  /// NodeDeleted - Handle nodes deleted from the graph. If the node being
1009  /// deleted is the current ISelPosition node, update ISelPosition.
1010  ///
1011  void NodeDeleted(SDNode *N, SDNode *E) override {
1012  if (ISelPosition == SelectionDAG::allnodes_iterator(N))
1013  ++ISelPosition;
1014  }
1015 
1016  /// NodeInserted - Handle new nodes inserted into the graph: propagate
1017  /// metadata from root nodes that also applies to new nodes, in case the root
1018  /// is later deleted.
1019  void NodeInserted(SDNode *N) override {
1020  SDNode *CurNode = &*ISelPosition;
1021  if (MDNode *MD = DAG.getPCSections(CurNode))
1022  DAG.addPCSections(N, MD);
1023  }
1024 };
1025 
1026 } // end anonymous namespace
1027 
1028 // This function is used to enforce the topological node id property
1029 // leveraged during instruction selection. Before the selection process all
1030 // nodes are given a non-negative id such that all nodes have a greater id than
1031 // their operands. As this holds transitively we can prune checks that a node N
1032 // is a predecessor of M another by not recursively checking through M's
1033 // operands if N's ID is larger than M's ID. This significantly improves
1034 // performance of various legality checks (e.g. IsLegalToFold / UpdateChains).
1035 
1036 // However, when we fuse multiple nodes into a single node during the
1037 // selection we may induce a predecessor relationship between inputs and
1038 // outputs of distinct nodes being merged, violating the topological property.
1039 // Should a fused node have a successor which has yet to be selected,
1040 // our legality checks would be incorrect. To avoid this we mark all unselected
1041 // successor nodes, i.e. id != -1, as invalid for pruning by bit-negating (x =>
1042 // (-(x+1))) the ids and modify our pruning check to ignore negative Ids of M.
1043 // We use bit-negation to more clearly enforce that node id -1 can only be
1044 // achieved by selected nodes. As the conversion is reversable to the original
1045 // Id, topological pruning can still be leveraged when looking for unselected
1046 // nodes. This method is called internally in all ISel replacement related
1047 // functions.
1050  Nodes.push_back(Node);
1051 
1052  while (!Nodes.empty()) {
1053  SDNode *N = Nodes.pop_back_val();
1054  for (auto *U : N->uses()) {
1055  auto UId = U->getNodeId();
1056  if (UId > 0) {
1057  InvalidateNodeId(U);
1058  Nodes.push_back(U);
1059  }
1060  }
1061  }
1062 }
1063 
1064 // InvalidateNodeId - As explained in EnforceNodeIdInvariant, mark a
1065 // NodeId with the equivalent node id which is invalid for topological
1066 // pruning.
1068  int InvalidId = -(N->getNodeId() + 1);
1069  N->setNodeId(InvalidId);
1070 }
1071 
1072 // getUninvalidatedNodeId - get original uninvalidated node id.
1074  int Id = N->getNodeId();
1075  if (Id < -1)
1076  return -(Id + 1);
1077  return Id;
1078 }
1079 
1080 void SelectionDAGISel::DoInstructionSelection() {
1081  LLVM_DEBUG(dbgs() << "===== Instruction selection begins: "
1082  << printMBBReference(*FuncInfo->MBB) << " '"
1083  << FuncInfo->MBB->getName() << "'\n");
1084 
1086 
1087  // Select target instructions for the DAG.
1088  {
1089  // Number all nodes with a topological order and set DAGSize.
1091 
1092  // Create a dummy node (which is not added to allnodes), that adds
1093  // a reference to the root node, preventing it from being deleted,
1094  // and tracking any changes of the root.
1097  ++ISelPosition;
1098 
1099  // Make sure that ISelPosition gets properly updated when nodes are deleted
1100  // in calls made from this function. New nodes inherit relevant metadata.
1101  ISelUpdater ISU(*CurDAG, ISelPosition);
1102 
1103  // The AllNodes list is now topological-sorted. Visit the
1104  // nodes by starting at the end of the list (the root of the
1105  // graph) and preceding back toward the beginning (the entry
1106  // node).
1107  while (ISelPosition != CurDAG->allnodes_begin()) {
1108  SDNode *Node = &*--ISelPosition;
1109  // Skip dead nodes. DAGCombiner is expected to eliminate all dead nodes,
1110  // but there are currently some corner cases that it misses. Also, this
1111  // makes it theoretically possible to disable the DAGCombiner.
1112  if (Node->use_empty())
1113  continue;
1114 
1115 #ifndef NDEBUG
1117  Nodes.push_back(Node);
1118 
1119  while (!Nodes.empty()) {
1120  auto N = Nodes.pop_back_val();
1121  if (N->getOpcode() == ISD::TokenFactor || N->getNodeId() < 0)
1122  continue;
1123  for (const SDValue &Op : N->op_values()) {
1124  if (Op->getOpcode() == ISD::TokenFactor)
1125  Nodes.push_back(Op.getNode());
1126  else {
1127  // We rely on topological ordering of node ids for checking for
1128  // cycles when fusing nodes during selection. All unselected nodes
1129  // successors of an already selected node should have a negative id.
1130  // This assertion will catch such cases. If this assertion triggers
1131  // it is likely you using DAG-level Value/Node replacement functions
1132  // (versus equivalent ISEL replacement) in backend-specific
1133  // selections. See comment in EnforceNodeIdInvariant for more
1134  // details.
1135  assert(Op->getNodeId() != -1 &&
1136  "Node has already selected predecessor node");
1137  }
1138  }
1139  }
1140 #endif
1141 
1142  // When we are using non-default rounding modes or FP exception behavior
1143  // FP operations are represented by StrictFP pseudo-operations. For
1144  // targets that do not (yet) understand strict FP operations directly,
1145  // we convert them to normal FP opcodes instead at this point. This
1146  // will allow them to be handled by existing target-specific instruction
1147  // selectors.
1148  if (!TLI->isStrictFPEnabled() && Node->isStrictFPOpcode()) {
1149  // For some opcodes, we need to call TLI->getOperationAction using
1150  // the first operand type instead of the result type. Note that this
1151  // must match what SelectionDAGLegalize::LegalizeOp is doing.
1152  EVT ActionVT;
1153  switch (Node->getOpcode()) {
1156  case ISD::STRICT_LRINT:
1157  case ISD::STRICT_LLRINT:
1158  case ISD::STRICT_LROUND:
1159  case ISD::STRICT_LLROUND:
1160  case ISD::STRICT_FSETCC:
1161  case ISD::STRICT_FSETCCS:
1162  ActionVT = Node->getOperand(1).getValueType();
1163  break;
1164  default:
1165  ActionVT = Node->getValueType(0);
1166  break;
1167  }
1168  if (TLI->getOperationAction(Node->getOpcode(), ActionVT)
1170  Node = CurDAG->mutateStrictFPToFP(Node);
1171  }
1172 
1173  LLVM_DEBUG(dbgs() << "\nISEL: Starting selection on root node: ";
1174  Node->dump(CurDAG));
1175 
1176  Select(Node);
1177  }
1178 
1179  CurDAG->setRoot(Dummy.getValue());
1180  }
1181 
1182  LLVM_DEBUG(dbgs() << "\n===== Instruction selection ends:\n");
1183 
1185 }
1186 
1188  for (const User *U : CPI->users()) {
1189  if (const IntrinsicInst *EHPtrCall = dyn_cast<IntrinsicInst>(U)) {
1190  Intrinsic::ID IID = EHPtrCall->getIntrinsicID();
1191  if (IID == Intrinsic::eh_exceptionpointer ||
1192  IID == Intrinsic::eh_exceptioncode)
1193  return true;
1194  }
1195  }
1196  return false;
1197 }
1198 
1199 // wasm.landingpad.index intrinsic is for associating a landing pad index number
1200 // with a catchpad instruction. Retrieve the landing pad index in the intrinsic
1201 // and store the mapping in the function.
1203  const CatchPadInst *CPI) {
1204  MachineFunction *MF = MBB->getParent();
1205  // In case of single catch (...), we don't emit LSDA, so we don't need
1206  // this information.
1207  bool IsSingleCatchAllClause =
1208  CPI->arg_size() == 1 &&
1209  cast<Constant>(CPI->getArgOperand(0))->isNullValue();
1210  // cathchpads for longjmp use an empty type list, e.g. catchpad within %0 []
1211  // and they don't need LSDA info
1212  bool IsCatchLongjmp = CPI->arg_size() == 0;
1213  if (!IsSingleCatchAllClause && !IsCatchLongjmp) {
1214  // Create a mapping from landing pad label to landing pad index.
1215  bool IntrFound = false;
1216  for (const User *U : CPI->users()) {
1217  if (const auto *Call = dyn_cast<IntrinsicInst>(U)) {
1218  Intrinsic::ID IID = Call->getIntrinsicID();
1219  if (IID == Intrinsic::wasm_landingpad_index) {
1220  Value *IndexArg = Call->getArgOperand(1);
1221  int Index = cast<ConstantInt>(IndexArg)->getZExtValue();
1223  IntrFound = true;
1224  break;
1225  }
1226  }
1227  }
1228  assert(IntrFound && "wasm.landingpad.index intrinsic not found!");
1229  (void)IntrFound;
1230  }
1231 }
1232 
1233 /// PrepareEHLandingPad - Emit an EH_LABEL, set up live-in registers, and
1234 /// do other setup for EH landing-pad blocks.
1235 bool SelectionDAGISel::PrepareEHLandingPad() {
1236  MachineBasicBlock *MBB = FuncInfo->MBB;
1237  const Constant *PersonalityFn = FuncInfo->Fn->getPersonalityFn();
1238  const BasicBlock *LLVMBB = MBB->getBasicBlock();
1239  const TargetRegisterClass *PtrRC =
1241 
1242  auto Pers = classifyEHPersonality(PersonalityFn);
1243 
1244  // Catchpads have one live-in register, which typically holds the exception
1245  // pointer or code.
1246  if (isFuncletEHPersonality(Pers)) {
1247  if (const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->getFirstNonPHI())) {
1248  if (hasExceptionPointerOrCodeUser(CPI)) {
1249  // Get or create the virtual register to hold the pointer or code. Mark
1250  // the live in physreg and copy into the vreg.
1251  MCPhysReg EHPhysReg = TLI->getExceptionPointerRegister(PersonalityFn);
1252  assert(EHPhysReg && "target lacks exception pointer register");
1253  MBB->addLiveIn(EHPhysReg);
1254  unsigned VReg = FuncInfo->getCatchPadExceptionPointerVReg(CPI, PtrRC);
1255  BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(),
1256  TII->get(TargetOpcode::COPY), VReg)
1257  .addReg(EHPhysReg, RegState::Kill);
1258  }
1259  }
1260  return true;
1261  }
1262 
1263  // Add a label to mark the beginning of the landing pad. Deletion of the
1264  // landing pad can thus be detected via the MachineModuleInfo.
1266 
1267  const MCInstrDesc &II = TII->get(TargetOpcode::EH_LABEL);
1268  BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(), II)
1269  .addSym(Label);
1270 
1271  // If the unwinder does not preserve all registers, ensure that the
1272  // function marks the clobbered registers as used.
1274  if (auto *RegMask = TRI.getCustomEHPadPreservedMask(*MF))
1276 
1277  if (Pers == EHPersonality::Wasm_CXX) {
1278  if (const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->getFirstNonPHI()))
1280  } else {
1281  // Assign the call site to the landing pad's begin label.
1282  MF->setCallSiteLandingPad(Label, SDB->LPadToCallSiteMap[MBB]);
1283  // Mark exception register as live in.
1284  if (unsigned Reg = TLI->getExceptionPointerRegister(PersonalityFn))
1285  FuncInfo->ExceptionPointerVirtReg = MBB->addLiveIn(Reg, PtrRC);
1286  // Mark exception selector register as live in.
1287  if (unsigned Reg = TLI->getExceptionSelectorRegister(PersonalityFn))
1288  FuncInfo->ExceptionSelectorVirtReg = MBB->addLiveIn(Reg, PtrRC);
1289  }
1290 
1291  return true;
1292 }
1293 
1294 /// isFoldedOrDeadInstruction - Return true if the specified instruction is
1295 /// side-effect free and is either dead or folded into a generated instruction.
1296 /// Return false if it needs to be emitted.
1298  const FunctionLoweringInfo &FuncInfo) {
1299  return !I->mayWriteToMemory() && // Side-effecting instructions aren't folded.
1300  !I->isTerminator() && // Terminators aren't folded.
1301  !isa<DbgInfoIntrinsic>(I) && // Debug instructions aren't folded.
1302  !I->isEHPad() && // EH pad instructions aren't folded.
1303  !FuncInfo.isExportedInst(I); // Exported instrs must be computed.
1304 }
1305 
1307  const Value *Address, DIExpression *Expr,
1308  DILocalVariable *Var, DebugLoc DbgLoc) {
1309  MachineFunction *MF = FuncInfo.MF;
1310  const DataLayout &DL = MF->getDataLayout();
1311 
1312  assert(Var && "Missing variable");
1313  assert(DbgLoc && "Missing location");
1314 
1315  // Look through casts and constant offset GEPs. These mostly come from
1316  // inalloca.
1317  APInt Offset(DL.getTypeSizeInBits(Address->getType()), 0);
1318  Address = Address->stripAndAccumulateInBoundsConstantOffsets(DL, Offset);
1319 
1320  // Check if the variable is a static alloca or a byval or inalloca
1321  // argument passed in memory. If it is not, then we will ignore this
1322  // intrinsic and handle this during isel like dbg.value.
1323  int FI = std::numeric_limits<int>::max();
1324  if (const auto *AI = dyn_cast<AllocaInst>(Address)) {
1325  auto SI = FuncInfo.StaticAllocaMap.find(AI);
1326  if (SI != FuncInfo.StaticAllocaMap.end())
1327  FI = SI->second;
1328  } else if (const auto *Arg = dyn_cast<Argument>(Address))
1329  FI = FuncInfo.getArgumentFrameIndex(Arg);
1330 
1331  if (FI == std::numeric_limits<int>::max())
1332  return;
1333 
1334  if (Offset.getBoolValue())
1336  Offset.getZExtValue());
1337 
1338  LLVM_DEBUG(dbgs() << "processDbgDeclare: setVariableDbgInfo Var=" << *Var
1339  << ", Expr=" << *Expr << ", FI=" << FI
1340  << ", DbgLoc=" << DbgLoc << "\n");
1341  MF->setVariableDbgInfo(Var, Expr, FI, DbgLoc);
1342 }
1343 
1344 /// Collect llvm.dbg.declare information. This is done after argument lowering
1345 /// in case the declarations refer to arguments.
1347  for (const BasicBlock &BB : *FuncInfo.Fn) {
1348  for (const Instruction &I : BB) {
1349  if (const DbgDeclareInst *DI = dyn_cast<DbgDeclareInst>(&I)) {
1350  Value *Address = DI->getAddress();
1351  if (!Address) {
1352  LLVM_DEBUG(dbgs() << "processDbgDeclares skipping " << *DI
1353  << " (bad address)\n");
1354  continue;
1355  }
1356  processDbgDeclare(FuncInfo, Address, DI->getExpression(),
1357  DI->getVariable(), DI->getDebugLoc());
1358  }
1359  }
1360  }
1361 }
1362 
1363 /// Collect single location variable information generated with assignment
1364 /// tracking. This is done after argument lowering in case the declarations
1365 /// refer to arguments.
1367  FunctionVarLocs const *FnVarLocs) {
1368  for (auto It = FnVarLocs->single_locs_begin(),
1369  End = FnVarLocs->single_locs_end();
1370  It != End; ++It)
1371  processDbgDeclare(FuncInfo, It->V, It->Expr,
1372  FnVarLocs->getDILocalVariable(It->VariableID), It->DL);
1373 }
1374 
1375 void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) {
1376  FastISelFailed = false;
1377  // Initialize the Fast-ISel state, if needed.
1378  FastISel *FastIS = nullptr;
1379  if (TM.Options.EnableFastISel) {
1380  LLVM_DEBUG(dbgs() << "Enabling fast-isel\n");
1381  FastIS = TLI->createFastISel(*FuncInfo, LibInfo);
1382  }
1383 
1385 
1386  // Lower arguments up front. An RPO iteration always visits the entry block
1387  // first.
1388  assert(*RPOT.begin() == &Fn.getEntryBlock());
1389  ++NumEntryBlocks;
1390 
1391  // Set up FuncInfo for ISel. Entry blocks never have PHIs.
1392  FuncInfo->MBB = FuncInfo->MBBMap[&Fn.getEntryBlock()];
1393  FuncInfo->InsertPt = FuncInfo->MBB->begin();
1394 
1396 
1397  if (!FastIS) {
1398  LowerArguments(Fn);
1399  } else {
1400  // See if fast isel can lower the arguments.
1401  FastIS->startNewBlock();
1402  if (!FastIS->lowerArguments()) {
1403  FastISelFailed = true;
1404  // Fast isel failed to lower these arguments
1405  ++NumFastIselFailLowerArguments;
1406 
1407  OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1408  Fn.getSubprogram(),
1409  &Fn.getEntryBlock());
1410  R << "FastISel didn't lower all arguments: "
1411  << ore::NV("Prototype", Fn.getType());
1413 
1414  // Use SelectionDAG argument lowering
1415  LowerArguments(Fn);
1416  CurDAG->setRoot(SDB->getControlRoot());
1417  SDB->clear();
1418  CodeGenAndEmitDAG();
1419  }
1420 
1421  // If we inserted any instructions at the beginning, make a note of
1422  // where they are, so we can be sure to emit subsequent instructions
1423  // after them.
1424  if (FuncInfo->InsertPt != FuncInfo->MBB->begin())
1425  FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt));
1426  else
1427  FastIS->setLastLocalValue(nullptr);
1428  }
1429 
1430  bool Inserted = SwiftError->createEntriesInEntryBlock(SDB->getCurDebugLoc());
1431 
1432  if (FastIS && Inserted)
1433  FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt));
1434 
1437  "expected AssignmentTrackingAnalysis pass results");
1439  } else {
1441  }
1442 
1443  // Iterate over all basic blocks in the function.
1444  StackProtector &SP = getAnalysis<StackProtector>();
1445  for (const BasicBlock *LLVMBB : RPOT) {
1446  if (OptLevel != CodeGenOpt::None) {
1447  bool AllPredsVisited = true;
1448  for (const BasicBlock *Pred : predecessors(LLVMBB)) {
1449  if (!FuncInfo->VisitedBBs.count(Pred)) {
1450  AllPredsVisited = false;
1451  break;
1452  }
1453  }
1454 
1455  if (AllPredsVisited) {
1456  for (const PHINode &PN : LLVMBB->phis())
1457  FuncInfo->ComputePHILiveOutRegInfo(&PN);
1458  } else {
1459  for (const PHINode &PN : LLVMBB->phis())
1460  FuncInfo->InvalidatePHILiveOutRegInfo(&PN);
1461  }
1462 
1463  FuncInfo->VisitedBBs.insert(LLVMBB);
1464  }
1465 
1466  BasicBlock::const_iterator const Begin =
1467  LLVMBB->getFirstNonPHI()->getIterator();
1468  BasicBlock::const_iterator const End = LLVMBB->end();
1469  BasicBlock::const_iterator BI = End;
1470 
1471  FuncInfo->MBB = FuncInfo->MBBMap[LLVMBB];
1472  if (!FuncInfo->MBB)
1473  continue; // Some blocks like catchpads have no code or MBB.
1474 
1475  // Insert new instructions after any phi or argument setup code.
1476  FuncInfo->InsertPt = FuncInfo->MBB->end();
1477 
1478  // Setup an EH landing-pad block.
1479  FuncInfo->ExceptionPointerVirtReg = 0;
1480  FuncInfo->ExceptionSelectorVirtReg = 0;
1481  if (LLVMBB->isEHPad())
1482  if (!PrepareEHLandingPad())
1483  continue;
1484 
1485  // Before doing SelectionDAG ISel, see if FastISel has been requested.
1486  if (FastIS) {
1487  if (LLVMBB != &Fn.getEntryBlock())
1488  FastIS->startNewBlock();
1489 
1490  unsigned NumFastIselRemaining = std::distance(Begin, End);
1491 
1492  // Pre-assign swifterror vregs.
1493  SwiftError->preassignVRegs(FuncInfo->MBB, Begin, End);
1494 
1495  // Do FastISel on as many instructions as possible.
1496  for (; BI != Begin; --BI) {
1497  const Instruction *Inst = &*std::prev(BI);
1498 
1499  // If we no longer require this instruction, skip it.
1500  if (isFoldedOrDeadInstruction(Inst, *FuncInfo) ||
1501  ElidedArgCopyInstrs.count(Inst)) {
1502  --NumFastIselRemaining;
1503  continue;
1504  }
1505 
1506  // Bottom-up: reset the insert pos at the top, after any local-value
1507  // instructions.
1508  FastIS->recomputeInsertPt();
1509 
1510  // Try to select the instruction with FastISel.
1511  if (FastIS->selectInstruction(Inst)) {
1512  --NumFastIselRemaining;
1513  ++NumFastIselSuccess;
1514  // If fast isel succeeded, skip over all the folded instructions, and
1515  // then see if there is a load right before the selected instructions.
1516  // Try to fold the load if so.
1517  const Instruction *BeforeInst = Inst;
1518  while (BeforeInst != &*Begin) {
1519  BeforeInst = &*std::prev(BasicBlock::const_iterator(BeforeInst));
1520  if (!isFoldedOrDeadInstruction(BeforeInst, *FuncInfo))
1521  break;
1522  }
1523  if (BeforeInst != Inst && isa<LoadInst>(BeforeInst) &&
1524  BeforeInst->hasOneUse() &&
1525  FastIS->tryToFoldLoad(cast<LoadInst>(BeforeInst), Inst)) {
1526  // If we succeeded, don't re-select the load.
1527  LLVM_DEBUG(dbgs()
1528  << "FastISel folded load: " << *BeforeInst << "\n");
1529  BI = std::next(BasicBlock::const_iterator(BeforeInst));
1530  --NumFastIselRemaining;
1531  ++NumFastIselSuccess;
1532  }
1533  continue;
1534  }
1535 
1536  FastISelFailed = true;
1537 
1538  // Then handle certain instructions as single-LLVM-Instruction blocks.
1539  // We cannot separate out GCrelocates to their own blocks since we need
1540  // to keep track of gc-relocates for a particular gc-statepoint. This is
1541  // done by SelectionDAGBuilder::LowerAsSTATEPOINT, called before
1542  // visitGCRelocate.
1543  if (isa<CallInst>(Inst) && !isa<GCStatepointInst>(Inst) &&
1544  !isa<GCRelocateInst>(Inst) && !isa<GCResultInst>(Inst)) {
1545  OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1546  Inst->getDebugLoc(), LLVMBB);
1547 
1548  R << "FastISel missed call";
1549 
1550  if (R.isEnabled() || EnableFastISelAbort) {
1551  std::string InstStrStorage;
1552  raw_string_ostream InstStr(InstStrStorage);
1553  InstStr << *Inst;
1554 
1555  R << ": " << InstStr.str();
1556  }
1557 
1559 
1560  if (!Inst->getType()->isVoidTy() && !Inst->getType()->isTokenTy() &&
1561  !Inst->use_empty()) {
1562  Register &R = FuncInfo->ValueMap[Inst];
1563  if (!R)
1564  R = FuncInfo->CreateRegs(Inst);
1565  }
1566 
1567  bool HadTailCall = false;
1568  MachineBasicBlock::iterator SavedInsertPt = FuncInfo->InsertPt;
1569  SelectBasicBlock(Inst->getIterator(), BI, HadTailCall);
1570 
1571  // If the call was emitted as a tail call, we're done with the block.
1572  // We also need to delete any previously emitted instructions.
1573  if (HadTailCall) {
1574  FastIS->removeDeadCode(SavedInsertPt, FuncInfo->MBB->end());
1575  --BI;
1576  break;
1577  }
1578 
1579  // Recompute NumFastIselRemaining as Selection DAG instruction
1580  // selection may have handled the call, input args, etc.
1581  unsigned RemainingNow = std::distance(Begin, BI);
1582  NumFastIselFailures += NumFastIselRemaining - RemainingNow;
1583  NumFastIselRemaining = RemainingNow;
1584  continue;
1585  }
1586 
1587  OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1588  Inst->getDebugLoc(), LLVMBB);
1589 
1590  bool ShouldAbort = EnableFastISelAbort;
1591  if (Inst->isTerminator()) {
1592  // Use a different message for terminator misses.
1593  R << "FastISel missed terminator";
1594  // Don't abort for terminator unless the level is really high
1595  ShouldAbort = (EnableFastISelAbort > 2);
1596  } else {
1597  R << "FastISel missed";
1598  }
1599 
1600  if (R.isEnabled() || EnableFastISelAbort) {
1601  std::string InstStrStorage;
1602  raw_string_ostream InstStr(InstStrStorage);
1603  InstStr << *Inst;
1604  R << ": " << InstStr.str();
1605  }
1606 
1607  reportFastISelFailure(*MF, *ORE, R, ShouldAbort);
1608 
1609  NumFastIselFailures += NumFastIselRemaining;
1610  break;
1611  }
1612 
1613  FastIS->recomputeInsertPt();
1614  }
1615 
1616  if (SP.shouldEmitSDCheck(*LLVMBB)) {
1617  bool FunctionBasedInstrumentation =
1619  SDB->SPDescriptor.initialize(LLVMBB, FuncInfo->MBBMap[LLVMBB],
1620  FunctionBasedInstrumentation);
1621  }
1622 
1623  if (Begin != BI)
1624  ++NumDAGBlocks;
1625  else
1626  ++NumFastIselBlocks;
1627 
1628  if (Begin != BI) {
1629  // Run SelectionDAG instruction selection on the remainder of the block
1630  // not handled by FastISel. If FastISel is not run, this is the entire
1631  // block.
1632  bool HadTailCall;
1633  SelectBasicBlock(Begin, BI, HadTailCall);
1634 
1635  // But if FastISel was run, we already selected some of the block.
1636  // If we emitted a tail-call, we need to delete any previously emitted
1637  // instruction that follows it.
1638  if (FastIS && HadTailCall && FuncInfo->InsertPt != FuncInfo->MBB->end())
1639  FastIS->removeDeadCode(FuncInfo->InsertPt, FuncInfo->MBB->end());
1640  }
1641 
1642  if (FastIS)
1643  FastIS->finishBasicBlock();
1644  FinishBasicBlock();
1645  FuncInfo->PHINodesToUpdate.clear();
1646  ElidedArgCopyInstrs.clear();
1647  }
1648 
1650 
1652 
1653  delete FastIS;
1654  SDB->clearDanglingDebugInfo();
1655  SDB->SPDescriptor.resetPerFunctionState();
1656 }
1657 
1658 void
1659 SelectionDAGISel::FinishBasicBlock() {
1660  LLVM_DEBUG(dbgs() << "Total amount of phi nodes to update: "
1661  << FuncInfo->PHINodesToUpdate.size() << "\n";
1662  for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e;
1663  ++i) dbgs()
1664  << "Node " << i << " : (" << FuncInfo->PHINodesToUpdate[i].first
1665  << ", " << FuncInfo->PHINodesToUpdate[i].second << ")\n");
1666 
1667  // Next, now that we know what the last MBB the LLVM BB expanded is, update
1668  // PHI nodes in successors.
1669  for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) {
1670  MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[i].first);
1671  assert(PHI->isPHI() &&
1672  "This is not a machine PHI node that we are updating!");
1673  if (!FuncInfo->MBB->isSuccessor(PHI->getParent()))
1674  continue;
1675  PHI.addReg(FuncInfo->PHINodesToUpdate[i].second).addMBB(FuncInfo->MBB);
1676  }
1677 
1678  // Handle stack protector.
1679  if (SDB->SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
1680  // The target provides a guard check function. There is no need to
1681  // generate error handling code or to split current basic block.
1682  MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB();
1683 
1684  // Add load and check to the basicblock.
1685  FuncInfo->MBB = ParentMBB;
1686  FuncInfo->InsertPt =
1687  findSplitPointForStackProtector(ParentMBB, *TII);
1688  SDB->visitSPDescriptorParent(SDB->SPDescriptor, ParentMBB);
1689  CurDAG->setRoot(SDB->getRoot());
1690  SDB->clear();
1691  CodeGenAndEmitDAG();
1692 
1693  // Clear the Per-BB State.
1694  SDB->SPDescriptor.resetPerBBState();
1695  } else if (SDB->SPDescriptor.shouldEmitStackProtector()) {
1696  MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB();
1697  MachineBasicBlock *SuccessMBB = SDB->SPDescriptor.getSuccessMBB();
1698 
1699  // Find the split point to split the parent mbb. At the same time copy all
1700  // physical registers used in the tail of parent mbb into virtual registers
1701  // before the split point and back into physical registers after the split
1702  // point. This prevents us needing to deal with Live-ins and many other
1703  // register allocation issues caused by us splitting the parent mbb. The
1704  // register allocator will clean up said virtual copies later on.
1705  MachineBasicBlock::iterator SplitPoint =
1706  findSplitPointForStackProtector(ParentMBB, *TII);
1707 
1708  // Splice the terminator of ParentMBB into SuccessMBB.
1709  SuccessMBB->splice(SuccessMBB->end(), ParentMBB,
1710  SplitPoint,
1711  ParentMBB->end());
1712 
1713  // Add compare/jump on neq/jump to the parent BB.
1714  FuncInfo->MBB = ParentMBB;
1715  FuncInfo->InsertPt = ParentMBB->end();
1716  SDB->visitSPDescriptorParent(SDB->SPDescriptor, ParentMBB);
1717  CurDAG->setRoot(SDB->getRoot());
1718  SDB->clear();
1719  CodeGenAndEmitDAG();
1720 
1721  // CodeGen Failure MBB if we have not codegened it yet.
1722  MachineBasicBlock *FailureMBB = SDB->SPDescriptor.getFailureMBB();
1723  if (FailureMBB->empty()) {
1724  FuncInfo->MBB = FailureMBB;
1725  FuncInfo->InsertPt = FailureMBB->end();
1726  SDB->visitSPDescriptorFailure(SDB->SPDescriptor);
1727  CurDAG->setRoot(SDB->getRoot());
1728  SDB->clear();
1729  CodeGenAndEmitDAG();
1730  }
1731 
1732  // Clear the Per-BB State.
1733  SDB->SPDescriptor.resetPerBBState();
1734  }
1735 
1736  // Lower each BitTestBlock.
1737  for (auto &BTB : SDB->SL->BitTestCases) {
1738  // Lower header first, if it wasn't already lowered
1739  if (!BTB.Emitted) {
1740  // Set the current basic block to the mbb we wish to insert the code into
1741  FuncInfo->MBB = BTB.Parent;
1742  FuncInfo->InsertPt = FuncInfo->MBB->end();
1743  // Emit the code
1744  SDB->visitBitTestHeader(BTB, FuncInfo->MBB);
1745  CurDAG->setRoot(SDB->getRoot());
1746  SDB->clear();
1747  CodeGenAndEmitDAG();
1748  }
1749 
1750  BranchProbability UnhandledProb = BTB.Prob;
1751  for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
1752  UnhandledProb -= BTB.Cases[j].ExtraProb;
1753  // Set the current basic block to the mbb we wish to insert the code into
1754  FuncInfo->MBB = BTB.Cases[j].ThisBB;
1755  FuncInfo->InsertPt = FuncInfo->MBB->end();
1756  // Emit the code
1757 
1758  // If all cases cover a contiguous range, it is not necessary to jump to
1759  // the default block after the last bit test fails. This is because the
1760  // range check during bit test header creation has guaranteed that every
1761  // case here doesn't go outside the range. In this case, there is no need
1762  // to perform the last bit test, as it will always be true. Instead, make
1763  // the second-to-last bit-test fall through to the target of the last bit
1764  // test, and delete the last bit test.
1765 
1766  MachineBasicBlock *NextMBB;
1767  if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
1768  // Second-to-last bit-test with contiguous range or omitted range
1769  // check: fall through to the target of the final bit test.
1770  NextMBB = BTB.Cases[j + 1].TargetBB;
1771  } else if (j + 1 == ej) {
1772  // For the last bit test, fall through to Default.
1773  NextMBB = BTB.Default;
1774  } else {
1775  // Otherwise, fall through to the next bit test.
1776  NextMBB = BTB.Cases[j + 1].ThisBB;
1777  }
1778 
1779  SDB->visitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j],
1780  FuncInfo->MBB);
1781 
1782  CurDAG->setRoot(SDB->getRoot());
1783  SDB->clear();
1784  CodeGenAndEmitDAG();
1785 
1786  if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
1787  // Since we're not going to use the final bit test, remove it.
1788  BTB.Cases.pop_back();
1789  break;
1790  }
1791  }
1792 
1793  // Update PHI Nodes
1794  for (const std::pair<MachineInstr *, unsigned> &P :
1795  FuncInfo->PHINodesToUpdate) {
1796  MachineInstrBuilder PHI(*MF, P.first);
1797  MachineBasicBlock *PHIBB = PHI->getParent();
1798  assert(PHI->isPHI() &&
1799  "This is not a machine PHI node that we are updating!");
1800  // This is "default" BB. We have two jumps to it. From "header" BB and
1801  // from last "case" BB, unless the latter was skipped.
1802  if (PHIBB == BTB.Default) {
1803  PHI.addReg(P.second).addMBB(BTB.Parent);
1804  if (!BTB.ContiguousRange) {
1805  PHI.addReg(P.second).addMBB(BTB.Cases.back().ThisBB);
1806  }
1807  }
1808  // One of "cases" BB.
1809  for (const SwitchCG::BitTestCase &BT : BTB.Cases) {
1810  MachineBasicBlock* cBB = BT.ThisBB;
1811  if (cBB->isSuccessor(PHIBB))
1812  PHI.addReg(P.second).addMBB(cBB);
1813  }
1814  }
1815  }
1816  SDB->SL->BitTestCases.clear();
1817 
1818  // If the JumpTable record is filled in, then we need to emit a jump table.
1819  // Updating the PHI nodes is tricky in this case, since we need to determine
1820  // whether the PHI is a successor of the range check MBB or the jump table MBB
1821  for (unsigned i = 0, e = SDB->SL->JTCases.size(); i != e; ++i) {
1822  // Lower header first, if it wasn't already lowered
1823  if (!SDB->SL->JTCases[i].first.Emitted) {
1824  // Set the current basic block to the mbb we wish to insert the code into
1825  FuncInfo->MBB = SDB->SL->JTCases[i].first.HeaderBB;
1826  FuncInfo->InsertPt = FuncInfo->MBB->end();
1827  // Emit the code
1828  SDB->visitJumpTableHeader(SDB->SL->JTCases[i].second,
1829  SDB->SL->JTCases[i].first, FuncInfo->MBB);
1830  CurDAG->setRoot(SDB->getRoot());
1831  SDB->clear();
1832  CodeGenAndEmitDAG();
1833  }
1834 
1835  // Set the current basic block to the mbb we wish to insert the code into
1836  FuncInfo->MBB = SDB->SL->JTCases[i].second.MBB;
1837  FuncInfo->InsertPt = FuncInfo->MBB->end();
1838  // Emit the code
1839  SDB->visitJumpTable(SDB->SL->JTCases[i].second);
1840  CurDAG->setRoot(SDB->getRoot());
1841  SDB->clear();
1842  CodeGenAndEmitDAG();
1843 
1844  // Update PHI Nodes
1845  for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size();
1846  pi != pe; ++pi) {
1847  MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first);
1848  MachineBasicBlock *PHIBB = PHI->getParent();
1849  assert(PHI->isPHI() &&
1850  "This is not a machine PHI node that we are updating!");
1851  // "default" BB. We can go there only from header BB.
1852  if (PHIBB == SDB->SL->JTCases[i].second.Default)
1853  PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second)
1854  .addMBB(SDB->SL->JTCases[i].first.HeaderBB);
1855  // JT BB. Just iterate over successors here
1856  if (FuncInfo->MBB->isSuccessor(PHIBB))
1857  PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(FuncInfo->MBB);
1858  }
1859  }
1860  SDB->SL->JTCases.clear();
1861 
1862  // If we generated any switch lowering information, build and codegen any
1863  // additional DAGs necessary.
1864  for (unsigned i = 0, e = SDB->SL->SwitchCases.size(); i != e; ++i) {
1865  // Set the current basic block to the mbb we wish to insert the code into
1866  FuncInfo->MBB = SDB->SL->SwitchCases[i].ThisBB;
1867  FuncInfo->InsertPt = FuncInfo->MBB->end();
1868 
1869  // Determine the unique successors.
1871  Succs.push_back(SDB->SL->SwitchCases[i].TrueBB);
1872  if (SDB->SL->SwitchCases[i].TrueBB != SDB->SL->SwitchCases[i].FalseBB)
1873  Succs.push_back(SDB->SL->SwitchCases[i].FalseBB);
1874 
1875  // Emit the code. Note that this could result in FuncInfo->MBB being split.
1876  SDB->visitSwitchCase(SDB->SL->SwitchCases[i], FuncInfo->MBB);
1877  CurDAG->setRoot(SDB->getRoot());
1878  SDB->clear();
1879  CodeGenAndEmitDAG();
1880 
1881  // Remember the last block, now that any splitting is done, for use in
1882  // populating PHI nodes in successors.
1883  MachineBasicBlock *ThisBB = FuncInfo->MBB;
1884 
1885  // Handle any PHI nodes in successors of this chunk, as if we were coming
1886  // from the original BB before switch expansion. Note that PHI nodes can
1887  // occur multiple times in PHINodesToUpdate. We have to be very careful to
1888  // handle them the right number of times.
1889  for (unsigned i = 0, e = Succs.size(); i != e; ++i) {
1890  FuncInfo->MBB = Succs[i];
1891  FuncInfo->InsertPt = FuncInfo->MBB->end();
1892  // FuncInfo->MBB may have been removed from the CFG if a branch was
1893  // constant folded.
1894  if (ThisBB->isSuccessor(FuncInfo->MBB)) {
1896  MBBI = FuncInfo->MBB->begin(), MBBE = FuncInfo->MBB->end();
1897  MBBI != MBBE && MBBI->isPHI(); ++MBBI) {
1899  // This value for this PHI node is recorded in PHINodesToUpdate.
1900  for (unsigned pn = 0; ; ++pn) {
1901  assert(pn != FuncInfo->PHINodesToUpdate.size() &&
1902  "Didn't find PHI entry!");
1903  if (FuncInfo->PHINodesToUpdate[pn].first == PHI) {
1904  PHI.addReg(FuncInfo->PHINodesToUpdate[pn].second).addMBB(ThisBB);
1905  break;
1906  }
1907  }
1908  }
1909  }
1910  }
1911  }
1912  SDB->SL->SwitchCases.clear();
1913 }
1914 
1915 /// Create the scheduler. If a specific scheduler was specified
1916 /// via the SchedulerRegistry, use it, otherwise select the
1917 /// one preferred by the target.
1918 ///
1919 ScheduleDAGSDNodes *SelectionDAGISel::CreateScheduler() {
1920  return ISHeuristic(this, OptLevel);
1921 }
1922 
1923 //===----------------------------------------------------------------------===//
1924 // Helper functions used by the generated instruction selector.
1925 //===----------------------------------------------------------------------===//
1926 // Calls to these methods are generated by tblgen.
1927 
1928 /// CheckAndMask - The isel is trying to match something like (and X, 255). If
1929 /// the dag combiner simplified the 255, we still want to match. RHS is the
1930 /// actual value in the DAG on the RHS of an AND, and DesiredMaskS is the value
1931 /// specified in the .td file (e.g. 255).
1933  int64_t DesiredMaskS) const {
1934  const APInt &ActualMask = RHS->getAPIntValue();
1935  const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS);
1936 
1937  // If the actual mask exactly matches, success!
1938  if (ActualMask == DesiredMask)
1939  return true;
1940 
1941  // If the actual AND mask is allowing unallowed bits, this doesn't match.
1942  if (!ActualMask.isSubsetOf(DesiredMask))
1943  return false;
1944 
1945  // Otherwise, the DAG Combiner may have proven that the value coming in is
1946  // either already zero or is not demanded. Check for known zero input bits.
1947  APInt NeededMask = DesiredMask & ~ActualMask;
1948  if (CurDAG->MaskedValueIsZero(LHS, NeededMask))
1949  return true;
1950 
1951  // TODO: check to see if missing bits are just not demanded.
1952 
1953  // Otherwise, this pattern doesn't match.
1954  return false;
1955 }
1956 
1957 /// CheckOrMask - The isel is trying to match something like (or X, 255). If
1958 /// the dag combiner simplified the 255, we still want to match. RHS is the
1959 /// actual value in the DAG on the RHS of an OR, and DesiredMaskS is the value
1960 /// specified in the .td file (e.g. 255).
1962  int64_t DesiredMaskS) const {
1963  const APInt &ActualMask = RHS->getAPIntValue();
1964  const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS);
1965 
1966  // If the actual mask exactly matches, success!
1967  if (ActualMask == DesiredMask)
1968  return true;
1969 
1970  // If the actual AND mask is allowing unallowed bits, this doesn't match.
1971  if (!ActualMask.isSubsetOf(DesiredMask))
1972  return false;
1973 
1974  // Otherwise, the DAG Combiner may have proven that the value coming in is
1975  // either already zero or is not demanded. Check for known zero input bits.
1976  APInt NeededMask = DesiredMask & ~ActualMask;
1978 
1979  // If all the missing bits in the or are already known to be set, match!
1980  if (NeededMask.isSubsetOf(Known.One))
1981  return true;
1982 
1983  // TODO: check to see if missing bits are just not demanded.
1984 
1985  // Otherwise, this pattern doesn't match.
1986  return false;
1987 }
1988 
1989 /// SelectInlineAsmMemoryOperands - Calls to this are automatically generated
1990 /// by tblgen. Others should not call it.
1992  const SDLoc &DL) {
1993  std::vector<SDValue> InOps;
1994  std::swap(InOps, Ops);
1995 
1996  Ops.push_back(InOps[InlineAsm::Op_InputChain]); // 0
1997  Ops.push_back(InOps[InlineAsm::Op_AsmString]); // 1
1998  Ops.push_back(InOps[InlineAsm::Op_MDNode]); // 2, !srcloc
1999  Ops.push_back(InOps[InlineAsm::Op_ExtraInfo]); // 3 (SideEffect, AlignStack)
2000 
2001  unsigned i = InlineAsm::Op_FirstOperand, e = InOps.size();
2002  if (InOps[e-1].getValueType() == MVT::Glue)
2003  --e; // Don't process a glue operand if it is here.
2004 
2005  while (i != e) {
2006  unsigned Flags = cast<ConstantSDNode>(InOps[i])->getZExtValue();
2007  if (!InlineAsm::isMemKind(Flags) && !InlineAsm::isFuncKind(Flags)) {
2008  // Just skip over this operand, copying the operands verbatim.
2009  Ops.insert(Ops.end(), InOps.begin()+i,
2010  InOps.begin()+i+InlineAsm::getNumOperandRegisters(Flags) + 1);
2011  i += InlineAsm::getNumOperandRegisters(Flags) + 1;
2012  } else {
2014  "Memory operand with multiple values?");
2015 
2016  unsigned TiedToOperand;
2017  if (InlineAsm::isUseOperandTiedToDef(Flags, TiedToOperand)) {
2018  // We need the constraint ID from the operand this is tied to.
2019  unsigned CurOp = InlineAsm::Op_FirstOperand;
2020  Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
2021  for (; TiedToOperand; --TiedToOperand) {
2022  CurOp += InlineAsm::getNumOperandRegisters(Flags)+1;
2023  Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
2024  }
2025  }
2026 
2027  // Otherwise, this is a memory operand. Ask the target to select it.
2028  std::vector<SDValue> SelOps;
2029  unsigned ConstraintID = InlineAsm::getMemoryConstraintID(Flags);
2030  if (SelectInlineAsmMemoryOperand(InOps[i+1], ConstraintID, SelOps))
2031  report_fatal_error("Could not match memory address. Inline asm"
2032  " failure!");
2033 
2034  // Add this to the output node.
2035  unsigned NewFlags =
2036  InlineAsm::isMemKind(Flags)
2038  : InlineAsm::getFlagWord(InlineAsm::Kind_Func, SelOps.size());
2039  NewFlags = InlineAsm::getFlagWordForMem(NewFlags, ConstraintID);
2040  Ops.push_back(CurDAG->getTargetConstant(NewFlags, DL, MVT::i32));
2041  llvm::append_range(Ops, SelOps);
2042  i += 2;
2043  }
2044  }
2045 
2046  // Add the glue input back if present.
2047  if (e != InOps.size())
2048  Ops.push_back(InOps.back());
2049 }
2050 
2051 /// findGlueUse - Return use of MVT::Glue value produced by the specified
2052 /// SDNode.
2053 ///
2055  unsigned FlagResNo = N->getNumValues()-1;
2056  for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
2057  SDUse &Use = I.getUse();
2058  if (Use.getResNo() == FlagResNo)
2059  return Use.getUser();
2060  }
2061  return nullptr;
2062 }
2063 
2064 /// findNonImmUse - Return true if "Def" is a predecessor of "Root" via a path
2065 /// beyond "ImmedUse". We may ignore chains as they are checked separately.
2066 static bool findNonImmUse(SDNode *Root, SDNode *Def, SDNode *ImmedUse,
2067  bool IgnoreChains) {
2070  // Only check if we have non-immediate uses of Def.
2071  if (ImmedUse->isOnlyUserOf(Def))
2072  return false;
2073 
2074  // We don't care about paths to Def that go through ImmedUse so mark it
2075  // visited and mark non-def operands as used.
2076  Visited.insert(ImmedUse);
2077  for (const SDValue &Op : ImmedUse->op_values()) {
2078  SDNode *N = Op.getNode();
2079  // Ignore chain deps (they are validated by
2080  // HandleMergeInputChains) and immediate uses
2081  if ((Op.getValueType() == MVT::Other && IgnoreChains) || N == Def)
2082  continue;
2083  if (!Visited.insert(N).second)
2084  continue;
2085  WorkList.push_back(N);
2086  }
2087 
2088  // Initialize worklist to operands of Root.
2089  if (Root != ImmedUse) {
2090  for (const SDValue &Op : Root->op_values()) {
2091  SDNode *N = Op.getNode();
2092  // Ignore chains (they are validated by HandleMergeInputChains)
2093  if ((Op.getValueType() == MVT::Other && IgnoreChains) || N == Def)
2094  continue;
2095  if (!Visited.insert(N).second)
2096  continue;
2097  WorkList.push_back(N);
2098  }
2099  }
2100 
2101  return SDNode::hasPredecessorHelper(Def, Visited, WorkList, 0, true);
2102 }
2103 
2104 /// IsProfitableToFold - Returns true if it's profitable to fold the specific
2105 /// operand node N of U during instruction selection that starts at Root.
2107  SDNode *Root) const {
2108  if (OptLevel == CodeGenOpt::None) return false;
2109  return N.hasOneUse();
2110 }
2111 
2112 /// IsLegalToFold - Returns true if the specific operand node N of
2113 /// U can be folded during instruction selection that starts at Root.
2115  CodeGenOpt::Level OptLevel,
2116  bool IgnoreChains) {
2117  if (OptLevel == CodeGenOpt::None) return false;
2118 
2119  // If Root use can somehow reach N through a path that that doesn't contain
2120  // U then folding N would create a cycle. e.g. In the following
2121  // diagram, Root can reach N through X. If N is folded into Root, then
2122  // X is both a predecessor and a successor of U.
2123  //
2124  // [N*] //
2125  // ^ ^ //
2126  // / \ //
2127  // [U*] [X]? //
2128  // ^ ^ //
2129  // \ / //
2130  // \ / //
2131  // [Root*] //
2132  //
2133  // * indicates nodes to be folded together.
2134  //
2135  // If Root produces glue, then it gets (even more) interesting. Since it
2136  // will be "glued" together with its glue use in the scheduler, we need to
2137  // check if it might reach N.
2138  //
2139  // [N*] //
2140  // ^ ^ //
2141  // / \ //
2142  // [U*] [X]? //
2143  // ^ ^ //
2144  // \ \ //
2145  // \ | //
2146  // [Root*] | //
2147  // ^ | //
2148  // f | //
2149  // | / //
2150  // [Y] / //
2151  // ^ / //
2152  // f / //
2153  // | / //
2154  // [GU] //
2155  //
2156  // If GU (glue use) indirectly reaches N (the load), and Root folds N
2157  // (call it Fold), then X is a predecessor of GU and a successor of
2158  // Fold. But since Fold and GU are glued together, this will create
2159  // a cycle in the scheduling graph.
2160 
2161  // If the node has glue, walk down the graph to the "lowest" node in the
2162  // glueged set.
2163  EVT VT = Root->getValueType(Root->getNumValues()-1);
2164  while (VT == MVT::Glue) {
2165  SDNode *GU = findGlueUse(Root);
2166  if (!GU)
2167  break;
2168  Root = GU;
2169  VT = Root->getValueType(Root->getNumValues()-1);
2170 
2171  // If our query node has a glue result with a use, we've walked up it. If
2172  // the user (which has already been selected) has a chain or indirectly uses
2173  // the chain, HandleMergeInputChains will not consider it. Because of
2174  // this, we cannot ignore chains in this predicate.
2175  IgnoreChains = false;
2176  }
2177 
2178  return !findNonImmUse(Root, N.getNode(), U, IgnoreChains);
2179 }
2180 
2181 void SelectionDAGISel::Select_INLINEASM(SDNode *N) {
2182  SDLoc DL(N);
2183 
2184  std::vector<SDValue> Ops(N->op_begin(), N->op_end());
2186 
2187  const EVT VTs[] = {MVT::Other, MVT::Glue};
2188  SDValue New = CurDAG->getNode(N->getOpcode(), DL, VTs, Ops);
2189  New->setNodeId(-1);
2190  ReplaceUses(N, New.getNode());
2192 }
2193 
2194 void SelectionDAGISel::Select_READ_REGISTER(SDNode *Op) {
2195  SDLoc dl(Op);
2196  MDNodeSDNode *MD = cast<MDNodeSDNode>(Op->getOperand(1));
2197  const MDString *RegStr = cast<MDString>(MD->getMD()->getOperand(0));
2198 
2199  EVT VT = Op->getValueType(0);
2200  LLT Ty = VT.isSimple() ? getLLTForMVT(VT.getSimpleVT()) : LLT();
2201  Register Reg =
2202  TLI->getRegisterByName(RegStr->getString().data(), Ty,
2205  Op->getOperand(0), dl, Reg, Op->getValueType(0));
2206  New->setNodeId(-1);
2207  ReplaceUses(Op, New.getNode());
2209 }
2210 
2211 void SelectionDAGISel::Select_WRITE_REGISTER(SDNode *Op) {
2212  SDLoc dl(Op);
2213  MDNodeSDNode *MD = cast<MDNodeSDNode>(Op->getOperand(1));
2214  const MDString *RegStr = cast<MDString>(MD->getMD()->getOperand(0));
2215 
2216  EVT VT = Op->getOperand(2).getValueType();
2217  LLT Ty = VT.isSimple() ? getLLTForMVT(VT.getSimpleVT()) : LLT();
2218 
2219  Register Reg = TLI->getRegisterByName(RegStr->getString().data(), Ty,
2222  Op->getOperand(0), dl, Reg, Op->getOperand(2));
2223  New->setNodeId(-1);
2224  ReplaceUses(Op, New.getNode());
2226 }
2227 
2228 void SelectionDAGISel::Select_UNDEF(SDNode *N) {
2229  CurDAG->SelectNodeTo(N, TargetOpcode::IMPLICIT_DEF, N->getValueType(0));
2230 }
2231 
2232 void SelectionDAGISel::Select_FREEZE(SDNode *N) {
2233  // TODO: We don't have FREEZE pseudo-instruction in MachineInstr-level now.
2234  // If FREEZE instruction is added later, the code below must be changed as
2235  // well.
2236  CurDAG->SelectNodeTo(N, TargetOpcode::COPY, N->getValueType(0),
2237  N->getOperand(0));
2238 }
2239 
2240 void SelectionDAGISel::Select_ARITH_FENCE(SDNode *N) {
2241  CurDAG->SelectNodeTo(N, TargetOpcode::ARITH_FENCE, N->getValueType(0),
2242  N->getOperand(0));
2243 }
2244 
2245 void SelectionDAGISel::Select_MEMBARRIER(SDNode *N) {
2246  CurDAG->SelectNodeTo(N, TargetOpcode::MEMBARRIER, N->getValueType(0),
2247  N->getOperand(0));
2248 }
2249 
2250 void SelectionDAGISel::pushStackMapLiveVariable(SmallVectorImpl<SDValue> &Ops,
2251  SDValue OpVal, SDLoc DL) {
2252  SDNode *OpNode = OpVal.getNode();
2253 
2254  // FrameIndex nodes should have been directly emitted to TargetFrameIndex
2255  // nodes at DAG-construction time.
2256  assert(OpNode->getOpcode() != ISD::FrameIndex);
2257 
2258  if (OpNode->getOpcode() == ISD::Constant) {
2259  Ops.push_back(
2260  CurDAG->getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
2261  Ops.push_back(
2262  CurDAG->getTargetConstant(cast<ConstantSDNode>(OpNode)->getZExtValue(),
2263  DL, OpVal.getValueType()));
2264  } else {
2265  Ops.push_back(OpVal);
2266  }
2267 }
2268 
2269 void SelectionDAGISel::Select_STACKMAP(SDNode *N) {
2271  auto *It = N->op_begin();
2272  SDLoc DL(N);
2273 
2274  // Stash the chain and glue operands so we can move them to the end.
2275  SDValue Chain = *It++;
2276  SDValue InFlag = *It++;
2277 
2278  // <id> operand.
2279  SDValue ID = *It++;
2280  assert(ID.getValueType() == MVT::i64);
2281  Ops.push_back(ID);
2282 
2283  // <numShadowBytes> operand.
2284  SDValue Shad = *It++;
2285  assert(Shad.getValueType() == MVT::i32);
2286  Ops.push_back(Shad);
2287 
2288  // Live variable operands.
2289  for (; It != N->op_end(); It++)
2290  pushStackMapLiveVariable(Ops, *It, DL);
2291 
2292  Ops.push_back(Chain);
2293  Ops.push_back(InFlag);
2294 
2296  CurDAG->SelectNodeTo(N, TargetOpcode::STACKMAP, NodeTys, Ops);
2297 }
2298 
2299 void SelectionDAGISel::Select_PATCHPOINT(SDNode *N) {
2301  auto *It = N->op_begin();
2302  SDLoc DL(N);
2303 
2304  // Cache arguments that will be moved to the end in the target node.
2305  SDValue Chain = *It++;
2306  std::optional<SDValue> Glue;
2307  if (It->getValueType() == MVT::Glue)
2308  Glue = *It++;
2309  SDValue RegMask = *It++;
2310 
2311  // <id> operand.
2312  SDValue ID = *It++;
2313  assert(ID.getValueType() == MVT::i64);
2314  Ops.push_back(ID);
2315 
2316  // <numShadowBytes> operand.
2317  SDValue Shad = *It++;
2318  assert(Shad.getValueType() == MVT::i32);
2319  Ops.push_back(Shad);
2320 
2321  // Add the callee.
2322  Ops.push_back(*It++);
2323 
2324  // Add <numArgs>.
2325  SDValue NumArgs = *It++;
2326  assert(NumArgs.getValueType() == MVT::i32);
2327  Ops.push_back(NumArgs);
2328 
2329  // Calling convention.
2330  Ops.push_back(*It++);
2331 
2332  // Push the args for the call.
2333  for (uint64_t I = cast<ConstantSDNode>(NumArgs)->getZExtValue(); I != 0; I--)
2334  Ops.push_back(*It++);
2335 
2336  // Now push the live variables.
2337  for (; It != N->op_end(); It++)
2338  pushStackMapLiveVariable(Ops, *It, DL);
2339 
2340  // Finally, the regmask, chain and (if present) glue are moved to the end.
2341  Ops.push_back(RegMask);
2342  Ops.push_back(Chain);
2343  if (Glue.has_value())
2344  Ops.push_back(*Glue);
2345 
2346  SDVTList NodeTys = N->getVTList();
2347  CurDAG->SelectNodeTo(N, TargetOpcode::PATCHPOINT, NodeTys, Ops);
2348 }
2349 
2350 /// GetVBR - decode a vbr encoding whose top bit is set.
2352 GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) {
2353  assert(Val >= 128 && "Not a VBR");
2354  Val &= 127; // Remove first vbr bit.
2355 
2356  unsigned Shift = 7;
2357  uint64_t NextBits;
2358  do {
2359  NextBits = MatcherTable[Idx++];
2360  Val |= (NextBits&127) << Shift;
2361  Shift += 7;
2362  } while (NextBits & 128);
2363 
2364  return Val;
2365 }
2366 
2367 /// When a match is complete, this method updates uses of interior chain results
2368 /// to use the new results.
2369 void SelectionDAGISel::UpdateChains(
2370  SDNode *NodeToMatch, SDValue InputChain,
2371  SmallVectorImpl<SDNode *> &ChainNodesMatched, bool isMorphNodeTo) {
2372  SmallVector<SDNode*, 4> NowDeadNodes;
2373 
2374  // Now that all the normal results are replaced, we replace the chain and
2375  // glue results if present.
2376  if (!ChainNodesMatched.empty()) {
2377  assert(InputChain.getNode() &&
2378  "Matched input chains but didn't produce a chain");
2379  // Loop over all of the nodes we matched that produced a chain result.
2380  // Replace all the chain results with the final chain we ended up with.
2381  for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) {
2382  SDNode *ChainNode = ChainNodesMatched[i];
2383  // If ChainNode is null, it's because we replaced it on a previous
2384  // iteration and we cleared it out of the map. Just skip it.
2385  if (!ChainNode)
2386  continue;
2387 
2388  assert(ChainNode->getOpcode() != ISD::DELETED_NODE &&
2389  "Deleted node left in chain");
2390 
2391  // Don't replace the results of the root node if we're doing a
2392  // MorphNodeTo.
2393  if (ChainNode == NodeToMatch && isMorphNodeTo)
2394  continue;
2395 
2396  SDValue ChainVal = SDValue(ChainNode, ChainNode->getNumValues()-1);
2397  if (ChainVal.getValueType() == MVT::Glue)
2398  ChainVal = ChainVal.getValue(ChainVal->getNumValues()-2);
2399  assert(ChainVal.getValueType() == MVT::Other && "Not a chain?");
2401  *CurDAG, [&](SDNode *N, SDNode *E) {
2402  std::replace(ChainNodesMatched.begin(), ChainNodesMatched.end(), N,
2403  static_cast<SDNode *>(nullptr));
2404  });
2405  if (ChainNode->getOpcode() != ISD::TokenFactor)
2406  ReplaceUses(ChainVal, InputChain);
2407 
2408  // If the node became dead and we haven't already seen it, delete it.
2409  if (ChainNode != NodeToMatch && ChainNode->use_empty() &&
2410  !llvm::is_contained(NowDeadNodes, ChainNode))
2411  NowDeadNodes.push_back(ChainNode);
2412  }
2413  }
2414 
2415  if (!NowDeadNodes.empty())
2416  CurDAG->RemoveDeadNodes(NowDeadNodes);
2417 
2418  LLVM_DEBUG(dbgs() << "ISEL: Match complete!\n");
2419 }
2420 
2421 /// HandleMergeInputChains - This implements the OPC_EmitMergeInputChains
2422 /// operation for when the pattern matched at least one node with a chains. The
2423 /// input vector contains a list of all of the chained nodes that we match. We
2424 /// must determine if this is a valid thing to cover (i.e. matching it won't
2425 /// induce cycles in the DAG) and if so, creating a TokenFactor node. that will
2426 /// be used as the input node chain for the generated nodes.
2427 static SDValue
2429  SelectionDAG *CurDAG) {
2430 
2433  SmallVector<SDValue, 3> InputChains;
2434  unsigned int Max = 8192;
2435 
2436  // Quick exit on trivial merge.
2437  if (ChainNodesMatched.size() == 1)
2438  return ChainNodesMatched[0]->getOperand(0);
2439 
2440  // Add chains that aren't already added (internal). Peek through
2441  // token factors.
2442  std::function<void(const SDValue)> AddChains = [&](const SDValue V) {
2443  if (V.getValueType() != MVT::Other)
2444  return;
2445  if (V->getOpcode() == ISD::EntryToken)
2446  return;
2447  if (!Visited.insert(V.getNode()).second)
2448  return;
2449  if (V->getOpcode() == ISD::TokenFactor) {
2450  for (const SDValue &Op : V->op_values())
2451  AddChains(Op);
2452  } else
2453  InputChains.push_back(V);
2454  };
2455 
2456  for (auto *N : ChainNodesMatched) {
2457  Worklist.push_back(N);
2458  Visited.insert(N);
2459  }
2460 
2461  while (!Worklist.empty())
2462  AddChains(Worklist.pop_back_val()->getOperand(0));
2463 
2464  // Skip the search if there are no chain dependencies.
2465  if (InputChains.size() == 0)
2466  return CurDAG->getEntryNode();
2467 
2468  // If one of these chains is a successor of input, we must have a
2469  // node that is both the predecessor and successor of the
2470  // to-be-merged nodes. Fail.
2471  Visited.clear();
2472  for (SDValue V : InputChains)
2473  Worklist.push_back(V.getNode());
2474 
2475  for (auto *N : ChainNodesMatched)
2476  if (SDNode::hasPredecessorHelper(N, Visited, Worklist, Max, true))
2477  return SDValue();
2478 
2479  // Return merged chain.
2480  if (InputChains.size() == 1)
2481  return InputChains[0];
2482  return CurDAG->getNode(ISD::TokenFactor, SDLoc(ChainNodesMatched[0]),
2483  MVT::Other, InputChains);
2484 }
2485 
2486 /// MorphNode - Handle morphing a node in place for the selector.
2487 SDNode *SelectionDAGISel::
2488 MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList,
2489  ArrayRef<SDValue> Ops, unsigned EmitNodeInfo) {
2490  // It is possible we're using MorphNodeTo to replace a node with no
2491  // normal results with one that has a normal result (or we could be
2492  // adding a chain) and the input could have glue and chains as well.
2493  // In this case we need to shift the operands down.
2494  // FIXME: This is a horrible hack and broken in obscure cases, no worse
2495  // than the old isel though.
2496  int OldGlueResultNo = -1, OldChainResultNo = -1;
2497 
2498  unsigned NTMNumResults = Node->getNumValues();
2499  if (Node->getValueType(NTMNumResults-1) == MVT::Glue) {
2500  OldGlueResultNo = NTMNumResults-1;
2501  if (NTMNumResults != 1 &&
2502  Node->getValueType(NTMNumResults-2) == MVT::Other)
2503  OldChainResultNo = NTMNumResults-2;
2504  } else if (Node->getValueType(NTMNumResults-1) == MVT::Other)
2505  OldChainResultNo = NTMNumResults-1;
2506 
2507  // Call the underlying SelectionDAG routine to do the transmogrification. Note
2508  // that this deletes operands of the old node that become dead.
2509  SDNode *Res = CurDAG->MorphNodeTo(Node, ~TargetOpc, VTList, Ops);
2510 
2511  // MorphNodeTo can operate in two ways: if an existing node with the
2512  // specified operands exists, it can just return it. Otherwise, it
2513  // updates the node in place to have the requested operands.
2514  if (Res == Node) {
2515  // If we updated the node in place, reset the node ID. To the isel,
2516  // this should be just like a newly allocated machine node.
2517  Res->setNodeId(-1);
2518  }
2519 
2520  unsigned ResNumResults = Res->getNumValues();
2521  // Move the glue if needed.
2522  if ((EmitNodeInfo & OPFL_GlueOutput) && OldGlueResultNo != -1 &&
2523  (unsigned)OldGlueResultNo != ResNumResults-1)
2524  ReplaceUses(SDValue(Node, OldGlueResultNo),
2525  SDValue(Res, ResNumResults - 1));
2526 
2527  if ((EmitNodeInfo & OPFL_GlueOutput) != 0)
2528  --ResNumResults;
2529 
2530  // Move the chain reference if needed.
2531  if ((EmitNodeInfo & OPFL_Chain) && OldChainResultNo != -1 &&
2532  (unsigned)OldChainResultNo != ResNumResults-1)
2533  ReplaceUses(SDValue(Node, OldChainResultNo),
2534  SDValue(Res, ResNumResults - 1));
2535 
2536  // Otherwise, no replacement happened because the node already exists. Replace
2537  // Uses of the old node with the new one.
2538  if (Res != Node) {
2539  ReplaceNode(Node, Res);
2540  } else {
2542  }
2543 
2544  return Res;
2545 }
2546 
2547 /// CheckSame - Implements OP_CheckSame.
2548 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2549 CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2550  const SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes) {
2551  // Accept if it is exactly the same as a previously recorded node.
2552  unsigned RecNo = MatcherTable[MatcherIndex++];
2553  assert(RecNo < RecordedNodes.size() && "Invalid CheckSame");
2554  return N == RecordedNodes[RecNo].first;
2555 }
2556 
2557 /// CheckChildSame - Implements OP_CheckChildXSame.
2559  const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2560  const SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes,
2561  unsigned ChildNo) {
2562  if (ChildNo >= N.getNumOperands())
2563  return false; // Match fails if out of range child #.
2564  return ::CheckSame(MatcherTable, MatcherIndex, N.getOperand(ChildNo),
2565  RecordedNodes);
2566 }
2567 
2568 /// CheckPatternPredicate - Implements OP_CheckPatternPredicate.
2569 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2570 CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2571  const SelectionDAGISel &SDISel) {
2572  return SDISel.CheckPatternPredicate(MatcherTable[MatcherIndex++]);
2573 }
2574 
2575 /// CheckNodePredicate - Implements OP_CheckNodePredicate.
2576 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2577 CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2578  const SelectionDAGISel &SDISel, SDNode *N) {
2579  return SDISel.CheckNodePredicate(N, MatcherTable[MatcherIndex++]);
2580 }
2581 
2582 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2583 CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2584  SDNode *N) {
2585  uint16_t Opc = MatcherTable[MatcherIndex++];
2586  Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
2587  return N->getOpcode() == Opc;
2588 }
2589 
2590 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2591 CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2592  const TargetLowering *TLI, const DataLayout &DL) {
2593  MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
2594  if (N.getValueType() == VT) return true;
2595 
2596  // Handle the case when VT is iPTR.
2597  return VT == MVT::iPTR && N.getValueType() == TLI->getPointerTy(DL);
2598 }
2599 
2600 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2601 CheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2602  SDValue N, const TargetLowering *TLI, const DataLayout &DL,
2603  unsigned ChildNo) {
2604  if (ChildNo >= N.getNumOperands())
2605  return false; // Match fails if out of range child #.
2606  return ::CheckType(MatcherTable, MatcherIndex, N.getOperand(ChildNo), TLI,
2607  DL);
2608 }
2609 
2610 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2611 CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2612  SDValue N) {
2613  return cast<CondCodeSDNode>(N)->get() ==
2614  (ISD::CondCode)MatcherTable[MatcherIndex++];
2615 }
2616 
2617 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2618 CheckChild2CondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2619  SDValue N) {
2620  if (2 >= N.getNumOperands())
2621  return false;
2622  return ::CheckCondCode(MatcherTable, MatcherIndex, N.getOperand(2));
2623 }
2624 
2625 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2626 CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2627  SDValue N, const TargetLowering *TLI, const DataLayout &DL) {
2628  MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
2629  if (cast<VTSDNode>(N)->getVT() == VT)
2630  return true;
2631 
2632  // Handle the case when VT is iPTR.
2633  return VT == MVT::iPTR && cast<VTSDNode>(N)->getVT() == TLI->getPointerTy(DL);
2634 }
2635 
2636 // Bit 0 stores the sign of the immediate. The upper bits contain the magnitude
2637 // shifted left by 1.
2639  if ((V & 1) == 0)
2640  return V >> 1;
2641  if (V != 1)
2642  return -(V >> 1);
2643  // There is no such thing as -0 with integers. "-0" really means MININT.
2644  return 1ULL << 63;
2645 }
2646 
2647 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2648 CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2649  SDValue N) {
2650  int64_t Val = MatcherTable[MatcherIndex++];
2651  if (Val & 128)
2652  Val = GetVBR(Val, MatcherTable, MatcherIndex);
2653 
2654  Val = decodeSignRotatedValue(Val);
2655 
2656  ConstantSDNode *C = dyn_cast<ConstantSDNode>(N);
2657  return C && C->getSExtValue() == Val;
2658 }
2659 
2660 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2661 CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2662  SDValue N, unsigned ChildNo) {
2663  if (ChildNo >= N.getNumOperands())
2664  return false; // Match fails if out of range child #.
2665  return ::CheckInteger(MatcherTable, MatcherIndex, N.getOperand(ChildNo));
2666 }
2667 
2668 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2669 CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2670  SDValue N, const SelectionDAGISel &SDISel) {
2671  int64_t Val = MatcherTable[MatcherIndex++];
2672  if (Val & 128)
2673  Val = GetVBR(Val, MatcherTable, MatcherIndex);
2674 
2675  if (N->getOpcode() != ISD::AND) return false;
2676 
2677  ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2678  return C && SDISel.CheckAndMask(N.getOperand(0), C, Val);
2679 }
2680 
2681 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2682 CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2683  const SelectionDAGISel &SDISel) {
2684  int64_t Val = MatcherTable[MatcherIndex++];
2685  if (Val & 128)
2686  Val = GetVBR(Val, MatcherTable, MatcherIndex);
2687 
2688  if (N->getOpcode() != ISD::OR) return false;
2689 
2690  ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2691  return C && SDISel.CheckOrMask(N.getOperand(0), C, Val);
2692 }
2693 
2694 /// IsPredicateKnownToFail - If we know how and can do so without pushing a
2695 /// scope, evaluate the current node. If the current predicate is known to
2696 /// fail, set Result=true and return anything. If the current predicate is
2697 /// known to pass, set Result=false and return the MatcherIndex to continue
2698 /// with. If the current predicate is unknown, set Result=false and return the
2699 /// MatcherIndex to continue with.
2700 static unsigned IsPredicateKnownToFail(const unsigned char *Table,
2701  unsigned Index, SDValue N,
2702  bool &Result,
2703  const SelectionDAGISel &SDISel,
2704  SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes) {
2705  switch (Table[Index++]) {
2706  default:
2707  Result = false;
2708  return Index-1; // Could not evaluate this predicate.
2710  Result = !::CheckSame(Table, Index, N, RecordedNodes);
2711  return Index;
2716  Result = !::CheckChildSame(Table, Index, N, RecordedNodes,
2718  return Index;
2720  Result = !::CheckPatternPredicate(Table, Index, SDISel);
2721  return Index;
2723  Result = !::CheckNodePredicate(Table, Index, SDISel, N.getNode());
2724  return Index;
2726  Result = !::CheckOpcode(Table, Index, N.getNode());
2727  return Index;
2729  Result = !::CheckType(Table, Index, N, SDISel.TLI,
2730  SDISel.CurDAG->getDataLayout());
2731  return Index;
2733  unsigned Res = Table[Index++];
2734  Result = !::CheckType(Table, Index, N.getValue(Res), SDISel.TLI,
2735  SDISel.CurDAG->getDataLayout());
2736  return Index;
2737  }
2746  Result = !::CheckChildType(
2747  Table, Index, N, SDISel.TLI, SDISel.CurDAG->getDataLayout(),
2749  return Index;
2751  Result = !::CheckCondCode(Table, Index, N);
2752  return Index;
2754  Result = !::CheckChild2CondCode(Table, Index, N);
2755  return Index;
2757  Result = !::CheckValueType(Table, Index, N, SDISel.TLI,
2758  SDISel.CurDAG->getDataLayout());
2759  return Index;
2761  Result = !::CheckInteger(Table, Index, N);
2762  return Index;
2768  Result = !::CheckChildInteger(Table, Index, N,
2770  return Index;
2772  Result = !::CheckAndImm(Table, Index, N, SDISel);
2773  return Index;
2775  Result = !::CheckOrImm(Table, Index, N, SDISel);
2776  return Index;
2777  }
2778 }
2779 
2780 namespace {
2781 
2782 struct MatchScope {
2783  /// FailIndex - If this match fails, this is the index to continue with.
2784  unsigned FailIndex;
2785 
2786  /// NodeStack - The node stack when the scope was formed.
2787  SmallVector<SDValue, 4> NodeStack;
2788 
2789  /// NumRecordedNodes - The number of recorded nodes when the scope was formed.
2790  unsigned NumRecordedNodes;
2791 
2792  /// NumMatchedMemRefs - The number of matched memref entries.
2793  unsigned NumMatchedMemRefs;
2794 
2795  /// InputChain/InputGlue - The current chain/glue
2796  SDValue InputChain, InputGlue;
2797 
2798  /// HasChainNodesMatched - True if the ChainNodesMatched list is non-empty.
2799  bool HasChainNodesMatched;
2800 };
2801 
2802 /// \A DAG update listener to keep the matching state
2803 /// (i.e. RecordedNodes and MatchScope) uptodate if the target is allowed to
2804 /// change the DAG while matching. X86 addressing mode matcher is an example
2805 /// for this.
2806 class MatchStateUpdater : public SelectionDAG::DAGUpdateListener
2807 {
2808  SDNode **NodeToMatch;
2810  SmallVectorImpl<MatchScope> &MatchScopes;
2811 
2812 public:
2813  MatchStateUpdater(SelectionDAG &DAG, SDNode **NodeToMatch,
2814  SmallVectorImpl<std::pair<SDValue, SDNode *>> &RN,
2816  : SelectionDAG::DAGUpdateListener(DAG), NodeToMatch(NodeToMatch),
2817  RecordedNodes(RN), MatchScopes(MS) {}
2818 
2819  void NodeDeleted(SDNode *N, SDNode *E) override {
2820  // Some early-returns here to avoid the search if we deleted the node or
2821  // if the update comes from MorphNodeTo (MorphNodeTo is the last thing we
2822  // do, so it's unnecessary to update matching state at that point).
2823  // Neither of these can occur currently because we only install this
2824  // update listener during matching a complex patterns.
2825  if (!E || E->isMachineOpcode())
2826  return;
2827  // Check if NodeToMatch was updated.
2828  if (N == *NodeToMatch)
2829  *NodeToMatch = E;
2830  // Performing linear search here does not matter because we almost never
2831  // run this code. You'd have to have a CSE during complex pattern
2832  // matching.
2833  for (auto &I : RecordedNodes)
2834  if (I.first.getNode() == N)
2835  I.first.setNode(E);
2836 
2837  for (auto &I : MatchScopes)
2838  for (auto &J : I.NodeStack)
2839  if (J.getNode() == N)
2840  J.setNode(E);
2841  }
2842 };
2843 
2844 } // end anonymous namespace
2845 
2847  const unsigned char *MatcherTable,
2848  unsigned TableSize) {
2849  // FIXME: Should these even be selected? Handle these cases in the caller?
2850  switch (NodeToMatch->getOpcode()) {
2851  default:
2852  break;
2853  case ISD::EntryToken: // These nodes remain the same.
2854  case ISD::BasicBlock:
2855  case ISD::Register:
2856  case ISD::RegisterMask:
2857  case ISD::HANDLENODE:
2858  case ISD::MDNODE_SDNODE:
2859  case ISD::TargetConstant:
2860  case ISD::TargetConstantFP:
2862  case ISD::TargetFrameIndex:
2864  case ISD::MCSymbol:
2866  case ISD::TargetJumpTable:
2869  case ISD::TokenFactor:
2870  case ISD::CopyFromReg:
2871  case ISD::CopyToReg:
2872  case ISD::EH_LABEL:
2873  case ISD::ANNOTATION_LABEL:
2874  case ISD::LIFETIME_START:
2875  case ISD::LIFETIME_END:
2876  case ISD::PSEUDO_PROBE:
2877  NodeToMatch->setNodeId(-1); // Mark selected.
2878  return;
2879  case ISD::AssertSext:
2880  case ISD::AssertZext:
2881  case ISD::AssertAlign:
2882  ReplaceUses(SDValue(NodeToMatch, 0), NodeToMatch->getOperand(0));
2883  CurDAG->RemoveDeadNode(NodeToMatch);
2884  return;
2885  case ISD::INLINEASM:
2886  case ISD::INLINEASM_BR:
2887  Select_INLINEASM(NodeToMatch);
2888  return;
2889  case ISD::READ_REGISTER:
2890  Select_READ_REGISTER(NodeToMatch);
2891  return;
2892  case ISD::WRITE_REGISTER:
2893  Select_WRITE_REGISTER(NodeToMatch);
2894  return;
2895  case ISD::UNDEF:
2896  Select_UNDEF(NodeToMatch);
2897  return;
2898  case ISD::FREEZE:
2899  Select_FREEZE(NodeToMatch);
2900  return;
2901  case ISD::ARITH_FENCE:
2902  Select_ARITH_FENCE(NodeToMatch);
2903  return;
2904  case ISD::MEMBARRIER:
2905  Select_MEMBARRIER(NodeToMatch);
2906  return;
2907  case ISD::STACKMAP:
2908  Select_STACKMAP(NodeToMatch);
2909  return;
2910  case ISD::PATCHPOINT:
2911  Select_PATCHPOINT(NodeToMatch);
2912  return;
2913  }
2914 
2915  assert(!NodeToMatch->isMachineOpcode() && "Node already selected!");
2916 
2917  // Set up the node stack with NodeToMatch as the only node on the stack.
2918  SmallVector<SDValue, 8> NodeStack;
2919  SDValue N = SDValue(NodeToMatch, 0);
2920  NodeStack.push_back(N);
2921 
2922  // MatchScopes - Scopes used when matching, if a match failure happens, this
2923  // indicates where to continue checking.
2924  SmallVector<MatchScope, 8> MatchScopes;
2925 
2926  // RecordedNodes - This is the set of nodes that have been recorded by the
2927  // state machine. The second value is the parent of the node, or null if the
2928  // root is recorded.
2929  SmallVector<std::pair<SDValue, SDNode*>, 8> RecordedNodes;
2930 
2931  // MatchedMemRefs - This is the set of MemRef's we've seen in the input
2932  // pattern.
2933  SmallVector<MachineMemOperand*, 2> MatchedMemRefs;
2934 
2935  // These are the current input chain and glue for use when generating nodes.
2936  // Various Emit operations change these. For example, emitting a copytoreg
2937  // uses and updates these.
2938  SDValue InputChain, InputGlue;
2939 
2940  // ChainNodesMatched - If a pattern matches nodes that have input/output
2941  // chains, the OPC_EmitMergeInputChains operation is emitted which indicates
2942  // which ones they are. The result is captured into this list so that we can
2943  // update the chain results when the pattern is complete.
2944  SmallVector<SDNode*, 3> ChainNodesMatched;
2945 
2946  LLVM_DEBUG(dbgs() << "ISEL: Starting pattern match\n");
2947 
2948  // Determine where to start the interpreter. Normally we start at opcode #0,
2949  // but if the state machine starts with an OPC_SwitchOpcode, then we
2950  // accelerate the first lookup (which is guaranteed to be hot) with the
2951  // OpcodeOffset table.
2952  unsigned MatcherIndex = 0;
2953 
2954  if (!OpcodeOffset.empty()) {
2955  // Already computed the OpcodeOffset table, just index into it.
2956  if (N.getOpcode() < OpcodeOffset.size())
2957  MatcherIndex = OpcodeOffset[N.getOpcode()];
2958  LLVM_DEBUG(dbgs() << " Initial Opcode index to " << MatcherIndex << "\n");
2959 
2960  } else if (MatcherTable[0] == OPC_SwitchOpcode) {
2961  // Otherwise, the table isn't computed, but the state machine does start
2962  // with an OPC_SwitchOpcode instruction. Populate the table now, since this
2963  // is the first time we're selecting an instruction.
2964  unsigned Idx = 1;
2965  while (true) {
2966  // Get the size of this case.
2967  unsigned CaseSize = MatcherTable[Idx++];
2968  if (CaseSize & 128)
2969  CaseSize = GetVBR(CaseSize, MatcherTable, Idx);
2970  if (CaseSize == 0) break;
2971 
2972  // Get the opcode, add the index to the table.
2973  uint16_t Opc = MatcherTable[Idx++];
2974  Opc |= (unsigned short)MatcherTable[Idx++] << 8;
2975  if (Opc >= OpcodeOffset.size())
2976  OpcodeOffset.resize((Opc+1)*2);
2977  OpcodeOffset[Opc] = Idx;
2978  Idx += CaseSize;
2979  }
2980 
2981  // Okay, do the lookup for the first opcode.
2982  if (N.getOpcode() < OpcodeOffset.size())
2983  MatcherIndex = OpcodeOffset[N.getOpcode()];
2984  }
2985 
2986  while (true) {
2987  assert(MatcherIndex < TableSize && "Invalid index");
2988 #ifndef NDEBUG
2989  unsigned CurrentOpcodeIndex = MatcherIndex;
2990 #endif
2991  BuiltinOpcodes Opcode = (BuiltinOpcodes)MatcherTable[MatcherIndex++];
2992  switch (Opcode) {
2993  case OPC_Scope: {
2994  // Okay, the semantics of this operation are that we should push a scope
2995  // then evaluate the first child. However, pushing a scope only to have
2996  // the first check fail (which then pops it) is inefficient. If we can
2997  // determine immediately that the first check (or first several) will
2998  // immediately fail, don't even bother pushing a scope for them.
2999  unsigned FailIndex;
3000 
3001  while (true) {
3002  unsigned NumToSkip = MatcherTable[MatcherIndex++];
3003  if (NumToSkip & 128)
3004  NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex);
3005  // Found the end of the scope with no match.
3006  if (NumToSkip == 0) {
3007  FailIndex = 0;
3008  break;
3009  }
3010 
3011  FailIndex = MatcherIndex+NumToSkip;
3012 
3013  unsigned MatcherIndexOfPredicate = MatcherIndex;
3014  (void)MatcherIndexOfPredicate; // silence warning.
3015 
3016  // If we can't evaluate this predicate without pushing a scope (e.g. if
3017  // it is a 'MoveParent') or if the predicate succeeds on this node, we
3018  // push the scope and evaluate the full predicate chain.
3019  bool Result;
3020  MatcherIndex = IsPredicateKnownToFail(MatcherTable, MatcherIndex, N,
3021  Result, *this, RecordedNodes);
3022  if (!Result)
3023  break;
3024 
3025  LLVM_DEBUG(
3026  dbgs() << " Skipped scope entry (due to false predicate) at "
3027  << "index " << MatcherIndexOfPredicate << ", continuing at "
3028  << FailIndex << "\n");
3029  ++NumDAGIselRetries;
3030 
3031  // Otherwise, we know that this case of the Scope is guaranteed to fail,
3032  // move to the next case.
3033  MatcherIndex = FailIndex;
3034  }
3035 
3036  // If the whole scope failed to match, bail.
3037  if (FailIndex == 0) break;
3038 
3039  // Push a MatchScope which indicates where to go if the first child fails
3040  // to match.
3041  MatchScope NewEntry;
3042  NewEntry.FailIndex = FailIndex;
3043  NewEntry.NodeStack.append(NodeStack.begin(), NodeStack.end());
3044  NewEntry.NumRecordedNodes = RecordedNodes.size();
3045  NewEntry.NumMatchedMemRefs = MatchedMemRefs.size();
3046  NewEntry.InputChain = InputChain;
3047  NewEntry.InputGlue = InputGlue;
3048  NewEntry.HasChainNodesMatched = !ChainNodesMatched.empty();
3049  MatchScopes.push_back(NewEntry);
3050  continue;
3051  }
3052  case OPC_RecordNode: {
3053  // Remember this node, it may end up being an operand in the pattern.
3054  SDNode *Parent = nullptr;
3055  if (NodeStack.size() > 1)
3056  Parent = NodeStack[NodeStack.size()-2].getNode();
3057  RecordedNodes.push_back(std::make_pair(N, Parent));
3058  continue;
3059  }
3060 
3064  case OPC_RecordChild6: case OPC_RecordChild7: {
3065  unsigned ChildNo = Opcode-OPC_RecordChild0;
3066  if (ChildNo >= N.getNumOperands())
3067  break; // Match fails if out of range child #.
3068 
3069  RecordedNodes.push_back(std::make_pair(N->getOperand(ChildNo),
3070  N.getNode()));
3071  continue;
3072  }
3073  case OPC_RecordMemRef:
3074  if (auto *MN = dyn_cast<MemSDNode>(N))
3075  MatchedMemRefs.push_back(MN->getMemOperand());
3076  else {
3077  LLVM_DEBUG(dbgs() << "Expected MemSDNode "; N->dump(CurDAG);
3078  dbgs() << '\n');
3079  }
3080 
3081  continue;
3082 
3083  case OPC_CaptureGlueInput:
3084  // If the current node has an input glue, capture it in InputGlue.
3085  if (N->getNumOperands() != 0 &&
3086  N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Glue)
3087  InputGlue = N->getOperand(N->getNumOperands()-1);
3088  continue;
3089 
3090  case OPC_MoveChild: {
3091  unsigned ChildNo = MatcherTable[MatcherIndex++];
3092  if (ChildNo >= N.getNumOperands())
3093  break; // Match fails if out of range child #.
3094  N = N.getOperand(ChildNo);
3095  NodeStack.push_back(N);
3096  continue;
3097  }
3098 
3099  case OPC_MoveChild0: case OPC_MoveChild1:
3100  case OPC_MoveChild2: case OPC_MoveChild3:
3101  case OPC_MoveChild4: case OPC_MoveChild5:
3102  case OPC_MoveChild6: case OPC_MoveChild7: {
3103  unsigned ChildNo = Opcode-OPC_MoveChild0;
3104  if (ChildNo >= N.getNumOperands())
3105  break; // Match fails if out of range child #.
3106  N = N.getOperand(ChildNo);
3107  NodeStack.push_back(N);
3108  continue;
3109  }
3110 
3111  case OPC_MoveParent:
3112  // Pop the current node off the NodeStack.
3113  NodeStack.pop_back();
3114  assert(!NodeStack.empty() && "Node stack imbalance!");
3115  N = NodeStack.back();
3116  continue;
3117 
3118  case OPC_CheckSame:
3119  if (!::CheckSame(MatcherTable, MatcherIndex, N, RecordedNodes)) break;
3120  continue;
3121 
3124  if (!::CheckChildSame(MatcherTable, MatcherIndex, N, RecordedNodes,
3125  Opcode-OPC_CheckChild0Same))
3126  break;
3127  continue;
3128 
3130  if (!::CheckPatternPredicate(MatcherTable, MatcherIndex, *this)) break;
3131  continue;
3132  case OPC_CheckPredicate:
3133  if (!::CheckNodePredicate(MatcherTable, MatcherIndex, *this,
3134  N.getNode()))
3135  break;
3136  continue;
3138  unsigned OpNum = MatcherTable[MatcherIndex++];
3140 
3141  for (unsigned i = 0; i < OpNum; ++i)
3142  Operands.push_back(RecordedNodes[MatcherTable[MatcherIndex++]].first);
3143 
3144  unsigned PredNo = MatcherTable[MatcherIndex++];
3145  if (!CheckNodePredicateWithOperands(N.getNode(), PredNo, Operands))
3146  break;
3147  continue;
3148  }
3149  case OPC_CheckComplexPat: {
3150  unsigned CPNum = MatcherTable[MatcherIndex++];
3151  unsigned RecNo = MatcherTable[MatcherIndex++];
3152  assert(RecNo < RecordedNodes.size() && "Invalid CheckComplexPat");
3153 
3154  // If target can modify DAG during matching, keep the matching state
3155  // consistent.
3156  std::unique_ptr<MatchStateUpdater> MSU;
3158  MSU.reset(new MatchStateUpdater(*CurDAG, &NodeToMatch, RecordedNodes,
3159  MatchScopes));
3160 
3161  if (!CheckComplexPattern(NodeToMatch, RecordedNodes[RecNo].second,
3162  RecordedNodes[RecNo].first, CPNum,
3163  RecordedNodes))
3164  break;
3165  continue;
3166  }
3167  case OPC_CheckOpcode:
3168  if (!::CheckOpcode(MatcherTable, MatcherIndex, N.getNode())) break;
3169  continue;
3170 
3171  case OPC_CheckType:
3172  if (!::CheckType(MatcherTable, MatcherIndex, N, TLI,
3173  CurDAG->getDataLayout()))
3174  break;
3175  continue;
3176 
3177  case OPC_CheckTypeRes: {
3178  unsigned Res = MatcherTable[MatcherIndex++];
3179  if (!::CheckType(MatcherTable, MatcherIndex, N.getValue(Res), TLI,
3180  CurDAG->getDataLayout()))
3181  break;
3182  continue;
3183  }
3184 
3185  case OPC_SwitchOpcode: {
3186  unsigned CurNodeOpcode = N.getOpcode();
3187  unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3188  unsigned CaseSize;
3189  while (true) {
3190  // Get the size of this case.
3191  CaseSize = MatcherTable[MatcherIndex++];
3192  if (CaseSize & 128)
3193  CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);
3194  if (CaseSize == 0) break;
3195 
3196  uint16_t Opc = MatcherTable[MatcherIndex++];
3197  Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
3198 
3199  // If the opcode matches, then we will execute this case.
3200  if (CurNodeOpcode == Opc)
3201  break;
3202 
3203  // Otherwise, skip over this case.
3204  MatcherIndex += CaseSize;
3205  }
3206 
3207  // If no cases matched, bail out.
3208  if (CaseSize == 0) break;
3209 
3210  // Otherwise, execute the case we found.
3211  LLVM_DEBUG(dbgs() << " OpcodeSwitch from " << SwitchStart << " to "
3212  << MatcherIndex << "\n");
3213  continue;
3214  }
3215 
3216  case OPC_SwitchType: {
3217  MVT CurNodeVT = N.getSimpleValueType();
3218  unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3219  unsigned CaseSize;
3220  while (true) {
3221  // Get the size of this case.
3222  CaseSize = MatcherTable[MatcherIndex++];
3223  if (CaseSize & 128)
3224  CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);
3225  if (CaseSize == 0) break;
3226 
3227  MVT CaseVT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3228  if (CaseVT == MVT::iPTR)
3229  CaseVT = TLI->getPointerTy(CurDAG->getDataLayout());
3230 
3231  // If the VT matches, then we will execute this case.
3232  if (CurNodeVT == CaseVT)
3233  break;
3234 
3235  // Otherwise, skip over this case.
3236  MatcherIndex += CaseSize;
3237  }
3238 
3239  // If no cases matched, bail out.
3240  if (CaseSize == 0) break;
3241 
3242  // Otherwise, execute the case we found.
3243  LLVM_DEBUG(dbgs() << " TypeSwitch[" << EVT(CurNodeVT).getEVTString()
3244  << "] from " << SwitchStart << " to " << MatcherIndex
3245  << '\n');
3246  continue;
3247  }
3252  if (!::CheckChildType(MatcherTable, MatcherIndex, N, TLI,
3253  CurDAG->getDataLayout(),
3254  Opcode - OPC_CheckChild0Type))
3255  break;
3256  continue;
3257  case OPC_CheckCondCode:
3258  if (!::CheckCondCode(MatcherTable, MatcherIndex, N)) break;
3259  continue;
3261  if (!::CheckChild2CondCode(MatcherTable, MatcherIndex, N)) break;
3262  continue;
3263  case OPC_CheckValueType:
3264  if (!::CheckValueType(MatcherTable, MatcherIndex, N, TLI,
3265  CurDAG->getDataLayout()))
3266  break;
3267  continue;
3268  case OPC_CheckInteger:
3269  if (!::CheckInteger(MatcherTable, MatcherIndex, N)) break;
3270  continue;
3274  if (!::CheckChildInteger(MatcherTable, MatcherIndex, N,
3275  Opcode-OPC_CheckChild0Integer)) break;
3276  continue;
3277  case OPC_CheckAndImm:
3278  if (!::CheckAndImm(MatcherTable, MatcherIndex, N, *this)) break;
3279  continue;
3280  case OPC_CheckOrImm:
3281  if (!::CheckOrImm(MatcherTable, MatcherIndex, N, *this)) break;
3282  continue;
3283  case OPC_CheckImmAllOnesV:
3284  if (!ISD::isConstantSplatVectorAllOnes(N.getNode()))
3285  break;
3286  continue;
3287  case OPC_CheckImmAllZerosV:
3288  if (!ISD::isConstantSplatVectorAllZeros(N.getNode()))
3289  break;
3290  continue;
3291 
3293  assert(NodeStack.size() != 1 && "No parent node");
3294  // Verify that all intermediate nodes between the root and this one have
3295  // a single use (ignoring chains, which are handled in UpdateChains).
3296  bool HasMultipleUses = false;
3297  for (unsigned i = 1, e = NodeStack.size()-1; i != e; ++i) {
3298  unsigned NNonChainUses = 0;
3299  SDNode *NS = NodeStack[i].getNode();
3300  for (auto UI = NS->use_begin(), UE = NS->use_end(); UI != UE; ++UI)
3301  if (UI.getUse().getValueType() != MVT::Other)
3302  if (++NNonChainUses > 1) {
3303  HasMultipleUses = true;
3304  break;
3305  }
3306  if (HasMultipleUses) break;
3307  }
3308  if (HasMultipleUses) break;
3309 
3310  // Check to see that the target thinks this is profitable to fold and that
3311  // we can fold it without inducing cycles in the graph.
3312  if (!IsProfitableToFold(N, NodeStack[NodeStack.size()-2].getNode(),
3313  NodeToMatch) ||
3314  !IsLegalToFold(N, NodeStack[NodeStack.size()-2].getNode(),
3315  NodeToMatch, OptLevel,
3316  true/*We validate our own chains*/))
3317  break;
3318 
3319  continue;
3320  }
3321  case OPC_EmitInteger:
3322  case OPC_EmitStringInteger: {
3324  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3325  int64_t Val = MatcherTable[MatcherIndex++];
3326  if (Val & 128)
3327  Val = GetVBR(Val, MatcherTable, MatcherIndex);
3328  if (Opcode == OPC_EmitInteger)
3329  Val = decodeSignRotatedValue(Val);
3330  RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3331  CurDAG->getTargetConstant(Val, SDLoc(NodeToMatch),
3332  VT), nullptr));
3333  continue;
3334  }
3335  case OPC_EmitRegister: {
3337  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3338  unsigned RegNo = MatcherTable[MatcherIndex++];
3339  RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3340  CurDAG->getRegister(RegNo, VT), nullptr));
3341  continue;
3342  }
3343  case OPC_EmitRegister2: {
3344  // For targets w/ more than 256 register names, the register enum
3345  // values are stored in two bytes in the matcher table (just like
3346  // opcodes).
3348  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3349  unsigned RegNo = MatcherTable[MatcherIndex++];
3350  RegNo |= MatcherTable[MatcherIndex++] << 8;
3351  RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3352  CurDAG->getRegister(RegNo, VT), nullptr));
3353  continue;
3354  }
3355 
3356  case OPC_EmitConvertToTarget: {
3357  // Convert from IMM/FPIMM to target version.
3358  unsigned RecNo = MatcherTable[MatcherIndex++];
3359  assert(RecNo < RecordedNodes.size() && "Invalid EmitConvertToTarget");
3360  SDValue Imm = RecordedNodes[RecNo].first;
3361 
3362  if (Imm->getOpcode() == ISD::Constant) {
3363  const ConstantInt *Val=cast<ConstantSDNode>(Imm)->getConstantIntValue();
3364  Imm = CurDAG->getTargetConstant(*Val, SDLoc(NodeToMatch),
3365  Imm.getValueType());
3366  } else if (Imm->getOpcode() == ISD::ConstantFP) {
3367  const ConstantFP *Val=cast<ConstantFPSDNode>(Imm)->getConstantFPValue();
3368  Imm = CurDAG->getTargetConstantFP(*Val, SDLoc(NodeToMatch),
3369  Imm.getValueType());
3370  }
3371 
3372  RecordedNodes.push_back(std::make_pair(Imm, RecordedNodes[RecNo].second));
3373  continue;
3374  }
3375 
3376  case OPC_EmitMergeInputChains1_0: // OPC_EmitMergeInputChains, 1, 0
3377  case OPC_EmitMergeInputChains1_1: // OPC_EmitMergeInputChains, 1, 1
3378  case OPC_EmitMergeInputChains1_2: { // OPC_EmitMergeInputChains, 1, 2
3379  // These are space-optimized forms of OPC_EmitMergeInputChains.
3380  assert(!InputChain.getNode() &&
3381  "EmitMergeInputChains should be the first chain producing node");
3382  assert(ChainNodesMatched.empty() &&
3383  "Should only have one EmitMergeInputChains per match");
3384 
3385  // Read all of the chained nodes.
3386  unsigned RecNo = Opcode - OPC_EmitMergeInputChains1_0;
3387  assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains");
3388  ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());
3389 
3390  // If the chained node is not the root, we can't fold it if it has
3391  // multiple uses.
3392  // FIXME: What if other value results of the node have uses not matched
3393  // by this pattern?
3394  if (ChainNodesMatched.back() != NodeToMatch &&
3395  !RecordedNodes[RecNo].first.hasOneUse()) {
3396  ChainNodesMatched.clear();
3397  break;
3398  }
3399 
3400  // Merge the input chains if they are not intra-pattern references.
3401  InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG);
3402 
3403  if (!InputChain.getNode())
3404  break; // Failed to merge.
3405  continue;
3406  }
3407 
3408  case OPC_EmitMergeInputChains: {
3409  assert(!InputChain.getNode() &&
3410  "EmitMergeInputChains should be the first chain producing node");
3411  // This node gets a list of nodes we matched in the input that have
3412  // chains. We want to token factor all of the input chains to these nodes
3413  // together. However, if any of the input chains is actually one of the
3414  // nodes matched in this pattern, then we have an intra-match reference.
3415  // Ignore these because the newly token factored chain should not refer to
3416  // the old nodes.
3417  unsigned NumChains = MatcherTable[MatcherIndex++];
3418  assert(NumChains != 0 && "Can't TF zero chains");
3419 
3420  assert(ChainNodesMatched.empty() &&
3421  "Should only have one EmitMergeInputChains per match");
3422 
3423  // Read all of the chained nodes.
3424  for (unsigned i = 0; i != NumChains; ++i) {
3425  unsigned RecNo = MatcherTable[MatcherIndex++];
3426  assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains");
3427  ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());
3428 
3429  // If the chained node is not the root, we can't fold it if it has
3430  // multiple uses.
3431  // FIXME: What if other value results of the node have uses not matched
3432  // by this pattern?
3433  if (ChainNodesMatched.back() != NodeToMatch &&
3434  !RecordedNodes[RecNo].first.hasOneUse()) {
3435  ChainNodesMatched.clear();
3436  break;
3437  }
3438  }
3439 
3440  // If the inner loop broke out, the match fails.
3441  if (ChainNodesMatched.empty())
3442  break;
3443 
3444  // Merge the input chains if they are not intra-pattern references.
3445  InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG);
3446 
3447  if (!InputChain.getNode())
3448  break; // Failed to merge.
3449 
3450  continue;
3451  }
3452 
3453  case OPC_EmitCopyToReg:
3454  case OPC_EmitCopyToReg2: {
3455  unsigned RecNo = MatcherTable[MatcherIndex++];
3456  assert(RecNo < RecordedNodes.size() && "Invalid EmitCopyToReg");
3457  unsigned DestPhysReg = MatcherTable[MatcherIndex++];
3458  if (Opcode == OPC_EmitCopyToReg2)
3459  DestPhysReg |= MatcherTable[MatcherIndex++] << 8;
3460 
3461  if (!InputChain.getNode())
3462  InputChain = CurDAG->getEntryNode();
3463 
3464  InputChain = CurDAG->getCopyToReg(InputChain, SDLoc(NodeToMatch),
3465  DestPhysReg, RecordedNodes[RecNo].first,
3466  InputGlue);
3467 
3468  InputGlue = InputChain.getValue(1);
3469  continue;
3470  }
3471 
3472  case OPC_EmitNodeXForm: {
3473  unsigned XFormNo = MatcherTable[MatcherIndex++];
3474  unsigned RecNo = MatcherTable[MatcherIndex++];
3475  assert(RecNo < RecordedNodes.size() && "Invalid EmitNodeXForm");
3476  SDValue Res = RunSDNodeXForm(RecordedNodes[RecNo].first, XFormNo);
3477  RecordedNodes.push_back(std::pair<SDValue,SDNode*>(Res, nullptr));
3478  continue;
3479  }
3480  case OPC_Coverage: {
3481  // This is emitted right before MorphNode/EmitNode.
3482  // So it should be safe to assume that this node has been selected
3483  unsigned index = MatcherTable[MatcherIndex++];
3484  index |= (MatcherTable[MatcherIndex++] << 8);
3485  dbgs() << "COVERED: " << getPatternForIndex(index) << "\n";
3486  dbgs() << "INCLUDED: " << getIncludePathForIndex(index) << "\n";
3487  continue;
3488  }
3489 
3490  case OPC_EmitNode: case OPC_MorphNodeTo:
3491  case OPC_EmitNode0: case OPC_EmitNode1: case OPC_EmitNode2:
3493  uint16_t TargetOpc = MatcherTable[MatcherIndex++];
3494  TargetOpc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
3495  unsigned EmitNodeInfo = MatcherTable[MatcherIndex++];
3496  // Get the result VT list.
3497  unsigned NumVTs;
3498  // If this is one of the compressed forms, get the number of VTs based
3499  // on the Opcode. Otherwise read the next byte from the table.
3500  if (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2)
3501  NumVTs = Opcode - OPC_MorphNodeTo0;
3502  else if (Opcode >= OPC_EmitNode0 && Opcode <= OPC_EmitNode2)
3503  NumVTs = Opcode - OPC_EmitNode0;
3504  else
3505  NumVTs = MatcherTable[MatcherIndex++];
3506  SmallVector<EVT, 4> VTs;
3507  for (unsigned i = 0; i != NumVTs; ++i) {
3509  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3510  if (VT == MVT::iPTR)
3511  VT = TLI->getPointerTy(CurDAG->getDataLayout()).SimpleTy;
3512  VTs.push_back(VT);
3513  }
3514 
3515  if (EmitNodeInfo & OPFL_Chain)
3516  VTs.push_back(MVT::Other);
3517  if (EmitNodeInfo & OPFL_GlueOutput)
3518  VTs.push_back(MVT::Glue);
3519 
3520  // This is hot code, so optimize the two most common cases of 1 and 2
3521  // results.
3522  SDVTList VTList;
3523  if (VTs.size() == 1)
3524  VTList = CurDAG->getVTList(VTs[0]);
3525  else if (VTs.size() == 2)
3526  VTList = CurDAG->getVTList(VTs[0], VTs[1]);
3527  else
3528  VTList = CurDAG->getVTList(VTs);
3529 
3530  // Get the operand list.
3531  unsigned NumOps = MatcherTable[MatcherIndex++];
3533  for (unsigned i = 0; i != NumOps; ++i) {
3534  unsigned RecNo = MatcherTable[MatcherIndex++];
3535  if (RecNo & 128)
3536  RecNo = GetVBR(RecNo, MatcherTable, MatcherIndex);
3537 
3538  assert(RecNo < RecordedNodes.size() && "Invalid EmitNode");
3539  Ops.push_back(RecordedNodes[RecNo].first);
3540  }
3541 
3542  // If there are variadic operands to add, handle them now.
3543  if (EmitNodeInfo & OPFL_VariadicInfo) {
3544  // Determine the start index to copy from.
3545  unsigned FirstOpToCopy = getNumFixedFromVariadicInfo(EmitNodeInfo);
3546  FirstOpToCopy += (EmitNodeInfo & OPFL_Chain) ? 1 : 0;
3547  assert(NodeToMatch->getNumOperands() >= FirstOpToCopy &&
3548  "Invalid variadic node");
3549  // Copy all of the variadic operands, not including a potential glue
3550  // input.
3551  for (unsigned i = FirstOpToCopy, e = NodeToMatch->getNumOperands();
3552  i != e; ++i) {
3553  SDValue V = NodeToMatch->getOperand(i);
3554  if (V.getValueType() == MVT::Glue) break;
3555  Ops.push_back(V);
3556  }
3557  }
3558 
3559  // If this has chain/glue inputs, add them.
3560  if (EmitNodeInfo & OPFL_Chain)
3561  Ops.push_back(InputChain);
3562  if ((EmitNodeInfo & OPFL_GlueInput) && InputGlue.getNode() != nullptr)
3563  Ops.push_back(InputGlue);
3564 
3565  // Check whether any matched node could raise an FP exception. Since all
3566  // such nodes must have a chain, it suffices to check ChainNodesMatched.
3567  // We need to perform this check before potentially modifying one of the
3568  // nodes via MorphNode.
3569  bool MayRaiseFPException =
3570  llvm::any_of(ChainNodesMatched, [this](SDNode *N) {
3571  return mayRaiseFPException(N) && !N->getFlags().hasNoFPExcept();
3572  });
3573 
3574  // Create the node.
3575  MachineSDNode *Res = nullptr;
3576  bool IsMorphNodeTo = Opcode == OPC_MorphNodeTo ||
3577  (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2);
3578  if (!IsMorphNodeTo) {
3579  // If this is a normal EmitNode command, just create the new node and
3580  // add the results to the RecordedNodes list.
3581  Res = CurDAG->getMachineNode(TargetOpc, SDLoc(NodeToMatch),
3582  VTList, Ops);
3583 
3584  // Add all the non-glue/non-chain results to the RecordedNodes list.
3585  for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
3586  if (VTs[i] == MVT::Other || VTs[i] == MVT::Glue) break;
3587  RecordedNodes.push_back(std::pair<SDValue,SDNode*>(SDValue(Res, i),
3588  nullptr));
3589  }
3590  } else {
3591  assert(NodeToMatch->getOpcode() != ISD::DELETED_NODE &&
3592  "NodeToMatch was removed partway through selection");
3594  SDNode *E) {
3596  auto &Chain = ChainNodesMatched;
3597  assert((!E || !is_contained(Chain, N)) &&
3598  "Chain node replaced during MorphNode");
3599  llvm::erase_value(Chain, N);
3600  });
3601  Res = cast<MachineSDNode>(MorphNode(NodeToMatch, TargetOpc, VTList,
3602  Ops, EmitNodeInfo));
3603  }
3604 
3605  // Set the NoFPExcept flag when no original matched node could
3606  // raise an FP exception, but the new node potentially might.
3608  SDNodeFlags Flags = Res->getFlags();
3609  Flags.setNoFPExcept(true);
3610  Res->setFlags(Flags);
3611  }
3612 
3613  // If the node had chain/glue results, update our notion of the current
3614  // chain and glue.
3615  if (EmitNodeInfo & OPFL_GlueOutput) {
3616  InputGlue = SDValue(Res, VTs.size()-1);
3617  if (EmitNodeInfo & OPFL_Chain)
3618  InputChain = SDValue(Res, VTs.size()-2);
3619  } else if (EmitNodeInfo & OPFL_Chain)
3620  InputChain = SDValue(Res, VTs.size()-1);
3621 
3622  // If the OPFL_MemRefs glue is set on this node, slap all of the
3623  // accumulated memrefs onto it.
3624  //
3625  // FIXME: This is vastly incorrect for patterns with multiple outputs
3626  // instructions that access memory and for ComplexPatterns that match
3627  // loads.
3628  if (EmitNodeInfo & OPFL_MemRefs) {
3629  // Only attach load or store memory operands if the generated
3630  // instruction may load or store.
3631  const MCInstrDesc &MCID = TII->get(TargetOpc);
3632  bool mayLoad = MCID.mayLoad();
3633  bool mayStore = MCID.mayStore();
3634 
3635  // We expect to have relatively few of these so just filter them into a
3636  // temporary buffer so that we can easily add them to the instruction.
3637  SmallVector<MachineMemOperand *, 4> FilteredMemRefs;
3638  for (MachineMemOperand *MMO : MatchedMemRefs) {
3639  if (MMO->isLoad()) {
3640  if (mayLoad)
3641  FilteredMemRefs.push_back(MMO);
3642  } else if (MMO->isStore()) {
3643  if (mayStore)
3644  FilteredMemRefs.push_back(MMO);
3645  } else {
3646  FilteredMemRefs.push_back(MMO);
3647  }
3648  }
3649 
3650  CurDAG->setNodeMemRefs(Res, FilteredMemRefs);
3651  }
3652 
3653  LLVM_DEBUG(if (!MatchedMemRefs.empty() && Res->memoperands_empty()) dbgs()
3654  << " Dropping mem operands\n";
3655  dbgs() << " " << (IsMorphNodeTo ? "Morphed" : "Created")
3656  << " node: ";
3657  Res->dump(CurDAG););
3658 
3659  // If this was a MorphNodeTo then we're completely done!
3660  if (IsMorphNodeTo) {
3661  // Update chain uses.
3662  UpdateChains(Res, InputChain, ChainNodesMatched, true);
3663  return;
3664  }
3665  continue;
3666  }
3667 
3668  case OPC_CompleteMatch: {
3669  // The match has been completed, and any new nodes (if any) have been
3670  // created. Patch up references to the matched dag to use the newly
3671  // created nodes.
3672  unsigned NumResults = MatcherTable[MatcherIndex++];
3673 
3674  for (unsigned i = 0; i != NumResults; ++i) {
3675  unsigned ResSlot = MatcherTable[MatcherIndex++];
3676  if (ResSlot & 128)
3677  ResSlot = GetVBR(ResSlot, MatcherTable, MatcherIndex);
3678 
3679  assert(ResSlot < RecordedNodes.size() && "Invalid CompleteMatch");
3680  SDValue Res = RecordedNodes[ResSlot].first;
3681 
3682  assert(i < NodeToMatch->getNumValues() &&
3683  NodeToMatch->getValueType(i) != MVT::Other &&
3684  NodeToMatch->getValueType(i) != MVT::Glue &&
3685  "Invalid number of results to complete!");
3686  assert((NodeToMatch->getValueType(i) == Res.getValueType() ||
3687  NodeToMatch->getValueType(i) == MVT::iPTR ||
3688  Res.getValueType() == MVT::iPTR ||
3689  NodeToMatch->getValueType(i).getSizeInBits() ==
3690  Res.getValueSizeInBits()) &&
3691  "invalid replacement");
3692  ReplaceUses(SDValue(NodeToMatch, i), Res);
3693  }
3694 
3695  // Update chain uses.
3696  UpdateChains(NodeToMatch, InputChain, ChainNodesMatched, false);
3697 
3698  // If the root node defines glue, we need to update it to the glue result.
3699  // TODO: This never happens in our tests and I think it can be removed /
3700  // replaced with an assert, but if we do it this the way the change is
3701  // NFC.
3702  if (NodeToMatch->getValueType(NodeToMatch->getNumValues() - 1) ==
3703  MVT::Glue &&
3704  InputGlue.getNode())
3705  ReplaceUses(SDValue(NodeToMatch, NodeToMatch->getNumValues() - 1),
3706  InputGlue);
3707 
3708  assert(NodeToMatch->use_empty() &&
3709  "Didn't replace all uses of the node?");
3710  CurDAG->RemoveDeadNode(NodeToMatch);
3711 
3712  return;
3713  }
3714  }
3715 
3716  // If the code reached this point, then the match failed. See if there is
3717  // another child to try in the current 'Scope', otherwise pop it until we
3718  // find a case to check.
3719  LLVM_DEBUG(dbgs() << " Match failed at index " << CurrentOpcodeIndex
3720  << "\n");
3721  ++NumDAGIselRetries;
3722  while (true) {
3723  if (MatchScopes.empty()) {
3724  CannotYetSelect(NodeToMatch);
3725  return;
3726  }
3727 
3728  // Restore the interpreter state back to the point where the scope was
3729  // formed.
3730  MatchScope &LastScope = MatchScopes.back();
3731  RecordedNodes.resize(LastScope.NumRecordedNodes);
3732  NodeStack.clear();
3733  NodeStack.append(LastScope.NodeStack.begin(), LastScope.NodeStack.end());
3734  N = NodeStack.back();
3735 
3736  if (LastScope.NumMatchedMemRefs != MatchedMemRefs.size())
3737  MatchedMemRefs.resize(LastScope.NumMatchedMemRefs);
3738  MatcherIndex = LastScope.FailIndex;
3739 
3740  LLVM_DEBUG(dbgs() << " Continuing at " << MatcherIndex << "\n");
3741 
3742  InputChain = LastScope.InputChain;
3743  InputGlue = LastScope.InputGlue;
3744  if (!LastScope.HasChainNodesMatched)
3745  ChainNodesMatched.clear();
3746 
3747  // Check to see what the offset is at the new MatcherIndex. If it is zero
3748  // we have reached the end of this scope, otherwise we have another child
3749  // in the current scope to try.
3750  unsigned NumToSkip = MatcherTable[MatcherIndex++];
3751  if (NumToSkip & 128)
3752  NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex);
3753 
3754  // If we have another child in this scope to match, update FailIndex and
3755  // try it.
3756  if (NumToSkip != 0) {
3757  LastScope.FailIndex = MatcherIndex+NumToSkip;
3758  break;
3759  }
3760 
3761  // End of this scope, pop it and try the next child in the containing
3762  // scope.
3763  MatchScopes.pop_back();
3764  }
3765  }
3766 }
3767 
3768 /// Return whether the node may raise an FP exception.
3770  // For machine opcodes, consult the MCID flag.
3771  if (N->isMachineOpcode()) {
3772  const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
3773  return MCID.mayRaiseFPException();
3774  }
3775 
3776  // For ISD opcodes, only StrictFP opcodes may raise an FP
3777  // exception.
3778  if (N->isTargetOpcode())
3779  return N->isTargetStrictFPOpcode();
3780  return N->isStrictFPOpcode();
3781 }
3782 
3784  assert(N->getOpcode() == ISD::OR && "Unexpected opcode");
3785  auto *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
3786  if (!C)
3787  return false;
3788 
3789  // Detect when "or" is used to add an offset to a stack object.
3790  if (auto *FN = dyn_cast<FrameIndexSDNode>(N->getOperand(0))) {
3791  MachineFrameInfo &MFI = MF->getFrameInfo();
3792  Align A = MFI.getObjectAlign(FN->getIndex());
3793  int32_t Off = C->getSExtValue();
3794  // If the alleged offset fits in the zero bits guaranteed by
3795  // the alignment, then this or is really an add.
3796  return (Off >= 0) && (((A.value() - 1) & Off) == unsigned(Off));
3797  }
3798  return false;
3799 }
3800 
3801 void SelectionDAGISel::CannotYetSelect(SDNode *N) {
3802  std::string msg;
3803  raw_string_ostream Msg(msg);
3804  Msg << "Cannot select: ";
3805 
3806  if (N->getOpcode() != ISD::INTRINSIC_W_CHAIN &&
3807  N->getOpcode() != ISD::INTRINSIC_WO_CHAIN &&
3808  N->getOpcode() != ISD::INTRINSIC_VOID) {
3809  N->printrFull(Msg, CurDAG);
3810  Msg << "\nIn function: " << MF->getName();
3811  } else {
3812  bool HasInputChain = N->getOperand(0).getValueType() == MVT::Other;
3813  unsigned iid =
3814  cast<ConstantSDNode>(N->getOperand(HasInputChain))->getZExtValue();
3815  if (iid < Intrinsic::num_intrinsics)
3816  Msg << "intrinsic %" << Intrinsic::getBaseName((Intrinsic::ID)iid);
3817  else if (const TargetIntrinsicInfo *TII = TM.getIntrinsicInfo())
3818  Msg << "target intrinsic %" << TII->getName(iid);
3819  else
3820  Msg << "unknown intrinsic #" << iid;
3821  }
3822  report_fatal_error(Twine(Msg.str()));
3823 }
llvm::DIExpression::prepend
static DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
Definition: DebugInfoMetadata.cpp:1632
llvm::FunctionLoweringInfo::Fn
const Function * Fn
Definition: FunctionLoweringInfo.h:54
llvm::Intrinsic::getBaseName
StringRef getBaseName(ID id)
Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....
Definition: Function.cpp:970
llvm::MachineInstr::isDebugValue
bool isDebugValue() const
Definition: MachineInstr.h:1268
llvm::SelectionDAGISel::OPC_MoveChild2
@ OPC_MoveChild2
Definition: SelectionDAGISel.h:124
i
i
Definition: README.txt:29
llvm::MachineFunction::hasInlineAsm
bool hasInlineAsm() const
Returns true if the function contains any inline assembly.
Definition: MachineFunction.h:745
ValueTypes.h
CheckSame
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * >> &RecordedNodes)
CheckSame - Implements OP_CheckSame.
Definition: SelectionDAGISel.cpp:2549
llvm::SelectionDAGISel::FastISelFailed
bool FastISelFailed
Definition: SelectionDAGISel.h:57
AssumptionCache.h
llvm::ConstantSDNode
Definition: SelectionDAGNodes.h:1586
llvm::SelectionDAGISel::OPC_CheckComplexPat
@ OPC_CheckComplexPat
Definition: SelectionDAGISel.h:146
llvm::BasicBlock::end
iterator end()
Definition: BasicBlock.h:316
llvm::ISD::STRICT_FSETCC
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:475
llvm::FunctionLoweringInfo::StaticAllocaMap
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
Definition: FunctionLoweringInfo.h:128
StackProtector.h
llvm::SelectionDAGISel::OPC_EmitInteger
@ OPC_EmitInteger
Definition: SelectionDAGISel.h:152
llvm::OptimizationRemarkMissed
Diagnostic information for missed-optimization remarks.
Definition: DiagnosticInfo.h:734
llvm::createDAGLinearizer
ScheduleDAGSDNodes * createDAGLinearizer(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createDAGLinearizer - This creates a "no-scheduling" scheduler which linearize the DAG using topologi...
Definition: ScheduleDAGFast.cpp:817
isFoldedOrDeadInstruction
static bool isFoldedOrDeadInstruction(const Instruction *I, const FunctionLoweringInfo &FuncInfo)
isFoldedOrDeadInstruction - Return true if the specified instruction is side-effect free and is eithe...
Definition: SelectionDAGISel.cpp:1297
llvm::MachineFunctionProperties::hasProperty
bool hasProperty(Property P) const
Definition: MachineFunction.h:193
llvm::ISD::INTRINSIC_VOID
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:199
llvm::SDUse
Represents a use of a SDNode.
Definition: SelectionDAGNodes.h:284
llvm::SelectionDAGISel::OPC_CheckChild2Integer
@ OPC_CheckChild2Integer
Definition: SelectionDAGISel.h:142
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:109
MachineInstr.h
llvm::SelectionDAGISel::TLI
const TargetLowering * TLI
Definition: SelectionDAGISel.h:56
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::SelectionDAGISel::getIncludePathForIndex
virtual StringRef getIncludePathForIndex(unsigned index)
getIncludePathForIndex - get the td source location of pattern instantiation
Definition: SelectionDAGISel.h:245
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
llvm::DiagnosticInfoISelFallback
Diagnostic information for ISel fallback path.
Definition: DiagnosticInfo.h:954
llvm::SelectionDAG::AssignTopologicalOrder
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
Definition: SelectionDAG.cpp:10736
llvm::FuncletPadInst::arg_size
unsigned arg_size() const
arg_size - Return the number of funcletpad arguments.
Definition: InstrTypes.h:2321
GCMetadata.h
llvm::SelectionDAGISel::TM
TargetMachine & TM
Definition: SelectionDAGISel.h:43
llvm::MCSymbol
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
llvm::ISD::PATCHPOINT
@ PATCHPOINT
Definition: ISDOpcodes.h:1303
llvm::SDNode::getValueType
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Definition: SelectionDAGNodes.h:986
UseMI
MachineInstrBuilder & UseMI
Definition: AArch64ExpandPseudoInsts.cpp:107
llvm::SelectionDAGISel::isOrEquivalentToAdd
bool isOrEquivalentToAdd(const SDNode *N) const
Definition: SelectionDAGISel.cpp:3783
llvm::ScheduleDAGSDNodes
ScheduleDAGSDNodes - A ScheduleDAG for scheduling SDNode-based DAGs.
Definition: ScheduleDAGSDNodes.h:46
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1106
llvm::SelectionDAGISel::OPC_EmitMergeInputChains1_2
@ OPC_EmitMergeInputChains1_2
Definition: SelectionDAGISel.h:160
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:114
llvm::ISD::OR
@ OR
Definition: ISDOpcodes.h:667
llvm::SelectionDAGISel::EnforceNodeIdInvariant
static void EnforceNodeIdInvariant(SDNode *N)
Definition: SelectionDAGISel.cpp:1048
llvm::TargetLowering::EmitInstrWithCustomInserter
virtual MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Definition: SelectionDAGISel.cpp:295
CheckChildSame
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * >> &RecordedNodes, unsigned ChildNo)
CheckChildSame - Implements OP_CheckChildXSame.
Definition: SelectionDAGISel.cpp:2558
llvm::Value::hasOneUse
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
llvm::MachineBasicBlock::getBasicBlock
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
Definition: MachineBasicBlock.h:213
llvm::SelectionDAGISel::CheckComplexPattern
virtual bool CheckComplexPattern(SDNode *Root, SDNode *Parent, SDValue N, unsigned PatternNo, SmallVectorImpl< std::pair< SDValue, SDNode * > > &Result)
Definition: SelectionDAGISel.h:288
llvm::FunctionVarLocs
Data structure describing the variable locations in a function.
Definition: AssignmentTrackingAnalysis.h:30
Metadata.h
llvm::ISD::LIFETIME_END
@ LIFETIME_END
Definition: ISDOpcodes.h:1229
llvm::SelectionDAGISel::OPC_RecordChild7
@ OPC_RecordChild7
Definition: SelectionDAGISel.h:120
llvm::SelectionDAGISel::PostprocessISelDAG
virtual void PostprocessISelDAG()
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
Definition: SelectionDAGISel.h:82
llvm::TargetLoweringBase::isStrictFPEnabled
bool isStrictFPEnabled() const
Return true if the target support strict float operation.
Definition: TargetLowering.h:341
PHI
Rewrite undef for PHI
Definition: AMDGPURewriteUndefForPHI.cpp:101
llvm::M68kBeads::Term
@ Term
Definition: M68kBaseInfo.h:90
IntrinsicInst.h
computeUsesMSVCFloatingPoint
static void computeUsesMSVCFloatingPoint(const Triple &TT, const Function &F, MachineModuleInfo &MMI)
Definition: SelectionDAGISel.cpp:357
hasExceptionPointerOrCodeUser
static bool hasExceptionPointerOrCodeUser(const CatchPadInst *CPI)
Definition: SelectionDAGISel.cpp:1187
DebugInfoMetadata.h
llvm::AssignmentTrackingAnalysis
Definition: AssignmentTrackingAnalysis.h:97
llvm::ISD::AssertSext
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
llvm::SelectionDAG::getCopyToReg
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:769
MCInstrDesc.h
llvm::SelectionDAGISel::DAGSize
unsigned DAGSize
DAGSize - Size of DAG being instruction selected.
Definition: SelectionDAGISel.h:203
EnableFastISelAbort
static cl::opt< int > EnableFastISelAbort("fast-isel-abort", cl::Hidden, cl::desc("Enable abort calls when \"fast\" instruction selection " "fails to lower an instruction: 0 disable the abort, 1 will " "abort but for args, calls and terminators, 2 will also " "abort for argument lowering, and 3 will never fallback " "to SelectionDAG."))
InstIterator.h
MachinePassRegistry.h
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:51
T
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:159
llvm::SelectionDAG::VerifyDAGDivergence
void VerifyDAGDivergence()
Definition: SelectionDAG.cpp:10655
llvm::SelectionDAGISel::OPC_EmitNode
@ OPC_EmitNode
Definition: SelectionDAGISel.h:164
llvm::Function
Definition: Function.h:59
llvm::ISD::ConstantFP
@ ConstantFP
Definition: ISDOpcodes.h:77
StringRef.h
llvm::ISD::LIFETIME_START
@ LIFETIME_START
This corresponds to the llvm.lifetime.
Definition: ISDOpcodes.h:1228
P
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Definition: README-SSE.txt:411
Pass.h
SchedulerRegistry.h
llvm::TargetSubtargetInfo::getInstrInfo
virtual const TargetInstrInfo * getInstrInfo() const
Definition: TargetSubtargetInfo.h:95
llvm::ISD::STRICT_UINT_TO_FP
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:449
llvm::SelectionDAGISel::SelectInlineAsmMemoryOperands
void SelectInlineAsmMemoryOperands(std::vector< SDValue > &Ops, const SDLoc &DL)
SelectInlineAsmMemoryOperands - Calls to this are automatically generated by tblgen.
Definition: SelectionDAGISel.cpp:1991
AssignmentTrackingAnalysis.h
llvm::InlineAsm::Op_MDNode
@ Op_MDNode
Definition: InlineAsm.h:220
llvm::StackProtector::copyToMachineFrameInfo
void copyToMachineFrameInfo(MachineFrameInfo &MFI) const
Definition: StackProtector.cpp:617
llvm::raw_string_ostream
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:642
llvm::SDNode::op_values
iterator_range< value_op_iterator > op_values() const
Definition: SelectionDAGNodes.h:944
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1199
llvm::TargetMachine::getO0WantsFastISel
bool getO0WantsFastISel()
Definition: TargetMachine.h:255
Statistic.h
llvm::isAssignmentTrackingEnabled
bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
Definition: DebugInfo.cpp:2033
llvm::MachineFunction::useDebugInstrRef
bool useDebugInstrRef() const
Returns true if the function's variable locations are tracked with instruction referencing.
Definition: MachineFunction.cpp:1216
InlineAsm.h
CheckType
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
Definition: SelectionDAGISel.cpp:2591
llvm::SelectionDAG::getVTList
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
Definition: SelectionDAG.cpp:9478
llvm::FastISel::finishBasicBlock
void finishBasicBlock()
Flush the local value map.
Definition: FastISel.cpp:136
llvm::printMBBReference
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Definition: MachineBasicBlock.cpp:118
llvm::FunctionPass::skipFunction
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Definition: Pass.cpp:174
llvm::Function::getSubprogram
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1625
llvm::MachineSDNode
An SDNode that represents everything that will be needed to construct a MachineInstr.
Definition: SelectionDAGNodes.h:2901
llvm::createDefaultScheduler
ScheduleDAGSDNodes * createDefaultScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createDefaultScheduler - This creates an instruction scheduler appropriate for the target.
Definition: SelectionDAGISel.cpp:254
ErrorHandling.h
llvm::FastISel::lowerArguments
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
Definition: FastISel.cpp:138
llvm::TargetTransformInfo
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Definition: TargetTransformInfo.h:172
LazyBlockFrequencyInfo.h
llvm::SelectionDAGISel::OPC_CheckOrImm
@ OPC_CheckOrImm
Definition: SelectionDAGISel.h:147
llvm::SelectionDAGISel::OPC_RecordChild3
@ OPC_RecordChild3
Definition: SelectionDAGISel.h:119
llvm::Function::getEntryBlock
const BasicBlock & getEntryBlock() const
Definition: Function.h:735
llvm::Sched::ILP
@ ILP
Definition: TargetLowering.h:103
llvm::SelectionDAG::allnodes_begin
allnodes_const_iterator allnodes_begin() const
Definition: SelectionDAG.h:527
llvm::SelectionDAG::getRoot
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:547
llvm::SelectionDAGISel::AC
AssumptionCache * AC
Definition: SelectionDAGISel.h:52
llvm::SelectionDAGISel::OPC_EmitNode1
@ OPC_EmitNode1
Definition: SelectionDAGISel.h:166
llvm::X86Disassembler::Reg
Reg
All possible values of the reg field in the ModR/M byte.
Definition: X86DisassemblerDecoder.h:462
llvm::SwiftErrorValueTracking::createEntriesInEntryBlock
bool createEntriesInEntryBlock(DebugLoc DbgLoc)
Create initial definitions of swifterror values in the entry block of the current function.
Definition: SwiftErrorValueTracking.cpp:115
llvm::SwiftErrorValueTracking::propagateVRegs
void propagateVRegs()
Propagate assigned swifterror vregs through a function, synthesizing PHI nodes when needed to maintai...
Definition: SwiftErrorValueTracking.cpp:147
llvm::ISD::EH_LABEL
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:1032
llvm::SelectionDAGISel::OPC_CheckChild1Integer
@ OPC_CheckChild1Integer
Definition: SelectionDAGISel.h:142
llvm::SelectionDAGISel::OPC_CheckFoldableChainNode
@ OPC_CheckFoldableChainNode
Definition: SelectionDAGISel.h:150
llvm::MachineRegisterInfo::defusechain_instr_iterator
defusechain_iterator - This class provides iterator support for machine operands in the function that...
Definition: MachineRegisterInfo.h:291
llvm::HandleSDNode
This class is used to form a handle around another node that is persistent and is updated across invo...
Definition: SelectionDAGNodes.h:1233
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
llvm::createFastDAGScheduler
ScheduleDAGSDNodes * createFastDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createFastDAGScheduler - This creates a "fast" scheduler.
Definition: ScheduleDAGFast.cpp:812
llvm::SelectionDAGISel::OPC_CheckChild1Same
@ OPC_CheckChild1Same
Definition: SelectionDAGISel.h:128
OptimizationRemarkEmitter.h
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:463
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
MachineBasicBlock.h
llvm::InlineAsm::Op_ExtraInfo
@ Op_ExtraInfo
Definition: InlineAsm.h:221
llvm::TargetSubtargetInfo::getRegisterInfo
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Definition: TargetSubtargetInfo.h:127
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:138
processDbgDeclare
static void processDbgDeclare(FunctionLoweringInfo &FuncInfo, const Value *Address, DIExpression *Expr, DILocalVariable *Var, DebugLoc DbgLoc)
Definition: SelectionDAGISel.cpp:1306
llvm::MVT::Glue
@ Glue
Definition: MachineValueType.h:282
llvm::SelectionDAGISel::OPC_MorphNodeTo2
@ OPC_MorphNodeTo2
Definition: SelectionDAGISel.h:169
llvm::SelectionDAGISel::OPC_MoveChild7
@ OPC_MoveChild7
Definition: SelectionDAGISel.h:125
llvm::SDNode::use_iterator
This class provides iterator support for SDUse operands that use a specific SDNode.
Definition: SelectionDAGNodes.h:745
APInt.h
llvm::TargetRegisterInfo
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Definition: TargetRegisterInfo.h:236
llvm::SelectionDAGISel::CheckPatternPredicate
virtual bool CheckPatternPredicate(unsigned PredNo) const
CheckPatternPredicate - This function is generated by tblgen in the target.
Definition: SelectionDAGISel.h:265
llvm::LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage
static void getLazyBFIAnalysisUsage(AnalysisUsage &AU)
Helper for client passes to set up the analysis usage on behalf of this pass.
Definition: LazyBlockFrequencyInfo.cpp:62
llvm::Function::getContext
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:315
llvm::DenseMapIterator
Definition: DenseMap.h:57
Shift
bool Shift
Definition: README.txt:468
llvm::InlineAsm::isUseOperandTiedToDef
static bool isUseOperandTiedToDef(unsigned Flag, unsigned &Idx)
isUseOperandTiedToDef - Return true if the flag of the inline asm operand indicates it is an use oper...
Definition: InlineAsm.h:369
ViewSchedDAGs
static cl::opt< bool > ViewSchedDAGs("view-sched-dags", cl::Hidden, cl::desc("Pop up a window to show sched dags as they are processed"))
DenseMap.h
llvm::TargetLowering::initializeSplitCSR
virtual void initializeSplitCSR(MachineBasicBlock *Entry) const
Perform necessary initialization to handle a subset of CSRs explicitly via copies.
Definition: TargetLowering.h:4058
llvm::TimePassesIsEnabled
bool TimePassesIsEnabled
If the user specifies the -time-passes argument on an LLVM tool command line then the value of this b...
Definition: PassTimingInfo.cpp:37
CheckChildInteger
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, unsigned ChildNo)
Definition: SelectionDAGISel.cpp:2661
llvm::SelectionDAGISel::OPC_CaptureGlueInput
@ OPC_CaptureGlueInput
Definition: SelectionDAGISel.h:122
llvm::SelectionDAGISel::OPC_EmitNodeXForm
@ OPC_EmitNodeXForm
Definition: SelectionDAGISel.h:163
TargetInstrInfo.h
llvm::MachineModuleInfo::setUsesMSVCFloatingPoint
void setUsesMSVCFloatingPoint(bool b)
Definition: MachineModuleInfo.h:182
llvm::MachineMemOperand
A description of a memory reference used in the backend.
Definition: MachineMemOperand.h:127
llvm::FunctionLoweringInfo::isExportedInst
bool isExportedInst(const Value *V) const
isExportedInst - Return true if the specified value is an instruction exported from its block.
Definition: FunctionLoweringInfo.h:198
EHPersonalities.h
llvm::TargetLoweringBase::getExceptionPointerRegister
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
Definition: TargetLowering.h:1831
llvm::RegisterScheduler::Registry
static MachinePassRegistry< FunctionPassCtor > Registry
RegisterScheduler class - Track the registration of instruction schedulers.
Definition: SchedulerRegistry.h:38
ViewDAGCombine1
static cl::opt< bool > ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the first " "dag combine pass"))
llvm::MachineFunction::setCallSiteLandingPad
void setCallSiteLandingPad(MCSymbol *Sym, ArrayRef< unsigned > Sites)
Map the landing pad's EH symbol to the call site indexes.
Definition: MachineFunction.cpp:809
llvm::SelectionDAGISel::OPC_CheckChild0Same
@ OPC_CheckChild0Same
Definition: SelectionDAGISel.h:128
llvm::SelectionDAGISel::OPFL_GlueInput
@ OPFL_GlueInput
Definition: SelectionDAGISel.h:178
LLVM_ATTRIBUTE_ALWAYS_INLINE
#define LLVM_ATTRIBUTE_ALWAYS_INLINE
LLVM_ATTRIBUTE_ALWAYS_INLINE - On compilers where we have a directive to do so, mark a method "always...
Definition: Compiler.h:230
llvm::MachineInstr::isCopy
bool isCopy() const
Definition: MachineInstr.h:1344
llvm::SelectionDAGISel::OPC_CheckPredicate
@ OPC_CheckPredicate
Definition: SelectionDAGISel.h:131
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:450
llvm::ore::NV
DiagnosticInfoOptimizationBase::Argument NV
Definition: OptimizationRemarkEmitter.h:136
llvm::SelectionDAGISel::OPC_Coverage
@ OPC_Coverage
Definition: SelectionDAGISel.h:172
llvm::ISD::isConstantSplatVectorAllZeros
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
Definition: SelectionDAG.cpp:220
llvm::max
Expected< ExpressionValue > max(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:337
llvm::TargetLowering::supportSplitCSR
virtual bool supportSplitCSR(MachineFunction *MF) const
Return true if the target supports that a subset of CSRs for the given machine function is handled ex...
Definition: TargetLowering.h:4048
llvm::SelectionDAG::RemoveDeadNodes
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
Definition: SelectionDAG.cpp:943
STLExtras.h
llvm::DIExpression
DWARF expression.
Definition: DebugInfoMetadata.h:2611
llvm::AfterLegalizeTypes
@ AfterLegalizeTypes
Definition: DAGCombine.h:17
RHS
Value * RHS
Definition: X86PartialReduction.cpp:76
llvm::SelectionDAGISel::OPC_CheckAndImm
@ OPC_CheckAndImm
Definition: SelectionDAGISel.h:147
llvm::SelectionDAGISel::OPC_EmitCopyToReg
@ OPC_EmitCopyToReg
Definition: SelectionDAGISel.h:161
llvm::SelectionDAGISel::OPC_SwitchType
@ OPC_SwitchType
Definition: SelectionDAGISel.h:137
llvm::Sched::Fast
@ Fast
Definition: TargetLowering.h:105
llvm::succ_empty
bool succ_empty(const Instruction *I)
Definition: CFG.h:255
llvm::SelectionDAGISel::OPC_CheckChild0Type
@ OPC_CheckChild0Type
Definition: SelectionDAGISel.h:138
replace
static void replace(Module &M, GlobalVariable *Old, GlobalVariable *New)
Definition: ConstantMerge.cpp:116
llvm::Sched::Linearize
@ Linearize
Definition: TargetLowering.h:106
llvm::SelectionDAGISel::OPC_CheckCondCode
@ OPC_CheckCondCode
Definition: SelectionDAGISel.h:144
SelectionDAG.h
llvm::SelectionDAGISel::OPC_CheckChild7Type
@ OPC_CheckChild7Type
Definition: SelectionDAGISel.h:140
llvm::initializeGCModuleInfoPass
void initializeGCModuleInfoPass(PassRegistry &)
processDbgDeclares
static void processDbgDeclares(FunctionLoweringInfo &FuncInfo)
Collect llvm.dbg.declare information.
Definition: SelectionDAGISel.cpp:1346
llvm::ISD::FREEZE
@ FREEZE
Definition: ISDOpcodes.h:216
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1628
llvm::SelectionDAG::clear
void clear()
Clear state and free memory necessary to make this SelectionDAG ready to process a new block.
Definition: SelectionDAG.cpp:1379
llvm::SelectionDAGISel::OPFL_VariadicInfo
@ OPFL_VariadicInfo
Definition: SelectionDAGISel.h:189
FilterDAGBasicBlockName
static cl::opt< std::string > FilterDAGBasicBlockName("filter-view-dags", cl::Hidden, cl::desc("Only display the basic block whose name " "matches this for all view-*-dags options"))
llvm::TargetLoweringBase::getOperationAction
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
Definition: TargetLowering.h:1113
llvm::initializeTargetLibraryInfoWrapperPassPass
void initializeTargetLibraryInfoWrapperPassPass(PassRegistry &)
llvm::SelectionDAGISel::OPC_CheckChild1Type
@ OPC_CheckChild1Type
Definition: SelectionDAGISel.h:138
llvm::SelectionDAGISel::OPC_EmitStringInteger
@ OPC_EmitStringInteger
Definition: SelectionDAGISel.h:153
llvm::MachineFunctionPass::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition: MachineFunctionPass.cpp:167
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
new
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM ID Predecessors according to mbb< bb27, 0x8b0a7c0 > Note ADDri is not a two address instruction its result reg1037 is an operand of the PHI node in bb76 and its operand reg1039 is the result of the PHI node We should treat it as a two address code and make sure the ADDri is scheduled after any node that reads reg1039 Use info(i.e. register scavenger) to assign it a free register to allow reuse the collector could move the objects and invalidate the derived pointer This is bad enough in the first but safe points can crop up unpredictably **array_addr i32 n y store obj * new
Definition: README.txt:125
F
#define F(x, y, z)
Definition: MD5.cpp:55
llvm::RISCVFenceField::R
@ R
Definition: RISCVBaseInfo.h:275
MachineRegisterInfo.h
KnownBits.h
llvm::SelectionDAGISel::PreprocessISelDAG
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
Definition: SelectionDAGISel.h:78
llvm::ISD::ARITH_FENCE
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1149
llvm::SelectionDAG::getRegister
SDValue getRegister(unsigned Reg, EVT VT)
Definition: SelectionDAG.cpp:2135
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:55
llvm::ISD::INLINEASM
@ INLINEASM
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:1024
llvm::EVT::isSimple
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:129
llvm::ISD::PSEUDO_PROBE
@ PSEUDO_PROBE
Pseudo probe for AutoFDO, as a place holder in a basic block to improve the sample counts quality.
Definition: ISDOpcodes.h:1248
llvm::TargetIntrinsicInfo
TargetIntrinsicInfo - Interface to description of machine instruction set.
Definition: TargetIntrinsicInfo.h:29
mapWasmLandingPadIndex
static void mapWasmLandingPadIndex(MachineBasicBlock *MBB, const CatchPadInst *CPI)
Definition: SelectionDAGISel.cpp:1202
CheckAndImm
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
Definition: SelectionDAGISel.cpp:2669
AliasAnalysis.h
UseMBPI
static cl::opt< bool > UseMBPI("use-mbpi", cl::desc("use Machine Branch Probability Info"), cl::init(true), cl::Hidden)
MachineValueType.h
llvm::MVT::SimpleValueType
SimpleValueType
Definition: MachineValueType.h:33
llvm::MachineFunction::setExposesReturnsTwice
void setExposesReturnsTwice(bool B)
setCallsSetJmp - Set a flag that indicates if there's a call to a "returns twice" function.
Definition: MachineFunction.h:740
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::classifyEHPersonality
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
Definition: EHPersonalities.cpp:22
llvm::MachineFunction::setWasmLandingPadIndex
void setWasmLandingPadIndex(const MachineBasicBlock *LPad, unsigned Index)
Map the landing pad to its index. Used for Wasm exception handling.
Definition: MachineFunction.h:1152
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:187
llvm::TargetLowering::AdjustInstrPostInstrSelection
virtual void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
Definition: SelectionDAGISel.cpp:305
llvm::SelectionDAGISel::OPC_EmitMergeInputChains1_0
@ OPC_EmitMergeInputChains1_0
Definition: SelectionDAGISel.h:158
llvm::SelectionDAGISel::ElidedArgCopyInstrs
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
Definition: SelectionDAGISel.h:58
llvm::SelectionDAGISel::OPC_MoveChild5
@ OPC_MoveChild5
Definition: SelectionDAGISel.h:125
Instruction.h
CommandLine.h
llvm::BeforeLegalizeTypes
@ BeforeLegalizeTypes
Definition: DAGCombine.h:16
LHS
Value * LHS
Definition: X86PartialReduction.cpp:75
TargetLowering.h
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:78
llvm::FuncletPadInst::getArgOperand
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th funcletpad argument.
Definition: InstrTypes.h:2337
llvm::SDNode::getOpcode
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
Definition: SelectionDAGNodes.h:644
llvm::MachineFunction::front
const MachineBasicBlock & front() const
Definition: MachineFunction.h:882
llvm::BlockFrequencyInfo
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Definition: BlockFrequencyInfo.h:37
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:682
reportFastISelFailure
static void reportFastISelFailure(MachineFunction &MF, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R, bool ShouldAbort)
Definition: SelectionDAGISel.cpp:663
llvm::SelectionDAGISel::OPC_RecordChild6
@ OPC_RecordChild6
Definition: SelectionDAGISel.h:120
llvm::MachineBasicBlock::insertAfter
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
Definition: MachineBasicBlock.h:958
llvm::SelectionDAGISel::OPC_RecordChild1
@ OPC_RecordChild1
Definition: SelectionDAGISel.h:119
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1141
ViewDAGCombineLT
static cl::opt< bool > ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the post " "legalize types dag combine pass"))
CheckValueType
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
Definition: SelectionDAGISel.cpp:2626
llvm::MachineFunctionProperties::Property::Selected
@ Selected
TargetMachine.h
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:24
CheckOrImm
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
Definition: SelectionDAGISel.cpp:2682
llvm::SelectionDAGISel::OPC_EmitNode0
@ OPC_EmitNode0
Definition: SelectionDAGISel.h:166
llvm::SelectionDAG
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:221
SelectionDAGNodes.h
Constants.h
llvm::SelectionDAGISel::RunSDNodeXForm
virtual SDValue RunSDNodeXForm(SDValue V, unsigned XFormNo)
Definition: SelectionDAGISel.h:294
llvm::ISD::Constant
@ Constant
Definition: ISDOpcodes.h:76
llvm::SelectionDAGISel::SDB
std::unique_ptr< SelectionDAGBuilder > SDB
Definition: SelectionDAGISel.h:50
llvm::SelectionDAGISel::OPC_CheckOpcode
@ OPC_CheckOpcode
Definition: SelectionDAGISel.h:133
llvm::SelectionDAGISel::OPC_CheckPatternPredicate
@ OPC_CheckPatternPredicate
Definition: SelectionDAGISel.h:130
llvm::DILocalVariable
Local variable.
Definition: DebugInfoMetadata.h:3162
llvm::BranchProbabilityInfoWrapperPass
Legacy analysis pass which computes BranchProbabilityInfo.
Definition: BranchProbabilityInfo.h:439
FunctionLoweringInfo.h
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::SelectionDAG::LegalizeTypes
bool LegalizeTypes()
This transforms the SelectionDAG into a SelectionDAG that only uses types natively supported by the t...
Definition: LegalizeTypes.cpp:1063
llvm::DIExpression::ApplyOffset
@ ApplyOffset
Definition: DebugInfoMetadata.h:2867
llvm::User
Definition: User.h:44
llvm::SelectionDAGISel::OPC_MoveChild
@ OPC_MoveChild
Definition: SelectionDAGISel.h:123
llvm::ISD::CopyToReg
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition: ISDOpcodes.h:203
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:34
llvm::SelectionDAGISel::RegInfo
MachineRegisterInfo * RegInfo
Definition: SelectionDAGISel.h:48
Intrinsics.h
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::MachineInstr::getOperand
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:526
llvm::SelectionDAGISel::SelectionDAGISel
SelectionDAGISel(char &ID, TargetMachine &tm, CodeGenOpt::Level OL=CodeGenOpt::Default)
Definition: SelectionDAGISel.cpp:316
llvm::SelectionDAGISel::OptLevel
CodeGenOpt::Level OptLevel
Definition: SelectionDAGISel.h:54
llvm::SelectionDAGISel::CheckNodePredicate
virtual bool CheckNodePredicate(SDNode *N, unsigned PredNo) const
CheckNodePredicate - This function is generated by tblgen in the target.
Definition: SelectionDAGISel.h:273
llvm::getLLTForMVT
LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
Definition: LowLevelType.cpp:64
llvm::TargetMachine::getIntrinsicInfo
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
Definition: TargetMachine.h:220
llvm::TargetLowering
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Definition: TargetLowering.h:3510
ViewLegalizeTypesDAGs
static cl::opt< bool > ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize types"))
llvm::KnownBits::One
APInt One
Definition: KnownBits.h:25
llvm::SelectionDAGISel::ComplexPatternFuncMutatesDAG
virtual bool ComplexPatternFuncMutatesDAG() const
Return true if complex patterns for this target can mutate the DAG.
Definition: SelectionDAGISel.h:303
llvm::TargetTransformInfo::hasBranchDivergence
bool hasBranchDivergence() const
Return true if branch divergence exists.
Definition: TargetTransformInfo.cpp:238
llvm::MachineBasicBlock::isSuccessor
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
Definition: MachineBasicBlock.cpp:934
llvm::StackProtector
Definition: StackProtector.h:37
llvm::SelectionDAG::MaskedValueIsZero
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
Definition: SelectionDAG.cpp:2547
llvm::createILPListDAGScheduler
ScheduleDAGSDNodes * createILPListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level)
createILPListDAGScheduler - This creates a bottom up register pressure aware list scheduler that trie...
Definition: ScheduleDAGRRList.cpp:3198
SI
@ SI
Definition: SIInstrInfo.cpp:7993
llvm::TargetRegisterClass
Definition: TargetRegisterInfo.h:45
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
llvm::SelectionDAG::Legalize
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
Definition: LegalizeDAG.cpp:5096
ViewLegalizeDAGs
static cl::opt< bool > ViewLegalizeDAGs("view-legalize-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize"))
CheckInteger
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
Definition: SelectionDAGISel.cpp:2648
llvm::MachineFunction::getProperties
const MachineFunctionProperties & getProperties() const
Get the function properties.
Definition: MachineFunction.h:763
TargetLibraryInfo.h
llvm::BitTracker
Definition: BitTracker.h:35
llvm::SelectionDAGISel::ReplaceNode
void ReplaceNode(SDNode *F, SDNode *T)
Replace all uses of F with T, then remove F from the DAG.
Definition: SelectionDAGISel.h:228
llvm::SelectionDAGISel::ORE
std::unique_ptr< OptimizationRemarkEmitter > ORE
Current optimization remark emitter.
Definition: SelectionDAGISel.h:62
llvm::SelectionDAGISel::OPC_MorphNodeTo1
@ OPC_MorphNodeTo1
Definition: SelectionDAGISel.h:169
llvm::dwarf::Index
Index
Definition: Dwarf.h:550
llvm::ISD::DELETED_NODE
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:44
llvm::MCInstrDesc
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:198
CheckCondCode
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
Definition: SelectionDAGISel.cpp:2611
llvm::SelectionDAG::setRoot
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:556
llvm::EVT::isInteger
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:144
llvm::Instruction
Definition: Instruction.h:41
llvm::InlineAsm::Op_InputChain
@ Op_InputChain
Definition: InlineAsm.h:218
llvm::MachineModuleInfo
This class contains meta information specific to a module.
Definition: MachineModuleInfo.h:74
llvm::ISD::MDNODE_SDNODE
@ MDNODE_SDNODE
MDNODE_SDNODE - This is a node that holdes an MDNode*, which is used to reference metadata in the IR.
Definition: ISDOpcodes.h:1101
llvm::createSourceListDAGScheduler
ScheduleDAGSDNodes * createSourceListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createBURRListDAGScheduler - This creates a bottom up list scheduler that schedules nodes in source c...
Definition: ScheduleDAGRRList.cpp:3168
llvm::SDNode::use_empty
bool use_empty() const
Return true if there are no uses of this node.
Definition: SelectionDAGNodes.h:715
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
llvm::BasicBlock::phis
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:372
llvm::STATISTIC
STATISTIC(NumFunctions, "Total number of functions")
decodeSignRotatedValue
static uint64_t decodeSignRotatedValue(uint64_t V)
Decode a signed value stored with the sign bit in the LSB for dense VBR encoding.
Definition: SelectionDAGISel.cpp:2638
CheckPatternPredicate
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel)
CheckPatternPredicate - Implements OP_CheckPatternPredicate.
Definition: SelectionDAGISel.cpp:2570
llvm::SelectionDAG::DAGNodeDeletedListener
Definition: SelectionDAG.h:329
ScheduleDAGSDNodes.h
llvm::ConstantFP
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:256
llvm::SmallVectorImpl::resize
void resize(size_type N)
Definition: SmallVector.h:642
llvm::Use::getUser
User * getUser() const
Returns the User that contains this Use.
Definition: Use.h:72
llvm::StringRef::data
const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:131
llvm::SelectionDAG::dump
void dump() const
Definition: SelectionDAGDumper.cpp:930
llvm::ISD::AssertAlign
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
Definition: ISDOpcodes.h:68
DebugLoc.h
SmallPtrSet.h
llvm::SwiftErrorValueTracking::preassignVRegs
void preassignVRegs(MachineBasicBlock *MBB, BasicBlock::const_iterator Begin, BasicBlock::const_iterator End)
Definition: SwiftErrorValueTracking.cpp:258
llvm::RegisterScheduler::FunctionPassCtor
ScheduleDAGSDNodes *(*)(SelectionDAGISel *, CodeGenOpt::Level) FunctionPassCtor
Definition: SchedulerRegistry.h:36
llvm::ISD::AND
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:666
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:687
llvm::ISD::TargetGlobalAddress
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
Definition: ISDOpcodes.h:164
llvm::BasicBlock::getFirstNonPHI
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:208
llvm::ISD::EntryToken
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:47
CheckNodePredicate
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel, SDNode *N)
CheckNodePredicate - Implements OP_CheckNodePredicate.
Definition: SelectionDAGISel.cpp:2577
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
findNonImmUse
static bool findNonImmUse(SDNode *Root, SDNode *Def, SDNode *ImmedUse, bool IgnoreChains)
findNonImmUse - Return true if "Def" is a predecessor of "Root" via a path beyond "ImmedUse".
Definition: SelectionDAGISel.cpp:2066
llvm::SelectionDAGISel::OPC_CheckSame
@ OPC_CheckSame
Definition: SelectionDAGISel.h:127
llvm::MachineRegisterInfo::getVRegDef
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
Definition: MachineRegisterInfo.cpp:398
ViewISelDAGs
static cl::opt< bool > ViewISelDAGs("view-isel-dags", cl::Hidden, cl::desc("Pop up a window to show isel dags as they are selected"))
llvm::SDValue::getValueSizeInBits
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
Definition: SelectionDAGNodes.h:199
llvm::ISD::CopyFromReg
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:208
llvm::RegisterPassParser
RegisterPassParser class - Handle the addition of new machine passes.
Definition: MachinePassRegistry.h:135
llvm::EHPersonality::Wasm_CXX
@ Wasm_CXX
llvm::MachineRegisterInfo::use_empty
bool use_empty(Register RegNo) const
use_empty - Return true if there are no instructions using the specified register.
Definition: MachineRegisterInfo.h:528
Statepoint.h
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm::erase_value
void erase_value(Container &C, ValueType V)
Wrapper function to remove a value from a container:
Definition: STLExtras.h:2006
Type.h
BranchProbability.h
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:94
llvm::TargetMachine::resetTargetOptions
void resetTargetOptions(const Function &F) const
Reset the target options based on the function's attributes.
Definition: TargetMachine.cpp:53
processSingleLocVars
static void processSingleLocVars(FunctionLoweringInfo &FuncInfo, FunctionVarLocs const *FnVarLocs)
Collect single location variable information generated with assignment tracking.
Definition: SelectionDAGISel.cpp:1366
llvm::Sched::RegPressure
@ RegPressure
Definition: TargetLowering.h:101
ViewSUnitDAGs
static cl::opt< bool > ViewSUnitDAGs("view-sunit-dags", cl::Hidden, cl::desc("Pop up a window to show SUnit dags after they are processed"))
llvm::InlineAsm::isMemKind
static bool isMemKind(unsigned Flag)
Definition: InlineAsm.h:301
llvm::OptLevelChanger::~OptLevelChanger
~OptLevelChanger()
Definition: SelectionDAGISel.cpp:238
llvm::TargetLowering::createFastISel
virtual FastISel * createFastISel(FunctionLoweringInfo &, const TargetLibraryInfo *) const
This method returns a target specific FastISel object, or null if the target does not support "fast" ...
Definition: TargetLowering.h:4572
Operands
mir Rename Register Operands
Definition: MIRNamerPass.cpp:74
llvm::MachineRegisterInfo::getRegClass
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
Definition: MachineRegisterInfo.h:661
llvm::MCID::MayRaiseFPException
@ MayRaiseFPException
Definition: MCInstrDesc.h:170
llvm::ISD::TargetConstantFP
@ TargetConstantFP
Definition: ISDOpcodes.h:159
llvm::predecessors
auto predecessors(const MachineBasicBlock *BB)
Definition: MachineSSAContext.h:30
llvm::SelectionDAGISel::InvalidateNodeId
static void InvalidateNodeId(SDNode *N)
Definition: SelectionDAGISel.cpp:1067
llvm::InlineAsm::Op_AsmString
@ Op_AsmString
Definition: InlineAsm.h:219
llvm::APInt::isSubsetOf
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1235
llvm::MDNode::getOperand
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1291
llvm::TargetLowering::insertCopiesSplitCSR
virtual void insertCopiesSplitCSR(MachineBasicBlock *Entry, const SmallVectorImpl< MachineBasicBlock * > &Exits) const
Insert explicit copies in entry and exit blocks.
Definition: TargetLowering.h:4066
llvm::MachineFunction::getMMI
MachineModuleInfo & getMMI() const
Definition: MachineFunction.h:623
llvm::Register::isVirtual
bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:91
llvm::ISD::STRICT_FSETCCS
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:476
llvm::MachineRegisterInfo::clearKillFlags
void clearKillFlags(Register Reg) const
clearKillFlags - Iterate over all the uses of the given register and clear the kill flag from the Mac...
Definition: MachineRegisterInfo.cpp:435
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:672
llvm::MCInstrDesc::mayLoad
bool mayLoad() const
Return true if this instruction could possibly read memory.
Definition: MCInstrDesc.h:437
llvm::ISD::WRITE_REGISTER
@ WRITE_REGISTER
Definition: ISDOpcodes.h:119
Timer.h
BasicBlock.h
llvm::FunctionVarLocs::single_locs_end
const VarLocInfo * single_locs_end() const
One past the last single-location variable location definition.
Definition: AssignmentTrackingAnalysis.h:65
llvm::cl::opt
Definition: CommandLine.h:1410
llvm::MachineFunction::setHasInlineAsm
void setHasInlineAsm(bool B)
Set a flag that indicates that the function contains inline assembly.
Definition: MachineFunction.h:750
llvm::SDNode::use_begin
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
Definition: SelectionDAGNodes.h:800
llvm::ISD::Register
@ Register
Definition: ISDOpcodes.h:74
llvm::SelectionDAGISel::OPC_CheckChild4Integer
@ OPC_CheckChild4Integer
Definition: SelectionDAGISel.h:143
llvm::instructions
inst_range instructions(Function *F)
Definition: InstIterator.h:133
llvm::FastISel::selectInstruction
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
Definition: FastISel.cpp:1478
llvm::SelectionDAGISel::OPC_CheckValueType
@ OPC_CheckValueType
Definition: SelectionDAGISel.h:145
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::SelectionDAG::RemoveDeadNode
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
Definition: SelectionDAG.cpp:997
llvm::SelectionDAGISel::OPC_CheckImmAllZerosV
@ OPC_CheckImmAllZerosV
Definition: SelectionDAGISel.h:149
BranchProbabilityInfo.h
llvm::SelectionDAGISel::OPC_CheckInteger
@ OPC_CheckInteger
Definition: SelectionDAGISel.h:141
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:340
llvm::SelectionDAGISel::IsProfitableToFold
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
Definition: SelectionDAGISel.cpp:2106
llvm::AfterLegalizeDAG
@ AfterLegalizeDAG
Definition: DAGCombine.h:19
CheckChild2CondCode
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChild2CondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
Definition: SelectionDAGISel.cpp:2618
Index
uint32_t Index
Definition: ELFObjHandler.cpp:83
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:66
index
splat index
Definition: README_ALTIVEC.txt:181
llvm::MachineInstrBuilder
Definition: MachineInstrBuilder.h:69
llvm::SelectionDAGISel::OPC_RecordMemRef
@ OPC_RecordMemRef
Definition: SelectionDAGISel.h:121
llvm::TargetLibraryInfoWrapperPass
Definition: TargetLibraryInfo.h:565
uint64_t
ProfileSummaryInfo.h
llvm::SelectionDAGISel::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition: SelectionDAGISel.cpp:336
llvm::TargetTransformInfoWrapperPass
Wrapper pass for TargetTransformInfo.
Definition: TargetTransformInfo.h:2663
llvm::SelectionDAGISel::TII
const TargetInstrInfo * TII
Definition: SelectionDAGISel.h:55
llvm::GlobalValue::getParent
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
llvm::MCInstrDesc::isCall
bool isCall() const
Return true if the instruction is a call.
Definition: MCInstrDesc.h:287
llvm::DbgDeclareInst
This represents the llvm.dbg.declare instruction.
Definition: IntrinsicInst.h:349
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::MachineRegisterInfo::use_instr_begin
use_instr_iterator use_instr_begin(Register RegNo) const
Definition: MachineRegisterInfo.h:499
llvm::SelectionDAGISel::OPC_RecordChild0
@ OPC_RecordChild0
Definition: SelectionDAGISel.h:119
HandleMergeInputChains
static SDValue HandleMergeInputChains(SmallVectorImpl< SDNode * > &ChainNodesMatched, SelectionDAG *CurDAG)
HandleMergeInputChains - This implements the OPC_EmitMergeInputChains operation for when the pattern ...
Definition: SelectionDAGISel.cpp:2428
llvm::ISD::AssertZext
@ AssertZext
Definition: ISDOpcodes.h:62
llvm::Function::hasGC
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
Definition: Function.h:305
llvm::ISD::STRICT_LRINT
@ STRICT_LRINT
Definition: ISDOpcodes.h:432
llvm::SelectionDAG::mutateStrictFPToFP
SDNode * mutateStrictFPToFP(SDNode *Node)
Mutate the specified strict FP node to its non-strict equivalent, unlinking the node from its chain a...
Definition: SelectionDAG.cpp:9868
llvm::SelectionDAGISel::FuncInfo
std::unique_ptr< FunctionLoweringInfo > FuncInfo
Definition: SelectionDAGISel.h:45
llvm::TargetRegisterInfo::getFrameRegister
virtual Register getFrameRegister(const MachineFunction &MF) const =0
Debug information queries.
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:31
llvm::SDNode::setNodeId
void setNodeId(int Id)
Set unique node id.
Definition: SelectionDAGNodes.h:728
llvm::SelectionDAG::getCopyFromReg
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:795
llvm::DenseMap< unsigned, unsigned >
llvm::SelectionDAGISel::IsLegalToFold
static bool IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, CodeGenOpt::Level OptLevel, bool IgnoreChains=false)
IsLegalToFold - Returns true if the specific operand node N of U can be folded during instruction sel...
Definition: SelectionDAGISel.cpp:2114
llvm::SelectionDAGISel::mayRaiseFPException
bool mayRaiseFPException(SDNode *Node) const
Return whether the node may raise an FP exception.
Definition: SelectionDAGISel.cpp:3769
DebugInfo.h
llvm::SelectionDAGISel::OPC_CheckChild2Same
@ OPC_CheckChild2Same
Definition: SelectionDAGISel.h:129
llvm::SDNode::getOperand
const SDValue & getOperand(unsigned Num) const
Definition: SelectionDAGNodes.h:921
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::SwiftErrorValueTracking
Definition: SwiftErrorValueTracking.h:34
llvm::MCPhysReg
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition: MCRegister.h:21
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:9172
CheckOpcode
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDNode *N)
Definition: SelectionDAGISel.cpp:2583
llvm::InlineAsm::Op_FirstOperand
@ Op_FirstOperand
Definition: InlineAsm.h:222
llvm::MachineFrameInfo::getObjectAlign
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
Definition: MachineFrameInfo.h:484
llvm::SelectionDAGISel::OPC_CheckChild4Type
@ OPC_CheckChild4Type
Definition: SelectionDAGISel.h:139
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
llvm::ProfileSummaryInfoWrapperPass
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
Definition: ProfileSummaryInfo.h:194
llvm::is_contained
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:1869
llvm::MachineRegisterInfo::liveins
ArrayRef< std::pair< MCRegister, Register > > liveins() const
Definition: MachineRegisterInfo.h:997
llvm::MachineFunction::finalizeDebugInstrRefs
void finalizeDebugInstrRefs()
Finalise any partially emitted debug instructions.
Definition: MachineFunction.cpp:1136
llvm::SelectionDAGISel::getPatternForIndex
virtual StringRef getPatternForIndex(unsigned index)
getPatternForIndex - Patterns selected by tablegen during ISEL
Definition: SelectionDAGISel.h:240
MachineFunctionPass.h
llvm::SelectionDAG::MorphNodeTo
SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
Definition: SelectionDAG.cpp:9815
llvm::MachineFunction::setUseDebugInstrRef
void setUseDebugInstrRef(bool UseInstrRef)
Set whether this function will use instruction referencing or not.
Definition: MachineFunction.cpp:1220
llvm::SelectionDAG::NewNodesMustHaveLegalTypes
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
Definition: SelectionDAG.h:383
llvm::Register::isVirtualRegister
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:71
llvm::SelectionDAGISel::OPC_CheckImmAllOnesV
@ OPC_CheckImmAllOnesV
Definition: SelectionDAGISel.h:148
llvm::SelectionDAGISel::OPC_SwitchOpcode
@ OPC_SwitchOpcode
Definition: SelectionDAGISel.h:134
llvm::DenseMapBase::find
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:150
llvm::MachineFunction::getName
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Definition: MachineFunction.cpp:582
llvm::SelectionDAGISel::OPC_CheckChild3Integer
@ OPC_CheckChild3Integer
Definition: SelectionDAGISel.h:143
llvm::ISD::STRICT_LROUND
@ STRICT_LROUND
Definition: ISDOpcodes.h:430
llvm::ISD::MEMBARRIER
@ MEMBARRIER
MEMBARRIER - Compiler barrier only; generate a no-op.
Definition: ISDOpcodes.h:1152
llvm::SDValue::getValue
SDValue getValue(unsigned R) const
Definition: SelectionDAGNodes.h:179
llvm::TargetMachine::Options
TargetOptions Options
Definition: TargetMachine.h:119
llvm::SelectionDAGISel::CheckOrMask
bool CheckOrMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const
CheckOrMask - The isel is trying to match something like (or X, 255).
Definition: SelectionDAGISel.cpp:1961
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::TargetMachine
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:78
llvm::SelectionDAGISel::~SelectionDAGISel
~SelectionDAGISel() override
Definition: SelectionDAGISel.cpp:331
llvm::FunctionLoweringInfo
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Definition: FunctionLoweringInfo.h:52
llvm::MVT::Other
@ Other
Definition: MachineValueType.h:42
llvm::SDNode::getNodeId
int getNodeId() const
Return the unique node id.
Definition: SelectionDAGNodes.h:725
llvm::createVLIWDAGScheduler
ScheduleDAGSDNodes * createVLIWDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createVLIWDAGScheduler - Scheduler for VLIW targets.
Definition: ScheduleDAGVLIW.cpp:269
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:853
llvm::MachineFunction::getFrameInfo
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Definition: MachineFunction.h:688
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition: MachineBasicBlock.h:265
llvm::OptimizationRemarkEmitter::emit
void emit(DiagnosticInfoOptimizationBase &OptDiag)
Output the remark via the diagnostic handler and to the optimization record file.
Definition: OptimizationRemarkEmitter.cpp:79
llvm::ISD::STACKMAP
@ STACKMAP
Definition: ISDOpcodes.h:1297
llvm::SelectionDAGISel::OPC_EmitMergeInputChains
@ OPC_EmitMergeInputChains
Definition: SelectionDAGISel.h:157
llvm::InlineAsm::getMemoryConstraintID
static unsigned getMemoryConstraintID(unsigned Flag)
Definition: InlineAsm.h:355
llvm::ISD::CondCode
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1428
llvm::ISD::BasicBlock
@ BasicBlock
Various leaf nodes.
Definition: ISDOpcodes.h:71
llvm::SelectionDAGISel::CurDAG
SelectionDAG * CurDAG
Definition: SelectionDAGISel.h:49
function
print Print MemDeps of function
Definition: MemDepPrinter.cpp:82
llvm::SelectionDAG::getMachineNode
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
Definition: SelectionDAG.cpp:9916
CodeGenCommonISel.h
MachineModuleInfo.h
llvm::ISD::TargetGlobalTLSAddress
@ TargetGlobalTLSAddress
Definition: ISDOpcodes.h:165
llvm::Sched::VLIW
@ VLIW
Definition: TargetLowering.h:104
llvm::MVT
Machine Value Type.
Definition: MachineValueType.h:31
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:97
llvm::SwiftErrorValueTracking::setFunction
void setFunction(MachineFunction &MF)
Initialize data structures for specified new function.
Definition: SwiftErrorValueTracking.cpp:79
llvm::FastISel
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:66
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:366
llvm::SelectionDAGISel::getUninvalidatedNodeId
static int getUninvalidatedNodeId(SDNode *N)
Definition: SelectionDAGISel.cpp:1073
llvm::SelectionDAG::setNodeMemRefs
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
Definition: SelectionDAG.cpp:9684
SwiftErrorValueTracking.h
llvm::MDNode
Metadata node.
Definition: Metadata.h:943
llvm::CodeGenOpt::None
@ None
-O0
Definition: CodeGen.h:58
llvm::ISD::STRICT_LLRINT
@ STRICT_LLRINT
Definition: ISDOpcodes.h:433
llvm::AfterLegalizeVectorOps
@ AfterLegalizeVectorOps
Definition: DAGCombine.h:18
llvm::MCInstrInfo::getName
StringRef getName(unsigned Opcode) const
Returns the name for the instructions with the given opcode.
Definition: MCInstrInfo.h:70
llvm::SelectionDAG::SelectNodeTo
SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
Definition: SelectionDAG.cpp:9708
llvm::TargetOptions::EnableFastISel
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
Definition: TargetOptions.h:232
llvm::SDNode::isMachineOpcode
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
Definition: SelectionDAGNodes.h:699
llvm::ISD::TargetBlockAddress
@ TargetBlockAddress
Definition: ISDOpcodes.h:170
llvm::SelectionDAGISel::OPC_CheckPredicateWithOperands
@ OPC_CheckPredicateWithOperands
Definition: SelectionDAGISel.h:132
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:75
llvm::MachineFunction
Definition: MachineFunction.h:258
llvm::MCInstrDesc::mayStore
bool mayStore() const
Return true if this instruction could possibly modify memory.
Definition: MCInstrDesc.h:443
llvm::ISD::ANNOTATION_LABEL
@ ANNOTATION_LABEL
ANNOTATION_LABEL - Represents a mid basic block label used by annotations.
Definition: ISDOpcodes.h:1038
llvm::SelectionDAG::getFunctionVarLocs
const FunctionVarLocs * getFunctionVarLocs() const
Returns the result of the AssignmentTrackingAnalysis pass if it's available, otherwise return nullptr...
Definition: SelectionDAG.h:480
llvm::SelectionDAGISel::CheckNodePredicateWithOperands
virtual bool CheckNodePredicateWithOperands(SDNode *N, unsigned PredNo, const SmallVectorImpl< SDValue > &Operands) const
CheckNodePredicateWithOperands - This function is generated by tblgen in the target.
Definition: SelectionDAGISel.h:282
llvm::MDNodeSDNode
Definition: SelectionDAGNodes.h:2148
llvm::createHybridListDAGScheduler
ScheduleDAGSDNodes * createHybridListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level)
createHybridListDAGScheduler - This creates a bottom up register pressure aware list scheduler that m...
Definition: ScheduleDAGRRList.cpp:3182
CFG.h
llvm::ISD::HANDLENODE
@ HANDLENODE
HANDLENODE node - Used as a handle for various purposes.
Definition: ISDOpcodes.h:1115
llvm::MachineBasicBlock::succ_empty
bool succ_empty() const
Definition: MachineBasicBlock.h:388
TargetOptions.h
SelectionDAGISel.h
ViewDAGCombine2
static cl::opt< bool > ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the second " "dag combine pass"))
llvm::ISD::TargetConstantPool
@ TargetConstantPool
Definition: ISDOpcodes.h:168
Scheduler
Machine Instruction Scheduler
Definition: MachineScheduler.cpp:243
llvm::NVPTXISD::Dummy
@ Dummy
Definition: NVPTXISelLowering.h:60
llvm::MachinePassRegistry
MachinePassRegistry - Track the registration of machine passes.
Definition: MachinePassRegistry.h:73
llvm::MachineBasicBlock::getFirstTerminator
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
Definition: MachineBasicBlock.cpp:240
llvm::SmallPtrSetImplBase::clear
void clear()
Definition: SmallPtrSet.h:95
llvm::Sched::Source
@ Source
Definition: TargetLowering.h:100
llvm::AssumptionCacheTracker
An immutable pass that tracks lazily created AssumptionCache objects.
Definition: AssumptionCache.h:202
llvm::initializeAAResultsWrapperPassPass
void initializeAAResultsWrapperPassPass(PassRegistry &)
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:33