LLVM  16.0.0git
SelectionDAGISel.cpp
Go to the documentation of this file.
1 //===- SelectionDAGISel.cpp - Implement the SelectionDAGISel class --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements the SelectionDAGISel class.
10 //
11 //===----------------------------------------------------------------------===//
12 
14 #include "ScheduleDAGSDNodes.h"
15 #include "SelectionDAGBuilder.h"
16 #include "llvm/ADT/APInt.h"
17 #include "llvm/ADT/DenseMap.h"
19 #include "llvm/ADT/STLExtras.h"
20 #include "llvm/ADT/SmallPtrSet.h"
21 #include "llvm/ADT/SmallVector.h"
22 #include "llvm/ADT/Statistic.h"
23 #include "llvm/ADT/StringRef.h"
27 #include "llvm/Analysis/CFG.h"
36 #include "llvm/CodeGen/FastISel.h"
54 #include "llvm/CodeGen/StackMaps.h"
62 #include "llvm/IR/BasicBlock.h"
63 #include "llvm/IR/Constants.h"
64 #include "llvm/IR/DataLayout.h"
66 #include "llvm/IR/DebugLoc.h"
67 #include "llvm/IR/DiagnosticInfo.h"
68 #include "llvm/IR/Function.h"
69 #include "llvm/IR/InlineAsm.h"
70 #include "llvm/IR/InstIterator.h"
71 #include "llvm/IR/Instruction.h"
72 #include "llvm/IR/Instructions.h"
73 #include "llvm/IR/IntrinsicInst.h"
74 #include "llvm/IR/Intrinsics.h"
75 #include "llvm/IR/IntrinsicsWebAssembly.h"
76 #include "llvm/IR/Metadata.h"
77 #include "llvm/IR/Statepoint.h"
78 #include "llvm/IR/Type.h"
79 #include "llvm/IR/User.h"
80 #include "llvm/IR/Value.h"
81 #include "llvm/InitializePasses.h"
82 #include "llvm/MC/MCInstrDesc.h"
83 #include "llvm/Pass.h"
85 #include "llvm/Support/Casting.h"
86 #include "llvm/Support/CodeGen.h"
88 #include "llvm/Support/Compiler.h"
89 #include "llvm/Support/Debug.h"
91 #include "llvm/Support/KnownBits.h"
93 #include "llvm/Support/Timer.h"
99 #include <algorithm>
100 #include <cassert>
101 #include <cstdint>
102 #include <iterator>
103 #include <limits>
104 #include <memory>
105 #include <string>
106 #include <utility>
107 #include <vector>
108 
109 using namespace llvm;
110 
111 #define DEBUG_TYPE "isel"
112 
113 STATISTIC(NumFastIselFailures, "Number of instructions fast isel failed on");
114 STATISTIC(NumFastIselSuccess, "Number of instructions fast isel selected");
115 STATISTIC(NumFastIselBlocks, "Number of blocks selected entirely by fast isel");
116 STATISTIC(NumDAGBlocks, "Number of blocks selected using DAG");
117 STATISTIC(NumDAGIselRetries,"Number of times dag isel has to try another path");
118 STATISTIC(NumEntryBlocks, "Number of entry blocks encountered");
119 STATISTIC(NumFastIselFailLowerArguments,
120  "Number of entry blocks where fast isel failed to lower arguments");
121 
123  "fast-isel-abort", cl::Hidden,
124  cl::desc("Enable abort calls when \"fast\" instruction selection "
125  "fails to lower an instruction: 0 disable the abort, 1 will "
126  "abort but for args, calls and terminators, 2 will also "
127  "abort for argument lowering, and 3 will never fallback "
128  "to SelectionDAG."));
129 
131  "fast-isel-report-on-fallback", cl::Hidden,
132  cl::desc("Emit a diagnostic when \"fast\" instruction selection "
133  "falls back to SelectionDAG."));
134 
135 static cl::opt<bool>
136 UseMBPI("use-mbpi",
137  cl::desc("use Machine Branch Probability Info"),
138  cl::init(true), cl::Hidden);
139 
140 #ifndef NDEBUG
142 FilterDAGBasicBlockName("filter-view-dags", cl::Hidden,
143  cl::desc("Only display the basic block whose name "
144  "matches this for all view-*-dags options"));
145 static cl::opt<bool>
146 ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden,
147  cl::desc("Pop up a window to show dags before the first "
148  "dag combine pass"));
149 static cl::opt<bool>
150 ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden,
151  cl::desc("Pop up a window to show dags before legalize types"));
152 static cl::opt<bool>
153  ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden,
154  cl::desc("Pop up a window to show dags before the post "
155  "legalize types dag combine pass"));
156 static cl::opt<bool>
157  ViewLegalizeDAGs("view-legalize-dags", cl::Hidden,
158  cl::desc("Pop up a window to show dags before legalize"));
159 static cl::opt<bool>
160 ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden,
161  cl::desc("Pop up a window to show dags before the second "
162  "dag combine pass"));
163 static cl::opt<bool>
164 ViewISelDAGs("view-isel-dags", cl::Hidden,
165  cl::desc("Pop up a window to show isel dags as they are selected"));
166 static cl::opt<bool>
167 ViewSchedDAGs("view-sched-dags", cl::Hidden,
168  cl::desc("Pop up a window to show sched dags as they are processed"));
169 static cl::opt<bool>
170 ViewSUnitDAGs("view-sunit-dags", cl::Hidden,
171  cl::desc("Pop up a window to show SUnit dags after they are processed"));
172 #else
173 static const bool ViewDAGCombine1 = false, ViewLegalizeTypesDAGs = false,
174  ViewDAGCombineLT = false, ViewLegalizeDAGs = false,
175  ViewDAGCombine2 = false, ViewISelDAGs = false,
176  ViewSchedDAGs = false, ViewSUnitDAGs = false;
177 #endif
178 
179 //===---------------------------------------------------------------------===//
180 ///
181 /// RegisterScheduler class - Track the registration of instruction schedulers.
182 ///
183 //===---------------------------------------------------------------------===//
186 
187 //===---------------------------------------------------------------------===//
188 ///
189 /// ISHeuristic command line option for instruction schedulers.
190 ///
191 //===---------------------------------------------------------------------===//
194 ISHeuristic("pre-RA-sched",
196  cl::desc("Instruction schedulers available (before register"
197  " allocation):"));
198 
199 static RegisterScheduler
200 defaultListDAGScheduler("default", "Best scheduler for the target",
202 
203 namespace llvm {
204 
205  //===--------------------------------------------------------------------===//
206  /// This class is used by SelectionDAGISel to temporarily override
207  /// the optimization level on a per-function basis.
209  SelectionDAGISel &IS;
210  CodeGenOpt::Level SavedOptLevel;
211  bool SavedFastISel;
212 
213  public:
215  CodeGenOpt::Level NewOptLevel) : IS(ISel) {
216  SavedOptLevel = IS.OptLevel;
217  SavedFastISel = IS.TM.Options.EnableFastISel;
218  if (NewOptLevel == SavedOptLevel)
219  return;
220  IS.OptLevel = NewOptLevel;
221  IS.TM.setOptLevel(NewOptLevel);
222  LLVM_DEBUG(dbgs() << "\nChanging optimization level for Function "
223  << IS.MF->getFunction().getName() << "\n");
224  LLVM_DEBUG(dbgs() << "\tBefore: -O" << SavedOptLevel << " ; After: -O"
225  << NewOptLevel << "\n");
226  if (NewOptLevel == CodeGenOpt::None) {
228  LLVM_DEBUG(
229  dbgs() << "\tFastISel is "
230  << (IS.TM.Options.EnableFastISel ? "enabled" : "disabled")
231  << "\n");
232  }
233  }
234 
236  if (IS.OptLevel == SavedOptLevel)
237  return;
238  LLVM_DEBUG(dbgs() << "\nRestoring optimization level for Function "
239  << IS.MF->getFunction().getName() << "\n");
240  LLVM_DEBUG(dbgs() << "\tBefore: -O" << IS.OptLevel << " ; After: -O"
241  << SavedOptLevel << "\n");
242  IS.OptLevel = SavedOptLevel;
243  IS.TM.setOptLevel(SavedOptLevel);
244  IS.TM.setFastISel(SavedFastISel);
245  }
246  };
247 
248  //===--------------------------------------------------------------------===//
249  /// createDefaultScheduler - This creates an instruction scheduler appropriate
250  /// for the target.
252  CodeGenOpt::Level OptLevel) {
253  const TargetLowering *TLI = IS->TLI;
254  const TargetSubtargetInfo &ST = IS->MF->getSubtarget();
255 
256  // Try first to see if the Target has its own way of selecting a scheduler
257  if (auto *SchedulerCtor = ST.getDAGScheduler(OptLevel)) {
258  return SchedulerCtor(IS, OptLevel);
259  }
260 
261  if (OptLevel == CodeGenOpt::None ||
262  (ST.enableMachineScheduler() && ST.enableMachineSchedDefaultSched()) ||
264  return createSourceListDAGScheduler(IS, OptLevel);
266  return createBURRListDAGScheduler(IS, OptLevel);
268  return createHybridListDAGScheduler(IS, OptLevel);
269  if (TLI->getSchedulingPreference() == Sched::VLIW)
270  return createVLIWDAGScheduler(IS, OptLevel);
271  if (TLI->getSchedulingPreference() == Sched::Fast)
272  return createFastDAGScheduler(IS, OptLevel);
274  return createDAGLinearizer(IS, OptLevel);
276  "Unknown sched type!");
277  return createILPListDAGScheduler(IS, OptLevel);
278  }
279 
280 } // end namespace llvm
281 
282 // EmitInstrWithCustomInserter - This method should be implemented by targets
283 // that mark instructions with the 'usesCustomInserter' flag. These
284 // instructions are special in various ways, which require special support to
285 // insert. The specified MachineInstr is created but not inserted into any
286 // basic blocks, and this method is called to expand it into a sequence of
287 // instructions, potentially also creating new basic blocks and control flow.
288 // When new basic blocks are inserted and the edges from MBB to its successors
289 // are modified, the method should insert pairs of <OldSucc, NewSucc> into the
290 // DenseMap.
293  MachineBasicBlock *MBB) const {
294 #ifndef NDEBUG
295  dbgs() << "If a target marks an instruction with "
296  "'usesCustomInserter', it must implement "
297  "TargetLowering::EmitInstrWithCustomInserter!\n";
298 #endif
299  llvm_unreachable(nullptr);
300 }
301 
303  SDNode *Node) const {
304  assert(!MI.hasPostISelHook() &&
305  "If a target marks an instruction with 'hasPostISelHook', "
306  "it must implement TargetLowering::AdjustInstrPostInstrSelection!");
307 }
308 
309 //===----------------------------------------------------------------------===//
310 // SelectionDAGISel code
311 //===----------------------------------------------------------------------===//
312 
314  : MachineFunctionPass(ID), TM(tm), FuncInfo(new FunctionLoweringInfo()),
315  SwiftError(new SwiftErrorValueTracking()),
316  CurDAG(new SelectionDAG(tm, OL)),
317  SDB(std::make_unique<SelectionDAGBuilder>(*CurDAG, *FuncInfo, *SwiftError,
318  OL)),
319  OptLevel(OL) {
325 }
326 
328  delete CurDAG;
329  delete SwiftError;
330 }
331 
333  if (OptLevel != CodeGenOpt::None)
344  if (OptLevel != CodeGenOpt::None)
347 }
348 
349 static void computeUsesMSVCFloatingPoint(const Triple &TT, const Function &F,
350  MachineModuleInfo &MMI) {
351  // Only needed for MSVC
352  if (!TT.isWindowsMSVCEnvironment())
353  return;
354 
355  // If it's already set, nothing to do.
356  if (MMI.usesMSVCFloatingPoint())
357  return;
358 
359  for (const Instruction &I : instructions(F)) {
360  if (I.getType()->isFPOrFPVectorTy()) {
361  MMI.setUsesMSVCFloatingPoint(true);
362  return;
363  }
364  for (const auto &Op : I.operands()) {
365  if (Op->getType()->isFPOrFPVectorTy()) {
366  MMI.setUsesMSVCFloatingPoint(true);
367  return;
368  }
369  }
370  }
371 }
372 
374  // If we already selected that function, we do not need to run SDISel.
375  if (mf.getProperties().hasProperty(
377  return false;
378  // Do some sanity-checking on the command-line options.
380  "-fast-isel-abort > 0 requires -fast-isel");
381 
382  const Function &Fn = mf.getFunction();
383  MF = &mf;
384 
385  // Decide what flavour of variable location debug-info will be used, before
386  // we change the optimisation level.
389 
390  // Reset the target options before resetting the optimization
391  // level below.
392  // FIXME: This is a horrible hack and should be processed via
393  // codegen looking at the optimization level explicitly when
394  // it wants to look at it.
396  // Reset OptLevel to None for optnone functions.
397  CodeGenOpt::Level NewOptLevel = OptLevel;
398  if (OptLevel != CodeGenOpt::None && skipFunction(Fn))
399  NewOptLevel = CodeGenOpt::None;
400  OptLevelChanger OLC(*this, NewOptLevel);
401 
404  RegInfo = &MF->getRegInfo();
405  LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(Fn);
406  GFI = Fn.hasGC() ? &getAnalysis<GCModuleInfo>().getFunctionInfo(Fn) : nullptr;
407  ORE = std::make_unique<OptimizationRemarkEmitter>(&Fn);
408  AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(mf.getFunction());
409  auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
410  BlockFrequencyInfo *BFI = nullptr;
411  if (PSI && PSI->hasProfileSummary() && OptLevel != CodeGenOpt::None)
412  BFI = &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI();
413 
414  LLVM_DEBUG(dbgs() << "\n\n\n=== " << Fn.getName() << "\n");
415 
416  CurDAG->init(*MF, *ORE, this, LibInfo,
417  getAnalysisIfAvailable<LegacyDivergenceAnalysis>(), PSI, BFI);
418  FuncInfo->set(Fn, *MF, CurDAG);
420 
421  // Now get the optional analyzes if we want to.
422  // This is based on the possibly changed OptLevel (after optnone is taken
423  // into account). That's unfortunate but OK because it just means we won't
424  // ask for passes that have been required anyway.
425 
427  FuncInfo->BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
428  else
429  FuncInfo->BPI = nullptr;
430 
431  if (OptLevel != CodeGenOpt::None)
432  AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
433  else
434  AA = nullptr;
435 
436  SDB->init(GFI, AA, AC, LibInfo);
437 
438  MF->setHasInlineAsm(false);
439 
440  FuncInfo->SplitCSR = false;
441 
442  // We split CSR if the target supports it for the given function
443  // and the function has only return exits.
445  FuncInfo->SplitCSR = true;
446 
447  // Collect all the return blocks.
448  for (const BasicBlock &BB : Fn) {
449  if (!succ_empty(&BB))
450  continue;
451 
452  const Instruction *Term = BB.getTerminator();
453  if (isa<UnreachableInst>(Term) || isa<ReturnInst>(Term))
454  continue;
455 
456  // Bail out if the exit block is not Return nor Unreachable.
457  FuncInfo->SplitCSR = false;
458  break;
459  }
460  }
461 
462  MachineBasicBlock *EntryMBB = &MF->front();
463  if (FuncInfo->SplitCSR)
464  // This performs initialization so lowering for SplitCSR will be correct.
465  TLI->initializeSplitCSR(EntryMBB);
466 
467  SelectAllBasicBlocks(Fn);
469  DiagnosticInfoISelFallback DiagFallback(Fn);
470  Fn.getContext().diagnose(DiagFallback);
471  }
472 
473  // Replace forward-declared registers with the registers containing
474  // the desired value.
475  // Note: it is important that this happens **before** the call to
476  // EmitLiveInCopies, since implementations can skip copies of unused
477  // registers. If we don't apply the reg fixups before, some registers may
478  // appear as unused and will be skipped, resulting in bad MI.
480  for (DenseMap<Register, Register>::iterator I = FuncInfo->RegFixups.begin(),
481  E = FuncInfo->RegFixups.end();
482  I != E; ++I) {
483  Register From = I->first;
484  Register To = I->second;
485  // If To is also scheduled to be replaced, find what its ultimate
486  // replacement is.
487  while (true) {
488  DenseMap<Register, Register>::iterator J = FuncInfo->RegFixups.find(To);
489  if (J == E)
490  break;
491  To = J->second;
492  }
493  // Make sure the new register has a sufficiently constrained register class.
496  // Replace it.
497 
498  // Replacing one register with another won't touch the kill flags.
499  // We need to conservatively clear the kill flags as a kill on the old
500  // register might dominate existing uses of the new register.
501  if (!MRI.use_empty(To))
503  MRI.replaceRegWith(From, To);
504  }
505 
506  // If the first basic block in the function has live ins that need to be
507  // copied into vregs, emit the copies into the top of the block before
508  // emitting the code for the block.
510  RegInfo->EmitLiveInCopies(EntryMBB, TRI, *TII);
511 
512  // Insert copies in the entry block and the return blocks.
513  if (FuncInfo->SplitCSR) {
515  // Collect all the return blocks.
516  for (MachineBasicBlock &MBB : mf) {
517  if (!MBB.succ_empty())
518  continue;
519 
521  if (Term != MBB.end() && Term->isReturn()) {
522  Returns.push_back(&MBB);
523  continue;
524  }
525  }
526  TLI->insertCopiesSplitCSR(EntryMBB, Returns);
527  }
528 
530  if (!FuncInfo->ArgDbgValues.empty())
531  for (std::pair<unsigned, unsigned> LI : RegInfo->liveins())
532  if (LI.second)
533  LiveInMap.insert(LI);
534 
535  // Insert DBG_VALUE instructions for function arguments to the entry block.
536  bool InstrRef = MF->useDebugInstrRef();
537  for (unsigned i = 0, e = FuncInfo->ArgDbgValues.size(); i != e; ++i) {
538  MachineInstr *MI = FuncInfo->ArgDbgValues[e - i - 1];
539  assert(MI->getOpcode() != TargetOpcode::DBG_VALUE_LIST &&
540  "Function parameters should not be described by DBG_VALUE_LIST.");
541  bool hasFI = MI->getOperand(0).isFI();
542  Register Reg =
543  hasFI ? TRI.getFrameRegister(*MF) : MI->getOperand(0).getReg();
545  EntryMBB->insert(EntryMBB->begin(), MI);
546  else {
548  if (Def) {
549  MachineBasicBlock::iterator InsertPos = Def;
550  // FIXME: VR def may not be in entry block.
551  Def->getParent()->insert(std::next(InsertPos), MI);
552  } else
553  LLVM_DEBUG(dbgs() << "Dropping debug info for dead vreg"
554  << Register::virtReg2Index(Reg) << "\n");
555  }
556 
557  // Don't try and extend through copies in instruction referencing mode.
558  if (InstrRef)
559  continue;
560 
561  // If Reg is live-in then update debug info to track its copy in a vreg.
563  if (LDI != LiveInMap.end()) {
564  assert(!hasFI && "There's no handling of frame pointer updating here yet "
565  "- add if needed");
566  MachineInstr *Def = RegInfo->getVRegDef(LDI->second);
567  MachineBasicBlock::iterator InsertPos = Def;
568  const MDNode *Variable = MI->getDebugVariable();
569  const MDNode *Expr = MI->getDebugExpression();
570  DebugLoc DL = MI->getDebugLoc();
571  bool IsIndirect = MI->isIndirectDebugValue();
572  if (IsIndirect)
573  assert(MI->getOperand(1).getImm() == 0 &&
574  "DBG_VALUE with nonzero offset");
575  assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) &&
576  "Expected inlined-at fields to agree");
577  assert(MI->getOpcode() != TargetOpcode::DBG_VALUE_LIST &&
578  "Didn't expect to see a DBG_VALUE_LIST here");
579  // Def is never a terminator here, so it is ok to increment InsertPos.
580  BuildMI(*EntryMBB, ++InsertPos, DL, TII->get(TargetOpcode::DBG_VALUE),
581  IsIndirect, LDI->second, Variable, Expr);
582 
583  // If this vreg is directly copied into an exported register then
584  // that COPY instructions also need DBG_VALUE, if it is the only
585  // user of LDI->second.
586  MachineInstr *CopyUseMI = nullptr;
588  UI = RegInfo->use_instr_begin(LDI->second),
589  E = RegInfo->use_instr_end(); UI != E; ) {
590  MachineInstr *UseMI = &*(UI++);
591  if (UseMI->isDebugValue()) continue;
592  if (UseMI->isCopy() && !CopyUseMI && UseMI->getParent() == EntryMBB) {
593  CopyUseMI = UseMI; continue;
594  }
595  // Otherwise this is another use or second copy use.
596  CopyUseMI = nullptr; break;
597  }
598  if (CopyUseMI &&
599  TRI.getRegSizeInBits(LDI->second, MRI) ==
600  TRI.getRegSizeInBits(CopyUseMI->getOperand(0).getReg(), MRI)) {
601  // Use MI's debug location, which describes where Variable was
602  // declared, rather than whatever is attached to CopyUseMI.
603  MachineInstr *NewMI =
604  BuildMI(*MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
605  CopyUseMI->getOperand(0).getReg(), Variable, Expr);
606  MachineBasicBlock::iterator Pos = CopyUseMI;
607  EntryMBB->insertAfter(Pos, NewMI);
608  }
609  }
610  }
611 
612  // For debug-info, in instruction referencing mode, we need to perform some
613  // post-isel maintenence.
616 
617  // Determine if there are any calls in this machine function.
618  MachineFrameInfo &MFI = MF->getFrameInfo();
619  for (const auto &MBB : *MF) {
620  if (MFI.hasCalls() && MF->hasInlineAsm())
621  break;
622 
623  for (const auto &MI : MBB) {
624  const MCInstrDesc &MCID = TII->get(MI.getOpcode());
625  if ((MCID.isCall() && !MCID.isReturn()) ||
626  MI.isStackAligningInlineAsm()) {
627  MFI.setHasCalls(true);
628  }
629  if (MI.isInlineAsm()) {
630  MF->setHasInlineAsm(true);
631  }
632  }
633  }
634 
635  // Determine if there is a call to setjmp in the machine function.
637 
638  // Determine if floating point is used for msvc
640 
641  // Release function-specific state. SDB and CurDAG are already cleared
642  // at this point.
643  FuncInfo->clear();
644 
645  LLVM_DEBUG(dbgs() << "*** MachineFunction at end of ISel ***\n");
646  LLVM_DEBUG(MF->print(dbgs()));
647 
648  return true;
649 }
650 
654  bool ShouldAbort) {
655  // Print the function name explicitly if we don't have a debug location (which
656  // makes the diagnostic less useful) or if we're going to emit a raw error.
657  if (!R.getLocation().isValid() || ShouldAbort)
658  R << (" (in function: " + MF.getName() + ")").str();
659 
660  if (ShouldAbort)
661  report_fatal_error(Twine(R.getMsg()));
662 
663  ORE.emit(R);
664  LLVM_DEBUG(dbgs() << R.getMsg() << "\n");
665 }
666 
667 void SelectionDAGISel::SelectBasicBlock(BasicBlock::const_iterator Begin,
669  bool &HadTailCall) {
670  // Allow creating illegal types during DAG building for the basic block.
672 
673  // Lower the instructions. If a call is emitted as a tail call, cease emitting
674  // nodes for this block.
675  for (BasicBlock::const_iterator I = Begin; I != End && !SDB->HasTailCall; ++I) {
676  if (!ElidedArgCopyInstrs.count(&*I))
677  SDB->visit(*I);
678  }
679 
680  // Make sure the root of the DAG is up-to-date.
681  CurDAG->setRoot(SDB->getControlRoot());
682  HadTailCall = SDB->HasTailCall;
683  SDB->resolveOrClearDbgInfo();
684  SDB->clear();
685 
686  // Final step, emit the lowered DAG as machine code.
687  CodeGenAndEmitDAG();
688 }
689 
690 void SelectionDAGISel::ComputeLiveOutVRegInfo() {
692  SmallVector<SDNode*, 128> Worklist;
693 
694  Worklist.push_back(CurDAG->getRoot().getNode());
695  Added.insert(CurDAG->getRoot().getNode());
696 
697  KnownBits Known;
698 
699  do {
700  SDNode *N = Worklist.pop_back_val();
701 
702  // Otherwise, add all chain operands to the worklist.
703  for (const SDValue &Op : N->op_values())
704  if (Op.getValueType() == MVT::Other && Added.insert(Op.getNode()).second)
705  Worklist.push_back(Op.getNode());
706 
707  // If this is a CopyToReg with a vreg dest, process it.
708  if (N->getOpcode() != ISD::CopyToReg)
709  continue;
710 
711  unsigned DestReg = cast<RegisterSDNode>(N->getOperand(1))->getReg();
712  if (!Register::isVirtualRegister(DestReg))
713  continue;
714 
715  // Ignore non-integer values.
716  SDValue Src = N->getOperand(2);
717  EVT SrcVT = Src.getValueType();
718  if (!SrcVT.isInteger())
719  continue;
720 
721  unsigned NumSignBits = CurDAG->ComputeNumSignBits(Src);
722  Known = CurDAG->computeKnownBits(Src);
723  FuncInfo->AddLiveOutRegInfo(DestReg, NumSignBits, Known);
724  } while (!Worklist.empty());
725 }
726 
727 void SelectionDAGISel::CodeGenAndEmitDAG() {
728  StringRef GroupName = "sdag";
729  StringRef GroupDescription = "Instruction Selection and Scheduling";
730  std::string BlockName;
731  bool MatchFilterBB = false; (void)MatchFilterBB;
732 #ifndef NDEBUG
734  getAnalysis<TargetTransformInfoWrapperPass>().getTTI(*FuncInfo->Fn);
735 #endif
736 
737  // Pre-type legalization allow creation of any node types.
739 
740 #ifndef NDEBUG
741  MatchFilterBB = (FilterDAGBasicBlockName.empty() ||
743  FuncInfo->MBB->getBasicBlock()->getName());
744 #endif
745 #ifdef NDEBUG
749 #endif
750  {
751  BlockName =
752  (MF->getName() + ":" + FuncInfo->MBB->getBasicBlock()->getName()).str();
753  }
754  LLVM_DEBUG(dbgs() << "Initial selection DAG: "
755  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
756  << "'\n";
757  CurDAG->dump());
758 
759 #ifndef NDEBUG
760  if (TTI.hasBranchDivergence())
762 #endif
763 
764  if (ViewDAGCombine1 && MatchFilterBB)
765  CurDAG->viewGraph("dag-combine1 input for " + BlockName);
766 
767  // Run the DAG combiner in pre-legalize mode.
768  {
769  NamedRegionTimer T("combine1", "DAG Combining 1", GroupName,
770  GroupDescription, TimePassesIsEnabled);
772  }
773 
774  LLVM_DEBUG(dbgs() << "Optimized lowered selection DAG: "
775  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
776  << "'\n";
777  CurDAG->dump());
778 
779 #ifndef NDEBUG
780  if (TTI.hasBranchDivergence())
782 #endif
783 
784  // Second step, hack on the DAG until it only uses operations and types that
785  // the target supports.
786  if (ViewLegalizeTypesDAGs && MatchFilterBB)
787  CurDAG->viewGraph("legalize-types input for " + BlockName);
788 
789  bool Changed;
790  {
791  NamedRegionTimer T("legalize_types", "Type Legalization", GroupName,
792  GroupDescription, TimePassesIsEnabled);
793  Changed = CurDAG->LegalizeTypes();
794  }
795 
796  LLVM_DEBUG(dbgs() << "Type-legalized selection DAG: "
797  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
798  << "'\n";
799  CurDAG->dump());
800 
801 #ifndef NDEBUG
802  if (TTI.hasBranchDivergence())
804 #endif
805 
806  // Only allow creation of legal node types.
808 
809  if (Changed) {
810  if (ViewDAGCombineLT && MatchFilterBB)
811  CurDAG->viewGraph("dag-combine-lt input for " + BlockName);
812 
813  // Run the DAG combiner in post-type-legalize mode.
814  {
815  NamedRegionTimer T("combine_lt", "DAG Combining after legalize types",
816  GroupName, GroupDescription, TimePassesIsEnabled);
818  }
819 
820  LLVM_DEBUG(dbgs() << "Optimized type-legalized selection DAG: "
821  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
822  << "'\n";
823  CurDAG->dump());
824 
825 #ifndef NDEBUG
826  if (TTI.hasBranchDivergence())
828 #endif
829  }
830 
831  {
832  NamedRegionTimer T("legalize_vec", "Vector Legalization", GroupName,
833  GroupDescription, TimePassesIsEnabled);
834  Changed = CurDAG->LegalizeVectors();
835  }
836 
837  if (Changed) {
838  LLVM_DEBUG(dbgs() << "Vector-legalized selection DAG: "
839  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
840  << "'\n";
841  CurDAG->dump());
842 
843 #ifndef NDEBUG
844  if (TTI.hasBranchDivergence())
846 #endif
847 
848  {
849  NamedRegionTimer T("legalize_types2", "Type Legalization 2", GroupName,
850  GroupDescription, TimePassesIsEnabled);
852  }
853 
854  LLVM_DEBUG(dbgs() << "Vector/type-legalized selection DAG: "
855  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
856  << "'\n";
857  CurDAG->dump());
858 
859 #ifndef NDEBUG
860  if (TTI.hasBranchDivergence())
862 #endif
863 
864  if (ViewDAGCombineLT && MatchFilterBB)
865  CurDAG->viewGraph("dag-combine-lv input for " + BlockName);
866 
867  // Run the DAG combiner in post-type-legalize mode.
868  {
869  NamedRegionTimer T("combine_lv", "DAG Combining after legalize vectors",
870  GroupName, GroupDescription, TimePassesIsEnabled);
872  }
873 
874  LLVM_DEBUG(dbgs() << "Optimized vector-legalized selection DAG: "
875  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
876  << "'\n";
877  CurDAG->dump());
878 
879 #ifndef NDEBUG
880  if (TTI.hasBranchDivergence())
882 #endif
883  }
884 
885  if (ViewLegalizeDAGs && MatchFilterBB)
886  CurDAG->viewGraph("legalize input for " + BlockName);
887 
888  {
889  NamedRegionTimer T("legalize", "DAG Legalization", GroupName,
890  GroupDescription, TimePassesIsEnabled);
891  CurDAG->Legalize();
892  }
893 
894  LLVM_DEBUG(dbgs() << "Legalized selection DAG: "
895  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
896  << "'\n";
897  CurDAG->dump());
898 
899 #ifndef NDEBUG
900  if (TTI.hasBranchDivergence())
902 #endif
903 
904  if (ViewDAGCombine2 && MatchFilterBB)
905  CurDAG->viewGraph("dag-combine2 input for " + BlockName);
906 
907  // Run the DAG combiner in post-legalize mode.
908  {
909  NamedRegionTimer T("combine2", "DAG Combining 2", GroupName,
910  GroupDescription, TimePassesIsEnabled);
912  }
913 
914  LLVM_DEBUG(dbgs() << "Optimized legalized selection DAG: "
915  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
916  << "'\n";
917  CurDAG->dump());
918 
919 #ifndef NDEBUG
920  if (TTI.hasBranchDivergence())
922 #endif
923 
924  if (OptLevel != CodeGenOpt::None)
925  ComputeLiveOutVRegInfo();
926 
927  if (ViewISelDAGs && MatchFilterBB)
928  CurDAG->viewGraph("isel input for " + BlockName);
929 
930  // Third, instruction select all of the operations to machine code, adding the
931  // code to the MachineBasicBlock.
932  {
933  NamedRegionTimer T("isel", "Instruction Selection", GroupName,
934  GroupDescription, TimePassesIsEnabled);
935  DoInstructionSelection();
936  }
937 
938  LLVM_DEBUG(dbgs() << "Selected selection DAG: "
939  << printMBBReference(*FuncInfo->MBB) << " '" << BlockName
940  << "'\n";
941  CurDAG->dump());
942 
943  if (ViewSchedDAGs && MatchFilterBB)
944  CurDAG->viewGraph("scheduler input for " + BlockName);
945 
946  // Schedule machine code.
947  ScheduleDAGSDNodes *Scheduler = CreateScheduler();
948  {
949  NamedRegionTimer T("sched", "Instruction Scheduling", GroupName,
950  GroupDescription, TimePassesIsEnabled);
951  Scheduler->Run(CurDAG, FuncInfo->MBB);
952  }
953 
954  if (ViewSUnitDAGs && MatchFilterBB)
955  Scheduler->viewGraph();
956 
957  // Emit machine code to BB. This can change 'BB' to the last block being
958  // inserted into.
959  MachineBasicBlock *FirstMBB = FuncInfo->MBB, *LastMBB;
960  {
961  NamedRegionTimer T("emit", "Instruction Creation", GroupName,
962  GroupDescription, TimePassesIsEnabled);
963 
964  // FuncInfo->InsertPt is passed by reference and set to the end of the
965  // scheduled instructions.
966  LastMBB = FuncInfo->MBB = Scheduler->EmitSchedule(FuncInfo->InsertPt);
967  }
968 
969  // If the block was split, make sure we update any references that are used to
970  // update PHI nodes later on.
971  if (FirstMBB != LastMBB)
972  SDB->UpdateSplitBlock(FirstMBB, LastMBB);
973 
974  // Free the scheduler state.
975  {
976  NamedRegionTimer T("cleanup", "Instruction Scheduling Cleanup", GroupName,
977  GroupDescription, TimePassesIsEnabled);
978  delete Scheduler;
979  }
980 
981  // Free the SelectionDAG state, now that we're finished with it.
982  CurDAG->clear();
983 }
984 
985 namespace {
986 
987 /// ISelUpdater - helper class to handle updates of the instruction selection
988 /// graph.
989 class ISelUpdater : public SelectionDAG::DAGUpdateListener {
990  SelectionDAG::allnodes_iterator &ISelPosition;
991 
992 public:
993  ISelUpdater(SelectionDAG &DAG, SelectionDAG::allnodes_iterator &isp)
994  : SelectionDAG::DAGUpdateListener(DAG), ISelPosition(isp) {}
995 
996  /// NodeDeleted - Handle nodes deleted from the graph. If the node being
997  /// deleted is the current ISelPosition node, update ISelPosition.
998  ///
999  void NodeDeleted(SDNode *N, SDNode *E) override {
1000  if (ISelPosition == SelectionDAG::allnodes_iterator(N))
1001  ++ISelPosition;
1002  }
1003 
1004  /// NodeInserted - Handle new nodes inserted into the graph: propagate
1005  /// metadata from root nodes that also applies to new nodes, in case the root
1006  /// is later deleted.
1007  void NodeInserted(SDNode *N) override {
1008  SDNode *CurNode = &*ISelPosition;
1009  if (MDNode *MD = DAG.getPCSections(CurNode))
1010  DAG.addPCSections(N, MD);
1011  }
1012 };
1013 
1014 } // end anonymous namespace
1015 
1016 // This function is used to enforce the topological node id property
1017 // leveraged during instruction selection. Before the selection process all
1018 // nodes are given a non-negative id such that all nodes have a greater id than
1019 // their operands. As this holds transitively we can prune checks that a node N
1020 // is a predecessor of M another by not recursively checking through M's
1021 // operands if N's ID is larger than M's ID. This significantly improves
1022 // performance of various legality checks (e.g. IsLegalToFold / UpdateChains).
1023 
1024 // However, when we fuse multiple nodes into a single node during the
1025 // selection we may induce a predecessor relationship between inputs and
1026 // outputs of distinct nodes being merged, violating the topological property.
1027 // Should a fused node have a successor which has yet to be selected,
1028 // our legality checks would be incorrect. To avoid this we mark all unselected
1029 // successor nodes, i.e. id != -1, as invalid for pruning by bit-negating (x =>
1030 // (-(x+1))) the ids and modify our pruning check to ignore negative Ids of M.
1031 // We use bit-negation to more clearly enforce that node id -1 can only be
1032 // achieved by selected nodes. As the conversion is reversable to the original
1033 // Id, topological pruning can still be leveraged when looking for unselected
1034 // nodes. This method is called internally in all ISel replacement related
1035 // functions.
1038  Nodes.push_back(Node);
1039 
1040  while (!Nodes.empty()) {
1041  SDNode *N = Nodes.pop_back_val();
1042  for (auto *U : N->uses()) {
1043  auto UId = U->getNodeId();
1044  if (UId > 0) {
1045  InvalidateNodeId(U);
1046  Nodes.push_back(U);
1047  }
1048  }
1049  }
1050 }
1051 
1052 // InvalidateNodeId - As explained in EnforceNodeIdInvariant, mark a
1053 // NodeId with the equivalent node id which is invalid for topological
1054 // pruning.
1056  int InvalidId = -(N->getNodeId() + 1);
1057  N->setNodeId(InvalidId);
1058 }
1059 
1060 // getUninvalidatedNodeId - get original uninvalidated node id.
1062  int Id = N->getNodeId();
1063  if (Id < -1)
1064  return -(Id + 1);
1065  return Id;
1066 }
1067 
1068 void SelectionDAGISel::DoInstructionSelection() {
1069  LLVM_DEBUG(dbgs() << "===== Instruction selection begins: "
1070  << printMBBReference(*FuncInfo->MBB) << " '"
1071  << FuncInfo->MBB->getName() << "'\n");
1072 
1074 
1075  // Select target instructions for the DAG.
1076  {
1077  // Number all nodes with a topological order and set DAGSize.
1079 
1080  // Create a dummy node (which is not added to allnodes), that adds
1081  // a reference to the root node, preventing it from being deleted,
1082  // and tracking any changes of the root.
1085  ++ISelPosition;
1086 
1087  // Make sure that ISelPosition gets properly updated when nodes are deleted
1088  // in calls made from this function. New nodes inherit relevant metadata.
1089  ISelUpdater ISU(*CurDAG, ISelPosition);
1090 
1091  // The AllNodes list is now topological-sorted. Visit the
1092  // nodes by starting at the end of the list (the root of the
1093  // graph) and preceding back toward the beginning (the entry
1094  // node).
1095  while (ISelPosition != CurDAG->allnodes_begin()) {
1096  SDNode *Node = &*--ISelPosition;
1097  // Skip dead nodes. DAGCombiner is expected to eliminate all dead nodes,
1098  // but there are currently some corner cases that it misses. Also, this
1099  // makes it theoretically possible to disable the DAGCombiner.
1100  if (Node->use_empty())
1101  continue;
1102 
1103 #ifndef NDEBUG
1105  Nodes.push_back(Node);
1106 
1107  while (!Nodes.empty()) {
1108  auto N = Nodes.pop_back_val();
1109  if (N->getOpcode() == ISD::TokenFactor || N->getNodeId() < 0)
1110  continue;
1111  for (const SDValue &Op : N->op_values()) {
1112  if (Op->getOpcode() == ISD::TokenFactor)
1113  Nodes.push_back(Op.getNode());
1114  else {
1115  // We rely on topological ordering of node ids for checking for
1116  // cycles when fusing nodes during selection. All unselected nodes
1117  // successors of an already selected node should have a negative id.
1118  // This assertion will catch such cases. If this assertion triggers
1119  // it is likely you using DAG-level Value/Node replacement functions
1120  // (versus equivalent ISEL replacement) in backend-specific
1121  // selections. See comment in EnforceNodeIdInvariant for more
1122  // details.
1123  assert(Op->getNodeId() != -1 &&
1124  "Node has already selected predecessor node");
1125  }
1126  }
1127  }
1128 #endif
1129 
1130  // When we are using non-default rounding modes or FP exception behavior
1131  // FP operations are represented by StrictFP pseudo-operations. For
1132  // targets that do not (yet) understand strict FP operations directly,
1133  // we convert them to normal FP opcodes instead at this point. This
1134  // will allow them to be handled by existing target-specific instruction
1135  // selectors.
1136  if (!TLI->isStrictFPEnabled() && Node->isStrictFPOpcode()) {
1137  // For some opcodes, we need to call TLI->getOperationAction using
1138  // the first operand type instead of the result type. Note that this
1139  // must match what SelectionDAGLegalize::LegalizeOp is doing.
1140  EVT ActionVT;
1141  switch (Node->getOpcode()) {
1144  case ISD::STRICT_LRINT:
1145  case ISD::STRICT_LLRINT:
1146  case ISD::STRICT_LROUND:
1147  case ISD::STRICT_LLROUND:
1148  case ISD::STRICT_FSETCC:
1149  case ISD::STRICT_FSETCCS:
1150  ActionVT = Node->getOperand(1).getValueType();
1151  break;
1152  default:
1153  ActionVT = Node->getValueType(0);
1154  break;
1155  }
1156  if (TLI->getOperationAction(Node->getOpcode(), ActionVT)
1158  Node = CurDAG->mutateStrictFPToFP(Node);
1159  }
1160 
1161  LLVM_DEBUG(dbgs() << "\nISEL: Starting selection on root node: ";
1162  Node->dump(CurDAG));
1163 
1164  Select(Node);
1165  }
1166 
1167  CurDAG->setRoot(Dummy.getValue());
1168  }
1169 
1170  LLVM_DEBUG(dbgs() << "\n===== Instruction selection ends:\n");
1171 
1173 }
1174 
1176  for (const User *U : CPI->users()) {
1177  if (const IntrinsicInst *EHPtrCall = dyn_cast<IntrinsicInst>(U)) {
1178  Intrinsic::ID IID = EHPtrCall->getIntrinsicID();
1179  if (IID == Intrinsic::eh_exceptionpointer ||
1180  IID == Intrinsic::eh_exceptioncode)
1181  return true;
1182  }
1183  }
1184  return false;
1185 }
1186 
1187 // wasm.landingpad.index intrinsic is for associating a landing pad index number
1188 // with a catchpad instruction. Retrieve the landing pad index in the intrinsic
1189 // and store the mapping in the function.
1191  const CatchPadInst *CPI) {
1192  MachineFunction *MF = MBB->getParent();
1193  // In case of single catch (...), we don't emit LSDA, so we don't need
1194  // this information.
1195  bool IsSingleCatchAllClause =
1196  CPI->getNumArgOperands() == 1 &&
1197  cast<Constant>(CPI->getArgOperand(0))->isNullValue();
1198  // cathchpads for longjmp use an empty type list, e.g. catchpad within %0 []
1199  // and they don't need LSDA info
1200  bool IsCatchLongjmp = CPI->getNumArgOperands() == 0;
1201  if (!IsSingleCatchAllClause && !IsCatchLongjmp) {
1202  // Create a mapping from landing pad label to landing pad index.
1203  bool IntrFound = false;
1204  for (const User *U : CPI->users()) {
1205  if (const auto *Call = dyn_cast<IntrinsicInst>(U)) {
1206  Intrinsic::ID IID = Call->getIntrinsicID();
1207  if (IID == Intrinsic::wasm_landingpad_index) {
1208  Value *IndexArg = Call->getArgOperand(1);
1209  int Index = cast<ConstantInt>(IndexArg)->getZExtValue();
1211  IntrFound = true;
1212  break;
1213  }
1214  }
1215  }
1216  assert(IntrFound && "wasm.landingpad.index intrinsic not found!");
1217  (void)IntrFound;
1218  }
1219 }
1220 
1221 /// PrepareEHLandingPad - Emit an EH_LABEL, set up live-in registers, and
1222 /// do other setup for EH landing-pad blocks.
1223 bool SelectionDAGISel::PrepareEHLandingPad() {
1224  MachineBasicBlock *MBB = FuncInfo->MBB;
1225  const Constant *PersonalityFn = FuncInfo->Fn->getPersonalityFn();
1226  const BasicBlock *LLVMBB = MBB->getBasicBlock();
1227  const TargetRegisterClass *PtrRC =
1229 
1230  auto Pers = classifyEHPersonality(PersonalityFn);
1231 
1232  // Catchpads have one live-in register, which typically holds the exception
1233  // pointer or code.
1234  if (isFuncletEHPersonality(Pers)) {
1235  if (const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->getFirstNonPHI())) {
1236  if (hasExceptionPointerOrCodeUser(CPI)) {
1237  // Get or create the virtual register to hold the pointer or code. Mark
1238  // the live in physreg and copy into the vreg.
1239  MCPhysReg EHPhysReg = TLI->getExceptionPointerRegister(PersonalityFn);
1240  assert(EHPhysReg && "target lacks exception pointer register");
1241  MBB->addLiveIn(EHPhysReg);
1242  unsigned VReg = FuncInfo->getCatchPadExceptionPointerVReg(CPI, PtrRC);
1243  BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(),
1244  TII->get(TargetOpcode::COPY), VReg)
1245  .addReg(EHPhysReg, RegState::Kill);
1246  }
1247  }
1248  return true;
1249  }
1250 
1251  // Add a label to mark the beginning of the landing pad. Deletion of the
1252  // landing pad can thus be detected via the MachineModuleInfo.
1254 
1255  const MCInstrDesc &II = TII->get(TargetOpcode::EH_LABEL);
1256  BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(), II)
1257  .addSym(Label);
1258 
1259  // If the unwinder does not preserve all registers, ensure that the
1260  // function marks the clobbered registers as used.
1262  if (auto *RegMask = TRI.getCustomEHPadPreservedMask(*MF))
1264 
1265  if (Pers == EHPersonality::Wasm_CXX) {
1266  if (const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->getFirstNonPHI()))
1268  } else {
1269  // Assign the call site to the landing pad's begin label.
1270  MF->setCallSiteLandingPad(Label, SDB->LPadToCallSiteMap[MBB]);
1271  // Mark exception register as live in.
1272  if (unsigned Reg = TLI->getExceptionPointerRegister(PersonalityFn))
1273  FuncInfo->ExceptionPointerVirtReg = MBB->addLiveIn(Reg, PtrRC);
1274  // Mark exception selector register as live in.
1275  if (unsigned Reg = TLI->getExceptionSelectorRegister(PersonalityFn))
1276  FuncInfo->ExceptionSelectorVirtReg = MBB->addLiveIn(Reg, PtrRC);
1277  }
1278 
1279  return true;
1280 }
1281 
1282 /// isFoldedOrDeadInstruction - Return true if the specified instruction is
1283 /// side-effect free and is either dead or folded into a generated instruction.
1284 /// Return false if it needs to be emitted.
1286  const FunctionLoweringInfo &FuncInfo) {
1287  return !I->mayWriteToMemory() && // Side-effecting instructions aren't folded.
1288  !I->isTerminator() && // Terminators aren't folded.
1289  !isa<DbgInfoIntrinsic>(I) && // Debug instructions aren't folded.
1290  !I->isEHPad() && // EH pad instructions aren't folded.
1291  !FuncInfo.isExportedInst(I); // Exported instrs must be computed.
1292 }
1293 
1294 /// Collect llvm.dbg.declare information. This is done after argument lowering
1295 /// in case the declarations refer to arguments.
1297  MachineFunction *MF = FuncInfo.MF;
1298  const DataLayout &DL = MF->getDataLayout();
1299  for (const BasicBlock &BB : *FuncInfo.Fn) {
1300  for (const Instruction &I : BB) {
1301  const DbgDeclareInst *DI = dyn_cast<DbgDeclareInst>(&I);
1302  if (!DI)
1303  continue;
1304 
1305  assert(DI->getVariable() && "Missing variable");
1306  assert(DI->getDebugLoc() && "Missing location");
1307  const Value *Address = DI->getAddress();
1308  if (!Address) {
1309  LLVM_DEBUG(dbgs() << "processDbgDeclares skipping " << *DI
1310  << " (bad address)\n");
1311  continue;
1312  }
1313 
1314  // Look through casts and constant offset GEPs. These mostly come from
1315  // inalloca.
1316  APInt Offset(DL.getTypeSizeInBits(Address->getType()), 0);
1317  Address = Address->stripAndAccumulateInBoundsConstantOffsets(DL, Offset);
1318 
1319  // Check if the variable is a static alloca or a byval or inalloca
1320  // argument passed in memory. If it is not, then we will ignore this
1321  // intrinsic and handle this during isel like dbg.value.
1322  int FI = std::numeric_limits<int>::max();
1323  if (const auto *AI = dyn_cast<AllocaInst>(Address)) {
1324  auto SI = FuncInfo.StaticAllocaMap.find(AI);
1325  if (SI != FuncInfo.StaticAllocaMap.end())
1326  FI = SI->second;
1327  } else if (const auto *Arg = dyn_cast<Argument>(Address))
1328  FI = FuncInfo.getArgumentFrameIndex(Arg);
1329 
1330  if (FI == std::numeric_limits<int>::max())
1331  continue;
1332 
1333  DIExpression *Expr = DI->getExpression();
1334  if (Offset.getBoolValue())
1336  Offset.getZExtValue());
1337  LLVM_DEBUG(dbgs() << "processDbgDeclares: setVariableDbgInfo FI=" << FI
1338  << ", " << *DI << "\n");
1339  MF->setVariableDbgInfo(DI->getVariable(), Expr, FI, DI->getDebugLoc());
1340  }
1341  }
1342 }
1343 
1344 void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) {
1345  FastISelFailed = false;
1346  // Initialize the Fast-ISel state, if needed.
1347  FastISel *FastIS = nullptr;
1348  if (TM.Options.EnableFastISel) {
1349  LLVM_DEBUG(dbgs() << "Enabling fast-isel\n");
1350  FastIS = TLI->createFastISel(*FuncInfo, LibInfo);
1351  if (FastIS)
1353  }
1354 
1356 
1357  // Lower arguments up front. An RPO iteration always visits the entry block
1358  // first.
1359  assert(*RPOT.begin() == &Fn.getEntryBlock());
1360  ++NumEntryBlocks;
1361 
1362  // Set up FuncInfo for ISel. Entry blocks never have PHIs.
1363  FuncInfo->MBB = FuncInfo->MBBMap[&Fn.getEntryBlock()];
1364  FuncInfo->InsertPt = FuncInfo->MBB->begin();
1365 
1367 
1368  if (!FastIS) {
1369  LowerArguments(Fn);
1370  } else {
1371  // See if fast isel can lower the arguments.
1372  FastIS->startNewBlock();
1373  if (!FastIS->lowerArguments()) {
1374  FastISelFailed = true;
1375  // Fast isel failed to lower these arguments
1376  ++NumFastIselFailLowerArguments;
1377 
1378  OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1379  Fn.getSubprogram(),
1380  &Fn.getEntryBlock());
1381  R << "FastISel didn't lower all arguments: "
1382  << ore::NV("Prototype", Fn.getType());
1384 
1385  // Use SelectionDAG argument lowering
1386  LowerArguments(Fn);
1387  CurDAG->setRoot(SDB->getControlRoot());
1388  SDB->clear();
1389  CodeGenAndEmitDAG();
1390  }
1391 
1392  // If we inserted any instructions at the beginning, make a note of
1393  // where they are, so we can be sure to emit subsequent instructions
1394  // after them.
1395  if (FuncInfo->InsertPt != FuncInfo->MBB->begin())
1396  FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt));
1397  else
1398  FastIS->setLastLocalValue(nullptr);
1399  }
1400 
1401  bool Inserted = SwiftError->createEntriesInEntryBlock(SDB->getCurDebugLoc());
1402 
1403  if (FastIS && Inserted)
1404  FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt));
1405 
1407 
1408  // Iterate over all basic blocks in the function.
1409  StackProtector &SP = getAnalysis<StackProtector>();
1410  for (const BasicBlock *LLVMBB : RPOT) {
1411  if (OptLevel != CodeGenOpt::None) {
1412  bool AllPredsVisited = true;
1413  for (const BasicBlock *Pred : predecessors(LLVMBB)) {
1414  if (!FuncInfo->VisitedBBs.count(Pred)) {
1415  AllPredsVisited = false;
1416  break;
1417  }
1418  }
1419 
1420  if (AllPredsVisited) {
1421  for (const PHINode &PN : LLVMBB->phis())
1422  FuncInfo->ComputePHILiveOutRegInfo(&PN);
1423  } else {
1424  for (const PHINode &PN : LLVMBB->phis())
1425  FuncInfo->InvalidatePHILiveOutRegInfo(&PN);
1426  }
1427 
1428  FuncInfo->VisitedBBs.insert(LLVMBB);
1429  }
1430 
1431  BasicBlock::const_iterator const Begin =
1432  LLVMBB->getFirstNonPHI()->getIterator();
1433  BasicBlock::const_iterator const End = LLVMBB->end();
1434  BasicBlock::const_iterator BI = End;
1435 
1436  FuncInfo->MBB = FuncInfo->MBBMap[LLVMBB];
1437  if (!FuncInfo->MBB)
1438  continue; // Some blocks like catchpads have no code or MBB.
1439 
1440  // Insert new instructions after any phi or argument setup code.
1441  FuncInfo->InsertPt = FuncInfo->MBB->end();
1442 
1443  // Setup an EH landing-pad block.
1444  FuncInfo->ExceptionPointerVirtReg = 0;
1445  FuncInfo->ExceptionSelectorVirtReg = 0;
1446  if (LLVMBB->isEHPad())
1447  if (!PrepareEHLandingPad())
1448  continue;
1449 
1450  // Before doing SelectionDAG ISel, see if FastISel has been requested.
1451  if (FastIS) {
1452  if (LLVMBB != &Fn.getEntryBlock())
1453  FastIS->startNewBlock();
1454 
1455  unsigned NumFastIselRemaining = std::distance(Begin, End);
1456 
1457  // Pre-assign swifterror vregs.
1458  SwiftError->preassignVRegs(FuncInfo->MBB, Begin, End);
1459 
1460  // Do FastISel on as many instructions as possible.
1461  for (; BI != Begin; --BI) {
1462  const Instruction *Inst = &*std::prev(BI);
1463 
1464  // If we no longer require this instruction, skip it.
1465  if (isFoldedOrDeadInstruction(Inst, *FuncInfo) ||
1466  ElidedArgCopyInstrs.count(Inst)) {
1467  --NumFastIselRemaining;
1468  continue;
1469  }
1470 
1471  // Bottom-up: reset the insert pos at the top, after any local-value
1472  // instructions.
1473  FastIS->recomputeInsertPt();
1474 
1475  // Try to select the instruction with FastISel.
1476  if (FastIS->selectInstruction(Inst)) {
1477  --NumFastIselRemaining;
1478  ++NumFastIselSuccess;
1479  // If fast isel succeeded, skip over all the folded instructions, and
1480  // then see if there is a load right before the selected instructions.
1481  // Try to fold the load if so.
1482  const Instruction *BeforeInst = Inst;
1483  while (BeforeInst != &*Begin) {
1484  BeforeInst = &*std::prev(BasicBlock::const_iterator(BeforeInst));
1485  if (!isFoldedOrDeadInstruction(BeforeInst, *FuncInfo))
1486  break;
1487  }
1488  if (BeforeInst != Inst && isa<LoadInst>(BeforeInst) &&
1489  BeforeInst->hasOneUse() &&
1490  FastIS->tryToFoldLoad(cast<LoadInst>(BeforeInst), Inst)) {
1491  // If we succeeded, don't re-select the load.
1492  LLVM_DEBUG(dbgs()
1493  << "FastISel folded load: " << *BeforeInst << "\n");
1494  BI = std::next(BasicBlock::const_iterator(BeforeInst));
1495  --NumFastIselRemaining;
1496  ++NumFastIselSuccess;
1497  }
1498  continue;
1499  }
1500 
1501  FastISelFailed = true;
1502 
1503  // Then handle certain instructions as single-LLVM-Instruction blocks.
1504  // We cannot separate out GCrelocates to their own blocks since we need
1505  // to keep track of gc-relocates for a particular gc-statepoint. This is
1506  // done by SelectionDAGBuilder::LowerAsSTATEPOINT, called before
1507  // visitGCRelocate.
1508  if (isa<CallInst>(Inst) && !isa<GCStatepointInst>(Inst) &&
1509  !isa<GCRelocateInst>(Inst) && !isa<GCResultInst>(Inst)) {
1510  OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1511  Inst->getDebugLoc(), LLVMBB);
1512 
1513  R << "FastISel missed call";
1514 
1515  if (R.isEnabled() || EnableFastISelAbort) {
1516  std::string InstStrStorage;
1517  raw_string_ostream InstStr(InstStrStorage);
1518  InstStr << *Inst;
1519 
1520  R << ": " << InstStr.str();
1521  }
1522 
1524 
1525  if (!Inst->getType()->isVoidTy() && !Inst->getType()->isTokenTy() &&
1526  !Inst->use_empty()) {
1527  Register &R = FuncInfo->ValueMap[Inst];
1528  if (!R)
1529  R = FuncInfo->CreateRegs(Inst);
1530  }
1531 
1532  bool HadTailCall = false;
1533  MachineBasicBlock::iterator SavedInsertPt = FuncInfo->InsertPt;
1534  SelectBasicBlock(Inst->getIterator(), BI, HadTailCall);
1535 
1536  // If the call was emitted as a tail call, we're done with the block.
1537  // We also need to delete any previously emitted instructions.
1538  if (HadTailCall) {
1539  FastIS->removeDeadCode(SavedInsertPt, FuncInfo->MBB->end());
1540  --BI;
1541  break;
1542  }
1543 
1544  // Recompute NumFastIselRemaining as Selection DAG instruction
1545  // selection may have handled the call, input args, etc.
1546  unsigned RemainingNow = std::distance(Begin, BI);
1547  NumFastIselFailures += NumFastIselRemaining - RemainingNow;
1548  NumFastIselRemaining = RemainingNow;
1549  continue;
1550  }
1551 
1552  OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1553  Inst->getDebugLoc(), LLVMBB);
1554 
1555  bool ShouldAbort = EnableFastISelAbort;
1556  if (Inst->isTerminator()) {
1557  // Use a different message for terminator misses.
1558  R << "FastISel missed terminator";
1559  // Don't abort for terminator unless the level is really high
1560  ShouldAbort = (EnableFastISelAbort > 2);
1561  } else {
1562  R << "FastISel missed";
1563  }
1564 
1565  if (R.isEnabled() || EnableFastISelAbort) {
1566  std::string InstStrStorage;
1567  raw_string_ostream InstStr(InstStrStorage);
1568  InstStr << *Inst;
1569  R << ": " << InstStr.str();
1570  }
1571 
1572  reportFastISelFailure(*MF, *ORE, R, ShouldAbort);
1573 
1574  NumFastIselFailures += NumFastIselRemaining;
1575  break;
1576  }
1577 
1578  FastIS->recomputeInsertPt();
1579  }
1580 
1581  if (SP.shouldEmitSDCheck(*LLVMBB)) {
1582  bool FunctionBasedInstrumentation =
1584  SDB->SPDescriptor.initialize(LLVMBB, FuncInfo->MBBMap[LLVMBB],
1585  FunctionBasedInstrumentation);
1586  }
1587 
1588  if (Begin != BI)
1589  ++NumDAGBlocks;
1590  else
1591  ++NumFastIselBlocks;
1592 
1593  if (Begin != BI) {
1594  // Run SelectionDAG instruction selection on the remainder of the block
1595  // not handled by FastISel. If FastISel is not run, this is the entire
1596  // block.
1597  bool HadTailCall;
1598  SelectBasicBlock(Begin, BI, HadTailCall);
1599 
1600  // But if FastISel was run, we already selected some of the block.
1601  // If we emitted a tail-call, we need to delete any previously emitted
1602  // instruction that follows it.
1603  if (FastIS && HadTailCall && FuncInfo->InsertPt != FuncInfo->MBB->end())
1604  FastIS->removeDeadCode(FuncInfo->InsertPt, FuncInfo->MBB->end());
1605  }
1606 
1607  if (FastIS)
1608  FastIS->finishBasicBlock();
1609  FinishBasicBlock();
1610  FuncInfo->PHINodesToUpdate.clear();
1611  ElidedArgCopyInstrs.clear();
1612  }
1613 
1615 
1617 
1618  delete FastIS;
1619  SDB->clearDanglingDebugInfo();
1620  SDB->SPDescriptor.resetPerFunctionState();
1621 }
1622 
1623 void
1624 SelectionDAGISel::FinishBasicBlock() {
1625  LLVM_DEBUG(dbgs() << "Total amount of phi nodes to update: "
1626  << FuncInfo->PHINodesToUpdate.size() << "\n";
1627  for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e;
1628  ++i) dbgs()
1629  << "Node " << i << " : (" << FuncInfo->PHINodesToUpdate[i].first
1630  << ", " << FuncInfo->PHINodesToUpdate[i].second << ")\n");
1631 
1632  // Next, now that we know what the last MBB the LLVM BB expanded is, update
1633  // PHI nodes in successors.
1634  for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) {
1635  MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[i].first);
1636  assert(PHI->isPHI() &&
1637  "This is not a machine PHI node that we are updating!");
1638  if (!FuncInfo->MBB->isSuccessor(PHI->getParent()))
1639  continue;
1640  PHI.addReg(FuncInfo->PHINodesToUpdate[i].second).addMBB(FuncInfo->MBB);
1641  }
1642 
1643  // Handle stack protector.
1644  if (SDB->SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
1645  // The target provides a guard check function. There is no need to
1646  // generate error handling code or to split current basic block.
1647  MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB();
1648 
1649  // Add load and check to the basicblock.
1650  FuncInfo->MBB = ParentMBB;
1651  FuncInfo->InsertPt =
1652  findSplitPointForStackProtector(ParentMBB, *TII);
1653  SDB->visitSPDescriptorParent(SDB->SPDescriptor, ParentMBB);
1654  CurDAG->setRoot(SDB->getRoot());
1655  SDB->clear();
1656  CodeGenAndEmitDAG();
1657 
1658  // Clear the Per-BB State.
1659  SDB->SPDescriptor.resetPerBBState();
1660  } else if (SDB->SPDescriptor.shouldEmitStackProtector()) {
1661  MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB();
1662  MachineBasicBlock *SuccessMBB = SDB->SPDescriptor.getSuccessMBB();
1663 
1664  // Find the split point to split the parent mbb. At the same time copy all
1665  // physical registers used in the tail of parent mbb into virtual registers
1666  // before the split point and back into physical registers after the split
1667  // point. This prevents us needing to deal with Live-ins and many other
1668  // register allocation issues caused by us splitting the parent mbb. The
1669  // register allocator will clean up said virtual copies later on.
1670  MachineBasicBlock::iterator SplitPoint =
1671  findSplitPointForStackProtector(ParentMBB, *TII);
1672 
1673  // Splice the terminator of ParentMBB into SuccessMBB.
1674  SuccessMBB->splice(SuccessMBB->end(), ParentMBB,
1675  SplitPoint,
1676  ParentMBB->end());
1677 
1678  // Add compare/jump on neq/jump to the parent BB.
1679  FuncInfo->MBB = ParentMBB;
1680  FuncInfo->InsertPt = ParentMBB->end();
1681  SDB->visitSPDescriptorParent(SDB->SPDescriptor, ParentMBB);
1682  CurDAG->setRoot(SDB->getRoot());
1683  SDB->clear();
1684  CodeGenAndEmitDAG();
1685 
1686  // CodeGen Failure MBB if we have not codegened it yet.
1687  MachineBasicBlock *FailureMBB = SDB->SPDescriptor.getFailureMBB();
1688  if (FailureMBB->empty()) {
1689  FuncInfo->MBB = FailureMBB;
1690  FuncInfo->InsertPt = FailureMBB->end();
1691  SDB->visitSPDescriptorFailure(SDB->SPDescriptor);
1692  CurDAG->setRoot(SDB->getRoot());
1693  SDB->clear();
1694  CodeGenAndEmitDAG();
1695  }
1696 
1697  // Clear the Per-BB State.
1698  SDB->SPDescriptor.resetPerBBState();
1699  }
1700 
1701  // Lower each BitTestBlock.
1702  for (auto &BTB : SDB->SL->BitTestCases) {
1703  // Lower header first, if it wasn't already lowered
1704  if (!BTB.Emitted) {
1705  // Set the current basic block to the mbb we wish to insert the code into
1706  FuncInfo->MBB = BTB.Parent;
1707  FuncInfo->InsertPt = FuncInfo->MBB->end();
1708  // Emit the code
1709  SDB->visitBitTestHeader(BTB, FuncInfo->MBB);
1710  CurDAG->setRoot(SDB->getRoot());
1711  SDB->clear();
1712  CodeGenAndEmitDAG();
1713  }
1714 
1715  BranchProbability UnhandledProb = BTB.Prob;
1716  for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
1717  UnhandledProb -= BTB.Cases[j].ExtraProb;
1718  // Set the current basic block to the mbb we wish to insert the code into
1719  FuncInfo->MBB = BTB.Cases[j].ThisBB;
1720  FuncInfo->InsertPt = FuncInfo->MBB->end();
1721  // Emit the code
1722 
1723  // If all cases cover a contiguous range, it is not necessary to jump to
1724  // the default block after the last bit test fails. This is because the
1725  // range check during bit test header creation has guaranteed that every
1726  // case here doesn't go outside the range. In this case, there is no need
1727  // to perform the last bit test, as it will always be true. Instead, make
1728  // the second-to-last bit-test fall through to the target of the last bit
1729  // test, and delete the last bit test.
1730 
1731  MachineBasicBlock *NextMBB;
1732  if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
1733  // Second-to-last bit-test with contiguous range or omitted range
1734  // check: fall through to the target of the final bit test.
1735  NextMBB = BTB.Cases[j + 1].TargetBB;
1736  } else if (j + 1 == ej) {
1737  // For the last bit test, fall through to Default.
1738  NextMBB = BTB.Default;
1739  } else {
1740  // Otherwise, fall through to the next bit test.
1741  NextMBB = BTB.Cases[j + 1].ThisBB;
1742  }
1743 
1744  SDB->visitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j],
1745  FuncInfo->MBB);
1746 
1747  CurDAG->setRoot(SDB->getRoot());
1748  SDB->clear();
1749  CodeGenAndEmitDAG();
1750 
1751  if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
1752  // Since we're not going to use the final bit test, remove it.
1753  BTB.Cases.pop_back();
1754  break;
1755  }
1756  }
1757 
1758  // Update PHI Nodes
1759  for (const std::pair<MachineInstr *, unsigned> &P :
1760  FuncInfo->PHINodesToUpdate) {
1761  MachineInstrBuilder PHI(*MF, P.first);
1762  MachineBasicBlock *PHIBB = PHI->getParent();
1763  assert(PHI->isPHI() &&
1764  "This is not a machine PHI node that we are updating!");
1765  // This is "default" BB. We have two jumps to it. From "header" BB and
1766  // from last "case" BB, unless the latter was skipped.
1767  if (PHIBB == BTB.Default) {
1768  PHI.addReg(P.second).addMBB(BTB.Parent);
1769  if (!BTB.ContiguousRange) {
1770  PHI.addReg(P.second).addMBB(BTB.Cases.back().ThisBB);
1771  }
1772  }
1773  // One of "cases" BB.
1774  for (const SwitchCG::BitTestCase &BT : BTB.Cases) {
1775  MachineBasicBlock* cBB = BT.ThisBB;
1776  if (cBB->isSuccessor(PHIBB))
1777  PHI.addReg(P.second).addMBB(cBB);
1778  }
1779  }
1780  }
1781  SDB->SL->BitTestCases.clear();
1782 
1783  // If the JumpTable record is filled in, then we need to emit a jump table.
1784  // Updating the PHI nodes is tricky in this case, since we need to determine
1785  // whether the PHI is a successor of the range check MBB or the jump table MBB
1786  for (unsigned i = 0, e = SDB->SL->JTCases.size(); i != e; ++i) {
1787  // Lower header first, if it wasn't already lowered
1788  if (!SDB->SL->JTCases[i].first.Emitted) {
1789  // Set the current basic block to the mbb we wish to insert the code into
1790  FuncInfo->MBB = SDB->SL->JTCases[i].first.HeaderBB;
1791  FuncInfo->InsertPt = FuncInfo->MBB->end();
1792  // Emit the code
1793  SDB->visitJumpTableHeader(SDB->SL->JTCases[i].second,
1794  SDB->SL->JTCases[i].first, FuncInfo->MBB);
1795  CurDAG->setRoot(SDB->getRoot());
1796  SDB->clear();
1797  CodeGenAndEmitDAG();
1798  }
1799 
1800  // Set the current basic block to the mbb we wish to insert the code into
1801  FuncInfo->MBB = SDB->SL->JTCases[i].second.MBB;
1802  FuncInfo->InsertPt = FuncInfo->MBB->end();
1803  // Emit the code
1804  SDB->visitJumpTable(SDB->SL->JTCases[i].second);
1805  CurDAG->setRoot(SDB->getRoot());
1806  SDB->clear();
1807  CodeGenAndEmitDAG();
1808 
1809  // Update PHI Nodes
1810  for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size();
1811  pi != pe; ++pi) {
1812  MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first);
1813  MachineBasicBlock *PHIBB = PHI->getParent();
1814  assert(PHI->isPHI() &&
1815  "This is not a machine PHI node that we are updating!");
1816  // "default" BB. We can go there only from header BB.
1817  if (PHIBB == SDB->SL->JTCases[i].second.Default)
1818  PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second)
1819  .addMBB(SDB->SL->JTCases[i].first.HeaderBB);
1820  // JT BB. Just iterate over successors here
1821  if (FuncInfo->MBB->isSuccessor(PHIBB))
1822  PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(FuncInfo->MBB);
1823  }
1824  }
1825  SDB->SL->JTCases.clear();
1826 
1827  // If we generated any switch lowering information, build and codegen any
1828  // additional DAGs necessary.
1829  for (unsigned i = 0, e = SDB->SL->SwitchCases.size(); i != e; ++i) {
1830  // Set the current basic block to the mbb we wish to insert the code into
1831  FuncInfo->MBB = SDB->SL->SwitchCases[i].ThisBB;
1832  FuncInfo->InsertPt = FuncInfo->MBB->end();
1833 
1834  // Determine the unique successors.
1836  Succs.push_back(SDB->SL->SwitchCases[i].TrueBB);
1837  if (SDB->SL->SwitchCases[i].TrueBB != SDB->SL->SwitchCases[i].FalseBB)
1838  Succs.push_back(SDB->SL->SwitchCases[i].FalseBB);
1839 
1840  // Emit the code. Note that this could result in FuncInfo->MBB being split.
1841  SDB->visitSwitchCase(SDB->SL->SwitchCases[i], FuncInfo->MBB);
1842  CurDAG->setRoot(SDB->getRoot());
1843  SDB->clear();
1844  CodeGenAndEmitDAG();
1845 
1846  // Remember the last block, now that any splitting is done, for use in
1847  // populating PHI nodes in successors.
1848  MachineBasicBlock *ThisBB = FuncInfo->MBB;
1849 
1850  // Handle any PHI nodes in successors of this chunk, as if we were coming
1851  // from the original BB before switch expansion. Note that PHI nodes can
1852  // occur multiple times in PHINodesToUpdate. We have to be very careful to
1853  // handle them the right number of times.
1854  for (unsigned i = 0, e = Succs.size(); i != e; ++i) {
1855  FuncInfo->MBB = Succs[i];
1856  FuncInfo->InsertPt = FuncInfo->MBB->end();
1857  // FuncInfo->MBB may have been removed from the CFG if a branch was
1858  // constant folded.
1859  if (ThisBB->isSuccessor(FuncInfo->MBB)) {
1861  MBBI = FuncInfo->MBB->begin(), MBBE = FuncInfo->MBB->end();
1862  MBBI != MBBE && MBBI->isPHI(); ++MBBI) {
1864  // This value for this PHI node is recorded in PHINodesToUpdate.
1865  for (unsigned pn = 0; ; ++pn) {
1866  assert(pn != FuncInfo->PHINodesToUpdate.size() &&
1867  "Didn't find PHI entry!");
1868  if (FuncInfo->PHINodesToUpdate[pn].first == PHI) {
1869  PHI.addReg(FuncInfo->PHINodesToUpdate[pn].second).addMBB(ThisBB);
1870  break;
1871  }
1872  }
1873  }
1874  }
1875  }
1876  }
1877  SDB->SL->SwitchCases.clear();
1878 }
1879 
1880 /// Create the scheduler. If a specific scheduler was specified
1881 /// via the SchedulerRegistry, use it, otherwise select the
1882 /// one preferred by the target.
1883 ///
1884 ScheduleDAGSDNodes *SelectionDAGISel::CreateScheduler() {
1885  return ISHeuristic(this, OptLevel);
1886 }
1887 
1888 //===----------------------------------------------------------------------===//
1889 // Helper functions used by the generated instruction selector.
1890 //===----------------------------------------------------------------------===//
1891 // Calls to these methods are generated by tblgen.
1892 
1893 /// CheckAndMask - The isel is trying to match something like (and X, 255). If
1894 /// the dag combiner simplified the 255, we still want to match. RHS is the
1895 /// actual value in the DAG on the RHS of an AND, and DesiredMaskS is the value
1896 /// specified in the .td file (e.g. 255).
1898  int64_t DesiredMaskS) const {
1899  const APInt &ActualMask = RHS->getAPIntValue();
1900  const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS);
1901 
1902  // If the actual mask exactly matches, success!
1903  if (ActualMask == DesiredMask)
1904  return true;
1905 
1906  // If the actual AND mask is allowing unallowed bits, this doesn't match.
1907  if (!ActualMask.isSubsetOf(DesiredMask))
1908  return false;
1909 
1910  // Otherwise, the DAG Combiner may have proven that the value coming in is
1911  // either already zero or is not demanded. Check for known zero input bits.
1912  APInt NeededMask = DesiredMask & ~ActualMask;
1913  if (CurDAG->MaskedValueIsZero(LHS, NeededMask))
1914  return true;
1915 
1916  // TODO: check to see if missing bits are just not demanded.
1917 
1918  // Otherwise, this pattern doesn't match.
1919  return false;
1920 }
1921 
1922 /// CheckOrMask - The isel is trying to match something like (or X, 255). If
1923 /// the dag combiner simplified the 255, we still want to match. RHS is the
1924 /// actual value in the DAG on the RHS of an OR, and DesiredMaskS is the value
1925 /// specified in the .td file (e.g. 255).
1927  int64_t DesiredMaskS) const {
1928  const APInt &ActualMask = RHS->getAPIntValue();
1929  const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS);
1930 
1931  // If the actual mask exactly matches, success!
1932  if (ActualMask == DesiredMask)
1933  return true;
1934 
1935  // If the actual AND mask is allowing unallowed bits, this doesn't match.
1936  if (!ActualMask.isSubsetOf(DesiredMask))
1937  return false;
1938 
1939  // Otherwise, the DAG Combiner may have proven that the value coming in is
1940  // either already zero or is not demanded. Check for known zero input bits.
1941  APInt NeededMask = DesiredMask & ~ActualMask;
1943 
1944  // If all the missing bits in the or are already known to be set, match!
1945  if (NeededMask.isSubsetOf(Known.One))
1946  return true;
1947 
1948  // TODO: check to see if missing bits are just not demanded.
1949 
1950  // Otherwise, this pattern doesn't match.
1951  return false;
1952 }
1953 
1954 /// SelectInlineAsmMemoryOperands - Calls to this are automatically generated
1955 /// by tblgen. Others should not call it.
1957  const SDLoc &DL) {
1958  std::vector<SDValue> InOps;
1959  std::swap(InOps, Ops);
1960 
1961  Ops.push_back(InOps[InlineAsm::Op_InputChain]); // 0
1962  Ops.push_back(InOps[InlineAsm::Op_AsmString]); // 1
1963  Ops.push_back(InOps[InlineAsm::Op_MDNode]); // 2, !srcloc
1964  Ops.push_back(InOps[InlineAsm::Op_ExtraInfo]); // 3 (SideEffect, AlignStack)
1965 
1966  unsigned i = InlineAsm::Op_FirstOperand, e = InOps.size();
1967  if (InOps[e-1].getValueType() == MVT::Glue)
1968  --e; // Don't process a glue operand if it is here.
1969 
1970  while (i != e) {
1971  unsigned Flags = cast<ConstantSDNode>(InOps[i])->getZExtValue();
1972  if (!InlineAsm::isMemKind(Flags)) {
1973  // Just skip over this operand, copying the operands verbatim.
1974  Ops.insert(Ops.end(), InOps.begin()+i,
1975  InOps.begin()+i+InlineAsm::getNumOperandRegisters(Flags) + 1);
1976  i += InlineAsm::getNumOperandRegisters(Flags) + 1;
1977  } else {
1979  "Memory operand with multiple values?");
1980 
1981  unsigned TiedToOperand;
1982  if (InlineAsm::isUseOperandTiedToDef(Flags, TiedToOperand)) {
1983  // We need the constraint ID from the operand this is tied to.
1984  unsigned CurOp = InlineAsm::Op_FirstOperand;
1985  Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
1986  for (; TiedToOperand; --TiedToOperand) {
1987  CurOp += InlineAsm::getNumOperandRegisters(Flags)+1;
1988  Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
1989  }
1990  }
1991 
1992  // Otherwise, this is a memory operand. Ask the target to select it.
1993  std::vector<SDValue> SelOps;
1994  unsigned ConstraintID = InlineAsm::getMemoryConstraintID(Flags);
1995  if (SelectInlineAsmMemoryOperand(InOps[i+1], ConstraintID, SelOps))
1996  report_fatal_error("Could not match memory address. Inline asm"
1997  " failure!");
1998 
1999  // Add this to the output node.
2000  unsigned NewFlags =
2002  NewFlags = InlineAsm::getFlagWordForMem(NewFlags, ConstraintID);
2003  Ops.push_back(CurDAG->getTargetConstant(NewFlags, DL, MVT::i32));
2004  llvm::append_range(Ops, SelOps);
2005  i += 2;
2006  }
2007  }
2008 
2009  // Add the glue input back if present.
2010  if (e != InOps.size())
2011  Ops.push_back(InOps.back());
2012 }
2013 
2014 /// findGlueUse - Return use of MVT::Glue value produced by the specified
2015 /// SDNode.
2016 ///
2018  unsigned FlagResNo = N->getNumValues()-1;
2019  for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
2020  SDUse &Use = I.getUse();
2021  if (Use.getResNo() == FlagResNo)
2022  return Use.getUser();
2023  }
2024  return nullptr;
2025 }
2026 
2027 /// findNonImmUse - Return true if "Def" is a predecessor of "Root" via a path
2028 /// beyond "ImmedUse". We may ignore chains as they are checked separately.
2029 static bool findNonImmUse(SDNode *Root, SDNode *Def, SDNode *ImmedUse,
2030  bool IgnoreChains) {
2033  // Only check if we have non-immediate uses of Def.
2034  if (ImmedUse->isOnlyUserOf(Def))
2035  return false;
2036 
2037  // We don't care about paths to Def that go through ImmedUse so mark it
2038  // visited and mark non-def operands as used.
2039  Visited.insert(ImmedUse);
2040  for (const SDValue &Op : ImmedUse->op_values()) {
2041  SDNode *N = Op.getNode();
2042  // Ignore chain deps (they are validated by
2043  // HandleMergeInputChains) and immediate uses
2044  if ((Op.getValueType() == MVT::Other && IgnoreChains) || N == Def)
2045  continue;
2046  if (!Visited.insert(N).second)
2047  continue;
2048  WorkList.push_back(N);
2049  }
2050 
2051  // Initialize worklist to operands of Root.
2052  if (Root != ImmedUse) {
2053  for (const SDValue &Op : Root->op_values()) {
2054  SDNode *N = Op.getNode();
2055  // Ignore chains (they are validated by HandleMergeInputChains)
2056  if ((Op.getValueType() == MVT::Other && IgnoreChains) || N == Def)
2057  continue;
2058  if (!Visited.insert(N).second)
2059  continue;
2060  WorkList.push_back(N);
2061  }
2062  }
2063 
2064  return SDNode::hasPredecessorHelper(Def, Visited, WorkList, 0, true);
2065 }
2066 
2067 /// IsProfitableToFold - Returns true if it's profitable to fold the specific
2068 /// operand node N of U during instruction selection that starts at Root.
2070  SDNode *Root) const {
2071  if (OptLevel == CodeGenOpt::None) return false;
2072  return N.hasOneUse();
2073 }
2074 
2075 /// IsLegalToFold - Returns true if the specific operand node N of
2076 /// U can be folded during instruction selection that starts at Root.
2078  CodeGenOpt::Level OptLevel,
2079  bool IgnoreChains) {
2080  if (OptLevel == CodeGenOpt::None) return false;
2081 
2082  // If Root use can somehow reach N through a path that that doesn't contain
2083  // U then folding N would create a cycle. e.g. In the following
2084  // diagram, Root can reach N through X. If N is folded into Root, then
2085  // X is both a predecessor and a successor of U.
2086  //
2087  // [N*] //
2088  // ^ ^ //
2089  // / \ //
2090  // [U*] [X]? //
2091  // ^ ^ //
2092  // \ / //
2093  // \ / //
2094  // [Root*] //
2095  //
2096  // * indicates nodes to be folded together.
2097  //
2098  // If Root produces glue, then it gets (even more) interesting. Since it
2099  // will be "glued" together with its glue use in the scheduler, we need to
2100  // check if it might reach N.
2101  //
2102  // [N*] //
2103  // ^ ^ //
2104  // / \ //
2105  // [U*] [X]? //
2106  // ^ ^ //
2107  // \ \ //
2108  // \ | //
2109  // [Root*] | //
2110  // ^ | //
2111  // f | //
2112  // | / //
2113  // [Y] / //
2114  // ^ / //
2115  // f / //
2116  // | / //
2117  // [GU] //
2118  //
2119  // If GU (glue use) indirectly reaches N (the load), and Root folds N
2120  // (call it Fold), then X is a predecessor of GU and a successor of
2121  // Fold. But since Fold and GU are glued together, this will create
2122  // a cycle in the scheduling graph.
2123 
2124  // If the node has glue, walk down the graph to the "lowest" node in the
2125  // glueged set.
2126  EVT VT = Root->getValueType(Root->getNumValues()-1);
2127  while (VT == MVT::Glue) {
2128  SDNode *GU = findGlueUse(Root);
2129  if (!GU)
2130  break;
2131  Root = GU;
2132  VT = Root->getValueType(Root->getNumValues()-1);
2133 
2134  // If our query node has a glue result with a use, we've walked up it. If
2135  // the user (which has already been selected) has a chain or indirectly uses
2136  // the chain, HandleMergeInputChains will not consider it. Because of
2137  // this, we cannot ignore chains in this predicate.
2138  IgnoreChains = false;
2139  }
2140 
2141  return !findNonImmUse(Root, N.getNode(), U, IgnoreChains);
2142 }
2143 
2144 void SelectionDAGISel::Select_INLINEASM(SDNode *N) {
2145  SDLoc DL(N);
2146 
2147  std::vector<SDValue> Ops(N->op_begin(), N->op_end());
2149 
2150  const EVT VTs[] = {MVT::Other, MVT::Glue};
2151  SDValue New = CurDAG->getNode(N->getOpcode(), DL, VTs, Ops);
2152  New->setNodeId(-1);
2153  ReplaceUses(N, New.getNode());
2155 }
2156 
2157 void SelectionDAGISel::Select_READ_REGISTER(SDNode *Op) {
2158  SDLoc dl(Op);
2159  MDNodeSDNode *MD = cast<MDNodeSDNode>(Op->getOperand(1));
2160  const MDString *RegStr = cast<MDString>(MD->getMD()->getOperand(0));
2161 
2162  EVT VT = Op->getValueType(0);
2163  LLT Ty = VT.isSimple() ? getLLTForMVT(VT.getSimpleVT()) : LLT();
2164  Register Reg =
2165  TLI->getRegisterByName(RegStr->getString().data(), Ty,
2168  Op->getOperand(0), dl, Reg, Op->getValueType(0));
2169  New->setNodeId(-1);
2170  ReplaceUses(Op, New.getNode());
2172 }
2173 
2174 void SelectionDAGISel::Select_WRITE_REGISTER(SDNode *Op) {
2175  SDLoc dl(Op);
2176  MDNodeSDNode *MD = cast<MDNodeSDNode>(Op->getOperand(1));
2177  const MDString *RegStr = cast<MDString>(MD->getMD()->getOperand(0));
2178 
2179  EVT VT = Op->getOperand(2).getValueType();
2180  LLT Ty = VT.isSimple() ? getLLTForMVT(VT.getSimpleVT()) : LLT();
2181 
2182  Register Reg = TLI->getRegisterByName(RegStr->getString().data(), Ty,
2185  Op->getOperand(0), dl, Reg, Op->getOperand(2));
2186  New->setNodeId(-1);
2187  ReplaceUses(Op, New.getNode());
2189 }
2190 
2191 void SelectionDAGISel::Select_UNDEF(SDNode *N) {
2192  CurDAG->SelectNodeTo(N, TargetOpcode::IMPLICIT_DEF, N->getValueType(0));
2193 }
2194 
2195 void SelectionDAGISel::Select_FREEZE(SDNode *N) {
2196  // TODO: We don't have FREEZE pseudo-instruction in MachineInstr-level now.
2197  // If FREEZE instruction is added later, the code below must be changed as
2198  // well.
2199  CurDAG->SelectNodeTo(N, TargetOpcode::COPY, N->getValueType(0),
2200  N->getOperand(0));
2201 }
2202 
2203 void SelectionDAGISel::Select_ARITH_FENCE(SDNode *N) {
2204  CurDAG->SelectNodeTo(N, TargetOpcode::ARITH_FENCE, N->getValueType(0),
2205  N->getOperand(0));
2206 }
2207 
2208 void SelectionDAGISel::pushStackMapLiveVariable(SmallVectorImpl<SDValue> &Ops,
2209  SDValue OpVal, SDLoc DL) {
2210  SDNode *OpNode = OpVal.getNode();
2211 
2212  // FrameIndex nodes should have been directly emitted to TargetFrameIndex
2213  // nodes at DAG-construction time.
2214  assert(OpNode->getOpcode() != ISD::FrameIndex);
2215 
2216  if (OpNode->getOpcode() == ISD::Constant) {
2217  Ops.push_back(
2218  CurDAG->getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
2219  Ops.push_back(
2220  CurDAG->getTargetConstant(cast<ConstantSDNode>(OpNode)->getZExtValue(),
2221  DL, OpVal.getValueType()));
2222  } else {
2223  Ops.push_back(OpVal);
2224  }
2225 }
2226 
2227 void SelectionDAGISel::Select_STACKMAP(SDNode *N) {
2229  auto *It = N->op_begin();
2230  SDLoc DL(N);
2231 
2232  // Stash the chain and glue operands so we can move them to the end.
2233  SDValue Chain = *It++;
2234  SDValue InFlag = *It++;
2235 
2236  // <id> operand.
2237  SDValue ID = *It++;
2238  assert(ID.getValueType() == MVT::i64);
2239  Ops.push_back(ID);
2240 
2241  // <numShadowBytes> operand.
2242  SDValue Shad = *It++;
2243  assert(Shad.getValueType() == MVT::i32);
2244  Ops.push_back(Shad);
2245 
2246  // Live variable operands.
2247  for (; It != N->op_end(); It++)
2248  pushStackMapLiveVariable(Ops, *It, DL);
2249 
2250  Ops.push_back(Chain);
2251  Ops.push_back(InFlag);
2252 
2254  CurDAG->SelectNodeTo(N, TargetOpcode::STACKMAP, NodeTys, Ops);
2255 }
2256 
2257 void SelectionDAGISel::Select_PATCHPOINT(SDNode *N) {
2259  auto *It = N->op_begin();
2260  SDLoc DL(N);
2261 
2262  // Cache arguments that will be moved to the end in the target node.
2263  SDValue Chain = *It++;
2264  Optional<SDValue> Glue;
2265  if (It->getValueType() == MVT::Glue)
2266  Glue = *It++;
2267  SDValue RegMask = *It++;
2268 
2269  // <id> operand.
2270  SDValue ID = *It++;
2271  assert(ID.getValueType() == MVT::i64);
2272  Ops.push_back(ID);
2273 
2274  // <numShadowBytes> operand.
2275  SDValue Shad = *It++;
2276  assert(Shad.getValueType() == MVT::i32);
2277  Ops.push_back(Shad);
2278 
2279  // Add the callee.
2280  Ops.push_back(*It++);
2281 
2282  // Add <numArgs>.
2283  SDValue NumArgs = *It++;
2284  assert(NumArgs.getValueType() == MVT::i32);
2285  Ops.push_back(NumArgs);
2286 
2287  // Calling convention.
2288  Ops.push_back(*It++);
2289 
2290  // Push the args for the call.
2291  for (uint64_t I = cast<ConstantSDNode>(NumArgs)->getZExtValue(); I != 0; I--)
2292  Ops.push_back(*It++);
2293 
2294  // Now push the live variables.
2295  for (; It != N->op_end(); It++)
2296  pushStackMapLiveVariable(Ops, *It, DL);
2297 
2298  // Finally, the regmask, chain and (if present) glue are moved to the end.
2299  Ops.push_back(RegMask);
2300  Ops.push_back(Chain);
2301  if (Glue.has_value())
2302  Ops.push_back(Glue.value());
2303 
2304  SDVTList NodeTys = N->getVTList();
2305  CurDAG->SelectNodeTo(N, TargetOpcode::PATCHPOINT, NodeTys, Ops);
2306 }
2307 
2308 /// GetVBR - decode a vbr encoding whose top bit is set.
2310 GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) {
2311  assert(Val >= 128 && "Not a VBR");
2312  Val &= 127; // Remove first vbr bit.
2313 
2314  unsigned Shift = 7;
2315  uint64_t NextBits;
2316  do {
2317  NextBits = MatcherTable[Idx++];
2318  Val |= (NextBits&127) << Shift;
2319  Shift += 7;
2320  } while (NextBits & 128);
2321 
2322  return Val;
2323 }
2324 
2325 /// When a match is complete, this method updates uses of interior chain results
2326 /// to use the new results.
2327 void SelectionDAGISel::UpdateChains(
2328  SDNode *NodeToMatch, SDValue InputChain,
2329  SmallVectorImpl<SDNode *> &ChainNodesMatched, bool isMorphNodeTo) {
2330  SmallVector<SDNode*, 4> NowDeadNodes;
2331 
2332  // Now that all the normal results are replaced, we replace the chain and
2333  // glue results if present.
2334  if (!ChainNodesMatched.empty()) {
2335  assert(InputChain.getNode() &&
2336  "Matched input chains but didn't produce a chain");
2337  // Loop over all of the nodes we matched that produced a chain result.
2338  // Replace all the chain results with the final chain we ended up with.
2339  for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) {
2340  SDNode *ChainNode = ChainNodesMatched[i];
2341  // If ChainNode is null, it's because we replaced it on a previous
2342  // iteration and we cleared it out of the map. Just skip it.
2343  if (!ChainNode)
2344  continue;
2345 
2346  assert(ChainNode->getOpcode() != ISD::DELETED_NODE &&
2347  "Deleted node left in chain");
2348 
2349  // Don't replace the results of the root node if we're doing a
2350  // MorphNodeTo.
2351  if (ChainNode == NodeToMatch && isMorphNodeTo)
2352  continue;
2353 
2354  SDValue ChainVal = SDValue(ChainNode, ChainNode->getNumValues()-1);
2355  if (ChainVal.getValueType() == MVT::Glue)
2356  ChainVal = ChainVal.getValue(ChainVal->getNumValues()-2);
2357  assert(ChainVal.getValueType() == MVT::Other && "Not a chain?");
2359  *CurDAG, [&](SDNode *N, SDNode *E) {
2360  std::replace(ChainNodesMatched.begin(), ChainNodesMatched.end(), N,
2361  static_cast<SDNode *>(nullptr));
2362  });
2363  if (ChainNode->getOpcode() != ISD::TokenFactor)
2364  ReplaceUses(ChainVal, InputChain);
2365 
2366  // If the node became dead and we haven't already seen it, delete it.
2367  if (ChainNode != NodeToMatch && ChainNode->use_empty() &&
2368  !llvm::is_contained(NowDeadNodes, ChainNode))
2369  NowDeadNodes.push_back(ChainNode);
2370  }
2371  }
2372 
2373  if (!NowDeadNodes.empty())
2374  CurDAG->RemoveDeadNodes(NowDeadNodes);
2375 
2376  LLVM_DEBUG(dbgs() << "ISEL: Match complete!\n");
2377 }
2378 
2379 /// HandleMergeInputChains - This implements the OPC_EmitMergeInputChains
2380 /// operation for when the pattern matched at least one node with a chains. The
2381 /// input vector contains a list of all of the chained nodes that we match. We
2382 /// must determine if this is a valid thing to cover (i.e. matching it won't
2383 /// induce cycles in the DAG) and if so, creating a TokenFactor node. that will
2384 /// be used as the input node chain for the generated nodes.
2385 static SDValue
2387  SelectionDAG *CurDAG) {
2388 
2391  SmallVector<SDValue, 3> InputChains;
2392  unsigned int Max = 8192;
2393 
2394  // Quick exit on trivial merge.
2395  if (ChainNodesMatched.size() == 1)
2396  return ChainNodesMatched[0]->getOperand(0);
2397 
2398  // Add chains that aren't already added (internal). Peek through
2399  // token factors.
2400  std::function<void(const SDValue)> AddChains = [&](const SDValue V) {
2401  if (V.getValueType() != MVT::Other)
2402  return;
2403  if (V->getOpcode() == ISD::EntryToken)
2404  return;
2405  if (!Visited.insert(V.getNode()).second)
2406  return;
2407  if (V->getOpcode() == ISD::TokenFactor) {
2408  for (const SDValue &Op : V->op_values())
2409  AddChains(Op);
2410  } else
2411  InputChains.push_back(V);
2412  };
2413 
2414  for (auto *N : ChainNodesMatched) {
2415  Worklist.push_back(N);
2416  Visited.insert(N);
2417  }
2418 
2419  while (!Worklist.empty())
2420  AddChains(Worklist.pop_back_val()->getOperand(0));
2421 
2422  // Skip the search if there are no chain dependencies.
2423  if (InputChains.size() == 0)
2424  return CurDAG->getEntryNode();
2425 
2426  // If one of these chains is a successor of input, we must have a
2427  // node that is both the predecessor and successor of the
2428  // to-be-merged nodes. Fail.
2429  Visited.clear();
2430  for (SDValue V : InputChains)
2431  Worklist.push_back(V.getNode());
2432 
2433  for (auto *N : ChainNodesMatched)
2434  if (SDNode::hasPredecessorHelper(N, Visited, Worklist, Max, true))
2435  return SDValue();
2436 
2437  // Return merged chain.
2438  if (InputChains.size() == 1)
2439  return InputChains[0];
2440  return CurDAG->getNode(ISD::TokenFactor, SDLoc(ChainNodesMatched[0]),
2441  MVT::Other, InputChains);
2442 }
2443 
2444 /// MorphNode - Handle morphing a node in place for the selector.
2445 SDNode *SelectionDAGISel::
2446 MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList,
2447  ArrayRef<SDValue> Ops, unsigned EmitNodeInfo) {
2448  // It is possible we're using MorphNodeTo to replace a node with no
2449  // normal results with one that has a normal result (or we could be
2450  // adding a chain) and the input could have glue and chains as well.
2451  // In this case we need to shift the operands down.
2452  // FIXME: This is a horrible hack and broken in obscure cases, no worse
2453  // than the old isel though.
2454  int OldGlueResultNo = -1, OldChainResultNo = -1;
2455 
2456  unsigned NTMNumResults = Node->getNumValues();
2457  if (Node->getValueType(NTMNumResults-1) == MVT::Glue) {
2458  OldGlueResultNo = NTMNumResults-1;
2459  if (NTMNumResults != 1 &&
2460  Node->getValueType(NTMNumResults-2) == MVT::Other)
2461  OldChainResultNo = NTMNumResults-2;
2462  } else if (Node->getValueType(NTMNumResults-1) == MVT::Other)
2463  OldChainResultNo = NTMNumResults-1;
2464 
2465  // Call the underlying SelectionDAG routine to do the transmogrification. Note
2466  // that this deletes operands of the old node that become dead.
2467  SDNode *Res = CurDAG->MorphNodeTo(Node, ~TargetOpc, VTList, Ops);
2468 
2469  // MorphNodeTo can operate in two ways: if an existing node with the
2470  // specified operands exists, it can just return it. Otherwise, it
2471  // updates the node in place to have the requested operands.
2472  if (Res == Node) {
2473  // If we updated the node in place, reset the node ID. To the isel,
2474  // this should be just like a newly allocated machine node.
2475  Res->setNodeId(-1);
2476  }
2477 
2478  unsigned ResNumResults = Res->getNumValues();
2479  // Move the glue if needed.
2480  if ((EmitNodeInfo & OPFL_GlueOutput) && OldGlueResultNo != -1 &&
2481  (unsigned)OldGlueResultNo != ResNumResults-1)
2482  ReplaceUses(SDValue(Node, OldGlueResultNo),
2483  SDValue(Res, ResNumResults - 1));
2484 
2485  if ((EmitNodeInfo & OPFL_GlueOutput) != 0)
2486  --ResNumResults;
2487 
2488  // Move the chain reference if needed.
2489  if ((EmitNodeInfo & OPFL_Chain) && OldChainResultNo != -1 &&
2490  (unsigned)OldChainResultNo != ResNumResults-1)
2491  ReplaceUses(SDValue(Node, OldChainResultNo),
2492  SDValue(Res, ResNumResults - 1));
2493 
2494  // Otherwise, no replacement happened because the node already exists. Replace
2495  // Uses of the old node with the new one.
2496  if (Res != Node) {
2497  ReplaceNode(Node, Res);
2498  } else {
2500  }
2501 
2502  return Res;
2503 }
2504 
2505 /// CheckSame - Implements OP_CheckSame.
2506 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2507 CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2508  const SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes) {
2509  // Accept if it is exactly the same as a previously recorded node.
2510  unsigned RecNo = MatcherTable[MatcherIndex++];
2511  assert(RecNo < RecordedNodes.size() && "Invalid CheckSame");
2512  return N == RecordedNodes[RecNo].first;
2513 }
2514 
2515 /// CheckChildSame - Implements OP_CheckChildXSame.
2517  const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2518  const SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes,
2519  unsigned ChildNo) {
2520  if (ChildNo >= N.getNumOperands())
2521  return false; // Match fails if out of range child #.
2522  return ::CheckSame(MatcherTable, MatcherIndex, N.getOperand(ChildNo),
2523  RecordedNodes);
2524 }
2525 
2526 /// CheckPatternPredicate - Implements OP_CheckPatternPredicate.
2527 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2528 CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2529  const SelectionDAGISel &SDISel) {
2530  return SDISel.CheckPatternPredicate(MatcherTable[MatcherIndex++]);
2531 }
2532 
2533 /// CheckNodePredicate - Implements OP_CheckNodePredicate.
2534 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2535 CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2536  const SelectionDAGISel &SDISel, SDNode *N) {
2537  return SDISel.CheckNodePredicate(N, MatcherTable[MatcherIndex++]);
2538 }
2539 
2540 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2541 CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2542  SDNode *N) {
2543  uint16_t Opc = MatcherTable[MatcherIndex++];
2544  Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
2545  return N->getOpcode() == Opc;
2546 }
2547 
2548 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2549 CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2550  const TargetLowering *TLI, const DataLayout &DL) {
2551  MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
2552  if (N.getValueType() == VT) return true;
2553 
2554  // Handle the case when VT is iPTR.
2555  return VT == MVT::iPTR && N.getValueType() == TLI->getPointerTy(DL);
2556 }
2557 
2558 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2559 CheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2560  SDValue N, const TargetLowering *TLI, const DataLayout &DL,
2561  unsigned ChildNo) {
2562  if (ChildNo >= N.getNumOperands())
2563  return false; // Match fails if out of range child #.
2564  return ::CheckType(MatcherTable, MatcherIndex, N.getOperand(ChildNo), TLI,
2565  DL);
2566 }
2567 
2568 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2569 CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2570  SDValue N) {
2571  return cast<CondCodeSDNode>(N)->get() ==
2572  (ISD::CondCode)MatcherTable[MatcherIndex++];
2573 }
2574 
2575 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2576 CheckChild2CondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2577  SDValue N) {
2578  if (2 >= N.getNumOperands())
2579  return false;
2580  return ::CheckCondCode(MatcherTable, MatcherIndex, N.getOperand(2));
2581 }
2582 
2583 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2584 CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2585  SDValue N, const TargetLowering *TLI, const DataLayout &DL) {
2586  MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
2587  if (cast<VTSDNode>(N)->getVT() == VT)
2588  return true;
2589 
2590  // Handle the case when VT is iPTR.
2591  return VT == MVT::iPTR && cast<VTSDNode>(N)->getVT() == TLI->getPointerTy(DL);
2592 }
2593 
2594 // Bit 0 stores the sign of the immediate. The upper bits contain the magnitude
2595 // shifted left by 1.
2597  if ((V & 1) == 0)
2598  return V >> 1;
2599  if (V != 1)
2600  return -(V >> 1);
2601  // There is no such thing as -0 with integers. "-0" really means MININT.
2602  return 1ULL << 63;
2603 }
2604 
2605 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2606 CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2607  SDValue N) {
2608  int64_t Val = MatcherTable[MatcherIndex++];
2609  if (Val & 128)
2610  Val = GetVBR(Val, MatcherTable, MatcherIndex);
2611 
2612  Val = decodeSignRotatedValue(Val);
2613 
2614  ConstantSDNode *C = dyn_cast<ConstantSDNode>(N);
2615  return C && C->getSExtValue() == Val;
2616 }
2617 
2618 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2619 CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2620  SDValue N, unsigned ChildNo) {
2621  if (ChildNo >= N.getNumOperands())
2622  return false; // Match fails if out of range child #.
2623  return ::CheckInteger(MatcherTable, MatcherIndex, N.getOperand(ChildNo));
2624 }
2625 
2626 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2627 CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2628  SDValue N, const SelectionDAGISel &SDISel) {
2629  int64_t Val = MatcherTable[MatcherIndex++];
2630  if (Val & 128)
2631  Val = GetVBR(Val, MatcherTable, MatcherIndex);
2632 
2633  if (N->getOpcode() != ISD::AND) return false;
2634 
2635  ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2636  return C && SDISel.CheckAndMask(N.getOperand(0), C, Val);
2637 }
2638 
2639 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
2640 CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2641  const SelectionDAGISel &SDISel) {
2642  int64_t Val = MatcherTable[MatcherIndex++];
2643  if (Val & 128)
2644  Val = GetVBR(Val, MatcherTable, MatcherIndex);
2645 
2646  if (N->getOpcode() != ISD::OR) return false;
2647 
2648  ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2649  return C && SDISel.CheckOrMask(N.getOperand(0), C, Val);
2650 }
2651 
2652 /// IsPredicateKnownToFail - If we know how and can do so without pushing a
2653 /// scope, evaluate the current node. If the current predicate is known to
2654 /// fail, set Result=true and return anything. If the current predicate is
2655 /// known to pass, set Result=false and return the MatcherIndex to continue
2656 /// with. If the current predicate is unknown, set Result=false and return the
2657 /// MatcherIndex to continue with.
2658 static unsigned IsPredicateKnownToFail(const unsigned char *Table,
2659  unsigned Index, SDValue N,
2660  bool &Result,
2661  const SelectionDAGISel &SDISel,
2662  SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes) {
2663  switch (Table[Index++]) {
2664  default:
2665  Result = false;
2666  return Index-1; // Could not evaluate this predicate.
2668  Result = !::CheckSame(Table, Index, N, RecordedNodes);
2669  return Index;
2674  Result = !::CheckChildSame(Table, Index, N, RecordedNodes,
2676  return Index;
2678  Result = !::CheckPatternPredicate(Table, Index, SDISel);
2679  return Index;
2681  Result = !::CheckNodePredicate(Table, Index, SDISel, N.getNode());
2682  return Index;
2684  Result = !::CheckOpcode(Table, Index, N.getNode());
2685  return Index;
2687  Result = !::CheckType(Table, Index, N, SDISel.TLI,
2688  SDISel.CurDAG->getDataLayout());
2689  return Index;
2691  unsigned Res = Table[Index++];
2692  Result = !::CheckType(Table, Index, N.getValue(Res), SDISel.TLI,
2693  SDISel.CurDAG->getDataLayout());
2694  return Index;
2695  }
2704  Result = !::CheckChildType(
2705  Table, Index, N, SDISel.TLI, SDISel.CurDAG->getDataLayout(),
2707  return Index;
2709  Result = !::CheckCondCode(Table, Index, N);
2710  return Index;
2712  Result = !::CheckChild2CondCode(Table, Index, N);
2713  return Index;
2715  Result = !::CheckValueType(Table, Index, N, SDISel.TLI,
2716  SDISel.CurDAG->getDataLayout());
2717  return Index;
2719  Result = !::CheckInteger(Table, Index, N);
2720  return Index;
2726  Result = !::CheckChildInteger(Table, Index, N,
2728  return Index;
2730  Result = !::CheckAndImm(Table, Index, N, SDISel);
2731  return Index;
2733  Result = !::CheckOrImm(Table, Index, N, SDISel);
2734  return Index;
2735  }
2736 }
2737 
2738 namespace {
2739 
2740 struct MatchScope {
2741  /// FailIndex - If this match fails, this is the index to continue with.
2742  unsigned FailIndex;
2743 
2744  /// NodeStack - The node stack when the scope was formed.
2745  SmallVector<SDValue, 4> NodeStack;
2746 
2747  /// NumRecordedNodes - The number of recorded nodes when the scope was formed.
2748  unsigned NumRecordedNodes;
2749 
2750  /// NumMatchedMemRefs - The number of matched memref entries.
2751  unsigned NumMatchedMemRefs;
2752 
2753  /// InputChain/InputGlue - The current chain/glue
2754  SDValue InputChain, InputGlue;
2755 
2756  /// HasChainNodesMatched - True if the ChainNodesMatched list is non-empty.
2757  bool HasChainNodesMatched;
2758 };
2759 
2760 /// \A DAG update listener to keep the matching state
2761 /// (i.e. RecordedNodes and MatchScope) uptodate if the target is allowed to
2762 /// change the DAG while matching. X86 addressing mode matcher is an example
2763 /// for this.
2764 class MatchStateUpdater : public SelectionDAG::DAGUpdateListener
2765 {
2766  SDNode **NodeToMatch;
2768  SmallVectorImpl<MatchScope> &MatchScopes;
2769 
2770 public:
2771  MatchStateUpdater(SelectionDAG &DAG, SDNode **NodeToMatch,
2772  SmallVectorImpl<std::pair<SDValue, SDNode *>> &RN,
2774  : SelectionDAG::DAGUpdateListener(DAG), NodeToMatch(NodeToMatch),
2775  RecordedNodes(RN), MatchScopes(MS) {}
2776 
2777  void NodeDeleted(SDNode *N, SDNode *E) override {
2778  // Some early-returns here to avoid the search if we deleted the node or
2779  // if the update comes from MorphNodeTo (MorphNodeTo is the last thing we
2780  // do, so it's unnecessary to update matching state at that point).
2781  // Neither of these can occur currently because we only install this
2782  // update listener during matching a complex patterns.
2783  if (!E || E->isMachineOpcode())
2784  return;
2785  // Check if NodeToMatch was updated.
2786  if (N == *NodeToMatch)
2787  *NodeToMatch = E;
2788  // Performing linear search here does not matter because we almost never
2789  // run this code. You'd have to have a CSE during complex pattern
2790  // matching.
2791  for (auto &I : RecordedNodes)
2792  if (I.first.getNode() == N)
2793  I.first.setNode(E);
2794 
2795  for (auto &I : MatchScopes)
2796  for (auto &J : I.NodeStack)
2797  if (J.getNode() == N)
2798  J.setNode(E);
2799  }
2800 };
2801 
2802 } // end anonymous namespace
2803 
2805  const unsigned char *MatcherTable,
2806  unsigned TableSize) {
2807  // FIXME: Should these even be selected? Handle these cases in the caller?
2808  switch (NodeToMatch->getOpcode()) {
2809  default:
2810  break;
2811  case ISD::EntryToken: // These nodes remain the same.
2812  case ISD::BasicBlock:
2813  case ISD::Register:
2814  case ISD::RegisterMask:
2815  case ISD::HANDLENODE:
2816  case ISD::MDNODE_SDNODE:
2817  case ISD::TargetConstant:
2818  case ISD::TargetConstantFP:
2820  case ISD::TargetFrameIndex:
2822  case ISD::MCSymbol:
2824  case ISD::TargetJumpTable:
2827  case ISD::TokenFactor:
2828  case ISD::CopyFromReg:
2829  case ISD::CopyToReg:
2830  case ISD::EH_LABEL:
2831  case ISD::ANNOTATION_LABEL:
2832  case ISD::LIFETIME_START:
2833  case ISD::LIFETIME_END:
2834  case ISD::PSEUDO_PROBE:
2835  NodeToMatch->setNodeId(-1); // Mark selected.
2836  return;
2837  case ISD::AssertSext:
2838  case ISD::AssertZext:
2839  case ISD::AssertAlign:
2840  ReplaceUses(SDValue(NodeToMatch, 0), NodeToMatch->getOperand(0));
2841  CurDAG->RemoveDeadNode(NodeToMatch);
2842  return;
2843  case ISD::INLINEASM:
2844  case ISD::INLINEASM_BR:
2845  Select_INLINEASM(NodeToMatch);
2846  return;
2847  case ISD::READ_REGISTER:
2848  Select_READ_REGISTER(NodeToMatch);
2849  return;
2850  case ISD::WRITE_REGISTER:
2851  Select_WRITE_REGISTER(NodeToMatch);
2852  return;
2853  case ISD::UNDEF:
2854  Select_UNDEF(NodeToMatch);
2855  return;
2856  case ISD::FREEZE:
2857  Select_FREEZE(NodeToMatch);
2858  return;
2859  case ISD::ARITH_FENCE:
2860  Select_ARITH_FENCE(NodeToMatch);
2861  return;
2862  case ISD::STACKMAP:
2863  Select_STACKMAP(NodeToMatch);
2864  return;
2865  case ISD::PATCHPOINT:
2866  Select_PATCHPOINT(NodeToMatch);
2867  return;
2868  }
2869 
2870  assert(!NodeToMatch->isMachineOpcode() && "Node already selected!");
2871 
2872  // Set up the node stack with NodeToMatch as the only node on the stack.
2873  SmallVector<SDValue, 8> NodeStack;
2874  SDValue N = SDValue(NodeToMatch, 0);
2875  NodeStack.push_back(N);
2876 
2877  // MatchScopes - Scopes used when matching, if a match failure happens, this
2878  // indicates where to continue checking.
2879  SmallVector<MatchScope, 8> MatchScopes;
2880 
2881  // RecordedNodes - This is the set of nodes that have been recorded by the
2882  // state machine. The second value is the parent of the node, or null if the
2883  // root is recorded.
2884  SmallVector<std::pair<SDValue, SDNode*>, 8> RecordedNodes;
2885 
2886  // MatchedMemRefs - This is the set of MemRef's we've seen in the input
2887  // pattern.
2888  SmallVector<MachineMemOperand*, 2> MatchedMemRefs;
2889 
2890  // These are the current input chain and glue for use when generating nodes.
2891  // Various Emit operations change these. For example, emitting a copytoreg
2892  // uses and updates these.
2893  SDValue InputChain, InputGlue;
2894 
2895  // ChainNodesMatched - If a pattern matches nodes that have input/output
2896  // chains, the OPC_EmitMergeInputChains operation is emitted which indicates
2897  // which ones they are. The result is captured into this list so that we can
2898  // update the chain results when the pattern is complete.
2899  SmallVector<SDNode*, 3> ChainNodesMatched;
2900 
2901  LLVM_DEBUG(dbgs() << "ISEL: Starting pattern match\n");
2902 
2903  // Determine where to start the interpreter. Normally we start at opcode #0,
2904  // but if the state machine starts with an OPC_SwitchOpcode, then we
2905  // accelerate the first lookup (which is guaranteed to be hot) with the
2906  // OpcodeOffset table.
2907  unsigned MatcherIndex = 0;
2908 
2909  if (!OpcodeOffset.empty()) {
2910  // Already computed the OpcodeOffset table, just index into it.
2911  if (N.getOpcode() < OpcodeOffset.size())
2912  MatcherIndex = OpcodeOffset[N.getOpcode()];
2913  LLVM_DEBUG(dbgs() << " Initial Opcode index to " << MatcherIndex << "\n");
2914 
2915  } else if (MatcherTable[0] == OPC_SwitchOpcode) {
2916  // Otherwise, the table isn't computed, but the state machine does start
2917  // with an OPC_SwitchOpcode instruction. Populate the table now, since this
2918  // is the first time we're selecting an instruction.
2919  unsigned Idx = 1;
2920  while (true) {
2921  // Get the size of this case.
2922  unsigned CaseSize = MatcherTable[Idx++];
2923  if (CaseSize & 128)
2924  CaseSize = GetVBR(CaseSize, MatcherTable, Idx);
2925  if (CaseSize == 0) break;
2926 
2927  // Get the opcode, add the index to the table.
2928  uint16_t Opc = MatcherTable[Idx++];
2929  Opc |= (unsigned short)MatcherTable[Idx++] << 8;
2930  if (Opc >= OpcodeOffset.size())
2931  OpcodeOffset.resize((Opc+1)*2);
2932  OpcodeOffset[Opc] = Idx;
2933  Idx += CaseSize;
2934  }
2935 
2936  // Okay, do the lookup for the first opcode.
2937  if (N.getOpcode() < OpcodeOffset.size())
2938  MatcherIndex = OpcodeOffset[N.getOpcode()];
2939  }
2940 
2941  while (true) {
2942  assert(MatcherIndex < TableSize && "Invalid index");
2943 #ifndef NDEBUG
2944  unsigned CurrentOpcodeIndex = MatcherIndex;
2945 #endif
2946  BuiltinOpcodes Opcode = (BuiltinOpcodes)MatcherTable[MatcherIndex++];
2947  switch (Opcode) {
2948  case OPC_Scope: {
2949  // Okay, the semantics of this operation are that we should push a scope
2950  // then evaluate the first child. However, pushing a scope only to have
2951  // the first check fail (which then pops it) is inefficient. If we can
2952  // determine immediately that the first check (or first several) will
2953  // immediately fail, don't even bother pushing a scope for them.
2954  unsigned FailIndex;
2955 
2956  while (true) {
2957  unsigned NumToSkip = MatcherTable[MatcherIndex++];
2958  if (NumToSkip & 128)
2959  NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex);
2960  // Found the end of the scope with no match.
2961  if (NumToSkip == 0) {
2962  FailIndex = 0;
2963  break;
2964  }
2965 
2966  FailIndex = MatcherIndex+NumToSkip;
2967 
2968  unsigned MatcherIndexOfPredicate = MatcherIndex;
2969  (void)MatcherIndexOfPredicate; // silence warning.
2970 
2971  // If we can't evaluate this predicate without pushing a scope (e.g. if
2972  // it is a 'MoveParent') or if the predicate succeeds on this node, we
2973  // push the scope and evaluate the full predicate chain.
2974  bool Result;
2975  MatcherIndex = IsPredicateKnownToFail(MatcherTable, MatcherIndex, N,
2976  Result, *this, RecordedNodes);
2977  if (!Result)
2978  break;
2979 
2980  LLVM_DEBUG(
2981  dbgs() << " Skipped scope entry (due to false predicate) at "
2982  << "index " << MatcherIndexOfPredicate << ", continuing at "
2983  << FailIndex << "\n");
2984  ++NumDAGIselRetries;
2985 
2986  // Otherwise, we know that this case of the Scope is guaranteed to fail,
2987  // move to the next case.
2988  MatcherIndex = FailIndex;
2989  }
2990 
2991  // If the whole scope failed to match, bail.
2992  if (FailIndex == 0) break;
2993 
2994  // Push a MatchScope which indicates where to go if the first child fails
2995  // to match.
2996  MatchScope NewEntry;
2997  NewEntry.FailIndex = FailIndex;
2998  NewEntry.NodeStack.append(NodeStack.begin(), NodeStack.end());
2999  NewEntry.NumRecordedNodes = RecordedNodes.size();
3000  NewEntry.NumMatchedMemRefs = MatchedMemRefs.size();
3001  NewEntry.InputChain = InputChain;
3002  NewEntry.InputGlue = InputGlue;
3003  NewEntry.HasChainNodesMatched = !ChainNodesMatched.empty();
3004  MatchScopes.push_back(NewEntry);
3005  continue;
3006  }
3007  case OPC_RecordNode: {
3008  // Remember this node, it may end up being an operand in the pattern.
3009  SDNode *Parent = nullptr;
3010  if (NodeStack.size() > 1)
3011  Parent = NodeStack[NodeStack.size()-2].getNode();
3012  RecordedNodes.push_back(std::make_pair(N, Parent));
3013  continue;
3014  }
3015 
3019  case OPC_RecordChild6: case OPC_RecordChild7: {
3020  unsigned ChildNo = Opcode-OPC_RecordChild0;
3021  if (ChildNo >= N.getNumOperands())
3022  break; // Match fails if out of range child #.
3023 
3024  RecordedNodes.push_back(std::make_pair(N->getOperand(ChildNo),
3025  N.getNode()));
3026  continue;
3027  }
3028  case OPC_RecordMemRef:
3029  if (auto *MN = dyn_cast<MemSDNode>(N))
3030  MatchedMemRefs.push_back(MN->getMemOperand());
3031  else {
3032  LLVM_DEBUG(dbgs() << "Expected MemSDNode "; N->dump(CurDAG);
3033  dbgs() << '\n');
3034  }
3035 
3036  continue;
3037 
3038  case OPC_CaptureGlueInput:
3039  // If the current node has an input glue, capture it in InputGlue.
3040  if (N->getNumOperands() != 0 &&
3041  N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Glue)
3042  InputGlue = N->getOperand(N->getNumOperands()-1);
3043  continue;
3044 
3045  case OPC_MoveChild: {
3046  unsigned ChildNo = MatcherTable[MatcherIndex++];
3047  if (ChildNo >= N.getNumOperands())
3048  break; // Match fails if out of range child #.
3049  N = N.getOperand(ChildNo);
3050  NodeStack.push_back(N);
3051  continue;
3052  }
3053 
3054  case OPC_MoveChild0: case OPC_MoveChild1:
3055  case OPC_MoveChild2: case OPC_MoveChild3:
3056  case OPC_MoveChild4: case OPC_MoveChild5:
3057  case OPC_MoveChild6: case OPC_MoveChild7: {
3058  unsigned ChildNo = Opcode-OPC_MoveChild0;
3059  if (ChildNo >= N.getNumOperands())
3060  break; // Match fails if out of range child #.
3061  N = N.getOperand(ChildNo);
3062  NodeStack.push_back(N);
3063  continue;
3064  }
3065 
3066  case OPC_MoveParent:
3067  // Pop the current node off the NodeStack.
3068  NodeStack.pop_back();
3069  assert(!NodeStack.empty() && "Node stack imbalance!");
3070  N = NodeStack.back();
3071  continue;
3072 
3073  case OPC_CheckSame:
3074  if (!::CheckSame(MatcherTable, MatcherIndex, N, RecordedNodes)) break;
3075  continue;
3076 
3079  if (!::CheckChildSame(MatcherTable, MatcherIndex, N, RecordedNodes,
3080  Opcode-OPC_CheckChild0Same))
3081  break;
3082  continue;
3083 
3085  if (!::CheckPatternPredicate(MatcherTable, MatcherIndex, *this)) break;
3086  continue;
3087  case OPC_CheckPredicate:
3088  if (!::CheckNodePredicate(MatcherTable, MatcherIndex, *this,
3089  N.getNode()))
3090  break;
3091  continue;
3093  unsigned OpNum = MatcherTable[MatcherIndex++];
3095 
3096  for (unsigned i = 0; i < OpNum; ++i)
3097  Operands.push_back(RecordedNodes[MatcherTable[MatcherIndex++]].first);
3098 
3099  unsigned PredNo = MatcherTable[MatcherIndex++];
3100  if (!CheckNodePredicateWithOperands(N.getNode(), PredNo, Operands))
3101  break;
3102  continue;
3103  }
3104  case OPC_CheckComplexPat: {
3105  unsigned CPNum = MatcherTable[MatcherIndex++];
3106  unsigned RecNo = MatcherTable[MatcherIndex++];
3107  assert(RecNo < RecordedNodes.size() && "Invalid CheckComplexPat");
3108 
3109  // If target can modify DAG during matching, keep the matching state
3110  // consistent.
3111  std::unique_ptr<MatchStateUpdater> MSU;
3113  MSU.reset(new MatchStateUpdater(*CurDAG, &NodeToMatch, RecordedNodes,
3114  MatchScopes));
3115 
3116  if (!CheckComplexPattern(NodeToMatch, RecordedNodes[RecNo].second,
3117  RecordedNodes[RecNo].first, CPNum,
3118  RecordedNodes))
3119  break;
3120  continue;
3121  }
3122  case OPC_CheckOpcode:
3123  if (!::CheckOpcode(MatcherTable, MatcherIndex, N.getNode())) break;
3124  continue;
3125 
3126  case OPC_CheckType:
3127  if (!::CheckType(MatcherTable, MatcherIndex, N, TLI,
3128  CurDAG->getDataLayout()))
3129  break;
3130  continue;
3131 
3132  case OPC_CheckTypeRes: {
3133  unsigned Res = MatcherTable[MatcherIndex++];
3134  if (!::CheckType(MatcherTable, MatcherIndex, N.getValue(Res), TLI,
3135  CurDAG->getDataLayout()))
3136  break;
3137  continue;
3138  }
3139 
3140  case OPC_SwitchOpcode: {
3141  unsigned CurNodeOpcode = N.getOpcode();
3142  unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3143  unsigned CaseSize;
3144  while (true) {
3145  // Get the size of this case.
3146  CaseSize = MatcherTable[MatcherIndex++];
3147  if (CaseSize & 128)
3148  CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);
3149  if (CaseSize == 0) break;
3150 
3151  uint16_t Opc = MatcherTable[MatcherIndex++];
3152  Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
3153 
3154  // If the opcode matches, then we will execute this case.
3155  if (CurNodeOpcode == Opc)
3156  break;
3157 
3158  // Otherwise, skip over this case.
3159  MatcherIndex += CaseSize;
3160  }
3161 
3162  // If no cases matched, bail out.
3163  if (CaseSize == 0) break;
3164 
3165  // Otherwise, execute the case we found.
3166  LLVM_DEBUG(dbgs() << " OpcodeSwitch from " << SwitchStart << " to "
3167  << MatcherIndex << "\n");
3168  continue;
3169  }
3170 
3171  case OPC_SwitchType: {
3172  MVT CurNodeVT = N.getSimpleValueType();
3173  unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3174  unsigned CaseSize;
3175  while (true) {
3176  // Get the size of this case.
3177  CaseSize = MatcherTable[MatcherIndex++];
3178  if (CaseSize & 128)
3179  CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);
3180  if (CaseSize == 0) break;
3181 
3182  MVT CaseVT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3183  if (CaseVT == MVT::iPTR)
3184  CaseVT = TLI->getPointerTy(CurDAG->getDataLayout());
3185 
3186  // If the VT matches, then we will execute this case.
3187  if (CurNodeVT == CaseVT)
3188  break;
3189 
3190  // Otherwise, skip over this case.
3191  MatcherIndex += CaseSize;
3192  }
3193 
3194  // If no cases matched, bail out.
3195  if (CaseSize == 0) break;
3196 
3197  // Otherwise, execute the case we found.
3198  LLVM_DEBUG(dbgs() << " TypeSwitch[" << EVT(CurNodeVT).getEVTString()
3199  << "] from " << SwitchStart << " to " << MatcherIndex
3200  << '\n');
3201  continue;
3202  }
3207  if (!::CheckChildType(MatcherTable, MatcherIndex, N, TLI,
3208  CurDAG->getDataLayout(),
3209  Opcode - OPC_CheckChild0Type))
3210  break;
3211  continue;
3212  case OPC_CheckCondCode:
3213  if (!::CheckCondCode(MatcherTable, MatcherIndex, N)) break;
3214  continue;
3216  if (!::CheckChild2CondCode(MatcherTable, MatcherIndex, N)) break;
3217  continue;
3218  case OPC_CheckValueType:
3219  if (!::CheckValueType(MatcherTable, MatcherIndex, N, TLI,
3220  CurDAG->getDataLayout()))
3221  break;
3222  continue;
3223  case OPC_CheckInteger:
3224  if (!::CheckInteger(MatcherTable, MatcherIndex, N)) break;
3225  continue;
3229  if (!::CheckChildInteger(MatcherTable, MatcherIndex, N,
3230  Opcode-OPC_CheckChild0Integer)) break;
3231  continue;
3232  case OPC_CheckAndImm:
3233  if (!::CheckAndImm(MatcherTable, MatcherIndex, N, *this)) break;
3234  continue;
3235  case OPC_CheckOrImm:
3236  if (!::CheckOrImm(MatcherTable, MatcherIndex, N, *this)) break;
3237  continue;
3238  case OPC_CheckImmAllOnesV:
3239  if (!ISD::isConstantSplatVectorAllOnes(N.getNode()))
3240  break;
3241  continue;
3242  case OPC_CheckImmAllZerosV:
3243  if (!ISD::isConstantSplatVectorAllZeros(N.getNode()))
3244  break;
3245  continue;
3246 
3248  assert(NodeStack.size() != 1 && "No parent node");
3249  // Verify that all intermediate nodes between the root and this one have
3250  // a single use (ignoring chains, which are handled in UpdateChains).
3251  bool HasMultipleUses = false;
3252  for (unsigned i = 1, e = NodeStack.size()-1; i != e; ++i) {
3253  unsigned NNonChainUses = 0;
3254  SDNode *NS = NodeStack[i].getNode();
3255  for (auto UI = NS->use_begin(), UE = NS->use_end(); UI != UE; ++UI)
3256  if (UI.getUse().getValueType() != MVT::Other)
3257  if (++NNonChainUses > 1) {
3258  HasMultipleUses = true;
3259  break;
3260  }
3261  if (HasMultipleUses) break;
3262  }
3263  if (HasMultipleUses) break;
3264 
3265  // Check to see that the target thinks this is profitable to fold and that
3266  // we can fold it without inducing cycles in the graph.
3267  if (!IsProfitableToFold(N, NodeStack[NodeStack.size()-2].getNode(),
3268  NodeToMatch) ||
3269  !IsLegalToFold(N, NodeStack[NodeStack.size()-2].getNode(),
3270  NodeToMatch, OptLevel,
3271  true/*We validate our own chains*/))
3272  break;
3273 
3274  continue;
3275  }
3276  case OPC_EmitInteger:
3277  case OPC_EmitStringInteger: {
3279  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3280  int64_t Val = MatcherTable[MatcherIndex++];
3281  if (Val & 128)
3282  Val = GetVBR(Val, MatcherTable, MatcherIndex);
3283  if (Opcode == OPC_EmitInteger)
3284  Val = decodeSignRotatedValue(Val);
3285  RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3286  CurDAG->getTargetConstant(Val, SDLoc(NodeToMatch),
3287  VT), nullptr));
3288  continue;
3289  }
3290  case OPC_EmitRegister: {
3292  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3293  unsigned RegNo = MatcherTable[MatcherIndex++];
3294  RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3295  CurDAG->getRegister(RegNo, VT), nullptr));
3296  continue;
3297  }
3298  case OPC_EmitRegister2: {
3299  // For targets w/ more than 256 register names, the register enum
3300  // values are stored in two bytes in the matcher table (just like
3301  // opcodes).
3303  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3304  unsigned RegNo = MatcherTable[MatcherIndex++];
3305  RegNo |= MatcherTable[MatcherIndex++] << 8;
3306  RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3307  CurDAG->getRegister(RegNo, VT), nullptr));
3308  continue;
3309  }
3310 
3311  case OPC_EmitConvertToTarget: {
3312  // Convert from IMM/FPIMM to target version.
3313  unsigned RecNo = MatcherTable[MatcherIndex++];
3314  assert(RecNo < RecordedNodes.size() && "Invalid EmitConvertToTarget");
3315  SDValue Imm = RecordedNodes[RecNo].first;
3316 
3317  if (Imm->getOpcode() == ISD::Constant) {
3318  const ConstantInt *Val=cast<ConstantSDNode>(Imm)->getConstantIntValue();
3319  Imm = CurDAG->getTargetConstant(*Val, SDLoc(NodeToMatch),
3320  Imm.getValueType());
3321  } else if (Imm->getOpcode() == ISD::ConstantFP) {
3322  const ConstantFP *Val=cast<ConstantFPSDNode>(Imm)->getConstantFPValue();
3323  Imm = CurDAG->getTargetConstantFP(*Val, SDLoc(NodeToMatch),
3324  Imm.getValueType());
3325  }
3326 
3327  RecordedNodes.push_back(std::make_pair(Imm, RecordedNodes[RecNo].second));
3328  continue;
3329  }
3330 
3331  case OPC_EmitMergeInputChains1_0: // OPC_EmitMergeInputChains, 1, 0
3332  case OPC_EmitMergeInputChains1_1: // OPC_EmitMergeInputChains, 1, 1
3333  case OPC_EmitMergeInputChains1_2: { // OPC_EmitMergeInputChains, 1, 2
3334  // These are space-optimized forms of OPC_EmitMergeInputChains.
3335  assert(!InputChain.getNode() &&
3336  "EmitMergeInputChains should be the first chain producing node");
3337  assert(ChainNodesMatched.empty() &&
3338  "Should only have one EmitMergeInputChains per match");
3339 
3340  // Read all of the chained nodes.
3341  unsigned RecNo = Opcode - OPC_EmitMergeInputChains1_0;
3342  assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains");
3343  ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());
3344 
3345  // If the chained node is not the root, we can't fold it if it has
3346  // multiple uses.
3347  // FIXME: What if other value results of the node have uses not matched
3348  // by this pattern?
3349  if (ChainNodesMatched.back() != NodeToMatch &&
3350  !RecordedNodes[RecNo].first.hasOneUse()) {
3351  ChainNodesMatched.clear();
3352  break;
3353  }
3354 
3355  // Merge the input chains if they are not intra-pattern references.
3356  InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG);
3357 
3358  if (!InputChain.getNode())
3359  break; // Failed to merge.
3360  continue;
3361  }
3362 
3363  case OPC_EmitMergeInputChains: {
3364  assert(!InputChain.getNode() &&
3365  "EmitMergeInputChains should be the first chain producing node");
3366  // This node gets a list of nodes we matched in the input that have
3367  // chains. We want to token factor all of the input chains to these nodes
3368  // together. However, if any of the input chains is actually one of the
3369  // nodes matched in this pattern, then we have an intra-match reference.
3370  // Ignore these because the newly token factored chain should not refer to
3371  // the old nodes.
3372  unsigned NumChains = MatcherTable[MatcherIndex++];
3373  assert(NumChains != 0 && "Can't TF zero chains");
3374 
3375  assert(ChainNodesMatched.empty() &&
3376  "Should only have one EmitMergeInputChains per match");
3377 
3378  // Read all of the chained nodes.
3379  for (unsigned i = 0; i != NumChains; ++i) {
3380  unsigned RecNo = MatcherTable[MatcherIndex++];
3381  assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains");
3382  ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());
3383 
3384  // If the chained node is not the root, we can't fold it if it has
3385  // multiple uses.
3386  // FIXME: What if other value results of the node have uses not matched
3387  // by this pattern?
3388  if (ChainNodesMatched.back() != NodeToMatch &&
3389  !RecordedNodes[RecNo].first.hasOneUse()) {
3390  ChainNodesMatched.clear();
3391  break;
3392  }
3393  }
3394 
3395  // If the inner loop broke out, the match fails.
3396  if (ChainNodesMatched.empty())
3397  break;
3398 
3399  // Merge the input chains if they are not intra-pattern references.
3400  InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG);
3401 
3402  if (!InputChain.getNode())
3403  break; // Failed to merge.
3404 
3405  continue;
3406  }
3407 
3408  case OPC_EmitCopyToReg:
3409  case OPC_EmitCopyToReg2: {
3410  unsigned RecNo = MatcherTable[MatcherIndex++];
3411  assert(RecNo < RecordedNodes.size() && "Invalid EmitCopyToReg");
3412  unsigned DestPhysReg = MatcherTable[MatcherIndex++];
3413  if (Opcode == OPC_EmitCopyToReg2)
3414  DestPhysReg |= MatcherTable[MatcherIndex++] << 8;
3415 
3416  if (!InputChain.getNode())
3417  InputChain = CurDAG->getEntryNode();
3418 
3419  InputChain = CurDAG->getCopyToReg(InputChain, SDLoc(NodeToMatch),
3420  DestPhysReg, RecordedNodes[RecNo].first,
3421  InputGlue);
3422 
3423  InputGlue = InputChain.getValue(1);
3424  continue;
3425  }
3426 
3427  case OPC_EmitNodeXForm: {
3428  unsigned XFormNo = MatcherTable[MatcherIndex++];
3429  unsigned RecNo = MatcherTable[MatcherIndex++];
3430  assert(RecNo < RecordedNodes.size() && "Invalid EmitNodeXForm");
3431  SDValue Res = RunSDNodeXForm(RecordedNodes[RecNo].first, XFormNo);
3432  RecordedNodes.push_back(std::pair<SDValue,SDNode*>(Res, nullptr));
3433  continue;
3434  }
3435  case OPC_Coverage: {
3436  // This is emitted right before MorphNode/EmitNode.
3437  // So it should be safe to assume that this node has been selected
3438  unsigned index = MatcherTable[MatcherIndex++];
3439  index |= (MatcherTable[MatcherIndex++] << 8);
3440  dbgs() << "COVERED: " << getPatternForIndex(index) << "\n";
3441  dbgs() << "INCLUDED: " << getIncludePathForIndex(index) << "\n";
3442  continue;
3443  }
3444 
3445  case OPC_EmitNode: case OPC_MorphNodeTo:
3446  case OPC_EmitNode0: case OPC_EmitNode1: case OPC_EmitNode2:
3448  uint16_t TargetOpc = MatcherTable[MatcherIndex++];
3449  TargetOpc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
3450  unsigned EmitNodeInfo = MatcherTable[MatcherIndex++];
3451  // Get the result VT list.
3452  unsigned NumVTs;
3453  // If this is one of the compressed forms, get the number of VTs based
3454  // on the Opcode. Otherwise read the next byte from the table.
3455  if (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2)
3456  NumVTs = Opcode - OPC_MorphNodeTo0;
3457  else if (Opcode >= OPC_EmitNode0 && Opcode <= OPC_EmitNode2)
3458  NumVTs = Opcode - OPC_EmitNode0;
3459  else
3460  NumVTs = MatcherTable[MatcherIndex++];
3461  SmallVector<EVT, 4> VTs;
3462  for (unsigned i = 0; i != NumVTs; ++i) {
3464  (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
3465  if (VT == MVT::iPTR)
3466  VT = TLI->getPointerTy(CurDAG->getDataLayout()).SimpleTy;
3467  VTs.push_back(VT);
3468  }
3469 
3470  if (EmitNodeInfo & OPFL_Chain)
3471  VTs.push_back(MVT::Other);
3472  if (EmitNodeInfo & OPFL_GlueOutput)
3473  VTs.push_back(MVT::Glue);
3474 
3475  // This is hot code, so optimize the two most common cases of 1 and 2
3476  // results.
3477  SDVTList VTList;
3478  if (VTs.size() == 1)
3479  VTList = CurDAG->getVTList(VTs[0]);
3480  else if (VTs.size() == 2)
3481  VTList = CurDAG->getVTList(VTs[0], VTs[1]);
3482  else
3483  VTList = CurDAG->getVTList(VTs);
3484 
3485  // Get the operand list.
3486  unsigned NumOps = MatcherTable[MatcherIndex++];
3488  for (unsigned i = 0; i != NumOps; ++i) {
3489  unsigned RecNo = MatcherTable[MatcherIndex++];
3490  if (RecNo & 128)
3491  RecNo = GetVBR(RecNo, MatcherTable, MatcherIndex);
3492 
3493  assert(RecNo < RecordedNodes.size() && "Invalid EmitNode");
3494  Ops.push_back(RecordedNodes[RecNo].first);
3495  }
3496 
3497  // If there are variadic operands to add, handle them now.
3498  if (EmitNodeInfo & OPFL_VariadicInfo) {
3499  // Determine the start index to copy from.
3500  unsigned FirstOpToCopy = getNumFixedFromVariadicInfo(EmitNodeInfo);
3501  FirstOpToCopy += (EmitNodeInfo & OPFL_Chain) ? 1 : 0;
3502  assert(NodeToMatch->getNumOperands() >= FirstOpToCopy &&
3503  "Invalid variadic node");
3504  // Copy all of the variadic operands, not including a potential glue
3505  // input.
3506  for (unsigned i = FirstOpToCopy, e = NodeToMatch->getNumOperands();
3507  i != e; ++i) {
3508  SDValue V = NodeToMatch->getOperand(i);
3509  if (V.getValueType() == MVT::Glue) break;
3510  Ops.push_back(V);
3511  }
3512  }
3513 
3514  // If this has chain/glue inputs, add them.
3515  if (EmitNodeInfo & OPFL_Chain)
3516  Ops.push_back(InputChain);
3517  if ((EmitNodeInfo & OPFL_GlueInput) && InputGlue.getNode() != nullptr)
3518  Ops.push_back(InputGlue);
3519 
3520  // Check whether any matched node could raise an FP exception. Since all
3521  // such nodes must have a chain, it suffices to check ChainNodesMatched.
3522  // We need to perform this check before potentially modifying one of the
3523  // nodes via MorphNode.
3524  bool MayRaiseFPException =
3525  llvm::any_of(ChainNodesMatched, [this](SDNode *N) {
3526  return mayRaiseFPException(N) && !N->getFlags().hasNoFPExcept();
3527  });
3528 
3529  // Create the node.
3530  MachineSDNode *Res = nullptr;
3531  bool IsMorphNodeTo = Opcode == OPC_MorphNodeTo ||
3532  (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2);
3533  if (!IsMorphNodeTo) {
3534  // If this is a normal EmitNode command, just create the new node and
3535  // add the results to the RecordedNodes list.
3536  Res = CurDAG->getMachineNode(TargetOpc, SDLoc(NodeToMatch),
3537  VTList, Ops);
3538 
3539  // Add all the non-glue/non-chain results to the RecordedNodes list.
3540  for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
3541  if (VTs[i] == MVT::Other || VTs[i] == MVT::Glue) break;
3542  RecordedNodes.push_back(std::pair<SDValue,SDNode*>(SDValue(Res, i),
3543  nullptr));
3544  }
3545  } else {
3546  assert(NodeToMatch->getOpcode() != ISD::DELETED_NODE &&
3547  "NodeToMatch was removed partway through selection");
3549  SDNode *E) {
3551  auto &Chain = ChainNodesMatched;
3552  assert((!E || !is_contained(Chain, N)) &&
3553  "Chain node replaced during MorphNode");
3554  llvm::erase_value(Chain, N);
3555  });
3556  Res = cast<MachineSDNode>(MorphNode(NodeToMatch, TargetOpc, VTList,
3557  Ops, EmitNodeInfo));
3558  }
3559 
3560  // Set the NoFPExcept flag when no original matched node could
3561  // raise an FP exception, but the new node potentially might.
3563  SDNodeFlags Flags = Res->getFlags();
3564  Flags.setNoFPExcept(true);
3565  Res->setFlags(Flags);
3566  }
3567 
3568  // If the node had chain/glue results, update our notion of the current
3569  // chain and glue.
3570  if (EmitNodeInfo & OPFL_GlueOutput) {
3571  InputGlue = SDValue(Res, VTs.size()-1);
3572  if (EmitNodeInfo & OPFL_Chain)
3573  InputChain = SDValue(Res, VTs.size()-2);
3574  } else if (EmitNodeInfo & OPFL_Chain)
3575  InputChain = SDValue(Res, VTs.size()-1);
3576 
3577  // If the OPFL_MemRefs glue is set on this node, slap all of the
3578  // accumulated memrefs onto it.
3579  //
3580  // FIXME: This is vastly incorrect for patterns with multiple outputs
3581  // instructions that access memory and for ComplexPatterns that match
3582  // loads.
3583  if (EmitNodeInfo & OPFL_MemRefs) {
3584  // Only attach load or store memory operands if the generated
3585  // instruction may load or store.
3586  const MCInstrDesc &MCID = TII->get(TargetOpc);
3587  bool mayLoad = MCID.mayLoad();
3588  bool mayStore = MCID.mayStore();
3589 
3590  // We expect to have relatively few of these so just filter them into a
3591  // temporary buffer so that we can easily add them to the instruction.
3592  SmallVector<MachineMemOperand *, 4> FilteredMemRefs;
3593  for (MachineMemOperand *MMO : MatchedMemRefs) {
3594  if (MMO->isLoad()) {
3595  if (mayLoad)
3596  FilteredMemRefs.push_back(MMO);
3597  } else if (MMO->isStore()) {
3598  if (mayStore)
3599  FilteredMemRefs.push_back(MMO);
3600  } else {
3601  FilteredMemRefs.push_back(MMO);
3602  }
3603  }
3604 
3605  CurDAG->setNodeMemRefs(Res, FilteredMemRefs);
3606  }
3607 
3608  LLVM_DEBUG(if (!MatchedMemRefs.empty() && Res->memoperands_empty()) dbgs()
3609  << " Dropping mem operands\n";
3610  dbgs() << " " << (IsMorphNodeTo ? "Morphed" : "Created")
3611  << " node: ";
3612  Res->dump(CurDAG););
3613 
3614  // If this was a MorphNodeTo then we're completely done!
3615  if (IsMorphNodeTo) {
3616  // Update chain uses.
3617  UpdateChains(Res, InputChain, ChainNodesMatched, true);
3618  return;
3619  }
3620  continue;
3621  }
3622 
3623  case OPC_CompleteMatch: {
3624  // The match has been completed, and any new nodes (if any) have been
3625  // created. Patch up references to the matched dag to use the newly
3626  // created nodes.
3627  unsigned NumResults = MatcherTable[MatcherIndex++];
3628 
3629  for (unsigned i = 0; i != NumResults; ++i) {
3630  unsigned ResSlot = MatcherTable[MatcherIndex++];
3631  if (ResSlot & 128)
3632  ResSlot = GetVBR(ResSlot, MatcherTable, MatcherIndex);
3633 
3634  assert(ResSlot < RecordedNodes.size() && "Invalid CompleteMatch");
3635  SDValue Res = RecordedNodes[ResSlot].first;
3636 
3637  assert(i < NodeToMatch->getNumValues() &&
3638  NodeToMatch->getValueType(i) != MVT::Other &&
3639  NodeToMatch->getValueType(i) != MVT::Glue &&
3640  "Invalid number of results to complete!");
3641  assert((NodeToMatch->getValueType(i) == Res.getValueType() ||
3642  NodeToMatch->getValueType(i) == MVT::iPTR ||
3643  Res.getValueType() == MVT::iPTR ||
3644  NodeToMatch->getValueType(i).getSizeInBits() ==
3645  Res.getValueSizeInBits()) &&
3646  "invalid replacement");
3647  ReplaceUses(SDValue(NodeToMatch, i), Res);
3648  }
3649 
3650  // Update chain uses.
3651  UpdateChains(NodeToMatch, InputChain, ChainNodesMatched, false);
3652 
3653  // If the root node defines glue, we need to update it to the glue result.
3654  // TODO: This never happens in our tests and I think it can be removed /
3655  // replaced with an assert, but if we do it this the way the change is
3656  // NFC.
3657  if (NodeToMatch->getValueType(NodeToMatch->getNumValues() - 1) ==
3658  MVT::Glue &&
3659  InputGlue.getNode())
3660  ReplaceUses(SDValue(NodeToMatch, NodeToMatch->getNumValues() - 1),
3661  InputGlue);
3662 
3663  assert(NodeToMatch->use_empty() &&
3664  "Didn't replace all uses of the node?");
3665  CurDAG->RemoveDeadNode(NodeToMatch);
3666 
3667  return;
3668  }
3669  }
3670 
3671  // If the code reached this point, then the match failed. See if there is
3672  // another child to try in the current 'Scope', otherwise pop it until we
3673  // find a case to check.
3674  LLVM_DEBUG(dbgs() << " Match failed at index " << CurrentOpcodeIndex
3675  << "\n");
3676  ++NumDAGIselRetries;
3677  while (true) {
3678  if (MatchScopes.empty()) {
3679  CannotYetSelect(NodeToMatch);
3680  return;
3681  }
3682 
3683  // Restore the interpreter state back to the point where the scope was
3684  // formed.
3685  MatchScope &LastScope = MatchScopes.back();
3686  RecordedNodes.resize(LastScope.NumRecordedNodes);
3687  NodeStack.clear();
3688  NodeStack.append(LastScope.NodeStack.begin(), LastScope.NodeStack.end());
3689  N = NodeStack.back();
3690 
3691  if (LastScope.NumMatchedMemRefs != MatchedMemRefs.size())
3692  MatchedMemRefs.resize(LastScope.NumMatchedMemRefs);
3693  MatcherIndex = LastScope.FailIndex;
3694 
3695  LLVM_DEBUG(dbgs() << " Continuing at " << MatcherIndex << "\n");
3696 
3697  InputChain = LastScope.InputChain;
3698  InputGlue = LastScope.InputGlue;
3699  if (!LastScope.HasChainNodesMatched)
3700  ChainNodesMatched.clear();
3701 
3702  // Check to see what the offset is at the new MatcherIndex. If it is zero
3703  // we have reached the end of this scope, otherwise we have another child
3704  // in the current scope to try.
3705  unsigned NumToSkip = MatcherTable[MatcherIndex++];
3706  if (NumToSkip & 128)
3707  NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex);
3708 
3709  // If we have another child in this scope to match, update FailIndex and
3710  // try it.
3711  if (NumToSkip != 0) {
3712  LastScope.FailIndex = MatcherIndex+NumToSkip;
3713  break;
3714  }
3715 
3716  // End of this scope, pop it and try the next child in the containing
3717  // scope.
3718  MatchScopes.pop_back();
3719  }
3720  }
3721 }
3722 
3723 /// Return whether the node may raise an FP exception.
3725  // For machine opcodes, consult the MCID flag.
3726  if (N->isMachineOpcode()) {
3727  const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
3728  return MCID.mayRaiseFPException();
3729  }
3730 
3731  // For ISD opcodes, only StrictFP opcodes may raise an FP
3732  // exception.
3733  if (N->isTargetOpcode())
3734  return N->isTargetStrictFPOpcode();
3735  return N->isStrictFPOpcode();
3736 }
3737 
3739  assert(N->getOpcode() == ISD::OR && "Unexpected opcode");
3740  auto *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
3741  if (!C)
3742  return false;
3743 
3744  // Detect when "or" is used to add an offset to a stack object.
3745  if (auto *FN = dyn_cast<FrameIndexSDNode>(N->getOperand(0))) {
3746  MachineFrameInfo &MFI = MF->getFrameInfo();
3747  Align A = MFI.getObjectAlign(FN->getIndex());
3748  int32_t Off = C->getSExtValue();
3749  // If the alleged offset fits in the zero bits guaranteed by
3750  // the alignment, then this or is really an add.
3751  return (Off >= 0) && (((A.value() - 1) & Off) == unsigned(Off));
3752  }
3753  return false;
3754 }
3755 
3756 void SelectionDAGISel::CannotYetSelect(SDNode *N) {
3757  std::string msg;
3758  raw_string_ostream Msg(msg);
3759  Msg << "Cannot select: ";
3760 
3761  if (N->getOpcode() != ISD::INTRINSIC_W_CHAIN &&
3762  N->getOpcode() != ISD::INTRINSIC_WO_CHAIN &&
3763  N->getOpcode() != ISD::INTRINSIC_VOID) {
3764  N->printrFull(Msg, CurDAG);
3765  Msg << "\nIn function: " << MF->getName();
3766  } else {
3767  bool HasInputChain = N->getOperand(0).getValueType() == MVT::Other;
3768  unsigned iid =
3769  cast<ConstantSDNode>(N->getOperand(HasInputChain))->getZExtValue();
3770  if (iid < Intrinsic::num_intrinsics)
3771  Msg << "intrinsic %" << Intrinsic::getBaseName((Intrinsic::ID)iid);
3772  else if (const TargetIntrinsicInfo *TII = TM.getIntrinsicInfo())
3773  Msg << "target intrinsic %" << TII->getName(iid);
3774  else
3775  Msg << "unknown intrinsic #" << iid;
3776  }
3777  report_fatal_error(Twine(Msg.str()));
3778 }
3779 
3780 char SelectionDAGISel::ID = 0;
llvm::DIExpression::prepend
static DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
Definition: DebugInfoMetadata.cpp:1449
llvm::FunctionLoweringInfo::Fn
const Function * Fn
Definition: FunctionLoweringInfo.h:54
llvm::Intrinsic::getBaseName
StringRef getBaseName(ID id)
Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....
Definition: Function.cpp:877
llvm::MachineInstr::isDebugValue
bool isDebugValue() const
Definition: MachineInstr.h:1261
llvm::SelectionDAGISel::OPC_MoveChild2
@ OPC_MoveChild2
Definition: SelectionDAGISel.h:127
i
i
Definition: README.txt:29
llvm::MachineFunction::hasInlineAsm
bool hasInlineAsm() const
Returns true if the function contains any inline assembly.
Definition: MachineFunction.h:729
ValueTypes.h
CheckSame
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * >> &RecordedNodes)
CheckSame - Implements OP_CheckSame.
Definition: SelectionDAGISel.cpp:2507
llvm::SelectionDAGISel::FastISelFailed
bool FastISelFailed
Definition: SelectionDAGISel.h:57
llvm::InlineAsm::Op_FirstOperand
@ Op_FirstOperand
Definition: InlineAsm.h:220
llvm::DbgVariableIntrinsic::getExpression
DIExpression * getExpression() const
Definition: IntrinsicInst.h:262
AssumptionCache.h
llvm::ConstantSDNode
Definition: SelectionDAGNodes.h:1581
llvm::SelectionDAGISel::OPC_CheckComplexPat
@ OPC_CheckComplexPat
Definition: SelectionDAGISel.h:149
llvm::BasicBlock::end
iterator end()
Definition: BasicBlock.h:308
llvm::ISD::STRICT_FSETCC
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:475
llvm::FunctionLoweringInfo::StaticAllocaMap
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
Definition: FunctionLoweringInfo.h:128
StackProtector.h
llvm::SelectionDAGISel::OPC_EmitInteger
@ OPC_EmitInteger
Definition: SelectionDAGISel.h:155
llvm::OptimizationRemarkMissed
Diagnostic information for missed-optimization remarks.
Definition: DiagnosticInfo.h:735
llvm::createDAGLinearizer
ScheduleDAGSDNodes * createDAGLinearizer(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createDAGLinearizer - This creates a "no-scheduling" scheduler which linearize the DAG using topologi...
Definition: ScheduleDAGFast.cpp:820
isFoldedOrDeadInstruction
static bool isFoldedOrDeadInstruction(const Instruction *I, const FunctionLoweringInfo &FuncInfo)
isFoldedOrDeadInstruction - Return true if the specified instruction is side-effect free and is eithe...
Definition: SelectionDAGISel.cpp:1285
llvm::MachineFunctionProperties::hasProperty
bool hasProperty(Property P) const
Definition: MachineFunction.h:192
llvm::ISD::INTRINSIC_VOID
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:199
llvm::SDUse
Represents a use of a SDNode.
Definition: SelectionDAGNodes.h:284
llvm::SelectionDAGISel::OPC_CheckChild2Integer
@ OPC_CheckChild2Integer
Definition: SelectionDAGISel.h:145
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:108
MachineInstr.h
llvm::SelectionDAGISel::UseInstrRefDebugInfo
bool UseInstrRefDebugInfo
Definition: SelectionDAGISel.h:59
llvm::SelectionDAGISel::TLI
const TargetLowering * TLI
Definition: SelectionDAGISel.h:56
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::SelectionDAGISel::getIncludePathForIndex
virtual StringRef getIncludePathForIndex(unsigned index)
getIncludePathForIndex - get the td source location of pattern instantiation
Definition: SelectionDAGISel.h:248
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
llvm::DbgDeclareInst::getAddress
Value * getAddress() const
Definition: IntrinsicInst.h:315
llvm::DiagnosticInfoISelFallback
Diagnostic information for ISel fallback path.
Definition: DiagnosticInfo.h:955
llvm::SelectionDAG::AssignTopologicalOrder
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
Definition: SelectionDAG.cpp:10574
GCMetadata.h
llvm::SelectionDAGISel::TM
TargetMachine & TM
Definition: SelectionDAGISel.h:43
llvm::MCSymbol
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
llvm::ISD::PATCHPOINT
@ PATCHPOINT
Definition: ISDOpcodes.h:1299
llvm::SDNode::getValueType
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Definition: SelectionDAGNodes.h:983
UseMI
MachineInstrBuilder & UseMI
Definition: AArch64ExpandPseudoInsts.cpp:105
llvm::SelectionDAGISel::isOrEquivalentToAdd
bool isOrEquivalentToAdd(const SDNode *N) const
Definition: SelectionDAGISel.cpp:3738
llvm::ScheduleDAGSDNodes
ScheduleDAGSDNodes - A ScheduleDAG for scheduling SDNode-based DAGs.
Definition: ScheduleDAGSDNodes.h:46
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1103
llvm::SelectionDAGISel::OPC_EmitMergeInputChains1_2
@ OPC_EmitMergeInputChains1_2
Definition: SelectionDAGISel.h:163
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::ISD::OR
@ OR
Definition: ISDOpcodes.h:667
llvm::SelectionDAGISel::EnforceNodeIdInvariant
static void EnforceNodeIdInvariant(SDNode *N)
Definition: SelectionDAGISel.cpp:1036
llvm::TargetLowering::EmitInstrWithCustomInserter
virtual MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Definition: SelectionDAGISel.cpp:292
CheckChildSame
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * >> &RecordedNodes, unsigned ChildNo)
CheckChildSame - Implements OP_CheckChildXSame.
Definition: SelectionDAGISel.cpp:2516
llvm::Value::hasOneUse
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
llvm::MachineBasicBlock::getBasicBlock
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
Definition: MachineBasicBlock.h:209
llvm::SelectionDAGISel::CheckComplexPattern
virtual bool CheckComplexPattern(SDNode *Root, SDNode *Parent, SDValue N, unsigned PatternNo, SmallVectorImpl< std::pair< SDValue, SDNode * > > &Result)
Definition: SelectionDAGISel.h:291
Metadata.h
llvm::ISD::LIFETIME_END
@ LIFETIME_END
Definition: ISDOpcodes.h:1225
llvm::SelectionDAGISel::OPC_RecordChild7
@ OPC_RecordChild7
Definition: SelectionDAGISel.h:123
llvm::SelectionDAGISel::PostprocessISelDAG
virtual void PostprocessISelDAG()
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
Definition: SelectionDAGISel.h:85
llvm::TargetLoweringBase::isStrictFPEnabled
bool isStrictFPEnabled() const
Return true if the target support strict float operation.
Definition: TargetLowering.h:337
PHI
Rewrite undef for PHI
Definition: AMDGPURewriteUndefForPHI.cpp:101
llvm::M68kBeads::Term
@ Term
Definition: M68kBaseInfo.h:71
IntrinsicInst.h
computeUsesMSVCFloatingPoint
static void computeUsesMSVCFloatingPoint(const Triple &TT, const Function &F, MachineModuleInfo &MMI)
Definition: SelectionDAGISel.cpp:349
hasExceptionPointerOrCodeUser
static bool hasExceptionPointerOrCodeUser(const CatchPadInst *CPI)
Definition: SelectionDAGISel.cpp:1175
DebugInfoMetadata.h
llvm::ISD::AssertSext
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
llvm::SelectionDAG::getCopyToReg
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:764
MCInstrDesc.h
llvm::SelectionDAGISel::DAGSize
unsigned DAGSize
DAGSize - Size of DAG being instruction selected.
Definition: SelectionDAGISel.h:206
EnableFastISelAbort
static cl::opt< int > EnableFastISelAbort("fast-isel-abort", cl::Hidden, cl::desc("Enable abort calls when \"fast\" instruction selection " "fails to lower an instruction: 0 disable the abort, 1 will " "abort but for args, calls and terminators, 2 will also " "abort for argument lowering, and 3 will never fallback " "to SelectionDAG."))
InstIterator.h
MachinePassRegistry.h
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:50
T
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:159
llvm::SelectionDAG::VerifyDAGDivergence
void VerifyDAGDivergence()
Definition: SelectionDAG.cpp:10493
llvm::SelectionDAGISel::OPC_EmitNode
@ OPC_EmitNode
Definition: SelectionDAGISel.h:167
llvm::Function
Definition: Function.h:60
llvm::ISD::ConstantFP
@ ConstantFP
Definition: ISDOpcodes.h:77
StringRef.h
llvm::ISD::LIFETIME_START
@ LIFETIME_START
This corresponds to the llvm.lifetime.
Definition: ISDOpcodes.h:1224
P
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Definition: README-SSE.txt:411
Pass.h
SchedulerRegistry.h
llvm::TargetSubtargetInfo::getInstrInfo
virtual const TargetInstrInfo * getInstrInfo() const
Definition: TargetSubtargetInfo.h:93
llvm::ISD::STRICT_UINT_TO_FP
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:449
llvm::SelectionDAGISel::SelectInlineAsmMemoryOperands
void SelectInlineAsmMemoryOperands(std::vector< SDValue > &Ops, const SDLoc &DL)
SelectInlineAsmMemoryOperands - Calls to this are automatically generated by tblgen.
Definition: SelectionDAGISel.cpp:1956
llvm::StackProtector::copyToMachineFrameInfo
void copyToMachineFrameInfo(MachineFrameInfo &MFI) const
Definition: StackProtector.cpp:599
llvm::raw_string_ostream
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:628
llvm::SDNode::op_values
iterator_range< value_op_iterator > op_values() const
Definition: SelectionDAGNodes.h:941
llvm::FuncletPadInst::getNumArgOperands
unsigned getNumArgOperands() const
getNumArgOperands - Return the number of funcletpad arguments.
Definition: InstrTypes.h:2383
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1181
llvm::TargetMachine::getO0WantsFastISel
bool getO0WantsFastISel()
Definition: TargetMachine.h:247
Statistic.h
llvm::SelectionDAG::init
void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, LegacyDivergenceAnalysis *Divergence, ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin)
Prepare this SelectionDAG to process code in the given MachineFunction.
Definition: SelectionDAG.cpp:1284
llvm::MachineFunction::useDebugInstrRef
bool useDebugInstrRef() const
Returns true if the function's variable locations should be tracked with instruction referencing.
Definition: MachineFunction.cpp:1243
InlineAsm.h
CheckType
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
Definition: SelectionDAGISel.cpp:2549
llvm::SelectionDAG::getVTList
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
Definition: SelectionDAG.cpp:9316
llvm::FastISel::finishBasicBlock
void finishBasicBlock()
Flush the local value map.
Definition: FastISel.cpp:136
llvm::printMBBReference
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Definition: MachineBasicBlock.cpp:117
llvm::FunctionPass::skipFunction
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Definition: Pass.cpp:173
llvm::Function::getSubprogram
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1583
llvm::MachineSDNode
An SDNode that represents everything that will be needed to construct a MachineInstr.
Definition: SelectionDAGNodes.h:2895
llvm::createDefaultScheduler
ScheduleDAGSDNodes * createDefaultScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createDefaultScheduler - This creates an instruction scheduler appropriate for the target.
Definition: SelectionDAGISel.cpp:251
ErrorHandling.h
llvm::FastISel::lowerArguments
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
Definition: FastISel.cpp:138
llvm::TargetTransformInfo
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Definition: TargetTransformInfo.h:172
LazyBlockFrequencyInfo.h
llvm::SelectionDAGISel::OPC_CheckOrImm
@ OPC_CheckOrImm
Definition: SelectionDAGISel.h:150
llvm::SelectionDAGISel::OPC_RecordChild3
@ OPC_RecordChild3
Definition: SelectionDAGISel.h:122
llvm::Function::getEntryBlock
const BasicBlock & getEntryBlock() const
Definition: Function.h:710
llvm::Sched::ILP
@ ILP
Definition: TargetLowering.h:101
llvm::SelectionDAG::allnodes_begin
allnodes_const_iterator allnodes_begin() const
Definition: SelectionDAG.h:522
llvm::SelectionDAG::getRoot
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:542
llvm::SelectionDAGISel::AC
AssumptionCache * AC
Definition: SelectionDAGISel.h:52
llvm::SelectionDAGISel::OPC_EmitNode1
@ OPC_EmitNode1
Definition: SelectionDAGISel.h:169
llvm::X86Disassembler::Reg
Reg
All possible values of the reg field in the ModR/M byte.
Definition: X86DisassemblerDecoder.h:462
llvm::SwiftErrorValueTracking::createEntriesInEntryBlock
bool createEntriesInEntryBlock(DebugLoc DbgLoc)
Create initial definitions of swifterror values in the entry block of the current function.
Definition: SwiftErrorValueTracking.cpp:115
llvm::SwiftErrorValueTracking::propagateVRegs
void propagateVRegs()
Propagate assigned swifterror vregs through a function, synthesizing PHI nodes when needed to maintai...
Definition: SwiftErrorValueTracking.cpp:147
llvm::ISD::EH_LABEL
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:1033
llvm::SelectionDAGISel::OPC_CheckChild1Integer
@ OPC_CheckChild1Integer
Definition: SelectionDAGISel.h:145
llvm::SelectionDAGISel::OPC_CheckFoldableChainNode
@ OPC_CheckFoldableChainNode
Definition: SelectionDAGISel.h:153
llvm::MachineRegisterInfo::defusechain_instr_iterator
defusechain_iterator - This class provides iterator support for machine operands in the function that...
Definition: MachineRegisterInfo.h:277
llvm::HandleSDNode
This class is used to form a handle around another node that is persistent and is updated across invo...
Definition: SelectionDAGNodes.h:1230
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
llvm::createFastDAGScheduler
ScheduleDAGSDNodes * createFastDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createFastDAGScheduler - This creates a "fast" scheduler.
Definition: ScheduleDAGFast.cpp:815
llvm::SelectionDAGISel::OPC_CheckChild1Same
@ OPC_CheckChild1Same
Definition: SelectionDAGISel.h:131
OptimizationRemarkEmitter.h
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:462
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
MachineBasicBlock.h
llvm::TargetSubtargetInfo::getRegisterInfo
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Definition: TargetSubtargetInfo.h:125
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:140
llvm::MVT::Glue
@ Glue
Definition: MachineValueType.h:272
llvm::SelectionDAGISel::OPC_MorphNodeTo2
@ OPC_MorphNodeTo2
Definition: SelectionDAGISel.h:172
llvm::SelectionDAGISel::OPC_MoveChild7
@ OPC_MoveChild7
Definition: SelectionDAGISel.h:128
llvm::SDNode::use_iterator
This class provides iterator support for SDUse operands that use a specific SDNode.
Definition: SelectionDAGNodes.h:743
APInt.h
llvm::TargetRegisterInfo
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Definition: TargetRegisterInfo.h:237
llvm::SelectionDAGISel::CheckPatternPredicate
virtual bool CheckPatternPredicate(unsigned PredNo) const
CheckPatternPredicate - This function is generated by tblgen in the target.
Definition: SelectionDAGISel.h:268
llvm::LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage
static void getLazyBFIAnalysisUsage(AnalysisUsage &AU)
Helper for client passes to set up the analysis usage on behalf of this pass.
Definition: LazyBlockFrequencyInfo.cpp:62
llvm::Function::getContext
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:320
llvm::DenseMapIterator
Definition: DenseMap.h:57
Shift
bool Shift
Definition: README.txt:468
llvm::InlineAsm::isUseOperandTiedToDef
static bool isUseOperandTiedToDef(unsigned Flag, unsigned &Idx)
isUseOperandTiedToDef - Return true if the flag of the inline asm operand indicates it is an use oper...
Definition: InlineAsm.h:361
ViewSchedDAGs
static cl::opt< bool > ViewSchedDAGs("view-sched-dags", cl::Hidden, cl::desc("Pop up a window to show sched dags as they are processed"))
DenseMap.h
llvm::TargetLowering::initializeSplitCSR
virtual void initializeSplitCSR(MachineBasicBlock *Entry) const
Perform necessary initialization to handle a subset of CSRs explicitly via copies.
Definition: TargetLowering.h:3978
llvm::TimePassesIsEnabled
bool TimePassesIsEnabled
If the user specifies the -time-passes argument on an LLVM tool command line then the value of this b...
Definition: PassTimingInfo.cpp:37
CheckChildInteger
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, unsigned ChildNo)
Definition: SelectionDAGISel.cpp:2619
llvm::SelectionDAGISel::OPC_CaptureGlueInput
@ OPC_CaptureGlueInput
Definition: SelectionDAGISel.h:125
llvm::SelectionDAGISel::OPC_EmitNodeXForm
@ OPC_EmitNodeXForm
Definition: SelectionDAGISel.h:166
TargetInstrInfo.h
llvm::MachineModuleInfo::setUsesMSVCFloatingPoint
void setUsesMSVCFloatingPoint(bool b)
Definition: MachineModuleInfo.h:186
llvm::MachineMemOperand
A description of a memory reference used in the backend.
Definition: MachineMemOperand.h:127
llvm::FunctionLoweringInfo::isExportedInst
bool isExportedInst(const Value *V) const
isExportedInst - Return true if the specified value is an instruction exported from its block.
Definition: FunctionLoweringInfo.h:198
EHPersonalities.h
llvm::TargetLoweringBase::getExceptionPointerRegister
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
Definition: TargetLowering.h:1826
llvm::RegisterScheduler::Registry
static MachinePassRegistry< FunctionPassCtor > Registry
RegisterScheduler class - Track the registration of instruction schedulers.
Definition: SchedulerRegistry.h:38
ViewDAGCombine1
static cl::opt< bool > ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the first " "dag combine pass"))
llvm::MachineFunction::setCallSiteLandingPad
void setCallSiteLandingPad(MCSymbol *Sym, ArrayRef< unsigned > Sites)
Map the landing pad's EH symbol to the call site indexes.
Definition: MachineFunction.cpp:861
llvm::SelectionDAGISel::OPC_CheckChild0Same
@ OPC_CheckChild0Same
Definition: SelectionDAGISel.h:131
llvm::DbgVariableIntrinsic::getVariable
DILocalVariable * getVariable() const
Definition: IntrinsicInst.h:258
llvm::Optional
Definition: APInt.h:33
LLVM_ATTRIBUTE_ALWAYS_INLINE
#define LLVM_ATTRIBUTE_ALWAYS_INLINE
LLVM_ATTRIBUTE_ALWAYS_INLINE - On compilers where we have a directive to do so, mark a method "always...
Definition: Compiler.h:230
llvm::MachineInstr::isCopy
bool isCopy() const
Definition: MachineInstr.h:1336
llvm::SelectionDAGISel::OPC_CheckPredicate
@ OPC_CheckPredicate
Definition: SelectionDAGISel.h:134
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:450
llvm::ore::NV
DiagnosticInfoOptimizationBase::Argument NV
Definition: OptimizationRemarkEmitter.h:136
llvm::SelectionDAGISel::OPC_Coverage
@ OPC_Coverage
Definition: SelectionDAGISel.h:175
llvm::ISD::isConstantSplatVectorAllZeros
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
Definition: SelectionDAG.cpp:219
llvm::InlineAsm::Op_AsmString
@ Op_AsmString
Definition: InlineAsm.h:217
llvm::max
Expected< ExpressionValue > max(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:337
llvm::TargetLowering::supportSplitCSR
virtual bool supportSplitCSR(MachineFunction *MF) const
Return true if the target supports that a subset of CSRs for the given machine function is handled ex...
Definition: TargetLowering.h:3968
llvm::SelectionDAG::RemoveDeadNodes
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
Definition: SelectionDAG.cpp:942
llvm::SelectionDAGISel::OPFL_GlueOutput
@ OPFL_GlueOutput
Definition: SelectionDAGISel.h:182
STLExtras.h
llvm::DIExpression
DWARF expression.
Definition: DebugInfoMetadata.h:2558
llvm::AfterLegalizeTypes
@ AfterLegalizeTypes
Definition: DAGCombine.h:17
RHS
Value * RHS
Definition: X86PartialReduction.cpp:76
llvm::SelectionDAGISel::OPC_CheckAndImm
@ OPC_CheckAndImm
Definition: SelectionDAGISel.h:150
llvm::SelectionDAGISel::OPC_EmitCopyToReg
@ OPC_EmitCopyToReg
Definition: SelectionDAGISel.h:164
llvm::SelectionDAGISel::OPC_SwitchType
@ OPC_SwitchType
Definition: SelectionDAGISel.h:140
llvm::Sched::Fast
@ Fast
Definition: TargetLowering.h:103
llvm::succ_empty
bool succ_empty(const Instruction *I)
Definition: CFG.h:255
llvm::SelectionDAGISel::OPC_CheckChild0Type
@ OPC_CheckChild0Type
Definition: SelectionDAGISel.h:141
replace
static void replace(Module &M, GlobalVariable *Old, GlobalVariable *New)
Definition: ConstantMerge.cpp:116
llvm::Sched::Linearize
@ Linearize
Definition: TargetLowering.h:104
llvm::SelectionDAGISel::OPC_CheckCondCode
@ OPC_CheckCondCode
Definition: SelectionDAGISel.h:147
SelectionDAG.h
llvm::SelectionDAGISel::OPC_CheckChild7Type
@ OPC_CheckChild7Type
Definition: SelectionDAGISel.h:143
llvm::initializeGCModuleInfoPass
void initializeGCModuleInfoPass(PassRegistry &)
processDbgDeclares
static void processDbgDeclares(FunctionLoweringInfo &FuncInfo)
Collect llvm.dbg.declare information.
Definition: SelectionDAGISel.cpp:1296
llvm::InlineAsm::Kind_Mem
@ Kind_Mem
Definition: InlineAsm.h:243
llvm::ISD::FREEZE
@ FREEZE
Definition: ISDOpcodes.h:216
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1628
llvm::SelectionDAG::clear
void clear()
Clear state and free memory necessary to make this SelectionDAG ready to process a new block.
Definition: SelectionDAG.cpp:1364
FilterDAGBasicBlockName
static cl::opt< std::string > FilterDAGBasicBlockName("filter-view-dags", cl::Hidden, cl::desc("Only display the basic block whose name " "matches this for all view-*-dags options"))
llvm::TargetLoweringBase::getOperationAction
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
Definition: TargetLowering.h:1109
llvm::initializeTargetLibraryInfoWrapperPassPass
void initializeTargetLibraryInfoWrapperPassPass(PassRegistry &)
llvm::SelectionDAGISel::OPC_CheckChild1Type
@ OPC_CheckChild1Type
Definition: SelectionDAGISel.h:141
llvm::SelectionDAGISel::OPC_EmitStringInteger
@ OPC_EmitStringInteger
Definition: SelectionDAGISel.h:156
llvm::MachineFunctionPass::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition: MachineFunctionPass.cpp:167
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
new
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM ID Predecessors according to mbb< bb27, 0x8b0a7c0 > Note ADDri is not a two address instruction its result reg1037 is an operand of the PHI node in bb76 and its operand reg1039 is the result of the PHI node We should treat it as a two address code and make sure the ADDri is scheduled after any node that reads reg1039 Use info(i.e. register scavenger) to assign it a free register to allow reuse the collector could move the objects and invalidate the derived pointer This is bad enough in the first but safe points can crop up unpredictably **array_addr i32 n y store obj * new
Definition: README.txt:125
F
#define F(x, y, z)
Definition: MD5.cpp:55
llvm::RISCVFenceField::R
@ R
Definition: RISCVBaseInfo.h:265
MachineRegisterInfo.h
KnownBits.h
llvm::SelectionDAGISel::PreprocessISelDAG
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
Definition: SelectionDAGISel.h:81
llvm::ISD::ARITH_FENCE
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1150
llvm::SelectionDAG::getRegister
SDValue getRegister(unsigned Reg, EVT VT)
Definition: SelectionDAG.cpp:2102
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:55
llvm::ISD::INLINEASM
@ INLINEASM
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:1025
llvm::EVT::isSimple
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:129
llvm::ISD::PSEUDO_PROBE
@ PSEUDO_PROBE
Pseudo probe for AutoFDO, as a place holder in a basic block to improve the sample counts quality.
Definition: ISDOpcodes.h:1244
llvm::TargetIntrinsicInfo
TargetIntrinsicInfo - Interface to description of machine instruction set.
Definition: TargetIntrinsicInfo.h:29
mapWasmLandingPadIndex
static void mapWasmLandingPadIndex(MachineBasicBlock *MBB, const CatchPadInst *CPI)
Definition: SelectionDAGISel.cpp:1190
CheckAndImm
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
Definition: SelectionDAGISel.cpp:2627
AliasAnalysis.h
UseMBPI
static cl::opt< bool > UseMBPI("use-mbpi", cl::desc("use Machine Branch Probability Info"), cl::init(true), cl::Hidden)
MachineValueType.h
llvm::MVT::SimpleValueType
SimpleValueType
Definition: MachineValueType.h:33
llvm::MachineFunction::setExposesReturnsTwice
void setExposesReturnsTwice(bool B)
setCallsSetJmp - Set a flag that indicates if there's a call to a "returns twice" function.
Definition: MachineFunction.h:724
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::classifyEHPersonality
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
Definition: EHPersonalities.cpp:22
llvm::MachineFunction::setWasmLandingPadIndex
void setWasmLandingPadIndex(const MachineBasicBlock *LPad, unsigned Index)
Map the landing pad to its index. Used for Wasm exception handling.
Definition: MachineFunction.h:1150
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:187
llvm::TargetLowering::AdjustInstrPostInstrSelection
virtual void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
Definition: SelectionDAGISel.cpp:302
llvm::SelectionDAGISel::OPC_EmitMergeInputChains1_0
@ OPC_EmitMergeInputChains1_0
Definition: SelectionDAGISel.h:161
llvm::SelectionDAGISel::ElidedArgCopyInstrs
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
Definition: SelectionDAGISel.h:58
llvm::SelectionDAGISel::OPC_MoveChild5
@ OPC_MoveChild5
Definition: SelectionDAGISel.h:128
Instruction.h
CommandLine.h
llvm::BeforeLegalizeTypes
@ BeforeLegalizeTypes
Definition: DAGCombine.h:16
LHS
Value * LHS
Definition: X86PartialReduction.cpp:75
TargetLowering.h
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
llvm::FuncletPadInst::getArgOperand
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th funcletpad argument.
Definition: InstrTypes.h:2399
llvm::SDNode::getOpcode
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
Definition: SelectionDAGNodes.h:642
llvm::MachineFunction::front
const MachineBasicBlock & front() const
Definition: MachineFunction.h:865
llvm::BlockFrequencyInfo
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Definition: BlockFrequencyInfo.h:37
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:666
reportFastISelFailure
static void reportFastISelFailure(MachineFunction &MF, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R, bool ShouldAbort)
Definition: SelectionDAGISel.cpp:651
llvm::SelectionDAGISel::OPC_RecordChild6
@ OPC_RecordChild6
Definition: SelectionDAGISel.h:123
llvm::MachineBasicBlock::insertAfter
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
Definition: MachineBasicBlock.h:930
llvm::SelectionDAGISel::OPC_RecordChild1
@ OPC_RecordChild1
Definition: SelectionDAGISel.h:122
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1138
ViewDAGCombineLT
static cl::opt< bool > ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the post " "legalize types dag combine pass"))
CheckValueType
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
Definition: SelectionDAGISel.cpp:2584
llvm::MachineFunctionProperties::Property::Selected
@ Selected
TargetMachine.h
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:24
CheckOrImm
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
Definition: SelectionDAGISel.cpp:2640
llvm::SelectionDAGISel::OPC_EmitNode0
@ OPC_EmitNode0
Definition: SelectionDAGISel.h:169
llvm::SelectionDAG
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:220
SelectionDAGNodes.h
Constants.h
llvm::SelectionDAGISel::RunSDNodeXForm
virtual SDValue RunSDNodeXForm(SDValue V, unsigned XFormNo)
Definition: SelectionDAGISel.h:297
llvm::ISD::Constant
@ Constant
Definition: ISDOpcodes.h:76
llvm::SelectionDAGISel::SDB
std::unique_ptr< SelectionDAGBuilder > SDB
Definition: SelectionDAGISel.h:50
llvm::SelectionDAGISel::OPC_CheckOpcode
@ OPC_CheckOpcode
Definition: SelectionDAGISel.h:136
llvm::SelectionDAGISel::OPC_CheckPatternPredicate
@ OPC_CheckPatternPredicate
Definition: SelectionDAGISel.h:133
llvm::BranchProbabilityInfoWrapperPass
Legacy analysis pass which computes BranchProbabilityInfo.
Definition: BranchProbabilityInfo.h:438
FunctionLoweringInfo.h
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::SelectionDAG::LegalizeTypes
bool LegalizeTypes()
This transforms the SelectionDAG into a SelectionDAG that only uses types natively supported by the t...
Definition: LegalizeTypes.cpp:1059
llvm::DIExpression::ApplyOffset
@ ApplyOffset
Definition: DebugInfoMetadata.h:2763
llvm::User
Definition: User.h:44
llvm::SelectionDAGISel::OPC_MoveChild
@ OPC_MoveChild
Definition: SelectionDAGISel.h:126
llvm::ISD::CopyToReg
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition: ISDOpcodes.h:203
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:34
llvm::SelectionDAGISel::RegInfo
MachineRegisterInfo * RegInfo
Definition: SelectionDAGISel.h:48
Intrinsics.h
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::MachineInstr::getOperand
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:526
llvm::SelectionDAGISel::OptLevel
CodeGenOpt::Level OptLevel
Definition: SelectionDAGISel.h:54
llvm::SelectionDAGISel::CheckNodePredicate
virtual bool CheckNodePredicate(SDNode *N, unsigned PredNo) const
CheckNodePredicate - This function is generated by tblgen in the target.
Definition: SelectionDAGISel.h:276
llvm::getLLTForMVT
LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
Definition: LowLevelType.cpp:64
llvm::TargetMachine::getIntrinsicInfo
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
Definition: TargetMachine.h:212
llvm::TargetLowering
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Definition: TargetLowering.h:3446
ViewLegalizeTypesDAGs
static cl::opt< bool > ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize types"))
llvm::KnownBits::One
APInt One
Definition: KnownBits.h:25
llvm::SelectionDAGISel::ComplexPatternFuncMutatesDAG
virtual bool ComplexPatternFuncMutatesDAG() const
Return true if complex patterns for this target can mutate the DAG.
Definition: SelectionDAGISel.h:306
llvm::TargetTransformInfo::hasBranchDivergence
bool hasBranchDivergence() const
Return true if branch divergence exists.
Definition: TargetTransformInfo.cpp:237
llvm::MachineBasicBlock::isSuccessor
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
Definition: MachineBasicBlock.cpp:924
llvm::StackProtector
Definition: StackProtector.h:36
llvm::SelectionDAG::MaskedValueIsZero
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
Definition: SelectionDAG.cpp:2513
llvm::createILPListDAGScheduler
ScheduleDAGSDNodes * createILPListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level)
createILPListDAGScheduler - This creates a bottom up register pressure aware list scheduler that trie...
Definition: ScheduleDAGRRList.cpp:3191
llvm::Register::isPhysicalRegister
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:65
llvm::InlineAsm::Op_MDNode
@ Op_MDNode
Definition: InlineAsm.h:218
llvm::TargetRegisterClass
Definition: TargetRegisterInfo.h:46
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
llvm::SelectionDAG::Legalize
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
Definition: LegalizeDAG.cpp:5066
ViewLegalizeDAGs
static cl::opt< bool > ViewLegalizeDAGs("view-legalize-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize"))
CheckInteger
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
Definition: SelectionDAGISel.cpp:2606
llvm::MachineFunction::getProperties
const MachineFunctionProperties & getProperties() const
Get the function properties.
Definition: MachineFunction.h:747
TargetLibraryInfo.h
llvm::BitTracker
Definition: BitTracker.h:35
llvm::SelectionDAGISel::ReplaceNode
void ReplaceNode(SDNode *F, SDNode *T)
Replace all uses of F with T, then remove F from the DAG.
Definition: SelectionDAGISel.h:231
llvm::SelectionDAGISel::ORE
std::unique_ptr< OptimizationRemarkEmitter > ORE
Current optimization remark emitter.
Definition: SelectionDAGISel.h:63
llvm::SelectionDAGISel::OPC_MorphNodeTo1
@ OPC_MorphNodeTo1
Definition: SelectionDAGISel.h:172
llvm::InlineAsm::Op_ExtraInfo
@ Op_ExtraInfo
Definition: InlineAsm.h:219
llvm::dwarf::Index
Index
Definition: Dwarf.h:472
llvm::ISD::DELETED_NODE
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:44
llvm::MCInstrDesc
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:197
CheckCondCode
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
Definition: SelectionDAGISel.cpp:2569
llvm::SelectionDAG::setRoot
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:551
llvm::EVT::isInteger
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:144
llvm::Instruction
Definition: Instruction.h:42
llvm::MachineModuleInfo
This class contains meta information specific to a module.
Definition: MachineModuleInfo.h:74
llvm::ISD::MDNODE_SDNODE
@ MDNODE_SDNODE
MDNODE_SDNODE - This is a node that holdes an MDNode*, which is used to reference metadata in the IR.
Definition: ISDOpcodes.h:1102
llvm::createSourceListDAGScheduler
ScheduleDAGSDNodes * createSourceListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createBURRListDAGScheduler - This creates a bottom up list scheduler that schedules nodes in source c...
Definition: ScheduleDAGRRList.cpp:3161
llvm::SDNode::use_empty
bool use_empty() const
Return true if there are no uses of this node.
Definition: SelectionDAGNodes.h:713
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
llvm::BasicBlock::phis
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:364
llvm::STATISTIC
STATISTIC(NumFunctions, "Total number of functions")
decodeSignRotatedValue
static uint64_t decodeSignRotatedValue(uint64_t V)
Decode a signed value stored with the sign bit in the LSB for dense VBR encoding.
Definition: SelectionDAGISel.cpp:2596
CheckPatternPredicate
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel)
CheckPatternPredicate - Implements OP_CheckPatternPredicate.
Definition: SelectionDAGISel.cpp:2528
llvm::SelectionDAG::DAGNodeDeletedListener
Definition: SelectionDAG.h:327
llvm::predecessors
auto predecessors(MachineBasicBlock *BB)
Definition: MachineSSAContext.h:30
ScheduleDAGSDNodes.h
llvm::ConstantFP
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:257
llvm::SmallVectorImpl::resize
void resize(size_type N)
Definition: SmallVector.h:624
llvm::Use::getUser
User * getUser() const
Returns the User that contains this Use.
Definition: Use.h:72
llvm::StringRef::data
const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:131
llvm::SelectionDAG::dump
void dump() const
Definition: SelectionDAGDumper.cpp:925
llvm::SelectionDAGISel::ID
static char ID
Definition: SelectionDAGISel.h:65
llvm::ISD::AssertAlign
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
Definition: ISDOpcodes.h:68
DebugLoc.h
SmallPtrSet.h
llvm::SwiftErrorValueTracking::preassignVRegs
void preassignVRegs(MachineBasicBlock *MBB, BasicBlock::const_iterator Begin, BasicBlock::const_iterator End)
Definition: SwiftErrorValueTracking.cpp:258
llvm::RegisterScheduler::FunctionPassCtor
ScheduleDAGSDNodes *(*)(SelectionDAGISel *, CodeGenOpt::Level) FunctionPassCtor
Definition: SchedulerRegistry.h:36
llvm::ISD::AND
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:666
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:669
llvm::ISD::TargetGlobalAddress
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
Definition: ISDOpcodes.h:164
llvm::BasicBlock::getFirstNonPHI
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:209
llvm::ISD::EntryToken
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:47
CheckNodePredicate
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel, SDNode *N)
CheckNodePredicate - Implements OP_CheckNodePredicate.
Definition: SelectionDAGISel.cpp:2535
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
findNonImmUse
static bool findNonImmUse(SDNode *Root, SDNode *Def, SDNode *ImmedUse, bool IgnoreChains)
findNonImmUse - Return true if "Def" is a predecessor of "Root" via a path beyond "ImmedUse".
Definition: SelectionDAGISel.cpp:2029
llvm::SelectionDAGISel::OPC_CheckSame
@ OPC_CheckSame
Definition: SelectionDAGISel.h:130
llvm::MachineRegisterInfo::getVRegDef
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
Definition: MachineRegisterInfo.cpp:396
ViewISelDAGs
static cl::opt< bool > ViewISelDAGs("view-isel-dags", cl::Hidden, cl::desc("Pop up a window to show isel dags as they are selected"))
llvm::SDValue::getValueSizeInBits
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
Definition: SelectionDAGNodes.h:199
llvm::ISD::CopyFromReg
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:208
llvm::RegisterPassParser
RegisterPassParser class - Handle the addition of new machine passes.
Definition: MachinePassRegistry.h:135
llvm::EHPersonality::Wasm_CXX
@ Wasm_CXX
llvm::MachineRegisterInfo::use_empty
bool use_empty(Register RegNo) const
use_empty - Return true if there are no instructions using the specified register.
Definition: MachineRegisterInfo.h:514
Statepoint.h
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm::erase_value
void erase_value(Container &C, ValueType V)
Wrapper function to remove a value from a container:
Definition: STLExtras.h:1810
Type.h
BranchProbability.h
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:94
llvm::TargetMachine::resetTargetOptions
void resetTargetOptions(const Function &F) const
Reset the target options based on the function's attributes.
Definition: TargetMachine.cpp:53
llvm::Sched::RegPressure
@ RegPressure
Definition: TargetLowering.h:99
ViewSUnitDAGs
static cl::opt< bool > ViewSUnitDAGs("view-sunit-dags", cl::Hidden, cl::desc("Pop up a window to show SUnit dags after they are processed"))
llvm::InlineAsm::isMemKind
static bool isMemKind(unsigned Flag)
Definition: InlineAsm.h:296
llvm::OptLevelChanger::~OptLevelChanger
~OptLevelChanger()
Definition: SelectionDAGISel.cpp:235
llvm::TargetLowering::createFastISel
virtual FastISel * createFastISel(FunctionLoweringInfo &, const TargetLibraryInfo *) const
This method returns a target specific FastISel object, or null if the target does not support "fast" ...
Definition: TargetLowering.h:4475
Operands
mir Rename Register Operands
Definition: MIRNamerPass.cpp:74
llvm::MachineRegisterInfo::getRegClass
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
Definition: MachineRegisterInfo.h:647
llvm::MCID::MayRaiseFPException
@ MayRaiseFPException
Definition: MCInstrDesc.h:169
llvm::ISD::TargetConstantFP
@ TargetConstantFP
Definition: ISDOpcodes.h:159
llvm::SelectionDAGISel::InvalidateNodeId
static void InvalidateNodeId(SDNode *N)
Definition: SelectionDAGISel.cpp:1055
llvm::APInt::isSubsetOf
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1207
llvm::MDNode::getOperand
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1292
llvm::TargetLowering::insertCopiesSplitCSR
virtual void insertCopiesSplitCSR(MachineBasicBlock *Entry, const SmallVectorImpl< MachineBasicBlock * > &Exits) const
Insert explicit copies in entry and exit blocks.
Definition: TargetLowering.h:3986
llvm::MachineFunction::getMMI
MachineModuleInfo & getMMI() const
Definition: MachineFunction.h:607
llvm::ISD::STRICT_FSETCCS
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:476
llvm::MachineRegisterInfo::clearKillFlags
void clearKillFlags(Register Reg) const
clearKillFlags - Iterate over all the uses of the given register and clear the kill flag from the Mac...
Definition: MachineRegisterInfo.cpp:433
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:656
llvm::MCInstrDesc::mayLoad
bool mayLoad() const
Return true if this instruction could possibly read memory.
Definition: MCInstrDesc.h:435
llvm::ISD::WRITE_REGISTER
@ WRITE_REGISTER
Definition: ISDOpcodes.h:119
Timer.h
BasicBlock.h
llvm::cl::opt
Definition: CommandLine.h:1400
llvm::MachineFunction::setHasInlineAsm
void setHasInlineAsm(bool B)
Set a flag that indicates that the function contains inline assembly.
Definition: MachineFunction.h:734
llvm::SDNode::use_begin
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
Definition: SelectionDAGNodes.h:797
llvm::ISD::Register
@ Register
Definition: ISDOpcodes.h:74
llvm::SelectionDAGISel::OPC_CheckChild4Integer
@ OPC_CheckChild4Integer
Definition: SelectionDAGISel.h:146
llvm::instructions
inst_range instructions(Function *F)
Definition: InstIterator.h:133
llvm::FastISel::selectInstruction
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
Definition: FastISel.cpp:1476
llvm::SelectionDAGISel::OPC_CheckValueType
@ OPC_CheckValueType
Definition: SelectionDAGISel.h:148
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::SelectionDAG::RemoveDeadNode
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
Definition: SelectionDAG.cpp:996
llvm::SelectionDAGISel::OPC_CheckImmAllZerosV
@ OPC_CheckImmAllZerosV
Definition: SelectionDAGISel.h:152
BranchProbabilityInfo.h
llvm::SelectionDAGISel::OPC_CheckInteger
@ OPC_CheckInteger
Definition: SelectionDAGISel.h:144
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:340
llvm::SelectionDAGISel::IsProfitableToFold
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
Definition: SelectionDAGISel.cpp:2069
llvm::AfterLegalizeDAG
@ AfterLegalizeDAG
Definition: DAGCombine.h:19
CheckChild2CondCode
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChild2CondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
Definition: SelectionDAGISel.cpp:2576
Index
uint32_t Index
Definition: ELFObjHandler.cpp:82
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:66
index
splat index
Definition: README_ALTIVEC.txt:181
llvm::MachineInstrBuilder
Definition: MachineInstrBuilder.h:69
llvm::SelectionDAGISel::OPC_RecordMemRef
@ OPC_RecordMemRef
Definition: SelectionDAGISel.h:124
llvm::TargetLibraryInfoWrapperPass
Definition: TargetLibraryInfo.h:474
uint64_t
ProfileSummaryInfo.h
llvm::SelectionDAGISel::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition: SelectionDAGISel.cpp:332
llvm::TargetTransformInfoWrapperPass
Wrapper pass for TargetTransformInfo.
Definition: TargetTransformInfo.h:2626
llvm::SelectionDAGISel::TII
const TargetInstrInfo * TII
Definition: SelectionDAGISel.h:55
llvm::GlobalValue::getParent
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:650
llvm::MCInstrDesc::isCall
bool isCall() const
Return true if the instruction is a call.
Definition: MCInstrDesc.h:285
llvm::DbgDeclareInst
This represents the llvm.dbg.declare instruction.
Definition: IntrinsicInst.h:313
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::MachineRegisterInfo::use_instr_begin
use_instr_iterator use_instr_begin(Register RegNo) const
Definition: MachineRegisterInfo.h:485
llvm::SelectionDAGISel::OPC_RecordChild0
@ OPC_RecordChild0
Definition: SelectionDAGISel.h:122
HandleMergeInputChains
static SDValue HandleMergeInputChains(SmallVectorImpl< SDNode * > &ChainNodesMatched, SelectionDAG *CurDAG)
HandleMergeInputChains - This implements the OPC_EmitMergeInputChains operation for when the pattern ...
Definition: SelectionDAGISel.cpp:2386
llvm::ISD::AssertZext
@ AssertZext
Definition: ISDOpcodes.h:62
llvm::Function::hasGC
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
Definition: Function.h:306
llvm::ISD::STRICT_LRINT
@ STRICT_LRINT
Definition: ISDOpcodes.h:432
llvm::SelectionDAG::mutateStrictFPToFP
SDNode * mutateStrictFPToFP(SDNode *Node)
Mutate the specified strict FP node to its non-strict equivalent, unlinking the node from its chain a...
Definition: SelectionDAG.cpp:9706
llvm::SelectionDAGISel::FuncInfo
std::unique_ptr< FunctionLoweringInfo > FuncInfo
Definition: SelectionDAGISel.h:45
llvm::TargetRegisterInfo::getFrameRegister
virtual Register getFrameRegister(const MachineFunction &MF) const =0
Debug information queries.
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:53
llvm::SDNode::setNodeId
void setNodeId(int Id)
Set unique node id.
Definition: SelectionDAGNodes.h:726
llvm::SelectionDAG::getCopyFromReg
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:790
llvm::DenseMap< unsigned, unsigned >
llvm::SelectionDAGISel::IsLegalToFold
static bool IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, CodeGenOpt::Level OptLevel, bool IgnoreChains=false)
IsLegalToFold - Returns true if the specific operand node N of U can be folded during instruction sel...
Definition: SelectionDAGISel.cpp:2077
llvm::SelectionDAGISel::mayRaiseFPException
bool mayRaiseFPException(SDNode *Node) const
Return whether the node may raise an FP exception.
Definition: SelectionDAGISel.cpp:3724
llvm::SelectionDAGISel::OPC_CheckChild2Same
@ OPC_CheckChild2Same
Definition: SelectionDAGISel.h:132
llvm::SDNode::getOperand
const SDValue & getOperand(unsigned Num) const
Definition: SelectionDAGNodes.h:918
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::SwiftErrorValueTracking
Definition: SwiftErrorValueTracking.h:34
llvm::MCPhysReg
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition: MCRegister.h:21
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:9010
CheckOpcode
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDNode *N)
Definition: SelectionDAGISel.cpp:2541
llvm::MachineFrameInfo::getObjectAlign
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
Definition: MachineFrameInfo.h:483
llvm::SelectionDAGISel::OPC_CheckChild4Type
@ OPC_CheckChild4Type
Definition: SelectionDAGISel.h:142
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:440
llvm::ProfileSummaryInfoWrapperPass
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
Definition: ProfileSummaryInfo.h:193
llvm::is_contained
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:1673
llvm::MachineRegisterInfo::liveins
ArrayRef< std::pair< MCRegister, Register > > liveins() const
Definition: MachineRegisterInfo.h:971
llvm::MachineFunction::finalizeDebugInstrRefs
void finalizeDebugInstrRefs()
Finalise any partially emitted debug instructions.
Definition: MachineFunction.cpp:1188
llvm::SelectionDAGISel::getPatternForIndex
virtual StringRef getPatternForIndex(unsigned index)
getPatternForIndex - Patterns selected by tablegen during ISEL
Definition: SelectionDAGISel.h:243
MachineFunctionPass.h
llvm::SelectionDAG::MorphNodeTo
SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
Definition: SelectionDAG.cpp:9653
llvm::SelectionDAG::NewNodesMustHaveLegalTypes
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
Definition: SelectionDAG.h:381
llvm::Register::isVirtualRegister
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:71
llvm::SelectionDAGISel::OPC_CheckImmAllOnesV
@ OPC_CheckImmAllOnesV
Definition: SelectionDAGISel.h:151
llvm::SelectionDAGISel::OPC_SwitchOpcode
@ OPC_SwitchOpcode
Definition: SelectionDAGISel.h:137
llvm::DenseMapBase::find
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:150
llvm::MachineFunction::getName
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Definition: MachineFunction.cpp:567
llvm::SelectionDAGISel::OPC_CheckChild3Integer
@ OPC_CheckChild3Integer
Definition: SelectionDAGISel.h:146
llvm::ISD::STRICT_LROUND
@ STRICT_LROUND
Definition: ISDOpcodes.h:430
llvm::SDValue::getValue
SDValue getValue(unsigned R) const
Definition: SelectionDAGNodes.h:179
llvm::TargetMachine::Options
TargetOptions Options
Definition: TargetMachine.h:118
llvm::SelectionDAGISel::CheckOrMask
bool CheckOrMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const
CheckOrMask - The isel is trying to match something like (or X, 255).
Definition: SelectionDAGISel.cpp:1926
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::TargetMachine
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:77
llvm::SelectionDAGISel::~SelectionDAGISel
~SelectionDAGISel() override
Definition: SelectionDAGISel.cpp:327
llvm::FunctionLoweringInfo
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Definition: FunctionLoweringInfo.h:52
llvm::MVT::Other
@ Other
Definition: MachineValueType.h:42
llvm::SDNode::getNodeId
int getNodeId() const
Return the unique node id.
Definition: SelectionDAGNodes.h:723
llvm::createVLIWDAGScheduler
ScheduleDAGSDNodes * createVLIWDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createVLIWDAGScheduler - Scheduler for VLIW targets.
Definition: ScheduleDAGVLIW.cpp:269
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:853
llvm::MachineFunction::getFrameInfo
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Definition: MachineFunction.h:672
SI
StandardInstrumentations SI(Debug, VerifyEach)
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition: MachineBasicBlock.h:261
llvm::OptimizationRemarkEmitter::emit
void emit(DiagnosticInfoOptimizationBase &OptDiag)
Output the remark via the diagnostic handler and to the optimization record file.
Definition: OptimizationRemarkEmitter.cpp:77
llvm::ISD::STACKMAP
@ STACKMAP
Definition: ISDOpcodes.h:1293
llvm::SelectionDAGISel::OPC_EmitMergeInputChains
@ OPC_EmitMergeInputChains
Definition: SelectionDAGISel.h:160
llvm::InlineAsm::getMemoryConstraintID
static unsigned getMemoryConstraintID(unsigned Flag)
Definition: InlineAsm.h:348
llvm::ISD::CondCode
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1424
llvm::ISD::BasicBlock
@ BasicBlock
Various leaf nodes.
Definition: ISDOpcodes.h:71
llvm::SelectionDAGISel::CurDAG
SelectionDAG * CurDAG
Definition: SelectionDAGISel.h:49
function
print Print MemDeps of function
Definition: MemDepPrinter.cpp:82
llvm::SelectionDAG::getMachineNode
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
Definition: SelectionDAG.cpp:9754
CodeGenCommonISel.h
MachineModuleInfo.h
llvm::ISD::TargetGlobalTLSAddress
@ TargetGlobalTLSAddress
Definition: ISDOpcodes.h:165
llvm::Sched::VLIW
@ VLIW
Definition: TargetLowering.h:102
llvm::MVT
Machine Value Type.
Definition: MachineValueType.h:31
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:97
llvm::SwiftErrorValueTracking::setFunction
void setFunction(MachineFunction &MF)
Initialize data structures for specified new function.
Definition: SwiftErrorValueTracking.cpp:79
llvm::FastISel
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:66
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:359
llvm::SelectionDAGISel::getUninvalidatedNodeId
static int getUninvalidatedNodeId(SDNode *N)
Definition: SelectionDAGISel.cpp:1061
llvm::SelectionDAGISel::SelectionDAGISel
SelectionDAGISel(TargetMachine &tm, CodeGenOpt::Level OL=CodeGenOpt::Default)
Definition: SelectionDAGISel.cpp:313
llvm::SelectionDAG::setNodeMemRefs
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
Definition: SelectionDAG.cpp:9522
SwiftErrorValueTracking.h
llvm::MDNode
Metadata node.
Definition: Metadata.h:944
llvm::ISD::STRICT_LLRINT
@ STRICT_LLRINT
Definition: ISDOpcodes.h:433
llvm::AfterLegalizeVectorOps
@ AfterLegalizeVectorOps
Definition: DAGCombine.h:18
llvm::MCInstrInfo::getName
StringRef getName(unsigned Opcode) const
Returns the name for the instructions with the given opcode.
Definition: MCInstrInfo.h:69
llvm::SelectionDAG::SelectNodeTo
SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
Definition: SelectionDAG.cpp:9546
llvm::TargetOptions::EnableFastISel
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
Definition: TargetOptions.h:232
llvm::SDNode::isMachineOpcode
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
Definition: SelectionDAGNodes.h:697
llvm::ISD::TargetBlockAddress
@ TargetBlockAddress
Definition: ISDOpcodes.h:170
llvm::SelectionDAGISel::OPC_CheckPredicateWithOperands
@ OPC_CheckPredicateWithOperands
Definition: SelectionDAGISel.h:135
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:75
llvm::MachineFunction
Definition: MachineFunction.h:257
llvm::MCInstrDesc::mayStore
bool mayStore() const
Return true if this instruction could possibly modify memory.
Definition: MCInstrDesc.h:441
llvm::ISD::ANNOTATION_LABEL
@ ANNOTATION_LABEL
ANNOTATION_LABEL - Represents a mid basic block label used by annotations.
Definition: ISDOpcodes.h:1039
llvm::SelectionDAGISel::CheckNodePredicateWithOperands
virtual bool CheckNodePredicateWithOperands(SDNode *N, unsigned PredNo, const SmallVectorImpl< SDValue > &Operands) const
CheckNodePredicateWithOperands - This function is generated by tblgen in the target.
Definition: SelectionDAGISel.h:285
llvm::MDNodeSDNode
Definition: SelectionDAGNodes.h:2142
llvm::createHybridListDAGScheduler
ScheduleDAGSDNodes * createHybridListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level)
createHybridListDAGScheduler - This creates a bottom up register pressure aware list scheduler that m...
Definition: ScheduleDAGRRList.cpp:3175
llvm::CodeGenOpt::None
@ None
Definition: CodeGen.h:53
CFG.h
llvm::ISD::HANDLENODE
@ HANDLENODE
HANDLENODE node - Used as a handle for various purposes.
Definition: ISDOpcodes.h:1116
llvm::MachineBasicBlock::succ_empty
bool succ_empty() const
Definition: MachineBasicBlock.h:384
TargetOptions.h
SelectionDAGISel.h
ViewDAGCombine2
static cl::opt< bool > ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the second " "dag combine pass"))
llvm::ISD::TargetConstantPool
@ TargetConstantPool
Definition: ISDOpcodes.h:168
Scheduler
Machine Instruction Scheduler
Definition: MachineScheduler.cpp:222
llvm::NVPTXISD::Dummy
@ Dummy
Definition: NVPTXISelLowering.h:60
llvm::MachinePassRegistry
MachinePassRegistry - Track the registration of machine passes.
Definition: MachinePassRegistry.h:73
llvm::MachineBasicBlock::getFirstTerminator
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
Definition: MachineBasicBlock.cpp:239
llvm::SmallPtrSetImplBase::clear
void clear()
Definition: SmallPtrSet.h:95
llvm::Sched::Source
@ Source
Definition: TargetLowering.h:98
llvm::AssumptionCacheTracker
An immutable pass that tracks lazily created AssumptionCache objects.
Definition: AssumptionCache.h:202
llvm::initializeAAResultsWrapperPassPass
void initializeAAResultsWrapperPassPass(PassRegistry &)
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::StackProtector::shouldEmitSDCheck
bool shouldEmitSDCheck(const BasicBlock &BB) const
Definition: StackProtector.cpp:595
llvm::OptimizationRemarkEmitter
The optimization diagnostic interface.
Definition: OptimizationRemarkEmitter.h:33
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1597
llvm::SDNode::use_end
static use_iterator use_end()
Definition: SelectionDAGNodes.h:801
llvm::MachineFrameInfo::setHasCalls
void setHasCalls(bool V)
Definition: MachineFrameInfo.h:606
DataLayout.h
llvm::SelectionDAGISel::OPC_CheckChild3Type
@ OPC_CheckChild3Type
Definition: SelectionDAGISel.h:142
llvm::MVT::i64
@ i64
Definition: MachineValueType.h:49
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
llvm::MachineBasicBlock::splice
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Definition: MachineBasicBlock.h:1009
llvm::SelectionDAGISel::SelectInlineAsmMemoryOperand
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
Definition: SelectionDAGISel.h:95
MBBI
MachineBasicBlock MachineBasicBlock::iterator MBBI
Definition: AArch64SLSHardening.cpp:75
llvm::SDNodeFlags::setNoFPExcept
void setNoFPExcept(bool b)
Definition: SelectionDAGNodes.h:428
llvm::SelectionDAGBuilder
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
Definition: SelectionDAGBuilder.h:95
llvm::InlineAsm::getFlagWord
static unsigned getFlagWord(unsigned Kind, unsigned NumOps)
Definition: InlineAsm.h:288
llvm::Sched::Hybrid
@ Hybrid
Definition: TargetLowering.h:100
llvm::ISD::STRICT_SINT_TO_FP
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:448
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::AnalysisUsage::addPreserved
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
Definition: PassAnalysisSupport.h:98
llvm::SelectionDAG::useInstrRefDebugInfo
void useInstrRefDebugInfo(bool Flag)
Signal whether instruction referencing variable locations are desired for this function's debug-info.
Definition: SelectionDAG.h:1847
llvm::ms_demangle::IntrinsicFunctionKind::New
@ New
llvm::BranchProbability
Definition: BranchProbability.h:30
Compiler.h
llvm::append_range
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
Definition: STLExtras.h:1818
llvm::TargetLowering::getRegisterByName
virtual Register getRegisterByName(const char *RegName, LLT Ty, const MachineFunction &MF) const
Return the register ID of the name passed in.
Definition: TargetLowering.h:4358
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:82
llvm::SelectionDAG::setFunctionLoweringInfo
void setFunctionLoweringInfo(FunctionLoweringInfo *FuncInfo)
Definition: SelectionDAG.h:455
TargetSubtargetInfo.h
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::FastISel::useInstrRefDebugInfo
void useInstrRefDebugInfo(bool Flag)
Signal whether instruction referencing variable locations are desired for this function's debug-inf