79#include "llvm/IR/IntrinsicsWebAssembly.h"
115#define DEBUG_TYPE "isel"
117STATISTIC(NumFastIselFailures,
"Number of instructions fast isel failed on");
118STATISTIC(NumFastIselSuccess,
"Number of instructions fast isel selected");
119STATISTIC(NumFastIselBlocks,
"Number of blocks selected entirely by fast isel");
120STATISTIC(NumDAGBlocks,
"Number of blocks selected using DAG");
121STATISTIC(NumDAGIselRetries,
"Number of times dag isel has to try another path");
122STATISTIC(NumEntryBlocks,
"Number of entry blocks encountered");
124 "Number of entry blocks where fast isel failed to lower arguments");
128 cl::desc(
"Enable abort calls when \"fast\" instruction selection "
129 "fails to lower an instruction: 0 disable the abort, 1 will "
130 "abort but for args, calls and terminators, 2 will also "
131 "abort for argument lowering, and 3 will never fallback "
132 "to SelectionDAG."));
136 cl::desc(
"Emit a diagnostic when \"fast\" instruction selection "
137 "falls back to SelectionDAG."));
141 cl::desc(
"use Machine Branch Probability Info"),
147 cl::desc(
"Only display the basic block whose name "
148 "matches this for all view-*-dags options"));
151 cl::desc(
"Pop up a window to show dags before the first "
152 "dag combine pass"));
155 cl::desc(
"Pop up a window to show dags before legalize types"));
158 cl::desc(
"Pop up a window to show dags before the post "
159 "legalize types dag combine pass"));
162 cl::desc(
"Pop up a window to show dags before legalize"));
165 cl::desc(
"Pop up a window to show dags before the second "
166 "dag combine pass"));
169 cl::desc(
"Pop up a window to show isel dags as they are selected"));
172 cl::desc(
"Pop up a window to show sched dags as they are processed"));
175 cl::desc(
"Pop up a window to show SUnit dags after they are processed"));
200 cl::desc(
"Instruction schedulers available (before register"
222 if (NewOptLevel == SavedOptLevel)
226 LLVM_DEBUG(
dbgs() <<
"\nChanging optimization level for Function "
228 LLVM_DEBUG(
dbgs() <<
"\tBefore: -O" << SavedOptLevel <<
" ; After: -O"
229 << NewOptLevel <<
"\n");
233 dbgs() <<
"\tFastISel is "
242 LLVM_DEBUG(
dbgs() <<
"\nRestoring optimization level for Function "
245 << SavedOptLevel <<
"\n");
261 if (
auto *SchedulerCtor = ST.getDAGScheduler(OptLevel)) {
262 return SchedulerCtor(IS, OptLevel);
266 (ST.enableMachineScheduler() && ST.enableMachineSchedDefaultSched()) ||
280 "Unknown sched type!");
299 dbgs() <<
"If a target marks an instruction with "
300 "'usesCustomInserter', it must implement "
301 "TargetLowering::EmitInstrWithCustomInserter!\n";
309 "If a target marks an instruction with 'hasPostISelHook', "
310 "it must implement TargetLowering::AdjustInstrPostInstrSelection!");
361 if (!TT.isWindowsMSVCEnvironment())
369 if (
I.getType()->isFPOrFPVectorTy()) {
373 for (
const auto &Op :
I.operands()) {
374 if (Op->getType()->isFPOrFPVectorTy()) {
389 "-fast-isel-abort > 0 requires -fast-isel");
414 LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(Fn);
415 GFI = Fn.
hasGC() ? &getAnalysis<GCModuleInfo>().getFunctionInfo(Fn) :
nullptr;
416 ORE = std::make_unique<OptimizationRemarkEmitter>(&Fn);
417 AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(mf.
getFunction());
418 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
421 BFI = &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI();
425 FnVarLocs = getAnalysis<AssignmentTrackingAnalysis>().getResults();
430 if (
auto *UAPass = getAnalysisIfAvailable<UniformityInfoWrapperPass>())
431 UA = &UAPass->getUniformityInfo();
442 FuncInfo->BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
447 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
468 if (isa<UnreachableInst>(Term) || isa<ReturnInst>(Term))
482 SelectAllBasicBlocks(Fn);
510 MRI.constrainRegClass(To,
MRI.getRegClass(
From));
516 if (!
MRI.use_empty(To))
536 if (Term !=
MBB.
end() && Term->isReturn()) {
545 if (!
FuncInfo->ArgDbgValues.empty())
551 for (
unsigned i = 0, e =
FuncInfo->ArgDbgValues.size(); i != e; ++i) {
553 assert(
MI->getOpcode() != TargetOpcode::DBG_VALUE_LIST &&
554 "Function parameters should not be described by DBG_VALUE_LIST.");
555 bool hasFI =
MI->getDebugOperand(0).isFI();
557 hasFI ?
TRI.getFrameRegister(*
MF) :
MI->getDebugOperand(0).getReg();
558 if (Reg.isPhysical())
565 Def->getParent()->insert(std::next(InsertPos),
MI);
577 if (LDI != LiveInMap.
end()) {
578 assert(!hasFI &&
"There's no handling of frame pointer updating here yet "
582 const MDNode *Variable =
MI->getDebugVariable();
583 const MDNode *Expr =
MI->getDebugExpression();
585 bool IsIndirect =
MI->isIndirectDebugValue();
587 assert(
MI->getDebugOffset().getImm() == 0 &&
588 "DBG_VALUE with nonzero offset");
589 assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(
DL) &&
590 "Expected inlined-at fields to agree");
591 assert(
MI->getOpcode() != TargetOpcode::DBG_VALUE_LIST &&
592 "Didn't expect to see a DBG_VALUE_LIST here");
595 IsIndirect, LDI->second, Variable, Expr);
607 CopyUseMI =
UseMI;
continue;
610 CopyUseMI =
nullptr;
break;
613 TRI.getRegSizeInBits(LDI->second,
MRI) ==
633 for (
const auto &
MBB : *
MF) {
637 for (
const auto &
MI :
MBB) {
640 MI.isStackAligningInlineAsm()) {
643 if (
MI.isInlineAsm()) {
671 if (!R.getLocation().isValid() || ShouldAbort)
672 R << (
" (in function: " + MF.
getName() +
")").str();
696 HadTailCall =
SDB->HasTailCall;
697 SDB->resolveOrClearDbgInfo();
704void SelectionDAGISel::ComputeLiveOutVRegInfo() {
717 for (
const SDValue &Op :
N->op_values())
718 if (
Op.getValueType() == MVT::Other &&
Added.insert(
Op.getNode()).second)
725 unsigned DestReg = cast<RegisterSDNode>(
N->getOperand(1))->getReg();
731 EVT SrcVT = Src.getValueType();
737 FuncInfo->AddLiveOutRegInfo(DestReg, NumSignBits, Known);
738 }
while (!Worklist.
empty());
741void SelectionDAGISel::CodeGenAndEmitDAG() {
743 StringRef GroupDescription =
"Instruction Selection and Scheduling";
744 std::string BlockName;
745 bool MatchFilterBB =
false; (void)MatchFilterBB;
748 getAnalysis<TargetTransformInfoWrapperPass>().getTTI(*
FuncInfo->Fn);
757 FuncInfo->MBB->getBasicBlock()->getName());
939 ComputeLiveOutVRegInfo();
949 DoInstructionSelection();
985 if (FirstMBB != LastMBB)
986 SDB->UpdateSplitBlock(FirstMBB, LastMBB);
1008 :
SelectionDAG::DAGUpdateListener(DAG), ISelPosition(isp) {}
1021 void NodeInserted(
SDNode *
N)
override {
1022 SDNode *CurNode = &*ISelPosition;
1023 if (
MDNode *MD = DAG.getPCSections(CurNode))
1024 DAG.addPCSections(
N, MD);
1054 while (!Nodes.
empty()) {
1056 for (
auto *U :
N->uses()) {
1057 auto UId = U->getNodeId();
1070 int InvalidId = -(
N->getNodeId() + 1);
1071 N->setNodeId(InvalidId);
1076 int Id =
N->getNodeId();
1082void SelectionDAGISel::DoInstructionSelection() {
1085 <<
FuncInfo->MBB->getName() <<
"'\n");
1103 ISelUpdater ISU(*
CurDAG, ISelPosition);
1110 SDNode *Node = &*--ISelPosition;
1114 if (Node->use_empty())
1121 while (!Nodes.
empty()) {
1125 for (
const SDValue &Op :
N->op_values()) {
1137 assert(Op->getNodeId() != -1 &&
1138 "Node has already selected predecessor node");
1155 switch (
Node->getOpcode()) {
1164 ActionVT =
Node->getOperand(1).getValueType();
1167 ActionVT =
Node->getValueType(0);
1175 LLVM_DEBUG(
dbgs() <<
"\nISEL: Starting selection on root node: ";
1191 if (
const IntrinsicInst *EHPtrCall = dyn_cast<IntrinsicInst>(U)) {
1193 if (IID == Intrinsic::eh_exceptionpointer ||
1194 IID == Intrinsic::eh_exceptioncode)
1209 bool IsSingleCatchAllClause =
1214 bool IsCatchLongjmp = CPI->
arg_size() == 0;
1215 if (!IsSingleCatchAllClause && !IsCatchLongjmp) {
1217 bool IntrFound =
false;
1219 if (
const auto *Call = dyn_cast<IntrinsicInst>(U)) {
1221 if (IID == Intrinsic::wasm_landingpad_index) {
1222 Value *IndexArg = Call->getArgOperand(1);
1223 int Index = cast<ConstantInt>(IndexArg)->getZExtValue();
1230 assert(IntrFound &&
"wasm.landingpad.index intrinsic not found!");
1237bool SelectionDAGISel::PrepareEHLandingPad() {
1249 if (
const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->
getFirstNonPHI())) {
1254 assert(EHPhysReg &&
"target lacks exception pointer register");
1256 unsigned VReg =
FuncInfo->getCatchPadExceptionPointerVReg(CPI, PtrRC);
1258 TII->
get(TargetOpcode::COPY), VReg)
1276 if (
auto *RegMask =
TRI.getCustomEHPadPreservedMask(*
MF))
1280 if (
const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->
getFirstNonPHI()))
1318 TII->
get(TargetOpcode::EH_LABEL))
1327 TII->
get(TargetOpcode::EH_LABEL))
1338 return !
I->mayWriteToMemory() &&
1339 !
I->isTerminator() &&
1340 !isa<DbgInfoIntrinsic>(
I) &&
1353 if (ArgIt ==
FuncInfo.ValueMap.end())
1355 Register ArgVReg = ArgIt->getSecond();
1358 for (
auto [PhysReg, VirtReg] :
FuncInfo.RegInfo->liveins())
1359 if (VirtReg == ArgVReg) {
1360 FuncInfo.MF->setVariableDbgInfo(Var, Expr, PhysReg, DbgLoc);
1361 LLVM_DEBUG(
dbgs() <<
"processDbgDeclare: setVariableDbgInfo Var=" << *Var
1362 <<
", Expr=" << *Expr <<
", MCRegister=" << PhysReg
1363 <<
", DbgLoc=" << DbgLoc <<
"\n");
1374 <<
" (bad address)\n");
1384 assert(Var &&
"Missing variable");
1385 assert(DbgLoc &&
"Missing location");
1395 int FI = std::numeric_limits<int>::max();
1396 if (
const auto *AI = dyn_cast<AllocaInst>(
Address)) {
1400 }
else if (
const auto *
Arg = dyn_cast<Argument>(
Address))
1403 if (FI == std::numeric_limits<int>::max())
1406 if (
Offset.getBoolValue())
1410 LLVM_DEBUG(
dbgs() <<
"processDbgDeclare: setVariableDbgInfo Var=" << *Var
1411 <<
", Expr=" << *Expr <<
", FI=" << FI
1412 <<
", DbgLoc=" << DbgLoc <<
"\n");
1421 const auto *DI = dyn_cast<DbgDeclareInst>(&
I);
1423 DI->getVariable(), DI->getDebugLoc()))
1424 FuncInfo.PreprocessedDbgDeclares.insert(DI);
1436 assert(!It->Values.hasArgList() &&
"Single loc variadic ops not supported");
1442void SelectionDAGISel::SelectAllBasicBlocks(
const Function &Fn) {
1472 ++NumFastIselFailLowerArguments;
1477 R <<
"FastISel didn't lower all arguments: "
1485 CodeGenAndEmitDAG();
1499 if (FastIS && Inserted)
1504 "expected AssignmentTrackingAnalysis pass results");
1514 bool AllPredsVisited =
true;
1516 if (!
FuncInfo->VisitedBBs.count(Pred)) {
1517 AllPredsVisited =
false;
1522 if (AllPredsVisited) {
1524 FuncInfo->ComputePHILiveOutRegInfo(&PN);
1527 FuncInfo->InvalidatePHILiveOutRegInfo(&PN);
1530 FuncInfo->VisitedBBs.insert(LLVMBB);
1546 FuncInfo->ExceptionPointerVirtReg = 0;
1547 FuncInfo->ExceptionSelectorVirtReg = 0;
1549 if (!PrepareEHLandingPad())
1557 unsigned NumFastIselRemaining = std::distance(Begin,
End);
1563 for (; BI != Begin; --BI) {
1569 --NumFastIselRemaining;
1579 --NumFastIselRemaining;
1580 ++NumFastIselSuccess;
1585 while (BeforeInst != &*Begin) {
1590 if (BeforeInst != Inst && isa<LoadInst>(BeforeInst) &&
1595 <<
"FastISel folded load: " << *BeforeInst <<
"\n");
1597 --NumFastIselRemaining;
1598 ++NumFastIselSuccess;
1610 if (isa<CallInst>(Inst) && !isa<GCStatepointInst>(Inst) &&
1611 !isa<GCRelocateInst>(Inst) && !isa<GCResultInst>(Inst)) {
1615 R <<
"FastISel missed call";
1618 std::string InstStrStorage;
1622 R <<
": " << InstStr.str();
1627 if (!Inst->getType()->isVoidTy() && !Inst->getType()->isTokenTy() &&
1628 !Inst->use_empty()) {
1634 bool HadTailCall =
false;
1636 SelectBasicBlock(Inst->getIterator(), BI, HadTailCall);
1648 unsigned RemainingNow = std::distance(Begin, BI);
1649 NumFastIselFailures += NumFastIselRemaining - RemainingNow;
1650 NumFastIselRemaining = RemainingNow;
1655 Inst->getDebugLoc(), LLVMBB);
1658 if (Inst->isTerminator()) {
1660 R <<
"FastISel missed terminator";
1664 R <<
"FastISel missed";
1668 std::string InstStrStorage;
1671 R <<
": " << InstStr.str();
1676 NumFastIselFailures += NumFastIselRemaining;
1684 bool FunctionBasedInstrumentation =
1686 SDB->SPDescriptor.initialize(LLVMBB,
FuncInfo->MBBMap[LLVMBB],
1687 FunctionBasedInstrumentation);
1693 ++NumFastIselBlocks;
1700 SelectBasicBlock(Begin, BI, HadTailCall);
1712 FuncInfo->PHINodesToUpdate.clear();
1718 reportIPToStateForBlocks(
MF);
1725 SDB->clearDanglingDebugInfo();
1726 SDB->SPDescriptor.resetPerFunctionState();
1730SelectionDAGISel::FinishBasicBlock() {
1732 <<
FuncInfo->PHINodesToUpdate.size() <<
"\n";
1733 for (
unsigned i = 0, e =
FuncInfo->PHINodesToUpdate.size(); i != e;
1735 <<
"Node " << i <<
" : (" <<
FuncInfo->PHINodesToUpdate[i].first
1736 <<
", " <<
FuncInfo->PHINodesToUpdate[i].second <<
")\n");
1740 for (
unsigned i = 0, e =
FuncInfo->PHINodesToUpdate.size(); i != e; ++i) {
1743 "This is not a machine PHI node that we are updating!");
1744 if (!
FuncInfo->MBB->isSuccessor(
PHI->getParent()))
1750 if (
SDB->SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
1759 SDB->visitSPDescriptorParent(
SDB->SPDescriptor, ParentMBB);
1762 CodeGenAndEmitDAG();
1765 SDB->SPDescriptor.resetPerBBState();
1766 }
else if (
SDB->SPDescriptor.shouldEmitStackProtector()) {
1780 SuccessMBB->
splice(SuccessMBB->
end(), ParentMBB,
1787 SDB->visitSPDescriptorParent(
SDB->SPDescriptor, ParentMBB);
1790 CodeGenAndEmitDAG();
1794 if (FailureMBB->
empty()) {
1797 SDB->visitSPDescriptorFailure(
SDB->SPDescriptor);
1800 CodeGenAndEmitDAG();
1804 SDB->SPDescriptor.resetPerBBState();
1808 for (
auto &BTB :
SDB->SL->BitTestCases) {
1818 CodeGenAndEmitDAG();
1822 for (
unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
1823 UnhandledProb -= BTB.Cases[
j].ExtraProb;
1838 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
1841 NextMBB = BTB.Cases[
j + 1].TargetBB;
1842 }
else if (j + 1 == ej) {
1844 NextMBB = BTB.Default;
1847 NextMBB = BTB.Cases[
j + 1].ThisBB;
1850 SDB->visitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j],
1855 CodeGenAndEmitDAG();
1857 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
1859 BTB.Cases.pop_back();
1865 for (
const std::pair<MachineInstr *, unsigned> &
P :
1870 "This is not a machine PHI node that we are updating!");
1873 if (PHIBB == BTB.Default) {
1874 PHI.addReg(
P.second).addMBB(BTB.Parent);
1875 if (!BTB.ContiguousRange) {
1876 PHI.addReg(
P.second).addMBB(BTB.Cases.back().ThisBB);
1883 PHI.addReg(
P.second).addMBB(cBB);
1887 SDB->SL->BitTestCases.clear();
1892 for (
unsigned i = 0, e =
SDB->SL->JTCases.size(); i != e; ++i) {
1894 if (!
SDB->SL->JTCases[i].first.Emitted) {
1896 FuncInfo->MBB =
SDB->SL->JTCases[i].first.HeaderBB;
1899 SDB->visitJumpTableHeader(
SDB->SL->JTCases[i].second,
1903 CodeGenAndEmitDAG();
1910 SDB->visitJumpTable(
SDB->SL->JTCases[i].second);
1913 CodeGenAndEmitDAG();
1916 for (
unsigned pi = 0, pe =
FuncInfo->PHINodesToUpdate.size();
1921 "This is not a machine PHI node that we are updating!");
1923 if (PHIBB ==
SDB->SL->JTCases[i].second.Default)
1925 .addMBB(
SDB->SL->JTCases[i].first.HeaderBB);
1927 if (
FuncInfo->MBB->isSuccessor(PHIBB))
1931 SDB->SL->JTCases.clear();
1935 for (
unsigned i = 0, e =
SDB->SL->SwitchCases.size(); i != e; ++i) {
1943 if (
SDB->SL->SwitchCases[i].TrueBB !=
SDB->SL->SwitchCases[i].FalseBB)
1950 CodeGenAndEmitDAG();
1960 for (
unsigned i = 0, e = Succs.
size(); i != e; ++i) {
1971 for (
unsigned pn = 0; ; ++pn) {
1973 "Didn't find PHI entry!");
1974 if (
FuncInfo->PHINodesToUpdate[pn].first ==
PHI) {
1975 PHI.addReg(
FuncInfo->PHINodesToUpdate[pn].second).addMBB(ThisBB);
1983 SDB->SL->SwitchCases.clear();
2004 int64_t DesiredMaskS)
const {
2005 const APInt &ActualMask =
RHS->getAPIntValue();
2006 const APInt &DesiredMask =
APInt(
LHS.getValueSizeInBits(), DesiredMaskS);
2009 if (ActualMask == DesiredMask)
2018 APInt NeededMask = DesiredMask & ~ActualMask;
2033 int64_t DesiredMaskS)
const {
2034 const APInt &ActualMask =
RHS->getAPIntValue();
2035 const APInt &DesiredMask =
APInt(
LHS.getValueSizeInBits(), DesiredMaskS);
2038 if (ActualMask == DesiredMask)
2047 APInt NeededMask = DesiredMask & ~ActualMask;
2064 std::vector<SDValue> InOps;
2073 if (InOps[e-1].getValueType() == MVT::Glue)
2077 unsigned Flags = cast<ConstantSDNode>(InOps[i])->getZExtValue();
2080 Ops.insert(Ops.end(), InOps.begin()+i,
2085 "Memory operand with multiple values?");
2087 unsigned TiedToOperand;
2091 Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
2092 for (; TiedToOperand; --TiedToOperand) {
2094 Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
2099 std::vector<SDValue> SelOps;
2118 if (e != InOps.size())
2119 Ops.push_back(InOps.back());
2126 unsigned FlagResNo =
N->getNumValues()-1;
2129 if (
Use.getResNo() == FlagResNo)
2138 bool IgnoreChains) {
2147 Visited.
insert(ImmedUse);
2152 if ((Op.getValueType() == MVT::Other && IgnoreChains) ||
N == Def)
2154 if (!Visited.
insert(
N).second)
2160 if (Root != ImmedUse) {
2164 if ((Op.getValueType() == MVT::Other && IgnoreChains) ||
N == Def)
2166 if (!Visited.
insert(
N).second)
2180 return N.hasOneUse();
2187 bool IgnoreChains) {
2235 while (VT == MVT::Glue) {
2246 IgnoreChains =
false;
2252void SelectionDAGISel::Select_INLINEASM(
SDNode *
N) {
2255 std::vector<SDValue> Ops(
N->op_begin(),
N->op_end());
2258 const EVT VTs[] = {MVT::Other, MVT::Glue};
2265void SelectionDAGISel::Select_READ_REGISTER(
SDNode *Op) {
2270 EVT VT =
Op->getValueType(0);
2276 Op->getOperand(0), dl, Reg,
Op->getValueType(0));
2282void SelectionDAGISel::Select_WRITE_REGISTER(
SDNode *Op) {
2287 EVT VT =
Op->getOperand(2).getValueType();
2293 Op->getOperand(0), dl, Reg,
Op->getOperand(2));
2299void SelectionDAGISel::Select_UNDEF(
SDNode *
N) {
2303void SelectionDAGISel::Select_FREEZE(
SDNode *
N) {
2311void SelectionDAGISel::Select_ARITH_FENCE(
SDNode *
N) {
2316void SelectionDAGISel::Select_MEMBARRIER(
SDNode *
N) {
2340void SelectionDAGISel::Select_STACKMAP(
SDNode *
N) {
2342 auto *It =
N->op_begin();
2351 assert(
ID.getValueType() == MVT::i64);
2360 for (; It !=
N->op_end(); It++)
2361 pushStackMapLiveVariable(Ops, *It,
DL);
2370void SelectionDAGISel::Select_PATCHPOINT(
SDNode *
N) {
2372 auto *It =
N->op_begin();
2377 std::optional<SDValue> Glue;
2378 if (It->getValueType() == MVT::Glue)
2384 assert(
ID.getValueType() == MVT::i64);
2404 for (
uint64_t I = cast<ConstantSDNode>(NumArgs)->getZExtValue();
I != 0;
I--)
2408 for (; It !=
N->op_end(); It++)
2409 pushStackMapLiveVariable(Ops, *It,
DL);
2414 if (Glue.has_value())
2424 assert(Val >= 128 &&
"Not a VBR");
2430 NextBits = MatcherTable[
Idx++];
2431 Val |= (NextBits&127) << Shift;
2433 }
while (NextBits & 128);
2440void SelectionDAGISel::UpdateChains(
2447 if (!ChainNodesMatched.
empty()) {
2449 "Matched input chains but didn't produce a chain");
2452 for (
unsigned i = 0, e = ChainNodesMatched.
size(); i != e; ++i) {
2453 SDNode *ChainNode = ChainNodesMatched[i];
2460 "Deleted node left in chain");
2464 if (ChainNode == NodeToMatch && isMorphNodeTo)
2473 std::replace(ChainNodesMatched.
begin(), ChainNodesMatched.
end(),
N,
2474 static_cast<SDNode *
>(
nullptr));
2480 if (ChainNode != NodeToMatch && ChainNode->
use_empty() &&
2486 if (!NowDeadNodes.
empty())
2505 unsigned int Max = 8192;
2508 if (ChainNodesMatched.
size() == 1)
2509 return ChainNodesMatched[0]->getOperand(0);
2513 std::function<void(
const SDValue)> AddChains = [&](
const SDValue V) {
2514 if (V.getValueType() != MVT::Other)
2518 if (!Visited.
insert(V.getNode()).second)
2521 for (
const SDValue &Op : V->op_values())
2527 for (
auto *
N : ChainNodesMatched) {
2532 while (!Worklist.
empty())
2536 if (InputChains.
size() == 0)
2546 for (
auto *
N : ChainNodesMatched)
2551 if (InputChains.
size() == 1)
2552 return InputChains[0];
2554 MVT::Other, InputChains);
2558SDNode *SelectionDAGISel::
2567 int OldGlueResultNo = -1, OldChainResultNo = -1;
2569 unsigned NTMNumResults =
Node->getNumValues();
2570 if (
Node->getValueType(NTMNumResults-1) == MVT::Glue) {
2571 OldGlueResultNo = NTMNumResults-1;
2572 if (NTMNumResults != 1 &&
2573 Node->getValueType(NTMNumResults-2) == MVT::Other)
2574 OldChainResultNo = NTMNumResults-2;
2575 }
else if (
Node->getValueType(NTMNumResults-1) == MVT::Other)
2576 OldChainResultNo = NTMNumResults-1;
2594 (
unsigned)OldGlueResultNo != ResNumResults-1)
2596 SDValue(Res, ResNumResults - 1));
2602 if ((EmitNodeInfo &
OPFL_Chain) && OldChainResultNo != -1 &&
2603 (
unsigned)OldChainResultNo != ResNumResults-1)
2605 SDValue(Res, ResNumResults - 1));
2623 unsigned RecNo = MatcherTable[MatcherIndex++];
2624 assert(RecNo < RecordedNodes.size() &&
"Invalid CheckSame");
2625 return N == RecordedNodes[RecNo].first;
2630 const unsigned char *MatcherTable,
unsigned &MatcherIndex,
SDValue N,
2633 if (ChildNo >=
N.getNumOperands())
2635 return ::CheckSame(MatcherTable, MatcherIndex,
N.getOperand(ChildNo),
2656 uint16_t Opc = MatcherTable[MatcherIndex++];
2657 Opc |= (
unsigned short)MatcherTable[MatcherIndex++] << 8;
2658 return N->getOpcode() == Opc;
2665 if (
N.getValueType() == VT)
return true;
2675 if (ChildNo >=
N.getNumOperands())
2677 return ::CheckType(MatcherTable, MatcherIndex,
N.getOperand(ChildNo), TLI,
2684 return cast<CondCodeSDNode>(
N)->get() ==
2691 if (2 >=
N.getNumOperands())
2693 return ::CheckCondCode(MatcherTable, MatcherIndex,
N.getOperand(2));
2700 if (cast<VTSDNode>(
N)->getVT() == VT)
2704 return VT == MVT::iPTR && cast<VTSDNode>(
N)->getVT() == TLI->
getPointerTy(
DL);
2721 int64_t Val = MatcherTable[MatcherIndex++];
2723 Val =
GetVBR(Val, MatcherTable, MatcherIndex);
2728 return C &&
C->getSExtValue() == Val;
2734 if (ChildNo >=
N.getNumOperands())
2736 return ::CheckInteger(MatcherTable, MatcherIndex,
N.getOperand(ChildNo));
2742 int64_t Val = MatcherTable[MatcherIndex++];
2744 Val =
GetVBR(Val, MatcherTable, MatcherIndex);
2746 if (
N->getOpcode() !=
ISD::AND)
return false;
2755 int64_t Val = MatcherTable[MatcherIndex++];
2757 Val =
GetVBR(Val, MatcherTable, MatcherIndex);
2759 if (
N->getOpcode() !=
ISD::OR)
return false;
2776 switch (Table[
Index++]) {
2804 unsigned Res = Table[
Index++];
2861 unsigned NumRecordedNodes;
2864 unsigned NumMatchedMemRefs;
2867 SDValue InputChain, InputGlue;
2870 bool HasChainNodesMatched;
2887 :
SelectionDAG::DAGUpdateListener(DAG), NodeToMatch(NodeToMatch),
2888 RecordedNodes(
RN), MatchScopes(MS) {}
2896 if (!
E ||
E->isMachineOpcode())
2899 if (
N == *NodeToMatch)
2904 for (
auto &
I : RecordedNodes)
2905 if (
I.first.getNode() ==
N)
2908 for (
auto &
I : MatchScopes)
2909 for (
auto &J :
I.NodeStack)
2910 if (J.getNode() ==
N)
2918 const unsigned char *MatcherTable,
2919 unsigned TableSize) {
2958 Select_INLINEASM(NodeToMatch);
2961 Select_READ_REGISTER(NodeToMatch);
2964 Select_WRITE_REGISTER(NodeToMatch);
2967 Select_UNDEF(NodeToMatch);
2970 Select_FREEZE(NodeToMatch);
2973 Select_ARITH_FENCE(NodeToMatch);
2976 Select_MEMBARRIER(NodeToMatch);
2979 Select_STACKMAP(NodeToMatch);
2982 Select_PATCHPOINT(NodeToMatch);
3009 SDValue InputChain, InputGlue;
3023 unsigned MatcherIndex = 0;
3025 if (!OpcodeOffset.empty()) {
3027 if (
N.getOpcode() < OpcodeOffset.size())
3028 MatcherIndex = OpcodeOffset[
N.getOpcode()];
3029 LLVM_DEBUG(
dbgs() <<
" Initial Opcode index to " << MatcherIndex <<
"\n");
3038 unsigned CaseSize = MatcherTable[
Idx++];
3040 CaseSize =
GetVBR(CaseSize, MatcherTable,
Idx);
3041 if (CaseSize == 0)
break;
3045 Opc |= (
unsigned short)MatcherTable[
Idx++] << 8;
3046 if (Opc >= OpcodeOffset.size())
3047 OpcodeOffset.resize((Opc+1)*2);
3048 OpcodeOffset[Opc] =
Idx;
3053 if (
N.getOpcode() < OpcodeOffset.size())
3054 MatcherIndex = OpcodeOffset[
N.getOpcode()];
3058 assert(MatcherIndex < TableSize &&
"Invalid index");
3060 unsigned CurrentOpcodeIndex = MatcherIndex;
3073 unsigned NumToSkip = MatcherTable[MatcherIndex++];
3074 if (NumToSkip & 128)
3075 NumToSkip =
GetVBR(NumToSkip, MatcherTable, MatcherIndex);
3077 if (NumToSkip == 0) {
3082 FailIndex = MatcherIndex+NumToSkip;
3084 unsigned MatcherIndexOfPredicate = MatcherIndex;
3085 (void)MatcherIndexOfPredicate;
3092 Result, *
this, RecordedNodes);
3097 dbgs() <<
" Skipped scope entry (due to false predicate) at "
3098 <<
"index " << MatcherIndexOfPredicate <<
", continuing at "
3099 << FailIndex <<
"\n");
3100 ++NumDAGIselRetries;
3104 MatcherIndex = FailIndex;
3108 if (FailIndex == 0)
break;
3112 MatchScope NewEntry;
3113 NewEntry.FailIndex = FailIndex;
3114 NewEntry.NodeStack.append(NodeStack.
begin(), NodeStack.
end());
3115 NewEntry.NumRecordedNodes = RecordedNodes.
size();
3116 NewEntry.NumMatchedMemRefs = MatchedMemRefs.
size();
3117 NewEntry.InputChain = InputChain;
3118 NewEntry.InputGlue = InputGlue;
3119 NewEntry.HasChainNodesMatched = !ChainNodesMatched.
empty();
3125 SDNode *Parent =
nullptr;
3126 if (NodeStack.
size() > 1)
3127 Parent = NodeStack[NodeStack.
size()-2].getNode();
3128 RecordedNodes.
push_back(std::make_pair(
N, Parent));
3137 if (ChildNo >=
N.getNumOperands())
3140 RecordedNodes.
push_back(std::make_pair(
N->getOperand(ChildNo),
3145 if (
auto *MN = dyn_cast<MemSDNode>(
N))
3146 MatchedMemRefs.
push_back(MN->getMemOperand());
3156 if (
N->getNumOperands() != 0 &&
3157 N->getOperand(
N->getNumOperands()-1).getValueType() == MVT::Glue)
3158 InputGlue =
N->getOperand(
N->getNumOperands()-1);
3162 unsigned ChildNo = MatcherTable[MatcherIndex++];
3163 if (ChildNo >=
N.getNumOperands())
3165 N =
N.getOperand(ChildNo);
3175 if (ChildNo >=
N.getNumOperands())
3177 N =
N.getOperand(ChildNo);
3185 assert(!NodeStack.
empty() &&
"Node stack imbalance!");
3186 N = NodeStack.
back();
3190 if (!
::CheckSame(MatcherTable, MatcherIndex,
N, RecordedNodes))
break;
3209 unsigned OpNum = MatcherTable[MatcherIndex++];
3212 for (
unsigned i = 0; i < OpNum; ++i)
3213 Operands.push_back(RecordedNodes[MatcherTable[MatcherIndex++]].first);
3215 unsigned PredNo = MatcherTable[MatcherIndex++];
3221 unsigned CPNum = MatcherTable[MatcherIndex++];
3222 unsigned RecNo = MatcherTable[MatcherIndex++];
3223 assert(RecNo < RecordedNodes.
size() &&
"Invalid CheckComplexPat");
3227 std::unique_ptr<MatchStateUpdater> MSU;
3229 MSU.reset(
new MatchStateUpdater(*
CurDAG, &NodeToMatch, RecordedNodes,
3233 RecordedNodes[RecNo].first, CPNum,
3239 if (!
::CheckOpcode(MatcherTable, MatcherIndex,
N.getNode()))
break;
3249 unsigned Res = MatcherTable[MatcherIndex++];
3257 unsigned CurNodeOpcode =
N.getOpcode();
3258 unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3262 CaseSize = MatcherTable[MatcherIndex++];
3264 CaseSize =
GetVBR(CaseSize, MatcherTable, MatcherIndex);
3265 if (CaseSize == 0)
break;
3267 uint16_t Opc = MatcherTable[MatcherIndex++];
3268 Opc |= (
unsigned short)MatcherTable[MatcherIndex++] << 8;
3271 if (CurNodeOpcode == Opc)
3275 MatcherIndex += CaseSize;
3279 if (CaseSize == 0)
break;
3282 LLVM_DEBUG(
dbgs() <<
" OpcodeSwitch from " << SwitchStart <<
" to "
3283 << MatcherIndex <<
"\n");
3288 MVT CurNodeVT =
N.getSimpleValueType();
3289 unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3293 CaseSize = MatcherTable[MatcherIndex++];
3295 CaseSize =
GetVBR(CaseSize, MatcherTable, MatcherIndex);
3296 if (CaseSize == 0)
break;
3299 if (CaseVT == MVT::iPTR)
3303 if (CurNodeVT == CaseVT)
3307 MatcherIndex += CaseSize;
3311 if (CaseSize == 0)
break;
3315 <<
"] from " << SwitchStart <<
" to " << MatcherIndex
3352 if (!
::CheckOrImm(MatcherTable, MatcherIndex,
N, *
this))
break;
3364 assert(NodeStack.
size() != 1 &&
"No parent node");
3367 bool HasMultipleUses =
false;
3368 for (
unsigned i = 1, e = NodeStack.
size()-1; i != e; ++i) {
3369 unsigned NNonChainUses = 0;
3370 SDNode *NS = NodeStack[i].getNode();
3372 if (UI.getUse().getValueType() != MVT::Other)
3373 if (++NNonChainUses > 1) {
3374 HasMultipleUses =
true;
3377 if (HasMultipleUses)
break;
3379 if (HasMultipleUses)
break;
3396 int64_t Val = MatcherTable[MatcherIndex++];
3398 Val =
GetVBR(Val, MatcherTable, MatcherIndex);
3401 RecordedNodes.
push_back(std::pair<SDValue, SDNode*>(
3409 unsigned RegNo = MatcherTable[MatcherIndex++];
3410 RecordedNodes.
push_back(std::pair<SDValue, SDNode*>(
3420 unsigned RegNo = MatcherTable[MatcherIndex++];
3421 RegNo |= MatcherTable[MatcherIndex++] << 8;
3422 RecordedNodes.
push_back(std::pair<SDValue, SDNode*>(
3429 unsigned RecNo = MatcherTable[MatcherIndex++];
3430 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitConvertToTarget");
3431 SDValue Imm = RecordedNodes[RecNo].first;
3434 const ConstantInt *Val=cast<ConstantSDNode>(Imm)->getConstantIntValue();
3436 Imm.getValueType());
3438 const ConstantFP *Val=cast<ConstantFPSDNode>(Imm)->getConstantFPValue();
3440 Imm.getValueType());
3443 RecordedNodes.
push_back(std::make_pair(Imm, RecordedNodes[RecNo].second));
3452 "EmitMergeInputChains should be the first chain producing node");
3454 "Should only have one EmitMergeInputChains per match");
3458 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitMergeInputChains");
3459 ChainNodesMatched.
push_back(RecordedNodes[RecNo].first.getNode());
3465 if (ChainNodesMatched.
back() != NodeToMatch &&
3466 !RecordedNodes[RecNo].first.hasOneUse()) {
3467 ChainNodesMatched.
clear();
3481 "EmitMergeInputChains should be the first chain producing node");
3488 unsigned NumChains = MatcherTable[MatcherIndex++];
3489 assert(NumChains != 0 &&
"Can't TF zero chains");
3492 "Should only have one EmitMergeInputChains per match");
3495 for (
unsigned i = 0; i != NumChains; ++i) {
3496 unsigned RecNo = MatcherTable[MatcherIndex++];
3497 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitMergeInputChains");
3498 ChainNodesMatched.
push_back(RecordedNodes[RecNo].first.getNode());
3504 if (ChainNodesMatched.
back() != NodeToMatch &&
3505 !RecordedNodes[RecNo].first.hasOneUse()) {
3506 ChainNodesMatched.
clear();
3512 if (ChainNodesMatched.
empty())
3526 unsigned RecNo = MatcherTable[MatcherIndex++];
3527 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitCopyToReg");
3528 unsigned DestPhysReg = MatcherTable[MatcherIndex++];
3530 DestPhysReg |= MatcherTable[MatcherIndex++] << 8;
3536 DestPhysReg, RecordedNodes[RecNo].first,
3539 InputGlue = InputChain.
getValue(1);
3544 unsigned XFormNo = MatcherTable[MatcherIndex++];
3545 unsigned RecNo = MatcherTable[MatcherIndex++];
3546 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitNodeXForm");
3548 RecordedNodes.
push_back(std::pair<SDValue,SDNode*>(Res,
nullptr));
3554 unsigned index = MatcherTable[MatcherIndex++];
3555 index |= (MatcherTable[MatcherIndex++] << 8);
3564 uint16_t TargetOpc = MatcherTable[MatcherIndex++];
3565 TargetOpc |= (
unsigned short)MatcherTable[MatcherIndex++] << 8;
3566 unsigned EmitNodeInfo = MatcherTable[MatcherIndex++];
3576 NumVTs = MatcherTable[MatcherIndex++];
3578 for (
unsigned i = 0; i != NumVTs; ++i) {
3581 if (VT == MVT::iPTR)
3594 if (VTs.
size() == 1)
3596 else if (VTs.
size() == 2)
3602 unsigned NumOps = MatcherTable[MatcherIndex++];
3604 for (
unsigned i = 0; i != NumOps; ++i) {
3605 unsigned RecNo = MatcherTable[MatcherIndex++];
3607 RecNo =
GetVBR(RecNo, MatcherTable, MatcherIndex);
3609 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitNode");
3610 Ops.
push_back(RecordedNodes[RecNo].first);
3617 FirstOpToCopy += (EmitNodeInfo &
OPFL_Chain) ? 1 : 0;
3619 "Invalid variadic node");
3622 for (
unsigned i = FirstOpToCopy, e = NodeToMatch->
getNumOperands();
3625 if (V.getValueType() == MVT::Glue)
break;
3640 bool MayRaiseFPException =
3649 if (!IsMorphNodeTo) {
3656 for (
unsigned i = 0, e = VTs.
size(); i != e; ++i) {
3657 if (VTs[i] == MVT::Other || VTs[i] == MVT::Glue)
break;
3663 "NodeToMatch was removed partway through selection");
3667 auto &Chain = ChainNodesMatched;
3669 "Chain node replaced during MorphNode");
3672 Res = cast<MachineSDNode>(MorphNode(NodeToMatch, TargetOpc, VTList,
3673 Ops, EmitNodeInfo));
3680 Flags.setNoFPExcept(
true);
3681 Res->setFlags(
Flags);
3703 bool mayLoad = MCID.
mayLoad();
3710 if (MMO->isLoad()) {
3713 }
else if (MMO->isStore()) {
3725 <<
" Dropping mem operands\n";
3726 dbgs() <<
" " << (IsMorphNodeTo ?
"Morphed" :
"Created")
3731 if (IsMorphNodeTo) {
3733 UpdateChains(Res, InputChain, ChainNodesMatched,
true);
3743 unsigned NumResults = MatcherTable[MatcherIndex++];
3745 for (
unsigned i = 0; i != NumResults; ++i) {
3746 unsigned ResSlot = MatcherTable[MatcherIndex++];
3748 ResSlot =
GetVBR(ResSlot, MatcherTable, MatcherIndex);
3750 assert(ResSlot < RecordedNodes.
size() &&
"Invalid CompleteMatch");
3751 SDValue Res = RecordedNodes[ResSlot].first;
3753 assert(i < NodeToMatch->getNumValues() &&
3756 "Invalid number of results to complete!");
3762 "invalid replacement");
3767 UpdateChains(NodeToMatch, InputChain, ChainNodesMatched,
false);
3780 "Didn't replace all uses of the node?");
3790 LLVM_DEBUG(
dbgs() <<
" Match failed at index " << CurrentOpcodeIndex
3792 ++NumDAGIselRetries;
3794 if (MatchScopes.
empty()) {
3795 CannotYetSelect(NodeToMatch);
3801 MatchScope &LastScope = MatchScopes.
back();
3802 RecordedNodes.
resize(LastScope.NumRecordedNodes);
3804 NodeStack.
append(LastScope.NodeStack.begin(), LastScope.NodeStack.end());
3805 N = NodeStack.
back();
3807 if (LastScope.NumMatchedMemRefs != MatchedMemRefs.
size())
3808 MatchedMemRefs.
resize(LastScope.NumMatchedMemRefs);
3809 MatcherIndex = LastScope.FailIndex;
3813 InputChain = LastScope.InputChain;
3814 InputGlue = LastScope.InputGlue;
3815 if (!LastScope.HasChainNodesMatched)
3816 ChainNodesMatched.
clear();
3821 unsigned NumToSkip = MatcherTable[MatcherIndex++];
3822 if (NumToSkip & 128)
3823 NumToSkip =
GetVBR(NumToSkip, MatcherTable, MatcherIndex);
3827 if (NumToSkip != 0) {
3828 LastScope.FailIndex = MatcherIndex+NumToSkip;
3842 if (
N->isMachineOpcode()) {
3849 if (
N->isTargetOpcode())
3850 return N->isTargetStrictFPOpcode();
3851 return N->isStrictFPOpcode();
3856 auto *
C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
3861 if (
auto *FN = dyn_cast<FrameIndexSDNode>(
N->getOperand(0))) {
3864 int32_t Off =
C->getSExtValue();
3867 return (Off >= 0) && (((
A.value() - 1) & Off) ==
unsigned(Off));
3872void SelectionDAGISel::CannotYetSelect(
SDNode *
N) {
3875 Msg <<
"Cannot select: ";
3881 Msg <<
"\nIn function: " <<
MF->
getName();
3883 bool HasInputChain =
N->getOperand(0).getValueType() == MVT::Other;
3885 cast<ConstantSDNode>(
N->getOperand(HasInputChain))->getZExtValue();
3886 if (iid < Intrinsic::num_intrinsics)
3889 Msg <<
"target intrinsic %" <<
TII->
getName(iid);
3891 Msg <<
"unknown intrinsic #" << iid;
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
amdgpu AMDGPU Register Bank Select
This file implements a class to represent arbitrary precision integral constant values and operations...
BlockVerifier::State From
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_ATTRIBUTE_ALWAYS_INLINE
LLVM_ATTRIBUTE_ALWAYS_INLINE - On compilers where we have a directive to do so, mark a method "always...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
This file defines the FastISel class.
mir Rename Register Operands
Machine Instruction Scheduler
unsigned const TargetRegisterInfo * TRI
typename CallsiteContextGraph< DerivedCCG, FuncTy, CallTy >::FuncInfo FuncInfo
print must be executed print the must be executed context for all instructions
const char LLVMTargetMachineRef TM
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
static cl::opt< bool > ViewSUnitDAGs("view-sunit-dags", cl::Hidden, cl::desc("Pop up a window to show SUnit dags after they are processed"))
static cl::opt< bool > ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the post " "legalize types dag combine pass"))
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes, unsigned ChildNo)
CheckChildSame - Implements OP_CheckChildXSame.
static uint64_t decodeSignRotatedValue(uint64_t V)
Decode a signed value stored with the sign bit in the LSB for dense VBR encoding.
static cl::opt< bool > ViewISelDAGs("view-isel-dags", cl::Hidden, cl::desc("Pop up a window to show isel dags as they are selected"))
static LLVM_ATTRIBUTE_ALWAYS_INLINE uint64_t GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx)
GetVBR - decode a vbr encoding whose top bit is set.
static void computeUsesMSVCFloatingPoint(const Triple &TT, const Function &F, MachineModuleInfo &MMI)
static void reportFastISelFailure(MachineFunction &MF, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R, bool ShouldAbort)
static cl::opt< bool > ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the second " "dag combine pass"))
static RegisterScheduler defaultListDAGScheduler("default", "Best scheduler for the target", createDefaultScheduler)
static unsigned IsPredicateKnownToFail(const unsigned char *Table, unsigned Index, SDValue N, bool &Result, const SelectionDAGISel &SDISel, SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes)
IsPredicateKnownToFail - If we know how and can do so without pushing a scope, evaluate the current n...
static cl::opt< int > EnableFastISelAbort("fast-isel-abort", cl::Hidden, cl::desc("Enable abort calls when \"fast\" instruction selection " "fails to lower an instruction: 0 disable the abort, 1 will " "abort but for args, calls and terminators, 2 will also " "abort for argument lowering, and 3 will never fallback " "to SelectionDAG."))
static void mapWasmLandingPadIndex(MachineBasicBlock *MBB, const CatchPadInst *CPI)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel, SDNode *N)
CheckNodePredicate - Implements OP_CheckNodePredicate.
static void processSingleLocVars(FunctionLoweringInfo &FuncInfo, FunctionVarLocs const *FnVarLocs)
Collect single location variable information generated with assignment tracking.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
static cl::opt< bool > UseMBPI("use-mbpi", cl::desc("use Machine Branch Probability Info"), cl::init(true), cl::Hidden)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes)
CheckSame - Implements OP_CheckSame.
static bool findNonImmUse(SDNode *Root, SDNode *Def, SDNode *ImmedUse, bool IgnoreChains)
findNonImmUse - Return true if "Def" is a predecessor of "Root" via a path beyond "ImmedUse".
static cl::opt< bool > ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the first " "dag combine pass"))
static bool processIfEntryValueDbgDeclare(FunctionLoweringInfo &FuncInfo, const Value *Arg, DIExpression *Expr, DILocalVariable *Var, DebugLoc DbgLoc)
static cl::opt< bool > ViewSchedDAGs("view-sched-dags", cl::Hidden, cl::desc("Pop up a window to show sched dags as they are processed"))
static void processDbgDeclares(FunctionLoweringInfo &FuncInfo)
Collect llvm.dbg.declare information.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL, unsigned ChildNo)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
static bool hasExceptionPointerOrCodeUser(const CatchPadInst *CPI)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChild2CondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel)
CheckPatternPredicate - Implements OP_CheckPatternPredicate.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
static cl::opt< bool > ViewLegalizeDAGs("view-legalize-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize"))
static cl::opt< bool > ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize types"))
static SDNode * findGlueUse(SDNode *N)
findGlueUse - Return use of MVT::Glue value produced by the specified SDNode.
static cl::opt< RegisterScheduler::FunctionPassCtor, false, RegisterPassParser< RegisterScheduler > > ISHeuristic("pre-RA-sched", cl::init(&createDefaultScheduler), cl::Hidden, cl::desc("Instruction schedulers available (before register" " allocation):"))
ISHeuristic command line option for instruction schedulers.
static cl::opt< bool > EnableFastISelFallbackReport("fast-isel-report-on-fallback", cl::Hidden, cl::desc("Emit a diagnostic when \"fast\" instruction selection " "falls back to SelectionDAG."))
static bool processDbgDeclare(FunctionLoweringInfo &FuncInfo, const Value *Address, DIExpression *Expr, DILocalVariable *Var, DebugLoc DbgLoc)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDNode *N)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, unsigned ChildNo)
static cl::opt< std::string > FilterDAGBasicBlockName("filter-view-dags", cl::Hidden, cl::desc("Only display the basic block whose name " "matches this for all view-*-dags options"))
static SDValue HandleMergeInputChains(SmallVectorImpl< SDNode * > &ChainNodesMatched, SelectionDAG *CurDAG)
HandleMergeInputChains - This implements the OPC_EmitMergeInputChains operation for when the pattern ...
static bool isFoldedOrDeadInstruction(const Instruction *I, const FunctionLoweringInfo &FuncInfo)
isFoldedOrDeadInstruction - Return true if the specified instruction is side-effect free and is eithe...
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This file describes how to lower LLVM code to machine code.
DEMANGLE_DUMP_METHOD void dump() const
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
Class for arbitrary precision integers.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
An immutable pass that tracks lazily created AssumptionCache objects.
LLVM Basic Block Representation.
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
InstListType::const_iterator const_iterator
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
bool isEHPad() const
Return true if this basic block is an exception handling block.
const Instruction * getFirstMayFaultInst() const
Returns the first potential AsynchEH faulty instruction currently it checks for loads/stores (which m...
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Legacy analysis pass which computes BranchProbabilityInfo.
ConstantFP - Floating Point Values [float, double].
This is the shared class of boolean and integer constants.
This is an important base class in LLVM.
bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
A parsed version of the target data layout string in and methods for querying it.
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Diagnostic information for ISel fallback path.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
void setLastLocalValue(MachineInstr *I)
Update the position of the last instruction emitted for materializing constants for use in the curren...
bool tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst)
We're checking to see if we can fold LI into FoldInst.
void removeDeadCode(MachineBasicBlock::iterator I, MachineBasicBlock::iterator E)
Remove all dead instructions between the I and E.
void startNewBlock()
Set the current block to which generated machine instructions will be appended.
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
void finishBasicBlock()
Flush the local value map.
void recomputeInsertPt()
Reset InsertPt to prepare for inserting instructions into the current block.
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
unsigned arg_size() const
arg_size - Return the number of funcletpad arguments.
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th funcletpad argument.
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Data structure describing the variable locations in a function.
const VarLocInfo * single_locs_begin() const
DILocalVariable * getDILocalVariable(const VarLocInfo *Loc) const
Return the DILocalVariable for the location definition represented by ID.
const VarLocInfo * single_locs_end() const
One past the last single-location variable location definition.
const BasicBlock & getEntryBlock() const
FunctionType * getFunctionType() const
Returns the FunctionType for me.
DISubprogram * getSubprogram() const
Get the attached subprogram.
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool callsFunctionThatReturnsTwice() const
callsFunctionThatReturnsTwice - Return true if the function has a call to setjmp or other function th...
An analysis pass which caches information about the entire Module.
Module * getParent()
Get the module that this global value is contained inside of...
This class is used to form a handle around another node that is persistent and is updated across invo...
static unsigned getFlagWord(unsigned Kind, unsigned NumOps)
static bool isMemKind(unsigned Flag)
static unsigned getNumOperandRegisters(unsigned Flag)
getNumOperandRegisters - Extract the number of registers field from the inline asm operand flag.
static unsigned getFlagWordForMem(unsigned InputFlag, unsigned Constraint)
Augment an existing flag word returned by getFlagWord with the constraint code for a memory constrain...
static bool isFuncKind(unsigned Flag)
static unsigned getMemoryConstraintID(unsigned Flag)
static bool isUseOperandTiedToDef(unsigned Flag, unsigned &Idx)
isUseOperandTiedToDef - Return true if the flag of the inline asm operand indicates it is an use oper...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
A wrapper class for inspecting calls to intrinsic functions.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
static void getLazyBFIAnalysisUsage(AnalysisUsage &AU)
Helper for client passes to set up the analysis usage on behalf of this pass.
MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
Describe properties that are true of each instruction in the target description file.
bool mayStore() const
Return true if this instruction could possibly modify memory.
bool mayLoad() const
Return true if this instruction could possibly read memory.
bool mayRaiseFPException() const
Return true if this instruction may raise a floating-point exception.
bool isCall() const
Return true if the instruction is a call.
bool isReturn() const
Return true if the instruction is a return.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
StringRef getName(unsigned Opcode) const
Returns the name for the instructions with the given opcode.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
const MDNode * getMD() const
const MDOperand & getOperand(unsigned I) const
StringRef getString() const
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
instr_iterator instr_end()
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasCalls() const
Return true if the current function has any function calls.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
bool hasProperty(Property P) const
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
bool useDebugInstrRef() const
Returns true if the function's variable locations are tracked with instruction referencing.
void setExposesReturnsTwice(bool B)
setCallsSetJmp - Set a flag that indicates if there's a call to a "returns twice" function.
void setHasInlineAsm(bool B)
Set a flag that indicates that the function contains inline assembly.
void setWasmLandingPadIndex(const MachineBasicBlock *LPad, unsigned Index)
Map the landing pad to its index. Used for Wasm exception handling.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
bool hasInlineAsm() const
Returns true if the function contains any inline assembly.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
void finalizeDebugInstrRefs()
Finalise any partially emitted debug instructions.
void setCallSiteLandingPad(MCSymbol *Sym, ArrayRef< unsigned > Sites)
Map the landing pad's EH symbol to the call site indexes.
void setUseDebugInstrRef(bool UseInstrRef)
Set whether this function will use instruction referencing or not.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
MCSymbol * addLandingPad(MachineBasicBlock *LandingPad)
Add a new panding pad, and extract the exception handling information from the landingpad instruction...
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineModuleInfo & getMMI() const
bool shouldUseDebugInstrRef() const
Determine whether, in the current machine configuration, we should use instruction referencing or not...
const MachineFunctionProperties & getProperties() const
Get the function properties.
void setVariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr, int Slot, const DILocation *Loc)
Collect information used to emit debugging information of a variable in a stack slot.
const MachineBasicBlock & front() const
void print(raw_ostream &OS, const SlotIndexes *=nullptr) const
print - Print out the MachineFunction in a format suitable for debugging to the specified stream.
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
bool isTerminator(QueryType Type=AnyInBundle) const
Returns true if this instruction part of the terminator for a basic block.
const MachineBasicBlock * getParent() const
bool isDebugValue() const
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
This class contains meta information specific to a module.
const MCContext & getContext() const
bool usesMSVCFloatingPoint() const
void setUsesMSVCFloatingPoint(bool b)
Register getReg() const
getReg - Returns the register number.
MachinePassRegistry - Track the registration of machine passes.
defusechain_iterator - This class provides iterator support for machine operands in the function that...
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
void EmitLiveInCopies(MachineBasicBlock *EntryMBB, const TargetRegisterInfo &TRI, const TargetInstrInfo &TII)
EmitLiveInCopies - Emit copies to initialize livein virtual registers into the given entry block.
use_instr_iterator use_instr_begin(Register RegNo) const
ArrayRef< std::pair< MCRegister, Register > > liveins() const
static use_instr_iterator use_instr_end()
void addPhysRegsUsedFromRegMask(const uint32_t *RegMask)
addPhysRegsUsedFromRegMask - Mark any registers not in RegMask as used.
An SDNode that represents everything that will be needed to construct a MachineInstr.
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
This class is used by SelectionDAGISel to temporarily override the optimization level on a per-functi...
OptLevelChanger(SelectionDAGISel &ISel, CodeGenOpt::Level NewOptLevel)
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
RegisterPassParser class - Handle the addition of new machine passes.
ScheduleDAGSDNodes *(*)(SelectionDAGISel *, CodeGenOpt::Level) FunctionPassCtor
static MachinePassRegistry< FunctionPassCtor > Registry
RegisterScheduler class - Track the registration of instruction schedulers.
Wrapper class representing virtual and physical registers.
static unsigned virtReg2Index(Register Reg)
Convert a virtual register number to a 0-based index.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< value_op_iterator > op_values() const
void setNodeId(int Id)
Set unique node id.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
ScheduleDAGSDNodes - A ScheduleDAG for scheduling SDNode-based DAGs.
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
bool CheckOrMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const
CheckOrMask - The isel is trying to match something like (or X, 255).
CodeGenOpt::Level OptLevel
const TargetLowering * TLI
virtual void PostprocessISelDAG()
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
virtual bool CheckNodePredicate(SDNode *N, unsigned PredNo) const
CheckNodePredicate - This function is generated by tblgen in the target.
std::unique_ptr< OptimizationRemarkEmitter > ORE
Current optimization remark emitter.
MachineRegisterInfo * RegInfo
unsigned DAGSize
DAGSize - Size of DAG being instruction selected.
@ OPC_CheckChild2CondCode