79#include "llvm/IR/IntrinsicsWebAssembly.h"
116#define DEBUG_TYPE "isel"
117#define ISEL_DUMP_DEBUG_TYPE DEBUG_TYPE "-dump"
119STATISTIC(NumFastIselFailures,
"Number of instructions fast isel failed on");
120STATISTIC(NumFastIselSuccess,
"Number of instructions fast isel selected");
121STATISTIC(NumFastIselBlocks,
"Number of blocks selected entirely by fast isel");
122STATISTIC(NumDAGBlocks,
"Number of blocks selected using DAG");
123STATISTIC(NumDAGIselRetries,
"Number of times dag isel has to try another path");
124STATISTIC(NumEntryBlocks,
"Number of entry blocks encountered");
126 "Number of entry blocks where fast isel failed to lower arguments");
130 cl::desc(
"Enable abort calls when \"fast\" instruction selection "
131 "fails to lower an instruction: 0 disable the abort, 1 will "
132 "abort but for args, calls and terminators, 2 will also "
133 "abort for argument lowering, and 3 will never fallback "
134 "to SelectionDAG."));
138 cl::desc(
"Emit a diagnostic when \"fast\" instruction selection "
139 "falls back to SelectionDAG."));
143 cl::desc(
"use Machine Branch Probability Info"),
149 cl::desc(
"Only display the basic block whose name "
150 "matches this for all view-*-dags options"));
153 cl::desc(
"Pop up a window to show dags before the first "
154 "dag combine pass"));
157 cl::desc(
"Pop up a window to show dags before legalize types"));
160 cl::desc(
"Pop up a window to show dags before the post "
161 "legalize types dag combine pass"));
164 cl::desc(
"Pop up a window to show dags before legalize"));
167 cl::desc(
"Pop up a window to show dags before the second "
168 "dag combine pass"));
171 cl::desc(
"Pop up a window to show isel dags as they are selected"));
174 cl::desc(
"Pop up a window to show sched dags as they are processed"));
177 cl::desc(
"Pop up a window to show SUnit dags after they are processed"));
186#define ISEL_DUMP(X) \
188 if (llvm::DebugFlag && \
189 (isCurrentDebugType(DEBUG_TYPE) || \
190 (isCurrentDebugType(ISEL_DUMP_DEBUG_TYPE) && MatchFilterFuncName))) { \
195#define ISEL_DUMP(X) do { } while (false)
215 cl::desc(
"Instruction schedulers available (before register"
228 return Arg.hasAttribute(Attribute::AttrKind::SwiftAsync);
247 if (NewOptLevel != SavedOptLevel) {
250 LLVM_DEBUG(
dbgs() <<
"\nChanging optimization level for Function "
252 LLVM_DEBUG(
dbgs() <<
"\tBefore: -O" <<
static_cast<int>(SavedOptLevel)
253 <<
" ; After: -O" <<
static_cast<int>(NewOptLevel)
261 dbgs() <<
"\tFastISel is "
269 LLVM_DEBUG(
dbgs() <<
"\nRestoring optimization level for Function "
272 <<
" ; After: -O" <<
static_cast<int>(SavedOptLevel) <<
"\n");
288 if (
auto *SchedulerCtor = ST.getDAGScheduler(OptLevel)) {
289 return SchedulerCtor(IS, OptLevel);
293 (ST.enableMachineScheduler() && ST.enableMachineSchedDefaultSched()) ||
307 "Unknown sched type!");
326 dbgs() <<
"If a target marks an instruction with "
327 "'usesCustomInserter', it must implement "
328 "TargetLowering::EmitInstrWithCustomInserter!\n";
336 "If a target marks an instruction with 'hasPostISelHook', "
337 "it must implement TargetLowering::AdjustInstrPostInstrSelection!");
388 if (!TT.isWindowsMSVCEnvironment())
396 if (
I.getType()->isFPOrFPVectorTy()) {
400 for (
const auto &
Op :
I.operands()) {
401 if (
Op->getType()->isFPOrFPVectorTy()) {
416 "-fast-isel-abort > 0 requires -fast-isel");
448 LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(Fn);
449 GFI = Fn.
hasGC() ? &getAnalysis<GCModuleInfo>().getFunctionInfo(Fn) :
nullptr;
450 ORE = std::make_unique<OptimizationRemarkEmitter>(&Fn);
451 AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(mf.
getFunction());
452 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
455 BFI = &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI();
459 FnVarLocs = getAnalysis<AssignmentTrackingAnalysis>().getResults();
464 if (
auto *UAPass = getAnalysisIfAvailable<UniformityInfoWrapperPass>())
465 UA = &UAPass->getUniformityInfo();
476 FuncInfo->BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
481 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
502 if (isa<UnreachableInst>(Term) || isa<ReturnInst>(Term))
516 SelectAllBasicBlocks(Fn);
544 MRI.constrainRegClass(To,
MRI.getRegClass(
From));
550 if (!
MRI.use_empty(To))
570 if (Term !=
MBB.
end() && Term->isReturn()) {
579 if (!
FuncInfo->ArgDbgValues.empty())
585 for (
unsigned i = 0, e =
FuncInfo->ArgDbgValues.size(); i != e; ++i) {
587 assert(
MI->getOpcode() != TargetOpcode::DBG_VALUE_LIST &&
588 "Function parameters should not be described by DBG_VALUE_LIST.");
589 bool hasFI =
MI->getDebugOperand(0).isFI();
591 hasFI ?
TRI.getFrameRegister(*
MF) :
MI->getDebugOperand(0).getReg();
592 if (Reg.isPhysical())
599 Def->getParent()->insert(std::next(InsertPos),
MI);
611 if (LDI != LiveInMap.
end()) {
612 assert(!hasFI &&
"There's no handling of frame pointer updating here yet "
616 const MDNode *Variable =
MI->getDebugVariable();
617 const MDNode *Expr =
MI->getDebugExpression();
619 bool IsIndirect =
MI->isIndirectDebugValue();
621 assert(
MI->getDebugOffset().getImm() == 0 &&
622 "DBG_VALUE with nonzero offset");
623 assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(
DL) &&
624 "Expected inlined-at fields to agree");
625 assert(
MI->getOpcode() != TargetOpcode::DBG_VALUE_LIST &&
626 "Didn't expect to see a DBG_VALUE_LIST here");
629 IsIndirect, LDI->second, Variable, Expr);
641 CopyUseMI =
UseMI;
continue;
644 CopyUseMI =
nullptr;
break;
647 TRI.getRegSizeInBits(LDI->second,
MRI) ==
667 for (
const auto &
MBB : *
MF) {
671 for (
const auto &
MI :
MBB) {
674 MI.isStackAligningInlineAsm()) {
677 if (
MI.isInlineAsm()) {
693 ISEL_DUMP(
dbgs() <<
"*** MachineFunction at end of ISel ***\n");
705 if (!R.getLocation().isValid() || ShouldAbort)
706 R << (
" (in function: " + MF.
getName() +
")").str();
728 SDB->visitDbgInfo(*
I);
733 HadTailCall =
SDB->HasTailCall;
734 SDB->resolveOrClearDbgInfo();
741void SelectionDAGISel::ComputeLiveOutVRegInfo() {
755 if (
Op.getValueType() == MVT::Other &&
Added.insert(
Op.getNode()).second)
762 unsigned DestReg = cast<RegisterSDNode>(
N->getOperand(1))->getReg();
768 EVT SrcVT = Src.getValueType();
774 FuncInfo->AddLiveOutRegInfo(DestReg, NumSignBits, Known);
775 }
while (!Worklist.
empty());
778void SelectionDAGISel::CodeGenAndEmitDAG() {
780 StringRef GroupDescription =
"Instruction Selection and Scheduling";
781 std::string BlockName;
782 bool MatchFilterBB =
false; (void)MatchFilterBB;
785 getAnalysis<TargetTransformInfoWrapperPass>().getTTI(*
FuncInfo->Fn);
794 FuncInfo->MBB->getBasicBlock()->getName());
871 ISEL_DUMP(
dbgs() <<
"\nOptimized type-legalized selection DAG: "
905 ISEL_DUMP(
dbgs() <<
"\nVector/type-legalized selection DAG: "
925 ISEL_DUMP(
dbgs() <<
"\nOptimized vector-legalized selection DAG: "
976 ComputeLiveOutVRegInfo();
986 DoInstructionSelection();
1022 if (FirstMBB != LastMBB)
1023 SDB->UpdateSplitBlock(FirstMBB, LastMBB);
1045 :
SelectionDAG::DAGUpdateListener(DAG), ISelPosition(isp) {}
1058 void NodeInserted(
SDNode *
N)
override {
1059 SDNode *CurNode = &*ISelPosition;
1060 if (
MDNode *MD = DAG.getPCSections(CurNode))
1061 DAG.addPCSections(
N, MD);
1091 while (!Nodes.
empty()) {
1093 for (
auto *U :
N->uses()) {
1094 auto UId = U->getNodeId();
1107 int InvalidId = -(
N->getNodeId() + 1);
1108 N->setNodeId(InvalidId);
1113 int Id =
N->getNodeId();
1119void SelectionDAGISel::DoInstructionSelection() {
1122 <<
FuncInfo->MBB->getName() <<
"'\n");
1140 ISelUpdater ISU(*
CurDAG, ISelPosition);
1147 SDNode *Node = &*--ISelPosition;
1151 if (Node->use_empty())
1158 while (!Nodes.
empty()) {
1175 "Node has already selected predecessor node");
1192 switch (
Node->getOpcode()) {
1201 ActionVT =
Node->getOperand(1).getValueType();
1204 ActionVT =
Node->getValueType(0);
1212 LLVM_DEBUG(
dbgs() <<
"\nISEL: Starting selection on root node: ";
1228 if (
const IntrinsicInst *EHPtrCall = dyn_cast<IntrinsicInst>(U)) {
1230 if (IID == Intrinsic::eh_exceptionpointer ||
1231 IID == Intrinsic::eh_exceptioncode)
1246 bool IsSingleCatchAllClause =
1251 bool IsCatchLongjmp = CPI->
arg_size() == 0;
1252 if (!IsSingleCatchAllClause && !IsCatchLongjmp) {
1254 bool IntrFound =
false;
1256 if (
const auto *Call = dyn_cast<IntrinsicInst>(U)) {
1258 if (IID == Intrinsic::wasm_landingpad_index) {
1259 Value *IndexArg = Call->getArgOperand(1);
1260 int Index = cast<ConstantInt>(IndexArg)->getZExtValue();
1267 assert(IntrFound &&
"wasm.landingpad.index intrinsic not found!");
1274bool SelectionDAGISel::PrepareEHLandingPad() {
1286 if (
const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->
getFirstNonPHI())) {
1291 assert(EHPhysReg &&
"target lacks exception pointer register");
1293 unsigned VReg =
FuncInfo->getCatchPadExceptionPointerVReg(CPI, PtrRC);
1295 TII->
get(TargetOpcode::COPY), VReg)
1313 if (
auto *RegMask =
TRI.getCustomEHPadPreservedMask(*
MF))
1317 if (
const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->
getFirstNonPHI()))
1355 TII->
get(TargetOpcode::EH_LABEL))
1364 TII->
get(TargetOpcode::EH_LABEL))
1375 return !
I->mayWriteToMemory() &&
1376 !
I->isTerminator() &&
1377 !isa<DbgInfoIntrinsic>(
I) &&
1389 auto ArgIt = FuncInfo.
ValueMap.find(Arg);
1390 if (ArgIt == FuncInfo.
ValueMap.end())
1392 Register ArgVReg = ArgIt->getSecond();
1396 if (VirtReg == ArgVReg) {
1400 LLVM_DEBUG(
dbgs() <<
"processDbgDeclare: setVariableDbgInfo Var=" << *Var
1401 <<
", Expr=" << *Expr <<
", MCRegister=" << PhysReg
1402 <<
", DbgLoc=" << DbgLoc <<
"\n");
1413 <<
" (bad address)\n");
1423 assert(Var &&
"Missing variable");
1424 assert(DbgLoc &&
"Missing location");
1434 int FI = std::numeric_limits<int>::max();
1435 if (
const auto *AI = dyn_cast<AllocaInst>(
Address)) {
1439 }
else if (
const auto *Arg = dyn_cast<Argument>(
Address))
1442 if (FI == std::numeric_limits<int>::max())
1445 if (
Offset.getBoolValue())
1449 LLVM_DEBUG(
dbgs() <<
"processDbgDeclare: setVariableDbgInfo Var=" << *Var
1450 <<
", Expr=" << *Expr <<
", FI=" << FI
1451 <<
", DbgLoc=" << DbgLoc <<
"\n");
1460 const auto *DI = dyn_cast<DbgDeclareInst>(&
I);
1462 DI->getVariable(), DI->getDebugLoc()))
1467 DVR.getExpression(), DVR.getVariable(),
1482 assert(!It->Values.hasArgList() &&
"Single loc variadic ops not supported");
1488void SelectionDAGISel::SelectAllBasicBlocks(
const Function &Fn) {
1518 ++NumFastIselFailLowerArguments;
1523 R <<
"FastISel didn't lower all arguments: "
1531 CodeGenAndEmitDAG();
1545 if (FastIS && Inserted)
1550 "expected AssignmentTrackingAnalysis pass results");
1560 bool AllPredsVisited =
true;
1562 if (!
FuncInfo->VisitedBBs.count(Pred)) {
1563 AllPredsVisited =
false;
1568 if (AllPredsVisited) {
1570 FuncInfo->ComputePHILiveOutRegInfo(&PN);
1573 FuncInfo->InvalidatePHILiveOutRegInfo(&PN);
1576 FuncInfo->VisitedBBs.insert(LLVMBB);
1592 FuncInfo->ExceptionPointerVirtReg = 0;
1593 FuncInfo->ExceptionSelectorVirtReg = 0;
1595 if (!PrepareEHLandingPad())
1603 unsigned NumFastIselRemaining = std::distance(Begin,
End);
1609 for (; BI != Begin; --BI) {
1615 --NumFastIselRemaining;
1626 --NumFastIselRemaining;
1627 ++NumFastIselSuccess;
1634 while (BeforeInst != &*Begin) {
1639 if (BeforeInst != Inst && isa<LoadInst>(BeforeInst) &&
1644 <<
"FastISel folded load: " << *BeforeInst <<
"\n");
1647 --NumFastIselRemaining;
1648 ++NumFastIselSuccess;
1660 if (isa<CallInst>(Inst) && !isa<GCStatepointInst>(Inst) &&
1661 !isa<GCRelocateInst>(Inst) && !isa<GCResultInst>(Inst)) {
1665 R <<
"FastISel missed call";
1668 std::string InstStrStorage;
1672 R <<
": " << InstStr.str();
1677 if (!Inst->getType()->isVoidTy() && !Inst->getType()->isTokenTy() &&
1678 !Inst->use_empty()) {
1684 bool HadTailCall =
false;
1686 SelectBasicBlock(Inst->getIterator(), BI, HadTailCall);
1698 unsigned RemainingNow = std::distance(Begin, BI);
1699 NumFastIselFailures += NumFastIselRemaining - RemainingNow;
1700 NumFastIselRemaining = RemainingNow;
1705 Inst->getDebugLoc(), LLVMBB);
1708 if (Inst->isTerminator()) {
1710 R <<
"FastISel missed terminator";
1714 R <<
"FastISel missed";
1718 std::string InstStrStorage;
1721 R <<
": " << InstStr.str();
1726 NumFastIselFailures += NumFastIselRemaining;
1734 bool FunctionBasedInstrumentation =
1736 SDB->SPDescriptor.initialize(LLVMBB,
FuncInfo->MBBMap[LLVMBB],
1737 FunctionBasedInstrumentation);
1743 ++NumFastIselBlocks;
1750 SelectBasicBlock(Begin, BI, HadTailCall);
1762 FuncInfo->PHINodesToUpdate.clear();
1768 reportIPToStateForBlocks(
MF);
1775 SDB->clearDanglingDebugInfo();
1776 SDB->SPDescriptor.resetPerFunctionState();
1780SelectionDAGISel::FinishBasicBlock() {
1782 <<
FuncInfo->PHINodesToUpdate.size() <<
"\n";
1783 for (
unsigned i = 0, e =
FuncInfo->PHINodesToUpdate.size(); i != e;
1785 <<
"Node " << i <<
" : (" <<
FuncInfo->PHINodesToUpdate[i].first
1786 <<
", " <<
FuncInfo->PHINodesToUpdate[i].second <<
")\n");
1790 for (
unsigned i = 0, e =
FuncInfo->PHINodesToUpdate.size(); i != e; ++i) {
1793 "This is not a machine PHI node that we are updating!");
1794 if (!
FuncInfo->MBB->isSuccessor(
PHI->getParent()))
1800 if (
SDB->SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
1809 SDB->visitSPDescriptorParent(
SDB->SPDescriptor, ParentMBB);
1812 CodeGenAndEmitDAG();
1815 SDB->SPDescriptor.resetPerBBState();
1816 }
else if (
SDB->SPDescriptor.shouldEmitStackProtector()) {
1830 SuccessMBB->
splice(SuccessMBB->
end(), ParentMBB,
1837 SDB->visitSPDescriptorParent(
SDB->SPDescriptor, ParentMBB);
1840 CodeGenAndEmitDAG();
1844 if (FailureMBB->
empty()) {
1847 SDB->visitSPDescriptorFailure(
SDB->SPDescriptor);
1850 CodeGenAndEmitDAG();
1854 SDB->SPDescriptor.resetPerBBState();
1858 for (
auto &BTB :
SDB->SL->BitTestCases) {
1868 CodeGenAndEmitDAG();
1872 for (
unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
1873 UnhandledProb -= BTB.Cases[
j].ExtraProb;
1888 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
1891 NextMBB = BTB.Cases[
j + 1].TargetBB;
1892 }
else if (j + 1 == ej) {
1894 NextMBB = BTB.Default;
1897 NextMBB = BTB.Cases[
j + 1].ThisBB;
1900 SDB->visitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j],
1905 CodeGenAndEmitDAG();
1907 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
1909 BTB.Cases.pop_back();
1915 for (
const std::pair<MachineInstr *, unsigned> &
P :
1920 "This is not a machine PHI node that we are updating!");
1923 if (PHIBB == BTB.Default) {
1924 PHI.addReg(
P.second).addMBB(BTB.Parent);
1925 if (!BTB.ContiguousRange) {
1926 PHI.addReg(
P.second).addMBB(BTB.Cases.back().ThisBB);
1933 PHI.addReg(
P.second).addMBB(cBB);
1937 SDB->SL->BitTestCases.clear();
1942 for (
unsigned i = 0, e =
SDB->SL->JTCases.size(); i != e; ++i) {
1944 if (!
SDB->SL->JTCases[i].first.Emitted) {
1946 FuncInfo->MBB =
SDB->SL->JTCases[i].first.HeaderBB;
1949 SDB->visitJumpTableHeader(
SDB->SL->JTCases[i].second,
1953 CodeGenAndEmitDAG();
1960 SDB->visitJumpTable(
SDB->SL->JTCases[i].second);
1963 CodeGenAndEmitDAG();
1966 for (
unsigned pi = 0, pe =
FuncInfo->PHINodesToUpdate.size();
1971 "This is not a machine PHI node that we are updating!");
1973 if (PHIBB ==
SDB->SL->JTCases[i].second.Default)
1975 .addMBB(
SDB->SL->JTCases[i].first.HeaderBB);
1977 if (
FuncInfo->MBB->isSuccessor(PHIBB))
1981 SDB->SL->JTCases.clear();
1985 for (
unsigned i = 0, e =
SDB->SL->SwitchCases.size(); i != e; ++i) {
1993 if (
SDB->SL->SwitchCases[i].TrueBB !=
SDB->SL->SwitchCases[i].FalseBB)
2000 CodeGenAndEmitDAG();
2010 for (
unsigned i = 0, e = Succs.
size(); i != e; ++i) {
2021 for (
unsigned pn = 0; ; ++pn) {
2023 "Didn't find PHI entry!");
2024 if (
FuncInfo->PHINodesToUpdate[pn].first ==
PHI) {
2025 PHI.addReg(
FuncInfo->PHINodesToUpdate[pn].second).addMBB(ThisBB);
2033 SDB->SL->SwitchCases.clear();
2054 int64_t DesiredMaskS)
const {
2055 const APInt &ActualMask =
RHS->getAPIntValue();
2056 const APInt &DesiredMask =
APInt(
LHS.getValueSizeInBits(), DesiredMaskS);
2059 if (ActualMask == DesiredMask)
2068 APInt NeededMask = DesiredMask & ~ActualMask;
2083 int64_t DesiredMaskS)
const {
2084 const APInt &ActualMask =
RHS->getAPIntValue();
2085 const APInt &DesiredMask =
APInt(
LHS.getValueSizeInBits(), DesiredMaskS);
2088 if (ActualMask == DesiredMask)
2097 APInt NeededMask = DesiredMask & ~ActualMask;
2114 std::vector<SDValue> InOps;
2123 if (InOps[e-1].getValueType() == MVT::Glue)
2128 if (!Flags.isMemKind() && !Flags.isFuncKind()) {
2130 Ops.insert(Ops.end(), InOps.begin() + i,
2131 InOps.begin() + i + Flags.getNumOperandRegisters() + 1);
2132 i += Flags.getNumOperandRegisters() + 1;
2134 assert(Flags.getNumOperandRegisters() == 1 &&
2135 "Memory operand with multiple values?");
2137 unsigned TiedToOperand;
2138 if (Flags.isUseOperandTiedToDef(TiedToOperand)) {
2142 for (; TiedToOperand; --TiedToOperand) {
2143 CurOp += Flags.getNumOperandRegisters() + 1;
2149 std::vector<SDValue> SelOps;
2151 Flags.getMemoryConstraintID();
2160 Flags.setMemConstraint(ConstraintID);
2168 if (e != InOps.size())
2169 Ops.push_back(InOps.back());
2176 unsigned FlagResNo =
N->getNumValues()-1;
2179 if (
Use.getResNo() == FlagResNo)
2188 bool IgnoreChains) {
2197 Visited.
insert(ImmedUse);
2202 if ((
Op.getValueType() == MVT::Other && IgnoreChains) ||
N == Def)
2204 if (!Visited.
insert(
N).second)
2210 if (Root != ImmedUse) {
2214 if ((
Op.getValueType() == MVT::Other && IgnoreChains) ||
N == Def)
2216 if (!Visited.
insert(
N).second)
2231 return N.hasOneUse();
2238 bool IgnoreChains) {
2287 while (VT == MVT::Glue) {
2298 IgnoreChains =
false;
2304void SelectionDAGISel::Select_INLINEASM(
SDNode *
N) {
2307 std::vector<SDValue> Ops(
N->op_begin(),
N->op_end());
2310 const EVT VTs[] = {MVT::Other, MVT::Glue};
2317void SelectionDAGISel::Select_READ_REGISTER(
SDNode *
Op) {
2322 EVT VT =
Op->getValueType(0);
2328 Op->getOperand(0), dl, Reg,
Op->getValueType(0));
2334void SelectionDAGISel::Select_WRITE_REGISTER(
SDNode *
Op) {
2339 EVT VT =
Op->getOperand(2).getValueType();
2345 Op->getOperand(0), dl, Reg,
Op->getOperand(2));
2351void SelectionDAGISel::Select_UNDEF(
SDNode *
N) {
2355void SelectionDAGISel::Select_FREEZE(
SDNode *
N) {
2363void SelectionDAGISel::Select_ARITH_FENCE(
SDNode *
N) {
2368void SelectionDAGISel::Select_MEMBARRIER(
SDNode *
N) {
2373void SelectionDAGISel::Select_CONVERGENCECTRL_ANCHOR(
SDNode *
N) {
2375 N->getValueType(0));
2378void SelectionDAGISel::Select_CONVERGENCECTRL_ENTRY(
SDNode *
N) {
2380 N->getValueType(0));
2383void SelectionDAGISel::Select_CONVERGENCECTRL_LOOP(
SDNode *
N) {
2385 N->getValueType(0),
N->getOperand(0));
2406void SelectionDAGISel::Select_STACKMAP(
SDNode *
N) {
2408 auto *It =
N->op_begin();
2417 assert(
ID.getValueType() == MVT::i64);
2426 for (; It !=
N->op_end(); It++)
2427 pushStackMapLiveVariable(Ops, *It,
DL);
2436void SelectionDAGISel::Select_PATCHPOINT(
SDNode *
N) {
2438 auto *It =
N->op_begin();
2443 std::optional<SDValue> Glue;
2444 if (It->getValueType() == MVT::Glue)
2450 assert(
ID.getValueType() == MVT::i64);
2474 for (; It !=
N->op_end(); It++)
2475 pushStackMapLiveVariable(Ops, *It,
DL);
2480 if (Glue.has_value())
2490 assert(Val >= 128 &&
"Not a VBR");
2496 NextBits = MatcherTable[
Idx++];
2497 Val |= (NextBits&127) << Shift;
2499 }
while (NextBits & 128);
2504void SelectionDAGISel::Select_JUMP_TABLE_DEBUG_INFO(
SDNode *
N) {
2508 dl, MVT::i64,
true));
2513void SelectionDAGISel::UpdateChains(
2520 if (!ChainNodesMatched.
empty()) {
2522 "Matched input chains but didn't produce a chain");
2525 for (
unsigned i = 0, e = ChainNodesMatched.
size(); i != e; ++i) {
2526 SDNode *ChainNode = ChainNodesMatched[i];
2533 "Deleted node left in chain");
2537 if (ChainNode == NodeToMatch && isMorphNodeTo)
2546 std::replace(ChainNodesMatched.
begin(), ChainNodesMatched.
end(),
N,
2547 static_cast<SDNode *
>(
nullptr));
2553 if (ChainNode != NodeToMatch && ChainNode->
use_empty() &&
2559 if (!NowDeadNodes.
empty())
2578 unsigned int Max = 8192;
2581 if (ChainNodesMatched.
size() == 1)
2582 return ChainNodesMatched[0]->getOperand(0);
2586 std::function<void(
const SDValue)> AddChains = [&](
const SDValue V) {
2587 if (V.getValueType() != MVT::Other)
2591 if (!Visited.
insert(V.getNode()).second)
2594 for (
const SDValue &
Op : V->op_values())
2600 for (
auto *
N : ChainNodesMatched) {
2605 while (!Worklist.
empty())
2609 if (InputChains.
size() == 0)
2619 for (
auto *
N : ChainNodesMatched)
2624 if (InputChains.
size() == 1)
2625 return InputChains[0];
2627 MVT::Other, InputChains);
2631SDNode *SelectionDAGISel::
2640 int OldGlueResultNo = -1, OldChainResultNo = -1;
2642 unsigned NTMNumResults =
Node->getNumValues();
2643 if (
Node->getValueType(NTMNumResults-1) == MVT::Glue) {
2644 OldGlueResultNo = NTMNumResults-1;
2645 if (NTMNumResults != 1 &&
2646 Node->getValueType(NTMNumResults-2) == MVT::Other)
2647 OldChainResultNo = NTMNumResults-2;
2648 }
else if (
Node->getValueType(NTMNumResults-1) == MVT::Other)
2649 OldChainResultNo = NTMNumResults-1;
2667 static_cast<unsigned>(OldGlueResultNo) != ResNumResults - 1)
2669 SDValue(Res, ResNumResults - 1));
2675 if ((EmitNodeInfo &
OPFL_Chain) && OldChainResultNo != -1 &&
2676 static_cast<unsigned>(OldChainResultNo) != ResNumResults - 1)
2678 SDValue(Res, ResNumResults - 1));
2696 unsigned RecNo = MatcherTable[MatcherIndex++];
2697 assert(RecNo < RecordedNodes.size() &&
"Invalid CheckSame");
2698 return N == RecordedNodes[RecNo].first;
2703 const unsigned char *MatcherTable,
unsigned &MatcherIndex,
SDValue N,
2706 if (ChildNo >=
N.getNumOperands())
2708 return ::CheckSame(MatcherTable, MatcherIndex,
N.getOperand(ChildNo),
2716 bool TwoBytePredNo =
2720 ? MatcherTable[MatcherIndex++]
2723 PredNo |= MatcherTable[MatcherIndex++] << 8;
2733 ? MatcherTable[MatcherIndex++]
2741 uint16_t Opc = MatcherTable[MatcherIndex++];
2742 Opc |=
static_cast<uint16_t>(MatcherTable[MatcherIndex++]) << 8;
2743 return N->getOpcode() == Opc;
2750 if (
N.getValueType() == VT)
2760 if (ChildNo >=
N.getNumOperands())
2762 return ::CheckType(VT,
N.getOperand(ChildNo), TLI,
DL);
2768 return cast<CondCodeSDNode>(
N)->get() ==
2775 if (2 >=
N.getNumOperands())
2777 return ::CheckCondCode(MatcherTable, MatcherIndex,
N.getOperand(2));
2785 if (cast<VTSDNode>(
N)->getVT() == VT)
2789 return VT == MVT::iPTR && cast<VTSDNode>(
N)->getVT() == TLI->
getPointerTy(
DL);
2806 int64_t Val = MatcherTable[MatcherIndex++];
2808 Val =
GetVBR(Val, MatcherTable, MatcherIndex);
2813 return C &&
C->getAPIntValue().trySExtValue() == Val;
2819 if (ChildNo >=
N.getNumOperands())
2821 return ::CheckInteger(MatcherTable, MatcherIndex,
N.getOperand(ChildNo));
2827 int64_t Val = MatcherTable[MatcherIndex++];
2829 Val =
GetVBR(Val, MatcherTable, MatcherIndex);
2831 if (
N->getOpcode() !=
ISD::AND)
return false;
2840 int64_t Val = MatcherTable[MatcherIndex++];
2842 Val =
GetVBR(Val, MatcherTable, MatcherIndex);
2844 if (
N->getOpcode() !=
ISD::OR)
return false;
2861 unsigned Opcode = Table[
Index++];
2921 unsigned Res = Table[
Index++];
2923 N.getValue(Res), SDISel.
TLI,
3009 unsigned NumRecordedNodes;
3012 unsigned NumMatchedMemRefs;
3015 SDValue InputChain, InputGlue;
3018 bool HasChainNodesMatched;
3035 :
SelectionDAG::DAGUpdateListener(DAG), NodeToMatch(NodeToMatch),
3036 RecordedNodes(
RN), MatchScopes(MS) {}
3044 if (!
E ||
E->isMachineOpcode())
3047 if (
N == *NodeToMatch)
3052 for (
auto &
I : RecordedNodes)
3053 if (
I.first.getNode() ==
N)
3056 for (
auto &
I : MatchScopes)
3057 for (
auto &J :
I.NodeStack)
3058 if (J.getNode() ==
N)
3066 const unsigned char *MatcherTable,
3067 unsigned TableSize) {
3106 Select_INLINEASM(NodeToMatch);
3109 Select_READ_REGISTER(NodeToMatch);
3112 Select_WRITE_REGISTER(NodeToMatch);
3115 Select_UNDEF(NodeToMatch);
3118 Select_FREEZE(NodeToMatch);
3121 Select_ARITH_FENCE(NodeToMatch);
3124 Select_MEMBARRIER(NodeToMatch);
3127 Select_STACKMAP(NodeToMatch);
3130 Select_PATCHPOINT(NodeToMatch);
3133 Select_JUMP_TABLE_DEBUG_INFO(NodeToMatch);
3136 Select_CONVERGENCECTRL_ANCHOR(NodeToMatch);
3139 Select_CONVERGENCECTRL_ENTRY(NodeToMatch);
3142 Select_CONVERGENCECTRL_LOOP(NodeToMatch);
3169 SDValue InputChain, InputGlue;
3183 unsigned MatcherIndex = 0;
3185 if (!OpcodeOffset.empty()) {
3187 if (
N.getOpcode() < OpcodeOffset.size())
3188 MatcherIndex = OpcodeOffset[
N.getOpcode()];
3189 LLVM_DEBUG(
dbgs() <<
" Initial Opcode index to " << MatcherIndex <<
"\n");
3198 unsigned CaseSize = MatcherTable[
Idx++];
3200 CaseSize =
GetVBR(CaseSize, MatcherTable,
Idx);
3201 if (CaseSize == 0)
break;
3205 Opc |=
static_cast<uint16_t>(MatcherTable[
Idx++]) << 8;
3206 if (Opc >= OpcodeOffset.size())
3207 OpcodeOffset.resize((Opc+1)*2);
3208 OpcodeOffset[Opc] =
Idx;
3213 if (
N.getOpcode() < OpcodeOffset.size())
3214 MatcherIndex = OpcodeOffset[
N.getOpcode()];
3218 assert(MatcherIndex < TableSize &&
"Invalid index");
3220 unsigned CurrentOpcodeIndex = MatcherIndex;
3234 unsigned NumToSkip = MatcherTable[MatcherIndex++];
3235 if (NumToSkip & 128)
3236 NumToSkip =
GetVBR(NumToSkip, MatcherTable, MatcherIndex);
3238 if (NumToSkip == 0) {
3243 FailIndex = MatcherIndex+NumToSkip;
3245 unsigned MatcherIndexOfPredicate = MatcherIndex;
3246 (void)MatcherIndexOfPredicate;
3253 Result, *
this, RecordedNodes);
3258 dbgs() <<
" Skipped scope entry (due to false predicate) at "
3259 <<
"index " << MatcherIndexOfPredicate <<
", continuing at "
3260 << FailIndex <<
"\n");
3261 ++NumDAGIselRetries;
3265 MatcherIndex = FailIndex;
3269 if (FailIndex == 0)
break;
3273 MatchScope NewEntry;
3274 NewEntry.FailIndex = FailIndex;
3275 NewEntry.NodeStack.append(NodeStack.
begin(), NodeStack.
end());
3276 NewEntry.NumRecordedNodes = RecordedNodes.
size();
3277 NewEntry.NumMatchedMemRefs = MatchedMemRefs.
size();
3278 NewEntry.InputChain = InputChain;
3279 NewEntry.InputGlue = InputGlue;
3280 NewEntry.HasChainNodesMatched = !ChainNodesMatched.
empty();
3286 SDNode *Parent =
nullptr;
3287 if (NodeStack.
size() > 1)
3288 Parent = NodeStack[NodeStack.
size()-2].getNode();
3289 RecordedNodes.
push_back(std::make_pair(
N, Parent));
3298 if (ChildNo >=
N.getNumOperands())
3301 RecordedNodes.
push_back(std::make_pair(
N->getOperand(ChildNo),
3306 if (
auto *MN = dyn_cast<MemSDNode>(
N))
3307 MatchedMemRefs.
push_back(MN->getMemOperand());
3317 if (
N->getNumOperands() != 0 &&
3318 N->getOperand(
N->getNumOperands()-1).getValueType() == MVT::Glue)
3319 InputGlue =
N->getOperand(
N->getNumOperands()-1);
3323 unsigned ChildNo = MatcherTable[MatcherIndex++];
3324 if (ChildNo >=
N.getNumOperands())
3326 N =
N.getOperand(ChildNo);
3336 if (ChildNo >=
N.getNumOperands())
3338 N =
N.getOperand(ChildNo);
3354 assert(!NodeStack.
empty() &&
"Node stack imbalance!");
3355 N = NodeStack.
back();
3358 ? MatcherTable[MatcherIndex++]
3360 if (SiblingNo >=
N.getNumOperands())
3362 N =
N.getOperand(SiblingNo);
3369 assert(!NodeStack.
empty() &&
"Node stack imbalance!");
3370 N = NodeStack.
back();
3374 if (!
::CheckSame(MatcherTable, MatcherIndex,
N, RecordedNodes))
break;
3411 unsigned OpNum = MatcherTable[MatcherIndex++];
3414 for (
unsigned i = 0; i < OpNum; ++i)
3415 Operands.push_back(RecordedNodes[MatcherTable[MatcherIndex++]].first);
3417 unsigned PredNo = MatcherTable[MatcherIndex++];
3432 ? MatcherTable[MatcherIndex++]
3434 unsigned RecNo = MatcherTable[MatcherIndex++];
3435 assert(RecNo < RecordedNodes.
size() &&
"Invalid CheckComplexPat");
3439 std::unique_ptr<MatchStateUpdater> MSU;
3441 MSU.reset(
new MatchStateUpdater(*
CurDAG, &NodeToMatch, RecordedNodes,
3445 RecordedNodes[RecNo].first, CPNum,
3451 if (!
::CheckOpcode(MatcherTable, MatcherIndex,
N.getNode()))
break;
3474 unsigned Res = MatcherTable[MatcherIndex++];
3483 unsigned CurNodeOpcode =
N.getOpcode();
3484 unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3488 CaseSize = MatcherTable[MatcherIndex++];
3490 CaseSize =
GetVBR(CaseSize, MatcherTable, MatcherIndex);
3491 if (CaseSize == 0)
break;
3493 uint16_t Opc = MatcherTable[MatcherIndex++];
3494 Opc |=
static_cast<uint16_t>(MatcherTable[MatcherIndex++]) << 8;
3497 if (CurNodeOpcode == Opc)
3501 MatcherIndex += CaseSize;
3505 if (CaseSize == 0)
break;
3508 LLVM_DEBUG(
dbgs() <<
" OpcodeSwitch from " << SwitchStart <<
" to "
3509 << MatcherIndex <<
"\n");
3514 MVT CurNodeVT =
N.getSimpleValueType();
3515 unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3519 CaseSize = MatcherTable[MatcherIndex++];
3521 CaseSize =
GetVBR(CaseSize, MatcherTable, MatcherIndex);
3522 if (CaseSize == 0)
break;
3526 if (CaseVT == MVT::iPTR)
3530 if (CurNodeVT == CaseVT)
3534 MatcherIndex += CaseSize;
3538 if (CaseSize == 0)
break;
3542 <<
"] from " << SwitchStart <<
" to " << MatcherIndex
3612 if (!
::CheckOrImm(MatcherTable, MatcherIndex,
N, *
this))
break;
3624 assert(NodeStack.
size() != 1 &&
"No parent node");
3627 bool HasMultipleUses =
false;
3628 for (
unsigned i = 1, e = NodeStack.
size()-1; i != e; ++i) {
3629 unsigned NNonChainUses = 0;
3630 SDNode *NS = NodeStack[i].getNode();
3632 if (UI.getUse().getValueType() != MVT::Other)
3633 if (++NNonChainUses > 1) {
3634 HasMultipleUses =
true;
3637 if (HasMultipleUses)
break;
3639 if (HasMultipleUses)
break;
3678 int64_t Val = MatcherTable[MatcherIndex++];
3680 Val =
GetVBR(Val, MatcherTable, MatcherIndex);
3683 RecordedNodes.
push_back(std::pair<SDValue, SDNode *>(
3702 unsigned RegNo = MatcherTable[MatcherIndex++];
3703 RecordedNodes.
push_back(std::pair<SDValue, SDNode *>(
3713 unsigned RegNo = MatcherTable[MatcherIndex++];
3714 RegNo |= MatcherTable[MatcherIndex++] << 8;
3715 RecordedNodes.
push_back(std::pair<SDValue, SDNode*>(
3731 ? MatcherTable[MatcherIndex++]
3733 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitConvertToTarget");
3734 SDValue Imm = RecordedNodes[RecNo].first;
3737 const ConstantInt *Val=cast<ConstantSDNode>(Imm)->getConstantIntValue();
3739 Imm.getValueType());
3741 const ConstantFP *Val=cast<ConstantFPSDNode>(Imm)->getConstantFPValue();
3743 Imm.getValueType());
3746 RecordedNodes.
push_back(std::make_pair(Imm, RecordedNodes[RecNo].second));
3755 "EmitMergeInputChains should be the first chain producing node");
3757 "Should only have one EmitMergeInputChains per match");
3761 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitMergeInputChains");
3762 ChainNodesMatched.
push_back(RecordedNodes[RecNo].first.getNode());
3768 if (ChainNodesMatched.
back() != NodeToMatch &&
3769 !RecordedNodes[RecNo].first.hasOneUse()) {
3770 ChainNodesMatched.
clear();
3784 "EmitMergeInputChains should be the first chain producing node");
3791 unsigned NumChains = MatcherTable[MatcherIndex++];
3792 assert(NumChains != 0 &&
"Can't TF zero chains");
3795 "Should only have one EmitMergeInputChains per match");
3798 for (
unsigned i = 0; i != NumChains; ++i) {
3799 unsigned RecNo = MatcherTable[MatcherIndex++];
3800 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitMergeInputChains");
3801 ChainNodesMatched.
push_back(RecordedNodes[RecNo].first.getNode());
3807 if (ChainNodesMatched.
back() != NodeToMatch &&
3808 !RecordedNodes[RecNo].first.hasOneUse()) {
3809 ChainNodesMatched.
clear();
3815 if (ChainNodesMatched.
empty())
3840 : MatcherTable[MatcherIndex++];
3841 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitCopyToReg");
3842 unsigned DestPhysReg = MatcherTable[MatcherIndex++];
3844 DestPhysReg |= MatcherTable[MatcherIndex++] << 8;
3850 DestPhysReg, RecordedNodes[RecNo].first,
3853 InputGlue = InputChain.
getValue(1);
3858 unsigned XFormNo = MatcherTable[MatcherIndex++];
3859 unsigned RecNo = MatcherTable[MatcherIndex++];
3860 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitNodeXForm");
3862 RecordedNodes.
push_back(std::pair<SDValue,SDNode*>(Res,
nullptr));
3868 unsigned index = MatcherTable[MatcherIndex++];
3869 index |= (MatcherTable[MatcherIndex++] << 8);
3901 uint16_t TargetOpc = MatcherTable[MatcherIndex++];
3902 TargetOpc |=
static_cast<uint16_t>(MatcherTable[MatcherIndex++]) << 8;
3903 unsigned EmitNodeInfo;
3922 EmitNodeInfo = MatcherTable[MatcherIndex++];
3947 NumVTs = MatcherTable[MatcherIndex++];
3949 for (
unsigned i = 0; i != NumVTs; ++i) {
3952 if (VT == MVT::iPTR)
3965 if (VTs.
size() == 1)
3967 else if (VTs.
size() == 2)
3973 unsigned NumOps = MatcherTable[MatcherIndex++];
3975 for (
unsigned i = 0; i != NumOps; ++i) {
3976 unsigned RecNo = MatcherTable[MatcherIndex++];
3978 RecNo =
GetVBR(RecNo, MatcherTable, MatcherIndex);
3980 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitNode");
3981 Ops.
push_back(RecordedNodes[RecNo].first);
3988 FirstOpToCopy += (EmitNodeInfo &
OPFL_Chain) ? 1 : 0;
3990 "Invalid variadic node");
3993 for (
unsigned i = FirstOpToCopy, e = NodeToMatch->
getNumOperands();
3996 if (V.getValueType() == MVT::Glue)
break;
4011 bool MayRaiseFPException =
4018 bool IsMorphNodeTo =
4021 if (!IsMorphNodeTo) {
4028 for (
unsigned i = 0, e = VTs.
size(); i != e; ++i) {
4029 if (VTs[i] == MVT::Other || VTs[i] == MVT::Glue)
break;
4035 "NodeToMatch was removed partway through selection");
4039 auto &Chain = ChainNodesMatched;
4041 "Chain node replaced during MorphNode");
4044 Res = cast<MachineSDNode>(MorphNode(NodeToMatch, TargetOpc, VTList,
4045 Ops, EmitNodeInfo));
4052 Flags.setNoFPExcept(
true);
4053 Res->setFlags(Flags);
4075 bool mayLoad = MCID.
mayLoad();
4082 if (MMO->isLoad()) {
4085 }
else if (MMO->isStore()) {
4097 <<
" Dropping mem operands\n";
4098 dbgs() <<
" " << (IsMorphNodeTo ?
"Morphed" :
"Created")
4103 if (IsMorphNodeTo) {
4105 UpdateChains(Res, InputChain, ChainNodesMatched,
true);
4115 unsigned NumResults = MatcherTable[MatcherIndex++];
4117 for (
unsigned i = 0; i != NumResults; ++i) {
4118 unsigned ResSlot = MatcherTable[MatcherIndex++];
4120 ResSlot =
GetVBR(ResSlot, MatcherTable, MatcherIndex);
4122 assert(ResSlot < RecordedNodes.
size() &&
"Invalid CompleteMatch");
4123 SDValue Res = RecordedNodes[ResSlot].first;
4125 assert(i < NodeToMatch->getNumValues() &&
4128 "Invalid number of results to complete!");
4134 "invalid replacement");
4139 UpdateChains(NodeToMatch, InputChain, ChainNodesMatched,
false);
4152 "Didn't replace all uses of the node?");
4162 LLVM_DEBUG(
dbgs() <<
" Match failed at index " << CurrentOpcodeIndex
4164 ++NumDAGIselRetries;
4166 if (MatchScopes.
empty()) {
4167 CannotYetSelect(NodeToMatch);
4173 MatchScope &LastScope = MatchScopes.
back();
4174 RecordedNodes.
resize(LastScope.NumRecordedNodes);
4176 NodeStack.
append(LastScope.NodeStack.begin(), LastScope.NodeStack.end());
4177 N = NodeStack.
back();
4179 if (LastScope.NumMatchedMemRefs != MatchedMemRefs.
size())
4180 MatchedMemRefs.
resize(LastScope.NumMatchedMemRefs);
4181 MatcherIndex = LastScope.FailIndex;
4185 InputChain = LastScope.InputChain;
4186 InputGlue = LastScope.InputGlue;
4187 if (!LastScope.HasChainNodesMatched)
4188 ChainNodesMatched.
clear();
4193 unsigned NumToSkip = MatcherTable[MatcherIndex++];
4194 if (NumToSkip & 128)
4195 NumToSkip =
GetVBR(NumToSkip, MatcherTable, MatcherIndex);
4199 if (NumToSkip != 0) {
4200 LastScope.FailIndex = MatcherIndex+NumToSkip;
4214 if (
N->isMachineOpcode()) {
4221 if (
N->isTargetOpcode())
4222 return N->isTargetStrictFPOpcode();
4223 return N->isStrictFPOpcode();
4228 auto *
C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
4233 if (
auto *FN = dyn_cast<FrameIndexSDNode>(
N->getOperand(0))) {
4236 int32_t Off =
C->getSExtValue();
4239 return (Off >= 0) && (((
A.value() - 1) & Off) ==
unsigned(Off));
4244void SelectionDAGISel::CannotYetSelect(
SDNode *
N) {
4247 Msg <<
"Cannot select: ";
4253 Msg <<
"\nIn function: " <<
MF->
getName();
4255 bool HasInputChain =
N->getOperand(0).getValueType() == MVT::Other;
4256 unsigned iid =
N->getConstantOperandVal(HasInputChain);
4257 if (iid < Intrinsic::num_intrinsics)
4260 Msg <<
"target intrinsic %" <<
TII->
getName(iid);
4262 Msg <<
"unknown intrinsic #" << iid;
unsigned const MachineRegisterInfo * MRI
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
MachineInstrBuilder & UseMI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
amdgpu AMDGPU Register Bank Select
This file implements a class to represent arbitrary precision integral constant values and operations...
Expand Atomic instructions
BlockVerifier::State From
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_ATTRIBUTE_ALWAYS_INLINE
LLVM_ATTRIBUTE_ALWAYS_INLINE - On compilers where we have a directive to do so, mark a method "always...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
This file defines the FastISel class.
mir Rename Register Operands
Machine Instruction Scheduler
unsigned const TargetRegisterInfo * TRI
const char LLVMTargetMachineRef TM
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
static cl::opt< bool > ViewSUnitDAGs("view-sunit-dags", cl::Hidden, cl::desc("Pop up a window to show SUnit dags after they are processed"))
static cl::opt< bool > ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the post " "legalize types dag combine pass"))
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckPatternPredicate(unsigned Opcode, const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel)
CheckPatternPredicate - Implements OP_CheckPatternPredicate.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes, unsigned ChildNo)
CheckChildSame - Implements OP_CheckChildXSame.
static uint64_t decodeSignRotatedValue(uint64_t V)
Decode a signed value stored with the sign bit in the LSB for dense VBR encoding.
static cl::opt< bool > ViewISelDAGs("view-isel-dags", cl::Hidden, cl::desc("Pop up a window to show isel dags as they are selected"))
static LLVM_ATTRIBUTE_ALWAYS_INLINE uint64_t GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx)
GetVBR - decode a vbr encoding whose top bit is set.
static void computeUsesMSVCFloatingPoint(const Triple &TT, const Function &F, MachineModuleInfo &MMI)
static void reportFastISelFailure(MachineFunction &MF, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R, bool ShouldAbort)
static cl::opt< bool > ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the second " "dag combine pass"))
static RegisterScheduler defaultListDAGScheduler("default", "Best scheduler for the target", createDefaultScheduler)
static unsigned IsPredicateKnownToFail(const unsigned char *Table, unsigned Index, SDValue N, bool &Result, const SelectionDAGISel &SDISel, SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes)
IsPredicateKnownToFail - If we know how and can do so without pushing a scope, evaluate the current n...
static cl::opt< int > EnableFastISelAbort("fast-isel-abort", cl::Hidden, cl::desc("Enable abort calls when \"fast\" instruction selection " "fails to lower an instruction: 0 disable the abort, 1 will " "abort but for args, calls and terminators, 2 will also " "abort for argument lowering, and 3 will never fallback " "to SelectionDAG."))
static void mapWasmLandingPadIndex(MachineBasicBlock *MBB, const CatchPadInst *CPI)
static void processSingleLocVars(FunctionLoweringInfo &FuncInfo, FunctionVarLocs const *FnVarLocs)
Collect single location variable information generated with assignment tracking.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
static cl::opt< bool > UseMBPI("use-mbpi", cl::desc("use Machine Branch Probability Info"), cl::init(true), cl::Hidden)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildType(MVT::SimpleValueType VT, SDValue N, const TargetLowering *TLI, const DataLayout &DL, unsigned ChildNo)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes)
CheckSame - Implements OP_CheckSame.
static bool dontUseFastISelFor(const Function &Fn)
static bool findNonImmUse(SDNode *Root, SDNode *Def, SDNode *ImmedUse, bool IgnoreChains)
findNonImmUse - Return true if "Def" is a predecessor of "Root" via a path beyond "ImmedUse".
static cl::opt< bool > ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the first " "dag combine pass"))
static bool processIfEntryValueDbgDeclare(FunctionLoweringInfo &FuncInfo, const Value *Arg, DIExpression *Expr, DILocalVariable *Var, DebugLoc DbgLoc)
static cl::opt< bool > ViewSchedDAGs("view-sched-dags", cl::Hidden, cl::desc("Pop up a window to show sched dags as they are processed"))
static void processDbgDeclares(FunctionLoweringInfo &FuncInfo)
Collect llvm.dbg.declare information.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckType(MVT::SimpleValueType VT, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
static bool hasExceptionPointerOrCodeUser(const CatchPadInst *CPI)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChild2CondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
static cl::opt< bool > ViewLegalizeDAGs("view-legalize-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize"))
static cl::opt< bool > ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize types"))
static SDNode * findGlueUse(SDNode *N)
findGlueUse - Return use of MVT::Glue value produced by the specified SDNode.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckNodePredicate(unsigned Opcode, const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel, SDNode *N)
CheckNodePredicate - Implements OP_CheckNodePredicate.
static cl::opt< RegisterScheduler::FunctionPassCtor, false, RegisterPassParser< RegisterScheduler > > ISHeuristic("pre-RA-sched", cl::init(&createDefaultScheduler), cl::Hidden, cl::desc("Instruction schedulers available (before register" " allocation):"))
ISHeuristic command line option for instruction schedulers.
static cl::opt< bool > EnableFastISelFallbackReport("fast-isel-report-on-fallback", cl::Hidden, cl::desc("Emit a diagnostic when \"fast\" instruction selection " "falls back to SelectionDAG."))
static bool processDbgDeclare(FunctionLoweringInfo &FuncInfo, const Value *Address, DIExpression *Expr, DILocalVariable *Var, DebugLoc DbgLoc)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDNode *N)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, unsigned ChildNo)
static cl::opt< std::string > FilterDAGBasicBlockName("filter-view-dags", cl::Hidden, cl::desc("Only display the basic block whose name " "matches this for all view-*-dags options"))
static SDValue HandleMergeInputChains(SmallVectorImpl< SDNode * > &ChainNodesMatched, SelectionDAG *CurDAG)
HandleMergeInputChains - This implements the OPC_EmitMergeInputChains operation for when the pattern ...
static bool isFoldedOrDeadInstruction(const Instruction *I, const FunctionLoweringInfo &FuncInfo)
isFoldedOrDeadInstruction - Return true if the specified instruction is side-effect free and is eithe...
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This file describes how to lower LLVM code to machine code.
DEMANGLE_DUMP_METHOD void dump() const
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
Class for arbitrary precision integers.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
An immutable pass that tracks lazily created AssumptionCache objects.
LLVM Basic Block Representation.
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
InstListType::const_iterator const_iterator
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
bool isEHPad() const
Return true if this basic block is an exception handling block.
const Instruction * getFirstMayFaultInst() const
Returns the first potential AsynchEH faulty instruction currently it checks for loads/stores (which m...
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Legacy analysis pass which computes BranchProbabilityInfo.
ConstantFP - Floating Point Values [float, double].
This is the shared class of boolean and integer constants.
This is an important base class in LLVM.
bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
static DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Record of a variable value-assignment, aka a non instruction representation of the dbg....
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Diagnostic information for ISel fallback path.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
void setLastLocalValue(MachineInstr *I)
Update the position of the last instruction emitted for materializing constants for use in the curren...
void handleDbgInfo(const Instruction *II)
Target-independent lowering of non-instruction debug info associated with this instruction.
bool tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst)
We're checking to see if we can fold LI into FoldInst.
void removeDeadCode(MachineBasicBlock::iterator I, MachineBasicBlock::iterator E)
Remove all dead instructions between the I and E.
void startNewBlock()
Set the current block to which generated machine instructions will be appended.
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
void finishBasicBlock()
Flush the local value map.
void recomputeInsertPt()
Reset InsertPt to prepare for inserting instructions into the current block.
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
unsigned arg_size() const
arg_size - Return the number of funcletpad arguments.
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th funcletpad argument.
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
SmallPtrSet< const DbgVariableRecord *, 8 > PreprocessedDVRDeclares
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
int getArgumentFrameIndex(const Argument *A)
getArgumentFrameIndex - Get frame index for the byval argument.
bool isExportedInst(const Value *V) const
isExportedInst - Return true if the specified value is an instruction exported from its block.
SmallPtrSet< const DbgDeclareInst *, 8 > PreprocessedDbgDeclares
Collection of dbg.declare instructions handled after argument lowering and before ISel proper.
DenseMap< const Value *, Register > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
MachineRegisterInfo * RegInfo
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Data structure describing the variable locations in a function.
const VarLocInfo * single_locs_begin() const
DILocalVariable * getDILocalVariable(const VarLocInfo *Loc) const
Return the DILocalVariable for the location definition represented by ID.
const VarLocInfo * single_locs_end() const
One past the last single-location variable location definition.
const BasicBlock & getEntryBlock() const
FunctionType * getFunctionType() const
Returns the FunctionType for me.
iterator_range< arg_iterator > args()
DISubprogram * getSubprogram() const
Get the attached subprogram.
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool callsFunctionThatReturnsTwice() const
callsFunctionThatReturnsTwice - Return true if the function has a call to setjmp or other function th...
An analysis pass which caches information about the entire Module.
Module * getParent()
Get the module that this global value is contained inside of...
This class is used to form a handle around another node that is persistent and is updated across invo...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
A wrapper class for inspecting calls to intrinsic functions.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
static void getLazyBFIAnalysisUsage(AnalysisUsage &AU)
Helper for client passes to set up the analysis usage on behalf of this pass.
MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
Describe properties that are true of each instruction in the target description file.
bool mayStore() const
Return true if this instruction could possibly modify memory.
bool mayLoad() const
Return true if this instruction could possibly read memory.
bool mayRaiseFPException() const
Return true if this instruction may raise a floating-point exception.
bool isCall() const
Return true if the instruction is a call.
bool isReturn() const
Return true if the instruction is a return.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
StringRef getName(unsigned Opcode) const
Returns the name for the instructions with the given opcode.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
const MDNode * getMD() const
const MDOperand & getOperand(unsigned I) const
StringRef getString() const
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
instr_iterator instr_end()
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasCalls() const
Return true if the current function has any function calls.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
bool hasProperty(Property P) const
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
bool useDebugInstrRef() const
Returns true if the function's variable locations are tracked with instruction referencing.
void setExposesReturnsTwice(bool B)
setCallsSetJmp - Set a flag that indicates if there's a call to a "returns twice" function.
void setHasInlineAsm(bool B)
Set a flag that indicates that the function contains inline assembly.
void setWasmLandingPadIndex(const MachineBasicBlock *LPad, unsigned Index)
Map the landing pad to its index. Used for Wasm exception handling.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
bool hasInlineAsm() const
Returns true if the function contains any inline assembly.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
void finalizeDebugInstrRefs()
Finalise any partially emitted debug instructions.
void setCallSiteLandingPad(MCSymbol *Sym, ArrayRef< unsigned > Sites)
Map the landing pad's EH symbol to the call site indexes.
void setUseDebugInstrRef(bool UseInstrRef)
Set whether this function will use instruction referencing or not.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
MCSymbol * addLandingPad(MachineBasicBlock *LandingPad)
Add a new panding pad, and extract the exception handling information from the landingpad instruction...
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineModuleInfo & getMMI() const
bool shouldUseDebugInstrRef() const
Determine whether, in the current machine configuration, we should use instruction referencing or not...
const MachineFunctionProperties & getProperties() const
Get the function properties.
void setVariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr, int Slot, const DILocation *Loc)
Collect information used to emit debugging information of a variable in a stack slot.
const MachineBasicBlock & front() const
void print(raw_ostream &OS, const SlotIndexes *=nullptr) const
print - Print out the MachineFunction in a format suitable for debugging to the specified stream.
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
bool isTerminator(QueryType Type=AnyInBundle) const
Returns true if this instruction part of the terminator for a basic block.
const MachineBasicBlock * getParent() const
bool isDebugValue() const
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
This class contains meta information specific to a module.
const MCContext & getContext() const
bool usesMSVCFloatingPoint() const
void setUsesMSVCFloatingPoint(bool b)
Register getReg() const
getReg - Returns the register number.
MachinePassRegistry - Track the registration of machine passes.
defusechain_iterator - This class provides iterator support for machine operands in the function that...
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
void EmitLiveInCopies(MachineBasicBlock *EntryMBB, const TargetRegisterInfo &TRI, const TargetInstrInfo &TII)
EmitLiveInCopies - Emit copies to initialize livein virtual registers into the given entry block.
use_instr_iterator use_instr_begin(Register RegNo) const
ArrayRef< std::pair< MCRegister, Register > > liveins() const
static use_instr_iterator use_instr_end()
void addPhysRegsUsedFromRegMask(const uint32_t *RegMask)
addPhysRegsUsedFromRegMask - Mark any registers not in RegMask as used.
An SDNode that represents everything that will be needed to construct a MachineInstr.
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
This class is used by SelectionDAGISel to temporarily override the optimization level on a per-functi...
OptLevelChanger(SelectionDAGISel &ISel, CodeGenOptLevel NewOptLevel)
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
RegisterPassParser class - Handle the addition of new machine passes.
ScheduleDAGSDNodes *(*)(SelectionDAGISel *, CodeGenOptLevel) FunctionPassCtor
static MachinePassRegistry< FunctionPassCtor > Registry
RegisterScheduler class - Track the registration of instruction schedulers.
Wrapper class representing virtual and physical registers.
static unsigned virtReg2Index(Register Reg)
Convert a virtual register number to a 0-based index.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< value_op_iterator > op_values() const
void setNodeId(int Id)
Set unique node id.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
ScheduleDAGSDNodes - A ScheduleDAG for scheduling SDNode-based DAGs.
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
std::unique_ptr< FunctionLoweringInfo > FuncInfo
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
bool CheckOrMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const
CheckOrMask - The isel is trying to match something like (or X, 255).
const TargetLowering * TLI
virtual void PostprocessISelDAG()
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
virtual bool CheckNodePredicate(SDNode *N, unsigned PredNo) const
CheckNodePredicate - This function is generated by tblgen in the target.
std::unique_ptr< OptimizationRemarkEmitter > ORE
Current optimization remark emitter.
MachineRegisterInfo * RegInfo
unsigned DAGSize
DAGSize - Size of DAG being instruction selected.
@ OPC_MorphNodeTo2GlueOutput
@ OPC_CheckPatternPredicate5
@ OPC_EmitCopyToRegTwoByte
@ OPC_MorphNodeTo2GlueInput
@ OPC_CheckChild2CondCode
@ OPC_CheckPatternPredicateTwoByte
@ OPC_CheckPatternPredicate1
@ OPC_MorphNodeTo1GlueOutput
@ OPC_EmitMergeInputChains1_1
@ OPC_CheckPatternPredicate2
@ OPC_EmitConvertToTarget2
@ OPC_EmitConvertToTarget0
@ OPC_CheckPatternPredicate4
@ OPC_EmitConvertToTarget1
@ OPC_CheckPatternPredicate
@ OPC_MorphNodeTo0GlueInput
@ OPC_CheckPatternPredicate6
@ OPC_MorphNodeTo0GlueOutput
@ OPC_CheckPatternPredicate7
@ OPC_EmitMergeInputChains
@ OPC_EmitMergeInputChains1_0
@ OPC_CheckFoldableChainNode
@ OPC_EmitConvertToTarget3
@ OPC_CheckPredicateWithOperands
@ OPC_EmitConvertToTarget4
@ OPC_EmitStringInteger32
@ OPC_EmitConvertToTarget7
@ OPC_EmitMergeInputChains1_2
@ OPC_EmitConvertToTarget5
@ OPC_CheckPatternPredicate0
@ OPC_MorphNodeTo1GlueInput
@ OPC_CheckPatternPredicate3
@ OPC_EmitConvertToTarget
@ OPC_EmitConvertToTarget6
bool isOrEquivalentToAdd(const SDNode *N) const
virtual bool CheckComplexPattern(SDNode *Root, SDNode *Parent, SDValue N, unsigned PatternNo, SmallVectorImpl< std::pair< SDValue, SDNode * > > &Result)
virtual bool CheckPatternPredicate(unsigned PredNo) const
CheckPatternPredicate - This function is generated by tblgen in the target.
static int getNumFixedFromVariadicInfo(unsigned Flags)
getNumFixedFromVariadicInfo - Transform an EmitNode flags word into the number of fixed arity values ...
const TargetLibraryInfo * LibInfo
static int getUninvalidatedNodeId(SDNode *N)
const TargetInstrInfo * TII
virtual bool CheckNodePredicateWithOperands(SDNode *N, unsigned PredNo, const SmallVectorImpl< SDValue > &Operands) const
CheckNodePredicateWithOperands - This function is generated by tblgen in the target.
void SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, unsigned TableSize)
static void EnforceNodeIdInvariant(SDNode *N)
void ReplaceUses(SDValue F, SDValue T)
ReplaceUses - replace all uses of the old node F with the use of the new node T.
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
virtual SDValue RunSDNodeXForm(SDValue V, unsigned XFormNo)
bool MatchFilterFuncName
True if the function currently processing is in the function printing list (i.e.
void SelectInlineAsmMemoryOperands(std::vector< SDValue > &Ops, const SDLoc &DL)
SelectInlineAsmMemoryOperands - Calls to this are automatically generated by tblgen.
static bool IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, CodeGenOptLevel OptLevel, bool IgnoreChains=false)
IsLegalToFold - Returns true if the specific operand node N of U can be folded during instruction sel...
virtual bool ComplexPatternFuncMutatesDAG() const
Return true if complex patterns for this target can mutate the DAG.
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
bool CheckAndMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const
CheckAndMask - The isel is trying to match something like (and X, 255).
SwiftErrorValueTracking * SwiftError
SelectionDAGISel(char &ID, TargetMachine &tm, CodeGenOptLevel OL=CodeGenOptLevel::Default)
virtual StringRef getPatternForIndex(unsigned index)
getPatternForIndex - Patterns selected by tablegen during ISEL
bool mayRaiseFPException(SDNode *Node) const
Return whether the node may raise an FP exception.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
std::unique_ptr< SelectionDAGBuilder > SDB
void ReplaceNode(SDNode *F, SDNode *T)
Replace all uses of F with T, then remove F from the DAG.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
static void InvalidateNodeId(SDNode *N)
~SelectionDAGISel() override
virtual StringRef getIncludePathForIndex(unsigned index)
getIncludePathForIndex - get the td source location of pattern instantiation
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
void setFunctionLoweringInfo(FunctionLoweringInfo *FuncInfo)
SDNode * mutateStrictFPToFP(SDNode *Node)
Mutate the specified strict FP node to its non-strict equivalent, unlinking the node from its chain a...
void VerifyDAGDivergence()
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
void salvageDebugInfo(SDNode &N)
To be invoked on an SDNode that is slated to be erased.
SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
allnodes_const_iterator allnodes_begin() const
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
void viewGraph(const std::string &Title)
Pop up a GraphViz/gv window with the DAG rendered using 'dot'.
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
void Combine(CombineLevel Level, AAResults *AA, CodeGenOptLevel OptLevel)
This iterates over the nodes in the SelectionDAG, folding certain types of nodes together,...
void clear()
Clear state and free memory necessary to make this SelectionDAG ready to process a new block.
SDValue getRegister(unsigned Reg, EVT VT)
void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, UniformityInfo *UA, ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin, FunctionVarLocs const *FnVarLocs)
Prepare this SelectionDAG to process code in the given MachineFunction.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
const FunctionVarLocs * getFunctionVarLocs() const
Returns the result of the AssignmentTrackingAnalysis pass if it's available, otherwise return nullptr...
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
ilist< SDNode >::iterator allnodes_iterator
bool LegalizeTypes()
This transforms the SelectionDAG into a SelectionDAG that only uses types natively supported by the t...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
bool shouldEmitSDCheck(const BasicBlock &BB) const
void copyToMachineFrameInfo(MachineFrameInfo &MFI) const
StringRef - Represent a constant reference to a string, i.e.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
bool createEntriesInEntryBlock(DebugLoc DbgLoc)
Create initial definitions of swifterror values in the entry block of the current function.
void setFunction(MachineFunction &MF)
Initialize data structures for specified new function.
void preassignVRegs(MachineBasicBlock *MBB, BasicBlock::const_iterator Begin, BasicBlock::const_iterator End)
void propagateVRegs()
Propagate assigned swifterror vregs through a function, synthesizing PHI nodes when needed to maintai...
TargetIntrinsicInfo - Interface to description of machine instruction set.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
Sched::Preference getSchedulingPreference() const
Return target scheduling preference.
bool isStrictFPEnabled() const
Return true if the target support strict float operation.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
virtual Register getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual Register getRegisterByName(const char *RegName, LLT Ty, const MachineFunction &MF) const
Return the register ID of the name passed in.
virtual void insertCopiesSplitCSR(MachineBasicBlock *Entry, const SmallVectorImpl< MachineBasicBlock * > &Exits) const
Insert explicit copies in entry and exit blocks.
virtual void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
virtual void initializeSplitCSR(MachineBasicBlock *Entry) const
Perform necessary initialization to handle a subset of CSRs explicitly via copies.
virtual MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
virtual FastISel * createFastISel(FunctionLoweringInfo &, const TargetLibraryInfo *) const
This method returns a target specific FastISel object, or null if the target does not support "fast" ...
virtual bool supportSplitCSR(MachineFunction *MF) const
Return true if the target supports that a subset of CSRs for the given machine function is handled ex...
Primary interface to the complete machine description for the target machine.
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
const Triple & getTargetTriple() const
void setFastISel(bool Enable)
bool getO0WantsFastISel()
void resetTargetOptions(const Function &F) const
Reset the target options based on the function's attributes.
void setOptLevel(CodeGenOptLevel Level)
Overrides the optimization level.
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetLowering * getTargetLowering() const
Triple - Helper class for working with autoconf configuration names.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
LLVM Value Representation.
bool hasOneUse() const
Return true if there is exactly one use of this value.
iterator_range< user_iterator > users()
StringRef getName() const
Return a constant reference to the value's name.
self_iterator getIterator()
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const CustomOperand< const MCSubtargetInfo & > Msg[]
@ C
The default llvm calling convention, compatible with C.
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
@ MDNODE_SDNODE
MDNODE_SDNODE - This is a node that holdes an MDNode*, which is used to reference metadata in the IR.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ JUMP_TABLE_DEBUG_INFO
JUMP_TABLE_DEBUG_INFO - Jumptable debug info.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ MEMBARRIER
MEMBARRIER - Compiler barrier only; generate a no-op.
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
@ ANNOTATION_LABEL
ANNOTATION_LABEL - Represents a mid basic block label used by annotations.
@ UNDEF
UNDEF - An undefined node.
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
@ BasicBlock
Various leaf nodes.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ LIFETIME_START
This corresponds to the llvm.lifetime.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ HANDLENODE
HANDLENODE node - Used as a handle for various purposes.
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ PSEUDO_PROBE
Pseudo probe for AutoFDO, as a place holder in a basic block to improve the sample counts quality.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ INLINEASM
INLINEASM - Represents an inline asm block.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
StringRef getBaseName(ID id)
Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....
@ Kill
The last use of a register.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
ScheduleDAGSDNodes * createDefaultScheduler(SelectionDAGISel *IS, CodeGenOptLevel OptLevel)
createDefaultScheduler - This creates an instruction scheduler appropriate for the target.
bool succ_empty(const Instruction *I)
ScheduleDAGSDNodes * createBURRListDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel OptLevel)
createBURRListDAGScheduler - This creates a bottom up register usage reduction list scheduler.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
ScheduleDAGSDNodes * createHybridListDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel)
createHybridListDAGScheduler - This creates a bottom up register pressure aware list scheduler that m...
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
MachineBasicBlock::iterator findSplitPointForStackProtector(MachineBasicBlock *BB, const TargetInstrInfo &TII)
Find the split point at which to splice the end of BB into its success stack protector check machine ...
bool TimePassesIsEnabled
If the user specifies the -time-passes argument on an LLVM tool command line then the value of this b...
LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
void initializeBranchProbabilityInfoWrapperPassPass(PassRegistry &)
ScheduleDAGSDNodes * createFastDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel OptLevel)
createFastDAGScheduler - This creates a "fast" scheduler.
void erase(Container &C, ValueType V)
Wrapper function to remove a value from a container:
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
ScheduleDAGSDNodes * createDAGLinearizer(SelectionDAGISel *IS, CodeGenOptLevel OptLevel)
createDAGLinearizer - This creates a "no-scheduling" scheduler which linearize the DAG using topologi...
void initializeAAResultsWrapperPassPass(PassRegistry &)
void initializeGCModuleInfoPass(PassRegistry &)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isFunctionInPrintList(StringRef FunctionName)
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
CodeGenOptLevel
Code generation optimization level.
bool isFuncletEHPersonality(EHPersonality Pers)
Returns true if this is a personality function that invokes handler funclets (which must return to it...
ScheduleDAGSDNodes * createSourceListDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel OptLevel)
createSourceListDAGScheduler - This creates a bottom up list scheduler that schedules nodes in source...
bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
ScheduleDAGSDNodes * createILPListDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel)
createILPListDAGScheduler - This creates a bottom up register pressure aware list scheduler that trie...
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
void initializeTargetLibraryInfoWrapperPassPass(PassRegistry &)
ScheduleDAGSDNodes * createVLIWDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel OptLevel)
createVLIWDAGScheduler - Scheduler for VLIW targets.
static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)
Filter the DbgRecord range to DbgVariableRecord types only and downcast.
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Implement std::hash so that hash_code can be used in STL containers.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
This class is basically a combination of TimeRegion and Timer.
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Clients of various APIs that cause global effects on the DAG can optionally implement this interface.
void addIPToStateRange(const InvokeInst *II, MCSymbol *InvokeBegin, MCSymbol *InvokeEnd)
DenseMap< const BasicBlock *, int > BlockToStateMap