84 #define DEBUG_TYPE "irtranslator"
90 cl::desc(
"Should enable CSE in irtranslator"),
111 if (!R.getLocation().isValid() || TPC.isGlobalISelAbortEnabled())
112 R << (
" (in function: " + MF.getName() +
")").str();
114 if (TPC.isGlobalISelAbortEnabled())
131 DILocationVerifier() =
default;
132 ~DILocationVerifier() =
default;
134 const Instruction *getCurrentInst()
const {
return CurrInst; }
135 void setCurrentInst(
const Instruction *Inst) { CurrInst = Inst; }
142 assert(getCurrentInst() &&
"Inserted instruction without a current MI");
147 <<
" was copied to " <<
MI);
153 MI.getDebugLoc().getLine() == 0) &&
154 "Line info was not transferred to all instructions");
158 #endif // ifndef NDEBUG
172 IRTranslator::allocateVRegs(
const Value &Val) {
173 auto VRegsIt = VMap.findVRegs(Val);
174 if (VRegsIt != VMap.vregs_end())
175 return *VRegsIt->second;
176 auto *Regs = VMap.getVRegs(Val);
177 auto *
Offsets = VMap.getOffsets(Val);
181 for (
unsigned i = 0;
i < SplitTys.size(); ++
i)
187 auto VRegsIt = VMap.findVRegs(Val);
188 if (VRegsIt != VMap.vregs_end())
189 return *VRegsIt->second;
192 return *VMap.getVRegs(Val);
195 auto *VRegs = VMap.getVRegs(Val);
196 auto *
Offsets = VMap.getOffsets(Val);
199 "Don't know how to create an empty vreg");
205 if (!isa<Constant>(Val)) {
206 for (
auto Ty : SplitTys)
213 auto &
C = cast<Constant>(Val);
215 while (
auto Elt =
C.getAggregateElement(Idx++)) {
216 auto EltRegs = getOrCreateVRegs(*Elt);
217 llvm::copy(EltRegs, std::back_inserter(*VRegs));
220 assert(SplitTys.size() == 1 &&
"unexpectedly split LLT");
222 bool Success = translate(cast<Constant>(Val), VRegs->front());
227 R <<
"unable to translate constant: " <<
ore::NV(
"Type", Val.
getType());
236 int IRTranslator::getOrCreateFrameIndex(
const AllocaInst &AI) {
237 auto MapEntry = FrameIndices.find(&AI);
238 if (MapEntry != FrameIndices.end())
239 return MapEntry->second;
243 ElementSize * cast<ConstantInt>(AI.
getArraySize())->getZExtValue();
246 Size = std::max<uint64_t>(
Size, 1u);
248 int &FI = FrameIndices[&AI];
255 return SI->getAlign();
256 if (
const LoadInst *LI = dyn_cast<LoadInst>(&
I))
257 return LI->getAlign();
264 R <<
"unable to translate memop: " <<
ore::NV(
"Opcode", &
I);
271 assert(
MBB &&
"BasicBlock was not encountered before");
276 assert(NewPred &&
"new predecessor must be a real MachineBasicBlock");
277 MachinePreds[Edge].push_back(NewPred);
280 bool IRTranslator::translateBinaryOp(
unsigned Opcode,
const User &U,
290 if (isa<Instruction>(U)) {
295 MIRBuilder.
buildInstr(Opcode, {Res}, {Op0, Op1}, Flags);
299 bool IRTranslator::translateUnaryOp(
unsigned Opcode,
const User &U,
304 if (isa<Instruction>(U)) {
308 MIRBuilder.
buildInstr(Opcode, {Res}, {Op0}, Flags);
313 return translateUnaryOp(TargetOpcode::G_FNEG, U, MIRBuilder);
316 bool IRTranslator::translateCompare(
const User &U,
318 auto *CI = dyn_cast<CmpInst>(&U);
324 cast<ConstantExpr>(U).getPredicate());
326 MIRBuilder.
buildICmp(Pred, Res, Op0, Op1);
334 assert(CI &&
"Instruction should be CmpInst");
335 MIRBuilder.
buildFCmp(Pred, Res, Op0, Op1,
350 VRegs = getOrCreateVRegs(*
Ret);
361 return CLI->
lowerReturn(MIRBuilder,
Ret, VRegs, FuncInfo, SwiftErrorVReg);
364 void IRTranslator::emitBranchForMergedCondition(
370 if (
const CmpInst *BOp = dyn_cast<CmpInst>(
Cond)) {
373 Condition = InvertCond ? IC->getInversePredicate() : IC->getPredicate();
376 Condition = InvertCond ?
FC->getInversePredicate() :
FC->getPredicate();
380 BOp->getOperand(1),
nullptr, TBB, FBB, CurBB,
381 CurBuilder->getDebugLoc(), TProb, FProb);
382 SL->SwitchCases.push_back(CB);
390 nullptr, TBB, FBB, CurBB, CurBuilder->getDebugLoc(), TProb, FProb);
391 SL->SwitchCases.push_back(CB);
396 return I->getParent() ==
BB;
400 void IRTranslator::findMergedConditions(
405 using namespace PatternMatch;
406 assert((Opc == Instruction::And || Opc == Instruction::Or) &&
407 "Expected Opc to be AND/OR");
413 findMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
419 const Value *BOpOp0, *BOpOp1;
433 if (BOpc == Instruction::And)
434 BOpc = Instruction::Or;
435 else if (BOpc == Instruction::Or)
436 BOpc = Instruction::And;
442 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->
hasOneUse();
446 emitBranchForMergedCondition(
Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb,
457 if (Opc == Instruction::Or) {
478 auto NewTrueProb = TProb / 2;
479 auto NewFalseProb = TProb / 2 + FProb;
481 findMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
482 NewFalseProb, InvertCond);
488 findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
489 Probs[1], InvertCond);
491 assert(Opc == Instruction::And &&
"Unknown merge op!");
511 auto NewTrueProb = TProb + FProb / 2;
512 auto NewFalseProb = FProb / 2;
514 findMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
515 NewFalseProb, InvertCond);
521 findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
522 Probs[1], InvertCond);
526 bool IRTranslator::shouldEmitAsBranches(
527 const std::vector<SwitchCG::CaseBlock> &Cases) {
529 if (Cases.size() != 2)
534 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
535 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
536 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
537 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
543 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
544 Cases[0].PredInfo.Pred == Cases[1].PredInfo.Pred &&
545 isa<Constant>(Cases[0].CmpRHS) &&
546 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
548 Cases[0].TrueBB == Cases[1].ThisBB)
551 Cases[0].FalseBB == Cases[1].ThisBB)
559 const BranchInst &BrInst = cast<BranchInst>(U);
560 auto &CurMBB = MIRBuilder.
getMBB();
565 if (!CurMBB.isLayoutSuccessor(Succ0MBB))
570 CurMBB.addSuccessor(&getMBB(*Succ));
598 using namespace PatternMatch;
599 const Instruction *CondI = dyn_cast<Instruction>(CondVal);
600 if (!TLI.isJumpExpensive() && CondI && CondI->
hasOneUse() &&
601 !BrInst.
hasMetadata(LLVMContext::MD_unpredictable)) {
604 const Value *BOp0, *BOp1;
606 Opcode = Instruction::And;
608 Opcode = Instruction::Or;
612 findMergedConditions(CondI, Succ0MBB, Succ1MBB, &CurMBB, &CurMBB, Opcode,
613 getEdgeProbability(&CurMBB, Succ0MBB),
614 getEdgeProbability(&CurMBB, Succ1MBB),
616 assert(SL->SwitchCases[0].ThisBB == &CurMBB &&
"Unexpected lowering!");
619 if (shouldEmitAsBranches(SL->SwitchCases)) {
621 emitSwitchCase(SL->SwitchCases[0], &CurMBB, *CurBuilder);
622 SL->SwitchCases.erase(SL->SwitchCases.begin());
628 for (
unsigned I = 1,
E = SL->SwitchCases.size();
I !=
E; ++
I)
629 MF->erase(SL->SwitchCases[
I].ThisBB);
631 SL->SwitchCases.clear();
638 nullptr, Succ0MBB, Succ1MBB, &CurMBB,
639 CurBuilder->getDebugLoc());
643 emitSwitchCase(CB, &CurMBB, *CurBuilder);
655 Prob = getEdgeProbability(Src, Dst);
667 auto SuccSize = std::max<uint32_t>(
succ_size(SrcBB), 1);
674 using namespace SwitchCG;
679 Clusters.reserve(
SI.getNumCases());
680 for (
auto &
I :
SI.cases()) {
682 assert(Succ &&
"Could not find successor mbb in mapping");
687 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
700 if (Clusters.empty()) {
707 SL->findJumpTables(Clusters, &
SI, DefaultMBB,
nullptr,
nullptr);
708 SL->findBitTestClusters(Clusters, &
SI);
711 dbgs() <<
"Case clusters: ";
712 for (
const CaseCluster &
C : Clusters) {
718 C.Low->getValue().print(
dbgs(),
true);
719 if (
C.Low !=
C.High) {
721 C.High->getValue().print(
dbgs(),
true);
728 assert(!Clusters.empty());
732 auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB);
733 WorkList.push_back({SwitchMBB,
First,
Last,
nullptr,
nullptr, DefaultProb});
737 while (!WorkList.empty()) {
738 SwitchWorkListItem
W = WorkList.back();
740 if (!lowerSwitchWorkItem(
W,
SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
749 assert(
JT.Reg != -1U &&
"Should lower JT Header first!");
771 Register SwitchOpReg = getOrCreateVReg(SValue);
773 auto Sub = MIB.
buildSub({SwitchTy}, SwitchOpReg, FirstCst);
781 JT.Reg = Sub.getReg(0);
792 auto Cst = getOrCreateVReg(
829 const auto *CI = dyn_cast<ConstantInt>(CB.
CmpRHS);
847 "Can only handle SLE ranges");
849 const APInt& Low = cast<ConstantInt>(CB.
CmpLHS)->getValue();
853 if (cast<ConstantInt>(CB.
CmpLHS)->isMinValue(
true)) {
859 auto Sub = MIB.
buildSub({CmpTy}, CmpOpReg, CondLHS);
894 bool FallthroughUnreachable) {
895 using namespace SwitchCG;
898 JumpTableHeader *JTH = &SL->JTCases[
I->JTCasesIndex].first;
904 CurMF->
insert(BBI, JumpMBB);
914 auto JumpProb =
I->Prob;
915 auto FallthroughProb = UnhandledProbs;
923 if (*
SI == DefaultMBB) {
924 JumpProb += DefaultProb / 2;
925 FallthroughProb -= DefaultProb / 2;
930 addMachineCFGPred({SwitchMBB->
getBasicBlock(), (*SI)->getBasicBlock()},
936 if (FallthroughUnreachable)
937 JTH->OmitRangeCheck =
true;
939 if (!JTH->OmitRangeCheck)
940 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
941 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
946 JTH->HeaderBB = CurMBB;
947 JT->Default = Fallthrough;
950 if (CurMBB == SwitchMBB) {
951 if (!emitJumpTableHeader(*
JT, *JTH, CurMBB))
960 bool FallthroughUnreachable,
965 using namespace SwitchCG;
966 const Value *RHS, *LHS, *MHS;
968 if (
I->Low ==
I->High) {
984 CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS,
I->MBB, Fallthrough,
987 emitSwitchCase(CB, SwitchMBB, MIB);
997 Register SwitchOpReg = getOrCreateVReg(*
B.SValue);
1001 auto RangeSub = MIB.
buildSub(SwitchOpTy, SwitchOpReg, MinValReg);
1004 LLT MaskTy = SwitchOpTy;
1005 for (
unsigned I = 0,
E =
B.Cases.size();
I !=
E; ++
I) {
1014 if (SwitchOpTy != MaskTy)
1022 if (!
B.OmitRangeCheck)
1023 addSuccessorWithProb(SwitchBB,
B.Default,
B.DefaultProb);
1024 addSuccessorWithProb(SwitchBB,
MBB,
B.Prob);
1028 if (!
B.OmitRangeCheck) {
1032 RangeSub, RangeCst);
1052 if (PopCount == 1) {
1055 auto MaskTrailingZeros =
1060 }
else if (PopCount ==
BB.Range) {
1062 auto MaskTrailingOnes =
1069 auto SwitchVal = MIB.
buildShl(SwitchTy, CstOne,
Reg);
1073 auto AndOp = MIB.
buildAnd(SwitchTy, SwitchVal, CstMask);
1080 addSuccessorWithProb(SwitchBB,
B.TargetBB,
B.ExtraProb);
1082 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
1090 addMachineCFGPred({
BB.Parent->getBasicBlock(),
B.TargetBB->getBasicBlock()},
1100 bool IRTranslator::lowerBitTestWorkItem(
1106 bool FallthroughUnreachable) {
1107 using namespace SwitchCG;
1110 BitTestBlock *BTB = &SL->BitTestCases[
I->BTCasesIndex];
1112 for (BitTestCase &BTC : BTB->Cases)
1113 CurMF->
insert(BBI, BTC.ThisBB);
1116 BTB->Parent = CurMBB;
1117 BTB->Default = Fallthrough;
1119 BTB->DefaultProb = UnhandledProbs;
1123 if (!BTB->ContiguousRange) {
1124 BTB->Prob += DefaultProb / 2;
1125 BTB->DefaultProb -= DefaultProb / 2;
1128 if (FallthroughUnreachable) {
1130 BTB->OmitRangeCheck =
true;
1134 if (CurMBB == SwitchMBB) {
1135 emitBitTestHeader(*BTB, SwitchMBB);
1136 BTB->Emitted =
true;
1146 using namespace SwitchCG;
1150 if (++BBI != FuncInfo.MF->end())
1159 [](
const CaseCluster &
a,
const CaseCluster &
b) {
1160 return a.Prob != b.Prob
1162 : a.Low->getValue().slt(b.Low->getValue());
1169 if (
I->Prob >
W.LastCluster->Prob)
1171 if (
I->Kind ==
CC_Range &&
I->MBB == NextMBB) {
1182 UnhandledProbs +=
I->Prob;
1186 bool FallthroughUnreachable =
false;
1188 if (
I ==
W.LastCluster) {
1190 Fallthrough = DefaultMBB;
1191 FallthroughUnreachable = isa<UnreachableInst>(
1195 CurMF->
insert(BBI, Fallthrough);
1197 UnhandledProbs -=
I->Prob;
1201 if (!lowerBitTestWorkItem(
W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1202 DefaultProb, UnhandledProbs,
I, Fallthrough,
1203 FallthroughUnreachable)) {
1211 if (!lowerJumpTableWorkItem(
W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1212 UnhandledProbs,
I, Fallthrough,
1213 FallthroughUnreachable)) {
1220 if (!lowerSwitchRangeWorkItem(
I,
Cond, Fallthrough,
1221 FallthroughUnreachable, UnhandledProbs,
1222 CurMBB, MIB, SwitchMBB)) {
1229 CurMBB = Fallthrough;
1235 bool IRTranslator::translateIndirectBr(
const User &U,
1249 if (!AddedSuccessors.
insert(Succ).second)
1258 if (
auto Arg = dyn_cast<Argument>(V))
1259 return Arg->hasSwiftErrorAttr();
1260 if (
auto AI = dyn_cast<AllocaInst>(V))
1266 const LoadInst &LI = cast<LoadInst>(U);
1278 assert(Regs.
size() == 1 &&
"swifterror should be single pointer");
1290 for (
unsigned i = 0;
i < Regs.
size(); ++
i) {
1295 Align BaseAlign = getMemOpAlign(LI);
1315 Register Base = getOrCreateVReg(*
SI.getPointerOperand());
1321 assert(Vals.
size() == 1 &&
"swifterror should be single pointer");
1324 SI.getPointerOperand());
1332 for (
unsigned i = 0;
i < Vals.
size(); ++
i) {
1337 Align BaseAlign = getMemOpAlign(
SI);
1339 SI.getAAMetadata(AAMetadata);
1343 SI.getSyncScopeID(),
SI.getOrdering());
1359 for (
auto Idx : EVI->indices())
1361 }
else if (
const InsertValueInst *IVI = dyn_cast<InsertValueInst>(&U)) {
1362 for (
auto Idx : IVI->indices())
1369 return 8 *
static_cast<uint64_t
>(
1370 DL.getIndexedOffsetInType(Src->
getType(), Indices));
1373 bool IRTranslator::translateExtractValue(
const User &U,
1380 auto &DstRegs = allocateVRegs(U);
1382 for (
unsigned i = 0;
i < DstRegs.size(); ++
i)
1383 DstRegs[
i] = SrcRegs[Idx++];
1388 bool IRTranslator::translateInsertValue(
const User &U,
1392 auto &DstRegs = allocateVRegs(U);
1396 auto InsertedIt = InsertedRegs.
begin();
1398 for (
unsigned i = 0;
i < DstRegs.size(); ++
i) {
1399 if (DstOffsets[
i] >=
Offset && InsertedIt != InsertedRegs.
end())
1400 DstRegs[
i] = *InsertedIt++;
1402 DstRegs[
i] = SrcRegs[
i];
1408 bool IRTranslator::translateSelect(
const User &U,
1419 for (
unsigned i = 0;
i < ResRegs.
size(); ++
i) {
1420 MIRBuilder.
buildSelect(ResRegs[
i], Tst, Op0Regs[
i], Op1Regs[
i], Flags);
1426 bool IRTranslator::translateCopy(
const User &U,
const Value &V,
1429 auto &Regs = *VMap.getVRegs(U);
1431 Regs.push_back(Src);
1432 VMap.getOffsets(U)->push_back(0);
1441 bool IRTranslator::translateBitCast(
const User &U,
1446 return translateCopy(U, *U.
getOperand(0), MIRBuilder);
1448 return translateCast(TargetOpcode::G_BITCAST, U, MIRBuilder);
1451 bool IRTranslator::translateCast(
unsigned Opcode,
const User &U,
1459 bool IRTranslator::translateGetElementPtr(
const User &U,
1462 Register BaseReg = getOrCreateVReg(Op0);
1470 unsigned VectorWidth = 0;
1471 if (
auto *VT = dyn_cast<VectorType>(U.
getType()))
1472 VectorWidth = cast<FixedVectorType>(VT)->getNumElements();
1476 if (VectorWidth && !PtrTy.
isVector()) {
1489 const Value *Idx = GTI.getOperand();
1490 if (
StructType *StTy = GTI.getStructTypeOrNull()) {
1491 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
1499 if (
const auto *CI = dyn_cast<ConstantInt>(Idx)) {
1500 Offset += ElementSize * CI->getSExtValue();
1506 BaseReg = MIRBuilder.
buildPtrAdd(PtrTy, BaseReg, OffsetMIB.getReg(0))
1511 Register IdxReg = getOrCreateVReg(*Idx);
1513 if (IdxTy != OffsetTy) {
1514 if (!IdxTy.
isVector() && VectorWidth) {
1525 if (ElementSize != 1) {
1531 GepOffsetReg = IdxReg;
1540 MIRBuilder.
buildPtrAdd(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0));
1544 MIRBuilder.
buildCopy(getOrCreateVReg(U), BaseReg);
1548 bool IRTranslator::translateMemFunc(
const CallInst &CI,
1558 unsigned MinPtrSize = UINT_MAX;
1559 for (
auto AI = CI.
arg_begin(), AE = CI.
arg_end(); std::next(AI) != AE; ++AI) {
1560 Register SrcReg = getOrCreateVReg(**AI);
1564 SrcRegs.push_back(SrcReg);
1570 Register &SizeOpReg = SrcRegs[SrcRegs.size() - 1];
1571 if (MRI->
getType(SizeOpReg) != SizeTy)
1584 if (
auto *MCI = dyn_cast<MemCpyInst>(&CI)) {
1585 DstAlign = MCI->getDestAlign().valueOrOne();
1586 SrcAlign = MCI->getSourceAlign().valueOrOne();
1587 }
else if (
auto *MMI = dyn_cast<MemMoveInst>(&CI)) {
1588 DstAlign = MMI->getDestAlign().valueOrOne();
1589 SrcAlign = MMI->getSourceAlign().valueOrOne();
1591 auto *MSI = cast<MemSetInst>(&CI);
1592 DstAlign = MSI->getDestAlign().valueOrOne();
1605 if (Opcode != TargetOpcode::G_MEMSET)
1613 void IRTranslator::getStackGuard(
Register DstReg,
1618 MIRBuilder.
buildInstr(TargetOpcode::LOAD_STACK_GUARD, {DstReg}, {});
1631 MIB.setMemRefs({
MemRef});
1634 bool IRTranslator::translateOverflowIntrinsic(
const CallInst &CI,
unsigned Op,
1638 Op, {ResRegs[0], ResRegs[1]},
1644 bool IRTranslator::translateFixedPointIntrinsic(
unsigned Op,
const CallInst &CI,
1646 Register Dst = getOrCreateVReg(CI);
1649 uint64_t Scale = cast<ConstantInt>(CI.
getOperand(2))->getZExtValue();
1650 MIRBuilder.
buildInstr(
Op, {Dst}, { Src0, Src1, Scale });
1658 case Intrinsic::bswap:
1659 return TargetOpcode::G_BSWAP;
1660 case Intrinsic::bitreverse:
1661 return TargetOpcode::G_BITREVERSE;
1662 case Intrinsic::fshl:
1663 return TargetOpcode::G_FSHL;
1664 case Intrinsic::fshr:
1665 return TargetOpcode::G_FSHR;
1667 return TargetOpcode::G_FCEIL;
1668 case Intrinsic::cos:
1669 return TargetOpcode::G_FCOS;
1670 case Intrinsic::ctpop:
1671 return TargetOpcode::G_CTPOP;
1672 case Intrinsic::exp:
1673 return TargetOpcode::G_FEXP;
1674 case Intrinsic::exp2:
1675 return TargetOpcode::G_FEXP2;
1676 case Intrinsic::fabs:
1677 return TargetOpcode::G_FABS;
1678 case Intrinsic::copysign:
1679 return TargetOpcode::G_FCOPYSIGN;
1681 return TargetOpcode::G_FMINNUM;
1683 return TargetOpcode::G_FMAXNUM;
1685 return TargetOpcode::G_FMINIMUM;
1687 return TargetOpcode::G_FMAXIMUM;
1688 case Intrinsic::canonicalize:
1689 return TargetOpcode::G_FCANONICALIZE;
1691 return TargetOpcode::G_FFLOOR;
1692 case Intrinsic::fma:
1693 return TargetOpcode::G_FMA;
1694 case Intrinsic::log:
1695 return TargetOpcode::G_FLOG;
1697 return TargetOpcode::G_FLOG2;
1698 case Intrinsic::log10:
1699 return TargetOpcode::G_FLOG10;
1700 case Intrinsic::nearbyint:
1701 return TargetOpcode::G_FNEARBYINT;
1702 case Intrinsic::pow:
1703 return TargetOpcode::G_FPOW;
1705 return TargetOpcode::G_FPOWI;
1706 case Intrinsic::rint:
1707 return TargetOpcode::G_FRINT;
1709 return TargetOpcode::G_INTRINSIC_ROUND;
1710 case Intrinsic::roundeven:
1711 return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
1712 case Intrinsic::sin:
1713 return TargetOpcode::G_FSIN;
1714 case Intrinsic::sqrt:
1715 return TargetOpcode::G_FSQRT;
1717 return TargetOpcode::G_INTRINSIC_TRUNC;
1718 case Intrinsic::readcyclecounter:
1719 return TargetOpcode::G_READCYCLECOUNTER;
1720 case Intrinsic::ptrmask:
1721 return TargetOpcode::G_PTRMASK;
1722 case Intrinsic::lrint:
1723 return TargetOpcode::G_INTRINSIC_LRINT;
1725 case Intrinsic::vector_reduce_fmin:
1726 return TargetOpcode::G_VECREDUCE_FMIN;
1727 case Intrinsic::vector_reduce_fmax:
1728 return TargetOpcode::G_VECREDUCE_FMAX;
1729 case Intrinsic::vector_reduce_add:
1730 return TargetOpcode::G_VECREDUCE_ADD;
1731 case Intrinsic::vector_reduce_mul:
1732 return TargetOpcode::G_VECREDUCE_MUL;
1733 case Intrinsic::vector_reduce_and:
1734 return TargetOpcode::G_VECREDUCE_AND;
1735 case Intrinsic::vector_reduce_or:
1736 return TargetOpcode::G_VECREDUCE_OR;
1737 case Intrinsic::vector_reduce_xor:
1738 return TargetOpcode::G_VECREDUCE_XOR;
1739 case Intrinsic::vector_reduce_smax:
1740 return TargetOpcode::G_VECREDUCE_SMAX;
1741 case Intrinsic::vector_reduce_smin:
1742 return TargetOpcode::G_VECREDUCE_SMIN;
1743 case Intrinsic::vector_reduce_umax:
1744 return TargetOpcode::G_VECREDUCE_UMAX;
1745 case Intrinsic::vector_reduce_umin:
1746 return TargetOpcode::G_VECREDUCE_UMIN;
1751 bool IRTranslator::translateSimpleIntrinsic(
const CallInst &CI,
1755 unsigned Op = getSimpleIntrinsicOpcode(
ID);
1764 VRegs.push_back(getOrCreateVReg(*
Arg));
1766 MIRBuilder.
buildInstr(
Op, {getOrCreateVReg(CI)}, VRegs,
1774 case Intrinsic::experimental_constrained_fadd:
1775 return TargetOpcode::G_STRICT_FADD;
1776 case Intrinsic::experimental_constrained_fsub:
1777 return TargetOpcode::G_STRICT_FSUB;
1778 case Intrinsic::experimental_constrained_fmul:
1779 return TargetOpcode::G_STRICT_FMUL;
1780 case Intrinsic::experimental_constrained_fdiv:
1781 return TargetOpcode::G_STRICT_FDIV;
1782 case Intrinsic::experimental_constrained_frem:
1783 return TargetOpcode::G_STRICT_FREM;
1784 case Intrinsic::experimental_constrained_fma:
1785 return TargetOpcode::G_STRICT_FMA;
1786 case Intrinsic::experimental_constrained_sqrt:
1787 return TargetOpcode::G_STRICT_FSQRT;
1793 bool IRTranslator::translateConstrainedFPIntrinsic(
1812 MIRBuilder.
buildInstr(Opcode, {getOrCreateVReg(FPI)}, VRegs, Flags);
1821 if (translateSimpleIntrinsic(CI,
ID, MIRBuilder))
1827 case Intrinsic::lifetime_start:
1828 case Intrinsic::lifetime_end: {
1843 for (
const Value *V : Allocas) {
1844 const AllocaInst *AI = dyn_cast<AllocaInst>(V);
1855 case Intrinsic::dbg_declare: {
1860 if (!Address || isa<UndefValue>(Address)) {
1867 "Expected inlined-at fields to agree");
1868 auto AI = dyn_cast<AllocaInst>(Address);
1882 case Intrinsic::dbg_label: {
1888 "Expected inlined-at fields to agree");
1893 case Intrinsic::vaend:
1897 case Intrinsic::vastart: {
1900 unsigned ListSize = TLI.getVaListSizeInBits(*DL) / 8;
1903 MIRBuilder.
buildInstr(TargetOpcode::G_VASTART, {}, {getOrCreateVReg(*Ptr)})
1906 ListSize,
Align(1)));
1909 case Intrinsic::dbg_value: {
1915 "Expected inlined-at fields to agree");
1920 }
else if (
const auto *CI = dyn_cast<Constant>(V)) {
1933 case Intrinsic::uadd_with_overflow:
1934 return translateOverflowIntrinsic(CI, TargetOpcode::G_UADDO, MIRBuilder);
1935 case Intrinsic::sadd_with_overflow:
1936 return translateOverflowIntrinsic(CI, TargetOpcode::G_SADDO, MIRBuilder);
1937 case Intrinsic::usub_with_overflow:
1938 return translateOverflowIntrinsic(CI, TargetOpcode::G_USUBO, MIRBuilder);
1939 case Intrinsic::ssub_with_overflow:
1940 return translateOverflowIntrinsic(CI, TargetOpcode::G_SSUBO, MIRBuilder);
1941 case Intrinsic::umul_with_overflow:
1942 return translateOverflowIntrinsic(CI, TargetOpcode::G_UMULO, MIRBuilder);
1943 case Intrinsic::smul_with_overflow:
1944 return translateOverflowIntrinsic(CI, TargetOpcode::G_SMULO, MIRBuilder);
1945 case Intrinsic::uadd_sat:
1946 return translateBinaryOp(TargetOpcode::G_UADDSAT, CI, MIRBuilder);
1947 case Intrinsic::sadd_sat:
1948 return translateBinaryOp(TargetOpcode::G_SADDSAT, CI, MIRBuilder);
1949 case Intrinsic::usub_sat:
1950 return translateBinaryOp(TargetOpcode::G_USUBSAT, CI, MIRBuilder);
1951 case Intrinsic::ssub_sat:
1952 return translateBinaryOp(TargetOpcode::G_SSUBSAT, CI, MIRBuilder);
1953 case Intrinsic::ushl_sat:
1954 return translateBinaryOp(TargetOpcode::G_USHLSAT, CI, MIRBuilder);
1955 case Intrinsic::sshl_sat:
1956 return translateBinaryOp(TargetOpcode::G_SSHLSAT, CI, MIRBuilder);
1958 return translateBinaryOp(TargetOpcode::G_UMIN, CI, MIRBuilder);
1960 return translateBinaryOp(TargetOpcode::G_UMAX, CI, MIRBuilder);
1962 return translateBinaryOp(TargetOpcode::G_SMIN, CI, MIRBuilder);
1964 return translateBinaryOp(TargetOpcode::G_SMAX, CI, MIRBuilder);
1967 return translateUnaryOp(TargetOpcode::G_ABS, CI, MIRBuilder);
1968 case Intrinsic::smul_fix:
1969 return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIX, CI, MIRBuilder);
1970 case Intrinsic::umul_fix:
1971 return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIX, CI, MIRBuilder);
1972 case Intrinsic::smul_fix_sat:
1973 return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
1974 case Intrinsic::umul_fix_sat:
1975 return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
1976 case Intrinsic::sdiv_fix:
1977 return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
1978 case Intrinsic::udiv_fix:
1979 return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
1980 case Intrinsic::sdiv_fix_sat:
1981 return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
1982 case Intrinsic::udiv_fix_sat:
1983 return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
1984 case Intrinsic::fmuladd: {
1987 Register Dst = getOrCreateVReg(CI);
1996 MIRBuilder.
buildFMA(Dst, Op0, Op1, Op2,
2007 case Intrinsic::convert_from_fp16:
2013 case Intrinsic::convert_to_fp16:
2020 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY);
2021 case Intrinsic::memmove:
2022 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMMOVE);
2023 case Intrinsic::memset:
2024 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMSET);
2025 case Intrinsic::eh_typeid_for: {
2032 case Intrinsic::objectsize:
2035 case Intrinsic::is_constant:
2038 case Intrinsic::stackguard:
2039 getStackGuard(getOrCreateVReg(CI), MIRBuilder);
2041 case Intrinsic::stackprotector: {
2044 getStackGuard(GuardVal, MIRBuilder);
2047 int FI = getOrCreateFrameIndex(*Slot);
2051 GuardVal, getOrCreateVReg(*Slot),
2058 case Intrinsic::stacksave: {
2072 case Intrinsic::stackrestore: {
2086 case Intrinsic::cttz:
2087 case Intrinsic::ctlz: {
2089 bool isTrailing =
ID == Intrinsic::cttz;
2090 unsigned Opcode = isTrailing
2091 ? Cst->
isZero() ? TargetOpcode::G_CTTZ
2092 : TargetOpcode::G_CTTZ_ZERO_UNDEF
2093 : Cst->
isZero() ? TargetOpcode::G_CTLZ
2094 : TargetOpcode::G_CTLZ_ZERO_UNDEF;
2095 MIRBuilder.
buildInstr(Opcode, {getOrCreateVReg(CI)},
2099 case Intrinsic::invariant_start: {
2105 case Intrinsic::invariant_end:
2107 case Intrinsic::expect:
2108 case Intrinsic::annotation:
2109 case Intrinsic::ptr_annotation:
2110 case Intrinsic::launder_invariant_group:
2111 case Intrinsic::strip_invariant_group: {
2113 MIRBuilder.
buildCopy(getOrCreateVReg(CI),
2117 case Intrinsic::assume:
2118 case Intrinsic::experimental_noalias_scope_decl:
2119 case Intrinsic::var_annotation:
2120 case Intrinsic::sideeffect:
2123 case Intrinsic::read_volatile_register:
2124 case Intrinsic::read_register: {
2127 .
buildInstr(TargetOpcode::G_READ_REGISTER, {getOrCreateVReg(CI)}, {})
2128 .addMetadata(cast<MDNode>(cast<MetadataAsValue>(
Arg)->getMetadata()));
2131 case Intrinsic::write_register: {
2133 MIRBuilder.
buildInstr(TargetOpcode::G_WRITE_REGISTER)
2134 .
addMetadata(cast<MDNode>(cast<MetadataAsValue>(
Arg)->getMetadata()))
2138 case Intrinsic::localescape: {
2146 if (isa<ConstantPointerNull>(
Arg))
2149 int FI = getOrCreateFrameIndex(*cast<AllocaInst>(
Arg));
2165 case Intrinsic::vector_reduce_fadd:
2166 case Intrinsic::vector_reduce_fmul: {
2169 Register Dst = getOrCreateVReg(CI);
2175 Opc =
ID == Intrinsic::vector_reduce_fadd
2176 ? TargetOpcode::G_VECREDUCE_SEQ_FADD
2177 : TargetOpcode::G_VECREDUCE_SEQ_FMUL;
2178 MIRBuilder.
buildInstr(Opc, {Dst}, {ScalarSrc, VecSrc},
2185 if (
ID == Intrinsic::vector_reduce_fadd) {
2186 Opc = TargetOpcode::G_VECREDUCE_FADD;
2187 ScalarOpc = TargetOpcode::G_FADD;
2189 Opc = TargetOpcode::G_VECREDUCE_FMUL;
2190 ScalarOpc = TargetOpcode::G_FMUL;
2195 MIRBuilder.
buildInstr(ScalarOpc, {Dst}, {ScalarSrc, Rdx},
2200 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
2201 case Intrinsic::INTRINSIC:
2202 #include "llvm/IR/ConstrainedOps.def"
2203 return translateConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(CI),
2210 bool IRTranslator::translateInlineAsm(
const CallBase &CB,
2217 dbgs() <<
"Inline asm lowering is not supported for this target yet\n");
2222 MIRBuilder, CB, [&](
const Value &Val) {
return getOrCreateVRegs(Val); });
2225 bool IRTranslator::translateCallBase(
const CallBase &CB,
2234 assert(SwiftInVReg == 0 &&
"Expected only one swift error argument");
2244 Args.push_back(getOrCreateVRegs(*
Arg));
2256 assert(!HasTailCall &&
"Can't tail call return twice from block?");
2265 const CallInst &CI = cast<CallInst>(U);
2270 if (
F && (
F->hasDLLImportStorageClass() ||
2272 F->hasExternalWeakLinkage())))
2280 return translateInlineAsm(CI, MIRBuilder);
2283 if (
F &&
F->isIntrinsic()) {
2284 ID =
F->getIntrinsicID();
2290 return translateCallBase(CI, MIRBuilder);
2294 if (translateKnownIntrinsic(CI,
ID, MIRBuilder))
2299 ResultRegs = getOrCreateVRegs(CI);
2305 if (isa<FPMathOperator>(CI))
2315 assert(CI->getBitWidth() <= 64 &&
2316 "large intrinsic immediates not handled");
2317 MIB.
addImm(CI->getSExtValue());
2321 }
else if (
auto MD = dyn_cast<MetadataAsValue>(
Arg.value())) {
2322 auto *MDN = dyn_cast<MDNode>(MD->getMetadata());
2328 if (VRegs.
size() > 1)
2336 TargetLowering::IntrinsicInfo
Info;
2339 Align Alignment =
Info.align.getValueOr(
2342 uint64_t
Size =
Info.memVT.getStoreSize();
2350 bool IRTranslator::findUnwindDestinations(
2370 if (isa<LandingPadInst>(Pad)) {
2372 UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
2375 if (isa<CleanupPadInst>(Pad)) {
2378 UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
2379 UnwindDests.
back().first->setIsEHScopeEntry();
2380 UnwindDests.back().first->setIsEHFuncletEntry();
2383 if (
auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2385 for (
const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2386 UnwindDests.emplace_back(&getMBB(*CatchPadBB), Prob);
2388 if (IsMSVCCXX || IsCoreCLR)
2389 UnwindDests.back().first->setIsEHFuncletEntry();
2391 UnwindDests.back().first->setIsEHScopeEntry();
2393 NewEHPadBB = CatchSwitch->getUnwindDest();
2399 if (BPI && NewEHPadBB)
2401 EHPadBB = NewEHPadBB;
2406 bool IRTranslator::translateInvoke(
const User &U,
2414 const Function *Fn =
I.getCalledFunction();
2415 if (
I.isInlineAsm())
2439 if (!translateCallBase(
I, MIRBuilder))
2452 if (!findUnwindDestinations(EHPadBB, EHPadBBProb, UnwindDests))
2456 &ReturnMBB = getMBB(*ReturnBB);
2458 addSuccessorWithProb(InvokeMBB, &ReturnMBB);
2459 for (
auto &UnwindDest : UnwindDests) {
2460 UnwindDest.first->setIsEHPad();
2461 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2465 MF->
addInvoke(&EHPadMBB, BeginSymbol, EndSymbol);
2466 MIRBuilder.
buildBr(ReturnMBB);
2470 bool IRTranslator::translateCallBr(
const User &U,
2476 bool IRTranslator::translateLandingPad(
const User &U,
2515 for (
Type *Ty : cast<StructType>(LP.
getType())->elements())
2517 assert(Tys.size() == 2 &&
"Only two-valued landingpads are supported");
2526 MIRBuilder.
buildCopy(ResRegs[0], ExceptionReg);
2534 MIRBuilder.
buildCopy(PtrVReg, SelectorReg);
2535 MIRBuilder.
buildCast(ResRegs[1], PtrVReg);
2540 bool IRTranslator::translateAlloca(
const User &U,
2542 auto &AI = cast<AllocaInst>(U);
2548 Register Res = getOrCreateVReg(AI);
2549 int FI = getOrCreateFrameIndex(AI);
2562 if (MRI->
getType(NumElts) != IntPtrTy) {
2573 MIRBuilder.
buildMul(AllocSize, NumElts, TySize);
2580 auto AllocAdd = MIRBuilder.
buildAdd(IntPtrTy, AllocSize, SAMinusOne,
2584 auto AlignedAlloc = MIRBuilder.
buildAnd(IntPtrTy, AllocAdd, AlignCst);
2588 Alignment =
Align(1);
2601 MIRBuilder.
buildInstr(TargetOpcode::G_VAARG, {getOrCreateVReg(U)},
2607 bool IRTranslator::translateInsertElement(
const User &U,
2611 if (cast<FixedVectorType>(U.
getType())->getNumElements() == 1)
2612 return translateCopy(U, *U.
getOperand(1), MIRBuilder);
2622 bool IRTranslator::translateExtractElement(
const User &U,
2627 return translateCopy(U, *U.
getOperand(0), MIRBuilder);
2634 if (
auto *CI = dyn_cast<ConstantInt>(U.
getOperand(1))) {
2635 if (CI->getBitWidth() != PreferredVecIdxWidth) {
2636 APInt NewIdx = CI->getValue().sextOrTrunc(PreferredVecIdxWidth);
2638 Idx = getOrCreateVReg(*NewIdxCI);
2651 bool IRTranslator::translateShuffleVector(
const User &U,
2654 if (
auto *SVI = dyn_cast<ShuffleVectorInst>(&U))
2655 Mask = SVI->getShuffleMask();
2657 Mask = cast<ConstantExpr>(U).getShuffleMask();
2660 .
buildInstr(TargetOpcode::G_SHUFFLE_VECTOR, {getOrCreateVReg(U)},
2663 .addShuffleMask(MaskAlloc);
2668 const PHINode &PI = cast<PHINode>(U);
2671 for (
auto Reg : getOrCreateVRegs(PI)) {
2672 auto MIB = MIRBuilder.
buildInstr(TargetOpcode::G_PHI, {
Reg}, {});
2676 PendingPHIs.emplace_back(&PI,
std::move(Insts));
2680 bool IRTranslator::translateAtomicCmpXchg(
const User &U,
2687 Type *ResType =
I.getType();
2688 Type *
ValType = ResType->Type::getStructElementType(0);
2690 auto Res = getOrCreateVRegs(
I);
2694 Register Cmp = getOrCreateVReg(*
I.getCompareOperand());
2695 Register NewVal = getOrCreateVReg(*
I.getNewValOperand());
2698 I.getAAMetadata(AAMetadata);
2701 OldValRes, SuccessRes,
Addr, Cmp, NewVal,
2705 I.getSyncScopeID(),
I.getSuccessOrdering(),
I.getFailureOrdering()));
2709 bool IRTranslator::translateAtomicRMW(
const User &U,
2715 Type *ResType =
I.getType();
2719 Register Val = getOrCreateVReg(*
I.getValOperand());
2721 unsigned Opcode = 0;
2722 switch (
I.getOperation()) {
2726 Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
2729 Opcode = TargetOpcode::G_ATOMICRMW_ADD;
2732 Opcode = TargetOpcode::G_ATOMICRMW_SUB;
2735 Opcode = TargetOpcode::G_ATOMICRMW_AND;
2738 Opcode = TargetOpcode::G_ATOMICRMW_NAND;
2741 Opcode = TargetOpcode::G_ATOMICRMW_OR;
2744 Opcode = TargetOpcode::G_ATOMICRMW_XOR;
2747 Opcode = TargetOpcode::G_ATOMICRMW_MAX;
2750 Opcode = TargetOpcode::G_ATOMICRMW_MIN;
2753 Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
2756 Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
2759 Opcode = TargetOpcode::G_ATOMICRMW_FADD;
2762 Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
2767 I.getAAMetadata(AAMetadata);
2770 Opcode, Res,
Addr, Val,
2773 getMemOpAlign(
I), AAMetadata,
nullptr,
2774 I.getSyncScopeID(),
I.getOrdering()));
2778 bool IRTranslator::translateFence(
const User &U,
2780 const FenceInst &Fence = cast<FenceInst>(U);
2786 bool IRTranslator::translateFreeze(
const User &U,
2792 "Freeze with different source and destination type?");
2794 for (
unsigned I = 0;
I < DstRegs.
size(); ++
I) {
2801 void IRTranslator::finishPendingPhis() {
2806 #endif // ifndef NDEBUG
2807 for (
auto &Phi : PendingPHIs) {
2808 const PHINode *PI = Phi.first;
2814 #endif // ifndef NDEBUG
2820 for (
auto Pred : getMachinePredBBs({IRPred, PI->
getParent()})) {
2824 for (
unsigned j = 0;
j < ValRegs.
size(); ++
j) {
2834 bool IRTranslator::valueIsSplit(
const Value &V,
2840 return SplitTys.size() > 1;
2843 bool IRTranslator::translate(
const Instruction &Inst) {
2849 Inst.
getContext(), 0, 0, DL.getScope(), DL.getInlinedAt()));
2851 EntryBuilder->setDebugLoc(
DebugLoc());
2858 #define HANDLE_INST(NUM, OPCODE, CLASS) \
2859 case Instruction::OPCODE: \
2860 return translate##OPCODE(Inst, *CurBuilder.get());
2861 #include "llvm/IR/Instruction.def"
2868 if (
auto CI = dyn_cast<ConstantInt>(&
C))
2869 EntryBuilder->buildConstant(
Reg, *CI);
2870 else if (
auto CF = dyn_cast<ConstantFP>(&
C))
2871 EntryBuilder->buildFConstant(
Reg, *CF);
2872 else if (isa<UndefValue>(
C))
2873 EntryBuilder->buildUndef(
Reg);
2874 else if (isa<ConstantPointerNull>(
C))
2875 EntryBuilder->buildConstant(
Reg, 0);
2876 else if (
auto GV = dyn_cast<GlobalValue>(&
C))
2877 EntryBuilder->buildGlobalValue(
Reg, GV);
2878 else if (
auto CAZ = dyn_cast<ConstantAggregateZero>(&
C)) {
2879 if (!CAZ->getType()->isVectorTy())
2882 if (CAZ->getNumElements() == 1)
2883 return translateCopy(
C, *CAZ->getElementValue(0u), *EntryBuilder.get());
2885 for (
unsigned i = 0;
i < CAZ->getNumElements(); ++
i) {
2886 Constant &Elt = *CAZ->getElementValue(
i);
2887 Ops.push_back(getOrCreateVReg(Elt));
2889 EntryBuilder->buildBuildVector(
Reg, Ops);
2890 }
else if (
auto CV = dyn_cast<ConstantDataVector>(&
C)) {
2892 if (CV->getNumElements() == 1)
2893 return translateCopy(
C, *CV->getElementAsConstant(0),
2894 *EntryBuilder.get());
2896 for (
unsigned i = 0;
i < CV->getNumElements(); ++
i) {
2897 Constant &Elt = *CV->getElementAsConstant(
i);
2898 Ops.push_back(getOrCreateVReg(Elt));
2900 EntryBuilder->buildBuildVector(
Reg, Ops);
2901 }
else if (
auto CE = dyn_cast<ConstantExpr>(&
C)) {
2902 switch(
CE->getOpcode()) {
2903 #define HANDLE_INST(NUM, OPCODE, CLASS) \
2904 case Instruction::OPCODE: \
2905 return translate##OPCODE(*CE, *EntryBuilder.get());
2906 #include "llvm/IR/Instruction.def"
2910 }
else if (
auto CV = dyn_cast<ConstantVector>(&
C)) {
2911 if (CV->getNumOperands() == 1)
2912 return translateCopy(
C, *CV->getOperand(0), *EntryBuilder.get());
2914 for (
unsigned i = 0;
i < CV->getNumOperands(); ++
i) {
2915 Ops.push_back(getOrCreateVReg(*CV->getOperand(
i)));
2917 EntryBuilder->buildBuildVector(
Reg, Ops);
2918 }
else if (
auto *BA = dyn_cast<BlockAddress>(&
C)) {
2919 EntryBuilder->buildBlockAddress(
Reg, BA);
2926 void IRTranslator::finalizeBasicBlock() {
2927 for (
auto &BTB : SL->BitTestCases) {
2930 emitBitTestHeader(BTB, BTB.Parent);
2933 for (
unsigned j = 0, ej = BTB.Cases.size();
j != ej; ++
j) {
2934 UnhandledProb -= BTB.Cases[
j].ExtraProb;
2946 if (BTB.ContiguousRange &&
j + 2 == ej) {
2949 NextMBB = BTB.Cases[
j + 1].TargetBB;
2950 }
else if (
j + 1 == ej) {
2952 NextMBB = BTB.Default;
2955 NextMBB = BTB.Cases[
j + 1].ThisBB;
2958 emitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[
j],
MBB);
2961 if (BTB.ContiguousRange &&
j + 2 == ej) {
2963 BTB.Cases.pop_back();
2969 CFGEdge HeaderToDefaultEdge = {BTB.Parent->getBasicBlock(),
2970 BTB.Default->getBasicBlock()};
2971 addMachineCFGPred(HeaderToDefaultEdge, BTB.Parent);
2972 if (!BTB.ContiguousRange) {
2973 addMachineCFGPred(HeaderToDefaultEdge, BTB.Cases.back().ThisBB);
2976 SL->BitTestCases.clear();
2978 for (
auto &JTCase : SL->JTCases) {
2980 if (!JTCase.first.Emitted)
2981 emitJumpTableHeader(JTCase.second, JTCase.first, JTCase.first.HeaderBB);
2983 emitJumpTable(JTCase.second, JTCase.second.MBB);
2985 SL->JTCases.clear();
2987 for (
auto &SwCase : SL->SwitchCases)
2988 emitSwitchCase(SwCase, &CurBuilder->getMBB(), *CurBuilder);
2989 SL->SwitchCases.clear();
2992 void IRTranslator::finalizeFunction() {
2995 PendingPHIs.clear();
2997 FrameIndices.clear();
2998 MachinePreds.clear();
3002 EntryBuilder.reset();
3016 const auto *CI = dyn_cast<CallInst>(&I);
3017 return CI && CI->isMustTailCall();
3027 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
3030 TPC = &getAnalysis<TargetPassConfig>();
3036 EntryBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
3038 EntryBuilder->setCSEInfo(CSEInfo);
3039 CurBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
3040 CurBuilder->setCSEInfo(CSEInfo);
3042 EntryBuilder = std::make_unique<MachineIRBuilder>();
3043 CurBuilder = std::make_unique<MachineIRBuilder>();
3046 CurBuilder->setMF(*MF);
3047 EntryBuilder->setMF(*MF);
3049 DL = &
F.getParent()->getDataLayout();
3050 ORE = std::make_unique<OptimizationRemarkEmitter>(&
F);
3052 TM.resetTargetOptions(
F);
3056 FuncInfo.
BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
3058 FuncInfo.
BPI =
nullptr;
3064 SL = std::make_unique<GISelSwitchLowering>(
this, FuncInfo);
3065 SL->init(TLI,
TM, *DL);
3069 assert(PendingPHIs.empty() &&
"stale PHIs");
3074 F.getSubprogram(), &
F.getEntryBlock());
3075 R <<
"unable to translate in big endian mode";
3080 auto FinalizeOnReturn =
make_scope_exit([
this]() { finalizeFunction(); });
3085 EntryBuilder->setMBB(*EntryBB);
3087 DebugLoc DbgLoc =
F.getEntryBlock().getFirstNonPHI()->getDebugLoc();
3091 bool IsVarArg =
F.isVarArg();
3092 bool HasMustTailInVarArgFn =
false;
3096 auto *&
MBB = BBToMBB[&
BB];
3101 if (
BB.hasAddressTaken())
3104 if (!HasMustTailInVarArgFn)
3111 EntryBB->addSuccessor(&getMBB(
F.front()));
3115 F.getSubprogram(), &
F.getEntryBlock());
3116 R <<
"unable to lower function: " <<
ore::NV(
"Prototype",
F.getType());
3127 VRegArgs.push_back(VRegs);
3129 if (
Arg.hasSwiftErrorAttr()) {
3130 assert(VRegs.
size() == 1 &&
"Too many vregs for Swift error");
3137 F.getSubprogram(), &
F.getEntryBlock());
3138 R <<
"unable to lower arguments: " <<
ore::NV(
"Prototype",
F.getType());
3145 if (EnableCSE && CSEInfo)
3152 #endif // ifndef NDEBUG
3159 CurBuilder->setMBB(
MBB);
3160 HasTailCall =
false;
3171 #endif // ifndef NDEBUG
3172 if (translate(Inst))
3177 R <<
"unable to translate instruction: " <<
ore::NV(
"Opcode", &Inst);
3179 if (ORE->allowExtraAnalysis(
"gisel-irtranslator")) {
3180 std::string InstStrStorage;
3184 R <<
": '" << InstStr.
str() <<
"'";
3191 finalizeBasicBlock();
3198 finishPendingPhis();
3205 assert(EntryBB->succ_size() == 1 &&
3206 "Custom BB used for lowering should have only one successor");
3210 "LLVM-IR entry block has a predecessor!?");
3213 NewEntryBB.
splice(NewEntryBB.
begin(), EntryBB, EntryBB->begin(),
3222 EntryBB->removeSuccessor(&NewEntryBB);
3227 "New entry wasn't next in the list of basic block!");