74#define DEBUG_TYPE "code-extractor"
82 cl::desc(
"Aggregate arguments to code-extracted functions"));
87 bool AllowVarArgs,
bool AllowAlloca) {
100 while (!ToVisit.
empty()) {
102 if (!Visited.
insert(Curr).second)
104 if (isa<BlockAddress const>(Curr))
107 if (isa<Instruction>(Curr) && cast<Instruction>(Curr)->
getParent() != &BB)
110 for (
auto const &U : Curr->
operands()) {
111 if (
auto *UU = dyn_cast<User>(U))
119 if (isa<AllocaInst>(
I)) {
125 if (
const auto *II = dyn_cast<InvokeInst>(
I)) {
128 if (
auto *UBB = II->getUnwindDest())
129 if (!Result.count(UBB))
136 if (
const auto *CSI = dyn_cast<CatchSwitchInst>(
I)) {
137 if (
auto *UBB = CSI->getUnwindDest())
138 if (!Result.count(UBB))
140 for (
const auto *HBB : CSI->handlers())
141 if (!Result.count(
const_cast<BasicBlock*
>(HBB)))
148 if (
const auto *CPI = dyn_cast<CatchPadInst>(
I)) {
149 for (
const auto *U : CPI->users())
150 if (
const auto *CRI = dyn_cast<CatchReturnInst>(U))
151 if (!Result.count(
const_cast<BasicBlock*
>(CRI->getParent())))
159 if (
const auto *CPI = dyn_cast<CleanupPadInst>(
I)) {
160 for (
const auto *U : CPI->users())
161 if (
const auto *CRI = dyn_cast<CleanupReturnInst>(U))
162 if (!Result.count(
const_cast<BasicBlock*
>(CRI->getParent())))
166 if (
const auto *CRI = dyn_cast<CleanupReturnInst>(
I)) {
167 if (
auto *UBB = CRI->getUnwindDest())
168 if (!Result.count(UBB))
173 if (
const CallInst *CI = dyn_cast<CallInst>(
I)) {
174 if (
const Function *
F = CI->getCalledFunction()) {
175 auto IID =
F->getIntrinsicID();
176 if (IID == Intrinsic::vastart) {
185 if (IID == Intrinsic::eh_typeid_for)
197 bool AllowVarArgs,
bool AllowAlloca) {
198 assert(!BBs.
empty() &&
"The set of blocks to extract must be non-empty");
208 if (!Result.insert(BB))
212 LLVM_DEBUG(
dbgs() <<
"Region front block: " << Result.front()->getName()
215 for (
auto *BB : Result) {
220 if (BB == Result.front()) {
222 LLVM_DEBUG(
dbgs() <<
"The first block cannot be an unwind block\n");
231 if (!Result.count(PBB)) {
232 LLVM_DEBUG(
dbgs() <<
"No blocks in this region may have entries from "
233 "outside the region except for the first block!\n"
234 <<
"Problematic source BB: " << BB->getName() <<
"\n"
235 <<
"Problematic destination BB: " << PBB->getName()
247 bool AllowVarArgs,
bool AllowAlloca,
248 BasicBlock *AllocationBlock, std::string Suffix,
249 bool ArgsInZeroAddressSpace)
251 BPI(BPI), AC(AC), AllocationBlock(AllocationBlock),
252 AllowVarArgs(AllowVarArgs),
254 Suffix(Suffix), ArgsInZeroAddressSpace(ArgsInZeroAddressSpace) {}
261 BPI(BPI), AC(AC), AllocationBlock(nullptr), AllowVarArgs(
false),
271 if (
Blocks.count(
I->getParent()))
280 if (isa<Argument>(V))
return true;
282 if (!
Blocks.count(
I->getParent()))
294 if (!CommonExitBlock) {
295 CommonExitBlock = Succ;
298 if (CommonExitBlock != Succ)
307 return CommonExitBlock;
312 for (
Instruction &II : BB.instructionsWithoutDebug())
313 if (
auto *AI = dyn_cast<AllocaInst>(&II))
314 Allocas.push_back(AI);
316 findSideEffectInfoForBlock(BB);
320void CodeExtractorAnalysisCache::findSideEffectInfoForBlock(
BasicBlock &BB) {
322 unsigned Opcode = II.getOpcode();
323 Value *MemAddr =
nullptr;
325 case Instruction::Store:
326 case Instruction::Load: {
327 if (Opcode == Instruction::Store) {
329 MemAddr = SI->getPointerOperand();
335 if (isa<Constant>(MemAddr))
338 if (!isa<AllocaInst>(
Base)) {
339 SideEffectingBlocks.insert(&BB);
342 BaseMemAddrs[&BB].insert(
Base);
350 SideEffectingBlocks.insert(&BB);
354 if (II.mayHaveSideEffects()) {
355 SideEffectingBlocks.insert(&BB);
365 if (SideEffectingBlocks.count(&BB))
367 auto It = BaseMemAddrs.find(&BB);
368 if (It != BaseMemAddrs.end())
369 return It->second.count(
Addr);
375 AllocaInst *AI = cast<AllocaInst>(
Addr->stripInBoundsConstantOffsets());
378 if (Blocks.count(&BB))
388 BasicBlock *SinglePredFromOutlineRegion =
nullptr;
389 assert(!Blocks.count(CommonExitBlock) &&
390 "Expect a block outside the region!");
392 if (!Blocks.count(Pred))
394 if (!SinglePredFromOutlineRegion) {
395 SinglePredFromOutlineRegion = Pred;
396 }
else if (SinglePredFromOutlineRegion != Pred) {
397 SinglePredFromOutlineRegion =
nullptr;
402 if (SinglePredFromOutlineRegion)
403 return SinglePredFromOutlineRegion;
409 while (
I != BB->
end()) {
410 PHINode *Phi = dyn_cast<PHINode>(
I);
422 assert(!getFirstPHI(CommonExitBlock) &&
"Phi not expected");
430 if (Blocks.count(Pred))
435 Blocks.insert(CommonExitBlock);
436 OldTargets.push_back(NewExitBlock);
437 return CommonExitBlock;
444CodeExtractor::LifetimeMarkerInfo
448 LifetimeMarkerInfo
Info;
458 Info.LifeStart = IntrInst;
464 Info.LifeEnd = IntrInst;
469 if (isa<DbgInfoIntrinsic>(IntrInst))
477 if (!
Info.LifeStart || !
Info.LifeEnd)
483 if ((
Info.SinkLifeStart ||
Info.HoistLifeEnd) &&
488 if (
Info.HoistLifeEnd && !ExitBlock)
500 auto moveOrIgnoreLifetimeMarkers =
501 [&](
const LifetimeMarkerInfo &LMI) ->
bool {
504 if (LMI.SinkLifeStart) {
507 SinkCands.
insert(LMI.LifeStart);
509 if (LMI.HoistLifeEnd) {
510 LLVM_DEBUG(
dbgs() <<
"Hoisting lifetime.end: " << *LMI.LifeEnd <<
"\n");
511 HoistCands.
insert(LMI.LifeEnd);
520 if (Blocks.count(BB))
529 LifetimeMarkerInfo MarkerInfo = getLifetimeMarkers(CEAC, AI, ExitBlock);
530 bool Moved = moveOrIgnoreLifetimeMarkers(MarkerInfo);
542 for (
User *U : AI->users()) {
546 if (U->stripInBoundsConstantOffsets() != AI)
550 for (
User *BU : Bitcast->users()) {
562 << *Bitcast <<
" in out-of-region lifetime marker "
563 << *IntrInst <<
"\n");
564 LifetimeBitcastUsers.
push_back(IntrInst);
574 I->replaceUsesOfWith(
I->getOperand(1), CastI);
580 for (
User *U : AI->users()) {
581 if (U->stripInBoundsConstantOffsets() == AI) {
583 LifetimeMarkerInfo LMI = getLifetimeMarkers(CEAC, Bitcast, ExitBlock);
599 if (Bitcasts.
empty())
602 LLVM_DEBUG(
dbgs() <<
"Sinking alloca (via bitcast): " << *AI <<
"\n");
604 for (
unsigned I = 0,
E = Bitcasts.
size();
I !=
E; ++
I) {
606 const LifetimeMarkerInfo &LMI = BitcastLifetimeInfo[
I];
608 "Unsafe to sink bitcast without lifetime markers");
609 moveOrIgnoreLifetimeMarkers(LMI);
611 LLVM_DEBUG(
dbgs() <<
"Sinking bitcast-of-alloca: " << *BitcastAddr
613 SinkCands.
insert(BitcastAddr);
627 if (AllowVarArgs &&
F->getFunctionType()->isVarArg()) {
628 auto containsVarArgIntrinsic = [](
const Instruction &
I) {
629 if (
const CallInst *CI = dyn_cast<CallInst>(&
I))
630 if (
const Function *Callee = CI->getCalledFunction())
631 return Callee->getIntrinsicID() == Intrinsic::vastart ||
632 Callee->getIntrinsicID() == Intrinsic::vaend;
636 for (
auto &BB : *
F) {
637 if (Blocks.count(&BB))
652 for (
auto &OI : II.operands()) {
658 for (
User *U : II.users())
670void CodeExtractor::severSplitPHINodesOfEntry(
BasicBlock *&Header) {
671 unsigned NumPredsFromRegion = 0;
672 unsigned NumPredsOutsideRegion = 0;
674 if (Header != &Header->getParent()->getEntryBlock()) {
675 PHINode *PN = dyn_cast<PHINode>(Header->begin());
683 ++NumPredsFromRegion;
685 ++NumPredsOutsideRegion;
689 if (NumPredsOutsideRegion <= 1)
return;
701 Blocks.remove(OldPred);
702 Blocks.insert(NewBB);
707 if (NumPredsFromRegion) {
720 for (AfterPHIs = OldPred->
begin(); isa<PHINode>(AfterPHIs); ++AfterPHIs) {
721 PHINode *PN = cast<PHINode>(AfterPHIs);
747void CodeExtractor::severSplitPHINodesOfExits(
752 for (
PHINode &PN : ExitBB->phis()) {
762 if (IncomingVals.
size() <= 1)
769 ExitBB->getName() +
".split",
770 ExitBB->getParent(), ExitBB);
774 if (Blocks.count(PredBB))
775 PredBB->getTerminator()->replaceUsesOfWith(ExitBB, NewBB);
777 Blocks.insert(NewBB);
784 for (
unsigned i : IncomingVals)
786 for (
unsigned i :
reverse(IncomingVals))
793void CodeExtractor::splitReturnBlocks() {
797 Block->splitBasicBlock(RI->getIterator(),
Block->getName() +
".ret");
815Function *CodeExtractor::constructFunction(
const ValueSet &inputs,
816 const ValueSet &outputs,
826 switch (NumExitBlocks) {
833 std::vector<Type *> ParamTy;
834 std::vector<Type *> AggParamTy;
835 ValueSet StructValues;
841 if (AggregateArgs && !ExcludeArgsFromAggregate.
contains(
value)) {
842 AggParamTy.push_back(
value->getType());
843 StructValues.insert(
value);
845 ParamTy.push_back(
value->getType());
849 for (
Value *output : outputs) {
851 if (AggregateArgs && !ExcludeArgsFromAggregate.
contains(output)) {
852 AggParamTy.push_back(output->getType());
853 StructValues.insert(output);
860 (ParamTy.size() + AggParamTy.size()) ==
861 (inputs.size() + outputs.size()) &&
862 "Number of scalar and aggregate params does not match inputs, outputs");
863 assert((StructValues.empty() || AggregateArgs) &&
864 "Expeced StructValues only with AggregateArgs set");
867 size_t NumScalarParams = ParamTy.size();
869 if (AggregateArgs && !AggParamTy.empty()) {
872 StructTy, ArgsInZeroAddressSpace ? 0 :
DL.getAllocaAddrSpace()));
876 dbgs() <<
"Function type: " << *RetTy <<
" f(";
877 for (
Type *i : ParamTy)
878 dbgs() << *i <<
", ";
883 RetTy, ParamTy, AllowVarArgs && oldFunction->
isVarArg());
885 std::string SuffixToUse =
892 oldFunction->
getName() +
"." + SuffixToUse, M);
903 if (Attr.isStringAttribute()) {
904 if (Attr.getKindAsString() ==
"thunk")
907 switch (Attr.getKindAsEnum()) {
910 case Attribute::AllocSize:
911 case Attribute::Builtin:
912 case Attribute::Convergent:
913 case Attribute::JumpTable:
914 case Attribute::Naked:
915 case Attribute::NoBuiltin:
916 case Attribute::NoMerge:
917 case Attribute::NoReturn:
918 case Attribute::NoSync:
919 case Attribute::ReturnsTwice:
920 case Attribute::Speculatable:
921 case Attribute::StackAlignment:
922 case Attribute::WillReturn:
923 case Attribute::AllocKind:
924 case Attribute::PresplitCoroutine:
925 case Attribute::Memory:
926 case Attribute::NoFPClass:
927 case Attribute::CoroDestroyOnlyWhenComplete:
930 case Attribute::AlwaysInline:
931 case Attribute::Cold:
932 case Attribute::DisableSanitizerInstrumentation:
933 case Attribute::FnRetThunkExtern:
935 case Attribute::NoRecurse:
936 case Attribute::InlineHint:
937 case Attribute::MinSize:
938 case Attribute::NoCallback:
939 case Attribute::NoDuplicate:
940 case Attribute::NoFree:
941 case Attribute::NoImplicitFloat:
942 case Attribute::NoInline:
943 case Attribute::NonLazyBind:
944 case Attribute::NoRedZone:
945 case Attribute::NoUnwind:
946 case Attribute::NoSanitizeBounds:
947 case Attribute::NoSanitizeCoverage:
948 case Attribute::NullPointerIsValid:
949 case Attribute::OptimizeForDebugging:
950 case Attribute::OptForFuzzing:
951 case Attribute::OptimizeNone:
952 case Attribute::OptimizeForSize:
953 case Attribute::SafeStack:
954 case Attribute::ShadowCallStack:
955 case Attribute::SanitizeAddress:
956 case Attribute::SanitizeMemory:
957 case Attribute::SanitizeThread:
958 case Attribute::SanitizeHWAddress:
959 case Attribute::SanitizeMemTag:
960 case Attribute::SpeculativeLoadHardening:
961 case Attribute::StackProtect:
962 case Attribute::StackProtectReq:
963 case Attribute::StackProtectStrong:
964 case Attribute::StrictFP:
965 case Attribute::UWTable:
966 case Attribute::VScaleRange:
967 case Attribute::NoCfCheck:
968 case Attribute::MustProgress:
969 case Attribute::NoProfile:
970 case Attribute::SkipProfile:
973 case Attribute::Alignment:
974 case Attribute::AllocatedPointer:
975 case Attribute::AllocAlign:
976 case Attribute::ByVal:
977 case Attribute::Dereferenceable:
978 case Attribute::DereferenceableOrNull:
979 case Attribute::ElementType:
980 case Attribute::InAlloca:
981 case Attribute::InReg:
982 case Attribute::Nest:
983 case Attribute::NoAlias:
984 case Attribute::NoCapture:
985 case Attribute::NoUndef:
986 case Attribute::NonNull:
987 case Attribute::Preallocated:
988 case Attribute::ReadNone:
989 case Attribute::ReadOnly:
990 case Attribute::Returned:
991 case Attribute::SExt:
992 case Attribute::StructRet:
993 case Attribute::SwiftError:
994 case Attribute::SwiftSelf:
995 case Attribute::SwiftAsync:
996 case Attribute::ZExt:
997 case Attribute::ImmArg:
998 case Attribute::ByRef:
999 case Attribute::WriteOnly:
1000 case Attribute::Writable:
1001 case Attribute::DeadOnUnwind:
1002 case Attribute::Range:
1013 newFunction->
insert(newFunction->
end(), newRootNode);
1022 for (
unsigned i = 0, e = inputs.size(), aggIdx = 0; i != e; ++i) {
1024 if (AggregateArgs && StructValues.contains(inputs[i])) {
1030 StructTy, &*AggAI,
Idx,
"gep_" + inputs[i]->
getName(), TI);
1032 "loadgep_" + inputs[i]->getName(), TI);
1035 RewriteVal = &*ScalarAI++;
1037 std::vector<User *>
Users(inputs[i]->user_begin(), inputs[i]->user_end());
1040 if (
Blocks.count(inst->getParent()))
1041 inst->replaceUsesOfWith(inputs[i], RewriteVal);
1045 if (NumScalarParams) {
1047 for (
unsigned i = 0, e = inputs.size(); i != e; ++i, ++ScalarAI)
1048 if (!StructValues.contains(inputs[i]))
1050 for (
unsigned i = 0, e = outputs.size(); i != e; ++i, ++ScalarAI)
1051 if (!StructValues.contains(outputs[i]))
1059 for (
auto &U :
Users)
1063 if (
I->isTerminator() &&
I->getFunction() == oldFunction &&
1064 !
Blocks.count(
I->getParent()))
1065 I->replaceUsesOfWith(header, newHeader);
1081 auto *II = dyn_cast<IntrinsicInst>(&
I);
1082 if (!II || !II->isLifetimeStartOrEnd())
1092 if (II->getIntrinsicID() == Intrinsic::lifetime_start)
1093 LifetimesStart.
insert(Mem);
1094 II->eraseFromParent();
1111 bool InsertBefore) {
1112 for (
Value *Mem : Objects) {
1115 "Input memory not defined in original function");
1122 Marker->insertBefore(Term);
1126 if (!LifetimesStart.
empty()) {
1127 insertMarkers(Intrinsic::lifetime_start, LifetimesStart,
1131 if (!LifetimesEnd.
empty()) {
1132 insertMarkers(Intrinsic::lifetime_end, LifetimesEnd,
1143 ValueSet &outputs) {
1146 std::vector<Value *> params, ReloadOutputs, Reloads;
1147 ValueSet StructValues;
1155 unsigned ScalarInputArgNo = 0;
1157 for (
Value *input : inputs) {
1158 if (AggregateArgs && !ExcludeArgsFromAggregate.
contains(input))
1159 StructValues.
insert(input);
1161 params.push_back(input);
1162 if (input->isSwiftError())
1163 SwiftErrorArgs.
push_back(ScalarInputArgNo);
1169 unsigned ScalarOutputArgNo = 0;
1170 for (
Value *output : outputs) {
1171 if (AggregateArgs && !ExcludeArgsFromAggregate.
contains(output)) {
1172 StructValues.insert(output);
1175 new AllocaInst(output->getType(),
DL.getAllocaAddrSpace(),
1176 nullptr, output->
getName() +
".loc",
1178 ReloadOutputs.push_back(alloca);
1179 params.push_back(alloca);
1180 ++ScalarOutputArgNo;
1186 unsigned NumAggregatedInputs = 0;
1187 if (AggregateArgs && !StructValues.empty()) {
1188 std::vector<Type *> ArgTypes;
1189 for (
Value *V : StructValues)
1190 ArgTypes.push_back(
V->getType());
1195 StructArgTy,
DL.getAllocaAddrSpace(),
nullptr,
"structArg",
1199 if (ArgsInZeroAddressSpace &&
DL.getAllocaAddrSpace() != 0) {
1201 Struct, PointerType ::get(Context, 0),
"structArg.ascast");
1202 StructSpaceCast->insertAfter(
Struct);
1203 params.push_back(StructSpaceCast);
1205 params.push_back(
Struct);
1208 for (
unsigned i = 0, e = StructValues.size(); i != e; ++i) {
1209 if (inputs.contains(StructValues[i])) {
1215 GEP->insertInto(codeReplacer, codeReplacer->
end());
1217 NumAggregatedInputs++;
1224 NumExitBlocks > 1 ?
"targetBlock" :
"");
1236 for (
unsigned SwiftErrArgNo : SwiftErrorArgs) {
1237 call->
addParamAttr(SwiftErrArgNo, Attribute::SwiftError);
1238 newFunction->
addParamAttr(SwiftErrArgNo, Attribute::SwiftError);
1243 for (
unsigned i = 0, e = outputs.size(), scalarIdx = 0,
1244 aggIdx = NumAggregatedInputs;
1246 Value *Output =
nullptr;
1247 if (AggregateArgs && StructValues.contains(outputs[i])) {
1253 GEP->insertInto(codeReplacer, codeReplacer->
end());
1257 Output = ReloadOutputs[scalarIdx];
1261 outputs[i]->
getName() +
".reload",
1263 Reloads.push_back(
load);
1264 std::vector<User *>
Users(outputs[i]->user_begin(), outputs[i]->user_end());
1275 codeReplacer, 0, codeReplacer);
1282 std::map<BasicBlock *, BasicBlock *> ExitBlockMap;
1286 unsigned switchVal = 0;
1288 if (
Blocks.count(OldTarget))
1290 BasicBlock *&NewTarget = ExitBlockMap[OldTarget];
1297 OldTarget->getName() +
".exitStub",
1299 unsigned SuccNum = switchVal++;
1301 Value *brVal =
nullptr;
1302 assert(NumExitBlocks < 0xffff &&
"too many exit blocks for switch");
1303 switch (NumExitBlocks) {
1329 BasicBlock *NewTarget = ExitBlockMap[OldTarget];
1330 assert(NewTarget &&
"Unknown target block!");
1341 std::advance(ScalarOutputArgBegin, ScalarInputArgNo);
1343 std::advance(AggOutputArgBegin, ScalarInputArgNo + ScalarOutputArgNo);
1345 for (
unsigned i = 0, e = outputs.size(), aggIdx = NumAggregatedInputs; i != e;
1347 auto *OutI = dyn_cast<Instruction>(outputs[i]);
1355 if (
auto *InvokeI = dyn_cast<InvokeInst>(OutI))
1356 InsertPt = InvokeI->getNormalDest()->getFirstInsertionPt();
1357 else if (
auto *Phi = dyn_cast<PHINode>(OutI))
1358 InsertPt =
Phi->getParent()->getFirstInsertionPt();
1360 InsertPt = std::next(OutI->getIterator());
1362 assert((InsertPt->getFunction() == newFunction ||
1363 Blocks.count(InsertPt->getParent())) &&
1364 "InsertPt should be in new function");
1365 if (AggregateArgs && StructValues.contains(outputs[i])) {
1367 "Number of aggregate output arguments should match "
1368 "the number of defined values");
1373 StructArgTy, &*AggOutputArgBegin,
Idx,
"gep_" + outputs[i]->
getName(),
1382 "Number of scalar output arguments should match "
1383 "the number of defined values");
1384 new StoreInst(outputs[i], &*ScalarOutputArgBegin, InsertPt);
1385 ++ScalarOutputArgBegin;
1391 switch (NumExitBlocks) {
1440void CodeExtractor::moveCodeToFunction(
Function *newFunction) {
1444 Block->removeFromParent();
1451 newFuncIt = newFunction->
insert(std::next(newFuncIt),
Block);
1455void CodeExtractor::calculateNewCallTerminatorWeights(
1467 Distribution BranchDist;
1474 BlockNode ExitNode(i);
1477 BranchDist.addExit(ExitNode, ExitFreq);
1483 if (BranchDist.Total == 0) {
1489 BranchDist.normalize();
1492 for (
unsigned I = 0,
E = BranchDist.Weights.size();
I <
E; ++
I) {
1493 const auto &Weight = BranchDist.Weights[
I];
1496 BranchWeights[Weight.TargetNode.Index] = Weight.Amount;
1498 EdgeProbabilities[Weight.TargetNode.Index] = BP;
1502 LLVMContext::MD_prof,
1514 if (DVI->getFunction() != &
F)
1515 DVI->eraseFromParent();
1517 if (DPV->getFunction() != &
F)
1518 DPV->eraseFromParent();
1541 assert(OldSP->getUnit() &&
"Missing compile unit for subprogram");
1547 DISubprogram::SPFlagOptimized |
1548 DISubprogram::SPFlagLocalToUnit;
1551 0, SPType, 0, DINode::FlagZero, SPFlags);
1554 auto IsInvalidLocation = [&NewFunc](
Value *Location) {
1558 (!isa<Constant>(Location) && !isa<Instruction>(Location)))
1560 Instruction *LocationInst = dyn_cast<Instruction>(Location);
1561 return LocationInst && LocationInst->
getFunction() != &NewFunc;
1576 DINode *&NewVar = RemappedMetadata[OldVar];
1579 *OldVar->getScope(), *NewSP, Ctx, Cache);
1581 NewScope, OldVar->
getName(), OldVar->getFile(), OldVar->getLine(),
1582 OldVar->getType(),
false, DINode::FlagZero,
1583 OldVar->getAlignInBits());
1585 return cast<DILocalVariable>(NewVar);
1588 auto UpdateDbgLabel = [&](
auto *LabelRecord) {
1591 if (LabelRecord->getDebugLoc().getInlinedAt())
1593 DILabel *OldLabel = LabelRecord->getLabel();
1594 DINode *&NewLabel = RemappedMetadata[OldLabel];
1597 *OldLabel->
getScope(), *NewSP, Ctx, Cache);
1601 LabelRecord->setLabel(cast<DILabel>(NewLabel));
1604 auto UpdateDbgRecordsOnInst = [&](
Instruction &
I) ->
void {
1605 for (
DbgRecord &DR :
I.getDbgRecordRange()) {
1606 if (
DPLabel *DPL = dyn_cast<DPLabel>(&DR)) {
1607 UpdateDbgLabel(DPL);
1611 DPValue &DPV = cast<DPValue>(DR);
1628 UpdateDbgRecordsOnInst(
I);
1630 auto *DII = dyn_cast<DbgInfoIntrinsic>(&
I);
1636 if (
auto *DLI = dyn_cast<DbgLabelInst>(&
I)) {
1637 UpdateDbgLabel(DLI);
1641 auto *DVI = cast<DbgVariableIntrinsic>(DII);
1643 if (
any_of(DVI->location_ops(), IsInvalidLocation)) {
1648 if (
auto *DAI = dyn_cast<DbgAssignIntrinsic>(DVI);
1649 DAI && IsInvalidLocation(DAI->getAddress())) {
1655 if (!DVI->getDebugLoc().getInlinedAt())
1656 DVI->setVariable(GetUpdatedDIVariable(DVI->getVariable()));
1659 for (
auto *DII : DebugIntrinsicsToDelete)
1660 DII->eraseFromParent();
1661 for (
auto *DPV : DPVsToDelete)
1662 DPV->getMarker()->MarkedInstr->dropOneDbgRecord(DPV);
1673 *NewSP, Ctx, Cache));
1676 auto updateLoopInfoLoc = [&Ctx, &Cache, NewSP](
Metadata *MD) ->
Metadata * {
1677 if (
auto *Loc = dyn_cast_or_null<DILocation>(MD))
1710 assert(BPI &&
"Both BPI and BFI are required to preserve profile info");
1723 if (
auto *AI = dyn_cast<AssumeInst>(&
I)) {
1726 AI->eraseFromParent();
1733 splitReturnBlocks();
1741 if (!
Blocks.count(Succ)) {
1751 NumExitBlocks = ExitBlocks.
size();
1755 if (!
Blocks.contains(OldTarget))
1756 OldTargets.push_back(OldTarget);
1760 severSplitPHINodesOfEntry(header);
1761 severSplitPHINodesOfExits(ExitBlocks);
1765 "codeRepl", oldFunction,
1783 if (!
I.getDebugLoc())
1787 if (isa<DbgInfoIntrinsic>(
I))
1789 BranchI->setDebugLoc(
I.getDebugLoc());
1794 BranchI->insertInto(newFuncRoot, newFuncRoot->
end());
1796 ValueSet SinkingCands, HoistingCands;
1798 findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);
1808 for (
auto *II : SinkingCands) {
1809 if (
auto *AI = dyn_cast<AllocaInst>(II)) {
1811 if (!FirstSunkAlloca)
1812 FirstSunkAlloca = AI;
1815 assert((SinkingCands.
empty() || FirstSunkAlloca) &&
1816 "Did not expect a sink candidate without any allocas");
1817 for (
auto *II : SinkingCands) {
1818 if (!isa<AllocaInst>(II)) {
1819 cast<Instruction>(II)->moveAfter(FirstSunkAlloca);
1823 if (!HoistingCands.
empty()) {
1826 for (
auto *II : HoistingCands)
1839 constructFunction(inputs, outputs, header, newFuncRoot, codeReplacer,
1852 emitCallAndSwitchStatement(newFunction, codeReplacer, inputs, outputs);
1854 moveCodeToFunction(newFunction);
1866 if (BFI && NumExitBlocks > 1)
1867 calculateNewCallTerminatorWeights(codeReplacer, ExitWeights, BPI);
1879 for (
PHINode &PN : ExitBB->phis()) {
1880 Value *IncomingCodeReplacerVal =
nullptr;
1887 if (!IncomingCodeReplacerVal) {
1892 "PHI has two incompatbile incoming values from codeRepl");
1903 return isa<ReturnInst>(Term) || isa<ResumeInst>(Term);
1909 newFunction->
dump();
1923 auto *
I = dyn_cast_or_null<CallInst>(AssumeVH);
1928 if (
I->getFunction() != &OldFunc)
1935 auto *AffectedCI = dyn_cast_or_null<CallInst>(AffectedValVH);
1938 if (AffectedCI->getFunction() != &OldFunc)
1940 auto *AssumedInst = cast<Instruction>(AffectedCI->getOperand(0));
1941 if (AssumedInst->getFunction() != &OldFunc)
1949 ExcludeArgsFromAggregate.
insert(Arg);
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
AMDGPU Mark last scratch load
Expand Atomic instructions
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Given that RA is a live value
This file defines the DenseMap class.
DenseMap< Block *, BlockRelaxAux > Blocks
static Function * getFunction(Constant *C)
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
iv Induction Variable Users
Move duplicate certain instructions close to their use
Module.h This file contains the declarations for the Module class.
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
This class represents a conversion between pointers from one address space to another.
an instruction to allocate memory on the stack
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
bool empty() const
empty - Check if the array is empty.
A cache of @llvm.assume calls within a function.
MutableArrayRef< ResultElem > assumptions()
Access the list of assumption handles currently tracked for this function.
void unregisterAssumption(AssumeInst *CI)
Remove an @llvm.assume intrinsic from this function's cache if it has been added to the cache earlier...
MutableArrayRef< ResultElem > assumptionsFor(const Value *V)
Access the list of assumptions which affect this value.
AttributeSet getFnAttrs() const
The function attributes are returned.
@ TombstoneKey
Use as Tombstone key for DenseMap of AttrKind.
@ None
No attributes have been set.
@ EmptyKey
Use as Empty key for DenseMap of AttrKind.
@ EndAttrKinds
Sentinel value useful for loops.
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
iterator_range< filter_iterator< BasicBlock::const_iterator, std::function< bool(const Instruction &)> > > instructionsWithoutDebug(bool SkipPseudoOp=true) const
Return a const iterator range over the instructions in the block, skipping any debug instructions.
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches,...
InstListType::const_iterator getFirstNonPHIIt() const
Iterator returning form of getFirstNonPHI.
InstListType::const_iterator const_iterator
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
const Function * getParent() const
Return the enclosing method, or null if none.
InstListType::iterator iterator
Instruction iterators...
LLVMContext & getContext() const
Get the context in which this basic block lives.
bool IsNewDbgInfoFormat
Flag recording whether or not this block stores debug-info in the form of intrinsic instructions (fal...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
std::optional< uint64_t > getProfileCountFromFreq(BlockFrequency Freq) const
Returns the estimated profile count of Freq.
void setBlockFreq(const BasicBlock *BB, BlockFrequency Freq)
BlockFrequency getBlockFreq(const BasicBlock *BB) const
getblockFreq - Return block frequency.
static BranchInst * Create(BasicBlock *IfTrue, BasicBlock::iterator InsertBefore)
Analysis providing branch probability information.
void setEdgeProbability(const BasicBlock *Src, const SmallVectorImpl< BranchProbability > &Probs)
Set the raw probabilities for all edges from the given block.
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
static BranchProbability getUnknown()
static BranchProbability getZero()
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr, BasicBlock::iterator InsertBefore)
This is the base class for all instructions that perform data casts.
static CastInst * CreatePointerCast(Value *S, Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd)
Create a BitCast AddrSpaceCast, or a PtrToInt cast instruction.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
DISubroutineType * createSubroutineType(DITypeRefArray ParameterTypes, DINode::DIFlags Flags=DINode::FlagZero, unsigned CC=0)
Create subroutine type.
void finalizeSubprogram(DISubprogram *SP)
Finalize a specific subprogram - no new variables may be added to this subprogram afterwards.
DISubprogram * createFunction(DIScope *Scope, StringRef Name, StringRef LinkageName, DIFile *File, unsigned LineNo, DISubroutineType *Ty, unsigned ScopeLine, DINode::DIFlags Flags=DINode::FlagZero, DISubprogram::DISPFlags SPFlags=DISubprogram::SPFlagZero, DITemplateParameterArray TParams=nullptr, DISubprogram *Decl=nullptr, DITypeArray ThrownTypes=nullptr, DINodeArray Annotations=nullptr, StringRef TargetFuncName="")
Create a new descriptor for the specified subprogram.
DITypeRefArray getOrCreateTypeArray(ArrayRef< Metadata * > Elements)
Get a DITypeRefArray, create one if required.
DILocalVariable * createAutoVariable(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNo, DIType *Ty, bool AlwaysPreserve=false, DINode::DIFlags Flags=DINode::FlagZero, uint32_t AlignInBits=0)
Create a new descriptor for an auto variable.
StringRef getName() const
DILocalScope * getScope() const
Get the local scope for this label.
static DILocalScope * cloneScopeForSubprogram(DILocalScope &RootScope, DISubprogram &NewSP, LLVMContext &Ctx, DenseMap< const MDNode *, MDNode * > &Cache)
Traverses the scope chain rooted at RootScope until it hits a Subprogram, recreating the chain with "...
Tagged DWARF-like metadata node.
StringRef getName() const
DISPFlags
Debug info subprogram flags.
Records a position in IR for a source label (DILabel).
Record of a variable value-assignment, aka a non instruction representation of the dbg....
Value * getAddress() const
void setVariable(DILocalVariable *NewVar)
iterator_range< location_op_iterator > location_ops() const
Get the locations corresponding to the variable referenced by the debug info intrinsic.
DILocalVariable * getVariable() const
A parsed version of the target data layout string in and methods for querying it.
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
This is the common base class for debug info intrinsics for variables.
static DebugLoc replaceInlinedAtSubprogram(const DebugLoc &DL, DISubprogram &NewSP, LLVMContext &Ctx, DenseMap< const MDNode *, MDNode * > &Cache)
Rebuild the entire inline-at chain by replacing the subprogram at the end of the chain with NewSP.
DILocation * getInlinedAt() const
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
DomTreeNodeBase< NodeT > * addNewBlock(NodeT *BB, NodeT *DomBB)
Add a new node to the dominator tree information.
DomTreeNodeBase< NodeT > * getNode(const NodeT *BB) const
getNode - return the (Post)DominatorTree node for the specified basic block.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Class to represent profile counts.
void addFnAttr(Attribute::AttrKind Kind)
Add function attributes to this function.
void setSubprogram(DISubprogram *SP)
Set the attached subprogram.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
const BasicBlock & getEntryBlock() const
const BasicBlock & front() const
DISubprogram * getSubprogram() const
Get the attached subprogram.
bool IsNewDbgInfoFormat
Is this function using intrinsics to record the position of debugging information,...
bool hasPersonalityFn() const
Check whether this function has a personality function.
Constant * getPersonalityFn() const
Get the personality function associated with this function.
void setPersonalityFn(Constant *Fn)
AttributeList getAttributes() const
Return the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
adds the attribute to the list of attributes for the given arg.
Function::iterator insert(Function::iterator Position, BasicBlock *BB)
Insert BB in the basic block list at Position.
Type * getReturnType() const
Returns the type of the ret val.
void setEntryCount(ProfileCount Count, const DenseSet< GlobalValue::GUID > *Imports=nullptr)
Set the entry count for this function.
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr, BasicBlock::iterator InsertBefore)
unsigned getAddressSpace() const
Module * getParent()
Get the module that this global value is contained inside of...
@ InternalLinkage
Rename collisions when linking (static functions).
bool isLifetimeStartOrEnd() const LLVM_READONLY
Return true if the instruction is a llvm.lifetime.start or llvm.lifetime.end marker.
unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const BasicBlock * getParent() const
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
const Function * getFunction() const
Return the function this instruction belongs to.
BasicBlock * getSuccessor(unsigned Idx) const LLVM_READONLY
Return the specified successor. This instruction must be a terminator.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
void setSuccessor(unsigned Idx, BasicBlock *BB)
Update the specified successor to point at the provided block.
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
InstListType::iterator insertInto(BasicBlock *ParentBB, InstListType::iterator It)
Inserts an unlinked instruction into ParentBB at position It and returns the iterator of the inserted...
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
Value * getPointerOperand()
Represents a single loop in the control flow graph.
MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight)
Return metadata containing two branch weights.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
StringRef getName() const
Return the name of the corresponding LLVM basic block, or an empty string.
A Module instance is used to store all the information related to an LLVM module.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
void setIncomingBlock(unsigned i, BasicBlock *BB)
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr, BasicBlock::iterator InsertBefore)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
Value * removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty=true)
Remove an incoming value.
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Return a value (possibly void), from a function.
static ReturnInst * Create(LLVMContext &C, Value *retVal, BasicBlock::iterator InsertBefore)
A vector that has set insertion semantics.
ArrayRef< value_type > getArrayRef() const
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
bool empty() const
Determine if the SetVector is empty or not.
bool insert(const value_type &X)
Insert a new element into the SetVector.
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
std::string str() const
str - Get the contents as an std::string.
constexpr bool empty() const
empty - Check if the string is empty.
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Type * getElementType(unsigned N) const
static SwitchInst * Create(Value *Value, BasicBlock *Default, unsigned NumCases, BasicBlock::iterator InsertBefore)
BasicBlock * getSuccessor(unsigned idx) const
void setCondition(Value *V)
void addCase(ConstantInt *OnVal, BasicBlock *Dest)
Add an entry to the switch instruction.
void setDefaultDest(BasicBlock *DefaultCase)
Value * getCondition() const
CaseIt removeCase(CaseIt I)
This method removes the specified case and its successor from the switch instruction.
The instances of the Type class are immutable: once they are created, they are never changed.
static IntegerType * getInt1Ty(LLVMContext &C)
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt16Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isVoidTy() const
Return true if this is 'void'.
bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
user_iterator user_begin()
void setName(const Twine &Name)
Change the name of the value.
const Value * stripInBoundsConstantOffsets() const
Strip off pointer casts and all-constant inbounds GEPs.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
const Value * stripInBoundsOffsets(function_ref< void(const Value *)> Func=[](const Value *) {}) const
Strip off pointer casts and inbounds GEPs.
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
void dump() const
Support for debugging, callable in GDB: V->dump()
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
NodeAddr< PhiNode * > Phi
This is an optimization pass for GlobalISel generic memory operations.
bool stripDebugInfo(Function &F)
Function::ProfileCount ProfileCount
bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
auto successors(const MachineBasicBlock *BB)
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
void findDbgUsers(SmallVectorImpl< DbgVariableIntrinsic * > &DbgInsts, Value *V, SmallVectorImpl< DPValue * > *DPValues=nullptr)
Finds the debug info intrinsics describing a value.
BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
auto predecessors(const MachineBasicBlock *BB)
void updateLoopMetadataDebugLocations(Instruction &I, function_ref< Metadata *(Metadata *)> Updater)
Update the debug locations contained within the MD_loop metadata attached to the instruction I,...
Representative of a block.
Distribution of unscaled probability weight.