74#define DEBUG_TYPE "code-extractor"
82 cl::desc(
"Aggregate arguments to code-extracted functions"));
87 bool AllowVarArgs,
bool AllowAlloca) {
100 while (!ToVisit.
empty()) {
102 if (!Visited.
insert(Curr).second)
104 if (isa<BlockAddress const>(Curr))
107 if (isa<Instruction>(Curr) && cast<Instruction>(Curr)->
getParent() != &BB)
110 for (
auto const &U : Curr->
operands()) {
111 if (
auto *UU = dyn_cast<User>(U))
119 if (isa<AllocaInst>(
I)) {
125 if (
const auto *
II = dyn_cast<InvokeInst>(
I)) {
128 if (
auto *UBB =
II->getUnwindDest())
129 if (!Result.count(UBB))
136 if (
const auto *CSI = dyn_cast<CatchSwitchInst>(
I)) {
137 if (
auto *UBB = CSI->getUnwindDest())
138 if (!Result.count(UBB))
140 for (
const auto *HBB : CSI->handlers())
141 if (!Result.count(
const_cast<BasicBlock*
>(HBB)))
148 if (
const auto *CPI = dyn_cast<CatchPadInst>(
I)) {
149 for (
const auto *U : CPI->users())
150 if (
const auto *CRI = dyn_cast<CatchReturnInst>(U))
151 if (!Result.count(
const_cast<BasicBlock*
>(CRI->getParent())))
159 if (
const auto *CPI = dyn_cast<CleanupPadInst>(
I)) {
160 for (
const auto *U : CPI->users())
161 if (
const auto *CRI = dyn_cast<CleanupReturnInst>(U))
162 if (!Result.count(
const_cast<BasicBlock*
>(CRI->getParent())))
166 if (
const auto *CRI = dyn_cast<CleanupReturnInst>(
I)) {
167 if (
auto *UBB = CRI->getUnwindDest())
168 if (!Result.count(UBB))
173 if (
const CallInst *CI = dyn_cast<CallInst>(
I)) {
174 if (
const Function *
F = CI->getCalledFunction()) {
175 auto IID =
F->getIntrinsicID();
176 if (IID == Intrinsic::vastart) {
185 if (IID == Intrinsic::eh_typeid_for)
197 bool AllowVarArgs,
bool AllowAlloca) {
198 assert(!BBs.
empty() &&
"The set of blocks to extract must be non-empty");
208 if (!Result.insert(BB))
212 LLVM_DEBUG(
dbgs() <<
"Region front block: " << Result.front()->getName()
215 for (
auto *BB : Result) {
220 if (BB == Result.front()) {
222 LLVM_DEBUG(
dbgs() <<
"The first block cannot be an unwind block\n");
231 if (!Result.count(PBB)) {
232 LLVM_DEBUG(
dbgs() <<
"No blocks in this region may have entries from "
233 "outside the region except for the first block!\n"
234 <<
"Problematic source BB: " << BB->getName() <<
"\n"
235 <<
"Problematic destination BB: " << PBB->getName()
247 bool AllowVarArgs,
bool AllowAlloca,
248 BasicBlock *AllocationBlock, std::string Suffix,
249 bool ArgsInZeroAddressSpace)
251 BPI(BPI), AC(AC), AllocationBlock(AllocationBlock),
252 AllowVarArgs(AllowVarArgs),
254 Suffix(Suffix), ArgsInZeroAddressSpace(ArgsInZeroAddressSpace) {}
261 BPI(BPI), AC(AC), AllocationBlock(nullptr), AllowVarArgs(
false),
271 if (
Blocks.count(
I->getParent()))
280 if (isa<Argument>(V))
return true;
282 if (!
Blocks.count(
I->getParent()))
294 if (!CommonExitBlock) {
295 CommonExitBlock = Succ;
298 if (CommonExitBlock != Succ)
307 return CommonExitBlock;
313 if (
auto *AI = dyn_cast<AllocaInst>(&
II))
314 Allocas.push_back(AI);
316 findSideEffectInfoForBlock(BB);
320void CodeExtractorAnalysisCache::findSideEffectInfoForBlock(
BasicBlock &BB) {
322 unsigned Opcode =
II.getOpcode();
323 Value *MemAddr =
nullptr;
325 case Instruction::Store:
326 case Instruction::Load: {
327 if (Opcode == Instruction::Store) {
329 MemAddr = SI->getPointerOperand();
335 if (isa<Constant>(MemAddr))
338 if (!isa<AllocaInst>(
Base)) {
339 SideEffectingBlocks.insert(&BB);
342 BaseMemAddrs[&BB].insert(
Base);
350 SideEffectingBlocks.insert(&BB);
354 if (
II.mayHaveSideEffects()) {
355 SideEffectingBlocks.insert(&BB);
365 if (SideEffectingBlocks.count(&BB))
367 auto It = BaseMemAddrs.find(&BB);
368 if (It != BaseMemAddrs.end())
369 return It->second.count(
Addr);
375 AllocaInst *AI = cast<AllocaInst>(
Addr->stripInBoundsConstantOffsets());
378 if (Blocks.count(&BB))
388 BasicBlock *SinglePredFromOutlineRegion =
nullptr;
389 assert(!Blocks.count(CommonExitBlock) &&
390 "Expect a block outside the region!");
392 if (!Blocks.count(Pred))
394 if (!SinglePredFromOutlineRegion) {
395 SinglePredFromOutlineRegion = Pred;
396 }
else if (SinglePredFromOutlineRegion != Pred) {
397 SinglePredFromOutlineRegion =
nullptr;
402 if (SinglePredFromOutlineRegion)
403 return SinglePredFromOutlineRegion;
409 while (
I != BB->
end()) {
410 PHINode *Phi = dyn_cast<PHINode>(
I);
422 assert(!getFirstPHI(CommonExitBlock) &&
"Phi not expected");
430 if (Blocks.count(Pred))
435 Blocks.insert(CommonExitBlock);
436 OldTargets.push_back(NewExitBlock);
437 return CommonExitBlock;
444CodeExtractor::LifetimeMarkerInfo
448 LifetimeMarkerInfo
Info;
458 Info.LifeStart = IntrInst;
464 Info.LifeEnd = IntrInst;
469 if (isa<DbgInfoIntrinsic>(IntrInst))
477 if (!
Info.LifeStart || !
Info.LifeEnd)
483 if ((
Info.SinkLifeStart ||
Info.HoistLifeEnd) &&
488 if (
Info.HoistLifeEnd && !ExitBlock)
500 auto moveOrIgnoreLifetimeMarkers =
501 [&](
const LifetimeMarkerInfo &LMI) ->
bool {
504 if (LMI.SinkLifeStart) {
507 SinkCands.
insert(LMI.LifeStart);
509 if (LMI.HoistLifeEnd) {
510 LLVM_DEBUG(
dbgs() <<
"Hoisting lifetime.end: " << *LMI.LifeEnd <<
"\n");
511 HoistCands.
insert(LMI.LifeEnd);
520 if (Blocks.count(BB))
529 LifetimeMarkerInfo MarkerInfo = getLifetimeMarkers(CEAC, AI, ExitBlock);
530 bool Moved = moveOrIgnoreLifetimeMarkers(MarkerInfo);
542 for (
User *U : AI->users()) {
546 if (U->stripInBoundsConstantOffsets() != AI)
550 for (
User *BU : Bitcast->users()) {
562 << *Bitcast <<
" in out-of-region lifetime marker "
563 << *IntrInst <<
"\n");
564 LifetimeBitcastUsers.
push_back(IntrInst);
574 I->replaceUsesOfWith(
I->getOperand(1), CastI);
580 for (
User *U : AI->users()) {
581 if (U->stripInBoundsConstantOffsets() == AI) {
583 LifetimeMarkerInfo LMI = getLifetimeMarkers(CEAC, Bitcast, ExitBlock);
599 if (Bitcasts.
empty())
602 LLVM_DEBUG(
dbgs() <<
"Sinking alloca (via bitcast): " << *AI <<
"\n");
604 for (
unsigned I = 0, E = Bitcasts.
size();
I != E; ++
I) {
606 const LifetimeMarkerInfo &LMI = BitcastLifetimeInfo[
I];
608 "Unsafe to sink bitcast without lifetime markers");
609 moveOrIgnoreLifetimeMarkers(LMI);
611 LLVM_DEBUG(
dbgs() <<
"Sinking bitcast-of-alloca: " << *BitcastAddr
613 SinkCands.
insert(BitcastAddr);
627 if (AllowVarArgs &&
F->getFunctionType()->isVarArg()) {
628 auto containsVarArgIntrinsic = [](
const Instruction &
I) {
629 if (
const CallInst *CI = dyn_cast<CallInst>(&
I))
630 if (
const Function *Callee = CI->getCalledFunction())
631 return Callee->getIntrinsicID() == Intrinsic::vastart ||
632 Callee->getIntrinsicID() == Intrinsic::vaend;
636 for (
auto &BB : *
F) {
637 if (Blocks.count(&BB))
652 for (
auto &OI :
II.operands()) {
658 for (
User *U :
II.users())
670void CodeExtractor::severSplitPHINodesOfEntry(
BasicBlock *&Header) {
671 unsigned NumPredsFromRegion = 0;
672 unsigned NumPredsOutsideRegion = 0;
674 if (Header != &Header->getParent()->getEntryBlock()) {
675 PHINode *PN = dyn_cast<PHINode>(Header->begin());
683 ++NumPredsFromRegion;
685 ++NumPredsOutsideRegion;
689 if (NumPredsOutsideRegion <= 1)
return;
701 Blocks.remove(OldPred);
702 Blocks.insert(NewBB);
707 if (NumPredsFromRegion) {
720 for (AfterPHIs = OldPred->
begin(); isa<PHINode>(AfterPHIs); ++AfterPHIs) {
721 PHINode *PN = cast<PHINode>(AfterPHIs);
747void CodeExtractor::severSplitPHINodesOfExits(
752 for (
PHINode &PN : ExitBB->phis()) {
762 if (IncomingVals.
size() <= 1)
769 ExitBB->getName() +
".split",
770 ExitBB->getParent(), ExitBB);
774 if (Blocks.count(PredBB))
775 PredBB->getTerminator()->replaceUsesOfWith(ExitBB, NewBB);
777 Blocks.insert(NewBB);
784 for (
unsigned i : IncomingVals)
786 for (
unsigned i :
reverse(IncomingVals))
793void CodeExtractor::splitReturnBlocks() {
797 Block->splitBasicBlock(RI->getIterator(),
Block->getName() +
".ret");
815Function *CodeExtractor::constructFunction(
const ValueSet &inputs,
816 const ValueSet &outputs,
826 switch (NumExitBlocks) {
833 std::vector<Type *> ParamTy;
834 std::vector<Type *> AggParamTy;
835 ValueSet StructValues;
841 if (AggregateArgs && !ExcludeArgsFromAggregate.
contains(
value)) {
842 AggParamTy.push_back(
value->getType());
843 StructValues.insert(
value);
845 ParamTy.push_back(
value->getType());
849 for (
Value *output : outputs) {
851 if (AggregateArgs && !ExcludeArgsFromAggregate.
contains(output)) {
852 AggParamTy.push_back(output->getType());
853 StructValues.insert(output);
860 (ParamTy.size() + AggParamTy.size()) ==
861 (inputs.size() + outputs.size()) &&
862 "Number of scalar and aggregate params does not match inputs, outputs");
863 assert((StructValues.empty() || AggregateArgs) &&
864 "Expeced StructValues only with AggregateArgs set");
867 size_t NumScalarParams = ParamTy.size();
869 if (AggregateArgs && !AggParamTy.empty()) {
872 StructTy, ArgsInZeroAddressSpace ? 0 :
DL.getAllocaAddrSpace()));
876 dbgs() <<
"Function type: " << *RetTy <<
" f(";
877 for (
Type *i : ParamTy)
878 dbgs() << *i <<
", ";
883 RetTy, ParamTy, AllowVarArgs && oldFunction->
isVarArg());
885 std::string SuffixToUse =
892 oldFunction->
getName() +
"." + SuffixToUse, M);
903 if (Attr.isStringAttribute()) {
904 if (Attr.getKindAsString() ==
"thunk")
907 switch (Attr.getKindAsEnum()) {
910 case Attribute::AllocSize:
911 case Attribute::Builtin:
912 case Attribute::Convergent:
913 case Attribute::JumpTable:
914 case Attribute::Naked:
915 case Attribute::NoBuiltin:
916 case Attribute::NoMerge:
917 case Attribute::NoReturn:
918 case Attribute::NoSync:
919 case Attribute::ReturnsTwice:
920 case Attribute::Speculatable:
921 case Attribute::StackAlignment:
922 case Attribute::WillReturn:
923 case Attribute::AllocKind:
924 case Attribute::PresplitCoroutine:
925 case Attribute::Memory:
926 case Attribute::NoFPClass:
927 case Attribute::CoroDestroyOnlyWhenComplete:
930 case Attribute::AlwaysInline:
931 case Attribute::Cold:
932 case Attribute::DisableSanitizerInstrumentation:
933 case Attribute::FnRetThunkExtern:
935 case Attribute::HybridPatchable:
936 case Attribute::NoRecurse:
937 case Attribute::InlineHint:
938 case Attribute::MinSize:
939 case Attribute::NoCallback:
940 case Attribute::NoDuplicate:
941 case Attribute::NoFree:
942 case Attribute::NoImplicitFloat:
943 case Attribute::NoInline:
944 case Attribute::NonLazyBind:
945 case Attribute::NoRedZone:
946 case Attribute::NoUnwind:
947 case Attribute::NoSanitizeBounds:
948 case Attribute::NoSanitizeCoverage:
949 case Attribute::NullPointerIsValid:
950 case Attribute::OptimizeForDebugging:
951 case Attribute::OptForFuzzing:
952 case Attribute::OptimizeNone:
953 case Attribute::OptimizeForSize:
954 case Attribute::SafeStack:
955 case Attribute::ShadowCallStack:
956 case Attribute::SanitizeAddress:
957 case Attribute::SanitizeMemory:
958 case Attribute::SanitizeNumericalStability:
959 case Attribute::SanitizeThread:
960 case Attribute::SanitizeHWAddress:
961 case Attribute::SanitizeMemTag:
962 case Attribute::SpeculativeLoadHardening:
963 case Attribute::StackProtect:
964 case Attribute::StackProtectReq:
965 case Attribute::StackProtectStrong:
966 case Attribute::StrictFP:
967 case Attribute::UWTable:
968 case Attribute::VScaleRange:
969 case Attribute::NoCfCheck:
970 case Attribute::MustProgress:
971 case Attribute::NoProfile:
972 case Attribute::SkipProfile:
975 case Attribute::Alignment:
976 case Attribute::AllocatedPointer:
977 case Attribute::AllocAlign:
978 case Attribute::ByVal:
979 case Attribute::Dereferenceable:
980 case Attribute::DereferenceableOrNull:
981 case Attribute::ElementType:
982 case Attribute::InAlloca:
983 case Attribute::InReg:
984 case Attribute::Nest:
985 case Attribute::NoAlias:
986 case Attribute::NoCapture:
987 case Attribute::NoUndef:
988 case Attribute::NonNull:
989 case Attribute::Preallocated:
990 case Attribute::ReadNone:
991 case Attribute::ReadOnly:
992 case Attribute::Returned:
993 case Attribute::SExt:
994 case Attribute::StructRet:
995 case Attribute::SwiftError:
996 case Attribute::SwiftSelf:
997 case Attribute::SwiftAsync:
998 case Attribute::ZExt:
999 case Attribute::ImmArg:
1000 case Attribute::ByRef:
1001 case Attribute::WriteOnly:
1002 case Attribute::Writable:
1003 case Attribute::DeadOnUnwind:
1004 case Attribute::Range:
1005 case Attribute::Initializes:
1017 if (NumExitBlocks == 0) {
1023 return isa<ReturnInst>(Term) || isa<ResumeInst>(Term);
1028 newFunction->
insert(newFunction->
end(), newRootNode);
1037 for (
unsigned i = 0, e = inputs.size(), aggIdx = 0; i != e; ++i) {
1039 if (AggregateArgs && StructValues.contains(inputs[i])) {
1045 StructTy, &*AggAI,
Idx,
"gep_" + inputs[i]->
getName(), TI);
1047 "loadgep_" + inputs[i]->getName(), TI);
1050 RewriteVal = &*ScalarAI++;
1052 std::vector<User *>
Users(inputs[i]->user_begin(), inputs[i]->user_end());
1055 if (
Blocks.count(inst->getParent()))
1056 inst->replaceUsesOfWith(inputs[i], RewriteVal);
1060 if (NumScalarParams) {
1062 for (
unsigned i = 0, e = inputs.size(); i != e; ++i, ++ScalarAI)
1063 if (!StructValues.contains(inputs[i]))
1065 for (
unsigned i = 0, e = outputs.size(); i != e; ++i, ++ScalarAI)
1066 if (!StructValues.contains(outputs[i]))
1074 for (
auto &U :
Users)
1078 if (
I->isTerminator() &&
I->getFunction() == oldFunction &&
1079 !
Blocks.count(
I->getParent()))
1080 I->replaceUsesOfWith(header, newHeader);
1096 auto *
II = dyn_cast<IntrinsicInst>(&
I);
1097 if (!
II || !
II->isLifetimeStartOrEnd())
1103 Value *Mem =
II->getOperand(1)->stripInBoundsOffsets();
1107 if (
II->getIntrinsicID() == Intrinsic::lifetime_start)
1108 LifetimesStart.
insert(Mem);
1109 II->eraseFromParent();
1126 bool InsertBefore) {
1127 for (
Value *Mem : Objects) {
1130 "Input memory not defined in original function");
1137 Marker->insertBefore(Term);
1141 if (!LifetimesStart.
empty()) {
1142 insertMarkers(Intrinsic::lifetime_start, LifetimesStart,
1146 if (!LifetimesEnd.
empty()) {
1147 insertMarkers(Intrinsic::lifetime_end, LifetimesEnd,
1158 ValueSet &outputs) {
1161 std::vector<Value *> params, ReloadOutputs, Reloads;
1162 ValueSet StructValues;
1170 unsigned ScalarInputArgNo = 0;
1172 for (
Value *input : inputs) {
1173 if (AggregateArgs && !ExcludeArgsFromAggregate.
contains(input))
1174 StructValues.
insert(input);
1176 params.push_back(input);
1177 if (input->isSwiftError())
1178 SwiftErrorArgs.
push_back(ScalarInputArgNo);
1184 unsigned ScalarOutputArgNo = 0;
1185 for (
Value *output : outputs) {
1186 if (AggregateArgs && !ExcludeArgsFromAggregate.
contains(output)) {
1187 StructValues.insert(output);
1190 new AllocaInst(output->getType(),
DL.getAllocaAddrSpace(),
1191 nullptr, output->
getName() +
".loc",
1193 ReloadOutputs.push_back(alloca);
1194 params.push_back(alloca);
1195 ++ScalarOutputArgNo;
1201 unsigned NumAggregatedInputs = 0;
1202 if (AggregateArgs && !StructValues.empty()) {
1203 std::vector<Type *> ArgTypes;
1204 for (
Value *V : StructValues)
1205 ArgTypes.push_back(
V->getType());
1210 StructArgTy,
DL.getAllocaAddrSpace(),
nullptr,
"structArg",
1214 if (ArgsInZeroAddressSpace &&
DL.getAllocaAddrSpace() != 0) {
1216 Struct, PointerType ::get(Context, 0),
"structArg.ascast");
1217 StructSpaceCast->insertAfter(
Struct);
1218 params.push_back(StructSpaceCast);
1220 params.push_back(
Struct);
1223 for (
unsigned i = 0, e = StructValues.size(); i != e; ++i) {
1224 if (inputs.contains(StructValues[i])) {
1230 GEP->insertInto(codeReplacer, codeReplacer->
end());
1232 NumAggregatedInputs++;
1239 NumExitBlocks > 1 ?
"targetBlock" :
"");
1251 for (
unsigned SwiftErrArgNo : SwiftErrorArgs) {
1252 call->
addParamAttr(SwiftErrArgNo, Attribute::SwiftError);
1253 newFunction->
addParamAttr(SwiftErrArgNo, Attribute::SwiftError);
1258 for (
unsigned i = 0, e = outputs.size(), scalarIdx = 0,
1259 aggIdx = NumAggregatedInputs;
1261 Value *Output =
nullptr;
1262 if (AggregateArgs && StructValues.contains(outputs[i])) {
1268 GEP->insertInto(codeReplacer, codeReplacer->
end());
1272 Output = ReloadOutputs[scalarIdx];
1276 outputs[i]->
getName() +
".reload",
1278 Reloads.push_back(
load);
1279 std::vector<User *>
Users(outputs[i]->user_begin(), outputs[i]->user_end());
1290 codeReplacer, 0, codeReplacer);
1297 std::map<BasicBlock *, BasicBlock *> ExitBlockMap;
1301 unsigned switchVal = 0;
1303 if (
Blocks.count(OldTarget))
1305 BasicBlock *&NewTarget = ExitBlockMap[OldTarget];
1312 OldTarget->getName() +
".exitStub",
1314 unsigned SuccNum = switchVal++;
1316 Value *brVal =
nullptr;
1317 assert(NumExitBlocks < 0xffff &&
"too many exit blocks for switch");
1318 switch (NumExitBlocks) {
1344 BasicBlock *NewTarget = ExitBlockMap[OldTarget];
1345 assert(NewTarget &&
"Unknown target block!");
1356 std::advance(ScalarOutputArgBegin, ScalarInputArgNo);
1358 std::advance(AggOutputArgBegin, ScalarInputArgNo + ScalarOutputArgNo);
1360 for (
unsigned i = 0, e = outputs.size(), aggIdx = NumAggregatedInputs; i != e;
1362 auto *OutI = dyn_cast<Instruction>(outputs[i]);
1370 if (
auto *InvokeI = dyn_cast<InvokeInst>(OutI))
1371 InsertPt = InvokeI->getNormalDest()->getFirstInsertionPt();
1372 else if (
auto *Phi = dyn_cast<PHINode>(OutI))
1373 InsertPt =
Phi->getParent()->getFirstInsertionPt();
1375 InsertPt = std::next(OutI->getIterator());
1377 assert((InsertPt->getFunction() == newFunction ||
1378 Blocks.count(InsertPt->getParent())) &&
1379 "InsertPt should be in new function");
1380 if (AggregateArgs && StructValues.contains(outputs[i])) {
1382 "Number of aggregate output arguments should match "
1383 "the number of defined values");
1388 StructArgTy, &*AggOutputArgBegin,
Idx,
"gep_" + outputs[i]->
getName(),
1397 "Number of scalar output arguments should match "
1398 "the number of defined values");
1399 new StoreInst(outputs[i], &*ScalarOutputArgBegin, InsertPt);
1400 ++ScalarOutputArgBegin;
1405 Type *OldFnRetTy = TheSwitch->
getParent()->getParent()->getReturnType();
1406 switch (NumExitBlocks) {
1414 }
else if (OldFnRetTy->
isVoidTy()) {
1459void CodeExtractor::moveCodeToFunction(
Function *newFunction) {
1463 Block->removeFromParent();
1470 newFuncIt = newFunction->
insert(std::next(newFuncIt),
Block);
1474void CodeExtractor::calculateNewCallTerminatorWeights(
1486 Distribution BranchDist;
1493 BlockNode ExitNode(i);
1496 BranchDist.addExit(ExitNode, ExitFreq);
1502 if (BranchDist.Total == 0) {
1508 BranchDist.normalize();
1511 for (
unsigned I = 0, E = BranchDist.Weights.size();
I < E; ++
I) {
1512 const auto &Weight = BranchDist.Weights[
I];
1515 BranchWeights[Weight.TargetNode.Index] = Weight.Amount;
1517 EdgeProbabilities[Weight.TargetNode.Index] = BP;
1521 LLVMContext::MD_prof,
1533 if (DVI->getFunction() != &
F)
1534 DVI->eraseFromParent();
1536 if (DVR->getFunction() != &
F)
1537 DVR->eraseFromParent();
1560 assert(OldSP->getUnit() &&
"Missing compile unit for subprogram");
1566 DISubprogram::SPFlagOptimized |
1567 DISubprogram::SPFlagLocalToUnit;
1570 0, SPType, 0, DINode::FlagZero, SPFlags);
1573 auto IsInvalidLocation = [&NewFunc](
Value *Location) {
1577 (!isa<Constant>(Location) && !isa<Instruction>(Location)))
1579 Instruction *LocationInst = dyn_cast<Instruction>(Location);
1580 return LocationInst && LocationInst->
getFunction() != &NewFunc;
1595 DINode *&NewVar = RemappedMetadata[OldVar];
1598 *OldVar->getScope(), *NewSP, Ctx, Cache);
1600 NewScope, OldVar->
getName(), OldVar->getFile(), OldVar->getLine(),
1601 OldVar->getType(),
false, DINode::FlagZero,
1602 OldVar->getAlignInBits());
1604 return cast<DILocalVariable>(NewVar);
1607 auto UpdateDbgLabel = [&](
auto *LabelRecord) {
1610 if (LabelRecord->getDebugLoc().getInlinedAt())
1612 DILabel *OldLabel = LabelRecord->getLabel();
1613 DINode *&NewLabel = RemappedMetadata[OldLabel];
1616 *OldLabel->
getScope(), *NewSP, Ctx, Cache);
1620 LabelRecord->setLabel(cast<DILabel>(NewLabel));
1623 auto UpdateDbgRecordsOnInst = [&](
Instruction &
I) ->
void {
1624 for (
DbgRecord &DR :
I.getDbgRecordRange()) {
1626 UpdateDbgLabel(DLR);
1647 UpdateDbgRecordsOnInst(
I);
1649 auto *DII = dyn_cast<DbgInfoIntrinsic>(&
I);
1655 if (
auto *DLI = dyn_cast<DbgLabelInst>(&
I)) {
1656 UpdateDbgLabel(DLI);
1660 auto *DVI = cast<DbgVariableIntrinsic>(DII);
1662 if (
any_of(DVI->location_ops(), IsInvalidLocation)) {
1667 if (
auto *DAI = dyn_cast<DbgAssignIntrinsic>(DVI);
1668 DAI && IsInvalidLocation(DAI->getAddress())) {
1674 if (!DVI->getDebugLoc().getInlinedAt())
1675 DVI->setVariable(GetUpdatedDIVariable(DVI->getVariable()));
1678 for (
auto *DII : DebugIntrinsicsToDelete)
1679 DII->eraseFromParent();
1680 for (
auto *DVR : DVRsToDelete)
1681 DVR->getMarker()->MarkedInstr->dropOneDbgRecord(DVR);
1693 *NewSP, Ctx, Cache));
1696 auto updateLoopInfoLoc = [&Ctx, &Cache, NewSP](
Metadata *MD) ->
Metadata * {
1697 if (
auto *Loc = dyn_cast_or_null<DILocation>(MD))
1731 assert(BPI &&
"Both BPI and BFI are required to preserve profile info");
1744 if (
auto *AI = dyn_cast<AssumeInst>(&
I)) {
1747 AI->eraseFromParent();
1754 splitReturnBlocks();
1762 if (!
Blocks.count(Succ)) {
1772 NumExitBlocks = ExitBlocks.
size();
1776 if (!
Blocks.contains(OldTarget))
1777 OldTargets.push_back(OldTarget);
1781 severSplitPHINodesOfEntry(header);
1782 severSplitPHINodesOfExits(ExitBlocks);
1786 "codeRepl", oldFunction,
1804 if (!
I.getDebugLoc())
1808 if (isa<DbgInfoIntrinsic>(
I))
1810 BranchI->setDebugLoc(
I.getDebugLoc());
1815 BranchI->insertInto(newFuncRoot, newFuncRoot->
end());
1817 ValueSet SinkingCands, HoistingCands;
1819 findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);
1829 for (
auto *
II : SinkingCands) {
1830 if (
auto *AI = dyn_cast<AllocaInst>(
II)) {
1832 if (!FirstSunkAlloca)
1833 FirstSunkAlloca = AI;
1836 assert((SinkingCands.
empty() || FirstSunkAlloca) &&
1837 "Did not expect a sink candidate without any allocas");
1838 for (
auto *
II : SinkingCands) {
1839 if (!isa<AllocaInst>(
II)) {
1840 cast<Instruction>(
II)->moveAfter(FirstSunkAlloca);
1844 if (!HoistingCands.
empty()) {
1847 for (
auto *
II : HoistingCands)
1860 constructFunction(inputs, outputs, header, newFuncRoot, codeReplacer,
1873 emitCallAndSwitchStatement(newFunction, codeReplacer, inputs, outputs);
1875 moveCodeToFunction(newFunction);
1887 if (BFI && NumExitBlocks > 1)
1888 calculateNewCallTerminatorWeights(codeReplacer, ExitWeights, BPI);
1900 for (
PHINode &PN : ExitBB->phis()) {
1901 Value *IncomingCodeReplacerVal =
nullptr;
1908 if (!IncomingCodeReplacerVal) {
1913 "PHI has two incompatbile incoming values from codeRepl");
1920 newFunction->
dump();
1934 auto *
I = dyn_cast_or_null<CallInst>(AssumeVH);
1939 if (
I->getFunction() != &OldFunc)
1946 auto *AffectedCI = dyn_cast_or_null<CallInst>(AffectedValVH);
1949 if (AffectedCI->getFunction() != &OldFunc)
1951 auto *AssumedInst = cast<Instruction>(AffectedCI->getOperand(0));
1952 if (AssumedInst->getFunction() != &OldFunc)
1960 ExcludeArgsFromAggregate.
insert(Arg);
AMDGPU Mark last scratch load
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Expand Atomic instructions
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
Analysis containing CSE Info
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Given that RA is a live value
This file defines the DenseMap class.
DenseMap< Block *, BlockRelaxAux > Blocks
static Function * getFunction(Constant *C)
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
iv Induction Variable Users
Move duplicate certain instructions close to their use
Module.h This file contains the declarations for the Module class.
uint64_t IntrinsicInst * II
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
This class represents a conversion between pointers from one address space to another.
an instruction to allocate memory on the stack
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
bool empty() const
empty - Check if the array is empty.
A cache of @llvm.assume calls within a function.
MutableArrayRef< ResultElem > assumptions()
Access the list of assumption handles currently tracked for this function.
void unregisterAssumption(AssumeInst *CI)
Remove an @llvm.assume intrinsic from this function's cache if it has been added to the cache earlier...
MutableArrayRef< ResultElem > assumptionsFor(const Value *V)
Access the list of assumptions which affect this value.
AttributeSet getFnAttrs() const
The function attributes are returned.
@ TombstoneKey
Use as Tombstone key for DenseMap of AttrKind.
@ None
No attributes have been set.
@ EmptyKey
Use as Empty key for DenseMap of AttrKind.
@ EndAttrKinds
Sentinel value useful for loops.
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
iterator_range< filter_iterator< BasicBlock::const_iterator, std::function< bool(const Instruction &)> > > instructionsWithoutDebug(bool SkipPseudoOp=true) const
Return a const iterator range over the instructions in the block, skipping any debug instructions.
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches,...
InstListType::const_iterator getFirstNonPHIIt() const
Iterator returning form of getFirstNonPHI.
InstListType::const_iterator const_iterator
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
const Function * getParent() const
Return the enclosing method, or null if none.
InstListType::iterator iterator
Instruction iterators...
LLVMContext & getContext() const
Get the context in which this basic block lives.
bool IsNewDbgInfoFormat
Flag recording whether or not this block stores debug-info in the form of intrinsic instructions (fal...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
std::optional< uint64_t > getProfileCountFromFreq(BlockFrequency Freq) const
Returns the estimated profile count of Freq.
void setBlockFreq(const BasicBlock *BB, BlockFrequency Freq)
BlockFrequency getBlockFreq(const BasicBlock *BB) const
getblockFreq - Return block frequency.
static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)
Analysis providing branch probability information.
void setEdgeProbability(const BasicBlock *Src, const SmallVectorImpl< BranchProbability > &Probs)
Set the raw probabilities for all edges from the given block.
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
static BranchProbability getUnknown()
static BranchProbability getZero()
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
This is the base class for all instructions that perform data casts.
static CastInst * CreatePointerCast(Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Create a BitCast, AddrSpaceCast or a PtrToInt cast instruction.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
DISubroutineType * createSubroutineType(DITypeRefArray ParameterTypes, DINode::DIFlags Flags=DINode::FlagZero, unsigned CC=0)
Create subroutine type.
void finalizeSubprogram(DISubprogram *SP)
Finalize a specific subprogram - no new variables may be added to this subprogram afterwards.
DISubprogram * createFunction(DIScope *Scope, StringRef Name, StringRef LinkageName, DIFile *File, unsigned LineNo, DISubroutineType *Ty, unsigned ScopeLine, DINode::DIFlags Flags=DINode::FlagZero, DISubprogram::DISPFlags SPFlags=DISubprogram::SPFlagZero, DITemplateParameterArray TParams=nullptr, DISubprogram *Decl=nullptr, DITypeArray ThrownTypes=nullptr, DINodeArray Annotations=nullptr, StringRef TargetFuncName="")
Create a new descriptor for the specified subprogram.
DITypeRefArray getOrCreateTypeArray(ArrayRef< Metadata * > Elements)
Get a DITypeRefArray, create one if required.
DILocalVariable * createAutoVariable(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNo, DIType *Ty, bool AlwaysPreserve=false, DINode::DIFlags Flags=DINode::FlagZero, uint32_t AlignInBits=0)
Create a new descriptor for an auto variable.
StringRef getName() const
DILocalScope * getScope() const
Get the local scope for this label.
static DILocalScope * cloneScopeForSubprogram(DILocalScope &RootScope, DISubprogram &NewSP, LLVMContext &Ctx, DenseMap< const MDNode *, MDNode * > &Cache)
Traverses the scope chain rooted at RootScope until it hits a Subprogram, recreating the chain with "...
Tagged DWARF-like metadata node.
StringRef getName() const
DISPFlags
Debug info subprogram flags.
A parsed version of the target data layout string in and methods for querying it.
Records a position in IR for a source label (DILabel).
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
This is the common base class for debug info intrinsics for variables.
Record of a variable value-assignment, aka a non instruction representation of the dbg....
void setVariable(DILocalVariable *NewVar)
Value * getAddress() const
DILocalVariable * getVariable() const
iterator_range< location_op_iterator > location_ops() const
Get the locations corresponding to the variable referenced by the debug info intrinsic.
static DebugLoc replaceInlinedAtSubprogram(const DebugLoc &DL, DISubprogram &NewSP, LLVMContext &Ctx, DenseMap< const MDNode *, MDNode * > &Cache)
Rebuild the entire inline-at chain by replacing the subprogram at the end of the chain with NewSP.
DILocation * getInlinedAt() const
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
DomTreeNodeBase< NodeT > * addNewBlock(NodeT *BB, NodeT *DomBB)
Add a new node to the dominator tree information.
DomTreeNodeBase< NodeT > * getNode(const NodeT *BB) const
getNode - return the (Post)DominatorTree node for the specified basic block.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Class to represent profile counts.
void addFnAttr(Attribute::AttrKind Kind)
Add function attributes to this function.
void setSubprogram(DISubprogram *SP)
Set the attached subprogram.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
const BasicBlock & getEntryBlock() const
const BasicBlock & front() const
DISubprogram * getSubprogram() const
Get the attached subprogram.
bool IsNewDbgInfoFormat
Is this function using intrinsics to record the position of debugging information,...
bool hasPersonalityFn() const
Check whether this function has a personality function.
Constant * getPersonalityFn() const
Get the personality function associated with this function.
void setPersonalityFn(Constant *Fn)
AttributeList getAttributes() const
Return the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
adds the attribute to the list of attributes for the given arg.
Function::iterator insert(Function::iterator Position, BasicBlock *BB)
Insert BB in the basic block list at Position.
bool doesNotReturn() const
Determine if the function cannot return.
void setEntryCount(ProfileCount Count, const DenseSet< GlobalValue::GUID > *Imports=nullptr)
Set the entry count for this function.
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
unsigned getAddressSpace() const
Module * getParent()
Get the module that this global value is contained inside of...
@ InternalLinkage
Rename collisions when linking (static functions).
bool isLifetimeStartOrEnd() const LLVM_READONLY
Return true if the instruction is a llvm.lifetime.start or llvm.lifetime.end marker.
unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
const Function * getFunction() const
Return the function this instruction belongs to.
BasicBlock * getSuccessor(unsigned Idx) const LLVM_READONLY
Return the specified successor. This instruction must be a terminator.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
void setSuccessor(unsigned Idx, BasicBlock *BB)
Update the specified successor to point at the provided block.
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
InstListType::iterator insertInto(BasicBlock *ParentBB, InstListType::iterator It)
Inserts an unlinked instruction into ParentBB at position It and returns the iterator of the inserted...
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
Value * getPointerOperand()
Represents a single loop in the control flow graph.
MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight, bool IsExpected=false)
Return metadata containing two branch weights.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
StringRef getName() const
Return the name of the corresponding LLVM basic block, or an empty string.
A Module instance is used to store all the information related to an LLVM module.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
void setIncomingBlock(unsigned i, BasicBlock *BB)
Value * removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty=true)
Remove an incoming value.
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Return a value (possibly void), from a function.
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, InsertPosition InsertBefore=nullptr)
A vector that has set insertion semantics.
ArrayRef< value_type > getArrayRef() const
size_type size() const
Determine the number of elements in the SetVector.
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
bool empty() const
Determine if the SetVector is empty or not.
bool insert(const value_type &X)
Insert a new element into the SetVector.
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
std::string str() const
str - Get the contents as an std::string.
constexpr bool empty() const
empty - Check if the string is empty.
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Type * getElementType(unsigned N) const
BasicBlock * getSuccessor(unsigned idx) const
static SwitchInst * Create(Value *Value, BasicBlock *Default, unsigned NumCases, InsertPosition InsertBefore=nullptr)
void setCondition(Value *V)
void addCase(ConstantInt *OnVal, BasicBlock *Dest)
Add an entry to the switch instruction.
void setDefaultDest(BasicBlock *DefaultCase)
Value * getCondition() const
CaseIt removeCase(CaseIt I)
This method removes the specified case and its successor from the switch instruction.
The instances of the Type class are immutable: once they are created, they are never changed.
static IntegerType * getInt1Ty(LLVMContext &C)
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt16Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isVoidTy() const
Return true if this is 'void'.
This function has undefined behavior.
bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
user_iterator user_begin()
void setName(const Twine &Name)
Change the name of the value.
const Value * stripInBoundsConstantOffsets() const
Strip off pointer casts and all-constant inbounds GEPs.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
void dump() const
Support for debugging, callable in GDB: V->dump()
const ParentTy * getParent() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
void remapAssignID(DenseMap< DIAssignID *, DIAssignID * > &Map, Instruction &I)
Replace DIAssignID uses and attachments with IDs from Map.
NodeAddr< PhiNode * > Phi
This is an optimization pass for GlobalISel generic memory operations.
bool stripDebugInfo(Function &F)
Function::ProfileCount ProfileCount
bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
void findDbgUsers(SmallVectorImpl< DbgVariableIntrinsic * > &DbgInsts, Value *V, SmallVectorImpl< DbgVariableRecord * > *DbgVariableRecords=nullptr)
Finds the debug info intrinsics describing a value.
auto successors(const MachineBasicBlock *BB)
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
auto predecessors(const MachineBasicBlock *BB)
void updateLoopMetadataDebugLocations(Instruction &I, function_ref< Metadata *(Metadata *)> Updater)
Update the debug locations contained within the MD_loop metadata attached to the instruction I,...
Representative of a block.
Distribution of unscaled probability weight.