Go to the documentation of this file.
23 #include "llvm/Config/llvm-config.h"
46 #define DEBUG_TYPE "coro-suspend-crossing"
52 class BlockToIndexMapping {
56 size_t size()
const {
return V.size(); }
66 assert(
I != V.end() && *
I ==
BB &&
"BasicBlockNumberng: Unknown block");
87 struct SuspendCrossingInfo {
88 BlockToIndexMapping Mapping;
104 return Block[Mapping.blockToIndex(
BB)];
113 size_t const DefIndex = Mapping.blockToIndex(DefBB);
114 size_t const UseIndex = Mapping.blockToIndex(UseBB);
116 bool const Result =
Block[UseIndex].Kills[DefIndex];
118 <<
" answer is " << Result <<
"\n");
122 bool isDefinitionAcrossSuspend(
BasicBlock *DefBB,
User *U)
const {
123 auto *
I = cast<Instruction>(U);
127 if (
auto *PN = dyn_cast<PHINode>(
I))
128 if (PN->getNumIncomingValues() > 1)
136 if (isa<CoroSuspendRetconInst>(
I) || isa<CoroSuspendAsyncInst>(
I)) {
138 assert(UseBB &&
"should have split coro.suspend into its own block");
141 return hasPathCrossingSuspendPoint(DefBB, UseBB);
144 bool isDefinitionAcrossSuspend(
Argument &A,
User *U)
const {
145 return isDefinitionAcrossSuspend(&
A.getParent()->getEntryBlock(), U);
149 auto *DefBB =
I.getParent();
154 if (isa<AnyCoroSuspendInst>(
I)) {
156 assert(DefBB &&
"should have split coro.suspend into its own block");
159 return isDefinitionAcrossSuspend(DefBB, U);
162 bool isDefinitionAcrossSuspend(
Value &V,
User *U)
const {
163 if (
auto *
Arg = dyn_cast<Argument>(&V))
164 return isDefinitionAcrossSuspend(*
Arg, U);
165 if (
auto *Inst = dyn_cast<Instruction>(&V))
166 return isDefinitionAcrossSuspend(*Inst, U);
169 "Coroutine could only collect Argument and Instruction now.");
174 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
178 for (
size_t I = 0,
N = BV.
size();
I <
N; ++
I)
180 dbgs() <<
" " << Mapping.indexToBlock(
I)->getName();
185 for (
size_t I = 0,
N =
Block.size();
I <
N; ++
I) {
187 dbgs() <<
B->getName() <<
":\n";
188 dump(
" Consumes", Block[
I].Consumes);
189 dump(
" Kills", Block[
I].Kills);
197 const size_t N = Mapping.size();
201 for (
size_t I = 0;
I <
N; ++
I) {
203 B.Consumes.resize(
N);
212 getBlockData(
CE->getParent()).End =
true;
220 auto &
B = getBlockData(SuspendBlock);
222 B.Kills |=
B.Consumes;
225 markSuspendBlock(CSI);
226 if (
auto *Save = CSI->getCoroSave())
227 markSuspendBlock(Save);
240 for (
size_t I = 0;
I <
N; ++
I) {
244 auto SuccNo = Mapping.blockToIndex(
SI);
249 auto SavedConsumes =
S.Consumes;
250 auto SavedKills =
S.Kills;
253 S.Consumes |=
B.Consumes;
259 S.Kills |=
B.Consumes;
264 S.Kills |=
S.Consumes;
274 S.Kills.reset(SuccNo);
278 Changed |= (
S.Kills != SavedKills) || (
S.Consumes != SavedConsumes);
280 if (
S.Kills != SavedKills) {
286 if (
S.Consumes != SavedConsumes) {
297 #undef DEBUG_TYPE // "coro-suspend-crossing"
298 #define DEBUG_TYPE "coro-frame"
301 class FrameTypeBuilder;
307 bool MayWriteBeforeCoroBegin;
310 bool MayWriteBeforeCoroBegin)
311 : Alloca(Alloca), Aliases(
std::move(Aliases)),
312 MayWriteBeforeCoroBegin(MayWriteBeforeCoroBegin) {}
314 struct FrameDataInfo {
324 for (
const auto &
P : Spills)
325 Defs.push_back(
P.first);
326 for (
const auto &
A : Allocas)
327 Defs.push_back(
A.Alloca);
332 auto Itr = FieldIndexMap.find(V);
333 assert(Itr != FieldIndexMap.end() &&
334 "Value does not have a frame field index");
339 assert((LayoutIndexUpdateStarted || FieldIndexMap.count(V) == 0) &&
340 "Cannot set the index for the same field twice.");
341 FieldIndexMap[V] =
Index;
345 auto Iter = FieldAlignMap.find(V);
346 assert(Iter != FieldAlignMap.end());
351 assert(FieldAlignMap.count(V) == 0);
352 FieldAlignMap.insert({V,
Align});
356 auto Iter = FieldOffsetMap.find(V);
357 assert(Iter != FieldOffsetMap.end());
362 assert(FieldOffsetMap.count(V) == 0);
363 FieldOffsetMap.insert({V,
Offset});
367 void updateLayoutIndex(FrameTypeBuilder &
B);
372 bool LayoutIndexUpdateStarted =
false;
388 dbgs() <<
"------------- " << Title <<
"--------------\n";
389 for (
const auto &
E : Spills) {
392 for (
auto *
I :
E.second)
398 dbgs() <<
"------------- Allocas --------------\n";
399 for (
const auto &
A : Allocas) {
406 using FieldIDType = size_t;
411 class FrameTypeBuilder {
417 FieldIDType LayoutFieldIndex;
426 bool IsFinished =
false;
441 bool IsHeader =
false) {
446 if (
auto *CI = dyn_cast<ConstantInt>(AI->
getArraySize()))
452 return addField(Ty, AI->
getAlign(), IsHeader);
482 void addFieldForAllocas(
const Function &
F, FrameDataInfo &FrameData,
487 bool IsHeader =
false,
488 bool IsSpillOfValue =
false) {
489 assert(!IsFinished &&
"adding fields to a finished builder");
490 assert(Ty &&
"must provide a type for a field");
497 if (FieldSize == 0) {
505 auto ABIAlign =
DL.getABITypeAlign(Ty);
507 (IsSpillOfValue && MaxFrameAlignment)
508 ? (*MaxFrameAlignment < ABIAlign ? *MaxFrameAlignment : ABIAlign)
510 if (!FieldAlignment) {
511 FieldAlignment = TyAlignment;
518 StructSize =
Offset + FieldSize;
522 Offset = OptimizedStructLayoutField::FlexibleOffset;
525 Fields.push_back({FieldSize,
Offset, Ty, 0, *FieldAlignment, TyAlignment});
526 return Fields.size() - 1;
533 assert(IsFinished &&
"not yet finished!");
537 Align getStructAlign()
const {
538 assert(IsFinished &&
"not yet finished!");
542 FieldIDType getLayoutFieldIndex(FieldIDType
Id)
const {
543 assert(IsFinished &&
"not yet finished!");
544 return Fields[
Id].LayoutFieldIndex;
547 Field getLayoutField(FieldIDType
Id)
const {
548 assert(IsFinished &&
"not yet finished!");
554 void FrameDataInfo::updateLayoutIndex(FrameTypeBuilder &
B) {
555 auto Updater = [&](
Value *
I) {
556 auto Field =
B.getLayoutField(getFieldIndex(
I));
557 setFieldIndex(
I,
Field.LayoutFieldIndex);
561 LayoutIndexUpdateStarted =
true;
562 for (
auto &
S : Spills)
564 for (
const auto &
A : Allocas)
566 LayoutIndexUpdateStarted =
false;
569 void FrameTypeBuilder::addFieldForAllocas(
const Function &
F,
570 FrameDataInfo &FrameData,
577 for (
auto AllocaList : NonOverlapedAllocas) {
578 auto *LargestAI = *AllocaList.begin();
579 FieldIDType
Id = addFieldForAlloca(LargestAI);
580 for (
auto *Alloca : AllocaList)
588 NonOverlapedAllocas.emplace_back(AllocaSetType(1, Alloca));
608 if (
auto *ConstSWI = dyn_cast<SwitchInst>(U)) {
609 auto *SWI =
const_cast<SwitchInst *
>(ConstSWI);
610 DefaultSuspendDest[SWI] = SWI->getDefaultDest();
611 SWI->setDefaultDest(SWI->getSuccessor(1));
616 auto ExtractAllocas = [&]() {
617 AllocaSetType Allocas;
620 Allocas.push_back(
A.Alloca);
624 StackLifetime::LivenessType::May);
625 StackLifetimeAnalyzer.run();
627 return StackLifetimeAnalyzer.getLiveRange(AI1).overlaps(
628 StackLifetimeAnalyzer.getLiveRange(AI2));
630 auto GetAllocaSize = [&](
const AllocaInfo &
A) {
632 assert(RetSize &&
"Variable Length Arrays (VLA) are not supported.\n");
640 sort(
FrameData.Allocas, [&](
const auto &Iter1,
const auto &Iter2) {
641 return GetAllocaSize(Iter1) > GetAllocaSize(Iter2);
649 for (
auto &AllocaSet : NonOverlapedAllocas) {
650 assert(!AllocaSet.empty() &&
"Processing Alloca Set is not empty.\n");
651 bool NoInference =
none_of(AllocaSet, [&](
auto Iter) {
652 return IsAllocaInferenre(Alloca, Iter);
660 bool Alignable = [&]() ->
bool {
661 auto *LargestAlloca = *AllocaSet.begin();
662 return LargestAlloca->getAlign().value() % Alloca->
getAlign().
value() ==
665 bool CouldMerge = NoInference && Alignable;
668 AllocaSet.push_back(Alloca);
673 NonOverlapedAllocas.emplace_back(AllocaSetType(1, Alloca));
678 for (
auto SwitchAndDefaultDest : DefaultSuspendDest) {
680 BasicBlock *DestBB = SwitchAndDefaultDest.second;
685 : NonOverlapedAllocas) {
686 if (AllocaSet.size() > 1) {
687 dbgs() <<
"In Function:" << F.getName() <<
"\n";
688 dbgs() <<
"Find Union Set "
690 dbgs() <<
"\tAllocas are \n";
691 for (auto Alloca : AllocaSet)
692 dbgs() <<
"\t\t" << *Alloca <<
"\n";
697 void FrameTypeBuilder::finish(
StructType *Ty) {
698 assert(!IsFinished &&
"already finished!");
703 LayoutFields.
reserve(Fields.size());
704 for (
auto &
Field : Fields) {
711 StructSize = SizeAndAlign.first;
712 StructAlign = SizeAndAlign.second;
715 return *
static_cast<Field *
>(
const_cast<void*
>(LayoutField.Id));
721 for (
auto &LayoutField : LayoutFields) {
722 auto &
F = getField(LayoutField);
723 if (!
isAligned(
F.TyAlignment, LayoutField.Offset))
731 FieldTypes.
reserve(LayoutFields.size() * 3 / 2);
733 for (
auto &LayoutField : LayoutFields) {
734 auto &
F = getField(LayoutField);
736 auto Offset = LayoutField.Offset;
742 if (
Offset != LastOffset) {
749 F.LayoutFieldIndex = FieldTypes.size();
751 FieldTypes.push_back(
F.Ty);
755 Ty->
setBody(FieldTypes, Packed);
759 auto Layout =
DL.getStructLayout(Ty);
760 for (
auto &
F : Fields) {
762 assert(Layout->getElementOffset(
F.LayoutFieldIndex) ==
F.Offset);
771 for (
auto *V : FrameData.getAllDefs()) {
772 if (DIVarCache.
find(V) != DIVarCache.
end())
780 DIVarCache.
insert({V, (*I)->getVariable()});
791 OS <<
"__int_" << cast<IntegerType>(Ty)->getBitWidth();
793 return MDName->getString();
801 return "__floating_type_";
804 if (
auto *PtrTy = dyn_cast<PointerType>(Ty)) {
805 if (PtrTy->isOpaque())
806 return "PointerType";
809 if (
Name ==
"UnknownType")
810 return "PointerType";
814 return MDName->getString();
818 if (!cast<StructType>(Ty)->hasName())
819 return "__LiteralStructType_";
825 if (Iter ==
'.' || Iter ==
':')
829 return MDName->getString();
832 return "UnknownType";
844 DIType *RetType =
nullptr;
847 auto BitWidth = cast<IntegerType>(Ty)->getBitWidth();
849 llvm::DINode::FlagArtificial);
853 llvm::DINode::FlagArtificial);
863 dwarf::DW_ATE_address,
864 llvm::DINode::FlagArtificial);
866 auto *DIStruct =
Builder.createStructType(
869 llvm::DINodeArray());
871 auto *StructTy = cast<StructType>(Ty);
873 for (
unsigned I = 0;
I < StructTy->getNumElements();
I++) {
875 Scope, LineNum, DITypeCache);
877 Elements.push_back(
Builder.createMemberType(
881 llvm::DINode::FlagArtificial, DITy));
884 Builder.replaceArrays(DIStruct,
Builder.getOrCreateArray(Elements));
893 dwarf::DW_ATE_address,
894 llvm::DINode::FlagArtificial);
897 DITypeCache.
insert({Ty, RetType});
914 FrameDataInfo &FrameData) {
919 if (!DIS || !DIS->getUnit() ||
925 "We could only build debug infomation for C++ coroutine now.\n");
931 "Coroutine with switch ABI should own Promise alloca");
942 unsigned LineNum = PromiseDIVariable->
getLine();
945 DIS,
"__coro_frame_ty", DFile, LineNum, Shape.
FrameSize * 8,
947 llvm::DINodeArray());
950 DataLayout Layout =
F.getParent()->getDataLayout();
955 unsigned ResumeIndex = coro::Shape::SwitchFieldIndex::Resume;
956 unsigned DestroyIndex = coro::Shape::SwitchFieldIndex::Destroy;
960 NameCache.
insert({ResumeIndex,
"__resume_fn"});
961 NameCache.
insert({DestroyIndex,
"__destroy_fn"});
962 NameCache.
insert({IndexIndex,
"__coro_index"});
969 TyCache.
insert({ResumeIndex,
972 dwarf::DW_ATE_address)});
976 dwarf::DW_ATE_address)});
985 dwarf::DW_ATE_unsigned_char)});
987 for (
auto *V : FrameData.getAllDefs()) {
988 if (DIVarCache.
find(V) == DIVarCache.
end())
991 auto Index = FrameData.getFieldIndex(V);
993 NameCache.
insert({
Index, DIVarCache[V]->getName()});
1000 OffsetCache.
insert({ResumeIndex, {8, 0}});
1001 OffsetCache.
insert({DestroyIndex, {8, 8}});
1006 for (
auto *V : FrameData.getAllDefs()) {
1007 auto Index = FrameData.getFieldIndex(V);
1010 {
Index, {FrameData.getAlign(V), FrameData.getOffset(V)}});
1018 unsigned UnknownTypeNum = 0;
1030 assert(Ty->
isSized() &&
"We can't handle type which is not sized.\n");
1032 AlignInBits = OffsetCache[
Index].first * 8;
1033 OffsetInBits = OffsetCache[
Index].second * 8;
1037 DITy = TyCache[
Index];
1039 DITy =
solveDIType(DBuilder, Ty, Layout, FrameDITy, LineNum, DITypeCache);
1040 assert(DITy &&
"SolveDIType shouldn't return nullptr.\n");
1047 FrameDITy,
Name, DFile, LineNum, SizeInBits, AlignInBits, OffsetInBits,
1048 llvm::DINode::FlagArtificial, DITy));
1054 DFile, LineNum, FrameDITy,
1055 true, DINode::FlagArtificial);
1065 if (
auto *SubProgram = dyn_cast<DISubprogram>(PromiseDIScope)) {
1066 auto RetainedNodes = SubProgram->getRetainedNodes();
1068 RetainedNodes.end());
1069 RetainedNodesVec.push_back(FrameDIVar);
1070 SubProgram->replaceOperandWith(
1074 DBuilder.insertDeclare(Shape.
FramePtr, FrameDIVar,
1088 FrameDataInfo &FrameData) {
1093 Name.append(
".Frame");
1094 return StructType::create(
C,
Name);
1099 if (Shape.
ABI == coro::ABI::Async)
1101 FrameTypeBuilder
B(
C,
DL, MaxFrameAlignment);
1106 if (Shape.
ABI == coro::ABI::Switch) {
1110 auto *FnPtrTy = FnTy->getPointerTo();
1114 (void)
B.addField(FnPtrTy,
None,
true);
1115 (void)
B.addField(FnPtrTy,
None,
true);
1121 FrameData.setFieldIndex(
1122 PromiseAlloca,
B.addFieldForAlloca(PromiseAlloca,
true));
1127 Type *IndexType = Type::getIntNTy(
C, IndexBits);
1129 SwitchIndexFieldId =
B.addField(IndexType,
None);
1131 assert(PromiseAlloca ==
nullptr &&
"lowering doesn't support promises");
1136 B.addFieldForAllocas(
F, FrameData, Shape);
1141 if (Shape.
ABI == coro::ABI::Switch && PromiseAlloca)
1144 FrameData.Allocas.emplace_back(
1147 for (
auto &
S : FrameData.Spills) {
1148 Type *FieldType =
S.first->getType();
1151 if (
const Argument *
A = dyn_cast<Argument>(
S.first))
1152 if (
A->hasByValAttr())
1153 FieldType =
A->getParamByValType();
1155 B.addField(FieldType,
None,
false ,
true );
1156 FrameData.setFieldIndex(
S.first,
Id);
1160 FrameData.updateLayoutIndex(
B);
1164 switch (Shape.
ABI) {
1165 case coro::ABI::Switch: {
1167 auto IndexField =
B.getLayoutField(*SwitchIndexFieldId);
1179 case coro::ABI::Retcon:
1180 case coro::ABI::RetconOnce: {
1183 = (
B.getStructSize() <=
Id->getStorageSize() &&
1184 B.getStructAlign() <=
Id->getStorageAlignment());
1187 case coro::ABI::Async: {
1197 "The alignment requirment of frame variables cannot be higher than "
1198 "the alignment of the async function context");
1236 const CoroBeginInst &CB,
const SuspendCrossingInfo &Checker,
1237 bool ShouldUseLifetimeStartInfo)
1239 ShouldUseLifetimeStartInfo(ShouldUseLifetimeStartInfo) {}
1246 if (PI.isEscaped() && !DT.dominates(&CoroBegin, PI.getEscapingInst())) {
1247 MayWriteBeforeCoroBegin =
true;
1269 if (
SI.getValueOperand() != U->get())
1282 auto IsSimpleStoreThenLoad = [&]() {
1283 auto *AI = dyn_cast<AllocaInst>(
SI.getPointerOperand());
1291 while (!StoreAliases.empty()) {
1293 for (
User *U :
I->users()) {
1296 if (
auto *LI = dyn_cast<LoadInst>(U)) {
1303 if (
auto *
S = dyn_cast<StoreInst>(U))
1304 if (
S->getPointerOperand() ==
I)
1306 if (
auto *II = dyn_cast<IntrinsicInst>(U))
1307 if (II->isLifetimeStartOrEnd())
1311 if (
auto *BI = dyn_cast<BitCastInst>(U)) {
1312 StoreAliases.push_back(BI);
1322 if (!IsSimpleStoreThenLoad())
1349 if (II.
getIntrinsicID() != Intrinsic::lifetime_start || !IsOffsetKnown ||
1352 LifetimeStarts.insert(&II);
1356 for (
unsigned Op = 0, OpCount = CB.
arg_size();
Op < OpCount; ++
Op)
1362 bool getShouldLiveOnFrame()
const {
1363 if (!ShouldLiveOnFrame)
1364 ShouldLiveOnFrame = computeShouldLiveOnFrame();
1365 return ShouldLiveOnFrame.getValue();
1368 bool getMayWriteBeforeCoroBegin()
const {
return MayWriteBeforeCoroBegin; }
1371 assert(getShouldLiveOnFrame() &&
"This method should only be called if the "
1372 "alloca needs to live on the frame.");
1373 for (
const auto &
P : AliasOffetMap)
1376 "created before CoroBegin.");
1377 return AliasOffetMap;
1383 const SuspendCrossingInfo &Checker;
1390 bool MayWriteBeforeCoroBegin{
false};
1391 bool ShouldUseLifetimeStartInfo{
true};
1395 bool computeShouldLiveOnFrame()
const {
1400 if (ShouldUseLifetimeStartInfo && !LifetimeStarts.empty()) {
1402 for (
auto *
S : LifetimeStarts)
1403 if (Checker.isDefinitionAcrossSuspend(*
S,
I))
1420 for (
auto *U1 :
Users)
1421 for (
auto *U2 :
Users)
1422 if (Checker.isDefinitionAcrossSuspend(*U1, U2))
1430 MayWriteBeforeCoroBegin =
true;
1434 for (
auto &U :
I.uses())
1444 if (DT.
dominates(&CoroBegin, &
I) || !usedAfterCoroBegin(
I))
1447 if (!IsOffsetKnown) {
1448 AliasOffetMap[&
I].
reset();
1450 auto Itr = AliasOffetMap.find(&
I);
1451 if (Itr == AliasOffetMap.end()) {
1453 }
else if (Itr->second.hasValue() && Itr->second.getValue() !=
Offset) {
1456 AliasOffetMap[&
I].reset();
1478 CleanupPadInst::Create(CatchSwitch->
getParentPad(), {},
"", CurrentBlock);
1480 CleanupReturnInst::Create(CleanupPad, NewBlock, CurrentBlock);
1490 cast<Instruction>(
Builder.CreateBitCast(CB, FramePtrTy,
"FramePtr"));
1527 auto GetFramePointer = [&](
Value *Orig) ->
Value * {
1528 FieldIDType
Index = FrameData.getFieldIndex(Orig);
1534 if (
auto *AI = dyn_cast<AllocaInst>(Orig)) {
1535 if (
auto *CI = dyn_cast<ConstantInt>(AI->
getArraySize())) {
1536 auto Count = CI->getValue().getZExtValue();
1545 auto GEP = cast<GetElementPtrInst>(
1547 if (isa<AllocaInst>(Orig)) {
1555 if (
GEP->getResultElementType() != Orig->getType())
1556 return Builder.CreateBitCast(
GEP, Orig->getType(),
1557 Orig->getName() +
Twine(
".cast"));
1562 for (
auto const &
E : FrameData.Spills) {
1564 auto SpillAlignment =
Align(FrameData.getAlign(
Def));
1568 Type *ByValTy =
nullptr;
1569 if (
auto *
Arg = dyn_cast<Argument>(
Def)) {
1577 Arg->getParent()->removeParamAttr(
Arg->getArgNo(), Attribute::NoCapture);
1579 if (
Arg->hasByValAttr())
1580 ByValTy =
Arg->getParamByValType();
1581 }
else if (
auto *CSI = dyn_cast<AnyCoroSuspendInst>(
Def)) {
1586 auto *
I = cast<Instruction>(
Def);
1591 }
else if (
auto *II = dyn_cast<InvokeInst>(
I)) {
1595 InsertPt = NewBB->getTerminator();
1596 }
else if (isa<PHINode>(
I)) {
1599 if (
auto *CSI = dyn_cast<CatchSwitchInst>(DefBlock->
getTerminator()))
1604 assert(!
I->isTerminator() &&
"unexpected terminator");
1607 InsertPt =
I->getNextNode();
1611 auto Index = FrameData.getFieldIndex(
Def);
1612 Builder.SetInsertPoint(InsertPt);
1613 auto *
G =
Builder.CreateConstInBoundsGEP2_32(
1621 Builder.CreateAlignedStore(
Def,
G, SpillAlignment);
1625 Value *CurrentReload =
nullptr;
1626 for (
auto *U :
E.second) {
1630 if (CurrentBlock != U->getParent()) {
1631 CurrentBlock = U->getParent();
1634 auto *
GEP = GetFramePointer(
E.first);
1635 GEP->setName(
E.first->getName() +
Twine(
".reload.addr"));
1637 CurrentReload =
GEP;
1639 CurrentReload =
Builder.CreateAlignedLoad(
1641 SpillAlignment,
E.first->getName() +
Twine(
".reload"));
1645 bool AllowUnresolved =
false;
1650 .insertDeclare(CurrentReload, DDI->getVariable(),
1651 DDI->getExpression(), DDI->getDebugLoc(),
1661 if (
auto *DI = dyn_cast<DbgAddrIntrinsic>(U)) {
1668 if (
auto *PN = dyn_cast<PHINode>(U)) {
1669 assert(PN->getNumIncomingValues() == 1 &&
1670 "unexpected number of incoming "
1671 "values in the PHINode");
1672 PN->replaceAllUsesWith(CurrentReload);
1673 PN->eraseFromParent();
1691 if (Shape.
ABI == coro::ABI::Retcon || Shape.
ABI == coro::ABI::RetconOnce ||
1692 Shape.
ABI == coro::ABI::Async) {
1694 Builder.SetInsertPoint(&SpillBlock->front());
1695 for (
const auto &
P : FrameData.Allocas) {
1697 auto *
G = GetFramePointer(Alloca);
1701 G->takeName(Alloca);
1715 for (
const auto &
A : FrameData.Allocas) {
1717 UsersToUpdate.
clear();
1719 auto *
I = cast<Instruction>(U);
1721 UsersToUpdate.push_back(
I);
1723 if (UsersToUpdate.empty())
1725 auto *
G = GetFramePointer(Alloca);
1730 for (
auto *DVI : DIs)
1731 DVI->replaceUsesOfWith(Alloca,
G);
1734 I->replaceUsesOfWith(Alloca,
G);
1737 for (
const auto &
A : FrameData.Allocas) {
1739 if (
A.MayWriteBeforeCoroBegin) {
1743 "Coroutines cannot handle copying of array allocas yet");
1745 auto *
G = GetFramePointer(Alloca);
1752 for (
const auto &Alias :
A.Aliases) {
1753 auto *
FramePtr = GetFramePointer(Alloca);
1756 auto &
Value = Alias.second.getValue();
1758 auto *AliasPtr =
Builder.CreateGEP(Type::getInt8Ty(
C), FramePtrRaw,
1760 auto *AliasPtrTyped =
1761 Builder.CreateBitCast(AliasPtr, Alias.first->getType());
1762 Alias.first->replaceUsesWithIf(
1763 AliasPtrTyped, [&](
Use &U) {
return DT.
dominates(CB, U); });
1773 PHINode *UntilPHI =
nullptr) {
1774 auto *PN = cast<PHINode>(&SuccBB->
front());
1776 int Index = PN->getBasicBlockIndex(InsertedBB);
1778 PHINode *InputV = PHINode::Create(
1780 &InsertedBB->
front());
1782 PN->setIncomingValue(
Index, InputV);
1783 PN = dyn_cast<PHINode>(PN->getNextNode());
1784 }
while (PN != UntilPHI);
1818 auto *UnreachBB = BasicBlock::Create(
1824 auto *NewCleanupPadBB =
1825 BasicBlock::Create(CleanupPadBB->
getContext(),
1827 CleanupPadBB->
getParent(), CleanupPadBB);
1828 Builder.SetInsertPoint(NewCleanupPadBB);
1829 auto *SwitchType =
Builder.getInt8Ty();
1830 auto *SetDispatchValuePN =
1834 auto *SwitchOnDispatch =
Builder.CreateSwitch(SetDispatchValuePN, UnreachBB,
1837 int SwitchIndex = 0;
1841 auto *CaseBB = BasicBlock::Create(CleanupPadBB->
getContext(),
1843 Twine(
".from.") + Pred->getName(),
1844 CleanupPadBB->
getParent(), CleanupPadBB);
1846 CaseBB->setName(CleanupPadBB->
getName() +
Twine(
".from.") +
1848 Builder.SetInsertPoint(CaseBB);
1849 Builder.CreateBr(CleanupPadBB);
1857 SetDispatchValuePN->addIncoming(SwitchConstant, Pred);
1858 SwitchOnDispatch->addCase(SwitchConstant, CaseBB);
1865 for (
auto &
BB :
F) {
1866 for (
auto &Phi :
BB.phis()) {
1867 if (Phi.getNumIncomingValues() == 1) {
1868 Worklist.push_back(&Phi);
1873 while (!Worklist.empty()) {
1904 if (
auto *CleanupPad =
1905 dyn_cast_or_null<CleanupPadInst>(
BB.getFirstNonPHI())) {
1909 dyn_cast<CatchSwitchInst>(Pred->getTerminator())) {
1912 assert(CS->getUnwindDest() == &
BB);
1922 if ((LandingPad = dyn_cast_or_null<LandingPadInst>(
BB.getFirstNonPHI()))) {
1926 ReplPHI = PHINode::Create(LandingPad->
getType(), 1,
"", LandingPad);
1936 IncomingBB->setName(
BB.getName() +
Twine(
".from.") + Pred->getName());
1954 if (
auto *PN = dyn_cast<PHINode>(&
BB.front()))
1955 if (PN->getNumIncomingValues() > 1)
1956 WorkList.push_back(&
BB);
1965 return isa<CastInst>(&V) || isa<GetElementPtrInst>(&V) ||
1966 isa<BinaryOperator>(&V) || isa<CmpInst>(&V) || isa<SelectInst>(&V);
1972 return isa<CoroIdInst>(&
I) || isa<CoroSaveInst>(&
I) ||
1973 isa<CoroSuspendInst>(&
I);
1979 const SpillInfo &Spills) {
1980 for (
const auto &
E : Spills) {
1988 bool IsInCoroSuspendBlock = isa<AnyCoroSuspendInst>(U);
1990 auto *InsertBlock = IsInCoroSuspendBlock
1993 CurrentMaterialization = cast<Instruction>(
Def)->
clone();
1994 CurrentMaterialization->
setName(
Def->getName());
1996 IsInCoroSuspendBlock ? InsertBlock->getTerminator()
1997 : &*InsertBlock->getFirstInsertionPt());
1999 if (
auto *PN = dyn_cast<PHINode>(U)) {
2000 assert(PN->getNumIncomingValues() == 1 &&
2001 "unexpected number of incoming "
2002 "values in the PHINode");
2003 PN->replaceAllUsesWith(CurrentMaterialization);
2004 PN->eraseFromParent();
2017 auto *
BB =
I->getParent();
2018 if (&
BB->front() ==
I) {
2019 if (
BB->getSinglePredecessor()) {
2024 return BB->splitBasicBlock(
I,
Name);
2035 return isa<AnyCoroSuspendInst>(
BB->front());
2070 if (
auto FI = dyn_cast<CoroAllocaFreeInst>(
User))
2071 VisitedOrFreeBBs.
insert(FI->getParent());
2080 unsigned depth = 3) {
2083 if (depth == 0)
return false;
2102 for (
auto U : AI->
users()) {
2103 auto FI = dyn_cast<CoroAllocaFreeInst>(U);
2118 for (
auto AI : LocalAllocas) {
2124 Value *StackSave =
nullptr;
2126 StackSave =
Builder.CreateCall(
2130 auto Alloca =
Builder.CreateAlloca(
Builder.getInt8Ty(), AI->getSize());
2133 for (
auto U : AI->
users()) {
2135 if (isa<CoroAllocaGetInst>(U)) {
2142 auto FI = cast<CoroAllocaFreeInst>(U);
2150 DeadInsts.push_back(cast<Instruction>(U));
2153 DeadInsts.push_back(AI);
2167 if (isa<CoroAllocaGetInst>(U)) {
2170 auto FI = cast<CoroAllocaFreeInst>(U);
2174 DeadInsts.push_back(cast<Instruction>(U));
2178 DeadInsts.push_back(AI);
2181 return cast<Instruction>(Alloc);
2204 {V->getType()},
false);
2226 auto ValueBeforeCall =
Builder.CreateLoad(ValueTy, Alloca);
2232 if (isa<CallInst>(
Call)) {
2235 auto Invoke = cast<InvokeInst>(
Call);
2236 Builder.SetInsertPoint(Invoke->getNormalDest()->getFirstNonPHIOrDbg());
2241 Builder.CreateStore(ValueAfterCall, Alloca);
2254 if (isa<LoadInst>(
User) || isa<StoreInst>(
User))
2258 auto Call = cast<Instruction>(
User);
2279 auto ArgTy = cast<PointerType>(
Arg.getType());
2282 auto ValueTy = ArgTy->isOpaque() ? PointerType::getUnqual(
F.getContext())
2283 : ArgTy->getNonOpaquePointerElementType();
2288 auto Alloca =
Builder.CreateAlloca(ValueTy, ArgTy->getAddressSpace());
2289 Arg.replaceAllUsesWith(Alloca);
2292 auto InitialValue = Constant::getNullValue(ValueTy);
2293 Builder.CreateStore(InitialValue, Alloca);
2303 auto FinalValue =
Builder.CreateLoad(ValueTy, Alloca);
2308 AllocasToPromote.push_back(Alloca);
2318 for (
auto &
Arg :
F.args()) {
2319 if (!
Arg.hasSwiftErrorAttr())
continue;
2326 for (
auto &Inst :
F.getEntryBlock()) {
2327 auto Alloca = dyn_cast<AllocaInst>(&Inst);
2333 AllocasToPromote.push_back(Alloca);
2339 if (!AllocasToPromote.empty()) {
2348 const FrameDataInfo &FrameData,
2356 for (
auto *
Def : FrameData.getAllDefs()) {
2357 for (
User *U :
Def->users()) {
2358 auto Inst = cast<Instruction>(U);
2359 if (Inst->getParent() != CoroBegin->
getParent() ||
2363 Worklist.push_back(Inst);
2367 while (!Worklist.empty()) {
2369 for (
User *U :
Def->users()) {
2370 auto Inst = cast<Instruction>(U);
2374 Worklist.push_back(Inst);
2395 SuspendCrossingInfo &Checker) {
2400 DomSet.
insert(&
F.getEntryBlock());
2404 "should have split coro.suspend into its own block");
2418 if (
auto* II = dyn_cast<IntrinsicInst>(
I))
2425 Lifetimes.push_back(U);
2444 Checker.isDefinitionAcrossSuspend(DomBB, UI)) {
2447 if (collectLifetimeStart(UI, AI))
2455 if (Valid && Lifetimes.size() != 0) {
2458 if (isa<AllocaInst>(Lifetimes[0]->getOperand(1)))
2460 auto *Int8PtrTy = Type::getInt8PtrTy(
F.getContext());
2461 return CastInst::Create(Instruction::BitCast, AI, Int8PtrTy,
"",
2462 DomBB->getTerminator());
2465 auto *NewLifetime = Lifetimes[0]->clone();
2466 NewLifetime->replaceUsesOfWith(NewLifetime->getOperand(1), NewBitCast);
2467 NewLifetime->insertBefore(DomBB->getTerminator());
2471 S->eraseFromParent();
2480 const SuspendCrossingInfo &Checker,
2483 auto *AI = dyn_cast<AllocaInst>(&
I);
2495 bool ShouldUseLifetimeStartInfo =
2496 (Shape.
ABI != coro::ABI::Async && Shape.
ABI != coro::ABI::Retcon &&
2497 Shape.
ABI != coro::ABI::RetconOnce);
2498 AllocaUseVisitor Visitor{
F.getParent()->getDataLayout(), DT,
2500 ShouldUseLifetimeStartInfo};
2501 Visitor.visitPtr(*AI);
2502 if (!Visitor.getShouldLiveOnFrame())
2505 Visitor.getMayWriteBeforeCoroBegin());
2514 auto InsertPt =
F->getEntryBlock().getFirstInsertionPt();
2515 while (isa<IntrinsicInst>(InsertPt))
2517 Builder.SetInsertPoint(&
F->getEntryBlock(), InsertPt);
2521 bool SkipOutermostLoad = !isa<DbgValueInst>(DVI);
2523 Value *OriginalStorage = Storage;
2524 while (
auto *Inst = dyn_cast_or_null<Instruction>(Storage)) {
2525 if (
auto *LdInst = dyn_cast<LoadInst>(Inst)) {
2526 Storage = LdInst->getOperand(0);
2533 if (!SkipOutermostLoad)
2534 Expr = DIExpression::prepend(Expr, DIExpression::DerefBefore);
2535 }
else if (
auto *StInst = dyn_cast<StoreInst>(Inst)) {
2536 Storage = StInst->getOperand(0);
2543 if (!
Op || !AdditionalValues.empty()) {
2549 Expr = DIExpression::appendOpsToArg(Expr, Ops, 0,
false);
2551 SkipOutermostLoad =
false;
2564 if (
auto *
Arg = dyn_cast<llvm::Argument>(Storage)) {
2565 auto &Cached = DbgPtrAllocaCache[Storage];
2568 Arg->getName() +
".debug");
2569 Builder.CreateStore(Storage, Cached);
2580 Expr = DIExpression::prepend(Expr, DIExpression::DerefBefore);
2588 if (!isa<DbgValueInst>(DVI) && !isa<DbgAddrIntrinsic>(DVI)) {
2589 if (
auto *II = dyn_cast<InvokeInst>(Storage))
2590 DVI->
moveBefore(II->getNormalDest()->getFirstNonPHI());
2591 else if (
auto *CBI = dyn_cast<CallBrInst>(Storage))
2592 DVI->
moveBefore(CBI->getDefaultDest()->getFirstNonPHI());
2593 else if (
auto *InsertPt = dyn_cast<Instruction>(Storage)) {
2594 assert(!InsertPt->isTerminator() &&
2595 "Unimaged terminator that could return a storage.");
2597 }
else if (isa<Argument>(Storage))
2598 DVI->
moveAfter(
F->getEntryBlock().getFirstNonPHI());
2607 if (
Shape.
ABI == coro::ABI::Switch &&
2616 if (
auto *Save = CSI->getCoroSave())
2629 if (
auto *AsyncEnd = dyn_cast<CoroAsyncEndInst>(CE)) {
2630 auto *MustTailCallFn = AsyncEnd->getMustTailCallFunction();
2631 if (!MustTailCallFn)
2651 SuspendCrossingInfo Checker(
F,
Shape);
2654 FrameDataInfo FrameData;
2660 for (
int Repeat = 0; Repeat < 4; ++Repeat) {
2664 for (
User *U :
I.users())
2665 if (Checker.isDefinitionAcrossSuspend(
I, U))
2666 Spills[&
I].push_back(cast<Instruction>(U));
2681 Shape.
ABI != coro::ABI::RetconOnce)
2690 for (
User *U :
A.users())
2691 if (Checker.isDefinitionAcrossSuspend(
A, U))
2692 FrameData.Spills[&
A].push_back(cast<Instruction>(U));
2702 if (
Shape.
ABI == coro::ABI::Switch &&
2707 if (
auto AI = dyn_cast<CoroAllocaAllocInst>(&
I)) {
2710 LocalAllocas.push_back(AI);
2721 for (
User *U : Alloc->users()) {
2722 if (Checker.isDefinitionAcrossSuspend(*Alloc, U))
2723 FrameData.Spills[Alloc].push_back(cast<Instruction>(U));
2729 if (isa<CoroAllocaGetInst>(
I))
2732 if (isa<AllocaInst>(
I))
2735 for (
User *U :
I.users())
2736 if (Checker.isDefinitionAcrossSuspend(
I, U)) {
2738 if (
I.getType()->isTokenTy())
2740 "token definition is separated from the use by a suspend point");
2741 FrameData.Spills[&
I].push_back(cast<Instruction>(U));
2749 for (
auto &Iter : FrameData.Spills) {
2750 auto *V = Iter.first;
2754 if (Checker.isDefinitionAcrossSuspend(*V, DVI))
2755 FrameData.Spills[V].push_back(DVI);
2760 if (
Shape.
ABI == coro::ABI::Retcon ||
Shape.
ABI == coro::ABI::RetconOnce ||
2770 for (
auto I : DeadInstructions)
2771 I->eraseFromParent();
Instruction * getInsertPtAfterFramePtr() const
static int alignTo(int Num, int PowOf2)
void enqueueUsers(Instruction &I)
Enqueue the users of this instruction in the visit worklist.
static BasicBlock * splitBlockIfNotFirst(Instruction *I, const Twine &Name)
DIExpression * getExpression() const
This class represents an incoming formal argument to a Function.
CallInst * createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, ArrayRef< Value * > Arguments, IRBuilder<> &)
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
This is an optimization pass for GlobalISel generic memory operations.
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
uint64_t getNumLocationOperands() const
Return the number of unique location operands referred to (via DW_OP_LLVM_arg) in this expression; th...
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
A parsed version of the target data layout string in and methods for querying it.
bool hasOneUse() const
Return true if there is exactly one use of this value.
uint64_t Offset
The offset of this field in the final layout.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
static void eliminateSwiftError(Function &F, coro::Shape &Shape)
Eliminate all problematic uses of swifterror arguments and allocas from the function.
SwitchLoweringStorage SwitchLowering
uint64_t getSizeInBits() const
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
const Function * getParent() const
Return the enclosing method, or null if none.
bool isPointerTy() const
True if this is an instance of PointerType.
static void dumpAllocas(const SmallVectorImpl< AllocaInfo > &Allocas)
ScalarTy getFixedSize() const
static void splitAround(Instruction *I, const Twine &Name)
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
static Value * emitSetSwiftErrorValue(IRBuilder<> &Builder, Value *V, coro::Shape &Shape)
Set the given value as the current swifterror value.
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
void setBody(ArrayRef< Type * > Elements, bool isPacked=false)
Specify a body for an opaque identified type.
This class represents a no-op cast from one type to another.
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
void visitIntrinsicInst(IntrinsicInst &I)
void findDbgUsers(SmallVectorImpl< DbgVariableIntrinsic * > &DbgInsts, Value *V)
Finds the debug info intrinsics describing a value.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
void replaceVariableLocationOp(Value *OldValue, Value *NewValue)
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
static void rewritePHIsForCleanupPad(BasicBlock *CleanupPadBB, CleanupPadInst *CleanupPad)
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
This represents the llvm.coro.alloca.alloc instruction.
uint32_t getAlignInBits() const
void visitBitCastInst(BitCastInst &I)
The instances of the Type class are immutable: once they are created, they are never changed.
static void eliminateSwiftErrorArgument(Function &F, Argument &Arg, coro::Shape &Shape, SmallVectorImpl< AllocaInst * > &AllocasToPromote)
"Eliminate" a swifterror argument by reducing it to the alloca case and then loading and storing in t...
const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
DILocalVariable * getVariable() const
This is the common base class for memset/memcpy/memmove.
static void collectFrameAllocas(Function &F, coro::Shape &Shape, const SuspendCrossingInfo &Checker, SmallVectorImpl< AllocaInfo > &Allocas)
AllocaInst * getPromiseAlloca() const
static StringRef solveTypeName(Type *Ty)
Create name for Type.
void PromoteMemToReg(ArrayRef< AllocaInst * > Allocas, DominatorTree &DT, AssumptionCache *AC=nullptr)
Promote the specified list of alloca instructions into scalar registers, inserting PHI nodes as appro...
auto successors(MachineBasicBlock *BB)
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
DIDerivedType * createMemberType(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNo, uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits, DINode::DIFlags Flags, DIType *Ty, DINodeArray Annotations=nullptr)
Create debugging information entry for a member.
LLVM_NODISCARD T pop_back_val()
static bool willLeaveFunctionImmediatelyAfter(BasicBlock *BB, unsigned depth=3)
After we split the coroutine, will the given basic block be along an obvious exit path for the resump...
void buildCoroutineFrame(Function &F, Shape &Shape)
bool isFloatingPointTy() const
Return true if this is one of the six floating-point types.
Type * getNonOpaquePointerElementType() const
Only use this method in code that is not reachable with opaque pointers, or part of deprecated method...
AllocaInst * PromiseAlloca
bool isScalable() const
Returns whether the size is scaled by a runtime quantity (vscale).
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
Value * getVariableLocationOp(unsigned OpIdx) const
uint64_t ContextHeaderSize
LLVM Basic Block Representation.
print must be executed print the must be executed context for all instructions
void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const
Deallocate memory according to the rules of the active lowering.
DILocalVariable * createAutoVariable(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNo, DIType *Ty, bool AlwaysPreserve=false, DINode::DIFlags Flags=DINode::FlagZero, uint32_t AlignInBits=0)
Create a new descriptor for an auto variable.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
void insertAfter(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately after the specified instruction.
BasicBlock * AllocaSpillBlock
static void cleanupSinglePredPHIs(Function &F)
bool IsFrameInlineInStorage
static void rewriteMaterializableInstructions(IRBuilder<> &IRB, const SpillInfo &Spills)
This class represents a conversion between pointers from one address space to another.
iterator begin()
Get an iterator to the beginning of the SetVector.
static Value * emitGetSwiftErrorValue(IRBuilder<> &Builder, Type *ValueTy, coro::Shape &Shape)
Get the current swifterror value.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
This represents the llvm.dbg.value instruction.
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
void visitGetElementPtrInst(GetElementPtrInst &GEPI)
(vector float) vec_cmpeq(*A, *B) C
static bool localAllocaNeedsStackSave(CoroAllocaAllocInst *AI)
size_type size() const
size - Returns the number of bits in this bitvector.
StringRef getStructName() const
static Instruction * lowerNonLocalAlloca(CoroAllocaAllocInst *AI, coro::Shape &Shape, SmallVectorImpl< Instruction * > &DeadInsts)
Turn the given coro.alloca.alloc call into a dynamic allocation.
void visitGetElementPtrInst(GetElementPtrInst &I)
This represents the llvm.coro.suspend instruction.
uint64_t getElementOffsetInBits(unsigned Idx) const
StringRef getName() const
iterator_range< use_iterator > uses()
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
static const unsigned FramePtr
static void eliminateSwiftErrorAlloca(Function &F, AllocaInst *Alloca, coro::Shape &Shape)
Eliminate a formerly-swifterror alloca by inserting the get/set intrinsics and attempting to MemToReg...
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
void setExpression(DIExpression *NewExpr)
SmallVector< AnyCoroEndInst *, 4 > CoroEnds
LLVM_NODISCARD detail::scope_exit< typename std::decay< Callable >::type > make_scope_exit(Callable &&F)
TinyPtrVector< DbgDeclareInst * > FindDbgDeclareUses(Value *V)
Like FindDbgAddrUses, but only returns dbg.declare intrinsics, not dbg.addr.
const Value * getArraySize() const
Get the number of elements allocated.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
auto predecessors(MachineBasicBlock *BB)
void setName(const Twine &Name)
Change the name of the value.
User * getUser() const
Returns the User that contains this Use.
static void sinkSpillUsesAfterCoroBegin(Function &F, const FrameDataInfo &FrameData, CoroBeginInst *CoroBegin)
retcon and retcon.once conventions assume that all spill uses can be sunk after the coro....
void visitPHINode(PHINode &I)
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
This struct is a compact representation of a valid (non-zero power of two) alignment.
BasicBlock * ehAwareSplitEdge(BasicBlock *BB, BasicBlock *Succ, LandingPadInst *OriginalPad=nullptr, PHINode *LandingPadReplacement=nullptr, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions(), const Twine &BBName="")
Split the edge connect the specficed blocks in the case that Succ is an Exception Handling Block.
void replaceArrays(DICompositeType *&T, DINodeArray Elements, DINodeArray TParams=DINodeArray())
Replace arrays on a composite type.
LLVM_NODISCARD std::string str() const
str - Get the contents as an std::string.
static void sinkLifetimeStartMarkers(Function &F, coro::Shape &Shape, SuspendCrossingInfo &Checker)
For each local variable that all of its user are only used inside one of suspended region,...
bool isSwiftError() const
Return true if this alloca is used as a swifterror argument to a call.
DILocalScope * getScope() const
Get the local scope for this variable.
AsyncLoweringStorage AsyncLowering
bool isIntegerTy() const
True if this is an instance of IntegerType.
void visitMemIntrinsic(MemIntrinsic &I)
static bool isCoroutineStructureIntrinsic(Instruction &I)
static void createFramePtr(coro::Shape &Shape)
A MapVector that performs no allocations if smaller than a certain size.
Align Alignment
The required alignment of this field.
This is the common base class for debug info intrinsics for variables.
SmallVector< CallInst *, 2 > SwiftErrorOps
void visitSelectInst(SelectInst &I)
An instruction for storing to memory.
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
static void dumpSpills(StringRef Title, const SpillInfo &Spills)
static StructType * buildFrameType(Function &F, coro::Shape &Shape, FrameDataInfo &FrameData)
void setDefaultDest(BasicBlock *DefaultCase)
UnaryFunction for_each(R &&Range, UnaryFunction F)
Provide wrappers to std::for_each which take ranges instead of having to pass begin/end explicitly.
Module * getParent()
Get the module that this global value is contained inside of...
This represents the llvm.dbg.declare instruction.
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
DIExpression * createExpression(ArrayRef< uint64_t > Addr=None)
Create a new descriptor for the specified variable which has a complex address expression for its add...
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
This is an important class for using LLVM in a threaded context.
void moveAfter(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
DILocation * get() const
Get the underlying DILocation.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static void buildFrameDebugInfo(Function &F, coro::Shape &Shape, FrameDataInfo &FrameData)
Build artificial debug info for C++ coroutine frames to allow users to inspect the contents of the fr...
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Compute live ranges of allocas.
Class to represent pointers.
static Align getAlign(GlobalVariable *GV)
void findDbgValues(SmallVectorImpl< DbgValueInst * > &DbgValues, Value *V)
Finds the llvm.dbg.value intrinsics describing a value.
static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape)
void updatePhiNodes(BasicBlock *DestBB, BasicBlock *OldPred, BasicBlock *NewPred, PHINode *Until=nullptr)
Replaces all uses of OldPred with the NewPred block in all PHINodes in a block.
void visit(Iterator Start, Iterator End)
unsigned Log2_64_Ceil(uint64_t Value)
Return the ceil log base 2 of the specified value, 64 if the value is zero.
bool doesNotCapture(unsigned OpNo) const
Determine whether this data operand is not captured.
static bool materializable(Instruction &V)
iterator find(const_arg_type_t< KeyT > Val)
static bool isSuspendReachableFrom(BasicBlock *From, VisitedBlocksSet &VisitedOrFreeBBs)
Does control flow starting at the given block ever reach a suspend instruction before reaching a bloc...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static DIType * solveDIType(DIBuilder &Builder, Type *Ty, const DataLayout &Layout, DIScope *Scope, unsigned LineNum, DenseMap< Type *, DIType * > &DITypeCache)
uint64_t getPrefTypeAlignment(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
StandardInstrumentations SI(Debug, VerifyEach)
void visitAddrSpaceCastInst(AddrSpaceCastInst &I)
uint64_t getAlignment() const
This class represents the LLVM 'select' instruction.
bool OptimizeFrame
This would only be true if optimization are enabled.
@ CE
Windows NT (Windows on ARM)
static Value * emitSetAndGetSwiftErrorValueAround(Instruction *Call, AllocaInst *Alloca, coro::Shape &Shape)
Set the swifterror value from the given alloca before a call, then put in back in the alloca afterwar...
bool insert(const value_type &X)
Insert a new element into the SetVector.
static bool isAligned(const Value *Base, const APInt &Offset, Align Alignment, const DataLayout &DL)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Scope
Defines the scope in which this symbol should be visible: Default – Visible in the public interface o...
Class to represent struct types.
StringRef - Represent a constant reference to a string, i.e.
StringRef str() const
Return a StringRef for the vector contents.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Type * getType() const
All values are typed, get the type of this value.
unsigned pred_size(const BasicBlock *BB)
Get the number of predecessors of BB.
const Function * getFunction() const
Return the function this instruction belongs to.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
A base class for visitors over the uses of a pointer value.
LLVMContext & getContext() const
All values hold a context through their type.
bool isArrayAllocation() const
Return true if there is an allocation size parameter to the allocation instruction that is not 1.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static void movePHIValuesToInsertedBlock(BasicBlock *SuccBB, BasicBlock *InsertedBB, BasicBlock *PredBB, PHINode *UntilPHI=nullptr)
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
void visitCallBase(CallBase &CB)
CoroBeginInst * CoroBegin
uint64_t Size
The required size of this field in bytes.
StringRef getName() const
Return a constant reference to the value's name.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
APInt Offset
The constant offset of the use if that is known.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
RetconLoweringStorage RetconLowering
bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
std::pair< uint64_t, Align > performOptimizedStructLayout(MutableArrayRef< OptimizedStructLayoutField > Fields)
Compute a layout for a struct containing the given fields, making a best-effort attempt to minimize t...
unsigned getNumElements() const
Random access to the elements.
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
const Instruction & front() const
LLVMContext & getContext() const
Get the context in which this basic block lives.
DICompositeType * createStructType(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber, uint64_t SizeInBits, uint32_t AlignInBits, DINode::DIFlags Flags, DIType *DerivedFrom, DINodeArray Elements, unsigned RunTimeLang=0, DIType *VTableHolder=nullptr, StringRef UniqueIdentifier="")
Create debugging information entry for a struct.
BasicBlock * SplitEdge(BasicBlock *From, BasicBlock *To, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="")
Split the edge connecting the specified blocks, and return the newly created basic block between From...
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static Instruction * splitBeforeCatchSwitch(CatchSwitchInst *CatchSwitch)
void setUnwindEdgeTo(Instruction *TI, BasicBlock *Succ)
Sets the unwind edge of an instruction to a particular successor.
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Base class for scope-like contexts.
void visitAddrSpaceCastInst(AddrSpaceCastInst &ASC)
Should compile to something r4 addze r3 instead we get
uint64_t value() const
This is a hole in the type system and should not be abused.
unsigned arg_size() const
Value * salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps, SmallVectorImpl< uint64_t > &Ops, SmallVectorImpl< Value * > &AdditionalValues)
bool isComplex() const
Return whether the location is computed on the expression stack, meaning it cannot be a simple regist...
constexpr unsigned BitWidth
#define LLVM_NODISCARD
LLVM_NODISCARD - Warn if a type or return value is discarded.
void sort(IteratorTy Start, IteratorTy End)
AMDGPU Lower Kernel Arguments
void salvageDebugInfo(Instruction &I)
Assuming the instruction I is going to be deleted, attempt to salvage debug users of I by writing the...
DINodeArray getOrCreateArray(ArrayRef< Metadata * > Elements)
Get a DINodeArray, create one if required.
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const
Allocate memory according to the rules of the active lowering.
CoroIdInst * getSwitchCoroId() const
static void rewritePHIs(BasicBlock &BB)
bool isCPlusPlus(SourceLanguage S)
StringRef str() const
Explicit conversion to StringRef.
Value * getParentPad() const
A wrapper class for inspecting calls to intrinsic functions.
static bool isLifetimeStart(const Instruction *Inst)
static void lowerLocalAllocas(ArrayRef< CoroAllocaAllocInst * > LocalAllocas, SmallVectorImpl< Instruction * > &DeadInsts)
Turn each of the given local allocas into a normal (dynamic) alloca instruction.
static bool isSuspendBlock(BasicBlock *BB)
void setAlignment(Align Align)
iterator end()
Get an iterator to the end of the SetVector.
AnyCoroIdRetconInst * getRetconCoroId() const
bool isStructTy() const
True if this is an instance of StructType.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
static bool isLocalAlloca(CoroAllocaAllocInst *AI)
Is the given alloca "local", i.e.
Value * getArgOperand(unsigned i) const
const BasicBlock * getParent() const
std::string to_string(const T &Value)
Align max(MaybeAlign Lhs, Align Rhs)
iv Induction Variable Users
A range adaptor for a pair of iterators.
A raw_ostream that writes to an SmallVector or SmallString.
void reserve(size_type NumEntries)
Grow the densemap so that it can contain at least NumEntries items before resizing again.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
This class represents the llvm.coro.begin instruction.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Type * getElementType(unsigned N) const
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
A SetVector that performs no allocations if smaller than a certain size.
void visitStoreInst(StoreInst &SI)
StringRef toStringRef(SmallVectorImpl< char > &Out) const
This returns the twine as a single StringRef if it can be represented as such.
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
DIBasicType * createBasicType(StringRef Name, uint64_t SizeInBits, unsigned Encoding, DINode::DIFlags Flags=DINode::FlagZero)
Create debugging information entry for a basic type.
void visitIntrinsicInst(IntrinsicInst &II)
BlockVerifier::State From
void takeName(Value *V)
Transfer the name from V to this value.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
an instruction to allocate memory on the stack
SmallPtrSet< BasicBlock *, 8 > VisitedBlocksSet
static void cacheDIVar(FrameDataInfo &FrameData, DenseMap< Value *, DILocalVariable * > &DIVarCache)
Align getContextAlignment() const
TinyPtrVector - This class is specialized for cases where there are normally 0 or 1 element in a vect...
void reserve(size_type N)
void visitBitCastInst(BitCastInst &BC)
LLVM Value Representation.
iterator_range< user_iterator > users()
void setSwiftError(bool V)
Specify whether this alloca is used to represent a swifterror.
bool isAllocaPromotable(const AllocaInst *AI)
Return true if this alloca is legal for promotion.
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
A Use represents the edge between a Value definition and its users.
reference emplace_back(ArgTypes &&... Args)
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.