78 #define DEBUG_TYPE "rewrite-statepoints-for-gc" 98 #ifdef EXPENSIVE_CHECKS 101 static bool ClobberNonLive =
false;
129 bool Changed =
false;
133 if (
F.isDeclaration() ||
F.empty())
162 class RewriteStatepointsForGCLegacyPass :
public ModulePass {
168 RewriteStatepointsForGCLegacyPass() :
ModulePass(ID), Impl() {
173 bool runOnModule(
Module &M)
override {
174 bool Changed =
false;
177 if (
F.isDeclaration() ||
F.empty())
186 getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
F);
188 getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
F);
189 auto &DT = getAnalysis<DominatorTreeWrapperPass>(
F).getDomTree();
218 return new RewriteStatepointsForGCLegacyPass();
222 "rewrite-statepoints-for-gc",
223 "Make relocations explicit at statepoints",
false,
false)
227 "rewrite-statepoints-for-gc",
228 "Make relocations explicit at statepoints",
false,
false)
293 "Found non-leaf call without deopt info!");
302 GCPtrLivenessData &
Data);
313 if (
auto *PT = dyn_cast<PointerType>(T))
317 return PT->getAddressSpace() == 1;
331 if (
auto VT = dyn_cast<VectorType>(T))
343 if (
VectorType *VT = dyn_cast<VectorType>(Ty))
345 if (
ArrayType *AT = dyn_cast<ArrayType>(Ty))
373 PartiallyConstructedSafepointRecord &Result) {
378 dbgs() <<
"Live Variables:\n";
379 for (
Value *V : LiveSet)
380 dbgs() <<
" " << V->getName() <<
" " << *V <<
"\n";
384 dbgs() <<
"Number live values: " << LiveSet.size() <<
"\n";
386 Result.LiveSet = LiveSet;
401 struct BaseDefiningValueResult {
407 const bool IsKnownBase;
409 BaseDefiningValueResult(
Value *BDV,
bool IsKnownBase)
410 : BDV(BDV), IsKnownBase(IsKnownBase) {
415 assert(!MustBeBase || MustBeBase == IsKnownBase);
433 static BaseDefiningValueResult
438 if (isa<Argument>(I))
440 return BaseDefiningValueResult(I,
true);
442 if (isa<Constant>(I))
448 if (isa<LoadInst>(I))
449 return BaseDefiningValueResult(I,
true);
451 if (isa<InsertElementInst>(I))
455 return BaseDefiningValueResult(I,
false);
457 if (isa<ShuffleVectorInst>(I))
463 return BaseDefiningValueResult(I,
false);
467 if (
auto *
GEP = dyn_cast<GetElementPtrInst>(I))
472 if (
auto *BC = dyn_cast<BitCastInst>(I))
478 if (isa<CallInst>(I) || isa<InvokeInst>(
I))
479 return BaseDefiningValueResult(I,
true);
483 assert((isa<SelectInst>(I) || isa<PHINode>(I)) &&
484 "unknown vector instruction - no base found for vector element");
485 return BaseDefiningValueResult(I,
false);
494 "Illegal to ask for the base pointer of a non-pointer type");
499 if (isa<Argument>(I))
502 return BaseDefiningValueResult(I,
true);
504 if (isa<Constant>(I)) {
515 return BaseDefiningValueResult(
519 if (
CastInst *CI = dyn_cast<CastInst>(I)) {
520 Value *
Def = CI->stripPointerCasts();
523 assert(cast<PointerType>(Def->
getType())->getAddressSpace() ==
524 cast<PointerType>(CI->getType())->getAddressSpace() &&
525 "unsupported addrspacecast");
529 assert(!isa<CastInst>(Def) &&
"shouldn't find another cast here");
533 if (isa<LoadInst>(I))
535 return BaseDefiningValueResult(I,
true);
542 switch (II->getIntrinsicID()) {
546 case Intrinsic::experimental_gc_statepoint:
548 case Intrinsic::experimental_gc_relocate:
553 case Intrinsic::gcroot:
558 "interaction with the gcroot mechanism is not supported");
564 if (isa<CallInst>(I) || isa<InvokeInst>(I))
565 return BaseDefiningValueResult(I,
true);
569 assert(!isa<LandingPadInst>(I) &&
"Landing Pad is unimplemented");
571 if (isa<AtomicCmpXchgInst>(I))
575 return BaseDefiningValueResult(I,
true);
577 assert(!isa<AtomicRMWInst>(I) &&
"Xchg handled above, all others are " 578 "binary ops which don't apply to pointers");
583 if (isa<ExtractValueInst>(I))
584 return BaseDefiningValueResult(I,
true);
588 assert(!isa<InsertValueInst>(I) &&
589 "Base pointer for a struct is meaningless");
595 if (isa<ExtractElementInst>(I))
599 return BaseDefiningValueResult(I,
false);
605 assert((isa<SelectInst>(I) || isa<PHINode>(I)) &&
606 "missing instruction case in findBaseDefiningValing");
607 return BaseDefiningValueResult(I,
false);
612 Value *&Cached = Cache[
I];
618 assert(Cache[I] !=
nullptr);
626 auto Found = Cache.find(Def);
627 if (Found != Cache.end()) {
629 return Found->second;
638 if (!isa<PHINode>(V) && !isa<SelectInst>(V) &&
639 !isa<ExtractElementInst>(V) && !isa<InsertElementInst>(V) &&
640 !isa<ShuffleVectorInst>(V)) {
644 if (isa<Instruction>(V) &&
645 cast<Instruction>(V)->getMetadata(
"is_base_value")) {
664 BDVState() : BaseValue(
nullptr) {}
667 : Status(Status), BaseValue(BaseValue) {
671 explicit BDVState(
Value *BaseValue) : Status(
Base), BaseValue(BaseValue) {}
673 Status getStatus()
const {
return Status; }
674 Value *getBaseValue()
const {
return BaseValue; }
676 bool isBase()
const {
return getStatus() ==
Base; }
677 bool isUnknown()
const {
return getStatus() ==
Unknown; }
678 bool isConflict()
const {
return getStatus() == Conflict; }
681 return BaseValue == Other.BaseValue && Status == Other.Status;
684 bool operator!=(
const BDVState &other)
const {
return !(*
this == other); }
693 switch (getStatus()) {
704 OS <<
" (" << getBaseValue() <<
" - " 705 << (getBaseValue() ? getBaseValue()->getName() :
"nullptr") <<
"): ";
723 switch (LHS.getStatus()) {
728 assert(LHS.getBaseValue() &&
"can't be null");
733 if (LHS.getBaseValue() == RHS.getBaseValue()) {
734 assert(LHS == RHS &&
"equality broken!");
737 return BDVState(BDVState::Conflict);
739 assert(RHS.isConflict() &&
"only three states!");
740 return BDVState(BDVState::Conflict);
742 case BDVState::Conflict:
750 static BDVState
meetBDVState(
const BDVState &LHS,
const BDVState &RHS) {
753 "Math is wrong: meet does not commute!");
790 auto isExpectedBDVType = [](
Value *BDV) {
791 return isa<PHINode>(BDV) || isa<SelectInst>(BDV) ||
792 isa<ExtractElementInst>(BDV) || isa<InsertElementInst>(BDV) ||
793 isa<ShuffleVectorInst>(BDV);
810 while (!Worklist.
empty()) {
814 auto visitIncomingValue = [&](
Value *InVal) {
820 assert(isExpectedBDVType(Base) &&
"the only non-base values " 821 "we see should be base defining values");
822 if (States.
insert(std::make_pair(Base, BDVState())).second)
825 if (
PHINode *PN = dyn_cast<PHINode>(Current)) {
826 for (
Value *InVal : PN->incoming_values())
827 visitIncomingValue(InVal);
828 }
else if (
SelectInst *
SI = dyn_cast<SelectInst>(Current)) {
829 visitIncomingValue(
SI->getTrueValue());
830 visitIncomingValue(
SI->getFalseValue());
831 }
else if (
auto *EE = dyn_cast<ExtractElementInst>(Current)) {
832 visitIncomingValue(EE->getVectorOperand());
833 }
else if (
auto *
IE = dyn_cast<InsertElementInst>(Current)) {
834 visitIncomingValue(
IE->getOperand(0));
835 visitIncomingValue(
IE->getOperand(1));
836 }
else if (
auto *SV = dyn_cast<ShuffleVectorInst>(Current)) {
837 visitIncomingValue(SV->getOperand(0));
838 visitIncomingValue(SV->getOperand(1));
848 for (
auto Pair : States) {
849 LLVM_DEBUG(
dbgs() <<
" " << Pair.second <<
" for " << *Pair.first <<
"\n");
855 auto getStateForBDV = [&](
Value *baseValue) {
857 return BDVState(baseValue);
858 auto I = States.find(baseValue);
859 assert(I != States.end() &&
"lookup failed!");
863 bool Progress =
true;
866 const size_t OldSize = States.size();
873 for (
auto Pair : States) {
874 Value *BDV = Pair.first;
879 auto getStateForInput = [&](
Value *V)
mutable {
881 return getStateForBDV(BDV);
886 NewState =
meetBDVState(NewState, getStateForInput(
SI->getTrueValue()));
889 }
else if (
PHINode *PN = dyn_cast<PHINode>(BDV)) {
890 for (
Value *Val : PN->incoming_values())
891 NewState =
meetBDVState(NewState, getStateForInput(Val));
892 }
else if (
auto *EE = dyn_cast<ExtractElementInst>(BDV)) {
896 meetBDVState(NewState, getStateForInput(EE->getVectorOperand()));
897 }
else if (
auto *
IE = dyn_cast<InsertElementInst>(BDV)){
900 NewState =
meetBDVState(NewState, getStateForInput(
IE->getOperand(0)));
901 NewState =
meetBDVState(NewState, getStateForInput(
IE->getOperand(1)));
905 auto *SV = cast<ShuffleVectorInst>(BDV);
906 NewState =
meetBDVState(NewState, getStateForInput(SV->getOperand(0)));
907 NewState =
meetBDVState(NewState, getStateForInput(SV->getOperand(1)));
910 BDVState OldState = States[BDV];
911 if (OldState != NewState) {
913 States[BDV] = NewState;
917 assert(OldSize == States.size() &&
918 "fixed point shouldn't be adding any new nodes to state");
923 for (
auto Pair : States) {
924 LLVM_DEBUG(
dbgs() <<
" " << Pair.second <<
" for " << *Pair.first <<
"\n");
930 for (
auto Pair : States) {
932 BDVState State = Pair.second;
934 assert(!State.isUnknown() &&
"Optimistic algorithm didn't complete!");
940 if (State.isBase() && isa<ExtractElementInst>(
I) &&
941 isa<VectorType>(State.getBaseValue()->getType())) {
942 auto *EE = cast<ExtractElementInst>(
I);
947 State.getBaseValue(), EE->getIndexOperand(),
"base_ee", EE);
949 States[
I] = BDVState(BDVState::Base, BaseInst);
955 assert(!isa<InsertElementInst>(I) || State.isConflict());
957 if (!State.isConflict())
963 if (isa<PHINode>(I)) {
966 assert(NumPreds > 0 &&
"how did we reach here");
974 }
else if (
auto *EE = dyn_cast<ExtractElementInst>(I)) {
979 }
else if (
auto *
IE = dyn_cast<InsertElementInst>(I)) {
986 auto *SV = cast<ShuffleVectorInst>(
I);
993 Instruction *BaseInst = MakeBaseInstPlaceholder(I);
996 States[
I] = BDVState(BDVState::Conflict, BaseInst);
1014 assert(States.count(BDV));
1015 Base = States[BDV].getBaseValue();
1017 assert(Base &&
"Can't be null");
1027 for (
auto Pair : States) {
1029 BDVState State = Pair.second;
1032 assert(!State.isUnknown() &&
"Optimistic algorithm didn't complete!");
1033 if (!State.isConflict())
1036 if (
PHINode *BasePHI = dyn_cast<PHINode>(State.getBaseValue())) {
1037 PHINode *PN = cast<PHINode>(BDV);
1039 for (
unsigned i = 0; i < NumPHIValues; i++) {
1052 int BlockIndex = BasePHI->getBasicBlockIndex(InBB);
1053 if (BlockIndex != -1) {
1054 Value *OldBase = BasePHI->getIncomingValue(BlockIndex);
1055 BasePHI->addIncoming(OldBase, InBB);
1058 Value *
Base = getBaseForInput(InVal,
nullptr);
1066 "Sanity -- findBaseOrBDV should be pure!");
1075 BasePHI->addIncoming(Base, InBB);
1077 assert(BasePHI->getNumIncomingValues() == NumPHIValues);
1079 dyn_cast<SelectInst>(State.getBaseValue())) {
1084 BaseSI->setTrueValue(getBaseForInput(SI->
getTrueValue(), BaseSI));
1085 BaseSI->setFalseValue(getBaseForInput(SI->
getFalseValue(), BaseSI));
1086 }
else if (
auto *BaseEE =
1087 dyn_cast<ExtractElementInst>(State.getBaseValue())) {
1088 Value *InVal = cast<ExtractElementInst>(BDV)->getVectorOperand();
1091 BaseEE->setOperand(0, getBaseForInput(InVal, BaseEE));
1092 }
else if (
auto *BaseIE = dyn_cast<InsertElementInst>(State.getBaseValue())){
1093 auto *BdvIE = cast<InsertElementInst>(BDV);
1094 auto UpdateOperand = [&](
int OperandIdx) {
1095 Value *InVal = BdvIE->getOperand(OperandIdx);
1096 Value *
Base = getBaseForInput(InVal, BaseIE);
1097 BaseIE->setOperand(OperandIdx, Base);
1102 auto *BaseSV = cast<ShuffleVectorInst>(State.getBaseValue());
1103 auto *BdvSV = cast<ShuffleVectorInst>(BDV);
1104 auto UpdateOperand = [&](
int OperandIdx) {
1105 Value *InVal = BdvSV->getOperand(OperandIdx);
1106 Value *
Base = getBaseForInput(InVal, BaseSV);
1107 BaseSV->setOperand(OperandIdx, Base);
1117 for (
auto Pair : States) {
1118 auto *BDV = Pair.first;
1119 Value *
Base = Pair.second.getBaseValue();
1124 dbgs() <<
"Updating base value cache" 1125 <<
" for: " << BDV->getName() <<
" from: " 1126 << (Cache.count(BDV) ? Cache[BDV]->getName().str() :
"none")
1127 <<
" to: " << Base->
getName() <<
"\n");
1129 if (Cache.count(BDV)) {
1131 "must be something we 'know' is a base pointer");
1135 "base relation should be stable");
1139 assert(Cache.count(Def));
1162 for (
Value *ptr : live) {
1164 assert(base &&
"failed to find base pointer");
1165 PointerToBase[ptr] = base;
1166 assert((!isa<Instruction>(base) || !isa<Instruction>(ptr) ||
1167 DT->
dominates(cast<Instruction>(base)->getParent(),
1168 cast<Instruction>(ptr)->
getParent())) &&
1169 "The base we found better dominate the derived pointer");
1177 PartiallyConstructedSafepointRecord &result) {
1182 errs() <<
"Base Pairs (w/o Relocation):\n";
1183 for (
auto &Pair : PointerToBase) {
1184 errs() <<
" derived ";
1185 Pair.first->printAsOperand(
errs(),
false);
1187 Pair.second->printAsOperand(
errs(),
false);
1192 result.PointerToBase = PointerToBase;
1199 PartiallyConstructedSafepointRecord &result);
1206 GCPtrLivenessData RevisedLivenessData;
1208 for (
size_t i = 0; i < records.
size(); i++) {
1209 struct PartiallyConstructedSafepointRecord &
info = records[i];
1231 "All PHI nodes should have been removed!");
1269 const int LiveStart,
1273 if (LiveVariables.
empty())
1278 assert(ValIt != LiveVec.
end() &&
"Val not found in LiveVec!");
1279 size_t Index = std::distance(LiveVec.
begin(), ValIt);
1280 assert(Index < LiveVec.
size() &&
"Bug in std::find?");
1292 auto getGCRelocateDecl = [&] (
Type *Ty) {
1294 auto AS = Ty->getScalarType()->getPointerAddressSpace();
1296 if (
auto *VT = dyn_cast<VectorType>(Ty))
1307 for (
unsigned i = 0; i < LiveVariables.
size(); i++) {
1310 Builder.
getInt32(LiveStart + FindIndex(LiveVariables, BasePtrs[i]));
1313 Type *Ty = LiveVariables[i]->getType();
1314 if (!TypeToDeclMap.
count(Ty))
1315 TypeToDeclMap[Ty] = getGCRelocateDecl(Ty);
1316 Function *GCRelocateDecl = TypeToDeclMap[Ty];
1320 GCRelocateDecl, {StatepointToken, BaseIdx, LiveIdx},
1332 class DeferredReplacement {
1335 bool IsDeoptimize =
false;
1337 DeferredReplacement() =
default;
1341 assert(Old != New && Old && New &&
1342 "Cannot RAUW equal values or to / from null!");
1344 DeferredReplacement
D;
1350 static DeferredReplacement createDelete(
Instruction *ToErase) {
1351 DeferredReplacement
D;
1356 static DeferredReplacement createDeoptimizeReplacement(
Instruction *Old) {
1360 "Only way to construct a deoptimize deferred replacement");
1362 DeferredReplacement
D;
1364 D.IsDeoptimize =
true;
1369 void doReplacement() {
1373 assert(OldI != NewI &&
"Disallowed at construction?!");
1374 assert((!IsDeoptimize || !New) &&
1375 "Deoptimize intrinsics are not replaced!");
1398 const char *DeoptLowering =
"deopt-lowering";
1410 return "live-through";
1417 PartiallyConstructedSafepointRecord &Result,
1418 std::vector<DeferredReplacement> &Replacements) {
1435 if (
auto TransitionBundle =
1438 TransitionArgs = TransitionBundle->Inputs;
1444 bool IsDeoptimize =
false;
1455 if (DeoptLowering.
equals(
"live-in"))
1458 assert(DeoptLowering.
equals(
"live-through") &&
"Unsupported value!");
1462 if (
Function *F = dyn_cast<Function>(CallTarget)) {
1482 IsDeoptimize =
true;
1488 if (
auto *CI = dyn_cast<CallInst>(Call)) {
1489 CallInst *SPCall = Builder.CreateGCStatepointCall(
1490 StatepointID, NumPatchBytes, CallTarget, Flags, CallArgs,
1491 TransitionArgs, DeoptArgs, GCArgs,
"safepoint_token");
1506 assert(CI->getNextNode() &&
"Not a terminator, must have next!");
1507 Builder.SetInsertPoint(CI->getNextNode());
1508 Builder.SetCurrentDebugLocation(CI->getNextNode()->getDebugLoc());
1510 auto *II = cast<InvokeInst>(Call);
1515 InvokeInst *SPInvoke = Builder.CreateGCStatepointInvoke(
1516 StatepointID, NumPatchBytes, CallTarget, II->getNormalDest(),
1517 II->getUnwindDest(), Flags, CallArgs, TransitionArgs, DeoptArgs, GCArgs,
1518 "statepoint_token");
1531 BasicBlock *UnwindBlock = II->getUnwindDest();
1534 "can't safely insert in this block!");
1537 Builder.SetCurrentDebugLocation(II->getDebugLoc());
1541 Result.UnwindToken = ExceptionalToken;
1548 BasicBlock *NormalDest = II->getNormalDest();
1551 "can't safely insert in this block!");
1558 assert(Token &&
"Should be set in one of the above branches!");
1564 Replacements.push_back(
1565 DeferredReplacement::createDeoptimizeReplacement(Call));
1567 Token->
setName(
"statepoint_token");
1581 Replacements.emplace_back(
1582 DeferredReplacement::createRAUW(Call, GCResult));
1584 Replacements.emplace_back(DeferredReplacement::createDelete(Call));
1588 Result.StatepointToken = Token;
1602 PartiallyConstructedSafepointRecord &Result,
1603 std::vector<DeferredReplacement> &Replacements) {
1604 const auto &LiveSet = Result.LiveSet;
1605 const auto &PointerToBase = Result.PointerToBase;
1609 LiveVec.
reserve(LiveSet.size());
1610 BaseVec.
reserve(LiveSet.size());
1611 for (
Value *L : LiveSet) {
1613 assert(PointerToBase.count(L));
1614 Value *
Base = PointerToBase.find(L)->second;
1633 for (
User *U : GCRelocs) {
1640 Value *Alloca = AllocaMap[OriginalValue];
1646 "Should always have one since it's not a terminator");
1648 Value *CastedRelocatedValue =
1649 Builder.CreateBitCast(Relocate,
1650 cast<AllocaInst>(Alloca)->getAllocatedType(),
1654 Store->
insertAfter(cast<Instruction>(CastedRelocatedValue));
1657 VisitedLiveValues.
insert(OriginalValue);
1668 for (
auto RematerializedValuePair: RematerializedValues) {
1669 Instruction *RematerializedValue = RematerializedValuePair.first;
1670 Value *OriginalValue = RematerializedValuePair.second;
1673 "Can not find alloca for rematerialized value");
1674 Value *Alloca = AllocaMap[OriginalValue];
1680 VisitedLiveValues.
insert(OriginalValue);
1692 int InitialAllocaNum = 0;
1694 if (isa<AllocaInst>(I))
1702 std::size_t NumRematerializedValues = 0;
1708 auto emitAllocaFor = [&](
Value *LiveValue) {
1712 AllocaMap[LiveValue] = Alloca;
1717 for (
Value *V : Live)
1721 for (
const auto &
Info : Records)
1722 for (
auto RematerializedValuePair :
Info.RematerializedValues) {
1723 Value *OriginalValue = RematerializedValuePair.second;
1724 if (AllocaMap.
count(OriginalValue) != 0)
1727 emitAllocaFor(OriginalValue);
1728 ++NumRematerializedValues;
1740 for (
const auto &
Info : Records) {
1751 if (isa<InvokeInst>(Statepoint)) {
1760 if (ClobberNonLive) {
1767 for (
auto Pair : AllocaMap) {
1772 if (VisitedLiveValues.
count(Def)) {
1779 for (
auto *AI : ToClobber) {
1780 auto PT = cast<PointerType>(AI->getAllocatedType());
1789 if (
auto II = dyn_cast<InvokeInst>(Statepoint)) {
1790 InsertClobbersAt(&*II->getNormalDest()->getFirstInsertionPt());
1791 InsertClobbersAt(&*II->getUnwindDest()->getFirstInsertionPt());
1793 InsertClobbersAt(cast<Instruction>(Statepoint)->getNextNode());
1799 for (
auto Pair : AllocaMap) {
1810 if (!isa<ConstantExpr>(U)) {
1821 auto Last = std::unique(Uses.
begin(), Uses.
end());
1825 if (isa<PHINode>(
Use)) {
1838 Use->replaceUsesOfWith(Def, Load);
1846 if (
Instruction *Inst = dyn_cast<Instruction>(Def)) {
1847 if (
InvokeInst *Invoke = dyn_cast<InvokeInst>(Inst)) {
1850 BasicBlock *NormalDest = Invoke->getNormalDest();
1853 assert(!Inst->isTerminator() &&
1854 "The only terminator that can produce a value is " 1855 "InvokeInst which is handled above.");
1859 assert(isa<Argument>(Def));
1864 assert(PromotableAllocas.
size() == Live.size() + NumRematerializedValues &&
1865 "we must have the same allocas with lives");
1866 if (!PromotableAllocas.
empty()) {
1873 if (isa<AllocaInst>(I))
1875 assert(InitialAllocaNum == 0 &&
"We must not introduce any extra allocas");
1900 if (isa<CallInst>(Call)) {
1908 auto *II = cast<InvokeInst>(Call);
1910 Func, Values,
"", &*II->getNormalDest()->getFirstInsertionPt()));
1912 Func, Values,
"", &*II->getUnwindDest()->getFirstInsertionPt()));
1918 GCPtrLivenessData OriginalLivenessData;
1920 for (
size_t i = 0; i < records.
size(); i++) {
1921 struct PartiallyConstructedSafepointRecord &
info = records[i];
1934 Value *CurrentValue) {
1938 GEP->getPointerOperand());
1941 if (
CastInst *CI = dyn_cast<CastInst>(CurrentValue)) {
1942 if (!CI->isNoopCast(CI->getModule()->getDataLayout()))
1952 return CurrentValue;
1963 if (
CastInst *CI = dyn_cast<CastInst>(Instr)) {
1964 assert(CI->isNoopCast(CI->getModule()->getDataLayout()) &&
1965 "non noop cast is found during rematerialization");
1967 Type *SrcTy = CI->getOperand(0)->getType();
1972 Type *ValTy =
GEP->getSourceElementType();
1978 if (!
GEP->hasAllConstantIndices())
1997 for (
unsigned i = 0; i < PhiNum; i++)
2003 for (
unsigned i = 0; i < PhiNum; i++) {
2006 if (CIVI == CurrentIncomingValues.
end())
2008 BasicBlock *CurrentIncomingBB = CIVI->second;
2020 PartiallyConstructedSafepointRecord &
Info,
2022 const unsigned int ChainLengthThreshold = 10;
2028 for (
Value *LiveValue: Info.LiveSet) {
2031 assert(Info.PointerToBase.count(LiveValue));
2032 Value *RootOfChain =
2037 if ( ChainToBase.
size() == 0 ||
2038 ChainToBase.
size() > ChainLengthThreshold)
2043 if (RootOfChain != Info.PointerToBase[LiveValue]) {
2046 if (!OrigRootPhi || !AlternateRootPhi)
2062 assert(Info.LiveSet.count(AlternateRootPhi));
2073 if (isa<InvokeInst>(Call)) {
2081 LiveValuesToBeDeleted.
push_back(LiveValue);
2091 auto rematerializeChain = [&ChainToBase](
2100 assert(isa<GetElementPtrInst>(Instr) || isa<CastInst>(Instr));
2104 ClonedValue->
setName(Instr->getName() +
".remat");
2108 if (LastClonedValue) {
2116 "incorrect use in rematerialization chain");
2119 assert(OpValue != RootOfChain && OpValue != AlternateLiveBase);
2128 if (RootOfChain != AlternateLiveBase)
2132 LastClonedValue = ClonedValue;
2136 return LastClonedValue;
2141 if (isa<CallInst>(Call)) {
2144 Instruction *RematerializedValue = rematerializeChain(
2145 InsertBefore, RootOfChain, Info.PointerToBase[LiveValue]);
2146 Info.RematerializedValues[RematerializedValue] = LiveValue;
2148 auto *Invoke = cast<InvokeInst>(
Call);
2151 &*Invoke->getNormalDest()->getFirstInsertionPt();
2153 &*Invoke->getUnwindDest()->getFirstInsertionPt();
2155 Instruction *NormalRematerializedValue = rematerializeChain(
2156 NormalInsertBefore, RootOfChain, Info.PointerToBase[LiveValue]);
2157 Instruction *UnwindRematerializedValue = rematerializeChain(
2158 UnwindInsertBefore, RootOfChain, Info.PointerToBase[LiveValue]);
2160 Info.RematerializedValues[NormalRematerializedValue] = LiveValue;
2161 Info.RematerializedValues[UnwindRematerializedValue] = LiveValue;
2166 for (
auto LiveValue: LiveValuesToBeDeleted) {
2167 Info.LiveSet.remove(LiveValue);
2176 std::set<CallBase *> Uniqued;
2177 Uniqued.insert(ToUpdate.
begin(), ToUpdate.
end());
2178 assert(Uniqued.size() == ToUpdate.
size() &&
"no duplicates please!");
2209 "support for FCA unimplemented");
2230 for (
size_t i = 0; i < Records.size(); i++) {
2231 PartiallyConstructedSafepointRecord &
info = Records[i];
2249 Holders.reserve(Holders.size() + Records.size());
2250 for (
size_t i = 0; i < Records.size(); i++) {
2251 PartiallyConstructedSafepointRecord &
Info = Records[i];
2254 for (
auto Pair : Info.PointerToBase)
2266 for (
auto &
Info : Records) {
2267 errs() <<
"Base Pairs: (w/Relocation)\n";
2268 for (
auto Pair :
Info.PointerToBase) {
2269 errs() <<
" derived ";
2270 Pair.first->printAsOperand(
errs(),
false);
2272 Pair.second->printAsOperand(
errs(),
false);
2286 for (
auto &
Info : Records)
2287 for (
auto &BasePair :
Info.PointerToBase)
2288 if (isa<Constant>(BasePair.second))
2289 Info.LiveSet.remove(BasePair.first);
2292 CI->eraseFromParent();
2299 for (
size_t i = 0; i < Records.size(); i++)
2305 std::vector<DeferredReplacement> Replacements;
2313 for (
size_t i = 0; i < Records.size(); i++)
2318 for (
auto &PR : Replacements)
2321 Replacements.clear();
2323 for (
auto &
Info : Records) {
2332 Info.LiveSet.clear();
2333 Info.PointerToBase.clear();
2338 for (
size_t i = 0; i < Records.size(); i++) {
2339 PartiallyConstructedSafepointRecord &
Info = Records[i];
2355 "statepoint must be reachable or liveness is meaningless");
2357 if (!isa<Instruction>(V))
2360 auto *LiveInst = cast<Instruction>(V);
2362 "unreachable values should never be live");
2364 "basic SSA liveness expectation violated by liveness analysis");
2372 for (
auto *Ptr : Live)
2374 "must be a gc pointer type");
2378 return !Records.empty();
2382 template <
typename AttrHolder>
2386 if (AH.getDereferenceableBytes(Index))
2388 AH.getDereferenceableBytes(Index)));
2389 if (AH.getDereferenceableOrNullBytes(Index))
2391 AH.getDereferenceableOrNullBytes(Index)));
2396 AH.setAttributes(AH.getAttributes().removeAttributes(Ctx, Index, R));
2403 if (isa<PointerType>(
A.getType()))
2415 if (!isa<LoadInst>(I) && !isa<StoreInst>(I))
2430 unsigned ValidMetadataAfterRS4GC[] = {LLVMContext::MD_tbaa,
2431 LLVMContext::MD_range,
2432 LLVMContext::MD_alias_scope,
2433 LLVMContext::MD_nontemporal,
2434 LLVMContext::MD_nonnull,
2435 LLVMContext::MD_align,
2436 LLVMContext::MD_type};
2460 if (
auto *II = dyn_cast<IntrinsicInst>(&I))
2461 if (II->getIntrinsicID() == Intrinsic::invariant_start) {
2462 InvariantStartInstructions.
push_back(II);
2466 if (
MDNode *
Tag = I.getMetadata(LLVMContext::MD_tbaa)) {
2468 I.setMetadata(LLVMContext::MD_tbaa, MutableTBAA);
2473 if (
auto *Call = dyn_cast<CallBase>(&I)) {
2474 for (
int i = 0,
e = Call->
arg_size(); i !=
e; i++)
2478 if (isa<PointerType>(Call->
getType()))
2484 for (
auto *II : InvariantStartInstructions) {
2486 II->eraseFromParent();
2495 const auto &FunctionGCName = F.
getGC();
2496 const StringRef StatepointExampleName(
"statepoint-example");
2498 return (StatepointExampleName == FunctionGCName) ||
2499 (CoreCLRName == FunctionGCName);
2520 "need function body to rewrite statepoints in");
2524 if (
const auto *Call = dyn_cast<CallBase>(&I))
2543 if (NeedsRewrite(I)) {
2549 "no unreachable blocks expected");
2550 ParsePointNeeded.
push_back(cast<CallBase>(&I));
2555 if (ParsePointNeeded.
empty())
2563 if (BB.getUniquePredecessor()) {
2582 if (
auto *BI = dyn_cast<BranchInst>(TI))
2583 if (BI->isConditional())
2584 return dyn_cast<Instruction>(BI->getCondition());
2590 if (
auto *Cond = getConditionInst(TI))
2593 if (isa<ICmpInst>(Cond) && Cond->hasOneUse()) {
2595 Cond->moveBefore(TI);
2605 if (!isa<GetElementPtrInst>(I))
2609 for (
unsigned i = 0; i < I.getNumOperands(); i++)
2621 I.setOperand(0, Splat);
2647 if (isa<PHINode>(I))
2651 for (
Value *V : I.operands()) {
2653 "support for FCA unimplemented");
2673 for (
auto &I : *Succ) {
2680 "support for FCA unimplemented");
2700 for (
Value *V : Live) {
2701 if (
auto *I = dyn_cast<Instruction>(V)) {
2705 if (TermOkay && TI == I)
2708 "basic SSA liveness expectation violated by liveness analysis");
2725 GCPtrLivenessData &Data) {
2731 Data.LiveSet[&BB].clear();
2736 assert(!Data.LiveSet[&BB].count(
Kill) &&
"live set contains kill");
2741 Data.LiveIn[&BB] = Data.LiveSet[&BB];
2742 Data.LiveIn[&BB].set_union(Data.LiveOut[&BB]);
2743 Data.LiveIn[&BB].set_subtract(Data.KillSet[&BB]);
2744 if (!Data.LiveIn[&BB].empty())
2749 while (!Worklist.
empty()) {
2755 const auto OldLiveOutSize = LiveOut.
size();
2757 assert(Data.LiveIn.count(Succ));
2761 if (OldLiveOutSize == LiveOut.
size()) {
2767 Data.LiveOut[BB] = LiveOut;
2774 assert(Data.LiveIn.count(BB));
2777 if (OldLiveIn.size() != LiveTmp.
size()) {
2778 Data.LiveIn[BB] = LiveTmp;
2796 assert(Data.LiveOut.count(BB));
2805 LiveOut.remove(Inst);
2806 Out.insert(LiveOut.begin(), LiveOut.end());
2811 PartiallyConstructedSafepointRecord &
Info) {
2817 for (
auto V : Updated)
2818 if (Info.PointerToBase.insert({V, V}).second) {
2820 "Can't find base for unexpected live value!");
2825 for (
auto V : Updated)
2826 assert(Info.PointerToBase.count(V) &&
2827 "Must be able to find base for live value!");
2833 for (
auto KVPair : Info.PointerToBase)
2834 if (!Updated.count(KVPair.first))
2835 ToErase.
insert(KVPair.first);
2837 for (
auto *V : ToErase)
2838 Info.PointerToBase.erase(V);
2841 for (
auto KVPair : Info.PointerToBase)
2842 assert(Updated.count(KVPair.first) &&
"record for non-live value");
2845 Info.LiveSet = Updated;
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
static void rematerializeLiveValues(CallBase *Call, PartiallyConstructedSafepointRecord &Info, TargetTransformInfo &TTI)
A parsed version of the target data layout string in and methods for querying it. ...
static void unique_unsorted(SmallVectorImpl< T > &Vec)
Implement a unique function which doesn't require we sort the input vector.
static void computeLiveOutSeed(BasicBlock *BB, SetVector< Value *> &LiveTmp)
static bool isHandledGCPointerType(Type *T)
static cl::opt< bool, true > ClobberNonLiveOverride("rs4gc-clobber-non-live", cl::location(ClobberNonLive), cl::Hidden)
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
MapVector< Value *, Value * > DefiningValueMapTy
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
static void insertRelocationStores(iterator_range< Value::user_iterator > GCRelocs, DenseMap< Value *, AllocaInst *> &AllocaMap, DenseSet< Value *> &VisitedLiveValues)
This class represents an incoming formal argument to a Function.
LLVM_NODISCARD std::string str() const
str - Get the contents as an std::string.
static Value * findBaseDefiningValueCached(Value *I, DefiningValueMapTy &Cache)
Returns the base defining value for this value.
static bool AreEquivalentPhiNodes(PHINode &OrigRootPhi, PHINode &AlternateRootPhi)
static BDVState meetBDVStateImpl(const BDVState &LHS, const BDVState &RHS)
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
LLVM_NODISCARD T pop_back_val()
This class represents lattice values for constants.
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds...
Instruction * StatepointToken
The new gc.statepoint instruction itself.
size_type size() const
Determine the number of elements in the SetVector.
static void stripInvalidMetadataFromInstruction(Instruction &I)
Certain metadata on instructions are invalid after running RS4GC.
Instruction * UnwindToken
Instruction to which exceptional gc relocates are attached Makes it easier to iterate through them du...
A Module instance is used to store all the information related to an LLVM module. ...
void dropUnknownNonDebugMetadata(ArrayRef< unsigned > KnownIDs)
Drop all unknown metadata except for debug locations.
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
StatepointDirectives parseStatepointDirectivesFromAttrs(AttributeList AS)
Parse out statepoint directives from the function attributes present in AS.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
static void checkBasicSSA(DominatorTree &DT, SetVector< Value *> &Live, Instruction *TI, bool TermOkay=false)
Check that the items in 'Live' dominate 'TI'.
void push_back(const T &Elt)
static ConstantAggregateZero * get(Type *Ty)
This provides a very simple, boring adaptor for a begin and end iterator into a range type...
This class represents a function call, abstracting a target machine's calling convention.
INITIALIZE_PASS_BEGIN(RewriteStatepointsForGCLegacyPass, "rewrite-statepoints-for-gc", "Make relocations explicit at statepoints", false, false) INITIALIZE_PASS_END(RewriteStatepointsForGCLegacyPass
DominatorTree & getDomTree()
Flush DomTree updates and return DomTree.
Optional< std::vector< StOtherPiece > > Other
static void insertUseHolderAfter(CallBase *Call, const ArrayRef< Value *> Values, SmallVectorImpl< CallInst *> &Holders)
Insert holders so that each Value is obviously live through the entire lifetime of the call...
MapVector< BasicBlock *, SetVector< Value * > > KillSet
Values defined in this block.
const Value * getTrueValue() const
The two locations do not alias at all.
Analysis pass providing the TargetTransformInfo.
This instruction constructs a fixed permutation of two input vectors.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", Instruction *InsertBefore=nullptr, Instruction *MDFrom=nullptr)
MapVector< BasicBlock *, SetVector< Value * > > LiveSet
Values used in this block (and thus live); does not included values killed within this block...
LLVMContext & getContext() const
All values hold a context through their type.
static void stripNonValidAttributesFromPrototype(Function &F)
MapVector< BasicBlock *, SetVector< Value * > > LiveIn
Values live into this basic block (i.e.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
This class implements a map that also provides access to all stored values in a deterministic order...
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Analysis pass which computes a DominatorTree.
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
An instruction for reading from memory.
reverse_iterator rbegin()
AttrBuilder & addAttribute(Attribute::AttrKind Val)
Add an attribute to the builder.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
bool isVectorTy() const
True if this is an instance of VectorType.
static void stripNonValidDataFromBody(Function &F)
static BaseDefiningValueResult findBaseDefiningValueOfVector(Value *I)
Return a base defining value for the 'Index' element of the given vector instruction 'I'...
void reserve(size_type N)
static void makeStatepointExplicit(DominatorTree &DT, CallBase *Call, PartiallyConstructedSafepointRecord &Result, std::vector< DeferredReplacement > &Replacements)
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
bool hasAttribute(unsigned Index, Attribute::AttrKind Kind) const
Return true if the attribute exists at the given index.
static void computeLiveInValues(DominatorTree &DT, Function &F, GCPtrLivenessData &Data)
Compute the live-in set for every basic block in the function.
iterator begin()
Instruction iterator methods.
Value * getArgOperand(unsigned i) const
unsigned getAllocaAddrSpace() const
AnalysisUsage & addRequired()
static InsertElementInst * Create(Value *Vec, Value *NewElt, Value *Idx, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
#define INITIALIZE_PASS_DEPENDENCY(depName)
static void makeStatepointExplicitImpl(CallBase *Call, const SmallVectorImpl< Value *> &BasePtrs, const SmallVectorImpl< Value *> &LiveVariables, PartiallyConstructedSafepointRecord &Result, std::vector< DeferredReplacement > &Replacements)
static void relocationViaAlloca(Function &F, DominatorTree &DT, ArrayRef< Value *> Live, ArrayRef< PartiallyConstructedSafepointRecord > Records)
Do all the relocation update via allocas and mem2reg.
This class represents the LLVM 'select' instruction.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
This is the base class for all instructions that perform data casts.
'undef' values are things that do not have specified contents.
static bool isKnownBaseResult(Value *V)
Given the result of a call to findBaseDefiningValue, or findBaseOrBDV, is it known to be a base point...
Class to represent struct types.
LLVMContext & getContext() const
Get the global data context.
static Value * findBasePointer(Value *I, DefiningValueMapTy &Cache)
For a given value or instruction, figure out what base ptr its derived from.
A Use represents the edge between a Value definition and its users.
Value * getDerivedPtr() const
static Value * findRematerializableChainToBasePointer(SmallVectorImpl< Instruction *> &ChainToBase, Value *CurrentValue)
static bool isGCPointerType(Type *T)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
This file contains the simple types necessary to represent the attributes associated with functions a...
AttributeSet getRetAttributes() const
The attributes for the ret value are returned.
static ArrayRef< Use > GetDeoptBundleOperands(const CallBase *Call)
void setName(const Twine &Name)
Change the name of the value.
arg_iterator gc_args_begin() const
bool remove(const value_type &X)
Remove an item from the set vector.
void initializeRewriteStatepointsForGCLegacyPassPass(PassRegistry &)
LLVMContext & getContext() const
Retrieve the LLVM context.
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
static cl::opt< bool > PrintLiveSet("spp-print-liveset", cl::Hidden, cl::init(false))
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following: ...
Type * getType() const
All values are typed, get the type of this value.
bool insert(const value_type &X)
Insert a new element into the SetVector.
bool isStatepoint(const CallBase *Call)
const T & getValue() const LLVM_LVALUE_FUNCTION
const BasicBlock * getUniquePredecessor() const
Return the predecessor of this block if it has a unique predecessor block.
Class to represent array types.
iterator_range< arg_iterator > gc_args() const
range adapter for gc arguments
This class represents a no-op cast from one type to another.
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
AttrBuilder & remove(const AttrBuilder &B)
Remove the attributes from the builder.
const std::string & getGC() const
An instruction for storing to memory.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
ModulePass * createRewriteStatepointsForGCLegacyPass()
StatepointLiveSetTy LiveSet
The set of values known to be live across this safepoint.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Function * getDeclaration(Module *M, ID id, ArrayRef< Type *> Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
Analysis containing CSE Info
Optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
static unsigned chainToBasePointerCost(SmallVectorImpl< Instruction *> &Chain, TargetTransformInfo &TTI)
SetVector< Value * > StatepointLiveSetTy
void replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
iterator find(const_arg_type_t< KeyT > Val)
static SetVector< Value * > computeKillSet(BasicBlock *BB)
bool isVoidTy() const
Return true if this is 'void'.
const BasicBlock & getEntryBlock() const
an instruction for type-safe pointer arithmetic to access elements of arrays and structs ...
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata *> MDs)
BasicBlock * SplitBlockPredecessors(BasicBlock *BB, ArrayRef< BasicBlock *> Preds, const char *Suffix, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, bool PreserveLCSSA=false)
This method introduces at least one new basic block into the function and moves some of the predecess...
initializer< Ty > init(const Ty &Val)
Type * getReturnType() const
Returns the type of the ret val.
static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData, CallBase *Call, PartiallyConstructedSafepointRecord &result)
Given an updated version of the dataflow liveness results, update the liveset and base pointer maps f...
static void RemoveNonValidAttrAtIndex(LLVMContext &Ctx, AttrHolder &AH, unsigned Index)
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
A set of analyses that are preserved following a run of a transformation pass.
static AttributeSet get(LLVMContext &C, const AttrBuilder &B)
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
unsigned arg_size() const
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction...
Value * getCalledValue() const
LLVM Basic Block Representation.
The instances of the Type class are immutable: once they are created, they are never changed...
This is an important class for using LLVM in a threaded context.
size_t size() const
size - Get the array size.
This function has undefined behavior.
This is an important base class in LLVM.
bool set_union(const STy &S)
Compute This := This u S, return whether 'This' changed.
static cl::opt< unsigned > RematerializationThreshold("spp-rematerialization-threshold", cl::Hidden, cl::init(6))
Value * getIncomingValueForBlock(const BasicBlock *BB) const
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static AttributeList legalizeCallAttributes(AttributeList AL)
std::pair< iterator, bool > insert(const ValueT &V)
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
bool runOnFunction(Function &F, DominatorTree &, TargetTransformInfo &, const TargetLibraryInfo &)
Optional< uint32_t > NumPatchBytes
Represent the analysis usage information of a pass.
static Type * getVoidTy(LLVMContext &C)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
RematerializedValueMapTy RematerializedValues
Record live values we are rematerialized instead of relocating.
void setCallingConv(CallingConv::ID CC)
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
static FunctionType * get(Type *Result, ArrayRef< Type *> Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
A specialization of it's base class for read-write access to a gc.statepoint.
Interval::pred_iterator pred_end(Interval *I)
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
self_iterator getIterator()
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
static void stripNonValidData(Module &M)
The IR fed into RewriteStatepointsForGC may have had attributes and metadata implying dereferenceabil...
void setTailCallKind(TailCallKind TCK)
const Function * getFunction() const
Return the function this instruction belongs to.
static bool containsGCPtrType(Type *Ty)
Returns true if this type contains a gc pointer whether we know how to handle that type or not...
auto remove_if(R &&Range, UnaryPredicate P) -> decltype(adl_begin(Range))
Provide wrappers to std::remove_if which take ranges instead of having to pass begin/end explicitly...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
static StringRef getDeoptLowering(CallBase *Call)
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
iterator erase(const_iterator CI)
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Attribute getAttribute(unsigned Index, Attribute::AttrKind Kind) const
Return the attribute object that exists at the given index.
static void findLiveSetAtInst(Instruction *inst, GCPtrLivenessData &Data, StatepointLiveSetTy &out)
Given results from the dataflow liveness computation, find the set of live Values at a particular ins...
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
bool hasFnAttr(Attribute::AttrKind Kind) const
Determine whether this call has the given attribute.
auto find(R &&Range, const T &Val) -> decltype(adl_begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
static Value * findBaseOrBDV(Value *I, DefiningValueMapTy &Cache)
Return a base pointer for this value if known.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
static bool shouldRewriteStatepointsIn(Function &F)
Returns true if this function should be rewritten by this pass.
void sort(IteratorTy Start, IteratorTy End)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
static cl::opt< bool > PrintBasePointers("spp-print-base-pointers", cl::Hidden, cl::init(false))
static cl::opt< bool > AllowStatepointWithNoDeoptInfo("rs4gc-allow-statepoint-with-no-deopt-info", cl::Hidden, cl::init(true))
A SetVector that performs no allocations if smaller than a certain size.
arg_iterator gc_args_end() const
AttrBuilder & removeAttribute(Attribute::AttrKind Val)
Remove an attribute from the builder.
bool callsGCLeafFunction(const CallBase *Call, const TargetLibraryInfo &TLI)
Return true if this call calls a gc leaf function.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
static BDVState meetBDVState(const BDVState &LHS, const BDVState &RHS)
bool dominates(const Instruction *Def, const Use &U) const
Return true if Def dominates a use in User.
Module.h This file contains the declarations for the Module class.
Provides information about what library functions are available for the current target.
Indicates that this statepoint is a transition from GC-aware code to code that is not GC-aware...
LLVM_NODISCARD T pop_back_val()
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static BasicBlock * normalizeForInvokeSafepoint(BasicBlock *BB, BasicBlock *InvokeParent, DominatorTree &DT)
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
FunctionCallee getOrInsertFunction(StringRef Name, FunctionType *T, AttributeList AttributeList)
Look up the specified function in the module symbol table.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
AttributeList getAttributes() const
Return the parameter attributes for this call.
static const Function * getCalledFunction(const Value *V, bool LookThroughBitCast, bool &IsNoBuiltin)
void set_subtract(const STy &S)
Compute This := This - B TODO: We should be able to use set_subtract from SetOperations.h, but SetVector interface is inconsistent with DenseSet.
Value handle that asserts if the Value is deleted.
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
unsigned getVectorNumElements() const
static bool ClobberNonLive
A range adaptor for a pair of iterators.
Class to represent vector types.
static std::string suffixed_name_or(Value *V, StringRef Suffix, StringRef DefaultName)
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
MapVector< AssertingVH< Instruction >, AssertingVH< Value > > RematerializedValueMapTy
bool isStatepointDirectiveAttr(Attribute Attr)
Return true if the Attr is an attribute that is a statepoint directive.
Optional< uint64_t > StatepointID
iterator_range< user_iterator > users()
static const uint64_t DefaultStatepointID
void FoldSingleEntryPHINodes(BasicBlock *BB, MemoryDependenceResults *MemDep=nullptr)
We know that BB has one predecessor.
iterator insert(iterator I, T &&Elt)
const Value * getFalseValue() const
LLVM_NODISCARD bool equals(StringRef RHS) const
equals - Check for string equality, this is more efficient than compare() when the relative ordering ...
Call sites that get wrapped by a gc.statepoint (currently only in RewriteStatepointsForGC and potenti...
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
bool operator!=(uint64_t V1, const APInt &V2)
static void CreateGCRelocates(ArrayRef< Value *> LiveVariables, const int LiveStart, ArrayRef< Value *> BasePtrs, Instruction *StatepointToken, IRBuilder<> Builder)
Helper function to place all gc relocates necessary for the given statepoint.
unsigned getNumUses() const
This method computes the number of uses of this Value.
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
static void findLiveReferences(Function &F, DominatorTree &DT, ArrayRef< CallBase *> toUpdate, MutableArrayRef< struct PartiallyConstructedSafepointRecord > records)
static cl::opt< bool > PrintLiveSetSize("spp-print-liveset-size", cl::Hidden, cl::init(false))
bool empty() const
Return true if the builder contains no target-independent attributes.
LLVM_NODISCARD bool empty() const
StringRef getValueAsString() const
Return the attribute's value as a string.
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
StringRef getName() const
Return a constant reference to the value's name.
static bool isUnhandledGCPointerType(Type *Ty)
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
void insertAfter(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately after the specified instruction...
unsigned pred_size(const BasicBlock *BB)
Get the number of predecessors of BB.
bool empty() const
Determine if the SetVector is empty or not.
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
static void findBasePointers(const StatepointLiveSetTy &live, MapVector< Value *, Value *> &PointerToBase, DominatorTree *DT, DefiningValueMapTy &DVCache)
iterator_range< value_op_iterator > operand_values()
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
ilist_iterator< OptionsT, !IsReverse, IsConst > getReverse() const
Get a reverse iterator to the same node.
void preserve()
Mark an analysis as preserved.
static void insertRematerializationStores(const RematerializedValueMapTy &RematerializedValues, DenseMap< Value *, AllocaInst *> &AllocaMap, DenseSet< Value *> &VisitedLiveValues)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value *> Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
static bool insertParsePoints(Function &F, DominatorTree &DT, TargetTransformInfo &TTI, SmallVectorImpl< CallBase *> &ToUpdate)
bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Remove all blocks that can not be reached from the function's entry.
raw_ostream & operator<<(raw_ostream &OS, const APInt &I)
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Analysis pass providing the TargetLibraryInfo.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Represents calls to the gc.relocate intrinsic.
Mark the deopt arguments associated with the statepoint as only being "live-in".
Module * getParent()
Get the module that this global value is contained inside of...
LLVM Value Representation.
A vector that has set insertion semantics.
succ_range successors(Instruction *I)
static const Function * getParent(const Value *V)
AttributeSet getFnAttributes() const
The function attributes are returned.
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
MapVector< BasicBlock *, SetVector< Value * > > LiveOut
Values live out of this basic block (i.e.
This class implements an extremely fast bulk output stream that can only output to a stream...
void PromoteMemToReg(ArrayRef< AllocaInst *> Allocas, DominatorTree &DT, AssumptionCache *AC=nullptr)
Promote the specified list of alloca instructions into scalar registers, inserting PHI nodes as appro...
StringRef - Represent a constant reference to a string, i.e.
inst_range instructions(Function *F)
A container for analyses that lazily runs them and caches their results.
const LandingPadInst * getLandingPadInst() const
Return the landingpad instruction associated with the landing pad.
Legacy analysis pass which computes a DominatorTree.
bool operator==(uint64_t V1, const APInt &V2)
MapVector< Value *, Value * > PointerToBase
Mapping from live pointers to a base-defining-value.
void setIncomingValue(unsigned i, Value *V)
static void analyzeParsePointLiveness(DominatorTree &DT, GCPtrLivenessData &OriginalLivenessData, CallBase *Call, PartiallyConstructedSafepointRecord &Result)
bool isEmpty() const
Return true if there are no attributes.
InstListType::reverse_iterator reverse_iterator
MDNode * createMutableTBAAAccessTag(MDNode *Tag)
Return mutable version of the given mutable or immutable TBAA access tag.
LocationClass< Ty > location(Ty &L)
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute >> Attrs)
Create an AttributeList with the specified parameters in it.
iterator_range< arg_iterator > args()
bool empty() const
empty - Check if the array is empty.
A wrapper class for inspecting calls to intrinsic functions.
const BasicBlock * getParent() const
an instruction to allocate memory on the stack
static BaseDefiningValueResult findBaseDefiningValue(Value *I)
Helper function for findBasePointer - Will return a value which either a) defines the base pointer fo...
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
unsigned gcArgsStartIdx() const