41 #include "llvm/Config/llvm-config.h"
62 #include "llvm/IR/IntrinsicsAArch64.h"
103 using namespace llvm;
106 #define DEBUG_TYPE "codegenprepare"
108 STATISTIC(NumBlocksElim,
"Number of blocks eliminated");
109 STATISTIC(NumPHIsElim,
"Number of trivial PHIs eliminated");
110 STATISTIC(NumGEPsElim,
"Number of GEPs converted to casts");
111 STATISTIC(NumCmpUses,
"Number of uses of Cmp expressions replaced with uses of "
113 STATISTIC(NumCastUses,
"Number of uses of Cast expressions replaced with uses "
115 STATISTIC(NumMemoryInsts,
"Number of memory instructions whose address "
116 "computations were sunk");
118 "Number of phis created when address "
119 "computations were sunk to memory instructions");
121 "Number of select created when address "
122 "computations were sunk to memory instructions");
123 STATISTIC(NumExtsMoved,
"Number of [s|z]ext instructions combined with loads");
124 STATISTIC(NumExtUses,
"Number of uses of [s|z]ext instructions optimized");
126 "Number of and mask instructions added to form ext loads");
127 STATISTIC(NumAndUses,
"Number of uses of and mask instructions optimized");
128 STATISTIC(NumRetsDup,
"Number of return instructions duplicated");
129 STATISTIC(NumDbgValueMoved,
"Number of debug value instructions moved");
130 STATISTIC(NumSelectsExpanded,
"Number of selects turned into branches");
131 STATISTIC(NumStoreExtractExposed,
"Number of store(extractelement) exposed");
135 cl::desc(
"Disable branch optimizations in CodeGenPrepare"));
139 cl::desc(
"Disable GC optimizations in CodeGenPrepare"));
143 cl::desc(
"Disable select to branch conversion."));
147 cl::desc(
"Address sinking in CGP using GEPs."));
151 cl::desc(
"Enable sinkinig and/cmp into branches."));
155 cl::desc(
"Disable store(extract) optimizations in CodeGenPrepare"));
159 cl::desc(
"Stress test store(extract) optimizations in CodeGenPrepare"));
163 cl::desc(
"Disable ext(promotable(ld)) -> promoted(ext(ld)) optimization in "
168 cl::desc(
"Stress test ext(promotable(ld)) -> promoted(ext(ld)) "
169 "optimization in CodeGenPrepare"));
173 cl::desc(
"Disable protection against removing loop preheaders"));
177 cl::desc(
"Use profile info to add section prefix for hot/cold functions"));
180 "profile-unknown-in-special-section",
cl::Hidden,
181 cl::desc(
"In profiling mode like sampleFDO, if a function doesn't have "
182 "profile, we cannot tell the function is cold for sure because "
183 "it may be a function newly added without ever being sampled. "
184 "With the flag enabled, compiler can put such profile unknown "
185 "functions into a special section, so runtime system can choose "
186 "to handle it in a different way than .text section, to save "
187 "RAM for example. "));
191 cl::desc(
"Use the basic-block-sections profile to determine the text "
192 "section prefix for hot functions. Functions with "
193 "basic-block-sections profile will be placed in `.text.hot` "
194 "regardless of their FDO profile info. Other functions won't be "
195 "impacted, i.e., their prefixes will be decided by FDO/sampleFDO "
200 cl::desc(
"Skip merging empty blocks if (frequency of empty block) / "
201 "(frequency of destination block) is greater than this ratio"));
205 cl::desc(
"Force store splitting no matter what the target query says."));
209 cl::desc(
"Enable merging of redundant sexts when one is dominating"
214 cl::desc(
"Disables combining addressing modes with different parts "
215 "in optimizeMemoryInst."));
219 cl::desc(
"Allow creation of Phis in Address sinking."));
223 cl::desc(
"Allow creation of selects in Address sinking."));
227 cl::desc(
"Allow combining of BaseReg field in Address sinking."));
231 cl::desc(
"Allow combining of BaseGV field in Address sinking."));
235 cl::desc(
"Allow combining of BaseOffs field in Address sinking."));
239 cl::desc(
"Allow combining of ScaledReg field in Address sinking."));
244 cl::desc(
"Enable splitting large offset of GEP."));
248 cl::desc(
"Enable ICMP_EQ to ICMP_S(L|G)T conversion."));
252 cl::desc(
"Enable BFI update verification for "
257 cl::desc(
"Enable converting phi types in CodeGenPrepare"));
276 class TypePromotionTransaction;
287 std::unique_ptr<BlockFrequencyInfo>
BFI;
288 std::unique_ptr<BranchProbabilityInfo> BPI;
303 SetOfInstrs InsertedInsts;
307 InstrToOrigTy PromotedInsts;
310 SetOfInstrs RemovedInsts;
330 ValueToSExts ValToSExtendedUses;
340 std::unique_ptr<DominatorTree> DT;
351 StringRef getPassName()
const override {
return "CodeGen Prepare"; }
364 template <
typename F>
365 void resetIteratorIfInvalidatedWhileCalling(
BasicBlock *
BB,
F f) {
369 Value *CurValue = &*CurInstIterator;
376 if (IterHandle != CurValue) {
377 CurInstIterator =
BB->begin();
385 DT = std::make_unique<DominatorTree>(
F);
389 void removeAllAssertingVHReferences(
Value *V);
392 bool eliminateMostlyEmptyBlocks(
Function &
F);
402 Type *AccessTy,
unsigned AddrSpace);
404 bool optimizeInlineAsmInst(
CallInst *CS);
416 bool optimizeExtractElementInst(
Instruction *Inst);
417 bool dupRetToEnableTailCallOpts(
BasicBlock *
BB,
bool &ModifiedDT);
423 bool tryToPromoteExts(TypePromotionTransaction &TPT,
426 unsigned CreatedInstsCost = 0);
428 bool splitLargeGEPOffsets();
432 bool performAddressTypePromotion(
434 bool AllowPromotionWithoutCommonHeader,
435 bool HasPromoted, TypePromotionTransaction &TPT,
437 bool splitBranchCondition(
Function &
F,
bool &ModifiedDT);
444 bool optimizeCmp(
CmpInst *Cmp,
bool &ModifiedDT);
445 bool combineToUSubWithOverflow(
CmpInst *Cmp,
bool &ModifiedDT);
446 bool combineToUAddWithOverflow(
CmpInst *Cmp,
bool &ModifiedDT);
455 "Optimize for code generation",
false,
false)
471 DL = &
F.getParent()->getDataLayout();
473 bool EverMadeChange =
false;
475 InsertedInsts.clear();
476 PromotedInsts.clear();
479 SubtargetInfo =
TM->getSubtargetImpl(
F);
480 TLI = SubtargetInfo->getTargetLowering();
481 TRI = SubtargetInfo->getRegisterInfo();
482 TLInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
F);
483 TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
F);
484 LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
487 PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
488 BBSectionsProfileReader =
489 getAnalysisIfAvailable<BasicBlockSectionsProfileReader>();
490 OptSize =
F.hasOptSize();
494 F.setSectionPrefix(
"hot");
499 if (
F.hasFnAttribute(Attribute::Hot) ||
501 F.setSectionPrefix(
"hot");
507 F.setSectionPrefix(
"unlikely");
510 F.setSectionPrefix(
"unknown");
519 while (
BB !=
nullptr) {
533 EverMadeChange |= eliminateAssumptions(
F);
537 EverMadeChange |= eliminateMostlyEmptyBlocks(
F);
539 bool ModifiedDT =
false;
541 EverMadeChange |= splitBranchCondition(
F, ModifiedDT);
548 bool MadeChange =
true;
553 bool ModifiedDTOnIteration =
false;
557 if (ModifiedDTOnIteration)
561 MadeChange |= mergeSExts(
F);
562 if (!LargeOffsetGEPMap.
empty())
563 MadeChange |= splitLargeGEPOffsets();
564 MadeChange |= optimizePhiTypes(
F);
567 eliminateFallThrough(
F);
573 EverMadeChange |= MadeChange;
574 SeenChainsForSExt.
clear();
575 ValToSExtendedUses.clear();
576 RemovedInsts.clear();
577 LargeOffsetGEPMap.
clear();
578 LargeOffsetGEPID.
clear();
593 if (!MadeChange)
continue;
601 MadeChange |= !WorkList.
empty();
602 while (!WorkList.
empty()) {
615 if (EverMadeChange || MadeChange)
616 MadeChange |= eliminateFallThrough(
F);
618 EverMadeChange |= MadeChange;
625 if (
auto *SP = dyn_cast<GCStatepointInst>(&
I))
626 Statepoints.push_back(SP);
627 for (
auto &
I : Statepoints)
628 EverMadeChange |= simplifyOffsetableRelocate(*
I);
633 EverMadeChange |= placeDbgValues(
F);
634 EverMadeChange |= placePseudoProbes(
F);
641 return EverMadeChange;
644 bool CodeGenPrepare::eliminateAssumptions(
Function &
F) {
645 bool MadeChange =
false;
647 CurInstIterator =
BB.begin();
648 while (CurInstIterator !=
BB.end()) {
650 if (
auto *Assume = dyn_cast<AssumeInst>(
I)) {
652 Value *Operand = Assume->getOperand(0);
653 Assume->eraseFromParent();
655 resetIteratorIfInvalidatedWhileCalling(&
BB, [&]() {
666 void CodeGenPrepare::removeAllAssertingVHReferences(
Value *V) {
667 LargeOffsetGEPMap.
erase(V);
668 NewGEPBases.
erase(V);
670 auto GEP = dyn_cast<GetElementPtrInst>(V);
676 auto VecI = LargeOffsetGEPMap.
find(
GEP->getPointerOperand());
677 if (VecI == LargeOffsetGEPMap.
end())
680 auto &GEPVector = VecI->second;
683 if (GEPVector.empty())
684 LargeOffsetGEPMap.
erase(VecI);
693 NewBFI.verifyMatch(*
BFI);
699 bool CodeGenPrepare::eliminateFallThrough(
Function &
F) {
700 bool Changed =
false;
706 Blocks.push_back(&Block);
709 for (
auto &Block : Blocks) {
710 auto *
BB = cast_or_null<BasicBlock>(Block);
718 if (!SinglePred || SinglePred ==
BB ||
BB->hasAddressTaken())
continue;
721 if (
Term && !
Term->isConditional()) {
733 for (
auto &Pred : Preds)
734 if (
auto *
BB = cast_or_null<BasicBlock>(Pred))
743 BranchInst *BI = dyn_cast<BranchInst>(
BB->getTerminator());
750 if (BBI !=
BB->begin()) {
752 while (isa<DbgInfoIntrinsic>(BBI)) {
753 if (BBI ==
BB->begin())
757 if (!isa<DbgInfoIntrinsic>(BBI) && !isa<PHINode>(BBI))
766 if (!canMergeBlocks(
BB, DestBB))
776 bool CodeGenPrepare::eliminateMostlyEmptyBlocks(
Function &
F) {
779 while (!LoopList.empty()) {
780 Loop *L = LoopList.pop_back_val();
783 Preheaders.
insert(Preheader);
786 bool MadeChange =
false;
792 Blocks.push_back(&Block);
794 for (
auto &Block : Blocks) {
798 BasicBlock *DestBB = findDestBlockOfMergeableEmptyBlock(
BB);
800 !isMergingEmptyBlockProfitable(
BB, DestBB, Preheaders.
count(
BB)))
803 eliminateMostlyEmptyBlock(
BB);
809 bool CodeGenPrepare::isMergingEmptyBlockProfitable(
BasicBlock *
BB,
817 !(
BB->getSinglePredecessor() &&
818 BB->getSinglePredecessor()->getSingleSuccessor()))
825 if (
auto *CBI = dyn_cast<CallBrInst>((Pred)->getTerminator()))
826 for (
unsigned i = 0,
e = CBI->getNumSuccessors();
i !=
e; ++
i)
827 if (DestBB == CBI->getSuccessor(
i))
845 if (
BB->getTerminator() !=
BB->getFirstNonPHIOrDbg())
859 if (!isa<PHINode>(DestBB->
begin()))
867 if (DestBBPred ==
BB)
871 return DestPN.getIncomingValueForBlock(BB) ==
872 DestPN.getIncomingValueForBlock(DestBBPred);
874 SameIncomingValueBBs.
insert(DestBBPred);
880 if (SameIncomingValueBBs.
count(Pred))
886 for (
auto *SameValueBB : SameIncomingValueBBs)
887 if (SameValueBB->getUniquePredecessor() == Pred &&
888 DestBB == findDestBlockOfMergeableEmptyBlock(SameValueBB))
889 BBFreq +=
BFI->getBlockFreq(SameValueBB);
898 bool CodeGenPrepare::canMergeBlocks(
const BasicBlock *
BB,
906 if (UI->
getParent() != DestBB || !isa<PHINode>(UI))
912 if (
const PHINode *UPN = dyn_cast<PHINode>(UI))
913 for (
unsigned I = 0,
E = UPN->getNumIncomingValues();
I !=
E; ++
I) {
916 Insn->getParent() != UPN->getIncomingBlock(
I))
926 const PHINode *DestBBPN = dyn_cast<PHINode>(DestBB->
begin());
927 if (!DestBBPN)
return true;
931 if (
const PHINode *BBPN = dyn_cast<PHINode>(
BB->begin())) {
933 for (
unsigned i = 0,
e = BBPN->getNumIncomingValues();
i !=
e; ++
i)
934 BBPreds.
insert(BBPN->getIncomingBlock(
i));
942 if (BBPreds.
count(Pred)) {
944 const Value *V1 = PN.getIncomingValueForBlock(Pred);
945 const Value *
V2 = PN.getIncomingValueForBlock(
BB);
948 if (
const PHINode *V2PN = dyn_cast<PHINode>(
V2))
949 if (V2PN->getParent() ==
BB)
950 V2 = V2PN->getIncomingValueForBlock(Pred);
953 if (V1 !=
V2)
return false;
963 void CodeGenPrepare::eliminateMostlyEmptyBlock(
BasicBlock *
BB) {
973 if (SinglePred != DestBB) {
975 "Single predecessor not the same as predecessor");
989 Value *InVal = PN.removeIncomingValue(
BB,
false);
993 PHINode *InValPhi = dyn_cast<PHINode>(InVal);
1002 if (
PHINode *BBPN = dyn_cast<PHINode>(
BB->begin())) {
1003 for (
unsigned i = 0,
e = BBPN->getNumIncomingValues();
i !=
e; ++
i)
1004 PN.addIncoming(InVal, BBPN->getIncomingBlock(
i));
1007 PN.addIncoming(InVal, Pred);
1014 BB->replaceAllUsesWith(DestBB);
1015 BB->eraseFromParent();
1031 for (
auto *ThisRelocate : AllRelocateCalls) {
1032 auto K = std::make_pair(ThisRelocate->getBasePtrIndex(),
1033 ThisRelocate->getDerivedPtrIndex());
1034 RelocateIdxMap.
insert(std::make_pair(K, ThisRelocate));
1036 for (
auto &Item : RelocateIdxMap) {
1037 std::pair<unsigned, unsigned>
Key = Item.first;
1038 if (
Key.first ==
Key.second)
1043 auto BaseKey = std::make_pair(
Key.first,
Key.first);
1046 auto MaybeBase = RelocateIdxMap.find(BaseKey);
1047 if (MaybeBase == RelocateIdxMap.end())
1052 RelocateInstMap[MaybeBase->second].push_back(
I);
1060 for (
unsigned i = 1;
i <
GEP->getNumOperands();
i++) {
1062 auto *
Op = dyn_cast<ConstantInt>(
GEP->getOperand(
i));
1063 if (!
Op ||
Op->getZExtValue() > 20)
1067 for (
unsigned i = 1;
i <
GEP->getNumOperands();
i++)
1068 OffsetV.push_back(
GEP->getOperand(
i));
1077 bool MadeChange =
false;
1085 &*R != RelocatedBase; ++R)
1086 if (
auto *RI = dyn_cast<GCRelocateInst>(R))
1095 "Not relocating a derived object of the original base object");
1096 if (ToReplace->getBasePtrIndex() == ToReplace->getDerivedPtrIndex()) {
1110 auto *Derived = dyn_cast<GetElementPtrInst>(ToReplace->getDerivedPtr());
1111 if (!Derived || Derived->getPointerOperand() !=
Base)
1120 "Should always have one since it's not a terminator");
1124 Builder.SetCurrentDebugLocation(ToReplace->getDebugLoc());
1147 Value *ActualRelocatedBase = RelocatedBase;
1148 if (RelocatedBase->
getType() !=
Base->getType()) {
1149 ActualRelocatedBase =
1150 Builder.CreateBitCast(RelocatedBase,
Base->getType());
1153 Derived->getSourceElementType(), ActualRelocatedBase,
makeArrayRef(OffsetV));
1157 Value *ActualReplacement = Replacement;
1158 if (Replacement->
getType() != ToReplace->getType()) {
1162 ToReplace->replaceAllUsesWith(ActualReplacement);
1163 ToReplace->eraseFromParent();
1188 bool MadeChange =
false;
1190 for (
auto *U :
I.users())
1193 AllRelocateCalls.push_back(Relocate);
1197 if (AllRelocateCalls.size() < 2)
1204 if (RelocateInstMap.
empty())
1207 for (
auto &Item : RelocateInstMap)
1221 bool MadeChange =
false;
1224 Use &TheUse = UI.getUse();
1231 UserBB = PN->getIncomingBlock(TheUse);
1239 if (
User->isEHPad())
1249 if (UserBB == DefBB)
continue;
1252 CastInst *&InsertedCast = InsertedCasts[UserBB];
1254 if (!InsertedCast) {
1258 CI->
getType(),
"", &*InsertPt);
1263 TheUse = InsertedCast;
1287 if (
auto *ASC = dyn_cast<AddrSpaceCastInst>(CI)) {
1289 ASC->getDestAddressSpace()))
1303 if (SrcVT.
bitsLT(DstVT))
return false;
1328 match(IVInc, m_ExtractValue<0>(m_Intrinsic<Intrinsic::uadd_with_overflow>(
1332 match(IVInc, m_ExtractValue<0>(m_Intrinsic<Intrinsic::usub_with_overflow>(
1350 if (!IVInc || LI->
getLoopFor(IVInc->getParent()) != L)
1355 return std::make_pair(IVInc, Step);
1360 auto *
I = dyn_cast<Instruction>(V);
1367 if (
auto *PN = dyn_cast<PHINode>(
LHS))
1369 return IVInc->first ==
I;
1373 bool CodeGenPrepare::replaceMathCmpWithIntrinsic(
BinaryOperator *BO,
1381 assert(L &&
"L should not be null after isIVIncrement()");
1398 if (BO->
getParent() !=
Cmp->getParent() && !IsReplacableIVIncrement(BO)) {
1422 IID == Intrinsic::usub_with_overflow) {
1423 assert(isa<Constant>(Arg1) &&
"Unexpected input for usubo");
1432 if ((BO->
getOpcode() != Instruction::Xor && &Iter == BO) || &Iter == Cmp) {
1437 assert(InsertPt !=
nullptr &&
"Parent block did not contain cmp or binop");
1440 Value *MathOV =
Builder.CreateBinaryIntrinsic(IID, Arg0, Arg1);
1441 if (BO->
getOpcode() != Instruction::Xor) {
1442 Value *Math =
Builder.CreateExtractValue(MathOV, 0,
"math");
1446 "Patterns with XOr should use the BO only in the compare");
1447 Value *OV =
Builder.CreateExtractValue(MathOV, 1,
"ov");
1448 Cmp->replaceAllUsesWith(OV);
1449 Cmp->eraseFromParent();
1459 Value *A = Cmp->getOperand(0), *
B = Cmp->getOperand(1);
1462 if (isa<Constant>(A))
1475 for (
User *U : A->users()) {
1477 Add = cast<BinaryOperator>(U);
1486 bool CodeGenPrepare::combineToUAddWithOverflow(
CmpInst *Cmp,
1494 A =
Add->getOperand(0);
1495 B =
Add->getOperand(1);
1500 Add->hasNUsesOrMore(2)))
1506 if (
Add->getParent() !=
Cmp->getParent() && !
Add->hasOneUse())
1509 if (!replaceMathCmpWithIntrinsic(Add, A,
B, Cmp,
1510 Intrinsic::uadd_with_overflow))
1518 bool CodeGenPrepare::combineToUSubWithOverflow(
CmpInst *Cmp,
1522 if (isa<Constant>(A) && isa<Constant>(
B))
1547 Value *CmpVariableOperand = isa<Constant>(A) ?
B :
A;
1549 for (
User *U : CmpVariableOperand->
users()) {
1552 Sub = cast<BinaryOperator>(U);
1557 const APInt *CmpC, *AddC;
1560 Sub = cast<BinaryOperator>(U);
1573 Cmp, Intrinsic::usub_with_overflow))
1598 bool MadeChange =
false;
1601 Use &TheUse = UI.getUse();
1608 if (isa<PHINode>(
User))
1616 if (UserBB == DefBB)
continue;
1619 CmpInst *&InsertedCmp = InsertedCmps[UserBB];
1626 Cmp->getOperand(0), Cmp->getOperand(1),
"",
1633 TheUse = InsertedCmp;
1639 if (Cmp->use_empty()) {
1640 Cmp->eraseFromParent();
1677 for (
User *U : Cmp->users()) {
1678 if (isa<BranchInst>(U))
1680 if (isa<SelectInst>(U) && cast<SelectInst>(U)->getCondition() == Cmp)
1699 if (CmpBB != FalseBB)
1702 Value *CmpOp0 = Cmp->getOperand(0), *CmpOp1 = Cmp->getOperand(1);
1716 for (
User *U : Cmp->users()) {
1717 if (
auto *BI = dyn_cast<BranchInst>(U)) {
1722 if (
auto *
SI = dyn_cast<SelectInst>(U)) {
1725 SI->swapProfMetadata();
1734 bool CodeGenPrepare::optimizeCmp(
CmpInst *Cmp,
bool &ModifiedDT) {
1738 if (combineToUAddWithOverflow(Cmp, ModifiedDT))
1741 if (combineToUSubWithOverflow(Cmp, ModifiedDT))
1757 SetOfInstrs &InsertedInsts) {
1760 assert(!InsertedInsts.count(AndI) &&
1761 "Attempting to optimize already optimized and instruction");
1762 (void) InsertedInsts;
1771 if (!isa<ConstantInt>(AndI->
getOperand(0)) &&
1776 for (
auto *U : AndI->
users()) {
1780 if (!isa<ICmpInst>(
User))
1784 if (!CmpC || !CmpC->
isZero())
1799 Use &TheUse = UI.getUse();
1817 TheUse = InsertedAnd;
1833 if (!isa<TruncInst>(
User)) {
1834 if (
User->getOpcode() != Instruction::And ||
1840 if ((Cimm & (Cimm + 1)).getBoolValue())
1853 auto *TruncI = cast<TruncInst>(
User);
1854 bool MadeChange =
false;
1857 TruncE = TruncI->user_end();
1858 TruncUI != TruncE;) {
1860 Use &TruncTheUse = TruncUI.getUse();
1861 Instruction *TruncUser = cast<Instruction>(*TruncUI);
1880 if (isa<PHINode>(TruncUser))
1885 if (UserBB == TruncUserBB)
1889 CastInst *&InsertedTrunc = InsertedTruncs[TruncUserBB];
1891 if (!InsertedShift && !InsertedTrunc) {
1895 if (ShiftI->
getOpcode() == Instruction::AShr)
1896 InsertedShift = BinaryOperator::CreateAShr(ShiftI->
getOperand(0), CI,
1899 InsertedShift = BinaryOperator::CreateLShr(ShiftI->
getOperand(0), CI,
1906 assert(TruncInsertPt != TruncUserBB->
end());
1909 TruncI->
getType(),
"", &*TruncInsertPt);
1910 InsertedTrunc->
setDebugLoc(TruncI->getDebugLoc());
1914 TruncTheUse = InsertedTrunc;
1947 bool MadeChange =
false;
1950 Use &TheUse = UI.getUse();
1956 if (isa<PHINode>(
User))
1964 if (UserBB == DefBB) {
1979 if (isa<TruncInst>(
User) && shiftIsLegal
1992 if (!InsertedShift) {
1996 if (ShiftI->
getOpcode() == Instruction::AShr)
1997 InsertedShift = BinaryOperator::CreateAShr(ShiftI->
getOperand(0), CI,
2000 InsertedShift = BinaryOperator::CreateLShr(ShiftI->
getOperand(0), CI,
2008 TheUse = InsertedShift;
2055 if (Ty->
isVectorTy() || SizeInBits >
DL->getLargestLegalIntTypeSizeInBits())
2085 Builder.CreateCondBr(Cmp, EndBlock, CallBlock);
2114 CurInstIterator =
BB->begin();
2121 if (optimizeInlineAsmInst(CI))
2130 for (
auto &
Arg : CI->
args()) {
2135 if (!
Arg->getType()->isPointerTy())
2138 cast<PointerType>(
Arg->getType())->getAddressSpace()),
2140 Value *Val =
Arg->stripAndAccumulateInBoundsConstantOffsets(*
DL, Offset);
2145 if ((AI = dyn_cast<AllocaInst>(Val)) && AI->
getAlign() < PrefAlign &&
2164 if (!MIDestAlign || DestAlign > *MIDestAlign)
2165 MI->setDestAlignment(DestAlign);
2167 MaybeAlign MTISrcAlign = MTI->getSourceAlign();
2169 if (!MTISrcAlign || SrcAlign > *MTISrcAlign)
2170 MTI->setSourceAlignment(SrcAlign);
2181 for (
auto &
Arg : CI->
args()) {
2182 if (!
Arg->getType()->isPointerTy())
2184 unsigned AS =
Arg->getType()->getPointerAddressSpace();
2185 return optimizeMemoryInst(CI,
Arg,
Arg->getType(), AS);
2192 case Intrinsic::assume:
2194 case Intrinsic::experimental_widenable_condition: {
2203 resetIteratorIfInvalidatedWhileCalling(
BB, [&]() {
2208 case Intrinsic::objectsize:
2210 case Intrinsic::is_constant:
2212 case Intrinsic::aarch64_stlxr:
2213 case Intrinsic::aarch64_stxr: {
2222 InsertedInsts.insert(ExtVal);
2226 case Intrinsic::launder_invariant_group:
2227 case Intrinsic::strip_invariant_group: {
2229 auto it = LargeOffsetGEPMap.
find(II);
2230 if (
it != LargeOffsetGEPMap.
end()) {
2235 LargeOffsetGEPMap[ArgVal].append(GEPs.begin(), GEPs.end());
2236 LargeOffsetGEPMap.
erase(II);
2243 case Intrinsic::cttz:
2244 case Intrinsic::ctlz:
2247 case Intrinsic::fshl:
2248 case Intrinsic::fshr:
2249 return optimizeFunnelShift(II);
2250 case Intrinsic::dbg_value:
2251 return fixupDbgValue(II);
2252 case Intrinsic::vscale: {
2256 Type *ScalableVectorTy =
2258 if (
DL->getTypeAllocSize(ScalableVectorTy).getKnownMinSize() == 8) {
2269 case Intrinsic::masked_gather:
2271 case Intrinsic::masked_scatter:
2278 while (!PtrOps.empty()) {
2281 if (optimizeMemoryInst(II, PtrVal, AccessTy, AS))
2334 bool CodeGenPrepare::dupRetToEnableTailCallOpts(
BasicBlock *
BB,
bool &ModifiedDT) {
2335 ReturnInst *RetI = dyn_cast<ReturnInst>(
BB->getTerminator());
2344 BCI = dyn_cast<BitCastInst>(V);
2348 EVI = dyn_cast<ExtractValueInst>(V);
2355 PN = dyn_cast<PHINode>(V);
2363 auto isLifetimeEndOrBitCastFor = [](
const Instruction *Inst) {
2364 const BitCastInst *BC = dyn_cast<BitCastInst>(Inst);
2368 if (
const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst))
2377 while (isa<DbgInfoIntrinsic>(BI) || BI == BCI || BI == EVI ||
2378 isa<PseudoProbeInst>(BI) || isLifetimeEndOrBitCastFor(BI))
2391 CallInst *CI = dyn_cast<CallInst>(IncomingVal);
2397 TailCallBBs.push_back(PredBB);
2402 if (!VisitedBBs.
insert(Pred).second)
2404 if (
Instruction *
I = Pred->rbegin()->getPrevNonDebugInstruction(
true)) {
2408 TailCallBBs.push_back(Pred);
2413 bool Changed =
false;
2414 for (
auto const &TailCallBB : TailCallBBs) {
2417 BranchInst *BI = dyn_cast<BranchInst>(TailCallBB->getTerminator());
2424 BFI->getBlockFreq(
BB) >=
BFI->getBlockFreq(TailCallBB));
2427 (
BFI->getBlockFreq(
BB) -
BFI->getBlockFreq(TailCallBB)).getFrequency());
2428 ModifiedDT = Changed =
true;
2434 BB->eraseFromParent();
2448 Value *BaseReg =
nullptr;
2449 Value *ScaledReg =
nullptr;
2450 Value *OriginalValue =
nullptr;
2451 bool InBounds =
true;
2455 BaseRegField = 0x01,
2457 BaseOffsField = 0x04,
2458 ScaledRegField = 0x08,
2460 MultipleFields = 0xff
2472 if (BaseReg && other.
BaseReg &&
2474 return MultipleFields;
2475 if (BaseGV && other.BaseGV &&
2476 BaseGV->getType() != other.BaseGV->getType())
2477 return MultipleFields;
2480 return MultipleFields;
2483 if (InBounds != other.InBounds)
2484 return MultipleFields;
2487 unsigned Result = NoField;
2490 if (BaseGV != other.BaseGV)
2492 if (BaseOffs != other.BaseOffs)
2495 Result |= ScaledRegField;
2498 if (Scale && other.
Scale && Scale != other.
Scale)
2502 return MultipleFields;
2504 return static_cast<FieldName
>(
Result);
2514 return !BaseOffs && !Scale && !(BaseGV && BaseReg);
2525 case ScaledRegField:
2532 void SetCombinedField(FieldName
Field,
Value *V,
2538 case ExtAddrMode::BaseRegField:
2541 case ExtAddrMode::BaseGVField:
2544 assert(BaseReg ==
nullptr);
2548 case ExtAddrMode::ScaledRegField:
2559 case ExtAddrMode::BaseOffsField:
2562 assert(ScaledReg ==
nullptr);
2580 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2582 bool NeedPlus =
false;
2587 OS << (NeedPlus ?
" + " :
"")
2589 BaseGV->printAsOperand(OS,
false);
2594 OS << (NeedPlus ?
" + " :
"")
2600 OS << (NeedPlus ?
" + " :
"")
2606 OS << (NeedPlus ?
" + " :
"")
2628 class TypePromotionTransaction {
2632 class TypePromotionAction {
2640 TypePromotionAction(
Instruction *Inst) : Inst(Inst) {}
2642 virtual ~TypePromotionAction() =
default;
2649 virtual void undo() = 0;
2654 virtual void commit() {
2660 class InsertionHandler {
2671 bool HasPrevInstruction;
2678 if (HasPrevInstruction)
2679 Point.PrevInst = &*--It;
2686 if (HasPrevInstruction) {
2691 Instruction *Position = &*Point.BB->getFirstInsertionPt();
2701 class InstructionMoveBefore :
public TypePromotionAction {
2703 InsertionHandler Position;
2708 : TypePromotionAction(Inst), Position(Inst) {
2709 LLVM_DEBUG(
dbgs() <<
"Do: move: " << *Inst <<
"\nbefore: " << *Before
2715 void undo()
override {
2717 Position.insert(Inst);
2722 class OperandSetter :
public TypePromotionAction {
2732 : TypePromotionAction(Inst), Idx(Idx) {
2734 <<
"for:" << *Inst <<
"\n"
2735 <<
"with:" << *NewVal <<
"\n");
2741 void undo()
override {
2743 <<
"for: " << *Inst <<
"\n"
2744 <<
"with: " << *Origin <<
"\n");
2751 class OperandsHider :
public TypePromotionAction {
2757 OperandsHider(
Instruction *Inst) : TypePromotionAction(Inst) {
2760 OriginalValues.
reserve(NumOpnds);
2761 for (
unsigned It = 0; It < NumOpnds; ++It) {
2764 OriginalValues.push_back(Val);
2773 void undo()
override {
2775 for (
unsigned It = 0, EndIt = OriginalValues.size(); It != EndIt; ++It)
2781 class TruncBuilder :
public TypePromotionAction {
2788 TruncBuilder(
Instruction *Opnd,
Type *Ty) : TypePromotionAction(Opnd) {
2791 Val =
Builder.CreateTrunc(Opnd, Ty,
"promoted");
2796 Value *getBuiltValue() {
return Val; }
2799 void undo()
override {
2801 if (
Instruction *IVal = dyn_cast<Instruction>(Val))
2802 IVal->eraseFromParent();
2807 class SExtBuilder :
public TypePromotionAction {
2815 : TypePromotionAction(InsertPt) {
2817 Val =
Builder.CreateSExt(Opnd, Ty,
"promoted");
2822 Value *getBuiltValue() {
return Val; }
2825 void undo()
override {
2827 if (
Instruction *IVal = dyn_cast<Instruction>(Val))
2828 IVal->eraseFromParent();
2833 class ZExtBuilder :
public TypePromotionAction {
2841 : TypePromotionAction(InsertPt) {
2844 Val =
Builder.CreateZExt(Opnd, Ty,
"promoted");
2849 Value *getBuiltValue() {
return Val; }
2852 void undo()
override {
2854 if (
Instruction *IVal = dyn_cast<Instruction>(Val))
2855 IVal->eraseFromParent();
2860 class TypeMutator :
public TypePromotionAction {
2867 : TypePromotionAction(Inst), OrigTy(Inst->
getType()) {
2868 LLVM_DEBUG(
dbgs() <<
"Do: MutateType: " << *Inst <<
" with " << *NewTy
2874 void undo()
override {
2875 LLVM_DEBUG(
dbgs() <<
"Undo: MutateType: " << *Inst <<
" with " << *OrigTy
2882 class UsesReplacer :
public TypePromotionAction {
2884 struct InstructionAndIdx {
2891 InstructionAndIdx(
Instruction *Inst,
unsigned Idx)
2892 : Inst(Inst), Idx(Idx) {}
2909 : TypePromotionAction(Inst),
New(
New) {
2910 LLVM_DEBUG(
dbgs() <<
"Do: UsersReplacer: " << *Inst <<
" with " << *New
2914 Instruction *UserI = cast<Instruction>(U.getUser());
2915 OriginalUses.push_back(InstructionAndIdx(UserI, U.getOperandNo()));
2926 void undo()
override {
2928 for (InstructionAndIdx &
Use : OriginalUses)
2929 Use.Inst->setOperand(
Use.Idx, Inst);
2934 for (
auto *DVI : DbgValues)
2935 DVI->replaceVariableLocationOp(New, Inst);
2940 class InstructionRemover :
public TypePromotionAction {
2942 InsertionHandler Inserter;
2946 OperandsHider Hider;
2949 UsesReplacer *Replacer =
nullptr;
2952 SetOfInstrs &RemovedInsts;
2959 InstructionRemover(
Instruction *Inst, SetOfInstrs &RemovedInsts,
2960 Value *New =
nullptr)
2961 : TypePromotionAction(Inst), Inserter(Inst), Hider(Inst),
2962 RemovedInsts(RemovedInsts) {
2964 Replacer =
new UsesReplacer(Inst, New);
2965 LLVM_DEBUG(
dbgs() <<
"Do: InstructionRemover: " << *Inst <<
"\n");
2966 RemovedInsts.insert(Inst);
2973 ~InstructionRemover()
override {
delete Replacer; }
2977 void undo()
override {
2978 LLVM_DEBUG(
dbgs() <<
"Undo: InstructionRemover: " << *Inst <<
"\n");
2979 Inserter.insert(Inst);
2983 RemovedInsts.erase(Inst);
2991 using ConstRestorationPt =
const TypePromotionAction *;
2993 TypePromotionTransaction(SetOfInstrs &RemovedInsts)
2994 : RemovedInsts(RemovedInsts) {}
3001 void rollback(ConstRestorationPt Point);
3004 ConstRestorationPt getRestorationPoint()
const;
3039 SetOfInstrs &RemovedInsts;
3044 void TypePromotionTransaction::setOperand(
Instruction *Inst,
unsigned Idx,
3046 Actions.push_back(std::make_unique<TypePromotionTransaction::OperandSetter>(
3047 Inst, Idx, NewVal));
3053 std::make_unique<TypePromotionTransaction::InstructionRemover>(
3054 Inst, RemovedInsts, NewVal));
3057 void TypePromotionTransaction::replaceAllUsesWith(
Instruction *Inst,
3060 std::make_unique<TypePromotionTransaction::UsesReplacer>(Inst, New));
3063 void TypePromotionTransaction::mutateType(
Instruction *Inst,
Type *NewTy) {
3065 std::make_unique<TypePromotionTransaction::TypeMutator>(Inst, NewTy));
3070 std::unique_ptr<TruncBuilder> Ptr(
new TruncBuilder(Opnd, Ty));
3071 Value *Val = Ptr->getBuiltValue();
3078 std::unique_ptr<SExtBuilder> Ptr(
new SExtBuilder(Inst, Opnd, Ty));
3079 Value *Val = Ptr->getBuiltValue();
3086 std::unique_ptr<ZExtBuilder> Ptr(
new ZExtBuilder(Inst, Opnd, Ty));
3087 Value *Val = Ptr->getBuiltValue();
3092 void TypePromotionTransaction::moveBefore(
Instruction *Inst,
3095 std::make_unique<TypePromotionTransaction::InstructionMoveBefore>(
3099 TypePromotionTransaction::ConstRestorationPt
3100 TypePromotionTransaction::getRestorationPoint()
const {
3101 return !Actions.empty() ? Actions.back().get() :
nullptr;
3104 bool TypePromotionTransaction::commit() {
3105 for (std::unique_ptr<TypePromotionAction> &Action : Actions)
3112 void TypePromotionTransaction::rollback(
3113 TypePromotionTransaction::ConstRestorationPt Point) {
3114 while (!Actions.empty() && Point != Actions.back().get()) {
3115 std::unique_ptr<TypePromotionAction> Curr = Actions.pop_back_val();
3125 class AddressingModeMatcher {
3144 const SetOfInstrs &InsertedInsts;
3147 InstrToOrigTy &PromotedInsts;
3150 TypePromotionTransaction &TPT;
3153 std::pair<AssertingVH<GetElementPtrInst>, int64_t> &LargeOffsetGEP;
3157 bool IgnoreProfitability;
3165 AddressingModeMatcher(
3170 const SetOfInstrs &InsertedInsts, InstrToOrigTy &PromotedInsts,
3171 TypePromotionTransaction &TPT,
3174 : AddrModeInsts(AMI), TLI(TLI),
TRI(
TRI),
3175 DL(
MI->getModule()->getDataLayout()), LI(LI), getDTFn(getDTFn),
3176 AccessTy(AT), AddrSpace(AS), MemoryInst(
MI),
AddrMode(AM),
3177 InsertedInsts(InsertedInsts), PromotedInsts(PromotedInsts), TPT(TPT),
3178 LargeOffsetGEP(LargeOffsetGEP), OptSize(OptSize), PSI(PSI),
BFI(
BFI) {
3179 IgnoreProfitability =
false;
3196 InstrToOrigTy &PromotedInsts, TypePromotionTransaction &TPT,
3201 bool Success = AddressingModeMatcher(
3202 AddrModeInsts, TLI,
TRI, LI, getDTFn, AccessTy, AS, MemoryInst, Result,
3203 InsertedInsts, PromotedInsts, TPT, LargeOffsetGEP, OptSize, PSI,
3204 BFI).matchAddr(V, 0);
3210 bool matchScaledValue(
Value *ScaleReg, int64_t Scale,
unsigned Depth);
3211 bool matchAddr(
Value *
Addr,
unsigned Depth);
3212 bool matchOperationAddr(
User *AddrInst,
unsigned Opcode,
unsigned Depth,
3213 bool *MovedAway =
nullptr);
3214 bool isProfitableToFoldIntoAddressingMode(
Instruction *
I,
3217 bool valueAlreadyLiveAtInst(
Value *Val,
Value *KnownLive1,
Value *KnownLive2);
3218 bool isPromotionProfitable(
unsigned NewCost,
unsigned OldCost,
3219 Value *PromotedOperand)
const;
3225 class PhiNodeSetIterator {
3226 PhiNodeSet *
const Set;
3227 size_t CurrentIndex = 0;
3232 PhiNodeSetIterator(PhiNodeSet *
const Set,
size_t Start);
3234 PhiNodeSetIterator& operator++();
3250 friend class PhiNodeSetIterator;
3253 using iterator = PhiNodeSetIterator;
3268 size_t FirstValidElement = 0;
3275 if (NodeMap.insert(std::make_pair(Ptr,
NodeList.size())).second) {
3286 if (NodeMap.erase(Ptr)) {
3287 SkipRemovedElements(FirstValidElement);
3297 FirstValidElement = 0;
3303 if (FirstValidElement == 0)
3304 SkipRemovedElements(FirstValidElement);
3305 return PhiNodeSetIterator(
this, FirstValidElement);
3309 iterator
end() {
return PhiNodeSetIterator(
this,
NodeList.size()); }
3312 size_t size()
const {
3313 return NodeMap.size();
3318 return NodeMap.count(Ptr);
3327 void SkipRemovedElements(
size_t &CurrentIndex) {
3328 while (CurrentIndex <
NodeList.size()) {
3329 auto it = NodeMap.find(
NodeList[CurrentIndex]);
3332 if (
it != NodeMap.end() &&
it->second == CurrentIndex)
3339 PhiNodeSetIterator::PhiNodeSetIterator(PhiNodeSet *
const Set,
size_t Start)
3340 : Set(Set), CurrentIndex(Start) {}
3344 "PhiNodeSet access out of range");
3345 return Set->NodeList[CurrentIndex];
3348 PhiNodeSetIterator& PhiNodeSetIterator::operator++() {
3350 "PhiNodeSet access out of range");
3352 Set->SkipRemovedElements(CurrentIndex);
3357 return CurrentIndex ==
RHS.CurrentIndex;
3361 return !((*this) ==
RHS);
3367 class SimplificationTracker {
3372 PhiNodeSet AllPhiNodes;
3382 auto SV = Storage.
find(V);
3383 if (SV == Storage.
end())
3392 WorkList.push_back(Val);
3393 while (!WorkList.empty()) {
3395 if (!Visited.
insert(
P).second)
3397 if (
auto *PI = dyn_cast<Instruction>(
P))
3399 for (
auto *U : PI->users())
3400 WorkList.push_back(cast<Value>(U));
3402 PI->replaceAllUsesWith(V);
3403 if (
auto *PHI = dyn_cast<PHINode>(PI))
3404 AllPhiNodes.erase(PHI);
3405 if (
auto *
Select = dyn_cast<SelectInst>(PI))
3407 PI->eraseFromParent();
3419 while (OldReplacement !=
From) {
3421 To = dyn_cast<PHINode>(OldReplacement);
3422 OldReplacement = Get(
From);
3424 assert(To && Get(To) == To &&
"Replacement PHI node is already replaced.");
3426 From->replaceAllUsesWith(To);
3427 AllPhiNodes.erase(
From);
3428 From->eraseFromParent();
3431 PhiNodeSet& newPhiNodes() {
return AllPhiNodes; }
3433 void insertNewPhi(
PHINode *PN) { AllPhiNodes.insert(PN); }
3437 unsigned countNewPhiNodes()
const {
return AllPhiNodes.size(); }
3439 unsigned countNewSelectNodes()
const {
return AllSelectNodes.
size(); }
3441 void destroyNewNodes(
Type *CommonType) {
3444 for (
auto *
I : AllPhiNodes) {
3445 I->replaceAllUsesWith(
Dummy);
3446 I->eraseFromParent();
3448 AllPhiNodes.clear();
3449 for (
auto *
I : AllSelectNodes) {
3450 I->replaceAllUsesWith(
Dummy);
3451 I->eraseFromParent();
3453 AllSelectNodes.clear();
3458 class AddressingModeCombiner {
3460 typedef std::pair<PHINode *, PHINode *> PHIPair;
3467 ExtAddrMode::FieldName DifferentField = ExtAddrMode::NoField;
3470 bool AllAddrModesTrivial =
true;
3473 Type *CommonType =
nullptr;
3483 : SQ(_SQ), Original(OriginalValue) {}
3487 return AddrModes[0];
3497 AllAddrModesTrivial = AllAddrModesTrivial && NewAddrMode.isTrivial();
3500 if (AddrModes.empty()) {
3508 ExtAddrMode::FieldName ThisDifferentField =
3509 AddrModes[0].compare(NewAddrMode);
3510 if (DifferentField == ExtAddrMode::NoField)
3511 DifferentField = ThisDifferentField;
3512 else if (DifferentField != ThisDifferentField)
3513 DifferentField = ExtAddrMode::MultipleFields;
3516 bool CanHandle = DifferentField != ExtAddrMode::MultipleFields;
3519 CanHandle = CanHandle && DifferentField != ExtAddrMode::ScaleField;
3524 CanHandle = CanHandle && (DifferentField != ExtAddrMode::BaseOffsField ||
3529 CanHandle = CanHandle && (DifferentField != ExtAddrMode::BaseGVField ||
3530 !NewAddrMode.HasBaseReg);
3547 bool combineAddrModes() {
3549 if (AddrModes.size() == 0)
3553 if (AddrModes.size() == 1 || DifferentField == ExtAddrMode::NoField)
3558 if (AllAddrModesTrivial)
3561 if (!addrModeCombiningAllowed())
3567 FoldAddrToValueMapping
Map;
3568 if (!initializeMap(Map))
3571 Value *CommonValue = findCommon(Map);
3573 AddrModes[0].SetCombinedField(DifferentField, CommonValue, AddrModes);
3574 return CommonValue !=
nullptr;
3583 bool initializeMap(FoldAddrToValueMapping &Map) {
3588 for (
auto &AM : AddrModes) {
3589 Value *DV = AM.GetFieldAsValue(DifferentField, IntPtrTy);
3592 if (CommonType && CommonType !=
Type)
3595 Map[AM.OriginalValue] = DV;
3597 NullValue.push_back(AM.OriginalValue);
3600 assert(CommonType &&
"At least one non-null value must be!");
3601 for (
auto *V : NullValue)
3602 Map[V] = Constant::getNullValue(CommonType);
3629 Value *findCommon(FoldAddrToValueMapping &Map) {
3637 SimplificationTracker
ST(SQ);
3642 InsertPlaceholders(Map, TraverseOrder,
ST);
3645 FillPlaceholders(Map, TraverseOrder,
ST);
3648 ST.destroyNewNodes(CommonType);
3653 unsigned PhiNotMatchedCount = 0;
3655 ST.destroyNewNodes(CommonType);
3659 auto *
Result =
ST.Get(
Map.find(Original)->second);
3661 NumMemoryInstsPhiCreated +=
ST.countNewPhiNodes() + PhiNotMatchedCount;
3662 NumMemoryInstsSelectCreated +=
ST.countNewSelectNodes();
3671 PhiNodeSet &PhiNodesToMatch) {
3673 Matcher.
insert({ PHI, Candidate });
3676 WorkList.push_back({ PHI, Candidate });
3678 while (!WorkList.empty()) {
3680 if (!Visited.
insert(Item).second)
3687 for (
auto B : Item.first->blocks()) {
3688 Value *FirstValue = Item.first->getIncomingValueForBlock(
B);
3689 Value *SecondValue = Item.second->getIncomingValueForBlock(
B);
3690 if (FirstValue == SecondValue)
3693 PHINode *FirstPhi = dyn_cast<PHINode>(FirstValue);
3694 PHINode *SecondPhi = dyn_cast<PHINode>(SecondValue);
3700 if (!FirstPhi || !SecondPhi || !PhiNodesToMatch.count(FirstPhi) ||
3705 if (Matcher.
count({ FirstPhi, SecondPhi }))
3710 if (MatchedPHIs.
insert(FirstPhi).second)
3711 Matcher.
insert({ FirstPhi, SecondPhi });
3713 WorkList.push_back({ FirstPhi, SecondPhi });
3722 bool MatchPhiSet(SimplificationTracker &
ST,
bool AllowNewPhiNodes,
3723 unsigned &PhiNotMatchedCount) {
3729 PhiNodeSet &PhiNodesToMatch =
ST.newPhiNodes();
3730 while (PhiNodesToMatch.size()) {
3731 PHINode *PHI = *PhiNodesToMatch.begin();
3734 WillNotMatch.
clear();
3735 WillNotMatch.
insert(PHI);
3738 bool IsMatched =
false;
3741 if (PhiNodesToMatch.count(&
P))
3743 if ((IsMatched = MatchPhiNode(PHI, &
P, Matched, PhiNodesToMatch)))
3748 for (
auto M : Matched)
3754 for (
auto MV : Matched)
3755 ST.ReplacePhi(MV.first, MV.second);
3760 if (!AllowNewPhiNodes)
3763 PhiNotMatchedCount += WillNotMatch.
size();
3764 for (
auto *
P : WillNotMatch)
3765 PhiNodesToMatch.erase(
P);
3770 void FillPlaceholders(FoldAddrToValueMapping &Map,
3772 SimplificationTracker &
ST) {
3773 while (!TraverseOrder.empty()) {
3775 assert(
Map.find(Current) !=
Map.end() &&
"No node to fill!!!");
3780 auto *CurrentSelect = cast<SelectInst>(Current);
3781 auto *TrueValue = CurrentSelect->getTrueValue();
3782 assert(
Map.find(TrueValue) !=
Map.end() &&
"No True Value!");
3783 Select->setTrueValue(
ST.Get(Map[TrueValue]));
3784 auto *FalseValue = CurrentSelect->getFalseValue();
3785 assert(
Map.find(FalseValue) !=
Map.end() &&
"No False Value!");
3786 Select->setFalseValue(
ST.Get(Map[FalseValue]));
3789 auto *PHI = cast<PHINode>(V);
3792 Value *PV = cast<PHINode>(Current)->getIncomingValueForBlock(
B);
3793 assert(
Map.find(PV) !=
Map.end() &&
"No predecessor Value!");
3797 Map[Current] =
ST.Simplify(V);
3806 void InsertPlaceholders(FoldAddrToValueMapping &Map,
3808 SimplificationTracker &
ST) {
3810 assert((isa<PHINode>(Original) || isa<SelectInst>(Original)) &&
3811 "Address must be a Phi or Select node");
3813 Worklist.push_back(Original);
3814 while (!Worklist.empty()) {
3817 if (
Map.find(Current) !=
Map.end())
3819 TraverseOrder.push_back(Current);
3823 if (
SelectInst *CurrentSelect = dyn_cast<SelectInst>(Current)) {
3828 CurrentSelect->getName(), CurrentSelect, CurrentSelect);
3832 Worklist.push_back(CurrentSelect->getTrueValue());
3833 Worklist.push_back(CurrentSelect->getFalseValue());
3836 PHINode *CurrentPhi = cast<PHINode>(Current);
3839 PHINode::Create(CommonType, PredCount,
"sunk_phi", CurrentPhi);
3841 ST.insertNewPhi(PHI);
3847 bool addrModeCombiningAllowed() {
3850 switch (DifferentField) {
3853 case ExtAddrMode::BaseRegField:
3855 case ExtAddrMode::BaseGVField:
3857 case ExtAddrMode::BaseOffsField:
3859 case ExtAddrMode::ScaledRegField:
3869 bool AddressingModeMatcher::matchScaledValue(
Value *ScaleReg, int64_t Scale,
3874 return matchAddr(ScaleReg,
Depth);
3889 TestAddrMode.
Scale += Scale;
3904 if (isa<Instruction>(ScaleReg) &&
3907 TestAddrMode.InBounds =
false;
3914 AddrModeInsts.push_back(cast<Instruction>(ScaleReg));
3924 auto GetConstantStep = [
this](
const Value * V)
3925 ->
Optional<std::pair<Instruction *, APInt> > {
3926 auto *PN = dyn_cast<PHINode>(V);
3938 if (
auto *OIVInc = dyn_cast<OverflowingBinaryOperator>(IVInc->first))
3939 if (OIVInc->hasNoSignedWrap() || OIVInc->hasNoUnsignedWrap())
3941 if (
auto *ConstantStep = dyn_cast<ConstantInt>(IVInc->second))
3942 return std::make_pair(IVInc->first, ConstantStep->getValue());
3958 if (
auto IVStep = GetConstantStep(ScaleReg)) {
3965 APInt Step = IVStep->second;
3967 if (
Offset.isSignedIntN(64)) {
3968 TestAddrMode.InBounds =
false;
3970 TestAddrMode.BaseOffs -=
Offset.getLimitedValue();
3975 getDTFn().
dominates(IVInc, MemoryInst)) {
3976 AddrModeInsts.push_back(cast<Instruction>(IVInc));
3995 switch (
I->getOpcode()) {
3996 case Instruction::BitCast:
3997 case Instruction::AddrSpaceCast:
3999 if (
I->getType() ==
I->getOperand(0)->getType())
4001 return I->getType()->isIntOrPtrTy();
4002 case Instruction::PtrToInt:
4005 case Instruction::IntToPtr:
4011 case Instruction::Shl:
4013 return isa<ConstantInt>(
I->getOperand(1));
4014 case Instruction::GetElementPtr:
4027 Instruction *PromotedInst = dyn_cast<Instruction>(Val);
4042 class TypePromotionHelper {
4045 static void addPromotedInst(InstrToOrigTy &PromotedInsts,
4048 ExtType ExtTy = IsSExt ? SignExtension : ZeroExtension;
4049 InstrToOrigTy::iterator It = PromotedInsts.find(ExtOpnd);
4050 if (It != PromotedInsts.end()) {
4053 if (It->second.getInt() == ExtTy)
4059 ExtTy = BothExtension;
4061 PromotedInsts[ExtOpnd] = TypeIsSExt(ExtOpnd->
getType(), ExtTy);
4068 static const Type *getOrigType(
const InstrToOrigTy &PromotedInsts,
4071 ExtType ExtTy = IsSExt ? SignExtension : ZeroExtension;
4072 InstrToOrigTy::const_iterator It = PromotedInsts.find(Opnd);
4073 if (It != PromotedInsts.end() && It->second.getInt() == ExtTy)
4074 return It->second.getPointer();
4089 static bool canGetThrough(
const Instruction *Inst,
Type *ConsideredExtType,
4090 const InstrToOrigTy &PromotedInsts,
bool IsSExt);
4094 static bool shouldExtOperand(
const Instruction *Inst,
int OpIdx) {
4095 return !(isa<SelectInst>(Inst) && OpIdx == 0);
4107 static Value *promoteOperandForTruncAndAnyExt(
4109 InstrToOrigTy &PromotedInsts,
unsigned &CreatedInstsCost,
4123 TypePromotionTransaction &TPT,
4124 InstrToOrigTy &PromotedInsts,
4125 unsigned &CreatedInstsCost,
4131 static Value *signExtendOperandForOther(
4133 InstrToOrigTy &PromotedInsts,
unsigned &CreatedInstsCost,
4136 return promoteOperandForOther(
Ext, TPT, PromotedInsts, CreatedInstsCost,
4137 Exts, Truncs, TLI,
true);
4141 static Value *zeroExtendOperandForOther(
4143 InstrToOrigTy &PromotedInsts,
unsigned &CreatedInstsCost,
4146 return promoteOperandForOther(
Ext, TPT, PromotedInsts, CreatedInstsCost,
4147 Exts, Truncs, TLI,
false);
4153 InstrToOrigTy &PromotedInsts,
4154 unsigned &CreatedInstsCost,
4168 static Action getAction(
Instruction *
Ext,
const SetOfInstrs &InsertedInsts,
4170 const InstrToOrigTy &PromotedInsts);
4175 bool TypePromotionHelper::canGetThrough(
const Instruction *Inst,
4176 Type *ConsideredExtType,
4177 const InstrToOrigTy &PromotedInsts,
4186 if (isa<ZExtInst>(Inst))
4190 if (IsSExt && isa<SExtInst>(Inst))
4195 if (
const auto *BinOp = dyn_cast<BinaryOperator>(Inst))
4196 if (isa<OverflowingBinaryOperator>(BinOp) &&
4197 ((!IsSExt && BinOp->hasNoUnsignedWrap()) ||
4198 (IsSExt && BinOp->hasNoSignedWrap())))
4202 if ((Inst->
getOpcode() == Instruction::And ||
4207 if (Inst->
getOpcode() == Instruction::Xor) {
4209 if (
const auto *Cst = dyn_cast<ConstantInt>(Inst->
getOperand(1)))
4210 if (!Cst->getValue().isAllOnes())
4219 if (Inst->
getOpcode() == Instruction::LShr && !IsSExt)
4228 const auto *ExtInst = cast<const Instruction>(*Inst->
user_begin());
4229 if (ExtInst->hasOneUse()) {
4230 const auto *AndInst = dyn_cast<const Instruction>(*ExtInst->user_begin());
4231 if (AndInst && AndInst->getOpcode() == Instruction::And) {
4232 const auto *Cst = dyn_cast<ConstantInt>(AndInst->getOperand(1));
4242 if (!isa<TruncInst>(Inst))
4256 Instruction *Opnd = dyn_cast<Instruction>(OpndVal);
4264 const Type *OpndType = getOrigType(PromotedInsts, Opnd, IsSExt);
4267 else if ((IsSExt && isa<SExtInst>(Opnd)) || (!IsSExt && isa<ZExtInst>(Opnd)))
4277 TypePromotionHelper::Action TypePromotionHelper::getAction(
4281 "Unexpected instruction type");
4282 Instruction *ExtOpnd = dyn_cast<Instruction>(
Ext->getOperand(0));
4284 bool IsSExt = isa<SExtInst>(
Ext);
4288 if (!ExtOpnd || !canGetThrough(ExtOpnd, ExtTy, PromotedInsts, IsSExt))
4294 if (isa<TruncInst>(ExtOpnd) && InsertedInsts.count(ExtOpnd))
4299 if (isa<SExtInst>(ExtOpnd) || isa<TruncInst>(ExtOpnd) ||
4300 isa<ZExtInst>(ExtOpnd))
4301 return promoteOperandForTruncAndAnyExt;
4307 return IsSExt ? signExtendOperandForOther : zeroExtendOperandForOther;
4310 Value *TypePromotionHelper::promoteOperandForTruncAndAnyExt(
4312 InstrToOrigTy &PromotedInsts,
unsigned &CreatedInstsCost,
4318 Value *ExtVal = SExt;
4319 bool HasMergedNonFreeExt =
false;
4320 if (isa<ZExtInst>(SExtOpnd)) {
4323 HasMergedNonFreeExt = !TLI.
isExtFree(SExtOpnd);
4326 TPT.replaceAllUsesWith(SExt, ZExt);
4327 TPT.eraseInstruction(SExt);
4332 TPT.setOperand(SExt, 0, SExtOpnd->
getOperand(0));
4334 CreatedInstsCost = 0;
4338 TPT.eraseInstruction(SExtOpnd);
4341 Instruction *ExtInst = dyn_cast<Instruction>(ExtVal);
4345 Exts->push_back(ExtInst);
4346 CreatedInstsCost = !TLI.
isExtFree(ExtInst) && !HasMergedNonFreeExt;
4354 TPT.eraseInstruction(ExtInst, NextVal);
4358 Value *TypePromotionHelper::promoteOperandForOther(
4360 InstrToOrigTy &PromotedInsts,
unsigned &CreatedInstsCost,
4367 CreatedInstsCost = 0;
4374 if (
Instruction *ITrunc = dyn_cast<Instruction>(Trunc)) {
4376 ITrunc->moveAfter(ExtOpnd);
4378 Truncs->push_back(ITrunc);
4381 TPT.replaceAllUsesWith(ExtOpnd, Trunc);
4384 TPT.setOperand(
Ext, 0, ExtOpnd);
4394 addPromotedInst(PromotedInsts, ExtOpnd, IsSExt);
4396 TPT.mutateType(ExtOpnd,
Ext->getType());
4398 TPT.replaceAllUsesWith(
Ext, ExtOpnd);
4403 for (
int OpIdx = 0, EndOpIdx = ExtOpnd->
getNumOperands(); OpIdx != EndOpIdx;
4407 !shouldExtOperand(ExtOpnd, OpIdx)) {
4413 if (
const ConstantInt *Cst = dyn_cast<ConstantInt>(Opnd)) {
4415 unsigned BitWidth =
Ext->getType()->getIntegerBitWidth();
4422 if (isa<UndefValue>(Opnd)) {
4433 Value *ValForExtOpnd = IsSExt ? TPT.createSExt(
Ext, Opnd,
Ext->getType())
4434 : TPT.createZExt(
Ext, Opnd,
Ext->getType());
4435 if (!isa<Instruction>(ValForExtOpnd)) {
4436 TPT.setOperand(ExtOpnd, OpIdx, ValForExtOpnd);
4439 ExtForOpnd = cast<Instruction>(ValForExtOpnd);
4442 Exts->push_back(ExtForOpnd);
4443 TPT.setOperand(ExtForOpnd, 0, Opnd);
4446 TPT.moveBefore(ExtForOpnd, ExtOpnd);
4447 TPT.setOperand(ExtOpnd, OpIdx, ExtForOpnd);
4448 CreatedInstsCost += !TLI.
isExtFree(ExtForOpnd);
4450 ExtForOpnd =
nullptr;
4452 if (ExtForOpnd ==
Ext) {
4454 TPT.eraseInstruction(
Ext);
4467 bool AddressingModeMatcher::isPromotionProfitable(
4468 unsigned NewCost,
unsigned OldCost,
Value *PromotedOperand)
const {
4469 LLVM_DEBUG(
dbgs() <<
"OldCost: " << OldCost <<
"\tNewCost: " << NewCost
4474 if (NewCost > OldCost)
4476 if (NewCost < OldCost)
4495 bool AddressingModeMatcher::matchOperationAddr(
User *AddrInst,
unsigned Opcode,
4499 if (
Depth >= 5)
return false;
4506 case Instruction::PtrToInt:
4509 case Instruction::IntToPtr: {
4511 auto PtrTy = MVT::getIntegerVT(
DL.getPointerSizeInBits(AS));
4517 case Instruction::BitCast:
4527 case Instruction::AddrSpaceCast: {
4538 unsigned OldSize = AddrModeInsts.size();
4543 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
4544 TPT.getRestorationPoint();
4553 AddrModeInsts.
resize(OldSize);
4554 TPT.rollback(LastKnownGood);
4563 AddrModeInsts.
resize(OldSize);
4564 TPT.rollback(LastKnownGood);
4571 case Instruction::Shl: {
4575 if (!
RHS ||
RHS->getBitWidth() > 64)
4577 int64_t Scale = Opcode == Instruction::Shl
4578 ? 1LL <<
RHS->getLimitedValue(
RHS->getBitWidth() - 1)
4579 :
RHS->getSExtValue();
4583 case Instruction::GetElementPtr: {
4586 int VariableOperand = -1;
4587 unsigned VariableScale = 0;
4589 int64_t ConstantOffset = 0;
4595 cast<ConstantInt>(AddrInst->
getOperand(
i))->getZExtValue();
4613 if (VariableOperand != -1)
4617 VariableOperand =
i;
4625 if (VariableOperand == -1) {
4626 AddrMode.BaseOffs += ConstantOffset;
4627 if (ConstantOffset == 0 ||
4631 if (!cast<GEPOperator>(AddrInst)->isInBounds())
4637 ConstantOffset > 0) {
4643 auto *BaseI = dyn_cast<Instruction>(
Base);
4644 auto *
GEP = cast<GetElementPtrInst>(AddrInst);
4645 if (isa<Argument>(
Base) || isa<GlobalValue>(
Base) ||
4646 (BaseI && !isa<CastInst>(BaseI) &&
4647 !isa<GetElementPtrInst>(BaseI))) {
4651 BaseI ? BaseI->
getParent() : &
GEP->getFunction()->getEntryBlock();
4653 LargeOffsetGEP = std::make_pair(
GEP, ConstantOffset);
4656 AddrMode.BaseOffs -= ConstantOffset;
4662 unsigned OldSize = AddrModeInsts.size();
4665 AddrMode.BaseOffs += ConstantOffset;
4666 if (!cast<GEPOperator>(AddrInst)->isInBounds())
4674 AddrModeInsts.
resize(OldSize);
4682 if (!matchScaledValue(AddrInst->
getOperand(VariableOperand), VariableScale,
4687 AddrModeInsts.
resize(OldSize);
4692 AddrMode.BaseOffs += ConstantOffset;
4693 if (!matchScaledValue(AddrInst->
getOperand(VariableOperand),
4694 VariableScale,
Depth)) {
4697 AddrModeInsts.
resize(OldSize);
4704 case Instruction::SExt:
4705 case Instruction::ZExt: {
4712 TypePromotionHelper::Action TPH =
4713 TypePromotionHelper::getAction(
Ext, InsertedInsts, TLI, PromotedInsts);
4717 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
4718 TPT.getRestorationPoint();
4719 unsigned CreatedInstsCost = 0;
4721 Value *PromotedOperand =
4722 TPH(
Ext, TPT, PromotedInsts, CreatedInstsCost,
nullptr,
nullptr, TLI);
4737 assert(PromotedOperand &&
4738 "TypePromotionHelper should have filtered out those cases");
4741 unsigned OldSize = AddrModeInsts.size();
4743 if (!matchAddr(PromotedOperand,
Depth) ||
4748 !isPromotionProfitable(CreatedInstsCost,
4749 ExtCost + (AddrModeInsts.size() - OldSize),
4752 AddrModeInsts.
resize(OldSize);
4753 LLVM_DEBUG(
dbgs() <<
"Sign extension does not pay off: rollback\n");
4754 TPT.rollback(LastKnownGood);
4768 bool AddressingModeMatcher::matchAddr(
Value *
Addr,
unsigned Depth) {
4771 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
4772 TPT.getRestorationPoint();
4791 unsigned OldSize = AddrModeInsts.size();
4794 bool MovedAway =
false;
4795 if (matchOperationAddr(
I,
I->getOpcode(),
Depth, &MovedAway)) {
4803 if (
I->hasOneUse() ||
4804 isProfitableToFoldIntoAddressingMode(
I, BackupAddrMode,
AddrMode)) {
4805 AddrModeInsts.push_back(
I);
4811 AddrModeInsts.
resize(OldSize);
4812 TPT.rollback(LastKnownGood);
4815 if (matchOperationAddr(CE,
CE->getOpcode(),
Depth))
4817 TPT.rollback(LastKnownGood);
4818 }
else if (isa<ConstantPointerNull>(
Addr)) {
4844 TPT.rollback(LastKnownGood);
4863 if (OpInfo.CallOperandVal == OpVal &&
4864 (OpInfo.ConstraintType != TargetLowering::C_Memory ||
4865 !OpInfo.isIndirect))
4885 if (!ConsideredInsts.
insert(
I).second)
4893 for (
Use &U :
I->uses()) {
4899 Instruction *UserI = cast<Instruction>(U.getUser());
4900 if (
LoadInst *LI = dyn_cast<LoadInst>(UserI)) {
4901 MemoryUses.push_back({U.get(), LI->getType()});
4908 MemoryUses.push_back({U.get(),
SI->getValueOperand()->getType()});
4915 MemoryUses.push_back({U.get(), RMW->getValOperand()->getType()});
4922 MemoryUses.push_back({U.get(), CmpX->getCompareOperand()->getType()});
4926 if (
CallInst *CI = dyn_cast<CallInst>(UserI)) {
4930 bool OptForSize = OptSize ||
4936 InlineAsm *IA = dyn_cast<InlineAsm>(CI->getCalledOperand());
4937 if (!IA)
return true;
4946 PSI,
BFI, SeenInsts))
4957 bool AddressingModeMatcher::valueAlreadyLiveAtInst(
Value *Val,
Value *KnownLive1,
4958 Value *KnownLive2) {
4960 if (Val ==
nullptr || Val == KnownLive1 || Val == KnownLive2)
4964 if (!isa<Instruction>(Val) && !isa<Argument>(Val))
return true;
4969 if (
AllocaInst *AI = dyn_cast<AllocaInst>(Val))
5000 bool AddressingModeMatcher::
5003 if (IgnoreProfitability)
return true;
5020 if (valueAlreadyLiveAtInst(ScaledReg, AMBefore.
BaseReg, AMBefore.
ScaledReg))
5021 ScaledReg =
nullptr;
5025 if (!BaseReg && !ScaledReg)
5048 for (
const std::pair<Value *, Type *> &Pair : MemoryUses) {
5050 Type *AddressAccessTy = Pair.second;
5051 unsigned AS =
Address->getType()->getPointerAddressSpace();
5057 std::pair<AssertingVH<GetElementPtrInst>, int64_t> LargeOffsetGEP(
nullptr,
5059 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
5060 TPT.getRestorationPoint();
5061 AddressingModeMatcher Matcher(MatchedAddrModeInsts, TLI,
TRI, LI, getDTFn,
5062 AddressAccessTy, AS, MemoryInst, Result,
5063 InsertedInsts, PromotedInsts, TPT,
5064 LargeOffsetGEP, OptSize, PSI,
BFI);
5065 Matcher.IgnoreProfitability =
true;
5066 bool Success = Matcher.matchAddr(Address, 0);
5072 TPT.rollback(LastKnownGood);
5078 MatchedAddrModeInsts.
clear();
5088 return I->getParent() !=
BB;
5112 Type *AccessTy,
unsigned AddrSpace) {
5119 worklist.push_back(
Addr);
5124 bool PhiOrSelectSeen =
false;
5127 AddressingModeCombiner AddrModes(SQ,
Addr);
5128 TypePromotionTransaction TPT(RemovedInsts);
5129 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
5130 TPT.getRestorationPoint();
5131 while (!worklist.empty()) {
5143 if (!Visited.
insert(V).second)
5147 if (
PHINode *
P = dyn_cast<PHINode>(V)) {
5149 PhiOrSelectSeen =
true;
5154 worklist.push_back(
SI->getFalseValue());
5155 worklist.push_back(
SI->getTrueValue());
5156 PhiOrSelectSeen =
true;
5163 AddrModeInsts.
clear();
5164 std::pair<AssertingVH<GetElementPtrInst>, int64_t> LargeOffsetGEP(
nullptr,
5169 auto getDTFn = [MemoryInst,
this]() ->
const DominatorTree & {
5171 return this->getDT(*
F);
5173 ExtAddrMode NewAddrMode = AddressingModeMatcher::Match(
5174 V, AccessTy, AddrSpace, MemoryInst, AddrModeInsts, *TLI, *LI, getDTFn,
5175 *
TRI, InsertedInsts, PromotedInsts, TPT, LargeOffsetGEP, OptSize, PSI,
5183 LargeOffsetGEPMap[
GEP->getPointerOperand()].push_back(LargeOffsetGEP);
5184 LargeOffsetGEPID.
insert(std::make_pair(
GEP, LargeOffsetGEPID.
size()));
5187 NewAddrMode.OriginalValue = V;
5188 if (!AddrModes.addNewAddrMode(NewAddrMode))
5195 if (!AddrModes.combineAddrModes()) {
5196 TPT.rollback(LastKnownGood);
5208 if (!PhiOrSelectSeen &&
none_of(AddrModeInsts, [&](
Value *V) {
5232 <<
" for " << *MemoryInst <<
"\n");
5234 SunkAddr =
Builder.CreatePointerCast(SunkAddr,
Addr->getType());
5236 SubtargetInfo->addrSinkUsingGEPs())) {
5240 <<
" for " << *MemoryInst <<
"\n");
5241 Type *IntPtrTy =
DL->getIntPtrType(
Addr->getType());
5242 Value *ResultPtr =
nullptr, *ResultIndex =
nullptr;
5253 if (ResultPtr ||
AddrMode.Scale != 1)
5284 if (!
DL->isNonIntegralPointerType(
Addr->getType())) {
5285 if (!ResultPtr &&
AddrMode.BaseReg) {
5289 }
else if (!ResultPtr &&
AddrMode.Scale == 1) {
5298 SunkAddr = Constant::getNullValue(
Addr->getType());
5299 }
else if (!ResultPtr) {
5303 Builder.getInt8PtrTy(
Addr->getType()->getPointerAddressSpace());
5314 V =
Builder.CreateIntCast(V, IntPtrTy,
true,
"sunkaddr");
5322 if (V->
getType() == IntPtrTy) {
5326 cast<IntegerType>(V->
getType())->getBitWidth() &&
5327 "We can't transform if ScaledReg is too narrow");
5328 V =
Builder.CreateTrunc(V, IntPtrTy,
"sunkaddr");
5335 ResultIndex =
Builder.CreateAdd(ResultIndex, V,
"sunkaddr");
5346 if (ResultPtr->
getType() != I8PtrTy)
5347 ResultPtr =
Builder.CreatePointerCast(ResultPtr, I8PtrTy);
5348 ResultPtr =
Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex,
5356 SunkAddr = ResultPtr;
5358 if (ResultPtr->
getType() != I8PtrTy)
5359 ResultPtr =
Builder.CreatePointerCast(ResultPtr, I8PtrTy);
5360 SunkAddr =
Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex,
"sunkaddr",
5365 SunkAddr =
Builder.CreatePointerCast(SunkAddr,
Addr->getType());
5372 PointerType *BasePtrTy = dyn_cast_or_null<PointerType>(BaseTy);
5373 PointerType *ScalePtrTy = dyn_cast_or_null<PointerType>(ScaleTy);
5374 if (
DL->isNonIntegralPointerType(
Addr->getType()) ||
5375 (BasePtrTy &&
DL->isNonIntegralPointerType(BasePtrTy)) ||
5376 (ScalePtrTy &&
DL->isNonIntegralPointerType(ScalePtrTy)) ||
5378 DL->isNonIntegralPointerType(
AddrMode.BaseGV->getType())))
5382 <<
" for " << *MemoryInst <<
"\n");
5383 Type *IntPtrTy =
DL->getIntPtrType(
Addr->getType());
5394 V =
Builder.CreatePtrToInt(V, IntPtrTy,
"sunkaddr");
5396 V =
Builder.CreateIntCast(V, IntPtrTy,
true,
"sunkaddr");
5403 if (V->
getType() == IntPtrTy) {
5406 V =
Builder.CreatePtrToInt(V, IntPtrTy,
"sunkaddr");
5407 }
else if (cast<IntegerType>(IntPtrTy)->
getBitWidth() <
5408 cast<IntegerType>(V->
getType())->getBitWidth()) {
5409 V =
Builder.CreateTrunc(V, IntPtrTy,
"sunkaddr");
5416 Instruction *
I = dyn_cast_or_null<Instruction>(Result);
5418 I->eraseFromParent();