80#define DEBUG_TYPE "argpromotion"
82STATISTIC(NumArgumentsPromoted,
"Number of pointer arguments promoted");
83STATISTIC(NumArgumentsDead,
"Number of dead pointer args eliminated");
95using OffsetAndArgPart = std::pair<int64_t, ArgPart>;
103 unsigned AddrSpace =
Ptr->getType()->getPointerAddressSpace();
105 if (!
Ptr->getType()->isOpaquePointerTy()) {
106 Type *OrigElemTy =
Ptr->getType()->getNonOpaquePointerElementType();
107 if (OrigOffset == 0 && OrigElemTy == ResElemTy)
111 APInt TmpOffset = OrigOffset;
112 Type *TmpTy = OrigElemTy;
114 DL.getGEPIndicesForOffset(TmpTy, TmpOffset);
115 if (TmpOffset == 0) {
117 while (TmpTy != ResElemTy) {
123 isa<StructType>(TmpTy) ? 32 : OrigOffset.
getBitWidth()));
131 if (OrigOffset != 0 || TmpTy == ResElemTy) {
139 if (OrigOffset != 0) {
156 std::vector<Type *> Params;
168 if (!ArgsToPromote.count(&*
I)) {
170 Params.push_back(
I->getType());
172 }
else if (
I->use_empty()) {
176 const auto &ArgParts = ArgsToPromote.find(&*
I)->second;
177 for (
const auto &Pair : ArgParts) {
178 Params.push_back(Pair.second.Ty);
181 ++NumArgumentsPromoted;
199 F->setSubprogram(
nullptr);
201 LLVM_DEBUG(
dbgs() <<
"ARG PROMOTION: Promoting to:" << *NF <<
"\n"
205 for (
auto *
I : Params)
206 if (
auto *VT = dyn_cast<llvm::VectorType>(
I))
207 LargestVectorWidth = std::max(
208 LargestVectorWidth, VT->getPrimitiveSizeInBits().getKnownMinValue());
217 F->getParent()->getFunctionList().insert(
F->getIterator(), NF);
226 while (!
F->use_empty()) {
227 CallBase &CB = cast<CallBase>(*
F->user_back());
237 ++
I, ++AI, ++ArgNo) {
238 if (!ArgsToPromote.count(&*
I)) {
241 }
else if (!
I->use_empty()) {
243 const auto &ArgParts = ArgsToPromote.find(&*
I)->second;
244 for (
const auto &Pair : ArgParts) {
248 Pair.second.Alignment, V->getName() +
".val");
249 if (Pair.second.MustExecInstr) {
250 LI->
setAAMetadata(Pair.second.MustExecInstr->getAAMetadata());
252 {LLVMContext::MD_dereferenceable,
253 LLVMContext::MD_dereferenceable_or_null,
254 LLVMContext::MD_noundef,
255 LLVMContext::MD_nontemporal});
262 {LLVMContext::MD_range, LLVMContext::MD_nonnull,
263 LLVMContext::MD_align});
269 assert(ArgsToPromote.count(&*
I) &&
I->use_empty());
275 for (; AI != CB.
arg_end(); ++AI, ++ArgNo) {
284 if (
InvokeInst *II = dyn_cast<InvokeInst>(&CB)) {
286 Args, OpBundles,
"", &CB);
289 NewCall->setTailCallKind(cast<CallInst>(&CB)->getTailCallKind());
296 NewCS->
copyMetadata(CB, {LLVMContext::MD_prof, LLVMContext::MD_dbg});
328 if (!ArgsToPromote.count(&
Arg)) {
331 Arg.replaceAllUsesWith(&*I2);
351 "Only arguments with a pointer type are promotable");
357 for (
const auto &Pair : ArgsToPromote.find(&
Arg)->second) {
358 int64_t
Offset = Pair.first;
359 const ArgPart &Part = Pair.second;
387 while (!Worklist.
empty()) {
389 if (isa<BitCastInst>(V) || isa<GetElementPtrInst>(V)) {
390 DeadInsts.
push_back(cast<Instruction>(V));
395 if (
auto *LI = dyn_cast<LoadInst>(V)) {
396 Value *
Ptr = LI->getPointerOperand();
401 if (
auto *
SI = dyn_cast<StoreInst>(V)) {
402 assert(!
SI->isVolatile() &&
"Volatile operations can't be promoted.");
413 I->eraseFromParent();
417 for (
const auto &Pair : OffsetToAlloca) {
419 "By design, only promotable allocas should be produced.");
425 <<
" alloca(s) are promotable by Mem2Reg\n");
427 if (!Allocas.
empty()) {
446 APInt Bytes(64, NeededDerefBytes);
455 CallBase &CB = cast<CallBase>(*U);
456 return isDereferenceableAndAlignedPointer(CB.getArgOperand(Arg->getArgNo()),
457 NeededAlign, Bytes, DL);
464 unsigned MaxElements,
bool IsRecursive,
467 if (
Arg->use_empty())
484 Align NeededAlign(1);
491 bool AreStoresAllowed =
Arg->getParamByValType() &&
Arg->getParamAlign();
496 auto HandleEndUser = [&](
auto *
I,
Type *Ty,
497 bool GuaranteedToExecute) -> std::optional<bool> {
509 if (
Offset.getSignificantBits() >= 64)
514 if (
Size.isScalable())
519 if (IsRecursive && Ty->isPointerTy())
522 int64_t Off =
Offset.getSExtValue();
524 Off, ArgPart{Ty,
I->getAlign(), GuaranteedToExecute ?
I :
nullptr});
525 ArgPart &Part = Pair.first->second;
526 bool OffsetNotSeenBefore = Pair.second;
530 if (MaxElements > 0 && ArgParts.
size() > MaxElements) {
532 <<
"more than " << MaxElements <<
" parts\n");
540 <<
"accessed as both " << *Part.Ty <<
" and " << *Ty
541 <<
" at offset " << Off <<
"\n");
551 if (!GuaranteedToExecute &&
552 (OffsetNotSeenBefore || Part.Alignment <
I->getAlign())) {
561 NeededDerefBytes = std::max(NeededDerefBytes, Off +
Size.getFixedValue());
562 NeededAlign = std::max(NeededAlign,
I->getAlign());
565 Part.Alignment = std::max(Part.Alignment,
I->getAlign());
571 std::optional<bool> Res{};
572 if (
LoadInst *LI = dyn_cast<LoadInst>(&
I))
573 Res = HandleEndUser(LI, LI->getType(),
true);
575 Res = HandleEndUser(
SI,
SI->getValueOperand()->getType(),
589 auto AppendUses = [&](
const Value *V) {
590 for (
const Use &U : V->uses())
591 if (Visited.
insert(&U).second)
595 while (!Worklist.
empty()) {
597 Value *V = U->getUser();
598 if (isa<BitCastInst>(V)) {
603 if (
auto *
GEP = dyn_cast<GetElementPtrInst>(V)) {
604 if (!
GEP->hasAllConstantIndices())
610 if (
auto *LI = dyn_cast<LoadInst>(V)) {
611 if (!*HandleEndUser(LI, LI->getType(),
false))
618 auto *
SI = dyn_cast<StoreInst>(V);
619 if (AreStoresAllowed &&
SI &&
621 if (!*HandleEndUser(
SI,
SI->getValueOperand()->getType(),
631 <<
"unknown user " << *V <<
"\n");
635 if (NeededDerefBytes || NeededAlign > 1) {
640 <<
"not dereferenceable or aligned\n");
645 if (ArgParts.
empty())
653 int64_t
Offset = ArgPartsVec[0].first;
654 for (
const auto &Pair : ArgPartsVec) {
658 Offset = Pair.first +
DL.getTypeStoreSize(Pair.second.Ty);
664 if (AreStoresAllowed)
706 CallBase *CB = dyn_cast<CallBase>(U.getUser());
710 const Function *Caller = CB->getCaller();
711 const Function *Callee = CB->getCalledFunction();
712 return TTI.areTypesABICompatible(Caller, Callee, Types);
721 unsigned MaxElements,
bool IsRecursive) {
725 if (
F->hasFnAttribute(Attribute::Naked))
729 if (!
F->hasLocalLinkage())
742 if (
F->getAttributes().hasAttrSomewhere(Attribute::InAlloca))
748 if (
I.getType()->isPointerTy())
750 if (PointerArgs.
empty())
756 for (
Use &U :
F->uses()) {
757 CallBase *CB = dyn_cast<CallBase>(U.getUser());
759 if (CB ==
nullptr || !CB->
isCallee(&U) ||
774 if (BB.getTerminatingMustTailCall())
784 unsigned NumArgsAfterPromote =
F->getFunctionType()->getNumParams();
785 for (
Argument *PtrArg : PointerArgs) {
788 if (PtrArg->hasStructRetAttr()) {
789 unsigned ArgNo = PtrArg->getArgNo();
790 F->removeParamAttr(ArgNo, Attribute::StructRet);
791 F->addParamAttr(ArgNo, Attribute::NoAlias);
792 for (
Use &U :
F->uses()) {
793 CallBase &CB = cast<CallBase>(*U.getUser());
802 if (
findArgParts(PtrArg,
DL, AAR, MaxElements, IsRecursive, ArgParts)) {
804 for (
const auto &Pair : ArgParts)
805 Types.push_back(Pair.second.Ty);
808 NumArgsAfterPromote += ArgParts.size() - 1;
809 ArgsToPromote.
insert({PtrArg, std::move(ArgParts)});
815 if (ArgsToPromote.
empty())
828 bool Changed =
false, LocalChange;
837 bool IsRecursive =
C.size() > 1;
850 C.getOuterRefSCC().replaceNodeFunction(
N, *NewF);
856 for (
auto *U : NewF->
users()) {
857 auto *UserF = cast<CallBase>(U)->getFunction();
862 Changed |= LocalChange;
863 }
while (LocalChange);
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
This file contains the simple types necessary to represent the attributes associated with functions a...
This is the interface for LLVM's primary stateless and local alias analysis.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
This file provides utility analysis objects describing memory locations.
FunctionAnalysisManager FAM
This header defines various interfaces for pass management in LLVM.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This defines the Use class.
A manager for alias analyses.
bool canInstructionRangeModRef(const Instruction &I1, const Instruction &I2, const MemoryLocation &Loc, const ModRefInfo Mode)
Check if it is possible for the execution of the specified instructions to mod(according to the mode)...
bool canBasicBlockModify(const BasicBlock &BB, const MemoryLocation &Loc)
Check if it is possible for execution of the specified basic block to modify the location Loc.
Class for arbitrary precision integers.
unsigned getBitWidth() const
Return the number of bits in the APInt.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
This templated class represents "all analyses that operate over <a particular IR unit>" (e....
an instruction to allocate memory on the stack
void setAlignment(Align Align)
A container for analyses that lazily runs them and caches their results.
void clear(IRUnitT &IR, llvm::StringRef Name)
Clear any cached analysis results for a single unit of IR.
void invalidate(IRUnitT &IR, const PreservedAnalyses &PA)
Invalidate cached analyses for an IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
A function analysis which provides an AssumptionCache.
AttributeSet getFnAttrs() const
The function attributes are returned.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
AttributeSet getParamAttrs(unsigned ArgNo) const
The attributes for the argument or parameter at the given index are returned.
LLVM Basic Block Representation.
const Instruction & front() const
Represents analyses that only rely on functions' control flow.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void setCallingConv(CallingConv::ID CC)
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
void removeParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Removes the attribute from the given argument.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
CallingConv::ID getCallingConv() const
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isCallee(Value::const_user_iterator UI) const
Determine whether the passed iterator points to the callee operand's Use.
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
FunctionType * getFunctionType() const
AttributeList getAttributes() const
Return the parameter attributes for this call.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
Function * getCaller()
Helper to get the caller (the parent function).
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
A parsed version of the target data layout string in and methods for querying it.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Analysis pass which computes a DominatorTree.
A proxy from a FunctionAnalysisManager to an SCC.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
void splice(Function::iterator ToIt, Function *FromF)
Transfer all blocks from FromF to this function at ToIt.
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
void setAttributes(AttributeList Attrs)
Set the attribute list for this Function.
void copyAttributesFrom(const Function *Src)
copyAttributesFrom - copy all additional attributes (those not needed to create a Function) from the ...
static Type * getTypeAtIndex(Type *Ty, Value *Idx)
Return the type of the element at the given index of an indexable type.
void copyMetadata(const GlobalObject *Src, unsigned Offset)
Copy metadata from Src, adjusting offsets by Offset.
Common base class shared among various IRBuilders.
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
PointerType * getInt8PtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer to an 8-bit integer value.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
ConstantInt * getInt(const APInt &AI)
Get a constant integer value.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
void setAAMetadata(const AAMDNodes &N)
Sets the AA metadata on this instruction from the AAMDNodes structure.
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
const Function * getFunction() const
Return the function this instruction belongs to.
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
static InvokeInst * Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef< Value * > Args, const Twine &NameStr, Instruction *InsertBefore=nullptr)
A node in the call graph.
An SCC of the call graph.
A lazily constructed view of the call graph of a module.
An instruction for reading from memory.
static unsigned getPointerOperandIndex()
Representation for a specific memory location.
static MemoryLocation get(const LoadInst *LI)
Return a location with information about the memory reference by the given instruction.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void preserveSet()
Mark an analysis set as preserved.
void preserve()
Mark an analysis as preserved.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
static unsigned getPointerOperandIndex()
Analysis pass providing the TargetTransformInfo.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
LLVM Value Representation.
void setName(const Twine &Name)
Change the name of the value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
StringRef getName() const
Return a constant reference to the value's name.
void takeName(Value *V)
Transfer the name from V to this value.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void updateMinLegalVectorWidthAttr(Function &Fn, uint64_t Width)
Update min-legal-vector-width if it is in Attribute and less than Width.
@ C
The default llvm calling convention, compatible with C.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
void PromoteMemToReg(ArrayRef< AllocaInst * > Allocas, DominatorTree &DT, AssumptionCache *AC=nullptr)
Promote the specified list of alloca instructions into scalar registers, inserting PHI nodes as appro...
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
bool isAligned(Align Lhs, uint64_t SizeInBytes)
Checks that SizeInBytes is a multiple of the alignment.
bool isDereferenceableAndAlignedPointer(const Value *V, Type *Ty, Align Alignment, const DataLayout &DL, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Returns true if V is always a dereferenceable pointer with alignment greater or equal than requested.
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
bool isAllocaPromotable(const AllocaInst *AI)
Return true if this alloca is legal for promotion.
void sort(IteratorTy Start, IteratorTy End)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
iterator_range< idf_ext_iterator< T, SetTy > > inverse_depth_first_ext(const T &G, SetTy &S)
bool RecursivelyDeleteTriviallyDeadInstructionsPermissive(SmallVectorImpl< WeakTrackingVH > &DeadInsts, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, std::function< void(Value *)> AboutToDeleteCallback=std::function< void(Value *)>())
Same functionality as RecursivelyDeleteTriviallyDeadInstructions, but allow instructions that are not...
bool isGuaranteedToTransferExecutionToSuccessor(const Instruction *I)
Return true if this function can prove that the instruction I will always transfer execution to one o...
auto predecessors(const MachineBasicBlock *BB)
This struct is a compact representation of a valid (non-zero power of two) alignment.
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Function object to check whether the first component of a container supported by std::get (like std::...