80#define DEBUG_TYPE "argpromotion"
82STATISTIC(NumArgumentsPromoted,
"Number of pointer arguments promoted");
83STATISTIC(NumArgumentsDead,
"Number of dead pointer args eliminated");
95using OffsetAndArgPart = std::pair<int64_t, ArgPart>;
118 std::vector<Type *> Params;
130 unsigned ArgNo = 0, NewArgNo = 0;
133 if (!ArgsToPromote.count(&*
I)) {
135 Params.push_back(
I->getType());
138 }
else if (
I->use_empty()) {
143 const auto &ArgParts = ArgsToPromote.find(&*
I)->second;
144 for (
const auto &Pair : ArgParts) {
145 Params.push_back(Pair.second.Ty);
148 ++NumArgumentsPromoted;
150 NewArgNo += ArgParts.size();
168 F->setSubprogram(
nullptr);
170 LLVM_DEBUG(
dbgs() <<
"ARG PROMOTION: Promoting to:" << *NF <<
"\n"
174 for (
auto *
I : Params)
175 if (
auto *VT = dyn_cast<llvm::VectorType>(
I))
176 LargestVectorWidth = std::max(
177 LargestVectorWidth, VT->getPrimitiveSizeInBits().getKnownMinValue());
186 unsigned Arg1 = NewArgIndices[AllocSize->first];
187 assert(Arg1 != (
unsigned)-1 &&
"allocsize cannot be promoted argument");
188 std::optional<unsigned> Arg2;
189 if (AllocSize->second) {
190 Arg2 = NewArgIndices[*AllocSize->second];
191 assert(Arg2 != (
unsigned)-1 &&
"allocsize cannot be promoted argument");
199 F->getParent()->getFunctionList().insert(
F->getIterator(), NF);
208 while (!
F->use_empty()) {
209 CallBase &CB = cast<CallBase>(*
F->user_back());
219 ++
I, ++AI, ++ArgNo) {
220 if (!ArgsToPromote.count(&*
I)) {
223 }
else if (!
I->use_empty()) {
225 const auto &ArgParts = ArgsToPromote.find(&*
I)->second;
226 for (
const auto &Pair : ArgParts) {
230 Pair.second.Alignment, V->getName() +
".val");
231 if (Pair.second.MustExecInstr) {
232 LI->
setAAMetadata(Pair.second.MustExecInstr->getAAMetadata());
234 {LLVMContext::MD_dereferenceable,
235 LLVMContext::MD_dereferenceable_or_null,
236 LLVMContext::MD_noundef,
237 LLVMContext::MD_nontemporal});
244 {LLVMContext::MD_range, LLVMContext::MD_nonnull,
245 LLVMContext::MD_align});
251 assert(ArgsToPromote.count(&*
I) &&
I->use_empty());
257 for (; AI != CB.
arg_end(); ++AI, ++ArgNo) {
266 if (
InvokeInst *II = dyn_cast<InvokeInst>(&CB)) {
268 Args, OpBundles,
"", &CB);
271 NewCall->setTailCallKind(cast<CallInst>(&CB)->getTailCallKind());
278 NewCS->
copyMetadata(CB, {LLVMContext::MD_prof, LLVMContext::MD_dbg});
310 if (!ArgsToPromote.count(&Arg)) {
332 assert(Arg.getType()->isPointerTy() &&
333 "Only arguments with a pointer type are promotable");
339 for (
const auto &Pair : ArgsToPromote.find(&Arg)->second) {
340 int64_t
Offset = Pair.first;
341 const ArgPart &Part = Pair.second;
347 Part.Ty,
nullptr, Arg.getName() +
"." +
Twine(
Offset) +
".allc");
359 assert(
Ptr == &Arg &&
"Not constant offset from arg?");
369 while (!Worklist.
empty()) {
371 if (isa<BitCastInst>(V) || isa<GetElementPtrInst>(V)) {
372 DeadInsts.
push_back(cast<Instruction>(V));
377 if (
auto *LI = dyn_cast<LoadInst>(V)) {
378 Value *
Ptr = LI->getPointerOperand();
383 if (
auto *SI = dyn_cast<StoreInst>(V)) {
384 assert(!SI->isVolatile() &&
"Volatile operations can't be promoted.");
385 Value *
Ptr = SI->getPointerOperand();
395 I->eraseFromParent();
399 for (
const auto &Pair : OffsetToAlloca) {
401 "By design, only promotable allocas should be produced.");
407 <<
" alloca(s) are promotable by Mem2Reg\n");
409 if (!Allocas.
empty()) {
427 const DataLayout &
DL = Callee->getParent()->getDataLayout();
428 APInt Bytes(64, NeededDerefBytes);
437 CallBase &CB = cast<CallBase>(*U);
438 return isDereferenceableAndAlignedPointer(CB.getArgOperand(Arg->getArgNo()),
439 NeededAlign, Bytes, DL);
446 unsigned MaxElements,
bool IsRecursive,
466 Align NeededAlign(1);
478 auto HandleEndUser = [&](
auto *
I,
Type *Ty,
479 bool GuaranteedToExecute) -> std::optional<bool> {
491 if (
Offset.getSignificantBits() >= 64)
496 if (
Size.isScalable())
501 if (IsRecursive && Ty->isPointerTy())
504 int64_t Off =
Offset.getSExtValue();
506 Off, ArgPart{Ty,
I->getAlign(), GuaranteedToExecute ?
I :
nullptr});
507 ArgPart &Part = Pair.first->second;
508 bool OffsetNotSeenBefore = Pair.second;
512 if (MaxElements > 0 && ArgParts.
size() > MaxElements) {
514 <<
"more than " << MaxElements <<
" parts\n");
522 <<
"accessed as both " << *Part.Ty <<
" and " << *Ty
523 <<
" at offset " << Off <<
"\n");
533 if (!GuaranteedToExecute &&
534 (OffsetNotSeenBefore || Part.Alignment <
I->getAlign())) {
543 NeededDerefBytes = std::max(NeededDerefBytes, Off +
Size.getFixedValue());
544 NeededAlign = std::max(NeededAlign,
I->getAlign());
547 Part.Alignment = std::max(Part.Alignment,
I->getAlign());
553 std::optional<bool> Res{};
554 if (
LoadInst *LI = dyn_cast<LoadInst>(&
I))
555 Res = HandleEndUser(LI, LI->getType(),
true);
556 else if (
StoreInst *SI = dyn_cast<StoreInst>(&
I))
557 Res = HandleEndUser(SI, SI->getValueOperand()->getType(),
571 auto AppendUses = [&](
const Value *V) {
572 for (
const Use &U : V->uses())
573 if (Visited.
insert(&U).second)
577 while (!Worklist.
empty()) {
579 Value *V = U->getUser();
580 if (isa<BitCastInst>(V)) {
585 if (
auto *
GEP = dyn_cast<GetElementPtrInst>(V)) {
586 if (!
GEP->hasAllConstantIndices())
592 if (
auto *LI = dyn_cast<LoadInst>(V)) {
593 if (!*HandleEndUser(LI, LI->getType(),
false))
600 auto *SI = dyn_cast<StoreInst>(V);
601 if (AreStoresAllowed && SI &&
603 if (!*HandleEndUser(SI, SI->getValueOperand()->getType(),
613 <<
"unknown user " << *V <<
"\n");
617 if (NeededDerefBytes || NeededAlign > 1) {
622 <<
"not dereferenceable or aligned\n");
627 if (ArgParts.
empty())
635 int64_t
Offset = ArgPartsVec[0].first;
636 for (
const auto &Pair : ArgPartsVec) {
640 Offset = Pair.first +
DL.getTypeStoreSize(Pair.second.Ty);
646 if (AreStoresAllowed)
688 CallBase *CB = dyn_cast<CallBase>(U.getUser());
692 const Function *Caller = CB->getCaller();
693 const Function *Callee = CB->getCalledFunction();
694 return TTI.areTypesABICompatible(Caller, Callee, Types);
703 unsigned MaxElements,
bool IsRecursive) {
707 if (
F->hasFnAttribute(Attribute::Naked))
711 if (!
F->hasLocalLinkage())
724 if (
F->getAttributes().hasAttrSomewhere(Attribute::InAlloca))
730 if (
I.getType()->isPointerTy())
732 if (PointerArgs.
empty())
738 for (
Use &U :
F->uses()) {
739 CallBase *CB = dyn_cast<CallBase>(U.getUser());
741 if (CB ==
nullptr || !CB->
isCallee(&U) ||
756 if (BB.getTerminatingMustTailCall())
766 unsigned NumArgsAfterPromote =
F->getFunctionType()->getNumParams();
767 for (
Argument *PtrArg : PointerArgs) {
770 if (PtrArg->hasStructRetAttr()) {
771 unsigned ArgNo = PtrArg->getArgNo();
772 F->removeParamAttr(ArgNo, Attribute::StructRet);
773 F->addParamAttr(ArgNo, Attribute::NoAlias);
774 for (
Use &U :
F->uses()) {
775 CallBase &CB = cast<CallBase>(*U.getUser());
784 if (
findArgParts(PtrArg,
DL, AAR, MaxElements, IsRecursive, ArgParts)) {
786 for (
const auto &Pair : ArgParts)
787 Types.push_back(Pair.second.Ty);
790 NumArgsAfterPromote += ArgParts.size() - 1;
791 ArgsToPromote.
insert({PtrArg, std::move(ArgParts)});
797 if (ArgsToPromote.
empty())
810 bool Changed =
false, LocalChange;
819 bool IsRecursive =
C.size() > 1;
832 C.getOuterRefSCC().replaceNodeFunction(
N, *NewF);
838 for (
auto *U : NewF->
users()) {
839 auto *UserF = cast<CallBase>(U)->getFunction();
844 Changed |= LocalChange;
845 }
while (LocalChange);
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file contains the simple types necessary to represent the attributes associated with functions a...
This is the interface for LLVM's primary stateless and local alias analysis.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
This file provides utility analysis objects describing memory locations.
FunctionAnalysisManager FAM
This header defines various interfaces for pass management in LLVM.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This defines the Use class.
A manager for alias analyses.
bool canInstructionRangeModRef(const Instruction &I1, const Instruction &I2, const MemoryLocation &Loc, const ModRefInfo Mode)
Check if it is possible for the execution of the specified instructions to mod(according to the mode)...
bool canBasicBlockModify(const BasicBlock &BB, const MemoryLocation &Loc)
Check if it is possible for execution of the specified basic block to modify the location Loc.
Class for arbitrary precision integers.
This templated class represents "all analyses that operate over <a particular IR unit>" (e....
an instruction to allocate memory on the stack
void setAlignment(Align Align)
A container for analyses that lazily runs them and caches their results.
void clear(IRUnitT &IR, llvm::StringRef Name)
Clear any cached analysis results for a single unit of IR.
void invalidate(IRUnitT &IR, const PreservedAnalyses &PA)
Invalidate cached analyses for an IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
This class represents an incoming formal argument to a Function.
const Function * getParent() const
Type * getParamByValType() const
If this is a byval argument, return its type.
MaybeAlign getParamAlign() const
If this is a byval or inalloca argument, return its alignment.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
A function analysis which provides an AssumptionCache.
AttributeSet getFnAttrs() const
The function attributes are returned.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
AttributeSet getParamAttrs(unsigned ArgNo) const
The attributes for the argument or parameter at the given index are returned.
std::optional< std::pair< unsigned, std::optional< unsigned > > > getAllocSizeArgs() const
static Attribute getWithAllocSizeArgs(LLVMContext &Context, unsigned ElemSizeArg, const std::optional< unsigned > &NumElemsArg)
LLVM Basic Block Representation.
const Instruction & front() const
Represents analyses that only rely on functions' control flow.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void setCallingConv(CallingConv::ID CC)
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
void removeParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Removes the attribute from the given argument.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
CallingConv::ID getCallingConv() const
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isCallee(Value::const_user_iterator UI) const
Determine whether the passed iterator points to the callee operand's Use.
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
FunctionType * getFunctionType() const
AttributeList getAttributes() const
Return the parameter attributes for this call.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
Function * getCaller()
Helper to get the caller (the parent function).
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
A parsed version of the target data layout string in and methods for querying it.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Analysis pass which computes a DominatorTree.
A proxy from a FunctionAnalysisManager to an SCC.
void addFnAttr(Attribute::AttrKind Kind)
Add function attributes to this function.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
void splice(Function::iterator ToIt, Function *FromF)
Transfer all blocks from FromF to this function at ToIt.
const BasicBlock & getEntryBlock() const
AttributeList getAttributes() const
Return the attribute list for this Function.
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
void setAttributes(AttributeList Attrs)
Set the attribute list for this Function.
void copyAttributesFrom(const Function *Src)
copyAttributesFrom - copy all additional attributes (those not needed to create a Function) from the ...
void copyMetadata(const GlobalObject *Src, unsigned Offset)
Copy metadata from Src, adjusting offsets by Offset.
Common base class shared among various IRBuilders.
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
ConstantInt * getInt(const APInt &AI)
Get a constant integer value.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
void setAAMetadata(const AAMDNodes &N)
Sets the AA metadata on this instruction from the AAMDNodes structure.
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
const Function * getFunction() const
Return the function this instruction belongs to.
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
static InvokeInst * Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef< Value * > Args, const Twine &NameStr, Instruction *InsertBefore=nullptr)
A node in the call graph.
An SCC of the call graph.
A lazily constructed view of the call graph of a module.
An instruction for reading from memory.
static unsigned getPointerOperandIndex()
Representation for a specific memory location.
static MemoryLocation get(const LoadInst *LI)
Return a location with information about the memory reference by the given instruction.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void preserveSet()
Mark an analysis set as preserved.
void preserve()
Mark an analysis as preserved.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
static unsigned getPointerOperandIndex()
Analysis pass providing the TargetTransformInfo.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
LLVM Value Representation.
void setName(const Twine &Name)
Change the name of the value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
StringRef getName() const
Return a constant reference to the value's name.
void takeName(Value *V)
Transfer the name from V to this value.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void updateMinLegalVectorWidthAttr(Function &Fn, uint64_t Width)
Update min-legal-vector-width if it is in Attribute and less than Width.
@ C
The default llvm calling convention, compatible with C.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
void PromoteMemToReg(ArrayRef< AllocaInst * > Allocas, DominatorTree &DT, AssumptionCache *AC=nullptr)
Promote the specified list of alloca instructions into scalar registers, inserting PHI nodes as appro...
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
bool isAligned(Align Lhs, uint64_t SizeInBytes)
Checks that SizeInBytes is a multiple of the alignment.
bool isDereferenceableAndAlignedPointer(const Value *V, Type *Ty, Align Alignment, const DataLayout &DL, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Returns true if V is always a dereferenceable pointer with alignment greater or equal than requested.
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
bool isAllocaPromotable(const AllocaInst *AI)
Return true if this alloca is legal for promotion.
void sort(IteratorTy Start, IteratorTy End)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
iterator_range< idf_ext_iterator< T, SetTy > > inverse_depth_first_ext(const T &G, SetTy &S)
bool RecursivelyDeleteTriviallyDeadInstructionsPermissive(SmallVectorImpl< WeakTrackingVH > &DeadInsts, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, std::function< void(Value *)> AboutToDeleteCallback=std::function< void(Value *)>())
Same functionality as RecursivelyDeleteTriviallyDeadInstructions, but allow instructions that are not...
bool isGuaranteedToTransferExecutionToSuccessor(const Instruction *I)
Return true if this function can prove that the instruction I will always transfer execution to one o...
auto predecessors(const MachineBasicBlock *BB)
This struct is a compact representation of a valid (non-zero power of two) alignment.
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Function object to check whether the first component of a container supported by std::get (like std::...