LLVM 20.0.0git
Classes | Macros | Functions | Variables
SROA.cpp File Reference

This transformation implements the well known scalar replacement of aggregates transformation. More...

#include "llvm/Transforms/Scalar/SROA.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/PointerIntPair.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/ADT/iterator.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/PtrUseVisitor.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/ConstantFolder.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DIBuilder.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugInfo.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GetElementPtrTypeIterator.h"
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstVisitor.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/PromoteMemToReg.h"
#include <algorithm>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <iterator>
#include <string>
#include <tuple>
#include <utility>
#include <variant>
#include <vector>

Go to the source code of this file.

Classes

class  AllocaSlices::partition_iterator
 An iterator over partitions of the alloca's slices. More...
 
class  AllocaSlices::SliceBuilder
 Builder for the alloca slices. More...
 

Macros

#define DEBUG_TYPE   "sroa"
 

Functions

 STATISTIC (NumAllocasAnalyzed, "Number of allocas analyzed for replacement")
 
 STATISTIC (NumAllocaPartitions, "Number of alloca partitions formed")
 
 STATISTIC (MaxPartitionsPerAlloca, "Maximum number of partitions per alloca")
 
 STATISTIC (NumAllocaPartitionUses, "Number of alloca partition uses rewritten")
 
 STATISTIC (MaxUsesPerAllocaPartition, "Maximum number of uses of a partition")
 
 STATISTIC (NumNewAllocas, "Number of new, smaller allocas introduced")
 
 STATISTIC (NumPromoted, "Number of allocas promoted to SSA values")
 
 STATISTIC (NumLoadsSpeculated, "Number of loads speculated to allow promotion")
 
 STATISTIC (NumLoadsPredicated, "Number of loads rewritten into predicated loads to allow promotion")
 
 STATISTIC (NumStoresPredicated, "Number of stores rewritten into predicated loads to allow promotion")
 
 STATISTIC (NumDeleted, "Number of instructions deleted")
 
 STATISTIC (NumVectorized, "Number of vectorized aggregates")
 
static FragCalcResult calculateFragment (DILocalVariable *Variable, uint64_t NewStorageSliceOffsetInBits, uint64_t NewStorageSliceSizeInBits, std::optional< DIExpression::FragmentInfo > StorageFragment, std::optional< DIExpression::FragmentInfo > CurrentFragment, DIExpression::FragmentInfo &Target)
 
static DebugVariable getAggregateVariable (DbgVariableIntrinsic *DVI)
 
static DebugVariable getAggregateVariable (DbgVariableRecord *DVR)
 
DbgVariableRecordUnwrapDbgInstPtr (DbgInstPtr P, DbgVariableRecord *Unused)
 Helpers for handling new and old debug info modes in migrateDebugInfo.
 
DbgAssignIntrinsicUnwrapDbgInstPtr (DbgInstPtr P, DbgAssignIntrinsic *Unused)
 
static void migrateDebugInfo (AllocaInst *OldAlloca, bool IsSplit, uint64_t OldAllocaOffsetInBits, uint64_t SliceSizeInBits, Instruction *OldInst, Instruction *Inst, Value *Dest, Value *Value, const DataLayout &DL)
 Find linked dbg.assign and generate a new one with the correct FragmentInfo.
 
static ValuefoldSelectInst (SelectInst &SI)
 
static ValuefoldPHINodeOrSelectInst (Instruction &I)
 A helper that folds a PHI node or a select.
 
static std::pair< Type *, IntegerType * > findCommonType (AllocaSlices::const_iterator B, AllocaSlices::const_iterator E, uint64_t EndOffset)
 Walk the range of a partitioning looking for a common type to cover this sequence of slices.
 
static bool isSafePHIToSpeculate (PHINode &PN)
 PHI instructions that use an alloca and are subsequently loaded can be rewritten to load both input pointers in the pred blocks and then PHI the results, allowing the load of the alloca to be promoted.
 
static void speculatePHINodeLoads (IRBuilderTy &IRB, PHINode &PN)
 
static SelectHandSpeculativity isSafeLoadOfSelectToSpeculate (LoadInst &LI, SelectInst &SI, bool PreserveCFG)
 
static void speculateSelectInstLoads (SelectInst &SI, LoadInst &LI, IRBuilderTy &IRB)
 
template<typename T >
static void rewriteMemOpOfSelect (SelectInst &SI, T &I, SelectHandSpeculativity Spec, DomTreeUpdater &DTU)
 
static void rewriteMemOpOfSelect (SelectInst &SelInst, Instruction &I, SelectHandSpeculativity Spec, DomTreeUpdater &DTU)
 
static bool rewriteSelectInstMemOps (SelectInst &SI, const RewriteableMemOps &Ops, IRBuilderTy &IRB, DomTreeUpdater *DTU)
 
static ValuegetAdjustedPtr (IRBuilderTy &IRB, const DataLayout &DL, Value *Ptr, APInt Offset, Type *PointerTy, const Twine &NamePrefix)
 Compute an adjusted pointer from Ptr by Offset bytes where the resulting pointer has PointerTy.
 
static Align getAdjustedAlignment (Instruction *I, uint64_t Offset)
 Compute the adjusted alignment for a load or store from an offset.
 
static bool canConvertValue (const DataLayout &DL, Type *OldTy, Type *NewTy)
 Test whether we can convert a value from the old to the new type.
 
static ValueconvertValue (const DataLayout &DL, IRBuilderTy &IRB, Value *V, Type *NewTy)
 Generic routine to convert an SSA value to a value of a different type.
 
static bool isVectorPromotionViableForSlice (Partition &P, const Slice &S, VectorType *Ty, uint64_t ElementSize, const DataLayout &DL)
 Test whether the given slice use can be promoted to a vector.
 
static bool checkVectorTypeForPromotion (Partition &P, VectorType *VTy, const DataLayout &DL)
 Test whether a vector type is viable for promotion.
 
static VectorTypecheckVectorTypesForPromotion (Partition &P, const DataLayout &DL, SmallVectorImpl< VectorType * > &CandidateTys, bool HaveCommonEltTy, Type *CommonEltTy, bool HaveVecPtrTy, bool HaveCommonVecPtrTy, VectorType *CommonVecPtrTy)
 Test whether any vector type in CandidateTys is viable for promotion.
 
static VectorTypecreateAndCheckVectorTypesForPromotion (SetVector< Type * > &OtherTys, ArrayRef< VectorType * > CandidateTysCopy, function_ref< void(Type *)> CheckCandidateType, Partition &P, const DataLayout &DL, SmallVectorImpl< VectorType * > &CandidateTys, bool &HaveCommonEltTy, Type *&CommonEltTy, bool &HaveVecPtrTy, bool &HaveCommonVecPtrTy, VectorType *&CommonVecPtrTy)
 
static VectorTypeisVectorPromotionViable (Partition &P, const DataLayout &DL)
 Test whether the given alloca partitioning and range of slices can be promoted to a vector.
 
static bool isIntegerWideningViableForSlice (const Slice &S, uint64_t AllocBeginOffset, Type *AllocaTy, const DataLayout &DL, bool &WholeAllocaOp)
 Test whether a slice of an alloca is valid for integer widening.
 
static bool isIntegerWideningViable (Partition &P, Type *AllocaTy, const DataLayout &DL)
 Test whether the given alloca partition's integer operations can be widened to promotable ones.
 
static ValueextractInteger (const DataLayout &DL, IRBuilderTy &IRB, Value *V, IntegerType *Ty, uint64_t Offset, const Twine &Name)
 
static ValueinsertInteger (const DataLayout &DL, IRBuilderTy &IRB, Value *Old, Value *V, uint64_t Offset, const Twine &Name)
 
static ValueextractVector (IRBuilderTy &IRB, Value *V, unsigned BeginIndex, unsigned EndIndex, const Twine &Name)
 
static ValueinsertVector (IRBuilderTy &IRB, Value *Old, Value *V, unsigned BeginIndex, const Twine &Name)
 
static TypestripAggregateTypeWrapping (const DataLayout &DL, Type *Ty)
 Strip aggregate type wrapping.
 
static TypegetTypePartition (const DataLayout &DL, Type *Ty, uint64_t Offset, uint64_t Size)
 Try to find a partition of the aggregate type passed in for a given offset and size.
 
const ValuegetAddress (const DbgVariableIntrinsic *DVI)
 
const ValuegetAddress (const DbgVariableRecord *DVR)
 
bool isKillAddress (const DbgVariableIntrinsic *DVI)
 
bool isKillAddress (const DbgVariableRecord *DVR)
 
const DIExpressiongetAddressExpression (const DbgVariableIntrinsic *DVI)
 
const DIExpressiongetAddressExpression (const DbgVariableRecord *DVR)
 
static DIExpressioncreateOrReplaceFragment (const DIExpression *Expr, DIExpression::FragmentInfo Frag, int64_t BitExtractOffset)
 Create or replace an existing fragment in a DIExpression with Frag.
 
static void insertNewDbgInst (DIBuilder &DIB, DbgDeclareInst *Orig, AllocaInst *NewAddr, DIExpression *NewAddrExpr, Instruction *BeforeInst, std::optional< DIExpression::FragmentInfo > NewFragment, int64_t BitExtractAdjustment)
 Insert a new dbg.declare.
 
static void insertNewDbgInst (DIBuilder &DIB, DbgAssignIntrinsic *Orig, AllocaInst *NewAddr, DIExpression *NewAddrExpr, Instruction *BeforeInst, std::optional< DIExpression::FragmentInfo > NewFragment, int64_t BitExtractAdjustment)
 Insert a new dbg.assign.
 
static void insertNewDbgInst (DIBuilder &DIB, DbgVariableRecord *Orig, AllocaInst *NewAddr, DIExpression *NewAddrExpr, Instruction *BeforeInst, std::optional< DIExpression::FragmentInfo > NewFragment, int64_t BitExtractAdjustment)
 Insert a new DbgRecord.
 
 INITIALIZE_PASS_BEGIN (SROALegacyPass, "sroa", "Scalar Replacement Of Aggregates", false, false) INITIALIZE_PASS_END(SROALegacyPass
 

Variables

static cl::opt< boolSROASkipMem2Reg ("sroa-skip-mem2reg", cl::init(false), cl::Hidden)
 Disable running mem2reg during SROA in order to test or debug SROA.
 
 sroa
 
Scalar Replacement Of Aggregates
 
Scalar Replacement Of false
 

Detailed Description

This transformation implements the well known scalar replacement of aggregates transformation.

It tries to identify promotable elements of an aggregate alloca, and promote them to registers. It will also try to convert uses of an element (or set of elements) of an alloca into a vector or bitfield-style integer scalar if appropriate.

It works to do this with minimal slicing of the alloca so that regions which are merely transferred in and out of external memory remain unchanged and are not decomposed to scalar code.

Because this also performs alloca promotion, it can be thought of as also serving the purpose of SSA formation. The algorithm iterates on the function until all opportunities for promotion have been realized.

Definition in file SROA.cpp.

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE   "sroa"

Definition at line 101 of file SROA.cpp.

Function Documentation

◆ calculateFragment()

static FragCalcResult calculateFragment ( DILocalVariable Variable,
uint64_t  NewStorageSliceOffsetInBits,
uint64_t  NewStorageSliceSizeInBits,
std::optional< DIExpression::FragmentInfo StorageFragment,
std::optional< DIExpression::FragmentInfo CurrentFragment,
DIExpression::FragmentInfo Target 
)
static

Definition at line 268 of file SROA.cpp.

References llvm::DIVariable::getSizeInBits(), and Size.

Referenced by migrateDebugInfo().

◆ canConvertValue()

static bool canConvertValue ( const DataLayout DL,
Type OldTy,
Type NewTy 
)
static

Test whether we can convert a value from the old to the new type.

This predicate should be used to guard calls to convertValue in order to ensure that we only try to convert viable values. The strategy is that we will peel off single element struct and array wrappings to get to an underlying value, and convert that value.

Definition at line 1906 of file SROA.cpp.

References assert(), DL, getBitWidth(), llvm::Type::getPointerAddressSpace(), llvm::Type::getScalarType(), llvm::Type::isIntegerTy(), llvm::Type::isPointerTy(), llvm::Type::isSingleValueType(), and llvm::Type::isTargetExtTy().

Referenced by convertValue(), isIntegerWideningViable(), isIntegerWideningViableForSlice(), and isVectorPromotionViableForSlice().

◆ checkVectorTypeForPromotion()

static bool checkVectorTypeForPromotion ( Partition &  P,
VectorType VTy,
const DataLayout DL 
)
static

Test whether a vector type is viable for promotion.

This implements the necessary checking for checkVectorTypesForPromotion (and thus isVectorPromotionViable) over all slices of the alloca for the given VectorType.

Definition at line 2096 of file SROA.cpp.

References assert(), DL, isVectorPromotionViableForSlice(), and P.

Referenced by checkVectorTypesForPromotion().

◆ checkVectorTypesForPromotion()

static VectorType * checkVectorTypesForPromotion ( Partition &  P,
const DataLayout DL,
SmallVectorImpl< VectorType * > &  CandidateTys,
bool  HaveCommonEltTy,
Type CommonEltTy,
bool  HaveVecPtrTy,
bool  HaveCommonVecPtrTy,
VectorType CommonVecPtrTy 
)
static

Test whether any vector type in CandidateTys is viable for promotion.

This implements the necessary checking for isVectorPromotionViable over all slices of the alloca for the given VectorType.

Definition at line 2125 of file SROA.cpp.

References assert(), checkVectorTypeForPromotion(), llvm::SmallVectorImpl< T >::clear(), DL, llvm::SmallVectorBase< Size_T >::empty(), llvm::erase_if(), P, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::sort(), and llvm::unique().

Referenced by createAndCheckVectorTypesForPromotion().

◆ convertValue()

static Value * convertValue ( const DataLayout DL,
IRBuilderTy &  IRB,
Value V,
Type NewTy 
)
static

Generic routine to convert an SSA value to a value of a different type.

This will try various different casting techniques, such as bitcasts, inttoptr, and ptrtoint casts. Use the canConvertValue predicate to test two types for viability with this routine.

Definition at line 1968 of file SROA.cpp.

References assert(), canConvertValue(), DL, llvm::Type::getPointerAddressSpace(), llvm::Type::isIntOrIntVectorTy(), and llvm::Type::isPtrOrPtrVectorTy().

◆ createAndCheckVectorTypesForPromotion()

static VectorType * createAndCheckVectorTypesForPromotion ( SetVector< Type * > &  OtherTys,
ArrayRef< VectorType * >  CandidateTysCopy,
function_ref< void(Type *)>  CheckCandidateType,
Partition &  P,
const DataLayout DL,
SmallVectorImpl< VectorType * > &  CandidateTys,
bool HaveCommonEltTy,
Type *&  CommonEltTy,
bool HaveVecPtrTy,
bool HaveCommonVecPtrTy,
VectorType *&  CommonVecPtrTy 
)
static

Definition at line 2211 of file SROA.cpp.

References assert(), checkVectorTypesForPromotion(), DL, P, and llvm::ArrayRef< T >::size().

Referenced by isVectorPromotionViable().

◆ createOrReplaceFragment()

static DIExpression * createOrReplaceFragment ( const DIExpression Expr,
DIExpression::FragmentInfo  Frag,
int64_t  BitExtractOffset 
)
static

Create or replace an existing fragment in a DIExpression with Frag.

If the expression already contains a DW_OP_LLVM_extract_bits_[sz]ext operation, add BitExtractOffset to the offset part.

Returns the new expression, or nullptr if this fails (see details below).

This function is similar to DIExpression::createFragmentExpression except for 3 important distinctions:

  1. The new fragment isn't relative to an existing fragment.
  2. It assumes the computed location is a memory location. This means we don't need to perform checks that creating the fragment preserves the expression semantics.
  3. Existing extract_bits are modified independently of fragment changes using BitExtractOffset. A change to the fragment offset or size may affect a bit extract. But a bit extract offset can change independently of the fragment dimensions.

Returns the new expression, or nullptr if one couldn't be created. Ideally this is only used to signal that a bit-extract has become zero-sized (and thus the new debug record has no size and can be dropped), however, it fails for other reasons too - see the FIXME below.

FIXME: To keep the change that introduces this function NFC it bails in some situations unecessarily, e.g. when fragment and bit extract sizes differ.

Definition at line 5038 of file SROA.cpp.

References assert(), llvm::dwarf::DW_OP_LLVM_extract_bits_sext, llvm::dwarf::DW_OP_LLVM_extract_bits_zext, llvm::dwarf::DW_OP_LLVM_fragment, llvm::DIExpression::expr_ops(), llvm::MDNode::getContext(), llvm::DbgVariableFragmentInfo::OffsetInBits, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::DbgVariableFragmentInfo::SizeInBits.

Referenced by insertNewDbgInst().

◆ extractInteger()

static Value * extractInteger ( const DataLayout DL,
IRBuilderTy &  IRB,
Value V,
IntegerType Ty,
uint64_t  Offset,
const Twine Name 
)
static

◆ extractVector()

static Value * extractVector ( IRBuilderTy &  IRB,
Value V,
unsigned  BeginIndex,
unsigned  EndIndex,
const Twine Name 
)
static

Definition at line 2528 of file SROA.cpp.

References assert(), llvm::dbgs(), getNumElements(), LLVM_DEBUG, and Name.

◆ findCommonType()

static std::pair< Type *, IntegerType * > findCommonType ( AllocaSlices::const_iterator  B,
AllocaSlices::const_iterator  E,
uint64_t  EndOffset 
)
static

Walk the range of a partitioning looking for a common type to cover this sequence of slices.

Definition at line 1463 of file SROA.cpp.

References B, E, llvm::IntegerType::getBitWidth(), llvm::Value::getType(), and I.

◆ foldPHINodeOrSelectInst()

static Value * foldPHINodeOrSelectInst ( Instruction I)
static

A helper that folds a PHI node or a select.

Definition at line 1001 of file SROA.cpp.

References foldSelectInst(), and I.

◆ foldSelectInst()

static Value * foldSelectInst ( SelectInst SI)
static

Definition at line 988 of file SROA.cpp.

Referenced by foldPHINodeOrSelectInst().

◆ getAddress() [1/2]

const Value * getAddress ( const DbgVariableIntrinsic DVI)

◆ getAddress() [2/2]

const Value * getAddress ( const DbgVariableRecord DVR)

◆ getAddressExpression() [1/2]

const DIExpression * getAddressExpression ( const DbgVariableIntrinsic DVI)

Definition at line 4999 of file SROA.cpp.

◆ getAddressExpression() [2/2]

const DIExpression * getAddressExpression ( const DbgVariableRecord DVR)

◆ getAdjustedAlignment()

static Align getAdjustedAlignment ( Instruction I,
uint64_t  Offset 
)
static

Compute the adjusted alignment for a load or store from an offset.

Definition at line 1896 of file SROA.cpp.

References llvm::commonAlignment(), llvm::getLoadStoreAlignment(), I, and llvm::Offset.

◆ getAdjustedPtr()

static Value * getAdjustedPtr ( IRBuilderTy &  IRB,
const DataLayout DL,
Value Ptr,
APInt  Offset,
Type PointerTy,
const Twine NamePrefix 
)
static

Compute an adjusted pointer from Ptr by Offset bytes where the resulting pointer has PointerTy.

Definition at line 1885 of file SROA.cpp.

References llvm::Offset, and Ptr.

◆ getAggregateVariable() [1/2]

static DebugVariable getAggregateVariable ( DbgVariableIntrinsic DVI)
static

◆ getAggregateVariable() [2/2]

static DebugVariable getAggregateVariable ( DbgVariableRecord DVR)
static

◆ getTypePartition()

static Type * getTypePartition ( const DataLayout DL,
Type Ty,
uint64_t  Offset,
uint64_t  Size 
)
static

Try to find a partition of the aggregate type passed in for a given offset and size.

This recurses through the aggregate type and tries to compute a subtype based on the offset and size. When the offset and size span a sub-section of an array, it will even compute a new array type for that sub-section, and the same for structs.

Note that this routine is very strict and tries to find a partition of the type which produces the exact right offset and size. It is not forgiving when the size or offset cause either end of type-based partition to be off. Also, this is a best-effort routine. It is reasonable to give up and not return a type if necessary.

Definition at line 4172 of file SROA.cpp.

References assert(), DL, llvm::StructType::element_begin(), llvm::StructType::element_end(), llvm::StructType::get(), llvm::Type::getContext(), llvm::StructLayout::getElementContainingOffset(), llvm::StructLayout::getElementOffset(), llvm::StructType::getElementType(), llvm::StructLayout::getSizeInBits(), llvm::StructLayout::getSizeInBytes(), getSizeInBytes(), getTypePartition(), llvm::StructType::isPacked(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isScalable(), llvm::Offset, Size, and stripAggregateTypeWrapping().

Referenced by getTypePartition().

◆ INITIALIZE_PASS_BEGIN()

INITIALIZE_PASS_BEGIN ( SROALegacyPass  ,
"sroa"  ,
"Scalar Replacement Of Aggregates"  ,
false  ,
false   
)

◆ insertInteger()

static Value * insertInteger ( const DataLayout DL,
IRBuilderTy &  IRB,
Value Old,
Value V,
uint64_t  Offset,
const Twine Name 
)
static

◆ insertNewDbgInst() [1/3]

static void insertNewDbgInst ( DIBuilder DIB,
DbgAssignIntrinsic Orig,
AllocaInst NewAddr,
DIExpression NewAddrExpr,
Instruction BeforeInst,
std::optional< DIExpression::FragmentInfo NewFragment,
int64_t  BitExtractAdjustment 
)
static

Insert a new dbg.assign.

Orig Original to copy debug loc, variable, value and value expression from. NewAddr Location's new base address. NewAddrExpr New expression to apply to address. BeforeInst Insert position. NewFragment New fragment (absolute, non-relative). BitExtractAdjustment Offset to apply to any extract_bits op.

Definition at line 5125 of file SROA.cpp.

References createOrReplaceFragment(), llvm::dbgs(), llvm::PointerUnion< PTs >::get(), llvm::Value::getContext(), llvm::Instruction::getDebugLoc(), llvm::DIAssignID::getDistinct(), llvm::DbgVariableIntrinsic::getExpression(), llvm::DbgValueInst::getValue(), llvm::DbgVariableIntrinsic::getVariable(), llvm::Instruction::hasMetadata(), llvm::DIBuilder::insertDbgAssign(), LLVM_DEBUG, and llvm::Instruction::setMetadata().

◆ insertNewDbgInst() [2/3]

static void insertNewDbgInst ( DIBuilder DIB,
DbgDeclareInst Orig,
AllocaInst NewAddr,
DIExpression NewAddrExpr,
Instruction BeforeInst,
std::optional< DIExpression::FragmentInfo NewFragment,
int64_t  BitExtractAdjustment 
)
static

Insert a new dbg.declare.

Orig Original to copy debug loc and variable from. NewAddr Location's new base address. NewAddrExpr New expression to apply to address. BeforeInst Insert position. NewFragment New fragment (absolute, non-relative). BitExtractAdjustment Offset to apply to any extract_bits op.

Definition at line 5102 of file SROA.cpp.

References createOrReplaceFragment(), llvm::Instruction::getDebugLoc(), and llvm::DbgVariableIntrinsic::getVariable().

◆ insertNewDbgInst() [3/3]

static void insertNewDbgInst ( DIBuilder DIB,
DbgVariableRecord Orig,
AllocaInst NewAddr,
DIExpression NewAddrExpr,
Instruction BeforeInst,
std::optional< DIExpression::FragmentInfo NewFragment,
int64_t  BitExtractAdjustment 
)
static

Insert a new DbgRecord.

Orig Original to copy record type, debug loc and variable from, and additionally value and value expression for dbg_assign records. NewAddr Location's new base address. NewAddrExpr New expression to apply to address. BeforeInst Insert position. NewFragment New fragment (absolute, non-relative). BitExtractAdjustment Offset to apply to any extract_bits op.

Definition at line 5165 of file SROA.cpp.

References llvm::DbgVariableRecord::createDVRDeclare(), llvm::DbgVariableRecord::createLinkedDVRAssign(), createOrReplaceFragment(), llvm::dbgs(), llvm::Value::getContext(), llvm::DbgRecord::getDebugLoc(), llvm::DIAssignID::getDistinct(), llvm::DbgVariableRecord::getExpression(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::DbgVariableRecord::getValue(), llvm::DbgVariableRecord::getVariable(), llvm::Instruction::hasMetadata(), llvm::DbgVariableRecord::isDbgAssign(), llvm::DbgVariableRecord::isDbgDeclare(), LLVM_DEBUG, and llvm::Instruction::setMetadata().

◆ insertVector()

static Value * insertVector ( IRBuilderTy &  IRB,
Value Old,
Value V,
unsigned  BeginIndex,
const Twine Name 
)
static

◆ isIntegerWideningViable()

static bool isIntegerWideningViable ( Partition &  P,
Type AllocaTy,
const DataLayout DL 
)
static

Test whether the given alloca partition's integer operations can be widened to promotable ones.

This is a quick test to check whether we can rewrite the integer loads and stores to a particular alloca into wider loads and stores and be able to promote the resulting alloca.

Definition at line 2429 of file SROA.cpp.

References canConvertValue(), DL, llvm::Type::getContext(), llvm::Type::getIntNTy(), isIntegerWideningViableForSlice(), llvm::IntegerType::MAX_INT_BITS, and P.

◆ isIntegerWideningViableForSlice()

static bool isIntegerWideningViableForSlice ( const Slice &  S,
uint64_t  AllocBeginOffset,
Type AllocaTy,
const DataLayout DL,
bool WholeAllocaOp 
)
static

Test whether a slice of an alloca is valid for integer widening.

This implements the necessary checking for the isIntegerWideningViable test below on a single slice of the alloca.

Definition at line 2336 of file SROA.cpp.

References canConvertValue(), DL, llvm::Value::getType(), II, llvm::LoadInst::isVolatile(), MI, and Size.

Referenced by isIntegerWideningViable().

◆ isKillAddress() [1/2]

bool isKillAddress ( const DbgVariableIntrinsic DVI)

Definition at line 4985 of file SROA.cpp.

◆ isKillAddress() [2/2]

bool isKillAddress ( const DbgVariableRecord DVR)

◆ isSafeLoadOfSelectToSpeculate()

static SelectHandSpeculativity isSafeLoadOfSelectToSpeculate ( LoadInst LI,
SelectInst SI,
bool  PreserveCFG 
)
static

◆ isSafePHIToSpeculate()

static bool isSafePHIToSpeculate ( PHINode PN)
static

PHI instructions that use an alloca and are subsequently loaded can be rewritten to load both input pointers in the pred blocks and then PHI the results, allowing the load of the alloca to be promoted.

From this: P2 = phi [i32* Alloca, i32* Other] V = load i32* P2 to: V1 = load i32* Alloca -> will be mem2reg'd ... V2 = load i32* Other ... V = phi [i32 V1, i32 V2]

We can do this to a select if its only uses are loads and if the operands to the select can be loaded unconditionally.

FIXME: This should be hoisted into a generic utility, likely in Transforms/Util/Local.h

Definition at line 1529 of file SROA.cpp.

References DL, llvm::LoadInst::getAlign(), llvm::Instruction::getDataLayout(), llvm::PHINode::getIncomingBlock(), llvm::PHINode::getIncomingValue(), llvm::PHINode::getNumIncomingValues(), llvm::Instruction::getNumSuccessors(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), Idx, llvm::isSafeToLoadUnconditionally(), llvm::LoadInst::isSimple(), llvm::Instruction::mayHaveSideEffects(), and llvm::Value::users().

◆ isVectorPromotionViable()

static VectorType * isVectorPromotionViable ( Partition &  P,
const DataLayout DL 
)
static

Test whether the given alloca partitioning and range of slices can be promoted to a vector.

This is a quick test to check whether we can rewrite a particular alloca partition (and its newly formed alloca) into a vector alloca with only whole-vector loads and stores such that it could be promoted to a vector SSA value. We only can ensure this for a limited set of operations, and we don't want to do the rewrites unless we are confident that the result will be promotable, so we have an early test here.

Definition at line 2255 of file SROA.cpp.

References llvm::SmallVectorImpl< T >::clear(), createAndCheckVectorTypesForPromotion(), DL, llvm::SmallVectorBase< Size_T >::empty(), llvm::Type::getScalarType(), llvm::Value::getType(), llvm::SetVector< T, Vector, Set, N >::insert(), llvm::Type::isPointerTy(), P, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

◆ isVectorPromotionViableForSlice()

static bool isVectorPromotionViableForSlice ( Partition &  P,
const Slice &  S,
VectorType Ty,
uint64_t  ElementSize,
const DataLayout DL 
)
static

Test whether the given slice use can be promoted to a vector.

This function is called to test each entry in a partition which is slated for a single slice.

Definition at line 2022 of file SROA.cpp.

References assert(), canConvertValue(), DL, llvm::FixedVectorType::get(), llvm::Type::getIntNTy(), getNumElements(), llvm::Value::getType(), II, llvm::Type::isIntegerTy(), llvm::Type::isStructTy(), llvm::LoadInst::isVolatile(), MI, and P.

Referenced by checkVectorTypeForPromotion().

◆ migrateDebugInfo()

static void migrateDebugInfo ( AllocaInst OldAlloca,
bool  IsSplit,
uint64_t  OldAllocaOffsetInBits,
uint64_t  SliceSizeInBits,
Instruction OldInst,
Instruction Inst,
Value Dest,
Value Value,
const DataLayout DL 
)
static

Find linked dbg.assign and generate a new one with the correct FragmentInfo.

Link Inst to the new dbg.assign. If Value is nullptr the value component is copied from the old dbg.assign to the new.

Parameters
OldAllocaAlloca for the variable before splitting.
IsSplitTrue if the store (not necessarily alloca) is being split.
OldAllocaOffsetInBitsOffset of the slice taken from OldAlloca.
SliceSizeInBitsNew number of bits being written to.
OldInstInstruction that is being split.
InstNew instruction performing this part of the split store.
DestStore destination.
ValueStored value.
DLDatalayout.

Map of aggregate variables to their fragment associated with OldAlloca.

Definition at line 349 of file SROA.cpp.

References assert(), calculateFragment(), llvm::DIExpression::createFragmentExpression(), llvm::dbgs(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::end(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::find(), llvm::for_each(), getAggregateVariable(), llvm::at::getAssignmentMarkers(), llvm::Value::getContext(), llvm::DIAssignID::getDistinct(), llvm::at::getDVRAssignmentMarkers(), llvm::Instruction::getMetadata(), llvm::Instruction::getModule(), llvm::DIBuilder::insertDbgAssign(), llvm::AllocaInst::isStaticAlloca(), LLVM_DEBUG, llvm::DbgVariableFragmentInfo::OffsetInBits, llvm::Instruction::setMetadata(), llvm::DbgVariableFragmentInfo::SizeInBits, and UnwrapDbgInstPtr().

◆ rewriteMemOpOfSelect() [1/2]

static void rewriteMemOpOfSelect ( SelectInst SelInst,
Instruction I,
SelectHandSpeculativity  Spec,
DomTreeUpdater DTU 
)
static

Definition at line 1840 of file SROA.cpp.

References I, llvm::llvm_unreachable_internal(), and rewriteMemOpOfSelect().

◆ rewriteMemOpOfSelect() [2/2]

template<typename T >
static void rewriteMemOpOfSelect ( SelectInst SI,
T I,
SelectHandSpeculativity  Spec,
DomTreeUpdater DTU 
)
static

◆ rewriteSelectInstMemOps()

static bool rewriteSelectInstMemOps ( SelectInst SI,
const RewriteableMemOps &  Ops,
IRBuilderTy &  IRB,
DomTreeUpdater DTU 
)
static

◆ speculatePHINodeLoads()

static void speculatePHINodeLoads ( IRBuilderTy &  IRB,
PHINode PN 
)
static

◆ speculateSelectInstLoads()

static void speculateSelectInstLoads ( SelectInst SI,
LoadInst LI,
IRBuilderTy &  IRB 
)
static

◆ STATISTIC() [1/12]

STATISTIC ( MaxPartitionsPerAlloca  ,
"Maximum number of partitions per alloca"   
)

◆ STATISTIC() [2/12]

STATISTIC ( MaxUsesPerAllocaPartition  ,
"Maximum number of uses of a partition"   
)

◆ STATISTIC() [3/12]

STATISTIC ( NumAllocaPartitions  ,
"Number of alloca partitions formed"   
)

◆ STATISTIC() [4/12]

STATISTIC ( NumAllocaPartitionUses  ,
"Number of alloca partition uses rewritten"   
)

◆ STATISTIC() [5/12]

STATISTIC ( NumAllocasAnalyzed  ,
"Number of allocas analyzed for replacement"   
)

◆ STATISTIC() [6/12]

STATISTIC ( NumDeleted  ,
"Number of instructions deleted"   
)

◆ STATISTIC() [7/12]

STATISTIC ( NumLoadsPredicated  ,
"Number of loads rewritten into predicated loads to allow promotion"   
)

◆ STATISTIC() [8/12]

STATISTIC ( NumLoadsSpeculated  ,
"Number of loads speculated to allow promotion"   
)

◆ STATISTIC() [9/12]

STATISTIC ( NumNewAllocas  ,
"Number of  new,
smaller allocas introduced"   
)

◆ STATISTIC() [10/12]

STATISTIC ( NumPromoted  ,
"Number of allocas promoted to SSA values"   
)

◆ STATISTIC() [11/12]

STATISTIC ( NumStoresPredicated  ,
"Number of stores rewritten into predicated loads to allow promotion"   
)

◆ STATISTIC() [12/12]

STATISTIC ( NumVectorized  ,
"Number of vectorized aggregates"   
)

◆ stripAggregateTypeWrapping()

static Type * stripAggregateTypeWrapping ( const DataLayout DL,
Type Ty 
)
static

Strip aggregate type wrapping.

This removes no-op aggregate types wrapping an underlying type. It will strip as many layers of types as it can without changing either the type size or the allocated size.

Definition at line 4134 of file SROA.cpp.

References DL, llvm::StructLayout::getElementContainingOffset(), llvm::Type::isSingleValueType(), and stripAggregateTypeWrapping().

Referenced by getTypePartition(), and stripAggregateTypeWrapping().

◆ UnwrapDbgInstPtr() [1/2]

DbgAssignIntrinsic * UnwrapDbgInstPtr ( DbgInstPtr  P,
DbgAssignIntrinsic Unused 
)

Definition at line 330 of file SROA.cpp.

References P.

◆ UnwrapDbgInstPtr() [2/2]

DbgVariableRecord * UnwrapDbgInstPtr ( DbgInstPtr  P,
DbgVariableRecord Unused 
)

Helpers for handling new and old debug info modes in migrateDebugInfo.

These overloads unwrap a DbgInstPtr {Instruction* | DbgRecord*} union based on the Unused parameter type.

Definition at line 326 of file SROA.cpp.

References P.

Referenced by migrateDebugInfo().

Variable Documentation

◆ Aggregates

Scalar Replacement Of Aggregates

Definition at line 5707 of file SROA.cpp.

◆ false

Scalar Replacement Of false

Definition at line 5708 of file SROA.cpp.

◆ sroa

sroa

Definition at line 5707 of file SROA.cpp.

◆ SROASkipMem2Reg

cl::opt< bool > SROASkipMem2Reg("sroa-skip-mem2reg", cl::init(false), cl::Hidden) ( "sroa-skip-mem2reg"  ,
cl::init(false)  ,
cl::Hidden   
)
static

Disable running mem2reg during SROA in order to test or debug SROA.