LowerMemIntrinsics.cpp File Reference

#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/ProfDataUtils.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include <limits>
#include <optional>

Go to the source code of this file.

Namespaces
namespace	llvm
	This is an optimization pass for GlobalISel generic memory operations.

Macros
#define	DEBUG_TYPE   "lower-mem-intrinsics"

Functions
static Value *	getRuntimeLoopRemainder (IRBuilderBase &B, Value *Len, Value *OpSize, unsigned OpSizeVal)
static Value *	getRuntimeLoopUnits (IRBuilderBase &B, Value *Len, Value *OpSize, unsigned OpSizeVal, Value *RTLoopRemainder=nullptr)
static LoopExpansionInfo	insertLoopExpansion (Instruction *InsertBefore, Value *Len, unsigned MainLoopStep, unsigned ResidualLoopStep, StringRef BBNamePrefix, std::optional< uint64_t > ExpectedUnits)
	Insert the control flow and loop counters for a memcpy/memset loop expansion.
static std::pair< Value *, Value * >	tryInsertCastToCommonAddrSpace (IRBuilderBase &B, Value *Addr1, Value *Addr2, const TargetTransformInfo &TTI)
static void	createMemMoveLoopUnknownSize (Instruction *InsertBefore, Value *SrcAddr, Value *DstAddr, Value *CopyLen, Align SrcAlign, Align DstAlign, bool SrcIsVolatile, bool DstIsVolatile, const TargetTransformInfo &TTI)
static void	createMemMoveLoopKnownSize (Instruction *InsertBefore, Value *SrcAddr, Value *DstAddr, ConstantInt *CopyLen, Align SrcAlign, Align DstAlign, bool SrcIsVolatile, bool DstIsVolatile, const TargetTransformInfo &TTI)
static Value *	createMemSetSplat (const DataLayout &DL, IRBuilderBase &B, Value *SetValue, Type *DstType)
	Create a Value of DstType that consists of a sequence of copies of SetValue, using bitcasts and a vector splat.
static void	createMemSetLoopKnownSize (Instruction *InsertBefore, Value *DstAddr, ConstantInt *Len, Value *SetValue, Align DstAlign, bool IsVolatile, const TargetTransformInfo *TTI, std::optional< uint64_t > AverageTripCount)
static void	createMemSetLoopUnknownSize (Instruction *InsertBefore, Value *DstAddr, Value *Len, Value *SetValue, Align DstAlign, bool IsVolatile, const TargetTransformInfo *TTI, std::optional< uint64_t > AverageTripCount)
static void	createMemSetPatternLoop (Instruction *InsertBefore, Value *DstAddr, Value *Len, Value *SetValue, Align DstAlign, bool IsVolatile, const TargetTransformInfo *TTI, std::optional< uint64_t > AverageTripCount)
template<typename T>
static bool	canOverlap (MemTransferBase< T > *Memcpy, ScalarEvolution *SE)

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "lower-mem-intrinsics"

Definition at line 24 of file LowerMemIntrinsics.cpp.

Function Documentation

canOverlap()

template<typename T>

bool canOverlap ( MemTransferBase< T > * Memcpy, ScalarEvolution * SE )

static

Definition at line 1410 of file LowerMemIntrinsics.cpp.

References llvm::MemTransferBase< BaseCL >::getRawSource(), llvm::ScalarEvolution::getSCEV(), llvm::CmpInst::ICMP_NE, and llvm::ScalarEvolution::isKnownPredicateAt().

Referenced by llvm::expandMemCpyAsLoop().

createMemMoveLoopKnownSize()

void createMemMoveLoopKnownSize ( Instruction * InsertBefore, Value * SrcAddr, Value * DstAddr, ConstantInt * CopyLen, Align SrcAlign, Align DstAlign, bool SrcIsVolatile, bool DstIsVolatile, const TargetTransformInfo & TTI )

static

Definition at line 937 of file LowerMemIntrinsics.cpp.

References llvm::PHINode::addIncoming(), llvm::alignDown(), assert(), llvm::cast(), llvm::commonAlignment(), llvm::IRBuilderBase::CreateAdd(), llvm::IRBuilderBase::CreateAlignedLoad(), llvm::IRBuilderBase::CreateAlignedStore(), llvm::IRBuilderBase::CreateCondBr(), llvm::IRBuilderBase::CreateICmpEQ(), llvm::IRBuilderBase::CreateICmpULT(), llvm::IRBuilderBase::CreateInBoundsGEP(), llvm::IRBuilderBase::CreatePHI(), llvm::IRBuilderBase::CreateSub(), DL, llvm::Instruction::eraseFromParent(), F, llvm::BasicBlock::getContext(), llvm::BasicBlock::getFirstNonPHIIt(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::Type::getInt8Ty(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::BasicBlock::getParent(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Instruction::getStableDebugLoc(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), llvm::ConstantInt::getZExtValue(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isFixed(), llvm::ConstantInt::isZero(), llvm::IRBuilderBase::SetCurrentDebugLocation(), llvm::IRBuilderBase::SetInsertPoint(), llvm::Value::setName(), llvm::BasicBlock::splitBasicBlock(), llvm::SplitBlockAndInsertIfThenElse(), and tryInsertCastToCommonAddrSpace().

Referenced by llvm::expandMemMoveAsLoop().

createMemMoveLoopUnknownSize()

void createMemMoveLoopUnknownSize ( Instruction * InsertBefore, Value * SrcAddr, Value * DstAddr, Value * CopyLen, Align SrcAlign, Align DstAlign, bool SrcIsVolatile, bool DstIsVolatile, const TargetTransformInfo & TTI )

static

Definition at line 670 of file LowerMemIntrinsics.cpp.

References llvm::PHINode::addIncoming(), llvm::cast(), llvm::commonAlignment(), llvm::BasicBlock::Create(), llvm::CondBrInst::Create(), llvm::IRBuilderBase::CreateAdd(), llvm::IRBuilderBase::CreateAlignedLoad(), llvm::IRBuilderBase::CreateAlignedStore(), llvm::IRBuilderBase::CreateCondBr(), llvm::IRBuilderBase::CreateICmpEQ(), llvm::IRBuilderBase::CreateICmpULT(), llvm::IRBuilderBase::CreateInBoundsGEP(), llvm::IRBuilderBase::CreatePHI(), llvm::IRBuilderBase::CreateSub(), llvm::IRBuilderBase::CreateUnreachable(), DL, llvm::Instruction::eraseFromParent(), F, llvm::BasicBlock::getContext(), llvm::Type::getInt8Ty(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::BasicBlock::getParent(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), getRuntimeLoopRemainder(), getRuntimeLoopUnits(), llvm::Instruction::getStableDebugLoc(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), llvm::IRBuilderBase::SetCurrentDebugLocation(), llvm::Value::setName(), llvm::SplitBlockAndInsertIfThenElse(), and tryInsertCastToCommonAddrSpace().

Referenced by llvm::expandMemMoveAsLoop().

createMemSetLoopKnownSize()

void createMemSetLoopKnownSize ( Instruction * InsertBefore, Value * DstAddr, ConstantInt * Len, Value * SetValue, Align DstAlign, bool IsVolatile, const TargetTransformInfo * TTI, std::optional< uint64_t > AverageTripCount )

static

Definition at line 1151 of file LowerMemIntrinsics.cpp.

References llvm::alignDown(), assert(), llvm::cast(), llvm::commonAlignment(), llvm::IRBuilderBase::CreateAlignedStore(), llvm::IRBuilderBase::CreateInBoundsGEP(), createMemSetSplat(), DL, llvm::BasicBlock::getContext(), llvm::Function::getDataLayout(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::Type::getInt8Ty(), llvm::BasicBlock::getParent(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Value::getType(), insertLoopExpansion(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isFixed(), and SetValue().

Referenced by llvm::expandMemSetAsLoop().

createMemSetLoopUnknownSize()

void createMemSetLoopUnknownSize ( Instruction * InsertBefore, Value * DstAddr, Value * Len, Value * SetValue, Align DstAlign, bool IsVolatile, const TargetTransformInfo * TTI, std::optional< uint64_t > AverageTripCount )

static

Definition at line 1250 of file LowerMemIntrinsics.cpp.

References assert(), llvm::cast(), llvm::commonAlignment(), llvm::IRBuilderBase::CreateAlignedStore(), llvm::IRBuilderBase::CreateInBoundsGEP(), createMemSetSplat(), DL, llvm::BasicBlock::getContext(), llvm::Function::getDataLayout(), llvm::Type::getInt8Ty(), llvm::BasicBlock::getParent(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Value::getType(), insertLoopExpansion(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isFixed(), and SetValue().

Referenced by llvm::expandMemSetAsLoop().

createMemSetPatternLoop()

void createMemSetPatternLoop ( Instruction * InsertBefore, Value * DstAddr, Value * Len, Value * SetValue, Align DstAlign, bool IsVolatile, const TargetTransformInfo * TTI, std::optional< uint64_t > AverageTripCount )

static

Definition at line 1310 of file LowerMemIntrinsics.cpp.

References assert(), llvm::cast(), llvm::commonAlignment(), llvm::IRBuilderBase::CreateAlignedStore(), llvm::IRBuilderBase::CreateInBoundsGEP(), createMemSetSplat(), DL, llvm::dyn_cast(), llvm::BasicBlock::getContext(), llvm::Function::getDataLayout(), llvm::BasicBlock::getParent(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), insertLoopExpansion(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isFixed(), and SetValue().

Referenced by llvm::expandMemSetPatternAsLoop().

createMemSetSplat()

Value * createMemSetSplat ( const DataLayout & DL, IRBuilderBase & B, Value * SetValue, Type * DstType )

static

Create a Value of DstType that consists of a sequence of copies of SetValue, using bitcasts and a vector splat.

Definition at line 1121 of file LowerMemIntrinsics.cpp.

References assert(), B(), DL, llvm::Type::getIntNTy(), llvm::Type::isFloatingPointTy(), llvm::Type::isIntegerTy(), and SetValue().

Referenced by createMemSetLoopKnownSize(), createMemSetLoopUnknownSize(), and createMemSetPatternLoop().

getRuntimeLoopRemainder()

Value * getRuntimeLoopRemainder ( IRBuilderBase & B, Value * Len, Value * OpSize, unsigned OpSizeVal )

static

Returns

Len urem OpSize, checking for optimization opportunities. OpSizeVal must be the integer value of the ConstantInt OpSize.

Definition at line 34 of file LowerMemIntrinsics.cpp.

References B(), and llvm::isPowerOf2_32().

Referenced by createMemMoveLoopUnknownSize(), getRuntimeLoopUnits(), and insertLoopExpansion().

getRuntimeLoopUnits()

Value * getRuntimeLoopUnits ( IRBuilderBase & B, Value * Len, Value * OpSize, unsigned OpSizeVal, Value * RTLoopRemainder = nullptr )

static

Returns

(Len udiv OpSize) mul OpSize, checking for optimization opportunities. If RTLoopRemainder is provided, it must be the result of getRuntimeLoopRemainder() with the same arguments.

Definition at line 46 of file LowerMemIntrinsics.cpp.

References B(), and getRuntimeLoopRemainder().

Referenced by createMemMoveLoopUnknownSize(), and insertLoopExpansion().

insertLoopExpansion()

LoopExpansionInfo insertLoopExpansion ( Instruction * InsertBefore, Value * Len, unsigned MainLoopStep, unsigned ResidualLoopStep, StringRef BBNamePrefix, std::optional< uint64_t > ExpectedUnits )

static

Insert the control flow and loop counters for a memcpy/memset loop expansion.

This function inserts IR corresponding to the following C code before InsertBefore:

LoopUnits = (Len / MainLoopStep) * MainLoopStep;
ResidualUnits = Len - LoopUnits;
MainLoopIndex = 0;
if (LoopUnits > 0) {
  do {
    // MainLoopIP
    MainLoopIndex += MainLoopStep;
  } while (MainLoopIndex < LoopUnits);
}
for (size_t i = 0; i < ResidualUnits; i += ResidualLoopStep) {
  ResidualLoopIndex = LoopUnits + i;
  // ResidualLoopIP
}

MainLoopStep and ResidualLoopStep determine by how many "units" the loop index is increased in each iteration of the main and residual loops, respectively. In most cases, the "unit" will be bytes, but larger units are useful for lowering memset.pattern.

The computation of LoopUnits and ResidualUnits is performed at compile time if Len is a ConstantInt. The second (residual) loop is omitted if ResidualLoopStep is 0 or equal to MainLoopStep. The generated MainLoopIP, MainLoopIndex, ResidualLoopIP, and ResidualLoopIndex are returned in a LoopExpansionInfo object.

If provided, ExpectedUnits is used as the expected number of units handled by the loop expansion when computing branch weights.

Definition at line 131 of file LowerMemIntrinsics.cpp.

References llvm::PHINode::addIncoming(), llvm::alignDown(), assert(), llvm::cast(), llvm::BasicBlock::Create(), llvm::IRBuilderBase::CreateAdd(), llvm::IRBuilderBase::CreateBr(), llvm::IRBuilderBase::CreateCondBr(), llvm::IRBuilderBase::CreateICmpNE(), llvm::IRBuilderBase::CreateICmpULT(), llvm::MDBuilder::createLikelyBranchWeights(), llvm::IRBuilderBase::CreatePHI(), DEBUG_TYPE, llvm::dyn_cast(), llvm::Instruction::eraseFromParent(), llvm::BasicBlock::getContext(), llvm::Function::getContext(), llvm::BasicBlock::getParent(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), getRuntimeLoopRemainder(), getRuntimeLoopUnits(), llvm::Instruction::getStableDebugLoc(), llvm::BasicBlock::getTerminator(), llvm::IRBuilderBase::SetCurrentDebugLocation(), llvm::setExplicitlyUnknownBranchWeightsIfProfiled(), llvm::setFittedBranchWeights(), llvm::CondBrInst::setSuccessor(), and llvm::BasicBlock::splitBasicBlock().

Referenced by llvm::createMemCpyLoopKnownSize(), createMemSetLoopKnownSize(), createMemSetLoopUnknownSize(), and createMemSetPatternLoop().

tryInsertCastToCommonAddrSpace()

std::pair< Value *, Value * > tryInsertCastToCommonAddrSpace ( IRBuilderBase & B, Value * Addr1, Value * Addr2, const TargetTransformInfo & TTI )

static

Definition at line 625 of file LowerMemIntrinsics.cpp.

References B(), llvm::cast(), llvm::Value::getType(), and llvm_unreachable.

Referenced by createMemMoveLoopKnownSize(), and createMemMoveLoopUnknownSize().