#include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/PriorityWorklist.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/DependenceAnalysis.h"
#include "llvm/Analysis/LoopAnalysisManager.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopNestAnalysis.h"
#include "llvm/Analysis/LoopPass.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/PassManager.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Scalar/LoopPassManager.h"
#include "llvm/Transforms/Utils/LoopPeel.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/UnrollLoop.h"
#include <cassert>
#include <cstdint>

Namespaces
namespace	llvm
	This is an optimization pass for GlobalISel generic memory operations.

Macros
#define	DEBUG_TYPE "loop-unroll-and-jam"

Functions
static MDNode *	getUnrollMetadataForLoop (const Loop *L, StringRef Name)

static bool	hasAnyUnrollPragma (const Loop *L, StringRef Prefix)

static bool	hasUnrollAndJamEnablePragma (const Loop *L)

static unsigned	unrollAndJamCountPragmaValue (const Loop *L)

static uint64_t	getUnrollAndJammedLoopSize (unsigned LoopSize, TargetTransformInfo::UnrollingPreferences &UP)

static bool	computeUnrollAndJamCount (Loop L, Loop SubLoop, const TargetTransformInfo &TTI, DominatorTree &DT, LoopInfo LI, AssumptionCache AC, ScalarEvolution &SE, const SmallPtrSetImpl< const Value * > &EphValues, OptimizationRemarkEmitter *ORE, unsigned OuterTripCount, unsigned OuterTripMultiple, const UnrollCostEstimator &OuterUCE, unsigned InnerTripCount, unsigned InnerLoopSize, TargetTransformInfo::UnrollingPreferences &UP, TargetTransformInfo::PeelingPreferences &PP)

static LoopUnrollResult	tryToUnrollAndJamLoop (Loop L, DominatorTree &DT, LoopInfo LI, ScalarEvolution &SE, const TargetTransformInfo &TTI, AssumptionCache &AC, DependenceInfo &DI, OptimizationRemarkEmitter &ORE, int OptLevel)

static bool	tryToUnrollAndJamLoop (LoopNest &LN, DominatorTree &DT, LoopInfo &LI, ScalarEvolution &SE, const TargetTransformInfo &TTI, AssumptionCache &AC, DependenceInfo &DI, OptimizationRemarkEmitter &ORE, int OptLevel, LPMUpdater &U)

Variables
static cl::opt< bool >	AllowUnrollAndJam ("allow-unroll-and-jam", cl::Hidden, cl::desc("Allows loops to be unroll-and-jammed."))

static cl::opt< unsigned >	UnrollAndJamCount ("unroll-and-jam-count", cl::Hidden, cl::desc("Use this unroll count for all loops including those with " "unroll_and_jam_count pragma values, for testing purposes"))

static cl::opt< unsigned >	UnrollAndJamThreshold ("unroll-and-jam-threshold", cl::init(60), cl::Hidden, cl::desc("Threshold to use for inner loop when doing unroll and jam."))

static cl::opt< unsigned >	PragmaUnrollAndJamThreshold ("pragma-unroll-and-jam-threshold", cl::init(1024), cl::Hidden, cl::desc("Unrolled size limit for loops with an unroll_and_jam(full) or " "unroll_count pragma."))


static const char *const	LLVMLoopUnrollAndJamFollowupAll

static const char *const	LLVMLoopUnrollAndJamFollowupInner

static const char *const	LLVMLoopUnrollAndJamFollowupOuter

static const char *const	LLVMLoopUnrollAndJamFollowupRemainderInner

static const char *const	LLVMLoopUnrollAndJamFollowupRemainderOuter

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE "loop-unroll-and-jam"

Definition at line 54 of file LoopUnrollAndJamPass.cpp.

Function Documentation

◆ computeUnrollAndJamCount()

static bool computeUnrollAndJamCount	(	Loop *	L,
		Loop *	SubLoop,
		const TargetTransformInfo &	TTI,
		DominatorTree &	DT,
		LoopInfo *	LI,
		AssumptionCache *	AC,
		ScalarEvolution &	SE,
		const SmallPtrSetImpl< const Value * > &	EphValues,
		OptimizationRemarkEmitter *	ORE,
		unsigned	OuterTripCount,
		unsigned	OuterTripMultiple,
		const UnrollCostEstimator &	OuterUCE,
		unsigned	InnerTripCount,
		unsigned	InnerLoopSize,
		TargetTransformInfo::UnrollingPreferences &	UP,
		TargetTransformInfo::PeelingPreferences &	PP
	)

static

Definition at line 151 of file LoopUnrollAndJamPass.cpp.

References llvm::TargetTransformInfo::UnrollingPreferences::AllowRemainder, llvm::computeUnrollCount(), llvm::TargetTransformInfo::UnrollingPreferences::Count, llvm::dbgs(), llvm::TargetTransformInfo::UnrollingPreferences::Force, llvm::LoopBase< BlockT, LoopT >::getBlocks(), llvm::UnrollCostEstimator::getRolledLoopSize(), llvm::ScalarEvolution::getSCEVAtScope(), getUnrollAndJammedLoopSize(), hasUnrollAndJamEnablePragma(), I, llvm::ScalarEvolution::isLoopInvariant(), LLVM_DEBUG, PragmaUnrollAndJamThreshold, llvm::TargetTransformInfo::UnrollingPreferences::Runtime, llvm::TargetTransformInfo::UnrollingPreferences::Threshold, UnrollAndJamCount, unrollAndJamCountPragmaValue(), and llvm::TargetTransformInfo::UnrollingPreferences::UnrollAndJamInnerLoopThreshold.

Referenced by tryToUnrollAndJamLoop().

◆ getUnrollAndJammedLoopSize()

static uint64_t getUnrollAndJammedLoopSize	(	unsigned	LoopSize,
		TargetTransformInfo::UnrollingPreferences &	UP
	)

static

Definition at line 143 of file LoopUnrollAndJamPass.cpp.

References assert(), llvm::TargetTransformInfo::UnrollingPreferences::BEInsns, and llvm::TargetTransformInfo::UnrollingPreferences::Count.

Referenced by computeUnrollAndJamCount().

◆ getUnrollMetadataForLoop()

static MDNode * getUnrollMetadataForLoop	(	const Loop *	L,
		StringRef	Name
	)

static

Definition at line 91 of file LoopUnrollAndJamPass.cpp.

References llvm::GetUnrollMetadata(), and Name.

Referenced by hasRuntimeUnrollDisablePragma(), hasUnrollAndJamEnablePragma(), hasUnrollEnablePragma(), hasUnrollFullPragma(), unrollAndJamCountPragmaValue(), and unrollCountPragmaValue().

◆ hasAnyUnrollPragma()

static bool hasAnyUnrollPragma	(	const Loop *	L,
		StringRef	Prefix
	)

static

Definition at line 99 of file LoopUnrollAndJamPass.cpp.

References assert(), llvm::MDNode::getOperand(), llvm::MDString::getString(), I, and llvm::StringRef::starts_with().

Referenced by tryToUnrollAndJamLoop().

◆ hasUnrollAndJamEnablePragma()

static bool hasUnrollAndJamEnablePragma ( const Loop * L )

static

Definition at line 122 of file LoopUnrollAndJamPass.cpp.

References getUnrollMetadataForLoop().

Referenced by computeUnrollAndJamCount().

◆ tryToUnrollAndJamLoop() [1/2]

static LoopUnrollResult tryToUnrollAndJamLoop	(	Loop *	L,
		DominatorTree &	DT,
		LoopInfo *	LI,
		ScalarEvolution &	SE,
		const TargetTransformInfo &	TTI,
		AssumptionCache &	AC,
		DependenceInfo &	DI,
		OptimizationRemarkEmitter &	ORE,
		int	OptLevel
	)

static

Definition at line 279 of file LoopUnrollAndJamPass.cpp.

References AllowUnrollAndJam, llvm::TargetTransformInfo::UnrollingPreferences::BEInsns, llvm::UnrollCostEstimator::canUnroll(), llvm::CodeMetrics::collectEphemeralValues(), computeUnrollAndJamCount(), llvm::UnrollCostEstimator::Convergent, llvm::TargetTransformInfo::UnrollingPreferences::Count, llvm::dbgs(), llvm::FullyUnrolled, llvm::gatherPeelingPreferences(), llvm::gatherUnrollingPreferences(), llvm::Loop::getLoopID(), llvm::LoopBase< BlockT, LoopT >::getLoopLatch(), llvm::UnrollCostEstimator::getRolledLoopSize(), llvm::ScalarEvolution::getSmallConstantTripCount(), llvm::ScalarEvolution::getSmallConstantTripMultiple(), hasAnyUnrollPragma(), llvm::hasUnrollAndJamTransformation(), llvm::isSafeToUnrollAndJam(), LLVM_DEBUG, LLVMLoopUnrollAndJamFollowupAll, LLVMLoopUnrollAndJamFollowupInner, LLVMLoopUnrollAndJamFollowupOuter, LLVMLoopUnrollAndJamFollowupRemainderInner, LLVMLoopUnrollAndJamFollowupRemainderOuter, llvm::makeFollowupLoopID(), llvm::UnrollCostEstimator::NumInlineCandidates, llvm::PartiallyUnrolled, llvm::Loop::setLoopID(), llvm::TM_Disable, llvm::TM_ForcedByUser, llvm::Unmodified, llvm::TargetTransformInfo::UnrollingPreferences::UnrollAndJam, llvm::TargetTransformInfo::UnrollingPreferences::UnrollAndJamInnerLoopThreshold, llvm::UnrollAndJamLoop(), UnrollAndJamThreshold, and llvm::TargetTransformInfo::UnrollingPreferences::UnrollRemainder.

Referenced by llvm::LoopUnrollAndJamPass::run(), and tryToUnrollAndJamLoop().

◆ tryToUnrollAndJamLoop() [2/2]

static bool tryToUnrollAndJamLoop	(	LoopNest &	LN,
		DominatorTree &	DT,
		LoopInfo &	LI,
		ScalarEvolution &	SE,
		const TargetTransformInfo &	TTI,
		AssumptionCache &	AC,
		DependenceInfo &	DI,
		OptimizationRemarkEmitter &	ORE,
		int	OptLevel,
		LPMUpdater &	U
	)

static

Definition at line 422 of file LoopUnrollAndJamPass.cpp.

References llvm::appendLoopsToWorklist(), llvm::PriorityWorklist< T, VectorT, MapT >::empty(), llvm::FullyUnrolled, llvm::LoopNest::getLoops(), llvm::LoopNest::getOutermostLoop(), Loops, llvm::PriorityWorklist< T, VectorT, MapT >::pop_back_val(), tryToUnrollAndJamLoop(), and llvm::Unmodified.

◆ unrollAndJamCountPragmaValue()

static unsigned unrollAndJamCountPragmaValue ( const Loop * L )

static

Definition at line 128 of file LoopUnrollAndJamPass.cpp.

References assert(), llvm::MDNode::getNumOperands(), llvm::MDNode::getOperand(), and getUnrollMetadataForLoop().

Referenced by computeUnrollAndJamCount().

Variable Documentation

◆ AllowUnrollAndJam

cl::opt< bool > AllowUnrollAndJam("allow-unroll-and-jam", cl::Hidden, cl::desc("Allows loops to be unroll-and-jammed."))	(	"allow-unroll-and-jam"	,
		cl::Hidden	,
		cl::desc("Allows loops to be unroll-and-jammed.")
	)

static

Referenced by tryToUnrollAndJamLoop().

◆ LLVMLoopUnrollAndJamFollowupAll

const char* const LLVMLoopUnrollAndJamFollowupAll

static

Initial value:

=

"llvm.loop.unroll_and_jam.followup_all"

Metadata attribute names

Definition at line 58 of file LoopUnrollAndJamPass.cpp.

Referenced by tryToUnrollAndJamLoop().

◆ LLVMLoopUnrollAndJamFollowupInner

const char* const LLVMLoopUnrollAndJamFollowupInner

static

Initial value:

=

"llvm.loop.unroll_and_jam.followup_inner"

Definition at line 60 of file LoopUnrollAndJamPass.cpp.

Referenced by tryToUnrollAndJamLoop().

◆ LLVMLoopUnrollAndJamFollowupOuter

const char* const LLVMLoopUnrollAndJamFollowupOuter

static

Initial value:

=

"llvm.loop.unroll_and_jam.followup_outer"

Definition at line 62 of file LoopUnrollAndJamPass.cpp.

Referenced by tryToUnrollAndJamLoop().

◆ LLVMLoopUnrollAndJamFollowupRemainderInner

const char* const LLVMLoopUnrollAndJamFollowupRemainderInner

static

Initial value:

=

"llvm.loop.unroll_and_jam.followup_remainder_inner"

Definition at line 64 of file LoopUnrollAndJamPass.cpp.

Referenced by tryToUnrollAndJamLoop().

◆ LLVMLoopUnrollAndJamFollowupRemainderOuter

const char* const LLVMLoopUnrollAndJamFollowupRemainderOuter

static

Initial value:

=

"llvm.loop.unroll_and_jam.followup_remainder_outer"

Definition at line 66 of file LoopUnrollAndJamPass.cpp.

Referenced by tryToUnrollAndJamLoop().

◆ PragmaUnrollAndJamThreshold

cl::opt< unsigned > PragmaUnrollAndJamThreshold("pragma-unroll-and-jam-threshold", cl::init(1024), cl::Hidden, cl::desc("Unrolled size limit for loops with an unroll_and_jam(full) or " "unroll_count pragma."))	(	"pragma-unroll-and-jam-threshold"	,
		cl::init(1024)	,
		cl::Hidden	,
		cl::desc("Unrolled size limit for loops with an unroll_and_jam(full) or " "unroll_count pragma.")
	)

static

Referenced by computeUnrollAndJamCount().

◆ UnrollAndJamCount

cl::opt< unsigned > UnrollAndJamCount("unroll-and-jam-count", cl::Hidden, cl::desc("Use this unroll count for all loops including those with " "unroll_and_jam_count pragma values, for testing purposes"))	(	"unroll-and-jam-count"	,
		cl::Hidden	,
		cl::desc("Use this unroll count for all loops including those with " "unroll_and_jam_count pragma values, for testing purposes")
	)

static

Referenced by computeUnrollAndJamCount().

◆ UnrollAndJamThreshold

cl::opt< unsigned > UnrollAndJamThreshold("unroll-and-jam-threshold", cl::init(60), cl::Hidden, cl::desc("Threshold to use for inner loop when doing unroll and jam."))	(	"unroll-and-jam-threshold"	,
		cl::init(60)	,
		cl::Hidden	,
		cl::desc("Threshold to use for inner loop when doing unroll and jam.")
	)

static

Referenced by tryToUnrollAndJamLoop().

Namespaces

Macros

Functions

Variables

Macro Definition Documentation

◆ DEBUG_TYPE

Function Documentation

◆ computeUnrollAndJamCount()

◆ getUnrollAndJammedLoopSize()

◆ getUnrollMetadataForLoop()

◆ hasAnyUnrollPragma()

◆ hasUnrollAndJamEnablePragma()

◆ tryToUnrollAndJamLoop() [1/2]

◆ tryToUnrollAndJamLoop() [2/2]

◆ unrollAndJamCountPragmaValue()

Variable Documentation

◆ AllowUnrollAndJam

◆ LLVMLoopUnrollAndJamFollowupAll

◆ LLVMLoopUnrollAndJamFollowupInner

◆ LLVMLoopUnrollAndJamFollowupOuter

◆ LLVMLoopUnrollAndJamFollowupRemainderInner

◆ LLVMLoopUnrollAndJamFollowupRemainderOuter

◆ PragmaUnrollAndJamThreshold

◆ UnrollAndJamCount

◆ UnrollAndJamThreshold