LoopUnrollAndJamPass.cpp File Reference

#include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/PriorityWorklist.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/DependenceAnalysis.h"
#include "llvm/Analysis/LoopAnalysisManager.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopNestAnalysis.h"
#include "llvm/Analysis/LoopPass.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/PassManager.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/PassRegistry.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Scalar/LoopPassManager.h"
#include "llvm/Transforms/Utils/LoopPeel.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/UnrollLoop.h"
#include <cassert>
#include <cstdint>

Include dependency graph for LoopUnrollAndJamPass.cpp:

Go to the source code of this file.

Namespaces
	llvm
	This is an optimization pass for GlobalISel generic memory operations.

Macros
#define	DEBUG_TYPE   "loop-unroll-and-jam"

Functions
static MDNode *	getUnrollMetadataForLoop (const Loop *L, StringRef Name)
static bool	hasAnyUnrollPragma (const Loop *L, StringRef Prefix)
static bool	hasUnrollAndJamEnablePragma (const Loop *L)
static unsigned	unrollAndJamCountPragmaValue (const Loop *L)
static uint64_t	getUnrollAndJammedLoopSize (unsigned LoopSize, TargetTransformInfo::UnrollingPreferences &UP)
static bool	computeUnrollAndJamCount (Loop *L, Loop *SubLoop, const TargetTransformInfo &TTI, DominatorTree &DT, LoopInfo *LI, AssumptionCache *AC, ScalarEvolution &SE, const SmallPtrSetImpl< const Value * > &EphValues, OptimizationRemarkEmitter *ORE, unsigned OuterTripCount, unsigned OuterTripMultiple, unsigned OuterLoopSize, unsigned InnerTripCount, unsigned InnerLoopSize, TargetTransformInfo::UnrollingPreferences &UP, TargetTransformInfo::PeelingPreferences &PP)
static LoopUnrollResult	tryToUnrollAndJamLoop (Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE, const TargetTransformInfo &TTI, AssumptionCache &AC, DependenceInfo &DI, OptimizationRemarkEmitter &ORE, int OptLevel)
static bool	tryToUnrollAndJamLoop (LoopNest &LN, DominatorTree &DT, LoopInfo &LI, ScalarEvolution &SE, const TargetTransformInfo &TTI, AssumptionCache &AC, DependenceInfo &DI, OptimizationRemarkEmitter &ORE, int OptLevel, LPMUpdater &U)
	INITIALIZE_PASS_BEGIN (LoopUnrollAndJam, "loop-unroll-and-jam", "Unroll and Jam loops", false, false) INITIALIZE_PASS_END(LoopUnrollAndJam

Variables
static cl::opt< bool >	AllowUnrollAndJam ("allow-unroll-and-jam", cl::Hidden, cl::desc("Allows loops to be unroll-and-jammed."))
static cl::opt< unsigned >	UnrollAndJamCount ("unroll-and-jam-count", cl::Hidden, cl::desc("Use this unroll count for all loops including those with " "unroll_and_jam_count pragma values, for testing purposes"))
static cl::opt< unsigned >	UnrollAndJamThreshold ("unroll-and-jam-threshold", cl::init(60), cl::Hidden, cl::desc("Threshold to use for inner loop when doing unroll and jam."))
static cl::opt< unsigned >	PragmaUnrollAndJamThreshold ("pragma-unroll-and-jam-threshold", cl::init(1024), cl::Hidden, cl::desc("Unrolled size limit for loops with an unroll_and_jam(full) or " "unroll_count pragma."))
loop unroll and	jam
loop unroll and Unroll and Jam	loops
loop unroll and Unroll and Jam	false
static const char *const	LLVMLoopUnrollAndJamFollowupAll
static const char *const	LLVMLoopUnrollAndJamFollowupInner
static const char *const	LLVMLoopUnrollAndJamFollowupOuter
static const char *const	LLVMLoopUnrollAndJamFollowupRemainderInner
static const char *const	LLVMLoopUnrollAndJamFollowupRemainderOuter

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "loop-unroll-and-jam"

Definition at line 60 of file LoopUnrollAndJamPass.cpp.

Function Documentation

computeUnrollAndJamCount()

static bool computeUnrollAndJamCount ( Loop *  L, Loop *  SubLoop, const TargetTransformInfo &  TTI, DominatorTree &  DT, LoopInfo *  LI, AssumptionCache *  AC, ScalarEvolution &  SE, const SmallPtrSetImpl< const Value * > &  EphValues, OptimizationRemarkEmitter *  ORE, unsigned  OuterTripCount, unsigned  OuterTripMultiple, unsigned  OuterLoopSize, unsigned  InnerTripCount, unsigned  InnerLoopSize, TargetTransformInfo::UnrollingPreferences &  UP, TargetTransformInfo::PeelingPreferences &  PP  )

static

Definition at line 157 of file LoopUnrollAndJamPass.cpp.

References llvm::TargetTransformInfo::UnrollingPreferences::AllowRemainder, BB, llvm::computeUnrollCount(), llvm::TargetTransformInfo::UnrollingPreferences::Count, llvm::dbgs(), llvm::TargetTransformInfo::UnrollingPreferences::Force, llvm::LoopBase< BlockT, LoopT >::getBlocks(), llvm::ScalarEvolution::getSCEVAtScope(), getUnrollAndJammedLoopSize(), hasUnrollAndJamEnablePragma(), I, llvm::ScalarEvolution::isLoopInvariant(), LLVM_DEBUG, PragmaUnrollAndJamThreshold, llvm::TargetTransformInfo::UnrollingPreferences::Runtime, llvm::TargetTransformInfo::UnrollingPreferences::Threshold, UnrollAndJamCount, unrollAndJamCountPragmaValue(), and llvm::TargetTransformInfo::UnrollingPreferences::UnrollAndJamInnerLoopThreshold.

Referenced by tryToUnrollAndJamLoop().

getUnrollAndJammedLoopSize()

static uint64_t getUnrollAndJammedLoopSize ( unsigned  LoopSize, TargetTransformInfo::UnrollingPreferences &  UP  )

static

Definition at line 149 of file LoopUnrollAndJamPass.cpp.

References assert(), llvm::TargetTransformInfo::UnrollingPreferences::BEInsns, and llvm::TargetTransformInfo::UnrollingPreferences::Count.

Referenced by computeUnrollAndJamCount().

getUnrollMetadataForLoop()

static MDNode* getUnrollMetadataForLoop ( const Loop *  L, StringRef  Name  )

static

Definition at line 97 of file LoopUnrollAndJamPass.cpp.

References llvm::Loop::getLoopID(), and llvm::GetUnrollMetadata().

Referenced by hasUnrollAndJamEnablePragma(), and unrollAndJamCountPragmaValue().

hasAnyUnrollPragma()

static bool hasAnyUnrollPragma ( const Loop *  L, StringRef  Prefix  )

static

Definition at line 105 of file LoopUnrollAndJamPass.cpp.

References assert(), E, llvm::Loop::getLoopID(), llvm::MDNode::getOperand(), I, llvm::cl::Prefix, and S.

Referenced by tryToUnrollAndJamLoop().

hasUnrollAndJamEnablePragma()

static bool hasUnrollAndJamEnablePragma ( const Loop *  L )

static

Definition at line 128 of file LoopUnrollAndJamPass.cpp.

References getUnrollMetadataForLoop().

Referenced by computeUnrollAndJamCount().

INITIALIZE_PASS_BEGIN()

INITIALIZE_PASS_BEGIN	(	LoopUnrollAndJam	,
		"loop-unroll-and-jam"	,
		"Unroll and Jam loops"	,
		false	,
		false
	)

tryToUnrollAndJamLoop() [1/2]

static LoopUnrollResult tryToUnrollAndJamLoop ( Loop *  L, DominatorTree &  DT, LoopInfo *  LI, ScalarEvolution &  SE, const TargetTransformInfo &  TTI, AssumptionCache &  AC, DependenceInfo &  DI, OptimizationRemarkEmitter &  ORE, int  OptLevel  )

static

Definition at line 283 of file LoopUnrollAndJamPass.cpp.

References AllowUnrollAndJam, llvm::ApproximateLoopSize(), llvm::TargetTransformInfo::UnrollingPreferences::BEInsns, llvm::CodeMetrics::collectEphemeralValues(), computeUnrollAndJamCount(), llvm::MCID::Convergent, llvm::TargetTransformInfo::UnrollingPreferences::Count, llvm::dbgs(), llvm::FullyUnrolled, llvm::gatherPeelingPreferences(), llvm::gatherUnrollingPreferences(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::Loop::getLoopID(), llvm::LoopBase< BlockT, LoopT >::getLoopLatch(), llvm::Value::getName(), llvm::cl::Option::getNumOccurrences(), llvm::BasicBlock::getParent(), llvm::ScalarEvolution::getSmallConstantTripCount(), llvm::ScalarEvolution::getSmallConstantTripMultiple(), llvm::LoopBase< BlockT, LoopT >::getSubLoops(), llvm::InstructionCost::getValue(), hasAnyUnrollPragma(), llvm::hasUnrollAndJamTransformation(), llvm::isSafeToUnrollAndJam(), llvm::InstructionCost::isValid(), LLVM_DEBUG, LLVMLoopUnrollAndJamFollowupAll, LLVMLoopUnrollAndJamFollowupInner, LLVMLoopUnrollAndJamFollowupOuter, LLVMLoopUnrollAndJamFollowupRemainderInner, LLVMLoopUnrollAndJamFollowupRemainderOuter, llvm::makeFollowupLoopID(), llvm::None, llvm::PartiallyUnrolled, llvm::Loop::setLoopAlreadyUnrolled(), llvm::Loop::setLoopID(), llvm::TM_Disable, llvm::TM_ForcedByUser, llvm::Unmodified, llvm::TargetTransformInfo::UnrollingPreferences::UnrollAndJam, llvm::TargetTransformInfo::UnrollingPreferences::UnrollAndJamInnerLoopThreshold, llvm::UnrollAndJamLoop(), UnrollAndJamThreshold, llvm::TargetTransformInfo::UnrollingPreferences::UnrollRemainder, and llvm::Optional< T >::value().

Referenced by llvm::LoopUnrollAndJamPass::run(), and tryToUnrollAndJamLoop().

tryToUnrollAndJamLoop() [2/2]

static bool tryToUnrollAndJamLoop ( LoopNest &  LN, DominatorTree &  DT, LoopInfo &  LI, ScalarEvolution &  SE, const TargetTransformInfo &  TTI, AssumptionCache &  AC, DependenceInfo &  DI, OptimizationRemarkEmitter &  ORE, int  OptLevel, LPMUpdater &  U  )

static

Definition at line 437 of file LoopUnrollAndJamPass.cpp.

References llvm::appendLoopsToWorklist(), llvm::PriorityWorklist< T, SmallVector< T, N >, SmallDenseMap< T, ptrdiff_t > >::empty(), llvm::FullyUnrolled, llvm::LoopNest::getLoops(), llvm::Loop::getName(), llvm::LoopNest::getOutermostLoop(), Loops, llvm::LPMUpdater::markLoopAsDeleted(), llvm::PriorityWorklist< T, SmallVector< T, N >, SmallDenseMap< T, ptrdiff_t > >::pop_back_val(), tryToUnrollAndJamLoop(), and llvm::Unmodified.

unrollAndJamCountPragmaValue()

static unsigned unrollAndJamCountPragmaValue ( const Loop *  L )

static

Definition at line 134 of file LoopUnrollAndJamPass.cpp.

References assert(), llvm::MDNode::getNumOperands(), llvm::MDNode::getOperand(), and getUnrollMetadataForLoop().

Referenced by computeUnrollAndJamCount().

Variable Documentation

AllowUnrollAndJam

cl::opt<bool> AllowUnrollAndJam("allow-unroll-and-jam", cl::Hidden, cl::desc("Allows loops to be unroll-and-jammed."))

static

Referenced by tryToUnrollAndJamLoop().

false

loop unroll and Unroll and Jam false

Definition at line 529 of file LoopUnrollAndJamPass.cpp.

jam

loop unroll and jam

Definition at line 528 of file LoopUnrollAndJamPass.cpp.

LLVMLoopUnrollAndJamFollowupAll

const char* const LLVMLoopUnrollAndJamFollowupAll

static

Initial value:

=
    "llvm.loop.unroll_and_jam.followup_all"

Metadata attribute names

Definition at line 64 of file LoopUnrollAndJamPass.cpp.

Referenced by tryToUnrollAndJamLoop().

LLVMLoopUnrollAndJamFollowupInner

const char* const LLVMLoopUnrollAndJamFollowupInner

static

Initial value:

=
    "llvm.loop.unroll_and_jam.followup_inner"

Definition at line 66 of file LoopUnrollAndJamPass.cpp.

Referenced by tryToUnrollAndJamLoop().

LLVMLoopUnrollAndJamFollowupOuter

const char* const LLVMLoopUnrollAndJamFollowupOuter

static

Initial value:

=
    "llvm.loop.unroll_and_jam.followup_outer"

Definition at line 68 of file LoopUnrollAndJamPass.cpp.

Referenced by tryToUnrollAndJamLoop().

LLVMLoopUnrollAndJamFollowupRemainderInner

const char* const LLVMLoopUnrollAndJamFollowupRemainderInner

static

Initial value:

=
    "llvm.loop.unroll_and_jam.followup_remainder_inner"

Definition at line 70 of file LoopUnrollAndJamPass.cpp.

Referenced by tryToUnrollAndJamLoop().

LLVMLoopUnrollAndJamFollowupRemainderOuter

const char* const LLVMLoopUnrollAndJamFollowupRemainderOuter

static

Initial value:

=
    "llvm.loop.unroll_and_jam.followup_remainder_outer"

Definition at line 72 of file LoopUnrollAndJamPass.cpp.

Referenced by tryToUnrollAndJamLoop().

loops

loop unroll and Unroll and Jam loops

Definition at line 529 of file LoopUnrollAndJamPass.cpp.

PragmaUnrollAndJamThreshold

cl::opt<unsigned> PragmaUnrollAndJamThreshold("pragma-unroll-and-jam-threshold", cl::init(1024), cl::Hidden, cl::desc("Unrolled size limit for loops with an unroll_and_jam(full) or " "unroll_count pragma."))

static

Referenced by computeUnrollAndJamCount().

UnrollAndJamCount

cl::opt<unsigned> UnrollAndJamCount("unroll-and-jam-count", cl::Hidden, cl::desc("Use this unroll count for all loops including those with " "unroll_and_jam_count pragma values, for testing purposes"))

static

Referenced by computeUnrollAndJamCount().

UnrollAndJamThreshold

cl::opt<unsigned> UnrollAndJamThreshold("unroll-and-jam-threshold", cl::init(60), cl::Hidden, cl::desc("Threshold to use for inner loop when doing unroll and jam."))

static

Referenced by tryToUnrollAndJamLoop().