40#define DEBUG_TYPE "loop-rotate"
43 "Number of loops not rotated due to the header size");
45 "Number of instructions hoisted into loop preheader");
47 "Number of instructions cloned into loop preheader");
52 cl::desc(
"Allow loop rotation multiple times in order to reach "
53 "a better latch exit"));
61 const unsigned MaxHeaderSize;
74 LoopRotate(
unsigned MaxHeaderSize,
LoopInfo *LI,
79 : MaxHeaderSize(MaxHeaderSize), LI(LI),
TTI(
TTI), AC(AC), DT(DT), SE(SE),
80 MSSAU(MSSAU), SQ(SQ), RotationOnly(RotationOnly),
81 IsUtilMode(IsUtilMode), PrepareForLTO(PrepareForLTO) {}
82 bool processLoop(
Loop *L);
85 bool rotateLoop(
Loop *L,
bool SimplifiedLatch);
86 bool simplifyLoopLatch(
Loop *L);
93 bool Inserted = VM.
insert({K, V}).second;
109 PN->removeIncomingValue(PN->getBasicBlockIndex(OrigPreheader));
114 for (
I = OrigHeader->
begin();
I !=
E; ++
I) {
115 Value *OrigHeaderVal = &*
I;
131 SSA.AddAvailableValue(OrigHeader, OrigHeaderVal);
132 SSA.AddAvailableValue(OrigPreheader, OrigPreHeaderVal);
138 Instruction *UserInst = cast<Instruction>(U.getUser());
139 if (!isa<PHINode>(UserInst)) {
144 if (UserBB == OrigHeader)
149 if (UserBB == OrigPreheader) {
150 U = OrigPreHeaderVal;
168 if (UserBB == OrigHeader)
176 if (UserBB == OrigPreheader)
177 NewVal = OrigPreHeaderVal;
178 else if (
SSA.HasValueForBlock(UserBB))
179 NewVal =
SSA.GetValueInMiddleOfBlock(UserBB);
182 DbgValue->replaceVariableLocationOp(OrigHeaderVal, NewVal);
187 for (
DPValue *DPV : DPValues) {
191 if (UserBB == OrigHeader)
199 if (UserBB == OrigPreheader)
200 NewVal = OrigPreHeaderVal;
201 else if (
SSA.HasValueForBlock(UserBB))
202 NewVal =
SSA.GetValueInMiddleOfBlock(UserBB);
205 DPV->replaceVariableLocationOp(OrigHeaderVal, NewVal);
215 BranchInst *BI = dyn_cast<BranchInst>(Header->getTerminator());
218 if (L->contains(HeaderExit))
221 for (
auto &Phi : Header->phis()) {
224 return cast<Instruction>(U)->getParent() != HeaderExit;
240 assert(Latch &&
"need latch");
247 if (L->contains(Exit))
255 L->getUniqueExitBlocks(Exits);
256 if (!Exits.
empty()) {
277 bool HasConditionalPreHeader,
280 if (WeightMD ==
nullptr)
291 if (Weights.
size() != 2)
293 uint32_t OrigLoopExitWeight = Weights[0];
294 uint32_t OrigLoopBackedgeWeight = Weights[1];
297 std::swap(OrigLoopExitWeight, OrigLoopBackedgeWeight);
326 if (OrigLoopExitWeight > 0 && OrigLoopBackedgeWeight > 0) {
328 if (HasConditionalPreHeader) {
330 if (OrigLoopBackedgeWeight >= OrigLoopExitWeight) {
339 if ((OrigLoopBackedgeWeight & HighBit) != 0 ||
340 (OrigLoopExitWeight & HighBit) != 0)
342 OrigLoopBackedgeWeight <<= 1;
343 OrigLoopExitWeight <<= 1;
348 ExitWeight0 = OrigLoopExitWeight - OrigLoopBackedgeWeight;
351 ExitWeight1 = OrigLoopExitWeight - ExitWeight0;
352 EnterWeight = ExitWeight1;
353 LoopBackWeight = OrigLoopBackedgeWeight - EnterWeight;
354 }
else if (OrigLoopExitWeight == 0) {
355 if (OrigLoopBackedgeWeight == 0) {
368 LoopBackWeight = OrigLoopBackedgeWeight;
372 assert(OrigLoopBackedgeWeight == 0 &&
"remaining case is backedge zero");
380 SuccsSwapped ? LoopBackWeight : ExitWeight1,
381 SuccsSwapped ? ExitWeight1 : LoopBackWeight,
384 if (HasConditionalPreHeader) {
385 const uint32_t PreHeaderBIWeights[] = {
386 SuccsSwapped ? EnterWeight : ExitWeight0,
387 SuccsSwapped ? ExitWeight0 : EnterWeight,
406bool LoopRotate::rotateLoop(
Loop *L,
bool SimplifiedLatch) {
408 if (
L->getBlocks().size() == 1)
411 bool Rotated =
false;
423 if (!
L->isLoopExiting(OrigHeader))
433 if (
L->isLoopExiting(OrigLatch) && !SimplifiedLatch && IsUtilMode ==
false &&
445 Metrics.analyzeBasicBlock(OrigHeader, *
TTI, EphValues, PrepareForLTO);
448 dbgs() <<
"LoopRotation: NOT rotating - contains non-duplicatable"
449 <<
" instructions: ";
454 LLVM_DEBUG(
dbgs() <<
"LoopRotation: NOT rotating - contains convergent "
459 if (!
Metrics.NumInsts.isValid()) {
460 LLVM_DEBUG(
dbgs() <<
"LoopRotation: NOT rotating - contains instructions"
461 " with invalid cost: ";
465 if (
Metrics.NumInsts > MaxHeaderSize) {
468 <<
" instructions, which is more than the threshold ("
469 << MaxHeaderSize <<
" instructions): ";
471 ++NumNotRotatedDueToHeaderSize;
477 if (PrepareForLTO &&
Metrics.NumInlineCandidates > 0)
486 if (!OrigPreheader || !
L->hasDedicatedExits())
495 SE->forgetTopmostLoop(L);
500 SE->forgetBlockAndLoopDispositions();
505 MSSAU->getMemorySSA()->verifyMemorySSA();
512 bool BISuccsSwapped =
L->contains(Exit);
515 assert(NewHeader &&
"Unable to determine new loop header");
516 assert(
L->contains(NewHeader) && !
L->contains(Exit) &&
517 "Unable to determine loop header and exit blocks");
522 "New header doesn't have one pred!");
532 for (;
PHINode *PN = dyn_cast<PHINode>(
I); ++
I)
542 using DbgIntrinsicHash =
543 std::pair<std::pair<hash_code, DILocalVariable *>,
DIExpression *>;
545 auto VarLocOps =
D->location_ops();
552 if (
auto *DII = dyn_cast<DbgVariableIntrinsic>(&
I))
553 DbgIntrinsics.
insert(makeHash(DII));
560 auto makeHashDPV = [](
const DPValue &
D) -> DbgIntrinsicHash {
561 auto VarLocOps =
D.location_ops();
567 for (
const DPValue &DPV :
I.getDbgValueRange())
568 DbgIntrinsics.
insert(makeHashDPV(DPV));
575 if (
auto *Decl = dyn_cast<NoAliasScopeDeclInst>(&
I))
578 Module *
M = OrigHeader->getModule();
599 std::optional<DPValue::self_iterator> NextDbgInst = std::nullopt;
612 !isa<DbgInfoIntrinsic>(Inst) && !isa<AllocaInst>(Inst)) {
621 NextDbgInst =
I->getDbgValueRange().begin();
630 C->insertBefore(LoopEntryBranch);
632 ++NumInstrsDuplicated;
635 auto Range =
C->cloneDebugInfoFrom(Inst, NextDbgInst);
638 if (DbgIntrinsics.
count(makeHashDPV(DPV)))
639 DPV.eraseFromParent();
642 NextDbgInst = std::nullopt;
650 if (
auto *DII = dyn_cast<DbgVariableIntrinsic>(
C))
651 if (DbgIntrinsics.
count(makeHash(DII))) {
652 C->eraseFromParent();
660 if (V && LI->replacementPreservesLCSSAForm(
C, V)) {
664 if (!
C->mayHaveSideEffects()) {
665 C->eraseFromParent();
675 if (
auto *II = dyn_cast<AssumeInst>(
C))
676 AC->registerAssumption(II);
684 if (!NoAliasDeclInstructions.
empty()) {
709 LLVM_DEBUG(
dbgs() <<
" Cloning llvm.experimental.noalias.scope.decl:"
722 NoAliasDeclScopes.
push_back(NAD->getScopeList());
736 cast<Instruction>(
ValueMap[*NoAliasDeclInstructions.begin()]);
737 auto *LastInst = &OrigPreheader->
back();
752 PHINode *PN = dyn_cast<PHINode>(BI); ++BI)
765 MSSAU->updateForClonedBlockIntoPred(OrigHeader, OrigPreheader,
778 if (!InsertedPHIs.
empty())
782 L->moveToHeader(NewHeader);
783 assert(
L->getHeader() == NewHeader &&
"Latch block is our new header");
790 Updates.
push_back({DominatorTree::Insert, OrigPreheader, Exit});
791 Updates.
push_back({DominatorTree::Insert, OrigPreheader, NewHeader});
792 Updates.
push_back({DominatorTree::Delete, OrigPreheader, OrigHeader});
795 MSSAU->applyUpdates(Updates, *DT,
true);
797 MSSAU->getMemorySSA()->verifyMemorySSA();
799 DT->applyUpdates(Updates);
813 const bool HasConditionalPreHeader =
814 !isa<ConstantInt>(
Cond) ||
819 if (HasConditionalPreHeader) {
827 OrigPreheader, NewHeader,
836 bool SplitLatchEdge =
false;
839 Loop *PredLoop = LI->getLoopFor(ExitPred);
840 if (!PredLoop || PredLoop->
contains(Exit) ||
841 isa<IndirectBrInst>(ExitPred->getTerminator()))
843 SplitLatchEdge |=
L->getLoopLatch() == ExitPred;
850 "Despite splitting all preds, failed to split latch exit?");
851 (void)SplitLatchEdge;
861 if (DT) DT->deleteEdge(OrigPreheader, Exit);
865 MSSAU->removeEdge(OrigPreheader, Exit);
868 assert(
L->getLoopPreheader() &&
"Invalid loop preheader after loop rotation");
869 assert(
L->getLoopLatch() &&
"Invalid loop latch after loop rotation");
872 MSSAU->getMemorySSA()->verifyMemorySSA();
885 MSSAU->getMemorySSA()->verifyMemorySSA();
892 SimplifiedLatch =
false;
910 bool seenIncrement =
false;
911 bool MultiExitLoop =
false;
913 if (!L->getExitingBlock())
914 MultiExitLoop =
true;
921 if (isa<DbgInfoIntrinsic>(
I))
924 switch (
I->getOpcode()) {
927 case Instruction::GetElementPtr:
929 if (!cast<GEPOperator>(
I)->hasAllConstantIndices())
933 case Instruction::Add:
934 case Instruction::Sub:
935 case Instruction::And:
936 case Instruction::Or:
937 case Instruction::Xor:
938 case Instruction::Shl:
939 case Instruction::LShr:
940 case Instruction::AShr: {
942 !isa<Constant>(
I->getOperand(0))
944 : !isa<Constant>(
I->getOperand(1)) ?
I->getOperand(1) :
nullptr;
952 auto *UserInst = cast<Instruction>(UseI);
953 if (!L->contains(UserInst))
960 seenIncrement =
true;
963 case Instruction::Trunc:
964 case Instruction::ZExt:
965 case Instruction::SExt:
981bool LoopRotate::simplifyLoopLatch(
Loop *L) {
991 if (!LastExit || !
L->isLoopExiting(LastExit))
1002 << LastExit->
getName() <<
"\n");
1010 SE->forgetBlockAndLoopDispositions();
1014 MSSAU->getMemorySSA()->verifyMemorySSA();
1020bool LoopRotate::processLoop(
Loop *L) {
1022 MDNode *LoopMD =
L->getLoopID();
1024 bool SimplifiedLatch =
false;
1030 SimplifiedLatch = simplifyLoopLatch(L);
1032 bool MadeChange = rotateLoop(L, SimplifiedLatch);
1033 assert((!MadeChange ||
L->isLoopExiting(
L->getLoopLatch())) &&
1034 "Loop latch should be exiting after loop-rotate.");
1038 if ((MadeChange || SimplifiedLatch) && LoopMD)
1039 L->setLoopID(LoopMD);
1041 return MadeChange || SimplifiedLatch;
1050 unsigned Threshold =
unsigned(-1),
1051 bool IsUtilMode =
true,
bool PrepareForLTO) {
1052 LoopRotate LR(Threshold, LI,
TTI, AC, DT, SE, MSSAU, SQ, RotationOnly,
1053 IsUtilMode, PrepareForLTO);
1054 return LR.processLoop(L);
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
static constexpr uint32_t ZeroTripCountWeights[]
static bool canRotateDeoptimizingLatchExit(Loop *L)
static bool shouldSpeculateInstrs(BasicBlock::iterator Begin, BasicBlock::iterator End, Loop *L)
Determine whether the instructions in this range may be safely and cheaply speculated.
static cl::opt< bool > MultiRotate("loop-rotate-multi", cl::init(false), cl::Hidden, cl::desc("Allow loop rotation multiple times in order to reach " "a better latch exit"))
static bool profitableToRotateLoopExitingLatch(Loop *L)
static void updateBranchWeights(BranchInst &PreHeaderBI, BranchInst &LoopBI, bool HasConditionalPreHeader, bool SuccsSwapped)
static void InsertNewValueIntoMap(ValueToValueMapTy &VM, Value *K, Value *V)
Insert (K, V) pair into the ValueToValueMap, and verify the key did not previously exist in the map,...
static void RewriteUsesOfClonedInstructions(BasicBlock *OrigHeader, BasicBlock *OrigPreheader, ValueToValueMapTy &ValueMap, ScalarEvolution *SE, SmallVectorImpl< PHINode * > *InsertedPHIs)
RewriteUsesOfClonedInstructions - We just cloned the instructions from the old header into the prehea...
This file exposes an interface to building/using memory SSA to walk memory instructions using a use/d...
This file contains the declarations for profiling metadata utility functions.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Class recording the (high level) value of a variable.
A cache of @llvm.assume calls within a function.
LLVM Basic Block Representation.
DPMarker * getMarker(InstListType::iterator It)
Return the DPMarker for the position given by It, so that DPValues can be inserted there.
iterator begin()
Instruction iterator methods.
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches,...
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const BasicBlock * getUniquePredecessor() const
Return the predecessor of this block if it has a unique predecessor block.
InstListType::iterator iterator
Instruction iterators...
LLVMContext & getContext() const
Get the context in which this basic block lives.
bool IsNewDbgInfoFormat
Flag recording whether or not this block stores debug-info in the form of intrinsic instructions (fal...
void moveBefore(BasicBlock *MovePos)
Unlink this basic block from its current function and insert it into the function that MovePos lives ...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
const CallInst * getPostdominatingDeoptimizeCall() const
Returns the call instruction calling @llvm.experimental.deoptimize that is present either in current ...
void flushTerminatorDbgValues()
Eject any debug-info trailing at the end of a block.
const Instruction & back() const
void removePredecessor(BasicBlock *Pred, bool KeepOneInputPHIs=false)
Update PHI nodes in this BasicBlock before removal of predecessor Pred.
Conditional or Unconditional Branch instruction.
bool isConditional() const
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
BasicBlock * getSuccessor(unsigned i) const
bool isUnconditional() const
Value * getCondition() const
const BasicBlock * getParent() const
Record of a variable value-assignment, aka a non instruction representation of the dbg....
This is the common base class for debug info intrinsics for variables.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
bool mayWriteToMemory() const LLVM_READONLY
Return true if this instruction may modify memory.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const BasicBlock * getParent() const
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
bool isTerminator() const
bool mayReadFromMemory() const LLVM_READONLY
Return true if this instruction may read memory.
iterator_range< simple_ilist< DPValue >::iterator > cloneDebugInfoFrom(const Instruction *From, std::optional< simple_ilist< DPValue >::iterator > FromHere=std::nullopt, bool InsertAtHead=false)
Clone any debug-info attached to From onto this instruction.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
Represents a single loop in the control flow graph.
A Module instance is used to store all the information related to an LLVM module.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
Value * getIncomingValueForBlock(const BasicBlock *BB) const
Helper class for SSA formation on a set of values defined in multiple blocks.
The main scalar evolution driver.
void forgetValue(Value *V)
This method should be called by the client when it has changed a value in a way that may effect its v...
Implements a dense probed hash-table based set with some number of buckets stored inline.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
ValueT lookup(const KeyT &Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
iterator_range< user_iterator > users()
iterator_range< use_iterator > uses()
StringRef getName() const
Return a constant reference to the value's name.
void dump() const
Support for debugging, callable in GDB: V->dump()
std::pair< iterator, bool > insert(const ValueT &V)
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
self_iterator getIterator()
@ C
The default llvm calling convention, compatible with C.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
bool RemoveRedundantDbgInstrs(BasicBlock *BB)
Try to remove redundant dbg.value instructions from given basic block.
auto successors(const MachineBasicBlock *BB)
MDNode * getBranchWeightMDNode(const Instruction &I)
Get the branch weights metadata node.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
void insertDebugValuesForPHIs(BasicBlock *BB, SmallVectorImpl< PHINode * > &InsertedPHIs)
Propagate dbg.value intrinsics through the newly inserted PHIs.
void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
Value * simplifyInstruction(Instruction *I, const SimplifyQuery &Q)
See if we can compute a simplified version of this instruction.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
@ RF_IgnoreMissingLocals
If this flag is set, the remapper ignores missing function-local entries (Argument,...
@ RF_NoModuleLevelChanges
If this flag is set, the remapper knows that only local values within a function (such as an instruct...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void RemapInstruction(Instruction *I, ValueToValueMapTy &VM, RemapFlags Flags=RF_None, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Convert the instruction operands from referencing the current values into those specified by VM.
void RemapDPValueRange(Module *M, iterator_range< DPValueIterator > Range, ValueToValueMapTy &VM, RemapFlags Flags=RF_None, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Remap the Values used in the DPValue V using the value map VM.
bool VerifyMemorySSA
Enables verification of MemorySSA.
bool MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, MemoryDependenceResults *MemDep=nullptr, bool PredecessorWithTwoSuccessors=false, DominatorTree *DT=nullptr)
Attempts to merge a block into its predecessor, if possible.
void findDbgValues(SmallVectorImpl< DbgValueInst * > &DbgValues, Value *V, SmallVectorImpl< DPValue * > *DPValues=nullptr)
Finds the llvm.dbg.value intrinsics describing a value.
BasicBlock * SplitCriticalEdge(Instruction *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions(), const Twine &BBName="")
If this edge is a critical edge, insert a new node to split the critical edge.
void cloneAndAdaptNoAliasScopes(ArrayRef< MDNode * > NoAliasDeclScopes, ArrayRef< BasicBlock * > NewBlocks, LLVMContext &Context, StringRef Ext)
Clone the specified noalias decl scopes.
bool FoldSingleEntryPHINodes(BasicBlock *BB, MemoryDependenceResults *MemDep=nullptr)
We know that BB has one predecessor.
bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Return true if the instruction does not have any effects besides calculating the result and does not ...
auto predecessors(const MachineBasicBlock *BB)
void extractFromBranchWeightMD(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)
Faster version of extractBranchWeights() that skips checks and must only be called with "branch_weigh...
bool LoopRotation(Loop *L, LoopInfo *LI, const TargetTransformInfo *TTI, AssumptionCache *AC, DominatorTree *DT, ScalarEvolution *SE, MemorySSAUpdater *MSSAU, const SimplifyQuery &SQ, bool RotationOnly, unsigned Threshold, bool IsUtilMode, bool PrepareForLTO=false)
Convert a loop into a loop with bottom test.
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Utility to calculate the size and a few similar metrics for a set of basic blocks.
static void collectEphemeralValues(const Loop *L, AssumptionCache *AC, SmallPtrSetImpl< const Value * > &EphValues)
Collect a loop's ephemeral values (those used only by an assume or similar intrinsics in the loop).
Option class for critical edge splitting.