LLVM 19.0.0git
LoopStrengthReduce.cpp
Go to the documentation of this file.
1//===- LoopStrengthReduce.cpp - Strength Reduce IVs in Loops --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This transformation analyzes and transforms the induction variables (and
10// computations derived from them) into forms suitable for efficient execution
11// on the target.
12//
13// This pass performs a strength reduction on array references inside loops that
14// have as one or more of their components the loop induction variable, it
15// rewrites expressions to take advantage of scaled-index addressing modes
16// available on the target, and it performs a variety of other optimizations
17// related to loop induction variables.
18//
19// Terminology note: this code has a lot of handling for "post-increment" or
20// "post-inc" users. This is not talking about post-increment addressing modes;
21// it is instead talking about code like this:
22//
23// %i = phi [ 0, %entry ], [ %i.next, %latch ]
24// ...
25// %i.next = add %i, 1
26// %c = icmp eq %i.next, %n
27//
28// The SCEV for %i is {0,+,1}<%L>. The SCEV for %i.next is {1,+,1}<%L>, however
29// it's useful to think about these as the same register, with some uses using
30// the value of the register before the add and some using it after. In this
31// example, the icmp is a post-increment user, since it uses %i.next, which is
32// the value of the induction variable after the increment. The other common
33// case of post-increment users is users outside the loop.
34//
35// TODO: More sophistication in the way Formulae are generated and filtered.
36//
37// TODO: Handle multiple loops at a time.
38//
39// TODO: Should the addressing mode BaseGV be changed to a ConstantExpr instead
40// of a GlobalValue?
41//
42// TODO: When truncation is free, truncate ICmp users' operands to make it a
43// smaller encoding (on x86 at least).
44//
45// TODO: When a negated register is used by an add (such as in a list of
46// multiple base registers, or as the increment expression in an addrec),
47// we may not actually need both reg and (-1 * reg) in registers; the
48// negation can be implemented by using a sub instead of an add. The
49// lack of support for taking this into consideration when making
50// register pressure decisions is partly worked around by the "Special"
51// use kind.
52//
53//===----------------------------------------------------------------------===//
54
56#include "llvm/ADT/APInt.h"
57#include "llvm/ADT/DenseMap.h"
58#include "llvm/ADT/DenseSet.h"
59#include "llvm/ADT/Hashing.h"
61#include "llvm/ADT/STLExtras.h"
62#include "llvm/ADT/SetVector.h"
65#include "llvm/ADT/SmallSet.h"
67#include "llvm/ADT/Statistic.h"
84#include "llvm/Config/llvm-config.h"
85#include "llvm/IR/BasicBlock.h"
86#include "llvm/IR/Constant.h"
87#include "llvm/IR/Constants.h"
90#include "llvm/IR/Dominators.h"
91#include "llvm/IR/GlobalValue.h"
92#include "llvm/IR/IRBuilder.h"
93#include "llvm/IR/InstrTypes.h"
94#include "llvm/IR/Instruction.h"
97#include "llvm/IR/Module.h"
98#include "llvm/IR/Operator.h"
99#include "llvm/IR/PassManager.h"
100#include "llvm/IR/Type.h"
101#include "llvm/IR/Use.h"
102#include "llvm/IR/User.h"
103#include "llvm/IR/Value.h"
104#include "llvm/IR/ValueHandle.h"
106#include "llvm/Pass.h"
107#include "llvm/Support/Casting.h"
110#include "llvm/Support/Debug.h"
120#include <algorithm>
121#include <cassert>
122#include <cstddef>
123#include <cstdint>
124#include <iterator>
125#include <limits>
126#include <map>
127#include <numeric>
128#include <optional>
129#include <utility>
130
131using namespace llvm;
132
133#define DEBUG_TYPE "loop-reduce"
134
135/// MaxIVUsers is an arbitrary threshold that provides an early opportunity for
136/// bail out. This threshold is far beyond the number of users that LSR can
137/// conceivably solve, so it should not affect generated code, but catches the
138/// worst cases before LSR burns too much compile time and stack space.
139static const unsigned MaxIVUsers = 200;
140
141/// Limit the size of expression that SCEV-based salvaging will attempt to
142/// translate into a DIExpression.
143/// Choose a maximum size such that debuginfo is not excessively increased and
144/// the salvaging is not too expensive for the compiler.
145static const unsigned MaxSCEVSalvageExpressionSize = 64;
146
147// Cleanup congruent phis after LSR phi expansion.
149 "enable-lsr-phielim", cl::Hidden, cl::init(true),
150 cl::desc("Enable LSR phi elimination"));
151
152// The flag adds instruction count to solutions cost comparison.
154 "lsr-insns-cost", cl::Hidden, cl::init(true),
155 cl::desc("Add instruction count to a LSR cost model"));
156
157// Flag to choose how to narrow complex lsr solution
159 "lsr-exp-narrow", cl::Hidden, cl::init(false),
160 cl::desc("Narrow LSR complex solution using"
161 " expectation of registers number"));
162
163// Flag to narrow search space by filtering non-optimal formulae with
164// the same ScaledReg and Scale.
166 "lsr-filter-same-scaled-reg", cl::Hidden, cl::init(true),
167 cl::desc("Narrow LSR search space by filtering non-optimal formulae"
168 " with the same ScaledReg and Scale"));
169
171 "lsr-preferred-addressing-mode", cl::Hidden, cl::init(TTI::AMK_None),
172 cl::desc("A flag that overrides the target's preferred addressing mode."),
174 "none",
175 "Don't prefer any addressing mode"),
177 "preindexed",
178 "Prefer pre-indexed addressing mode"),
180 "postindexed",
181 "Prefer post-indexed addressing mode")));
182
184 "lsr-complexity-limit", cl::Hidden,
185 cl::init(std::numeric_limits<uint16_t>::max()),
186 cl::desc("LSR search space complexity limit"));
187
189 "lsr-setupcost-depth-limit", cl::Hidden, cl::init(7),
190 cl::desc("The limit on recursion depth for LSRs setup cost"));
191
193 "lsr-term-fold", cl::Hidden,
194 cl::desc("Attempt to replace primary IV with other IV."));
195
197 "lsr-drop-solution", cl::Hidden, cl::init(false),
198 cl::desc("Attempt to drop solution if it is less profitable"));
199
200STATISTIC(NumTermFold,
201 "Number of terminating condition fold recognized and performed");
202
203#ifndef NDEBUG
204// Stress test IV chain generation.
206 "stress-ivchain", cl::Hidden, cl::init(false),
207 cl::desc("Stress test LSR IV chains"));
208#else
209static bool StressIVChain = false;
210#endif
211
212namespace {
213
214struct MemAccessTy {
215 /// Used in situations where the accessed memory type is unknown.
216 static const unsigned UnknownAddressSpace =
217 std::numeric_limits<unsigned>::max();
218
219 Type *MemTy = nullptr;
220 unsigned AddrSpace = UnknownAddressSpace;
221
222 MemAccessTy() = default;
223 MemAccessTy(Type *Ty, unsigned AS) : MemTy(Ty), AddrSpace(AS) {}
224
225 bool operator==(MemAccessTy Other) const {
226 return MemTy == Other.MemTy && AddrSpace == Other.AddrSpace;
227 }
228
229 bool operator!=(MemAccessTy Other) const { return !(*this == Other); }
230
231 static MemAccessTy getUnknown(LLVMContext &Ctx,
232 unsigned AS = UnknownAddressSpace) {
233 return MemAccessTy(Type::getVoidTy(Ctx), AS);
234 }
235
236 Type *getType() { return MemTy; }
237};
238
239/// This class holds data which is used to order reuse candidates.
240class RegSortData {
241public:
242 /// This represents the set of LSRUse indices which reference
243 /// a particular register.
244 SmallBitVector UsedByIndices;
245
246 void print(raw_ostream &OS) const;
247 void dump() const;
248};
249
250} // end anonymous namespace
251
252#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
253void RegSortData::print(raw_ostream &OS) const {
254 OS << "[NumUses=" << UsedByIndices.count() << ']';
255}
256
257LLVM_DUMP_METHOD void RegSortData::dump() const {
258 print(errs()); errs() << '\n';
259}
260#endif
261
262namespace {
263
264/// Map register candidates to information about how they are used.
265class RegUseTracker {
266 using RegUsesTy = DenseMap<const SCEV *, RegSortData>;
267
268 RegUsesTy RegUsesMap;
270
271public:
272 void countRegister(const SCEV *Reg, size_t LUIdx);
273 void dropRegister(const SCEV *Reg, size_t LUIdx);
274 void swapAndDropUse(size_t LUIdx, size_t LastLUIdx);
275
276 bool isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const;
277
278 const SmallBitVector &getUsedByIndices(const SCEV *Reg) const;
279
280 void clear();
281
284
285 iterator begin() { return RegSequence.begin(); }
286 iterator end() { return RegSequence.end(); }
287 const_iterator begin() const { return RegSequence.begin(); }
288 const_iterator end() const { return RegSequence.end(); }
289};
290
291} // end anonymous namespace
292
293void
294RegUseTracker::countRegister(const SCEV *Reg, size_t LUIdx) {
295 std::pair<RegUsesTy::iterator, bool> Pair =
296 RegUsesMap.insert(std::make_pair(Reg, RegSortData()));
297 RegSortData &RSD = Pair.first->second;
298 if (Pair.second)
299 RegSequence.push_back(Reg);
300 RSD.UsedByIndices.resize(std::max(RSD.UsedByIndices.size(), LUIdx + 1));
301 RSD.UsedByIndices.set(LUIdx);
302}
303
304void
305RegUseTracker::dropRegister(const SCEV *Reg, size_t LUIdx) {
306 RegUsesTy::iterator It = RegUsesMap.find(Reg);
307 assert(It != RegUsesMap.end());
308 RegSortData &RSD = It->second;
309 assert(RSD.UsedByIndices.size() > LUIdx);
310 RSD.UsedByIndices.reset(LUIdx);
311}
312
313void
314RegUseTracker::swapAndDropUse(size_t LUIdx, size_t LastLUIdx) {
315 assert(LUIdx <= LastLUIdx);
316
317 // Update RegUses. The data structure is not optimized for this purpose;
318 // we must iterate through it and update each of the bit vectors.
319 for (auto &Pair : RegUsesMap) {
320 SmallBitVector &UsedByIndices = Pair.second.UsedByIndices;
321 if (LUIdx < UsedByIndices.size())
322 UsedByIndices[LUIdx] =
323 LastLUIdx < UsedByIndices.size() ? UsedByIndices[LastLUIdx] : false;
324 UsedByIndices.resize(std::min(UsedByIndices.size(), LastLUIdx));
325 }
326}
327
328bool
329RegUseTracker::isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const {
330 RegUsesTy::const_iterator I = RegUsesMap.find(Reg);
331 if (I == RegUsesMap.end())
332 return false;
333 const SmallBitVector &UsedByIndices = I->second.UsedByIndices;
334 int i = UsedByIndices.find_first();
335 if (i == -1) return false;
336 if ((size_t)i != LUIdx) return true;
337 return UsedByIndices.find_next(i) != -1;
338}
339
340const SmallBitVector &RegUseTracker::getUsedByIndices(const SCEV *Reg) const {
341 RegUsesTy::const_iterator I = RegUsesMap.find(Reg);
342 assert(I != RegUsesMap.end() && "Unknown register!");
343 return I->second.UsedByIndices;
344}
345
346void RegUseTracker::clear() {
347 RegUsesMap.clear();
348 RegSequence.clear();
349}
350
351namespace {
352
353/// This class holds information that describes a formula for computing
354/// satisfying a use. It may include broken-out immediates and scaled registers.
355struct Formula {
356 /// Global base address used for complex addressing.
357 GlobalValue *BaseGV = nullptr;
358
359 /// Base offset for complex addressing.
360 int64_t BaseOffset = 0;
361
362 /// Whether any complex addressing has a base register.
363 bool HasBaseReg = false;
364
365 /// The scale of any complex addressing.
366 int64_t Scale = 0;
367
368 /// The list of "base" registers for this use. When this is non-empty. The
369 /// canonical representation of a formula is
370 /// 1. BaseRegs.size > 1 implies ScaledReg != NULL and
371 /// 2. ScaledReg != NULL implies Scale != 1 || !BaseRegs.empty().
372 /// 3. The reg containing recurrent expr related with currect loop in the
373 /// formula should be put in the ScaledReg.
374 /// #1 enforces that the scaled register is always used when at least two
375 /// registers are needed by the formula: e.g., reg1 + reg2 is reg1 + 1 * reg2.
376 /// #2 enforces that 1 * reg is reg.
377 /// #3 ensures invariant regs with respect to current loop can be combined
378 /// together in LSR codegen.
379 /// This invariant can be temporarily broken while building a formula.
380 /// However, every formula inserted into the LSRInstance must be in canonical
381 /// form.
383
384 /// The 'scaled' register for this use. This should be non-null when Scale is
385 /// not zero.
386 const SCEV *ScaledReg = nullptr;
387
388 /// An additional constant offset which added near the use. This requires a
389 /// temporary register, but the offset itself can live in an add immediate
390 /// field rather than a register.
391 int64_t UnfoldedOffset = 0;
392
393 Formula() = default;
394
395 void initialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE);
396
397 bool isCanonical(const Loop &L) const;
398
399 void canonicalize(const Loop &L);
400
401 bool unscale();
402
403 bool hasZeroEnd() const;
404
405 size_t getNumRegs() const;
406 Type *getType() const;
407
408 void deleteBaseReg(const SCEV *&S);
409
410 bool referencesReg(const SCEV *S) const;
411 bool hasRegsUsedByUsesOtherThan(size_t LUIdx,
412 const RegUseTracker &RegUses) const;
413
414 void print(raw_ostream &OS) const;
415 void dump() const;
416};
417
418} // end anonymous namespace
419
420/// Recursion helper for initialMatch.
421static void DoInitialMatch(const SCEV *S, Loop *L,
424 ScalarEvolution &SE) {
425 // Collect expressions which properly dominate the loop header.
426 if (SE.properlyDominates(S, L->getHeader())) {
427 Good.push_back(S);
428 return;
429 }
430
431 // Look at add operands.
432 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
433 for (const SCEV *S : Add->operands())
434 DoInitialMatch(S, L, Good, Bad, SE);
435 return;
436 }
437
438 // Look at addrec operands.
439 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S))
440 if (!AR->getStart()->isZero() && AR->isAffine()) {
441 DoInitialMatch(AR->getStart(), L, Good, Bad, SE);
442 DoInitialMatch(SE.getAddRecExpr(SE.getConstant(AR->getType(), 0),
443 AR->getStepRecurrence(SE),
444 // FIXME: AR->getNoWrapFlags()
445 AR->getLoop(), SCEV::FlagAnyWrap),
446 L, Good, Bad, SE);
447 return;
448 }
449
450 // Handle a multiplication by -1 (negation) if it didn't fold.
451 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S))
452 if (Mul->getOperand(0)->isAllOnesValue()) {
454 const SCEV *NewMul = SE.getMulExpr(Ops);
455
458 DoInitialMatch(NewMul, L, MyGood, MyBad, SE);
459 const SCEV *NegOne = SE.getSCEV(ConstantInt::getAllOnesValue(
460 SE.getEffectiveSCEVType(NewMul->getType())));
461 for (const SCEV *S : MyGood)
462 Good.push_back(SE.getMulExpr(NegOne, S));
463 for (const SCEV *S : MyBad)
464 Bad.push_back(SE.getMulExpr(NegOne, S));
465 return;
466 }
467
468 // Ok, we can't do anything interesting. Just stuff the whole thing into a
469 // register and hope for the best.
470 Bad.push_back(S);
471}
472
473/// Incorporate loop-variant parts of S into this Formula, attempting to keep
474/// all loop-invariant and loop-computable values in a single base register.
475void Formula::initialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE) {
478 DoInitialMatch(S, L, Good, Bad, SE);
479 if (!Good.empty()) {
480 const SCEV *Sum = SE.getAddExpr(Good);
481 if (!Sum->isZero())
482 BaseRegs.push_back(Sum);
483 HasBaseReg = true;
484 }
485 if (!Bad.empty()) {
486 const SCEV *Sum = SE.getAddExpr(Bad);
487 if (!Sum->isZero())
488 BaseRegs.push_back(Sum);
489 HasBaseReg = true;
490 }
491 canonicalize(*L);
492}
493
494static bool containsAddRecDependentOnLoop(const SCEV *S, const Loop &L) {
495 return SCEVExprContains(S, [&L](const SCEV *S) {
496 return isa<SCEVAddRecExpr>(S) && (cast<SCEVAddRecExpr>(S)->getLoop() == &L);
497 });
498}
499
500/// Check whether or not this formula satisfies the canonical
501/// representation.
502/// \see Formula::BaseRegs.
503bool Formula::isCanonical(const Loop &L) const {
504 if (!ScaledReg)
505 return BaseRegs.size() <= 1;
506
507 if (Scale != 1)
508 return true;
509
510 if (Scale == 1 && BaseRegs.empty())
511 return false;
512
513 if (containsAddRecDependentOnLoop(ScaledReg, L))
514 return true;
515
516 // If ScaledReg is not a recurrent expr, or it is but its loop is not current
517 // loop, meanwhile BaseRegs contains a recurrent expr reg related with current
518 // loop, we want to swap the reg in BaseRegs with ScaledReg.
519 return none_of(BaseRegs, [&L](const SCEV *S) {
521 });
522}
523
524/// Helper method to morph a formula into its canonical representation.
525/// \see Formula::BaseRegs.
526/// Every formula having more than one base register, must use the ScaledReg
527/// field. Otherwise, we would have to do special cases everywhere in LSR
528/// to treat reg1 + reg2 + ... the same way as reg1 + 1*reg2 + ...
529/// On the other hand, 1*reg should be canonicalized into reg.
530void Formula::canonicalize(const Loop &L) {
531 if (isCanonical(L))
532 return;
533
534 if (BaseRegs.empty()) {
535 // No base reg? Use scale reg with scale = 1 as such.
536 assert(ScaledReg && "Expected 1*reg => reg");
537 assert(Scale == 1 && "Expected 1*reg => reg");
538 BaseRegs.push_back(ScaledReg);
539 Scale = 0;
540 ScaledReg = nullptr;
541 return;
542 }
543
544 // Keep the invariant sum in BaseRegs and one of the variant sum in ScaledReg.
545 if (!ScaledReg) {
546 ScaledReg = BaseRegs.pop_back_val();
547 Scale = 1;
548 }
549
550 // If ScaledReg is an invariant with respect to L, find the reg from
551 // BaseRegs containing the recurrent expr related with Loop L. Swap the
552 // reg with ScaledReg.
553 if (!containsAddRecDependentOnLoop(ScaledReg, L)) {
554 auto I = find_if(BaseRegs, [&L](const SCEV *S) {
556 });
557 if (I != BaseRegs.end())
558 std::swap(ScaledReg, *I);
559 }
560 assert(isCanonical(L) && "Failed to canonicalize?");
561}
562
563/// Get rid of the scale in the formula.
564/// In other words, this method morphes reg1 + 1*reg2 into reg1 + reg2.
565/// \return true if it was possible to get rid of the scale, false otherwise.
566/// \note After this operation the formula may not be in the canonical form.
567bool Formula::unscale() {
568 if (Scale != 1)
569 return false;
570 Scale = 0;
571 BaseRegs.push_back(ScaledReg);
572 ScaledReg = nullptr;
573 return true;
574}
575
576bool Formula::hasZeroEnd() const {
577 if (UnfoldedOffset || BaseOffset)
578 return false;
579 if (BaseRegs.size() != 1 || ScaledReg)
580 return false;
581 return true;
582}
583
584/// Return the total number of register operands used by this formula. This does
585/// not include register uses implied by non-constant addrec strides.
586size_t Formula::getNumRegs() const {
587 return !!ScaledReg + BaseRegs.size();
588}
589
590/// Return the type of this formula, if it has one, or null otherwise. This type
591/// is meaningless except for the bit size.
592Type *Formula::getType() const {
593 return !BaseRegs.empty() ? BaseRegs.front()->getType() :
594 ScaledReg ? ScaledReg->getType() :
595 BaseGV ? BaseGV->getType() :
596 nullptr;
597}
598
599/// Delete the given base reg from the BaseRegs list.
600void Formula::deleteBaseReg(const SCEV *&S) {
601 if (&S != &BaseRegs.back())
602 std::swap(S, BaseRegs.back());
603 BaseRegs.pop_back();
604}
605
606/// Test if this formula references the given register.
607bool Formula::referencesReg(const SCEV *S) const {
608 return S == ScaledReg || is_contained(BaseRegs, S);
609}
610
611/// Test whether this formula uses registers which are used by uses other than
612/// the use with the given index.
613bool Formula::hasRegsUsedByUsesOtherThan(size_t LUIdx,
614 const RegUseTracker &RegUses) const {
615 if (ScaledReg)
616 if (RegUses.isRegUsedByUsesOtherThan(ScaledReg, LUIdx))
617 return true;
618 for (const SCEV *BaseReg : BaseRegs)
619 if (RegUses.isRegUsedByUsesOtherThan(BaseReg, LUIdx))
620 return true;
621 return false;
622}
623
624#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
625void Formula::print(raw_ostream &OS) const {
626 bool First = true;
627 if (BaseGV) {
628 if (!First) OS << " + "; else First = false;
629 BaseGV->printAsOperand(OS, /*PrintType=*/false);
630 }
631 if (BaseOffset != 0) {
632 if (!First) OS << " + "; else First = false;
633 OS << BaseOffset;
634 }
635 for (const SCEV *BaseReg : BaseRegs) {
636 if (!First) OS << " + "; else First = false;
637 OS << "reg(" << *BaseReg << ')';
638 }
639 if (HasBaseReg && BaseRegs.empty()) {
640 if (!First) OS << " + "; else First = false;
641 OS << "**error: HasBaseReg**";
642 } else if (!HasBaseReg && !BaseRegs.empty()) {
643 if (!First) OS << " + "; else First = false;
644 OS << "**error: !HasBaseReg**";
645 }
646 if (Scale != 0) {
647 if (!First) OS << " + "; else First = false;
648 OS << Scale << "*reg(";
649 if (ScaledReg)
650 OS << *ScaledReg;
651 else
652 OS << "<unknown>";
653 OS << ')';
654 }
655 if (UnfoldedOffset != 0) {
656 if (!First) OS << " + ";
657 OS << "imm(" << UnfoldedOffset << ')';
658 }
659}
660
661LLVM_DUMP_METHOD void Formula::dump() const {
662 print(errs()); errs() << '\n';
663}
664#endif
665
666/// Return true if the given addrec can be sign-extended without changing its
667/// value.
669 Type *WideTy =
671 return isa<SCEVAddRecExpr>(SE.getSignExtendExpr(AR, WideTy));
672}
673
674/// Return true if the given add can be sign-extended without changing its
675/// value.
676static bool isAddSExtable(const SCEVAddExpr *A, ScalarEvolution &SE) {
677 Type *WideTy =
678 IntegerType::get(SE.getContext(), SE.getTypeSizeInBits(A->getType()) + 1);
679 return isa<SCEVAddExpr>(SE.getSignExtendExpr(A, WideTy));
680}
681
682/// Return true if the given mul can be sign-extended without changing its
683/// value.
684static bool isMulSExtable(const SCEVMulExpr *M, ScalarEvolution &SE) {
685 Type *WideTy =
687 SE.getTypeSizeInBits(M->getType()) * M->getNumOperands());
688 return isa<SCEVMulExpr>(SE.getSignExtendExpr(M, WideTy));
689}
690
691/// Return an expression for LHS /s RHS, if it can be determined and if the
692/// remainder is known to be zero, or null otherwise. If IgnoreSignificantBits
693/// is true, expressions like (X * Y) /s Y are simplified to X, ignoring that
694/// the multiplication may overflow, which is useful when the result will be
695/// used in a context where the most significant bits are ignored.
696static const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS,
697 ScalarEvolution &SE,
698 bool IgnoreSignificantBits = false) {
699 // Handle the trivial case, which works for any SCEV type.
700 if (LHS == RHS)
701 return SE.getConstant(LHS->getType(), 1);
702
703 // Handle a few RHS special cases.
704 const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS);
705 if (RC) {
706 const APInt &RA = RC->getAPInt();
707 // Handle x /s -1 as x * -1, to give ScalarEvolution a chance to do
708 // some folding.
709 if (RA.isAllOnes()) {
710 if (LHS->getType()->isPointerTy())
711 return nullptr;
712 return SE.getMulExpr(LHS, RC);
713 }
714 // Handle x /s 1 as x.
715 if (RA == 1)
716 return LHS;
717 }
718
719 // Check for a division of a constant by a constant.
720 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(LHS)) {
721 if (!RC)
722 return nullptr;
723 const APInt &LA = C->getAPInt();
724 const APInt &RA = RC->getAPInt();
725 if (LA.srem(RA) != 0)
726 return nullptr;
727 return SE.getConstant(LA.sdiv(RA));
728 }
729
730 // Distribute the sdiv over addrec operands, if the addrec doesn't overflow.
731 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS)) {
732 if ((IgnoreSignificantBits || isAddRecSExtable(AR, SE)) && AR->isAffine()) {
733 const SCEV *Step = getExactSDiv(AR->getStepRecurrence(SE), RHS, SE,
734 IgnoreSignificantBits);
735 if (!Step) return nullptr;
736 const SCEV *Start = getExactSDiv(AR->getStart(), RHS, SE,
737 IgnoreSignificantBits);
738 if (!Start) return nullptr;
739 // FlagNW is independent of the start value, step direction, and is
740 // preserved with smaller magnitude steps.
741 // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
742 return SE.getAddRecExpr(Start, Step, AR->getLoop(), SCEV::FlagAnyWrap);
743 }
744 return nullptr;
745 }
746
747 // Distribute the sdiv over add operands, if the add doesn't overflow.
748 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(LHS)) {
749 if (IgnoreSignificantBits || isAddSExtable(Add, SE)) {
751 for (const SCEV *S : Add->operands()) {
752 const SCEV *Op = getExactSDiv(S, RHS, SE, IgnoreSignificantBits);
753 if (!Op) return nullptr;
754 Ops.push_back(Op);
755 }
756 return SE.getAddExpr(Ops);
757 }
758 return nullptr;
759 }
760
761 // Check for a multiply operand that we can pull RHS out of.
762 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(LHS)) {
763 if (IgnoreSignificantBits || isMulSExtable(Mul, SE)) {
764 // Handle special case C1*X*Y /s C2*X*Y.
765 if (const SCEVMulExpr *MulRHS = dyn_cast<SCEVMulExpr>(RHS)) {
766 if (IgnoreSignificantBits || isMulSExtable(MulRHS, SE)) {
767 const SCEVConstant *LC = dyn_cast<SCEVConstant>(Mul->getOperand(0));
768 const SCEVConstant *RC =
769 dyn_cast<SCEVConstant>(MulRHS->getOperand(0));
770 if (LC && RC) {
772 SmallVector<const SCEV *, 4> ROps(drop_begin(MulRHS->operands()));
773 if (LOps == ROps)
774 return getExactSDiv(LC, RC, SE, IgnoreSignificantBits);
775 }
776 }
777 }
778
780 bool Found = false;
781 for (const SCEV *S : Mul->operands()) {
782 if (!Found)
783 if (const SCEV *Q = getExactSDiv(S, RHS, SE,
784 IgnoreSignificantBits)) {
785 S = Q;
786 Found = true;
787 }
788 Ops.push_back(S);
789 }
790 return Found ? SE.getMulExpr(Ops) : nullptr;
791 }
792 return nullptr;
793 }
794
795 // Otherwise we don't know.
796 return nullptr;
797}
798
799/// If S involves the addition of a constant integer value, return that integer
800/// value, and mutate S to point to a new SCEV with that value excluded.
801static int64_t ExtractImmediate(const SCEV *&S, ScalarEvolution &SE) {
802 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) {
803 if (C->getAPInt().getSignificantBits() <= 64) {
804 S = SE.getConstant(C->getType(), 0);
805 return C->getValue()->getSExtValue();
806 }
807 } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
808 SmallVector<const SCEV *, 8> NewOps(Add->operands());
809 int64_t Result = ExtractImmediate(NewOps.front(), SE);
810 if (Result != 0)
811 S = SE.getAddExpr(NewOps);
812 return Result;
813 } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
814 SmallVector<const SCEV *, 8> NewOps(AR->operands());
815 int64_t Result = ExtractImmediate(NewOps.front(), SE);
816 if (Result != 0)
817 S = SE.getAddRecExpr(NewOps, AR->getLoop(),
818 // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
820 return Result;
821 }
822 return 0;
823}
824
825/// If S involves the addition of a GlobalValue address, return that symbol, and
826/// mutate S to point to a new SCEV with that value excluded.
828 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
829 if (GlobalValue *GV = dyn_cast<GlobalValue>(U->getValue())) {
830 S = SE.getConstant(GV->getType(), 0);
831 return GV;
832 }
833 } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
834 SmallVector<const SCEV *, 8> NewOps(Add->operands());
835 GlobalValue *Result = ExtractSymbol(NewOps.back(), SE);
836 if (Result)
837 S = SE.getAddExpr(NewOps);
838 return Result;
839 } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
840 SmallVector<const SCEV *, 8> NewOps(AR->operands());
841 GlobalValue *Result = ExtractSymbol(NewOps.front(), SE);
842 if (Result)
843 S = SE.getAddRecExpr(NewOps, AR->getLoop(),
844 // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
846 return Result;
847 }
848 return nullptr;
849}
850
851/// Returns true if the specified instruction is using the specified value as an
852/// address.
854 Instruction *Inst, Value *OperandVal) {
855 bool isAddress = isa<LoadInst>(Inst);
856 if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
857 if (SI->getPointerOperand() == OperandVal)
858 isAddress = true;
859 } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) {
860 // Addressing modes can also be folded into prefetches and a variety
861 // of intrinsics.
862 switch (II->getIntrinsicID()) {
863 case Intrinsic::memset:
864 case Intrinsic::prefetch:
865 case Intrinsic::masked_load:
866 if (II->getArgOperand(0) == OperandVal)
867 isAddress = true;
868 break;
869 case Intrinsic::masked_store:
870 if (II->getArgOperand(1) == OperandVal)
871 isAddress = true;
872 break;
873 case Intrinsic::memmove:
874 case Intrinsic::memcpy:
875 if (II->getArgOperand(0) == OperandVal ||
876 II->getArgOperand(1) == OperandVal)
877 isAddress = true;
878 break;
879 default: {
880 MemIntrinsicInfo IntrInfo;
881 if (TTI.getTgtMemIntrinsic(II, IntrInfo)) {
882 if (IntrInfo.PtrVal == OperandVal)
883 isAddress = true;
884 }
885 }
886 }
887 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Inst)) {
888 if (RMW->getPointerOperand() == OperandVal)
889 isAddress = true;
890 } else if (AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(Inst)) {
891 if (CmpX->getPointerOperand() == OperandVal)
892 isAddress = true;
893 }
894 return isAddress;
895}
896
897/// Return the type of the memory being accessed.
898static MemAccessTy getAccessType(const TargetTransformInfo &TTI,
899 Instruction *Inst, Value *OperandVal) {
900 MemAccessTy AccessTy = MemAccessTy::getUnknown(Inst->getContext());
901
902 // First get the type of memory being accessed.
903 if (Type *Ty = Inst->getAccessType())
904 AccessTy.MemTy = Ty;
905
906 // Then get the pointer address space.
907 if (const StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
908 AccessTy.AddrSpace = SI->getPointerAddressSpace();
909 } else if (const LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
910 AccessTy.AddrSpace = LI->getPointerAddressSpace();
911 } else if (const AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Inst)) {
912 AccessTy.AddrSpace = RMW->getPointerAddressSpace();
913 } else if (const AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(Inst)) {
914 AccessTy.AddrSpace = CmpX->getPointerAddressSpace();
915 } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) {
916 switch (II->getIntrinsicID()) {
917 case Intrinsic::prefetch:
918 case Intrinsic::memset:
919 AccessTy.AddrSpace = II->getArgOperand(0)->getType()->getPointerAddressSpace();
920 AccessTy.MemTy = OperandVal->getType();
921 break;
922 case Intrinsic::memmove:
923 case Intrinsic::memcpy:
924 AccessTy.AddrSpace = OperandVal->getType()->getPointerAddressSpace();
925 AccessTy.MemTy = OperandVal->getType();
926 break;
927 case Intrinsic::masked_load:
928 AccessTy.AddrSpace =
929 II->getArgOperand(0)->getType()->getPointerAddressSpace();
930 break;
931 case Intrinsic::masked_store:
932 AccessTy.AddrSpace =
933 II->getArgOperand(1)->getType()->getPointerAddressSpace();
934 break;
935 default: {
936 MemIntrinsicInfo IntrInfo;
937 if (TTI.getTgtMemIntrinsic(II, IntrInfo) && IntrInfo.PtrVal) {
938 AccessTy.AddrSpace
939 = IntrInfo.PtrVal->getType()->getPointerAddressSpace();
940 }
941
942 break;
943 }
944 }
945 }
946
947 return AccessTy;
948}
949
950/// Return true if this AddRec is already a phi in its loop.
951static bool isExistingPhi(const SCEVAddRecExpr *AR, ScalarEvolution &SE) {
952 for (PHINode &PN : AR->getLoop()->getHeader()->phis()) {
953 if (SE.isSCEVable(PN.getType()) &&
954 (SE.getEffectiveSCEVType(PN.getType()) ==
955 SE.getEffectiveSCEVType(AR->getType())) &&
956 SE.getSCEV(&PN) == AR)
957 return true;
958 }
959 return false;
960}
961
962/// Check if expanding this expression is likely to incur significant cost. This
963/// is tricky because SCEV doesn't track which expressions are actually computed
964/// by the current IR.
965///
966/// We currently allow expansion of IV increments that involve adds,
967/// multiplication by constants, and AddRecs from existing phis.
968///
969/// TODO: Allow UDivExpr if we can find an existing IV increment that is an
970/// obvious multiple of the UDivExpr.
971static bool isHighCostExpansion(const SCEV *S,
973 ScalarEvolution &SE) {
974 // Zero/One operand expressions
975 switch (S->getSCEVType()) {
976 case scUnknown:
977 case scConstant:
978 case scVScale:
979 return false;
980 case scTruncate:
981 return isHighCostExpansion(cast<SCEVTruncateExpr>(S)->getOperand(),
982 Processed, SE);
983 case scZeroExtend:
984 return isHighCostExpansion(cast<SCEVZeroExtendExpr>(S)->getOperand(),
985 Processed, SE);
986 case scSignExtend:
987 return isHighCostExpansion(cast<SCEVSignExtendExpr>(S)->getOperand(),
988 Processed, SE);
989 default:
990 break;
991 }
992
993 if (!Processed.insert(S).second)
994 return false;
995
996 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
997 for (const SCEV *S : Add->operands()) {
998 if (isHighCostExpansion(S, Processed, SE))
999 return true;
1000 }
1001 return false;
1002 }
1003
1004 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) {
1005 if (Mul->getNumOperands() == 2) {
1006 // Multiplication by a constant is ok
1007 if (isa<SCEVConstant>(Mul->getOperand(0)))
1008 return isHighCostExpansion(Mul->getOperand(1), Processed, SE);
1009
1010 // If we have the value of one operand, check if an existing
1011 // multiplication already generates this expression.
1012 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Mul->getOperand(1))) {
1013 Value *UVal = U->getValue();
1014 for (User *UR : UVal->users()) {
1015 // If U is a constant, it may be used by a ConstantExpr.
1016 Instruction *UI = dyn_cast<Instruction>(UR);
1017 if (UI && UI->getOpcode() == Instruction::Mul &&
1018 SE.isSCEVable(UI->getType())) {
1019 return SE.getSCEV(UI) == Mul;
1020 }
1021 }
1022 }
1023 }
1024 }
1025
1026 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
1027 if (isExistingPhi(AR, SE))
1028 return false;
1029 }
1030
1031 // Fow now, consider any other type of expression (div/mul/min/max) high cost.
1032 return true;
1033}
1034
1035namespace {
1036
1037class LSRUse;
1038
1039} // end anonymous namespace
1040
1041/// Check if the addressing mode defined by \p F is completely
1042/// folded in \p LU at isel time.
1043/// This includes address-mode folding and special icmp tricks.
1044/// This function returns true if \p LU can accommodate what \p F
1045/// defines and up to 1 base + 1 scaled + offset.
1046/// In other words, if \p F has several base registers, this function may
1047/// still return true. Therefore, users still need to account for
1048/// additional base registers and/or unfolded offsets to derive an
1049/// accurate cost model.
1051 const LSRUse &LU, const Formula &F);
1052
1053// Get the cost of the scaling factor used in F for LU.
1055 const LSRUse &LU, const Formula &F,
1056 const Loop &L);
1057
1058namespace {
1059
1060/// This class is used to measure and compare candidate formulae.
1061class Cost {
1062 const Loop *L = nullptr;
1063 ScalarEvolution *SE = nullptr;
1064 const TargetTransformInfo *TTI = nullptr;
1067
1068public:
1069 Cost() = delete;
1070 Cost(const Loop *L, ScalarEvolution &SE, const TargetTransformInfo &TTI,
1072 L(L), SE(&SE), TTI(&TTI), AMK(AMK) {
1073 C.Insns = 0;
1074 C.NumRegs = 0;
1075 C.AddRecCost = 0;
1076 C.NumIVMuls = 0;
1077 C.NumBaseAdds = 0;
1078 C.ImmCost = 0;
1079 C.SetupCost = 0;
1080 C.ScaleCost = 0;
1081 }
1082
1083 bool isLess(const Cost &Other) const;
1084
1085 void Lose();
1086
1087#ifndef NDEBUG
1088 // Once any of the metrics loses, they must all remain losers.
1089 bool isValid() {
1090 return ((C.Insns | C.NumRegs | C.AddRecCost | C.NumIVMuls | C.NumBaseAdds
1091 | C.ImmCost | C.SetupCost | C.ScaleCost) != ~0u)
1092 || ((C.Insns & C.NumRegs & C.AddRecCost & C.NumIVMuls & C.NumBaseAdds
1093 & C.ImmCost & C.SetupCost & C.ScaleCost) == ~0u);
1094 }
1095#endif
1096
1097 bool isLoser() {
1098 assert(isValid() && "invalid cost");
1099 return C.NumRegs == ~0u;
1100 }
1101
1102 void RateFormula(const Formula &F,
1104 const DenseSet<const SCEV *> &VisitedRegs,
1105 const LSRUse &LU,
1106 SmallPtrSetImpl<const SCEV *> *LoserRegs = nullptr);
1107
1108 void print(raw_ostream &OS) const;
1109 void dump() const;
1110
1111private:
1112 void RateRegister(const Formula &F, const SCEV *Reg,
1114 void RatePrimaryRegister(const Formula &F, const SCEV *Reg,
1117};
1118
1119/// An operand value in an instruction which is to be replaced with some
1120/// equivalent, possibly strength-reduced, replacement.
1121struct LSRFixup {
1122 /// The instruction which will be updated.
1123 Instruction *UserInst = nullptr;
1124
1125 /// The operand of the instruction which will be replaced. The operand may be
1126 /// used more than once; every instance will be replaced.
1127 Value *OperandValToReplace = nullptr;
1128
1129 /// If this user is to use the post-incremented value of an induction
1130 /// variable, this set is non-empty and holds the loops associated with the
1131 /// induction variable.
1132 PostIncLoopSet PostIncLoops;
1133
1134 /// A constant offset to be added to the LSRUse expression. This allows
1135 /// multiple fixups to share the same LSRUse with different offsets, for
1136 /// example in an unrolled loop.
1137 int64_t Offset = 0;
1138
1139 LSRFixup() = default;
1140
1141 bool isUseFullyOutsideLoop(const Loop *L) const;
1142
1143 void print(raw_ostream &OS) const;
1144 void dump() const;
1145};
1146
1147/// A DenseMapInfo implementation for holding DenseMaps and DenseSets of sorted
1148/// SmallVectors of const SCEV*.
1149struct UniquifierDenseMapInfo {
1150 static SmallVector<const SCEV *, 4> getEmptyKey() {
1152 V.push_back(reinterpret_cast<const SCEV *>(-1));
1153 return V;
1154 }
1155
1156 static SmallVector<const SCEV *, 4> getTombstoneKey() {
1158 V.push_back(reinterpret_cast<const SCEV *>(-2));
1159 return V;
1160 }
1161
1162 static unsigned getHashValue(const SmallVector<const SCEV *, 4> &V) {
1163 return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
1164 }
1165
1166 static bool isEqual(const SmallVector<const SCEV *, 4> &LHS,
1168 return LHS == RHS;
1169 }
1170};
1171
1172/// This class holds the state that LSR keeps for each use in IVUsers, as well
1173/// as uses invented by LSR itself. It includes information about what kinds of
1174/// things can be folded into the user, information about the user itself, and
1175/// information about how the use may be satisfied. TODO: Represent multiple
1176/// users of the same expression in common?
1177class LSRUse {
1178 DenseSet<SmallVector<const SCEV *, 4>, UniquifierDenseMapInfo> Uniquifier;
1179
1180public:
1181 /// An enum for a kind of use, indicating what types of scaled and immediate
1182 /// operands it might support.
1183 enum KindType {
1184 Basic, ///< A normal use, with no folding.
1185 Special, ///< A special case of basic, allowing -1 scales.
1186 Address, ///< An address use; folding according to TargetLowering
1187 ICmpZero ///< An equality icmp with both operands folded into one.
1188 // TODO: Add a generic icmp too?
1189 };
1190
1191 using SCEVUseKindPair = PointerIntPair<const SCEV *, 2, KindType>;
1192
1193 KindType Kind;
1194 MemAccessTy AccessTy;
1195
1196 /// The list of operands which are to be replaced.
1198
1199 /// Keep track of the min and max offsets of the fixups.
1200 int64_t MinOffset = std::numeric_limits<int64_t>::max();
1201 int64_t MaxOffset = std::numeric_limits<int64_t>::min();
1202
1203 /// This records whether all of the fixups using this LSRUse are outside of
1204 /// the loop, in which case some special-case heuristics may be used.
1205 bool AllFixupsOutsideLoop = true;
1206
1207 /// RigidFormula is set to true to guarantee that this use will be associated
1208 /// with a single formula--the one that initially matched. Some SCEV
1209 /// expressions cannot be expanded. This allows LSR to consider the registers
1210 /// used by those expressions without the need to expand them later after
1211 /// changing the formula.
1212 bool RigidFormula = false;
1213
1214 /// This records the widest use type for any fixup using this
1215 /// LSRUse. FindUseWithSimilarFormula can't consider uses with different max
1216 /// fixup widths to be equivalent, because the narrower one may be relying on
1217 /// the implicit truncation to truncate away bogus bits.
1218 Type *WidestFixupType = nullptr;
1219
1220 /// A list of ways to build a value that can satisfy this user. After the
1221 /// list is populated, one of these is selected heuristically and used to
1222 /// formulate a replacement for OperandValToReplace in UserInst.
1223 SmallVector<Formula, 12> Formulae;
1224
1225 /// The set of register candidates used by all formulae in this LSRUse.
1227
1228 LSRUse(KindType K, MemAccessTy AT) : Kind(K), AccessTy(AT) {}
1229
1230 LSRFixup &getNewFixup() {
1231 Fixups.push_back(LSRFixup());
1232 return Fixups.back();
1233 }
1234
1235 void pushFixup(LSRFixup &f) {
1236 Fixups.push_back(f);
1237 if (f.Offset > MaxOffset)
1238 MaxOffset = f.Offset;
1239 if (f.Offset < MinOffset)
1240 MinOffset = f.Offset;
1241 }
1242
1243 bool HasFormulaWithSameRegs(const Formula &F) const;
1244 float getNotSelectedProbability(const SCEV *Reg) const;
1245 bool InsertFormula(const Formula &F, const Loop &L);
1246 void DeleteFormula(Formula &F);
1247 void RecomputeRegs(size_t LUIdx, RegUseTracker &Reguses);
1248
1249 void print(raw_ostream &OS) const;
1250 void dump() const;
1251};
1252
1253} // end anonymous namespace
1254
1256 LSRUse::KindType Kind, MemAccessTy AccessTy,
1257 GlobalValue *BaseGV, int64_t BaseOffset,
1258 bool HasBaseReg, int64_t Scale,
1259 Instruction *Fixup = nullptr);
1260
1261static unsigned getSetupCost(const SCEV *Reg, unsigned Depth) {
1262 if (isa<SCEVUnknown>(Reg) || isa<SCEVConstant>(Reg))
1263 return 1;
1264 if (Depth == 0)
1265 return 0;
1266 if (const auto *S = dyn_cast<SCEVAddRecExpr>(Reg))
1267 return getSetupCost(S->getStart(), Depth - 1);
1268 if (auto S = dyn_cast<SCEVIntegralCastExpr>(Reg))
1269 return getSetupCost(S->getOperand(), Depth - 1);
1270 if (auto S = dyn_cast<SCEVNAryExpr>(Reg))
1271 return std::accumulate(S->operands().begin(), S->operands().end(), 0,
1272 [&](unsigned i, const SCEV *Reg) {
1273 return i + getSetupCost(Reg, Depth - 1);
1274 });
1275 if (auto S = dyn_cast<SCEVUDivExpr>(Reg))
1276 return getSetupCost(S->getLHS(), Depth - 1) +
1277 getSetupCost(S->getRHS(), Depth - 1);
1278 return 0;
1279}
1280
1281/// Tally up interesting quantities from the given register.
1282void Cost::RateRegister(const Formula &F, const SCEV *Reg,
1284 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Reg)) {
1285 // If this is an addrec for another loop, it should be an invariant
1286 // with respect to L since L is the innermost loop (at least
1287 // for now LSR only handles innermost loops).
1288 if (AR->getLoop() != L) {
1289 // If the AddRec exists, consider it's register free and leave it alone.
1290 if (isExistingPhi(AR, *SE) && AMK != TTI::AMK_PostIndexed)
1291 return;
1292
1293 // It is bad to allow LSR for current loop to add induction variables
1294 // for its sibling loops.
1295 if (!AR->getLoop()->contains(L)) {
1296 Lose();
1297 return;
1298 }
1299
1300 // Otherwise, it will be an invariant with respect to Loop L.
1301 ++C.NumRegs;
1302 return;
1303 }
1304
1305 unsigned LoopCost = 1;
1306 if (TTI->isIndexedLoadLegal(TTI->MIM_PostInc, AR->getType()) ||
1307 TTI->isIndexedStoreLegal(TTI->MIM_PostInc, AR->getType())) {
1308
1309 // If the step size matches the base offset, we could use pre-indexed
1310 // addressing.
1311 if (AMK == TTI::AMK_PreIndexed) {
1312 if (auto *Step = dyn_cast<SCEVConstant>(AR->getStepRecurrence(*SE)))
1313 if (Step->getAPInt() == F.BaseOffset)
1314 LoopCost = 0;
1315 } else if (AMK == TTI::AMK_PostIndexed) {
1316 const SCEV *LoopStep = AR->getStepRecurrence(*SE);
1317 if (isa<SCEVConstant>(LoopStep)) {
1318 const SCEV *LoopStart = AR->getStart();
1319 if (!isa<SCEVConstant>(LoopStart) &&
1320 SE->isLoopInvariant(LoopStart, L))
1321 LoopCost = 0;
1322 }
1323 }
1324 }
1325 C.AddRecCost += LoopCost;
1326
1327 // Add the step value register, if it needs one.
1328 // TODO: The non-affine case isn't precisely modeled here.
1329 if (!AR->isAffine() || !isa<SCEVConstant>(AR->getOperand(1))) {
1330 if (!Regs.count(AR->getOperand(1))) {
1331 RateRegister(F, AR->getOperand(1), Regs);
1332 if (isLoser())
1333 return;
1334 }
1335 }
1336 }
1337 ++C.NumRegs;
1338
1339 // Rough heuristic; favor registers which don't require extra setup
1340 // instructions in the preheader.
1341 C.SetupCost += getSetupCost(Reg, SetupCostDepthLimit);
1342 // Ensure we don't, even with the recusion limit, produce invalid costs.
1343 C.SetupCost = std::min<unsigned>(C.SetupCost, 1 << 16);
1344
1345 C.NumIVMuls += isa<SCEVMulExpr>(Reg) &&
1346 SE->hasComputableLoopEvolution(Reg, L);
1347}
1348
1349/// Record this register in the set. If we haven't seen it before, rate
1350/// it. Optional LoserRegs provides a way to declare any formula that refers to
1351/// one of those regs an instant loser.
1352void Cost::RatePrimaryRegister(const Formula &F, const SCEV *Reg,
1354 SmallPtrSetImpl<const SCEV *> *LoserRegs) {
1355 if (LoserRegs && LoserRegs->count(Reg)) {
1356 Lose();
1357 return;
1358 }
1359 if (Regs.insert(Reg).second) {
1360 RateRegister(F, Reg, Regs);
1361 if (LoserRegs && isLoser())
1362 LoserRegs->insert(Reg);
1363 }
1364}
1365
1366void Cost::RateFormula(const Formula &F,
1368 const DenseSet<const SCEV *> &VisitedRegs,
1369 const LSRUse &LU,
1370 SmallPtrSetImpl<const SCEV *> *LoserRegs) {
1371 if (isLoser())
1372 return;
1373 assert(F.isCanonical(*L) && "Cost is accurate only for canonical formula");
1374 // Tally up the registers.
1375 unsigned PrevAddRecCost = C.AddRecCost;
1376 unsigned PrevNumRegs = C.NumRegs;
1377 unsigned PrevNumBaseAdds = C.NumBaseAdds;
1378 if (const SCEV *ScaledReg = F.ScaledReg) {
1379 if (VisitedRegs.count(ScaledReg)) {
1380 Lose();
1381 return;
1382 }
1383 RatePrimaryRegister(F, ScaledReg, Regs, LoserRegs);
1384 if (isLoser())
1385 return;
1386 }
1387 for (const SCEV *BaseReg : F.BaseRegs) {
1388 if (VisitedRegs.count(BaseReg)) {
1389 Lose();
1390 return;
1391 }
1392 RatePrimaryRegister(F, BaseReg, Regs, LoserRegs);
1393 if (isLoser())
1394 return;
1395 }
1396
1397 // Determine how many (unfolded) adds we'll need inside the loop.
1398 size_t NumBaseParts = F.getNumRegs();
1399 if (NumBaseParts > 1)
1400 // Do not count the base and a possible second register if the target
1401 // allows to fold 2 registers.
1402 C.NumBaseAdds +=
1403 NumBaseParts - (1 + (F.Scale && isAMCompletelyFolded(*TTI, LU, F)));
1404 C.NumBaseAdds += (F.UnfoldedOffset != 0);
1405
1406 // Accumulate non-free scaling amounts.
1407 C.ScaleCost += *getScalingFactorCost(*TTI, LU, F, *L).getValue();
1408
1409 // Tally up the non-zero immediates.
1410 for (const LSRFixup &Fixup : LU.Fixups) {
1411 int64_t O = Fixup.Offset;
1412 int64_t Offset = (uint64_t)O + F.BaseOffset;
1413 if (F.BaseGV)
1414 C.ImmCost += 64; // Handle symbolic values conservatively.
1415 // TODO: This should probably be the pointer size.
1416 else if (Offset != 0)
1417 C.ImmCost += APInt(64, Offset, true).getSignificantBits();
1418
1419 // Check with target if this offset with this instruction is
1420 // specifically not supported.
1421 if (LU.Kind == LSRUse::Address && Offset != 0 &&
1422 !isAMCompletelyFolded(*TTI, LSRUse::Address, LU.AccessTy, F.BaseGV,
1423 Offset, F.HasBaseReg, F.Scale, Fixup.UserInst))
1424 C.NumBaseAdds++;
1425 }
1426
1427 // If we don't count instruction cost exit here.
1428 if (!InsnsCost) {
1429 assert(isValid() && "invalid cost");
1430 return;
1431 }
1432
1433 // Treat every new register that exceeds TTI.getNumberOfRegisters() - 1 as
1434 // additional instruction (at least fill).
1435 // TODO: Need distinguish register class?
1436 unsigned TTIRegNum = TTI->getNumberOfRegisters(
1437 TTI->getRegisterClassForType(false, F.getType())) - 1;
1438 if (C.NumRegs > TTIRegNum) {
1439 // Cost already exceeded TTIRegNum, then only newly added register can add
1440 // new instructions.
1441 if (PrevNumRegs > TTIRegNum)
1442 C.Insns += (C.NumRegs - PrevNumRegs);
1443 else
1444 C.Insns += (C.NumRegs - TTIRegNum);
1445 }
1446
1447 // If ICmpZero formula ends with not 0, it could not be replaced by
1448 // just add or sub. We'll need to compare final result of AddRec.
1449 // That means we'll need an additional instruction. But if the target can
1450 // macro-fuse a compare with a branch, don't count this extra instruction.
1451 // For -10 + {0, +, 1}:
1452 // i = i + 1;
1453 // cmp i, 10
1454 //
1455 // For {-10, +, 1}:
1456 // i = i + 1;
1457 if (LU.Kind == LSRUse::ICmpZero && !F.hasZeroEnd() &&
1458 !TTI->canMacroFuseCmp())
1459 C.Insns++;
1460 // Each new AddRec adds 1 instruction to calculation.
1461 C.Insns += (C.AddRecCost - PrevAddRecCost);
1462
1463 // BaseAdds adds instructions for unfolded registers.
1464 if (LU.Kind != LSRUse::ICmpZero)
1465 C.Insns += C.NumBaseAdds - PrevNumBaseAdds;
1466 assert(isValid() && "invalid cost");
1467}
1468
1469/// Set this cost to a losing value.
1470void Cost::Lose() {
1471 C.Insns = std::numeric_limits<unsigned>::max();
1472 C.NumRegs = std::numeric_limits<unsigned>::max();
1473 C.AddRecCost = std::numeric_limits<unsigned>::max();
1474 C.NumIVMuls = std::numeric_limits<unsigned>::max();
1475 C.NumBaseAdds = std::numeric_limits<unsigned>::max();
1476 C.ImmCost = std::numeric_limits<unsigned>::max();
1477 C.SetupCost = std::numeric_limits<unsigned>::max();
1478 C.ScaleCost = std::numeric_limits<unsigned>::max();
1479}
1480
1481/// Choose the lower cost.
1482bool Cost::isLess(const Cost &Other) const {
1483 if (InsnsCost.getNumOccurrences() > 0 && InsnsCost &&
1484 C.Insns != Other.C.Insns)
1485 return C.Insns < Other.C.Insns;
1486 return TTI->isLSRCostLess(C, Other.C);
1487}
1488
1489#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1490void Cost::print(raw_ostream &OS) const {
1491 if (InsnsCost)
1492 OS << C.Insns << " instruction" << (C.Insns == 1 ? " " : "s ");
1493 OS << C.NumRegs << " reg" << (C.NumRegs == 1 ? "" : "s");
1494 if (C.AddRecCost != 0)
1495 OS << ", with addrec cost " << C.AddRecCost;
1496 if (C.NumIVMuls != 0)
1497 OS << ", plus " << C.NumIVMuls << " IV mul"
1498 << (C.NumIVMuls == 1 ? "" : "s");
1499 if (C.NumBaseAdds != 0)
1500 OS << ", plus " << C.NumBaseAdds << " base add"
1501 << (C.NumBaseAdds == 1 ? "" : "s");
1502 if (C.ScaleCost != 0)
1503 OS << ", plus " << C.ScaleCost << " scale cost";
1504 if (C.ImmCost != 0)
1505 OS << ", plus " << C.ImmCost << " imm cost";
1506 if (C.SetupCost != 0)
1507 OS << ", plus " << C.SetupCost << " setup cost";
1508}
1509
1510LLVM_DUMP_METHOD void Cost::dump() const {
1511 print(errs()); errs() << '\n';
1512}
1513#endif
1514
1515/// Test whether this fixup always uses its value outside of the given loop.
1516bool LSRFixup::isUseFullyOutsideLoop(const Loop *L) const {
1517 // PHI nodes use their value in their incoming blocks.
1518 if (const PHINode *PN = dyn_cast<PHINode>(UserInst)) {
1519 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
1520 if (PN->getIncomingValue(i) == OperandValToReplace &&
1521 L->contains(PN->getIncomingBlock(i)))
1522 return false;
1523 return true;
1524 }
1525
1526 return !L->contains(UserInst);
1527}
1528
1529#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1530void LSRFixup::print(raw_ostream &OS) const {
1531 OS << "UserInst=";
1532 // Store is common and interesting enough to be worth special-casing.
1533 if (StoreInst *Store = dyn_cast<StoreInst>(UserInst)) {
1534 OS << "store ";
1535 Store->getOperand(0)->printAsOperand(OS, /*PrintType=*/false);
1536 } else if (UserInst->getType()->isVoidTy())
1537 OS << UserInst->getOpcodeName();
1538 else
1539 UserInst->printAsOperand(OS, /*PrintType=*/false);
1540
1541 OS << ", OperandValToReplace=";
1542 OperandValToReplace->printAsOperand(OS, /*PrintType=*/false);
1543
1544 for (const Loop *PIL : PostIncLoops) {
1545 OS << ", PostIncLoop=";
1546 PIL->getHeader()->printAsOperand(OS, /*PrintType=*/false);
1547 }
1548
1549 if (Offset != 0)
1550 OS << ", Offset=" << Offset;
1551}
1552
1553LLVM_DUMP_METHOD void LSRFixup::dump() const {
1554 print(errs()); errs() << '\n';
1555}
1556#endif
1557
1558/// Test whether this use as a formula which has the same registers as the given
1559/// formula.
1560bool LSRUse::HasFormulaWithSameRegs(const Formula &F) const {
1562 if (F.ScaledReg) Key.push_back(F.ScaledReg);
1563 // Unstable sort by host order ok, because this is only used for uniquifying.
1564 llvm::sort(Key);
1565 return Uniquifier.count(Key);
1566}
1567
1568/// The function returns a probability of selecting formula without Reg.
1569float LSRUse::getNotSelectedProbability(const SCEV *Reg) const {
1570 unsigned FNum = 0;
1571 for (const Formula &F : Formulae)
1572 if (F.referencesReg(Reg))
1573 FNum++;
1574 return ((float)(Formulae.size() - FNum)) / Formulae.size();
1575}
1576
1577/// If the given formula has not yet been inserted, add it to the list, and
1578/// return true. Return false otherwise. The formula must be in canonical form.
1579bool LSRUse::InsertFormula(const Formula &F, const Loop &L) {
1580 assert(F.isCanonical(L) && "Invalid canonical representation");
1581
1582 if (!Formulae.empty() && RigidFormula)
1583 return false;
1584
1586 if (F.ScaledReg) Key.push_back(F.ScaledReg);
1587 // Unstable sort by host order ok, because this is only used for uniquifying.
1588 llvm::sort(Key);
1589
1590 if (!Uniquifier.insert(Key).second)
1591 return false;
1592
1593 // Using a register to hold the value of 0 is not profitable.
1594 assert((!F.ScaledReg || !F.ScaledReg->isZero()) &&
1595 "Zero allocated in a scaled register!");
1596#ifndef NDEBUG
1597 for (const SCEV *BaseReg : F.BaseRegs)
1598 assert(!BaseReg->isZero() && "Zero allocated in a base register!");
1599#endif
1600
1601 // Add the formula to the list.
1602 Formulae.push_back(F);
1603
1604 // Record registers now being used by this use.
1605 Regs.insert(F.BaseRegs.begin(), F.BaseRegs.end());
1606 if (F.ScaledReg)
1607 Regs.insert(F.ScaledReg);
1608
1609 return true;
1610}
1611
1612/// Remove the given formula from this use's list.
1613void LSRUse::DeleteFormula(Formula &F) {
1614 if (&F != &Formulae.back())
1615 std::swap(F, Formulae.back());
1616 Formulae.pop_back();
1617}
1618
1619/// Recompute the Regs field, and update RegUses.
1620void LSRUse::RecomputeRegs(size_t LUIdx, RegUseTracker &RegUses) {
1621 // Now that we've filtered out some formulae, recompute the Regs set.
1622 SmallPtrSet<const SCEV *, 4> OldRegs = std::move(Regs);
1623 Regs.clear();
1624 for (const Formula &F : Formulae) {
1625 if (F.ScaledReg) Regs.insert(F.ScaledReg);
1626 Regs.insert(F.BaseRegs.begin(), F.BaseRegs.end());
1627 }
1628
1629 // Update the RegTracker.
1630 for (const SCEV *S : OldRegs)
1631 if (!Regs.count(S))
1632 RegUses.dropRegister(S, LUIdx);
1633}
1634
1635#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1636void LSRUse::print(raw_ostream &OS) const {
1637 OS << "LSR Use: Kind=";
1638 switch (Kind) {
1639 case Basic: OS << "Basic"; break;
1640 case Special: OS << "Special"; break;
1641 case ICmpZero: OS << "ICmpZero"; break;
1642 case Address:
1643 OS << "Address of ";
1644 if (AccessTy.MemTy->isPointerTy())
1645 OS << "pointer"; // the full pointer type could be really verbose
1646 else {
1647 OS << *AccessTy.MemTy;
1648 }
1649
1650 OS << " in addrspace(" << AccessTy.AddrSpace << ')';
1651 }
1652
1653 OS << ", Offsets={";
1654 bool NeedComma = false;
1655 for (const LSRFixup &Fixup : Fixups) {
1656 if (NeedComma) OS << ',';
1657 OS << Fixup.Offset;
1658 NeedComma = true;
1659 }
1660 OS << '}';
1661
1662 if (AllFixupsOutsideLoop)
1663 OS << ", all-fixups-outside-loop";
1664
1665 if (WidestFixupType)
1666 OS << ", widest fixup type: " << *WidestFixupType;
1667}
1668
1669LLVM_DUMP_METHOD void LSRUse::dump() const {
1670 print(errs()); errs() << '\n';
1671}
1672#endif
1673
1675 LSRUse::KindType Kind, MemAccessTy AccessTy,
1676 GlobalValue *BaseGV, int64_t BaseOffset,
1677 bool HasBaseReg, int64_t Scale,
1678 Instruction *Fixup/*= nullptr*/) {
1679 switch (Kind) {
1680 case LSRUse::Address:
1681 return TTI.isLegalAddressingMode(AccessTy.MemTy, BaseGV, BaseOffset,
1682 HasBaseReg, Scale, AccessTy.AddrSpace, Fixup);
1683
1684 case LSRUse::ICmpZero:
1685 // There's not even a target hook for querying whether it would be legal to
1686 // fold a GV into an ICmp.
1687 if (BaseGV)
1688 return false;
1689
1690 // ICmp only has two operands; don't allow more than two non-trivial parts.
1691 if (Scale != 0 && HasBaseReg && BaseOffset != 0)
1692 return false;
1693
1694 // ICmp only supports no scale or a -1 scale, as we can "fold" a -1 scale by
1695 // putting the scaled register in the other operand of the icmp.
1696 if (Scale != 0 && Scale != -1)
1697 return false;
1698
1699 // If we have low-level target information, ask the target if it can fold an
1700 // integer immediate on an icmp.
1701 if (BaseOffset != 0) {
1702 // We have one of:
1703 // ICmpZero BaseReg + BaseOffset => ICmp BaseReg, -BaseOffset
1704 // ICmpZero -1*ScaleReg + BaseOffset => ICmp ScaleReg, BaseOffset
1705 // Offs is the ICmp immediate.
1706 if (Scale == 0)
1707 // The cast does the right thing with
1708 // std::numeric_limits<int64_t>::min().
1709 BaseOffset = -(uint64_t)BaseOffset;
1710 return TTI.isLegalICmpImmediate(BaseOffset);
1711 }
1712
1713 // ICmpZero BaseReg + -1*ScaleReg => ICmp BaseReg, ScaleReg
1714 return true;
1715
1716 case LSRUse::Basic:
1717 // Only handle single-register values.
1718 return !BaseGV && Scale == 0 && BaseOffset == 0;
1719
1720 case LSRUse::Special:
1721 // Special case Basic to handle -1 scales.
1722 return !BaseGV && (Scale == 0 || Scale == -1) && BaseOffset == 0;
1723 }
1724
1725 llvm_unreachable("Invalid LSRUse Kind!");
1726}
1727
1729 int64_t MinOffset, int64_t MaxOffset,
1730 LSRUse::KindType Kind, MemAccessTy AccessTy,
1731 GlobalValue *BaseGV, int64_t BaseOffset,
1732 bool HasBaseReg, int64_t Scale) {
1733 // Check for overflow.
1734 if (((int64_t)((uint64_t)BaseOffset + MinOffset) > BaseOffset) !=
1735 (MinOffset > 0))
1736 return false;
1737 MinOffset = (uint64_t)BaseOffset + MinOffset;
1738 if (((int64_t)((uint64_t)BaseOffset + MaxOffset) > BaseOffset) !=
1739 (MaxOffset > 0))
1740 return false;
1741 MaxOffset = (uint64_t)BaseOffset + MaxOffset;
1742
1743 return isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, MinOffset,
1744 HasBaseReg, Scale) &&
1745 isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, MaxOffset,
1746 HasBaseReg, Scale);
1747}
1748
1750 int64_t MinOffset, int64_t MaxOffset,
1751 LSRUse::KindType Kind, MemAccessTy AccessTy,
1752 const Formula &F, const Loop &L) {
1753 // For the purpose of isAMCompletelyFolded either having a canonical formula
1754 // or a scale not equal to zero is correct.
1755 // Problems may arise from non canonical formulae having a scale == 0.
1756 // Strictly speaking it would best to just rely on canonical formulae.
1757 // However, when we generate the scaled formulae, we first check that the
1758 // scaling factor is profitable before computing the actual ScaledReg for
1759 // compile time sake.
1760 assert((F.isCanonical(L) || F.Scale != 0));
1761 return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy,
1762 F.BaseGV, F.BaseOffset, F.HasBaseReg, F.Scale);
1763}
1764
1765/// Test whether we know how to expand the current formula.
1766static bool isLegalUse(const TargetTransformInfo &TTI, int64_t MinOffset,
1767 int64_t MaxOffset, LSRUse::KindType Kind,
1768 MemAccessTy AccessTy, GlobalValue *BaseGV,
1769 int64_t BaseOffset, bool HasBaseReg, int64_t Scale) {
1770 // We know how to expand completely foldable formulae.
1771 return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy, BaseGV,
1772 BaseOffset, HasBaseReg, Scale) ||
1773 // Or formulae that use a base register produced by a sum of base
1774 // registers.
1775 (Scale == 1 &&
1776 isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy,
1777 BaseGV, BaseOffset, true, 0));
1778}
1779
1780static bool isLegalUse(const TargetTransformInfo &TTI, int64_t MinOffset,
1781 int64_t MaxOffset, LSRUse::KindType Kind,
1782 MemAccessTy AccessTy, const Formula &F) {
1783 return isLegalUse(TTI, MinOffset, MaxOffset, Kind, AccessTy, F.BaseGV,
1784 F.BaseOffset, F.HasBaseReg, F.Scale);
1785}
1786
1788 const LSRUse &LU, const Formula &F) {
1789 // Target may want to look at the user instructions.
1790 if (LU.Kind == LSRUse::Address && TTI.LSRWithInstrQueries()) {
1791 for (const LSRFixup &Fixup : LU.Fixups)
1792 if (!isAMCompletelyFolded(TTI, LSRUse::Address, LU.AccessTy, F.BaseGV,
1793 (F.BaseOffset + Fixup.Offset), F.HasBaseReg,
1794 F.Scale, Fixup.UserInst))
1795 return false;
1796 return true;
1797 }
1798
1799 return isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind,
1800 LU.AccessTy, F.BaseGV, F.BaseOffset, F.HasBaseReg,
1801 F.Scale);
1802}
1803
1805 const LSRUse &LU, const Formula &F,
1806 const Loop &L) {
1807 if (!F.Scale)
1808 return 0;
1809
1810 // If the use is not completely folded in that instruction, we will have to
1811 // pay an extra cost only for scale != 1.
1812 if (!isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind,
1813 LU.AccessTy, F, L))
1814 return F.Scale != 1;
1815
1816 switch (LU.Kind) {
1817 case LSRUse::Address: {
1818 // Check the scaling factor cost with both the min and max offsets.
1819 InstructionCost ScaleCostMinOffset = TTI.getScalingFactorCost(
1820 LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MinOffset, F.HasBaseReg,
1821 F.Scale, LU.AccessTy.AddrSpace);
1822 InstructionCost ScaleCostMaxOffset = TTI.getScalingFactorCost(
1823 LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MaxOffset, F.HasBaseReg,
1824 F.Scale, LU.AccessTy.AddrSpace);
1825
1826 assert(ScaleCostMinOffset.isValid() && ScaleCostMaxOffset.isValid() &&
1827 "Legal addressing mode has an illegal cost!");
1828 return std::max(ScaleCostMinOffset, ScaleCostMaxOffset);
1829 }
1830 case LSRUse::ICmpZero:
1831 case LSRUse::Basic:
1832 case LSRUse::Special:
1833 // The use is completely folded, i.e., everything is folded into the
1834 // instruction.
1835 return 0;
1836 }
1837
1838 llvm_unreachable("Invalid LSRUse Kind!");
1839}
1840
1842 LSRUse::KindType Kind, MemAccessTy AccessTy,
1843 GlobalValue *BaseGV, int64_t BaseOffset,
1844 bool HasBaseReg) {
1845 // Fast-path: zero is always foldable.
1846 if (BaseOffset == 0 && !BaseGV) return true;
1847
1848 // Conservatively, create an address with an immediate and a
1849 // base and a scale.
1850 int64_t Scale = Kind == LSRUse::ICmpZero ? -1 : 1;
1851
1852 // Canonicalize a scale of 1 to a base register if the formula doesn't
1853 // already have a base register.
1854 if (!HasBaseReg && Scale == 1) {
1855 Scale = 0;
1856 HasBaseReg = true;
1857 }
1858
1859 return isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, BaseOffset,
1860 HasBaseReg, Scale);
1861}
1862
1864 ScalarEvolution &SE, int64_t MinOffset,
1865 int64_t MaxOffset, LSRUse::KindType Kind,
1866 MemAccessTy AccessTy, const SCEV *S,
1867 bool HasBaseReg) {
1868 // Fast-path: zero is always foldable.
1869 if (S->isZero()) return true;
1870
1871 // Conservatively, create an address with an immediate and a
1872 // base and a scale.
1873 int64_t BaseOffset = ExtractImmediate(S, SE);
1874 GlobalValue *BaseGV = ExtractSymbol(S, SE);
1875
1876 // If there's anything else involved, it's not foldable.
1877 if (!S->isZero()) return false;
1878
1879 // Fast-path: zero is always foldable.
1880 if (BaseOffset == 0 && !BaseGV) return true;
1881
1882 // Conservatively, create an address with an immediate and a
1883 // base and a scale.
1884 int64_t Scale = Kind == LSRUse::ICmpZero ? -1 : 1;
1885
1886 return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy, BaseGV,
1887 BaseOffset, HasBaseReg, Scale);
1888}
1889
1890namespace {
1891
1892/// An individual increment in a Chain of IV increments. Relate an IV user to
1893/// an expression that computes the IV it uses from the IV used by the previous
1894/// link in the Chain.
1895///
1896/// For the head of a chain, IncExpr holds the absolute SCEV expression for the
1897/// original IVOperand. The head of the chain's IVOperand is only valid during
1898/// chain collection, before LSR replaces IV users. During chain generation,
1899/// IncExpr can be used to find the new IVOperand that computes the same
1900/// expression.
1901struct IVInc {
1902 Instruction *UserInst;
1903 Value* IVOperand;
1904 const SCEV *IncExpr;
1905
1906 IVInc(Instruction *U, Value *O, const SCEV *E)
1907 : UserInst(U), IVOperand(O), IncExpr(E) {}
1908};
1909
1910// The list of IV increments in program order. We typically add the head of a
1911// chain without finding subsequent links.
1912struct IVChain {
1914 const SCEV *ExprBase = nullptr;
1915
1916 IVChain() = default;
1917 IVChain(const IVInc &Head, const SCEV *Base)
1918 : Incs(1, Head), ExprBase(Base) {}
1919
1921
1922 // Return the first increment in the chain.
1923 const_iterator begin() const {
1924 assert(!Incs.empty());
1925 return std::next(Incs.begin());
1926 }
1927 const_iterator end() const {
1928 return Incs.end();
1929 }
1930
1931 // Returns true if this chain contains any increments.
1932 bool hasIncs() const { return Incs.size() >= 2; }
1933
1934 // Add an IVInc to the end of this chain.
1935 void add(const IVInc &X) { Incs.push_back(X); }
1936
1937 // Returns the last UserInst in the chain.
1938 Instruction *tailUserInst() const { return Incs.back().UserInst; }
1939
1940 // Returns true if IncExpr can be profitably added to this chain.
1941 bool isProfitableIncrement(const SCEV *OperExpr,
1942 const SCEV *IncExpr,
1944};
1945
1946/// Helper for CollectChains to track multiple IV increment uses. Distinguish
1947/// between FarUsers that definitely cross IV increments and NearUsers that may
1948/// be used between IV increments.
1949struct ChainUsers {
1952};
1953
1954/// This class holds state for the main loop strength reduction logic.
1955class LSRInstance {
1956 IVUsers &IU;
1957 ScalarEvolution &SE;
1958 DominatorTree &DT;
1959 LoopInfo &LI;
1960 AssumptionCache &AC;
1961 TargetLibraryInfo &TLI;
1962 const TargetTransformInfo &TTI;
1963 Loop *const L;
1964 MemorySSAUpdater *MSSAU;
1966 mutable SCEVExpander Rewriter;
1967 bool Changed = false;
1968
1969 /// This is the insert position that the current loop's induction variable
1970 /// increment should be placed. In simple loops, this is the latch block's
1971 /// terminator. But in more complicated cases, this is a position which will
1972 /// dominate all the in-loop post-increment users.
1973 Instruction *IVIncInsertPos = nullptr;
1974
1975 /// Interesting factors between use strides.
1976 ///
1977 /// We explicitly use a SetVector which contains a SmallSet, instead of the
1978 /// default, a SmallDenseSet, because we need to use the full range of
1979 /// int64_ts, and there's currently no good way of doing that with
1980 /// SmallDenseSet.
1982
1983 /// The cost of the current SCEV, the best solution by LSR will be dropped if
1984 /// the solution is not profitable.
1985 Cost BaselineCost;
1986
1987 /// Interesting use types, to facilitate truncation reuse.
1989
1990 /// The list of interesting uses.
1992
1993 /// Track which uses use which register candidates.
1994 RegUseTracker RegUses;
1995
1996 // Limit the number of chains to avoid quadratic behavior. We don't expect to
1997 // have more than a few IV increment chains in a loop. Missing a Chain falls
1998 // back to normal LSR behavior for those uses.
1999 static const unsigned MaxChains = 8;
2000
2001 /// IV users can form a chain of IV increments.
2003
2004 /// IV users that belong to profitable IVChains.
2006
2007 /// Induction variables that were generated and inserted by the SCEV Expander.
2008 SmallVector<llvm::WeakVH, 2> ScalarEvolutionIVs;
2009
2010 void OptimizeShadowIV();
2011 bool FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse);
2012 ICmpInst *OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse);
2013 void OptimizeLoopTermCond();
2014
2015 void ChainInstruction(Instruction *UserInst, Instruction *IVOper,
2016 SmallVectorImpl<ChainUsers> &ChainUsersVec);
2017 void FinalizeChain(IVChain &Chain);
2018 void CollectChains();
2019 void GenerateIVChain(const IVChain &Chain,
2021
2022 void CollectInterestingTypesAndFactors();
2023 void CollectFixupsAndInitialFormulae();
2024
2025 // Support for sharing of LSRUses between LSRFixups.
2027 UseMapTy UseMap;
2028
2029 bool reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg,
2030 LSRUse::KindType Kind, MemAccessTy AccessTy);
2031
2032 std::pair<size_t, int64_t> getUse(const SCEV *&Expr, LSRUse::KindType Kind,
2033 MemAccessTy AccessTy);
2034
2035 void DeleteUse(LSRUse &LU, size_t LUIdx);
2036
2037 LSRUse *FindUseWithSimilarFormula(const Formula &F, const LSRUse &OrigLU);
2038
2039 void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
2040 void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
2041 void CountRegisters(const Formula &F, size_t LUIdx);
2042 bool InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F);
2043
2044 void CollectLoopInvariantFixupsAndFormulae();
2045
2046 void GenerateReassociations(LSRUse &LU, unsigned LUIdx, Formula Base,
2047 unsigned Depth = 0);
2048
2049 void GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
2050 const Formula &Base, unsigned Depth,
2051 size_t Idx, bool IsScaledReg = false);
2052 void GenerateCombinations(LSRUse &LU, unsigned LUIdx, Formula Base);
2053 void GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
2054 const Formula &Base, size_t Idx,
2055 bool IsScaledReg = false);
2056 void GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
2057 void GenerateConstantOffsetsImpl(LSRUse &LU, unsigned LUIdx,
2058 const Formula &Base,
2059 const SmallVectorImpl<int64_t> &Worklist,
2060 size_t Idx, bool IsScaledReg = false);
2061 void GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
2062 void GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, Formula Base);
2063 void GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base);
2064 void GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base);
2065 void GenerateCrossUseConstantOffsets();
2066 void GenerateAllReuseFormulae();
2067
2068 void FilterOutUndesirableDedicatedRegisters();
2069
2070 size_t EstimateSearchSpaceComplexity() const;
2071 void NarrowSearchSpaceByDetectingSupersets();
2072 void NarrowSearchSpaceByCollapsingUnrolledCode();
2073 void NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters();
2074 void NarrowSearchSpaceByFilterFormulaWithSameScaledReg();
2075 void NarrowSearchSpaceByFilterPostInc();
2076 void NarrowSearchSpaceByDeletingCostlyFormulas();
2077 void NarrowSearchSpaceByPickingWinnerRegs();
2078 void NarrowSearchSpaceUsingHeuristics();
2079
2080 void SolveRecurse(SmallVectorImpl<const Formula *> &Solution,
2081 Cost &SolutionCost,
2083 const Cost &CurCost,
2084 const SmallPtrSet<const SCEV *, 16> &CurRegs,
2085 DenseSet<const SCEV *> &VisitedRegs) const;
2086 void Solve(SmallVectorImpl<const Formula *> &Solution) const;
2087
2089 HoistInsertPosition(BasicBlock::iterator IP,
2090 const SmallVectorImpl<Instruction *> &Inputs) const;
2091 BasicBlock::iterator AdjustInsertPositionForExpand(BasicBlock::iterator IP,
2092 const LSRFixup &LF,
2093 const LSRUse &LU) const;
2094
2095 Value *Expand(const LSRUse &LU, const LSRFixup &LF, const Formula &F,
2097 SmallVectorImpl<WeakTrackingVH> &DeadInsts) const;
2098 void RewriteForPHI(PHINode *PN, const LSRUse &LU, const LSRFixup &LF,
2099 const Formula &F,
2100 SmallVectorImpl<WeakTrackingVH> &DeadInsts) const;
2101 void Rewrite(const LSRUse &LU, const LSRFixup &LF, const Formula &F,
2102 SmallVectorImpl<WeakTrackingVH> &DeadInsts) const;
2103 void ImplementSolution(const SmallVectorImpl<const Formula *> &Solution);
2104
2105public:
2106 LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE, DominatorTree &DT,
2108 TargetLibraryInfo &TLI, MemorySSAUpdater *MSSAU);
2109
2110 bool getChanged() const { return Changed; }
2111 const SmallVectorImpl<WeakVH> &getScalarEvolutionIVs() const {
2112 return ScalarEvolutionIVs;
2113 }
2114
2115 void print_factors_and_types(raw_ostream &OS) const;
2116 void print_fixups(raw_ostream &OS) const;
2117 void print_uses(raw_ostream &OS) const;
2118 void print(raw_ostream &OS) const;
2119 void dump() const;
2120};
2121
2122} // end anonymous namespace
2123
2124/// If IV is used in a int-to-float cast inside the loop then try to eliminate
2125/// the cast operation.
2126void LSRInstance::OptimizeShadowIV() {
2127 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L);
2128 if (isa<SCEVCouldNotCompute>(BackedgeTakenCount))
2129 return;
2130
2131 for (IVUsers::const_iterator UI = IU.begin(), E = IU.end();
2132 UI != E; /* empty */) {
2133 IVUsers::const_iterator CandidateUI = UI;
2134 ++UI;
2135 Instruction *ShadowUse = CandidateUI->getUser();
2136 Type *DestTy = nullptr;
2137 bool IsSigned = false;
2138
2139 /* If shadow use is a int->float cast then insert a second IV
2140 to eliminate this cast.
2141
2142 for (unsigned i = 0; i < n; ++i)
2143 foo((double)i);
2144
2145 is transformed into
2146
2147 double d = 0.0;
2148 for (unsigned i = 0; i < n; ++i, ++d)
2149 foo(d);
2150 */
2151 if (UIToFPInst *UCast = dyn_cast<UIToFPInst>(CandidateUI->getUser())) {
2152 IsSigned = false;
2153 DestTy = UCast->getDestTy();
2154 }
2155 else if (SIToFPInst *SCast = dyn_cast<SIToFPInst>(CandidateUI->getUser())) {
2156 IsSigned = true;
2157 DestTy = SCast->getDestTy();
2158 }
2159 if (!DestTy) continue;
2160
2161 // If target does not support DestTy natively then do not apply
2162 // this transformation.
2163 if (!TTI.isTypeLegal(DestTy)) continue;
2164
2165 PHINode *PH = dyn_cast<PHINode>(ShadowUse->getOperand(0));
2166 if (!PH) continue;
2167 if (PH->getNumIncomingValues() != 2) continue;
2168
2169 // If the calculation in integers overflows, the result in FP type will
2170 // differ. So we only can do this transformation if we are guaranteed to not
2171 // deal with overflowing values
2172 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(PH));
2173 if (!AR) continue;
2174 if (IsSigned && !AR->hasNoSignedWrap()) continue;
2175 if (!IsSigned && !AR->hasNoUnsignedWrap()) continue;
2176
2177 Type *SrcTy = PH->getType();
2178 int Mantissa = DestTy->getFPMantissaWidth();
2179 if (Mantissa == -1) continue;
2180 if ((int)SE.getTypeSizeInBits(SrcTy) > Mantissa)
2181 continue;
2182
2183 unsigned Entry, Latch;
2184 if (PH->getIncomingBlock(0) == L->getLoopPreheader()) {
2185 Entry = 0;
2186 Latch = 1;
2187 } else {
2188 Entry = 1;
2189 Latch = 0;
2190 }
2191
2192 ConstantInt *Init = dyn_cast<ConstantInt>(PH->getIncomingValue(Entry));
2193 if (!Init) continue;
2194 Constant *NewInit = ConstantFP::get(DestTy, IsSigned ?
2195 (double)Init->getSExtValue() :
2196 (double)Init->getZExtValue());
2197
2198 BinaryOperator *Incr =
2199 dyn_cast<BinaryOperator>(PH->getIncomingValue(Latch));
2200 if (!Incr) continue;
2201 if (Incr->getOpcode() != Instruction::Add
2202 && Incr->getOpcode() != Instruction::Sub)
2203 continue;
2204
2205 /* Initialize new IV, double d = 0.0 in above example. */
2206 ConstantInt *C = nullptr;
2207 if (Incr->getOperand(0) == PH)
2208 C = dyn_cast<ConstantInt>(Incr->getOperand(1));
2209 else if (Incr->getOperand(1) == PH)
2210 C = dyn_cast<ConstantInt>(Incr->getOperand(0));
2211 else
2212 continue;
2213
2214 if (!C) continue;
2215
2216 // Ignore negative constants, as the code below doesn't handle them
2217 // correctly. TODO: Remove this restriction.
2218 if (!C->getValue().isStrictlyPositive())
2219 continue;
2220
2221 /* Add new PHINode. */
2222 PHINode *NewPH = PHINode::Create(DestTy, 2, "IV.S.", PH->getIterator());
2223
2224 /* create new increment. '++d' in above example. */
2225 Constant *CFP = ConstantFP::get(DestTy, C->getZExtValue());
2227 Incr->getOpcode() == Instruction::Add ? Instruction::FAdd
2228 : Instruction::FSub,
2229 NewPH, CFP, "IV.S.next.", Incr->getIterator());
2230
2231 NewPH->addIncoming(NewInit, PH->getIncomingBlock(Entry));
2232 NewPH->addIncoming(NewIncr, PH->getIncomingBlock(Latch));
2233
2234 /* Remove cast operation */
2235 ShadowUse->replaceAllUsesWith(NewPH);
2236 ShadowUse->eraseFromParent();
2237 Changed = true;
2238 break;
2239 }
2240}
2241
2242/// If Cond has an operand that is an expression of an IV, set the IV user and
2243/// stride information and return true, otherwise return false.
2244bool LSRInstance::FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse) {
2245 for (IVStrideUse &U : IU)
2246 if (U.getUser() == Cond) {
2247 // NOTE: we could handle setcc instructions with multiple uses here, but
2248 // InstCombine does it as well for simple uses, it's not clear that it
2249 // occurs enough in real life to handle.
2250 CondUse = &U;
2251 return true;
2252 }
2253 return false;
2254}
2255
2256/// Rewrite the loop's terminating condition if it uses a max computation.
2257///
2258/// This is a narrow solution to a specific, but acute, problem. For loops
2259/// like this:
2260///
2261/// i = 0;
2262/// do {
2263/// p[i] = 0.0;
2264/// } while (++i < n);
2265///
2266/// the trip count isn't just 'n', because 'n' might not be positive. And
2267/// unfortunately this can come up even for loops where the user didn't use
2268/// a C do-while loop. For example, seemingly well-behaved top-test loops
2269/// will commonly be lowered like this:
2270///
2271/// if (n > 0) {
2272/// i = 0;
2273/// do {
2274/// p[i] = 0.0;
2275/// } while (++i < n);
2276/// }
2277///
2278/// and then it's possible for subsequent optimization to obscure the if
2279/// test in such a way that indvars can't find it.
2280///
2281/// When indvars can't find the if test in loops like this, it creates a
2282/// max expression, which allows it to give the loop a canonical
2283/// induction variable:
2284///
2285/// i = 0;
2286/// max = n < 1 ? 1 : n;
2287/// do {
2288/// p[i] = 0.0;
2289/// } while (++i != max);
2290///
2291/// Canonical induction variables are necessary because the loop passes
2292/// are designed around them. The most obvious example of this is the
2293/// LoopInfo analysis, which doesn't remember trip count values. It
2294/// expects to be able to rediscover the trip count each time it is
2295/// needed, and it does this using a simple analysis that only succeeds if
2296/// the loop has a canonical induction variable.
2297///
2298/// However, when it comes time to generate code, the maximum operation
2299/// can be quite costly, especially if it's inside of an outer loop.
2300///
2301/// This function solves this problem by detecting this type of loop and
2302/// rewriting their conditions from ICMP_NE back to ICMP_SLT, and deleting
2303/// the instructions for the maximum computation.
2304ICmpInst *LSRInstance::OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse) {
2305 // Check that the loop matches the pattern we're looking for.
2306 if (Cond->getPredicate() != CmpInst::ICMP_EQ &&
2307 Cond->getPredicate() != CmpInst::ICMP_NE)
2308 return Cond;
2309
2310 SelectInst *Sel = dyn_cast<SelectInst>(Cond->getOperand(1));
2311 if (!Sel || !Sel->hasOneUse()) return Cond;
2312
2313 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L);
2314 if (isa<SCEVCouldNotCompute>(BackedgeTakenCount))
2315 return Cond;
2316 const SCEV *One = SE.getConstant(BackedgeTakenCount->getType(), 1);
2317
2318 // Add one to the backedge-taken count to get the trip count.
2319 const SCEV *IterationCount = SE.getAddExpr(One, BackedgeTakenCount);
2320 if (IterationCount != SE.getSCEV(Sel)) return Cond;
2321
2322 // Check for a max calculation that matches the pattern. There's no check
2323 // for ICMP_ULE here because the comparison would be with zero, which
2324 // isn't interesting.
2325 CmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE;
2326 const SCEVNAryExpr *Max = nullptr;
2327 if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(BackedgeTakenCount)) {
2328 Pred = ICmpInst::ICMP_SLE;
2329 Max = S;
2330 } else if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(IterationCount)) {
2331 Pred = ICmpInst::ICMP_SLT;
2332 Max = S;
2333 } else if (const SCEVUMaxExpr *U = dyn_cast<SCEVUMaxExpr>(IterationCount)) {
2334 Pred = ICmpInst::ICMP_ULT;
2335 Max = U;
2336 } else {
2337 // No match; bail.
2338 return Cond;
2339 }
2340
2341 // To handle a max with more than two operands, this optimization would
2342 // require additional checking and setup.
2343 if (Max->getNumOperands() != 2)
2344 return Cond;
2345
2346 const SCEV *MaxLHS = Max->getOperand(0);
2347 const SCEV *MaxRHS = Max->getOperand(1);
2348
2349 // ScalarEvolution canonicalizes constants to the left. For < and >, look
2350 // for a comparison with 1. For <= and >=, a comparison with zero.
2351 if (!MaxLHS ||
2352 (ICmpInst::isTrueWhenEqual(Pred) ? !MaxLHS->isZero() : (MaxLHS != One)))
2353 return Cond;
2354
2355 // Check the relevant induction variable for conformance to
2356 // the pattern.
2357 const SCEV *IV = SE.getSCEV(Cond->getOperand(0));
2358 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(IV);
2359 if (!AR || !AR->isAffine() ||
2360 AR->getStart() != One ||
2361 AR->getStepRecurrence(SE) != One)
2362 return Cond;
2363
2364 assert(AR->getLoop() == L &&
2365 "Loop condition operand is an addrec in a different loop!");
2366
2367 // Check the right operand of the select, and remember it, as it will
2368 // be used in the new comparison instruction.
2369 Value *NewRHS = nullptr;
2370 if (ICmpInst::isTrueWhenEqual(Pred)) {
2371 // Look for n+1, and grab n.
2372 if (AddOperator *BO = dyn_cast<AddOperator>(Sel->getOperand(1)))
2373 if (ConstantInt *BO1 = dyn_cast<ConstantInt>(BO->getOperand(1)))
2374 if (BO1->isOne() && SE.getSCEV(BO->getOperand(0)) == MaxRHS)
2375 NewRHS = BO->getOperand(0);
2376 if (AddOperator *BO = dyn_cast<AddOperator>(Sel->getOperand(2)))
2377 if (ConstantInt *BO1 = dyn_cast<ConstantInt>(BO->getOperand(1)))
2378 if (BO1->isOne() && SE.getSCEV(BO->getOperand(0)) == MaxRHS)
2379 NewRHS = BO->getOperand(0);
2380 if (!NewRHS)
2381 return Cond;
2382 } else if (SE.getSCEV(Sel->getOperand(1)) == MaxRHS)
2383 NewRHS = Sel->getOperand(1);
2384 else if (SE.getSCEV(Sel->getOperand(2)) == MaxRHS)
2385 NewRHS = Sel->getOperand(2);
2386 else if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(MaxRHS))
2387 NewRHS = SU->getValue();
2388 else
2389 // Max doesn't match expected pattern.
2390 return Cond;
2391
2392 // Determine the new comparison opcode. It may be signed or unsigned,
2393 // and the original comparison may be either equality or inequality.
2394 if (Cond->getPredicate() == CmpInst::ICMP_EQ)
2395 Pred = CmpInst::getInversePredicate(Pred);
2396
2397 // Ok, everything looks ok to change the condition into an SLT or SGE and
2398 // delete the max calculation.
2399 ICmpInst *NewCond = new ICmpInst(Cond->getIterator(), Pred,
2400 Cond->getOperand(0), NewRHS, "scmp");
2401
2402 // Delete the max calculation instructions.
2403 NewCond->setDebugLoc(Cond->getDebugLoc());
2404 Cond->replaceAllUsesWith(NewCond);
2405 CondUse->setUser(NewCond);
2406 Instruction *Cmp = cast<Instruction>(Sel->getOperand(0));
2407 Cond->eraseFromParent();
2408 Sel->eraseFromParent();
2409 if (Cmp->use_empty())
2410 Cmp->eraseFromParent();
2411 return NewCond;
2412}
2413
2414/// Change loop terminating condition to use the postinc iv when possible.
2415void
2416LSRInstance::OptimizeLoopTermCond() {
2418
2419 // We need a different set of heuristics for rotated and non-rotated loops.
2420 // If a loop is rotated then the latch is also the backedge, so inserting
2421 // post-inc expressions just before the latch is ideal. To reduce live ranges
2422 // it also makes sense to rewrite terminating conditions to use post-inc
2423 // expressions.
2424 //
2425 // If the loop is not rotated then the latch is not a backedge; the latch
2426 // check is done in the loop head. Adding post-inc expressions before the
2427 // latch will cause overlapping live-ranges of pre-inc and post-inc expressions
2428 // in the loop body. In this case we do *not* want to use post-inc expressions
2429 // in the latch check, and we want to insert post-inc expressions before
2430 // the backedge.
2431 BasicBlock *LatchBlock = L->getLoopLatch();
2432 SmallVector<BasicBlock*, 8> ExitingBlocks;
2433 L->getExitingBlocks(ExitingBlocks);
2434 if (!llvm::is_contained(ExitingBlocks, LatchBlock)) {
2435 // The backedge doesn't exit the loop; treat this as a head-tested loop.
2436 IVIncInsertPos = LatchBlock->getTerminator();
2437 return;
2438 }
2439
2440 // Otherwise treat this as a rotated loop.
2441 for (BasicBlock *ExitingBlock : ExitingBlocks) {
2442 // Get the terminating condition for the loop if possible. If we
2443 // can, we want to change it to use a post-incremented version of its
2444 // induction variable, to allow coalescing the live ranges for the IV into
2445 // one register value.
2446
2447 BranchInst *TermBr = dyn_cast<BranchInst>(ExitingBlock->getTerminator());
2448 if (!TermBr)
2449 continue;
2450 // FIXME: Overly conservative, termination condition could be an 'or' etc..
2451 if (TermBr->isUnconditional() || !isa<ICmpInst>(TermBr->getCondition()))
2452 continue;
2453
2454 // Search IVUsesByStride to find Cond's IVUse if there is one.
2455 IVStrideUse *CondUse = nullptr;
2456 ICmpInst *Cond = cast<ICmpInst>(TermBr->getCondition());
2457 if (!FindIVUserForCond(Cond, CondUse))
2458 continue;
2459
2460 // If the trip count is computed in terms of a max (due to ScalarEvolution
2461 // being unable to find a sufficient guard, for example), change the loop
2462 // comparison to use SLT or ULT instead of NE.
2463 // One consequence of doing this now is that it disrupts the count-down
2464 // optimization. That's not always a bad thing though, because in such
2465 // cases it may still be worthwhile to avoid a max.
2466 Cond = OptimizeMax(Cond, CondUse);
2467
2468 // If this exiting block dominates the latch block, it may also use
2469 // the post-inc value if it won't be shared with other uses.
2470 // Check for dominance.
2471 if (!DT.dominates(ExitingBlock, LatchBlock))
2472 continue;
2473
2474 // Conservatively avoid trying to use the post-inc value in non-latch
2475 // exits if there may be pre-inc users in intervening blocks.
2476 if (LatchBlock != ExitingBlock)
2477 for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI)
2478 // Test if the use is reachable from the exiting block. This dominator
2479 // query is a conservative approximation of reachability.
2480 if (&*UI != CondUse &&
2481 !DT.properlyDominates(UI->getUser()->getParent(), ExitingBlock)) {
2482 // Conservatively assume there may be reuse if the quotient of their
2483 // strides could be a legal scale.
2484 const SCEV *A = IU.getStride(*CondUse, L);
2485 const SCEV *B = IU.getStride(*UI, L);
2486 if (!A || !B) continue;
2487 if (SE.getTypeSizeInBits(A->getType()) !=
2488 SE.getTypeSizeInBits(B->getType())) {
2489 if (SE.getTypeSizeInBits(A->getType()) >
2490 SE.getTypeSizeInBits(B->getType()))
2491 B = SE.getSignExtendExpr(B, A->getType());
2492 else
2493 A = SE.getSignExtendExpr(A, B->getType());
2494 }
2495 if (const SCEVConstant *D =
2496 dyn_cast_or_null<SCEVConstant>(getExactSDiv(B, A, SE))) {
2497 const ConstantInt *C = D->getValue();
2498 // Stride of one or negative one can have reuse with non-addresses.
2499 if (C->isOne() || C->isMinusOne())
2500 goto decline_post_inc;
2501 // Avoid weird situations.
2502 if (C->getValue().getSignificantBits() >= 64 ||
2503 C->getValue().isMinSignedValue())
2504 goto decline_post_inc;
2505 // Check for possible scaled-address reuse.
2506 if (isAddressUse(TTI, UI->getUser(), UI->getOperandValToReplace())) {
2507 MemAccessTy AccessTy = getAccessType(
2508 TTI, UI->getUser(), UI->getOperandValToReplace());
2509 int64_t Scale = C->getSExtValue();
2510 if (TTI.isLegalAddressingMode(AccessTy.MemTy, /*BaseGV=*/nullptr,
2511 /*BaseOffset=*/0,
2512 /*HasBaseReg=*/true, Scale,
2513 AccessTy.AddrSpace))
2514 goto decline_post_inc;
2515 Scale = -Scale;
2516 if (TTI.isLegalAddressingMode(AccessTy.MemTy, /*BaseGV=*/nullptr,
2517 /*BaseOffset=*/0,
2518 /*HasBaseReg=*/true, Scale,
2519 AccessTy.AddrSpace))
2520 goto decline_post_inc;
2521 }
2522 }
2523 }
2524
2525 LLVM_DEBUG(dbgs() << " Change loop exiting icmp to use postinc iv: "
2526 << *Cond << '\n');
2527
2528 // It's possible for the setcc instruction to be anywhere in the loop, and
2529 // possible for it to have multiple users. If it is not immediately before
2530 // the exiting block branch, move it.
2531 if (Cond->getNextNonDebugInstruction() != TermBr) {
2532 if (Cond->hasOneUse()) {
2533 Cond->moveBefore(TermBr);
2534 } else {
2535 // Clone the terminating condition and insert into the loopend.
2536 ICmpInst *OldCond = Cond;
2537 Cond = cast<ICmpInst>(Cond->clone());
2538 Cond->setName(L->getHeader()->getName() + ".termcond");
2539 Cond->insertInto(ExitingBlock, TermBr->getIterator());
2540
2541 // Clone the IVUse, as the old use still exists!
2542 CondUse = &IU.AddUser(Cond, CondUse->getOperandValToReplace());
2543 TermBr->replaceUsesOfWith(OldCond, Cond);
2544 }
2545 }
2546
2547 // If we get to here, we know that we can transform the setcc instruction to
2548 // use the post-incremented version of the IV, allowing us to coalesce the
2549 // live ranges for the IV correctly.
2550 CondUse->transformToPostInc(L);
2551 Changed = true;
2552
2553 PostIncs.insert(Cond);
2554 decline_post_inc:;
2555 }
2556
2557 // Determine an insertion point for the loop induction variable increment. It
2558 // must dominate all the post-inc comparisons we just set up, and it must
2559 // dominate the loop latch edge.
2560 IVIncInsertPos = L->getLoopLatch()->getTerminator();
2561 for (Instruction *Inst : PostIncs)
2562 IVIncInsertPos = DT.findNearestCommonDominator(IVIncInsertPos, Inst);
2563}
2564
2565/// Determine if the given use can accommodate a fixup at the given offset and
2566/// other details. If so, update the use and return true.
2567bool LSRInstance::reconcileNewOffset(LSRUse &LU, int64_t NewOffset,
2568 bool HasBaseReg, LSRUse::KindType Kind,
2569 MemAccessTy AccessTy) {
2570 int64_t NewMinOffset = LU.MinOffset;
2571 int64_t NewMaxOffset = LU.MaxOffset;
2572 MemAccessTy NewAccessTy = AccessTy;
2573
2574 // Check for a mismatched kind. It's tempting to collapse mismatched kinds to
2575 // something conservative, however this can pessimize in the case that one of
2576 // the uses will have all its uses outside the loop, for example.
2577 if (LU.Kind != Kind)
2578 return false;
2579
2580 // Check for a mismatched access type, and fall back conservatively as needed.
2581 // TODO: Be less conservative when the type is similar and can use the same
2582 // addressing modes.
2583 if (Kind == LSRUse::Address) {
2584 if (AccessTy.MemTy != LU.AccessTy.MemTy) {
2585 NewAccessTy = MemAccessTy::getUnknown(AccessTy.MemTy->getContext(),
2586 AccessTy.AddrSpace);
2587 }
2588 }
2589
2590 // Conservatively assume HasBaseReg is true for now.
2591 if (NewOffset < LU.MinOffset) {
2592 if (!isAlwaysFoldable(TTI, Kind, NewAccessTy, /*BaseGV=*/nullptr,
2593 LU.MaxOffset - NewOffset, HasBaseReg))
2594 return false;
2595 NewMinOffset = NewOffset;
2596 } else if (NewOffset > LU.MaxOffset) {
2597 if (!isAlwaysFoldable(TTI, Kind, NewAccessTy, /*BaseGV=*/nullptr,
2598 NewOffset - LU.MinOffset, HasBaseReg))
2599 return false;
2600 NewMaxOffset = NewOffset;
2601 }
2602
2603 // Update the use.
2604 LU.MinOffset = NewMinOffset;
2605 LU.MaxOffset = NewMaxOffset;
2606 LU.AccessTy = NewAccessTy;
2607 return true;
2608}
2609
2610/// Return an LSRUse index and an offset value for a fixup which needs the given
2611/// expression, with the given kind and optional access type. Either reuse an
2612/// existing use or create a new one, as needed.
2613std::pair<size_t, int64_t> LSRInstance::getUse(const SCEV *&Expr,
2614 LSRUse::KindType Kind,
2615 MemAccessTy AccessTy) {
2616 const SCEV *Copy = Expr;
2617 int64_t Offset = ExtractImmediate(Expr, SE);
2618
2619 // Basic uses can't accept any offset, for example.
2620 if (!isAlwaysFoldable(TTI, Kind, AccessTy, /*BaseGV=*/ nullptr,
2621 Offset, /*HasBaseReg=*/ true)) {
2622 Expr = Copy;
2623 Offset = 0;
2624 }
2625
2626 std::pair<UseMapTy::iterator, bool> P =
2627 UseMap.insert(std::make_pair(LSRUse::SCEVUseKindPair(Expr, Kind), 0));
2628 if (!P.second) {
2629 // A use already existed with this base.
2630 size_t LUIdx = P.first->second;
2631 LSRUse &LU = Uses[LUIdx];
2632 if (reconcileNewOffset(LU, Offset, /*HasBaseReg=*/true, Kind, AccessTy))
2633 // Reuse this use.
2634 return std::make_pair(LUIdx, Offset);
2635 }
2636
2637 // Create a new use.
2638 size_t LUIdx = Uses.size();
2639 P.first->second = LUIdx;
2640 Uses.push_back(LSRUse(Kind, AccessTy));
2641 LSRUse &LU = Uses[LUIdx];
2642
2643 LU.MinOffset = Offset;
2644 LU.MaxOffset = Offset;
2645 return std::make_pair(LUIdx, Offset);
2646}
2647
2648/// Delete the given use from the Uses list.
2649void LSRInstance::DeleteUse(LSRUse &LU, size_t LUIdx) {
2650 if (&LU != &Uses.back())
2651 std::swap(LU, Uses.back());
2652 Uses.pop_back();
2653
2654 // Update RegUses.
2655 RegUses.swapAndDropUse(LUIdx, Uses.size());
2656}
2657
2658/// Look for a use distinct from OrigLU which is has a formula that has the same
2659/// registers as the given formula.
2660LSRUse *
2661LSRInstance::FindUseWithSimilarFormula(const Formula &OrigF,
2662 const LSRUse &OrigLU) {
2663 // Search all uses for the formula. This could be more clever.
2664 for (LSRUse &LU : Uses) {
2665 // Check whether this use is close enough to OrigLU, to see whether it's
2666 // worthwhile looking through its formulae.
2667 // Ignore ICmpZero uses because they may contain formulae generated by
2668 // GenerateICmpZeroScales, in which case adding fixup offsets may
2669 // be invalid.
2670 if (&LU != &OrigLU &&
2671 LU.Kind != LSRUse::ICmpZero &&
2672 LU.Kind == OrigLU.Kind && OrigLU.AccessTy == LU.AccessTy &&
2673 LU.WidestFixupType == OrigLU.WidestFixupType &&
2674 LU.HasFormulaWithSameRegs(OrigF)) {
2675 // Scan through this use's formulae.
2676 for (const Formula &F : LU.Formulae) {
2677 // Check to see if this formula has the same registers and symbols
2678 // as OrigF.
2679 if (F.BaseRegs == OrigF.BaseRegs &&
2680 F.ScaledReg == OrigF.ScaledReg &&
2681 F.BaseGV == OrigF.BaseGV &&
2682 F.Scale == OrigF.Scale &&
2683 F.UnfoldedOffset == OrigF.UnfoldedOffset) {
2684 if (F.BaseOffset == 0)
2685 return &LU;
2686 // This is the formula where all the registers and symbols matched;
2687 // there aren't going to be any others. Since we declined it, we
2688 // can skip the rest of the formulae and proceed to the next LSRUse.
2689 break;
2690 }
2691 }
2692 }
2693 }
2694
2695 // Nothing looked good.
2696 return nullptr;
2697}
2698
2699void LSRInstance::CollectInterestingTypesAndFactors() {
2701
2702 // Collect interesting types and strides.
2704 for (const IVStrideUse &U : IU) {
2705 const SCEV *Expr = IU.getExpr(U);
2706 if (!Expr)
2707 continue;
2708
2709 // Collect interesting types.
2710 Types.insert(SE.getEffectiveSCEVType(Expr->getType()));
2711
2712 // Add strides for mentioned loops.
2713 Worklist.push_back(Expr);
2714 do {
2715 const SCEV *S = Worklist.pop_back_val();
2716 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
2717 if (AR->getLoop() == L)
2718 Strides.insert(AR->getStepRecurrence(SE));
2719 Worklist.push_back(AR->getStart());
2720 } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
2721 append_range(Worklist, Add->operands());
2722 }
2723 } while (!Worklist.empty());
2724 }
2725
2726 // Compute interesting factors from the set of interesting strides.
2728 I = Strides.begin(), E = Strides.end(); I != E; ++I)
2730 std::next(I); NewStrideIter != E; ++NewStrideIter) {
2731 const SCEV *OldStride = *I;
2732 const SCEV *NewStride = *NewStrideIter;
2733
2734 if (SE.getTypeSizeInBits(OldStride->getType()) !=
2735 SE.getTypeSizeInBits(NewStride->getType())) {
2736 if (SE.getTypeSizeInBits(OldStride->getType()) >
2737 SE.getTypeSizeInBits(NewStride->getType()))
2738 NewStride = SE.getSignExtendExpr(NewStride, OldStride->getType());
2739 else
2740 OldStride = SE.getSignExtendExpr(OldStride, NewStride->getType());
2741 }
2742 if (const SCEVConstant *Factor =
2743 dyn_cast_or_null<SCEVConstant>(getExactSDiv(NewStride, OldStride,
2744 SE, true))) {
2745 if (Factor->getAPInt().getSignificantBits() <= 64 && !Factor->isZero())
2746 Factors.insert(Factor->getAPInt().getSExtValue());
2747 } else if (const SCEVConstant *Factor =
2748 dyn_cast_or_null<SCEVConstant>(getExactSDiv(OldStride,
2749 NewStride,
2750 SE, true))) {
2751 if (Factor->getAPInt().getSignificantBits() <= 64 && !Factor->isZero())
2752 Factors.insert(Factor->getAPInt().getSExtValue());
2753 }
2754 }
2755
2756 // If all uses use the same type, don't bother looking for truncation-based
2757 // reuse.
2758 if (Types.size() == 1)
2759 Types.clear();
2760
2761 LLVM_DEBUG(print_factors_and_types(dbgs()));
2762}
2763
2764/// Helper for CollectChains that finds an IV operand (computed by an AddRec in
2765/// this loop) within [OI,OE) or returns OE. If IVUsers mapped Instructions to
2766/// IVStrideUses, we could partially skip this.
2767static User::op_iterator
2769 Loop *L, ScalarEvolution &SE) {
2770 for(; OI != OE; ++OI) {
2771 if (Instruction *Oper = dyn_cast<Instruction>(*OI)) {
2772 if (!SE.isSCEVable(Oper->getType()))
2773 continue;
2774
2775 if (const SCEVAddRecExpr *AR =
2776 dyn_cast<SCEVAddRecExpr>(SE.getSCEV(Oper))) {
2777 if (AR->getLoop() == L)
2778 break;
2779 }
2780 }
2781 }
2782 return OI;
2783}
2784
2785/// IVChain logic must consistently peek base TruncInst operands, so wrap it in
2786/// a convenient helper.
2788 if (TruncInst *Trunc = dyn_cast<TruncInst>(Oper))
2789 return Trunc->getOperand(0);
2790 return Oper;
2791}
2792
2793/// Return an approximation of this SCEV expression's "base", or NULL for any
2794/// constant. Returning the expression itself is conservative. Returning a
2795/// deeper subexpression is more precise and valid as long as it isn't less
2796/// complex than another subexpression. For expressions involving multiple
2797/// unscaled values, we need to return the pointer-type SCEVUnknown. This avoids
2798/// forming chains across objects, such as: PrevOper==a[i], IVOper==b[i],
2799/// IVInc==b-a.
2800///
2801/// Since SCEVUnknown is the rightmost type, and pointers are the rightmost
2802/// SCEVUnknown, we simply return the rightmost SCEV operand.
2803static const SCEV *getExprBase(const SCEV *S) {
2804 switch (S->getSCEVType()) {
2805 default: // including scUnknown.
2806 return S;
2807 case scConstant:
2808 case scVScale:
2809 return nullptr;
2810 case scTruncate:
2811 return getExprBase(cast<SCEVTruncateExpr>(S)->getOperand());
2812 case scZeroExtend:
2813 return getExprBase(cast<SCEVZeroExtendExpr>(S)->getOperand());
2814 case scSignExtend:
2815 return getExprBase(cast<SCEVSignExtendExpr>(S)->getOperand());
2816 case scAddExpr: {
2817 // Skip over scaled operands (scMulExpr) to follow add operands as long as
2818 // there's nothing more complex.
2819 // FIXME: not sure if we want to recognize negation.
2820 const SCEVAddExpr *Add = cast<SCEVAddExpr>(S);
2821 for (const SCEV *SubExpr : reverse(Add->operands())) {
2822 if (SubExpr->getSCEVType() == scAddExpr)
2823 return getExprBase(SubExpr);
2824
2825 if (SubExpr->getSCEVType() != scMulExpr)
2826 return SubExpr;
2827 }
2828 return S; // all operands are scaled, be conservative.
2829 }
2830 case scAddRecExpr:
2831 return getExprBase(cast<SCEVAddRecExpr>(S)->getStart());
2832 }
2833 llvm_unreachable("Unknown SCEV kind!");
2834}
2835
2836/// Return true if the chain increment is profitable to expand into a loop
2837/// invariant value, which may require its own register. A profitable chain
2838/// increment will be an offset relative to the same base. We allow such offsets
2839/// to potentially be used as chain increment as long as it's not obviously
2840/// expensive to expand using real instructions.
2841bool IVChain::isProfitableIncrement(const SCEV *OperExpr,
2842 const SCEV *IncExpr,
2843 ScalarEvolution &SE) {
2844 // Aggressively form chains when -stress-ivchain.
2845 if (StressIVChain)
2846 return true;
2847
2848 // Do not replace a constant offset from IV head with a nonconstant IV
2849 // increment.
2850 if (!isa<SCEVConstant>(IncExpr)) {
2851 const SCEV *HeadExpr = SE.getSCEV(getWideOperand(Incs[0].IVOperand));
2852 if (isa<SCEVConstant>(SE.getMinusSCEV(OperExpr, HeadExpr)))
2853 return false;
2854 }
2855
2857 return !isHighCostExpansion(IncExpr, Processed, SE);
2858}
2859
2860/// Return true if the number of registers needed for the chain is estimated to
2861/// be less than the number required for the individual IV users. First prohibit
2862/// any IV users that keep the IV live across increments (the Users set should
2863/// be empty). Next count the number and type of increments in the chain.
2864///
2865/// Chaining IVs can lead to considerable code bloat if ISEL doesn't
2866/// effectively use postinc addressing modes. Only consider it profitable it the
2867/// increments can be computed in fewer registers when chained.
2868///
2869/// TODO: Consider IVInc free if it's already used in another chains.
2870static bool isProfitableChain(IVChain &Chain,
2872 ScalarEvolution &SE,
2873 const TargetTransformInfo &TTI) {
2874 if (StressIVChain)
2875 return true;
2876
2877 if (!Chain.hasIncs())
2878 return false;
2879
2880 if (!Users.empty()) {
2881 LLVM_DEBUG(dbgs() << "Chain: " << *Chain.Incs[0].UserInst << " users:\n";
2882 for (Instruction *Inst
2883 : Users) { dbgs() << " " << *Inst << "\n"; });
2884 return false;
2885 }
2886 assert(!Chain.Incs.empty() && "empty IV chains are not allowed");
2887
2888 // The chain itself may require a register, so intialize cost to 1.
2889 int cost = 1;
2890
2891 // A complete chain likely eliminates the need for keeping the original IV in
2892 // a register. LSR does not currently know how to form a complete chain unless
2893 // the header phi already exists.
2894 if (isa<PHINode>(Chain.tailUserInst())
2895 && SE.getSCEV(Chain.tailUserInst()) == Chain.Incs[0].IncExpr) {
2896 --cost;
2897 }
2898 const SCEV *LastIncExpr = nullptr;
2899 unsigned NumConstIncrements = 0;
2900 unsigned NumVarIncrements = 0;
2901 unsigned NumReusedIncrements = 0;
2902
2903 if (TTI.isProfitableLSRChainElement(Chain.Incs[0].UserInst))
2904 return true;
2905
2906 for (const IVInc &Inc : Chain) {
2907 if (TTI.isProfitableLSRChainElement(Inc.UserInst))
2908 return true;
2909 if (Inc.IncExpr->isZero())
2910 continue;
2911
2912 // Incrementing by zero or some constant is neutral. We assume constants can
2913 // be folded into an addressing mode or an add's immediate operand.
2914 if (isa<SCEVConstant>(Inc.IncExpr)) {
2915 ++NumConstIncrements;
2916 continue;
2917 }
2918
2919 if (Inc.IncExpr == LastIncExpr)
2920 ++NumReusedIncrements;
2921 else
2922 ++NumVarIncrements;
2923
2924 LastIncExpr = Inc.IncExpr;
2925 }
2926 // An IV chain with a single increment is handled by LSR's postinc
2927 // uses. However, a chain with multiple increments requires keeping the IV's
2928 // value live longer than it needs to be if chained.
2929 if (NumConstIncrements > 1)
2930 --cost;
2931
2932 // Materializing increment expressions in the preheader that didn't exist in
2933 // the original code may cost a register. For example, sign-extended array
2934 // indices can produce ridiculous increments like this:
2935 // IV + ((sext i32 (2 * %s) to i64) + (-1 * (sext i32 %s to i64)))
2936 cost += NumVarIncrements;
2937
2938 // Reusing variable increments likely saves a register to hold the multiple of
2939 // the stride.
2940 cost -= NumReusedIncrements;
2941
2942 LLVM_DEBUG(dbgs() << "Chain: " << *Chain.Incs[0].UserInst << " Cost: " << cost
2943 << "\n");
2944
2945 return cost < 0;
2946}
2947
2948/// Add this IV user to an existing chain or make it the head of a new chain.
2949void LSRInstance::ChainInstruction(Instruction *UserInst, Instruction *IVOper,
2950 SmallVectorImpl<ChainUsers> &ChainUsersVec) {
2951 // When IVs are used as types of varying widths, they are generally converted
2952 // to a wider type with some uses remaining narrow under a (free) trunc.
2953 Value *const NextIV = getWideOperand(IVOper);
2954 const SCEV *const OperExpr = SE.getSCEV(NextIV);
2955 const SCEV *const OperExprBase = getExprBase(OperExpr);
2956
2957 // Visit all existing chains. Check if its IVOper can be computed as a
2958 // profitable loop invariant increment from the last link in the Chain.
2959 unsigned ChainIdx = 0, NChains = IVChainVec.size();
2960 const SCEV *LastIncExpr = nullptr;
2961 for (; ChainIdx < NChains; ++ChainIdx) {
2962 IVChain &Chain = IVChainVec[ChainIdx];
2963
2964 // Prune the solution space aggressively by checking that both IV operands
2965 // are expressions that operate on the same unscaled SCEVUnknown. This
2966 // "base" will be canceled by the subsequent getMinusSCEV call. Checking
2967 // first avoids creating extra SCEV expressions.
2968 if (!StressIVChain && Chain.ExprBase != OperExprBase)
2969 continue;
2970
2971 Value *PrevIV = getWideOperand(Chain.Incs.back().IVOperand);
2972 if (PrevIV->getType() != NextIV->getType())
2973 continue;
2974
2975 // A phi node terminates a chain.
2976 if (isa<PHINode>(UserInst) && isa<PHINode>(Chain.tailUserInst()))
2977 continue;
2978
2979 // The increment must be loop-invariant so it can be kept in a register.
2980 const SCEV *PrevExpr = SE.getSCEV(PrevIV);
2981 const SCEV *IncExpr = SE.getMinusSCEV(OperExpr, PrevExpr);
2982 if (isa<SCEVCouldNotCompute>(IncExpr) || !SE.isLoopInvariant(IncExpr, L))
2983 continue;
2984
2985 if (Chain.isProfitableIncrement(OperExpr, IncExpr, SE)) {
2986 LastIncExpr = IncExpr;
2987 break;
2988 }
2989 }
2990 // If we haven't found a chain, create a new one, unless we hit the max. Don't
2991 // bother for phi nodes, because they must be last in the chain.
2992 if (ChainIdx == NChains) {
2993 if (isa<PHINode>(UserInst))
2994 return;
2995 if (NChains >= MaxChains && !StressIVChain) {
2996 LLVM_DEBUG(dbgs() << "IV Chain Limit\n");
2997 return;
2998 }
2999 LastIncExpr = OperExpr;
3000 // IVUsers may have skipped over sign/zero extensions. We don't currently
3001 // attempt to form chains involving extensions unless they can be hoisted
3002 // into this loop's AddRec.
3003 if (!isa<SCEVAddRecExpr>(LastIncExpr))
3004 return;
3005 ++NChains;
3006 IVChainVec.push_back(IVChain(IVInc(UserInst, IVOper, LastIncExpr),
3007 OperExprBase));
3008 ChainUsersVec.resize(NChains);
3009 LLVM_DEBUG(dbgs() << "IV Chain#" << ChainIdx << " Head: (" << *UserInst
3010 << ") IV=" << *LastIncExpr << "\n");
3011 } else {
3012 LLVM_DEBUG(dbgs() << "IV Chain#" << ChainIdx << " Inc: (" << *UserInst
3013 << ") IV+" << *LastIncExpr << "\n");
3014 // Add this IV user to the end of the chain.
3015 IVChainVec[ChainIdx].add(IVInc(UserInst, IVOper, LastIncExpr));
3016 }
3017 IVChain &Chain = IVChainVec[ChainIdx];
3018
3019 SmallPtrSet<Instruction*,4> &NearUsers = ChainUsersVec[ChainIdx].NearUsers;
3020 // This chain's NearUsers become FarUsers.
3021 if (!LastIncExpr->isZero()) {
3022 ChainUsersVec[ChainIdx].FarUsers.insert(NearUsers.begin(),
3023 NearUsers.end());
3024 NearUsers.clear();
3025 }
3026
3027 // All other uses of IVOperand become near uses of the chain.
3028 // We currently ignore intermediate values within SCEV expressions, assuming
3029 // they will eventually be used be the current chain, or can be computed
3030 // from one of the chain increments. To be more precise we could
3031 // transitively follow its user and only add leaf IV users to the set.
3032 for (User *U : IVOper->users()) {
3033 Instruction *OtherUse = dyn_cast<Instruction>(U);
3034 if (!OtherUse)
3035 continue;
3036 // Uses in the chain will no longer be uses if the chain is formed.
3037 // Include the head of the chain in this iteration (not Chain.begin()).
3038 IVChain::const_iterator IncIter = Chain.Incs.begin();
3039 IVChain::const_iterator IncEnd = Chain.Incs.end();
3040 for( ; IncIter != IncEnd; ++IncIter) {
3041 if (IncIter->UserInst == OtherUse)
3042 break;
3043 }
3044 if (IncIter != IncEnd)
3045 continue;
3046
3047 if (SE.isSCEVable(OtherUse->getType())
3048 && !isa<SCEVUnknown>(SE.getSCEV(OtherUse))
3049 && IU.isIVUserOrOperand(OtherUse)) {
3050 continue;
3051 }
3052 NearUsers.insert(OtherUse);
3053 }
3054
3055 // Since this user is part of the chain, it's no longer considered a use
3056 // of the chain.
3057 ChainUsersVec[ChainIdx].FarUsers.erase(UserInst);
3058}
3059
3060/// Populate the vector of Chains.
3061///
3062/// This decreases ILP at the architecture level. Targets with ample registers,
3063/// multiple memory ports, and no register renaming probably don't want
3064/// this. However, such targets should probably disable LSR altogether.
3065///
3066/// The job of LSR is to make a reasonable choice of induction variables across
3067/// the loop. Subsequent passes can easily "unchain" computation exposing more
3068/// ILP *within the loop* if the target wants it.
3069///
3070/// Finding the best IV chain is potentially a scheduling problem. Since LSR
3071/// will not reorder memory operations, it will recognize this as a chain, but
3072/// will generate redundant IV increments. Ideally this would be corrected later
3073/// by a smart scheduler:
3074/// = A[i]
3075/// = A[i+x]
3076/// A[i] =
3077/// A[i+x] =
3078///
3079/// TODO: Walk the entire domtree within this loop, not just the path to the
3080/// loop latch. This will discover chains on side paths, but requires
3081/// maintaining multiple copies of the Chains state.
3082void LSRInstance::CollectChains() {
3083 LLVM_DEBUG(dbgs() << "Collecting IV Chains.\n");
3084 SmallVector<ChainUsers, 8> ChainUsersVec;
3085
3087 BasicBlock *LoopHeader = L->getHeader();
3088 for (DomTreeNode *Rung = DT.getNode(L->getLoopLatch());
3089 Rung->getBlock() != LoopHeader; Rung = Rung->getIDom()) {
3090 LatchPath.push_back(Rung->getBlock());
3091 }
3092 LatchPath.push_back(LoopHeader);
3093
3094 // Walk the instruction stream from the loop header to the loop latch.
3095 for (BasicBlock *BB : reverse(LatchPath)) {
3096 for (Instruction &I : *BB) {
3097 // Skip instructions that weren't seen by IVUsers analysis.
3098 if (isa<PHINode>(I) || !IU.isIVUserOrOperand(&I))
3099 continue;
3100
3101 // Ignore users that are part of a SCEV expression. This way we only
3102 // consider leaf IV Users. This effectively rediscovers a portion of
3103 // IVUsers analysis but in program order this time.
3104 if (SE.isSCEVable(I.getType()) && !isa<SCEVUnknown>(SE.getSCEV(&I)))
3105 continue;
3106
3107 // Remove this instruction from any NearUsers set it may be in.
3108 for (unsigned ChainIdx = 0, NChains = IVChainVec.size();
3109 ChainIdx < NChains; ++ChainIdx) {
3110 ChainUsersVec[ChainIdx].NearUsers.erase(&I);
3111 }
3112 // Search for operands that can be chained.
3113 SmallPtrSet<Instruction*, 4> UniqueOperands;
3114 User::op_iterator IVOpEnd = I.op_end();
3115 User::op_iterator IVOpIter = findIVOperand(I.op_begin(), IVOpEnd, L, SE);
3116 while (IVOpIter != IVOpEnd) {
3117 Instruction *IVOpInst = cast<Instruction>(*IVOpIter);
3118 if (UniqueOperands.insert(IVOpInst).second)
3119 ChainInstruction(&I, IVOpInst, ChainUsersVec);
3120 IVOpIter = findIVOperand(std::next(IVOpIter), IVOpEnd, L, SE);
3121 }
3122 } // Continue walking down the instructions.
3123 } // Continue walking down the domtree.
3124 // Visit phi backedges to determine if the chain can generate the IV postinc.
3125 for (PHINode &PN : L->getHeader()->phis()) {
3126 if (!SE.isSCEVable(PN.getType()))
3127 continue;
3128
3129 Instruction *IncV =
3130 dyn_cast<Instruction>(PN.getIncomingValueForBlock(L->getLoopLatch()));
3131 if (IncV)
3132 ChainInstruction(&PN, IncV, ChainUsersVec);
3133 }
3134 // Remove any unprofitable chains.
3135 unsigned ChainIdx = 0;
3136 for (unsigned UsersIdx = 0, NChains = IVChainVec.size();
3137 UsersIdx < NChains; ++UsersIdx) {
3138 if (!isProfitableChain(IVChainVec[UsersIdx],
3139 ChainUsersVec[UsersIdx].FarUsers, SE, TTI))
3140 continue;
3141 // Preserve the chain at UsesIdx.
3142 if (ChainIdx != UsersIdx)
3143 IVChainVec[ChainIdx] = IVChainVec[UsersIdx];
3144 FinalizeChain(IVChainVec[ChainIdx]);
3145 ++ChainIdx;
3146 }
3147 IVChainVec.resize(ChainIdx);
3148}
3149
3150void LSRInstance::FinalizeChain(IVChain &Chain) {
3151 assert(!Chain.Incs.empty() && "empty IV chains are not allowed");
3152 LLVM_DEBUG(dbgs() << "Final Chain: " << *Chain.Incs[0].UserInst << "\n");
3153
3154 for (const IVInc &Inc : Chain) {
3155 LLVM_DEBUG(dbgs() << " Inc: " << *Inc.UserInst << "\n");
3156 auto UseI = find(Inc.UserInst->operands(), Inc.IVOperand);
3157 assert(UseI != Inc.UserInst->op_end() && "cannot find IV operand");
3158 IVIncSet.insert(UseI);
3159 }
3160}
3161
3162/// Return true if the IVInc can be folded into an addressing mode.
3163static bool canFoldIVIncExpr(const SCEV *IncExpr, Instruction *UserInst,
3164 Value *Operand, const TargetTransformInfo &TTI) {
3165 const SCEVConstant *IncConst = dyn_cast<SCEVConstant>(IncExpr);
3166 if (!IncConst || !isAddressUse(TTI, UserInst, Operand))
3167 return false;
3168
3169 if (IncConst->getAPInt().getSignificantBits() > 64)
3170 return false;
3171
3172 MemAccessTy AccessTy = getAccessType(TTI, UserInst, Operand);
3173 int64_t IncOffset = IncConst->getValue()->getSExtValue();
3174 if (!isAlwaysFoldable(TTI, LSRUse::Address, AccessTy, /*BaseGV=*/nullptr,
3175 IncOffset, /*HasBaseReg=*/false))
3176 return false;
3177
3178 return true;
3179}
3180
3181/// Generate an add or subtract for each IVInc in a chain to materialize the IV
3182/// user's operand from the previous IV user's operand.
3183void LSRInstance::GenerateIVChain(const IVChain &Chain,
3185 // Find the new IVOperand for the head of the chain. It may have been replaced
3186 // by LSR.
3187 const IVInc &Head = Chain.Incs[0];
3188 User::op_iterator IVOpEnd = Head.UserInst->op_end();
3189 // findIVOperand returns IVOpEnd if it can no longer find a valid IV user.
3190 User::op_iterator IVOpIter = findIVOperand(Head.UserInst->op_begin(),
3191 IVOpEnd, L, SE);
3192 Value *IVSrc = nullptr;
3193 while (IVOpIter != IVOpEnd) {
3194 IVSrc = getWideOperand(*IVOpIter);
3195
3196 // If this operand computes the expression that the chain needs, we may use
3197 // it. (Check this after setting IVSrc which is used below.)
3198 //
3199 // Note that if Head.IncExpr is wider than IVSrc, then this phi is too
3200 // narrow for the chain, so we can no longer use it. We do allow using a
3201 // wider phi, assuming the LSR checked for free truncation. In that case we
3202 // should already have a truncate on this operand such that
3203 // getSCEV(IVSrc) == IncExpr.
3204 if (SE.getSCEV(*IVOpIter) == Head.IncExpr
3205 || SE.getSCEV(IVSrc) == Head.IncExpr) {
3206 break;
3207 }
3208 IVOpIter = findIVOperand(std::next(IVOpIter), IVOpEnd, L, SE);
3209 }
3210 if (IVOpIter == IVOpEnd) {
3211 // Gracefully give up on this chain.
3212 LLVM_DEBUG(dbgs() << "Concealed chain head: " << *Head.UserInst << "\n");
3213 return;
3214 }
3215 assert(IVSrc && "Failed to find IV chain source");
3216
3217 LLVM_DEBUG(dbgs() << "Generate chain at: " << *IVSrc << "\n");
3218 Type *IVTy = IVSrc->getType();
3219 Type *IntTy = SE.getEffectiveSCEVType(IVTy);
3220 const SCEV *LeftOverExpr = nullptr;
3221 for (const IVInc &Inc : Chain) {
3222 Instruction *InsertPt = Inc.UserInst;
3223 if (isa<PHINode>(InsertPt))
3224 InsertPt = L->getLoopLatch()->getTerminator();
3225
3226 // IVOper will replace the current IV User's operand. IVSrc is the IV
3227 // value currently held in a register.
3228 Value *IVOper = IVSrc;
3229 if (!Inc.IncExpr->isZero()) {
3230 // IncExpr was the result of subtraction of two narrow values, so must
3231 // be signed.
3232 const SCEV *IncExpr = SE.getNoopOrSignExtend(Inc.IncExpr, IntTy);
3233 LeftOverExpr = LeftOverExpr ?
3234 SE.getAddExpr(LeftOverExpr, IncExpr) : IncExpr;
3235 }
3236 if (LeftOverExpr && !LeftOverExpr->isZero()) {
3237 // Expand the IV increment.
3238 Rewriter.clearPostInc();
3239 Value *IncV = Rewriter.expandCodeFor(LeftOverExpr, IntTy, InsertPt);
3240 const SCEV *IVOperExpr = SE.getAddExpr(SE.getUnknown(IVSrc),
3241 SE.getUnknown(IncV));
3242 IVOper = Rewriter.expandCodeFor(IVOperExpr, IVTy, InsertPt);
3243
3244 // If an IV increment can't be folded, use it as the next IV value.
3245 if (!canFoldIVIncExpr(LeftOverExpr, Inc.UserInst, Inc.IVOperand, TTI)) {
3246 assert(IVTy == IVOper->getType() && "inconsistent IV increment type");
3247 IVSrc = IVOper;
3248 LeftOverExpr = nullptr;
3249 }
3250 }
3251 Type *OperTy = Inc.IVOperand->getType();
3252 if (IVTy != OperTy) {
3253 assert(SE.getTypeSizeInBits(IVTy) >= SE.getTypeSizeInBits(OperTy) &&
3254 "cannot extend a chained IV");
3255 IRBuilder<> Builder(InsertPt);
3256 IVOper = Builder.CreateTruncOrBitCast(IVOper, OperTy, "lsr.chain");
3257 }
3258 Inc.UserInst->replaceUsesOfWith(Inc.IVOperand, IVOper);
3259 if (auto *OperandIsInstr = dyn_cast<Instruction>(Inc.IVOperand))
3260 DeadInsts.emplace_back(OperandIsInstr);
3261 }
3262 // If LSR created a new, wider phi, we may also replace its postinc. We only
3263 // do this if we also found a wide value for the head of the chain.
3264 if (isa<PHINode>(Chain.tailUserInst())) {
3265 for (PHINode &Phi : L->getHeader()->phis()) {
3266 if (Phi.getType() != IVSrc->getType())
3267 continue;
3268 Instruction *PostIncV = dyn_cast<Instruction>(
3269 Phi.getIncomingValueForBlock(L->getLoopLatch()));
3270 if (!PostIncV || (SE.getSCEV(PostIncV) != SE.getSCEV(IVSrc)))
3271 continue;
3272 Value *IVOper = IVSrc;
3273 Type *PostIncTy = PostIncV->getType();
3274 if (IVTy != PostIncTy) {
3275 assert(PostIncTy->isPointerTy() && "mixing int/ptr IV types");
3276 IRBuilder<> Builder(L->getLoopLatch()->getTerminator());
3277 Builder.SetCurrentDebugLocation(PostIncV->getDebugLoc());
3278 IVOper = Builder.CreatePointerCast(IVSrc, PostIncTy, "lsr.chain");
3279 }
3280 Phi.replaceUsesOfWith(PostIncV, IVOper);
3281 DeadInsts.emplace_back(PostIncV);
3282 }
3283 }
3284}
3285
3286void LSRInstance::CollectFixupsAndInitialFormulae() {
3287 BranchInst *ExitBranch = nullptr;
3288 bool SaveCmp = TTI.canSaveCmp(L, &ExitBranch, &SE, &LI, &DT, &AC, &TLI);
3289
3290 // For calculating baseline cost
3292 DenseSet<const SCEV *> VisitedRegs;
3293 DenseSet<size_t> VisitedLSRUse;
3294
3295 for (const IVStrideUse &U : IU) {
3296 Instruction *UserInst = U.getUser();
3297 // Skip IV users that are part of profitable IV Chains.
3298 User::op_iterator UseI =
3299 find(UserInst->operands(), U.getOperandValToReplace());
3300 assert(UseI != UserInst->op_end() && "cannot find IV operand");
3301 if (IVIncSet.count(UseI)) {
3302 LLVM_DEBUG(dbgs() << "Use is in profitable chain: " << **UseI << '\n');
3303 continue;
3304 }
3305
3306 LSRUse::KindType Kind = LSRUse::Basic;
3307 MemAccessTy AccessTy;
3308 if (isAddressUse(TTI, UserInst, U.getOperandValToReplace())) {
3309 Kind = LSRUse::Address;
3310 AccessTy = getAccessType(TTI, UserInst, U.getOperandValToReplace());
3311 }
3312
3313 const SCEV *S = IU.getExpr(U);
3314 if (!S)
3315 continue;
3316 PostIncLoopSet TmpPostIncLoops = U.getPostIncLoops();
3317
3318 // Equality (== and !=) ICmps are special. We can rewrite (i == N) as
3319 // (N - i == 0), and this allows (N - i) to be the expression that we work
3320 // with rather than just N or i, so we can consider the register
3321 // requirements for both N and i at the same time. Limiting this code to
3322 // equality icmps is not a problem because all interesting loops use
3323 // equality icmps, thanks to IndVarSimplify.
3324 if (ICmpInst *CI = dyn_cast<ICmpInst>(UserInst)) {
3325 // If CI can be saved in some target, like replaced inside hardware loop
3326 // in PowerPC, no need to generate initial formulae for it.
3327 if (SaveCmp && CI == dyn_cast<ICmpInst>(ExitBranch->getCondition()))
3328 continue;
3329 if (CI->isEquality()) {
3330 // Swap the operands if needed to put the OperandValToReplace on the
3331 // left, for consistency.
3332 Value *NV = CI->getOperand(1);
3333 if (NV == U.getOperandValToReplace()) {
3334 CI->setOperand(1, CI->getOperand(0));
3335 CI->setOperand(0, NV);
3336 NV = CI->getOperand(1);
3337 Changed = true;
3338 }
3339
3340 // x == y --> x - y == 0
3341 const SCEV *N = SE.getSCEV(NV);
3342 if (SE.isLoopInvariant(N, L) && Rewriter.isSafeToExpand(N) &&
3343 (!NV->getType()->isPointerTy() ||
3344 SE.getPointerBase(N) == SE.getPointerBase(S))) {
3345 // S is normalized, so normalize N before folding it into S
3346 // to keep the result normalized.
3347 N = normalizeForPostIncUse(N, TmpPostIncLoops, SE);
3348 if (!N)
3349 continue;
3350 Kind = LSRUse::ICmpZero;
3351 S = SE.getMinusSCEV(N, S);
3352 } else if (L->isLoopInvariant(NV) &&
3353 (!isa<Instruction>(NV) ||
3354 DT.dominates(cast<Instruction>(NV), L->getHeader())) &&
3355 !NV->getType()->isPointerTy()) {
3356 // If we can't generally expand the expression (e.g. it contains
3357 // a divide), but it is already at a loop invariant point before the
3358 // loop, wrap it in an unknown (to prevent the expander from trying
3359 // to re-expand in a potentially unsafe way.) The restriction to
3360 // integer types is required because the unknown hides the base, and
3361 // SCEV can't compute the difference of two unknown pointers.
3362 N = SE.getUnknown(NV);
3363 N = normalizeForPostIncUse(N, TmpPostIncLoops, SE);
3364 if (!N)
3365 continue;
3366 Kind = LSRUse::ICmpZero;
3367 S = SE.getMinusSCEV(N, S);
3368 assert(!isa<SCEVCouldNotCompute>(S));
3369 }
3370
3371 // -1 and the negations of all interesting strides (except the negation
3372 // of -1) are now also interesting.
3373 for (size_t i = 0, e = Factors.size(); i != e; ++i)
3374 if (Factors[i] != -1)
3375 Factors.insert(-(uint64_t)Factors[i]);
3376 Factors.insert(-1);
3377 }
3378 }
3379
3380 // Get or create an LSRUse.
3381 std::pair<size_t, int64_t> P = getUse(S, Kind, AccessTy);
3382 size_t LUIdx = P.first;
3383 int64_t Offset = P.second;
3384 LSRUse &LU = Uses[LUIdx];
3385
3386 // Record the fixup.
3387 LSRFixup &LF = LU.getNewFixup();
3388 LF.UserInst = UserInst;
3389 LF.OperandValToReplace = U.getOperandValToReplace();
3390 LF.PostIncLoops = TmpPostIncLoops;
3391 LF.Offset = Offset;
3392 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
3393
3394 // Create SCEV as Formula for calculating baseline cost
3395 if (!VisitedLSRUse.count(LUIdx) && !LF.isUseFullyOutsideLoop(L)) {
3396 Formula F;
3397 F.initialMatch(S, L, SE);
3398 BaselineCost.RateFormula(F, Regs, VisitedRegs, LU);
3399 VisitedLSRUse.insert(LUIdx);
3400 }
3401
3402 if (!LU.WidestFixupType ||
3403 SE.getTypeSizeInBits(LU.WidestFixupType) <
3404 SE.getTypeSizeInBits(LF.OperandValToReplace->getType()))
3405 LU.WidestFixupType = LF.OperandValToReplace->getType();
3406
3407 // If this is the first use of this LSRUse, give it a formula.
3408 if (LU.Formulae.empty()) {
3409 InsertInitialFormula(S, LU, LUIdx);
3410 CountRegisters(LU.Formulae.back(), LUIdx);
3411 }
3412 }
3413
3414 LLVM_DEBUG(print_fixups(dbgs()));
3415}
3416
3417/// Insert a formula for the given expression into the given use, separating out
3418/// loop-variant portions from loop-invariant and loop-computable portions.
3419void LSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU,
3420 size_t LUIdx) {
3421 // Mark uses whose expressions cannot be expanded.
3422 if (!Rewriter.isSafeToExpand(S))
3423 LU.RigidFormula = true;
3424
3425 Formula F;
3426 F.initialMatch(S, L, SE);
3427 bool Inserted = InsertFormula(LU, LUIdx, F);
3428 assert(Inserted && "Initial formula already exists!"); (void)Inserted;
3429}
3430
3431/// Insert a simple single-register formula for the given expression into the
3432/// given use.
3433void
3434LSRInstance::InsertSupplementalFormula(const SCEV *S,
3435 LSRUse &LU, size_t LUIdx) {
3436 Formula F;
3437 F.BaseRegs.push_back(S);
3438 F.HasBaseReg = true;
3439 bool Inserted = InsertFormula(LU, LUIdx, F);
3440 assert(Inserted && "Supplemental formula already exists!"); (void)Inserted;
3441}
3442
3443/// Note which registers are used by the given formula, updating RegUses.
3444void LSRInstance::CountRegisters(const Formula &F, size_t LUIdx) {
3445 if (F.ScaledReg)
3446 RegUses.countRegister(F.ScaledReg, LUIdx);
3447 for (const SCEV *BaseReg : F.BaseRegs)
3448 RegUses.countRegister(BaseReg, LUIdx);
3449}
3450
3451/// If the given formula has not yet been inserted, add it to the list, and
3452/// return true. Return false otherwise.
3453bool LSRInstance::InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F) {
3454 // Do not insert formula that we will not be able to expand.
3455 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F) &&
3456 "Formula is illegal");
3457
3458 if (!LU.InsertFormula(F, *L))
3459 return false;
3460
3461 CountRegisters(F, LUIdx);
3462 return true;
3463}
3464
3465/// Check for other uses of loop-invariant values which we're tracking. These
3466/// other uses will pin these values in registers, making them less profitable
3467/// for elimination.
3468/// TODO: This currently misses non-constant addrec step registers.
3469/// TODO: Should this give more weight to users inside the loop?
3470void
3471LSRInstance::CollectLoopInvariantFixupsAndFormulae() {
3472 SmallVector<const SCEV *, 8> Worklist(RegUses.begin(), RegUses.end());
3474
3475 // Don't collect outside uses if we are favoring postinc - the instructions in
3476 // the loop are more important than the ones outside of it.
3477 if (AMK == TTI::AMK_PostIndexed)
3478 return;
3479
3480 while (!Worklist.empty()) {
3481 const SCEV *S = Worklist.pop_back_val();
3482
3483 // Don't process the same SCEV twice
3484 if (!Visited.insert(S).second)
3485 continue;
3486
3487 if (const SCEVNAryExpr *N = dyn_cast<SCEVNAryExpr>(S))
3488 append_range(Worklist, N->operands());
3489 else if (const SCEVIntegralCastExpr *C = dyn_cast<SCEVIntegralCastExpr>(S))
3490 Worklist.push_back(C->getOperand());
3491 else if (const SCEVUDivExpr *D = dyn_cast<SCEVUDivExpr>(S)) {
3492 Worklist.push_back(D->getLHS());
3493 Worklist.push_back(D->getRHS());
3494 } else if (const SCEVUnknown *US = dyn_cast<SCEVUnknown>(S)) {
3495 const Value *V = US->getValue();
3496 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
3497 // Look for instructions defined outside the loop.
3498 if (L->contains(Inst)) continue;
3499 } else if (isa<Constant>(V))
3500 // Constants can be re-materialized.
3501 continue;
3502 for (const Use &U : V->uses()) {
3503 const Instruction *UserInst = dyn_cast<Instruction>(U.getUser());
3504 // Ignore non-instructions.
3505 if (!UserInst)
3506 continue;
3507 // Don't bother if the instruction is an EHPad.
3508 if (UserInst->isEHPad())
3509 continue;
3510 // Ignore instructions in other functions (as can happen with
3511 // Constants).
3512 if (UserInst->getParent()->getParent() != L->getHeader()->getParent())
3513 continue;
3514 // Ignore instructions not dominated by the loop.
3515 const BasicBlock *UseBB = !isa<PHINode>(UserInst) ?
3516 UserInst->getParent() :
3517 cast<PHINode>(UserInst)->getIncomingBlock(
3519 if (!DT.dominates(L->getHeader(), UseBB))
3520 continue;
3521 // Don't bother if the instruction is in a BB which ends in an EHPad.
3522 if (UseBB->getTerminator()->isEHPad())
3523 continue;
3524
3525 // Ignore cases in which the currently-examined value could come from
3526 // a basic block terminated with an EHPad. This checks all incoming
3527 // blocks of the phi node since it is possible that the same incoming
3528 // value comes from multiple basic blocks, only some of which may end
3529 // in an EHPad. If any of them do, a subsequent rewrite attempt by this
3530 // pass would try to insert instructions into an EHPad, hitting an
3531 // assertion.
3532 if (isa<PHINode>(UserInst)) {
3533 const auto *PhiNode = cast<PHINode>(UserInst);
3534 bool HasIncompatibleEHPTerminatedBlock = false;
3535 llvm::Value *ExpectedValue = U;
3536 for (unsigned int I = 0; I < PhiNode->getNumIncomingValues(); I++) {
3537 if (PhiNode->getIncomingValue(I) == ExpectedValue) {
3538 if (PhiNode->getIncomingBlock(I)->getTerminator()->isEHPad()) {
3539 HasIncompatibleEHPTerminatedBlock = true;
3540 break;
3541 }
3542 }
3543 }
3544 if (HasIncompatibleEHPTerminatedBlock) {
3545 continue;
3546 }
3547 }
3548
3549 // Don't bother rewriting PHIs in catchswitch blocks.
3550 if (isa<CatchSwitchInst>(UserInst->getParent()->getTerminator()))
3551 continue;
3552 // Ignore uses which are part of other SCEV expressions, to avoid
3553 // analyzing them multiple times.
3554 if (SE.isSCEVable(UserInst->getType())) {
3555 const SCEV *UserS = SE.getSCEV(const_cast<Instruction *>(UserInst));
3556 // If the user is a no-op, look through to its uses.
3557 if (!isa<SCEVUnknown>(UserS))
3558 continue;
3559 if (UserS == US) {
3560 Worklist.push_back(
3561 SE.getUnknown(const_cast<Instruction *>(UserInst)));
3562 continue;
3563 }
3564 }
3565 // Ignore icmp instructions which are already being analyzed.
3566 if (const ICmpInst *ICI = dyn_cast<ICmpInst>(UserInst)) {
3567 unsigned OtherIdx = !U.getOperandNo();
3568 Value *OtherOp = const_cast<Value *>(ICI->getOperand(OtherIdx));
3569 if (SE.hasComputableLoopEvolution(SE.getSCEV(OtherOp), L))
3570 continue;
3571 }
3572
3573 std::pair<size_t, int64_t> P = getUse(
3574 S, LSRUse::Basic, MemAccessTy());
3575 size_t LUIdx = P.first;
3576 int64_t Offset = P.second;
3577 LSRUse &LU = Uses[LUIdx];
3578 LSRFixup &LF = LU.getNewFixup();
3579 LF.UserInst = const_cast<Instruction *>(UserInst);
3580 LF.OperandValToReplace = U;
3581 LF.Offset = Offset;
3582 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
3583 if (!LU.WidestFixupType ||
3584 SE.getTypeSizeInBits(LU.WidestFixupType) <
3585 SE.getTypeSizeInBits(LF.OperandValToReplace->getType()))
3586 LU.WidestFixupType = LF.OperandValToReplace->getType();
3587 InsertSupplementalFormula(US, LU, LUIdx);
3588 CountRegisters(LU.Formulae.back(), Uses.size() - 1);
3589 break;
3590 }
3591 }
3592 }
3593}
3594
3595/// Split S into subexpressions which can be pulled out into separate
3596/// registers. If C is non-null, multiply each subexpression by C.
3597///
3598/// Return remainder expression after factoring the subexpressions captured by
3599/// Ops. If Ops is complete, return NULL.
3600static const SCEV *CollectSubexprs(const SCEV *S, const SCEVConstant *C,
3602 const Loop *L,
3603 ScalarEvolution &SE,
3604 unsigned Depth = 0) {
3605 // Arbitrarily cap recursion to protect compile time.
3606 if (Depth >= 3)
3607 return S;
3608
3609 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
3610 // Break out add operands.
3611 for (const SCEV *S : Add->operands()) {
3612 const SCEV *Remainder = CollectSubexprs(S, C, Ops, L, SE, Depth+1);
3613 if (Remainder)
3614 Ops.push_back(C ? SE.getMulExpr(C, Remainder) : Remainder);
3615 }
3616 return nullptr;
3617 } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
3618 // Split a non-zero base out of an addrec.
3619 if (AR->getStart()->isZero() || !AR->isAffine())
3620 return S;
3621
3622 const SCEV *Remainder = CollectSubexprs(AR->getStart(),
3623 C, Ops, L, SE, Depth+1);
3624 // Split the non-zero AddRec unless it is part of a nested recurrence that
3625 // does not pertain to this loop.
3626 if (Remainder && (AR->getLoop() == L || !isa<SCEVAddRecExpr>(Remainder))) {
3627 Ops.push_back(C ? SE.getMulExpr(C, Remainder) : Remainder);
3628 Remainder = nullptr;
3629 }
3630 if (Remainder != AR->getStart()) {
3631 if (!Remainder)
3632 Remainder = SE.getConstant(AR->getType(), 0);
3633 return SE.getAddRecExpr(Remainder,
3634 AR->getStepRecurrence(SE),
3635 AR->getLoop(),
3636 //FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
3638 }
3639 } else if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) {
3640 // Break (C * (a + b + c)) into C*a + C*b + C*c.
3641 if (Mul->getNumOperands() != 2)
3642 return S;
3643 if (const SCEVConstant *Op0 =
3644 dyn_cast<SCEVConstant>(Mul->getOperand(0))) {
3645 C = C ? cast<SCEVConstant>(SE.getMulExpr(C, Op0)) : Op0;
3646 const SCEV *Remainder =
3647 CollectSubexprs(Mul->getOperand(1), C, Ops, L, SE, Depth+1);
3648 if (Remainder)
3649 Ops.push_back(SE.getMulExpr(C, Remainder));
3650 return nullptr;
3651 }
3652 }
3653 return S;
3654}
3655
3656/// Return true if the SCEV represents a value that may end up as a
3657/// post-increment operation.
3659 LSRUse &LU, const SCEV *S, const Loop *L,
3660 ScalarEvolution &SE) {
3661 if (LU.Kind != LSRUse::Address ||
3662 !LU.AccessTy.getType()->isIntOrIntVectorTy())
3663 return false;
3664 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S);
3665 if (!AR)
3666 return false;
3667 const SCEV *LoopStep = AR->getStepRecurrence(SE);
3668 if (!isa<SCEVConstant>(LoopStep))
3669 return false;
3670 // Check if a post-indexed load/store can be used.
3673 const SCEV *LoopStart = AR->getStart();
3674 if (!isa<SCEVConstant>(LoopStart) && SE.isLoopInvariant(LoopStart, L))
3675 return true;
3676 }
3677 return false;
3678}
3679
3680/// Helper function for LSRInstance::GenerateReassociations.
3681void LSRInstance::GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
3682 const Formula &Base,
3683 unsigned Depth, size_t Idx,
3684 bool IsScaledReg) {
3685 const SCEV *BaseReg = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
3686 // Don't generate reassociations for the base register of a value that
3687 // may generate a post-increment operator. The reason is that the
3688 // reassociations cause extra base+register formula to be created,
3689 // and possibly chosen, but the post-increment is more efficient.
3690 if (AMK == TTI::AMK_PostIndexed && mayUsePostIncMode(TTI, LU, BaseReg, L, SE))
3691 return;
3693 const SCEV *Remainder = CollectSubexprs(BaseReg, nullptr, AddOps, L, SE);
3694 if (Remainder)
3695 AddOps.push_back(Remainder);
3696
3697 if (AddOps.size() == 1)
3698 return;
3699
3701 JE = AddOps.end();
3702 J != JE; ++J) {
3703 // Loop-variant "unknown" values are uninteresting; we won't be able to
3704 // do anything meaningful with them.
3705 if (isa<SCEVUnknown>(*J) && !SE.isLoopInvariant(*J, L))
3706 continue;
3707
3708 // Don't pull a constant into a register if the constant could be folded
3709 // into an immediate field.
3710 if (isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind,
3711 LU.AccessTy, *J, Base.getNumRegs() > 1))
3712 continue;
3713
3714 // Collect all operands except *J.
3715 SmallVector<const SCEV *, 8> InnerAddOps(
3716 ((const SmallVector<const SCEV *, 8> &)AddOps).begin(), J);
3717 InnerAddOps.append(std::next(J),
3718 ((const SmallVector<const SCEV *, 8> &)AddOps).end());
3719
3720 // Don't leave just a constant behind in a register if the constant could
3721 // be folded into an immediate field.
3722 if (InnerAddOps.size() == 1 &&
3723 isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind,
3724 LU.AccessTy, InnerAddOps[0], Base.getNumRegs() > 1))
3725 continue;
3726
3727 const SCEV *InnerSum = SE.getAddExpr(InnerAddOps);
3728 if (InnerSum->isZero())
3729 continue;
3730 Formula F = Base;
3731
3732 // Add the remaining pieces of the add back into the new formula.
3733 const SCEVConstant *InnerSumSC = dyn_cast<SCEVConstant>(InnerSum);
3734 if (InnerSumSC && SE.getTypeSizeInBits(InnerSumSC->getType()) <= 64 &&
3735 TTI.isLegalAddImmediate((uint64_t)F.UnfoldedOffset +
3736 InnerSumSC->getValue()->getZExtValue())) {
3737 F.UnfoldedOffset =
3738 (uint64_t)F.UnfoldedOffset + InnerSumSC->getValue()->getZExtValue();
3739 if (IsScaledReg)
3740 F.ScaledReg = nullptr;
3741 else
3742 F.BaseRegs.erase(F.BaseRegs.begin() + Idx);
3743 } else if (IsScaledReg)
3744 F.ScaledReg = InnerSum;
3745 else
3746 F.BaseRegs[Idx] = InnerSum;
3747
3748 // Add J as its own register, or an unfolded immediate.
3749 const SCEVConstant *SC = dyn_cast<SCEVConstant>(*J);
3750 if (SC && SE.getTypeSizeInBits(SC->getType()) <= 64 &&
3751 TTI.isLegalAddImmediate((uint64_t)F.UnfoldedOffset +
3752 SC->getValue()->getZExtValue()))
3753 F.UnfoldedOffset =
3754 (uint64_t)F.UnfoldedOffset + SC->getValue()->getZExtValue();
3755 else
3756 F.BaseRegs.push_back(*J);
3757 // We may have changed the number of register in base regs, adjust the
3758 // formula accordingly.
3759 F.canonicalize(*L);
3760
3761 if (InsertFormula(LU, LUIdx, F))
3762 // If that formula hadn't been seen before, recurse to find more like
3763 // it.
3764 // Add check on Log16(AddOps.size()) - same as Log2_32(AddOps.size()) >> 2)
3765 // Because just Depth is not enough to bound compile time.
3766 // This means that every time AddOps.size() is greater 16^x we will add
3767 // x to Depth.
3768 GenerateReassociations(LU, LUIdx, LU.Formulae.back(),
3769 Depth + 1 + (Log2_32(AddOps.size()) >> 2));
3770 }
3771}
3772
3773/// Split out subexpressions from adds and the bases of addrecs.
3774void LSRInstance::GenerateReassociations(LSRUse &LU, unsigned LUIdx,
3775 Formula Base, unsigned Depth) {
3776 assert(Base.isCanonical(*L) && "Input must be in the canonical form");
3777 // Arbitrarily cap recursion to protect compile time.
3778 if (Depth >= 3)
3779 return;
3780
3781 for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i)
3782 GenerateReassociationsImpl(LU, LUIdx, Base, Depth, i);
3783
3784 if (Base.Scale == 1)
3785 GenerateReassociationsImpl(LU, LUIdx, Base, Depth,
3786 /* Idx */ -1, /* IsScaledReg */ true);
3787}
3788
3789/// Generate a formula consisting of all of the loop-dominating registers added
3790/// into a single register.
3791void LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx,
3792 Formula Base) {
3793 // This method is only interesting on a plurality of registers.
3794 if (Base.BaseRegs.size() + (Base.Scale == 1) +
3795 (Base.UnfoldedOffset != 0) <= 1)
3796 return;
3797
3798 // Flatten the representation, i.e., reg1 + 1*reg2 => reg1 + reg2, before
3799 // processing the formula.
3800 Base.unscale();
3802 Formula NewBase = Base;
3803 NewBase.BaseRegs.clear();
3804 Type *CombinedIntegerType = nullptr;
3805 for (const SCEV *BaseReg : Base.BaseRegs) {
3806 if (SE.properlyDominates(BaseReg, L->getHeader()) &&
3807 !SE.hasComputableLoopEvolution(BaseReg, L)) {
3808 if (!CombinedIntegerType)
3809 CombinedIntegerType = SE.getEffectiveSCEVType(BaseReg->getType());
3810 Ops.push_back(BaseReg);
3811 }
3812 else
3813 NewBase.BaseRegs.push_back(BaseReg);
3814 }
3815
3816 // If no register is relevant, we're done.
3817 if (Ops.size() == 0)
3818 return;
3819
3820 // Utility function for generating the required variants of the combined
3821 // registers.
3822 auto GenerateFormula = [&](const SCEV *Sum) {
3823 Formula F = NewBase;
3824
3825 // TODO: If Sum is zero, it probably means ScalarEvolution missed an
3826 // opportunity to fold something. For now, just ignore such cases
3827 // rather than proceed with zero in a register.
3828 if (Sum->isZero())
3829 return;
3830
3831 F.BaseRegs.push_back(Sum);
3832 F.canonicalize(*L);
3833 (void)InsertFormula(LU, LUIdx, F);
3834 };
3835
3836 // If we collected at least two registers, generate a formula combining them.
3837 if (Ops.size() > 1) {
3838 SmallVector<const SCEV *, 4> OpsCopy(Ops); // Don't let SE modify Ops.
3839 GenerateFormula(SE.getAddExpr(OpsCopy));
3840 }
3841
3842 // If we have an unfolded offset, generate a formula combining it with the
3843 // registers collected.
3844 if (NewBase.UnfoldedOffset) {
3845 assert(CombinedIntegerType && "Missing a type for the unfolded offset");
3846 Ops.push_back(SE.getConstant(CombinedIntegerType, NewBase.UnfoldedOffset,
3847 true));
3848 NewBase.UnfoldedOffset = 0;
3849 GenerateFormula(SE.getAddExpr(Ops));
3850 }
3851}
3852
3853/// Helper function for LSRInstance::GenerateSymbolicOffsets.
3854void LSRInstance::GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
3855 const Formula &Base, size_t Idx,
3856 bool IsScaledReg) {
3857 const SCEV *G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
3858 GlobalValue *GV = ExtractSymbol(G, SE);
3859 if (G->isZero() || !GV)
3860 return;
3861 Formula F = Base;
3862 F.BaseGV = GV;
3863 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F))
3864 return;
3865 if (IsScaledReg)
3866 F.ScaledReg = G;
3867 else
3868 F.BaseRegs[Idx] = G;
3869 (void)InsertFormula(LU, LUIdx, F);
3870}
3871
3872/// Generate reuse formulae using symbolic offsets.
3873void LSRInstance::GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx,
3874 Formula Base) {
3875 // We can't add a symbolic offset if the address already contains one.
3876 if (Base.BaseGV) return;
3877
3878 for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i)
3879 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, i);
3880 if (Base.Scale == 1)
3881 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, /* Idx */ -1,
3882 /* IsScaledReg */ true);
3883}
3884
3885/// Helper function for LSRInstance::GenerateConstantOffsets.
3886void LSRInstance::GenerateConstantOffsetsImpl(
3887 LSRUse &LU, unsigned LUIdx, const Formula &Base,
3888 const SmallVectorImpl<int64_t> &Worklist, size_t Idx, bool IsScaledReg) {
3889
3890 auto GenerateOffset = [&](const SCEV *G, int64_t Offset) {
3891 Formula F = Base;
3892 F.BaseOffset = (uint64_t)Base.BaseOffset - Offset;
3893
3894 if (isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F)) {
3895 // Add the offset to the base register.
3896 const SCEV *NewG = SE.getAddExpr(SE.getConstant(G->getType(), Offset), G);
3897 // If it cancelled out, drop the base register, otherwise update it.
3898 if (NewG->isZero()) {
3899 if (IsScaledReg) {
3900 F.Scale = 0;
3901 F.ScaledReg = nullptr;
3902 } else
3903 F.deleteBaseReg(F.BaseRegs[Idx]);
3904 F.canonicalize(*L);
3905 } else if (IsScaledReg)
3906 F.ScaledReg = NewG;
3907 else
3908 F.BaseRegs[Idx] = NewG;
3909
3910 (void)InsertFormula(LU, LUIdx, F);
3911 }
3912 };
3913
3914 const SCEV *G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
3915
3916 // With constant offsets and constant steps, we can generate pre-inc
3917 // accesses by having the offset equal the step. So, for access #0 with a
3918 // step of 8, we generate a G - 8 base which would require the first access
3919 // to be ((G - 8) + 8),+,8. The pre-indexed access then updates the pointer
3920 // for itself and hopefully becomes the base for other accesses. This means
3921 // means that a single pre-indexed access can be generated to become the new
3922 // base pointer for each iteration of the loop, resulting in no extra add/sub
3923 // instructions for pointer updating.
3924 if (AMK == TTI::AMK_PreIndexed && LU.Kind == LSRUse::Address) {
3925 if (auto *GAR = dyn_cast<SCEVAddRecExpr>(G)) {
3926 if (auto *StepRec =
3927 dyn_cast<SCEVConstant>(GAR->getStepRecurrence(SE))) {
3928 const APInt &StepInt = StepRec->getAPInt();
3929 int64_t Step = StepInt.isNegative() ?
3930 StepInt.getSExtValue() : StepInt.getZExtValue();
3931
3932 for (int64_t Offset : Worklist) {
3933 Offset -= Step;
3934 GenerateOffset(G, Offset);
3935 }
3936 }
3937 }
3938 }
3939 for (int64_t Offset : Worklist)
3940 GenerateOffset(G, Offset);
3941
3942 int64_t Imm = ExtractImmediate(G, SE);
3943 if (G->isZero() || Imm == 0)
3944 return;
3945 Formula F = Base;
3946 F.BaseOffset = (uint64_t)F.BaseOffset + Imm;
3947 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F))
3948 return;
3949 if (IsScaledReg) {
3950 F.ScaledReg = G;
3951 } else {
3952 F.BaseRegs[Idx] = G;
3953 // We may generate non canonical Formula if G is a recurrent expr reg
3954 // related with current loop while F.ScaledReg is not.
3955 F.canonicalize(*L);
3956 }
3957 (void)InsertFormula(LU, LUIdx, F);
3958}
3959
3960/// GenerateConstantOffsets - Generate reuse formulae using symbolic offsets.
3961void LSRInstance::GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx,
3962 Formula Base) {
3963 // TODO: For now, just add the min and max offset, because it usually isn't
3964 // worthwhile looking at everything inbetween.
3965 SmallVector<int64_t, 2> Worklist;
3966 Worklist.push_back(LU.MinOffset);
3967 if (LU.MaxOffset != LU.MinOffset)
3968 Worklist.push_back(LU.MaxOffset);
3969
3970 for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i)
3971 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, i);
3972 if (Base.Scale == 1)
3973 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, /* Idx */ -1,
3974 /* IsScaledReg */ true);
3975}
3976
3977/// For ICmpZero, check to see if we can scale up the comparison. For example, x
3978/// == y -> x*c == y*c.
3979void LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx,
3980 Formula Base) {
3981 if (LU.Kind != LSRUse::ICmpZero) return;
3982
3983 // Determine the integer type for the base formula.
3984 Type *IntTy = Base.getType();
3985 if (!IntTy) return;
3986 if (SE.getTypeSizeInBits(IntTy) > 64) return;
3987
3988 // Don't do this if there is more than one offset.
3989 if (LU.MinOffset != LU.MaxOffset) return;
3990
3991 // Check if transformation is valid. It is illegal to multiply pointer.
3992 if (Base.ScaledReg && Base.ScaledReg->getType()->isPointerTy())
3993 return;
3994 for (const SCEV *BaseReg : Base.BaseRegs)
3995 if (BaseReg->getType()->isPointerTy())
3996 return;
3997 assert(!Base.BaseGV && "ICmpZero use is not legal!");
3998
3999 // Check each interesting stride.
4000 for (int64_t Factor : Factors) {
4001 // Check that Factor can be represented by IntTy
4002 if (!ConstantInt::isValueValidForType(IntTy, Factor))
4003 continue;
4004 // Check that the multiplication doesn't overflow.
4005 if (Base.BaseOffset == std::numeric_limits<int64_t>::min() && Factor == -1)
4006 continue;
4007 int64_t NewBaseOffset = (uint64_t)Base.BaseOffset * Factor;
4008 assert(Factor != 0 && "Zero factor not expected!");
4009 if (NewBaseOffset / Factor != Base.BaseOffset)
4010 continue;
4011 // If the offset will be truncated at this use, check that it is in bounds.
4012 if (!IntTy->isPointerTy() &&
4013 !ConstantInt::isValueValidForType(IntTy, NewBaseOffset))
4014 continue;
4015
4016 // Check that multiplying with the use offset doesn't overflow.
4017 int64_t Offset = LU.MinOffset;
4018 if (Offset == std::numeric_limits<int64_t>::min() && Factor == -1)
4019 continue;
4020 Offset = (uint64_t)Offset * Factor;
4021 if (Offset / Factor != LU.MinOffset)
4022 continue;
4023 // If the offset will be truncated at this use, check that it is in bounds.
4024 if (!IntTy->isPointerTy() &&
4026 continue;
4027
4028 Formula F = Base;
4029 F.BaseOffset = NewBaseOffset;
4030
4031 // Check that this scale is legal.
4032 if (!isLegalUse(TTI, Offset, Offset, LU.Kind, LU.AccessTy, F))
4033 continue;
4034
4035 // Compensate for the use having MinOffset built into it.
4036 F.BaseOffset = (uint64_t)F.BaseOffset + Offset - LU.MinOffset;
4037
4038 const SCEV *FactorS = SE.getConstant(IntTy, Factor);
4039
4040 // Check that multiplying with each base register doesn't overflow.
4041 for (size_t i = 0, e = F.BaseRegs.size(); i != e; ++i) {
4042 F.BaseRegs[i] = SE.getMulExpr(F.BaseRegs[i], FactorS);
4043 if (getExactSDiv(F.BaseRegs[i], FactorS, SE) != Base.BaseRegs[i])
4044 goto next;
4045 }
4046
4047 // Check that multiplying with the scaled register doesn't overflow.
4048 if (F.ScaledReg) {
4049 F.ScaledReg = SE.getMulExpr(F.ScaledReg, FactorS);
4050 if (getExactSDiv(F.ScaledReg, FactorS, SE) != Base.ScaledReg)
4051 continue;
4052 }
4053
4054 // Check that multiplying with the unfolded offset doesn't overflow.
4055 if (F.UnfoldedOffset != 0) {
4056 if (F.UnfoldedOffset == std::numeric_limits<int64_t>::min() &&
4057 Factor == -1)
4058 continue;
4059 F.UnfoldedOffset = (uint64_t)F.UnfoldedOffset * Factor;
4060 if (F.UnfoldedOffset / Factor != Base.UnfoldedOffset)
4061 continue;
4062 // If the offset will be truncated, check that it is in bounds.
4063 if (!IntTy->isPointerTy() &&
4064 !ConstantInt::isValueValidForType(IntTy, F.UnfoldedOffset))
4065 continue;
4066 }
4067
4068 // If we make it here and it's legal, add it.
4069 (void)InsertFormula(LU, LUIdx, F);
4070 next:;
4071 }
4072}
4073
4074/// Generate stride factor reuse formulae by making use of scaled-offset address
4075/// modes, for example.
4076void LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) {
4077 // Determine the integer type for the base formula.
4078 Type *IntTy = Base.getType();
4079 if (!IntTy) return;
4080
4081 // If this Formula already has a scaled register, we can't add another one.
4082 // Try to unscale the formula to generate a better scale.
4083 if (Base.Scale != 0 && !Base.unscale())
4084 return;
4085
4086 assert(Base.Scale == 0 && "unscale did not did its job!");
4087
4088 // Check each interesting stride.
4089 for (int64_t Factor : Factors) {
4090 Base.Scale = Factor;
4091 Base.HasBaseReg = Base.BaseRegs.size() > 1;
4092 // Check whether this scale is going to be legal.
4093 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
4094 Base)) {
4095 // As a special-case, handle special out-of-loop Basic users specially.
4096 // TODO: Reconsider this special case.
4097 if (LU.Kind == LSRUse::Basic &&
4098 isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LSRUse::Special,
4099 LU.AccessTy, Base) &&
4100 LU.AllFixupsOutsideLoop)
4101 LU.Kind = LSRUse::Special;
4102 else
4103 continue;
4104 }
4105 // For an ICmpZero, negating a solitary base register won't lead to
4106 // new solutions.
4107 if (LU.Kind == LSRUse::ICmpZero &&
4108 !Base.HasBaseReg && Base.BaseOffset == 0 && !Base.BaseGV)
4109 continue;
4110 // For each addrec base reg, if its loop is current loop, apply the scale.
4111 for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) {
4112 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Base.BaseRegs[i]);
4113 if (AR && (AR->getLoop() == L || LU.AllFixupsOutsideLoop)) {
4114 const SCEV *FactorS = SE.getConstant(IntTy, Factor);
4115 if (FactorS->isZero())
4116 continue;
4117 // Divide out the factor, ignoring high bits, since we'll be
4118 // scaling the value back up in the end.
4119 if (const SCEV *Quotient = getExactSDiv(AR, FactorS, SE, true))
4120 if (!Quotient->isZero()) {
4121 // TODO: This could be optimized to avoid all the copying.
4122 Formula F = Base;
4123 F.ScaledReg = Quotient;
4124 F.deleteBaseReg(F.BaseRegs[i]);
4125 // The canonical representation of 1*reg is reg, which is already in
4126 // Base. In that case, do not try to insert the formula, it will be
4127 // rejected anyway.
4128 if (F.Scale == 1 && (F.BaseRegs.empty() ||
4129 (AR->getLoop() != L && LU.AllFixupsOutsideLoop)))
4130 continue;
4131 // If AllFixupsOutsideLoop is true and F.Scale is 1, we may generate
4132 // non canonical Formula with ScaledReg's loop not being L.
4133 if (F.Scale == 1 && LU.AllFixupsOutsideLoop)
4134 F.canonicalize(*L);
4135 (void)InsertFormula(LU, LUIdx, F);
4136 }
4137 }
4138 }
4139 }
4140}
4141
4142/// Extend/Truncate \p Expr to \p ToTy considering post-inc uses in \p Loops.
4143/// For all PostIncLoopSets in \p Loops, first de-normalize \p Expr, then
4144/// perform the extension/truncate and normalize again, as the normalized form
4145/// can result in folds that are not valid in the post-inc use contexts. The
4146/// expressions for all PostIncLoopSets must match, otherwise return nullptr.
4147static const SCEV *
4149 const SCEV *Expr, Type *ToTy,
4150 ScalarEvolution &SE) {
4151 const SCEV *Result = nullptr;
4152 for (auto &L : Loops) {
4153 auto *DenormExpr = denormalizeForPostIncUse(Expr, L, SE);
4154 const SCEV *NewDenormExpr = SE.getAnyExtendExpr(DenormExpr, ToTy);
4155 const SCEV *New = normalizeForPostIncUse(NewDenormExpr, L, SE);
4156 if (!New || (Result && New != Result))
4157 return nullptr;
4158 Result = New;
4159 }
4160
4161 assert(Result && "failed to create expression");
4162 return Result;
4163}
4164
4165/// Generate reuse formulae from different IV types.
4166void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) {
4167 // Don't bother truncating symbolic values.
4168 if (Base.BaseGV) return;
4169
4170 // Determine the integer type for the base formula.
4171 Type *DstTy = Base.getType();
4172 if (!DstTy) return;
4173 if (DstTy->isPointerTy())
4174 return;
4175
4176 // It is invalid to extend a pointer type so exit early if ScaledReg or
4177 // any of the BaseRegs are pointers.
4178 if (Base.ScaledReg && Base.ScaledReg->getType()->isPointerTy())
4179 return;
4180 if (any_of(Base.BaseRegs,
4181 [](const SCEV *S) { return S->getType()->isPointerTy(); }))
4182 return;
4183
4185 for (auto &LF : LU.Fixups)
4186 Loops.push_back(LF.PostIncLoops);
4187
4188 for (Type *SrcTy : Types) {
4189 if (SrcTy != DstTy && TTI.isTruncateFree(SrcTy, DstTy)) {
4190 Formula F = Base;
4191
4192 // Sometimes SCEV is able to prove zero during ext transform. It may
4193 // happen if SCEV did not do all possible transforms while creating the
4194 // initial node (maybe due to depth limitations), but it can do them while
4195 // taking ext.
4196 if (F.ScaledReg) {
4197 const SCEV *NewScaledReg =
4198 getAnyExtendConsideringPostIncUses(Loops, F.ScaledReg, SrcTy, SE);
4199 if (!NewScaledReg || NewScaledReg->isZero())
4200 continue;
4201 F.ScaledReg = NewScaledReg;
4202 }
4203 bool HasZeroBaseReg = false;
4204 for (const SCEV *&BaseReg : F.BaseRegs) {
4205 const SCEV *NewBaseReg =
4206 getAnyExtendConsideringPostIncUses(Loops, BaseReg, SrcTy, SE);
4207 if (!NewBaseReg || NewBaseReg->isZero()) {
4208 HasZeroBaseReg = true;
4209 break;
4210 }
4211 BaseReg = NewBaseReg;
4212 }
4213 if (HasZeroBaseReg)
4214 continue;
4215
4216 // TODO: This assumes we've done basic processing on all uses and
4217 // have an idea what the register usage is.
4218 if (!F.hasRegsUsedByUsesOtherThan(LUIdx, RegUses))
4219 continue;
4220
4221 F.canonicalize(*L);
4222 (void)InsertFormula(LU, LUIdx, F);
4223 }
4224 }
4225}
4226
4227namespace {
4228
4229/// Helper class for GenerateCrossUseConstantOffsets. It's used to defer
4230/// modifications so that the search phase doesn't have to worry about the data
4231/// structures moving underneath it.
4232struct WorkItem {
4233 size_t LUIdx;
4234 int64_t Imm;
4235 const SCEV *OrigReg;
4236
4237 WorkItem(size_t LI, int64_t I, const SCEV *R)
4238 : LUIdx(LI), Imm(I), OrigReg(R) {}
4239
4240 void print(raw_ostream &OS) const;
4241 void dump() const;
4242};
4243
4244} // end anonymous namespace
4245
4246#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4247void WorkItem::print(raw_ostream &OS) const {
4248 OS << "in formulae referencing " << *OrigReg << " in use " << LUIdx
4249 << " , add offset " << Imm;
4250}
4251
4252LLVM_DUMP_METHOD void WorkItem::dump() const {
4253 print(errs()); errs() << '\n';
4254}
4255#endif
4256
4257/// Look for registers which are a constant distance apart and try to form reuse
4258/// opportunities between them.
4259void LSRInstance::GenerateCrossUseConstantOffsets() {
4260 // Group the registers by their value without any added constant offset.
4261 using ImmMapTy = std::map<int64_t, const SCEV *>;
4262
4266 for (const SCEV *Use : RegUses) {
4267 const SCEV *Reg = Use; // Make a copy for ExtractImmediate to modify.
4268 int64_t Imm = ExtractImmediate(Reg, SE);
4269 auto Pair = Map.insert(std::make_pair(Reg, ImmMapTy()));
4270 if (Pair.second)
4271 Sequence.push_back(Reg);
4272 Pair.first->second.insert(std::make_pair(Imm, Use));
4273 UsedByIndicesMap[Reg] |= RegUses.getUsedByIndices(Use);
4274 }
4275
4276 // Now examine each set of registers with the same base value. Build up
4277 // a list of work to do and do the work in a separate step so that we're
4278 // not adding formulae and register counts while we're searching.
4279 SmallVector<WorkItem, 32> WorkItems;
4280 SmallSet<std::pair<size_t, int64_t>, 32> UniqueItems;
4281 for (const SCEV *Reg : Sequence) {
4282 const ImmMapTy &Imms = Map.find(Reg)->second;
4283
4284 // It's not worthwhile looking for reuse if there's only one offset.
4285 if (Imms.size() == 1)
4286 continue;
4287
4288 LLVM_DEBUG(dbgs() << "Generating cross-use offsets for " << *Reg << ':';
4289 for (const auto &Entry
4290 : Imms) dbgs()
4291 << ' ' << Entry.first;
4292 dbgs() << '\n');
4293
4294 // Examine each offset.
4295 for (ImmMapTy::const_iterator J = Imms.begin(), JE = Imms.end();
4296 J != JE; ++J) {
4297 const SCEV *OrigReg = J->second;
4298
4299 int64_t JImm = J->first;
4300 const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(OrigReg);
4301
4302 if (!isa<SCEVConstant>(OrigReg) &&
4303 UsedByIndicesMap[Reg].count() == 1) {
4304 LLVM_DEBUG(dbgs() << "Skipping cross-use reuse for " << *OrigReg
4305 << '\n');
4306 continue;
4307 }
4308
4309 // Conservatively examine offsets between this orig reg a few selected
4310 // other orig regs.
4311 int64_t First = Imms.begin()->first;
4312 int64_t Last = std::prev(Imms.end())->first;
4313 // Compute (First + Last) / 2 without overflow using the fact that
4314 // First + Last = 2 * (First + Last) + (First ^ Last).
4315 int64_t Avg = (First & Last) + ((First ^ Last) >> 1);
4316 // If the result is negative and First is odd and Last even (or vice versa),
4317 // we rounded towards -inf. Add 1 in that case, to round towards 0.
4318 Avg = Avg + ((First ^ Last) & ((uint64_t)Avg >> 63));
4319 ImmMapTy::const_iterator OtherImms[] = {
4320 Imms.begin(), std::prev(Imms.end()),
4321 Imms.lower_bound(Avg)};
4322 for (const auto &M : OtherImms) {
4323 if (M == J || M == JE) continue;
4324
4325 // Compute the difference between the two.
4326 int64_t Imm = (uint64_t)JImm - M->first;
4327 for (unsigned LUIdx : UsedByIndices.set_bits())
4328 // Make a memo of this use, offset, and register tuple.
4329 if (UniqueItems.insert(std::make_pair(LUIdx, Imm)).second)
4330 WorkItems.push_back(WorkItem(LUIdx, Imm, OrigReg));
4331 }
4332 }
4333 }
4334
4335 Map.clear();
4336 Sequence.clear();
4337 UsedByIndicesMap.clear();
4338 UniqueItems.clear();
4339
4340 // Now iterate through the worklist and add new formulae.
4341 for (const WorkItem &WI : WorkItems) {
4342 size_t LUIdx = WI.LUIdx;
4343 LSRUse &LU = Uses[LUIdx];
4344 int64_t Imm = WI.Imm;
4345 const SCEV *OrigReg = WI.OrigReg;
4346
4347 Type *IntTy = SE.getEffectiveSCEVType(OrigReg->getType());
4348 const SCEV *NegImmS = SE.getSCEV(ConstantInt::get(IntTy, -(uint64_t)Imm));
4349 unsigned BitWidth = SE.getTypeSizeInBits(IntTy);
4350
4351 // TODO: Use a more targeted data structure.
4352 for (size_t L = 0, LE = LU.Formulae.size(); L != LE; ++L) {
4353 Formula F = LU.Formulae[L];
4354 // FIXME: The code for the scaled and unscaled registers looks
4355 // very similar but slightly different. Investigate if they
4356 // could be merged. That way, we would not have to unscale the
4357 // Formula.
4358 F.unscale();
4359 // Use the immediate in the scaled register.
4360 if (F.ScaledReg == OrigReg) {
4361 int64_t Offset = (uint64_t)F.BaseOffset + Imm * (uint64_t)F.Scale;
4362 // Don't create 50 + reg(-50).
4363 if (F.referencesReg(SE.getSCEV(
4364 ConstantInt::get(IntTy, -(uint64_t)Offset))))
4365 continue;
4366 Formula NewF = F;
4367 NewF.BaseOffset = Offset;
4368 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
4369 NewF))
4370 continue;
4371 NewF.ScaledReg = SE.getAddExpr(NegImmS, NewF.ScaledReg);
4372
4373 // If the new scale is a constant in a register, and adding the constant
4374 // value to the immediate would produce a value closer to zero than the
4375 // immediate itself, then the formula isn't worthwhile.
4376 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(NewF.ScaledReg))
4377 if (C->getValue()->isNegative() != (NewF.BaseOffset < 0) &&
4378 (C->getAPInt().abs() * APInt(BitWidth, F.Scale))
4379 .ule(std::abs(NewF.BaseOffset)))
4380 continue;
4381
4382 // OK, looks good.
4383 NewF.canonicalize(*this->L);
4384 (void)InsertFormula(LU, LUIdx, NewF);
4385 } else {
4386 // Use the immediate in a base register.
4387 for (size_t N = 0, NE = F.BaseRegs.size(); N != NE; ++N) {
4388 const SCEV *BaseReg = F.BaseRegs[N];
4389 if (BaseReg != OrigReg)
4390 continue;
4391 Formula NewF = F;
4392 NewF.BaseOffset = (uint64_t)NewF.BaseOffset + Imm;
4393 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset,
4394 LU.Kind, LU.AccessTy, NewF)) {
4395 if (AMK == TTI::AMK_PostIndexed &&
4396 mayUsePostIncMode(TTI, LU, OrigReg, this->L, SE))
4397 continue;
4398 if (!TTI.isLegalAddImmediate((uint64_t)NewF.UnfoldedOffset + Imm))
4399 continue;
4400 NewF = F;
4401 NewF.UnfoldedOffset = (uint64_t)NewF.UnfoldedOffset + Imm;
4402 }
4403 NewF.BaseRegs[N] = SE.getAddExpr(NegImmS, BaseReg);
4404
4405 // If the new formula has a constant in a register, and adding the
4406 // constant value to the immediate would produce a value closer to
4407 // zero than the immediate itself, then the formula isn't worthwhile.
4408 for (const SCEV *NewReg : NewF.BaseRegs)
4409 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(NewReg))
4410 if ((C->getAPInt() + NewF.BaseOffset)
4411 .abs()
4412 .slt(std::abs(NewF.BaseOffset)) &&
4413 (C->getAPInt() + NewF.BaseOffset).countr_zero() >=
4414 (unsigned)llvm::countr_zero<uint64_t>(NewF.BaseOffset))
4415 goto skip_formula;
4416
4417 // Ok, looks good.
4418 NewF.canonicalize(*this->L);
4419 (void)InsertFormula(LU, LUIdx, NewF);
4420 break;
4421 skip_formula:;
4422 }
4423 }
4424 }
4425 }
4426}
4427
4428/// Generate formulae for each use.
4429void
4430LSRInstance::GenerateAllReuseFormulae() {
4431 // This is split into multiple loops so that hasRegsUsedByUsesOtherThan
4432 // queries are more precise.
4433 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4434 LSRUse &LU = Uses[LUIdx];
4435 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4436 GenerateReassociations(LU, LUIdx, LU.Formulae[i]);
4437 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4438 GenerateCombinations(LU, LUIdx, LU.Formulae[i]);
4439 }
4440 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4441 LSRUse &LU = Uses[LUIdx];
4442 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4443 GenerateSymbolicOffsets(LU, LUIdx, LU.Formulae[i]);
4444 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4445 GenerateConstantOffsets(LU, LUIdx, LU.Formulae[i]);
4446 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4447 GenerateICmpZeroScales(LU, LUIdx, LU.Formulae[i]);
4448 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4449 GenerateScales(LU, LUIdx, LU.Formulae[i]);
4450 }
4451 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4452 LSRUse &LU = Uses[LUIdx];
4453 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4454 GenerateTruncates(LU, LUIdx, LU.Formulae[i]);
4455 }
4456
4457 GenerateCrossUseConstantOffsets();
4458
4459 LLVM_DEBUG(dbgs() << "\n"
4460 "After generating reuse formulae:\n";
4461 print_uses(dbgs()));
4462}
4463
4464/// If there are multiple formulae with the same set of registers used
4465/// by other uses, pick the best one and delete the others.
4466void LSRInstance::FilterOutUndesirableDedicatedRegisters() {
4467 DenseSet<const SCEV *> VisitedRegs;
4470#ifndef NDEBUG
4471 bool ChangedFormulae = false;
4472#endif
4473
4474 // Collect the best formula for each unique set of shared registers. This
4475 // is reset for each use.
4476 using BestFormulaeTy =
4477 DenseMap<SmallVector<const SCEV *, 4>, size_t, UniquifierDenseMapInfo>;
4478
4479 BestFormulaeTy BestFormulae;
4480
4481 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4482 LSRUse &LU = Uses[LUIdx];
4483 LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs());
4484 dbgs() << '\n');
4485
4486 bool Any = false;
4487 for (size_t FIdx = 0, NumForms = LU.Formulae.size();
4488 FIdx != NumForms; ++FIdx) {
4489 Formula &F = LU.Formulae[FIdx];
4490
4491 // Some formulas are instant losers. For example, they may depend on
4492 // nonexistent AddRecs from other loops. These need to be filtered
4493 // immediately, otherwise heuristics could choose them over others leading
4494 // to an unsatisfactory solution. Passing LoserRegs into RateFormula here
4495 // avoids the need to recompute this information across formulae using the
4496 // same bad AddRec. Passing LoserRegs is also essential unless we remove
4497 // the corresponding bad register from the Regs set.
4498 Cost CostF(L, SE, TTI, AMK);
4499 Regs.clear();
4500 CostF.RateFormula(F, Regs, VisitedRegs, LU, &LoserRegs);
4501 if (CostF.isLoser()) {
4502 // During initial formula generation, undesirable formulae are generated
4503 // by uses within other loops that have some non-trivial address mode or
4504 // use the postinc form of the IV. LSR needs to provide these formulae
4505 // as the basis of rediscovering the desired formula that uses an AddRec
4506 // corresponding to the existing phi. Once all formulae have been
4507 // generated, these initial losers may be pruned.
4508 LLVM_DEBUG(dbgs() << " Filtering loser "; F.print(dbgs());
4509 dbgs() << "\n");
4510 }
4511 else {
4513 for (const SCEV *Reg : F.BaseRegs) {
4514 if (RegUses.isRegUsedByUsesOtherThan(Reg, LUIdx))
4515 Key.push_back(Reg);
4516 }
4517 if (F.ScaledReg &&
4518 RegUses.isRegUsedByUsesOtherThan(F.ScaledReg, LUIdx))
4519 Key.push_back(F.ScaledReg);
4520 // Unstable sort by host order ok, because this is only used for
4521 // uniquifying.
4522 llvm::sort(Key);
4523
4524 std::pair<BestFormulaeTy::const_iterator, bool> P =
4525 BestFormulae.insert(std::make_pair(Key, FIdx));
4526 if (P.second)
4527 continue;
4528
4529 Formula &Best = LU.Formulae[P.first->second];
4530
4531 Cost CostBest(L, SE, TTI, AMK);
4532 Regs.clear();
4533 CostBest.RateFormula(Best, Regs, VisitedRegs, LU);
4534 if (CostF.isLess(CostBest))
4535 std::swap(F, Best);
4536 LLVM_DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs());
4537 dbgs() << "\n"
4538 " in favor of formula ";
4539 Best.print(dbgs()); dbgs() << '\n');
4540 }
4541#ifndef NDEBUG
4542 ChangedFormulae = true;
4543#endif
4544 LU.DeleteFormula(F);
4545 --FIdx;
4546 --NumForms;
4547 Any = true;
4548 }
4549
4550 // Now that we've filtered out some formulae, recompute the Regs set.
4551 if (Any)
4552 LU.RecomputeRegs(LUIdx, RegUses);
4553
4554 // Reset this to prepare for the next use.
4555 BestFormulae.clear();
4556 }
4557
4558 LLVM_DEBUG(if (ChangedFormulae) {
4559 dbgs() << "\n"
4560 "After filtering out undesirable candidates:\n";
4561 print_uses(dbgs());
4562 });
4563}
4564
4565/// Estimate the worst-case number of solutions the solver might have to
4566/// consider. It almost never considers this many solutions because it prune the
4567/// search space, but the pruning isn't always sufficient.
4568size_t LSRInstance::EstimateSearchSpaceComplexity() const {
4569 size_t Power = 1;
4570 for (const LSRUse &LU : Uses) {
4571 size_t FSize = LU.Formulae.size();
4572 if (FSize >= ComplexityLimit) {
4573 Power = ComplexityLimit;
4574 break;
4575 }
4576 Power *= FSize;
4577 if (Power >= ComplexityLimit)
4578 break;
4579 }
4580 return Power;
4581}
4582
4583/// When one formula uses a superset of the registers of another formula, it
4584/// won't help reduce register pressure (though it may not necessarily hurt
4585/// register pressure); remove it to simplify the system.
4586void LSRInstance::NarrowSearchSpaceByDetectingSupersets() {
4587 if (EstimateSearchSpaceComplexity() >= ComplexityLimit) {
4588 LLVM_DEBUG(dbgs() << "The search space is too complex.\n");
4589
4590 LLVM_DEBUG(dbgs() << "Narrowing the search space by eliminating formulae "
4591 "which use a superset of registers used by other "
4592 "formulae.\n");
4593
4594 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4595 LSRUse &LU = Uses[LUIdx];
4596 bool Any = false;
4597 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
4598 Formula &F = LU.Formulae[i];
4599 // Look for a formula with a constant or GV in a register. If the use
4600 // also has a formula with that same value in an immediate field,
4601 // delete the one that uses a register.
4603 I = F.BaseRegs.begin(), E = F.BaseRegs.end(); I != E; ++I) {
4604 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(*I)) {
4605 Formula NewF = F;
4606 //FIXME: Formulas should store bitwidth to do wrapping properly.
4607 // See PR41034.
4608 NewF.BaseOffset += (uint64_t)C->getValue()->getSExtValue();
4609 NewF.BaseRegs.erase(NewF.BaseRegs.begin() +
4610 (I - F.BaseRegs.begin()));
4611 if (LU.HasFormulaWithSameRegs(NewF)) {
4612 LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs());
4613 dbgs() << '\n');
4614 LU.DeleteFormula(F);
4615 --i;
4616 --e;
4617 Any = true;
4618 break;
4619 }
4620 } else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(*I)) {
4621 if (GlobalValue *GV = dyn_cast<GlobalValue>(U->getValue()))
4622 if (!F.BaseGV) {
4623 Formula NewF = F;
4624 NewF.BaseGV = GV;
4625 NewF.BaseRegs.erase(NewF.BaseRegs.begin() +
4626 (I - F.BaseRegs.begin()));
4627 if (LU.HasFormulaWithSameRegs(NewF)) {
4628 LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs());
4629 dbgs() << '\n');
4630 LU.DeleteFormula(F);
4631 --i;
4632 --e;
4633 Any = true;
4634 break;
4635 }
4636 }
4637 }
4638 }
4639 }
4640 if (Any)
4641 LU.RecomputeRegs(LUIdx, RegUses);
4642 }
4643
4644 LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
4645 }
4646}
4647
4648/// When there are many registers for expressions like A, A+1, A+2, etc.,
4649/// allocate a single register for them.
4650void LSRInstance::NarrowSearchSpaceByCollapsingUnrolledCode() {
4651 if (EstimateSearchSpaceComplexity() < ComplexityLimit)
4652 return;
4653
4654 LLVM_DEBUG(
4655 dbgs() << "The search space is too complex.\n"
4656 "Narrowing the search space by assuming that uses separated "
4657 "by a constant offset will use the same registers.\n");
4658
4659 // This is especially useful for unrolled loops.
4660
4661 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4662 LSRUse &LU = Uses[LUIdx];
4663 for (const Formula &F : LU.Formulae) {
4664 if (F.BaseOffset == 0 || (F.Scale != 0 && F.Scale != 1))
4665 continue;
4666
4667 LSRUse *LUThatHas = FindUseWithSimilarFormula(F, LU);
4668 if (!LUThatHas)
4669 continue;
4670
4671 if (!reconcileNewOffset(*LUThatHas, F.BaseOffset, /*HasBaseReg=*/ false,
4672 LU.Kind, LU.AccessTy))
4673 continue;
4674
4675 LLVM_DEBUG(dbgs() << " Deleting use "; LU.print(dbgs()); dbgs() << '\n');
4676
4677 LUThatHas->AllFixupsOutsideLoop &= LU.AllFixupsOutsideLoop;
4678
4679 // Transfer the fixups of LU to LUThatHas.
4680 for (LSRFixup &Fixup : LU.Fixups) {
4681 Fixup.Offset += F.BaseOffset;
4682 LUThatHas->pushFixup(Fixup);
4683 LLVM_DEBUG(dbgs() << "New fixup has offset " << Fixup.Offset << '\n');
4684 }
4685
4686 // Delete formulae from the new use which are no longer legal.
4687 bool Any = false;
4688 for (size_t i = 0, e = LUThatHas->Formulae.size(); i != e; ++i) {
4689 Formula &F = LUThatHas->Formulae[i];
4690 if (!isLegalUse(TTI, LUThatHas->MinOffset, LUThatHas->MaxOffset,
4691 LUThatHas->Kind, LUThatHas->AccessTy, F)) {
4692 LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << '\n');
4693 LUThatHas->DeleteFormula(F);
4694 --i;
4695 --e;
4696 Any = true;
4697 }
4698 }
4699
4700 if (Any)
4701 LUThatHas->RecomputeRegs(LUThatHas - &Uses.front(), RegUses);
4702
4703 // Delete the old use.
4704 DeleteUse(LU, LUIdx);
4705 --LUIdx;
4706 --NumUses;
4707 break;
4708 }
4709 }
4710
4711 LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
4712}
4713
4714/// Call FilterOutUndesirableDedicatedRegisters again, if necessary, now that
4715/// we've done more filtering, as it may be able to find more formulae to
4716/// eliminate.
4717void LSRInstance::NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(){
4718 if (EstimateSearchSpaceComplexity() >= ComplexityLimit) {
4719 LLVM_DEBUG(dbgs() << "The search space is too complex.\n");
4720
4721 LLVM_DEBUG(dbgs() << "Narrowing the search space by re-filtering out "
4722 "undesirable dedicated registers.\n");
4723
4724 FilterOutUndesirableDedicatedRegisters();
4725
4726 LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
4727 }
4728}
4729
4730/// If a LSRUse has multiple formulae with the same ScaledReg and Scale.
4731/// Pick the best one and delete the others.
4732/// This narrowing heuristic is to keep as many formulae with different
4733/// Scale and ScaledReg pair as possible while narrowing the search space.
4734/// The benefit is that it is more likely to find out a better solution
4735/// from a formulae set with more Scale and ScaledReg variations than
4736/// a formulae set with the same Scale and ScaledReg. The picking winner
4737/// reg heuristic will often keep the formulae with the same Scale and
4738/// ScaledReg and filter others, and we want to avoid that if possible.
4739void LSRInstance::NarrowSearchSpaceByFilterFormulaWithSameScaledReg() {
4740 if (EstimateSearchSpaceComplexity() < ComplexityLimit)
4741 return;
4742
4743 LLVM_DEBUG(
4744 dbgs() << "The search space is too complex.\n"
4745 "Narrowing the search space by choosing the best Formula "
4746 "from the Formulae with the same Scale and ScaledReg.\n");
4747
4748 // Map the "Scale * ScaledReg" pair to the best formula of current LSRUse.
4749 using BestFormulaeTy = DenseMap<std::pair<const SCEV *, int64_t>, size_t>;
4750
4751 BestFormulaeTy BestFormulae;
4752#ifndef NDEBUG
4753 bool ChangedFormulae = false;
4754#endif
4755 DenseSet<const SCEV *> VisitedRegs;
4757
4758 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4759 LSRUse &LU = Uses[LUIdx];
4760 LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs());
4761 dbgs() << '\n');
4762
4763 // Return true if Formula FA is better than Formula FB.
4764 auto IsBetterThan = [&](Formula &FA, Formula &FB) {
4765 // First we will try to choose the Formula with fewer new registers.
4766 // For a register used by current Formula, the more the register is
4767 // shared among LSRUses, the less we increase the register number
4768 // counter of the formula.
4769 size_t FARegNum = 0;
4770 for (const SCEV *Reg : FA.BaseRegs) {
4771 const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(Reg);
4772 FARegNum += (NumUses - UsedByIndices.count() + 1);
4773 }
4774 size_t FBRegNum = 0;
4775 for (const SCEV *Reg : FB.BaseRegs) {
4776 const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(Reg);
4777 FBRegNum += (NumUses - UsedByIndices.count() + 1);
4778 }
4779 if (FARegNum != FBRegNum)
4780 return FARegNum < FBRegNum;
4781
4782 // If the new register numbers are the same, choose the Formula with
4783 // less Cost.
4784 Cost CostFA(L, SE, TTI, AMK);
4785 Cost CostFB(L, SE, TTI, AMK);
4786 Regs.clear();
4787 CostFA.RateFormula(FA, Regs, VisitedRegs, LU);
4788 Regs.clear();
4789 CostFB.RateFormula(FB, Regs, VisitedRegs, LU);
4790 return CostFA.isLess(CostFB);
4791 };
4792
4793 bool Any = false;
4794 for (size_t FIdx = 0, NumForms = LU.Formulae.size(); FIdx != NumForms;
4795 ++FIdx) {
4796 Formula &F = LU.Formulae[FIdx];
4797 if (!F.ScaledReg)
4798 continue;
4799 auto P = BestFormulae.insert({{F.ScaledReg, F.Scale}, FIdx});
4800 if (P.second)
4801 continue;
4802
4803 Formula &Best = LU.Formulae[P.first->second];
4804 if (IsBetterThan(F, Best))
4805 std::swap(F, Best);
4806 LLVM_DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs());
4807 dbgs() << "\n"
4808 " in favor of formula ";
4809 Best.print(dbgs()); dbgs() << '\n');
4810#ifndef NDEBUG
4811 ChangedFormulae = true;
4812#endif
4813 LU.DeleteFormula(F);
4814 --FIdx;
4815 --NumForms;
4816 Any = true;
4817 }
4818 if (Any)
4819 LU.RecomputeRegs(LUIdx, RegUses);
4820
4821 // Reset this to prepare for the next use.
4822 BestFormulae.clear();
4823 }
4824
4825 LLVM_DEBUG(if (ChangedFormulae) {
4826 dbgs() << "\n"
4827 "After filtering out undesirable candidates:\n";
4828 print_uses(dbgs());
4829 });
4830}
4831
4832/// If we are over the complexity limit, filter out any post-inc prefering
4833/// variables to only post-inc values.
4834void LSRInstance::NarrowSearchSpaceByFilterPostInc() {
4835 if (AMK != TTI::AMK_PostIndexed)
4836 return;
4837 if (EstimateSearchSpaceComplexity() < ComplexityLimit)
4838 return;
4839
4840 LLVM_DEBUG(dbgs() << "The search space is too complex.\n"
4841 "Narrowing the search space by choosing the lowest "
4842 "register Formula for PostInc Uses.\n");
4843
4844 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4845 LSRUse &LU = Uses[LUIdx];
4846
4847 if (LU.Kind != LSRUse::Address)
4848 continue;
4849 if (!TTI.isIndexedLoadLegal(TTI.MIM_PostInc, LU.AccessTy.getType()) &&
4850 !TTI.isIndexedStoreLegal(TTI.MIM_PostInc, LU.AccessTy.getType()))
4851 continue;
4852
4853 size_t MinRegs = std::numeric_limits<size_t>::max();
4854 for (const Formula &F : LU.Formulae)
4855 MinRegs = std::min(F.getNumRegs(), MinRegs);
4856
4857 bool Any = false;
4858 for (size_t FIdx = 0, NumForms = LU.Formulae.size(); FIdx != NumForms;
4859 ++FIdx) {
4860 Formula &F = LU.Formulae[FIdx];
4861 if (F.getNumRegs() > MinRegs) {
4862 LLVM_DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs());
4863 dbgs() << "\n");
4864 LU.DeleteFormula(F);
4865 --FIdx;
4866 --NumForms;
4867 Any = true;
4868 }
4869 }
4870 if (Any)
4871 LU.RecomputeRegs(LUIdx, RegUses);
4872
4873 if (EstimateSearchSpaceComplexity() < ComplexityLimit)
4874 break;
4875 }
4876
4877 LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
4878}
4879
4880/// The function delete formulas with high registers number expectation.
4881/// Assuming we don't know the value of each formula (already delete
4882/// all inefficient), generate probability of not selecting for each
4883/// register.
4884/// For example,
4885/// Use1:
4886/// reg(a) + reg({0,+,1})
4887/// reg(a) + reg({-1,+,1}) + 1
4888/// reg({a,+,1})
4889/// Use2:
4890/// reg(b) + reg({0,+,1})
4891/// reg(b) + reg({-1,+,1}) + 1
4892/// reg({b,+,1})
4893/// Use3:
4894/// reg(c) + reg(b) + reg({0,+,1})
4895/// reg(c) + reg({b,+,1})
4896///
4897/// Probability of not selecting
4898/// Use1 Use2 Use3
4899/// reg(a) (1/3) * 1 * 1
4900/// reg(b) 1 * (1/3) * (1/2)
4901/// reg({0,+,1}) (2/3) * (2/3) * (1/2)
4902/// reg({-1,+,1}) (2/3) * (2/3) * 1
4903/// reg({a,+,1}) (2/3) * 1 * 1
4904/// reg({b,+,1}) 1 * (2/3) * (2/3)
4905/// reg(c) 1 * 1 * 0
4906///
4907/// Now count registers number mathematical expectation for each formula:
4908/// Note that for each use we exclude probability if not selecting for the use.
4909/// For example for Use1 probability for reg(a) would be just 1 * 1 (excluding
4910/// probabilty 1/3 of not selecting for Use1).
4911/// Use1:
4912/// reg(a) + reg({0,+,1}) 1 + 1/3 -- to be deleted
4913/// reg(a) + reg({-1,+,1}) + 1 1 + 4/9 -- to be deleted
4914/// reg({a,+,1}) 1
4915/// Use2:
4916/// reg(b) + reg({0,+,1}) 1/2 + 1/3 -- to be deleted
4917/// reg(b) + reg({-1,+,1}) + 1 1/2 + 2/3 -- to be deleted
4918/// reg({b,+,1}) 2/3
4919/// Use3:
4920/// reg(c) + reg(b) + reg({0,+,1}) 1 + 1/3 + 4/9 -- to be deleted
4921/// reg(c) + reg({b,+,1}) 1 + 2/3
4922void LSRInstance::NarrowSearchSpaceByDeletingCostlyFormulas() {
4923 if (EstimateSearchSpaceComplexity() < ComplexityLimit)
4924 return;
4925 // Ok, we have too many of formulae on our hands to conveniently handle.
4926 // Use a rough heuristic to thin out the list.
4927
4928 // Set of Regs wich will be 100% used in final solution.
4929 // Used in each formula of a solution (in example above this is reg(c)).
4930 // We can skip them in calculations.
4932 LLVM_DEBUG(dbgs() << "The search space is too complex.\n");
4933
4934 // Map each register to probability of not selecting
4935 DenseMap <const SCEV *, float> RegNumMap;
4936 for (const SCEV *Reg : RegUses) {
4937 if (UniqRegs.count(Reg))
4938 continue;
4939 float PNotSel = 1;
4940 for (const LSRUse &LU : Uses) {
4941 if (!LU.Regs.count(Reg))
4942 continue;
4943 float P = LU.getNotSelectedProbability(Reg);
4944 if (P != 0.0)
4945 PNotSel *= P;
4946 else
4947 UniqRegs.insert(Reg);
4948 }
4949 RegNumMap.insert(std::make_pair(Reg, PNotSel));
4950 }
4951
4952 LLVM_DEBUG(
4953 dbgs() << "Narrowing the search space by deleting costly formulas\n");
4954
4955 // Delete formulas where registers number expectation is high.
4956 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4957 LSRUse &LU = Uses[LUIdx];
4958 // If nothing to delete - continue.
4959 if (LU.Formulae.size() < 2)
4960 continue;
4961 // This is temporary solution to test performance. Float should be
4962 // replaced with round independent type (based on integers) to avoid
4963 // different results for different target builds.
4964 float FMinRegNum = LU.Formulae[0].getNumRegs();
4965 float FMinARegNum = LU.Formulae[0].getNumRegs();
4966 size_t MinIdx = 0;
4967 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
4968 Formula &F = LU.Formulae[i];
4969 float FRegNum = 0;
4970 float FARegNum = 0;
4971 for (const SCEV *BaseReg : F.BaseRegs) {
4972 if (UniqRegs.count(BaseReg))
4973 continue;
4974 FRegNum += RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg);
4975 if (isa<SCEVAddRecExpr>(BaseReg))
4976 FARegNum +=
4977 RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg);
4978 }
4979 if (const SCEV *ScaledReg = F.ScaledReg) {
4980 if (!UniqRegs.count(ScaledReg)) {
4981 FRegNum +=
4982 RegNumMap[ScaledReg] / LU.getNotSelectedProbability(ScaledReg);
4983 if (isa<SCEVAddRecExpr>(ScaledReg))
4984 FARegNum +=
4985 RegNumMap[ScaledReg] / LU.getNotSelectedProbability(ScaledReg);
4986 }
4987 }
4988 if (FMinRegNum > FRegNum ||
4989 (FMinRegNum == FRegNum && FMinARegNum > FARegNum)) {
4990 FMinRegNum = FRegNum;
4991 FMinARegNum = FARegNum;
4992 MinIdx = i;
4993 }
4994 }
4995 LLVM_DEBUG(dbgs() << " The formula "; LU.Formulae[MinIdx].print(dbgs());
4996 dbgs() << " with min reg num " << FMinRegNum << '\n');
4997 if (MinIdx != 0)
4998 std::swap(LU.Formulae[MinIdx], LU.Formulae[0]);
4999 while (LU.Formulae.size() != 1) {
5000 LLVM_DEBUG(dbgs() << " Deleting "; LU.Formulae.back().print(dbgs());
5001 dbgs() << '\n');
5002 LU.Formulae.pop_back();
5003 }
5004 LU.RecomputeRegs(LUIdx, RegUses);
5005 assert(LU.Formulae.size() == 1 && "Should be exactly 1 min regs formula");
5006 Formula &F = LU.Formulae[0];
5007 LLVM_DEBUG(dbgs() << " Leaving only "; F.print(dbgs()); dbgs() << '\n');
5008 // When we choose the formula, the regs become unique.
5009 UniqRegs.insert(F.BaseRegs.begin(), F.BaseRegs.end());
5010 if (F.ScaledReg)
5011 UniqRegs.insert(F.ScaledReg);
5012 }
5013 LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5014}
5015
5016// Check if Best and Reg are SCEVs separated by a constant amount C, and if so
5017// would the addressing offset +C would be legal where the negative offset -C is
5018// not.
5020 ScalarEvolution &SE, const SCEV *Best,
5021 const SCEV *Reg,
5022 MemAccessTy AccessType) {
5023 if (Best->getType() != Reg->getType() ||
5024 (isa<SCEVAddRecExpr>(Best) && isa<SCEVAddRecExpr>(Reg) &&
5025 cast<SCEVAddRecExpr>(Best)->getLoop() !=
5026 cast<SCEVAddRecExpr>(Reg)->getLoop()))
5027 return false;
5028 const auto *Diff = dyn_cast<SCEVConstant>(SE.getMinusSCEV(Best, Reg));
5029 if (!Diff)
5030 return false;
5031
5033 AccessType.MemTy, /*BaseGV=*/nullptr,
5034 /*BaseOffset=*/Diff->getAPInt().getSExtValue(),
5035 /*HasBaseReg=*/true, /*Scale=*/0, AccessType.AddrSpace) &&
5037 AccessType.MemTy, /*BaseGV=*/nullptr,
5038 /*BaseOffset=*/-Diff->getAPInt().getSExtValue(),
5039 /*HasBaseReg=*/true, /*Scale=*/0, AccessType.AddrSpace);
5040}
5041
5042/// Pick a register which seems likely to be profitable, and then in any use
5043/// which has any reference to that register, delete all formulae which do not
5044/// reference that register.
5045void LSRInstance::NarrowSearchSpaceByPickingWinnerRegs() {
5046 // With all other options exhausted, loop until the system is simple
5047 // enough to handle.
5049 while (EstimateSearchSpaceComplexity() >= ComplexityLimit) {
5050 // Ok, we have too many of formulae on our hands to conveniently handle.
5051 // Use a rough heuristic to thin out the list.
5052 LLVM_DEBUG(dbgs() << "The search space is too complex.\n");
5053
5054 // Pick the register which is used by the most LSRUses, which is likely
5055 // to be a good reuse register candidate.
5056 const SCEV *Best = nullptr;
5057 unsigned BestNum = 0;
5058 for (const SCEV *Reg : RegUses) {
5059 if (Taken.count(Reg))
5060 continue;
5061 if (!Best) {
5062 Best = Reg;
5063 BestNum = RegUses.getUsedByIndices(Reg).count();
5064 } else {
5065 unsigned Count = RegUses.getUsedByIndices(Reg).count();
5066 if (Count > BestNum) {
5067 Best = Reg;
5068 BestNum = Count;
5069 }
5070
5071 // If the scores are the same, but the Reg is simpler for the target
5072 // (for example {x,+,1} as opposed to {x+C,+,1}, where the target can
5073 // handle +C but not -C), opt for the simpler formula.
5074 if (Count == BestNum) {
5075 int LUIdx = RegUses.getUsedByIndices(Reg).find_first();
5076 if (LUIdx >= 0 && Uses[LUIdx].Kind == LSRUse::Address &&
5077 IsSimplerBaseSCEVForTarget(TTI, SE, Best, Reg,
5078 Uses[LUIdx].AccessTy)) {
5079 Best = Reg;
5080 BestNum = Count;
5081 }
5082 }
5083 }
5084 }
5085 assert(Best && "Failed to find best LSRUse candidate");
5086
5087 LLVM_DEBUG(dbgs() << "Narrowing the search space by assuming " << *Best
5088 << " will yield profitable reuse.\n");
5089 Taken.insert(Best);
5090
5091 // In any use with formulae which references this register, delete formulae
5092 // which don't reference it.
5093 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5094 LSRUse &LU = Uses[LUIdx];
5095 if (!LU.Regs.count(Best)) continue;
5096
5097 bool Any = false;
5098 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
5099 Formula &F = LU.Formulae[i];
5100 if (!F.referencesReg(Best)) {
5101 LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << '\n');
5102 LU.DeleteFormula(F);
5103 --e;
5104 --i;
5105 Any = true;
5106 assert(e != 0 && "Use has no formulae left! Is Regs inconsistent?");
5107 continue;
5108 }
5109 }
5110
5111 if (Any)
5112 LU.RecomputeRegs(LUIdx, RegUses);
5113 }
5114
5115 LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5116 }
5117}
5118
5119/// If there are an extraordinary number of formulae to choose from, use some
5120/// rough heuristics to prune down the number of formulae. This keeps the main
5121/// solver from taking an extraordinary amount of time in some worst-case
5122/// scenarios.
5123void LSRInstance::NarrowSearchSpaceUsingHeuristics() {
5124 NarrowSearchSpaceByDetectingSupersets();
5125 NarrowSearchSpaceByCollapsingUnrolledCode();
5126 NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters();
5128 NarrowSearchSpaceByFilterFormulaWithSameScaledReg();
5129 NarrowSearchSpaceByFilterPostInc();
5130 if (LSRExpNarrow)
5131 NarrowSearchSpaceByDeletingCostlyFormulas();
5132 else
5133 NarrowSearchSpaceByPickingWinnerRegs();
5134}
5135
5136/// This is the recursive solver.
5137void LSRInstance::SolveRecurse(SmallVectorImpl<const Formula *> &Solution,
5138 Cost &SolutionCost,
5140 const Cost &CurCost,
5141 const SmallPtrSet<const SCEV *, 16> &CurRegs,
5142 DenseSet<const SCEV *> &VisitedRegs) const {
5143 // Some ideas:
5144 // - prune more:
5145 // - use more aggressive filtering
5146 // - sort the formula so that the most profitable solutions are found first
5147 // - sort the uses too
5148 // - search faster:
5149 // - don't compute a cost, and then compare. compare while computing a cost
5150 // and bail early.
5151 // - track register sets with SmallBitVector
5152
5153 const LSRUse &LU = Uses[Workspace.size()];
5154
5155 // If this use references any register that's already a part of the
5156 // in-progress solution, consider it a requirement that a formula must
5157 // reference that register in order to be considered. This prunes out
5158 // unprofitable searching.
5160 for (const SCEV *S : CurRegs)
5161 if (LU.Regs.count(S))
5162 ReqRegs.insert(S);
5163
5165 Cost NewCost(L, SE, TTI, AMK);
5166 for (const Formula &F : LU.Formulae) {
5167 // Ignore formulae which may not be ideal in terms of register reuse of
5168 // ReqRegs. The formula should use all required registers before
5169 // introducing new ones.
5170 // This can sometimes (notably when trying to favour postinc) lead to
5171 // sub-optimial decisions. There it is best left to the cost modelling to
5172 // get correct.
5173 if (AMK != TTI::AMK_PostIndexed || LU.Kind != LSRUse::Address) {
5174 int NumReqRegsToFind = std::min(F.getNumRegs(), ReqRegs.size());
5175 for (const SCEV *Reg : ReqRegs) {
5176 if ((F.ScaledReg && F.ScaledReg == Reg) ||
5177 is_contained(F.BaseRegs, Reg)) {
5178 --NumReqRegsToFind;
5179 if (NumReqRegsToFind == 0)
5180 break;
5181 }
5182 }
5183 if (NumReqRegsToFind != 0) {
5184 // If none of the formulae satisfied the required registers, then we could
5185 // clear ReqRegs and try again. Currently, we simply give up in this case.
5186 continue;
5187 }
5188 }
5189
5190 // Evaluate the cost of the current formula. If it's already worse than
5191 // the current best, prune the search at that point.
5192 NewCost = CurCost;
5193 NewRegs = CurRegs;
5194 NewCost.RateFormula(F, NewRegs, VisitedRegs, LU);
5195 if (NewCost.isLess(SolutionCost)) {
5196 Workspace.push_back(&F);
5197 if (Workspace.size() != Uses.size()) {
5198 SolveRecurse(Solution, SolutionCost, Workspace, NewCost,
5199 NewRegs, VisitedRegs);
5200 if (F.getNumRegs() == 1 && Workspace.size() == 1)
5201 VisitedRegs.insert(F.ScaledReg ? F.ScaledReg : F.BaseRegs[0]);
5202 } else {
5203 LLVM_DEBUG(dbgs() << "New best at "; NewCost.print(dbgs());
5204 dbgs() << ".\nRegs:\n";
5205 for (const SCEV *S : NewRegs) dbgs()
5206 << "- " << *S << "\n";
5207 dbgs() << '\n');
5208
5209 SolutionCost = NewCost;
5210 Solution = Workspace;
5211 }
5212 Workspace.pop_back();
5213 }
5214 }
5215}
5216
5217/// Choose one formula from each use. Return the results in the given Solution
5218/// vector.
5219void LSRInstance::Solve(SmallVectorImpl<const Formula *> &Solution) const {
5221 Cost SolutionCost(L, SE, TTI, AMK);
5222 SolutionCost.Lose();
5223 Cost CurCost(L, SE, TTI, AMK);
5225 DenseSet<const SCEV *> VisitedRegs;
5226 Workspace.reserve(Uses.size());
5227
5228 // SolveRecurse does all the work.
5229 SolveRecurse(Solution, SolutionCost, Workspace, CurCost,
5230 CurRegs, VisitedRegs);
5231 if (Solution.empty()) {
5232 LLVM_DEBUG(dbgs() << "\nNo Satisfactory Solution\n");
5233 return;
5234 }
5235
5236 // Ok, we've now made all our decisions.
5237 LLVM_DEBUG(dbgs() << "\n"
5238 "The chosen solution requires ";
5239 SolutionCost.print(dbgs()); dbgs() << ":\n";
5240 for (size_t i = 0, e = Uses.size(); i != e; ++i) {
5241 dbgs() << " ";
5242 Uses[i].print(dbgs());
5243 dbgs() << "\n"
5244 " ";
5245 Solution[i]->print(dbgs());
5246 dbgs() << '\n';
5247 });
5248
5249 assert(Solution.size() == Uses.size() && "Malformed solution!");
5250
5251 if (BaselineCost.isLess(SolutionCost)) {
5252 LLVM_DEBUG(dbgs() << "The baseline solution requires ";
5253 BaselineCost.print(dbgs()); dbgs() << "\n");
5255 LLVM_DEBUG(
5256 dbgs() << "Baseline is more profitable than chosen solution, "
5257 "add option 'lsr-drop-solution' to drop LSR solution.\n");
5258 else {
5259 LLVM_DEBUG(dbgs() << "Baseline is more profitable than chosen "
5260 "solution, dropping LSR solution.\n";);
5261 Solution.clear();
5262 }
5263 }
5264}
5265
5266/// Helper for AdjustInsertPositionForExpand. Climb up the dominator tree far as
5267/// we can go while still being dominated by the input positions. This helps
5268/// canonicalize the insert position, which encourages sharing.
5270LSRInstance::HoistInsertPosition(BasicBlock::iterator IP,
5271 const SmallVectorImpl<Instruction *> &Inputs)
5272 const {
5273 Instruction *Tentative = &*IP;
5274 while (true) {
5275 bool AllDominate = true;
5276 Instruction *BetterPos = nullptr;
5277 // Don't bother attempting to insert before a catchswitch, their basic block
5278 // cannot have other non-PHI instructions.
5279 if (isa<CatchSwitchInst>(Tentative))
5280 return IP;
5281
5282 for (Instruction *Inst : Inputs) {
5283 if (Inst == Tentative || !DT.dominates(Inst, Tentative)) {
5284 AllDominate = false;
5285 break;
5286 }
5287 // Attempt to find an insert position in the middle of the block,
5288 // instead of at the end, so that it can be used for other expansions.
5289 if (Tentative->getParent() == Inst->getParent() &&
5290 (!BetterPos || !DT.dominates(Inst, BetterPos)))
5291 BetterPos = &*std::next(BasicBlock::iterator(Inst));
5292 }
5293 if (!AllDominate)
5294 break;
5295 if (BetterPos)
5296 IP = BetterPos->getIterator();
5297 else
5298 IP = Tentative->getIterator();
5299
5300 const Loop *IPLoop = LI.getLoopFor(IP->getParent());
5301 unsigned IPLoopDepth = IPLoop ? IPLoop->getLoopDepth() : 0;
5302
5303 BasicBlock *IDom;
5304 for (DomTreeNode *Rung = DT.getNode(IP->getParent()); ; ) {
5305 if (!Rung) return IP;
5306 Rung = Rung->getIDom();
5307 if (!Rung) return IP;
5308 IDom = Rung->getBlock();
5309
5310 // Don't climb into a loop though.
5311 const Loop *IDomLoop = LI.getLoopFor(IDom);
5312 unsigned IDomDepth = IDomLoop ? IDomLoop->getLoopDepth() : 0;
5313 if (IDomDepth <= IPLoopDepth &&
5314 (IDomDepth != IPLoopDepth || IDomLoop == IPLoop))
5315 break;
5316 }
5317
5318 Tentative = IDom->getTerminator();
5319 }
5320
5321 return IP;
5322}
5323
5324/// Determine an input position which will be dominated by the operands and
5325/// which will dominate the result.
5326BasicBlock::iterator LSRInstance::AdjustInsertPositionForExpand(
5327 BasicBlock::iterator LowestIP, const LSRFixup &LF, const LSRUse &LU) const {
5328 // Collect some instructions which must be dominated by the
5329 // expanding replacement. These must be dominated by any operands that
5330 // will be required in the expansion.
5332 if (Instruction *I = dyn_cast<Instruction>(LF.OperandValToReplace))
5333 Inputs.push_back(I);
5334 if (LU.Kind == LSRUse::ICmpZero)
5335 if (Instruction *I =
5336 dyn_cast<Instruction>(cast<ICmpInst>(LF.UserInst)->getOperand(1)))
5337 Inputs.push_back(I);
5338 if (LF.PostIncLoops.count(L)) {
5339 if (LF.isUseFullyOutsideLoop(L))
5340 Inputs.push_back(L->getLoopLatch()->getTerminator());
5341 else
5342 Inputs.push_back(IVIncInsertPos);
5343 }
5344 // The expansion must also be dominated by the increment positions of any
5345 // loops it for which it is using post-inc mode.
5346 for (const Loop *PIL : LF.PostIncLoops) {
5347 if (PIL == L) continue;
5348
5349 // Be dominated by the loop exit.
5350 SmallVector<BasicBlock *, 4> ExitingBlocks;
5351 PIL->getExitingBlocks(ExitingBlocks);
5352 if (!ExitingBlocks.empty()) {
5353 BasicBlock *BB = ExitingBlocks[0];
5354 for (unsigned i = 1, e = ExitingBlocks.size(); i != e; ++i)
5355 BB = DT.findNearestCommonDominator(BB, ExitingBlocks[i]);
5356 Inputs.push_back(BB->getTerminator());
5357 }
5358 }
5359
5360 assert(!isa<PHINode>(LowestIP) && !LowestIP->isEHPad()
5361 && !isa<DbgInfoIntrinsic>(LowestIP) &&
5362 "Insertion point must be a normal instruction");
5363
5364 // Then, climb up the immediate dominator tree as far as we can go while
5365 // still being dominated by the input positions.
5366 BasicBlock::iterator IP = HoistInsertPosition(LowestIP, Inputs);
5367
5368 // Don't insert instructions before PHI nodes.
5369 while (isa<PHINode>(IP)) ++IP;
5370
5371 // Ignore landingpad instructions.
5372 while (IP->isEHPad()) ++IP;
5373
5374 // Ignore debug intrinsics.
5375 while (isa<DbgInfoIntrinsic>(IP)) ++IP;
5376
5377 // Set IP below instructions recently inserted by SCEVExpander. This keeps the
5378 // IP consistent across expansions and allows the previously inserted
5379 // instructions to be reused by subsequent expansion.
5380 while (Rewriter.isInsertedInstruction(&*IP) && IP != LowestIP)
5381 ++IP;
5382
5383 return IP;
5384}
5385
5386/// Emit instructions for the leading candidate expression for this LSRUse (this
5387/// is called "expanding").
5388Value *LSRInstance::Expand(const LSRUse &LU, const LSRFixup &LF,
5389 const Formula &F, BasicBlock::iterator IP,
5390 SmallVectorImpl<WeakTrackingVH> &DeadInsts) const {
5391 if (LU.RigidFormula)
5392 return LF.OperandValToReplace;
5393
5394 // Determine an input position which will be dominated by the operands and
5395 // which will dominate the result.
5396 IP = AdjustInsertPositionForExpand(IP, LF, LU);
5397 Rewriter.setInsertPoint(&*IP);
5398
5399 // Inform the Rewriter if we have a post-increment use, so that it can
5400 // perform an advantageous expansion.
5401 Rewriter.setPostInc(LF.PostIncLoops);
5402
5403 // This is the type that the user actually needs.
5404 Type *OpTy = LF.OperandValToReplace->getType();
5405 // This will be the type that we'll initially expand to.
5406 Type *Ty = F.getType();
5407 if (!Ty)
5408 // No type known; just expand directly to the ultimate type.
5409 Ty = OpTy;
5410 else if (SE.getEffectiveSCEVType(Ty) == SE.getEffectiveSCEVType(OpTy))
5411 // Expand directly to the ultimate type if it's the right size.
5412 Ty = OpTy;
5413 // This is the type to do integer arithmetic in.
5414 Type *IntTy = SE.getEffectiveSCEVType(Ty);
5415
5416 // Build up a list of operands to add together to form the full base.
5418
5419 // Expand the BaseRegs portion.
5420 for (const SCEV *Reg : F.BaseRegs) {
5421 assert(!Reg->isZero() && "Zero allocated in a base register!");
5422
5423 // If we're expanding for a post-inc user, make the post-inc adjustment.
5424 Reg = denormalizeForPostIncUse(Reg, LF.PostIncLoops, SE);
5425 Ops.push_back(SE.getUnknown(Rewriter.expandCodeFor(Reg, nullptr)));
5426 }
5427
5428 // Expand the ScaledReg portion.
5429 Value *ICmpScaledV = nullptr;
5430 if (F.Scale != 0) {
5431 const SCEV *ScaledS = F.ScaledReg;
5432
5433 // If we're expanding for a post-inc user, make the post-inc adjustment.
5434 PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops);
5435 ScaledS = denormalizeForPostIncUse(ScaledS, Loops, SE);
5436
5437 if (LU.Kind == LSRUse::ICmpZero) {
5438 // Expand ScaleReg as if it was part of the base regs.
5439 if (F.Scale == 1)
5440 Ops.push_back(
5441 SE.getUnknown(Rewriter.expandCodeFor(ScaledS, nullptr)));
5442 else {
5443 // An interesting way of "folding" with an icmp is to use a negated
5444 // scale, which we'll implement by inserting it into the other operand
5445 // of the icmp.
5446 assert(F.Scale == -1 &&
5447 "The only scale supported by ICmpZero uses is -1!");
5448 ICmpScaledV = Rewriter.expandCodeFor(ScaledS, nullptr);
5449 }
5450 } else {
5451 // Otherwise just expand the scaled register and an explicit scale,
5452 // which is expected to be matched as part of the address.
5453
5454 // Flush the operand list to suppress SCEVExpander hoisting address modes.
5455 // Unless the addressing mode will not be folded.
5456 if (!Ops.empty() && LU.Kind == LSRUse::Address &&
5457 isAMCompletelyFolded(TTI, LU, F)) {
5458 Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), nullptr);
5459 Ops.clear();
5460 Ops.push_back(SE.getUnknown(FullV));
5461 }
5462 ScaledS = SE.getUnknown(Rewriter.expandCodeFor(ScaledS, nullptr));
5463 if (F.Scale != 1)
5464 ScaledS =
5465 SE.getMulExpr(ScaledS, SE.getConstant(ScaledS->getType(), F.Scale));
5466 Ops.push_back(ScaledS);
5467 }
5468 }
5469
5470 // Expand the GV portion.
5471 if (F.BaseGV) {
5472 // Flush the operand list to suppress SCEVExpander hoisting.
5473 if (!Ops.empty()) {
5474 Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), IntTy);
5475 Ops.clear();
5476 Ops.push_back(SE.getUnknown(FullV));
5477 }
5478 Ops.push_back(SE.getUnknown(F.BaseGV));
5479 }
5480
5481 // Flush the operand list to suppress SCEVExpander hoisting of both folded and
5482 // unfolded offsets. LSR assumes they both live next to their uses.
5483 if (!Ops.empty()) {
5484 Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty);
5485 Ops.clear();
5486 Ops.push_back(SE.getUnknown(FullV));
5487 }
5488
5489 // Expand the immediate portion.
5490 int64_t Offset = (uint64_t)F.BaseOffset + LF.Offset;
5491 if (Offset != 0) {
5492 if (LU.Kind == LSRUse::ICmpZero) {
5493 // The other interesting way of "folding" with an ICmpZero is to use a
5494 // negated immediate.
5495 if (!ICmpScaledV)
5496 ICmpScaledV = ConstantInt::get(IntTy, -(uint64_t)Offset);
5497 else {
5498 Ops.push_back(SE.getUnknown(ICmpScaledV));
5499 ICmpScaledV = ConstantInt::get(IntTy, Offset);
5500 }
5501 } else {
5502 // Just add the immediate values. These again are expected to be matched
5503 // as part of the address.
5505 }
5506 }
5507
5508 // Expand the unfolded offset portion.
5509 int64_t UnfoldedOffset = F.UnfoldedOffset;
5510 if (UnfoldedOffset != 0) {
5511 // Just add the immediate values.
5513 UnfoldedOffset)));
5514 }
5515
5516 // Emit instructions summing all the operands.
5517 const SCEV *FullS = Ops.empty() ?
5518 SE.getConstant(IntTy, 0) :
5519 SE.getAddExpr(Ops);
5520 Value *FullV = Rewriter.expandCodeFor(FullS, Ty);
5521
5522 // We're done expanding now, so reset the rewriter.
5523 Rewriter.clearPostInc();
5524
5525 // An ICmpZero Formula represents an ICmp which we're handling as a
5526 // comparison against zero. Now that we've expanded an expression for that
5527 // form, update the ICmp's other operand.
5528 if (LU.Kind == LSRUse::ICmpZero) {
5529 ICmpInst *CI = cast<ICmpInst>(LF.UserInst);
5530 if (auto *OperandIsInstr = dyn_cast<Instruction>(CI->getOperand(1)))
5531 DeadInsts.emplace_back(OperandIsInstr);
5532 assert(!F.BaseGV && "ICmp does not support folding a global value and "
5533 "a scale at the same time!");
5534 if (F.Scale == -1) {
5535 if (ICmpScaledV->getType() != OpTy) {
5537 CastInst::getCastOpcode(ICmpScaledV, false, OpTy, false),
5538 ICmpScaledV, OpTy, "tmp", CI->getIterator());
5539 ICmpScaledV = Cast;
5540 }
5541 CI->setOperand(1, ICmpScaledV);
5542 } else {
5543 // A scale of 1 means that the scale has been expanded as part of the
5544 // base regs.
5545 assert((F.Scale == 0 || F.Scale == 1) &&
5546 "ICmp does not support folding a global value and "
5547 "a scale at the same time!");
5549 -(uint64_t)Offset);
5550 if (C->getType() != OpTy) {
5552 CastInst::getCastOpcode(C, false, OpTy, false), C, OpTy,
5553 CI->getModule()->getDataLayout());
5554 assert(C && "Cast of ConstantInt should have folded");
5555 }
5556
5557 CI->setOperand(1, C);
5558 }
5559 }
5560
5561 return FullV;
5562}
5563
5564/// Helper for Rewrite. PHI nodes are special because the use of their operands
5565/// effectively happens in their predecessor blocks, so the expression may need
5566/// to be expanded in multiple places.
5567void LSRInstance::RewriteForPHI(
5568 PHINode *PN, const LSRUse &LU, const LSRFixup &LF, const Formula &F,
5569 SmallVectorImpl<WeakTrackingVH> &DeadInsts) const {
5571
5572 // Inserting instructions in the loop and using them as PHI's input could
5573 // break LCSSA in case if PHI's parent block is not a loop exit (i.e. the
5574 // corresponding incoming block is not loop exiting). So collect all such
5575 // instructions to form LCSSA for them later.
5576 SmallVector<Instruction *, 4> InsertedNonLCSSAInsts;
5577
5578 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
5579 if (PN->getIncomingValue(i) == LF.OperandValToReplace) {
5580 bool needUpdateFixups = false;
5581 BasicBlock *BB = PN->getIncomingBlock(i);
5582
5583 // If this is a critical edge, split the edge so that we do not insert
5584 // the code on all predecessor/successor paths. We do this unless this
5585 // is the canonical backedge for this loop, which complicates post-inc
5586 // users.
5587 if (e != 1 && BB->getTerminator()->getNumSuccessors() > 1 &&
5588 !isa<IndirectBrInst>(BB->getTerminator()) &&
5589 !isa<CatchSwitchInst>(BB->getTerminator())) {
5590 BasicBlock *Parent = PN->getParent();
5591 Loop *PNLoop = LI.getLoopFor(Parent);
5592 if (!PNLoop || Parent != PNLoop->getHeader()) {
5593 // Split the critical edge.
5594 BasicBlock *NewBB = nullptr;
5595 if (!Parent->isLandingPad()) {
5596 NewBB =
5597 SplitCriticalEdge(BB, Parent,
5598 CriticalEdgeSplittingOptions(&DT, &LI, MSSAU)
5599 .setMergeIdenticalEdges()
5600 .setKeepOneInputPHIs());
5601 } else {
5603 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
5604 SplitLandingPadPredecessors(Parent, BB, "", "", NewBBs, &DTU, &LI);
5605 NewBB = NewBBs[0];
5606 }
5607 // If NewBB==NULL, then SplitCriticalEdge refused to split because all
5608 // phi predecessors are identical. The simple thing to do is skip
5609 // splitting in this case rather than complicate the API.
5610 if (NewBB) {
5611 // If PN is outside of the loop and BB is in the loop, we want to
5612 // move the block to be immediately before the PHI block, not
5613 // immediately after BB.
5614 if (L->contains(BB) && !L->contains(PN))
5615 NewBB->moveBefore(PN->getParent());
5616
5617 // Splitting the edge can reduce the number of PHI entries we have.
5618 e = PN->getNumIncomingValues();
5619 BB = NewBB;
5620 i = PN->getBasicBlockIndex(BB);
5621
5622 needUpdateFixups = true;
5623 }
5624 }
5625 }
5626
5627 std::pair<DenseMap<BasicBlock *, Value *>::iterator, bool> Pair =
5628 Inserted.insert(std::make_pair(BB, static_cast<Value *>(nullptr)));
5629 if (!Pair.second)
5630 PN->setIncomingValue(i, Pair.first->second);
5631 else {
5632 Value *FullV =
5633 Expand(LU, LF, F, BB->getTerminator()->getIterator(), DeadInsts);
5634
5635 // If this is reuse-by-noop-cast, insert the noop cast.
5636 Type *OpTy = LF.OperandValToReplace->getType();
5637 if (FullV->getType() != OpTy)
5638 FullV = CastInst::Create(
5639 CastInst::getCastOpcode(FullV, false, OpTy, false), FullV,
5640 LF.OperandValToReplace->getType(), "tmp",
5641 BB->getTerminator()->getIterator());
5642
5643 // If the incoming block for this value is not in the loop, it means the
5644 // current PHI is not in a loop exit, so we must create a LCSSA PHI for
5645 // the inserted value.
5646 if (auto *I = dyn_cast<Instruction>(FullV))
5647 if (L->contains(I) && !L->contains(BB))
5648 InsertedNonLCSSAInsts.push_back(I);
5649
5650 PN->setIncomingValue(i, FullV);
5651 Pair.first->second = FullV;
5652 }
5653
5654 // If LSR splits critical edge and phi node has other pending
5655 // fixup operands, we need to update those pending fixups. Otherwise
5656 // formulae will not be implemented completely and some instructions
5657 // will not be eliminated.
5658 if (needUpdateFixups) {
5659 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx)
5660 for (LSRFixup &Fixup : Uses[LUIdx].Fixups)
5661 // If fixup is supposed to rewrite some operand in the phi
5662 // that was just updated, it may be already moved to
5663 // another phi node. Such fixup requires update.
5664 if (Fixup.UserInst == PN) {
5665 // Check if the operand we try to replace still exists in the
5666 // original phi.
5667 bool foundInOriginalPHI = false;
5668 for (const auto &val : PN->incoming_values())
5669 if (val == Fixup.OperandValToReplace) {
5670 foundInOriginalPHI = true;
5671 break;
5672 }
5673
5674 // If fixup operand found in original PHI - nothing to do.
5675 if (foundInOriginalPHI)
5676 continue;
5677
5678 // Otherwise it might be moved to another PHI and requires update.
5679 // If fixup operand not found in any of the incoming blocks that
5680 // means we have already rewritten it - nothing to do.
5681 for (const auto &Block : PN->blocks())
5682 for (BasicBlock::iterator I = Block->begin(); isa<PHINode>(I);
5683 ++I) {
5684 PHINode *NewPN = cast<PHINode>(I);
5685 for (const auto &val : NewPN->incoming_values())
5686 if (val == Fixup.OperandValToReplace)
5687 Fixup.UserInst = NewPN;
5688 }
5689 }
5690 }
5691 }
5692
5693 formLCSSAForInstructions(InsertedNonLCSSAInsts, DT, LI, &SE);
5694}
5695
5696/// Emit instructions for the leading candidate expression for this LSRUse (this
5697/// is called "expanding"), and update the UserInst to reference the newly
5698/// expanded value.
5699void LSRInstance::Rewrite(const LSRUse &LU, const LSRFixup &LF,
5700 const Formula &F,
5701 SmallVectorImpl<WeakTrackingVH> &DeadInsts) const {
5702 // First, find an insertion point that dominates UserInst. For PHI nodes,
5703 // find the nearest block which dominates all the relevant uses.
5704 if (PHINode *PN = dyn_cast<PHINode>(LF.UserInst)) {
5705 RewriteForPHI(PN, LU, LF, F, DeadInsts);
5706 } else {
5707 Value *FullV = Expand(LU, LF, F, LF.UserInst->getIterator(), DeadInsts);
5708
5709 // If this is reuse-by-noop-cast, insert the noop cast.
5710 Type *OpTy = LF.OperandValToReplace->getType();
5711 if (FullV->getType() != OpTy) {
5712 Instruction *Cast =
5713 CastInst::Create(CastInst::getCastOpcode(FullV, false, OpTy, false),
5714 FullV, OpTy, "tmp", LF.UserInst->getIterator());
5715 FullV = Cast;
5716 }
5717
5718 // Update the user. ICmpZero is handled specially here (for now) because
5719 // Expand may have updated one of the operands of the icmp already, and
5720 // its new value may happen to be equal to LF.OperandValToReplace, in
5721 // which case doing replaceUsesOfWith leads to replacing both operands
5722 // with the same value. TODO: Reorganize this.
5723 if (LU.Kind == LSRUse::ICmpZero)
5724 LF.UserInst->setOperand(0, FullV);
5725 else
5726 LF.UserInst->replaceUsesOfWith(LF.OperandValToReplace, FullV);
5727 }
5728
5729 if (auto *OperandIsInstr = dyn_cast<Instruction>(LF.OperandValToReplace))
5730 DeadInsts.emplace_back(OperandIsInstr);
5731}
5732
5733// Trying to hoist the IVInc to loop header if all IVInc users are in
5734// the loop header. It will help backend to generate post index load/store
5735// when the latch block is different from loop header block.
5736static bool canHoistIVInc(const TargetTransformInfo &TTI, const LSRFixup &Fixup,
5737 const LSRUse &LU, Instruction *IVIncInsertPos,
5738 Loop *L) {
5739 if (LU.Kind != LSRUse::Address)
5740 return false;
5741
5742 // For now this code do the conservative optimization, only work for
5743 // the header block. Later we can hoist the IVInc to the block post
5744 // dominate all users.
5745 BasicBlock *LHeader = L->getHeader();
5746 if (IVIncInsertPos->getParent() == LHeader)
5747 return false;
5748
5749 if (!Fixup.OperandValToReplace ||
5750 any_of(Fixup.OperandValToReplace->users(), [&LHeader](User *U) {
5751 Instruction *UI = cast<Instruction>(U);
5752 return UI->getParent() != LHeader;
5753 }))
5754 return false;
5755
5756 Instruction *I = Fixup.UserInst;
5757 Type *Ty = I->getType();
5758 return Ty->isIntegerTy() &&
5759 ((isa<LoadInst>(I) && TTI.isIndexedLoadLegal(TTI.MIM_PostInc, Ty)) ||
5760 (isa<StoreInst>(I) && TTI.isIndexedStoreLegal(TTI.MIM_PostInc, Ty)));
5761}
5762
5763/// Rewrite all the fixup locations with new values, following the chosen
5764/// solution.
5765void LSRInstance::ImplementSolution(
5766 const SmallVectorImpl<const Formula *> &Solution) {
5767 // Keep track of instructions we may have made dead, so that
5768 // we can remove them after we are done working.
5770
5771 // Mark phi nodes that terminate chains so the expander tries to reuse them.
5772 for (const IVChain &Chain : IVChainVec) {
5773 if (PHINode *PN = dyn_cast<PHINode>(Chain.tailUserInst()))
5774 Rewriter.setChainedPhi(PN);
5775 }
5776
5777 // Expand the new value definitions and update the users.
5778 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx)
5779 for (const LSRFixup &Fixup : Uses[LUIdx].Fixups) {
5780 Instruction *InsertPos =
5781 canHoistIVInc(TTI, Fixup, Uses[LUIdx], IVIncInsertPos, L)
5782 ? L->getHeader()->getTerminator()
5783 : IVIncInsertPos;
5784 Rewriter.setIVIncInsertPos(L, InsertPos);
5785 Rewrite(Uses[LUIdx], Fixup, *Solution[LUIdx], DeadInsts);
5786 Changed = true;
5787 }
5788
5789 for (const IVChain &Chain : IVChainVec) {
5790 GenerateIVChain(Chain, DeadInsts);
5791 Changed = true;
5792 }
5793
5794 for (const WeakVH &IV : Rewriter.getInsertedIVs())
5795 if (IV && dyn_cast<Instruction>(&*IV)->getParent())
5796 ScalarEvolutionIVs.push_back(IV);
5797
5798 // Clean up after ourselves. This must be done before deleting any
5799 // instructions.
5800 Rewriter.clear();
5801
5803 &TLI, MSSAU);
5804
5805 // In our cost analysis above, we assume that each addrec consumes exactly
5806 // one register, and arrange to have increments inserted just before the
5807 // latch to maximimize the chance this is true. However, if we reused
5808 // existing IVs, we now need to move the increments to match our
5809 // expectations. Otherwise, our cost modeling results in us having a
5810 // chosen a non-optimal result for the actual schedule. (And yes, this
5811 // scheduling decision does impact later codegen.)
5812 for (PHINode &PN : L->getHeader()->phis()) {
5813 BinaryOperator *BO = nullptr;
5814 Value *Start = nullptr, *Step = nullptr;
5815 if (!matchSimpleRecurrence(&PN, BO, Start, Step))
5816 continue;
5817
5818 switch (BO->getOpcode()) {
5819 case Instruction::Sub:
5820 if (BO->getOperand(0) != &PN)
5821 // sub is non-commutative - match handling elsewhere in LSR
5822 continue;
5823 break;
5824 case Instruction::Add:
5825 break;
5826 default:
5827 continue;
5828 };
5829
5830 if (!isa<Constant>(Step))
5831 // If not a constant step, might increase register pressure
5832 // (We assume constants have been canonicalized to RHS)
5833 continue;
5834
5835 if (BO->getParent() == IVIncInsertPos->getParent())
5836 // Only bother moving across blocks. Isel can handle block local case.
5837 continue;
5838
5839 // Can we legally schedule inc at the desired point?
5840 if (!llvm::all_of(BO->uses(),
5841 [&](Use &U) {return DT.dominates(IVIncInsertPos, U);}))
5842 continue;
5843 BO->moveBefore(IVIncInsertPos);
5844 Changed = true;
5845 }
5846
5847
5848}
5849
5850LSRInstance::LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE,
5851 DominatorTree &DT, LoopInfo &LI,
5854 : IU(IU), SE(SE), DT(DT), LI(LI), AC(AC), TLI(TLI), TTI(TTI), L(L),
5855 MSSAU(MSSAU), AMK(PreferredAddresingMode.getNumOccurrences() > 0
5857 : TTI.getPreferredAddressingMode(L, &SE)),
5858 Rewriter(SE, L->getHeader()->getModule()->getDataLayout(), "lsr", false),
5859 BaselineCost(L, SE, TTI, AMK) {
5860 // If LoopSimplify form is not available, stay out of trouble.
5861 if (!L->isLoopSimplifyForm())
5862 return;
5863
5864 // If there's no interesting work to be done, bail early.
5865 if (IU.empty()) return;
5866
5867 // If there's too much analysis to be done, bail early. We won't be able to
5868 // model the problem anyway.
5869 unsigned NumUsers = 0;
5870 for (const IVStrideUse &U : IU) {
5871 if (++NumUsers > MaxIVUsers) {
5872 (void)U;
5873 LLVM_DEBUG(dbgs() << "LSR skipping loop, too many IV Users in " << U
5874 << "\n");
5875 return;
5876 }
5877 // Bail out if we have a PHI on an EHPad that gets a value from a
5878 // CatchSwitchInst. Because the CatchSwitchInst cannot be split, there is
5879 // no good place to stick any instructions.
5880 if (auto *PN = dyn_cast<PHINode>(U.getUser())) {
5881 auto *FirstNonPHI = PN->getParent()->getFirstNonPHI();
5882 if (isa<FuncletPadInst>(FirstNonPHI) ||
5883 isa<CatchSwitchInst>(FirstNonPHI))
5884 for (BasicBlock *PredBB : PN->blocks())
5885 if (isa<CatchSwitchInst>(PredBB->getFirstNonPHI()))
5886 return;
5887 }
5888 }
5889
5890 LLVM_DEBUG(dbgs() << "\nLSR on loop ";
5891 L->getHeader()->printAsOperand(dbgs(), /*PrintType=*/false);
5892 dbgs() << ":\n");
5893
5894 // Configure SCEVExpander already now, so the correct mode is used for
5895 // isSafeToExpand() checks.
5896#ifndef NDEBUG
5897 Rewriter.setDebugType(DEBUG_TYPE);
5898#endif
5899 Rewriter.disableCanonicalMode();
5900 Rewriter.enableLSRMode();
5901
5902 // First, perform some low-level loop optimizations.
5903 OptimizeShadowIV();
5904 OptimizeLoopTermCond();
5905
5906 // If loop preparation eliminates all interesting IV users, bail.
5907 if (IU.empty()) return;
5908
5909 // Skip nested loops until we can model them better with formulae.
5910 if (!L->isInnermost()) {
5911 LLVM_DEBUG(dbgs() << "LSR skipping outer loop " << *L << "\n");
5912 return;
5913 }
5914
5915 // Start collecting data and preparing for the solver.
5916 // If number of registers is not the major cost, we cannot benefit from the
5917 // current profitable chain optimization which is based on number of
5918 // registers.
5919 // FIXME: add profitable chain optimization for other kinds major cost, for
5920 // example number of instructions.
5922 CollectChains();
5923 CollectInterestingTypesAndFactors();
5924 CollectFixupsAndInitialFormulae();
5925 CollectLoopInvariantFixupsAndFormulae();
5926
5927 if (Uses.empty())
5928 return;
5929
5930 LLVM_DEBUG(dbgs() << "LSR found " << Uses.size() << " uses:\n";
5931 print_uses(dbgs()));
5932
5933 // Now use the reuse data to generate a bunch of interesting ways
5934 // to formulate the values needed for the uses.
5935 GenerateAllReuseFormulae();
5936
5937 FilterOutUndesirableDedicatedRegisters();
5938 NarrowSearchSpaceUsingHeuristics();
5939
5941 Solve(Solution);
5942
5943 // Release memory that is no longer needed.
5944 Factors.clear();
5945 Types.clear();
5946 RegUses.clear();
5947
5948 if (Solution.empty())
5949 return;
5950
5951#ifndef NDEBUG
5952 // Formulae should be legal.
5953 for (const LSRUse &LU : Uses) {
5954 for (const Formula &F : LU.Formulae)
5955 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
5956 F) && "Illegal formula generated!");
5957 };
5958#endif
5959
5960 // Now that we've decided what we want, make it so.
5961 ImplementSolution(Solution);
5962}
5963
5964#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
5965void LSRInstance::print_factors_and_types(raw_ostream &OS) const {
5966 if (Factors.empty() && Types.empty()) return;
5967
5968 OS << "LSR has identified the following interesting factors and types: ";
5969 bool First = true;
5970
5971 for (int64_t Factor : Factors) {
5972 if (!First) OS << ", ";
5973 First = false;
5974 OS << '*' << Factor;
5975 }
5976
5977 for (Type *Ty : Types) {
5978 if (!First) OS << ", ";
5979 First = false;
5980 OS << '(' << *Ty << ')';
5981 }
5982 OS << '\n';
5983}
5984
5985void LSRInstance::print_fixups(raw_ostream &OS) const {
5986 OS << "LSR is examining the following fixup sites:\n";
5987 for (const LSRUse &LU : Uses)
5988 for (const LSRFixup &LF : LU.Fixups) {
5989 dbgs() << " ";
5990 LF.print(OS);
5991 OS << '\n';
5992 }
5993}
5994
5995void LSRInstance::print_uses(raw_ostream &OS) const {
5996 OS << "LSR is examining the following uses:\n";
5997 for (const LSRUse &LU : Uses) {
5998 dbgs() << " ";
5999 LU.print(OS);
6000 OS << '\n';
6001 for (const Formula &F : LU.Formulae) {
6002 OS << " ";
6003 F.print(OS);
6004 OS << '\n';
6005 }
6006 }
6007}
6008
6009void LSRInstance::print(raw_ostream &OS) const {
6010 print_factors_and_types(OS);
6011 print_fixups(OS);
6012 print_uses(OS);
6013}
6014
6015LLVM_DUMP_METHOD void LSRInstance::dump() const {
6016 print(errs()); errs() << '\n';
6017}
6018#endif
6019
6020namespace {
6021
6022class LoopStrengthReduce : public LoopPass {
6023public:
6024 static char ID; // Pass ID, replacement for typeid
6025
6026 LoopStrengthReduce();
6027
6028private:
6029 bool runOnLoop(Loop *L, LPPassManager &LPM) override;
6030 void getAnalysisUsage(AnalysisUsage &AU) const override;
6031};
6032
6033} // end anonymous namespace
6034
6035LoopStrengthReduce::LoopStrengthReduce() : LoopPass(ID) {
6037}
6038
6039void LoopStrengthReduce::getAnalysisUsage(AnalysisUsage &AU) const {
6040 // We split critical edges, so we change the CFG. However, we do update
6041 // many analyses if they are around.
6043
6053 // Requiring LoopSimplify a second time here prevents IVUsers from running
6054 // twice, since LoopSimplify was invalidated by running ScalarEvolution.
6060}
6061
6062namespace {
6063
6064/// Enables more convenient iteration over a DWARF expression vector.
6066ToDwarfOpIter(SmallVectorImpl<uint64_t> &Expr) {
6071 return {Begin, End};
6072}
6073
6074struct SCEVDbgValueBuilder {
6075 SCEVDbgValueBuilder() = default;
6076 SCEVDbgValueBuilder(const SCEVDbgValueBuilder &Base) { clone(Base); }
6077
6078 void clone(const SCEVDbgValueBuilder &Base) {
6079 LocationOps = Base.LocationOps;
6080 Expr = Base.Expr;
6081 }
6082
6083 void clear() {
6084 LocationOps.clear();
6085 Expr.clear();
6086 }
6087
6088 /// The DIExpression as we translate the SCEV.
6090 /// The location ops of the DIExpression.
6091 SmallVector<Value *, 2> LocationOps;
6092
6093 void pushOperator(uint64_t Op) { Expr.push_back(Op); }
6094 void pushUInt(uint64_t Operand) { Expr.push_back(Operand); }
6095
6096 /// Add a DW_OP_LLVM_arg to the expression, followed by the index of the value
6097 /// in the set of values referenced by the expression.
6098 void pushLocation(llvm::Value *V) {
6100 auto *It = llvm::find(LocationOps, V);
6101 unsigned ArgIndex = 0;
6102 if (It != LocationOps.end()) {
6103 ArgIndex = std::distance(LocationOps.begin(), It);
6104 } else {
6105 ArgIndex = LocationOps.size();
6106 LocationOps.push_back(V);
6107 }
6108 Expr.push_back(ArgIndex);
6109 }
6110
6111 void pushValue(const SCEVUnknown *U) {
6112 llvm::Value *V = cast<SCEVUnknown>(U)->getValue();
6113 pushLocation(V);
6114 }
6115
6116 bool pushConst(const SCEVConstant *C) {
6117 if (C->getAPInt().getSignificantBits() > 64)
6118 return false;
6119 Expr.push_back(llvm::dwarf::DW_OP_consts);
6120 Expr.push_back(C->getAPInt().getSExtValue());
6121 return true;
6122 }
6123
6124 // Iterating the expression as DWARF ops is convenient when updating
6125 // DWARF_OP_LLVM_args.
6127 return ToDwarfOpIter(Expr);
6128 }
6129
6130 /// Several SCEV types are sequences of the same arithmetic operator applied
6131 /// to constants and values that may be extended or truncated.
6132 bool pushArithmeticExpr(const llvm::SCEVCommutativeExpr *CommExpr,
6133 uint64_t DwarfOp) {
6134 assert((isa<llvm::SCEVAddExpr>(CommExpr) || isa<SCEVMulExpr>(CommExpr)) &&
6135 "Expected arithmetic SCEV type");
6136 bool Success = true;
6137 unsigned EmitOperator = 0;
6138 for (const auto &Op : CommExpr->operands()) {
6139 Success &= pushSCEV(Op);
6140
6141 if (EmitOperator >= 1)
6142 pushOperator(DwarfOp);
6143 ++EmitOperator;
6144 }
6145 return Success;
6146 }
6147
6148 // TODO: Identify and omit noop casts.
6149 bool pushCast(const llvm::SCEVCastExpr *C, bool IsSigned) {
6150 const llvm::SCEV *Inner = C->getOperand(0);
6151 const llvm::Type *Type = C->getType();
6152 uint64_t ToWidth = Type->getIntegerBitWidth();
6153 bool Success = pushSCEV(Inner);
6154 uint64_t CastOps[] = {dwarf::DW_OP_LLVM_convert, ToWidth,
6155 IsSigned ? llvm::dwarf::DW_ATE_signed
6156 : llvm::dwarf::DW_ATE_unsigned};
6157 for (const auto &Op : CastOps)
6158 pushOperator(Op);
6159 return Success;
6160 }
6161
6162 // TODO: MinMax - although these haven't been encountered in the test suite.
6163 bool pushSCEV(const llvm::SCEV *S) {
6164 bool Success = true;
6165 if (const SCEVConstant *StartInt = dyn_cast<SCEVConstant>(S)) {
6166 Success &= pushConst(StartInt);
6167
6168 } else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
6169 if (!U->getValue())
6170 return false;
6171 pushLocation(U->getValue());
6172
6173 } else if (const SCEVMulExpr *MulRec = dyn_cast<SCEVMulExpr>(S)) {
6174 Success &= pushArithmeticExpr(MulRec, llvm::dwarf::DW_OP_mul);
6175
6176 } else if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) {
6177 Success &= pushSCEV(UDiv->getLHS());
6178 Success &= pushSCEV(UDiv->getRHS());
6179 pushOperator(llvm::dwarf::DW_OP_div);
6180
6181 } else if (const SCEVCastExpr *Cast = dyn_cast<SCEVCastExpr>(S)) {
6182 // Assert if a new and unknown SCEVCastEXpr type is encountered.
6183 assert((isa<SCEVZeroExtendExpr>(Cast) || isa<SCEVTruncateExpr>(Cast) ||
6184 isa<SCEVPtrToIntExpr>(Cast) || isa<SCEVSignExtendExpr>(Cast)) &&
6185 "Unexpected cast type in SCEV.");
6186 Success &= pushCast(Cast, (isa<SCEVSignExtendExpr>(Cast)));
6187
6188 } else if (const SCEVAddExpr *AddExpr = dyn_cast<SCEVAddExpr>(S)) {
6189 Success &= pushArithmeticExpr(AddExpr, llvm::dwarf::DW_OP_plus);
6190
6191 } else if (isa<SCEVAddRecExpr>(S)) {
6192 // Nested SCEVAddRecExpr are generated by nested loops and are currently
6193 // unsupported.
6194 return false;
6195
6196 } else {
6197 return false;
6198 }
6199 return Success;
6200 }
6201
6202 /// Return true if the combination of arithmetic operator and underlying
6203 /// SCEV constant value is an identity function.
6204 bool isIdentityFunction(uint64_t Op, const SCEV *S) {
6205 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) {
6206 if (C->getAPInt().getSignificantBits() > 64)
6207 return false;
6208 int64_t I = C->getAPInt().getSExtValue();
6209 switch (Op) {
6210 case llvm::dwarf::DW_OP_plus:
6211 case llvm::dwarf::DW_OP_minus:
6212 return I == 0;
6213 case llvm::dwarf::DW_OP_mul:
6214 case llvm::dwarf::DW_OP_div:
6215 return I == 1;
6216 }
6217 }
6218 return false;
6219 }
6220
6221 /// Convert a SCEV of a value to a DIExpression that is pushed onto the
6222 /// builder's expression stack. The stack should already contain an
6223 /// expression for the iteration count, so that it can be multiplied by
6224 /// the stride and added to the start.
6225 /// Components of the expression are omitted if they are an identity function.
6226 /// Chain (non-affine) SCEVs are not supported.
6227 bool SCEVToValueExpr(const llvm::SCEVAddRecExpr &SAR, ScalarEvolution &SE) {
6228 assert(SAR.isAffine() && "Expected affine SCEV");
6229 // TODO: Is this check needed?
6230 if (isa<SCEVAddRecExpr>(SAR.getStart()))
6231 return false;
6232
6233 const SCEV *Start = SAR.getStart();
6234 const SCEV *Stride = SAR.getStepRecurrence(SE);
6235
6236 // Skip pushing arithmetic noops.
6237 if (!isIdentityFunction(llvm::dwarf::DW_OP_mul, Stride)) {
6238 if (!pushSCEV(Stride))
6239 return false;
6240 pushOperator(llvm::dwarf::DW_OP_mul);
6241 }
6242 if (!isIdentityFunction(llvm::dwarf::DW_OP_plus, Start)) {
6243 if (!pushSCEV(Start))
6244 return false;
6245 pushOperator(llvm::dwarf::DW_OP_plus);
6246 }
6247 return true;
6248 }
6249
6250 /// Create an expression that is an offset from a value (usually the IV).
6251 void createOffsetExpr(int64_t Offset, Value *OffsetValue) {
6252 pushLocation(OffsetValue);
6254 LLVM_DEBUG(
6255 dbgs() << "scev-salvage: Generated IV offset expression. Offset: "
6256 << std::to_string(Offset) << "\n");
6257 }
6258
6259 /// Combine a translation of the SCEV and the IV to create an expression that
6260 /// recovers a location's value.
6261 /// returns true if an expression was created.
6262 bool createIterCountExpr(const SCEV *S,
6263 const SCEVDbgValueBuilder &IterationCount,
6264 ScalarEvolution &SE) {
6265 // SCEVs for SSA values are most frquently of the form
6266 // {start,+,stride}, but sometimes they are ({start,+,stride} + %a + ..).
6267 // This is because %a is a PHI node that is not the IV. However, these
6268 // SCEVs have not been observed to result in debuginfo-lossy optimisations,
6269 // so its not expected this point will be reached.
6270 if (!isa<SCEVAddRecExpr>(S))
6271 return false;
6272
6273 LLVM_DEBUG(dbgs() << "scev-salvage: Location to salvage SCEV: " << *S
6274 << '\n');
6275
6276 const auto *Rec = cast<SCEVAddRecExpr>(S);
6277 if (!Rec->isAffine())
6278 return false;
6279
6281 return false;
6282
6283 // Initialise a new builder with the iteration count expression. In
6284 // combination with the value's SCEV this enables recovery.
6285 clone(IterationCount);
6286 if (!SCEVToValueExpr(*Rec, SE))
6287 return false;
6288
6289 return true;
6290 }
6291
6292 /// Convert a SCEV of a value to a DIExpression that is pushed onto the
6293 /// builder's expression stack. The stack should already contain an
6294 /// expression for the iteration count, so that it can be multiplied by
6295 /// the stride and added to the start.
6296 /// Components of the expression are omitted if they are an identity function.
6297 bool SCEVToIterCountExpr(const llvm::SCEVAddRecExpr &SAR,
6298 ScalarEvolution &SE) {
6299 assert(SAR.isAffine() && "Expected affine SCEV");
6300 if (isa<SCEVAddRecExpr>(SAR.getStart())) {
6301 LLVM_DEBUG(dbgs() << "scev-salvage: IV SCEV. Unsupported nested AddRec: "
6302 << SAR << '\n');
6303 return false;
6304 }
6305 const SCEV *Start = SAR.getStart();
6306 const SCEV *Stride = SAR.getStepRecurrence(SE);
6307
6308 // Skip pushing arithmetic noops.
6309 if (!isIdentityFunction(llvm::dwarf::DW_OP_minus, Start)) {
6310 if (!pushSCEV(Start))
6311 return false;
6312 pushOperator(llvm::dwarf::DW_OP_minus);
6313 }
6314 if (!isIdentityFunction(llvm::dwarf::DW_OP_div, Stride)) {
6315 if (!pushSCEV(Stride))
6316 return false;
6317 pushOperator(llvm::dwarf::DW_OP_div);
6318 }
6319 return true;
6320 }
6321
6322 // Append the current expression and locations to a location list and an
6323 // expression list. Modify the DW_OP_LLVM_arg indexes to account for
6324 // the locations already present in the destination list.
6325 void appendToVectors(SmallVectorImpl<uint64_t> &DestExpr,
6326 SmallVectorImpl<Value *> &DestLocations) {
6327 assert(!DestLocations.empty() &&
6328 "Expected the locations vector to contain the IV");
6329 // The DWARF_OP_LLVM_arg arguments of the expression being appended must be
6330 // modified to account for the locations already in the destination vector.
6331 // All builders contain the IV as the first location op.
6332 assert(!LocationOps.empty() &&
6333 "Expected the location ops to contain the IV.");
6334 // DestIndexMap[n] contains the index in DestLocations for the nth
6335 // location in this SCEVDbgValueBuilder.
6336 SmallVector<uint64_t, 2> DestIndexMap;
6337 for (const auto &Op : LocationOps) {
6338 auto It = find(DestLocations, Op);
6339 if (It != DestLocations.end()) {
6340 // Location already exists in DestLocations, reuse existing ArgIndex.
6341 DestIndexMap.push_back(std::distance(DestLocations.begin(), It));
6342 continue;
6343 }
6344 // Location is not in DestLocations, add it.
6345 DestIndexMap.push_back(DestLocations.size());
6346 DestLocations.push_back(Op);
6347 }
6348
6349 for (const auto &Op : expr_ops()) {
6350 if (Op.getOp() != dwarf::DW_OP_LLVM_arg) {
6351 Op.appendToVector(DestExpr);
6352 continue;
6353 }
6354
6356 // `DW_OP_LLVM_arg n` represents the nth LocationOp in this SCEV,
6357 // DestIndexMap[n] contains its new index in DestLocations.
6358 uint64_t NewIndex = DestIndexMap[Op.getArg(0)];
6359 DestExpr.push_back(NewIndex);
6360 }
6361 }
6362};
6363
6364/// Holds all the required data to salvage a dbg.value using the pre-LSR SCEVs
6365/// and DIExpression.
6366struct DVIRecoveryRec {
6367 DVIRecoveryRec(DbgValueInst *DbgValue)
6368 : DbgRef(DbgValue), Expr(DbgValue->getExpression()),
6369 HadLocationArgList(false) {}
6370 DVIRecoveryRec(DPValue *DPV)
6371 : DbgRef(DPV), Expr(DPV->getExpression()), HadLocationArgList(false) {}
6372
6374 DIExpression *Expr;
6375 bool HadLocationArgList;
6376 SmallVector<WeakVH, 2> LocationOps;
6379
6380 void clear() {
6381 for (auto &RE : RecoveryExprs)
6382 RE.reset();
6383 RecoveryExprs.clear();
6384 }
6385
6386 ~DVIRecoveryRec() { clear(); }
6387};
6388} // namespace
6389
6390/// Returns the total number of DW_OP_llvm_arg operands in the expression.
6391/// This helps in determining if a DIArglist is necessary or can be omitted from
6392/// the dbg.value.
6394 auto expr_ops = ToDwarfOpIter(Expr);
6395 unsigned Count = 0;
6396 for (auto Op : expr_ops)
6397 if (Op.getOp() == dwarf::DW_OP_LLVM_arg)
6398 Count++;
6399 return Count;
6400}
6401
6402/// Overwrites DVI with the location and Ops as the DIExpression. This will
6403/// create an invalid expression if Ops has any dwarf::DW_OP_llvm_arg operands,
6404/// because a DIArglist is not created for the first argument of the dbg.value.
6405template <typename T>
6406static void updateDVIWithLocation(T &DbgVal, Value *Location,
6408 assert(numLLVMArgOps(Ops) == 0 && "Expected expression that does not "
6409 "contain any DW_OP_llvm_arg operands.");
6410 DbgVal.setRawLocation(ValueAsMetadata::get(Location));
6411 DbgVal.setExpression(DIExpression::get(DbgVal.getContext(), Ops));
6412 DbgVal.setExpression(DIExpression::get(DbgVal.getContext(), Ops));
6413}
6414
6415/// Overwrite DVI with locations placed into a DIArglist.
6416template <typename T>
6417static void updateDVIWithLocations(T &DbgVal,
6418 SmallVectorImpl<Value *> &Locations,
6420 assert(numLLVMArgOps(Ops) != 0 &&
6421 "Expected expression that references DIArglist locations using "
6422 "DW_OP_llvm_arg operands.");
6424 for (Value *V : Locations)
6425 MetadataLocs.push_back(ValueAsMetadata::get(V));
6426 auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(MetadataLocs);
6427 DbgVal.setRawLocation(llvm::DIArgList::get(DbgVal.getContext(), ValArrayRef));
6428 DbgVal.setExpression(DIExpression::get(DbgVal.getContext(), Ops));
6429}
6430
6431/// Write the new expression and new location ops for the dbg.value. If possible
6432/// reduce the szie of the dbg.value intrinsic by omitting DIArglist. This
6433/// can be omitted if:
6434/// 1. There is only a single location, refenced by a single DW_OP_llvm_arg.
6435/// 2. The DW_OP_LLVM_arg is the first operand in the expression.
6436static void UpdateDbgValueInst(DVIRecoveryRec &DVIRec,
6437 SmallVectorImpl<Value *> &NewLocationOps,
6439 auto UpdateDbgValueInstImpl = [&](auto *DbgVal) {
6440 unsigned NumLLVMArgs = numLLVMArgOps(NewExpr);
6441 if (NumLLVMArgs == 0) {
6442 // Location assumed to be on the stack.
6443 updateDVIWithLocation(*DbgVal, NewLocationOps[0], NewExpr);
6444 } else if (NumLLVMArgs == 1 && NewExpr[0] == dwarf::DW_OP_LLVM_arg) {
6445 // There is only a single DW_OP_llvm_arg at the start of the expression,
6446 // so it can be omitted along with DIArglist.
6447 assert(NewExpr[1] == 0 &&
6448 "Lone LLVM_arg in a DIExpression should refer to location-op 0.");
6450 updateDVIWithLocation(*DbgVal, NewLocationOps[0], ShortenedOps);
6451 } else {
6452 // Multiple DW_OP_llvm_arg, so DIArgList is strictly necessary.
6453 updateDVIWithLocations(*DbgVal, NewLocationOps, NewExpr);
6454 }
6455
6456 // If the DIExpression was previously empty then add the stack terminator.
6457 // Non-empty expressions have only had elements inserted into them and so
6458 // the terminator should already be present e.g. stack_value or fragment.
6459 DIExpression *SalvageExpr = DbgVal->getExpression();
6460 if (!DVIRec.Expr->isComplex() && SalvageExpr->isComplex()) {
6461 SalvageExpr =
6462 DIExpression::append(SalvageExpr, {dwarf::DW_OP_stack_value});
6463 DbgVal->setExpression(SalvageExpr);
6464 }
6465 };
6466 if (isa<DbgValueInst *>(DVIRec.DbgRef))
6467 UpdateDbgValueInstImpl(cast<DbgValueInst *>(DVIRec.DbgRef));
6468 else
6469 UpdateDbgValueInstImpl(cast<DPValue *>(DVIRec.DbgRef));
6470}
6471
6472/// Cached location ops may be erased during LSR, in which case a poison is
6473/// required when restoring from the cache. The type of that location is no
6474/// longer available, so just use int8. The poison will be replaced by one or
6475/// more locations later when a SCEVDbgValueBuilder selects alternative
6476/// locations to use for the salvage.
6478 return (VH) ? VH : PoisonValue::get(llvm::Type::getInt8Ty(C));
6479}
6480
6481/// Restore the DVI's pre-LSR arguments. Substitute undef for any erased values.
6482static void restorePreTransformState(DVIRecoveryRec &DVIRec) {
6483 auto RestorePreTransformStateImpl = [&](auto *DbgVal) {
6484 LLVM_DEBUG(dbgs() << "scev-salvage: restore dbg.value to pre-LSR state\n"
6485 << "scev-salvage: post-LSR: " << *DbgVal << '\n');
6486 assert(DVIRec.Expr && "Expected an expression");
6487 DbgVal->setExpression(DVIRec.Expr);
6488
6489 // Even a single location-op may be inside a DIArgList and referenced with
6490 // DW_OP_LLVM_arg, which is valid only with a DIArgList.
6491 if (!DVIRec.HadLocationArgList) {
6492 assert(DVIRec.LocationOps.size() == 1 &&
6493 "Unexpected number of location ops.");
6494 // LSR's unsuccessful salvage attempt may have added DIArgList, which in
6495 // this case was not present before, so force the location back to a
6496 // single uncontained Value.
6497 Value *CachedValue =
6498 getValueOrPoison(DVIRec.LocationOps[0], DbgVal->getContext());
6499 DbgVal->setRawLocation(ValueAsMetadata::get(CachedValue));
6500 } else {
6502 for (WeakVH VH : DVIRec.LocationOps) {
6503 Value *CachedValue = getValueOrPoison(VH, DbgVal->getContext());
6504 MetadataLocs.push_back(ValueAsMetadata::get(CachedValue));
6505 }
6506 auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(MetadataLocs);
6507 DbgVal->setRawLocation(
6508 llvm::DIArgList::get(DbgVal->getContext(), ValArrayRef));
6509 }
6510 LLVM_DEBUG(dbgs() << "scev-salvage: pre-LSR: " << *DbgVal << '\n');
6511 };
6512 if (isa<DbgValueInst *>(DVIRec.DbgRef))
6513 RestorePreTransformStateImpl(cast<DbgValueInst *>(DVIRec.DbgRef));
6514 else
6515 RestorePreTransformStateImpl(cast<DPValue *>(DVIRec.DbgRef));
6516}
6517
6519 llvm::PHINode *LSRInductionVar, DVIRecoveryRec &DVIRec,
6520 const SCEV *SCEVInductionVar,
6521 SCEVDbgValueBuilder IterCountExpr) {
6522
6523 if (isa<DbgValueInst *>(DVIRec.DbgRef)
6524 ? !cast<DbgValueInst *>(DVIRec.DbgRef)->isKillLocation()
6525 : !cast<DPValue *>(DVIRec.DbgRef)->isKillLocation())
6526 return false;
6527
6528 // LSR may have caused several changes to the dbg.value in the failed salvage
6529 // attempt. So restore the DIExpression, the location ops and also the
6530 // location ops format, which is always DIArglist for multiple ops, but only
6531 // sometimes for a single op.
6533
6534 // LocationOpIndexMap[i] will store the post-LSR location index of
6535 // the non-optimised out location at pre-LSR index i.
6536 SmallVector<int64_t, 2> LocationOpIndexMap;
6537 LocationOpIndexMap.assign(DVIRec.LocationOps.size(), -1);
6538 SmallVector<Value *, 2> NewLocationOps;
6539 NewLocationOps.push_back(LSRInductionVar);
6540
6541 for (unsigned i = 0; i < DVIRec.LocationOps.size(); i++) {
6542 WeakVH VH = DVIRec.LocationOps[i];
6543 // Place the locations not optimised out in the list first, avoiding
6544 // inserts later. The map is used to update the DIExpression's
6545 // DW_OP_LLVM_arg arguments as the expression is updated.
6546 if (VH && !isa<UndefValue>(VH)) {
6547 NewLocationOps.push_back(VH);
6548 LocationOpIndexMap[i] = NewLocationOps.size() - 1;
6549 LLVM_DEBUG(dbgs() << "scev-salvage: Location index " << i
6550 << " now at index " << LocationOpIndexMap[i] << "\n");
6551 continue;
6552 }
6553
6554 // It's possible that a value referred to in the SCEV may have been
6555 // optimised out by LSR.
6556 if (SE.containsErasedValue(DVIRec.SCEVs[i]) ||
6557 SE.containsUndefs(DVIRec.SCEVs[i])) {
6558 LLVM_DEBUG(dbgs() << "scev-salvage: SCEV for location at index: " << i
6559 << " refers to a location that is now undef or erased. "
6560 "Salvage abandoned.\n");
6561 return false;
6562 }
6563
6564 LLVM_DEBUG(dbgs() << "scev-salvage: salvaging location at index " << i
6565 << " with SCEV: " << *DVIRec.SCEVs[i] << "\n");
6566
6567 DVIRec.RecoveryExprs[i] = std::make_unique<SCEVDbgValueBuilder>();
6568 SCEVDbgValueBuilder *SalvageExpr = DVIRec.RecoveryExprs[i].get();
6569
6570 // Create an offset-based salvage expression if possible, as it requires
6571 // less DWARF ops than an iteration count-based expression.
6572 if (std::optional<APInt> Offset =
6573 SE.computeConstantDifference(DVIRec.SCEVs[i], SCEVInductionVar)) {
6574 if (Offset->getSignificantBits() <= 64)
6575 SalvageExpr->createOffsetExpr(Offset->getSExtValue(), LSRInductionVar);
6576 } else if (!SalvageExpr->createIterCountExpr(DVIRec.SCEVs[i], IterCountExpr,
6577 SE))
6578 return false;
6579 }
6580
6581 // Merge the DbgValueBuilder generated expressions and the original
6582 // DIExpression, place the result into an new vector.
6584 if (DVIRec.Expr->getNumElements() == 0) {
6585 assert(DVIRec.RecoveryExprs.size() == 1 &&
6586 "Expected only a single recovery expression for an empty "
6587 "DIExpression.");
6588 assert(DVIRec.RecoveryExprs[0] &&
6589 "Expected a SCEVDbgSalvageBuilder for location 0");
6590 SCEVDbgValueBuilder *B = DVIRec.RecoveryExprs[0].get();
6591 B->appendToVectors(NewExpr, NewLocationOps);
6592 }
6593 for (const auto &Op : DVIRec.Expr->expr_ops()) {
6594 // Most Ops needn't be updated.
6595 if (Op.getOp() != dwarf::DW_OP_LLVM_arg) {
6596 Op.appendToVector(NewExpr);
6597 continue;
6598 }
6599
6600 uint64_t LocationArgIndex = Op.getArg(0);
6601 SCEVDbgValueBuilder *DbgBuilder =
6602 DVIRec.RecoveryExprs[LocationArgIndex].get();
6603 // The location doesn't have s SCEVDbgValueBuilder, so LSR did not
6604 // optimise it away. So just translate the argument to the updated
6605 // location index.
6606 if (!DbgBuilder) {
6607 NewExpr.push_back(dwarf::DW_OP_LLVM_arg);
6608 assert(LocationOpIndexMap[Op.getArg(0)] != -1 &&
6609 "Expected a positive index for the location-op position.");
6610 NewExpr.push_back(LocationOpIndexMap[Op.getArg(0)]);
6611 continue;
6612 }
6613 // The location has a recovery expression.
6614 DbgBuilder->appendToVectors(NewExpr, NewLocationOps);
6615 }
6616
6617 UpdateDbgValueInst(DVIRec, NewLocationOps, NewExpr);
6618 if (isa<DbgValueInst *>(DVIRec.DbgRef))
6619 LLVM_DEBUG(dbgs() << "scev-salvage: Updated DVI: "
6620 << *cast<DbgValueInst *>(DVIRec.DbgRef) << "\n");
6621 else
6622 LLVM_DEBUG(dbgs() << "scev-salvage: Updated DVI: "
6623 << *cast<DPValue *>(DVIRec.DbgRef) << "\n");
6624 return true;
6625}
6626
6627/// Obtain an expression for the iteration count, then attempt to salvage the
6628/// dbg.value intrinsics.
6630 llvm::Loop *L, ScalarEvolution &SE, llvm::PHINode *LSRInductionVar,
6631 SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> &DVIToUpdate) {
6632 if (DVIToUpdate.empty())
6633 return;
6634
6635 const llvm::SCEV *SCEVInductionVar = SE.getSCEV(LSRInductionVar);
6636 assert(SCEVInductionVar &&
6637 "Anticipated a SCEV for the post-LSR induction variable");
6638
6639 if (const SCEVAddRecExpr *IVAddRec =
6640 dyn_cast<SCEVAddRecExpr>(SCEVInductionVar)) {
6641 if (!IVAddRec->isAffine())
6642 return;
6643
6644 // Prevent translation using excessive resources.
6645 if (IVAddRec->getExpressionSize() > MaxSCEVSalvageExpressionSize)
6646 return;
6647
6648 // The iteration count is required to recover location values.
6649 SCEVDbgValueBuilder IterCountExpr;
6650 IterCountExpr.pushLocation(LSRInductionVar);
6651 if (!IterCountExpr.SCEVToIterCountExpr(*IVAddRec, SE))
6652 return;
6653
6654 LLVM_DEBUG(dbgs() << "scev-salvage: IV SCEV: " << *SCEVInductionVar
6655 << '\n');
6656
6657 for (auto &DVIRec : DVIToUpdate) {
6658 SalvageDVI(L, SE, LSRInductionVar, *DVIRec, SCEVInductionVar,
6659 IterCountExpr);
6660 }
6661 }
6662}
6663
6664/// Identify and cache salvageable DVI locations and expressions along with the
6665/// corresponding SCEV(s). Also ensure that the DVI is not deleted between
6666/// cacheing and salvaging.
6668 Loop *L, ScalarEvolution &SE,
6669 SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> &SalvageableDVISCEVs,
6670 SmallSet<AssertingVH<DbgValueInst>, 2> &DVIHandles) {
6671 for (const auto &B : L->getBlocks()) {
6672 for (auto &I : *B) {
6673 auto ProcessDbgValue = [&](auto *DbgVal) -> bool {
6674 // Ensure that if any location op is undef that the dbg.vlue is not
6675 // cached.
6676 if (DbgVal->isKillLocation())
6677 return false;
6678
6679 // Check that the location op SCEVs are suitable for translation to
6680 // DIExpression.
6681 const auto &HasTranslatableLocationOps =
6682 [&](const auto *DbgValToTranslate) -> bool {
6683 for (const auto LocOp : DbgValToTranslate->location_ops()) {
6684 if (!LocOp)
6685 return false;
6686
6687 if (!SE.isSCEVable(LocOp->getType()))
6688 return false;
6689
6690 const SCEV *S = SE.getSCEV(LocOp);
6691 if (SE.containsUndefs(S))
6692 return false;
6693 }
6694 return true;
6695 };
6696
6697 if (!HasTranslatableLocationOps(DbgVal))
6698 return false;
6699
6700 std::unique_ptr<DVIRecoveryRec> NewRec =
6701 std::make_unique<DVIRecoveryRec>(DbgVal);
6702 // Each location Op may need a SCEVDbgValueBuilder in order to recover
6703 // it. Pre-allocating a vector will enable quick lookups of the builder
6704 // later during the salvage.
6705 NewRec->RecoveryExprs.resize(DbgVal->getNumVariableLocationOps());
6706 for (const auto LocOp : DbgVal->location_ops()) {
6707 NewRec->SCEVs.push_back(SE.getSCEV(LocOp));
6708 NewRec->LocationOps.push_back(LocOp);
6709 NewRec->HadLocationArgList = DbgVal->hasArgList();
6710 }
6711 SalvageableDVISCEVs.push_back(std::move(NewRec));
6712 return true;
6713 };
6714 for (DPValue &DPV : filterDbgVars(I.getDbgRecordRange())) {
6715 if (DPV.isDbgValue() || DPV.isDbgAssign())
6716 ProcessDbgValue(&DPV);
6717 }
6718 auto DVI = dyn_cast<DbgValueInst>(&I);
6719 if (!DVI)
6720 continue;
6721 if (ProcessDbgValue(DVI))
6722 DVIHandles.insert(DVI);
6723 }
6724 }
6725}
6726
6727/// Ideally pick the PHI IV inserted by ScalarEvolutionExpander. As a fallback
6728/// any PHi from the loop header is usable, but may have less chance of
6729/// surviving subsequent transforms.
6731 const LSRInstance &LSR) {
6732
6733 auto IsSuitableIV = [&](PHINode *P) {
6734 if (!SE.isSCEVable(P->getType()))
6735 return false;
6736 if (const SCEVAddRecExpr *Rec = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(P)))
6737 return Rec->isAffine() && !SE.containsUndefs(SE.getSCEV(P));
6738 return false;
6739 };
6740
6741 // For now, just pick the first IV that was generated and inserted by
6742 // ScalarEvolution. Ideally pick an IV that is unlikely to be optimised away
6743 // by subsequent transforms.
6744 for (const WeakVH &IV : LSR.getScalarEvolutionIVs()) {
6745 if (!IV)
6746 continue;
6747
6748 // There should only be PHI node IVs.
6749 PHINode *P = cast<PHINode>(&*IV);
6750
6751 if (IsSuitableIV(P))
6752 return P;
6753 }
6754
6755 for (PHINode &P : L.getHeader()->phis()) {
6756 if (IsSuitableIV(&P))
6757 return &P;
6758 }
6759 return nullptr;
6760}
6761
6762static std::optional<std::tuple<PHINode *, PHINode *, const SCEV *, bool>>
6764 const LoopInfo &LI, const TargetTransformInfo &TTI) {
6765 if (!L->isInnermost()) {
6766 LLVM_DEBUG(dbgs() << "Cannot fold on non-innermost loop\n");
6767 return std::nullopt;
6768 }
6769 // Only inspect on simple loop structure
6770 if (!L->isLoopSimplifyForm()) {
6771 LLVM_DEBUG(dbgs() << "Cannot fold on non-simple loop\n");
6772 return std::nullopt;
6773 }
6774
6776 LLVM_DEBUG(dbgs() << "Cannot fold on backedge that is loop variant\n");
6777 return std::nullopt;
6778 }
6779
6780 BasicBlock *LoopLatch = L->getLoopLatch();
6781 BranchInst *BI = dyn_cast<BranchInst>(LoopLatch->getTerminator());
6782 if (!BI || BI->isUnconditional())
6783 return std::nullopt;
6784 auto *TermCond = dyn_cast<ICmpInst>(BI->getCondition());
6785 if (!TermCond) {
6786 LLVM_DEBUG(
6787 dbgs() << "Cannot fold on branching condition that is not an ICmpInst");
6788 return std::nullopt;
6789 }
6790 if (!TermCond->hasOneUse()) {
6791 LLVM_DEBUG(
6792 dbgs()
6793 << "Cannot replace terminating condition with more than one use\n");
6794 return std::nullopt;
6795 }
6796
6797 BinaryOperator *LHS = dyn_cast<BinaryOperator>(TermCond->getOperand(0));
6798 Value *RHS = TermCond->getOperand(1);
6799 if (!LHS || !L->isLoopInvariant(RHS))
6800 // We could pattern match the inverse form of the icmp, but that is
6801 // non-canonical, and this pass is running *very* late in the pipeline.
6802 return std::nullopt;
6803
6804 // Find the IV used by the current exit condition.
6805 PHINode *ToFold;
6806 Value *ToFoldStart, *ToFoldStep;
6807 if (!matchSimpleRecurrence(LHS, ToFold, ToFoldStart, ToFoldStep))
6808 return std::nullopt;
6809
6810 // Ensure the simple recurrence is a part of the current loop.
6811 if (ToFold->getParent() != L->getHeader())
6812 return std::nullopt;
6813
6814 // If that IV isn't dead after we rewrite the exit condition in terms of
6815 // another IV, there's no point in doing the transform.
6816 if (!isAlmostDeadIV(ToFold, LoopLatch, TermCond))
6817 return std::nullopt;
6818
6819 // Inserting instructions in the preheader has a runtime cost, scale
6820 // the allowed cost with the loops trip count as best we can.
6821 const unsigned ExpansionBudget = [&]() {
6822 unsigned Budget = 2 * SCEVCheapExpansionBudget;
6823 if (unsigned SmallTC = SE.getSmallConstantMaxTripCount(L))
6824 return std::min(Budget, SmallTC);
6825 if (std::optional<unsigned> SmallTC = getLoopEstimatedTripCount(L))
6826 return std::min(Budget, *SmallTC);
6827 // Unknown trip count, assume long running by default.
6828 return Budget;
6829 }();
6830
6831 const SCEV *BECount = SE.getBackedgeTakenCount(L);
6832 const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
6833 SCEVExpander Expander(SE, DL, "lsr_fold_term_cond");
6834
6835 PHINode *ToHelpFold = nullptr;
6836 const SCEV *TermValueS = nullptr;
6837 bool MustDropPoison = false;
6838 auto InsertPt = L->getLoopPreheader()->getTerminator();
6839 for (PHINode &PN : L->getHeader()->phis()) {
6840 if (ToFold == &PN)
6841 continue;
6842
6843 if (!SE.isSCEVable(PN.getType())) {
6844 LLVM_DEBUG(dbgs() << "IV of phi '" << PN
6845 << "' is not SCEV-able, not qualified for the "
6846 "terminating condition folding.\n");
6847 continue;
6848 }
6849 const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(&PN));
6850 // Only speculate on affine AddRec
6851 if (!AddRec || !AddRec->isAffine()) {
6852 LLVM_DEBUG(dbgs() << "SCEV of phi '" << PN
6853 << "' is not an affine add recursion, not qualified "
6854 "for the terminating condition folding.\n");
6855 continue;
6856 }
6857
6858 // Check that we can compute the value of AddRec on the exiting iteration
6859 // without soundness problems. evaluateAtIteration internally needs
6860 // to multiply the stride of the iteration number - which may wrap around.
6861 // The issue here is subtle because computing the result accounting for
6862 // wrap is insufficient. In order to use the result in an exit test, we
6863 // must also know that AddRec doesn't take the same value on any previous
6864 // iteration. The simplest case to consider is a candidate IV which is
6865 // narrower than the trip count (and thus original IV), but this can
6866 // also happen due to non-unit strides on the candidate IVs.
6867 if (!AddRec->hasNoSelfWrap() ||
6868 !SE.isKnownNonZero(AddRec->getStepRecurrence(SE)))
6869 continue;
6870
6871 const SCEVAddRecExpr *PostInc = AddRec->getPostIncExpr(SE);
6872 const SCEV *TermValueSLocal = PostInc->evaluateAtIteration(BECount, SE);
6873 if (!Expander.isSafeToExpand(TermValueSLocal)) {
6874 LLVM_DEBUG(
6875 dbgs() << "Is not safe to expand terminating value for phi node" << PN
6876 << "\n");
6877 continue;
6878 }
6879
6880 if (Expander.isHighCostExpansion(TermValueSLocal, L, ExpansionBudget,
6881 &TTI, InsertPt)) {
6882 LLVM_DEBUG(
6883 dbgs() << "Is too expensive to expand terminating value for phi node"
6884 << PN << "\n");
6885 continue;
6886 }
6887
6888 // The candidate IV may have been otherwise dead and poison from the
6889 // very first iteration. If we can't disprove that, we can't use the IV.
6890 if (!mustExecuteUBIfPoisonOnPathTo(&PN, LoopLatch->getTerminator(), &DT)) {
6891 LLVM_DEBUG(dbgs() << "Can not prove poison safety for IV "
6892 << PN << "\n");
6893 continue;
6894 }
6895
6896 // The candidate IV may become poison on the last iteration. If this
6897 // value is not branched on, this is a well defined program. We're
6898 // about to add a new use to this IV, and we have to ensure we don't
6899 // insert UB which didn't previously exist.
6900 bool MustDropPoisonLocal = false;
6901 Instruction *PostIncV =
6902 cast<Instruction>(PN.getIncomingValueForBlock(LoopLatch));
6903 if (!mustExecuteUBIfPoisonOnPathTo(PostIncV, LoopLatch->getTerminator(),
6904 &DT)) {
6905 LLVM_DEBUG(dbgs() << "Can not prove poison safety to insert use"
6906 << PN << "\n");
6907
6908 // If this is a complex recurrance with multiple instructions computing
6909 // the backedge value, we might need to strip poison flags from all of
6910 // them.
6911 if (PostIncV->getOperand(0) != &PN)
6912 continue;
6913
6914 // In order to perform the transform, we need to drop the poison generating
6915 // flags on this instruction (if any).
6916 MustDropPoisonLocal = PostIncV->hasPoisonGeneratingFlags();
6917 }
6918
6919 // We pick the last legal alternate IV. We could expore choosing an optimal
6920 // alternate IV if we had a decent heuristic to do so.
6921 ToHelpFold = &PN;
6922 TermValueS = TermValueSLocal;
6923 MustDropPoison = MustDropPoisonLocal;
6924 }
6925
6926 LLVM_DEBUG(if (ToFold && !ToHelpFold) dbgs()
6927 << "Cannot find other AddRec IV to help folding\n";);
6928
6929 LLVM_DEBUG(if (ToFold && ToHelpFold) dbgs()
6930 << "\nFound loop that can fold terminating condition\n"
6931 << " BECount (SCEV): " << *SE.getBackedgeTakenCount(L) << "\n"
6932 << " TermCond: " << *TermCond << "\n"
6933 << " BrandInst: " << *BI << "\n"
6934 << " ToFold: " << *ToFold << "\n"
6935 << " ToHelpFold: " << *ToHelpFold << "\n");
6936
6937 if (!ToFold || !ToHelpFold)
6938 return std::nullopt;
6939 return std::make_tuple(ToFold, ToHelpFold, TermValueS, MustDropPoison);
6940}
6941
6943 DominatorTree &DT, LoopInfo &LI,
6944 const TargetTransformInfo &TTI,
6946 MemorySSA *MSSA) {
6947
6948 // Debug preservation - before we start removing anything identify which DVI
6949 // meet the salvageable criteria and store their DIExpression and SCEVs.
6950 SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> SalvageableDVIRecords;
6952 DbgGatherSalvagableDVI(L, SE, SalvageableDVIRecords, DVIHandles);
6953
6954 bool Changed = false;
6955 std::unique_ptr<MemorySSAUpdater> MSSAU;
6956 if (MSSA)
6957 MSSAU = std::make_unique<MemorySSAUpdater>(MSSA);
6958
6959 // Run the main LSR transformation.
6960 const LSRInstance &Reducer =
6961 LSRInstance(L, IU, SE, DT, LI, TTI, AC, TLI, MSSAU.get());
6962 Changed |= Reducer.getChanged();
6963
6964 // Remove any extra phis created by processing inner loops.
6965 Changed |= DeleteDeadPHIs(L->getHeader(), &TLI, MSSAU.get());
6966 if (EnablePhiElim && L->isLoopSimplifyForm()) {
6968 const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
6969 SCEVExpander Rewriter(SE, DL, "lsr", false);
6970#ifndef NDEBUG
6971 Rewriter.setDebugType(DEBUG_TYPE);
6972#endif
6973 unsigned numFolded = Rewriter.replaceCongruentIVs(L, &DT, DeadInsts, &TTI);
6974 Rewriter.clear();
6975 if (numFolded) {
6976 Changed = true;
6978 MSSAU.get());
6979 DeleteDeadPHIs(L->getHeader(), &TLI, MSSAU.get());
6980 }
6981 }
6982 // LSR may at times remove all uses of an induction variable from a loop.
6983 // The only remaining use is the PHI in the exit block.
6984 // When this is the case, if the exit value of the IV can be calculated using
6985 // SCEV, we can replace the exit block PHI with the final value of the IV and
6986 // skip the updates in each loop iteration.
6987 if (L->isRecursivelyLCSSAForm(DT, LI) && L->getExitBlock()) {
6989 const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
6990 SCEVExpander Rewriter(SE, DL, "lsr", true);
6991 int Rewrites = rewriteLoopExitValues(L, &LI, &TLI, &SE, &TTI, Rewriter, &DT,
6992 UnusedIndVarInLoop, DeadInsts);
6993 Rewriter.clear();
6994 if (Rewrites) {
6995 Changed = true;
6997 MSSAU.get());
6998 DeleteDeadPHIs(L->getHeader(), &TLI, MSSAU.get());
6999 }
7000 }
7001
7002 const bool EnableFormTerm = [&] {
7004 case cl::BOU_TRUE:
7005 return true;
7006 case cl::BOU_FALSE:
7007 return false;
7008 case cl::BOU_UNSET:
7010 }
7011 llvm_unreachable("Unhandled cl::boolOrDefault enum");
7012 }();
7013
7014 if (EnableFormTerm) {
7015 if (auto Opt = canFoldTermCondOfLoop(L, SE, DT, LI, TTI)) {
7016 auto [ToFold, ToHelpFold, TermValueS, MustDrop] = *Opt;
7017
7018 Changed = true;
7019 NumTermFold++;
7020
7021 BasicBlock *LoopPreheader = L->getLoopPreheader();
7022 BasicBlock *LoopLatch = L->getLoopLatch();
7023
7024 (void)ToFold;
7025 LLVM_DEBUG(dbgs() << "To fold phi-node:\n"
7026 << *ToFold << "\n"
7027 << "New term-cond phi-node:\n"
7028 << *ToHelpFold << "\n");
7029
7030 Value *StartValue = ToHelpFold->getIncomingValueForBlock(LoopPreheader);
7031 (void)StartValue;
7032 Value *LoopValue = ToHelpFold->getIncomingValueForBlock(LoopLatch);
7033
7034 // See comment in canFoldTermCondOfLoop on why this is sufficient.
7035 if (MustDrop)
7036 cast<Instruction>(LoopValue)->dropPoisonGeneratingFlags();
7037
7038 // SCEVExpander for both use in preheader and latch
7039 const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
7040 SCEVExpander Expander(SE, DL, "lsr_fold_term_cond");
7041
7042 assert(Expander.isSafeToExpand(TermValueS) &&
7043 "Terminating value was checked safe in canFoldTerminatingCondition");
7044
7045 // Create new terminating value at loop preheader
7046 Value *TermValue = Expander.expandCodeFor(TermValueS, ToHelpFold->getType(),
7047 LoopPreheader->getTerminator());
7048
7049 LLVM_DEBUG(dbgs() << "Start value of new term-cond phi-node:\n"
7050 << *StartValue << "\n"
7051 << "Terminating value of new term-cond phi-node:\n"
7052 << *TermValue << "\n");
7053
7054 // Create new terminating condition at loop latch
7055 BranchInst *BI = cast<BranchInst>(LoopLatch->getTerminator());
7056 ICmpInst *OldTermCond = cast<ICmpInst>(BI->getCondition());
7057 IRBuilder<> LatchBuilder(LoopLatch->getTerminator());
7058 Value *NewTermCond =
7059 LatchBuilder.CreateICmp(CmpInst::ICMP_EQ, LoopValue, TermValue,
7060 "lsr_fold_term_cond.replaced_term_cond");
7061 // Swap successors to exit loop body if IV equals to new TermValue
7062 if (BI->getSuccessor(0) == L->getHeader())
7063 BI->swapSuccessors();
7064
7065 LLVM_DEBUG(dbgs() << "Old term-cond:\n"
7066 << *OldTermCond << "\n"
7067 << "New term-cond:\n" << *NewTermCond << "\n");
7068
7069 BI->setCondition(NewTermCond);
7070
7071 Expander.clear();
7072 OldTermCond->eraseFromParent();
7073 DeleteDeadPHIs(L->getHeader(), &TLI, MSSAU.get());
7074 }
7075 }
7076
7077 if (SalvageableDVIRecords.empty())
7078 return Changed;
7079
7080 // Obtain relevant IVs and attempt to rewrite the salvageable DVIs with
7081 // expressions composed using the derived iteration count.
7082 // TODO: Allow for multiple IV references for nested AddRecSCEVs
7083 for (const auto &L : LI) {
7084 if (llvm::PHINode *IV = GetInductionVariable(*L, SE, Reducer))
7085 DbgRewriteSalvageableDVIs(L, SE, IV, SalvageableDVIRecords);
7086 else {
7087 LLVM_DEBUG(dbgs() << "scev-salvage: SCEV salvaging not possible. An IV "
7088 "could not be identified.\n");
7089 }
7090 }
7091
7092 for (auto &Rec : SalvageableDVIRecords)
7093 Rec->clear();
7094 SalvageableDVIRecords.clear();
7095 DVIHandles.clear();
7096 return Changed;
7097}
7098
7099bool LoopStrengthReduce::runOnLoop(Loop *L, LPPassManager & /*LPM*/) {
7100 if (skipLoop(L))
7101 return false;
7102
7103 auto &IU = getAnalysis<IVUsersWrapperPass>().getIU();
7104 auto &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
7105 auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
7106 auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
7107 const auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
7108 *L->getHeader()->getParent());
7109 auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
7110 *L->getHeader()->getParent());
7111 auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
7112 *L->getHeader()->getParent());
7113 auto *MSSAAnalysis = getAnalysisIfAvailable<MemorySSAWrapperPass>();
7114 MemorySSA *MSSA = nullptr;
7115 if (MSSAAnalysis)
7116 MSSA = &MSSAAnalysis->getMSSA();
7117 return ReduceLoopStrength(L, IU, SE, DT, LI, TTI, AC, TLI, MSSA);
7118}
7119
7122 LPMUpdater &) {
7123 if (!ReduceLoopStrength(&L, AM.getResult<IVUsersAnalysis>(L, AR), AR.SE,
7124 AR.DT, AR.LI, AR.TTI, AR.AC, AR.TLI, AR.MSSA))
7125 return PreservedAnalyses::all();
7126
7127 auto PA = getLoopPassPreservedAnalyses();
7128 if (AR.MSSA)
7129 PA.preserve<MemorySSAAnalysis>();
7130 return PA;
7131}
7132
7133char LoopStrengthReduce::ID = 0;
7134
7135INITIALIZE_PASS_BEGIN(LoopStrengthReduce, "loop-reduce",
7136 "Loop Strength Reduction", false, false)
7142INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
7143INITIALIZE_PASS_END(LoopStrengthReduce, "loop-reduce",
7144 "Loop Strength Reduction", false, false)
7145
7146Pass *llvm::createLoopStrengthReducePass() { return new LoopStrengthReduce(); }
#define Success
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file implements a class to represent arbitrary precision integral constant values and operations...
@ PostInc
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
static bool isEqual(const Function &Caller, const Function &Callee)
static const Function * getParent(const Value *V)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Definition: CommandLine.h:693
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Definition: Compiler.h:529
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static void clear(coro::Shape &Shape)
Definition: Coroutines.cpp:148
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static bool isCanonical(const MDString *S)
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
This file contains constants used for implementing Dwarf debug support.
std::optional< std::vector< StOtherPiece > > Other
Definition: ELFYAML.cpp:1290
bool End
Definition: ELF_riscv.cpp:480
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
Rewrite Partial Register Uses
Hexagon Hardware Loops
iv Induction Variable Users
Definition: IVUsers.cpp:48
This header provides classes for managing per-loop analyses.
static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE, llvm::PHINode *LSRInductionVar, DVIRecoveryRec &DVIRec, const SCEV *SCEVInductionVar, SCEVDbgValueBuilder IterCountExpr)
static std::optional< std::tuple< PHINode *, PHINode *, const SCEV *, bool > > canFoldTermCondOfLoop(Loop *L, ScalarEvolution &SE, DominatorTree &DT, const LoopInfo &LI, const TargetTransformInfo &TTI)
static Value * getWideOperand(Value *Oper)
IVChain logic must consistently peek base TruncInst operands, so wrap it in a convenient helper.
static bool isAddSExtable(const SCEVAddExpr *A, ScalarEvolution &SE)
Return true if the given add can be sign-extended without changing its value.
static bool mayUsePostIncMode(const TargetTransformInfo &TTI, LSRUse &LU, const SCEV *S, const Loop *L, ScalarEvolution &SE)
Return true if the SCEV represents a value that may end up as a post-increment operation.
static void restorePreTransformState(DVIRecoveryRec &DVIRec)
Restore the DVI's pre-LSR arguments. Substitute undef for any erased values.
static bool containsAddRecDependentOnLoop(const SCEV *S, const Loop &L)
static User::op_iterator findIVOperand(User::op_iterator OI, User::op_iterator OE, Loop *L, ScalarEvolution &SE)
Helper for CollectChains that finds an IV operand (computed by an AddRec in this loop) within [OI,...
static bool isMulSExtable(const SCEVMulExpr *M, ScalarEvolution &SE)
Return true if the given mul can be sign-extended without changing its value.
static const unsigned MaxSCEVSalvageExpressionSize
Limit the size of expression that SCEV-based salvaging will attempt to translate into a DIExpression.
static bool isExistingPhi(const SCEVAddRecExpr *AR, ScalarEvolution &SE)
Return true if this AddRec is already a phi in its loop.
static InstructionCost getScalingFactorCost(const TargetTransformInfo &TTI, const LSRUse &LU, const Formula &F, const Loop &L)
static cl::opt< bool > InsnsCost("lsr-insns-cost", cl::Hidden, cl::init(true), cl::desc("Add instruction count to a LSR cost model"))
static cl::opt< bool > StressIVChain("stress-ivchain", cl::Hidden, cl::init(false), cl::desc("Stress test LSR IV chains"))
static bool isAddressUse(const TargetTransformInfo &TTI, Instruction *Inst, Value *OperandVal)
Returns true if the specified instruction is using the specified value as an address.
static GlobalValue * ExtractSymbol(const SCEV *&S, ScalarEvolution &SE)
If S involves the addition of a GlobalValue address, return that symbol, and mutate S to point to a n...
static void updateDVIWithLocation(T &DbgVal, Value *Location, SmallVectorImpl< uint64_t > &Ops)
Overwrites DVI with the location and Ops as the DIExpression.
static cl::opt< TTI::AddressingModeKind > PreferredAddresingMode("lsr-preferred-addressing-mode", cl::Hidden, cl::init(TTI::AMK_None), cl::desc("A flag that overrides the target's preferred addressing mode."), cl::values(clEnumValN(TTI::AMK_None, "none", "Don't prefer any addressing mode"), clEnumValN(TTI::AMK_PreIndexed, "preindexed", "Prefer pre-indexed addressing mode"), clEnumValN(TTI::AMK_PostIndexed, "postindexed", "Prefer post-indexed addressing mode")))
static const SCEV * getExprBase(const SCEV *S)
Return an approximation of this SCEV expression's "base", or NULL for any constant.
static llvm::PHINode * GetInductionVariable(const Loop &L, ScalarEvolution &SE, const LSRInstance &LSR)
Ideally pick the PHI IV inserted by ScalarEvolutionExpander.
static cl::opt< cl::boolOrDefault > AllowTerminatingConditionFoldingAfterLSR("lsr-term-fold", cl::Hidden, cl::desc("Attempt to replace primary IV with other IV."))
static bool IsSimplerBaseSCEVForTarget(const TargetTransformInfo &TTI, ScalarEvolution &SE, const SCEV *Best, const SCEV *Reg, MemAccessTy AccessType)
loop reduce
static const unsigned MaxIVUsers
MaxIVUsers is an arbitrary threshold that provides an early opportunity for bail out.
static cl::opt< bool > AllowDropSolutionIfLessProfitable("lsr-drop-solution", cl::Hidden, cl::init(false), cl::desc("Attempt to drop solution if it is less profitable"))
static bool isHighCostExpansion(const SCEV *S, SmallPtrSetImpl< const SCEV * > &Processed, ScalarEvolution &SE)
Check if expanding this expression is likely to incur significant cost.
static Value * getValueOrPoison(WeakVH &VH, LLVMContext &C)
Cached location ops may be erased during LSR, in which case a poison is required when restoring from ...
static MemAccessTy getAccessType(const TargetTransformInfo &TTI, Instruction *Inst, Value *OperandVal)
Return the type of the memory being accessed.
static unsigned numLLVMArgOps(SmallVectorImpl< uint64_t > &Expr)
Returns the total number of DW_OP_llvm_arg operands in the expression.
static bool isAlwaysFoldable(const TargetTransformInfo &TTI, LSRUse::KindType Kind, MemAccessTy AccessTy, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg)
static void DbgRewriteSalvageableDVIs(llvm::Loop *L, ScalarEvolution &SE, llvm::PHINode *LSRInductionVar, SmallVector< std::unique_ptr< DVIRecoveryRec >, 2 > &DVIToUpdate)
Obtain an expression for the iteration count, then attempt to salvage the dbg.value intrinsics.
static cl::opt< bool > EnablePhiElim("enable-lsr-phielim", cl::Hidden, cl::init(true), cl::desc("Enable LSR phi elimination"))
static void DbgGatherSalvagableDVI(Loop *L, ScalarEvolution &SE, SmallVector< std::unique_ptr< DVIRecoveryRec >, 2 > &SalvageableDVISCEVs, SmallSet< AssertingVH< DbgValueInst >, 2 > &DVIHandles)
Identify and cache salvageable DVI locations and expressions along with the corresponding SCEV(s).
static bool isAddRecSExtable(const SCEVAddRecExpr *AR, ScalarEvolution &SE)
Return true if the given addrec can be sign-extended without changing its value.
static bool canHoistIVInc(const TargetTransformInfo &TTI, const LSRFixup &Fixup, const LSRUse &LU, Instruction *IVIncInsertPos, Loop *L)
static void DoInitialMatch(const SCEV *S, Loop *L, SmallVectorImpl< const SCEV * > &Good, SmallVectorImpl< const SCEV * > &Bad, ScalarEvolution &SE)
Recursion helper for initialMatch.
static bool isAMCompletelyFolded(const TargetTransformInfo &TTI, const LSRUse &LU, const Formula &F)
Check if the addressing mode defined by F is completely folded in LU at isel time.
static cl::opt< bool > LSRExpNarrow("lsr-exp-narrow", cl::Hidden, cl::init(false), cl::desc("Narrow LSR complex solution using" " expectation of registers number"))
static cl::opt< bool > FilterSameScaledReg("lsr-filter-same-scaled-reg", cl::Hidden, cl::init(true), cl::desc("Narrow LSR search space by filtering non-optimal formulae" " with the same ScaledReg and Scale"))
static bool isLegalUse(const TargetTransformInfo &TTI, int64_t MinOffset, int64_t MaxOffset, LSRUse::KindType Kind, MemAccessTy AccessTy, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale)
Test whether we know how to expand the current formula.
static void updateDVIWithLocations(T &DbgVal, SmallVectorImpl< Value * > &Locations, SmallVectorImpl< uint64_t > &Ops)
Overwrite DVI with locations placed into a DIArglist.
static cl::opt< unsigned > ComplexityLimit("lsr-complexity-limit", cl::Hidden, cl::init(std::numeric_limits< uint16_t >::max()), cl::desc("LSR search space complexity limit"))
static void UpdateDbgValueInst(DVIRecoveryRec &DVIRec, SmallVectorImpl< Value * > &NewLocationOps, SmallVectorImpl< uint64_t > &NewExpr)
Write the new expression and new location ops for the dbg.value.
static int64_t ExtractImmediate(const SCEV *&S, ScalarEvolution &SE)
If S involves the addition of a constant integer value, return that integer value,...
static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE, DominatorTree &DT, LoopInfo &LI, const TargetTransformInfo &TTI, AssumptionCache &AC, TargetLibraryInfo &TLI, MemorySSA *MSSA)
static bool isProfitableChain(IVChain &Chain, SmallPtrSetImpl< Instruction * > &Users, ScalarEvolution &SE, const TargetTransformInfo &TTI)
Return true if the number of registers needed for the chain is estimated to be less than the number r...
static const SCEV * CollectSubexprs(const SCEV *S, const SCEVConstant *C, SmallVectorImpl< const SCEV * > &Ops, const Loop *L, ScalarEvolution &SE, unsigned Depth=0)
Split S into subexpressions which can be pulled out into separate registers.
static const SCEV * getExactSDiv(const SCEV *LHS, const SCEV *RHS, ScalarEvolution &SE, bool IgnoreSignificantBits=false)
Return an expression for LHS /s RHS, if it can be determined and if the remainder is known to be zero...
static bool canFoldIVIncExpr(const SCEV *IncExpr, Instruction *UserInst, Value *Operand, const TargetTransformInfo &TTI)
Return true if the IVInc can be folded into an addressing mode.
#define DEBUG_TYPE
static const SCEV * getAnyExtendConsideringPostIncUses(ArrayRef< PostIncLoopSet > Loops, const SCEV *Expr, Type *ToTy, ScalarEvolution &SE)
Extend/Truncate Expr to ToTy considering post-inc uses in Loops.
static unsigned getSetupCost(const SCEV *Reg, unsigned Depth)
static cl::opt< unsigned > SetupCostDepthLimit("lsr-setupcost-depth-limit", cl::Hidden, cl::init(7), cl::desc("The limit on recursion depth for LSRs setup cost"))
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
unsigned Reg
This file exposes an interface to building/using memory SSA to walk memory instructions using a use/d...
Module.h This file contains the declarations for the Module class.
#define P(N)
PowerPC TLS Dynamic Call Fixup
if(VerifyEach)
This header defines various interfaces for pass management in LLVM.
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:59
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
This file defines the PointerIntPair class.
const SmallVectorImpl< MachineOperand > & Cond
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SI optimize exec mask operations pre RA
This file contains some templates that are useful if you are working with the STL at all.
raw_pwrite_stream & OS
This file implements a set that has insertion order iteration characteristics.
This file implements the SmallBitVector class.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:40
This pass exposes codegen information to IR-level passes.
This defines the Use class.
Virtual Register Rewriter
Definition: VirtRegMap.cpp:237
Value * RHS
Value * LHS
BinaryOperator * Mul
static const uint32_t IV[8]
Definition: blake3_impl.h:78
Class recording the (high level) value of a variable.
Class for arbitrary precision integers.
Definition: APInt.h:76
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1491
bool isNegative() const
Determine sign of this APInt.
Definition: APInt.h:307
APInt sdiv(const APInt &RHS) const
Signed division function for APInt.
Definition: APInt.cpp:1650
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
Definition: APInt.h:1482
APInt srem(const APInt &RHS) const
Function for signed remainder operation.
Definition: APInt.cpp:1742
int64_t getSExtValue() const
Get sign extended value.
Definition: APInt.h:1513
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:348
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:500
Represent the analysis usage information of a pass.
AnalysisUsage & addRequiredID(const void *ID)
Definition: Pass.cpp:283
AnalysisUsage & addPreservedID(const void *ID)
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
Definition: Any.h:28
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
Value handle that asserts if the Value is deleted.
Definition: ValueHandle.h:264
An immutable pass that tracks lazily created AssumptionCache objects.
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:539
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:748
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:498
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:347
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:205
InstListType::iterator iterator
Instruction iterators...
Definition: BasicBlock.h:164
void moveBefore(BasicBlock *MovePos)
Unlink this basic block from its current function and insert it into the function that MovePos lives ...
Definition: BasicBlock.h:357
bool isLandingPad() const
Return true if this basic block is a landing pad.
Definition: BasicBlock.cpp:659
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:220
static BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name, BasicBlock::iterator InsertBefore)
Construct a binary instruction, given the opcode and the two operands.
BinaryOps getOpcode() const
Definition: InstrTypes.h:491
Conditional or Unconditional Branch instruction.
void setCondition(Value *V)
void swapSuccessors()
Swap the successors of this branch instruction.
BasicBlock * getSuccessor(unsigned i) const
bool isUnconditional() const
Value * getCondition() const
static Instruction::CastOps getCastOpcode(const Value *Val, bool SrcIsSigned, Type *Ty, bool DstIsSigned)
Returns the opcode necessary to cast Val into Ty using usual casting rules.
static CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name, BasicBlock::iterator InsertBefore)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:965
@ ICMP_EQ
equal
Definition: InstrTypes.h:986
@ ICMP_NE
not equal
Definition: InstrTypes.h:987
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition: InstrTypes.h:1090
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
static bool isValueValidForType(Type *Ty, uint64_t V)
This static method returns true if the type Ty is big enough to represent the value V.
Definition: Constants.cpp:1588
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
Definition: Constants.h:122
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Definition: Constants.h:159
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:153
This is an important base class in LLVM.
Definition: Constant.h:41
static DIArgList * get(LLVMContext &Context, ArrayRef< ValueAsMetadata * > Args)
An iterator for expression operands.
DWARF expression.
static DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
static void appendOffset(SmallVectorImpl< uint64_t > &Ops, int64_t Offset)
Append Ops with operations to apply the Offset.
bool isComplex() const
Return whether the location is computed on the expression stack, meaning it cannot be a simple regist...
Record of a variable value-assignment, aka a non instruction representation of the dbg....
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
This represents the llvm.dbg.value instruction.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:220
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
NodeT * getBlock() const
Legacy analysis pass which computes a DominatorTree.
Definition: Dominators.h:317
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:162
This instruction compares its operands according to the predicate given to the constructor.
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2334
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2649
IVStrideUse - Keep track of one use of a strided induction variable.
Definition: IVUsers.h:34
void transformToPostInc(const Loop *L)
transformToPostInc - Transform the expression to post-inc form for the given loop.
Definition: IVUsers.cpp:367
Value * getOperandValToReplace() const
getOperandValToReplace - Return the Value of the operand in the user instruction that this IVStrideUs...
Definition: IVUsers.h:53
void setUser(Instruction *NewUser)
setUser - Assign a new user instruction for this use.
Definition: IVUsers.h:47
Analysis pass that exposes the IVUsers for a loop.
Definition: IVUsers.h:183
ilist< IVStrideUse >::const_iterator const_iterator
Definition: IVUsers.h:141
bool empty() const
Definition: IVUsers.h:146
void print(raw_ostream &OS) const
std::optional< CostType > getValue() const
This function is intended to be used as sparingly as possible, since the class provides the full rang...
unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:453
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:80
bool isEHPad() const
Return true if the instruction is a variety of EH-block.
Definition: Instruction.h:801
const BasicBlock * getParent() const
Definition: Instruction.h:151
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Type * getAccessType() const LLVM_READONLY
Return the type this instruction accesses in memory, if any.
bool hasPoisonGeneratingFlags() const LLVM_READONLY
Return true if this operator has flags which may cause this instruction to evaluate to poison despite...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:251
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
Definition: Instruction.h:450
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:278
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:47
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
This class provides an interface for updating the loop pass manager based on mutations to the loop ne...
An instruction for reading from memory.
Definition: Instructions.h:184
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
BlockT * getHeader() const
unsigned getLoopDepth() const
Return the nesting level of this loop.
The legacy pass manager's analysis pass to compute loop information.
Definition: LoopInfo.h:593
virtual bool runOnLoop(Loop *L, LPPassManager &LPM)=0
PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR, LPMUpdater &U)
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:44
An analysis that produces MemorySSA for a function.
Definition: MemorySSA.h:923
Legacy analysis pass which computes MemorySSA.
Definition: MemorySSA.h:980
Encapsulates MemorySSA, including all data associated with memory accesses.
Definition: MemorySSA.h:700
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.h:287
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
iterator_range< const_block_iterator > blocks() const
op_range incoming_values()
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr, BasicBlock::iterator InsertBefore)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
void setIncomingValue(unsigned i, Value *V)
Value * getIncomingValueForBlock(const BasicBlock *BB) const
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
static unsigned getIncomingValueNumForOperand(unsigned i)
int getBasicBlockIndex(const BasicBlock *BB) const
Return the first index of the specified basic block in the value list for this PHI.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:94
virtual void getAnalysisUsage(AnalysisUsage &) const
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
Definition: Pass.cpp:98
PointerIntPair - This class implements a pair of a pointer and small integer.
A discriminated union of two or more pointer types, with the discriminator in the low bit of the poin...
Definition: PointerUnion.h:118
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1827
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:109
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:115
This node represents an addition of some number of SCEVs.
This node represents a polynomial recurrence on the trip count of the specified loop.
const SCEV * getStepRecurrence(ScalarEvolution &SE) const
Constructs and returns the recurrence indicating how much this expression steps by.
bool isAffine() const
Return true if this represents an expression A + B*x where A and B are loop invariant values.
const SCEVAddRecExpr * getPostIncExpr(ScalarEvolution &SE) const
Return an expression representing the value of this expression one iteration of the loop ahead.
This is the base class for unary cast operator classes.
This node is the base class for n'ary commutative operators.
This class represents a constant integer value.
ConstantInt * getValue() const
const APInt & getAPInt() const
This class uses information about analyze scalars to rewrite expressions in canonical form.
bool isSafeToExpand(const SCEV *S) const
Return true if the given expression is safe to expand in the sense that all materialized values are s...
bool isHighCostExpansion(ArrayRef< const SCEV * > Exprs, Loop *L, unsigned Budget, const TargetTransformInfo *TTI, const Instruction *At)
Return true for expressions that can't be evaluated at runtime within given Budget.
void clear()
Erase the contents of the InsertedExpressions map so that users trying to expand the same expression ...
Value * expandCodeFor(const SCEV *SH, Type *Ty, BasicBlock::iterator I)
Insert code to directly compute the specified SCEV expression into the program.
This is the base class for unary integral cast operator classes.
This node represents multiplication of some number of SCEVs.
This node is a base class providing common functionality for n'ary operators.
ArrayRef< const SCEV * > operands() const
This class represents a signed maximum selection.
This class represents a binary unsigned division operation.
This class represents an unsigned maximum selection.
This means that we are dealing with an entirely unknown SCEV value, and only represent it as its LLVM...
This class represents an analyzed expression in the program.
ArrayRef< const SCEV * > operands() const
Return operands of this SCEV expression.
unsigned short getExpressionSize() const
bool isZero() const
Return true if the expression is a constant zero.
SCEVTypes getSCEVType() const
Type * getType() const
Return the LLVM type of this SCEV expression.
This class represents a cast from signed integer to floating point.
The main scalar evolution driver.
bool isKnownNonZero(const SCEV *S)
Test if the given expression is known to be non-zero.
const SCEV * getBackedgeTakenCount(const Loop *L, ExitCountKind Kind=Exact)
If the specified loop has a predictable backedge-taken count, return it, otherwise return a SCEVCould...
uint64_t getTypeSizeInBits(Type *Ty) const
Return the size in bits of the specified type, for which isSCEVable must return true.
const SCEV * getConstant(ConstantInt *V)
const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
const SCEV * getNoopOrSignExtend(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
unsigned getSmallConstantMaxTripCount(const Loop *L)
Returns the upper bound of the loop trip count as a normal unsigned value.
bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
const SCEV * getAddRecExpr(const SCEV *Start, const SCEV *Step, const Loop *L, SCEV::NoWrapFlags Flags)
Get an add recurrence expression for the specified loop.
bool isSCEVable(Type *Ty) const
Test if values of the given type are analyzable within the SCEV framework.
Type * getEffectiveSCEVType(Type *Ty) const
Return a type with the same bitwidth as the given type and which represents how SCEV will treat the g...
const SCEV * getMinusSCEV(const SCEV *LHS, const SCEV *RHS, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Return LHS-RHS.
bool hasLoopInvariantBackedgeTakenCount(const Loop *L)
Return true if the specified loop has an analyzable loop-invariant backedge-taken count.
const SCEV * getAnyExtendExpr(const SCEV *Op, Type *Ty)
getAnyExtendExpr - Return a SCEV for the given operand extended with unspecified bits out to the give...
bool containsUndefs(const SCEV *S) const
Return true if the SCEV expression contains an undef value.
const SCEV * getSignExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth=0)
bool hasComputableLoopEvolution(const SCEV *S, const Loop *L)
Return true if the given SCEV changes value in a known way in the specified loop.
const SCEV * getPointerBase(const SCEV *V)
Transitively follow the chain of pointer-type operands until reaching a SCEV that does not have a sin...
const SCEV * getMulExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Get a canonical multiply expression, or something simpler if possible.
const SCEV * getUnknown(Value *V)
std::optional< APInt > computeConstantDifference(const SCEV *LHS, const SCEV *RHS)
Compute LHS - RHS and returns the result as an APInt if it is a constant, and std::nullopt if it isn'...
bool properlyDominates(const SCEV *S, const BasicBlock *BB)
Return true if elements that makes up the given SCEV properly dominate the specified basic block.
const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Get a canonical add expression, or something simpler if possible.
bool containsErasedValue(const SCEV *S) const
Return true if the SCEV expression contains a Value that has been optimised out and is now a nullptr.
LLVMContext & getContext() const
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
Definition: SetVector.h:57
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:98
iterator end()
Get an iterator to the end of the SetVector.
Definition: SetVector.h:113
iterator begin()
Get an iterator to the beginning of the SetVector.
Definition: SetVector.h:103
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
int find_first() const
Returns the index of the first set bit, -1 if none of the bits are set.
iterator_range< const_set_bits_iterator > set_bits() const
int find_next(unsigned Prev) const
Returns the index of the next set bit following the "Prev" bit.
size_type size() const
Returns the number of bits in this bitvector.
void resize(unsigned N, bool t=false)
Grow or shrink the bitvector.
size_type count() const
Returns the number of bits which are set.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:321
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:360
iterator end() const
Definition: SmallPtrSet.h:385
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
iterator begin() const
Definition: SmallPtrSet.h:380
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:370
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
void clear()
Definition: SmallSet.h:218
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:179
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void assign(size_type NumElts, ValueParamT Elt)
Definition: SmallVector.h:717
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:950
void reserve(size_type N)
Definition: SmallVector.h:676
iterator erase(const_iterator CI)
Definition: SmallVector.h:750
typename SuperClass::const_iterator const_iterator
Definition: SmallVector.h:591
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:818
typename SuperClass::iterator iterator
Definition: SmallVector.h:590
void resize(size_type N)
Definition: SmallVector.h:651
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
An instruction for storing to memory.
Definition: Instructions.h:317
Provides information about what library functions are available for the current target.
Wrapper pass for TargetTransformInfo.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) const
bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1, const TargetTransformInfo::LSRCost &C2) const
Return true if LSR cost of C1 is lower than C2.
bool shouldFoldTerminatingConditionAfterLSR() const
Return true if LSR should attempts to replace a use of an otherwise dead primary IV in the latch cond...
bool isProfitableLSRChainElement(Instruction *I) const
bool LSRWithInstrQueries() const
Return true if the loop strength reduce pass should make Instruction* based TTI queries to isLegalAdd...
bool isIndexedStoreLegal(enum MemIndexedMode Mode, Type *Ty) const
bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace=0, Instruction *I=nullptr) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
unsigned getRegisterClassForType(bool Vector, Type *Ty=nullptr) const
bool isIndexedLoadLegal(enum MemIndexedMode Mode, Type *Ty) const
bool isLegalICmpImmediate(int64_t Imm) const
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
bool isTypeLegal(Type *Ty) const
Return true if this type is legal.
bool isLegalAddImmediate(int64_t Imm) const
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, LoopInfo *LI, DominatorTree *DT, AssumptionCache *AC, TargetLibraryInfo *LibInfo) const
Return true if the target can save a compare for loop count, for example hardware loop saves a compar...
InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace=0) const
Return the cost of the scaling factor used in the addressing mode represented by AM for this target,...
unsigned getNumberOfRegisters(unsigned ClassID) const
bool isNumRegsMajorCostOfLSR() const
Return true if LSR major cost is number of registers.
@ MIM_PostInc
Post-incrementing.
bool canMacroFuseCmp() const
Return true if the target can fuse a compare and branch.
bool isTruncateFree(Type *Ty1, Type *Ty2) const
Return true if it's free to truncate a value of type Ty1 to type Ty2.
This class represents a truncation of integer types.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
unsigned getIntegerBitWidth() const
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:255
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static Type * getVoidTy(LLVMContext &C)
int getFPMantissaWidth() const
Return the width of the mantissa of this type.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:228
This class represents a cast unsigned integer to floating point.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
op_range operands()
Definition: User.h:242
bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Definition: User.cpp:21
void setOperand(unsigned i, Value *Val)
Definition: User.h:174
Value * getOperand(unsigned i) const
Definition: User.h:169
unsigned getNumOperands() const
Definition: User.h:191
op_iterator op_end()
Definition: User.h:236
static ValueAsMetadata * get(Value *V)
Definition: Metadata.cpp:492
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:534
iterator_range< user_iterator > users()
Definition: Value.h:421
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1074
iterator_range< use_iterator > uses()
Definition: Value.h:376
A nullable Value handle that is nullable.
Definition: ValueHandle.h:144
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:97
self_iterator getIterator()
Definition: ilist_node.h:109
A range adaptor for a pair of iterators.
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Key
PAL metadata keys.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
Reg
All possible values of the reg field in the ModR/M byte.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
Definition: CommandLine.h:718
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
Definition: Dwarf.h:146
@ DW_OP_LLVM_convert
Only used in LLVM metadata.
Definition: Dwarf.h:142
constexpr double e
Definition: MathExtras.h:31
Sequence
A sequence of states that a pointer may go through in which an objc_retain and objc_release are actua...
Definition: PtrState.h:41
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< PhiNode * > Phi
Definition: RDFGraph.h:390
const_iterator begin(StringRef path, Style style=Style::native)
Get begin iterator over path.
Definition: Path.cpp:227
const_iterator end(StringRef path)
Get end iterator over path.
Definition: Path.cpp:236
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
bool mustExecuteUBIfPoisonOnPathTo(Instruction *Root, Instruction *OnPathTo, DominatorTree *DT)
Return true if undefined behavior would provable be executed on the path to OnPathTo if Root produced...
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:329
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
@ Offset
Definition: DWP.cpp:456
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1751
std::optional< unsigned > getLoopEstimatedTripCount(Loop *L, unsigned *EstimatedLoopInvocationWeight=nullptr)
Returns a loop's estimated trip count based on branch weight metadata.
Definition: LoopUtils.cpp:849
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1731
APFloat abs(APFloat X)
Returns the absolute value of the argument.
Definition: APFloat.h:1381
bool operator!=(uint64_t V1, const APInt &V2)
Definition: APInt.h:2043
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition: STLExtras.h:2082
char & LoopSimplifyID
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:215
bool matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO, Value *&Start, Value *&Step)
Attempt to match a simple first order recurrence cycle of the form: iv = phi Ty [Start,...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1738
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:313
bool DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Examine each PHI in the given block and delete it if it is dead.
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:428
const SCEV * denormalizeForPostIncUse(const SCEV *S, const PostIncLoopSet &Loops, ScalarEvolution &SE)
Denormalize S to be post-increment for all loops present in Loops.
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1656
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1745
cl::opt< unsigned > SCEVCheapExpansionBudget
Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
void SplitLandingPadPredecessors(BasicBlock *OrigBB, ArrayRef< BasicBlock * > Preds, const char *Suffix, const char *Suffix2, SmallVectorImpl< BasicBlock * > &NewBBs, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, bool PreserveLCSSA=false)
This method transforms the landing pad, OrigBB, by introducing two new basic blocks into the function...
const SCEV * normalizeForPostIncUse(const SCEV *S, const PostIncLoopSet &Loops, ScalarEvolution &SE, bool CheckInvertible=true)
Normalize S to be post-increment for all loops present in Loops.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Add
Sum of integers.
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:1923
Pass * createLoopStrengthReducePass()
BasicBlock * SplitCriticalEdge(Instruction *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions(), const Twine &BBName="")
If this edge is a critical edge, insert a new node to split the critical edge.
bool RecursivelyDeleteTriviallyDeadInstructionsPermissive(SmallVectorImpl< WeakTrackingVH > &DeadInsts, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, std::function< void(Value *)> AboutToDeleteCallback=std::function< void(Value *)>())
Same functionality as RecursivelyDeleteTriviallyDeadInstructions, but allow instructions that are not...
Definition: Local.cpp:548
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:191
bool formLCSSAForInstructions(SmallVectorImpl< Instruction * > &Worklist, const DominatorTree &DT, const LoopInfo &LI, ScalarEvolution *SE, SmallVectorImpl< PHINode * > *PHIsToRemove=nullptr, SmallVectorImpl< PHINode * > *InsertedPHIs=nullptr)
Ensures LCSSA form for every instruction from the Worklist in the scope of innermost containing loop.
Definition: LCSSA.cpp:77
void initializeLoopStrengthReducePass(PassRegistry &)
PreservedAnalyses getLoopPassPreservedAnalyses()
Returns the minimum set of Analyses that all loop passes must preserve.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1758
bool isAlmostDeadIV(PHINode *IV, BasicBlock *LatchBlock, Value *Cond)
Return true if the induction variable IV in a Loop whose latch is LatchBlock would become dead if the...
Definition: LoopUtils.cpp:469
int rewriteLoopExitValues(Loop *L, LoopInfo *LI, TargetLibraryInfo *TLI, ScalarEvolution *SE, const TargetTransformInfo *TTI, SCEVExpander &Rewriter, DominatorTree *DT, ReplaceExitVal ReplaceExitValue, SmallVector< WeakTrackingVH, 16 > &DeadInsts)
If the final value of any expressions that are recurrent in the loop can be computed,...
Definition: LoopUtils.cpp:1416
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition: STLExtras.h:1888
@ UnusedIndVarInLoop
Definition: LoopUtils.h:465
static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)
Filter the DbgRecord range to DPValue types only and downcast.
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
Definition: Hashing.h:491
bool SCEVExprContains(const SCEV *Root, PredTy Pred)
Return true if any node in Root satisfies the predicate Pred.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
Option class for critical edge splitting.
The adaptor from a function pass to a loop pass computes these analyses and makes them available to t...
Information about a load/store intrinsic defined by the target.
Value * PtrVal
This is the pointer that the intrinsic is loading from or storing to.