Index: llvm/include/llvm/Analysis/ScalarEvolution.h =================================================================== --- llvm/include/llvm/Analysis/ScalarEvolution.h +++ llvm/include/llvm/Analysis/ScalarEvolution.h @@ -1117,11 +1117,13 @@ /// Return the "disposition" of the given SCEV with respect to the given /// loop. - LoopDisposition getLoopDisposition(const SCEV *S, const Loop *L) const; + LoopDisposition getLoopDisposition(const SCEV *S, const Loop *L, + bool UseCache = true) const; /// Return true if the value of the given SCEV is unchanging in the /// specified loop. - bool isLoopInvariant(const SCEV *S, const Loop *L) const; + bool isLoopInvariant(const SCEV *S, const Loop *L, + bool UseCache = true) const; /// Determine if the SCEV can be evaluated at loop's entry. It is true if it /// doesn't depend on a SCEVUnknown of an instruction which is dominated by @@ -1515,7 +1517,8 @@ } /// Compute a LoopDisposition value. - LoopDisposition computeLoopDisposition(const SCEV *S, const Loop *L) const; + LoopDisposition computeLoopDisposition(const SCEV *S, const Loop *L, + bool UseCache) const; /// Memoized computeBlockDisposition results. DenseMap< Index: llvm/lib/Analysis/ScalarEvolution.cpp =================================================================== --- llvm/lib/Analysis/ScalarEvolution.cpp +++ llvm/lib/Analysis/ScalarEvolution.cpp @@ -13396,26 +13396,32 @@ } ScalarEvolution::LoopDisposition -ScalarEvolution::getLoopDisposition(const SCEV *S, const Loop *L) const { - auto &Values = LoopDispositions[S]; - for (auto &V : Values) { - if (V.getPointer() == L) - return V.getInt(); - } - Values.emplace_back(L, LoopVariant); - LoopDisposition D = computeLoopDisposition(S, L); - auto &Values2 = LoopDispositions[S]; - for (auto &V : llvm::reverse(Values2)) { - if (V.getPointer() == L) { - V.setInt(D); - break; +ScalarEvolution::getLoopDisposition(const SCEV *S, const Loop *L, + bool UseCache) const { + if (UseCache) { + auto &Values = LoopDispositions[S]; + for (auto &V : Values) { + if (V.getPointer() == L) + return V.getInt(); + } + Values.emplace_back(L, LoopVariant); + } + LoopDisposition D = computeLoopDisposition(S, L, UseCache); + if (UseCache) { + auto &Values2 = LoopDispositions[S]; + for (auto &V : llvm::reverse(Values2)) { + if (V.getPointer() == L) { + V.setInt(D); + break; + } } } return D; } ScalarEvolution::LoopDisposition -ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) const { +ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L, + bool UseCache) const { switch (S->getSCEVType()) { case scConstant: return LoopInvariant; @@ -13423,7 +13429,7 @@ case scTruncate: case scZeroExtend: case scSignExtend: - return getLoopDisposition(cast(S)->getOperand(), L); + return getLoopDisposition(cast(S)->getOperand(), L, UseCache); case scAddRecExpr: { const SCEVAddRecExpr *AR = cast(S); @@ -13448,7 +13454,7 @@ // This recurrence is variant w.r.t. L if any of its operands // are variant. for (const auto *Op : AR->operands()) - if (!isLoopInvariant(Op, L)) + if (!isLoopInvariant(Op, L, UseCache)) return LoopVariant; // Otherwise it's loop-invariant. @@ -13463,7 +13469,7 @@ case scSequentialUMinExpr: { bool HasVarying = false; for (const auto *Op : cast(S)->operands()) { - LoopDisposition D = getLoopDisposition(Op, L); + LoopDisposition D = getLoopDisposition(Op, L, UseCache); if (D == LoopVariant) return LoopVariant; if (D == LoopComputable) @@ -13473,10 +13479,10 @@ } case scUDivExpr: { const SCEVUDivExpr *UDiv = cast(S); - LoopDisposition LD = getLoopDisposition(UDiv->getLHS(), L); + LoopDisposition LD = getLoopDisposition(UDiv->getLHS(), L, UseCache); if (LD == LoopVariant) return LoopVariant; - LoopDisposition RD = getLoopDisposition(UDiv->getRHS(), L); + LoopDisposition RD = getLoopDisposition(UDiv->getRHS(), L, UseCache); if (RD == LoopVariant) return LoopVariant; return (LD == LoopInvariant && RD == LoopInvariant) ? @@ -13496,8 +13502,9 @@ llvm_unreachable("Unknown SCEV kind!"); } -bool ScalarEvolution::isLoopInvariant(const SCEV *S, const Loop *L) const { - return getLoopDisposition(S, L) == LoopInvariant; +bool ScalarEvolution::isLoopInvariant(const SCEV *S, const Loop *L, + bool UseCache) const { + return getLoopDisposition(S, L, UseCache) == LoopInvariant; } bool ScalarEvolution::hasComputableLoopEvolution(const SCEV *S, @@ -13972,6 +13979,25 @@ }; VerifyBECountUsers(/* Predicated */ false); VerifyBECountUsers(/* Predicated */ true); + + // Verify intergity of loop disposition cache. + for (auto &It : LoopDispositions) { + const SCEV *S = It.first; + auto &Values = It.second; + for (auto &V : Values) { + auto CachedDisposition = V.getInt(); + auto *Loop = V.getPointer(); + auto RecomputedLoopDisposition = + getLoopDisposition(S, Loop, /*UseCache*/ false); + if (CachedDisposition != RecomputedLoopDisposition) { + dbgs() << "Cached dispoisition of " << *S << " for loop " << *Loop + << " is incorrect: cached " + << loopDispositionToStr(CachedDisposition) << ", actual " + << loopDispositionToStr(RecomputedLoopDisposition) << "\n"; + std::abort(); + } + } + } } bool ScalarEvolution::invalidate( Index: llvm/test/Transforms/LoopFusion/double_loop_nest_inner_guard.ll =================================================================== --- llvm/test/Transforms/LoopFusion/double_loop_nest_inner_guard.ll +++ llvm/test/Transforms/LoopFusion/double_loop_nest_inner_guard.ll @@ -1,4 +1,7 @@ ; RUN: opt -S -loop-fusion < %s 2>&1 | FileCheck %s +; XFAIL: * +; REQUIRES: asserts +; Fails due to incorrect cached loop disposition. ; Verify that LoopFusion can fuse two double-loop nests with guarded inner ; loops. Loops are in canonical form. Index: llvm/test/Transforms/LoopFusion/loop_nest.ll =================================================================== --- llvm/test/Transforms/LoopFusion/loop_nest.ll +++ llvm/test/Transforms/LoopFusion/loop_nest.ll @@ -1,4 +1,7 @@ ; RUN: opt -S -loop-fusion < %s | FileCheck %s +; XFAIL: * +; REQUIRES: asserts +; Fails due to incorrect cached loop disposition. ; ; int A[1024][1024]; ; int B[1024][1024]; Index: llvm/test/Transforms/LoopFusion/triple_loop_nest_inner_guard.ll =================================================================== --- llvm/test/Transforms/LoopFusion/triple_loop_nest_inner_guard.ll +++ llvm/test/Transforms/LoopFusion/triple_loop_nest_inner_guard.ll @@ -1,4 +1,7 @@ ; RUN: opt -S -loop-fusion < %s 2>&1 | FileCheck %s +; XFAIL: * +; REQUIRES: asserts +; Fails due to incorrect cached loop disposition. ; Verify that LoopFusion can fuse two triple-loop nests with guarded inner ; loops. Loops are in canonical form.