Index: llvm/include/llvm/Analysis/ScalarEvolution.h =================================================================== --- llvm/include/llvm/Analysis/ScalarEvolution.h +++ llvm/include/llvm/Analysis/ScalarEvolution.h @@ -537,6 +537,9 @@ std::pair getStrengthenedNoWrapFlagsFromBinOp(const OverflowingBinaryOperator *OBO); + /// Notify SCEV that \p User directly uses SCEVs in \p Ops. + void registerUser(const SCEV *User, ArrayRef Ops); + /// Return a SCEV expression for the full generality of the specified /// expression. const SCEV *getSCEV(Value *V); @@ -1498,6 +1501,9 @@ /// Compute a BlockDisposition value. BlockDisposition computeBlockDisposition(const SCEV *S, const BasicBlock *BB); + /// Stores all SCEV that use a given SCEV as its direct operand. + DenseMap > SCEVUsers; + /// Memoized results from getRange DenseMap UnsignedRanges; @@ -1883,6 +1889,8 @@ /// Drop memoized information computed for S. void forgetMemoizedResults(const SCEV *S); + void forgetMemoizedResultsImpl(const SCEV *S); + /// Return an existing SCEV for V if there is one, otherwise return nullptr. const SCEV *getExistingSCEV(Value *V); Index: llvm/lib/Analysis/ScalarEvolution.cpp =================================================================== --- llvm/lib/Analysis/ScalarEvolution.cpp +++ llvm/lib/Analysis/ScalarEvolution.cpp @@ -1101,6 +1101,7 @@ SCEVPtrToIntExpr(ID.Intern(SCEVAllocator), Op, IntPtrTy); UniqueSCEVs.InsertNode(S, IP); addToLoopUseLists(S); + registerUser(S, { Op }); return S; } @@ -1221,6 +1222,7 @@ new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator), Op, Ty); UniqueSCEVs.InsertNode(S, IP); addToLoopUseLists(S); + registerUser(S, { Op }); return S; } @@ -1275,6 +1277,7 @@ Op, Ty); UniqueSCEVs.InsertNode(S, IP); addToLoopUseLists(S); + registerUser(S, { Op }); return S; } @@ -1604,6 +1607,7 @@ Op, Ty); UniqueSCEVs.InsertNode(S, IP); addToLoopUseLists(S); + registerUser(S, { Op }); return S; } @@ -1873,6 +1877,7 @@ Op, Ty); UniqueSCEVs.InsertNode(S, IP); addToLoopUseLists(S); + registerUser(S, { Op }); return S; } @@ -1912,6 +1917,7 @@ Op, Ty); UniqueSCEVs.InsertNode(S, IP); addToLoopUseLists(S); + registerUser(S, { Op }); return S; } @@ -2109,6 +2115,7 @@ Op, Ty); UniqueSCEVs.InsertNode(S, IP); addToLoopUseLists(S); + registerUser(S, { Op }); return S; } @@ -2894,6 +2901,7 @@ SCEVAddExpr(ID.Intern(SCEVAllocator), O, Ops.size()); UniqueSCEVs.InsertNode(S, IP); addToLoopUseLists(S); + registerUser(S, Ops); } S->setNoWrapFlags(Flags); return S; @@ -2917,6 +2925,7 @@ SCEVAddRecExpr(ID.Intern(SCEVAllocator), O, Ops.size(), L); UniqueSCEVs.InsertNode(S, IP); addToLoopUseLists(S); + registerUser(S, Ops); } setNoWrapFlags(S, Flags); return S; @@ -2939,6 +2948,7 @@ O, Ops.size()); UniqueSCEVs.InsertNode(S, IP); addToLoopUseLists(S); + registerUser(S, Ops); } S->setNoWrapFlags(Flags); return S; @@ -3448,6 +3458,7 @@ LHS, RHS); UniqueSCEVs.InsertNode(S, IP); addToLoopUseLists(S); + registerUser(S, { LHS, RHS }); return S; } @@ -3842,6 +3853,7 @@ UniqueSCEVs.InsertNode(S, IP); addToLoopUseLists(S); + registerUser(S, Ops); return S; } @@ -4085,6 +4097,17 @@ return false; } +void ScalarEvolution::registerUser(const SCEV *User, + ArrayRef Ops) { + for (auto *Op : Ops) { + if (!SCEVUsers.count(Op)) { + SmallPtrSet Users; + SCEVUsers[Op] = std::move(Users); + } + SCEVUsers[Op].insert(User); + } +} + /// Return an existing SCEV if it exists, otherwise analyze the expression and /// create a new one. const SCEV *ScalarEvolution::getSCEV(Value *V) { @@ -12374,6 +12397,7 @@ LoopDispositions(std::move(Arg.LoopDispositions)), LoopPropertiesCache(std::move(Arg.LoopPropertiesCache)), BlockDispositions(std::move(Arg.BlockDispositions)), + SCEVUsers(std::move(Arg.SCEVUsers)), UnsignedRanges(std::move(Arg.UnsignedRanges)), SignedRanges(std::move(Arg.SignedRanges)), UniqueSCEVs(std::move(Arg.UniqueSCEVs)), @@ -12782,8 +12806,24 @@ return SCEVExprContains(S, [&](const SCEV *Expr) { return Expr == Op; }); } -void -ScalarEvolution::forgetMemoizedResults(const SCEV *S) { +void ScalarEvolution::forgetMemoizedResults(const SCEV *S) { + SmallPtrSet Visited; + SmallVector Worklist; + Visited.insert(S); + Worklist.push_back(S); + while (!Worklist.empty()) { + const SCEV *Curr = Worklist.pop_back_val(); + auto Users = SCEVUsers.find(Curr); + if (Users != SCEVUsers.end()) + for (auto *User : Users->second) + if (Visited.insert(User).second) + Worklist.push_back(User); + } + for (auto *S : Visited) + forgetMemoizedResultsImpl(S); +} + +void ScalarEvolution::forgetMemoizedResultsImpl(const SCEV *S) { ValuesAtScopes.erase(S); LoopDispositions.erase(S); BlockDispositions.erase(S);