diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h --- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h +++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h @@ -28,12 +28,6 @@ namespace llvm { extern cl::opt SCEVCheapExpansionBudget; -/// Return true if the given expression is safe to expand in the sense that -/// all materialized values are safe to speculate anywhere their operands are -/// defined, and the expander is capable of expanding the expression. -/// CanonicalMode indicates whether the expander will be used in canonical mode. -bool isSafeToExpand(const SCEV *S, ScalarEvolution &SE, bool CanonicalMode); - /// struct for holding enough information to help calculate the cost of the /// given SCEV when expanded into IR. struct SCEVOperand { diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp --- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp +++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp @@ -1950,6 +1950,7 @@ Loop *const L; MemorySSAUpdater *MSSAU; TTI::AddressingModeKind AMK; + mutable SCEVExpander Rewriter; bool Changed = false; /// This is the insert position that the current loop's induction variable @@ -1998,7 +1999,7 @@ SmallVectorImpl &ChainUsersVec); void FinalizeChain(IVChain &Chain); void CollectChains(); - void GenerateIVChain(const IVChain &Chain, SCEVExpander &Rewriter, + void GenerateIVChain(const IVChain &Chain, SmallVectorImpl &DeadInsts); void CollectInterestingTypesAndFactors(); @@ -2068,22 +2069,19 @@ void Solve(SmallVectorImpl &Solution) const; BasicBlock::iterator - HoistInsertPosition(BasicBlock::iterator IP, - const SmallVectorImpl &Inputs) const; - BasicBlock::iterator - AdjustInsertPositionForExpand(BasicBlock::iterator IP, - const LSRFixup &LF, - const LSRUse &LU, - SCEVExpander &Rewriter) const; + HoistInsertPosition(BasicBlock::iterator IP, + const SmallVectorImpl &Inputs) const; + BasicBlock::iterator AdjustInsertPositionForExpand(BasicBlock::iterator IP, + const LSRFixup &LF, + const LSRUse &LU) const; Value *Expand(const LSRUse &LU, const LSRFixup &LF, const Formula &F, - BasicBlock::iterator IP, SCEVExpander &Rewriter, + BasicBlock::iterator IP, SmallVectorImpl &DeadInsts) const; void RewriteForPHI(PHINode *PN, const LSRUse &LU, const LSRFixup &LF, - const Formula &F, SCEVExpander &Rewriter, + const Formula &F, SmallVectorImpl &DeadInsts) const; void Rewrite(const LSRUse &LU, const LSRFixup &LF, const Formula &F, - SCEVExpander &Rewriter, SmallVectorImpl &DeadInsts) const; void ImplementSolution(const SmallVectorImpl &Solution); @@ -3183,7 +3181,7 @@ /// Generate an add or subtract for each IVInc in a chain to materialize the IV /// user's operand from the previous IV user's operand. -void LSRInstance::GenerateIVChain(const IVChain &Chain, SCEVExpander &Rewriter, +void LSRInstance::GenerateIVChain(const IVChain &Chain, SmallVectorImpl &DeadInsts) { // Find the new IVOperand for the head of the chain. It may have been replaced // by LSR. @@ -3335,8 +3333,7 @@ // x == y --> x - y == 0 const SCEV *N = SE.getSCEV(NV); - if (SE.isLoopInvariant(N, L) && - isSafeToExpand(N, SE, /* CanonicalMode */ false) && + if (SE.isLoopInvariant(N, L) && Rewriter.isSafeToExpand(N) && (!NV->getType()->isPointerTy() || SE.getPointerBase(N) == SE.getPointerBase(S))) { // S is normalized, so normalize N before folding it into S @@ -3386,10 +3383,10 @@ /// Insert a formula for the given expression into the given use, separating out /// loop-variant portions from loop-invariant and loop-computable portions. -void -LSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx) { +void LSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU, + size_t LUIdx) { // Mark uses whose expressions cannot be expanded. - if (!isSafeToExpand(S, SE, /*CanonicalMode*/ false)) + if (!Rewriter.isSafeToExpand(S)) LU.RigidFormula = true; Formula F; @@ -5207,11 +5204,8 @@ /// Determine an input position which will be dominated by the operands and /// which will dominate the result. -BasicBlock::iterator -LSRInstance::AdjustInsertPositionForExpand(BasicBlock::iterator LowestIP, - const LSRFixup &LF, - const LSRUse &LU, - SCEVExpander &Rewriter) const { +BasicBlock::iterator LSRInstance::AdjustInsertPositionForExpand( + BasicBlock::iterator LowestIP, const LSRFixup &LF, const LSRUse &LU) const { // Collect some instructions which must be dominated by the // expanding replacement. These must be dominated by any operands that // will be required in the expansion. @@ -5274,14 +5268,13 @@ /// is called "expanding"). Value *LSRInstance::Expand(const LSRUse &LU, const LSRFixup &LF, const Formula &F, BasicBlock::iterator IP, - SCEVExpander &Rewriter, SmallVectorImpl &DeadInsts) const { if (LU.RigidFormula) return LF.OperandValToReplace; // Determine an input position which will be dominated by the operands and // which will dominate the result. - IP = AdjustInsertPositionForExpand(IP, LF, LU, Rewriter); + IP = AdjustInsertPositionForExpand(IP, LF, LU); Rewriter.setInsertPoint(&*IP); // Inform the Rewriter if we have a post-increment use, so that it can @@ -5453,7 +5446,7 @@ /// to be expanded in multiple places. void LSRInstance::RewriteForPHI( PHINode *PN, const LSRUse &LU, const LSRFixup &LF, const Formula &F, - SCEVExpander &Rewriter, SmallVectorImpl &DeadInsts) const { + SmallVectorImpl &DeadInsts) const { DenseMap Inserted; for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) if (PN->getIncomingValue(i) == LF.OperandValToReplace) { @@ -5508,8 +5501,8 @@ if (!Pair.second) PN->setIncomingValue(i, Pair.first->second); else { - Value *FullV = Expand(LU, LF, F, BB->getTerminator()->getIterator(), - Rewriter, DeadInsts); + Value *FullV = + Expand(LU, LF, F, BB->getTerminator()->getIterator(), DeadInsts); // If this is reuse-by-noop-cast, insert the noop cast. Type *OpTy = LF.OperandValToReplace->getType(); @@ -5568,15 +5561,14 @@ /// is called "expanding"), and update the UserInst to reference the newly /// expanded value. void LSRInstance::Rewrite(const LSRUse &LU, const LSRFixup &LF, - const Formula &F, SCEVExpander &Rewriter, + const Formula &F, SmallVectorImpl &DeadInsts) const { // First, find an insertion point that dominates UserInst. For PHI nodes, // find the nearest block which dominates all the relevant uses. if (PHINode *PN = dyn_cast(LF.UserInst)) { - RewriteForPHI(PN, LU, LF, F, Rewriter, DeadInsts); + RewriteForPHI(PN, LU, LF, F, DeadInsts); } else { - Value *FullV = - Expand(LU, LF, F, LF.UserInst->getIterator(), Rewriter, DeadInsts); + Value *FullV = Expand(LU, LF, F, LF.UserInst->getIterator(), DeadInsts); // If this is reuse-by-noop-cast, insert the noop cast. Type *OpTy = LF.OperandValToReplace->getType(); @@ -5610,13 +5602,6 @@ // we can remove them after we are done working. SmallVector DeadInsts; - SCEVExpander Rewriter(SE, L->getHeader()->getModule()->getDataLayout(), "lsr", - false); -#ifndef NDEBUG - Rewriter.setDebugType(DEBUG_TYPE); -#endif - Rewriter.disableCanonicalMode(); - Rewriter.enableLSRMode(); Rewriter.setIVIncInsertPos(L, IVIncInsertPos); // Mark phi nodes that terminate chains so the expander tries to reuse them. @@ -5628,12 +5613,12 @@ // Expand the new value definitions and update the users. for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) for (const LSRFixup &Fixup : Uses[LUIdx].Fixups) { - Rewrite(Uses[LUIdx], Fixup, *Solution[LUIdx], Rewriter, DeadInsts); + Rewrite(Uses[LUIdx], Fixup, *Solution[LUIdx], DeadInsts); Changed = true; } for (const IVChain &Chain : IVChainVec) { - GenerateIVChain(Chain, Rewriter, DeadInsts); + GenerateIVChain(Chain, DeadInsts); Changed = true; } @@ -5698,8 +5683,10 @@ const TargetTransformInfo &TTI, AssumptionCache &AC, TargetLibraryInfo &TLI, MemorySSAUpdater *MSSAU) : IU(IU), SE(SE), DT(DT), LI(LI), AC(AC), TLI(TLI), TTI(TTI), L(L), - MSSAU(MSSAU), AMK(PreferredAddresingMode.getNumOccurrences() > 0 ? - PreferredAddresingMode : TTI.getPreferredAddressingMode(L, &SE)) { + MSSAU(MSSAU), AMK(PreferredAddresingMode.getNumOccurrences() > 0 + ? PreferredAddresingMode + : TTI.getPreferredAddressingMode(L, &SE)), + Rewriter(SE, L->getHeader()->getModule()->getDataLayout(), "lsr", false) { // If LoopSimplify form is not available, stay out of trouble. if (!L->isLoopSimplifyForm()) return; @@ -5734,6 +5721,14 @@ L->getHeader()->printAsOperand(dbgs(), /*PrintType=*/false); dbgs() << ":\n"); + // Configure SCEVExpander already now, so the correct mode is used for + // isSafeToExpand() checks. +#ifndef NDEBUG + Rewriter.setDebugType(DEBUG_TYPE); +#endif + Rewriter.disableCanonicalMode(); + Rewriter.enableLSRMode(); + // First, perform some low-level loop optimizations. OptimizeShadowIV(); OptimizeLoopTermCond(); diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp --- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -2557,10 +2557,6 @@ return User->getOperand(OpIdx); } -bool SCEVExpander::isSafeToExpand(const SCEV *S) const { - return llvm::isSafeToExpand(S, SE, CanonicalMode); -} - namespace { // Search for a SCEV subexpression that is not safe to expand. Any expression // that may expand to a !isSafeToSpeculativelyExecute value is unsafe, namely @@ -2616,8 +2612,7 @@ }; } // namespace -namespace llvm { -bool isSafeToExpand(const SCEV *S, ScalarEvolution &SE, bool CanonicalMode) { +bool SCEVExpander::isSafeToExpand(const SCEV *S) const { SCEVFindUnsafe Search(SE, CanonicalMode); visitAll(S, Search); return !Search.IsUnsafe; @@ -2675,4 +2670,3 @@ I->eraseFromParent(); } } -}