Index: lib/Transforms/Scalar/LoopUnrollPass.cpp =================================================================== --- lib/Transforms/Scalar/LoopUnrollPass.cpp +++ lib/Transforms/Scalar/LoopUnrollPass.cpp @@ -246,187 +246,6 @@ } namespace { -/// \brief SCEV expressions visitor used for finding expressions that would -/// become constants if the loop L is unrolled. -struct FindConstantPointers { - /// \brief Shows whether the expression is ConstAddress+Constant or not. - bool IndexIsConstant; - - /// \brief Used for filtering out SCEV expressions with two or more AddRec - /// subexpressions. - /// - /// Used to filter out complicated SCEV expressions, having several AddRec - /// sub-expressions. We don't handle them, because unrolling one loop - /// would help to replace only one of these inductions with a constant, and - /// consequently, the expression would remain non-constant. - bool HaveSeenAR; - - /// \brief If the SCEV expression becomes ConstAddress+Constant, this value - /// holds ConstAddress. Otherwise, it's nullptr. - Value *BaseAddress; - - /// \brief The loop, which we try to completely unroll. - const Loop *L; - - ScalarEvolution &SE; - - FindConstantPointers(const Loop *L, ScalarEvolution &SE) - : IndexIsConstant(true), HaveSeenAR(false), BaseAddress(nullptr), - L(L), SE(SE) {} - - /// Examine the given expression S and figure out, if it can be a part of an - /// expression, that could become a constant after the loop is unrolled. - /// The routine sets IndexIsConstant and HaveSeenAR according to the analysis - /// results. - /// \returns true if we need to examine subexpressions, and false otherwise. - bool follow(const SCEV *S) { - if (const SCEVUnknown *SC = dyn_cast(S)) { - // We've reached the leaf node of SCEV, it's most probably just a - // variable. - // If it's the only one SCEV-subexpression, then it might be a base - // address of an index expression. - // If we've already recorded base address, then just give up on this SCEV - // - it's too complicated. - if (BaseAddress) { - IndexIsConstant = false; - return false; - } - BaseAddress = SC->getValue(); - return false; - } - if (isa(S)) - return false; - if (const SCEVAddRecExpr *AR = dyn_cast(S)) { - // If the current SCEV expression is AddRec, and its loop isn't the loop - // we are about to unroll, then we won't get a constant address after - // unrolling, and thus, won't be able to eliminate the load. - if (AR->getLoop() != L) { - IndexIsConstant = false; - return false; - } - // We don't handle multiple AddRecs here, so give up in this case. - if (HaveSeenAR) { - IndexIsConstant = false; - return false; - } - HaveSeenAR = true; - } - - // Continue traversal. - return true; - } - bool isDone() const { return !IndexIsConstant; } -}; -} // End anonymous namespace. - -namespace { -/// \brief A cache of SCEV results used to optimize repeated queries to SCEV on -/// the same set of instructions. -/// -/// The primary cost this saves is the cost of checking the validity of a SCEV -/// every time it is looked up. However, in some cases we can provide a reduced -/// and especially useful model for an instruction based upon SCEV that is -/// non-trivial to compute but more useful to clients. -class SCEVCache { -public: - /// \brief Struct to represent a GEP whose start and step are known fixed - /// offsets from a base address due to SCEV's analysis. - struct GEPDescriptor { - Value *BaseAddr = nullptr; - unsigned Start = 0; - unsigned Step = 0; - }; - - Optional getGEPDescriptor(GetElementPtrInst *GEP); - - SCEVCache(const Loop &L, ScalarEvolution &SE) : L(L), SE(SE) {} - -private: - const Loop &L; - ScalarEvolution &SE; - - SmallDenseMap GEPDescriptors; -}; -} // End anonymous namespace. - -/// \brief Get a simplified descriptor for a GEP instruction. -/// -/// Where possible, this produces a simplified descriptor for a GEP instruction -/// using SCEV analysis of the containing loop. If this isn't possible, it -/// returns an empty optional. -/// -/// The model is a base address, an initial offset, and a per-iteration step. -/// This fits very common patterns of GEPs inside loops and is something we can -/// use to simulate the behavior of a particular iteration of a loop. -/// -/// This is a cached interface. The first call may do non-trivial work to -/// compute the result, but all subsequent calls will return a fast answer -/// based on a cached result. This includes caching negative results. -Optional -SCEVCache::getGEPDescriptor(GetElementPtrInst *GEP) { - decltype(GEPDescriptors)::iterator It; - bool Inserted; - - std::tie(It, Inserted) = GEPDescriptors.insert({GEP, {}}); - - if (!Inserted) { - if (!It->second.BaseAddr) - return None; - - return It->second; - } - - // We've inserted a new record into the cache, so compute the GEP descriptor - // if possible. - Value *V = cast(GEP); - if (!SE.isSCEVable(V->getType())) - return None; - const SCEV *S = SE.getSCEV(V); - - // FIXME: It'd be nice if the worklist and set used by the - // SCEVTraversal could be re-used between loop iterations, but the - // interface doesn't support that. There is no way to clear the visited - // sets between uses. - FindConstantPointers Visitor(&L, SE); - SCEVTraversal T(Visitor); - - // Try to find (BaseAddress+Step+Offset) tuple. - // If succeeded, save it to the cache - it might help in folding - // loads. - T.visitAll(S); - if (!Visitor.IndexIsConstant || !Visitor.BaseAddress) - return None; - - const SCEV *BaseAddrSE = SE.getSCEV(Visitor.BaseAddress); - if (BaseAddrSE->getType() != S->getType()) - return None; - const SCEV *OffSE = SE.getMinusSCEV(S, BaseAddrSE); - const SCEVAddRecExpr *AR = dyn_cast(OffSE); - - if (!AR) - return None; - - const SCEVConstant *StepSE = - dyn_cast(AR->getStepRecurrence(SE)); - const SCEVConstant *StartSE = dyn_cast(AR->getStart()); - if (!StepSE || !StartSE) - return None; - - // Check and skip caching if doing so would require lots of bits to - // avoid overflow. - APInt Start = StartSE->getValue()->getValue(); - APInt Step = StepSE->getValue()->getValue(); - if (Start.getActiveBits() > 32 || Step.getActiveBits() > 32) - return None; - - // We found a cacheable SCEV model for the GEP. - It->second.BaseAddr = Visitor.BaseAddress; - It->second.Start = Start.getLimitedValue(); - It->second.Step = Step.getLimitedValue(); - return It->second; -} - -namespace { // This class is used to get an estimate of the optimization effects that we // could get from complete loop unrolling. It comes from the fact that some // loads might be replaced with concrete constant values and that could trigger @@ -450,13 +269,24 @@ public: UnrolledInstAnalyzer(unsigned Iteration, DenseMap &SimplifiedValues, - SCEVCache &SC) - : Iteration(Iteration), SimplifiedValues(SimplifiedValues), SC(SC) {} + const Loop *L, ScalarEvolution &SE) + : Iteration(Iteration), SimplifiedValues(SimplifiedValues), L(L), SE(SE) { + } // Allow access to the initial visit method. using Base::visit; private: + /// \brief A cache of pointer bases corresponding to GEP (or derived from GEP) + /// instructions. + /// + /// In order to find the base pointer one needs to perform non-trivial + /// traversal of the corresponding SCEV expression, so it's good to have the + /// results saved. + /// The instruction doesn't have to be GetElementPtrInst - for instance, it + /// could be PHINode. That's why we use a generic Instruction type here. + SmallDenseMap GEPPointerBases; + /// \brief Number of currently simulated iteration. /// /// If an expression is ConstAddress+Constant, then the Constant is @@ -471,11 +301,54 @@ // post-unrolling. DenseMap &SimplifiedValues; - // We use a cache to wrap all our SCEV queries. - SCEVCache &SC; + const Loop *L; + ScalarEvolution &SE; + + /// \brief Try to simplify instruction \param I using its SCEV expression. + /// + /// The idea is that some AddRec expressions become constants, which then + /// could trigger folding of other instructions. However, that only happens + /// for expressions whose start value is also constant, which isn't always the + /// case. In another common and important case the start value is just some + /// address (i.e. SCEVUnknown) - in this case we compute the offset and save + /// it instead. + bool SCEVSimplify(Instruction *I) { + if (!SE.isSCEVable(I->getType())) + return false; + + const SCEV *S = SE.getSCEV(I); + if (auto *SC = dyn_cast(S)) { + SimplifiedValues[I] = SC->getValue(); + return true; + } + + if (auto *AR = dyn_cast(S)) { + const SCEV *IterationNumber = SE.getConstant(APInt(64, Iteration)); + const SCEV *ValueAtIteration = + AR->evaluateAtIteration(IterationNumber, SE); + // Check if the AddRec expression becomes a constant. + if (auto *SC = dyn_cast(ValueAtIteration)) { + SimplifiedValues[I] = SC->getValue(); + return true; + } + // Check if the offset from the base address becomes a constant. + auto *Base = dyn_cast(SE.getPointerBase(S)); + if (!Base) + return false; + if (auto *Offset = + dyn_cast(SE.getMinusSCEV(ValueAtIteration, Base))) { + GEPPointerBases[I] = Base->getValue(); + SimplifiedValues[I] = Offset->getValue(); + return true; + } + } + return false; + } /// Base case for the instruction visitor. - bool visitInstruction(Instruction &I) { return false; }; + bool visitInstruction(Instruction &I) { + return SCEVSimplify(&I); + } /// TODO: Add visitors for other instruction types, e.g. ZExt, SExt. @@ -485,6 +358,8 @@ /// simplifications effects to a separate class, since we have a very similar /// code in InlineCost already. bool visitBinaryOperator(BinaryOperator &I) { + if (SCEVSimplify(&I)) + return true; Value *LHS = I.getOperand(0), *RHS = I.getOperand(1); if (!isa(LHS)) if (Constant *SimpleLHS = SimplifiedValues.lookup(LHS)) @@ -509,18 +384,16 @@ /// Try to fold load I. bool visitLoad(LoadInst &I) { Value *AddrOp = I.getPointerOperand(); - if (!isa(AddrOp)) - if (Constant *SimplifiedAddrOp = SimplifiedValues.lookup(AddrOp)) - AddrOp = SimplifiedAddrOp; - auto *GEP = dyn_cast(AddrOp); if (!GEP) return false; - auto OptionalGEPDesc = SC.getGEPDescriptor(GEP); - if (!OptionalGEPDesc) + + ConstantInt *SimplifiedAddrOp = + dyn_cast_or_null(SimplifiedValues.lookup(AddrOp)); + if (!SimplifiedAddrOp) return false; - auto GV = dyn_cast(OptionalGEPDesc->BaseAddr); + auto *GV = dyn_cast_or_null(GEPPointerBases.lookup(GEP)); // We're only interested in loads that can be completely folded to a // constant. if (!GV || !GV->hasInitializer()) @@ -535,9 +408,7 @@ // low and both the start and step are 32-bit integers. We use signed // integers so that UBSan will catch if a bug sneaks into the code. int ElemSize = CDS->getElementType()->getPrimitiveSizeInBits() / 8U; - int64_t Index = ((int64_t)OptionalGEPDesc->Start + - (int64_t)OptionalGEPDesc->Step * (int64_t)Iteration) / - ElemSize; + int64_t Index = SimplifiedAddrOp->getLimitedValue() / ElemSize; if (Index >= CDS->getNumElements()) { // FIXME: For now we conservatively ignore out of bound accesses, but // we're allowed to perform the optimization in this case. @@ -593,10 +464,6 @@ SmallSetVector BBWorklist; DenseMap SimplifiedValues; - // Use a cache to access SCEV expressions so that we don't pay the cost on - // each iteration. This cache is lazily self-populating. - SCEVCache SC(*L, SE); - unsigned NumberOfOptimizedInstructions = 0; unsigned UnrolledLoopSize = 0; @@ -606,7 +473,7 @@ // we literally have to go through all loop's iterations. for (unsigned Iteration = 0; Iteration < TripCount; ++Iteration) { SimplifiedValues.clear(); - UnrolledInstAnalyzer Analyzer(Iteration, SimplifiedValues, SC); + UnrolledInstAnalyzer Analyzer(Iteration, SimplifiedValues, L, SE); BBWorklist.clear(); BBWorklist.insert(L->getHeader());