diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -580,7 +580,7 @@ void fixupIVUsers(PHINode *OrigPhi, const InductionDescriptor &II, Value *VectorTripCount, Value *EndValue, BasicBlock *MiddleBlock, BasicBlock *VectorHeader, - VPlan &Plan); + VPlan &Plan, VPTransformState &State); /// Handle all cross-iteration phis in the header. void fixCrossIterationPHIs(VPTransformState &State); @@ -3342,7 +3342,8 @@ const InductionDescriptor &II, Value *VectorTripCount, Value *EndValue, BasicBlock *MiddleBlock, - BasicBlock *VectorHeader, VPlan &Plan) { + BasicBlock *VectorHeader, VPlan &Plan, + VPTransformState &State) { // There are two kinds of external IV usages - those that use the value // computed in the last iteration (the PHI) and those that use the penultimate // value (the value that feeds into the phi from the loop latch). @@ -3370,7 +3371,6 @@ auto *UI = cast(U); if (!OrigLoop->contains(UI)) { assert(isa(UI) && "Expected LCSSA form"); - IRBuilder<> B(MiddleBlock->getTerminator()); // Fast-math-flags propagate from the original induction instruction. @@ -3380,8 +3380,11 @@ Value *CountMinusOne = B.CreateSub( VectorTripCount, ConstantInt::get(VectorTripCount->getType(), 1)); CountMinusOne->setName("cmo"); - Value *Step = CreateStepValue(II.getStep(), *PSE.getSE(), - VectorHeader->getTerminator()); + + VPValue *StepVPV = Plan.getSCEVExpansion(II.getStep()); + assert(StepVPV && "step must have been expanded during VPlan execution"); + Value *Step = StepVPV->isLiveIn() ? StepVPV->getLiveInIRValue() + : State.get(StepVPV, 0); Value *Escape = emitTransformedIndex(B, CountMinusOne, II.getStartValue(), Step, II); Escape->setName("ind.escape"); @@ -3740,7 +3743,7 @@ fixupIVUsers(Entry.first, Entry.second, getOrCreateVectorTripCount(VectorLoop->getLoopPreheader()), IVEndValues[Entry.first], LoopMiddleBlock, - VectorLoop->getHeader(), Plan); + VectorLoop->getHeader(), Plan, State); } // Fix LCSSA phis not already fixed earlier. Extracts may need to be generated diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2254,6 +2254,11 @@ /// Values used outside the plan. MapVector LiveOuts; + /// Mapping from SCEVs to the VPValues representing their expansions. + /// NOTE: This mapping is temporary and will be removed once all users have + /// been modeled in VPlan directly. + DenseMap SCEVToExpansion; + public: VPlan(VPBasicBlock *Entry = nullptr) : Entry(Entry) { if (Entry) @@ -2417,6 +2422,19 @@ return LiveOuts; } + VPValue *getSCEVExpansion(const SCEV *S) const { + auto I = SCEVToExpansion.find(S); + if (I == SCEVToExpansion.end()) + return nullptr; + return I->second; + } + + void addSCEVExpansion(const SCEV *S, VPValue *V) { + assert(SCEVToExpansion.find(S) == SCEVToExpansion.end() && + "SCEV already expanded"); + SCEVToExpansion[S] = V; + } + private: /// Add to the given dominator tree the header block and every new basic block /// that was created between it and the latch block, inclusive. diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1132,13 +1132,19 @@ VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE) { + if (auto *Expanded = Plan.getSCEVExpansion(Expr)) + return Expanded; + VPValue *Expanded = nullptr; if (auto *E = dyn_cast(Expr)) - return Plan.getVPValueOrAddLiveIn(E->getValue()); - if (auto *E = dyn_cast(Expr)) - return Plan.getVPValueOrAddLiveIn(E->getValue()); - - VPBasicBlock *Preheader = Plan.getEntry(); - VPExpandSCEVRecipe *Step = new VPExpandSCEVRecipe(Expr, SE); - Preheader->appendRecipe(Step); - return Step; + Expanded = Plan.getVPValueOrAddLiveIn(E->getValue()); + else if (auto *E = dyn_cast(Expr)) + Expanded = Plan.getVPValueOrAddLiveIn(E->getValue()); + else { + + VPBasicBlock *Preheader = Plan.getEntry(); + Expanded = new VPExpandSCEVRecipe(Expr, SE); + Preheader->appendRecipe(Expanded->getDefiningRecipe()); + } + Plan.addSCEVExpansion(Expr, Expanded); + return Expanded; } diff --git a/llvm/test/Transforms/LoopVectorize/pr58811-scev-expansion.ll b/llvm/test/Transforms/LoopVectorize/pr58811-scev-expansion.ll --- a/llvm/test/Transforms/LoopVectorize/pr58811-scev-expansion.ll +++ b/llvm/test/Transforms/LoopVectorize/pr58811-scev-expansion.ll @@ -1,9 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s -; REQUIRES: asserts -; XFAIL: * - define void @test1_pr58811() { ; CHECK-LABEL: @test1_pr58811( ; CHECK-NEXT: entry: