diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -431,8 +431,9 @@ /// Vectorize a single GetElementPtrInst based on information gathered and /// decisions taken during planning. - void widenGEP(GetElementPtrInst *GEP, unsigned UF, unsigned VF, - bool IsPtrLoopInvariant, SmallBitVector &IsIndexLoopInvariant); + void widenGEP(GetElementPtrInst *GEP, VPUser &Indices, unsigned UF, + unsigned VF, bool IsPtrLoopInvariant, + SmallBitVector &IsIndexLoopInvariant, VPTransformState &State); /// Vectorize a single PHINode in a block. This method handles the induction /// variable canonicalization. It supports both VF = 1 for unrolled loops and @@ -4075,9 +4076,11 @@ } } -void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP, unsigned UF, - unsigned VF, bool IsPtrLoopInvariant, - SmallBitVector &IsIndexLoopInvariant) { +void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP, VPUser &Operands, + unsigned UF, unsigned VF, + bool IsPtrLoopInvariant, + SmallBitVector &IsIndexLoopInvariant, + VPTransformState &State) { // Construct a vector GEP by widening the operands of the scalar GEP as // necessary. We mark the vector GEP 'inbounds' if appropriate. A GEP // results in a vector of pointers when at least one operand of the GEP @@ -4121,12 +4124,12 @@ // Collect all the indices for the new GEP. If any index is // loop-invariant, we won't broadcast it. SmallVector Indices; - for (auto Index : enumerate(GEP->indices())) { - Value *User = Index.value().get(); + for (auto Index : enumerate(Operands.operands())) { + VPValue *Operand = Index.value(); if (IsIndexLoopInvariant[Index.index()]) - Indices.push_back(User); + Indices.push_back(State.get(Operand, {0, 0})); else - Indices.push_back(getOrCreateVectorValue(User, Part)); + Indices.push_back(State.get(Operand, Part)); } // Create the new GEP. Note that this GEP may be a scalar if VF == 1, @@ -7137,7 +7140,8 @@ return nullptr; if (auto GEP = dyn_cast(Instr)) - return new VPWidenGEPRecipe(GEP, OrigLoop); + return new VPWidenGEPRecipe(GEP, Plan->mapToVPValues(GEP->indices()), + OrigLoop); if (auto *SI = dyn_cast(Instr)) { bool InvariantCond = @@ -7446,8 +7450,8 @@ } void VPWidenGEPRecipe::execute(VPTransformState &State) { - State.ILV->widenGEP(GEP, State.UF, State.VF, IsPtrLoopInvariant, - IsIndexLoopInvariant); + State.ILV->widenGEP(GEP, Indices, State.UF, State.VF, IsPtrLoopInvariant, + IsIndexLoopInvariant, State); } void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -854,12 +854,18 @@ /// A recipe for handling GEP instructions. class VPWidenGEPRecipe : public VPRecipeBase { GetElementPtrInst *GEP; + + /// Hold VPValues for the indices of the GEP. + VPUser Indices; + bool IsPtrLoopInvariant; SmallBitVector IsIndexLoopInvariant; public: - VPWidenGEPRecipe(GetElementPtrInst *GEP, Loop *OrigLoop) - : VPRecipeBase(VPWidenGEPSC), GEP(GEP), + template + VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range Indices, + Loop *OrigLoop) + : VPRecipeBase(VPWidenGEPSC), GEP(GEP), Indices(Indices), IsIndexLoopInvariant(GEP->getNumIndices(), false) { IsPtrLoopInvariant = OrigLoop->isLoopInvariant(GEP->getPointerOperand()); for (auto Index : enumerate(GEP->indices())) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -70,7 +70,8 @@ } else NewRecipe = new VPWidenPHIRecipe(Phi); } else if (GetElementPtrInst *GEP = dyn_cast(Inst)) { - NewRecipe = new VPWidenGEPRecipe(GEP, OrigLoop); + NewRecipe = new VPWidenGEPRecipe( + GEP, Plan->mapToVPValues(GEP->indices()), OrigLoop); } else NewRecipe = new VPWidenRecipe(*Inst, Plan->mapToVPValues(Inst->operands()));