diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -484,7 +484,7 @@ /// Vectorize a single GetElementPtrInst based on information gathered and /// decisions taken during planning. - void widenGEP(GetElementPtrInst *GEP, VPUser &Indices, unsigned UF, + void widenGEP(GetElementPtrInst *GEP, VPDef &Def, unsigned UF, ElementCount VF, bool IsPtrLoopInvariant, SmallBitVector &IsIndexLoopInvariant, VPTransformState &State); @@ -4287,7 +4287,7 @@ } } -void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP, VPUser &Operands, +void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP, VPDef &Def, unsigned UF, ElementCount VF, bool IsPtrLoopInvariant, SmallBitVector &IsIndexLoopInvariant, @@ -4328,14 +4328,14 @@ for (unsigned Part = 0; Part < UF; ++Part) { // The pointer operand of the new GEP. If it's loop-invariant, we // won't broadcast it. - auto *Ptr = IsPtrLoopInvariant ? State.get(Operands.getOperand(0), {0, 0}) - : State.get(Operands.getOperand(0), Part); + auto *Ptr = IsPtrLoopInvariant ? State.get(Def.getOperand(0), {0, 0}) + : State.get(Def.getOperand(0), Part); // Collect all the indices for the new GEP. If any index is // loop-invariant, we won't broadcast it. SmallVector Indices; - for (unsigned I = 1, E = Operands.getNumOperands(); I < E; I++) { - VPValue *Operand = Operands.getOperand(I); + for (unsigned I = 1, E = Def.getNumOperands(); I < E; I++) { + VPValue *Operand = Def.getOperand(I); if (IsIndexLoopInvariant[I - 1]) Indices.push_back(State.get(Operand, {0, 0})); else @@ -8001,7 +8001,8 @@ } void VPWidenGEPRecipe::execute(VPTransformState &State) { - State.ILV->widenGEP(GEP, *this, State.UF, State.VF, IsPtrLoopInvariant, + State.ILV->widenGEP(cast(getUnderlyingInstr()), *this, + State.UF, State.VF, IsPtrLoopInvariant, IsIndexLoopInvariant, State); } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -896,23 +896,25 @@ }; /// A recipe for handling GEP instructions. -class VPWidenGEPRecipe : public VPRecipeBase, public VPUser { - GetElementPtrInst *GEP; - +class VPWidenGEPRecipe : public VPRecipeBase, public VPDef { bool IsPtrLoopInvariant; SmallBitVector IsIndexLoopInvariant; public: template VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range Operands) - : VPRecipeBase(VPWidenGEPSC), VPUser(Operands), GEP(GEP), - IsIndexLoopInvariant(GEP->getNumIndices(), false) {} + : VPRecipeBase(VPRecipeBase::VPWidenGEPSC), VPDef(Operands), + IsIndexLoopInvariant(GEP->getNumIndices(), false) { + + new VPValue(GEP, this); + } template VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range Operands, Loop *OrigLoop) - : VPRecipeBase(VPWidenGEPSC), VPUser(Operands), GEP(GEP), + : VPRecipeBase(VPWidenGEPSC), VPDef(Operands), IsIndexLoopInvariant(GEP->getNumIndices(), false) { + new VPValue(GEP, this); IsPtrLoopInvariant = OrigLoop->isLoopInvariant(GEP->getPointerOperand()); for (auto Index : enumerate(GEP->indices())) IsIndexLoopInvariant[Index.index()] = diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -116,6 +116,8 @@ return V->getVPValue(); if (auto *V = dyn_cast(this)) return V->getVPValue(); + if (auto *V = dyn_cast(this)) + return V->getVPValue(); return nullptr; } @@ -128,6 +130,8 @@ return V->getVPValue(); if (auto *V = dyn_cast(this)) return V->getVPValue(); + if (auto *V = dyn_cast(this)) + return V->getVPValue(); return nullptr; } @@ -877,7 +881,7 @@ for (size_t I = 0; I < IndicesNumber; ++I) O << "[" << (IsIndexLoopInvariant[I] ? "Inv" : "Var") << "]"; O << "\\l\""; - O << " +\n" << Indent << "\" " << VPlanIngredient(GEP); + O << " +\n" << Indent << "\" " << VPlanIngredient(getUnderlyingInstr()); } void VPWidenPHIRecipe::print(raw_ostream &O, const Twine &Indent, diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -88,7 +88,8 @@ VPInstructionSC, VPMemoryInstructionSC, VPVWidenCallSC, - VPVWidenSelectSC + VPVWidenSelectSC, + VPVWidenGEPSC }; VPValue(Value *UV = nullptr, VPDef *Def = nullptr)