diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -439,8 +439,8 @@ /// Vectorize a single GetElementPtrInst based on information gathered and /// decisions taken during planning. - void widenGEP(GetElementPtrInst *GEP, VPUser &Indices, unsigned UF, - unsigned VF, bool IsPtrLoopInvariant, + void widenGEP(GetElementPtrInst *GEP, VPValue *VPDef, VPUser &Indices, + unsigned UF, unsigned VF, bool IsPtrLoopInvariant, SmallBitVector &IsIndexLoopInvariant, VPTransformState &State); /// Vectorize a single PHINode in a block. This method handles the induction @@ -4108,8 +4108,8 @@ } } -void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP, VPUser &Operands, - unsigned UF, unsigned VF, +void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP, VPValue *VPDef, + VPUser &Operands, unsigned UF, unsigned VF, bool IsPtrLoopInvariant, SmallBitVector &IsIndexLoopInvariant, VPTransformState &State) { @@ -7217,9 +7217,12 @@ if (!shouldWiden(Instr, Range)) return nullptr; - if (auto GEP = dyn_cast(Instr)) - return new VPWidenGEPRecipe(GEP, Plan->mapToVPValues(GEP->operands()), - OrigLoop); + if (auto GEP = dyn_cast(Instr)) { + auto *WidenG = new VPWidenGEPRecipe( + GEP, Plan->mapToVPValues(GEP->operands()), OrigLoop); + Plan->addVPValue(GEP, WidenG); + return WidenG; + } if (auto *SI = dyn_cast(Instr)) { bool InvariantCond = @@ -7530,8 +7533,9 @@ } void VPWidenGEPRecipe::execute(VPTransformState &State) { - State.ILV->widenGEP(GEP, User, State.UF, State.VF, IsPtrLoopInvariant, - IsIndexLoopInvariant, State); + State.ILV->widenGEP(getUnderlyingInstruction(), this, User, State.UF, + State.VF, IsPtrLoopInvariant, IsIndexLoopInvariant, + State); } void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -872,9 +872,7 @@ }; /// A recipe for handling GEP instructions. -class VPWidenGEPRecipe : public VPRecipeBase { - GetElementPtrInst *GEP; - +class VPWidenGEPRecipe : public VPRecipeBase, public VPValue { /// Hold VPValues for the base and indices of the GEP. VPUser User; @@ -885,8 +883,8 @@ template VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range Operands, Loop *OrigLoop) - : VPRecipeBase(VPWidenGEPSC), GEP(GEP), User(Operands), - IsIndexLoopInvariant(GEP->getNumIndices(), false) { + : VPRecipeBase(VPWidenGEPSC), VPValue(VPValue::VPVWidenGEPSC, GEP), + User(Operands), IsIndexLoopInvariant(GEP->getNumIndices(), false) { IsPtrLoopInvariant = OrigLoop->isLoopInvariant(GEP->getPointerOperand()); for (auto Index : enumerate(GEP->indices())) IsIndexLoopInvariant[Index.index()] = @@ -905,6 +903,13 @@ /// Print the recipe. void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override; + + GetElementPtrInst *getUnderlyingInstruction() { + return cast(getUnderlyingValue()); + } + const GetElementPtrInst *getUnderlyingInstruction() const { + return cast(getUnderlyingValue()); + } }; /// A recipe for handling phi nodes of integer and floating-point inductions, diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -779,7 +779,8 @@ for (size_t I = 0; I < IndicesNumber; ++I) O << "[" << (IsIndexLoopInvariant[I] ? "Inv" : "Var") << "]"; O << "\\l\""; - O << " +\n" << Indent << "\" " << VPlanIngredient(GEP); + O << " +\n" + << Indent << "\" " << VPlanIngredient(getUnderlyingInstruction()); } void VPWidenPHIRecipe::print(raw_ostream &O, const Twine &Indent, diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -81,7 +81,8 @@ VPInstructionSC, VPMemoryInstructionSC, VPVWidenCallSC, - VPVWidenSelectSC + VPVWidenSelectSC, + VPVWidenGEPSC }; VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV) {}