diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -485,8 +485,8 @@ /// Vectorize a single GetElementPtrInst based on information gathered and /// decisions taken during planning. - void widenGEP(GetElementPtrInst *GEP, VPUser &Indices, unsigned UF, - ElementCount VF, bool IsPtrLoopInvariant, + void widenGEP(GetElementPtrInst *GEP, VPValue *VPDef, VPUser &Indices, + unsigned UF, ElementCount VF, bool IsPtrLoopInvariant, SmallBitVector &IsIndexLoopInvariant, VPTransformState &State); /// Vectorize a single PHINode in a block. This method handles the induction @@ -4288,9 +4288,9 @@ } } -void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP, VPUser &Operands, - unsigned UF, ElementCount VF, - bool IsPtrLoopInvariant, +void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP, VPValue *VPDef, + VPUser &Operands, unsigned UF, + ElementCount VF, bool IsPtrLoopInvariant, SmallBitVector &IsIndexLoopInvariant, VPTransformState &State) { // Construct a vector GEP by widening the operands of the scalar GEP as @@ -8004,7 +8004,8 @@ } void VPWidenGEPRecipe::execute(VPTransformState &State) { - State.ILV->widenGEP(GEP, *this, State.UF, State.VF, IsPtrLoopInvariant, + State.ILV->widenGEP(cast(getUnderlyingInstr()), this, + *this, State.UF, State.VF, IsPtrLoopInvariant, IsIndexLoopInvariant, State); } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -906,22 +906,21 @@ }; /// A recipe for handling GEP instructions. -class VPWidenGEPRecipe : public VPRecipeBase, public VPUser { - GetElementPtrInst *GEP; - +class VPWidenGEPRecipe : public VPRecipeBase, public VPValue, public VPUser { bool IsPtrLoopInvariant; SmallBitVector IsIndexLoopInvariant; public: template VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range Operands) - : VPRecipeBase(VPWidenGEPSC), VPUser(Operands), GEP(GEP), - IsIndexLoopInvariant(GEP->getNumIndices(), false) {} + : VPRecipeBase(VPRecipeBase::VPWidenGEPSC), VPValue(VPWidenGEPSC, GEP), + VPUser(Operands), IsIndexLoopInvariant(GEP->getNumIndices(), false) {} template VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range Operands, Loop *OrigLoop) - : VPRecipeBase(VPWidenGEPSC), VPUser(Operands), GEP(GEP), + : VPRecipeBase(VPRecipeBase::VPWidenGEPSC), + VPValue(VPValue::VPVWidenGEPSC, GEP), VPUser(Operands), IsIndexLoopInvariant(GEP->getNumIndices(), false) { IsPtrLoopInvariant = OrigLoop->isLoopInvariant(GEP->getPointerOperand()); for (auto Index : enumerate(GEP->indices())) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -110,6 +110,8 @@ return V; if (auto *V = dyn_cast(this)) return V; + if (auto *V = dyn_cast(this)) + return V; return nullptr; } @@ -122,6 +124,8 @@ return V; if (auto *V = dyn_cast(this)) return V; + if (auto *V = dyn_cast(this)) + return V; return nullptr; } @@ -882,8 +886,11 @@ size_t IndicesNumber = IsIndexLoopInvariant.size(); for (size_t I = 0; I < IndicesNumber; ++I) O << "[" << (IsIndexLoopInvariant[I] ? "Inv" : "Var") << "]"; - O << "\\l\""; - O << " +\n" << Indent << "\" " << VPlanIngredient(GEP); + + O << " "; + printAsOperand(O, SlotTracker); + O << " = getelementptr "; + printOperands(O, SlotTracker); } void VPWidenPHIRecipe::print(raw_ostream &O, const Twine &Indent, diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -83,7 +83,8 @@ VPInstructionSC, VPMemoryInstructionSC, VPVWidenCallSC, - VPVWidenSelectSC + VPVWidenSelectSC, + VPVWidenGEPSC }; VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV) {} diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll @@ -36,4 +36,38 @@ ret void } +define void @print_widen_gep_and_select(i64 %n, float* noalias %y, float* noalias %x, float* %z) nounwind uwtable { +; CHECK: N0 [label = +; CHECK-NEXT: "for.body:\n" + +; CHECK-NEXT: "WIDEN-INDUCTION %iv = phi %iv.next, 0\l" + +; CHECK-NEXT: "WIDEN-GEP Inv[Var] ir<%arrayidx> = getelementptr ir<%y>, ir<%iv>\l" + +; CHECK-NEXT: "WIDEN ir<%lv> = load ir<%arrayidx>\l" + +; CHECK-NEXT: "WIDEN\l"" %cmp = icmp %arrayidx, %z\l" + +; CHECK-NEXT: "WIDEN-SELECT ir<%sel> = select ir<%cmp>, ir<1.000000e+01>, ir<2.000000e+01>\l" + +; CHECK-NEXT: "WIDEN\l"" %add = fadd %lv, %sel\l" + +; CHECK-NEXT: "CLONE %arrayidx2 = getelementptr %x, %iv\l" + +; CHECK-NEXT: "WIDEN store ir<%arrayidx2>, ir<%add>\l" +; CHECK-NEXT: ] + +entry: + %cmp6 = icmp sgt i64 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float, float* %y, i64 %iv + %lv = load float, float* %arrayidx, align 4 + %cmp = icmp eq float* %arrayidx, %z + %sel = select i1 %cmp, float 10.0, float 20.0 + %add = fadd float %lv, %sel + %arrayidx2 = getelementptr inbounds float, float* %x, i64 %iv + store float %add, float* %arrayidx2, align 4 + %iv.next = add i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + declare float @llvm.sqrt.f32(float) nounwind readnone