diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -468,8 +468,8 @@ void widenCallInstruction(CallInst &I, VPDef &Def, VPTransformState &State); /// Widen a single select instruction within the innermost loop. - void widenSelectInstruction(SelectInst &I, VPUser &Operands, - bool InvariantCond, VPTransformState &State); + void widenSelectInstruction(SelectInst &I, VPDef &VPDef, bool InvariantCond, + VPTransformState &State); /// Fix the vectorized code, taking care of header phi's, live-outs, and more. void fixVectorizedLoop(); @@ -4684,8 +4684,7 @@ } } -void InnerLoopVectorizer::widenSelectInstruction(SelectInst &I, - VPUser &Operands, +void InnerLoopVectorizer::widenSelectInstruction(SelectInst &I, VPDef &Def, bool InvariantCond, VPTransformState &State) { setDebugLocFromInst(Builder, &I); @@ -4695,15 +4694,14 @@ // We have to take the 'vectorized' value and pick the first lane. // Instcombine will make this a no-op. auto *InvarCond = - InvariantCond ? State.get(Operands.getOperand(0), {0, 0}) : nullptr; + InvariantCond ? State.get(Def.getOperand(0), {0, 0}) : nullptr; for (unsigned Part = 0; Part < UF; ++Part) { - Value *Cond = - InvarCond ? InvarCond : State.get(Operands.getOperand(0), Part); - Value *Op0 = State.get(Operands.getOperand(1), Part); - Value *Op1 = State.get(Operands.getOperand(2), Part); + Value *Cond = InvarCond ? InvarCond : State.get(Def.getOperand(0), Part); + Value *Op0 = State.get(Def.getOperand(1), Part); + Value *Op1 = State.get(Def.getOperand(2), Part); Value *Sel = Builder.CreateSelect(Cond, Op0, Op1); - VectorLoopValueMap.setVectorValue(&I, Part, Sel); + State.set(Def.getVPValue(), &I, Sel, Part); addMetadata(Sel, &I); } } @@ -7633,7 +7631,10 @@ NeedDef.insert(Legal->getPrimaryInduction()); for (auto &Reduction : Legal->getReductionVars()) { NeedDef.insert(Reduction.first); - NeedDef.insert(Reduction.second.getLoopExitInstr()); + // VPWidenSelect is a VPValue already, there is no need to add a separate + // VPValue. + if (!isa(Reduction.second.getLoopExitInstr())) + NeedDef.insert(Reduction.second.getLoopExitInstr()); } } @@ -7991,7 +7992,8 @@ } void VPWidenSelectRecipe::execute(VPTransformState &State) { - State.ILV->widenSelectInstruction(Ingredient, *this, InvariantCond, State); + State.ILV->widenSelectInstruction(*cast(getUnderlyingInstr()), + *this, InvariantCond, State); } void VPWidenRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -866,10 +866,7 @@ }; /// A recipe for widening select instructions. -class VPWidenSelectRecipe : public VPRecipeBase, public VPUser { -private: - /// Hold the select to be widened. - SelectInst &Ingredient; +class VPWidenSelectRecipe : public VPRecipeBase, public VPDef { /// Is the condition of the select loop invariant? bool InvariantCond; @@ -878,8 +875,10 @@ template VPWidenSelectRecipe(SelectInst &I, iterator_range Operands, bool InvariantCond) - : VPRecipeBase(VPWidenSelectSC), VPUser(Operands), Ingredient(I), - InvariantCond(InvariantCond) {} + : VPRecipeBase(VPRecipeBase::VPWidenSelectSC), VPDef(Operands), + InvariantCond(InvariantCond) { + new VPValue(&I, this); + } ~VPWidenSelectRecipe() override = default; diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -114,6 +114,8 @@ return V; if (auto *V = dyn_cast(this)) return V->getVPValue(); + if (auto *V = dyn_cast(this)) + return V->getVPValue(); return nullptr; } @@ -124,6 +126,8 @@ return V; if (auto *V = dyn_cast(this)) return V->getVPValue(); + if (auto *V = dyn_cast(this)) + return V->getVPValue(); return nullptr; } @@ -837,8 +841,15 @@ void VPWidenSelectRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { - O << "\"WIDEN-SELECT" << VPlanIngredient(&Ingredient) - << (InvariantCond ? " (condition is loop invariant)" : ""); + O << "\"WIDEN-SELECT "; + getVPValue()->printAsOperand(O, SlotTracker); + O << " = select "; + getOperand(0)->printAsOperand(O, SlotTracker); + O << ", "; + getOperand(1)->printAsOperand(O, SlotTracker); + O << ", "; + getOperand(2)->printAsOperand(O, SlotTracker); + O << (InvariantCond ? " (condition is loop invariant)" : ""); } void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent, diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -83,7 +83,13 @@ /// are actually instantiated. Values of this enumeration are kept in the /// SubclassID field of the VPValue objects. They are used for concrete /// type identification. - enum { VPValueSC, VPInstructionSC, VPMemoryInstructionSC, VPVWidenCallSC }; + enum { + VPValueSC, + VPInstructionSC, + VPMemoryInstructionSC, + VPVWidenCallSC, + VPVWidenSelectSC + }; VPValue(Value *UV = nullptr, VPDef *Def = nullptr) : VPValue(VPValueSC, UV, Def) {} diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll --- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll +++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll @@ -39,7 +39,7 @@ ; CHECK-NEXT: "loop:\n" + ; CHECK-NEXT: "WIDEN-INDUCTION %iv = phi 0, %iv.next\l" + ; CHECK-NEXT: "WIDEN\l"" %cond0 = icmp %iv, 13\l" + -; CHECK-NEXT: "WIDEN-SELECT%s = select %cond0, 10, 20\l" +; CHECK-NEXT: "WIDEN-SELECT ir<%s> = select ir<%cond0>, ir<10>, ir<20>\l" ; CHECK-NEXT: ] define void @test() { entry: