diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -467,7 +467,7 @@ VPTransformState &State); /// Widen a single select instruction within the innermost loop. - void widenSelectInstruction(SelectInst &I, VPUser &Operands, + void widenSelectInstruction(SelectInst &I, VPValue *VPDef, VPUser &Operands, bool InvariantCond, VPTransformState &State); /// Fix the vectorized code, taking care of header phi's, live-outs, and more. @@ -4680,7 +4680,7 @@ } } -void InnerLoopVectorizer::widenSelectInstruction(SelectInst &I, +void InnerLoopVectorizer::widenSelectInstruction(SelectInst &I, VPValue *VPDef, VPUser &Operands, bool InvariantCond, VPTransformState &State) { @@ -4699,7 +4699,7 @@ Value *Op0 = State.get(Operands.getOperand(1), Part); Value *Op1 = State.get(Operands.getOperand(2), Part); Value *Sel = Builder.CreateSelect(Cond, Op0, Op1); - VectorLoopValueMap.setVectorValue(&I, Part, Sel); + State.set(VPDef, &I, Sel, Part); addMetadata(Sel, &I); } } @@ -7614,7 +7614,10 @@ NeedDef.insert(Legal->getPrimaryInduction()); for (auto &Reduction : Legal->getReductionVars()) { NeedDef.insert(Reduction.first); - NeedDef.insert(Reduction.second.getLoopExitInstr()); + // VPWidenSelect is a VPValue already, there is no need to add a separate + // VPValue. + if (!isa(Reduction.second.getLoopExitInstr())) + NeedDef.insert(Reduction.second.getLoopExitInstr()); } } @@ -7970,7 +7973,8 @@ } void VPWidenSelectRecipe::execute(VPTransformState &State) { - State.ILV->widenSelectInstruction(Ingredient, *this, InvariantCond, State); + State.ILV->widenSelectInstruction(*getUnderlyingInstruction(), this, *this, + InvariantCond, State); } void VPWidenRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -860,10 +860,7 @@ }; /// A recipe for widening select instructions. -class VPWidenSelectRecipe : public VPRecipeBase, public VPUser { -private: - /// Hold the select to be widened. - SelectInst &Ingredient; +class VPWidenSelectRecipe : public VPRecipeBase, public VPValue, public VPUser { /// Is the condition of the select loop invariant? bool InvariantCond; @@ -872,8 +869,8 @@ template VPWidenSelectRecipe(SelectInst &I, iterator_range Operands, bool InvariantCond) - : VPRecipeBase(VPWidenSelectSC), VPUser(Operands), Ingredient(I), - InvariantCond(InvariantCond) {} + : VPRecipeBase(VPWidenSelectSC), VPValue(VPValue::VPVWidenSelectSC, &I), + VPUser(Operands), InvariantCond(InvariantCond) {} ~VPWidenSelectRecipe() override = default; @@ -888,6 +885,10 @@ /// Print the recipe. void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override; + + SelectInst *getUnderlyingInstruction() { + return cast(getUnderlyingValue()); + } }; /// A recipe for handling GEP instructions. diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -108,6 +108,8 @@ return Widen; if (auto *V = dyn_cast(this)) return V; + if (auto *V = dyn_cast(this)) + return V; return nullptr; } // Get the top-most entry block of \p Start. This is the entry block of the @@ -818,8 +820,15 @@ void VPWidenSelectRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { - O << "\"WIDEN-SELECT" << VPlanIngredient(&Ingredient) - << (InvariantCond ? " (condition is loop invariant)" : ""); + O << "\"WIDEN "; + printAsOperand(O, SlotTracker); + O << " = select "; + getOperand(0)->printAsOperand(O, SlotTracker); + O << ", "; + getOperand(1)->printAsOperand(O, SlotTracker); + O << ", "; + getOperand(2)->printAsOperand(O, SlotTracker); + O << (InvariantCond ? " (condition is loop invariant)" : ""); } void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent, diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -78,7 +78,13 @@ /// are actually instantiated. Values of this enumeration are kept in the /// SubclassID field of the VPValue objects. They are used for concrete /// type identification. - enum { VPValueSC, VPInstructionSC, VPMemoryInstructionSC, VPVWidenCallSC }; + enum { + VPValueSC, + VPInstructionSC, + VPMemoryInstructionSC, + VPVWidenCallSC, + VPVWidenSelectSC + }; VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV) {} VPValue(const VPValue &) = delete; diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll --- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll +++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll @@ -39,7 +39,7 @@ ; CHECK-NEXT: "loop:\n" + ; CHECK-NEXT: "WIDEN-INDUCTION %iv = phi 0, %iv.next\l" + ; CHECK-NEXT: "WIDEN\l"" %cond0 = icmp %iv, 13\l" + -; CHECK-NEXT: "WIDEN-SELECT%s = select %cond0, 10, 20\l" +; CHECK-NEXT: "WIDEN ir<%s> = select ir<%cond0>, ir<10>, ir<20>\l" ; CHECK-NEXT: ] define void @test() { entry: