diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -423,7 +423,7 @@ VPTransformState &State); /// Widen a single select instruction within the innermost loop. - void widenSelectInstruction(SelectInst &I, VPUser &Operands, + void widenSelectInstruction(SelectInst &I, VPValue *VPDef, VPUser &Operands, bool InvariantCond, VPTransformState &State); /// Fix the vectorized code, taking care of header phi's, live-outs, and more. @@ -4495,7 +4495,7 @@ } } -void InnerLoopVectorizer::widenSelectInstruction(SelectInst &I, +void InnerLoopVectorizer::widenSelectInstruction(SelectInst &I, VPValue *VPDef, VPUser &Operands, bool InvariantCond, VPTransformState &State) { @@ -4514,7 +4514,7 @@ Value *Op0 = State.get(Operands.getOperand(1), Part); Value *Op1 = State.get(Operands.getOperand(2), Part); Value *Sel = Builder.CreateSelect(Cond, Op0, Op1); - VectorLoopValueMap.setVectorValue(&I, Part, Sel); + State.set(VPDef, &I, Sel, Part); addMetadata(Sel, &I); } } @@ -7224,8 +7224,10 @@ if (auto *SI = dyn_cast(Instr)) { bool InvariantCond = PSE.getSE()->isLoopInvariant(PSE.getSCEV(SI->getOperand(0)), OrigLoop); - return new VPWidenSelectRecipe(*SI, Plan->mapToVPValues(SI->operands()), - InvariantCond); + auto *WidenSelect = new VPWidenSelectRecipe( + *SI, Plan->mapToVPValues(SI->operands()), InvariantCond); + Plan->addVPValue(SI, WidenSelect); + return WidenSelect; } return tryToWiden(Instr, *Plan); @@ -7519,7 +7521,8 @@ } void VPWidenSelectRecipe::execute(VPTransformState &State) { - State.ILV->widenSelectInstruction(Ingredient, User, InvariantCond, State); + State.ILV->widenSelectInstruction(*getUnderlyingInstruction(), this, User, + InvariantCond, State); } void VPWidenRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -837,11 +837,8 @@ }; /// A recipe for widening select instructions. -class VPWidenSelectRecipe : public VPRecipeBase { +class VPWidenSelectRecipe : public VPRecipeBase, public VPValue { private: - /// Hold the select to be widened. - SelectInst &Ingredient; - /// Hold VPValues for the operands of the select. VPUser User; @@ -852,8 +849,8 @@ template VPWidenSelectRecipe(SelectInst &I, iterator_range Operands, bool InvariantCond) - : VPRecipeBase(VPWidenSelectSC), Ingredient(I), User(Operands), - InvariantCond(InvariantCond) {} + : VPRecipeBase(VPWidenSelectSC), VPValue(VPValue::VPVWidenSelectSC, &I), + User(Operands), InvariantCond(InvariantCond) {} ~VPWidenSelectRecipe() override = default; @@ -868,6 +865,10 @@ /// Print the recipe. void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override; + + SelectInst *getUnderlyingInstruction() { + return cast(getUnderlyingValue()); + } }; /// A recipe for handling GEP instructions. diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -743,8 +743,15 @@ void VPWidenSelectRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { - O << "\"WIDEN-SELECT" << VPlanIngredient(&Ingredient) - << (InvariantCond ? " (condition is loop invariant)" : ""); + O << "\"WIDEN "; + printAsOperand(O, SlotTracker); + O << " = select "; + User.getOperand(0)->printAsOperand(O, SlotTracker); + O << ", "; + User.getOperand(1)->printAsOperand(O, SlotTracker); + O << ", "; + User.getOperand(2)->printAsOperand(O, SlotTracker); + O << (InvariantCond ? " (condition is loop invariant)" : ""); } void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent, diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -76,7 +76,13 @@ /// are actually instantiated. Values of this enumeration are kept in the /// SubclassID field of the VPValue objects. They are used for concrete /// type identification. - enum { VPValueSC, VPInstructionSC, VPMemoryInstructionSC, VPVWidenCallSC }; + enum { + VPValueSC, + VPInstructionSC, + VPMemoryInstructionSC, + VPVWidenCallSC, + VPVWidenSelectSC + }; VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV) {} VPValue(const VPValue &) = delete;