diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -415,7 +415,8 @@ VPTransformState &State); /// Widen a single select instruction within the innermost loop. - void widenSelectInstruction(SelectInst &I, bool InvariantCond); + void widenSelectInstruction(SelectInst &I, VPUser &Operands, + bool InvariantCond, VPTransformState &State); /// Fix the vectorized code, taking care of header phi's, live-outs, and more. void fixVectorizedLoop(); @@ -4417,7 +4418,9 @@ } void InnerLoopVectorizer::widenSelectInstruction(SelectInst &I, - bool InvariantCond) { + VPUser &Operands, + bool InvariantCond, + VPTransformState &State) { setDebugLocFromInst(Builder, &I); // The condition can be loop invariant but still defined inside the @@ -4425,12 +4428,12 @@ // We have to take the 'vectorized' value and pick the first lane. // Instcombine will make this a no-op. - auto *ScalarCond = getOrCreateScalarValue(I.getOperand(0), {0, 0}); + auto *ScalarCond = State.get(Operands.getOperand(0), {0, 0}); for (unsigned Part = 0; Part < UF; ++Part) { - Value *Cond = getOrCreateVectorValue(I.getOperand(0), Part); - Value *Op0 = getOrCreateVectorValue(I.getOperand(1), Part); - Value *Op1 = getOrCreateVectorValue(I.getOperand(2), Part); + Value *Cond = State.get(Operands.getOperand(0), Part); + Value *Op0 = State.get(Operands.getOperand(1), Part); + Value *Op1 = State.get(Operands.getOperand(2), Part); Value *Sel = Builder.CreateSelect(InvariantCond ? ScalarCond : Cond, Op0, Op1); VectorLoopValueMap.setVectorValue(&I, Part, Sel); @@ -7139,7 +7142,8 @@ if (auto *SI = dyn_cast(Instr)) { bool InvariantCond = PSE.getSE()->isLoopInvariant(PSE.getSCEV(SI->getOperand(0)), OrigLoop); - return new VPWidenSelectRecipe(*SI, InvariantCond); + return new VPWidenSelectRecipe(*SI, Plan->mapToVPValues(SI->operands()), + InvariantCond); } return tryToWiden(Instr, *Plan); @@ -7434,7 +7438,7 @@ } void VPWidenSelectRecipe::execute(VPTransformState &State) { - State.ILV->widenSelectInstruction(Ingredient, InvariantCond); + State.ILV->widenSelectInstruction(Ingredient, User, InvariantCond, State); } void VPWidenRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -823,15 +823,17 @@ /// Hold the select to be widened. SelectInst &Ingredient; - /// Is the condition of the select loop invariant? - bool InvariantCond; - /// Hold VPValues for the operands of the select. VPUser User; + /// Is the condition of the select loop invariant? + bool InvariantCond; + public: - VPWidenSelectRecipe(SelectInst &I, bool InvariantCond) - : VPRecipeBase(VPWidenSelectSC), Ingredient(I), + template + VPWidenSelectRecipe(SelectInst &I, iterator_range Operands, + bool InvariantCond) + : VPRecipeBase(VPWidenSelectSC), Ingredient(I), User(Operands), InvariantCond(InvariantCond) {} ~VPWidenSelectRecipe() override = default;