diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -6884,17 +6884,18 @@ // duplications since this is a simple recursive scan, but future // optimizations will clean it up. - SmallVector Masks; + SmallVector Operands; unsigned NumIncoming = Phi->getNumIncomingValues(); for (unsigned In = 0; In < NumIncoming; In++) { VPValue *EdgeMask = createEdgeMask(Phi->getIncomingBlock(In), Phi->getParent(), Plan); assert((EdgeMask || NumIncoming == 1) && "Multiple predecessors with one having a full mask"); + Operands.push_back(Plan->getOrAddVPValue(Phi->getIncomingValue(In))); if (EdgeMask) - Masks.push_back(EdgeMask); + Operands.push_back(EdgeMask); } - return new VPBlendRecipe(Phi, Masks); + return new VPBlendRecipe(Phi, Operands); } VPWidenCallRecipe * @@ -7436,10 +7437,8 @@ // duplications since this is a simple recursive scan, but future // optimizations will clean it up. - unsigned NumIncoming = Phi->getNumIncomingValues(); + unsigned NumIncoming = getNumIncomingValues(); - assert((User || NumIncoming == 1) && - "Multiple predecessors with predecessors having a full mask"); // Generate a sequence of selects of the form: // SELECT(Mask3, In3, // SELECT(Mask2, In2, @@ -7449,14 +7448,13 @@ for (unsigned Part = 0; Part < State.UF; ++Part) { // We might have single edge PHIs (blocks) - use an identity // 'select' for the first PHI operand. - Value *In0 = - State.ILV->getOrCreateVectorValue(Phi->getIncomingValue(In), Part); + Value *In0 = State.get(getIncomingValue(In), Part); if (In == 0) Entry[Part] = In0; // Initialize with the first incoming value. else { // Select between the current value and the previous incoming edge // based on the incoming mask. - Value *Cond = State.get(User->getOperand(In), Part); + Value *Cond = State.get(getMask(In), Part); Entry[Part] = State.Builder.CreateSelect(Cond, In0, Entry[Part], "predphi"); } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -902,17 +902,18 @@ private: PHINode *Phi; - /// The blend operation is a User of a mask, if not null. - std::unique_ptr User; + /// The blend operation is a User of the incoming values and of their + /// respective masks, ordered [I0, M0, I1, M1, ...]. Note that a single value + /// would be incoming with a full mask for which there is no VPValue. + VPUser User; public: - VPBlendRecipe(PHINode *Phi, ArrayRef Masks) - : VPRecipeBase(VPBlendSC), Phi(Phi) { - assert((Phi->getNumIncomingValues() == 1 || - Phi->getNumIncomingValues() == Masks.size()) && - "Expected the same number of incoming values and masks"); - if (!Masks.empty()) - User.reset(new VPUser(Masks)); + VPBlendRecipe(PHINode *Phi, ArrayRef Operands) + : VPRecipeBase(VPBlendSC), Phi(Phi), User(Operands) { + assert(((Operands.size() == 1) || + (Operands.size() > 2 && Operands.size() % 2 == 0)) && + "Expected either a single incoming value or a greater than two and " + "even number of operands"); } /// Method to support type inquiry through isa, cast, and dyn_cast. @@ -920,6 +921,20 @@ return V->getVPRecipeID() == VPRecipeBase::VPBlendSC; } + /// Return the number of incoming values, taking into account that a single + /// incoming value has no mask. + unsigned getNumIncomingValues() const { + return (User.getNumOperands() + 1) / 2; + } + + /// Return incoming value number \p Idx. + VPValue *getIncomingValue(unsigned Idx) const { + return User.getOperand(Idx * 2); + } + + /// Return mask number \p Idx. + VPValue *getMask(unsigned Idx) const { return User.getOperand(Idx * 2 + 1); } + /// Generate the phi/select nodes. void execute(VPTransformState &State) override; diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -756,17 +756,17 @@ O << " +\n" << Indent << "\"BLEND "; Phi->printAsOperand(O, false); O << " ="; - if (!User) { + if (getNumIncomingValues() == 1) { // Not a User of any mask: not really blending, this is a // single-predecessor phi. O << " "; - Phi->getIncomingValue(0)->printAsOperand(O, false); + getIncomingValue(0)->printAsOperand(O, SlotTracker); } else { - for (unsigned I = 0, E = User->getNumOperands(); I < E; ++I) { + for (unsigned I = 0, E = getNumIncomingValues(); I < E; ++I) { O << " "; - Phi->getIncomingValue(I)->printAsOperand(O, false); + getIncomingValue(I)->printAsOperand(O, SlotTracker); O << "/"; - User->getOperand(I)->printAsOperand(O, SlotTracker); + getMask(I)->printAsOperand(O, SlotTracker); } } O << "\\l\"";