diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -13982,7 +13982,7 @@ /// r *= v1 + v2 + v3 + v4 /// In such a case start looking for a tree rooted in the first '+'. /// \Returns the new root if found, which may be nullptr if not an instruction. -static Instruction *tryGetScondaryReductionRoot(PHINode *Phi, +static Instruction *tryGetSecondaryReductionRoot(PHINode *Phi, Instruction *Root) { assert((isa(Root) || isa(Root) || isa(Root)) && @@ -13998,16 +13998,49 @@ return nullptr; } +/// \p Returns the first operand of \p I that does not match \p Phi. If +/// operand is not an instruction it returns nullptr. +static Instruction *getNonPhiOperand(Instruction *I, PHINode *Phi) { + Value *Op0 = nullptr; + Value *Op1 = nullptr; + matchRdxBop(I, Op0, Op1); + assert(Op0 && Op1 && "Expected Binop"); + Instruction *Op = dyn_cast(Op0); + if (Op == Phi) + Op = dyn_cast(Op1); + return Op; +} + +/// \Returns true if \p I is a candidate instruction for reduction vectorization. +static bool isReductionCandidate(Instruction *I) { + bool IsSelect = match(I, m_Select(m_Value(), m_Value(), m_Value())); + Value *B0 = nullptr, *B1 = nullptr; + bool IsBinop = matchRdxBop(I, B0, B1); + return IsBinop || IsSelect; +} + bool SLPVectorizerPass::vectorizeHorReduction( PHINode *P, Instruction *Root, BasicBlock *BB, BoUpSLP &R, TargetTransformInfo *TTI, SmallVectorImpl &PostponedInsts) { if (!ShouldVectorizeHor) return false; + bool TryOperandsAsNewSeeds = P != nullptr; if (!isa(Root)) - P = nullptr; + TryOperandsAsNewSeeds = false; if (Root->getParent() != BB || isa(Root)) return false; + + // If we can find a secondary reduction root, use that instead. + Instruction *NewRoot = nullptr; + if (isReductionCandidate(Root)) { + assert((!TryOperandsAsNewSeeds || is_contained(P->operands(), Root)) && + "Phi needs to use the binary operator"); + if (TryOperandsAsNewSeeds && + HorizontalReduction::getRdxKind(Root) != RecurKind::None) + NewRoot = tryGetSecondaryReductionRoot(P, Root); + } + // Start analysis starting from Root instruction. If horizontal reduction is // found, try to vectorize it. If it is not a horizontal reduction or // vectorization is not possible or not effective, and currently analyzed @@ -14020,69 +14053,45 @@ // If a horizintal reduction was not matched or vectorized we collect // instructions for possible later attempts for vectorization. std::queue> Stack; - Stack.emplace(Root, 0); + Stack.emplace(NewRoot ? NewRoot : Root, 0); SmallPtrSet VisitedInstrs; bool Res = false; - auto &&TryToReduce = [this, TTI, &P, &R](Instruction *Inst, Value *&B0, - Value *&B1) -> Value * { - if (R.isAnalyzedReductionRoot(Inst)) - return nullptr; - bool IsBinop = matchRdxBop(Inst, B0, B1); - bool IsSelect = match(Inst, m_Select(m_Value(), m_Value(), m_Value())); - if (IsBinop || IsSelect) { - assert((!P || is_contained(P->operands(), Inst)) && - "Phi needs to use the binary operator"); - if (P && HorizontalReduction::getRdxKind(Inst) != RecurKind::None) - if (Instruction *NewRoot = tryGetScondaryReductionRoot(P, Inst)) - Inst = NewRoot; - - HorizontalReduction HorRdx; - if (HorRdx.matchAssociativeReduction(Inst, *SE, *DL, *TLI)) - return HorRdx.tryToReduce(R, TTI, *TLI); - } - return nullptr; - }; while (!Stack.empty()) { Instruction *Inst; unsigned Level; std::tie(Inst, Level) = Stack.front(); Stack.pop(); + // Do not try to analyze instruction that has already been vectorized. // This may happen when we vectorize instruction operands on a previous // iteration while stack was populated before that happened. if (R.isDeleted(Inst)) continue; - Value *B0 = nullptr, *B1 = nullptr; - if (Value *V = TryToReduce(Inst, B0, B1)) { + Value *VectorizedV = nullptr; + if (!R.isAnalyzedReductionRoot(Inst) && isReductionCandidate(Inst)) { + HorizontalReduction HorRdx; + if (HorRdx.matchAssociativeReduction(Inst, *SE, *DL, *TLI)) + VectorizedV = HorRdx.tryToReduce(R, TTI, *TLI); + } + if (VectorizedV != nullptr) { Res = true; - // Set P to nullptr to avoid re-analysis of phi node in - // matchAssociativeReduction function unless this is the root node. - P = nullptr; - if (auto *I = dyn_cast(V)) { + if (auto *I = dyn_cast(VectorizedV)) { // Try to find another reduction. Stack.emplace(I, Level); continue; } } else { - bool IsBinop = B0 && B1; - if (P && IsBinop) { - Inst = dyn_cast(B0); - if (Inst == P) - Inst = dyn_cast(B1); - if (!Inst) { - // Set P to nullptr to avoid re-analysis of phi node in - // matchAssociativeReduction function unless this is the root node. - P = nullptr; - continue; - } + // We could not vectorize `Inst` so try to use it as a future seed. + Instruction *FutureSeed = Inst; + if (TryOperandsAsNewSeeds && Inst == Root) { + FutureSeed = getNonPhiOperand(Root, P); + if (!FutureSeed) + break; } - // Set P to nullptr to avoid re-analysis of phi node in - // matchAssociativeReduction function unless this is the root node. - P = nullptr; // Do not collect CmpInst or InsertElementInst/InsertValueInst as their // analysis is done separately. - if (!isa(Inst)) - PostponedInsts.push_back(Inst); + if (!isa(FutureSeed)) + PostponedInsts.push_back(FutureSeed); } // Try to vectorize operands.