Index: llvm/include/llvm/Analysis/IVDescriptors.h =================================================================== --- llvm/include/llvm/Analysis/IVDescriptors.h +++ llvm/include/llvm/Analysis/IVDescriptors.h @@ -78,12 +78,14 @@ FastMathFlags FMF, Instruction *ExactFP, Type *RT, bool Signed, bool Ordered, SmallPtrSetImpl &CI, + SmallVectorImpl &RdxChain, unsigned MinWidthCastToRecurTy) : StartValue(Start), LoopExitInstr(Exit), Kind(K), FMF(FMF), ExactFPMathInst(ExactFP), RecurrenceType(RT), IsSigned(Signed), IsOrdered(Ordered), MinWidthCastToRecurrenceType(MinWidthCastToRecurTy) { CastInsts.insert(CI.begin(), CI.end()); + ReductionChain.assign(RdxChain); } /// This POD struct holds information about a potential recurrence operation. @@ -253,6 +255,12 @@ /// recurrence. const SmallPtrSet &getCastInsts() const { return CastInsts; } + /// If found, returns the chain of operations from Phi to LoopExitInst that + /// can be treated as a set of reduction instructions for in-loop reductions. + const SmallVector &getReductionChain() const { + return ReductionChain; + } + /// Returns the minimum width used by the recurrence in bits. unsigned getMinWidthCastToRecurrenceTypeInBits() const { return MinWidthCastToRecurrenceType; @@ -266,8 +274,9 @@ /// Attempts to find a chain of operations from Phi to LoopExitInst that can /// be treated as a set of reductions instructions for in-loop reductions. - SmallVector getReductionOpChain(PHINode *Phi, - Loop *L) const; + static SmallVector getReductionOpChain(PHINode *OrigPhi, Loop *L, + Instruction *LoopExitInstr, + unsigned RedOp); /// Returns true if the instruction is a call to the llvm.fmuladd intrinsic. static bool isFMulAddIntrinsic(Instruction *I) { @@ -298,6 +307,8 @@ bool IsOrdered = false; // Instructions used for type-promoting the recurrence. SmallPtrSet CastInsts; + // The chain of operations between the reduction Phi and the exit instruction. + SmallVector ReductionChain; // The minimum width used by the recurrence. unsigned MinWidthCastToRecurrenceType; }; Index: llvm/lib/Analysis/IVDescriptors.cpp =================================================================== --- llvm/lib/Analysis/IVDescriptors.cpp +++ llvm/lib/Analysis/IVDescriptors.cpp @@ -218,8 +218,7 @@ !RecurrenceDescriptor::isFMulAddIntrinsic(Exit)) return false; - // Ensure the exit instruction has only one user other than the reduction PHI - if (Exit != ExactFPMathInst || Exit->hasNUsesOrMore(3)) + if (Exit != ExactFPMathInst) return false; // The only pattern accepted is the one in which the reduction PHI @@ -480,8 +479,14 @@ if (!FoundStartPHI || !FoundReduxOp || !ExitInstruction) return false; - const bool IsOrdered = checkOrderedReduction( - Kind, ReduxDesc.getExactFPMathInst(), ExitInstruction, Phi); + SmallVector ReductionChain; + ReductionChain = + getReductionOpChain(Phi, TheLoop, ExitInstruction, getOpcode(Kind)); + + bool IsOrdered = false; + if (!ReductionChain.empty()) + IsOrdered = checkOrderedReduction(Kind, ReduxDesc.getExactFPMathInst(), + ExitInstruction, Phi); if (Start != Phi) { // If the starting value is not the same as the phi node, we speculatively @@ -540,7 +545,7 @@ // Save the description of this reduction variable. RecurrenceDescriptor RD(RdxStart, ExitInstruction, Kind, FMF, ReduxDesc.getExactFPMathInst(), RecurrenceType, - IsSigned, IsOrdered, CastInsts, + IsSigned, IsOrdered, CastInsts, ReductionChain, MinWidthCastToRecurrenceType); RedDes = RD; @@ -1033,9 +1038,10 @@ } SmallVector -RecurrenceDescriptor::getReductionOpChain(PHINode *Phi, Loop *L) const { +RecurrenceDescriptor::getReductionOpChain(PHINode *Phi, Loop *L, + Instruction *LoopExitInstr, + unsigned RedOp) { SmallVector ReductionOperations; - unsigned RedOp = getOpcode(Kind); // Search down from the Phi to the LoopExitInstr, looking for instructions // with a single user of the correct type for the reduction. @@ -1084,7 +1090,7 @@ // check the opcode is correct (and dont allow them to be Subs) and that they // have expected to have the expected number of uses. They will have one use // from the phi and one from a LCSSA value, no matter the type. - if (!isCorrectOpcode(LoopExitInstr) || !LoopExitInstr->hasNUses(2)) + if (!isCorrectOpcode(LoopExitInstr)) return {}; // Check that the Phi has one (or two for min/max) uses. Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7737,7 +7737,7 @@ // finding the chain of operations that leads from the phi to the loop // exit value. SmallVector ReductionOperations = - RdxDesc.getReductionOpChain(Phi, TheLoop); + RdxDesc.getReductionChain(); bool InLoop = !ReductionOperations.empty(); if (InLoop) { InLoopReductionChains[Phi] = ReductionOperations;