diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1584,11 +1584,6 @@ return InLoopReductionChains; } - /// Returns true if the Phi is part of an inloop reduction. - bool isInLoopReduction(PHINode *Phi) const { - return InLoopReductionChains.count(Phi); - } - /// Estimate cost of an intrinsic call instruction CI if it were vectorized /// with factor VF. Return the cost of the instruction, including /// scalarization overhead if it's needed. @@ -4283,7 +4278,8 @@ TrackingVH ReductionStartValue = RdxDesc.getRecurrenceStartValue(); Instruction *LoopExitInst = RdxDesc.getLoopExitInstr(); setDebugLocFromInst(Builder, ReductionStartValue); - bool IsInLoopReductionPhi = Cost->isInLoopReduction(OrigPhi); + bool IsInLoopReductionPhi = PhiR->getNumOperands() == 2 && + isa(PhiR->getOperand(1)); VPValue *LoopExitInstDef = State.Plan->getVPValue(LoopExitInst); // This is the vector-clone of the value that leaves the loop. @@ -4693,7 +4689,8 @@ if (RdxDesc || Legal->isFirstOrderRecurrence(P)) { Value *Iden = nullptr; bool ScalarPHI = - (State.VF.isScalar()) || Cost->isInLoopReduction(cast(PN)); + (State.VF.isScalar()) || (PhiR->getNumOperands() == 2 && + isa(PhiR->getOperand(1))); Type *VecTy = ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF); @@ -9039,11 +9036,15 @@ if (CM.foldTailByMasking() && !Legal->getReductionVars().empty()) { Builder.setInsertPoint(VPBB); auto *Cond = RecipeBuilder.createBlockInMask(OrigLoop->getHeader(), Plan); - for (auto &Reduction : Legal->getReductionVars()) { - if (CM.isInLoopReduction(Reduction.first)) + for (VPRecipeBase &R : Plan->getEntry()->getEntryBasicBlock()->phis()) { + VPWidenPHIRecipe *PhiR = dyn_cast(&R); + if (!PhiR || !PhiR->getRecurrenceDescriptor() || + isa(PhiR->getOperand(1))) continue; - VPValue *Phi = Plan->getOrAddVPValue(Reduction.first); - VPValue *Red = Plan->getOrAddVPValue(Reduction.second.getLoopExitInstr()); + + VPValue *Phi = PhiR; + VPValue *Red = Plan->getOrAddVPValue( + PhiR->getRecurrenceDescriptor()->getLoopExitInstr()); Builder.createNaryOp(Instruction::Select, {Cond, Red, Phi}); } }