diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -512,7 +512,8 @@ /// Vectorize a single PHINode in a block. This method handles the induction /// variable canonicalization. It supports both VF = 1 for unrolled loops and /// arbitrary length vectors. - void widenPHIInstruction(Instruction *PN, unsigned UF, ElementCount VF); + void widenPHIInstruction(Instruction *PN, Value *StartV, unsigned UF, + ElementCount VF); /// A helper function to scalarize a single Instruction in the innermost loop. /// Generates a sequence of scalar instances for each lane between \p MinLane @@ -4148,8 +4149,6 @@ } void InnerLoopVectorizer::fixReduction(PHINode *Phi) { - Constant *Zero = Builder.getInt32(0); - // Get it's reduction variable descriptor. assert(Legal->isReductionVariable(Phi) && "Unable to find the reduction variable"); @@ -4170,37 +4169,6 @@ // This is the vector-clone of the value that leaves the loop. Type *VecTy = getOrCreateVectorValue(LoopExitInst, 0)->getType(); - // Find the reduction identity variable. Zero for addition, or, xor, - // one for multiplication, -1 for And. - Value *Identity; - Value *VectorStart; - if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) { - // MinMax reduction have the start value as their identify. - if (VF.isScalar() || IsInLoopReductionPhi) { - VectorStart = Identity = ReductionStartValue; - } else { - VectorStart = Identity = - Builder.CreateVectorSplat(VF, ReductionStartValue, "minmax.ident"); - } - } else { - // Handle other reduction kinds: - Constant *Iden = RecurrenceDescriptor::getRecurrenceIdentity( - RK, VecTy->getScalarType()); - if (VF.isScalar() || IsInLoopReductionPhi) { - Identity = Iden; - // This vector is the Identity vector where the first element is the - // incoming scalar reduction. - VectorStart = ReductionStartValue; - } else { - Identity = ConstantVector::getSplat(VF, Iden); - - // This vector is the Identity vector where the first element is the - // incoming scalar reduction. - VectorStart = - Builder.CreateInsertElement(Identity, ReductionStartValue, Zero); - } - } - // Wrap flags are in general invalid after vectorization, clear them. clearReductionWrapFlags(RdxDesc); @@ -4214,10 +4182,6 @@ for (unsigned Part = 0; Part < UF; ++Part) { Value *VecRdxPhi = getOrCreateVectorValue(Phi, Part); Value *Val = getOrCreateVectorValue(LoopVal, Part); - // Make sure to add the reduction start value only to the - // first unroll part. - Value *StartVal = (Part == 0) ? VectorStart : Identity; - cast(VecRdxPhi)->addIncoming(StartVal, LoopVectorPreHeader); cast(VecRdxPhi) ->addIncoming(Val, LI->getLoopFor(LoopVectorBody)->getLoopLatch()); } @@ -4592,8 +4556,8 @@ } } -void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN, unsigned UF, - ElementCount VF) { +void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN, Value *StartV, + unsigned UF, ElementCount VF) { assert(!VF.isScalable() && "scalable vectors not yet supported."); PHINode *P = cast(PN); if (EnableVPlanNativePath) { @@ -4618,15 +4582,50 @@ // stage #1: We create a new vector PHI node with no incoming edges. We'll use // this value when we vectorize all of the instructions that use the PHI. if (Legal->isReductionVariable(P) || Legal->isFirstOrderRecurrence(P)) { + Value *Iden = nullptr; + bool ScalarPHI = + (VF.isScalar()) || Cost->isInLoopReduction(cast(PN)); + Type *VecTy = + ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), VF); + + if (Legal->isReductionVariable(P)) { + RecurrenceDescriptor RdxDesc = Legal->getReductionVars()[P]; + RecurKind RK = RdxDesc.getRecurrenceKind(); + if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) { + // MinMax reduction have the start value as their identify. + if (ScalarPHI) { + StartV = Iden = StartV; + } else { + IRBuilderBase::InsertPointGuard IPBuilder(Builder); + Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator()); + StartV = Iden = Builder.CreateVectorSplat(VF, StartV, "minmax.ident"); + } + } else { + Constant *IdenC = RecurrenceDescriptor::getRecurrenceIdentity( + RK, VecTy->getScalarType()); + Iden = IdenC; + + if (!ScalarPHI) { + Iden = ConstantVector::getSplat(VF, IdenC); + IRBuilderBase::InsertPointGuard IPBuilder(Builder); + Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator()); + Constant *Zero = Builder.getInt32(0); + StartV = Builder.CreateInsertElement(Iden, StartV, Zero); + } + } + } + for (unsigned Part = 0; Part < UF; ++Part) { // This is phase one of vectorizing PHIs. - bool ScalarPHI = - (VF.isScalar()) || Cost->isInLoopReduction(cast(PN)); - Type *VecTy = - ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), VF); Value *EntryPart = PHINode::Create( VecTy, 2, "vec.phi", &*LoopVectorBody->getFirstInsertionPt()); VectorLoopValueMap.setVectorValue(P, Part, EntryPart); + if (StartV) { + // Make sure to add the reduction start value only to the + // first unroll part. + Value *StartVal = (Part == 0) ? StartV : Iden; + cast(EntryPart)->addIncoming(StartVal, LoopVectorPreHeader); + } } return; } @@ -8316,7 +8315,14 @@ return tryToBlend(Phi, Plan); if ((Recipe = tryToOptimizeInductionPHI(Phi, *Plan))) return Recipe; - return new VPWidenPHIRecipe(Phi); + + VPValue *StartV = nullptr; + if (Legal->isReductionVariable(Phi)) { + RecurrenceDescriptor RdxDesc = Legal->getReductionVars()[Phi]; + StartV = Plan->getOrAddVPValue(RdxDesc.getRecurrenceStartValue()); + } + + return new VPWidenPHIRecipe(Phi, StartV); } if (isa(Instr) && (Recipe = tryToOptimizeInductionTruncate( @@ -8719,7 +8725,9 @@ } void VPWidenPHIRecipe::execute(VPTransformState &State) { - State.ILV->widenPHIInstruction(Phi, State.UF, State.VF); + Value *StartV = + getStartValue() ? getStartValue()->getLiveInIRValue() : nullptr; + State.ILV->widenPHIInstruction(Phi, StartV, State.UF, State.VF); } void VPBlendRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -947,11 +947,18 @@ }; /// A recipe for handling all phi nodes except for integer and FP inductions. -class VPWidenPHIRecipe : public VPRecipeBase { +/// Reduction and first-order recurrence phis have the start value as first +/// operand. +class VPWidenPHIRecipe : public VPRecipeBase, public VPUser { PHINode *Phi; public: - VPWidenPHIRecipe(PHINode *Phi) : VPRecipeBase(VPWidenPHISC), Phi(Phi) { + /// Create a VPWidenPHIRecipe for \p Phi. For reduction or first-order + /// recurrence phis, the initial value must be provided as \p Start. + VPWidenPHIRecipe(PHINode *Phi, VPValue *Start) + : VPRecipeBase(VPWidenPHISC), VPUser(), Phi(Phi) { + if (Start) + addOperand(Start); new VPValue(Phi, this); } ~VPWidenPHIRecipe() override = default; @@ -967,6 +974,10 @@ /// Print the recipe. void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override; + + VPValue *getStartValue() { + return getNumOperands() == 0 ? nullptr : getOperand(0); + } }; /// A recipe for vectorizing a phi-node as a sequence of mask-based select diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -72,7 +72,7 @@ VPValue *Start = Plan->getOrAddVPValue(II.getStartValue()); NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start); } else - NewRecipe = new VPWidenPHIRecipe(Phi); + NewRecipe = new VPWidenPHIRecipe(Phi, nullptr); } else if (GetElementPtrInst *GEP = dyn_cast(Inst)) { NewRecipe = new VPWidenGEPRecipe( GEP, Plan->mapToVPValues(GEP->operands()), OrigLoop);