diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -503,11 +503,11 @@ unsigned UF, ElementCount VF, bool IsPtrLoopInvariant, SmallBitVector &IsIndexLoopInvariant, VPTransformState &State); - /// Vectorize a single PHINode in a block. This method handles the induction - /// variable canonicalization. It supports both VF = 1 for unrolled loops and - /// arbitrary length vectors. - void widenPHIInstruction(Instruction *PN, RecurrenceDescriptor *RdxDesc, - VPWidenPHIRecipe *PhiR, VPTransformState &State); + /// Vectorize a single first-order recurrence or pointer induction PHINode in + /// a block. This method handles the induction variable canonicalization. It + /// supports both VF = 1 for unrolled loops and arbitrary length vectors. + void widenPHIInstruction(Instruction *PN, VPWidenPHIRecipe *PhiR, + VPTransformState &State); /// A helper function to scalarize a single Instruction in the innermost loop. /// Generates a sequence of scalar instances for each lane between \p MinLane @@ -596,7 +596,7 @@ /// Fix a reduction cross-iteration phi. This is the second phase of /// vectorizing this phi node. - void fixReduction(VPWidenPHIRecipe *Phi, VPTransformState &State); + void fixReduction(VPReductionPHIRecipe *Phi, VPTransformState &State); /// Clear NSW/NUW flags from reduction instructions if necessary. void clearReductionWrapFlags(const RecurrenceDescriptor &RdxDesc, @@ -4135,8 +4135,8 @@ if (!PhiR) continue; auto *OrigPhi = cast(PhiR->getUnderlyingValue()); - if (PhiR->getRecurrenceDescriptor()) { - fixReduction(PhiR, State); + if (auto *ReductionPhi = dyn_cast(PhiR)) { + fixReduction(ReductionPhi, State); } else if (Legal->isFirstOrderRecurrence(OrigPhi)) fixFirstOrderRecurrence(PhiR, State); } @@ -4320,19 +4320,18 @@ LCSSAPhi.addIncoming(ExtractForPhiUsedOutsideLoop, LoopMiddleBlock); } -void InnerLoopVectorizer::fixReduction(VPWidenPHIRecipe *PhiR, +void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR, VPTransformState &State) { PHINode *OrigPhi = cast(PhiR->getUnderlyingValue()); // Get it's reduction variable descriptor. assert(Legal->isReductionVariable(OrigPhi) && "Unable to find the reduction variable"); - const RecurrenceDescriptor &RdxDesc = *PhiR->getRecurrenceDescriptor(); + const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor(); RecurKind RK = RdxDesc.getRecurrenceKind(); TrackingVH ReductionStartValue = RdxDesc.getRecurrenceStartValue(); Instruction *LoopExitInst = RdxDesc.getLoopExitInstr(); setDebugLocFromInst(ReductionStartValue); - bool IsInLoopReductionPhi = Cost->isInLoopReduction(OrigPhi); VPValue *LoopExitInstDef = State.Plan->getVPValue(LoopExitInst); // This is the vector-clone of the value that leaves the loop. @@ -4347,14 +4346,11 @@ // any loop invariant values. BasicBlock *VectorLoopLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch(); - bool IsOrdered = IsInLoopReductionPhi && Cost->useOrderedReductions(RdxDesc); - - for (unsigned Part = 0; Part < UF; ++Part) { - if (IsOrdered && Part > 0) - break; + unsigned LastPartForNewPhi = PhiR->isOrdered() ? 1 : UF; + for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) { Value *VecRdxPhi = State.get(PhiR->getVPSingleValue(), Part); Value *Val = State.get(PhiR->getBackedgeValue(), Part); - if (IsOrdered) + if (PhiR->isOrdered()) Val = State.get(PhiR->getBackedgeValue(), UF - 1); cast(VecRdxPhi)->addIncoming(Val, VectorLoopLatch); @@ -4373,7 +4369,7 @@ // a Select choosing between the vectorized LoopExitInst and vectorized Phi, // instead of the former. For an inloop reduction the reduction will already // be predicated, and does not need to be handled here. - if (Cost->foldTailByMasking() && !IsInLoopReductionPhi) { + if (Cost->foldTailByMasking() && !PhiR->isInLoop()) { for (unsigned Part = 0; Part < UF; ++Part) { Value *VecLoopExitInst = State.get(LoopExitInstDef, Part); Value *Sel = nullptr; @@ -4408,7 +4404,7 @@ // then extend the loop exit value to enable InstCombine to evaluate the // entire expression in the smaller type. if (VF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) { - assert(!IsInLoopReductionPhi && "Unexpected truncated inloop reduction!"); + assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!"); Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), VF); Builder.SetInsertPoint( LI->getLoopFor(LoopVectorBody)->getLoopLatch()->getTerminator()); @@ -4446,7 +4442,7 @@ // terminate on this line. This is the easiest way to ensure we don't // accidentally cause an extra step back into the loop while debugging. setDebugLocFromInst(LoopMiddleBlock->getTerminator()); - if (IsOrdered) + if (PhiR->isOrdered()) ReducedPartRdx = State.get(LoopExitInstDef, UF - 1); else { // Floating-point operations should have some FMF to enable the reduction. @@ -4465,7 +4461,7 @@ // Create the reduction after the loop. Note that inloop reductions create the // target reduction in the loop using a Reduction recipe. - if (VF.isVector() && !IsInLoopReductionPhi) { + if (VF.isVector() && !PhiR->isInLoop()) { ReducedPartRdx = createTargetReduction(Builder, TTI, RdxDesc, ReducedPartRdx); // If the reduction can be performed in a smaller type, we need to extend @@ -4729,7 +4725,6 @@ } void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN, - RecurrenceDescriptor *RdxDesc, VPWidenPHIRecipe *PhiR, VPTransformState &State) { PHINode *P = cast(PN); @@ -4755,68 +4750,21 @@ // Phi nodes have cycles, so we need to vectorize them in two stages. This is // stage #1: We create a new vector PHI node with no incoming edges. We'll use // this value when we vectorize all of the instructions that use the PHI. - if (RdxDesc || Legal->isFirstOrderRecurrence(P)) { - bool ScalarPHI = - (State.VF.isScalar()) || Cost->isInLoopReduction(cast(PN)); - Type *VecTy = - ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF); - - bool IsOrdered = Cost->isInLoopReduction(cast(PN)) && - Cost->useOrderedReductions(*RdxDesc); - unsigned LastPartForNewPhi = IsOrdered ? 1 : State.UF; - for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) { + if (Legal->isFirstOrderRecurrence(P)) { + Type *VecTy = State.VF.isScalar() + ? PN->getType() + : VectorType::get(PN->getType(), State.VF); + + for (unsigned Part = 0; Part < State.UF; ++Part) { Value *EntryPart = PHINode::Create( VecTy, 2, "vec.phi", &*LoopVectorBody->getFirstInsertionPt()); State.set(PhiR, EntryPart, Part); } - if (Legal->isFirstOrderRecurrence(P)) return; - VPValue *StartVPV = PhiR->getStartValue(); - Value *StartV = StartVPV->getLiveInIRValue(); - - Value *Iden = nullptr; - - assert(Legal->isReductionVariable(P) && StartV && - "RdxDesc should only be set for reduction variables; in that case " - "a StartV is also required"); - RecurKind RK = RdxDesc->getRecurrenceKind(); - if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) { - // MinMax reduction have the start value as their identify. - if (ScalarPHI) { - Iden = StartV; - } else { - IRBuilderBase::InsertPointGuard IPBuilder(Builder); - Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator()); - StartV = Iden = - Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident"); - } - } else { - Constant *IdenC = RecurrenceDescriptor::getRecurrenceIdentity( - RK, VecTy->getScalarType(), RdxDesc->getFastMathFlags()); - Iden = IdenC; - - if (!ScalarPHI) { - Iden = ConstantVector::getSplat(State.VF, IdenC); - IRBuilderBase::InsertPointGuard IPBuilder(Builder); - Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator()); - Constant *Zero = Builder.getInt32(0); - StartV = Builder.CreateInsertElement(Iden, StartV, Zero); - } - } - - for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) { - Value *EntryPart = State.get(PhiR, Part); - // Make sure to add the reduction start value only to the - // first unroll part. - Value *StartVal = (Part == 0) ? StartV : Iden; - cast(EntryPart)->addIncoming(StartVal, LoopVectorPreHeader); - } - - return; } assert(!Legal->isReductionVariable(P) && - "reductions should be handled above"); + "reductions should be handled elsewhere"); setDebugLocFromInst(P); @@ -8978,7 +8926,9 @@ RecurrenceDescriptor &RdxDesc = Legal->getReductionVars()[Phi]; assert(RdxDesc.getRecurrenceStartValue() == Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader())); - PhiRecipe = new VPWidenPHIRecipe(Phi, RdxDesc, *StartV); + PhiRecipe = new VPReductionPHIRecipe(Phi, RdxDesc, *StartV, + CM.isInLoopReduction(Phi), + CM.useOrderedReductions(RdxDesc)); } else { PhiRecipe = new VPWidenPHIRecipe(Phi, *StartV); } @@ -9493,8 +9443,8 @@ } void VPWidenPHIRecipe::execute(VPTransformState &State) { - State.ILV->widenPHIInstruction(cast(getUnderlyingValue()), RdxDesc, - this, State); + State.ILV->widenPHIInstruction(cast(getUnderlyingValue()), this, + State); } void VPBlendRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -306,6 +306,13 @@ /// BasicBlock, used for placing the newly created BasicBlocks. BasicBlock *LastBB = nullptr; + /// The IR BasicBlock that is the preheader of the vector loop in the output + /// IR. + /// FIXME: The vector preheader should also be modeled in VPlan, so any code + /// that needs to be added to the preheader gets directly generated by + /// VPlan. There should be no need to manage a pointer to the IR BasicBlock. + BasicBlock *VectorPreHeader = nullptr; + /// A mapping of each VPBasicBlock to the corresponding BasicBlock. In case /// of replication, maps the BasicBlock of the last replica created. SmallDenseMap VPBB2IRBB; @@ -730,8 +737,7 @@ /// Returns true for PHI-like recipes. bool isPhi() const { - return getVPDefID() == VPWidenIntOrFpInductionSC || getVPDefID() == VPWidenPHISC || - getVPDefID() == VPPredInstPHISC || getVPDefID() == VPWidenCanonicalIVSC; + return getVPDefID() >= VPFirstPHISC && getVPDefID() <= VPLastPHISC; } /// Returns true if the recipe may read from memory. @@ -1044,45 +1050,39 @@ } }; -/// A recipe for handling all phi nodes except for integer and FP inductions. -/// For reduction PHIs, RdxDesc must point to the corresponding recurrence -/// descriptor. For reductions and first-order recurrences, the start value is -/// the first operand of the recipe and the incoming value from the backedge is -/// the second operand. In the VPlan native path, all incoming VPValues & -/// VPBasicBlock pairs are managed in the recipe directly. +/// A recipe for handling first order recurrences and pointer inductions. For +/// first-order recurrences, the start value is the first operand of the recipe +/// and the incoming value from the backedge is the second operand. It also +/// serves as base class for VPReductionPHIRecipe. In the VPlan native path, all +/// incoming VPValues & VPBasicBlock pairs are managed in the recipe directly. class VPWidenPHIRecipe : public VPRecipeBase, public VPValue { - /// Descriptor for a reduction PHI. - RecurrenceDescriptor *RdxDesc = nullptr; - /// List of incoming blocks. Only used in the VPlan native path. SmallVector IncomingBlocks; +protected: + VPWidenPHIRecipe(unsigned char VPVID, unsigned char VPDefID, PHINode *Phi) + : VPRecipeBase(VPDefID, {}), VPValue(VPVID, Phi, this) {} + public: - /// Create a new VPWidenPHIRecipe for the reduction \p Phi described by \p - /// RdxDesc. - VPWidenPHIRecipe(PHINode *Phi, RecurrenceDescriptor &RdxDesc, VPValue &Start) - : VPWidenPHIRecipe(Phi) { - this->RdxDesc = &RdxDesc; - addOperand(&Start); - } + /// Create a VPWidenPHIRecipe for \p Phi + VPWidenPHIRecipe(PHINode *Phi) + : VPWidenPHIRecipe(VPVWidenPHISC, VPWidenPHISC, Phi) {} /// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start. VPWidenPHIRecipe(PHINode *Phi, VPValue &Start) : VPWidenPHIRecipe(Phi) { addOperand(&Start); } - /// Create a VPWidenPHIRecipe for \p Phi - VPWidenPHIRecipe(PHINode *Phi) - : VPRecipeBase(VPWidenPHISC, {}), - VPValue(VPValue::VPVWidenPHISC, Phi, this) {} ~VPWidenPHIRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPDef *D) { - return D->getVPDefID() == VPRecipeBase::VPWidenPHISC; + static inline bool classof(const VPRecipeBase *B) { + return B->getVPDefID() == VPRecipeBase::VPWidenPHISC || + B->getVPDefID() == VPRecipeBase::VPReductionPHISC; } static inline bool classof(const VPValue *V) { - return V->getVPValueID() == VPValue::VPVWidenPHISC; + return V->getVPValueID() == VPValue::VPVWidenPHISC || + V->getVPValueID() == VPValue::VPVReductionPHISC; } /// Generate the phi/select nodes. @@ -1117,8 +1117,60 @@ /// Returns the \p I th incoming VPBasicBlock. VPBasicBlock *getIncomingBlock(unsigned I) { return IncomingBlocks[I]; } +}; + +/// A recipe for handling reduction phis. The start value is the first operand +/// of the recipe and the incoming value from the backedge is the second +/// operand. +class VPReductionPHIRecipe : public VPWidenPHIRecipe { + /// Descriptor for the reduction. + RecurrenceDescriptor &RdxDesc; + + /// The phi is part of an in-loop reduction. + bool IsInLoop; + + /// The phi is part of an ordered reduction. Requires IsInLoop to be true. + bool IsOrdered; + +public: + /// Create a new VPReductionPHIRecipe for the reduction \p Phi described by \p + /// RdxDesc. + VPReductionPHIRecipe(PHINode *Phi, RecurrenceDescriptor &RdxDesc, + VPValue &Start, bool IsInLoop = false, + bool IsOrdered = false) + : VPWidenPHIRecipe(VPVReductionPHISC, VPReductionPHISC, Phi), + RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered) { + assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop"); + addOperand(&Start); + } - RecurrenceDescriptor *getRecurrenceDescriptor() { return RdxDesc; } + /// Method to support type inquiry through isa, cast, and dyn_cast. + static inline bool classof(const VPRecipeBase *R) { + return R->getVPDefID() == VPRecipeBase::VPReductionPHISC; + } + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPValue::VPVReductionPHISC; + } + static inline bool classof(const VPWidenPHIRecipe *R) { + return R->getVPDefID() == VPRecipeBase::VPReductionPHISC; + } + + /// Generate the phi/select nodes. + void execute(VPTransformState &State) override; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + /// Print the recipe. + void print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const override; +#endif + + RecurrenceDescriptor &getRecurrenceDescriptor() { return RdxDesc; } + + /// Returns true, if the phi is part of an ordered reduction. + bool isOrdered() const { return IsOrdered; } + + /// Returns true, if the phi is part of an in-loop reduction. + bool isInLoop() const { return IsInLoop; } }; /// A recipe for vectorizing a phi-node as a sequence of mask-based select @@ -1257,10 +1309,6 @@ return V->getVPValueID() == VPValue::VPVReductionSC; } - static inline bool classof(const VPDef *D) { - return D->getVPDefID() == VPRecipeBase::VPReductionSC; - } - /// Generate the reduction in the loop void execute(VPTransformState &State) override; diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -763,6 +763,7 @@ State->VPValue2Value[Entry.second] = Entry.first; BasicBlock *VectorPreHeaderBB = State->CFG.PrevBB; + State->CFG.VectorPreHeader = VectorPreHeaderBB; BasicBlock *VectorHeaderBB = VectorPreHeaderBB->getSingleSuccessor(); assert(VectorHeaderBB && "Loop preheader does not have a single successor."); @@ -1114,6 +1115,74 @@ printOperands(O, SlotTracker); } +void VPReductionPHIRecipe::print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const { + O << Indent << "WIDEN-REDUCTION-PHI "; + + printAsOperand(O, SlotTracker); + O << " = phi "; + printOperands(O, SlotTracker); +} + +void VPReductionPHIRecipe::execute(VPTransformState &State) { + PHINode *PN = cast(getUnderlyingValue()); + auto &Builder = State.Builder; + + // In order to support recurrences we need to be able to vectorize Phi nodes. + // Phi nodes have cycles, so we need to vectorize them in two stages. This is + // stage #1: We create a new vector PHI node with no incoming edges. We'll use + // this value when we vectorize all of the instructions that use the PHI. + bool ScalarPHI = State.VF.isScalar() || IsInLoop; + Type *VecTy = + ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF); + + BasicBlock *HeaderBB = State.CFG.PrevBB; + assert(State.LI->getLoopFor(HeaderBB)->getHeader() == HeaderBB && + "recipe must be in the vector loop header"); + unsigned LastPartForNewPhi = isOrdered() ? 1 : State.UF; + for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) { + Value *EntryPart = + PHINode::Create(VecTy, 2, "vec.phi", &*HeaderBB->getFirstInsertionPt()); + State.set(this, EntryPart, Part); + } + VPValue *StartVPV = getStartValue(); + Value *StartV = StartVPV->getLiveInIRValue(); + + Value *Iden = nullptr; + RecurKind RK = RdxDesc.getRecurrenceKind(); + if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) { + // MinMax reduction have the start value as their identify. + if (ScalarPHI) { + Iden = StartV; + } else { + IRBuilderBase::InsertPointGuard IPBuilder(Builder); + Builder.SetInsertPoint(State.CFG.VectorPreHeader->getTerminator()); + StartV = Iden = + Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident"); + } + } else { + Constant *IdenC = RecurrenceDescriptor::getRecurrenceIdentity( + RK, VecTy->getScalarType(), RdxDesc.getFastMathFlags()); + Iden = IdenC; + + if (!ScalarPHI) { + Iden = ConstantVector::getSplat(State.VF, IdenC); + IRBuilderBase::InsertPointGuard IPBuilder(Builder); + Builder.SetInsertPoint(State.CFG.VectorPreHeader->getTerminator()); + Constant *Zero = Builder.getInt32(0); + StartV = Builder.CreateInsertElement(Iden, StartV, Zero); + } + } + + for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) { + Value *EntryPart = State.get(this, Part); + // Make sure to add the reduction start value only to the + // first unroll part. + Value *StartVal = (Part == 0) ? StartV : Iden; + cast(EntryPart)->addIncoming(StartVal, State.CFG.VectorPreHeader); + } +} + void VPBlendRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { O << Indent << "BLEND "; diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -234,8 +234,7 @@ auto *UI = dyn_cast(U); if (!UI) continue; - auto *PhiR = dyn_cast(UI); - if (PhiR && !PhiR->getRecurrenceDescriptor()) + if (isa(UI) && !isa(UI)) return true; } return false; diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -90,18 +90,22 @@ /// type identification. enum { VPValueSC, - VPVBlendSC, VPVInstructionSC, VPVMemoryInstructionSC, - VPVPredInstPHI, VPVReductionSC, VPVReplicateSC, VPVWidenSC, VPVWidenCallSC, VPVWidenGEPSC, - VPVWidenIntOrFpIndcutionSC, - VPVWidenPHISC, VPVWidenSelectSC, + + // Phi-like VPValues. Need to be kept together. + VPVBlendSC, + VPVWidenPHISC, + VPVWidenCanonicalIVSC, + VPVWidenIntOrFpInductionSC, + VPVPredInstPHI, + VPVReductionPHISC, }; VPValue(Value *UV = nullptr, VPDef *Def = nullptr) @@ -314,21 +318,26 @@ /// SubclassID field of the VPRecipeBase objects. They are used for concrete /// type identification. using VPRecipeTy = enum { - VPBlendSC, VPBranchOnMaskSC, VPInstructionSC, VPInterleaveSC, - VPPredInstPHISC, VPReductionSC, VPReplicateSC, VPWidenCallSC, - VPWidenCanonicalIVSC, VPWidenGEPSC, - VPWidenIntOrFpInductionSC, VPWidenMemoryInstructionSC, - VPWidenPHISC, VPWidenSC, - VPWidenSelectSC + VPWidenSelectSC, + + // Phi-like recipes. Need to be kept together. + VPBlendSC, + VPWidenPHISC, + VPWidenCanonicalIVSC, + VPWidenIntOrFpInductionSC, + VPPredInstPHISC, + VPReductionPHISC, + VPFirstPHISC = VPBlendSC, + VPLastPHISC = VPReductionPHISC, }; VPDef(const unsigned char SC) : SubclassID(SC) {} diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll @@ -143,7 +143,7 @@ ; CHECK-NEXT: loop: ; CHECK-NEXT: WIDEN-PHI ir<%recur> = phi ir<0>, ir<%recur.next> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next -; CHECK-NEXT: WIDEN-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next> ; CHECK-NEXT: EMIT vp<%4> = icmp ule ir<%iv> vp<%0> ; CHECK-NEXT: Successor(s): loop.0 ; CHECK-EMPTY: diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll @@ -79,7 +79,7 @@ ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { ; CHECK-NEXT: for.body: ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0 -; CHECK-NEXT: WIDEN-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next> ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%y>, ir<%iv> ; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx> ; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + reduce.fadd (ir<%lv>)