diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4166,14 +4166,10 @@ // the incoming edges. VPBasicBlock *Header = State.Plan->getEntry()->getEntryBasicBlock(); for (VPRecipeBase &R : Header->phis()) { - auto *PhiR = dyn_cast(&R); - if (!PhiR) - continue; - auto *OrigPhi = cast(PhiR->getUnderlyingValue()); - if (auto *ReductionPhi = dyn_cast(PhiR)) { + if (auto *ReductionPhi = dyn_cast(&R)) fixReduction(ReductionPhi, State); - } else if (Legal->isFirstOrderRecurrence(OrigPhi)) - fixFirstOrderRecurrence(PhiR, State); + else if (auto *FOR = dyn_cast(&R)) + fixFirstOrderRecurrence(FOR, State); } } @@ -4202,7 +4198,7 @@ // // In this example, s1 is a recurrence because it's value depends on the // previous iteration. In the first phase of vectorization, we created a - // temporary value for s1. We now complete the vectorization and produce the + // vector phi v1 for s1. We now complete the vectorization and produce the // shorthand vector IR shown below (for VF = 4, UF = 1). // // vector.ph: @@ -4228,82 +4224,19 @@ // After execution completes the vector loop, we extract the next value of // the recurrence (x) to use as the initial value in the scalar loop. - auto *ScalarInit = PhiR->getStartValue()->getLiveInIRValue(); - auto *IdxTy = Builder.getInt32Ty(); - auto *One = ConstantInt::get(IdxTy, 1); - - // Create a vector from the initial value. - auto *VectorInit = ScalarInit; - if (VF.isVector()) { - Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator()); - auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, VF); - auto *LastIdx = Builder.CreateSub(RuntimeVF, One); - VectorInit = Builder.CreateInsertElement( - PoisonValue::get(VectorType::get(VectorInit->getType(), VF)), - VectorInit, LastIdx, "vector.recur.init"); - } - - VPValue *PreviousDef = PhiR->getBackedgeValue(); - // We constructed a temporary phi node in the first phase of vectorization. - // This phi node will eventually be deleted. - Builder.SetInsertPoint(cast(State.get(PhiR, 0))); - - // Create a phi node for the new recurrence. The current value will either be - // the initial value inserted into a vector or loop-varying vector value. - auto *VecPhi = Builder.CreatePHI(VectorInit->getType(), 2, "vector.recur"); - VecPhi->addIncoming(VectorInit, LoopVectorPreHeader); - - // Get the vectorized previous value of the last part UF - 1. It appears last - // among all unrolled iterations, due to the order of their construction. - Value *PreviousLastPart = State.get(PreviousDef, UF - 1); - - // Find and set the insertion point after the previous value if it is an - // instruction. - BasicBlock::iterator InsertPt; - // Note that the previous value may have been constant-folded so it is not - // guaranteed to be an instruction in the vector loop. - // FIXME: Loop invariant values do not form recurrences. We should deal with - // them earlier. - if (LI->getLoopFor(LoopVectorBody)->isLoopInvariant(PreviousLastPart)) - InsertPt = LoopVectorBody->getFirstInsertionPt(); - else { - Instruction *PreviousInst = cast(PreviousLastPart); - if (isa(PreviousLastPart)) - // If the previous value is a phi node, we should insert after all the phi - // nodes in the block containing the PHI to avoid breaking basic block - // verification. Note that the basic block may be different to - // LoopVectorBody, in case we predicate the loop. - InsertPt = PreviousInst->getParent()->getFirstInsertionPt(); - else - InsertPt = ++PreviousInst->getIterator(); - } - Builder.SetInsertPoint(&*InsertPt); - - // The vector from which to take the initial value for the current iteration - // (actual or unrolled). Initially, this is the vector phi node. - Value *Incoming = VecPhi; - - // Shuffle the current and previous vector and update the vector parts. - for (unsigned Part = 0; Part < UF; ++Part) { - Value *PreviousPart = State.get(PreviousDef, Part); - Value *PhiPart = State.get(PhiR, Part); - auto *Shuffle = VF.isVector() - ? Builder.CreateVectorSplice(Incoming, PreviousPart, -1) - : Incoming; - PhiPart->replaceAllUsesWith(Shuffle); - cast(PhiPart)->eraseFromParent(); - State.reset(PhiR, Shuffle, Part); - Incoming = PreviousPart; - } + auto *VecPhi = cast(State.get(PhiR, 0)); // Fix the latch value of the new recurrence in the vector loop. + VPValue *PreviousDef = PhiR->getBackedgeValue(); + Value *Incoming = State.get(PreviousDef, UF - 1); VecPhi->addIncoming(Incoming, LI->getLoopFor(LoopVectorBody)->getLoopLatch()); // Extract the last vector element in the middle block. This will be the // initial value for the recurrence when jumping to the scalar loop. auto *ExtractForScalar = Incoming; if (VF.isVector()) { + auto *One = ConstantInt::get(IdxTy, 1); Builder.SetInsertPoint(LoopMiddleBlock->getTerminator()); auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, VF); auto *LastIdx = Builder.CreateSub(RuntimeVF, One); @@ -4332,6 +4265,7 @@ Builder.SetInsertPoint(&*LoopScalarPreHeader->begin()); PHINode *Phi = cast(PhiR->getUnderlyingValue()); auto *Start = Builder.CreatePHI(Phi->getType(), 2, "scalar.recur.init"); + auto *ScalarInit = PhiR->getStartValue()->getLiveInIRValue(); for (auto *BB : predecessors(LoopScalarPreHeader)) { auto *Incoming = BB == LoopMiddleBlock ? ExtractForScalar : ScalarInit; Start->addIncoming(Incoming, BB); @@ -4787,18 +4721,6 @@ // Phi nodes have cycles, so we need to vectorize them in two stages. This is // stage #1: We create a new vector PHI node with no incoming edges. We'll use // this value when we vectorize all of the instructions that use the PHI. - if (Legal->isFirstOrderRecurrence(P)) { - Type *VecTy = State.VF.isScalar() - ? PN->getType() - : VectorType::get(PN->getType(), State.VF); - - for (unsigned Part = 0; Part < State.UF; ++Part) { - Value *EntryPart = PHINode::Create( - VecTy, 2, "vec.phi", &*LoopVectorBody->getFirstInsertionPt()); - State.set(PhiR, EntryPart, Part); - } - return; - } assert(!Legal->isReductionVariable(P) && "reductions should be handled elsewhere"); @@ -9073,7 +8995,7 @@ CM.isInLoopReduction(Phi), CM.useOrderedReductions(RdxDesc)); } else { - PhiRecipe = new VPWidenPHIRecipe(Phi, *StartV); + PhiRecipe = new VPFirstOrderRecurrencePHIRecipe(Phi, *StartV); } // Record the incoming value from the backedge, so we can add the incoming @@ -9314,23 +9236,22 @@ // --------------------------------------------------------------------------- // Apply Sink-After legal constraints. + auto GetReplicateRegion = [](VPRecipeBase *R) -> VPRegionBlock * { + auto *Region = dyn_cast_or_null(R->getParent()->getParent()); + if (Region && Region->isReplicator()) { + assert(Region->getNumSuccessors() == 1 && + Region->getNumPredecessors() == 1 && "Expected SESE region!"); + assert(R->getParent()->size() == 1 && + "A recipe in an original replicator region must be the only " + "recipe in its block"); + return Region; + } + return nullptr; + }; for (auto &Entry : SinkAfter) { VPRecipeBase *Sink = RecipeBuilder.getRecipe(Entry.first); VPRecipeBase *Target = RecipeBuilder.getRecipe(Entry.second); - auto GetReplicateRegion = [](VPRecipeBase *R) -> VPRegionBlock * { - auto *Region = - dyn_cast_or_null(R->getParent()->getParent()); - if (Region && Region->isReplicator()) { - assert(Region->getNumSuccessors() == 1 && - Region->getNumPredecessors() == 1 && "Expected SESE region!"); - assert(R->getParent()->size() == 1 && - "A recipe in an original replicator region must be the only " - "recipe in its block"); - return Region; - } - return nullptr; - }; auto *TargetRegion = GetReplicateRegion(Target); auto *SinkRegion = GetReplicateRegion(Sink); if (!SinkRegion) { @@ -9362,8 +9283,8 @@ VPBlockUtils::connectBlocks(SinkRegion, TargetSucc); } else { // The sink source is in a replicate region, we need to move the whole - // replicate region, which should only contain a single recipe in the main - // block. + // replicate region, which should only contain a single recipe in the + // main block. auto *SplitBlock = Target->getParent()->splitAt(std::next(Target->getIterator())); @@ -9377,6 +9298,29 @@ } } + // Introduce a recipe to combine the incoming and previous values of a + // first-order recurrence. + for (VPRecipeBase &R : Plan->getEntry()->getEntryBasicBlock()->phis()) { + auto *RecurPhi = dyn_cast(&R); + if (!RecurPhi) + continue; + + auto *RecurSplice = cast( + Builder.createNaryOp(VPInstruction::FirstOrderRecurrenceSplice, + {RecurPhi, RecurPhi->getBackedgeValue()})); + + VPRecipeBase *PrevRecipe = RecurPhi->getBackedgeRecipe(); + if (auto *Region = GetReplicateRegion(PrevRecipe)) { + VPBasicBlock *Succ = cast(Region->getSingleSuccessor()); + RecurSplice->moveBefore(*Succ, Succ->getFirstNonPhi()); + } else + RecurSplice->moveAfter(PrevRecipe); + RecurPhi->replaceAllUsesWith(RecurSplice); + // Set the first operand of RecurSplice to RecurPhi again, after replacing + // all users. + RecurSplice->setOperand(0, RecurPhi); + } + // Interleave memory: for each Interleave Group we marked earlier as relevant // for this VPlan, replace the Recipes widening its memory instructions with a // single VPInterleaveRecipe at its insertion point. diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -776,7 +776,10 @@ public: /// VPlan opcodes, extending LLVM IR with idiomatics instructions. enum { - Not = Instruction::OtherOpsEnd + 1, + FirstOrderRecurrenceSplice = + Instruction::OtherOpsEnd + 1, // Combines the incoming and previous + // values of a first-order recurrence. + Not, ICmpULE, SLPLoad, SLPStore, @@ -1060,8 +1063,12 @@ SmallVector IncomingBlocks; protected: - VPWidenPHIRecipe(unsigned char VPVID, unsigned char VPDefID, PHINode *Phi) - : VPRecipeBase(VPDefID, {}), VPValue(VPVID, Phi, this) {} + VPWidenPHIRecipe(unsigned char VPVID, unsigned char VPDefID, PHINode *Phi, + VPValue *Start = nullptr) + : VPRecipeBase(VPDefID, {}), VPValue(VPVID, Phi, this) { + if (Start) + addOperand(Start); + } public: /// Create a VPWidenPHIRecipe for \p Phi @@ -1078,10 +1085,12 @@ /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPRecipeBase *B) { return B->getVPDefID() == VPRecipeBase::VPWidenPHISC || + B->getVPDefID() == VPRecipeBase::VPFirstOrderRecurrencePHISC || B->getVPDefID() == VPRecipeBase::VPReductionPHISC; } static inline bool classof(const VPValue *V) { return V->getVPValueID() == VPValue::VPVWidenPHISC || + V->getVPValueID() == VPValue::VPVFirstOrderRecurrencePHISC || V->getVPValueID() == VPValue::VPVReductionPHISC; } @@ -1106,6 +1115,11 @@ return getOperand(1); } + /// Returns the backedge value as a recipe. The backedge value is guaranteed to be a recipe. + VPRecipeBase *getBackedgeRecipe() { + return cast(getBackedgeValue()->getDef()); + } + /// Adds a pair (\p IncomingV, \p IncomingBlock) to the phi. void addIncoming(VPValue *IncomingV, VPBasicBlock *IncomingBlock) { addOperand(IncomingV); @@ -1119,6 +1133,34 @@ VPBasicBlock *getIncomingBlock(unsigned I) { return IncomingBlocks[I]; } }; +/// A recipe for handling first-order recurrence phis. The start value is the +/// first operand of the recipe and the incoming value from the backedge is the +/// second operand. +struct VPFirstOrderRecurrencePHIRecipe : public VPWidenPHIRecipe { + VPFirstOrderRecurrencePHIRecipe(PHINode *Phi, VPValue &Start) + : VPWidenPHIRecipe(VPVFirstOrderRecurrencePHISC, + VPFirstOrderRecurrencePHISC, Phi, &Start) {} + + /// Method to support type inquiry through isa, cast, and dyn_cast. + static inline bool classof(const VPRecipeBase *R) { + return R->getVPDefID() == VPRecipeBase::VPFirstOrderRecurrencePHISC; + } + static inline bool classof(const VPWidenPHIRecipe *D) { + return D->getVPDefID() == VPRecipeBase::VPFirstOrderRecurrencePHISC; + } + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPValue::VPVFirstOrderRecurrencePHISC; + } + + void execute(VPTransformState &State) override; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + /// Print the recipe. + void print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const override; +#endif +}; + /// A recipe for handling reduction phis. The start value is the first operand /// of the recipe and the incoming value from the backedge is the second /// operand. @@ -1138,10 +1180,9 @@ VPReductionPHIRecipe(PHINode *Phi, RecurrenceDescriptor &RdxDesc, VPValue &Start, bool IsInLoop = false, bool IsOrdered = false) - : VPWidenPHIRecipe(VPVReductionPHISC, VPReductionPHISC, Phi), + : VPWidenPHIRecipe(VPVReductionPHISC, VPReductionPHISC, Phi, &Start), RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered) { assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop"); - addOperand(&Start); } ~VPReductionPHIRecipe() override = default; diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -687,6 +687,30 @@ State.set(this, Call, Part); break; } + case VPInstruction::FirstOrderRecurrenceSplice: { + // Generate code to combine the previous and current values in vector v3. + // + // vector.ph: + // v_init = vector(..., ..., ..., a[-1]) + // br vector.body + // + // vector.body + // i = phi [0, vector.ph], [i+4, vector.body] + // v1 = phi [v_init, vector.ph], [v2, vector.body] + // v2 = a[i, i+1, i+2, i+3]; + // v3 = vector(v1(3), v2(0, 1, 2)) + + // For the first part, use the recurrence phi (v1), otherwise v2. + auto *V1 = State.get(getOperand(0), 0); + Value *PartMinus1 = Part == 0 ? V1 : State.get(getOperand(1), Part - 1); + if (!PartMinus1->getType()->isVectorTy()) { + State.set(this, PartMinus1, Part); + } else { + Value *V2 = State.get(getOperand(1), Part); + State.set(this, Builder.CreateVectorSplice(PartMinus1, V2, -1), Part); + } + break; + } default: llvm_unreachable("Unsupported opcode for instruction"); } @@ -729,7 +753,9 @@ case VPInstruction::ActiveLaneMask: O << "active lane mask"; break; - + case VPInstruction::FirstOrderRecurrenceSplice: + O << "first-order splice"; + break; default: O << Instruction::getOpcodeName(getOpcode()); } @@ -1222,6 +1248,43 @@ } #endif +void VPFirstOrderRecurrencePHIRecipe::execute(VPTransformState &State) { + auto &Builder = State.Builder; + // Create a vector from the initial value. + auto *VectorInit = getStartValue()->getLiveInIRValue(); + + Type *VecTy = State.VF.isScalar() + ? VectorInit->getType() + : VectorType::get(VectorInit->getType(), State.VF); + + if (State.VF.isVector()) { + auto *IdxTy = Builder.getInt32Ty(); + auto *One = ConstantInt::get(IdxTy, 1); + IRBuilder<>::InsertPointGuard Guard(Builder); + Builder.SetInsertPoint(State.CFG.VectorPreHeader->getTerminator()); + auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, State.VF); + auto *LastIdx = Builder.CreateSub(RuntimeVF, One); + VectorInit = Builder.CreateInsertElement( + PoisonValue::get(VecTy), VectorInit, LastIdx, "vector.recur.init"); + } + + // Create a phi node for the new recurrence. + PHINode *EntryPart = PHINode::Create( + VecTy, 2, "vector.recur", &*State.CFG.PrevBB->getFirstInsertionPt()); + EntryPart->addIncoming(VectorInit, State.CFG.VectorPreHeader); + State.set(this, EntryPart, 0); +} + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +void VPFirstOrderRecurrencePHIRecipe::print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const { + O << Indent << "FIRST-ORDER-RECURRENCE-PHI "; + printAsOperand(O, SlotTracker); + O << " = phi "; + printOperands(O, SlotTracker); +} +#endif + void VPReductionPHIRecipe::execute(VPTransformState &State) { PHINode *PN = cast(getUnderlyingValue()); auto &Builder = State.Builder; diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -221,27 +221,6 @@ // region. Such dependencies should be rejected during earlier dependence // checks, which guarantee accesses can be re-ordered for vectorization. // - // If a recipe is used by a first-order recurrence phi, we cannot move it at - // the moment: a recipe R feeding a first order recurrence phi must allow - // for a *vector* shuffle to be inserted immediately after it, and therefore - // if R is *scalarized and predicated* it must appear last in its basic - // block. In addition, other recipes may need to "sink after" R, so best if - // R not be moved at all. - auto IsImmovableRecipe = [](VPRecipeBase &R) { - assert(R.getNumDefinedValues() <= 1 && - "no multi-defs are expected in predicated blocks"); - for (VPUser *U : R.getVPSingleValue()->users()) { - auto *UI = dyn_cast(U); - if (!UI) - continue; - if (isa(UI) && !isa(UI)) - return true; - } - return false; - }; - if (any_of(*Then1, IsImmovableRecipe)) - continue; - // Move recipes to the successor region. for (VPRecipeBase &ToMove : make_early_inc_range(reverse(*Then1))) ToMove.moveBefore(*Then2, Then2->getFirstNonPhi()); diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -101,6 +101,7 @@ // Phi-like VPValues. Need to be kept together. VPVBlendSC, + VPVFirstOrderRecurrencePHISC, VPVWidenPHISC, VPVWidenCanonicalIVSC, VPVWidenIntOrFpInductionSC, @@ -331,6 +332,7 @@ // Phi-like recipes. Need to be kept together. VPBlendSC, + VPFirstOrderRecurrencePHISC, VPWidenPHISC, VPWidenCanonicalIVSC, VPWidenIntOrFpInductionSC, diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll @@ -16,9 +16,9 @@ ; CHECK-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[IDX_PHI_TRANS]], align 4 ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[DOTPRE]], i32 3 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i32 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer -; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[DOTPRE]], i32 3 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -100,9 +100,9 @@ ; CHECK-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[IDX_PHI_TRANS]], align 4 ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[DOTPRE]], i32 3 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i32 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer -; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[DOTPRE]], i32 3 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -183,9 +183,9 @@ ; CHECK-NEXT: [[DOTPRE:%.*]] = load i32, i32* [[IDX_PHI_TRANS]], align 4 ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[DOTPRE]], i32 3 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i32 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer -; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[DOTPRE]], i32 3 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -502,7 +502,7 @@ ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], ; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 @@ -523,7 +523,7 @@ ; CHECK-NEXT: store i16 [[SCALAR_RECUR]], i16* [[B]], align 4 ; CHECK-NEXT: [[INC]] = add nsw i32 [[I]], 1 ; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i32 [[I]], 2096 -; CHECK-NEXT: br i1 [[CMP2]], label [[FOR_COND]], label [[IF_END]], !llvm.loop [[LOOP7:![0-9]+]] +; CHECK-NEXT: br i1 [[CMP2]], label [[FOR_COND]], label [[IF_END]], !llvm.loop [[LOOP9:![0-9]+]] ; CHECK: if.end: ; CHECK-NEXT: [[REC_LCSSA:%.*]] = phi i16 [ [[SCALAR_RECUR]], [[FOR_BODY]] ], [ [[SCALAR_RECUR]], [[FOR_COND]] ] ; CHECK-NEXT: ret i16 [[REC_LCSSA]] @@ -586,7 +586,7 @@ ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], ; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2 @@ -607,7 +607,7 @@ ; CHECK-NEXT: store i16 [[SCALAR_RECUR]], i16* [[B]], align 4 ; CHECK-NEXT: [[INC]] = add nsw i32 [[I]], 1 ; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i32 [[I]], 2096 -; CHECK-NEXT: br i1 [[CMP2]], label [[FOR_COND]], label [[IF_END]], !llvm.loop [[LOOP9:![0-9]+]] +; CHECK-NEXT: br i1 [[CMP2]], label [[FOR_COND]], label [[IF_END]], !llvm.loop [[LOOP11:![0-9]+]] ; CHECK: if.end: ; CHECK-NEXT: [[REC_LCSSA:%.*]] = phi i16 [ [[SCALAR_RECUR]], [[FOR_COND]] ], [ 10, [[FOR_BODY]] ] ; CHECK-NEXT: ret i16 [[REC_LCSSA]] diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll @@ -10,7 +10,7 @@ ; CHECK-LABEL: sink_replicate_region_1 ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { ; CHECK-NEXT: loop: -; CHECK-NEXT: WIDEN-PHI ir<%0> = phi ir<0>, ir<%conv> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next ; CHECK-NEXT: EMIT vp<%3> = icmp ule ir<%iv> vp<%0> ; CHECK-NEXT: Successor(s): loop.0 @@ -37,6 +37,7 @@ ; CHECK-EMPTY: ; CHECK-NEXT: loop.1: ; CHECK-NEXT: WIDEN ir<%conv> = sext vp<%6> +; CHECK-NEXT: EMIT vp<%8> = first-order splice ir<%0> ir<%conv> ; CHECK-NEXT: Successor(s): pred.srem ; CHECK-EMPTY: ; CHECK-NEXT: pred.srem: { @@ -46,17 +47,17 @@ ; CHECK-NEXT: CondBit: vp<%3> (loop) ; CHECK-EMPTY: ; CHECK-NEXT: pred.srem.if: -; CHECK-NEXT: REPLICATE ir<%rem> = srem ir<%0>, ir<%x> (S->V) +; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<%8>, ir<%x> (S->V) ; CHECK-NEXT: Successor(s): pred.srem.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.srem.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%9> = ir<%rem> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%10> = ir<%rem> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.1.split ; CHECK-EMPTY: ; CHECK-NEXT: loop.1.split: -; CHECK-NEXT: WIDEN ir<%add> = add ir<%conv>, vp<%9> +; CHECK-NEXT: WIDEN ir<%add> = add ir<%conv>, vp<%10> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -83,13 +84,14 @@ ; CHECK-LABEL: sink_replicate_region_2 ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { ; CHECK-NEXT: loop: -; CHECK-NEXT: WIDEN-PHI ir<%recur> = phi ir<0>, ir<%recur.next> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next ; CHECK-NEXT: EMIT vp<%3> = icmp ule ir<%iv> vp<%0> ; CHECK-NEXT: Successor(s): loop.0 ; CHECK-EMPTY: ; CHECK-NEXT: loop.0: ; CHECK-NEXT: WIDEN ir<%recur.next> = sext ir<%y> +; CHECK-NEXT: EMIT vp<%5> = first-order splice ir<%recur> ir<%recur.next> ; CHECK-NEXT: Successor(s): loop.0.split ; CHECK-EMPTY: ; CHECK-NEXT: loop.0.split: @@ -102,14 +104,14 @@ ; CHECK-NEXT: CondBit: vp<%3> (loop) ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%rem> = srem ir<%recur>, ir<%x> +; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<%5>, ir<%x> ; CHECK-NEXT: REPLICATE ir<%add> = add ir<%rem>, ir<%recur.next> ; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, ir<%iv> ; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%9> = ir<%rem> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%10> = ir<%rem> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.1 @@ -141,7 +143,7 @@ ; CHECK-LABEL: sink_replicate_region_3_reduction ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { ; CHECK-NEXT: loop: -; CHECK-NEXT: WIDEN-PHI ir<%recur> = phi ir<0>, ir<%recur.next> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next ; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next> ; CHECK-NEXT: EMIT vp<%4> = icmp ule ir<%iv> vp<%0> @@ -149,6 +151,7 @@ ; CHECK-EMPTY: ; CHECK-NEXT: loop.0: ; CHECK-NEXT: WIDEN ir<%recur.next> = sext ir<%y> +; CHECK-NEXT: EMIT vp<%6> = first-order splice ir<%recur> ir<%recur.next> ; CHECK-NEXT: Successor(s): pred.srem ; CHECK-EMPTY: ; CHECK-NEXT: pred.srem: { @@ -158,19 +161,19 @@ ; CHECK-NEXT: CondBit: vp<%4> (loop) ; CHECK-EMPTY: ; CHECK-NEXT: pred.srem.if: -; CHECK-NEXT: REPLICATE ir<%rem> = srem ir<%recur>, ir<%x> (S->V) +; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<%6>, ir<%x> (S->V) ; CHECK-NEXT: Successor(s): pred.srem.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.srem.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%7> = ir<%rem> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%8> = ir<%rem> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.0.split ; CHECK-EMPTY: ; CHECK-NEXT: loop.0.split: -; CHECK-NEXT: WIDEN ir<%add> = add vp<%7>, ir<%recur.next> +; CHECK-NEXT: WIDEN ir<%add> = add vp<%8>, ir<%recur.next> ; CHECK-NEXT: WIDEN ir<%and.red.next> = and ir<%and.red>, ir<%add> -; CHECK-NEXT: EMIT vp<%10> = select vp<%4> ir<%and.red.next> ir<%and.red> +; CHECK-NEXT: EMIT vp<%11> = select vp<%4> ir<%and.red.next> ir<%and.red> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -200,7 +203,7 @@ ; CHECK-LABEL: sink_replicate_region_4_requires_split_at_end_of_block ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { ; CHECK-NEXT: loop: -; CHECK-NEXT: WIDEN-PHI ir<%0> = phi ir<0>, ir<%conv> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next ; CHECK-NEXT: EMIT vp<%3> = icmp ule ir<%iv> vp<%0> ; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, ir<%iv> @@ -227,6 +230,7 @@ ; CHECK-EMPTY: ; CHECK-NEXT: loop.1: ; CHECK-NEXT: WIDEN ir<%conv> = sext vp<%6> +; CHECK-NEXT: EMIT vp<%8> = first-order splice ir<%0> ir<%conv> ; CHECK-NEXT: Successor(s): loop.1.split ; CHECK: loop.1.split: @@ -239,19 +243,19 @@ ; CHECK-NEXT: CondBit: vp<%3> (loop) ; CHECK: pred.load.if: -; CHECK-NEXT: REPLICATE ir<%rem> = srem ir<%0>, ir<%x> (S->V) +; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<%8>, ir<%x> (S->V) ; CHECK-NEXT: REPLICATE ir<%lv.2> = load ir<%gep> (S->V) ; CHECK-NEXT: Successor(s): pred.load.continue ; CHECK: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%10> = ir<%rem> -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%11> = ir<%lv.2> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%11> = ir<%rem> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%12> = ir<%lv.2> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK: loop.2: -; CHECK-NEXT: WIDEN ir<%add.1> = add ir<%conv>, vp<%10> -; CHECK-NEXT: WIDEN ir<%conv.lv.2> = sext vp<%11> +; CHECK-NEXT: WIDEN ir<%add.1> = add ir<%conv>, vp<%11> +; CHECK-NEXT: WIDEN ir<%conv.lv.2> = sext vp<%12> ; CHECK-NEXT: WIDEN ir<%add> = add ir<%add.1>, ir<%conv.lv.2> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -283,7 +287,7 @@ ; CHECK-LABEL: sink_replicate_region_after_replicate_region ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { ; CHECK-NEXT: loop: -; CHECK-NEXT: WIDEN-PHI ir<%recur> = phi ir<0>, ir<%recur.next> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next ; CHECK-NEXT: EMIT vp<%3> = icmp ule ir<%iv> vp<%0> ; CHECK-NEXT: Successor(s): loop.0 @@ -293,6 +297,7 @@ ; CHECK-EMPTY: ; CHECK-NEXT: loop.1: ; CHECK-NEXT: WIDEN ir<%recur.next> = sext ir<%y> +; CHECK-NEXT: EMIT vp<%5> = first-order splice ir<%recur> ir<%recur.next> ; CHECK-NEXT: Successor(s): pred.srem ; CHECK-EMPTY: ; CHECK-NEXT: pred.srem: { @@ -302,11 +307,11 @@ ; CHECK-NEXT: CondBit: vp<%3> (loop) ; CHECK-EMPTY: ; CHECK-NEXT: pred.srem.if: -; CHECK-NEXT: REPLICATE ir<%rem> = srem ir<%recur>, ir<%x> +; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<%5>, ir<%x> ; CHECK-NEXT: Successor(s): pred.srem.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.srem.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%6> = ir<%rem> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%7> = ir<%rem> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.1.split @@ -321,13 +326,13 @@ ; CHECK-NEXT: CondBit: vp<%3> (loop) ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%rem.div> = sdiv ir<20>, vp<%6> +; CHECK-NEXT: REPLICATE ir<%rem.div> = sdiv ir<20>, vp<%7> ; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, ir<%iv> ; CHECK-NEXT: REPLICATE store ir<%rem.div>, ir<%gep> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%10> = ir<%rem.div> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%11> = ir<%rem.div> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.2 diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll @@ -337,7 +337,8 @@ ; UNROLL-NO-IC-LABEL: @constant_folded_previous_value( ; UNROLL-NO-IC: vector.body: ; UNROLL-NO-IC: [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ , %vector.ph ], [ , %vector.body ] -; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> , <4 x i32> +; UNROLL-NO-IC: [[TMP0:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> , <4 x i32> +; CHECK-NO-IC-NEXT: add nuw i64 ; UNROLL-NO-IC: br i1 {{.*}}, label %middle.block, label %vector.body ; define void @constant_folded_previous_value() { @@ -641,10 +642,10 @@ ; SINK-AFTER-NEXT: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] ; SINK-AFTER-NEXT: %vec.ind = phi <4 x i16> [ , %vector.ph ], [ %vec.ind.next, %vector.body ] ; SINK-AFTER-NEXT: %vector.recur = phi <4 x i16> [ , %vector.ph ], [ %3, %vector.body ] -; SINK-AFTER-NEXT: %vector.recur2 = phi <4 x i32> [ , %vector.ph ], [ %1, %vector.body ] +; SINK-AFTER-NEXT: %vector.recur1 = phi <4 x i32> [ , %vector.ph ], [ %1, %vector.body ] ; SINK-AFTER-NEXT: %0 = add <4 x i16> %vec.ind, ; SINK-AFTER-NEXT: %1 = zext <4 x i16> %0 to <4 x i32> -; SINK-AFTER-NEXT: %2 = shufflevector <4 x i32> %vector.recur2, <4 x i32> %1, <4 x i32> +; SINK-AFTER-NEXT: %2 = shufflevector <4 x i32> %vector.recur1, <4 x i32> %1, <4 x i32> ; SINK-AFTER-NEXT: %3 = add <4 x i16> %0, ; SINK-AFTER-NEXT: %4 = shufflevector <4 x i16> %vector.recur, <4 x i16> %3, <4 x i32> ; SINK-AFTER-NEXT: %5 = sub <4 x i16> %4, @@ -704,30 +705,30 @@ ; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_UDIV_IF]] ] ; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP2]], i32 1 ; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_UDIV_IF4:%.*]], label [[PRED_UDIV_CONTINUE5:%.*]] -; CHECK: pred.udiv.if4: +; CHECK: pred.udiv.if3: ; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[OFFSET_IDX]], -1 ; CHECK-NEXT: [[TMP9:%.*]] = udiv i32 219220132, [[TMP8]] ; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[TMP9]], i32 1 ; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE5]] -; CHECK: pred.udiv.continue5: +; CHECK: pred.udiv.continue4: ; CHECK-NEXT: [[TMP11:%.*]] = phi <4 x i32> [ [[TMP6]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP10]], [[PRED_UDIV_IF4]] ] ; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2 ; CHECK-NEXT: br i1 [[TMP12]], label [[PRED_UDIV_IF6:%.*]], label [[PRED_UDIV_CONTINUE7:%.*]] -; CHECK: pred.udiv.if6: +; CHECK: pred.udiv.if5: ; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[OFFSET_IDX]], -2 ; CHECK-NEXT: [[TMP14:%.*]] = udiv i32 219220132, [[TMP13]] ; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP14]], i32 2 ; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE7]] -; CHECK: pred.udiv.continue7: +; CHECK: pred.udiv.continue6: ; CHECK-NEXT: [[TMP16:%.*]] = phi <4 x i32> [ [[TMP11]], [[PRED_UDIV_CONTINUE5]] ], [ [[TMP15]], [[PRED_UDIV_IF6]] ] ; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP2]], i32 3 ; CHECK-NEXT: br i1 [[TMP17]], label [[PRED_UDIV_IF8:%.*]], label [[PRED_UDIV_CONTINUE9]] -; CHECK: pred.udiv.if8: +; CHECK: pred.udiv.if7: ; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[OFFSET_IDX]], -3 ; CHECK-NEXT: [[TMP19:%.*]] = udiv i32 219220132, [[TMP18]] ; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP19]], i32 3 ; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE9]] -; CHECK: pred.udiv.continue9: +; CHECK: pred.udiv.continue8: ; CHECK-NEXT: [[TMP21]] = phi <4 x i32> [ [[TMP16]], [[PRED_UDIV_CONTINUE7]] ], [ [[TMP20]], [[PRED_UDIV_IF8]] ] ; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP21]], <4 x i32> ; CHECK-NEXT: [[TMP23]] = add <4 x i32> [[VEC_PHI1]], [[TMP22]] @@ -798,27 +799,27 @@ ; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_UDIV_IF]] ] ; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP5]], i32 1 ; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]] -; CHECK: pred.udiv.if5: +; CHECK: pred.udiv.if4: ; CHECK-NEXT: [[TMP11:%.*]] = udiv i32 219220132, [[TMP2]] ; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP9]], i32 [[TMP11]], i32 1 ; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE6]] -; CHECK: pred.udiv.continue6: +; CHECK: pred.udiv.continue5: ; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x i32> [ [[TMP9]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP12]], [[PRED_UDIV_IF5]] ] ; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP5]], i32 2 ; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8:%.*]] -; CHECK: pred.udiv.if7: +; CHECK: pred.udiv.if6: ; CHECK-NEXT: [[TMP15:%.*]] = udiv i32 219220132, [[TMP3]] ; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP15]], i32 2 ; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE8]] -; CHECK: pred.udiv.continue8: +; CHECK: pred.udiv.continue7: ; CHECK-NEXT: [[TMP17:%.*]] = phi <4 x i32> [ [[TMP13]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP16]], [[PRED_UDIV_IF7]] ] ; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP5]], i32 3 ; CHECK-NEXT: br i1 [[TMP18]], label [[PRED_UDIV_IF9:%.*]], label [[PRED_UDIV_CONTINUE10:%.*]] -; CHECK: pred.udiv.if9: +; CHECK: pred.udiv.if8: ; CHECK-NEXT: [[TMP19:%.*]] = udiv i32 219220132, [[TMP4]] ; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP19]], i32 3 ; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE10]] -; CHECK: pred.udiv.continue10: +; CHECK: pred.udiv.continue9: ; CHECK-NEXT: [[TMP21]] = phi <4 x i32> [ [[TMP17]], [[PRED_UDIV_CONTINUE8]] ], [ [[TMP20]], [[PRED_UDIV_IF9]] ] ; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP21]], <4 x i32> ; CHECK-NEXT: [[TMP23]] = add <4 x i32> [[VEC_PHI4]], [[TMP22]] @@ -832,31 +833,31 @@ ; CHECK: pred.store.continue: ; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP5]], i32 1 ; CHECK-NEXT: br i1 [[TMP27]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12:%.*]] -; CHECK: pred.store.if11: +; CHECK: pred.store.if10: ; CHECK-NEXT: [[TMP28:%.*]] = or i32 [[INDEX]], 1 ; CHECK-NEXT: [[TMP29:%.*]] = sext i32 [[TMP28]] to i64 ; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP29]] ; CHECK-NEXT: store i32 [[TMP2]], i32* [[TMP30]], align 4 ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE12]] -; CHECK: pred.store.continue12: +; CHECK: pred.store.continue11: ; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i1> [[TMP5]], i32 2 ; CHECK-NEXT: br i1 [[TMP31]], label [[PRED_STORE_IF13:%.*]], label [[PRED_STORE_CONTINUE14:%.*]] -; CHECK: pred.store.if13: +; CHECK: pred.store.if12: ; CHECK-NEXT: [[TMP32:%.*]] = or i32 [[INDEX]], 2 ; CHECK-NEXT: [[TMP33:%.*]] = sext i32 [[TMP32]] to i64 ; CHECK-NEXT: [[TMP34:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP33]] ; CHECK-NEXT: store i32 [[TMP3]], i32* [[TMP34]], align 4 ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE14]] -; CHECK: pred.store.continue14: +; CHECK: pred.store.continue13: ; CHECK-NEXT: [[TMP35:%.*]] = extractelement <4 x i1> [[TMP5]], i32 3 ; CHECK-NEXT: br i1 [[TMP35]], label [[PRED_STORE_IF15:%.*]], label [[PRED_STORE_CONTINUE16]] -; CHECK: pred.store.if15: +; CHECK: pred.store.if14: ; CHECK-NEXT: [[TMP36:%.*]] = or i32 [[INDEX]], 3 ; CHECK-NEXT: [[TMP37:%.*]] = sext i32 [[TMP36]] to i64 ; CHECK-NEXT: [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP37]] ; CHECK-NEXT: store i32 [[TMP4]], i32* [[TMP38]], align 4 ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE16]] -; CHECK: pred.store.continue16: +; CHECK: pred.store.continue15: ; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 ; CHECK-NEXT: [[VEC_IND_NEXT3]] = add <4 x i32> [[VEC_IND2]], ; CHECK-NEXT: [[TMP39:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll --- a/llvm/test/Transforms/LoopVectorize/induction.ll +++ b/llvm/test/Transforms/LoopVectorize/induction.ll @@ -860,10 +860,10 @@ ; CHECK-NEXT: %vec.phi = phi <2 x i64> ; CHECK-NEXT: %vec.ind = phi <2 x i64> [ , %vector.ph ], [ %vec.ind.next, %vector.body ] ; CHECK-NEXT: %vec.ind2 = phi <2 x i32> [ , %vector.ph ], [ %vec.ind.next3, %vector.body ] -; CHECK-NEXT: %vector.recur = phi <2 x i32> [ , %vector.ph ], [ %vec.ind5, %vector.body ] -; CHECK-NEXT: %vec.ind5 = phi <2 x i32> [ , %vector.ph ], [ %vec.ind.next6, %vector.body ] -; CHECK-NEXT: %vec.ind7 = phi <2 x i32> [ , %vector.ph ], [ %vec.ind.next8, %vector.body ] -; CHECK-NEXT: shufflevector <2 x i32> %vector.recur, <2 x i32> %vec.ind5, <2 x i32> +; CHECK-NEXT: %vector.recur = phi <2 x i32> [ , %vector.ph ], [ %vec.ind4, %vector.body ] +; CHECK-NEXT: %vec.ind4 = phi <2 x i32> [ , %vector.ph ], [ %vec.ind.next5, %vector.body ] +; CHECK-NEXT: %vec.ind6 = phi <2 x i32> [ , %vector.ph ], [ %vec.ind.next7, %vector.body ] +; CHECK-NEXT: shufflevector <2 x i32> %vector.recur, <2 x i32> %vec.ind4, <2 x i32> entry: br label %loop diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll --- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll +++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll @@ -1536,11 +1536,11 @@ ; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i16>, <8 x i16>* [[TMP10]], align 4 ; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i16> [[WIDE_VEC]], <8 x i16> poison, <4 x i32> ; CHECK-NEXT: [[STRIDED_VEC8]] = shufflevector <8 x i16> [[WIDE_VEC]], <8 x i16> poison, <4 x i32> -; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[STRIDED_VEC8]], <4 x i32> -; CHECK-NEXT: [[TMP12:%.*]] = sext <4 x i16> [[STRIDED_VEC]] to <4 x i32> -; CHECK-NEXT: [[TMP13:%.*]] = sext <4 x i16> [[TMP11]] to <4 x i32> +; CHECK-NEXT: [[TMP11:%.*]] = sext <4 x i16> [[STRIDED_VEC]] to <4 x i32> +; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[STRIDED_VEC8]], <4 x i32> +; CHECK-NEXT: [[TMP13:%.*]] = sext <4 x i16> [[TMP12]] to <4 x i32> ; CHECK-NEXT: [[TMP14:%.*]] = sext <4 x i16> [[STRIDED_VEC8]] to <4 x i32> -; CHECK-NEXT: [[TMP15:%.*]] = mul nsw <4 x i32> [[TMP13]], [[TMP12]] +; CHECK-NEXT: [[TMP15:%.*]] = mul nsw <4 x i32> [[TMP13]], [[TMP11]] ; CHECK-NEXT: [[TMP16:%.*]] = mul nsw <4 x i32> [[TMP15]], [[TMP14]] ; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] ; CHECK-NEXT: [[TMP18:%.*]] = bitcast i32* [[TMP17]] to <4 x i32>* diff --git a/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll --- a/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll +++ b/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll @@ -172,8 +172,9 @@ ; CHECK-VF4UF2-LABEL: @constant_folded_previous_value ; CHECK-VF4UF2: vector.body ; CHECK-VF4UF2: %[[VECTOR_RECUR:.*]] = phi [ %vector.recur.init, %vector.ph ], [ shufflevector ( insertelement ( undef, i64 1, i32 0), undef, zeroinitializer), %vector.body ] -; CHECK-VF4UF2-NEXT: %[[SPLICE1:.*]] = call @llvm.experimental.vector.splice.nxv4i64( %vector.recur, shufflevector ( insertelement ( undef, i64 1, i32 0), undef, zeroinitializer), i32 -1) +; CHECK-VF4UF2: %[[SPLICE1:.*]] = call @llvm.experimental.vector.splice.nxv4i64( %vector.recur, shufflevector ( insertelement ( undef, i64 1, i32 0), undef, zeroinitializer), i32 -1) ; CHECK-VF4UF2: %[[SPLICE2:.*]] = call @llvm.experimental.vector.splice.nxv4i64( shufflevector ( insertelement ( undef, i64 1, i32 0), undef, zeroinitializer), shufflevector ( insertelement ( undef, i64 1, i32 0), undef, zeroinitializer), i32 -1) +; CHECK-VF4UF2: br i1 {{.*}}, label %middle.block, label %vector.body entry: br label %scalar.body @@ -197,15 +198,17 @@ define i32 @extract_second_last_iteration(i32* %cval, i32 %x) { ; CHECK-VF4UF2-LABEL: @extract_second_last_iteration ; CHECK-VF4UF2: vector.ph -; CHECK-VF4UF2: %[[SPLAT_INS1:.*]] = insertelement poison, i32 %x, i32 0 -; CHECK-VF4UF2: %[[SPLAT1:.*]] = shufflevector %[[SPLAT_INS1]], poison, zeroinitializer -; CHECK-VF4UF2: %[[SPLAT_INS2:.*]] = insertelement poison, i32 %x, i32 0 -; CHECK-VF4UF2: %[[SPLAT2:.*]] = shufflevector %[[SPLAT_INS2]], poison, zeroinitializer +; CHECK-VF4UF2: call i32 @llvm.vscale.i32() +; CHECK-VF4UF2: call i32 @llvm.vscale.i32() ; CHECK-VF4UF2: %[[VSCALE1:.*]] = call i32 @llvm.vscale.i32() ; CHECK-VF4UF2: %[[MUL1:.*]] = mul i32 %[[VSCALE1]], 4 ; CHECK-VF4UF2: %[[SUB1:.*]] = sub i32 %[[MUL1]], 1 ; CHECK-VF4UF2: %[[VEC_RECUR_INIT:.*]] = insertelement poison, i32 0, i32 %[[SUB1]] -; CHECK-VF4UF2: vector.body +; CHECK-VF4UF2: %[[SPLAT_INS1:.*]] = insertelement poison, i32 %x, i32 0 +; CHECK-VF4UF2: %[[SPLAT1:.*]] = shufflevector %[[SPLAT_INS1]], poison, zeroinitializer +; CHECK-VF4UF2: %[[SPLAT_INS2:.*]] = insertelement poison, i32 %x, i32 0 +; CHECK-VF4UF2: %[[SPLAT2:.*]] = shufflevector %[[SPLAT_INS2]], poison, zeroinitializer +; ; CHECK-VF4UF2: vector.body ; CHECK-VF4UF2: %[[VEC_RECUR:.*]] = phi [ %[[VEC_RECUR_INIT]], %vector.ph ], [ %[[ADD2:.*]], %vector.body ] ; CHECK-VF4UF2: %[[ADD1:.*]] = add %{{.*}}, %[[SPLAT1]] ; CHECK-VF4UF2: %[[ADD2]] = add %{{.*}}, %[[SPLAT2]] diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll @@ -766,7 +766,7 @@ ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { ; CHECK-NEXT: loop: ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next -; CHECK-NEXT: WIDEN-PHI ir<%for> = phi ir<0>, ir<%lv.a> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, ir<%lv.a> ; CHECK-NEXT: EMIT vp<%3> = icmp ule ir<%iv> vp<%0> ; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%iv> ; CHECK-NEXT: Successor(s): pred.load @@ -788,6 +788,7 @@ ; CHECK-NEXT: Successor(s): loop.0 ; CHECK-EMPTY: ; CHECK-NEXT: loop.0: +; CHECK-NEXT: EMIT vp<%7> = first-order splice ir<%for> ir<%lv.a> ; CHECK-NEXT: Successor(s): loop.1 ; CHECK-EMPTY: ; CHECK-NEXT: loop.1: @@ -800,12 +801,12 @@ ; CHECK-NEXT: CondBit: vp<%3> (loop) ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%div> = sdiv ir<%for>, vp<%6> +; CHECK-NEXT: REPLICATE ir<%div> = sdiv vp<%7>, vp<%6> ; CHECK-NEXT: REPLICATE store ir<%div>, ir<%gep.a> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%9> = ir<%div> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%10> = ir<%div> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.2