[LV] Move recurrence backedge fixup code to VPlan::execute (NFC).

[fhahn]

Summary

As suggested in D105008, move the code that fixes up the backedge value
for first order recurrences to VPlan::execute.

Now all that remains in fixFirstOrderRecurrences is the code responsible
for creating the exit values in the middle block.

[Ayal]

This looks good to me, with a couple of minor comments; thanks for following-up!

Would also be good to hook-up other header phis here as well.

llvm/lib/Transforms/Vectorize/VPlan.cpp

818

recurrenxes

Mention that they have a single part, regardless of UF?

Use VectorLatchBB?

[fhahn]

llvm/lib/Transforms/Vectorize/VPlan.cpp

818

Thanks Ayal! I'll address those in the committed version.

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

protected:
 
   /// Create a new induction variable inside L.
   PHINode *createInductionVariable(Loop *L, Value *Start, Value *End,
                                    Value *Step, Instruction *DL);
 
   /// Handle all cross-iteration phis in the header.
   void fixCrossIterationPHIs(VPTransformState &State);
 
-  /// Fix a first-order recurrence. This is the second phase of vectorizing
-  /// this phi node.
+  /// Create the exit value of first order recurrences in the middle block and
+  /// update their users.
   void fixFirstOrderRecurrence(VPWidenPHIRecipe *PhiR, VPTransformState &State);
 
   /// Fix a reduction cross-iteration phi. This is the second phase of
   /// vectorizing this phi node.
   void fixReduction(VPReductionPHIRecipe *Phi, VPTransformState &State);
 
   /// Clear NSW/NUW flags from reduction instructions if necessary.
   void clearReductionWrapFlags(const RecurrenceDescriptor &RdxDesc,
void InnerLoopVectorizer::fixFirstOrderRecurrence(VPWidenPHIRecipe *PhiR,
   //
   //   scalar.ph:
   //     s_init = phi [x, middle.block], [a[-1], otherwise]
   //     br scalar.body
   //
   // After execution completes the vector loop, we extract the next value of
   // the recurrence (x) to use as the initial value in the scalar loop.
 
-  auto *IdxTy = Builder.getInt32Ty();
-  auto *VecPhi = cast<PHINode>(State.get(PhiR, 0));
-
-  // Fix the latch value of the new recurrence in the vector loop.
-  VPValue *PreviousDef = PhiR->getBackedgeValue();
-  Value *Incoming = State.get(PreviousDef, UF - 1);
-  VecPhi->addIncoming(Incoming, LI->getLoopFor(LoopVectorBody)->getLoopLatch());
-
   // Extract the last vector element in the middle block. This will be the
   // initial value for the recurrence when jumping to the scalar loop.
+  VPValue *PreviousDef = PhiR->getBackedgeValue();
+  Value *Incoming = State.get(PreviousDef, UF - 1);
   auto *ExtractForScalar = Incoming;
+  auto *IdxTy = Builder.getInt32Ty();
   if (VF.isVector()) {
     auto *One = ConstantInt::get(IdxTy, 1);
     Builder.SetInsertPoint(LoopMiddleBlock->getTerminator());
     auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, VF);
     auto *LastIdx = Builder.CreateSub(RuntimeVF, One);
     ExtractForScalar = Builder.CreateExtractElement(ExtractForScalar, LastIdx,
                                                     "vector.recur.extract");
   }

llvm/lib/Transforms/Vectorize/VPlan.cpp

void VPlan::execute(VPTransformState *State) {
   // 2. Generate code in loop body.
   State->CFG.PrevVPBB = nullptr;
   State->CFG.PrevBB = VectorHeaderBB;
   State->CFG.LastBB = VectorLatchBB;
 
   for (VPBlockBase *Block : depth_first(Entry))
     Block->execute(State);
 
+  // Fix the latch value of the first-order recurrences in the vector loop. Only
+  // a single part is generated, regardless of the UF.
+  VPBasicBlock *Header = Entry->getEntryBasicBlock();
+  for (VPRecipeBase &R : Header->phis()) {
+    if (auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R)) {
+      auto *VecPhi = cast<PHINode>(State->get(FOR, 0));
+
+      VPValue *PreviousDef = FOR->getBackedgeValue();
+      Value *Incoming = State->get(PreviousDef, State->UF - 1);
+      VecPhi->addIncoming(Incoming, VectorLatchBB);
+    }
+  }
+
   // Setup branch terminator successors for VPBBs in VPBBsToFix based on
   // VPBB's successors.
   for (auto VPBB : State->CFG.VPBBsToFix) {
     assert(EnableVPlanNativePath &&
            "Unexpected VPBBsToFix in non VPlan-native path");
     BasicBlock *BB = State->CFG.VPBB2IRBB[VPBB];
     assert(BB && "Unexpected null basic block for VPBB");