[LV] Create createInductionResumeValue helper.

[fhahn]

Summary

Factor out the logic to create induction resume values for a specific
induction. This will be used in D92132 to support widened IVs during
epilogue vectorization.

[Ayal]

LGTM, adding minor nits.

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

3179

nit: indent?

3193

Probably meant to collect all BCResumeVals created and return them, but is currently dead code - better leave for subsequent patch and out of this refactoring patch?

[fhahn]

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

3179

Yes, it looks like clang-format-diff doesn't handle formatting here properly for some reason. Will fix manually.

3193

This was left over from an earlier version, removed!

[fhahn]

Rebased and nits addressed, will land soon.

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

public:
   /// this is needed because each iteration in the loop corresponds to a SIMD
   /// element.
   virtual Value *getBroadcastInstrs(Value *V);
 
   // Returns the resume value (bc.merge.rdx) for a reduction as
   // generated by fixReduction.
   PHINode *getReductionResumeValue(const RecurrenceDescriptor &RdxDesc);
 
+  PHINode *createInductionResumeValue(
+      PHINode *OrigPhi, const InductionDescriptor &ID,
+      ArrayRef<BasicBlock *> BypassBlocks,
+      std::pair<BasicBlock *, Value *> AdditionalBypass = {nullptr, nullptr});
+
 protected:
   friend class LoopVectorizationPlanner;
 
   /// A small list of PHINodes.
   using PhiVector = SmallVector<PHINode *, 4>;
 
   /// A type for scalarized values in the new loop. Each value from the
   /// original loop, when scalarized, is represented by UF x VF scalar values
void InnerLoopVectorizer::createVectorLoopSkeleton(StringRef Prefix) {
   // created during VPlan exection.
   if (!Cost->requiresScalarEpilogue(VF))
     // If there is an epilogue which must run, there's no edge from the
     // middle block to exit blocks  and thus no need to update the immediate
     // dominator of the exit blocks.
     DT->changeImmediateDominator(LoopExitBlock, LoopMiddleBlock);
 }
 
-void InnerLoopVectorizer::createInductionResumeValues(
+PHINode *InnerLoopVectorizer::createInductionResumeValue(
+    PHINode *OrigPhi, const InductionDescriptor &II,
+    ArrayRef<BasicBlock *> BypassBlocks,
     std::pair<BasicBlock *, Value *> AdditionalBypass) {
-  assert(((AdditionalBypass.first && AdditionalBypass.second) ||
-          (!AdditionalBypass.first && !AdditionalBypass.second)) &&
-         "Inconsistent information about additional bypass.");
-
   Value *VectorTripCount = getOrCreateVectorTripCount(LoopVectorPreHeader);
   assert(VectorTripCount && "Expected valid arguments");
-  // We are going to resume the execution of the scalar loop.
-  // Go over all of the induction variables that we found and fix the
-  // PHIs that are left in the scalar version of the loop.
-  // The starting values of PHI nodes depend on the counter of the last
-  // iteration in the vectorized loop.
-  // If we come from a bypass edge then we need to start from the original
-  // start value.
-  Instruction *OldInduction = Legal->getPrimaryInduction();
-  for (const auto &InductionEntry : Legal->getInductionVars()) {
-    PHINode *OrigPhi = InductionEntry.first;
-    InductionDescriptor II = InductionEntry.second;
 
-    Value *&EndValue = IVEndValues[OrigPhi];
-    Value *EndValueFromAdditionalBypass = AdditionalBypass.second;
-    if (OrigPhi == OldInduction) {
-      // We know what the end value is.
-      EndValue = VectorTripCount;
-    } else {
-      IRBuilder<> B(LoopVectorPreHeader->getTerminator());
+  Instruction *OldInduction = Legal->getPrimaryInduction();
 
-      // Fast-math-flags propagate from the original induction instruction.
-      if (II.getInductionBinOp() && isa<FPMathOperator>(II.getInductionBinOp()))
-        B.setFastMathFlags(II.getInductionBinOp()->getFastMathFlags());
 
-      Type *StepType = II.getStep()->getType();
-      Instruction::CastOps CastOp =
-          CastInst::getCastOpcode(VectorTripCount, true, StepType, true);
-      Value *VTC = B.CreateCast(CastOp, VectorTripCount, StepType, "cast.vtc");
-      Value *Step =
-          CreateStepValue(II.getStep(), *PSE.getSE(), &*B.GetInsertPoint());
-      EndValue = emitTransformedIndex(B, VTC, II.getStartValue(), Step, II);
-      EndValue->setName("ind.end");
 
-      // Compute the end value for the additional bypass (if applicable).
-      if (AdditionalBypass.first) {
-        B.SetInsertPoint(&(*AdditionalBypass.first->getFirstInsertionPt()));
-        CastOp = CastInst::getCastOpcode(AdditionalBypass.second, true,
-                                         StepType, true);
-        Value *Step =
-            CreateStepValue(II.getStep(), *PSE.getSE(), &*B.GetInsertPoint());
-        VTC =
-            B.CreateCast(CastOp, AdditionalBypass.second, StepType, "cast.vtc");
-        EndValueFromAdditionalBypass =
-            emitTransformedIndex(B, VTC, II.getStartValue(), Step, II);
-        EndValueFromAdditionalBypass->setName("ind.end");
-      }
-    }
+      CastOp = CastInst::getCastOpcode(AdditionalBypass.second, true, StepType,
+                                       true);
+      VTC = B.CreateCast(CastOp, AdditionalBypass.second, StepType, "cast.vtc");
 
     // Create phi nodes to merge from the  backedge-taken check block.
     PHINode *BCResumeVal =
         PHINode::Create(OrigPhi->getType(), 3, "bc.resume.val",
                         LoopScalarPreHeader->getTerminator());
     // Copy original phi DL over to the new one.
     BCResumeVal->setDebugLoc(OrigPhi->getDebugLoc());
 
     // The new PHI merges the original incoming value, in case of a bypass,
     // or the value at the end of the vectorized loop.
     BCResumeVal->addIncoming(EndValue, LoopMiddleBlock);
 
     // Fix the scalar body counter (PHI node).
     // The old induction's phi node in the scalar body needs the truncated
     // value.
-    for (BasicBlock *BB : LoopBypassBlocks)
-      BCResumeVal->addIncoming(II.getStartValue(), BB);
+    for (BasicBlock *BB : BypassBlocks)
 
     if (AdditionalBypass.first)
       BCResumeVal->setIncomingValueForBlock(AdditionalBypass.first,
                                             EndValueFromAdditionalBypass);
+    return BCResumeVal;
+}
 
+DenseMap<const InductionDescriptor *, Value *>
+InnerLoopVectorizer::createInductionResumeValues(
+    std::pair<BasicBlock *, Value *> AdditionalBypass) {
+    assert(((AdditionalBypass.first && AdditionalBypass.second) ||
+            (!AdditionalBypass.first && !AdditionalBypass.second)) &&
+           "Inconsistent information about additional bypass.");
+    DenseMap<const InductionDescriptor *, Value *> ResumeValues;
+    // We are going to resume the execution of the scalar loop.
+    // Go over all of the induction variables that we found and fix the
+    // PHIs that are left in the scalar version of the loop.
+    // The starting values of PHI nodes depend on the counter of the last
+    // iteration in the vectorized loop.
+    // If we come from a bypass edge then we need to start from the original
+    // start value.
+    for (const auto &InductionEntry : Legal->getInductionVars()) {
+      PHINode *OrigPhi = InductionEntry.first;
+      const InductionDescriptor &II = InductionEntry.second;
+      PHINode *BCResumeVal = createInductionResumeValue(
+          OrigPhi, II, LoopBypassBlocks, AdditionalBypass);
       OrigPhi->setIncomingValueForBlock(LoopScalarPreHeader, BCResumeVal);
     }
 }
 
 BasicBlock *InnerLoopVectorizer::completeLoopSkeleton(MDNode *OrigLoopID) {
   // The trip counts should be cached by now.
   Value *Count = getOrCreateTripCount(LoopVectorPreHeader);
   Value *VectorTripCount = getOrCreateVectorTripCount(LoopVectorPreHeader);
 
   auto *ScalarLatchTerm = OrigLoop->getLoopLatch()->getTerminator();