[VPlan, VP] WIP: Build and execute VP recipes

[vkmr]

Summary

This patch implements building VPlan with VP Recipes.

Note: Marking it as WIP since the VP recipe execute() methods are defined (to satisfy lld) but have empty implementation. While the execution implementation waits on D99355 the VP recipe building part can be reviewed.

This patch is the fourth patch in the series of patches implementing D99750.

[simoll]

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

9614

I suppose there could be an llvm_unreachable here to document this missing code path (same for the other recipes).

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

VPValue *VPRecipeBuilder::createBlockInMask(BasicBlock *BB, VPlanPtr &Plan) {
   // All-one mask is modelled as no-mask following the convention for masked
   // load/store/gather/scatter. Initialize BlockMask to no-mask.
   VPValue *BlockMask = nullptr;
 
   if (OrigLoop->getHeader() == BB) {
     if (!CM.blockNeedsPredication(BB))
       return BlockMaskCache[BB] = BlockMask; // Loop incoming mask is all-one.
 
+    // if header block needs predication then it is only because tail-folding is
+    // enabled. If we are using VP intrinsics for a target with vector length
+    // predication support, this mask (icmp ule %IV %BTC) becomes redundant with
+    // EVL, which means unless we are using VP intrinsics without vector length
+    // predication support we can replace this mask with an all-true mask for
+    // possibly better latency.
+    if (CM.preferVPIntrinsics() &&
+        PreferPredicateWithVPIntrinsics !=
+            PreferVPIntrinsicsTy::WithoutAVLSupport) {
+      BlockMask = Builder.createNaryOp(VPInstruction::AllTrueMask, {});
+      return BlockMaskCache[BB] = BlockMask;
+    }
+
     // Create the block in mask as the first non-phi instruction in the block.
     VPBuilder::InsertPointGuard Guard(Builder);
     auto NewInsertionPoint = Builder.getInsertBlock()->getFirstNonPhi();
     Builder.setInsertPoint(Builder.getInsertBlock(), NewInsertionPoint);
 
     // Introduce the early-exit compare IV <= BTC to form header block mask.
     // This is used instead of IV < TC because TC may wrap, unlike BTC.
     // Start by constructing the desired canonical IV.
for (auto *Predecessor : predecessors(BB)) {
     }
 
     BlockMask = Builder.createOr(BlockMask, EdgeMask);
   }
 
   return BlockMaskCache[BB] = BlockMask;
 }
 
-VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(Instruction *I,
-                                                ArrayRef<VPValue *> Operands,
-                                                VFRange &Range,
-                                                VPlanPtr &Plan) {
+VPValue *VPRecipeBuilder::getOrCreateEVL(VPlanPtr &Plan) {
+  if (EVL)
+    return EVL;
+
+  if (PreferPredicateWithVPIntrinsics ==
+      PreferVPIntrinsicsTy::WithoutAVLSupport) {
+    EVL = Plan->getOrCreateRuntimeVF();
+    return EVL;
+  }
+
+  VPBuilder::InsertPointGuard Guard(Builder);
+  auto *HeaderBB = Plan->getEntry()->getSingleSuccessor()->getEntryBasicBlock();
+  auto NewInsertionPoint = HeaderBB->getFirstNonPhi();
+  Builder.setInsertPoint(HeaderBB, NewInsertionPoint);
+
+  VPValue *IV = getOrCreateIV(Builder.getInsertBlock(), Plan);
+  VPValue *TC = Plan->getOrCreateTripCount();
+  auto *EVLRecipe = new VPWidenEVLRecipe(IV, TC);
+  Builder.getInsertBlock()->insert(EVLRecipe, Builder.getInsertPoint());
+  EVL = EVLRecipe->getEVL();
+  return EVL;
+}
+
+bool VPRecipeBuilder::validateWidenMemory(Instruction *I,
+                                          VFRange &Range) const {
   assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
          "Must be called with either a load or store");
 
   auto willWiden = [&](ElementCount VF) -> bool {
     if (VF.isScalar())
       return false;
     LoopVectorizationCostModel::InstWidening Decision =
         CM.getWideningDecision(I, VF);
     assert(Decision != LoopVectorizationCostModel::CM_Unknown &&
            "CM decision should be taken at this point.");
     if (Decision == LoopVectorizationCostModel::CM_Interleave)
       return true;
     if (CM.isScalarAfterVectorization(I, VF) ||
         CM.isProfitableToScalarize(I, VF))
       return false;
     return Decision != LoopVectorizationCostModel::CM_Scalarize;
   };
 
-  if (!LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range))
+  return LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range);
+}
+
+VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(Instruction *I,
+                                                ArrayRef<VPValue *> Operands,
+                                                VFRange &Range,
+                                                VPlanPtr &Plan) {
+  if (!validateWidenMemory(I, Range))
     return nullptr;
 
   VPValue *Mask = nullptr;
   if (Legal->isMaskRequired(I))
     Mask = createBlockInMask(I->getParent(), Plan);
 
   if (LoadInst *Load = dyn_cast<LoadInst>(I))
     return new VPWidenMemoryInstructionRecipe(*Load, Operands[0], Mask);
 
   StoreInst *Store = cast<StoreInst>(I);
   return new VPWidenMemoryInstructionRecipe(*Store, Operands[1], Operands[0],
                                             Mask);
 }
 
+VPRecipeBase *
+VPRecipeBuilder::tryToPredicatedWidenMemory(Instruction *I,
+                                            ArrayRef<VPValue *> Operands,
+                                            VFRange &Range, VPlanPtr &Plan) {
+  if (!validateWidenMemory(I, Range))
+    return nullptr;
+
+  VPValue *Mask = createBlockInMask(I->getParent(), Plan);
+  VPValue *EVL = getOrCreateEVL(Plan);
+  if (LoadInst *Load = dyn_cast<LoadInst>(I))
+    return new VPPredicatedWidenMemoryInstructionRecipe(*Load, Operands[0],
+                                                        Mask, EVL);
+
+  StoreInst *Store = cast<StoreInst>(I);
+  return new VPPredicatedWidenMemoryInstructionRecipe(*Store, Operands[1],
+                                                      Operands[0], Mask, EVL);
+}
+
 VPWidenIntOrFpInductionRecipe *
 VPRecipeBuilder::tryToOptimizeInductionPHI(PHINode *Phi,
                                            ArrayRef<VPValue *> Operands) const {
   // Check if this is an integer or fp induction. If so, build the recipe that
   // produces its scalar and vector values.
   InductionDescriptor II = Legal->getInductionVars().lookup(Phi);
   if (II.getKind() == InductionDescriptor::IK_IntInduction ||
       II.getKind() == InductionDescriptor::IK_FpInduction) {
bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const {
   auto WillScalarize = [this, I](ElementCount VF) -> bool {
     return CM.isScalarAfterVectorization(I, VF) ||
            CM.isProfitableToScalarize(I, VF) || CM.isScalarWithPredication(I);
   };
   return !LoopVectorizationPlanner::getDecisionAndClampRange(WillScalarize,
                                                              Range);
 }
 
-VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
-                                           ArrayRef<VPValue *> Operands) const {
+bool VPRecipeBuilder::preferPredicatedWiden() const {
+  return CM.preferVPIntrinsics();
+}
+
+bool VPRecipeBuilder::validateWiden(Instruction *I) const {
   auto IsVectorizableOpcode = [](unsigned Opcode) {
     switch (Opcode) {
     case Instruction::Add:
     case Instruction::And:
     case Instruction::AShr:
     case Instruction::BitCast:
     case Instruction::FAdd:
     case Instruction::FCmp:
auto IsVectorizableOpcode = [](unsigned Opcode) {
     case Instruction::URem:
     case Instruction::Xor:
     case Instruction::ZExt:
       return true;
     }
     return false;
   };
 
-  if (!IsVectorizableOpcode(I->getOpcode()))
+  return IsVectorizableOpcode(I->getOpcode());
+}
+
+VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
+                                           ArrayRef<VPValue *> Operands) const {
+  if (!validateWiden(I))
     return nullptr;
 
   // Success: widen this instruction.
   return new VPWidenRecipe(*I, make_range(Operands.begin(), Operands.end()));
 }
 
 void VPRecipeBuilder::fixHeaderPhis() {
   BasicBlock *OrigLatch = OrigLoop->getLoopLatch();
   for (VPWidenPHIRecipe *R : PhisToFix) {
     auto *PN = cast<PHINode>(R->getUnderlyingValue());
     VPRecipeBase *IncR =
         getRecipe(cast<Instruction>(PN->getIncomingValueForBlock(OrigLatch)));
     R->addOperand(IncR->getVPSingleValue());
   }
 }
 
+VPPredicatedWidenRecipe *VPRecipeBuilder::tryToPredicatedWiden(
+    Instruction *I, ArrayRef<VPValue *> Operands, VPlanPtr &Plan) {
+  if (!validateWiden(I))
+    return nullptr;
+
+  VPValue *Mask = createBlockInMask(I->getParent(), Plan);
+  VPValue *EVL = getOrCreateEVL(Plan);
+  return new VPPredicatedWidenRecipe(
+      *I, make_range(Operands.begin(), Operands.end()), Mask, EVL);
+}
+
 VPBasicBlock *VPRecipeBuilder::handleReplication(
     Instruction *I, VFRange &Range, VPBasicBlock *VPBB,
     VPlanPtr &Plan) {
   bool IsUniform = LoopVectorizationPlanner::getDecisionAndClampRange(
       [&](ElementCount VF) { return CM.isUniformAfterVectorization(I, VF); },
       Range);
 
   bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange(

 VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
                                         ArrayRef<VPValue *> Operands,
                                         VFRange &Range, VPlanPtr &Plan) {
   // First, check for specific widening recipes that deal with calls, memory
   // operations, inductions and Phi nodes.
   if (auto *CI = dyn_cast<CallInst>(Instr))
     return toVPRecipeResult(tryToWidenCall(CI, Operands, Range));
 
-  if (isa<LoadInst>(Instr) || isa<StoreInst>(Instr))
+  if (isa<LoadInst>(Instr) || isa<StoreInst>(Instr)) {
+    if (preferPredicatedWiden())
+      return toVPRecipeResult(
+          tryToPredicatedWidenMemory(Instr, Operands, Range, Plan));
+
     return toVPRecipeResult(tryToWidenMemory(Instr, Operands, Range, Plan));
+  }
 
   VPRecipeBase *Recipe;
   if (auto Phi = dyn_cast<PHINode>(Instr)) {
     if (Phi->getParent() != OrigLoop->getHeader())
       return tryToBlend(Phi, Operands, Plan);
     if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands)))
       return toVPRecipeResult(Recipe);
 
VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
 
   if (auto *SI = dyn_cast<SelectInst>(Instr)) {
     bool InvariantCond =
         PSE.getSE()->isLoopInvariant(PSE.getSCEV(SI->getOperand(0)), OrigLoop);
     return toVPRecipeResult(new VPWidenSelectRecipe(
         *SI, make_range(Operands.begin(), Operands.end()), InvariantCond));
   }
 
+  if (preferPredicatedWiden())
+    return toVPRecipeResult(tryToPredicatedWiden(Instr, Operands, Plan));
+
   return toVPRecipeResult(tryToWiden(Instr, Operands));
 }
 
 void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
                                                         ElementCount MaxVF) {
   assert(OrigLoop->isInnermost() && "Inner loop expected.");
 
   // Collect instructions from the original loop that will become trivially dead
void VPWidenSelectRecipe::execute(VPTransformState &State) {
   State.ILV->widenSelectInstruction(*cast<SelectInst>(getUnderlyingInstr()),
                                     this, *this, InvariantCond, State);
 }
 
 void VPWidenRecipe::execute(VPTransformState &State) {
   State.ILV->widenInstruction(*getUnderlyingInstr(), this, *this, State);
 }
 
+void VPPredicatedWidenRecipe::execute(VPTransformState &State) {
+  // TODO: Implement widening
+}
+
 void VPWidenGEPRecipe::execute(VPTransformState &State) {
   State.ILV->widenGEP(cast<GetElementPtrInst>(getUnderlyingInstr()), this,
                       *this, State.UF, State.VF, IsPtrLoopInvariant,
                       IsIndexLoopInvariant, State);
 }
 
 void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
   assert(!State.Instance && "Int or FP induction being replicated.");

 
 void VPWidenMemoryInstructionRecipe::execute(VPTransformState &State) {
   VPValue *StoredValue = isStore() ? getStoredValue() : nullptr;
   State.ILV->vectorizeMemoryInstruction(
       &Ingredient, State, StoredValue ? nullptr : getVPSingleValue(), getAddr(),
       StoredValue, getMask());
 }
 
+void VPPredicatedWidenMemoryInstructionRecipe::execute(
+    VPTransformState &State) {
+  // TODO: Implement widening
+}
+
+void VPWidenEVLRecipe::execute(VPTransformState &State) {
+  // TODO: Implement widening
+}
+
 // Determine how to lower the scalar epilogue, which depends on 1) optimising
 // for minimum code-size, 2) predicate compiler options, 3) loop hints forcing
 // predication, and 4) a TTI hook that analyses whether the loop is suitable
 // for predication.
 static ScalarEpilogueLowering getScalarEpilogueLowering(
     Function *F, Loop *L, LoopVectorizeHints &Hints, ProfileSummaryInfo *PSI,
     BlockFrequencyInfo *BFI, TargetTransformInfo *TTI, TargetLibraryInfo *TLI,
     AssumptionCache *AC, LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT,

llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h

class VPRecipeBuilder {
 
   /// The profitablity analysis.
   LoopVectorizationCostModel &CM;
 
   PredicatedScalarEvolution &PSE;
 
   VPBuilder &Builder;
 
+  VPValue *EVL = nullptr;
+
   /// When we if-convert we need to create edge masks. We have to cache values
   /// so that we don't end up with exponential recursion/IR. Note that
   /// if-conversion currently takes place during VPlan-construction, so these
   /// caches are only used at that stage.
   using EdgeMaskCacheTy =
       DenseMap<std::pair<BasicBlock *, BasicBlock *>, VPValue *>;
   using BlockMaskCacheTy = DenseMap<BasicBlock *, VPValue *>;
   EdgeMaskCacheTy EdgeMaskCache;
class VPRecipeBuilder {
   bool shouldWiden(Instruction *I, VFRange &Range) const;
 
   /// Check if the load or store instruction \p I should widened for \p
   /// Range.Start and potentially masked. Such instructions are handled by a
   /// recipe that takes an additional VPInstruction for the mask.
   VPRecipeBase *tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
                                  VFRange &Range, VPlanPtr &Plan);
 
+  /// Similar to tryToWidenMemory, but create a predicated recipe. The
+  /// predicated recipe takes mandatory mask and EVL VPInstructions.
+  VPRecipeBase *tryToPredicatedWidenMemory(Instruction *I,
+                                           ArrayRef<VPValue *> Operands,
+                                           VFRange &Range, VPlanPtr &Plan);
+
+  /// Helper method used by tryToWidenMemory and tryToPredicatedWidenMemory to
+  /// validate if a memory instructions can be widened.
+  bool validateWidenMemory(Instruction *I, VFRange &Range) const;
+
   /// Check if an induction recipe should be constructed for \I. If so build and
   /// return it. If not, return null.
   VPWidenIntOrFpInductionRecipe *
   tryToOptimizeInductionPHI(PHINode *Phi, ArrayRef<VPValue *> Operands) const;
 
   /// Optimize the special case where the operand of \p I is a constant integer
   /// induction variable.
   VPWidenIntOrFpInductionRecipe *
class VPRecipeBuilder {
   VPWidenCallRecipe *tryToWidenCall(CallInst *CI, ArrayRef<VPValue *> Operands,
                                     VFRange &Range) const;
 
   /// Check if \p I has an opcode that can be widened and return a VPWidenRecipe
   /// if it can. The function should only be called if the cost-model indicates
   /// that widening should be performed.
   VPWidenRecipe *tryToWiden(Instruction *I, ArrayRef<VPValue *> Operands) const;
 
+  /// Similar to tryToWiden, but widen to VP intrinsics.
+  VPPredicatedWidenRecipe *tryToPredicatedWiden(Instruction *I,
+                                                ArrayRef<VPValue *> Operands,
+                                                VPlanPtr &Plan);
+
+  /// Helper method used by tryToWiden and tryToPredicatedWiden to validate if
+  /// an instruction can be widened.
+  bool validateWiden(Instruction *I) const;
+
   /// Return a VPRecipeOrValueTy with VPRecipeBase * being set. This can be used to force the use as VPRecipeBase* for recipe sub-types that also inherit from VPValue.
   VPRecipeOrVPValueTy toVPRecipeResult(VPRecipeBase *R) const { return R; }
 
+  /// Create recipes that will expand to VP intrinsics.
+  bool preferPredicatedWiden() const;
+
   /// Insert and Cache Induction Variable
   VPValue *getOrCreateIV(VPBasicBlock *VPBB, VPlanPtr &Plan);
 
 public:
   VPRecipeBuilder(Loop *OrigLoop, const TargetLibraryInfo *TLI,
                   LoopVectorizationLegality *Legal,
                   LoopVectorizationCostModel &CM,
                   PredicatedScalarEvolution &PSE, VPBuilder &Builder)
public:
   /// that the header block of the loop is set to True. It returns the *entry*
   /// mask for the block BB.
   VPValue *createBlockInMask(BasicBlock *BB, VPlanPtr &Plan);
 
   /// A helper function that computes the predicate of the edge between SRC
   /// and DST.
   VPValue *createEdgeMask(BasicBlock *Src, BasicBlock *Dst, VPlanPtr &Plan);
 
+  /// A helper function that computes the Explicit(Active) Vector Length for the
+  /// current vector iteration.
+  VPValue *getOrCreateEVL(VPlanPtr &Plan);
+
   /// Mark given ingredient for recording its recipe once one is created for
   /// it.
   void recordRecipeOf(Instruction *I) {
     assert((!Ingredient2Recipe.count(I) || Ingredient2Recipe[I] == nullptr) &&
            "Recipe already set for ingredient");
     Ingredient2Recipe[I] = nullptr;
   }
 

llvm/test/Transforms/LoopVectorize/vplan-vp-intrinsics.ll

+; REQUIRES: asserts
+
+; RUN: opt -loop-vectorize -debug-only=loop-vectorize -force-vector-width=4 \
+; RUN: -prefer-predicate-with-vp-intrinsics=without-active-vector-length-support \
+; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \
+; RUN: -mattr=+avx512f -disable-output  %s 2>&1 | FileCheck --check-prefix=WITHOUT-AVL %s
+
+; RUN: opt -loop-vectorize -debug-only=loop-vectorize -force-vector-width=4 \
+; RUN: -prefer-predicate-with-vp-intrinsics=if-active-vector-length-support \
+; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \
+; RUN: -mattr=+avx512f -disable-output  %s 2>&1 | FileCheck --check-prefix=IF-AVL %s
+
+; RUN: opt -loop-vectorize -debug-only=loop-vectorize -force-vector-width=4 \
+; RUN: -prefer-predicate-with-vp-intrinsics=force-active-vector-length-support \
+; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \
+; RUN: -mattr=+avx512f -disable-output  %s 2>&1 | FileCheck --check-prefix=FORCE-AVL %s
+
+; RUN: opt -loop-vectorize -debug-only=loop-vectorize -force-vector-width=4 \
+; RUN: -prefer-predicate-with-vp-intrinsics=no-predication \
+; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \
+; RUN: -mattr=+avx512f -disable-output  %s 2>&1 | FileCheck --check-prefix=NO-VP %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nofree norecurse nounwind uwtable
+define dso_local void @foo(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32* noalias nocapture readonly %c, i64 %N) local_unnamed_addr {
+; WITHOUT-AVL: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; WITHOUT-AVL-NEXT: for.body:
+; WITHOUT-AVL-NEXT:   WIDEN-INDUCTION %indvars.iv = phi 0, %indvars.iv.next
+; WITHOUT-AVL-NEXT:   EMIT vp<%3> = icmp ule ir<%indvars.iv> vp<%0>
+; WITHOUT-AVL-NEXT:   CLONE ir<%arrayidx> = getelementptr ir<%b>, ir<%indvars.iv>
+; WITHOUT-AVL-NEXT:   PREDICATED-WIDEN ir<%0> = load ir<%arrayidx>, vp<%3>, vp<%1>
+; WITHOUT-AVL-NEXT:   CLONE ir<%arrayidx2> = getelementptr ir<%c>, ir<%indvars.iv>
+; WITHOUT-AVL-NEXT:   PREDICATED-WIDEN ir<%1> = load ir<%arrayidx2>, vp<%3>, vp<%1>
+; WITHOUT-AVL-NEXT:   PREDICATED-WIDEN ir<%add> = add ir<%1>, ir<%0>, vp<%3>, vp<%1>
+; WITHOUT-AVL-NEXT:   CLONE ir<%arrayidx4> = getelementptr ir<%a>, ir<%indvars.iv>
+; WITHOUT-AVL-NEXT:   PREDICATED-WIDEN store ir<%arrayidx4>, ir<%add>, vp<%3>, vp<%1>
+; WITHOUT-AVL-NEXT: No successors
+; WITHOUT-AVL-NEXT: }
+
+; IF-AVL: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; IF-AVL-NEXT: for.body:
+; IF-AVL-NEXT:   WIDEN-INDUCTION %indvars.iv = phi 0, %indvars.iv.next
+; IF-AVL-NEXT:   EMIT vp<%2> = icmp ule ir<%indvars.iv> vp<%0>
+; IF-AVL-NEXT:   CLONE ir<%arrayidx> = getelementptr ir<%b>, ir<%indvars.iv>
+; IF-AVL-NEXT:   WIDEN ir<%0> = load ir<%arrayidx>, vp<%2>
+; IF-AVL-NEXT:   CLONE ir<%arrayidx2> = getelementptr ir<%c>, ir<%indvars.iv>
+; IF-AVL-NEXT:   WIDEN ir<%1> = load ir<%arrayidx2>, vp<%2>
+; IF-AVL-NEXT:   WIDEN ir<%add> = add ir<%1>, ir<%0>
+; IF-AVL-NEXT:   CLONE ir<%arrayidx4> = getelementptr ir<%a>, ir<%indvars.iv>
+; IF-AVL-NEXT:   WIDEN store ir<%arrayidx4>, ir<%add>, vp<%2>
+; IF-AVL-NEXT: No successors
+; IF-AVL-NEXT: }
+
+; FORCE-AVL: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; FORCE-AVL-NEXT: for.body:
+; FORCE-AVL-NEXT:   WIDEN-INDUCTION %indvars.iv = phi 0, %indvars.iv.next
+; FORCE-AVL-NEXT:   EMIT vp<%2> = GENERATE-EXPLICIT-VECTOR-LENGTH
+; FORCE-AVL-NEXT:   CLONE ir<%arrayidx> = getelementptr ir<%b>, ir<%indvars.iv>
+; FORCE-AVL-NEXT:   EMIT vp<%4> = all true mask
+; FORCE-AVL-NEXT:   PREDICATED-WIDEN ir<%0> = load ir<%arrayidx>, vp<%4>, vp<%2>
+; FORCE-AVL-NEXT:   CLONE ir<%arrayidx2> = getelementptr ir<%c>, ir<%indvars.iv>
+; FORCE-AVL-NEXT:   PREDICATED-WIDEN ir<%1> = load ir<%arrayidx2>, vp<%4>, vp<%2>
+; FORCE-AVL-NEXT:   PREDICATED-WIDEN ir<%add> = add ir<%1>, ir<%0>, vp<%4>, vp<%2>
+; FORCE-AVL-NEXT:   CLONE ir<%arrayidx4> = getelementptr ir<%a>, ir<%indvars.iv>
+; FORCE-AVL-NEXT:   PREDICATED-WIDEN store ir<%arrayidx4>, ir<%add>, vp<%4>, vp<%2>
+; FORCE-AVL-NEXT: No successors
+; FORCE-AVL-NEXT: }
+
+; NO-VP: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; NO-VP-NEXT: for.body:
+; NO-VP-NEXT:   WIDEN-INDUCTION %indvars.iv = phi 0, %indvars.iv.next
+; NO-VP-NEXT:   EMIT vp<%2> = icmp ule ir<%indvars.iv> vp<%0>
+; NO-VP-NEXT:   CLONE ir<%arrayidx> = getelementptr ir<%b>, ir<%indvars.iv>
+; NO-VP-NEXT:   WIDEN ir<%0> = load ir<%arrayidx>, vp<%2>
+; NO-VP-NEXT:   CLONE ir<%arrayidx2> = getelementptr ir<%c>, ir<%indvars.iv>
+; NO-VP-NEXT:   WIDEN ir<%1> = load ir<%arrayidx2>, vp<%2>
+; NO-VP-NEXT:   WIDEN ir<%add> = add ir<%1>, ir<%0>
+; NO-VP-NEXT:   CLONE ir<%arrayidx4> = getelementptr ir<%a>, ir<%indvars.iv>
+; NO-VP-NEXT:   WIDEN store ir<%arrayidx4>, ir<%add>, vp<%2>
+; NO-VP-NEXT: No successors
+; NO-VP-NEXT: }
+
+entry:
+  %cmp10 = icmp sgt i64 %N, 0
+  br i1 %cmp10, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.body
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %c, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx4 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx4, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %N
+  br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body
+}