[ARM][MVE] Tail-predication: support constant trip count

[SjoerdMeijer]

Summary

Let's also support the easy/easier case: constant trip counts :)

And while I was at it, a minor optimisation: I don't think we need to run Cleanup when there are no NewPredicates; it wasn't doing anything other than iterating over empty sets, but thought not calling it all was more obvious; I had to move the initialisation of ClonedVCTPInExitBlock for that.

[samparker]

llvm/lib/Target/ARM/MVETailPredication.cpp

444–445

But the number of elements isn't equal to the trip count!

[SjoerdMeijer]

llvm/lib/Target/ARM/MVETailPredication.cpp

444–445

Ha, the easy case, what a joke. Had myself convinced codegen was okay, but isn't. This indeed needs some more thinking how to approach this.

[SjoerdMeijer]

Thanks for catching that. This now gets the scalar loop iteration count from the icmp, which has as a RHS value a constant vector, i.e. a splat value with the scalar loop iteration count. This is returned as the computed number of elements for loops with constant tripcounts.

[samparker]

llvm/lib/Target/ARM/MVETailPredication.cpp

355

Looks like this could just be in MatchICmpConstVec? Save us from calling MatchICmpConstVec twice and reduces the logic involved.

640

With those functions combined, we could then just pass the constant value to isTailPredicate. So something like:

Value *NumElements = nullptr
if (!MatchICmpConstVec(Predicate, NumElements) && !ComputeElements(Predicate))
  continue;

This would allow our logic to diverge early, but maybe only once, instead of weaving between the two in the other two functions.

[SjoerdMeijer]

Thanks for the suggestion, I have refactored things: we now have 2 functions ComputeRuntimeElements() and ComputeConstElements that match the number of elements and feed that to isTailPredicate(), so that none of this decision making has to be done in isTailPredicate() anymore.

[SjoerdMeijer]

Minor fix in the assert in ComputeConstElements which looked at the wrong constant.

[samparker]

llvm/lib/Target/ARM/MVETailPredication.cpp

271

Doesn't look like you need ICmp or Shuffle here.

277

Combine with ComputeRuntimeElements? I'm finding really hard to follow what's producing what and why some things are pointers and others of pointer pointers...

654

if MatchElemCountLoopSetup was merged with ComputeRuntimeElements, we can also get rid of the duplicated call to isTailPredicate and the lambda.

if (NumElements = ComputeConst...)
else if (NumElements = ComputeRuntime...)
else continue;

if (!isTailPredicate...) continue;

[SjoerdMeijer]

Introduced a struct for the tripcount pattern matching bookkeeping, simplifying passing values around and the main loop driving the analysis.

[samparker]

Nice. LGTM.

llvm/lib/Target/ARM/MVETailPredication.cpp

 #define DESC "Transform predicated vector loops to use MVE tail predication"
 
 cl::opt<bool>
 DisableTailPredication("disable-mve-tail-predication", cl::Hidden,
                        cl::init(true),
                        cl::desc("Disable MVE Tail Predication"));
 namespace {
 
+// Bookkeeping for pattern matching the loop trip count and the number of
+// elements processed by the loop.
+struct TripCountPattern {
+  // The Predicate used by the masked loads/stores, i.e. an icmp instruction
+  // which calculates active/inactive lanes
+  Instruction *Predicate = nullptr;
+
+  // The add instruction that increments the IV
+  Value *TripCount = nullptr;
+
+  // The number of elements processed by the vector loop.
+  Value *NumElements = nullptr;
+
+  VectorType *VecTy = nullptr;
+  Instruction *Shuffle = nullptr;
+  Instruction *Induction = nullptr;
+
+  TripCountPattern(Instruction *P, Value *TC, VectorType *VT)
+      : Predicate(P), TripCount(TC), VecTy(VT){};
+};
+
 class MVETailPredication : public LoopPass {
   SmallVector<IntrinsicInst*, 4> MaskedInsts;
   Loop *L = nullptr;
   LoopInfo *LI = nullptr;
   const DataLayout *DL;
   DominatorTree *DT = nullptr;
   ScalarEvolution *SE = nullptr;
   TargetTransformInfo *TTI = nullptr;
void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addRequired<TargetLibraryInfoWrapperPass>();
     AU.addPreserved<LoopInfoWrapperPass>();
     AU.setPreservesCFG();
   }
 
   bool runOnLoop(Loop *L, LPPassManager&) override;
 
 private:
-
   /// Perform the relevant checks on the loop and convert if possible.
   bool TryConvert(Value *TripCount);
 
   /// Return whether this is a vectorized loop, that contains masked
   /// load/stores.
   bool IsPredicatedVectorLoop();
 
   /// Compute a value for the total number of elements that the predicated
-  /// loop will process.
-  Value *ComputeElements(Value *TripCount, VectorType *VecTy);
+  /// loop will process if it is a runtime value.
+  bool ComputeRuntimeElements(TripCountPattern &TCP);
 
   /// Is the icmp that generates an i1 vector, based upon a loop counter
   /// and a limit that is defined outside the loop.
-  bool isTailPredicate(Instruction *Predicate, Value *NumElements);
+  bool isTailPredicate(TripCountPattern &TCP);
 
   /// Insert the intrinsic to represent the effect of tail predication.
-  void InsertVCTPIntrinsic(Instruction *Predicate,
-                           DenseMap<Instruction*, Instruction*> &NewPredicates,
-                           VectorType *VecTy,
-                           Value *NumElements);
+  void InsertVCTPIntrinsic(TripCountPattern &TCP,
+                           DenseMap<Instruction *, Instruction *> &NewPredicates);
 
   /// Rematerialize the iteration count in exit blocks, which enables
   /// ARMLowOverheadLoops to better optimise away loop update statements inside
   /// hardware-loops.
   void RematerializeIterCount();
 };
 
 } // end namespace
for (auto &I : *BB) {
         break;
       }
     }
   }
 
   if (!Decrement)
     return false;
 
+  ClonedVCTPInExitBlock = false;
   LLVM_DEBUG(dbgs() << "ARM TP: Running on Loop: " << *L << *Setup << "\n"
              << *Decrement << "\n");
 
   if (TryConvert(Setup->getArgOperand(0))) {
     if (ClonedVCTPInExitBlock)
       RematerializeIterCount();
     return true;
   }
 
   return false;
 }
 
-bool MVETailPredication::isTailPredicate(Instruction *I, Value *NumElements) {
-  // Look for the following:
+// Pattern match predicates/masks and determine if they use the loop induction
+// variable to control the number of elements processed by the loop. If so,
+// the loop is a candidate for tail-predication.
+bool MVETailPredication::isTailPredicate(TripCountPattern &TCP) {
+  using namespace PatternMatch;
 
-  // %trip.count.minus.1 = add i32 %N, -1
-  // %broadcast.splatinsert10 = insertelement <4 x i32> undef,
-  //                                          i32 %trip.count.minus.1, i32 0
-  // %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10,
-  //                                    <4 x i32> undef,
-  //                                    <4 x i32> zeroinitializer
-  // ...
-  // ...
+  // Pattern match the loop body and find the add with takes the index iv
+  // and adds a constant vector to it:
+  //
+  // vector.body:
+  // ..
   // %index = phi i32
   // %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
   // %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert,
   //                                  <4 x i32> undef,
   //                                  <4 x i32> zeroinitializer
   // %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
   // %pred = icmp ule <4 x i32> %induction, %broadcast.splat11
 
-  // And return whether V == %pred.
-
-  using namespace PatternMatch;
-
-  CmpInst::Predicate Pred;
-  Instruction *Shuffle = nullptr;
-  Instruction *Induction = nullptr;
-
-  // The vector icmp
-  if (!match(I, m_ICmp(Pred, m_Instruction(Induction),
-                       m_Instruction(Shuffle))) ||
-      Pred != ICmpInst::ICMP_ULE)
-    return false;
-
-  // First find the stuff outside the loop which is setting up the limit
-  // vector....
-  // The invariant shuffle that broadcast the limit into a vector.
-  Instruction *Insert = nullptr;
-  if (!match(Shuffle, m_ShuffleVector(m_Instruction(Insert), m_Undef(),
-                                      m_Zero())))
-    return false;
-
-  // Insert the limit into a vector.
-  Instruction *BECount = nullptr;
-  if (!match(Insert, m_InsertElement(m_Undef(), m_Instruction(BECount),
-                                     m_Zero())))
-    return false;
-
-  // The limit calculation, backedge count.
-  Value *TripCount = nullptr;
-  if (!match(BECount, m_Add(m_Value(TripCount), m_AllOnes())))
-    return false;
-
-  if (TripCount != NumElements || !L->isLoopInvariant(BECount))
-    return false;
-
-  // Now back to searching inside the loop body...
-  // Find the add with takes the index iv and adds a constant vector to it.
   Instruction *BroadcastSplat = nullptr;
   Constant *Const = nullptr;
-  if (!match(Induction, m_Add(m_Instruction(BroadcastSplat),
-                              m_Constant(Const))))
+  if (!match(TCP.Induction,
+             m_Add(m_Instruction(BroadcastSplat), m_Constant(Const))))
     return false;
 
   // Check that we're adding <0, 1, 2, 3...
   if (auto *CDS = dyn_cast<ConstantDataSequential>(Const)) {
     for (unsigned i = 0; i < CDS->getNumElements(); ++i) {
       if (CDS->getElementAsInteger(i) != i)
         return false;
     }
   } else
     return false;
 
+  Instruction *Insert = nullptr;
   // The shuffle which broadcasts the index iv into a vector.
-  if (!match(BroadcastSplat, m_ShuffleVector(m_Instruction(Insert), m_Undef(),
-                                             m_Zero())))
+  if (!match(BroadcastSplat,
+             m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_Zero())))
     return false;
 
   // The insert element which initialises a vector with the index iv.
   Instruction *IV = nullptr;
   if (!match(Insert, m_InsertElement(m_Undef(), m_Instruction(IV), m_Zero())))
     return false;
 
   // The index iv.
bool MVETailPredication::isTailPredicate(TripCountPattern &TCP) {
 
   Instruction *LHS = nullptr;
   if (!match(InLoop, m_Add(m_Instruction(LHS), m_SpecificInt(Lanes))))
     return false;
 
   return LHS == Phi;
 }
 
-static VectorType* getVectorType(IntrinsicInst *I) {
+static VectorType *getVectorType(IntrinsicInst *I) {
   unsigned TypeOp = I->getIntrinsicID() == Intrinsic::masked_load ? 0 : 1;
   auto *PtrTy = cast<PointerType>(I->getOperand(TypeOp)->getType());
   return cast<VectorType>(PtrTy->getElementType());
 }
 
 bool MVETailPredication::IsPredicatedVectorLoop() {
   // Check that the loop contains at least one masked load/store intrinsic.
   // We only support 'normal' vector instructions - other than masked
for (auto &I : *BB) {
         }
       }
     }
   }
 
   return !MaskedInsts.empty();
 }
 
-Value* MVETailPredication::ComputeElements(Value *TripCount,
-                                           VectorType *VecTy) {
-  const SCEV *TripCountSE = SE->getSCEV(TripCount);
-  ConstantInt *VF = ConstantInt::get(cast<IntegerType>(TripCount->getType()),
-                                     VecTy->getNumElements());
+// Pattern match the predicate, which is an icmp with a constant vector of this
+// form:
+//
+//   icmp ult <4 x i32> %induction, <i32 32002, i32 32002, i32 32002, i32 32002>
+//
+// and return the constant, i.e. 32002 in this example. This is assumed to be
+// the scalar loop iteration count: the number of loop elements by the
+// the vector loop. Further checks are performed in function isTailPredicate(),
+// to verify 'induction' behaves as an induction variable.
+//
+static bool ComputeConstElements(TripCountPattern &TCP) {
+  if (!dyn_cast<ConstantInt>(TCP.TripCount))
+    return false;
+
+  ConstantInt *VF = ConstantInt::get(
+      cast<IntegerType>(TCP.TripCount->getType()), TCP.VecTy->getNumElements());
+  using namespace PatternMatch;
+  CmpInst::Predicate CC;
+
+  if (!match(TCP.Predicate, m_ICmp(CC, m_Instruction(TCP.Induction),
+                                   m_AnyIntegralConstant())) ||
+      CC != ICmpInst::ICMP_ULT)
+    return false;
+
+  LLVM_DEBUG(dbgs() << "ARM TP: icmp with constants: "; TCP.Predicate->dump(););
+  Value *ConstVec = TCP.Predicate->getOperand(1);
+
+  auto *CDS = dyn_cast<ConstantDataSequential>(ConstVec);
+  if (!CDS || CDS->getNumElements() != VF->getSExtValue())
+    return false;
+
+  if ((TCP.NumElements = CDS->getSplatValue())) {
+    assert(dyn_cast<ConstantInt>(TCP.NumElements)->getSExtValue() %
+                   VF->getSExtValue() !=
+               0 &&
+           "tail-predication: trip count should not be a multiple of the VF");
+    LLVM_DEBUG(dbgs() << "ARM TP: Found const elem count: " << *TCP.NumElements
+                      << "\n");
+    return true;
+  }
+  return false;
+}
+
+// Pattern match the loop iteration count setup:
+//
+// %trip.count.minus.1 = add i32 %N, -1
+// %broadcast.splatinsert10 = insertelement <4 x i32> undef,
+//                                          i32 %trip.count.minus.1, i32 0
+// %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10,
+//                                    <4 x i32> undef,
+//                                    <4 x i32> zeroinitializer
+// ..
+// vector.body:
+// ..
+//
+static bool MatchElemCountLoopSetup(Loop *L, Instruction *Shuffle,
+                                    Value *NumElements) {
+  using namespace PatternMatch;
+  Instruction *Insert = nullptr;
+
+  if (!match(Shuffle,
+             m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_Zero())))
+    return false;
+
+  // Insert the limit into a vector.
+  Instruction *BECount = nullptr;
+  if (!match(Insert,
+             m_InsertElement(m_Undef(), m_Instruction(BECount), m_Zero())))
+    return false;
+
+  // The limit calculation, backedge count.
+  Value *TripCount = nullptr;
+  if (!match(BECount, m_Add(m_Value(TripCount), m_AllOnes())))
+    return false;
+
+  if (TripCount != NumElements || !L->isLoopInvariant(BECount))
+    return false;
+
+  return true;
+}
+
+bool MVETailPredication::ComputeRuntimeElements(TripCountPattern &TCP) {
+  using namespace PatternMatch;
+  const SCEV *TripCountSE = SE->getSCEV(TCP.TripCount);
+  ConstantInt *VF = ConstantInt::get(
+      cast<IntegerType>(TCP.TripCount->getType()), TCP.VecTy->getNumElements());
 
   if (VF->equalsInt(1))
-    return nullptr;
+    return false;
+
+  CmpInst::Predicate Pred;
+  if (!match(TCP.Predicate, m_ICmp(Pred, m_Instruction(TCP.Induction),
+                                   m_Instruction(TCP.Shuffle))) ||
+      Pred != ICmpInst::ICMP_ULE)
+    return false;
 
-  // TODO: Support constant trip counts.
-  auto VisitAdd = [&](const SCEVAddExpr *S) -> const SCEVMulExpr* {
+  LLVM_DEBUG(dbgs() << "Computing number of elements for vector trip count: ";
+             TCP.TripCount->dump());
+
+  // Otherwise, continue and try to pattern match the vector iteration
+  // count expression
+  auto VisitAdd = [&](const SCEVAddExpr *S) -> const SCEVMulExpr * {
     if (auto *Const = dyn_cast<SCEVConstant>(S->getOperand(0))) {
       if (Const->getAPInt() != -VF->getValue())
         return nullptr;
     } else
       return nullptr;
     return dyn_cast<SCEVMulExpr>(S->getOperand(1));
   };
 
-  auto VisitMul = [&](const SCEVMulExpr *S) -> const SCEVUDivExpr* {
+  auto VisitMul = [&](const SCEVMulExpr *S) -> const SCEVUDivExpr * {
     if (auto *Const = dyn_cast<SCEVConstant>(S->getOperand(0))) {
       if (Const->getValue() != VF)
         return nullptr;
     } else
       return nullptr;
     return dyn_cast<SCEVUDivExpr>(S->getOperand(1));
   };
 
-  auto VisitDiv = [&](const SCEVUDivExpr *S) -> const SCEV* {
+  auto VisitDiv = [&](const SCEVUDivExpr *S) -> const SCEV * {
     if (auto *Const = dyn_cast<SCEVConstant>(S->getRHS())) {
       if (Const->getValue() != VF)
         return nullptr;
     } else
       return nullptr;
 
     if (auto *RoundUp = dyn_cast<SCEVAddExpr>(S->getLHS())) {
       if (auto *Const = dyn_cast<SCEVConstant>(RoundUp->getOperand(0))) {
if (auto *TC = dyn_cast<SCEVAddExpr>(TripCountSE))
     if (auto *Div = dyn_cast<SCEVUDivExpr>(TC->getOperand(1)))
       if (auto *Add = dyn_cast<SCEVAddExpr>(Div->getLHS()))
         if (auto *Mul = VisitAdd(Add))
           if (auto *Div = VisitMul(Mul))
             if (auto *Res = VisitDiv(Div))
               Elems = Res;
 
   if (!Elems)
-    return nullptr;
+    return false;
 
   Instruction *InsertPt = L->getLoopPreheader()->getTerminator();
   if (!isSafeToExpandAt(Elems, InsertPt, *SE))
-    return nullptr;
+    return false;
 
   auto DL = L->getHeader()->getModule()->getDataLayout();
   SCEVExpander Expander(*SE, DL, "elements");
-  return Expander.expandCodeFor(Elems, Elems->getType(), InsertPt);
+  TCP.NumElements = Expander.expandCodeFor(Elems, Elems->getType(), InsertPt);
+
+  if (!MatchElemCountLoopSetup(L, TCP.Shuffle, TCP.NumElements))
+    return false;
+
+  return true;
 }
 
 // Look through the exit block to see whether there's a duplicate predicate
 // instruction. This can happen when we need to perform a select on values
 // from the last and previous iteration. Instead of doing a straight
 // replacement of that predicate with the vctp, clone the vctp and place it
 // in the block. This means that the VPR doesn't have to be live into the
 // exit block which should make it easier to convert this loop into a proper
static bool Cleanup(DenseMap<Instruction*, Instruction*> &NewPredicates,
   }
 
   for (auto I : L->blocks())
     DeleteDeadPHIs(I);
 
   return ClonedVCTPInExitBlock;
 }
 
-void MVETailPredication::InsertVCTPIntrinsic(Instruction *Predicate,
-    DenseMap<Instruction*, Instruction*> &NewPredicates,
-    VectorType *VecTy, Value *NumElements) {
+void MVETailPredication::InsertVCTPIntrinsic(TripCountPattern &TCP,
+    DenseMap<Instruction*, Instruction*> &NewPredicates) {
   IRBuilder<> Builder(L->getHeader()->getFirstNonPHI());
   Module *M = L->getHeader()->getModule();
   Type *Ty = IntegerType::get(M->getContext(), 32);
 
   // Insert a phi to count the number of elements processed by the loop.
   PHINode *Processed = Builder.CreatePHI(Ty, 2);
-  Processed->addIncoming(NumElements, L->getLoopPreheader());
+  Processed->addIncoming(TCP.NumElements, L->getLoopPreheader());
 
   // Insert the intrinsic to represent the effect of tail predication.
-  Builder.SetInsertPoint(cast<Instruction>(Predicate));
+  Builder.SetInsertPoint(cast<Instruction>(TCP.Predicate));
   ConstantInt *Factor =
-    ConstantInt::get(cast<IntegerType>(Ty), VecTy->getNumElements());
+    ConstantInt::get(cast<IntegerType>(Ty), TCP.VecTy->getNumElements());
 
   Intrinsic::ID VCTPID;
-  switch (VecTy->getNumElements()) {
+  switch (TCP.VecTy->getNumElements()) {
   default:
     llvm_unreachable("unexpected number of lanes");
   case 4:  VCTPID = Intrinsic::arm_mve_vctp32; break;
   case 8:  VCTPID = Intrinsic::arm_mve_vctp16; break;
   case 16: VCTPID = Intrinsic::arm_mve_vctp8; break;
 
     // FIXME: vctp64 currently not supported because the predicate
     // vector wants to be <2 x i1>, but v2i1 is not a legal MVE
     // type, so problems happen at isel time.
     // Intrinsic::arm_mve_vctp64 exists for ACLE intrinsics
     // purposes, but takes a v4i1 instead of a v2i1.
   }
   Function *VCTP = Intrinsic::getDeclaration(M, VCTPID);
   Value *TailPredicate = Builder.CreateCall(VCTP, Processed);
-  Predicate->replaceAllUsesWith(TailPredicate);
-  NewPredicates[Predicate] = cast<Instruction>(TailPredicate);
+  TCP.Predicate->replaceAllUsesWith(TailPredicate);
+  NewPredicates[TCP.Predicate] = cast<Instruction>(TailPredicate);
 
   // Add the incoming value to the new phi.
   // TODO: This add likely already exists in the loop.
   Value *Remaining = Builder.CreateSub(Processed, Factor);
   Processed->addIncoming(Remaining, L->getLoopLatch());
   LLVM_DEBUG(dbgs() << "ARM TP: Insert processed elements phi: "
              << *Processed << "\n"
              << "ARM TP: Inserted VCTP: " << *TailPredicate << "\n");
 }
 
 bool MVETailPredication::TryConvert(Value *TripCount) {
   if (!IsPredicatedVectorLoop()) {
-    LLVM_DEBUG(dbgs() << "ARM TP: no masked instructions in loop");
+    LLVM_DEBUG(dbgs() << "ARM TP: no masked instructions in loop.\n");
     return false;
   }
 
   LLVM_DEBUG(dbgs() << "ARM TP: Found predicated vector loop.\n");
 
   // Walk through the masked intrinsics and try to find whether the predicate
   // operand is generated from an induction variable.
   SetVector<Instruction*> Predicates;
   DenseMap<Instruction*, Instruction*> NewPredicates;
 
   for (auto *I : MaskedInsts) {
     Intrinsic::ID ID = I->getIntrinsicID();
     unsigned PredOp = ID == Intrinsic::masked_load ? 2 : 3;
     auto *Predicate = dyn_cast<Instruction>(I->getArgOperand(PredOp));
     if (!Predicate || Predicates.count(Predicate))
       continue;
 
-    VectorType *VecTy = getVectorType(I);
-    Value *NumElements = ComputeElements(TripCount, VecTy);
-    if (!NumElements)
+    TripCountPattern TCP(Predicate, TripCount, getVectorType(I));
+
+    if (!(ComputeConstElements(TCP) || ComputeRuntimeElements(TCP)))
       continue;
 
-    if (!isTailPredicate(Predicate, NumElements)) {
+    if (!isTailPredicate(TCP)) {
       LLVM_DEBUG(dbgs() << "ARM TP: Not tail predicate: " << *Predicate << "\n");
       continue;
     }
 
     LLVM_DEBUG(dbgs() << "ARM TP: Found tail predicate: " << *Predicate << "\n");
     Predicates.insert(Predicate);
-
-    InsertVCTPIntrinsic(Predicate, NewPredicates, VecTy, NumElements);
+    InsertVCTPIntrinsic(TCP, NewPredicates);
   }
 
+  if (!NewPredicates.size())
+    return false;
+
   // Now clean up.
   ClonedVCTPInExitBlock = Cleanup(NewPredicates, Predicates, L);
   return true;
 }
 
 Pass *llvm::createMVETailPredicationPass() {
   return new MVETailPredication();
 }
 
 char MVETailPredication::ID = 0;
 
 INITIALIZE_PASS_BEGIN(MVETailPredication, DEBUG_TYPE, DESC, false, false)
 INITIALIZE_PASS_END(MVETailPredication, DEBUG_TYPE, DESC, false, false)

llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-const.ll

+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve %s -S -o - | FileCheck %s
+
+define dso_local void @foo(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C, i32* noalias nocapture readnone %D, i32 %N) local_unnamed_addr #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.set.loop.iterations.i32(i32 8001)
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[LSR_IV14:%.*]] = phi i32* [ [[SCEVGEP15:%.*]], [[VECTOR_BODY]] ], [ [[A:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[LSR_IV11:%.*]] = phi i32* [ [[SCEVGEP12:%.*]], [[VECTOR_BODY]] ], [ [[C:%.*]], [[ENTRY]] ]
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i32* [ [[SCEVGEP:%.*]], [[VECTOR_BODY]] ], [ [[B:%.*]], [[ENTRY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ 8001, [[ENTRY]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ 32002, [[ENTRY]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[LSR_IV1416:%.*]] = bitcast i32* [[LSR_IV14]] to <4 x i32>*
+; CHECK-NEXT:    [[LSR_IV1113:%.*]] = bitcast i32* [[LSR_IV11]] to <4 x i32>*
+; CHECK-NEXT:    [[LSR_IV10:%.*]] = bitcast i32* [[LSR_IV]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.vctp32(i32 [[TMP1]])
+; CHECK-NEXT:    [[TMP3]] = sub i32 [[TMP1]], 4
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[LSR_IV10]], i32 4, <4 x i1> [[TMP2]], <4 x i32> undef)
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[LSR_IV1113]], i32 4, <4 x i1> [[TMP2]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD9]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[TMP4]], <4 x i32>* [[LSR_IV1416]], i32 4, <4 x i1> [[TMP2]])
+; CHECK-NEXT:    [[SCEVGEP]] = getelementptr i32, i32* [[LSR_IV]], i32 4
+; CHECK-NEXT:    [[SCEVGEP12]] = getelementptr i32, i32* [[LSR_IV11]], i32 4
+; CHECK-NEXT:    [[SCEVGEP15]] = getelementptr i32, i32* [[LSR_IV14]], i32 4
+; CHECK-NEXT:    [[TMP5]] = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 [[TMP0]], i32 1)
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i32 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void @llvm.set.loop.iterations.i32(i32 8001)
+  br label %vector.body
+
+vector.body:
+  %lsr.iv14 = phi i32* [ %scevgep15, %vector.body ], [ %A, %entry ]
+  %lsr.iv11 = phi i32* [ %scevgep12, %vector.body ], [ %C, %entry ]
+  %lsr.iv = phi i32* [ %scevgep, %vector.body ], [ %B, %entry ]
+  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
+  %0 = phi i32 [ 8001, %entry ], [ %3, %vector.body ]
+  %lsr.iv1416 = bitcast i32* %lsr.iv14 to <4 x i32>*
+  %lsr.iv1113 = bitcast i32* %lsr.iv11 to <4 x i32>*
+  %lsr.iv10 = bitcast i32* %lsr.iv to <4 x i32>*
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %1 = icmp ult <4 x i32> %induction, <i32 32002, i32 32002, i32 32002, i32 32002>
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %lsr.iv10, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %wide.masked.load9 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %lsr.iv1113, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %2 = add nsw <4 x i32> %wide.masked.load9, %wide.masked.load
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %2, <4 x i32>* %lsr.iv1416, i32 4, <4 x i1> %1)
+  %index.next = add i32 %index, 4
+  %scevgep = getelementptr i32, i32* %lsr.iv, i32 4
+  %scevgep12 = getelementptr i32, i32* %lsr.iv11, i32 4
+  %scevgep15 = getelementptr i32, i32* %lsr.iv14, i32 4
+  %3 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %0, i32 1)
+  %4 = icmp ne i32 %3, 0
+  br i1 %4, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+}
+
+; Silly test case: the loop count is constant and a multiple of the vectorisation
+; factor. So, the vectoriser should not produce masked loads/stores and there's
+; nothing to tail-predicate here, just checking.
+define dso_local void @foo2(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C, i32* noalias nocapture readnone %D, i32 %N) local_unnamed_addr #0 {
+; CHECK-LABEL: @foo2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.set.loop.iterations.i32(i32 2000)
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[LSR_IV14:%.*]] = phi i32* [ [[SCEVGEP15:%.*]], [[VECTOR_BODY]] ], [ [[A:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[LSR_IV11:%.*]] = phi i32* [ [[SCEVGEP12:%.*]], [[VECTOR_BODY]] ], [ [[C:%.*]], [[ENTRY]] ]
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i32* [ [[SCEVGEP:%.*]], [[VECTOR_BODY]] ], [ [[B:%.*]], [[ENTRY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ 2000, [[ENTRY]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[LSR_IV1416:%.*]] = bitcast i32* [[LSR_IV14]] to <4 x i32>*
+; CHECK-NEXT:    [[LSR_IV1113:%.*]] = bitcast i32* [[LSR_IV11]] to <4 x i32>*
+; CHECK-NEXT:    [[LSR_IV10:%.*]] = bitcast i32* [[LSR_IV]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[LSR_IV10]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD9:%.*]] = load <4 x i32>, <4 x i32>* [[LSR_IV1113]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD9]], [[WIDE_LOAD]]
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[LSR_IV1416]], align 4
+; CHECK-NEXT:    [[SCEVGEP]] = getelementptr i32, i32* [[LSR_IV]], i32 4
+; CHECK-NEXT:    [[SCEVGEP12]] = getelementptr i32, i32* [[LSR_IV11]], i32 4
+; CHECK-NEXT:    [[SCEVGEP15]] = getelementptr i32, i32* [[LSR_IV14]], i32 4
+; CHECK-NEXT:    [[TMP2]] = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 [[TMP0]], i32 1)
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[TMP3]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void @llvm.set.loop.iterations.i32(i32 2000)
+  br label %vector.body
+
+vector.body:
+  %lsr.iv14 = phi i32* [ %scevgep15, %vector.body ], [ %A, %entry ]
+  %lsr.iv11 = phi i32* [ %scevgep12, %vector.body ], [ %C, %entry ]
+  %lsr.iv = phi i32* [ %scevgep, %vector.body ], [ %B, %entry ]
+  %0 = phi i32 [ 2000, %entry ], [ %2, %vector.body ]
+  %lsr.iv1416 = bitcast i32* %lsr.iv14 to <4 x i32>*
+  %lsr.iv1113 = bitcast i32* %lsr.iv11 to <4 x i32>*
+  %lsr.iv10 = bitcast i32* %lsr.iv to <4 x i32>*
+  %wide.load = load <4 x i32>, <4 x i32>* %lsr.iv10, align 4
+  %wide.load9 = load <4 x i32>, <4 x i32>* %lsr.iv1113, align 4
+  %1 = add nsw <4 x i32> %wide.load9, %wide.load
+  store <4 x i32> %1, <4 x i32>* %lsr.iv1416, align 4
+  %scevgep = getelementptr i32, i32* %lsr.iv, i32 4
+  %scevgep12 = getelementptr i32, i32* %lsr.iv11, i32 4
+  %scevgep15 = getelementptr i32, i32* %lsr.iv14, i32 4
+  %2 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %0, i32 1)
+  %3 = icmp ne i32 %2, 0
+  br i1 %3, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+}
+
+; Check that the icmp is a ult
+define dso_local void @foo3(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C, i32* noalias nocapture readnone %D, i32 %N) local_unnamed_addr #0 {
+; CHECK-LABEL: @foo3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.set.loop.iterations.i32(i32 8001)
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[LSR_IV14:%.*]] = phi i32* [ [[SCEVGEP15:%.*]], [[VECTOR_BODY]] ], [ [[A:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[LSR_IV11:%.*]] = phi i32* [ [[SCEVGEP12:%.*]], [[VECTOR_BODY]] ], [ [[C:%.*]], [[ENTRY]] ]
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i32* [ [[SCEVGEP:%.*]], [[VECTOR_BODY]] ], [ [[B:%.*]], [[ENTRY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ 8001, [[ENTRY]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[LSR_IV1416:%.*]] = bitcast i32* [[LSR_IV14]] to <4 x i32>*
+; CHECK-NEXT:    [[LSR_IV1113:%.*]] = bitcast i32* [[LSR_IV11]] to <4 x i32>*
+; CHECK-NEXT:    [[LSR_IV10:%.*]] = bitcast i32* [[LSR_IV]] to <4 x i32>*
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> undef, i32 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <4 x i32> [[INDUCTION]], <i32 32002, i32 32002, i32 32002, i32 32002>
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[LSR_IV10]], i32 4, <4 x i1> [[TMP1]], <4 x i32> undef)
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[LSR_IV1113]], i32 4, <4 x i1> [[TMP1]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD9]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[TMP2]], <4 x i32>* [[LSR_IV1416]], i32 4, <4 x i1> [[TMP1]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[SCEVGEP]] = getelementptr i32, i32* [[LSR_IV]], i32 4
+; CHECK-NEXT:    [[SCEVGEP12]] = getelementptr i32, i32* [[LSR_IV11]], i32 4
+; CHECK-NEXT:    [[SCEVGEP15]] = getelementptr i32, i32* [[LSR_IV14]], i32 4
+; CHECK-NEXT:    [[TMP3]] = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 [[TMP0]], i32 1)
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[TMP4]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void @llvm.set.loop.iterations.i32(i32 8001)
+  br label %vector.body
+
+vector.body:
+  %lsr.iv14 = phi i32* [ %scevgep15, %vector.body ], [ %A, %entry ]
+  %lsr.iv11 = phi i32* [ %scevgep12, %vector.body ], [ %C, %entry ]
+  %lsr.iv = phi i32* [ %scevgep, %vector.body ], [ %B, %entry ]
+  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
+  %0 = phi i32 [ 8001, %entry ], [ %3, %vector.body ]
+  %lsr.iv1416 = bitcast i32* %lsr.iv14 to <4 x i32>*
+  %lsr.iv1113 = bitcast i32* %lsr.iv11 to <4 x i32>*
+  %lsr.iv10 = bitcast i32* %lsr.iv to <4 x i32>*
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+
+; UGT here:
+  %1 = icmp ugt <4 x i32> %induction, <i32 32002, i32 32002, i32 32002, i32 32002>
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %lsr.iv10, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %wide.masked.load9 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %lsr.iv1113, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %2 = add nsw <4 x i32> %wide.masked.load9, %wide.masked.load
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %2, <4 x i32>* %lsr.iv1416, i32 4, <4 x i1> %1)
+  %index.next = add i32 %index, 4
+  %scevgep = getelementptr i32, i32* %lsr.iv, i32 4
+  %scevgep12 = getelementptr i32, i32* %lsr.iv11, i32 4
+  %scevgep15 = getelementptr i32, i32* %lsr.iv14, i32 4
+  %3 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %0, i32 1)
+  %4 = icmp ne i32 %3, 0
+  br i1 %4, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+}
+
+; Check that this loop behaves as expected, i.e, that the loop increment is
+; an increment and not a decrement.
+define dso_local void @foo4(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C, i32* noalias nocapture readnone %D, i32 %N) local_unnamed_addr #0 {
+; CHECK-LABEL: @foo4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.set.loop.iterations.i32(i32 8001)
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[LSR_IV14:%.*]] = phi i32* [ [[SCEVGEP15:%.*]], [[VECTOR_BODY]] ], [ [[A:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[LSR_IV11:%.*]] = phi i32* [ [[SCEVGEP12:%.*]], [[VECTOR_BODY]] ], [ [[C:%.*]], [[ENTRY]] ]
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i32* [ [[SCEVGEP:%.*]], [[VECTOR_BODY]] ], [ [[B:%.*]], [[ENTRY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ 8001, [[ENTRY]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[LSR_IV1416:%.*]] = bitcast i32* [[LSR_IV14]] to <4 x i32>*
+; CHECK-NEXT:    [[LSR_IV1113:%.*]] = bitcast i32* [[LSR_IV11]] to <4 x i32>*
+; CHECK-NEXT:    [[LSR_IV10:%.*]] = bitcast i32* [[LSR_IV]] to <4 x i32>*
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> undef, i32 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <4 x i32> [[INDUCTION]], <i32 32002, i32 32002, i32 32002, i32 32002>
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[LSR_IV10]], i32 4, <4 x i1> [[TMP1]], <4 x i32> undef)
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[LSR_IV1113]], i32 4, <4 x i1> [[TMP1]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD9]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[TMP2]], <4 x i32>* [[LSR_IV1416]], i32 4, <4 x i1> [[TMP1]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = sub i32 [[INDEX]], 4
+; CHECK-NEXT:    [[SCEVGEP]] = getelementptr i32, i32* [[LSR_IV]], i32 4
+; CHECK-NEXT:    [[SCEVGEP12]] = getelementptr i32, i32* [[LSR_IV11]], i32 4
+; CHECK-NEXT:    [[SCEVGEP15]] = getelementptr i32, i32* [[LSR_IV14]], i32 4
+; CHECK-NEXT:    [[TMP3]] = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 [[TMP0]], i32 1)
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[TMP4]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void @llvm.set.loop.iterations.i32(i32 8001)
+  br label %vector.body
+
+vector.body:
+  %lsr.iv14 = phi i32* [ %scevgep15, %vector.body ], [ %A, %entry ]
+  %lsr.iv11 = phi i32* [ %scevgep12, %vector.body ], [ %C, %entry ]
+  %lsr.iv = phi i32* [ %scevgep, %vector.body ], [ %B, %entry ]
+  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
+  %0 = phi i32 [ 8001, %entry ], [ %3, %vector.body ]
+  %lsr.iv1416 = bitcast i32* %lsr.iv14 to <4 x i32>*
+  %lsr.iv1113 = bitcast i32* %lsr.iv11 to <4 x i32>*
+  %lsr.iv10 = bitcast i32* %lsr.iv to <4 x i32>*
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %1 = icmp ult <4 x i32> %induction, <i32 32002, i32 32002, i32 32002, i32 32002>
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %lsr.iv10, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %wide.masked.load9 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %lsr.iv1113, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %2 = add nsw <4 x i32> %wide.masked.load9, %wide.masked.load
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %2, <4 x i32>* %lsr.iv1416, i32 4, <4 x i1> %1)
+
+; Counting down:
+  %index.next = sub i32 %index, 4
+  %scevgep = getelementptr i32, i32* %lsr.iv, i32 4
+  %scevgep12 = getelementptr i32, i32* %lsr.iv11, i32 4
+  %scevgep15 = getelementptr i32, i32* %lsr.iv14, i32 4
+  %3 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %0, i32 1)
+  %4 = icmp ne i32 %3, 0
+  br i1 %4, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+}
+
+define dso_local void @foo5(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C, i32* noalias nocapture readnone %D, i32 %N) local_unnamed_addr #0 {
+; CHECK-LABEL: @foo5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.set.loop.iterations.i32(i32 8001)
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[LSR_IV14:%.*]] = phi i32* [ [[SCEVGEP15:%.*]], [[VECTOR_BODY]] ], [ [[A:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[LSR_IV11:%.*]] = phi i32* [ [[SCEVGEP12:%.*]], [[VECTOR_BODY]] ], [ [[C:%.*]], [[ENTRY]] ]
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i32* [ [[SCEVGEP:%.*]], [[VECTOR_BODY]] ], [ [[B:%.*]], [[ENTRY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ 8001, [[ENTRY]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[LSR_IV1416:%.*]] = bitcast i32* [[LSR_IV14]] to <4 x i32>*
+; CHECK-NEXT:    [[LSR_IV1113:%.*]] = bitcast i32* [[LSR_IV11]] to <4 x i32>*
+; CHECK-NEXT:    [[LSR_IV10:%.*]] = bitcast i32* [[LSR_IV]] to <4 x i32>*
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> undef, i32 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <4 x i32> [[INDUCTION]], <i32 0, i32 3200, i32 32002, i32 32002>
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[LSR_IV10]], i32 4, <4 x i1> [[TMP1]], <4 x i32> undef)
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[LSR_IV1113]], i32 4, <4 x i1> [[TMP1]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD9]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[TMP2]], <4 x i32>* [[LSR_IV1416]], i32 4, <4 x i1> [[TMP1]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[SCEVGEP]] = getelementptr i32, i32* [[LSR_IV]], i32 4
+; CHECK-NEXT:    [[SCEVGEP12]] = getelementptr i32, i32* [[LSR_IV11]], i32 4
+; CHECK-NEXT:    [[SCEVGEP15]] = getelementptr i32, i32* [[LSR_IV14]], i32 4
+; CHECK-NEXT:    [[TMP3]] = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 [[TMP0]], i32 1)
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[TMP4]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void @llvm.set.loop.iterations.i32(i32 8001)
+  br label %vector.body
+
+vector.body:
+  %lsr.iv14 = phi i32* [ %scevgep15, %vector.body ], [ %A, %entry ]
+  %lsr.iv11 = phi i32* [ %scevgep12, %vector.body ], [ %C, %entry ]
+  %lsr.iv = phi i32* [ %scevgep, %vector.body ], [ %B, %entry ]
+  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
+  %0 = phi i32 [ 8001, %entry ], [ %3, %vector.body ]
+  %lsr.iv1416 = bitcast i32* %lsr.iv14 to <4 x i32>*
+  %lsr.iv1113 = bitcast i32* %lsr.iv11 to <4 x i32>*
+  %lsr.iv10 = bitcast i32* %lsr.iv to <4 x i32>*
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+
+; non-uniform constant vector here:
+  %1 = icmp ult <4 x i32> %induction, <i32 0, i32 3200, i32 32002, i32 32002>
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %lsr.iv10, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %wide.masked.load9 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %lsr.iv1113, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %2 = add nsw <4 x i32> %wide.masked.load9, %wide.masked.load
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %2, <4 x i32>* %lsr.iv1416, i32 4, <4 x i1> %1)
+  %index.next = add i32 %index, 4
+  %scevgep = getelementptr i32, i32* %lsr.iv, i32 4
+  %scevgep12 = getelementptr i32, i32* %lsr.iv11, i32 4
+  %scevgep15 = getelementptr i32, i32* %lsr.iv14, i32 4
+  %3 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %0, i32 1)
+  %4 = icmp ne i32 %3, 0
+  br i1 %4, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+}
+
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>) #1
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>) #2
+declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 , i32 )
+declare void @llvm.set.loop.iterations.i32(i32)