[ARM][MVE] Enable tail predication for loops containing MVE gather/scatters

[anwel]

Summary

Currently, MVE tail predication will only tail predicate a loop if there only are predicated loads and stores present. This patch widens the scope to also accept gathers and scatters, such that loops that are auto-vectorized with the option -enable-arm-maskedgatscat (and actually end up containing an MVE gather or scatter) can be tail predicated.

[dmgreen]

I was looking at this pass for something else recently. I'm not sure if the legality checks are really necessary in here. We might want to change it to just check for active lane masks and convert them to vctp's (when legal). I think that will always be better in terms of code quality than the expansion of the active lane mask, no matter if we end up transforming to a tail predicated loop or not.

That's not really relevant for this patch though, which looks OK as far as I can tell.

llvm/lib/Target/ARM/MVETailPredication.cpp

10–1

This comment can be removed now.

10–1

Can this entire function use:

• The return type of a load/gather
• The value type of a store/scatter

?

16

I don't think this pass should ever see valid Intrinsic::masked_scatter's that have not been transformed into some form of Intrinsic::arm_mve_vstr_...

[samparker]

llvm/lib/Target/ARM/ARMBaseInstrInfo.h

22

Like Dave also said, do we really need the generic IR opcodes in these helpers?

[anwel]

Changed getVectorType.

llvm/lib/Target/ARM/ARMBaseInstrInfo.h

22

In the way this function is intended to work, no we don't. Checking for the generic IR opcodes in the case the function is used for here might seem a bit overkill, but not harmful to the intentions of the check (please do correct me if I'm mistaken).
It is however only a re-use of the function, which was originally intended for a use in MVEGatherScatterLowering, for checking whether all users of a getelementptr instruction are gathers or scatters. Knowing this allows for certain transformations based on assumptions that can be made about such addresses. In that case it is important to include the generic ones, because at the time the check is made not all gathers and scatters have been transformed yet.
I was trying to avoid code duplication with this, but if your advice is that it would be better to separate the functionality more clearly, I'm happy to introduce an isMVEGather etc

llvm/lib/Target/ARM/MVETailPredication.cpp

10–1

Yes, if I see it correctly, it can. I changed it to do that.

[samparker]

llvm/lib/Target/ARM/ARMBaseInstrInfo.h

22

Okay, sounds fair enough to me.

[dmgreen]

Sounds good to me then, thanks.

llvm/lib/Target/ARM/ARMBaseInstrInfo.h

 #define LLVM_LIB_TARGET_ARM_ARMBASEINSTRINFO_H
 
 #include "MCTargetDesc/ARMBaseInfo.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsARM.h"
 #include <array>
 #include <cstdint>
 
 #define GET_INSTRINFO_HEADER
 #include "ARMGenInstrInfo.inc"
case ARMII::AddrModeT2_i8:
     return std::abs(Imm) < (((1 << 8) * 1) - 1);
   case ARMII::AddrModeT2_i12:
     return Imm >= 0 && Imm < (((1 << 12) * 1) - 1);
   default:
     llvm_unreachable("Unhandled Addressing mode");
   }
 }
 
-// Return true if the given intrinsic is a gather or scatter
-inline bool isGatherScatter(IntrinsicInst *IntInst) {
+// Return true if the given intrinsic is a gather
+inline bool isGather(IntrinsicInst *IntInst) {
   if (IntInst == nullptr)
     return false;
   unsigned IntrinsicID = IntInst->getIntrinsicID();
   return (IntrinsicID == Intrinsic::masked_gather ||
           IntrinsicID == Intrinsic::arm_mve_vldr_gather_base ||
           IntrinsicID == Intrinsic::arm_mve_vldr_gather_base_predicated ||
           IntrinsicID == Intrinsic::arm_mve_vldr_gather_base_wb ||
           IntrinsicID == Intrinsic::arm_mve_vldr_gather_base_wb_predicated ||
           IntrinsicID == Intrinsic::arm_mve_vldr_gather_offset ||
-          IntrinsicID == Intrinsic::arm_mve_vldr_gather_offset_predicated ||
-          IntrinsicID == Intrinsic::masked_scatter ||
+          IntrinsicID == Intrinsic::arm_mve_vldr_gather_offset_predicated);
+}
+
+// Return true if the given intrinsic is a scatter
+inline bool isScatter(IntrinsicInst *IntInst) {
+  if (IntInst == nullptr)
+    return false;
+  unsigned IntrinsicID = IntInst->getIntrinsicID();
+  return (IntrinsicID == Intrinsic::masked_scatter ||
           IntrinsicID == Intrinsic::arm_mve_vstr_scatter_base ||
           IntrinsicID == Intrinsic::arm_mve_vstr_scatter_base_predicated ||
           IntrinsicID == Intrinsic::arm_mve_vstr_scatter_base_wb ||
           IntrinsicID == Intrinsic::arm_mve_vstr_scatter_base_wb_predicated ||
           IntrinsicID == Intrinsic::arm_mve_vstr_scatter_offset ||
           IntrinsicID == Intrinsic::arm_mve_vstr_scatter_offset_predicated);
 }
 
+// Return true if the given intrinsic is a gather or scatter
+inline bool isGatherScatter(IntrinsicInst *IntInst) {
+  if (IntInst == nullptr)
+    return false;
+  return isGather(IntInst) || isScatter(IntInst);
+}
+
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_ARM_ARMBASEINSTRINFO_H

llvm/lib/Target/ARM/MVETailPredication.cpp

 //===- MVETailPredication.cpp - MVE Tail Predication ------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 /// \file
 /// Armv8.1m introduced MVE, M-Profile Vector Extension, and low-overhead
 /// branches to help accelerate DSP applications. These two extensions,
 /// combined with a new form of predication called tail-predication, can be used
 /// to provide implicit vector predication within a low-overhead loop.
 /// This is implicit because the predicate of active/inactive lanes is
 /// calculated by hardware, and thus does not need to be explicitly passed
 /// to vector instructions. The instructions responsible for this are the
 /// DLSTP and WLSTP instructions, which setup a tail-predicated loop and the
 /// the total number of data elements processed by the loop. The loop-end
 /// LETP instruction is responsible for decrementing and setting the remaining
 /// elements to be processed and generating the mask of active lanes.
 ///
 /// The HardwareLoops pass inserts intrinsics identifying loops that the
 /// backend will attempt to convert into a low-overhead loop. The vectorizer is
 /// responsible for generating a vectorized loop in which the lanes are

 }
 
 static bool IsMasked(Instruction *I) {
   auto *Call = dyn_cast<IntrinsicInst>(I);
   if (!Call)
     return false;
 
   Intrinsic::ID ID = Call->getIntrinsicID();
-  // TODO: Support gather/scatter expand/compress operations.
-  return ID == Intrinsic::masked_store || ID == Intrinsic::masked_load;
+  return ID == Intrinsic::masked_store || ID == Intrinsic::masked_load ||
+         isGatherScatter(Call);
 }
 
 bool MVETailPredication::runOnLoop(Loop *L, LPPassManager&) {
   if (skipLoop(L) || !EnableTailPredication)
     return false;
 
   MaskedInsts.clear();
   Function &F = *L->getHeader()->getParent();
if (!TryConvert(Setup->getArgOperand(0))) {
     LLVM_DEBUG(dbgs() << "ARM TP: Can't tail-predicate this loop.\n");
     return false;
   }
 
   return true;
 }
 
 static FixedVectorType *getVectorType(IntrinsicInst *I) {
-  unsigned TypeOp = I->getIntrinsicID() == Intrinsic::masked_load ? 0 : 1;
-  auto *PtrTy = cast<PointerType>(I->getOperand(TypeOp)->getType());
-  auto *VecTy = cast<FixedVectorType>(PtrTy->getElementType());
+  unsigned ID = I->getIntrinsicID();
+  FixedVectorType *VecTy;
+  if (ID == Intrinsic::masked_load || isGather(I)) {
+    if (ID == Intrinsic::arm_mve_vldr_gather_base_wb_predicated)
+      // then the type is a StructType
+      VecTy = dyn_cast<FixedVectorType>(I->getType()->getContainedType(0));
+    else
+      VecTy = dyn_cast<FixedVectorType>(I->getType());
+  } else if (ID == Intrinsic::masked_store) {
+    VecTy = dyn_cast<FixedVectorType>(I->getOperand(0)->getType());
+  } else {
+    VecTy = dyn_cast<FixedVectorType>(I->getOperand(2)->getType());
+  }
   assert(VecTy && "No scalable vectors expected here");
   return VecTy;
 }
 
 bool MVETailPredication::IsPredicatedVectorLoop() {
   // Check that the loop contains at least one masked load/store intrinsic.
   // We only support 'normal' vector instructions - other than masked
   // load/stores.
for (auto &I : *BB) {
       case Intrinsic::ceil:
       case Intrinsic::fabs:
         if (ST->hasMVEFloatOps())
           continue;
         break;
       default:
         break;
       }
-
       if (IsMasked(&I)) {
         auto *VecTy = getVectorType(Int);
         unsigned Lanes = VecTy->getNumElements();
         unsigned ElementWidth = VecTy->getScalarSizeInBits();
         // MVE vectors are 128-bit, but don't support 128 x i1.
         // TODO: Can we support vectors larger than 128-bits?
         unsigned MaxWidth = TTI->getRegisterBitWidth(true);
         if (Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth)
bool MVETailPredication::TryConvert(Value *TripCount) {
   }
 
   LLVM_DEBUG(dbgs() << "ARM TP: Found predicated vector loop.\n");
   SetVector<Instruction*> Predicates;
 
   // Walk through the masked intrinsics and try to find whether the predicate
   // operand is generated by intrinsic @llvm.get.active.lane.mask().
   for (auto *I : MaskedInsts) {
-    unsigned PredOp = I->getIntrinsicID() == Intrinsic::masked_load ? 2 : 3;
+    unsigned PredOp =
+        (I->getIntrinsicID() == Intrinsic::masked_load || isGather(I)) ? 2 : 3;
     auto *Predicate = dyn_cast<Instruction>(I->getArgOperand(PredOp));
     if (!Predicate || Predicates.count(Predicate))
       continue;
 
     auto *ActiveLaneMask = dyn_cast<IntrinsicInst>(Predicate);
     if (!ActiveLaneMask ||
         ActiveLaneMask->getIntrinsicID() != Intrinsic::get_active_lane_mask)
       continue;

llvm/test/CodeGen/Thumb2/mve-gather-scatter-tailpred.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -enable-arm-maskedldst -enable-mem-access-versioning=false -enable-arm-maskedgatscat -tail-predication=force-enabled %s -o - | FileCheck %s
+
+define dso_local void @mve_gather_qi_wb(i32* noalias nocapture readonly %A, i32* noalias nocapture readonly %B, i32* noalias nocapture %C, i32 %n, i32 %m, i32 %l) {
+; CHECK-LABEL: mve_gather_qi_wb:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    adr r4, .LCPI0_0
+; CHECK-NEXT:    add.w r12, r0, r3, lsl #2
+; CHECK-NEXT:    vldrw.u32 q0, [r4]
+; CHECK-NEXT:    adds r0, r3, #1
+; CHECK-NEXT:    vmov.i32 q2, #0x0
+; CHECK-NEXT:    vadd.i32 q0, q0, r1
+; CHECK-NEXT:    adds r1, r3, #4
+; CHECK-NEXT:    dlstp.32 lr, r0
+; CHECK-NEXT:  .LBB0_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q1, [r12], #16
+; CHECK-NEXT:    vldrw.u32 q3, [q0, #80]!
+; CHECK-NEXT:    vmul.i32 q1, q3, q1
+; CHECK-NEXT:    vadd.i32 q2, q2, q1
+; CHECK-NEXT:    letp lr, .LBB0_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    str.w r0, [r2, r1, lsl #2]
+; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI0_0:
+; CHECK-NEXT:    .long 4294967228 @ 0xffffffbc
+; CHECK-NEXT:    .long 4294967248 @ 0xffffffd0
+; CHECK-NEXT:    .long 4294967268 @ 0xffffffe4
+; CHECK-NEXT:    .long 4294967288 @ 0xfffffff8
+entry:                                  ; preds = %middle.
+  %add.us.us = add i32 4, %n
+  %arrayidx.us.us = getelementptr inbounds i32, i32* %C, i32 %add.us.us
+  br label %vector.body
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %entry ], [ %7, %vector.body ]
+  %vec.ind = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %entry ], [ %vec.ind.next, %vector.body ]
+  %0 = add i32 %index, %n
+  %1 = getelementptr inbounds i32, i32* %A, i32 %0
+  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %n)
+  %2 = bitcast i32* %1 to <4 x i32>*
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)
+  %3 = mul <4 x i32> %vec.ind, <i32 5, i32 5, i32 5, i32 5>
+  %4 = add <4 x i32> %3, <i32 3, i32 3, i32 3, i32 3>
+  %5 = getelementptr inbounds i32, i32* %B, <4 x i32> %4
+  %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %5, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)
+  %6 = mul nsw <4 x i32> %wide.masked.gather, %wide.masked.load
+  %7 = add <4 x i32> %vec.phi, %6
+  %index.next = add i32 %index, 4
+  %vec.ind.next = add <4 x i32> %vec.ind, <i32 4, i32 4, i32 4, i32 4>
+  %8 = icmp eq i32 %index.next, 5000
+  br i1 %8, label %middle.block, label %vector.body
+middle.block:                                     ; preds = %vector.body
+  %9 = select <4 x i1> %active.lane.mask, <4 x i32> %7, <4 x i32> %vec.phi
+  %10 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %9)
+  store i32 %10, i32* %arrayidx.us.us, align 4
+  %inc21.us.us = add nuw i32 4, 1
+  %exitcond81.not = icmp eq i32 %inc21.us.us, %n
+  br label %end
+end:                                 ; preds = %middle.block
+  ret void
+}
+
+define dso_local void @mve_gatherscatter_offset(i32* noalias nocapture readonly %A, i32* noalias nocapture readonly %B, i32* noalias nocapture %C, i32 %n, i32 %m, i32 %l) {
+; CHECK-LABEL: mve_gatherscatter_offset:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    adr r4, .LCPI1_0
+; CHECK-NEXT:    add.w r12, r0, r3, lsl #2
+; CHECK-NEXT:    adds r0, r3, #1
+; CHECK-NEXT:    vldrw.u32 q1, [r4]
+; CHECK-NEXT:    adds r3, #4
+; CHECK-NEXT:    vmov.i32 q3, #0x0
+; CHECK-NEXT:    vmov.i32 q0, #0x14
+; CHECK-NEXT:    dlstp.32 lr, r0
+; CHECK-NEXT:  .LBB1_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q2, [r1, q1, uxtw #2]
+; CHECK-NEXT:    vldrw.u32 q4, [r12], #16
+; CHECK-NEXT:    vmul.i32 q2, q2, q4
+; CHECK-NEXT:    vstrw.32 q2, [r1, q1, uxtw #2]
+; CHECK-NEXT:    vadd.i32 q1, q1, q0
+; CHECK-NEXT:    vadd.i32 q3, q3, q2
+; CHECK-NEXT:    letp lr, .LBB1_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vmov q0, q3
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    str.w r0, [r2, r3, lsl #2]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI1_0:
+; CHECK-NEXT:    .long 3 @ 0x3
+; CHECK-NEXT:    .long 8 @ 0x8
+; CHECK-NEXT:    .long 13 @ 0xd
+; CHECK-NEXT:    .long 18 @ 0x12
+entry:                                  ; preds = %middle.
+  %add.us.us = add i32 4, %n
+  %arrayidx.us.us = getelementptr inbounds i32, i32* %C, i32 %add.us.us
+  br label %vector.body
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %entry ], [ %7, %vector.body ]
+  %vec.ind = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %entry ], [ %vec.ind.next, %vector.body ]
+  %0 = add i32 %index, %n
+  %1 = getelementptr inbounds i32, i32* %A, i32 %0
+  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %n)
+  %2 = bitcast i32* %1 to <4 x i32>*
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)
+  %3 = mul <4 x i32> %vec.ind, <i32 5, i32 5, i32 5, i32 5>
+  %4 = add <4 x i32> %3, <i32 3, i32 3, i32 3, i32 3>
+  %5 = getelementptr inbounds i32, i32* %B, <4 x i32> %4
+  %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %5, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)
+  %6 = mul nsw <4 x i32> %wide.masked.gather, %wide.masked.load
+  call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %6, <4 x i32*> %5, i32 4, <4 x i1> %active.lane.mask)
+  %7 = add <4 x i32> %vec.phi, %6
+  %index.next = add i32 %index, 4
+  %vec.ind.next = add <4 x i32> %vec.ind, <i32 4, i32 4, i32 4, i32 4>
+  %8 = icmp eq i32 %index.next, 5000
+  br i1 %8, label %middle.block, label %vector.body
+middle.block:                                     ; preds = %vector.body
+  %9 = select <4 x i1> %active.lane.mask, <4 x i32> %7, <4 x i32> %vec.phi
+  %10 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %9)
+  store i32 %10, i32* %arrayidx.us.us, align 4
+  %inc21.us.us = add nuw i32 4, 1
+  %exitcond81.not = icmp eq i32 %inc21.us.us, %n
+  br label %end
+end:                                 ; preds = %middle.block
+  ret void
+}
+define dso_local void @mve_scatter_qi(i32* noalias nocapture readonly %A, i32* noalias nocapture readonly %B, i32* noalias nocapture %C, i32 %n, i32 %m, i32 %l) {
+; CHECK-LABEL: mve_scatter_qi:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    adr r4, .LCPI2_0
+; CHECK-NEXT:    add.w r12, r0, r3, lsl #2
+; CHECK-NEXT:    vldrw.u32 q0, [r4]
+; CHECK-NEXT:    adds r0, r3, #1
+; CHECK-NEXT:    vmov.i32 q3, #0x0
+; CHECK-NEXT:    vadd.i32 q0, q0, r1
+; CHECK-NEXT:    adds r1, r3, #4
+; CHECK-NEXT:    vmov.i32 q2, #0x3
+; CHECK-NEXT:    dlstp.32 lr, r0
+; CHECK-NEXT:  .LBB2_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q1, [r12], #16
+; CHECK-NEXT:    vmul.i32 q1, q1, q2
+; CHECK-NEXT:    vstrw.32 q1, [q0, #80]!
+; CHECK-NEXT:    vadd.i32 q3, q3, q1
+; CHECK-NEXT:    letp lr, .LBB2_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vmov q0, q3
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    str.w r0, [r2, r1, lsl #2]
+; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI2_0:
+; CHECK-NEXT:    .long 4294967228 @ 0xffffffbc
+; CHECK-NEXT:    .long 4294967248 @ 0xffffffd0
+; CHECK-NEXT:    .long 4294967268 @ 0xffffffe4
+; CHECK-NEXT:    .long 4294967288 @ 0xfffffff8
+entry:                                  ; preds = %middle.
+  %add.us.us = add i32 4, %n
+  %arrayidx.us.us = getelementptr inbounds i32, i32* %C, i32 %add.us.us
+  br label %vector.body
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %entry ], [ %7, %vector.body ]
+  %vec.ind = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %entry ], [ %vec.ind.next, %vector.body ]
+  %0 = add i32 %index, %n
+  %1 = getelementptr inbounds i32, i32* %A, i32 %0
+  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %n)
+  %2 = bitcast i32* %1 to <4 x i32>*
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)
+  %3 = mul <4 x i32> %vec.ind, <i32 5, i32 5, i32 5, i32 5>
+  %4 = add <4 x i32> %3, <i32 3, i32 3, i32 3, i32 3>
+  %5 = getelementptr inbounds i32, i32* %B, <4 x i32> %4
+  %6 = mul nsw <4 x i32> <i32 3, i32 3, i32 3, i32 3>, %wide.masked.load
+  call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %6, <4 x i32*> %5, i32 4, <4 x i1> %active.lane.mask)
+  %7 = add <4 x i32> %vec.phi, %6
+  %index.next = add i32 %index, 4
+  %vec.ind.next = add <4 x i32> %vec.ind, <i32 4, i32 4, i32 4, i32 4>
+  %8 = icmp eq i32 %index.next, 5000
+  br i1 %8, label %middle.block, label %vector.body
+middle.block:                                     ; preds = %vector.body
+  %9 = select <4 x i1> %active.lane.mask, <4 x i32> %7, <4 x i32> %vec.phi
+  %10 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %9)
+  store i32 %10, i32* %arrayidx.us.us, align 4
+  %inc21.us.us = add nuw i32 4, 1
+  %exitcond81.not = icmp eq i32 %inc21.us.us, %n
+  br label %end
+end:                                 ; preds = %middle.block
+  ret void
+}
+
+declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>)
+declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>)
+declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32, <4 x i1>, <4 x i32>)
+declare void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32>, <4 x i32*>, i32, <4 x i1>)