[ARM,MVE] Add ACLE intrinsics for the vaddv/vaddlv family.

[simon_tatham]

Summary

I've implemented them as target-specific IR intrinsics rather than
using @llvm.experimental.vector.reduce.add, on the grounds that the
'experimental' intrinsic doesn't currently have much code generation
benefit, and my replacements encapsulate the sign- or zero-extension
so that you don't expose the illegal MVE vector type (<4 x i64>) in
IR.

The machine instructions come in two versions: with and without an
input accumulator. My new IR intrinsics, like the 'experimental' one,
don't take an accumulator parameter: we represent that by just adding
on the input value using an ordinary i32 or i64 add. So if you write
the vaddvaq C-language intrinsic with an input accumulator of zero,
it can be optimised to VADDV, and conversely, if you write something
like x += vaddvq(y) then that can be combined into VADDVA.

Most of this is achieved in isel lowering, by converting these IR
intrinsics into the existing ARMISD::VADDV family of custom SDNode
types. For the difficult case (64-bit accumulators), isel lowering
already implements the optimization of folding an addition into a
VADDLV to make a VADDLVA; so once we've made a VADDLV, our job is
already done, except that I had to introduce a parallel set of ARMISD
nodes for the predicated forms of VADDLV.

For the simpler VADDV, we handle the predicated form by just leaving
the IR intrinsic alone and matching it in an ordinary dag pattern.

[dmgreen]

Sounds great, from what I can see. The predicated lowering looks useful when/if we try and get predicated vecreduce's working.

We'll have to do the same for VMLA at some point.
LGTM

clang/include/clang/Basic/arm_mve.td

multiclass MVEBinaryVectorHoriz64<dag subtract, dag exchange, string xsuffix> {
   defm "" : MVEBinaryVectorHoriz64Base<subtract, exchange, xsuffix, "vmlldava">;
 }
 
 multiclass MVEBinaryVectorHoriz64R<dag subtract, dag exchange, string xsuffix> {
   defm "" : MVEBinaryVectorHoriz64Base<subtract, exchange, xsuffix,
                                        "vrmlldavha">;
 }
 
+multiclass VADDV<bit acc, bit pred, string intbase, Type Scalar> {
+  defvar accArg = !if(acc, (args Scalar:$acc), (args));
+  defvar predArg = !if(pred, (args Predicate:$pred), (args));
+  defvar intrinsic = !if(pred,
+      IRInt<intbase # "_predicated", [Vector, Predicate]>,
+      IRInt<intbase, [Vector]>);
+  defvar intCG = !con((intrinsic $v, (unsignedflag Scalar)),
+                      !if(pred, (? $pred), (?)));
+  defvar accCG = !if(acc, (add intCG, $acc), intCG);
+
+  def "": Intrinsic<Scalar, !con(accArg, (args Vector:$v), predArg), accCG>;
+}
+
+let params = T.Int in {
+defm vaddvq    : VADDV<0, 0, "addv", Scalar32>;
+defm vaddvaq   : VADDV<1, 0, "addv", Scalar32>;
+defm vaddvq_p  : VADDV<0, 1, "addv", Scalar32>;
+defm vaddvaq_p : VADDV<1, 1, "addv", Scalar32>;
+}
+
+let params = [s32, u32] in {
+defm vaddlvq    : VADDV<0, 0, "addlv", Scalar64>;
+defm vaddlvaq   : VADDV<1, 0, "addlv", Scalar64>;
+defm vaddlvq_p  : VADDV<0, 1, "addlv", Scalar64>;
+defm vaddlvaq_p : VADDV<1, 1, "addlv", Scalar64>;
+}
+
 let params = T.Int in {
 def vabavq : Intrinsic<u32, (args u32:$a, Vector:$b, Vector:$c),
     (IRInt<"vabav", [Vector]> (unsignedflag Scalar), $a, $b, $c)>;
 def vabavq_p : Intrinsic<u32, (args u32:$a, Vector:$b, Vector:$c,
                                     Predicate:$pred),
     (IRInt<"vabav_predicated", [Vector, Predicate]>
                                (unsignedflag Scalar), $a, $b, $c, $pred)>;
 

clang/test/CodeGen/arm-mve-intrinsics/vaddv.c

+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+ // RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve -mfloat-abi hard -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
+ // RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve -mfloat-abi hard -O0 -disable-O0-optnone -DPOLYMORPHIC -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
+
+#include <arm_mve.h>
+
+// CHECK-LABEL: @test_vaddvq_s8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.arm.mve.addv.v16i8(<16 x i8> [[A:%.*]], i32 0)
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+int32_t test_vaddvq_s8(int8x16_t a) {
+#ifdef POLYMORPHIC
+  return vaddvq(a);
+#else  /* POLYMORPHIC */
+  return vaddvq_s8(a);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvq_s16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.arm.mve.addv.v8i16(<8 x i16> [[A:%.*]], i32 0)
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+int32_t test_vaddvq_s16(int16x8_t a) {
+#ifdef POLYMORPHIC
+  return vaddvq(a);
+#else  /* POLYMORPHIC */
+  return vaddvq_s16(a);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvq_s32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.arm.mve.addv.v4i32(<4 x i32> [[A:%.*]], i32 0)
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+int32_t test_vaddvq_s32(int32x4_t a) {
+#ifdef POLYMORPHIC
+  return vaddvq(a);
+#else  /* POLYMORPHIC */
+  return vaddvq_s32(a);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvq_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.arm.mve.addv.v16i8(<16 x i8> [[A:%.*]], i32 1)
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+uint32_t test_vaddvq_u8(uint8x16_t a) {
+#ifdef POLYMORPHIC
+  return vaddvq(a);
+#else  /* POLYMORPHIC */
+  return vaddvq_u8(a);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvq_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.arm.mve.addv.v8i16(<8 x i16> [[A:%.*]], i32 1)
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+uint32_t test_vaddvq_u16(uint16x8_t a) {
+#ifdef POLYMORPHIC
+  return vaddvq(a);
+#else  /* POLYMORPHIC */
+  return vaddvq_u16(a);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvq_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.arm.mve.addv.v4i32(<4 x i32> [[A:%.*]], i32 1)
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+uint32_t test_vaddvq_u32(uint32x4_t a) {
+#ifdef POLYMORPHIC
+  return vaddvq(a);
+#else  /* POLYMORPHIC */
+  return vaddvq_u32(a);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvaq_s8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.arm.mve.addv.v16i8(<16 x i8> [[B:%.*]], i32 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[TMP0]], [[A:%.*]]
+// CHECK-NEXT:    ret i32 [[TMP1]]
+//
+int32_t test_vaddvaq_s8(int32_t a, int8x16_t b) {
+#ifdef POLYMORPHIC
+  return vaddvaq(a, b);
+#else  /* POLYMORPHIC */
+  return vaddvaq_s8(a, b);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvaq_s16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.arm.mve.addv.v8i16(<8 x i16> [[B:%.*]], i32 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[TMP0]], [[A:%.*]]
+// CHECK-NEXT:    ret i32 [[TMP1]]
+//
+int32_t test_vaddvaq_s16(int32_t a, int16x8_t b) {
+#ifdef POLYMORPHIC
+  return vaddvaq(a, b);
+#else  /* POLYMORPHIC */
+  return vaddvaq_s16(a, b);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvaq_s32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.arm.mve.addv.v4i32(<4 x i32> [[B:%.*]], i32 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[TMP0]], [[A:%.*]]
+// CHECK-NEXT:    ret i32 [[TMP1]]
+//
+int32_t test_vaddvaq_s32(int32_t a, int32x4_t b) {
+#ifdef POLYMORPHIC
+  return vaddvaq(a, b);
+#else  /* POLYMORPHIC */
+  return vaddvaq_s32(a, b);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvaq_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.arm.mve.addv.v16i8(<16 x i8> [[B:%.*]], i32 1)
+// CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[TMP0]], [[A:%.*]]
+// CHECK-NEXT:    ret i32 [[TMP1]]
+//
+uint32_t test_vaddvaq_u8(uint32_t a, uint8x16_t b) {
+#ifdef POLYMORPHIC
+  return vaddvaq(a, b);
+#else  /* POLYMORPHIC */
+  return vaddvaq_u8(a, b);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvaq_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.arm.mve.addv.v8i16(<8 x i16> [[B:%.*]], i32 1)
+// CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[TMP0]], [[A:%.*]]
+// CHECK-NEXT:    ret i32 [[TMP1]]
+//
+uint32_t test_vaddvaq_u16(uint32_t a, uint16x8_t b) {
+#ifdef POLYMORPHIC
+  return vaddvaq(a, b);
+#else  /* POLYMORPHIC */
+  return vaddvaq_u16(a, b);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvaq_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.arm.mve.addv.v4i32(<4 x i32> [[B:%.*]], i32 1)
+// CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[TMP0]], [[A:%.*]]
+// CHECK-NEXT:    ret i32 [[TMP1]]
+//
+uint32_t test_vaddvaq_u32(uint32_t a, uint32x4_t b) {
+#ifdef POLYMORPHIC
+  return vaddvaq(a, b);
+#else  /* POLYMORPHIC */
+  return vaddvaq_u32(a, b);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvq_p_s8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.arm.mve.addv.predicated.v16i8.v16i1(<16 x i8> [[A:%.*]], i32 0, <16 x i1> [[TMP1]])
+// CHECK-NEXT:    ret i32 [[TMP2]]
+//
+int32_t test_vaddvq_p_s8(int8x16_t a, mve_pred16_t p) {
+#ifdef POLYMORPHIC
+  return vaddvq_p(a, p);
+#else  /* POLYMORPHIC */
+  return vaddvq_p_s8(a, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvq_p_s16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.arm.mve.addv.predicated.v8i16.v8i1(<8 x i16> [[A:%.*]], i32 0, <8 x i1> [[TMP1]])
+// CHECK-NEXT:    ret i32 [[TMP2]]
+//
+int32_t test_vaddvq_p_s16(int16x8_t a, mve_pred16_t p) {
+#ifdef POLYMORPHIC
+  return vaddvq_p(a, p);
+#else  /* POLYMORPHIC */
+  return vaddvq_p_s16(a, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvq_p_s32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> [[A:%.*]], i32 0, <4 x i1> [[TMP1]])
+// CHECK-NEXT:    ret i32 [[TMP2]]
+//
+int32_t test_vaddvq_p_s32(int32x4_t a, mve_pred16_t p) {
+#ifdef POLYMORPHIC
+  return vaddvq_p(a, p);
+#else  /* POLYMORPHIC */
+  return vaddvq_p_s32(a, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvq_p_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.arm.mve.addv.predicated.v16i8.v16i1(<16 x i8> [[A:%.*]], i32 1, <16 x i1> [[TMP1]])
+// CHECK-NEXT:    ret i32 [[TMP2]]
+//
+uint32_t test_vaddvq_p_u8(uint8x16_t a, mve_pred16_t p) {
+#ifdef POLYMORPHIC
+  return vaddvq_p(a, p);
+#else  /* POLYMORPHIC */
+  return vaddvq_p_u8(a, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvq_p_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.arm.mve.addv.predicated.v8i16.v8i1(<8 x i16> [[A:%.*]], i32 1, <8 x i1> [[TMP1]])
+// CHECK-NEXT:    ret i32 [[TMP2]]
+//
+uint32_t test_vaddvq_p_u16(uint16x8_t a, mve_pred16_t p) {
+#ifdef POLYMORPHIC
+  return vaddvq_p(a, p);
+#else  /* POLYMORPHIC */
+  return vaddvq_p_u16(a, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvq_p_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> [[A:%.*]], i32 1, <4 x i1> [[TMP1]])
+// CHECK-NEXT:    ret i32 [[TMP2]]
+//
+uint32_t test_vaddvq_p_u32(uint32x4_t a, mve_pred16_t p) {
+#ifdef POLYMORPHIC
+  return vaddvq_p(a, p);
+#else  /* POLYMORPHIC */
+  return vaddvq_p_u32(a, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvaq_p_s8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.arm.mve.addv.predicated.v16i8.v16i1(<16 x i8> [[B:%.*]], i32 0, <16 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], [[A:%.*]]
+// CHECK-NEXT:    ret i32 [[TMP3]]
+//
+int32_t test_vaddvaq_p_s8(int32_t a, int8x16_t b, mve_pred16_t p) {
+#ifdef POLYMORPHIC
+  return vaddvaq_p(a, b, p);
+#else  /* POLYMORPHIC */
+  return vaddvaq_p_s8(a, b, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvaq_p_s16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.arm.mve.addv.predicated.v8i16.v8i1(<8 x i16> [[B:%.*]], i32 0, <8 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], [[A:%.*]]
+// CHECK-NEXT:    ret i32 [[TMP3]]
+//
+int32_t test_vaddvaq_p_s16(int32_t a, int16x8_t b, mve_pred16_t p) {
+#ifdef POLYMORPHIC
+  return vaddvaq_p(a, b, p);
+#else  /* POLYMORPHIC */
+  return vaddvaq_p_s16(a, b, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvaq_p_s32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> [[B:%.*]], i32 0, <4 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], [[A:%.*]]
+// CHECK-NEXT:    ret i32 [[TMP3]]
+//
+int32_t test_vaddvaq_p_s32(int32_t a, int32x4_t b, mve_pred16_t p) {
+#ifdef POLYMORPHIC
+  return vaddvaq_p(a, b, p);
+#else  /* POLYMORPHIC */
+  return vaddvaq_p_s32(a, b, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvaq_p_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.arm.mve.addv.predicated.v16i8.v16i1(<16 x i8> [[B:%.*]], i32 1, <16 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], [[A:%.*]]
+// CHECK-NEXT:    ret i32 [[TMP3]]
+//
+uint32_t test_vaddvaq_p_u8(uint32_t a, uint8x16_t b, mve_pred16_t p) {
+#ifdef POLYMORPHIC
+  return vaddvaq_p(a, b, p);
+#else  /* POLYMORPHIC */
+  return vaddvaq_p_u8(a, b, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvaq_p_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.arm.mve.addv.predicated.v8i16.v8i1(<8 x i16> [[B:%.*]], i32 1, <8 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], [[A:%.*]]
+// CHECK-NEXT:    ret i32 [[TMP3]]
+//
+uint32_t test_vaddvaq_p_u16(uint32_t a, uint16x8_t b, mve_pred16_t p) {
+#ifdef POLYMORPHIC
+  return vaddvaq_p(a, b, p);
+#else  /* POLYMORPHIC */
+  return vaddvaq_p_u16(a, b, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddvaq_p_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> [[B:%.*]], i32 1, <4 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], [[A:%.*]]
+// CHECK-NEXT:    ret i32 [[TMP3]]
+//
+uint32_t test_vaddvaq_p_u32(uint32_t a, uint32x4_t b, mve_pred16_t p) {
+#ifdef POLYMORPHIC
+  return vaddvaq_p(a, b, p);
+#else  /* POLYMORPHIC */
+  return vaddvaq_p_u32(a, b, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddlvq_s32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.arm.mve.addlv.v4i32(<4 x i32> [[A:%.*]], i32 0)
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+int64_t test_vaddlvq_s32(int32x4_t a) {
+#ifdef POLYMORPHIC
+  return vaddlvq(a);
+#else  /* POLYMORPHIC */
+  return vaddlvq_s32(a);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddlvq_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.arm.mve.addlv.v4i32(<4 x i32> [[A:%.*]], i32 1)
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+uint64_t test_vaddlvq_u32(uint32x4_t a) {
+#ifdef POLYMORPHIC
+  return vaddlvq(a);
+#else  /* POLYMORPHIC */
+  return vaddlvq_u32(a);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddlvaq_s32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.arm.mve.addlv.v4i32(<4 x i32> [[B:%.*]], i32 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], [[A:%.*]]
+// CHECK-NEXT:    ret i64 [[TMP1]]
+//
+int64_t test_vaddlvaq_s32(int64_t a, int32x4_t b) {
+#ifdef POLYMORPHIC
+  return vaddlvaq(a, b);
+#else  /* POLYMORPHIC */
+  return vaddlvaq_s32(a, b);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddlvaq_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.arm.mve.addlv.v4i32(<4 x i32> [[B:%.*]], i32 1)
+// CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], [[A:%.*]]
+// CHECK-NEXT:    ret i64 [[TMP1]]
+//
+uint64_t test_vaddlvaq_u32(uint64_t a, uint32x4_t b) {
+#ifdef POLYMORPHIC
+  return vaddlvaq(a, b);
+#else  /* POLYMORPHIC */
+  return vaddlvaq_u32(a, b);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddlvq_p_s32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.arm.mve.addlv.predicated.v4i32.v4i1(<4 x i32> [[A:%.*]], i32 0, <4 x i1> [[TMP1]])
+// CHECK-NEXT:    ret i64 [[TMP2]]
+//
+int64_t test_vaddlvq_p_s32(int32x4_t a, mve_pred16_t p) {
+#ifdef POLYMORPHIC
+  return vaddlvq_p(a, p);
+#else  /* POLYMORPHIC */
+  return vaddlvq_p_s32(a, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddlvq_p_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.arm.mve.addlv.predicated.v4i32.v4i1(<4 x i32> [[A:%.*]], i32 1, <4 x i1> [[TMP1]])
+// CHECK-NEXT:    ret i64 [[TMP2]]
+//
+uint64_t test_vaddlvq_p_u32(uint32x4_t a, mve_pred16_t p) {
+#ifdef POLYMORPHIC
+  return vaddlvq_p(a, p);
+#else  /* POLYMORPHIC */
+  return vaddlvq_p_u32(a, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddlvaq_p_s32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.arm.mve.addlv.predicated.v4i32.v4i1(<4 x i32> [[B:%.*]], i32 0, <4 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], [[A:%.*]]
+// CHECK-NEXT:    ret i64 [[TMP3]]
+//
+int64_t test_vaddlvaq_p_s32(int64_t a, int32x4_t b, mve_pred16_t p) {
+#ifdef POLYMORPHIC
+  return vaddlvaq_p(a, b, p);
+#else  /* POLYMORPHIC */
+  return vaddlvaq_p_s32(a, b, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vaddlvaq_p_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.arm.mve.addlv.predicated.v4i32.v4i1(<4 x i32> [[B:%.*]], i32 1, <4 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], [[A:%.*]]
+// CHECK-NEXT:    ret i64 [[TMP3]]
+//
+uint64_t test_vaddlvaq_p_u32(uint64_t a, uint32x4_t b, mve_pred16_t p) {
+#ifdef POLYMORPHIC
+  return vaddlvaq_p(a, b, p);
+#else  /* POLYMORPHIC */
+  return vaddlvaq_p_u32(a, b, p);
+#endif /* POLYMORPHIC */
+}
+

llvm/include/llvm/IR/IntrinsicsARM.td

multiclass MVE_minmaxv {
   defm nmv: MVEPredicated<[llvm_anyfloat_ty],
      [LLVMMatchType<0>, llvm_anyvector_ty]>;
   defm nmav: MVEPredicated<[llvm_anyfloat_ty],
      [LLVMMatchType<0>, llvm_anyvector_ty]>;
 }
 defm int_arm_mve_min: MVE_minmaxv;
 defm int_arm_mve_max: MVE_minmaxv;
 
+defm int_arm_mve_addv: MVEPredicated<[llvm_i32_ty],
+   [llvm_anyvector_ty, llvm_i32_ty /* unsigned */]>;
+defm int_arm_mve_addlv: MVEPredicated<[llvm_i64_ty],
+   [llvm_anyvector_ty, llvm_i32_ty /* unsigned */]>;
+
 // Intrinsic with a predicated and a non-predicated case. The predicated case
 // has two additional parameters: inactive (the value for inactive lanes, can
 // be undef) and predicate.
 multiclass MVEMXPredicated<list<LLVMType> rets, list<LLVMType> flags,
                            list<LLVMType> params, LLVMType inactive,
                            LLVMType predicate,
                            list<IntrinsicProperty> props = [IntrNoMem]> {
   def "":          Intrinsic<rets, flags # params, props>;

llvm/lib/Target/ARM/ARMISelLowering.h

enum NodeType : unsigned {
       VTBL2,        // 2-register shuffle with mask
       VMOVN,        // MVE vmovn
 
       // Vector multiply long:
       VMULLs,       // ...signed
       VMULLu,       // ...unsigned
 
       // MVE reductions
-      VADDVs,
-      VADDVu,
-      VADDLVs,
-      VADDLVu,
-      VADDLVAs,
-      VADDLVAu,
+      VADDVs,       // sign- or zero-extend the elements of a vector to i32,
+      VADDVu,       //   add them all together, and return an i32 of their sum
+      VADDLVs,      // sign- or zero-extend elements to i64 and sum, returning
+      VADDLVu,      //   the low and high 32-bit halves of the sum
+      VADDLVAs,     // same as VADDLV[su] but also add an input accumulator
+      VADDLVAu,     //   provided as low and high halves
+      VADDLVps,     // same as VADDLVs but with a v4i1 predicate mask
+      VADDLVpu,     // same as VADDLVu but with a v4i1 predicate mask
+      VADDLVAps,    // same as VADDLVps but with a v4i1 predicate mask
+      VADDLVApu,    // same as VADDLVpu but with a v4i1 predicate mask
       VMLAVs,
       VMLAVu,
       VMLALVs,
       VMLALVu,
       VMLALVAs,
       VMLALVAu,
 
       SMULWB,       // Signed multiply word by half word, bottom

llvm/lib/Target/ARM/ARMISelLowering.cpp

const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case ARMISD::VMULLs:        return "ARMISD::VMULLs";
   case ARMISD::VMULLu:        return "ARMISD::VMULLu";
   case ARMISD::VADDVs:        return "ARMISD::VADDVs";
   case ARMISD::VADDVu:        return "ARMISD::VADDVu";
   case ARMISD::VADDLVs:       return "ARMISD::VADDLVs";
   case ARMISD::VADDLVu:       return "ARMISD::VADDLVu";
   case ARMISD::VADDLVAs:      return "ARMISD::VADDLVAs";
   case ARMISD::VADDLVAu:      return "ARMISD::VADDLVAu";
+  case ARMISD::VADDLVps:      return "ARMISD::VADDLVps";
+  case ARMISD::VADDLVpu:      return "ARMISD::VADDLVpu";
+  case ARMISD::VADDLVAps:     return "ARMISD::VADDLVAps";
+  case ARMISD::VADDLVApu:     return "ARMISD::VADDLVApu";
   case ARMISD::VMLAVs:        return "ARMISD::VMLAVs";
   case ARMISD::VMLAVu:        return "ARMISD::VMLAVu";
   case ARMISD::VMLALVs:       return "ARMISD::VMLALVs";
   case ARMISD::VMLALVu:       return "ARMISD::VMLALVu";
   case ARMISD::VMLALVAs:      return "ARMISD::VMLALVAs";
   case ARMISD::VMLALVAu:      return "ARMISD::VMLALVAu";
   case ARMISD::UMAAL:         return "ARMISD::UMAAL";
   case ARMISD::UMLAL:         return "ARMISD::UMLAL";
auto MakeVecReduce = [&](unsigned Opcode, unsigned OpcodeA, SDValue NA,
     if (NB->getOpcode() != ISD::BUILD_PAIR)
       return SDValue();
     SDValue VecRed = NB->getOperand(0);
     if (VecRed->getOpcode() != Opcode || VecRed.getResNo() != 0 ||
         NB->getOperand(1) != SDValue(VecRed.getNode(), 1))
       return SDValue();
 
     SDLoc dl(N);
-    SDValue Lo = DCI.DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, NA,
-                                 DCI.DAG.getConstant(0, dl, MVT::i32));
-    SDValue Hi = DCI.DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, NA,
-                                 DCI.DAG.getConstant(1, dl, MVT::i32));
-    SDValue Red =
-        VecRed->getNumOperands() == 1
-            ? DCI.DAG.getNode(OpcodeA, dl,
-                              DCI.DAG.getVTList({MVT::i32, MVT::i32}), Lo, Hi,
-                              VecRed->getOperand(0))
-            : DCI.DAG.getNode(OpcodeA, dl,
-                              DCI.DAG.getVTList({MVT::i32, MVT::i32}), Lo, Hi,
-                              VecRed->getOperand(0), VecRed->getOperand(1));
+    SmallVector<SDValue, 4> Ops;
+    Ops.push_back(DCI.DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, NA,
+                                  DCI.DAG.getConstant(0, dl, MVT::i32)));
+    Ops.push_back(DCI.DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, NA,
+                                  DCI.DAG.getConstant(1, dl, MVT::i32)));
+    for (unsigned i = 0, e = VecRed.getNumOperands(); i < e; i++)
+      Ops.push_back(VecRed->getOperand(i));
+    SDValue Red = DCI.DAG.getNode(OpcodeA, dl,
+                                  DCI.DAG.getVTList({MVT::i32, MVT::i32}), Ops);
     return DCI.DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Red,
                            SDValue(Red.getNode(), 1));
   };
 
   if (SDValue M = MakeVecReduce(ARMISD::VADDLVs, ARMISD::VADDLVAs, N0, N1))
     return M;
   if (SDValue M = MakeVecReduce(ARMISD::VADDLVu, ARMISD::VADDLVAu, N0, N1))
     return M;
   if (SDValue M = MakeVecReduce(ARMISD::VADDLVs, ARMISD::VADDLVAs, N1, N0))
     return M;
   if (SDValue M = MakeVecReduce(ARMISD::VADDLVu, ARMISD::VADDLVAu, N1, N0))
     return M;
+  if (SDValue M = MakeVecReduce(ARMISD::VADDLVps, ARMISD::VADDLVAps, N0, N1))
+    return M;
+  if (SDValue M = MakeVecReduce(ARMISD::VADDLVpu, ARMISD::VADDLVApu, N0, N1))
+    return M;
+  if (SDValue M = MakeVecReduce(ARMISD::VADDLVps, ARMISD::VADDLVAps, N1, N0))
+    return M;
+  if (SDValue M = MakeVecReduce(ARMISD::VADDLVpu, ARMISD::VADDLVApu, N1, N0))
+    return M;
   if (SDValue M = MakeVecReduce(ARMISD::VMLALVs, ARMISD::VMLALVAs, N0, N1))
     return M;
   if (SDValue M = MakeVecReduce(ARMISD::VMLALVu, ARMISD::VMLALVAu, N0, N1))
     return M;
   if (SDValue M = MakeVecReduce(ARMISD::VMLALVs, ARMISD::VMLALVAs, N1, N0))
     return M;
   if (SDValue M = MakeVecReduce(ARMISD::VMLALVu, ARMISD::VMLALVAu, N1, N0))
     return M;
case Intrinsic::arm_mve_maxav_predicated: {
     // These intrinsics all take an i32 scalar operand which is narrowed to the
     // size of a single lane of the vector type they take as the other input.
     unsigned BitWidth = N->getOperand(2)->getValueType(0).getScalarSizeInBits();
     APInt DemandedMask = APInt::getLowBitsSet(32, BitWidth);
     if (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI))
       return SDValue();
     break;
   }
+
+  case Intrinsic::arm_mve_addv: {
+    // Turn this intrinsic straight into the appropriate ARMISD::VADDV node,
+    // which allow PerformADDVecReduce to turn it into VADDLV when possible.
+    bool Unsigned = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
+    unsigned Opc = Unsigned ? ARMISD::VADDVu : ARMISD::VADDVs;
+    return DAG.getNode(Opc, SDLoc(N), N->getVTList(), N->getOperand(1));
+  }
+
+  case Intrinsic::arm_mve_addlv:
+  case Intrinsic::arm_mve_addlv_predicated: {
+    // Same for these, but ARMISD::VADDLV has to be followed by a BUILD_PAIR
+    // which recombines the two outputs into an i64
+    bool Unsigned = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
+    unsigned Opc = IntNo == Intrinsic::arm_mve_addlv ?
+                    (Unsigned ? ARMISD::VADDLVu : ARMISD::VADDLVs) :
+                    (Unsigned ? ARMISD::VADDLVpu : ARMISD::VADDLVps);
+
+    SmallVector<SDValue, 4> Ops;
+    for (unsigned i = 1, e = N->getNumOperands(); i < e; i++)
+      if (i != 2)                      // skip the unsigned flag
+        Ops.push_back(N->getOperand(i));
+
+    SDLoc dl(N);
+    SDValue val = DAG.getNode(Opc, dl, {MVT::i32, MVT::i32}, Ops);
+    return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, val.getValue(0),
+                       val.getValue(1));
+  }
   }
 
   return SDValue();
 }
 
 /// PerformShiftCombine - Checks for immediate versions of vector shifts and
 /// lowers them.  As with the vector shift intrinsics, this is done during DAG
 /// combining instead of DAG legalizing because the build_vectors for 64-bit

llvm/lib/Target/ARM/ARMInstrMVE.td

class MVE_VADDV<string iname, string suffix, dag iops, string cstr,
   let Inst{15-13} = Rda{3-1};
   let Inst{12} = 0b0;
   let Inst{8-6} = 0b100;
   let Inst{5} = A;
   let Inst{3-1} = Qm{2-0};
   let Inst{0} = 0b0;
 }
 
-multiclass MVE_VADDV_A<string suffix, bit U, bits<2> size,
-                       list<dag> pattern=[]> {
-  def acc    : MVE_VADDV<"vaddva", suffix,
+def ARMVADDVs       : SDNode<"ARMISD::VADDVs", SDTVecReduce>;
+def ARMVADDVu       : SDNode<"ARMISD::VADDVu", SDTVecReduce>;
+
+multiclass MVE_VADDV_A<MVEVectorVTInfo VTI> {
+  def acc    : MVE_VADDV<"vaddva", VTI.Suffix,
                          (ins tGPREven:$Rda_src, MQPR:$Qm), "$Rda = $Rda_src",
-                         0b1, U, size, pattern>;
-  def no_acc : MVE_VADDV<"vaddv", suffix,
+                         0b1, VTI.Unsigned, VTI.Size>;
+  def no_acc : MVE_VADDV<"vaddv", VTI.Suffix,
                          (ins MQPR:$Qm), "",
-                         0b0, U, size, pattern>;
-}
+                         0b0, VTI.Unsigned, VTI.Size>;
 
-defm MVE_VADDVs8  : MVE_VADDV_A<"s8",  0b0, 0b00>;
-defm MVE_VADDVs16 : MVE_VADDV_A<"s16", 0b0, 0b01>;
-defm MVE_VADDVs32 : MVE_VADDV_A<"s32", 0b0, 0b10>;
-defm MVE_VADDVu8  : MVE_VADDV_A<"u8",  0b1, 0b00>;
-defm MVE_VADDVu16 : MVE_VADDV_A<"u16", 0b1, 0b01>;
-defm MVE_VADDVu32 : MVE_VADDV_A<"u32", 0b1, 0b10>;
-
-def ARMVADDVs       : SDNode<"ARMISD::VADDVs", SDTVecReduce>;
-def ARMVADDVu       : SDNode<"ARMISD::VADDVu", SDTVecReduce>;
+  defvar InstA = !cast<Instruction>(NAME # "acc");
+  defvar InstN = !cast<Instruction>(NAME # "no_acc");
 
   let Predicates = [HasMVEInt] in {
-  def : Pat<(i32 (vecreduce_add (v4i32 MQPR:$src))),
-            (i32 (MVE_VADDVu32no_acc $src))>;
-  def : Pat<(i32 (vecreduce_add (v8i16 MQPR:$src))),
-            (i32 (MVE_VADDVu16no_acc $src))>;
-  def : Pat<(i32 (vecreduce_add (v16i8 MQPR:$src))),
-            (i32 (MVE_VADDVu8no_acc $src))>;
-
-  def : Pat<(i32 (ARMVADDVs (v8i16 MQPR:$src))),
-            (i32 (MVE_VADDVs16no_acc $src))>;
-  def : Pat<(i32 (ARMVADDVu (v8i16 MQPR:$src))),
-            (i32 (MVE_VADDVu16no_acc $src))>;
-  def : Pat<(i32 (ARMVADDVs (v16i8 MQPR:$src))),
-            (i32 (MVE_VADDVs8no_acc $src))>;
-  def : Pat<(i32 (ARMVADDVu (v16i8 MQPR:$src))),
-            (i32 (MVE_VADDVu8no_acc $src))>;
-
-  def : Pat<(i32 (add (i32 (vecreduce_add (v4i32 MQPR:$src1))), (i32 tGPREven:$src2))),
-            (i32 (MVE_VADDVu32acc $src2, $src1))>;
-  def : Pat<(i32 (add (i32 (vecreduce_add (v8i16 MQPR:$src1))), (i32 tGPREven:$src2))),
-            (i32 (MVE_VADDVu16acc $src2, $src1))>;
-  def : Pat<(i32 (add (i32 (vecreduce_add (v16i8 MQPR:$src1))), (i32 tGPREven:$src2))),
-            (i32 (MVE_VADDVu8acc $src2, $src1))>;
-
-  def : Pat<(i32 (add (i32 (ARMVADDVs (v8i16 MQPR:$src1))), (i32 tGPREven:$src2))),
-            (i32 (MVE_VADDVs16acc $src2, $src1))>;
-  def : Pat<(i32 (add (i32 (ARMVADDVu (v8i16 MQPR:$src1))), (i32 tGPREven:$src2))),
-            (i32 (MVE_VADDVu16acc $src2, $src1))>;
-  def : Pat<(i32 (add (i32 (ARMVADDVs (v16i8 MQPR:$src1))), (i32 tGPREven:$src2))),
-            (i32 (MVE_VADDVs8acc $src2, $src1))>;
-  def : Pat<(i32 (add (i32 (ARMVADDVu (v16i8 MQPR:$src1))), (i32 tGPREven:$src2))),
-            (i32 (MVE_VADDVu8acc $src2, $src1))>;
+    if VTI.Unsigned then {
+      def : Pat<(i32 (vecreduce_add (VTI.Vec MQPR:$vec))),
+                (i32 (InstN $vec))>;
+      def : Pat<(i32 (ARMVADDVu (VTI.Vec MQPR:$vec))),
+                (i32 (InstN $vec))>;
+      def : Pat<(i32 (add (i32 (vecreduce_add (VTI.Vec MQPR:$vec))),
+                          (i32 tGPREven:$acc))),
+                (i32 (InstA $acc, $vec))>;
+      def : Pat<(i32 (add (i32 (ARMVADDVu (VTI.Vec MQPR:$vec))),
+                          (i32 tGPREven:$acc))),
+                (i32 (InstA $acc, $vec))>;
+    } else {
+      def : Pat<(i32 (ARMVADDVs (VTI.Vec MQPR:$vec))),
+                (i32 (InstN $vec))>;
+      def : Pat<(i32 (add (i32 (ARMVADDVs (VTI.Vec MQPR:$vec))),
+                          (i32 tGPREven:$acc))),
+                (i32 (InstA $acc, $vec))>;
     }
 
+    def : Pat<(i32 (int_arm_mve_addv_predicated (VTI.Vec MQPR:$vec),
+                                                (i32 VTI.Unsigned),
+                                                (VTI.Pred VCCR:$pred))),
+              (i32 (InstN $vec, ARMVCCThen, $pred))>;
+    def : Pat<(i32 (add (int_arm_mve_addv_predicated (VTI.Vec MQPR:$vec),
+                                                     (i32 VTI.Unsigned),
+                                                     (VTI.Pred VCCR:$pred)),
+                        (i32 tGPREven:$acc))),
+              (i32 (InstA $acc, $vec, ARMVCCThen, $pred))>;
+  }
+}
+
+defm MVE_VADDVs8  : MVE_VADDV_A<MVE_v16s8>;
+defm MVE_VADDVs16 : MVE_VADDV_A<MVE_v8s16>;
+defm MVE_VADDVs32 : MVE_VADDV_A<MVE_v4s32>;
+defm MVE_VADDVu8  : MVE_VADDV_A<MVE_v16u8>;
+defm MVE_VADDVu16 : MVE_VADDV_A<MVE_v8u16>;
+defm MVE_VADDVu32 : MVE_VADDV_A<MVE_v4u32>;
+
 class MVE_VADDLV<string iname, string suffix, dag iops, string cstr,
                bit A, bit U, list<dag> pattern=[]>
   : MVE_rDest<(outs tGPREven:$RdaLo, tGPROdd:$RdaHi), iops, NoItinerary, iname,
               suffix, "$RdaLo, $RdaHi, $Qm", cstr, pattern> {
   bits<3> Qm;
   bits<4> RdaLo;
   bits<4> RdaHi;
 
   let Inst{28} = U;
   let Inst{22-20} = RdaHi{3-1};
   let Inst{19-18} = 0b10;
   let Inst{17-16} = 0b01;
   let Inst{15-13} = RdaLo{3-1};
   let Inst{12} = 0b0;
   let Inst{8-6} = 0b100;
   let Inst{5} = A;
   let Inst{3-1} = Qm{2-0};
   let Inst{0} = 0b0;
 }
 
-multiclass MVE_VADDLV_A<string suffix, bit U, list<dag> pattern=[]> {
-  def acc    : MVE_VADDLV<"vaddlva", suffix,
-                        (ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, MQPR:$Qm),
-                        "$RdaLo = $RdaLo_src,$RdaHi = $RdaHi_src",
-                        0b1, U, pattern>;
-  def no_acc : MVE_VADDLV<"vaddlv", suffix,
-                        (ins MQPR:$Qm), "",
-                        0b0, U, pattern>;
-}
-
-
-defm MVE_VADDLVs32 : MVE_VADDLV_A<"s32", 0b0>;
-defm MVE_VADDLVu32 : MVE_VADDLV_A<"u32", 0b1>;
-
 def SDTVecReduceL : SDTypeProfile<2, 1, [    // VADDLV
   SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>
 ]>;
 def SDTVecReduceLA : SDTypeProfile<2, 3, [    // VADDLVA
   SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisInt<3>,
   SDTCisVec<4>
 ]>;
-def ARMVADDLVs      : SDNode<"ARMISD::VADDLVs", SDTVecReduceL>;
-def ARMVADDLVu      : SDNode<"ARMISD::VADDLVu", SDTVecReduceL>;
-def ARMVADDLVAs     : SDNode<"ARMISD::VADDLVAs", SDTVecReduceLA>;
-def ARMVADDLVAu     : SDNode<"ARMISD::VADDLVAu", SDTVecReduceLA>;
+def SDTVecReduceLP : SDTypeProfile<2, 2, [    // VADDLVp
+  SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>, SDTCisVec<2>
+]>;
+def SDTVecReduceLPA : SDTypeProfile<2, 4, [    // VADDLVAp
+  SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisInt<3>,
+  SDTCisVec<4>, SDTCisVec<5>
+]>;
 
-let Predicates = [HasMVEInt] in {
-  def : Pat<(ARMVADDLVs (v4i32 MQPR:$val1)),
-            (MVE_VADDLVs32no_acc (v4i32 MQPR:$val1))>;
-  def : Pat<(ARMVADDLVu (v4i32 MQPR:$val1)),
-            (MVE_VADDLVu32no_acc (v4i32 MQPR:$val1))>;
-
-  def : Pat<(ARMVADDLVAs tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1)),
-            (MVE_VADDLVs32acc tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1))>;
-  def : Pat<(ARMVADDLVAu tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1)),
-            (MVE_VADDLVu32acc tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1))>;
+multiclass MVE_VADDLV_A<MVEVectorVTInfo VTI> {
+  def acc    : MVE_VADDLV<"vaddlva", VTI.Suffix,
+                        (ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, MQPR:$Qm),
+                        "$RdaLo = $RdaLo_src,$RdaHi = $RdaHi_src",
+                        0b1, VTI.Unsigned>;
+  def no_acc : MVE_VADDLV<"vaddlv", VTI.Suffix,
+                        (ins MQPR:$Qm), "",
+                        0b0, VTI.Unsigned>;
+
+  defvar InstA = !cast<Instruction>(NAME # "acc");
+  defvar InstN = !cast<Instruction>(NAME # "no_acc");
+
+  defvar letter = VTI.SuffixLetter;
+  defvar ARMVADDLV = SDNode<"ARMISD::VADDLV" # letter, SDTVecReduceL>;
+  defvar ARMVADDLVA = SDNode<"ARMISD::VADDLVA" # letter, SDTVecReduceLA>;
+  defvar ARMVADDLVp = SDNode<"ARMISD::VADDLVp" # letter, SDTVecReduceLP>;
+  defvar ARMVADDLVAp = SDNode<"ARMISD::VADDLVAp" # letter, SDTVecReduceLPA>;
+
+  let Predicates = [HasMVEInt] in {
+    def : Pat<(ARMVADDLV (v4i32 MQPR:$vec)),
+              (InstN (v4i32 MQPR:$vec))>;
+    def : Pat<(ARMVADDLVA tGPREven:$acclo, tGPROdd:$acchi, (v4i32 MQPR:$vec)),
+              (InstA tGPREven:$acclo, tGPROdd:$acchi, (v4i32 MQPR:$vec))>;
+    def : Pat<(ARMVADDLVp (v4i32 MQPR:$vec), (VTI.Pred VCCR:$pred)),
+              (InstN (v4i32 MQPR:$vec), ARMVCCThen, (VTI.Pred VCCR:$pred))>;
+    def : Pat<(ARMVADDLVAp tGPREven:$acclo, tGPROdd:$acchi, (v4i32 MQPR:$vec),
+                           (VTI.Pred VCCR:$pred)),
+              (InstA tGPREven:$acclo, tGPROdd:$acchi, (v4i32 MQPR:$vec),
+                     ARMVCCThen, (VTI.Pred VCCR:$pred))>;
   }
+}
+
+defm MVE_VADDLVs32 : MVE_VADDLV_A<MVE_v4s32>;
+defm MVE_VADDLVu32 : MVE_VADDLV_A<MVE_v4u32>;
 
 class MVE_VMINMAXNMV<string iname, string suffix, bit sz,
                      bit bit_17, bit bit_7, list<dag> pattern=[]>
   : MVE_rDest<(outs rGPR:$RdaDest), (ins rGPR:$RdaSrc, MQPR:$Qm),
               NoItinerary, iname, suffix, "$RdaSrc, $Qm",
               "$RdaDest = $RdaSrc", pattern> {
   bits<3> Qm;
   bits<4> RdaDest;

llvm/test/CodeGen/Thumb2/mve-intrinsics/vaddv.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc i32 @test_vaddvq_s8(<16 x i8> %a) {
+; CHECK-LABEL: test_vaddvq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddv.s8 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call i32 @llvm.arm.mve.addv.v16i8(<16 x i8> %a, i32 0)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvq_s16(<8 x i16> %a) {
+; CHECK-LABEL: test_vaddvq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddv.s16 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call i32 @llvm.arm.mve.addv.v8i16(<8 x i16> %a, i32 0)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvq_s32(<4 x i32> %a) {
+; CHECK-LABEL: test_vaddvq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddv.s32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call i32 @llvm.arm.mve.addv.v4i32(<4 x i32> %a, i32 0)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvq_u8(<16 x i8> %a) {
+; CHECK-LABEL: test_vaddvq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddv.u8 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call i32 @llvm.arm.mve.addv.v16i8(<16 x i8> %a, i32 1)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvq_u16(<8 x i16> %a) {
+; CHECK-LABEL: test_vaddvq_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddv.u16 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call i32 @llvm.arm.mve.addv.v8i16(<8 x i16> %a, i32 1)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvq_u32(<4 x i32> %a) {
+; CHECK-LABEL: test_vaddvq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call i32 @llvm.arm.mve.addv.v4i32(<4 x i32> %a, i32 1)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvaq_s8(i32 %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vaddvaq_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddva.s8 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call i32 @llvm.arm.mve.addv.v16i8(<16 x i8> %b, i32 0)
+  %1 = add i32 %0, %a
+  ret i32 %1
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvaq_s16(i32 %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vaddvaq_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddva.s16 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call i32 @llvm.arm.mve.addv.v8i16(<8 x i16> %b, i32 0)
+  %1 = add i32 %0, %a
+  ret i32 %1
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvaq_s32(i32 %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vaddvaq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddva.s32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call i32 @llvm.arm.mve.addv.v4i32(<4 x i32> %b, i32 0)
+  %1 = add i32 %0, %a
+  ret i32 %1
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvaq_u8(i32 %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vaddvaq_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddva.u8 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call i32 @llvm.arm.mve.addv.v16i8(<16 x i8> %b, i32 1)
+  %1 = add i32 %0, %a
+  ret i32 %1
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvaq_u16(i32 %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vaddvaq_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddva.u16 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call i32 @llvm.arm.mve.addv.v8i16(<8 x i16> %b, i32 1)
+  %1 = add i32 %0, %a
+  ret i32 %1
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvaq_u32(i32 %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vaddvaq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddva.u32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call i32 @llvm.arm.mve.addv.v4i32(<4 x i32> %b, i32 1)
+  %1 = add i32 %0, %a
+  ret i32 %1
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvq_p_s8(<16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vaddvq_p_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddvt.s8 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call i32 @llvm.arm.mve.addv.predicated.v16i8.v16i1(<16 x i8> %a, i32 0, <16 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvq_p_s16(<8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vaddvq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddvt.s16 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call i32 @llvm.arm.mve.addv.predicated.v8i16.v8i1(<8 x i16> %a, i32 0, <8 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvq_p_s32(<4 x i32> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vaddvq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddvt.s32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %a, i32 0, <4 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvq_p_u8(<16 x i8> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vaddvq_p_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddvt.u8 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call i32 @llvm.arm.mve.addv.predicated.v16i8.v16i1(<16 x i8> %a, i32 1, <16 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvq_p_u16(<8 x i16> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vaddvq_p_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddvt.u16 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call i32 @llvm.arm.mve.addv.predicated.v8i16.v8i1(<8 x i16> %a, i32 1, <8 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvq_p_u32(<4 x i32> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vaddvq_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddvt.u32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %a, i32 1, <4 x i1> %1)
+  ret i32 %2
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvaq_p_s8(i32 %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vaddvaq_p_s8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddvat.s8 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call i32 @llvm.arm.mve.addv.predicated.v16i8.v16i1(<16 x i8> %b, i32 0, <16 x i1> %1)
+  %3 = add i32 %2, %a
+  ret i32 %3
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvaq_p_s16(i32 %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vaddvaq_p_s16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddvat.s16 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call i32 @llvm.arm.mve.addv.predicated.v8i16.v8i1(<8 x i16> %b, i32 0, <8 x i1> %1)
+  %3 = add i32 %2, %a
+  ret i32 %3
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvaq_p_s32(i32 %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vaddvaq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddvat.s32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %b, i32 0, <4 x i1> %1)
+  %3 = add i32 %2, %a
+  ret i32 %3
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvaq_p_u8(i32 %a, <16 x i8> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vaddvaq_p_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddvat.u8 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call i32 @llvm.arm.mve.addv.predicated.v16i8.v16i1(<16 x i8> %b, i32 1, <16 x i1> %1)
+  %3 = add i32 %2, %a
+  ret i32 %3
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvaq_p_u16(i32 %a, <8 x i16> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vaddvaq_p_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddvat.u16 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call i32 @llvm.arm.mve.addv.predicated.v8i16.v8i1(<8 x i16> %b, i32 1, <8 x i1> %1)
+  %3 = add i32 %2, %a
+  ret i32 %3
+}
+
+define arm_aapcs_vfpcc i32 @test_vaddvaq_p_u32(i32 %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vaddvaq_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddvat.u32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %b, i32 1, <4 x i1> %1)
+  %3 = add i32 %2, %a
+  ret i32 %3
+}
+
+define arm_aapcs_vfpcc i64 @test_vaddlvq_s32(<4 x i32> %a) {
+; CHECK-LABEL: test_vaddlvq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddlv.s32 r0, r1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call i64 @llvm.arm.mve.addlv.v4i32(<4 x i32> %a, i32 0)
+  ret i64 %0
+}
+
+define arm_aapcs_vfpcc i64 @test_vaddlvq_u32(<4 x i32> %a) {
+; CHECK-LABEL: test_vaddlvq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddlv.u32 r0, r1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call i64 @llvm.arm.mve.addlv.v4i32(<4 x i32> %a, i32 1)
+  ret i64 %0
+}
+
+define arm_aapcs_vfpcc i64 @test_vaddlvaq_s32(i64 %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vaddlvaq_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddlva.s32 r0, r1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call i64 @llvm.arm.mve.addlv.v4i32(<4 x i32> %b, i32 0)
+  %1 = add i64 %0, %a
+  ret i64 %1
+}
+
+define arm_aapcs_vfpcc i64 @test_vaddlvaq_u32(i64 %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vaddlvaq_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vaddlva.u32 r0, r1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call i64 @llvm.arm.mve.addlv.v4i32(<4 x i32> %b, i32 1)
+  %1 = add i64 %0, %a
+  ret i64 %1
+}
+
+define arm_aapcs_vfpcc i64 @test_vaddlvq_p_s32(<4 x i32> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vaddlvq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddlvt.s32 r0, r1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call i64 @llvm.arm.mve.addlv.predicated.v4i32.v4i1(<4 x i32> %a, i32 0, <4 x i1> %1)
+  ret i64 %2
+}
+
+define arm_aapcs_vfpcc i64 @test_vaddlvq_p_u32(<4 x i32> %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vaddlvq_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddlvt.u32 r0, r1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call i64 @llvm.arm.mve.addlv.predicated.v4i32.v4i1(<4 x i32> %a, i32 1, <4 x i1> %1)
+  ret i64 %2
+}
+
+define arm_aapcs_vfpcc i64 @test_vaddlvaq_p_s32(i64 %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vaddlvaq_p_s32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddlvat.s32 r0, r1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call i64 @llvm.arm.mve.addlv.predicated.v4i32.v4i1(<4 x i32> %b, i32 0, <4 x i1> %1)
+  %3 = add i64 %2, %a
+  ret i64 %3
+}
+
+define arm_aapcs_vfpcc i64 @test_vaddlvaq_p_u32(i64 %a, <4 x i32> %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vaddlvaq_p_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddlvat.u32 r0, r1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call i64 @llvm.arm.mve.addlv.predicated.v4i32.v4i1(<4 x i32> %b, i32 1, <4 x i1> %1)
+  %3 = add i64 %2, %a
+  ret i64 %3
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32)
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32)
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
+
+declare i32 @llvm.arm.mve.addv.v16i8(<16 x i8>, i32)
+declare i32 @llvm.arm.mve.addv.v8i16(<8 x i16>, i32)
+declare i32 @llvm.arm.mve.addv.v4i32(<4 x i32>, i32)
+declare i64 @llvm.arm.mve.addlv.v4i32(<4 x i32>, i32)
+
+declare i32 @llvm.arm.mve.addv.predicated.v16i8.v16i1(<16 x i8>, i32, <16 x i1>)
+declare i32 @llvm.arm.mve.addv.predicated.v8i16.v8i1(<8 x i16>, i32, <8 x i1>)
+declare i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32>, i32, <4 x i1>)
+declare i64 @llvm.arm.mve.addlv.predicated.v4i32.v4i1(<4 x i32>, i32, <4 x i1>)