diff --git a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
--- a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
+++ b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
@@ -1,13 +1,267 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple aarch64-none-linux-gnu -mattr=+dotprod    < %s | FileCheck %s
 
+declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
+declare i32 @llvm.vector.reduce.add.v5i32(<5 x i32>)
 declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)
 declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)
+declare i32 @llvm.vector.reduce.add.v24i32(<24 x i32>)
+declare i32 @llvm.vector.reduce.add.v25i32(<25 x i32>)
 declare i32 @llvm.vector.reduce.add.v32i32(<32 x i32>)
 declare i32 @llvm.vector.reduce.add.v33i32(<33 x i32>)
 declare i32 @llvm.vector.reduce.add.v48i32(<48 x i32>)
 declare i32 @llvm.vector.reduce.add.v64i32(<64 x i32>)
 
+define i32 @test_udot_v4i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-LABEL: test_udot_v4i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr s0, [x1]
+; CHECK-NEXT:    ldr s1, [x0]
+; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-NEXT:    umull v0.4s, v0.4h, v1.4h
+; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    add w0, w8, w2
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <4 x i8>, ptr %a
+  %1 = zext <4 x i8> %0 to <4 x i32>
+  %2 = load <4 x i8>, ptr %b
+  %3 = zext <4 x i8> %2 to <4 x i32>
+  %4 = mul nuw nsw <4 x i32> %3, %1
+  %5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %4)
+  %op.extra = add i32 %5, %sum
+  ret i32 %op.extra
+}
+
+define i32 @test_udot_v4i8_nomla(ptr nocapture readonly %a1) {
+; CHECK-LABEL: test_udot_v4i8_nomla:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr s0, [x0]
+; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <4 x i8>, ptr %a1
+  %1 = zext <4 x i8> %0 to <4 x i32>
+  %2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %1)
+  ret i32 %2
+}
+define i32 @test_sdot_v4i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-LABEL: test_sdot_v4i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr s0, [x1]
+; CHECK-NEXT:    ldr s1, [x0]
+; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-NEXT:    smull v0.4s, v0.4h, v1.4h
+; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    add w0, w8, w2
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <4 x i8>, ptr %a
+  %1 = sext <4 x i8> %0 to <4 x i32>
+  %2 = load <4 x i8>, ptr %b
+  %3 = sext <4 x i8> %2 to <4 x i32>
+  %4 = mul nsw <4 x i32> %3, %1
+  %5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %4)
+  %op.extra = add nsw i32 %5, %sum
+  ret i32 %op.extra
+}
+
+define i32 @test_sdot_v4i8_double(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) {
+; CHECK-LABEL: test_sdot_v4i8_double:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ushll v3.4s, v3.4h, #0
+; CHECK-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-NEXT:    shl v3.4s, v3.4s, #24
+; CHECK-NEXT:    shl v2.4s, v2.4s, #24
+; CHECK-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-NEXT:    sshr v3.4s, v3.4s, #24
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    sshr v2.4s, v2.4s, #24
+; CHECK-NEXT:    shl v0.4s, v0.4s, #24
+; CHECK-NEXT:    shl v1.4s, v1.4s, #24
+; CHECK-NEXT:    mul v2.4s, v2.4s, v3.4s
+; CHECK-NEXT:    sshr v0.4s, v0.4s, #24
+; CHECK-NEXT:    sshr v1.4s, v1.4s, #24
+; CHECK-NEXT:    mla v2.4s, v0.4s, v1.4s
+; CHECK-NEXT:    addv s0, v2.4s
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
+entry:
+  %az = sext <4 x i8> %a to <4 x i32>
+  %bz = sext <4 x i8> %b to <4 x i32>
+  %m1 = mul nuw nsw <4 x i32> %az, %bz
+  %r1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m1)
+  %cz = sext <4 x i8> %c to <4 x i32>
+  %dz = sext <4 x i8> %d to <4 x i32>
+  %m2 = mul nuw nsw <4 x i32> %cz, %dz
+  %r2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m2)
+  %x = add i32 %r1, %r2
+  ret i32 %x
+}
+
+define i32 @test_sdot_v4i8_double_nomla(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) {
+; CHECK-LABEL: test_sdot_v4i8_double_nomla:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    ushll v1.4s, v2.4h, #0
+; CHECK-NEXT:    shl v0.4s, v0.4s, #24
+; CHECK-NEXT:    shl v1.4s, v1.4s, #24
+; CHECK-NEXT:    sshr v0.4s, v0.4s, #24
+; CHECK-NEXT:    ssra v0.4s, v1.4s, #24
+; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
+entry:
+  %az = sext <4 x i8> %a to <4 x i32>
+  %r1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %az)
+  %cz = sext <4 x i8> %c to <4 x i32>
+  %r2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %cz)
+  %x = add i32 %r1, %r2
+  ret i32 %x
+}
+
+define i32 @test_udot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-LABEL: test_udot_v5i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr d0, [x1]
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-NEXT:    umull2 v2.4s, v0.8h, v1.8h
+; CHECK-NEXT:    mov v2.s[1], wzr
+; CHECK-NEXT:    mov v2.s[2], wzr
+; CHECK-NEXT:    mov v2.s[3], wzr
+; CHECK-NEXT:    umlal v2.4s, v0.4h, v1.4h
+; CHECK-NEXT:    addv s0, v2.4s
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    add w0, w8, w2
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <5 x i8>, ptr %a
+  %1 = zext <5 x i8> %0 to <5 x i32>
+  %2 = load <5 x i8>, ptr %b
+  %3 = zext <5 x i8> %2 to <5 x i32>
+  %4 = mul nuw nsw <5 x i32> %3, %1
+  %5 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %4)
+  %op.extra = add i32 %5, %sum
+  ret i32 %op.extra
+}
+
+define i32 @test_udot_v5i8_nomla(ptr nocapture readonly %a1) {
+; CHECK-LABEL: test_udot_v5i8_nomla:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-NEXT:    ushll2 v1.4s, v0.8h, #0
+; CHECK-NEXT:    mov v1.s[1], wzr
+; CHECK-NEXT:    mov v1.s[2], wzr
+; CHECK-NEXT:    mov v1.s[3], wzr
+; CHECK-NEXT:    uaddw v0.4s, v1.4s, v0.4h
+; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <5 x i8>, ptr %a1
+  %1 = zext <5 x i8> %0 to <5 x i32>
+  %2 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %1)
+  ret i32 %2
+}
+define i32 @test_sdot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-LABEL: test_sdot_v5i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr d0, [x1]
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-NEXT:    smull2 v2.4s, v0.8h, v1.8h
+; CHECK-NEXT:    mov v2.s[1], wzr
+; CHECK-NEXT:    mov v2.s[2], wzr
+; CHECK-NEXT:    mov v2.s[3], wzr
+; CHECK-NEXT:    smlal v2.4s, v0.4h, v1.4h
+; CHECK-NEXT:    addv s0, v2.4s
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    add w0, w8, w2
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <5 x i8>, ptr %a
+  %1 = sext <5 x i8> %0 to <5 x i32>
+  %2 = load <5 x i8>, ptr %b
+  %3 = sext <5 x i8> %2 to <5 x i32>
+  %4 = mul nsw <5 x i32> %3, %1
+  %5 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %4)
+  %op.extra = add nsw i32 %5, %sum
+  ret i32 %op.extra
+}
+
+define i32 @test_sdot_v5i8_double(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8> %d) {
+; CHECK-LABEL: test_sdot_v5i8_double:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sshll v2.8h, v2.8b, #0
+; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-NEXT:    sshll v3.8h, v3.8b, #0
+; CHECK-NEXT:    smull2 v4.4s, v0.8h, v1.8h
+; CHECK-NEXT:    smull2 v5.4s, v2.8h, v3.8h
+; CHECK-NEXT:    mov v4.s[1], wzr
+; CHECK-NEXT:    mov v5.s[1], wzr
+; CHECK-NEXT:    mov v4.s[2], wzr
+; CHECK-NEXT:    mov v5.s[2], wzr
+; CHECK-NEXT:    mov v4.s[3], wzr
+; CHECK-NEXT:    mov v5.s[3], wzr
+; CHECK-NEXT:    smlal v4.4s, v0.4h, v1.4h
+; CHECK-NEXT:    smlal v5.4s, v2.4h, v3.4h
+; CHECK-NEXT:    add v0.4s, v4.4s, v5.4s
+; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
+entry:
+  %az = sext <5 x i8> %a to <5 x i32>
+  %bz = sext <5 x i8> %b to <5 x i32>
+  %m1 = mul nuw nsw <5 x i32> %az, %bz
+  %r1 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %m1)
+  %cz = sext <5 x i8> %c to <5 x i32>
+  %dz = sext <5 x i8> %d to <5 x i32>
+  %m2 = mul nuw nsw <5 x i32> %cz, %dz
+  %r2 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %m2)
+  %x = add i32 %r1, %r2
+  ret i32 %x
+}
+
+define i32 @test_sdot_v5i8_double_nomla(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8> %d) {
+; CHECK-LABEL: test_sdot_v5i8_double_nomla:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-NEXT:    sshll v1.8h, v2.8b, #0
+; CHECK-NEXT:    sshll2 v2.4s, v0.8h, #0
+; CHECK-NEXT:    sshll2 v3.4s, v1.8h, #0
+; CHECK-NEXT:    mov v2.s[1], wzr
+; CHECK-NEXT:    mov v3.s[1], wzr
+; CHECK-NEXT:    mov v2.s[2], wzr
+; CHECK-NEXT:    mov v3.s[2], wzr
+; CHECK-NEXT:    mov v2.s[3], wzr
+; CHECK-NEXT:    mov v3.s[3], wzr
+; CHECK-NEXT:    saddw v0.4s, v2.4s, v0.4h
+; CHECK-NEXT:    saddw v1.4s, v3.4s, v1.4h
+; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
+entry:
+  %az = sext <5 x i8> %a to <5 x i32>
+  %r1 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %az)
+  %cz = sext <5 x i8> %c to <5 x i32>
+  %r2 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %cz)
+  %x = add i32 %r1, %r2
+  ret i32 %x
+}
+
 define i32 @test_udot_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) {
 ; CHECK-LABEL: test_udot_v8i8:
 ; CHECK:       // %bb.0: // %entry
@@ -254,74 +508,974 @@
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
 entry:
-  %az = sext <8 x i8> %a to <8 x i32>
-  %bz = sext <8 x i8> %b to <8 x i32>
-  %m1 = mul nuw nsw <8 x i32> %az, %bz
-  %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m1)
-  %cz = sext <8 x i8> %c to <8 x i32>
-  %dz = sext <8 x i8> %d to <8 x i32>
-  %m2 = mul nuw nsw <8 x i32> %cz, %dz
-  %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m2)
-  %x = add i32 %r1, %r2
-  ret i32 %x
+  %az = sext <8 x i8> %a to <8 x i32>
+  %bz = sext <8 x i8> %b to <8 x i32>
+  %m1 = mul nuw nsw <8 x i32> %az, %bz
+  %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m1)
+  %cz = sext <8 x i8> %c to <8 x i32>
+  %dz = sext <8 x i8> %d to <8 x i32>
+  %m2 = mul nuw nsw <8 x i32> %cz, %dz
+  %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m2)
+  %x = add i32 %r1, %r2
+  ret i32 %x
+}
+
+define i32 @test_sdot_v8i8_double_nomla(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
+; CHECK-LABEL: test_sdot_v8i8_double_nomla:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v1.8b, #1
+; CHECK-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-NEXT:    sdot v3.2s, v2.8b, v1.8b
+; CHECK-NEXT:    sdot v3.2s, v0.8b, v1.8b
+; CHECK-NEXT:    addp v0.2s, v3.2s, v3.2s
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
+entry:
+  %az = sext <8 x i8> %a to <8 x i32>
+  %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %az)
+  %cz = sext <8 x i8> %c to <8 x i32>
+  %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %cz)
+  %x = add i32 %r1, %r2
+  ret i32 %x
+}
+
+define i32 @test_sdot_v16i8_double(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
+; CHECK-LABEL: test_sdot_v16i8_double:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-NEXT:    sdot v4.4s, v2.16b, v3.16b
+; CHECK-NEXT:    sdot v4.4s, v0.16b, v1.16b
+; CHECK-NEXT:    addv s0, v4.4s
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
+entry:
+  %az = sext <16 x i8> %a to <16 x i32>
+  %bz = sext <16 x i8> %b to <16 x i32>
+  %m1 = mul nuw nsw <16 x i32> %az, %bz
+  %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m1)
+  %cz = sext <16 x i8> %c to <16 x i32>
+  %dz = sext <16 x i8> %d to <16 x i32>
+  %m2 = mul nuw nsw <16 x i32> %cz, %dz
+  %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m2)
+  %x = add i32 %r1, %r2
+  ret i32 %x
+}
+
+define i32 @test_sdot_v16i8_double_nomla(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
+; CHECK-LABEL: test_sdot_v16i8_double_nomla:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v1.16b, #1
+; CHECK-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-NEXT:    sdot v3.4s, v2.16b, v1.16b
+; CHECK-NEXT:    sdot v3.4s, v0.16b, v1.16b
+; CHECK-NEXT:    addv s0, v3.4s
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
+entry:
+  %az = sext <16 x i8> %a to <16 x i32>
+  %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %az)
+  %cz = sext <16 x i8> %c to <16 x i32>
+  %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %cz)
+  %x = add i32 %r1, %r2
+  ret i32 %x
+}
+
+define i32 @test_udot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-LABEL: test_udot_v24i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr q0, [x1]
+; CHECK-NEXT:    ldr q1, [x0]
+; CHECK-NEXT:    ldr d2, [x0, #16]
+; CHECK-NEXT:    ushll v5.8h, v0.8b, #0
+; CHECK-NEXT:    ldr d3, [x1, #16]
+; CHECK-NEXT:    ushll v4.8h, v1.8b, #0
+; CHECK-NEXT:    ushll v2.8h, v2.8b, #0
+; CHECK-NEXT:    umull2 v6.4s, v5.8h, v4.8h
+; CHECK-NEXT:    umull v4.4s, v5.4h, v4.4h
+; CHECK-NEXT:    ushll v3.8h, v3.8b, #0
+; CHECK-NEXT:    ushll2 v1.8h, v1.16b, #0
+; CHECK-NEXT:    umlal2 v6.4s, v3.8h, v2.8h
+; CHECK-NEXT:    umlal v4.4s, v3.4h, v2.4h
+; CHECK-NEXT:    ushll2 v0.8h, v0.16b, #0
+; CHECK-NEXT:    umlal2 v6.4s, v0.8h, v1.8h
+; CHECK-NEXT:    umlal v4.4s, v0.4h, v1.4h
+; CHECK-NEXT:    add v0.4s, v4.4s, v6.4s
+; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    add w0, w8, w2
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <24 x i8>, ptr %a
+  %1 = zext <24 x i8> %0 to <24 x i32>
+  %2 = load <24 x i8>, ptr %b
+  %3 = zext <24 x i8> %2 to <24 x i32>
+  %4 = mul nuw nsw <24 x i32> %3, %1
+  %5 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %4)
+  %op.extra = add i32 %5, %sum
+  ret i32 %op.extra
+}
+
+define i32 @test_udot_v24i8_nomla(ptr nocapture readonly %a1) {
+; CHECK-LABEL: test_udot_v24i8_nomla:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr d0, [x0, #16]
+; CHECK-NEXT:    ldr q1, [x0]
+; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-NEXT:    ushll v2.8h, v1.8b, #0
+; CHECK-NEXT:    ushll2 v1.8h, v1.16b, #0
+; CHECK-NEXT:    uaddl2 v3.4s, v2.8h, v0.8h
+; CHECK-NEXT:    uaddl v0.4s, v2.4h, v0.4h
+; CHECK-NEXT:    uaddw2 v2.4s, v3.4s, v1.8h
+; CHECK-NEXT:    uaddw v0.4s, v0.4s, v1.4h
+; CHECK-NEXT:    add v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <24 x i8>, ptr %a1
+  %1 = zext <24 x i8> %0 to <24 x i32>
+  %2 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %1)
+  ret i32 %2
+}
+define i32 @test_sdot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-LABEL: test_sdot_v24i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr q0, [x1]
+; CHECK-NEXT:    ldr q1, [x0]
+; CHECK-NEXT:    ldr d2, [x0, #16]
+; CHECK-NEXT:    sshll v5.8h, v0.8b, #0
+; CHECK-NEXT:    ldr d3, [x1, #16]
+; CHECK-NEXT:    sshll v4.8h, v1.8b, #0
+; CHECK-NEXT:    sshll v2.8h, v2.8b, #0
+; CHECK-NEXT:    smull2 v6.4s, v5.8h, v4.8h
+; CHECK-NEXT:    smull v4.4s, v5.4h, v4.4h
+; CHECK-NEXT:    sshll v3.8h, v3.8b, #0
+; CHECK-NEXT:    sshll2 v1.8h, v1.16b, #0
+; CHECK-NEXT:    smlal2 v6.4s, v3.8h, v2.8h
+; CHECK-NEXT:    smlal v4.4s, v3.4h, v2.4h
+; CHECK-NEXT:    sshll2 v0.8h, v0.16b, #0
+; CHECK-NEXT:    smlal2 v6.4s, v0.8h, v1.8h
+; CHECK-NEXT:    smlal v4.4s, v0.4h, v1.4h
+; CHECK-NEXT:    add v0.4s, v4.4s, v6.4s
+; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    add w0, w8, w2
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <24 x i8>, ptr %a
+  %1 = sext <24 x i8> %0 to <24 x i32>
+  %2 = load <24 x i8>, ptr %b
+  %3 = sext <24 x i8> %2 to <24 x i32>
+  %4 = mul nsw <24 x i32> %3, %1
+  %5 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %4)
+  %op.extra = add nsw i32 %5, %sum
+  ret i32 %op.extra
+}
+
+define i32 @test_sdot_v24i8_double(<24 x i8> %a, <24 x i8> %b, <24 x i8> %c, <24 x i8> %d) {
+; CHECK-LABEL: test_sdot_v24i8_double:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr b0, [sp]
+; CHECK-NEXT:    add x8, sp, #8
+; CHECK-NEXT:    ldr b3, [sp, #64]
+; CHECK-NEXT:    add x9, sp, #16
+; CHECK-NEXT:    ldr b1, [sp, #192]
+; CHECK-NEXT:    add x10, sp, #200
+; CHECK-NEXT:    ld1 { v0.b }[1], [x8]
+; CHECK-NEXT:    add x8, sp, #72
+; CHECK-NEXT:    ldr b5, [sp, #256]
+; CHECK-NEXT:    fmov s6, w0
+; CHECK-NEXT:    ld1 { v1.b }[1], [x10]
+; CHECK-NEXT:    add x10, sp, #264
+; CHECK-NEXT:    ld1 { v3.b }[1], [x8]
+; CHECK-NEXT:    add x8, sp, #24
+; CHECK-NEXT:    ld1 { v0.b }[2], [x9]
+; CHECK-NEXT:    add x9, sp, #80
+; CHECK-NEXT:    ld1 { v5.b }[1], [x10]
+; CHECK-NEXT:    add x10, sp, #208
+; CHECK-NEXT:    ldr b16, [sp, #128]
+; CHECK-NEXT:    add x11, sp, #56
+; CHECK-NEXT:    ld1 { v3.b }[2], [x9]
+; CHECK-NEXT:    add x9, sp, #88
+; CHECK-NEXT:    ld1 { v0.b }[3], [x8]
+; CHECK-NEXT:    add x8, sp, #32
+; CHECK-NEXT:    ld1 { v1.b }[2], [x10]
+; CHECK-NEXT:    add x10, sp, #136
+; CHECK-NEXT:    mov v6.b[1], w1
+; CHECK-NEXT:    ldr b2, [sp, #384]
+; CHECK-NEXT:    ld1 { v3.b }[3], [x9]
+; CHECK-NEXT:    add x9, sp, #40
+; CHECK-NEXT:    ld1 { v0.b }[4], [x8]
+; CHECK-NEXT:    add x8, sp, #96
+; CHECK-NEXT:    ld1 { v16.b }[1], [x10]
+; CHECK-NEXT:    add x10, sp, #144
+; CHECK-NEXT:    mov v6.b[2], w2
+; CHECK-NEXT:    ldr b4, [sp, #448]
+; CHECK-NEXT:    ld1 { v3.b }[4], [x8]
+; CHECK-NEXT:    add x8, sp, #48
+; CHECK-NEXT:    ld1 { v0.b }[5], [x9]
+; CHECK-NEXT:    add x9, sp, #104
+; CHECK-NEXT:    ld1 { v16.b }[2], [x10]
+; CHECK-NEXT:    add x10, sp, #120
+; CHECK-NEXT:    mov v6.b[3], w3
+; CHECK-NEXT:    add x12, sp, #584
+; CHECK-NEXT:    ld1 { v3.b }[5], [x9]
+; CHECK-NEXT:    add x9, sp, #216
+; CHECK-NEXT:    ld1 { v0.b }[6], [x8]
+; CHECK-NEXT:    add x8, sp, #112
+; CHECK-NEXT:    ld1 { v1.b }[3], [x9]
+; CHECK-NEXT:    add x9, sp, #272
+; CHECK-NEXT:    ld1 { v3.b }[6], [x8]
+; CHECK-NEXT:    add x8, sp, #224
+; CHECK-NEXT:    ld1 { v0.b }[7], [x11]
+; CHECK-NEXT:    add x11, sp, #232
+; CHECK-NEXT:    ld1 { v5.b }[2], [x9]
+; CHECK-NEXT:    add x9, sp, #152
+; CHECK-NEXT:    ld1 { v1.b }[4], [x8]
+; CHECK-NEXT:    add x8, sp, #280
+; CHECK-NEXT:    ld1 { v3.b }[7], [x10]
+; CHECK-NEXT:    add x10, sp, #240
+; CHECK-NEXT:    ld1 { v16.b }[3], [x9]
+; CHECK-NEXT:    add x9, sp, #288
+; CHECK-NEXT:    ld1 { v5.b }[3], [x8]
+; CHECK-NEXT:    add x8, sp, #160
+; CHECK-NEXT:    ld1 { v1.b }[5], [x11]
+; CHECK-NEXT:    add x11, sp, #248
+; CHECK-NEXT:    mov v6.b[4], w4
+; CHECK-NEXT:    ld1 { v16.b }[4], [x8]
+; CHECK-NEXT:    add x8, sp, #168
+; CHECK-NEXT:    ld1 { v5.b }[4], [x9]
+; CHECK-NEXT:    add x9, sp, #296
+; CHECK-NEXT:    ld1 { v1.b }[6], [x10]
+; CHECK-NEXT:    add x10, sp, #392
+; CHECK-NEXT:    mov v6.b[5], w5
+; CHECK-NEXT:    ld1 { v16.b }[5], [x8]
+; CHECK-NEXT:    add x8, sp, #176
+; CHECK-NEXT:    ld1 { v2.b }[1], [x10]
+; CHECK-NEXT:    add x10, sp, #400
+; CHECK-NEXT:    ld1 { v5.b }[5], [x9]
+; CHECK-NEXT:    add x9, sp, #304
+; CHECK-NEXT:    mov v6.b[6], w6
+; CHECK-NEXT:    ld1 { v1.b }[7], [x11]
+; CHECK-NEXT:    ld1 { v16.b }[6], [x8]
+; CHECK-NEXT:    add x8, sp, #184
+; CHECK-NEXT:    ld1 { v2.b }[2], [x10]
+; CHECK-NEXT:    add x10, sp, #456
+; CHECK-NEXT:    ld1 { v5.b }[6], [x9]
+; CHECK-NEXT:    add x9, sp, #312
+; CHECK-NEXT:    mov v6.b[7], w7
+; CHECK-NEXT:    add x11, sp, #648
+; CHECK-NEXT:    ld1 { v16.b }[7], [x8]
+; CHECK-NEXT:    add x8, sp, #408
+; CHECK-NEXT:    ld1 { v4.b }[1], [x10]
+; CHECK-NEXT:    add x10, sp, #328
+; CHECK-NEXT:    ld1 { v5.b }[7], [x9]
+; CHECK-NEXT:    add x9, sp, #416
+; CHECK-NEXT:    ld1 { v2.b }[3], [x8]
+; CHECK-NEXT:    add x8, sp, #464
+; CHECK-NEXT:    sshll v6.8h, v6.8b, #0
+; CHECK-NEXT:    sshll v17.8h, v16.8b, #0
+; CHECK-NEXT:    ld1 { v4.b }[2], [x8]
+; CHECK-NEXT:    add x8, sp, #472
+; CHECK-NEXT:    ld1 { v2.b }[4], [x9]
+; CHECK-NEXT:    add x9, sp, #424
+; CHECK-NEXT:    sshll v7.8h, v3.8b, #0
+; CHECK-NEXT:    sshll v16.8h, v5.8b, #0
+; CHECK-NEXT:    smull2 v3.4s, v6.8h, v17.8h
+; CHECK-NEXT:    ld1 { v4.b }[3], [x8]
+; CHECK-NEXT:    smull v5.4s, v6.4h, v17.4h
+; CHECK-NEXT:    ldr b6, [sp, #320]
+; CHECK-NEXT:    ld1 { v2.b }[5], [x9]
+; CHECK-NEXT:    add x9, sp, #480
+; CHECK-NEXT:    add x8, sp, #432
+; CHECK-NEXT:    ldr b17, [sp, #512]
+; CHECK-NEXT:    ld1 { v6.b }[1], [x10]
+; CHECK-NEXT:    add x10, sp, #520
+; CHECK-NEXT:    ld1 { v4.b }[4], [x9]
+; CHECK-NEXT:    add x9, sp, #336
+; CHECK-NEXT:    ld1 { v2.b }[6], [x8]
+; CHECK-NEXT:    add x8, sp, #488
+; CHECK-NEXT:    ld1 { v17.b }[1], [x10]
+; CHECK-NEXT:    add x10, sp, #440
+; CHECK-NEXT:    ld1 { v6.b }[2], [x9]
+; CHECK-NEXT:    add x9, sp, #528
+; CHECK-NEXT:    ld1 { v4.b }[5], [x8]
+; CHECK-NEXT:    add x8, sp, #344
+; CHECK-NEXT:    smlal2 v3.4s, v7.8h, v16.8h
+; CHECK-NEXT:    ld1 { v2.b }[7], [x10]
+; CHECK-NEXT:    smlal v5.4s, v7.4h, v16.4h
+; CHECK-NEXT:    ldr b7, [sp, #640]
+; CHECK-NEXT:    ld1 { v17.b }[2], [x9]
+; CHECK-NEXT:    add x9, sp, #352
+; CHECK-NEXT:    ld1 { v6.b }[3], [x8]
+; CHECK-NEXT:    add x8, sp, #536
+; CHECK-NEXT:    ld1 { v7.b }[1], [x11]
+; CHECK-NEXT:    add x11, sp, #656
+; CHECK-NEXT:    ldr b16, [sp, #576]
+; CHECK-NEXT:    ld1 { v17.b }[3], [x8]
+; CHECK-NEXT:    add x8, sp, #544
+; CHECK-NEXT:    ld1 { v6.b }[4], [x9]
+; CHECK-NEXT:    add x9, sp, #360
+; CHECK-NEXT:    ld1 { v16.b }[1], [x12]
+; CHECK-NEXT:    add x12, sp, #592
+; CHECK-NEXT:    ld1 { v7.b }[2], [x11]
+; CHECK-NEXT:    add x11, sp, #664
+; CHECK-NEXT:    ld1 { v17.b }[4], [x8]
+; CHECK-NEXT:    add x8, sp, #552
+; CHECK-NEXT:    ld1 { v6.b }[5], [x9]
+; CHECK-NEXT:    add x9, sp, #368
+; CHECK-NEXT:    ld1 { v16.b }[2], [x12]
+; CHECK-NEXT:    add x12, sp, #600
+; CHECK-NEXT:    ld1 { v7.b }[3], [x11]
+; CHECK-NEXT:    add x11, sp, #672
+; CHECK-NEXT:    ld1 { v17.b }[5], [x8]
+; CHECK-NEXT:    add x8, sp, #560
+; CHECK-NEXT:    ld1 { v6.b }[6], [x9]
+; CHECK-NEXT:    add x9, sp, #376
+; CHECK-NEXT:    ld1 { v16.b }[3], [x12]
+; CHECK-NEXT:    add x12, sp, #608
+; CHECK-NEXT:    ld1 { v7.b }[4], [x11]
+; CHECK-NEXT:    add x11, sp, #680
+; CHECK-NEXT:    ld1 { v17.b }[6], [x8]
+; CHECK-NEXT:    add x8, sp, #568
+; CHECK-NEXT:    ld1 { v6.b }[7], [x9]
+; CHECK-NEXT:    add x9, sp, #496
+; CHECK-NEXT:    ld1 { v16.b }[4], [x12]
+; CHECK-NEXT:    add x12, sp, #616
+; CHECK-NEXT:    ld1 { v7.b }[5], [x11]
+; CHECK-NEXT:    add x11, sp, #688
+; CHECK-NEXT:    ld1 { v4.b }[6], [x9]
+; CHECK-NEXT:    add x9, sp, #696
+; CHECK-NEXT:    ld1 { v17.b }[7], [x8]
+; CHECK-NEXT:    add x8, sp, #504
+; CHECK-NEXT:    ld1 { v16.b }[5], [x12]
+; CHECK-NEXT:    ld1 { v7.b }[6], [x11]
+; CHECK-NEXT:    ld1 { v4.b }[7], [x8]
+; CHECK-NEXT:    add x8, sp, #624
+; CHECK-NEXT:    sshll v6.8h, v6.8b, #0
+; CHECK-NEXT:    sshll v17.8h, v17.8b, #0
+; CHECK-NEXT:    ld1 { v7.b }[7], [x9]
+; CHECK-NEXT:    ld1 { v16.b }[6], [x8]
+; CHECK-NEXT:    add x8, sp, #632
+; CHECK-NEXT:    smull v18.4s, v6.4h, v17.4h
+; CHECK-NEXT:    smull2 v6.4s, v6.8h, v17.8h
+; CHECK-NEXT:    sshll v4.8h, v4.8b, #0
+; CHECK-NEXT:    sshll v7.8h, v7.8b, #0
+; CHECK-NEXT:    ld1 { v16.b }[7], [x8]
+; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-NEXT:    smlal2 v6.4s, v4.8h, v7.8h
+; CHECK-NEXT:    smlal v18.4s, v4.4h, v7.4h
+; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-NEXT:    sshll v2.8h, v2.8b, #0
+; CHECK-NEXT:    sshll v4.8h, v16.8b, #0
+; CHECK-NEXT:    smlal2 v3.4s, v0.8h, v1.8h
+; CHECK-NEXT:    smlal v5.4s, v0.4h, v1.4h
+; CHECK-NEXT:    smlal2 v6.4s, v2.8h, v4.8h
+; CHECK-NEXT:    smlal v18.4s, v2.4h, v4.4h
+; CHECK-NEXT:    add v0.4s, v5.4s, v3.4s
+; CHECK-NEXT:    add v1.4s, v18.4s, v6.4s
+; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
+entry:
+  %az = sext <24 x i8> %a to <24 x i32>
+  %bz = sext <24 x i8> %b to <24 x i32>
+  %m1 = mul nuw nsw <24 x i32> %az, %bz
+  %r1 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %m1)
+  %cz = sext <24 x i8> %c to <24 x i32>
+  %dz = sext <24 x i8> %d to <24 x i32>
+  %m2 = mul nuw nsw <24 x i32> %cz, %dz
+  %r2 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %m2)
+  %x = add i32 %r1, %r2
+  ret i32 %x
+}
+
+define i32 @test_sdot_v24i8_double_nomla(<24 x i8> %a, <24 x i8> %b, <24 x i8> %c, <24 x i8> %d) {
+; CHECK-LABEL: test_sdot_v24i8_double_nomla:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr b0, [sp]
+; CHECK-NEXT:    add x8, sp, #8
+; CHECK-NEXT:    add x9, sp, #16
+; CHECK-NEXT:    ldr b1, [sp, #64]
+; CHECK-NEXT:    ldr b2, [sp, #384]
+; CHECK-NEXT:    add x10, sp, #24
+; CHECK-NEXT:    ld1 { v0.b }[1], [x8]
+; CHECK-NEXT:    add x8, sp, #72
+; CHECK-NEXT:    ldr b3, [sp, #448]
+; CHECK-NEXT:    fmov s4, w0
+; CHECK-NEXT:    ldr b5, [sp, #320]
+; CHECK-NEXT:    add x12, sp, #472
+; CHECK-NEXT:    ld1 { v1.b }[1], [x8]
+; CHECK-NEXT:    add x8, sp, #456
+; CHECK-NEXT:    ld1 { v0.b }[2], [x9]
+; CHECK-NEXT:    add x9, sp, #392
+; CHECK-NEXT:    mov v4.b[1], w1
+; CHECK-NEXT:    add x11, sp, #56
+; CHECK-NEXT:    ld1 { v3.b }[1], [x8]
+; CHECK-NEXT:    add x8, sp, #32
+; CHECK-NEXT:    ld1 { v2.b }[1], [x9]
+; CHECK-NEXT:    add x9, sp, #80
+; CHECK-NEXT:    ld1 { v0.b }[3], [x10]
+; CHECK-NEXT:    add x10, sp, #40
+; CHECK-NEXT:    mov v4.b[2], w2
+; CHECK-NEXT:    ld1 { v1.b }[2], [x9]
+; CHECK-NEXT:    add x9, sp, #328
+; CHECK-NEXT:    ld1 { v0.b }[4], [x8]
+; CHECK-NEXT:    add x8, sp, #88
+; CHECK-NEXT:    ld1 { v5.b }[1], [x9]
+; CHECK-NEXT:    add x9, sp, #48
+; CHECK-NEXT:    mov v4.b[3], w3
+; CHECK-NEXT:    ld1 { v1.b }[3], [x8]
+; CHECK-NEXT:    add x8, sp, #464
+; CHECK-NEXT:    ld1 { v0.b }[5], [x10]
+; CHECK-NEXT:    add x10, sp, #96
+; CHECK-NEXT:    ld1 { v3.b }[2], [x8]
+; CHECK-NEXT:    add x8, sp, #104
+; CHECK-NEXT:    ld1 { v1.b }[4], [x10]
+; CHECK-NEXT:    add x10, sp, #336
+; CHECK-NEXT:    ld1 { v0.b }[6], [x9]
+; CHECK-NEXT:    add x9, sp, #112
+; CHECK-NEXT:    mov v4.b[4], w4
+; CHECK-NEXT:    ld1 { v5.b }[2], [x10]
+; CHECK-NEXT:    add x10, sp, #344
+; CHECK-NEXT:    ld1 { v1.b }[5], [x8]
+; CHECK-NEXT:    add x8, sp, #400
+; CHECK-NEXT:    ld1 { v3.b }[3], [x12]
+; CHECK-NEXT:    ld1 { v0.b }[7], [x11]
+; CHECK-NEXT:    add x11, sp, #480
+; CHECK-NEXT:    ld1 { v2.b }[2], [x8]
+; CHECK-NEXT:    add x8, sp, #352
+; CHECK-NEXT:    ld1 { v5.b }[3], [x10]
+; CHECK-NEXT:    add x10, sp, #488
+; CHECK-NEXT:    ld1 { v1.b }[6], [x9]
+; CHECK-NEXT:    add x9, sp, #408
+; CHECK-NEXT:    ld1 { v3.b }[4], [x11]
+; CHECK-NEXT:    add x11, sp, #416
+; CHECK-NEXT:    mov v4.b[5], w5
+; CHECK-NEXT:    ld1 { v2.b }[3], [x9]
+; CHECK-NEXT:    add x9, sp, #120
+; CHECK-NEXT:    ld1 { v5.b }[4], [x8]
+; CHECK-NEXT:    add x8, sp, #360
+; CHECK-NEXT:    ld1 { v3.b }[5], [x10]
+; CHECK-NEXT:    add x10, sp, #496
+; CHECK-NEXT:    ld1 { v1.b }[7], [x9]
+; CHECK-NEXT:    add x9, sp, #424
+; CHECK-NEXT:    ld1 { v2.b }[4], [x11]
+; CHECK-NEXT:    ld1 { v5.b }[5], [x8]
+; CHECK-NEXT:    add x8, sp, #368
+; CHECK-NEXT:    mov v4.b[6], w6
+; CHECK-NEXT:    ld1 { v3.b }[6], [x10]
+; CHECK-NEXT:    add x10, sp, #504
+; CHECK-NEXT:    ld1 { v2.b }[5], [x9]
+; CHECK-NEXT:    add x9, sp, #432
+; CHECK-NEXT:    ld1 { v5.b }[6], [x8]
+; CHECK-NEXT:    add x8, sp, #376
+; CHECK-NEXT:    mov v4.b[7], w7
+; CHECK-NEXT:    ld1 { v3.b }[7], [x10]
+; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-NEXT:    ld1 { v2.b }[6], [x9]
+; CHECK-NEXT:    ld1 { v5.b }[7], [x8]
+; CHECK-NEXT:    add x8, sp, #440
+; CHECK-NEXT:    sshll v4.8h, v4.8b, #0
+; CHECK-NEXT:    sshll v3.8h, v3.8b, #0
+; CHECK-NEXT:    ld1 { v2.b }[7], [x8]
+; CHECK-NEXT:    sshll v5.8h, v5.8b, #0
+; CHECK-NEXT:    saddl2 v6.4s, v4.8h, v1.8h
+; CHECK-NEXT:    saddl v1.4s, v4.4h, v1.4h
+; CHECK-NEXT:    saddl2 v4.4s, v5.8h, v3.8h
+; CHECK-NEXT:    saddl v3.4s, v5.4h, v3.4h
+; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-NEXT:    sshll v2.8h, v2.8b, #0
+; CHECK-NEXT:    saddw2 v5.4s, v6.4s, v0.8h
+; CHECK-NEXT:    saddw v0.4s, v1.4s, v0.4h
+; CHECK-NEXT:    saddw2 v1.4s, v4.4s, v2.8h
+; CHECK-NEXT:    saddw v2.4s, v3.4s, v2.4h
+; CHECK-NEXT:    add v0.4s, v0.4s, v5.4s
+; CHECK-NEXT:    add v1.4s, v2.4s, v1.4s
+; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
+entry:
+  %az = sext <24 x i8> %a to <24 x i32>
+  %r1 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %az)
+  %cz = sext <24 x i8> %c to <24 x i32>
+  %r2 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %cz)
+  %x = add i32 %r1, %r2
+  ret i32 %x
+}
+
+
+define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-LABEL: test_udot_v25i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldp q3, q0, [x1]
+; CHECK-NEXT:    ushll v6.8h, v3.8b, #0
+; CHECK-NEXT:    ushll2 v3.8h, v3.16b, #0
+; CHECK-NEXT:    ldp q2, q1, [x0]
+; CHECK-NEXT:    ushll2 v5.8h, v0.16b, #0
+; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-NEXT:    ushll2 v4.8h, v1.16b, #0
+; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-NEXT:    umull v4.4s, v5.4h, v4.4h
+; CHECK-NEXT:    ushll2 v5.8h, v2.16b, #0
+; CHECK-NEXT:    ushll v2.8h, v2.8b, #0
+; CHECK-NEXT:    mov v4.s[1], wzr
+; CHECK-NEXT:    umull2 v7.4s, v6.8h, v2.8h
+; CHECK-NEXT:    umull v2.4s, v6.4h, v2.4h
+; CHECK-NEXT:    mov v4.s[2], wzr
+; CHECK-NEXT:    umlal2 v7.4s, v0.8h, v1.8h
+; CHECK-NEXT:    umlal v2.4s, v0.4h, v1.4h
+; CHECK-NEXT:    mov v4.s[3], wzr
+; CHECK-NEXT:    umlal2 v7.4s, v3.8h, v5.8h
+; CHECK-NEXT:    umlal v4.4s, v3.4h, v5.4h
+; CHECK-NEXT:    add v0.4s, v2.4s, v7.4s
+; CHECK-NEXT:    add v0.4s, v0.4s, v4.4s
+; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    add w0, w8, w2
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <25 x i8>, ptr %a
+  %1 = zext <25 x i8> %0 to <25 x i32>
+  %2 = load <25 x i8>, ptr %b
+  %3 = zext <25 x i8> %2 to <25 x i32>
+  %4 = mul nuw nsw <25 x i32> %3, %1
+  %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4)
+  %op.extra = add i32 %5, %sum
+  ret i32 %op.extra
+}
+
+define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) {
+; CHECK-LABEL: test_udot_v25i8_nomla:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldp q1, q0, [x0]
+; CHECK-NEXT:    ushll v3.8h, v1.8b, #0
+; CHECK-NEXT:    ushll2 v1.8h, v1.16b, #0
+; CHECK-NEXT:    ushll2 v2.8h, v0.16b, #0
+; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-NEXT:    uaddl2 v4.4s, v3.8h, v0.8h
+; CHECK-NEXT:    mov v2.s[1], wzr
+; CHECK-NEXT:    uaddl v0.4s, v3.4h, v0.4h
+; CHECK-NEXT:    uaddw2 v3.4s, v4.4s, v1.8h
+; CHECK-NEXT:    mov v2.s[2], wzr
+; CHECK-NEXT:    add v0.4s, v0.4s, v3.4s
+; CHECK-NEXT:    mov v2.s[3], wzr
+; CHECK-NEXT:    uaddw v1.4s, v2.4s, v1.4h
+; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <25 x i8>, ptr %a1
+  %1 = zext <25 x i8> %0 to <25 x i32>
+  %2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %1)
+  ret i32 %2
 }
-
-define i32 @test_sdot_v8i8_double_nomla(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
-; CHECK-LABEL: test_sdot_v8i8_double_nomla:
+define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-LABEL: test_sdot_v25i8:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.8b, #1
-; CHECK-NEXT:    movi v3.2d, #0000000000000000
-; CHECK-NEXT:    sdot v3.2s, v2.8b, v1.8b
-; CHECK-NEXT:    sdot v3.2s, v0.8b, v1.8b
-; CHECK-NEXT:    addp v0.2s, v3.2s, v3.2s
-; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ldp q3, q0, [x1]
+; CHECK-NEXT:    sshll v6.8h, v3.8b, #0
+; CHECK-NEXT:    sshll2 v3.8h, v3.16b, #0
+; CHECK-NEXT:    ldp q2, q1, [x0]
+; CHECK-NEXT:    sshll2 v5.8h, v0.16b, #0
+; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-NEXT:    sshll2 v4.8h, v1.16b, #0
+; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-NEXT:    smull v4.4s, v5.4h, v4.4h
+; CHECK-NEXT:    sshll2 v5.8h, v2.16b, #0
+; CHECK-NEXT:    sshll v2.8h, v2.8b, #0
+; CHECK-NEXT:    mov v4.s[1], wzr
+; CHECK-NEXT:    smull2 v7.4s, v6.8h, v2.8h
+; CHECK-NEXT:    smull v2.4s, v6.4h, v2.4h
+; CHECK-NEXT:    mov v4.s[2], wzr
+; CHECK-NEXT:    smlal2 v7.4s, v0.8h, v1.8h
+; CHECK-NEXT:    smlal v2.4s, v0.4h, v1.4h
+; CHECK-NEXT:    mov v4.s[3], wzr
+; CHECK-NEXT:    smlal2 v7.4s, v3.8h, v5.8h
+; CHECK-NEXT:    smlal v4.4s, v3.4h, v5.4h
+; CHECK-NEXT:    add v0.4s, v2.4s, v7.4s
+; CHECK-NEXT:    add v0.4s, v0.4s, v4.4s
+; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    add w0, w8, w2
 ; CHECK-NEXT:    ret
 entry:
-  %az = sext <8 x i8> %a to <8 x i32>
-  %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %az)
-  %cz = sext <8 x i8> %c to <8 x i32>
-  %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %cz)
-  %x = add i32 %r1, %r2
-  ret i32 %x
+  %0 = load <25 x i8>, ptr %a
+  %1 = sext <25 x i8> %0 to <25 x i32>
+  %2 = load <25 x i8>, ptr %b
+  %3 = sext <25 x i8> %2 to <25 x i32>
+  %4 = mul nsw <25 x i32> %3, %1
+  %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4)
+  %op.extra = add nsw i32 %5, %sum
+  ret i32 %op.extra
 }
 
-define i32 @test_sdot_v16i8_double(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
-; CHECK-LABEL: test_sdot_v16i8_double:
+define i32 @test_sdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) {
+; CHECK-LABEL: test_sdot_v25i8_double:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v4.2d, #0000000000000000
-; CHECK-NEXT:    sdot v4.4s, v2.16b, v3.16b
-; CHECK-NEXT:    sdot v4.4s, v0.16b, v1.16b
-; CHECK-NEXT:    addv s0, v4.4s
+; CHECK-NEXT:    ldr b2, [sp, #64]
+; CHECK-NEXT:    add x8, sp, #72
+; CHECK-NEXT:    ldr b0, [sp]
+; CHECK-NEXT:    add x9, sp, #80
+; CHECK-NEXT:    ldr b4, [sp, #264]
+; CHECK-NEXT:    add x10, sp, #272
+; CHECK-NEXT:    ld1 { v2.b }[1], [x8]
+; CHECK-NEXT:    add x8, sp, #8
+; CHECK-NEXT:    add x11, sp, #288
+; CHECK-NEXT:    add x12, sp, #40
+; CHECK-NEXT:    ld1 { v4.b }[1], [x10]
+; CHECK-NEXT:    add x10, sp, #280
+; CHECK-NEXT:    ld1 { v0.b }[1], [x8]
+; CHECK-NEXT:    add x8, sp, #88
+; CHECK-NEXT:    ld1 { v2.b }[2], [x9]
+; CHECK-NEXT:    add x9, sp, #16
+; CHECK-NEXT:    ldr b1, [sp, #200]
+; CHECK-NEXT:    fmov s6, w0
+; CHECK-NEXT:    ld1 { v4.b }[2], [x10]
+; CHECK-NEXT:    add x10, sp, #112
+; CHECK-NEXT:    ld1 { v0.b }[2], [x9]
+; CHECK-NEXT:    add x9, sp, #96
+; CHECK-NEXT:    ld1 { v2.b }[3], [x8]
+; CHECK-NEXT:    add x8, sp, #24
+; CHECK-NEXT:    ldr b16, [sp, #136]
+; CHECK-NEXT:    ld1 { v4.b }[3], [x11]
+; CHECK-NEXT:    add x11, sp, #208
+; CHECK-NEXT:    ld1 { v0.b }[3], [x8]
+; CHECK-NEXT:    add x8, sp, #104
+; CHECK-NEXT:    ld1 { v2.b }[4], [x9]
+; CHECK-NEXT:    add x9, sp, #32
+; CHECK-NEXT:    ld1 { v1.b }[1], [x11]
+; CHECK-NEXT:    add x11, sp, #312
+; CHECK-NEXT:    ldr b3, [sp, #464]
+; CHECK-NEXT:    ld1 { v0.b }[4], [x9]
+; CHECK-NEXT:    add x9, sp, #48
+; CHECK-NEXT:    ld1 { v2.b }[5], [x8]
+; CHECK-NEXT:    add x8, sp, #120
+; CHECK-NEXT:    mov v6.b[1], w1
+; CHECK-NEXT:    ldr b7, [sp, #336]
+; CHECK-NEXT:    ldr b19, [sp, #536]
+; CHECK-NEXT:    ld1 { v0.b }[5], [x12]
+; CHECK-NEXT:    add x12, sp, #56
+; CHECK-NEXT:    ld1 { v2.b }[6], [x10]
+; CHECK-NEXT:    add x10, sp, #144
+; CHECK-NEXT:    mov v6.b[2], w2
+; CHECK-NEXT:    ldr b5, [sp, #128]
+; CHECK-NEXT:    ldr b17, [sp, #328]
+; CHECK-NEXT:    ld1 { v0.b }[6], [x9]
+; CHECK-NEXT:    add x9, sp, #216
+; CHECK-NEXT:    ld1 { v16.b }[1], [x10]
+; CHECK-NEXT:    add x10, sp, #296
+; CHECK-NEXT:    ld1 { v2.b }[7], [x8]
+; CHECK-NEXT:    add x8, sp, #152
+; CHECK-NEXT:    ld1 { v1.b }[2], [x9]
+; CHECK-NEXT:    add x9, sp, #224
+; CHECK-NEXT:    ld1 { v4.b }[4], [x10]
+; CHECK-NEXT:    add x10, sp, #304
+; CHECK-NEXT:    ld1 { v16.b }[2], [x8]
+; CHECK-NEXT:    add x8, sp, #160
+; CHECK-NEXT:    mov v6.b[3], w3
+; CHECK-NEXT:    ldr b20, [sp, #528]
+; CHECK-NEXT:    ld1 { v1.b }[3], [x9]
+; CHECK-NEXT:    add x9, sp, #232
+; CHECK-NEXT:    ld1 { v4.b }[5], [x10]
+; CHECK-NEXT:    add x10, sp, #472
+; CHECK-NEXT:    ld1 { v16.b }[3], [x8]
+; CHECK-NEXT:    add x8, sp, #168
+; CHECK-NEXT:    mov v6.b[4], w4
+; CHECK-NEXT:    ld1 { v0.b }[7], [x12]
+; CHECK-NEXT:    ld1 { v1.b }[4], [x9]
+; CHECK-NEXT:    add x9, sp, #240
+; CHECK-NEXT:    ld1 { v3.b }[1], [x10]
+; CHECK-NEXT:    add x10, sp, #480
+; CHECK-NEXT:    ld1 { v16.b }[4], [x8]
+; CHECK-NEXT:    add x8, sp, #176
+; CHECK-NEXT:    mov v6.b[5], w5
+; CHECK-NEXT:    ld1 { v4.b }[6], [x11]
+; CHECK-NEXT:    ld1 { v1.b }[5], [x9]
+; CHECK-NEXT:    add x9, sp, #248
+; CHECK-NEXT:    ld1 { v3.b }[2], [x10]
+; CHECK-NEXT:    add x10, sp, #488
+; CHECK-NEXT:    ld1 { v16.b }[5], [x8]
+; CHECK-NEXT:    add x8, sp, #184
+; CHECK-NEXT:    mov v6.b[6], w6
+; CHECK-NEXT:    add x11, sp, #320
+; CHECK-NEXT:    ld1 { v1.b }[6], [x9]
+; CHECK-NEXT:    add x9, sp, #256
+; CHECK-NEXT:    ld1 { v3.b }[3], [x10]
+; CHECK-NEXT:    add x10, sp, #512
+; CHECK-NEXT:    ld1 { v16.b }[6], [x8]
+; CHECK-NEXT:    add x8, sp, #496
+; CHECK-NEXT:    mov v6.b[7], w7
+; CHECK-NEXT:    ld1 { v4.b }[7], [x11]
+; CHECK-NEXT:    ld1 { v1.b }[7], [x9]
+; CHECK-NEXT:    add x9, sp, #192
+; CHECK-NEXT:    ld1 { v3.b }[4], [x8]
+; CHECK-NEXT:    add x8, sp, #504
+; CHECK-NEXT:    sshll v5.8h, v5.8b, #0
+; CHECK-NEXT:    add x11, sp, #672
+; CHECK-NEXT:    ld1 { v16.b }[7], [x9]
+; CHECK-NEXT:    add x9, sp, #344
+; CHECK-NEXT:    sshll v6.8h, v6.8b, #0
+; CHECK-NEXT:    ld1 { v3.b }[5], [x8]
+; CHECK-NEXT:    add x8, sp, #352
+; CHECK-NEXT:    ld1 { v7.b }[1], [x9]
+; CHECK-NEXT:    add x9, sp, #544
+; CHECK-NEXT:    sshll v16.8h, v16.8b, #0
+; CHECK-NEXT:    sshll v17.8h, v17.8b, #0
+; CHECK-NEXT:    ld1 { v19.b }[1], [x9]
+; CHECK-NEXT:    add x9, sp, #360
+; CHECK-NEXT:    ld1 { v7.b }[2], [x8]
+; CHECK-NEXT:    add x8, sp, #552
+; CHECK-NEXT:    smull2 v18.4s, v6.8h, v16.8h
+; CHECK-NEXT:    ld1 { v3.b }[6], [x10]
+; CHECK-NEXT:    smull v6.4s, v6.4h, v16.4h
+; CHECK-NEXT:    ldr b16, [sp, #400]
+; CHECK-NEXT:    ld1 { v19.b }[2], [x8]
+; CHECK-NEXT:    add x8, sp, #560
+; CHECK-NEXT:    ld1 { v7.b }[3], [x9]
+; CHECK-NEXT:    add x9, sp, #368
+; CHECK-NEXT:    add x10, sp, #408
+; CHECK-NEXT:    sshll v2.8h, v2.8b, #0
+; CHECK-NEXT:    ld1 { v19.b }[3], [x8]
+; CHECK-NEXT:    add x8, sp, #568
+; CHECK-NEXT:    ld1 { v7.b }[4], [x9]
+; CHECK-NEXT:    add x9, sp, #376
+; CHECK-NEXT:    sshll v4.8h, v4.8b, #0
+; CHECK-NEXT:    ld1 { v16.b }[1], [x10]
+; CHECK-NEXT:    smull v5.4s, v5.4h, v17.4h
+; CHECK-NEXT:    ldr b17, [sp, #664]
+; CHECK-NEXT:    ld1 { v19.b }[4], [x8]
+; CHECK-NEXT:    add x8, sp, #576
+; CHECK-NEXT:    ld1 { v7.b }[5], [x9]
+; CHECK-NEXT:    add x9, sp, #384
+; CHECK-NEXT:    add x10, sp, #416
+; CHECK-NEXT:    ld1 { v17.b }[1], [x11]
+; CHECK-NEXT:    smlal v6.4s, v2.4h, v4.4h
+; CHECK-NEXT:    add x11, sp, #680
+; CHECK-NEXT:    ld1 { v19.b }[5], [x8]
+; CHECK-NEXT:    add x8, sp, #584
+; CHECK-NEXT:    ld1 { v7.b }[6], [x9]
+; CHECK-NEXT:    add x9, sp, #392
+; CHECK-NEXT:    smlal2 v18.4s, v2.8h, v4.8h
+; CHECK-NEXT:    ldr b2, [sp, #600]
+; CHECK-NEXT:    ld1 { v16.b }[2], [x10]
+; CHECK-NEXT:    add x10, sp, #424
+; CHECK-NEXT:    ld1 { v19.b }[6], [x8]
+; CHECK-NEXT:    add x8, sp, #592
+; CHECK-NEXT:    ld1 { v7.b }[7], [x9]
+; CHECK-NEXT:    add x9, sp, #608
+; CHECK-NEXT:    ld1 { v17.b }[2], [x11]
+; CHECK-NEXT:    add x11, sp, #520
+; CHECK-NEXT:    ld1 { v16.b }[3], [x10]
+; CHECK-NEXT:    add x10, sp, #688
+; CHECK-NEXT:    ld1 { v2.b }[1], [x9]
+; CHECK-NEXT:    add x9, sp, #616
+; CHECK-NEXT:    ld1 { v19.b }[7], [x8]
+; CHECK-NEXT:    add x8, sp, #432
+; CHECK-NEXT:    ld1 { v17.b }[3], [x10]
+; CHECK-NEXT:    add x10, sp, #696
+; CHECK-NEXT:    sshll v4.8h, v7.8b, #0
+; CHECK-NEXT:    ld1 { v3.b }[7], [x11]
+; CHECK-NEXT:    ld1 { v2.b }[2], [x9]
+; CHECK-NEXT:    add x9, sp, #624
+; CHECK-NEXT:    sshll v7.8h, v19.8b, #0
+; CHECK-NEXT:    ld1 { v16.b }[4], [x8]
+; CHECK-NEXT:    ld1 { v17.b }[4], [x10]
+; CHECK-NEXT:    add x10, sp, #704
+; CHECK-NEXT:    smull2 v19.4s, v4.8h, v7.8h
+; CHECK-NEXT:    add x8, sp, #440
+; CHECK-NEXT:    ld1 { v2.b }[3], [x9]
+; CHECK-NEXT:    add x9, sp, #632
+; CHECK-NEXT:    smull v4.4s, v4.4h, v7.4h
+; CHECK-NEXT:    ldr b7, [sp, #728]
+; CHECK-NEXT:    sshll v20.8h, v20.8b, #0
+; CHECK-NEXT:    ld1 { v17.b }[5], [x10]
+; CHECK-NEXT:    add x10, sp, #712
+; CHECK-NEXT:    ld1 { v16.b }[5], [x8]
+; CHECK-NEXT:    sshll v7.8h, v7.8b, #0
+; CHECK-NEXT:    ld1 { v2.b }[4], [x9]
+; CHECK-NEXT:    add x9, sp, #640
+; CHECK-NEXT:    add x8, sp, #448
+; CHECK-NEXT:    smull v7.4s, v20.4h, v7.4h
+; CHECK-NEXT:    ld1 { v17.b }[6], [x10]
+; CHECK-NEXT:    mov v5.s[1], wzr
+; CHECK-NEXT:    add x10, sp, #720
+; CHECK-NEXT:    ld1 { v2.b }[5], [x9]
+; CHECK-NEXT:    add x9, sp, #648
+; CHECK-NEXT:    ld1 { v16.b }[6], [x8]
+; CHECK-NEXT:    add x8, sp, #456
+; CHECK-NEXT:    mov v7.s[1], wzr
+; CHECK-NEXT:    ld1 { v17.b }[7], [x10]
+; CHECK-NEXT:    mov v5.s[2], wzr
+; CHECK-NEXT:    ld1 { v2.b }[6], [x9]
+; CHECK-NEXT:    add x9, sp, #656
+; CHECK-NEXT:    ld1 { v16.b }[7], [x8]
+; CHECK-NEXT:    mov v7.s[2], wzr
+; CHECK-NEXT:    sshll v3.8h, v3.8b, #0
+; CHECK-NEXT:    ld1 { v2.b }[7], [x9]
+; CHECK-NEXT:    sshll v17.8h, v17.8b, #0
+; CHECK-NEXT:    mov v5.s[3], wzr
+; CHECK-NEXT:    mov v7.s[3], wzr
+; CHECK-NEXT:    smlal v4.4s, v3.4h, v17.4h
+; CHECK-NEXT:    smlal2 v19.4s, v3.8h, v17.8h
+; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-NEXT:    sshll v3.8h, v16.8b, #0
+; CHECK-NEXT:    sshll v2.8h, v2.8b, #0
+; CHECK-NEXT:    smlal v5.4s, v0.4h, v1.4h
+; CHECK-NEXT:    smlal v7.4s, v3.4h, v2.4h
+; CHECK-NEXT:    smlal2 v18.4s, v0.8h, v1.8h
+; CHECK-NEXT:    smlal2 v19.4s, v3.8h, v2.8h
+; CHECK-NEXT:    add v0.4s, v6.4s, v5.4s
+; CHECK-NEXT:    add v1.4s, v4.4s, v7.4s
+; CHECK-NEXT:    add v0.4s, v0.4s, v18.4s
+; CHECK-NEXT:    add v1.4s, v1.4s, v19.4s
+; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    addv s0, v0.4s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
 entry:
-  %az = sext <16 x i8> %a to <16 x i32>
-  %bz = sext <16 x i8> %b to <16 x i32>
-  %m1 = mul nuw nsw <16 x i32> %az, %bz
-  %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m1)
-  %cz = sext <16 x i8> %c to <16 x i32>
-  %dz = sext <16 x i8> %d to <16 x i32>
-  %m2 = mul nuw nsw <16 x i32> %cz, %dz
-  %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m2)
+  %az = sext <25 x i8> %a to <25 x i32>
+  %bz = sext <25 x i8> %b to <25 x i32>
+  %m1 = mul nuw nsw <25 x i32> %az, %bz
+  %r1 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m1)
+  %cz = sext <25 x i8> %c to <25 x i32>
+  %dz = sext <25 x i8> %d to <25 x i32>
+  %m2 = mul nuw nsw <25 x i32> %cz, %dz
+  %r2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m2)
   %x = add i32 %r1, %r2
   ret i32 %x
 }
 
-define i32 @test_sdot_v16i8_double_nomla(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
-; CHECK-LABEL: test_sdot_v16i8_double_nomla:
+define i32 @test_sdot_v25i8_double_nomla(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) {
+; CHECK-LABEL: test_sdot_v25i8_double_nomla:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.16b, #1
-; CHECK-NEXT:    movi v3.2d, #0000000000000000
-; CHECK-NEXT:    sdot v3.4s, v2.16b, v1.16b
-; CHECK-NEXT:    sdot v3.4s, v0.16b, v1.16b
-; CHECK-NEXT:    addv s0, v3.4s
+; CHECK-NEXT:    ldr b0, [sp, #64]
+; CHECK-NEXT:    add x8, sp, #72
+; CHECK-NEXT:    ldr b2, [sp]
+; CHECK-NEXT:    add x9, sp, #8
+; CHECK-NEXT:    ldr b3, [sp, #464]
+; CHECK-NEXT:    add x10, sp, #472
+; CHECK-NEXT:    ld1 { v0.b }[1], [x8]
+; CHECK-NEXT:    add x8, sp, #80
+; CHECK-NEXT:    ld1 { v2.b }[1], [x9]
+; CHECK-NEXT:    add x9, sp, #16
+; CHECK-NEXT:    add x11, sp, #104
+; CHECK-NEXT:    ld1 { v3.b }[1], [x10]
+; CHECK-NEXT:    add x10, sp, #480
+; CHECK-NEXT:    add x12, sp, #32
+; CHECK-NEXT:    ld1 { v0.b }[2], [x8]
+; CHECK-NEXT:    add x8, sp, #88
+; CHECK-NEXT:    ld1 { v2.b }[2], [x9]
+; CHECK-NEXT:    add x9, sp, #24
+; CHECK-NEXT:    ldr b4, [sp, #128]
+; CHECK-NEXT:    fmov s1, w0
+; CHECK-NEXT:    ld1 { v3.b }[2], [x10]
+; CHECK-NEXT:    add x10, sp, #48
+; CHECK-NEXT:    ld1 { v0.b }[3], [x8]
+; CHECK-NEXT:    add x8, sp, #96
+; CHECK-NEXT:    ld1 { v2.b }[3], [x9]
+; CHECK-NEXT:    add x9, sp, #120
+; CHECK-NEXT:    sshll v4.8h, v4.8b, #0
+; CHECK-NEXT:    ldr b6, [sp, #400]
+; CHECK-NEXT:    mov v1.b[1], w1
+; CHECK-NEXT:    ldr b16, [sp, #528]
+; CHECK-NEXT:    ld1 { v0.b }[4], [x8]
+; CHECK-NEXT:    add x8, sp, #112
+; CHECK-NEXT:    ld1 { v2.b }[4], [x12]
+; CHECK-NEXT:    add x12, sp, #40
+; CHECK-NEXT:    sshll v5.4s, v4.4h, #0
+; CHECK-NEXT:    ldr b4, [sp, #336]
+; CHECK-NEXT:    mov v1.b[2], w2
+; CHECK-NEXT:    ld1 { v0.b }[5], [x11]
+; CHECK-NEXT:    add x11, sp, #488
+; CHECK-NEXT:    ld1 { v2.b }[5], [x12]
+; CHECK-NEXT:    sshll v16.8h, v16.8b, #0
+; CHECK-NEXT:    ld1 { v3.b }[3], [x11]
+; CHECK-NEXT:    add x11, sp, #344
+; CHECK-NEXT:    ld1 { v0.b }[6], [x8]
+; CHECK-NEXT:    add x8, sp, #496
+; CHECK-NEXT:    ld1 { v2.b }[6], [x10]
+; CHECK-NEXT:    add x10, sp, #352
+; CHECK-NEXT:    ld1 { v4.b }[1], [x11]
+; CHECK-NEXT:    add x11, sp, #416
+; CHECK-NEXT:    ld1 { v3.b }[4], [x8]
+; CHECK-NEXT:    add x8, sp, #504
+; CHECK-NEXT:    ld1 { v0.b }[7], [x9]
+; CHECK-NEXT:    add x9, sp, #408
+; CHECK-NEXT:    mov v1.b[3], w3
+; CHECK-NEXT:    ld1 { v4.b }[2], [x10]
+; CHECK-NEXT:    add x10, sp, #360
+; CHECK-NEXT:    ld1 { v6.b }[1], [x9]
+; CHECK-NEXT:    add x9, sp, #56
+; CHECK-NEXT:    ld1 { v3.b }[5], [x8]
+; CHECK-NEXT:    add x8, sp, #512
+; CHECK-NEXT:    mov v1.b[4], w4
+; CHECK-NEXT:    ld1 { v2.b }[7], [x9]
+; CHECK-NEXT:    add x9, sp, #424
+; CHECK-NEXT:    ld1 { v6.b }[2], [x11]
+; CHECK-NEXT:    ld1 { v4.b }[3], [x10]
+; CHECK-NEXT:    add x10, sp, #368
+; CHECK-NEXT:    ld1 { v3.b }[6], [x8]
+; CHECK-NEXT:    add x8, sp, #376
+; CHECK-NEXT:    mov v1.b[5], w5
+; CHECK-NEXT:    ld1 { v6.b }[3], [x9]
+; CHECK-NEXT:    add x9, sp, #432
+; CHECK-NEXT:    ld1 { v4.b }[4], [x10]
+; CHECK-NEXT:    add x10, sp, #440
+; CHECK-NEXT:    sshll v16.4s, v16.4h, #0
+; CHECK-NEXT:    mov v1.b[6], w6
+; CHECK-NEXT:    ld1 { v6.b }[4], [x9]
+; CHECK-NEXT:    add x9, sp, #520
+; CHECK-NEXT:    ld1 { v4.b }[5], [x8]
+; CHECK-NEXT:    add x8, sp, #384
+; CHECK-NEXT:    mov v5.s[1], wzr
+; CHECK-NEXT:    mov v16.s[1], wzr
+; CHECK-NEXT:    ld1 { v3.b }[7], [x9]
+; CHECK-NEXT:    ld1 { v6.b }[5], [x10]
+; CHECK-NEXT:    add x10, sp, #448
+; CHECK-NEXT:    ld1 { v4.b }[6], [x8]
+; CHECK-NEXT:    add x8, sp, #392
+; CHECK-NEXT:    mov v1.b[7], w7
+; CHECK-NEXT:    add x9, sp, #456
+; CHECK-NEXT:    mov v5.s[2], wzr
+; CHECK-NEXT:    ld1 { v6.b }[6], [x10]
+; CHECK-NEXT:    mov v16.s[2], wzr
+; CHECK-NEXT:    ld1 { v4.b }[7], [x8]
+; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-NEXT:    ld1 { v6.b }[7], [x9]
+; CHECK-NEXT:    mov v5.s[3], wzr
+; CHECK-NEXT:    mov v16.s[3], wzr
+; CHECK-NEXT:    saddl v7.4s, v1.4h, v0.4h
+; CHECK-NEXT:    saddl2 v0.4s, v1.8h, v0.8h
+; CHECK-NEXT:    sshll v1.8h, v3.8b, #0
+; CHECK-NEXT:    sshll v3.8h, v4.8b, #0
+; CHECK-NEXT:    sshll v2.8h, v2.8b, #0
+; CHECK-NEXT:    sshll v4.8h, v6.8b, #0
+; CHECK-NEXT:    saddl v6.4s, v3.4h, v1.4h
+; CHECK-NEXT:    saddl2 v1.4s, v3.8h, v1.8h
+; CHECK-NEXT:    saddw v5.4s, v5.4s, v2.4h
+; CHECK-NEXT:    saddw v3.4s, v16.4s, v4.4h
+; CHECK-NEXT:    saddw2 v0.4s, v0.4s, v2.8h
+; CHECK-NEXT:    saddw2 v1.4s, v1.4s, v4.8h
+; CHECK-NEXT:    add v5.4s, v7.4s, v5.4s
+; CHECK-NEXT:    add v2.4s, v6.4s, v3.4s
+; CHECK-NEXT:    add v0.4s, v5.4s, v0.4s
+; CHECK-NEXT:    add v1.4s, v2.4s, v1.4s
+; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    addv s0, v0.4s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
 entry:
-  %az = sext <16 x i8> %a to <16 x i32>
-  %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %az)
-  %cz = sext <16 x i8> %c to <16 x i32>
-  %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %cz)
+  %az = sext <25 x i8> %a to <25 x i32>
+  %r1 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %az)
+  %cz = sext <25 x i8> %c to <25 x i32>
+  %r2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %cz)
   %x = add i32 %r1, %r2
   ret i32 %x
 }
@@ -2231,4 +3385,4 @@
   %r2 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %cz)
   %x = add i32 %r1, %r2
   ret i32 %x
-}
\ No newline at end of file
+}