This is an archive of the discontinued LLVM Phabricator instance.

Differential D141922

[AArch64] Add additional tests for dotreduce to check for `v4i8` and `v24i8`
ClosedPublic

Authored by zjaffal on Jan 17 2023, 5:15 AM.

Details

Reviewers
fhahn
dmgreen
Commits
rG627400b11c28: [AArch64] Add additional tests for dotreduce to check for `v4i8` and `v24i8`

Diff Detail

Event Timeline

zjaffal created this revision.Jan 17 2023, 5:15 AM
Herald added a project: Restricted Project. · View Herald TranscriptJan 17 2023, 5:15 AM
Herald added subscribers: StephenFan, kristof.beyls. · View Herald Transcript
zjaffal requested review of this revision.Jan 17 2023, 5:15 AM
Herald added a project: Restricted Project. · View Herald TranscriptJan 17 2023, 5:15 AM
Herald added a subscriber: llvm-commits. · View Herald Transcript
zjaffal mentioned this in D141693: [AArch64] turn extended vecreduce bigger than v16i8 into udot/sdot.Jan 17 2023, 5:17 AM
Harbormaster completed remote builds in B208216: Diff 489777.Jan 17 2023, 6:54 AM
fhahn accepted this revision.Jan 31 2023, 1:51 AM

LGTM, thanks! If we don't already have tests for something like v25i8/v6i8 it would be good to add them as well before committing.

This revision is now accepted and ready to land.Jan 31 2023, 1:51 AM
This revision was landed with ongoing or failed builds.Jan 31 2023, 2:17 AM
Closed by commit rG627400b11c28: [AArch64] Add additional tests for dotreduce to check for `v4i8` and `v24i8` (authored by zjaffal). · Explain Why
This revision was automatically updated to reflect the committed changes.
zjaffal added a commit: rG627400b11c28: [AArch64] Add additional tests for dotreduce to check for `v4i8` and `v24i8`.

Revision Contents

PathSize
llvm/
test/
CodeGen/
AArch64/
1256 lines

Diff 493532

llvm/test/CodeGen/AArch64/neon-dotreduce.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple aarch64-none-linux-gnu -mattr=+dotprod < %s | FileCheck %s ; RUN: llc -mtriple aarch64-none-linux-gnu -mattr=+dotprod < %s | FileCheck %s
declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
declare i32 @llvm.vector.reduce.add.v5i32(<5 x i32>)
declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>) declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)
declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>) declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)
declare i32 @llvm.vector.reduce.add.v24i32(<24 x i32>)
declare i32 @llvm.vector.reduce.add.v25i32(<25 x i32>)
declare i32 @llvm.vector.reduce.add.v32i32(<32 x i32>) declare i32 @llvm.vector.reduce.add.v32i32(<32 x i32>)
declare i32 @llvm.vector.reduce.add.v33i32(<33 x i32>) declare i32 @llvm.vector.reduce.add.v33i32(<33 x i32>)
declare i32 @llvm.vector.reduce.add.v48i32(<48 x i32>) declare i32 @llvm.vector.reduce.add.v48i32(<48 x i32>)
declare i32 @llvm.vector.reduce.add.v64i32(<64 x i32>) declare i32 @llvm.vector.reduce.add.v64i32(<64 x i32>)
define i32 @test_udot_v4i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-LABEL: test_udot_v4i8:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldr s0, [x1]
; CHECK-NEXT: ldr s1, [x0]
; CHECK-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-NEXT: ushll v1.8h, v1.8b, #0
; CHECK-NEXT: umull v0.4s, v0.4h, v1.4h
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: add w0, w8, w2
; CHECK-NEXT: ret
entry:
%0 = load <4 x i8>, ptr %a
%1 = zext <4 x i8> %0 to <4 x i32>
%2 = load <4 x i8>, ptr %b
%3 = zext <4 x i8> %2 to <4 x i32>
%4 = mul nuw nsw <4 x i32> %3, %1
%5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %4)
%op.extra = add i32 %5, %sum
ret i32 %op.extra
}
define i32 @test_udot_v4i8_nomla(ptr nocapture readonly %a1) {
; CHECK-LABEL: test_udot_v4i8_nomla:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldr s0, [x0]
; CHECK-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w0, s0
; CHECK-NEXT: ret
entry:
%0 = load <4 x i8>, ptr %a1
%1 = zext <4 x i8> %0 to <4 x i32>
%2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %1)
ret i32 %2
}
define i32 @test_sdot_v4i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-LABEL: test_sdot_v4i8:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldr s0, [x1]
; CHECK-NEXT: ldr s1, [x0]
; CHECK-NEXT: sshll v0.8h, v0.8b, #0
; CHECK-NEXT: sshll v1.8h, v1.8b, #0
; CHECK-NEXT: smull v0.4s, v0.4h, v1.4h
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: add w0, w8, w2
; CHECK-NEXT: ret
entry:
%0 = load <4 x i8>, ptr %a
%1 = sext <4 x i8> %0 to <4 x i32>
%2 = load <4 x i8>, ptr %b
%3 = sext <4 x i8> %2 to <4 x i32>
%4 = mul nsw <4 x i32> %3, %1
%5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %4)
%op.extra = add nsw i32 %5, %sum
ret i32 %op.extra
}
define i32 @test_sdot_v4i8_double(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) {
; CHECK-LABEL: test_sdot_v4i8_double:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ushll v3.4s, v3.4h, #0
; CHECK-NEXT: ushll v2.4s, v2.4h, #0
; CHECK-NEXT: shl v3.4s, v3.4s, #24
; CHECK-NEXT: shl v2.4s, v2.4s, #24
; CHECK-NEXT: ushll v1.4s, v1.4h, #0
; CHECK-NEXT: sshr v3.4s, v3.4s, #24
; CHECK-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-NEXT: sshr v2.4s, v2.4s, #24
; CHECK-NEXT: shl v0.4s, v0.4s, #24
; CHECK-NEXT: shl v1.4s, v1.4s, #24
; CHECK-NEXT: mul v2.4s, v2.4s, v3.4s
; CHECK-NEXT: sshr v0.4s, v0.4s, #24
; CHECK-NEXT: sshr v1.4s, v1.4s, #24
; CHECK-NEXT: mla v2.4s, v0.4s, v1.4s
; CHECK-NEXT: addv s0, v2.4s
; CHECK-NEXT: fmov w0, s0
; CHECK-NEXT: ret
entry:
%az = sext <4 x i8> %a to <4 x i32>
%bz = sext <4 x i8> %b to <4 x i32>
%m1 = mul nuw nsw <4 x i32> %az, %bz
%r1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m1)
%cz = sext <4 x i8> %c to <4 x i32>
%dz = sext <4 x i8> %d to <4 x i32>
%m2 = mul nuw nsw <4 x i32> %cz, %dz
%r2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m2)
%x = add i32 %r1, %r2
ret i32 %x
}
define i32 @test_sdot_v4i8_double_nomla(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) {
; CHECK-LABEL: test_sdot_v4i8_double_nomla:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-NEXT: ushll v1.4s, v2.4h, #0
; CHECK-NEXT: shl v0.4s, v0.4s, #24
; CHECK-NEXT: shl v1.4s, v1.4s, #24
; CHECK-NEXT: sshr v0.4s, v0.4s, #24
; CHECK-NEXT: ssra v0.4s, v1.4s, #24
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w0, s0
; CHECK-NEXT: ret
entry:
%az = sext <4 x i8> %a to <4 x i32>
%r1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %az)
%cz = sext <4 x i8> %c to <4 x i32>
%r2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %cz)
%x = add i32 %r1, %r2
ret i32 %x
}
define i32 @test_udot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-LABEL: test_udot_v5i8:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldr d0, [x1]
; CHECK-NEXT: ldr d1, [x0]
; CHECK-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-NEXT: ushll v1.8h, v1.8b, #0
; CHECK-NEXT: umull2 v2.4s, v0.8h, v1.8h
; CHECK-NEXT: mov v2.s[1], wzr
; CHECK-NEXT: mov v2.s[2], wzr
; CHECK-NEXT: mov v2.s[3], wzr
; CHECK-NEXT: umlal v2.4s, v0.4h, v1.4h
; CHECK-NEXT: addv s0, v2.4s
; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: add w0, w8, w2
; CHECK-NEXT: ret
entry:
%0 = load <5 x i8>, ptr %a
%1 = zext <5 x i8> %0 to <5 x i32>
%2 = load <5 x i8>, ptr %b
%3 = zext <5 x i8> %2 to <5 x i32>
%4 = mul nuw nsw <5 x i32> %3, %1
%5 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %4)
%op.extra = add i32 %5, %sum
ret i32 %op.extra
}
define i32 @test_udot_v5i8_nomla(ptr nocapture readonly %a1) {
; CHECK-LABEL: test_udot_v5i8_nomla:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldr d0, [x0]
; CHECK-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-NEXT: ushll2 v1.4s, v0.8h, #0
; CHECK-NEXT: mov v1.s[1], wzr
; CHECK-NEXT: mov v1.s[2], wzr
; CHECK-NEXT: mov v1.s[3], wzr
; CHECK-NEXT: uaddw v0.4s, v1.4s, v0.4h
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w0, s0
; CHECK-NEXT: ret
entry:
%0 = load <5 x i8>, ptr %a1
%1 = zext <5 x i8> %0 to <5 x i32>
%2 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %1)
ret i32 %2
}
define i32 @test_sdot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-LABEL: test_sdot_v5i8:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldr d0, [x1]
; CHECK-NEXT: ldr d1, [x0]
; CHECK-NEXT: sshll v0.8h, v0.8b, #0
; CHECK-NEXT: sshll v1.8h, v1.8b, #0
; CHECK-NEXT: smull2 v2.4s, v0.8h, v1.8h
; CHECK-NEXT: mov v2.s[1], wzr
; CHECK-NEXT: mov v2.s[2], wzr
; CHECK-NEXT: mov v2.s[3], wzr
; CHECK-NEXT: smlal v2.4s, v0.4h, v1.4h
; CHECK-NEXT: addv s0, v2.4s
; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: add w0, w8, w2
; CHECK-NEXT: ret
entry:
%0 = load <5 x i8>, ptr %a
%1 = sext <5 x i8> %0 to <5 x i32>
%2 = load <5 x i8>, ptr %b
%3 = sext <5 x i8> %2 to <5 x i32>
%4 = mul nsw <5 x i32> %3, %1
%5 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %4)
%op.extra = add nsw i32 %5, %sum
ret i32 %op.extra
}
define i32 @test_sdot_v5i8_double(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8> %d) {
; CHECK-LABEL: test_sdot_v5i8_double:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: sshll v2.8h, v2.8b, #0
; CHECK-NEXT: sshll v0.8h, v0.8b, #0
; CHECK-NEXT: sshll v1.8h, v1.8b, #0
; CHECK-NEXT: sshll v3.8h, v3.8b, #0
; CHECK-NEXT: smull2 v4.4s, v0.8h, v1.8h
; CHECK-NEXT: smull2 v5.4s, v2.8h, v3.8h
; CHECK-NEXT: mov v4.s[1], wzr
; CHECK-NEXT: mov v5.s[1], wzr
; CHECK-NEXT: mov v4.s[2], wzr
; CHECK-NEXT: mov v5.s[2], wzr
; CHECK-NEXT: mov v4.s[3], wzr
; CHECK-NEXT: mov v5.s[3], wzr
; CHECK-NEXT: smlal v4.4s, v0.4h, v1.4h
; CHECK-NEXT: smlal v5.4s, v2.4h, v3.4h
; CHECK-NEXT: add v0.4s, v4.4s, v5.4s
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w0, s0
; CHECK-NEXT: ret
entry:
%az = sext <5 x i8> %a to <5 x i32>
%bz = sext <5 x i8> %b to <5 x i32>
%m1 = mul nuw nsw <5 x i32> %az, %bz
%r1 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %m1)
%cz = sext <5 x i8> %c to <5 x i32>
%dz = sext <5 x i8> %d to <5 x i32>
%m2 = mul nuw nsw <5 x i32> %cz, %dz
%r2 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %m2)
%x = add i32 %r1, %r2
ret i32 %x
}
define i32 @test_sdot_v5i8_double_nomla(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8> %d) {
; CHECK-LABEL: test_sdot_v5i8_double_nomla:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: sshll v0.8h, v0.8b, #0
; CHECK-NEXT: sshll v1.8h, v2.8b, #0
; CHECK-NEXT: sshll2 v2.4s, v0.8h, #0
; CHECK-NEXT: sshll2 v3.4s, v1.8h, #0
; CHECK-NEXT: mov v2.s[1], wzr
; CHECK-NEXT: mov v3.s[1], wzr
; CHECK-NEXT: mov v2.s[2], wzr
; CHECK-NEXT: mov v3.s[2], wzr
; CHECK-NEXT: mov v2.s[3], wzr
; CHECK-NEXT: mov v3.s[3], wzr
; CHECK-NEXT: saddw v0.4s, v2.4s, v0.4h
; CHECK-NEXT: saddw v1.4s, v3.4s, v1.4h
; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w0, s0
; CHECK-NEXT: ret
entry:
%az = sext <5 x i8> %a to <5 x i32>
%r1 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %az)
%cz = sext <5 x i8> %c to <5 x i32>
%r2 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %cz)
%x = add i32 %r1, %r2
ret i32 %x
}
define i32 @test_udot_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) { define i32 @test_udot_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) {
; CHECK-LABEL: test_udot_v8i8: ; CHECK-LABEL: test_udot_v8i8:
; CHECK: // %bb.0: // %entry ; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v0.2d, #0000000000000000 ; CHECK-NEXT: movi v0.2d, #0000000000000000
; CHECK-NEXT: ldr d1, [x0] ; CHECK-NEXT: ldr d1, [x0]
; CHECK-NEXT: ldr d2, [x1] ; CHECK-NEXT: ldr d2, [x1]
; CHECK-NEXT: udot v0.2s, v2.8b, v1.8b ; CHECK-NEXT: udot v0.2s, v2.8b, v1.8b
; CHECK-NEXT: addp v0.2s, v0.2s, v0.2s ; CHECK-NEXT: addp v0.2s, v0.2s, v0.2s
▲ Show 20 Lines • Show All 302 Lines • ▼ Show 20 Linesentry:
%az = sext <16 x i8> %a to <16 x i32> %az = sext <16 x i8> %a to <16 x i32>
%r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %az) %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %az)
%cz = sext <16 x i8> %c to <16 x i32> %cz = sext <16 x i8> %c to <16 x i32>
%r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %cz) %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %cz)
%x = add i32 %r1, %r2 %x = add i32 %r1, %r2
ret i32 %x ret i32 %x
} }
define i32 @test_udot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-LABEL: test_udot_v24i8:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldr q0, [x1]
; CHECK-NEXT: ldr q1, [x0]
; CHECK-NEXT: ldr d2, [x0, #16]
; CHECK-NEXT: ushll v5.8h, v0.8b, #0
; CHECK-NEXT: ldr d3, [x1, #16]
; CHECK-NEXT: ushll v4.8h, v1.8b, #0
; CHECK-NEXT: ushll v2.8h, v2.8b, #0
; CHECK-NEXT: umull2 v6.4s, v5.8h, v4.8h
; CHECK-NEXT: umull v4.4s, v5.4h, v4.4h
; CHECK-NEXT: ushll v3.8h, v3.8b, #0
; CHECK-NEXT: ushll2 v1.8h, v1.16b, #0
; CHECK-NEXT: umlal2 v6.4s, v3.8h, v2.8h
; CHECK-NEXT: umlal v4.4s, v3.4h, v2.4h
; CHECK-NEXT: ushll2 v0.8h, v0.16b, #0
; CHECK-NEXT: umlal2 v6.4s, v0.8h, v1.8h
; CHECK-NEXT: umlal v4.4s, v0.4h, v1.4h
; CHECK-NEXT: add v0.4s, v4.4s, v6.4s
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: add w0, w8, w2
; CHECK-NEXT: ret
entry:
%0 = load <24 x i8>, ptr %a
%1 = zext <24 x i8> %0 to <24 x i32>
%2 = load <24 x i8>, ptr %b
%3 = zext <24 x i8> %2 to <24 x i32>
%4 = mul nuw nsw <24 x i32> %3, %1
%5 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %4)
%op.extra = add i32 %5, %sum
ret i32 %op.extra
}
define i32 @test_udot_v24i8_nomla(ptr nocapture readonly %a1) {
; CHECK-LABEL: test_udot_v24i8_nomla:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldr d0, [x0, #16]
; CHECK-NEXT: ldr q1, [x0]
; CHECK-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-NEXT: ushll v2.8h, v1.8b, #0
; CHECK-NEXT: ushll2 v1.8h, v1.16b, #0
; CHECK-NEXT: uaddl2 v3.4s, v2.8h, v0.8h
; CHECK-NEXT: uaddl v0.4s, v2.4h, v0.4h
; CHECK-NEXT: uaddw2 v2.4s, v3.4s, v1.8h
; CHECK-NEXT: uaddw v0.4s, v0.4s, v1.4h
; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w0, s0
; CHECK-NEXT: ret
entry:
%0 = load <24 x i8>, ptr %a1
%1 = zext <24 x i8> %0 to <24 x i32>
%2 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %1)
ret i32 %2
}
define i32 @test_sdot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-LABEL: test_sdot_v24i8:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldr q0, [x1]
; CHECK-NEXT: ldr q1, [x0]
; CHECK-NEXT: ldr d2, [x0, #16]
; CHECK-NEXT: sshll v5.8h, v0.8b, #0
; CHECK-NEXT: ldr d3, [x1, #16]
; CHECK-NEXT: sshll v4.8h, v1.8b, #0
; CHECK-NEXT: sshll v2.8h, v2.8b, #0
; CHECK-NEXT: smull2 v6.4s, v5.8h, v4.8h
; CHECK-NEXT: smull v4.4s, v5.4h, v4.4h
; CHECK-NEXT: sshll v3.8h, v3.8b, #0
; CHECK-NEXT: sshll2 v1.8h, v1.16b, #0
; CHECK-NEXT: smlal2 v6.4s, v3.8h, v2.8h
; CHECK-NEXT: smlal v4.4s, v3.4h, v2.4h
; CHECK-NEXT: sshll2 v0.8h, v0.16b, #0
; CHECK-NEXT: smlal2 v6.4s, v0.8h, v1.8h
; CHECK-NEXT: smlal v4.4s, v0.4h, v1.4h
; CHECK-NEXT: add v0.4s, v4.4s, v6.4s
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: add w0, w8, w2
; CHECK-NEXT: ret
entry:
%0 = load <24 x i8>, ptr %a
%1 = sext <24 x i8> %0 to <24 x i32>
%2 = load <24 x i8>, ptr %b
%3 = sext <24 x i8> %2 to <24 x i32>
%4 = mul nsw <24 x i32> %3, %1
%5 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %4)
%op.extra = add nsw i32 %5, %sum
ret i32 %op.extra
}
define i32 @test_sdot_v24i8_double(<24 x i8> %a, <24 x i8> %b, <24 x i8> %c, <24 x i8> %d) {
; CHECK-LABEL: test_sdot_v24i8_double:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldr b0, [sp]
; CHECK-NEXT: add x8, sp, #8
; CHECK-NEXT: ldr b3, [sp, #64]
; CHECK-NEXT: add x9, sp, #16
; CHECK-NEXT: ldr b1, [sp, #192]
; CHECK-NEXT: add x10, sp, #200
; CHECK-NEXT: ld1 { v0.b }[1], [x8]
; CHECK-NEXT: add x8, sp, #72
; CHECK-NEXT: ldr b5, [sp, #256]
; CHECK-NEXT: fmov s6, w0
; CHECK-NEXT: ld1 { v1.b }[1], [x10]
; CHECK-NEXT: add x10, sp, #264
; CHECK-NEXT: ld1 { v3.b }[1], [x8]
; CHECK-NEXT: add x8, sp, #24
; CHECK-NEXT: ld1 { v0.b }[2], [x9]
; CHECK-NEXT: add x9, sp, #80
; CHECK-NEXT: ld1 { v5.b }[1], [x10]
; CHECK-NEXT: add x10, sp, #208
; CHECK-NEXT: ldr b16, [sp, #128]
; CHECK-NEXT: add x11, sp, #56
; CHECK-NEXT: ld1 { v3.b }[2], [x9]
; CHECK-NEXT: add x9, sp, #88
; CHECK-NEXT: ld1 { v0.b }[3], [x8]
; CHECK-NEXT: add x8, sp, #32
; CHECK-NEXT: ld1 { v1.b }[2], [x10]
; CHECK-NEXT: add x10, sp, #136
; CHECK-NEXT: mov v6.b[1], w1
; CHECK-NEXT: ldr b2, [sp, #384]
; CHECK-NEXT: ld1 { v3.b }[3], [x9]
; CHECK-NEXT: add x9, sp, #40
; CHECK-NEXT: ld1 { v0.b }[4], [x8]
; CHECK-NEXT: add x8, sp, #96
; CHECK-NEXT: ld1 { v16.b }[1], [x10]
; CHECK-NEXT: add x10, sp, #144
; CHECK-NEXT: mov v6.b[2], w2
; CHECK-NEXT: ldr b4, [sp, #448]
; CHECK-NEXT: ld1 { v3.b }[4], [x8]
; CHECK-NEXT: add x8, sp, #48
; CHECK-NEXT: ld1 { v0.b }[5], [x9]
; CHECK-NEXT: add x9, sp, #104
; CHECK-NEXT: ld1 { v16.b }[2], [x10]
; CHECK-NEXT: add x10, sp, #120
; CHECK-NEXT: mov v6.b[3], w3
; CHECK-NEXT: add x12, sp, #584
; CHECK-NEXT: ld1 { v3.b }[5], [x9]
; CHECK-NEXT: add x9, sp, #216
; CHECK-NEXT: ld1 { v0.b }[6], [x8]
; CHECK-NEXT: add x8, sp, #112
; CHECK-NEXT: ld1 { v1.b }[3], [x9]
; CHECK-NEXT: add x9, sp, #272
; CHECK-NEXT: ld1 { v3.b }[6], [x8]
; CHECK-NEXT: add x8, sp, #224
; CHECK-NEXT: ld1 { v0.b }[7], [x11]
; CHECK-NEXT: add x11, sp, #232
; CHECK-NEXT: ld1 { v5.b }[2], [x9]
; CHECK-NEXT: add x9, sp, #152
; CHECK-NEXT: ld1 { v1.b }[4], [x8]
; CHECK-NEXT: add x8, sp, #280
; CHECK-NEXT: ld1 { v3.b }[7], [x10]
; CHECK-NEXT: add x10, sp, #240
; CHECK-NEXT: ld1 { v16.b }[3], [x9]
; CHECK-NEXT: add x9, sp, #288
; CHECK-NEXT: ld1 { v5.b }[3], [x8]
; CHECK-NEXT: add x8, sp, #160
; CHECK-NEXT: ld1 { v1.b }[5], [x11]
; CHECK-NEXT: add x11, sp, #248
; CHECK-NEXT: mov v6.b[4], w4
; CHECK-NEXT: ld1 { v16.b }[4], [x8]
; CHECK-NEXT: add x8, sp, #168
; CHECK-NEXT: ld1 { v5.b }[4], [x9]
; CHECK-NEXT: add x9, sp, #296
; CHECK-NEXT: ld1 { v1.b }[6], [x10]
; CHECK-NEXT: add x10, sp, #392
; CHECK-NEXT: mov v6.b[5], w5
; CHECK-NEXT: ld1 { v16.b }[5], [x8]
; CHECK-NEXT: add x8, sp, #176
; CHECK-NEXT: ld1 { v2.b }[1], [x10]
; CHECK-NEXT: add x10, sp, #400
; CHECK-NEXT: ld1 { v5.b }[5], [x9]
; CHECK-NEXT: add x9, sp, #304
; CHECK-NEXT: mov v6.b[6], w6
; CHECK-NEXT: ld1 { v1.b }[7], [x11]
; CHECK-NEXT: ld1 { v16.b }[6], [x8]
; CHECK-NEXT: add x8, sp, #184
; CHECK-NEXT: ld1 { v2.b }[2], [x10]
; CHECK-NEXT: add x10, sp, #456
; CHECK-NEXT: ld1 { v5.b }[6], [x9]
; CHECK-NEXT: add x9, sp, #312
; CHECK-NEXT: mov v6.b[7], w7
; CHECK-NEXT: add x11, sp, #648
; CHECK-NEXT: ld1 { v16.b }[7], [x8]
; CHECK-NEXT: add x8, sp, #408
; CHECK-NEXT: ld1 { v4.b }[1], [x10]
; CHECK-NEXT: add x10, sp, #328
; CHECK-NEXT: ld1 { v5.b }[7], [x9]
; CHECK-NEXT: add x9, sp, #416
; CHECK-NEXT: ld1 { v2.b }[3], [x8]
; CHECK-NEXT: add x8, sp, #464
; CHECK-NEXT: sshll v6.8h, v6.8b, #0
; CHECK-NEXT: sshll v17.8h, v16.8b, #0
; CHECK-NEXT: ld1 { v4.b }[2], [x8]
; CHECK-NEXT: add x8, sp, #472
; CHECK-NEXT: ld1 { v2.b }[4], [x9]
; CHECK-NEXT: add x9, sp, #424
; CHECK-NEXT: sshll v7.8h, v3.8b, #0
; CHECK-NEXT: sshll v16.8h, v5.8b, #0
; CHECK-NEXT: smull2 v3.4s, v6.8h, v17.8h
; CHECK-NEXT: ld1 { v4.b }[3], [x8]
; CHECK-NEXT: smull v5.4s, v6.4h, v17.4h
; CHECK-NEXT: ldr b6, [sp, #320]
; CHECK-NEXT: ld1 { v2.b }[5], [x9]
; CHECK-NEXT: add x9, sp, #480
; CHECK-NEXT: add x8, sp, #432
; CHECK-NEXT: ldr b17, [sp, #512]
; CHECK-NEXT: ld1 { v6.b }[1], [x10]
; CHECK-NEXT: add x10, sp, #520
; CHECK-NEXT: ld1 { v4.b }[4], [x9]
; CHECK-NEXT: add x9, sp, #336
; CHECK-NEXT: ld1 { v2.b }[6], [x8]
; CHECK-NEXT: add x8, sp, #488
; CHECK-NEXT: ld1 { v17.b }[1], [x10]
; CHECK-NEXT: add x10, sp, #440
; CHECK-NEXT: ld1 { v6.b }[2], [x9]
; CHECK-NEXT: add x9, sp, #528
; CHECK-NEXT: ld1 { v4.b }[5], [x8]
; CHECK-NEXT: add x8, sp, #344
; CHECK-NEXT: smlal2 v3.4s, v7.8h, v16.8h
; CHECK-NEXT: ld1 { v2.b }[7], [x10]
; CHECK-NEXT: smlal v5.4s, v7.4h, v16.4h
; CHECK-NEXT: ldr b7, [sp, #640]
; CHECK-NEXT: ld1 { v17.b }[2], [x9]
; CHECK-NEXT: add x9, sp, #352
; CHECK-NEXT: ld1 { v6.b }[3], [x8]
; CHECK-NEXT: add x8, sp, #536
; CHECK-NEXT: ld1 { v7.b }[1], [x11]
; CHECK-NEXT: add x11, sp, #656
; CHECK-NEXT: ldr b16, [sp, #576]
; CHECK-NEXT: ld1 { v17.b }[3], [x8]
; CHECK-NEXT: add x8, sp, #544
; CHECK-NEXT: ld1 { v6.b }[4], [x9]
; CHECK-NEXT: add x9, sp, #360
; CHECK-NEXT: ld1 { v16.b }[1], [x12]
; CHECK-NEXT: add x12, sp, #592
; CHECK-NEXT: ld1 { v7.b }[2], [x11]
; CHECK-NEXT: add x11, sp, #664
; CHECK-NEXT: ld1 { v17.b }[4], [x8]
; CHECK-NEXT: add x8, sp, #552
; CHECK-NEXT: ld1 { v6.b }[5], [x9]
; CHECK-NEXT: add x9, sp, #368
; CHECK-NEXT: ld1 { v16.b }[2], [x12]
; CHECK-NEXT: add x12, sp, #600
; CHECK-NEXT: ld1 { v7.b }[3], [x11]
; CHECK-NEXT: add x11, sp, #672
; CHECK-NEXT: ld1 { v17.b }[5], [x8]
; CHECK-NEXT: add x8, sp, #560
; CHECK-NEXT: ld1 { v6.b }[6], [x9]
; CHECK-NEXT: add x9, sp, #376
; CHECK-NEXT: ld1 { v16.b }[3], [x12]
; CHECK-NEXT: add x12, sp, #608
; CHECK-NEXT: ld1 { v7.b }[4], [x11]
; CHECK-NEXT: add x11, sp, #680
; CHECK-NEXT: ld1 { v17.b }[6], [x8]
; CHECK-NEXT: add x8, sp, #568
; CHECK-NEXT: ld1 { v6.b }[7], [x9]
; CHECK-NEXT: add x9, sp, #496
; CHECK-NEXT: ld1 { v16.b }[4], [x12]
; CHECK-NEXT: add x12, sp, #616
; CHECK-NEXT: ld1 { v7.b }[5], [x11]
; CHECK-NEXT: add x11, sp, #688
; CHECK-NEXT: ld1 { v4.b }[6], [x9]
; CHECK-NEXT: add x9, sp, #696
; CHECK-NEXT: ld1 { v17.b }[7], [x8]
; CHECK-NEXT: add x8, sp, #504
; CHECK-NEXT: ld1 { v16.b }[5], [x12]
; CHECK-NEXT: ld1 { v7.b }[6], [x11]
; CHECK-NEXT: ld1 { v4.b }[7], [x8]
; CHECK-NEXT: add x8, sp, #624
; CHECK-NEXT: sshll v6.8h, v6.8b, #0
; CHECK-NEXT: sshll v17.8h, v17.8b, #0
; CHECK-NEXT: ld1 { v7.b }[7], [x9]
; CHECK-NEXT: ld1 { v16.b }[6], [x8]
; CHECK-NEXT: add x8, sp, #632
; CHECK-NEXT: smull v18.4s, v6.4h, v17.4h
; CHECK-NEXT: smull2 v6.4s, v6.8h, v17.8h
; CHECK-NEXT: sshll v4.8h, v4.8b, #0
; CHECK-NEXT: sshll v7.8h, v7.8b, #0
; CHECK-NEXT: ld1 { v16.b }[7], [x8]
; CHECK-NEXT: sshll v0.8h, v0.8b, #0
; CHECK-NEXT: smlal2 v6.4s, v4.8h, v7.8h
; CHECK-NEXT: smlal v18.4s, v4.4h, v7.4h
; CHECK-NEXT: sshll v1.8h, v1.8b, #0
; CHECK-NEXT: sshll v2.8h, v2.8b, #0
; CHECK-NEXT: sshll v4.8h, v16.8b, #0
; CHECK-NEXT: smlal2 v3.4s, v0.8h, v1.8h
; CHECK-NEXT: smlal v5.4s, v0.4h, v1.4h
; CHECK-NEXT: smlal2 v6.4s, v2.8h, v4.8h
; CHECK-NEXT: smlal v18.4s, v2.4h, v4.4h
; CHECK-NEXT: add v0.4s, v5.4s, v3.4s
; CHECK-NEXT: add v1.4s, v18.4s, v6.4s
; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w0, s0
; CHECK-NEXT: ret
entry:
%az = sext <24 x i8> %a to <24 x i32>
%bz = sext <24 x i8> %b to <24 x i32>
%m1 = mul nuw nsw <24 x i32> %az, %bz
%r1 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %m1)
%cz = sext <24 x i8> %c to <24 x i32>
%dz = sext <24 x i8> %d to <24 x i32>
%m2 = mul nuw nsw <24 x i32> %cz, %dz
%r2 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %m2)
%x = add i32 %r1, %r2
ret i32 %x
}
define i32 @test_sdot_v24i8_double_nomla(<24 x i8> %a, <24 x i8> %b, <24 x i8> %c, <24 x i8> %d) {
; CHECK-LABEL: test_sdot_v24i8_double_nomla:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldr b0, [sp]
; CHECK-NEXT: add x8, sp, #8
; CHECK-NEXT: add x9, sp, #16
; CHECK-NEXT: ldr b1, [sp, #64]
; CHECK-NEXT: ldr b2, [sp, #384]
; CHECK-NEXT: add x10, sp, #24
; CHECK-NEXT: ld1 { v0.b }[1], [x8]
; CHECK-NEXT: add x8, sp, #72
; CHECK-NEXT: ldr b3, [sp, #448]
; CHECK-NEXT: fmov s4, w0
; CHECK-NEXT: ldr b5, [sp, #320]
; CHECK-NEXT: add x12, sp, #472
; CHECK-NEXT: ld1 { v1.b }[1], [x8]
; CHECK-NEXT: add x8, sp, #456
; CHECK-NEXT: ld1 { v0.b }[2], [x9]
; CHECK-NEXT: add x9, sp, #392
; CHECK-NEXT: mov v4.b[1], w1
; CHECK-NEXT: add x11, sp, #56
; CHECK-NEXT: ld1 { v3.b }[1], [x8]
; CHECK-NEXT: add x8, sp, #32
; CHECK-NEXT: ld1 { v2.b }[1], [x9]
; CHECK-NEXT: add x9, sp, #80
; CHECK-NEXT: ld1 { v0.b }[3], [x10]
; CHECK-NEXT: add x10, sp, #40
; CHECK-NEXT: mov v4.b[2], w2
; CHECK-NEXT: ld1 { v1.b }[2], [x9]
; CHECK-NEXT: add x9, sp, #328
; CHECK-NEXT: ld1 { v0.b }[4], [x8]
; CHECK-NEXT: add x8, sp, #88
; CHECK-NEXT: ld1 { v5.b }[1], [x9]
; CHECK-NEXT: add x9, sp, #48
; CHECK-NEXT: mov v4.b[3], w3
; CHECK-NEXT: ld1 { v1.b }[3], [x8]
; CHECK-NEXT: add x8, sp, #464
; CHECK-NEXT: ld1 { v0.b }[5], [x10]
; CHECK-NEXT: add x10, sp, #96
; CHECK-NEXT: ld1 { v3.b }[2], [x8]
; CHECK-NEXT: add x8, sp, #104
; CHECK-NEXT: ld1 { v1.b }[4], [x10]
; CHECK-NEXT: add x10, sp, #336
; CHECK-NEXT: ld1 { v0.b }[6], [x9]
; CHECK-NEXT: add x9, sp, #112
; CHECK-NEXT: mov v4.b[4], w4
; CHECK-NEXT: ld1 { v5.b }[2], [x10]
; CHECK-NEXT: add x10, sp, #344
; CHECK-NEXT: ld1 { v1.b }[5], [x8]
; CHECK-NEXT: add x8, sp, #400
; CHECK-NEXT: ld1 { v3.b }[3], [x12]
; CHECK-NEXT: ld1 { v0.b }[7], [x11]
; CHECK-NEXT: add x11, sp, #480
; CHECK-NEXT: ld1 { v2.b }[2], [x8]
; CHECK-NEXT: add x8, sp, #352
; CHECK-NEXT: ld1 { v5.b }[3], [x10]
; CHECK-NEXT: add x10, sp, #488
; CHECK-NEXT: ld1 { v1.b }[6], [x9]
; CHECK-NEXT: add x9, sp, #408
; CHECK-NEXT: ld1 { v3.b }[4], [x11]
; CHECK-NEXT: add x11, sp, #416
; CHECK-NEXT: mov v4.b[5], w5
; CHECK-NEXT: ld1 { v2.b }[3], [x9]
; CHECK-NEXT: add x9, sp, #120
; CHECK-NEXT: ld1 { v5.b }[4], [x8]
; CHECK-NEXT: add x8, sp, #360
; CHECK-NEXT: ld1 { v3.b }[5], [x10]
; CHECK-NEXT: add x10, sp, #496
; CHECK-NEXT: ld1 { v1.b }[7], [x9]
; CHECK-NEXT: add x9, sp, #424
; CHECK-NEXT: ld1 { v2.b }[4], [x11]
; CHECK-NEXT: ld1 { v5.b }[5], [x8]
; CHECK-NEXT: add x8, sp, #368
; CHECK-NEXT: mov v4.b[6], w6
; CHECK-NEXT: ld1 { v3.b }[6], [x10]
; CHECK-NEXT: add x10, sp, #504
; CHECK-NEXT: ld1 { v2.b }[5], [x9]
; CHECK-NEXT: add x9, sp, #432
; CHECK-NEXT: ld1 { v5.b }[6], [x8]
; CHECK-NEXT: add x8, sp, #376
; CHECK-NEXT: mov v4.b[7], w7
; CHECK-NEXT: ld1 { v3.b }[7], [x10]
; CHECK-NEXT: sshll v1.8h, v1.8b, #0
; CHECK-NEXT: ld1 { v2.b }[6], [x9]
; CHECK-NEXT: ld1 { v5.b }[7], [x8]
; CHECK-NEXT: add x8, sp, #440
; CHECK-NEXT: sshll v4.8h, v4.8b, #0
; CHECK-NEXT: sshll v3.8h, v3.8b, #0
; CHECK-NEXT: ld1 { v2.b }[7], [x8]
; CHECK-NEXT: sshll v5.8h, v5.8b, #0
; CHECK-NEXT: saddl2 v6.4s, v4.8h, v1.8h
; CHECK-NEXT: saddl v1.4s, v4.4h, v1.4h
; CHECK-NEXT: saddl2 v4.4s, v5.8h, v3.8h
; CHECK-NEXT: saddl v3.4s, v5.4h, v3.4h
; CHECK-NEXT: sshll v0.8h, v0.8b, #0
; CHECK-NEXT: sshll v2.8h, v2.8b, #0
; CHECK-NEXT: saddw2 v5.4s, v6.4s, v0.8h
; CHECK-NEXT: saddw v0.4s, v1.4s, v0.4h
; CHECK-NEXT: saddw2 v1.4s, v4.4s, v2.8h
; CHECK-NEXT: saddw v2.4s, v3.4s, v2.4h
; CHECK-NEXT: add v0.4s, v0.4s, v5.4s
; CHECK-NEXT: add v1.4s, v2.4s, v1.4s
; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w0, s0
; CHECK-NEXT: ret
entry:
%az = sext <24 x i8> %a to <24 x i32>
%r1 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %az)
%cz = sext <24 x i8> %c to <24 x i32>
%r2 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %cz)
%x = add i32 %r1, %r2
ret i32 %x
}
define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-LABEL: test_udot_v25i8:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldp q3, q0, [x1]
; CHECK-NEXT: ushll v6.8h, v3.8b, #0
; CHECK-NEXT: ushll2 v3.8h, v3.16b, #0
; CHECK-NEXT: ldp q2, q1, [x0]
; CHECK-NEXT: ushll2 v5.8h, v0.16b, #0
; CHECK-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-NEXT: ushll2 v4.8h, v1.16b, #0
; CHECK-NEXT: ushll v1.8h, v1.8b, #0
; CHECK-NEXT: umull v4.4s, v5.4h, v4.4h
; CHECK-NEXT: ushll2 v5.8h, v2.16b, #0
; CHECK-NEXT: ushll v2.8h, v2.8b, #0
; CHECK-NEXT: mov v4.s[1], wzr
; CHECK-NEXT: umull2 v7.4s, v6.8h, v2.8h
; CHECK-NEXT: umull v2.4s, v6.4h, v2.4h
; CHECK-NEXT: mov v4.s[2], wzr
; CHECK-NEXT: umlal2 v7.4s, v0.8h, v1.8h
; CHECK-NEXT: umlal v2.4s, v0.4h, v1.4h
; CHECK-NEXT: mov v4.s[3], wzr
; CHECK-NEXT: umlal2 v7.4s, v3.8h, v5.8h
; CHECK-NEXT: umlal v4.4s, v3.4h, v5.4h
; CHECK-NEXT: add v0.4s, v2.4s, v7.4s
; CHECK-NEXT: add v0.4s, v0.4s, v4.4s
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: add w0, w8, w2
; CHECK-NEXT: ret
entry:
%0 = load <25 x i8>, ptr %a
%1 = zext <25 x i8> %0 to <25 x i32>
%2 = load <25 x i8>, ptr %b
%3 = zext <25 x i8> %2 to <25 x i32>
%4 = mul nuw nsw <25 x i32> %3, %1
%5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4)
%op.extra = add i32 %5, %sum
ret i32 %op.extra
}
define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) {
; CHECK-LABEL: test_udot_v25i8_nomla:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldp q1, q0, [x0]
; CHECK-NEXT: ushll v3.8h, v1.8b, #0
; CHECK-NEXT: ushll2 v1.8h, v1.16b, #0
; CHECK-NEXT: ushll2 v2.8h, v0.16b, #0
; CHECK-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-NEXT: ushll v2.4s, v2.4h, #0
; CHECK-NEXT: uaddl2 v4.4s, v3.8h, v0.8h
; CHECK-NEXT: mov v2.s[1], wzr
; CHECK-NEXT: uaddl v0.4s, v3.4h, v0.4h
; CHECK-NEXT: uaddw2 v3.4s, v4.4s, v1.8h
; CHECK-NEXT: mov v2.s[2], wzr
; CHECK-NEXT: add v0.4s, v0.4s, v3.4s
; CHECK-NEXT: mov v2.s[3], wzr
; CHECK-NEXT: uaddw v1.4s, v2.4s, v1.4h
; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w0, s0
; CHECK-NEXT: ret
entry:
%0 = load <25 x i8>, ptr %a1
%1 = zext <25 x i8> %0 to <25 x i32>
%2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %1)
ret i32 %2
}
define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-LABEL: test_sdot_v25i8:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldp q3, q0, [x1]
; CHECK-NEXT: sshll v6.8h, v3.8b, #0
; CHECK-NEXT: sshll2 v3.8h, v3.16b, #0
; CHECK-NEXT: ldp q2, q1, [x0]
; CHECK-NEXT: sshll2 v5.8h, v0.16b, #0
; CHECK-NEXT: sshll v0.8h, v0.8b, #0
; CHECK-NEXT: sshll2 v4.8h, v1.16b, #0
; CHECK-NEXT: sshll v1.8h, v1.8b, #0
; CHECK-NEXT: smull v4.4s, v5.4h, v4.4h
; CHECK-NEXT: sshll2 v5.8h, v2.16b, #0
; CHECK-NEXT: sshll v2.8h, v2.8b, #0
; CHECK-NEXT: mov v4.s[1], wzr
; CHECK-NEXT: smull2 v7.4s, v6.8h, v2.8h
; CHECK-NEXT: smull v2.4s, v6.4h, v2.4h
; CHECK-NEXT: mov v4.s[2], wzr
; CHECK-NEXT: smlal2 v7.4s, v0.8h, v1.8h
; CHECK-NEXT: smlal v2.4s, v0.4h, v1.4h
; CHECK-NEXT: mov v4.s[3], wzr
; CHECK-NEXT: smlal2 v7.4s, v3.8h, v5.8h
; CHECK-NEXT: smlal v4.4s, v3.4h, v5.4h
; CHECK-NEXT: add v0.4s, v2.4s, v7.4s
; CHECK-NEXT: add v0.4s, v0.4s, v4.4s
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: add w0, w8, w2
; CHECK-NEXT: ret
entry:
%0 = load <25 x i8>, ptr %a
%1 = sext <25 x i8> %0 to <25 x i32>
%2 = load <25 x i8>, ptr %b
%3 = sext <25 x i8> %2 to <25 x i32>
%4 = mul nsw <25 x i32> %3, %1
%5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4)
%op.extra = add nsw i32 %5, %sum
ret i32 %op.extra
}
define i32 @test_sdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) {
; CHECK-LABEL: test_sdot_v25i8_double:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldr b2, [sp, #64]
; CHECK-NEXT: add x8, sp, #72
; CHECK-NEXT: ldr b0, [sp]
; CHECK-NEXT: add x9, sp, #80
; CHECK-NEXT: ldr b4, [sp, #264]
; CHECK-NEXT: add x10, sp, #272
; CHECK-NEXT: ld1 { v2.b }[1], [x8]
; CHECK-NEXT: add x8, sp, #8
; CHECK-NEXT: add x11, sp, #288
; CHECK-NEXT: add x12, sp, #40
; CHECK-NEXT: ld1 { v4.b }[1], [x10]
; CHECK-NEXT: add x10, sp, #280
; CHECK-NEXT: ld1 { v0.b }[1], [x8]
; CHECK-NEXT: add x8, sp, #88
; CHECK-NEXT: ld1 { v2.b }[2], [x9]
; CHECK-NEXT: add x9, sp, #16
; CHECK-NEXT: ldr b1, [sp, #200]
; CHECK-NEXT: fmov s6, w0
; CHECK-NEXT: ld1 { v4.b }[2], [x10]
; CHECK-NEXT: add x10, sp, #112
; CHECK-NEXT: ld1 { v0.b }[2], [x9]
; CHECK-NEXT: add x9, sp, #96
; CHECK-NEXT: ld1 { v2.b }[3], [x8]
; CHECK-NEXT: add x8, sp, #24
; CHECK-NEXT: ldr b16, [sp, #136]
; CHECK-NEXT: ld1 { v4.b }[3], [x11]
; CHECK-NEXT: add x11, sp, #208
; CHECK-NEXT: ld1 { v0.b }[3], [x8]
; CHECK-NEXT: add x8, sp, #104
; CHECK-NEXT: ld1 { v2.b }[4], [x9]
; CHECK-NEXT: add x9, sp, #32
; CHECK-NEXT: ld1 { v1.b }[1], [x11]
; CHECK-NEXT: add x11, sp, #312
; CHECK-NEXT: ldr b3, [sp, #464]
; CHECK-NEXT: ld1 { v0.b }[4], [x9]
; CHECK-NEXT: add x9, sp, #48
; CHECK-NEXT: ld1 { v2.b }[5], [x8]
; CHECK-NEXT: add x8, sp, #120
; CHECK-NEXT: mov v6.b[1], w1
; CHECK-NEXT: ldr b7, [sp, #336]
; CHECK-NEXT: ldr b19, [sp, #536]
; CHECK-NEXT: ld1 { v0.b }[5], [x12]
; CHECK-NEXT: add x12, sp, #56
; CHECK-NEXT: ld1 { v2.b }[6], [x10]
; CHECK-NEXT: add x10, sp, #144
; CHECK-NEXT: mov v6.b[2], w2
; CHECK-NEXT: ldr b5, [sp, #128]
; CHECK-NEXT: ldr b17, [sp, #328]
; CHECK-NEXT: ld1 { v0.b }[6], [x9]
; CHECK-NEXT: add x9, sp, #216
; CHECK-NEXT: ld1 { v16.b }[1], [x10]
; CHECK-NEXT: add x10, sp, #296
; CHECK-NEXT: ld1 { v2.b }[7], [x8]
; CHECK-NEXT: add x8, sp, #152
; CHECK-NEXT: ld1 { v1.b }[2], [x9]
; CHECK-NEXT: add x9, sp, #224
; CHECK-NEXT: ld1 { v4.b }[4], [x10]
; CHECK-NEXT: add x10, sp, #304
; CHECK-NEXT: ld1 { v16.b }[2], [x8]
; CHECK-NEXT: add x8, sp, #160
; CHECK-NEXT: mov v6.b[3], w3
; CHECK-NEXT: ldr b20, [sp, #528]
; CHECK-NEXT: ld1 { v1.b }[3], [x9]
; CHECK-NEXT: add x9, sp, #232
; CHECK-NEXT: ld1 { v4.b }[5], [x10]
; CHECK-NEXT: add x10, sp, #472
; CHECK-NEXT: ld1 { v16.b }[3], [x8]
; CHECK-NEXT: add x8, sp, #168
; CHECK-NEXT: mov v6.b[4], w4
; CHECK-NEXT: ld1 { v0.b }[7], [x12]
; CHECK-NEXT: ld1 { v1.b }[4], [x9]
; CHECK-NEXT: add x9, sp, #240
; CHECK-NEXT: ld1 { v3.b }[1], [x10]
; CHECK-NEXT: add x10, sp, #480
; CHECK-NEXT: ld1 { v16.b }[4], [x8]
; CHECK-NEXT: add x8, sp, #176
; CHECK-NEXT: mov v6.b[5], w5
; CHECK-NEXT: ld1 { v4.b }[6], [x11]
; CHECK-NEXT: ld1 { v1.b }[5], [x9]
; CHECK-NEXT: add x9, sp, #248
; CHECK-NEXT: ld1 { v3.b }[2], [x10]
; CHECK-NEXT: add x10, sp, #488
; CHECK-NEXT: ld1 { v16.b }[5], [x8]
; CHECK-NEXT: add x8, sp, #184
; CHECK-NEXT: mov v6.b[6], w6
; CHECK-NEXT: add x11, sp, #320
; CHECK-NEXT: ld1 { v1.b }[6], [x9]
; CHECK-NEXT: add x9, sp, #256
; CHECK-NEXT: ld1 { v3.b }[3], [x10]
; CHECK-NEXT: add x10, sp, #512
; CHECK-NEXT: ld1 { v16.b }[6], [x8]
; CHECK-NEXT: add x8, sp, #496
; CHECK-NEXT: mov v6.b[7], w7
; CHECK-NEXT: ld1 { v4.b }[7], [x11]
; CHECK-NEXT: ld1 { v1.b }[7], [x9]
; CHECK-NEXT: add x9, sp, #192
; CHECK-NEXT: ld1 { v3.b }[4], [x8]
; CHECK-NEXT: add x8, sp, #504
; CHECK-NEXT: sshll v5.8h, v5.8b, #0
; CHECK-NEXT: add x11, sp, #672
; CHECK-NEXT: ld1 { v16.b }[7], [x9]
; CHECK-NEXT: add x9, sp, #344
; CHECK-NEXT: sshll v6.8h, v6.8b, #0
; CHECK-NEXT: ld1 { v3.b }[5], [x8]
; CHECK-NEXT: add x8, sp, #352
; CHECK-NEXT: ld1 { v7.b }[1], [x9]
; CHECK-NEXT: add x9, sp, #544
; CHECK-NEXT: sshll v16.8h, v16.8b, #0
; CHECK-NEXT: sshll v17.8h, v17.8b, #0
; CHECK-NEXT: ld1 { v19.b }[1], [x9]
; CHECK-NEXT: add x9, sp, #360
; CHECK-NEXT: ld1 { v7.b }[2], [x8]
; CHECK-NEXT: add x8, sp, #552
; CHECK-NEXT: smull2 v18.4s, v6.8h, v16.8h
; CHECK-NEXT: ld1 { v3.b }[6], [x10]
; CHECK-NEXT: smull v6.4s, v6.4h, v16.4h
; CHECK-NEXT: ldr b16, [sp, #400]
; CHECK-NEXT: ld1 { v19.b }[2], [x8]
; CHECK-NEXT: add x8, sp, #560
; CHECK-NEXT: ld1 { v7.b }[3], [x9]
; CHECK-NEXT: add x9, sp, #368
; CHECK-NEXT: add x10, sp, #408
; CHECK-NEXT: sshll v2.8h, v2.8b, #0
; CHECK-NEXT: ld1 { v19.b }[3], [x8]
; CHECK-NEXT: add x8, sp, #568
; CHECK-NEXT: ld1 { v7.b }[4], [x9]
; CHECK-NEXT: add x9, sp, #376
; CHECK-NEXT: sshll v4.8h, v4.8b, #0
; CHECK-NEXT: ld1 { v16.b }[1], [x10]
; CHECK-NEXT: smull v5.4s, v5.4h, v17.4h
; CHECK-NEXT: ldr b17, [sp, #664]
; CHECK-NEXT: ld1 { v19.b }[4], [x8]
; CHECK-NEXT: add x8, sp, #576
; CHECK-NEXT: ld1 { v7.b }[5], [x9]
; CHECK-NEXT: add x9, sp, #384
; CHECK-NEXT: add x10, sp, #416
; CHECK-NEXT: ld1 { v17.b }[1], [x11]
; CHECK-NEXT: smlal v6.4s, v2.4h, v4.4h
; CHECK-NEXT: add x11, sp, #680
; CHECK-NEXT: ld1 { v19.b }[5], [x8]
; CHECK-NEXT: add x8, sp, #584
; CHECK-NEXT: ld1 { v7.b }[6], [x9]
; CHECK-NEXT: add x9, sp, #392
; CHECK-NEXT: smlal2 v18.4s, v2.8h, v4.8h
; CHECK-NEXT: ldr b2, [sp, #600]
; CHECK-NEXT: ld1 { v16.b }[2], [x10]
; CHECK-NEXT: add x10, sp, #424
; CHECK-NEXT: ld1 { v19.b }[6], [x8]
; CHECK-NEXT: add x8, sp, #592
; CHECK-NEXT: ld1 { v7.b }[7], [x9]
; CHECK-NEXT: add x9, sp, #608
; CHECK-NEXT: ld1 { v17.b }[2], [x11]
; CHECK-NEXT: add x11, sp, #520
; CHECK-NEXT: ld1 { v16.b }[3], [x10]
; CHECK-NEXT: add x10, sp, #688
; CHECK-NEXT: ld1 { v2.b }[1], [x9]
; CHECK-NEXT: add x9, sp, #616
; CHECK-NEXT: ld1 { v19.b }[7], [x8]
; CHECK-NEXT: add x8, sp, #432
; CHECK-NEXT: ld1 { v17.b }[3], [x10]
; CHECK-NEXT: add x10, sp, #696
; CHECK-NEXT: sshll v4.8h, v7.8b, #0
; CHECK-NEXT: ld1 { v3.b }[7], [x11]
; CHECK-NEXT: ld1 { v2.b }[2], [x9]
; CHECK-NEXT: add x9, sp, #624
; CHECK-NEXT: sshll v7.8h, v19.8b, #0
; CHECK-NEXT: ld1 { v16.b }[4], [x8]
; CHECK-NEXT: ld1 { v17.b }[4], [x10]
; CHECK-NEXT: add x10, sp, #704
; CHECK-NEXT: smull2 v19.4s, v4.8h, v7.8h
; CHECK-NEXT: add x8, sp, #440
; CHECK-NEXT: ld1 { v2.b }[3], [x9]
; CHECK-NEXT: add x9, sp, #632
; CHECK-NEXT: smull v4.4s, v4.4h, v7.4h
; CHECK-NEXT: ldr b7, [sp, #728]
; CHECK-NEXT: sshll v20.8h, v20.8b, #0
; CHECK-NEXT: ld1 { v17.b }[5], [x10]
; CHECK-NEXT: add x10, sp, #712
; CHECK-NEXT: ld1 { v16.b }[5], [x8]
; CHECK-NEXT: sshll v7.8h, v7.8b, #0
; CHECK-NEXT: ld1 { v2.b }[4], [x9]
; CHECK-NEXT: add x9, sp, #640
; CHECK-NEXT: add x8, sp, #448
; CHECK-NEXT: smull v7.4s, v20.4h, v7.4h
; CHECK-NEXT: ld1 { v17.b }[6], [x10]
; CHECK-NEXT: mov v5.s[1], wzr
; CHECK-NEXT: add x10, sp, #720
; CHECK-NEXT: ld1 { v2.b }[5], [x9]
; CHECK-NEXT: add x9, sp, #648
; CHECK-NEXT: ld1 { v16.b }[6], [x8]
; CHECK-NEXT: add x8, sp, #456
; CHECK-NEXT: mov v7.s[1], wzr
; CHECK-NEXT: ld1 { v17.b }[7], [x10]
; CHECK-NEXT: mov v5.s[2], wzr
; CHECK-NEXT: ld1 { v2.b }[6], [x9]
; CHECK-NEXT: add x9, sp, #656
; CHECK-NEXT: ld1 { v16.b }[7], [x8]
; CHECK-NEXT: mov v7.s[2], wzr
; CHECK-NEXT: sshll v3.8h, v3.8b, #0
; CHECK-NEXT: ld1 { v2.b }[7], [x9]
; CHECK-NEXT: sshll v17.8h, v17.8b, #0
; CHECK-NEXT: mov v5.s[3], wzr
; CHECK-NEXT: mov v7.s[3], wzr
; CHECK-NEXT: smlal v4.4s, v3.4h, v17.4h
; CHECK-NEXT: smlal2 v19.4s, v3.8h, v17.8h
; CHECK-NEXT: sshll v0.8h, v0.8b, #0
; CHECK-NEXT: sshll v1.8h, v1.8b, #0
; CHECK-NEXT: sshll v3.8h, v16.8b, #0
; CHECK-NEXT: sshll v2.8h, v2.8b, #0
; CHECK-NEXT: smlal v5.4s, v0.4h, v1.4h
; CHECK-NEXT: smlal v7.4s, v3.4h, v2.4h
; CHECK-NEXT: smlal2 v18.4s, v0.8h, v1.8h
; CHECK-NEXT: smlal2 v19.4s, v3.8h, v2.8h
; CHECK-NEXT: add v0.4s, v6.4s, v5.4s
; CHECK-NEXT: add v1.4s, v4.4s, v7.4s
; CHECK-NEXT: add v0.4s, v0.4s, v18.4s
; CHECK-NEXT: add v1.4s, v1.4s, v19.4s
; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w0, s0
; CHECK-NEXT: ret
entry:
%az = sext <25 x i8> %a to <25 x i32>
%bz = sext <25 x i8> %b to <25 x i32>
%m1 = mul nuw nsw <25 x i32> %az, %bz
%r1 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m1)
%cz = sext <25 x i8> %c to <25 x i32>
%dz = sext <25 x i8> %d to <25 x i32>
%m2 = mul nuw nsw <25 x i32> %cz, %dz
%r2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m2)
%x = add i32 %r1, %r2
ret i32 %x
}
define i32 @test_sdot_v25i8_double_nomla(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) {
; CHECK-LABEL: test_sdot_v25i8_double_nomla:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldr b0, [sp, #64]
; CHECK-NEXT: add x8, sp, #72
; CHECK-NEXT: ldr b2, [sp]
; CHECK-NEXT: add x9, sp, #8
; CHECK-NEXT: ldr b3, [sp, #464]
; CHECK-NEXT: add x10, sp, #472
; CHECK-NEXT: ld1 { v0.b }[1], [x8]
; CHECK-NEXT: add x8, sp, #80
; CHECK-NEXT: ld1 { v2.b }[1], [x9]
; CHECK-NEXT: add x9, sp, #16
; CHECK-NEXT: add x11, sp, #104
; CHECK-NEXT: ld1 { v3.b }[1], [x10]
; CHECK-NEXT: add x10, sp, #480
; CHECK-NEXT: add x12, sp, #32
; CHECK-NEXT: ld1 { v0.b }[2], [x8]
; CHECK-NEXT: add x8, sp, #88
; CHECK-NEXT: ld1 { v2.b }[2], [x9]
; CHECK-NEXT: add x9, sp, #24
; CHECK-NEXT: ldr b4, [sp, #128]
; CHECK-NEXT: fmov s1, w0
; CHECK-NEXT: ld1 { v3.b }[2], [x10]
; CHECK-NEXT: add x10, sp, #48
; CHECK-NEXT: ld1 { v0.b }[3], [x8]
; CHECK-NEXT: add x8, sp, #96
; CHECK-NEXT: ld1 { v2.b }[3], [x9]
; CHECK-NEXT: add x9, sp, #120
; CHECK-NEXT: sshll v4.8h, v4.8b, #0
; CHECK-NEXT: ldr b6, [sp, #400]
; CHECK-NEXT: mov v1.b[1], w1
; CHECK-NEXT: ldr b16, [sp, #528]
; CHECK-NEXT: ld1 { v0.b }[4], [x8]
; CHECK-NEXT: add x8, sp, #112
; CHECK-NEXT: ld1 { v2.b }[4], [x12]
; CHECK-NEXT: add x12, sp, #40
; CHECK-NEXT: sshll v5.4s, v4.4h, #0
; CHECK-NEXT: ldr b4, [sp, #336]
; CHECK-NEXT: mov v1.b[2], w2
; CHECK-NEXT: ld1 { v0.b }[5], [x11]
; CHECK-NEXT: add x11, sp, #488
; CHECK-NEXT: ld1 { v2.b }[5], [x12]
; CHECK-NEXT: sshll v16.8h, v16.8b, #0
; CHECK-NEXT: ld1 { v3.b }[3], [x11]
; CHECK-NEXT: add x11, sp, #344
; CHECK-NEXT: ld1 { v0.b }[6], [x8]
; CHECK-NEXT: add x8, sp, #496
; CHECK-NEXT: ld1 { v2.b }[6], [x10]
; CHECK-NEXT: add x10, sp, #352
; CHECK-NEXT: ld1 { v4.b }[1], [x11]
; CHECK-NEXT: add x11, sp, #416
; CHECK-NEXT: ld1 { v3.b }[4], [x8]
; CHECK-NEXT: add x8, sp, #504
; CHECK-NEXT: ld1 { v0.b }[7], [x9]
; CHECK-NEXT: add x9, sp, #408
; CHECK-NEXT: mov v1.b[3], w3
; CHECK-NEXT: ld1 { v4.b }[2], [x10]
; CHECK-NEXT: add x10, sp, #360
; CHECK-NEXT: ld1 { v6.b }[1], [x9]
; CHECK-NEXT: add x9, sp, #56
; CHECK-NEXT: ld1 { v3.b }[5], [x8]
; CHECK-NEXT: add x8, sp, #512
; CHECK-NEXT: mov v1.b[4], w4
; CHECK-NEXT: ld1 { v2.b }[7], [x9]
; CHECK-NEXT: add x9, sp, #424
; CHECK-NEXT: ld1 { v6.b }[2], [x11]
; CHECK-NEXT: ld1 { v4.b }[3], [x10]
; CHECK-NEXT: add x10, sp, #368
; CHECK-NEXT: ld1 { v3.b }[6], [x8]
; CHECK-NEXT: add x8, sp, #376
; CHECK-NEXT: mov v1.b[5], w5
; CHECK-NEXT: ld1 { v6.b }[3], [x9]
; CHECK-NEXT: add x9, sp, #432
; CHECK-NEXT: ld1 { v4.b }[4], [x10]
; CHECK-NEXT: add x10, sp, #440
; CHECK-NEXT: sshll v16.4s, v16.4h, #0
; CHECK-NEXT: mov v1.b[6], w6
; CHECK-NEXT: ld1 { v6.b }[4], [x9]
; CHECK-NEXT: add x9, sp, #520
; CHECK-NEXT: ld1 { v4.b }[5], [x8]
; CHECK-NEXT: add x8, sp, #384
; CHECK-NEXT: mov v5.s[1], wzr
; CHECK-NEXT: mov v16.s[1], wzr
; CHECK-NEXT: ld1 { v3.b }[7], [x9]
; CHECK-NEXT: ld1 { v6.b }[5], [x10]
; CHECK-NEXT: add x10, sp, #448
; CHECK-NEXT: ld1 { v4.b }[6], [x8]
; CHECK-NEXT: add x8, sp, #392
; CHECK-NEXT: mov v1.b[7], w7
; CHECK-NEXT: add x9, sp, #456
; CHECK-NEXT: mov v5.s[2], wzr
; CHECK-NEXT: ld1 { v6.b }[6], [x10]
; CHECK-NEXT: mov v16.s[2], wzr
; CHECK-NEXT: ld1 { v4.b }[7], [x8]
; CHECK-NEXT: sshll v0.8h, v0.8b, #0
; CHECK-NEXT: sshll v1.8h, v1.8b, #0
; CHECK-NEXT: ld1 { v6.b }[7], [x9]
; CHECK-NEXT: mov v5.s[3], wzr
; CHECK-NEXT: mov v16.s[3], wzr
; CHECK-NEXT: saddl v7.4s, v1.4h, v0.4h
; CHECK-NEXT: saddl2 v0.4s, v1.8h, v0.8h
; CHECK-NEXT: sshll v1.8h, v3.8b, #0
; CHECK-NEXT: sshll v3.8h, v4.8b, #0
; CHECK-NEXT: sshll v2.8h, v2.8b, #0
; CHECK-NEXT: sshll v4.8h, v6.8b, #0
; CHECK-NEXT: saddl v6.4s, v3.4h, v1.4h
; CHECK-NEXT: saddl2 v1.4s, v3.8h, v1.8h
; CHECK-NEXT: saddw v5.4s, v5.4s, v2.4h
; CHECK-NEXT: saddw v3.4s, v16.4s, v4.4h
; CHECK-NEXT: saddw2 v0.4s, v0.4s, v2.8h
; CHECK-NEXT: saddw2 v1.4s, v1.4s, v4.8h
; CHECK-NEXT: add v5.4s, v7.4s, v5.4s
; CHECK-NEXT: add v2.4s, v6.4s, v3.4s
; CHECK-NEXT: add v0.4s, v5.4s, v0.4s
; CHECK-NEXT: add v1.4s, v2.4s, v1.4s
; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w0, s0
; CHECK-NEXT: ret
entry:
%az = sext <25 x i8> %a to <25 x i32>
%r1 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %az)
%cz = sext <25 x i8> %c to <25 x i32>
%r2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %cz)
%x = add i32 %r1, %r2
ret i32 %x
}
define i32 @test_udot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { define i32 @test_udot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-LABEL: test_udot_v32i8: ; CHECK-LABEL: test_udot_v32i8:
; CHECK: // %bb.0: // %entry ; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldp q0, q3, [x1] ; CHECK-NEXT: ldp q0, q3, [x1]
; CHECK-NEXT: ushll2 v7.8h, v0.16b, #0 ; CHECK-NEXT: ushll2 v7.8h, v0.16b, #0
; CHECK-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-NEXT: ldp q2, q1, [x0] ; CHECK-NEXT: ldp q2, q1, [x0]
; CHECK-NEXT: ushll v6.8h, v3.8b, #0 ; CHECK-NEXT: ushll v6.8h, v3.8b, #0
▲ Show 20 Lines • Show All 1,889 Lines • ▼ Show 20 Lines
; CHECK-NEXT: ret ; CHECK-NEXT: ret
entry: entry:
%az = sext <64 x i8> %a to <64 x i32> %az = sext <64 x i8> %a to <64 x i32>
%r1 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %az) %r1 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %az)
%cz = sext <64 x i8> %c to <64 x i32> %cz = sext <64 x i8> %c to <64 x i32>
%r2 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %cz) %r2 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %cz)
%x = add i32 %r1, %r2 %x = add i32 %r1, %r2
ret i32 %x ret i32 %x
} }
No newline at end of file