[AArch64][SVE2] Add the SVE2.1 signed and unsigned 2-way dot instructions

[david-arm]

Summary

This patch adds the assembly/disassembly for the following instructions:

SDOT : Signed integer 2-way dot product indexed and non-indexed
UDOT : Unsigned integer 2-way dot product, indexed and non-indexed

The reference can be found here:
https://developer.arm.com/documentation/ddi0602/2022-09

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

3585–3588

For clarity, can you create a follow up patch to ensure all the SVE dot instructions use this format? This presumably means adding 4WAY.

llvm/test/MC/AArch64/SVE2p1/sdot-diagnostics.s

2

Given the positive tests include movprfx, is it worth adding negative tests to ensure a suitable error is given when prefixed with a predicated movprfx?

[sdesmalen]

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

3585–3588

We should be using VG2 or VG4 for that purpose, that aligns with what we've done for SME2

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

3585–3588

VG2/4 relates to instruction operands that are multi-register. Here 2WAY relates to the instruction's operation, which in this case only take single register operands. For earlier versions of SVE there was no distinction because all the dot instructions were 4WAY. This now changes with SVE2p1, hence my suggestion.

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

3585–3588

Just noting D135676 shows a different naming scheme (e.g. BtoH and BtoS). This also works to distinguish between 2-way and 4-way so either are good options but we need to choose one and stick with it.

[sdesmalen]

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

3585–3588

Here 2WAY relates to the instruction's operation

Of course, I should have looked more closely before I posted.

This also works to distinguish between 2-way and 4-way so either are good options but we need to choose one and stick with it.

In that case, I'd prefer to align with D135676 and use _HtoS.

[david-arm]

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

3585–3588

Renamed to be SDOT_ZZZ_HtoS, etc.

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

 let Predicates = [HasSVE2p1_or_HasSME2] in {
 defm FCLAMP_ZZZ : sve2p1_fclamp<"fclamp">;
 def FDOT_ZZZ_S  : sve_float_dot<0b0, "fdot">;
 def FDOT_ZZZI_S : sve_float_dot_indexed<0b0, "fdot">;
 def BFMLSLB_ZZZ_S : sve_bfloat_matmul_longvecl<0b0, 0b1, "bfmlslb">;
 def BFMLSLT_ZZZ_S : sve_bfloat_matmul_longvecl<0b1, 0b1, "bfmlslt">;
 def BFMLSLB_ZZZI_S : sve_bfloat_matmul_longvecl_idx<0b0, 0b1, "bfmlslb">;
 def BFMLSLT_ZZZI_S : sve_bfloat_matmul_longvecl_idx<0b1, 0b1, "bfmlslt">;
+
+def SDOT_ZZZ_HtoS  : sve2p1_two_way_dot_vv<"sdot", 0b0>;
+def UDOT_ZZZ_HtoS  : sve2p1_two_way_dot_vv<"udot", 0b1>;
+def SDOT_ZZZI_HtoS : sve2p1_two_way_dot_vvi<"sdot", 0b0>;
+def UDOT_ZZZI_HtoS : sve2p1_two_way_dot_vvi<"udot", 0b1>;
 } // End HasSVE2p1_or_HasSME2

llvm/lib/Target/AArch64/SVEInstrFormats.td

class sve2p1_fclamp<string asm, bits<2> sz, ZPRRegOp zpr_ty>
   let ElementSize = zpr_ty.ElementSize;
 }
 
 multiclass sve2p1_fclamp<string asm> {
   def _H : sve2p1_fclamp<asm, 0b01, ZPR16>;
   def _S : sve2p1_fclamp<asm, 0b10, ZPR32>;
   def _D : sve2p1_fclamp<asm, 0b11, ZPR64>;
 }
+
+// SVE two-way dot product
+class sve2p1_two_way_dot_vv<string mnemonic, bit u>
+    : I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm),
+        mnemonic, "\t$Zda, $Zn, $Zm",
+        "", []>, Sched<[]> {
+  bits<5> Zda;
+  bits<5> Zn;
+  bits<5> Zm;
+  let Inst{31-21} = 0b01000100000;
+  let Inst{20-16} = Zm;
+  let Inst{15-11} = 0b11001;
+  let Inst{10}    = u;
+  let Inst{9-5}   = Zn;
+  let Inst{4-0}   = Zda;
+
+  let Constraints = "$Zda = $_Zda";
+  let DestructiveInstType = DestructiveOther;
+}
+
+
+// SVE two-way dot product (indexed)
+class sve2p1_two_way_dot_vvi<string mnemonic, bit u>
+    : I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexS:$i2),
+        mnemonic, "\t$Zda, $Zn, $Zm$i2",
+        "", []>, Sched<[]> {
+  bits<5> Zda;
+  bits<5> Zn;
+  bits<3> Zm;
+  bits<2> i2;
+  let Inst{31-21} = 0b01000100100;
+  let Inst{20-19} = i2;
+  let Inst{18-16} = Zm;
+  let Inst{15-11} = 0b11001;
+  let Inst{10}    = u;
+  let Inst{9-5}   = Zn;
+  let Inst{4-0}   = Zda;
+
+  let Constraints = "$Zda = $_Zda";
+  let DestructiveInstType = DestructiveOther;
+}

llvm/test/MC/AArch64/SVE/sdot-diagnostics.s

 // RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve  2>&1 < %s| FileCheck %s
 
 
 // ------------------------------------------------------------------------- //
 // Invalid element size
 
-sdot  z0.s, z1.h, z31.h
+sdot  z0.b, z1.h, z31.h
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
-// CHECK-NEXT: sdot  z0.s, z1.h, z31.h
+// CHECK-NEXT: sdot  z0.b, z1.h, z31.h
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 sdot  z0.d, z1.b, z31.b
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
 // CHECK-NEXT: sdot  z0.d, z1.b, z31.b
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 sdot  z0.d, z1.s, z31.s

llvm/test/MC/AArch64/SVE/udot-diagnostics.s

 // RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve  2>&1 < %s| FileCheck %s
 
 
 // ------------------------------------------------------------------------- //
 // Invalid element size
 
-udot  z0.s, z1.h, z31.h
+udot  z0.b, z1.h, z31.h
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
-// CHECK-NEXT: udot  z0.s, z1.h, z31.h
+// CHECK-NEXT: udot  z0.b, z1.h, z31.h
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 udot  z0.d, z1.b, z31.b
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
 // CHECK-NEXT: udot  z0.d, z1.b, z31.b
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 udot  z0.d, z1.s, z31.s

llvm/test/MC/AArch64/SVE2p1/sdot-diagnostics.s

+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 2>&1 < %s | FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid vector lane index
+
+sdot z0.s, z0.h, z0.h[8]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 3].
+// CHECK-NEXT: sdot z0.s, z0.h, z0.h[8]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+sdot z0.s, z0.h, z0.h[-1]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 3].
+// CHECK-NEXT: sdot z0.s, z0.h, z0.h[-1]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z31
+sdot z0.s, z0.h, z0.h[8]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 3].
+// CHECK-NEXT: sdot z0.s, z0.h, z0.h[8]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z31
+sdot z0.s, z0.h, z0.h[-1]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 3].
+// CHECK-NEXT: sdot z0.s, z0.h, z0.h[-1]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid vector suffix
+
+sdot z0.h, z0.s, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: sdot z0.h, z0.s, z0.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+sdot z0.b, z0.h, z0.h[1]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: sdot z0.b, z0.h, z0.h[1]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:

llvm/test/MC/AArch64/SVE2p1/sdot.s

+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sme2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2 < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sme2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+movprfx z23, z31
+sdot    z23.s, z13.h, z0.h[1]  // 01000100-10001000-11001001-10110111
+// CHECK-INST:  movprfx z23, z31
+// CHECK-INST: sdot    z23.s, z13.h, z0.h[1]
+// CHECK-ENCODING: [0xb7,0xc9,0x88,0x44]
+// CHECK-ERROR: instruction requires: sme2 or sve2p1
+// CHECK-UNKNOWN: 4488c9b7 <unknown>
+
+sdot    z0.s, z0.h, z0.h[0]  // 01000100-10000000-11001000-00000000
+// CHECK-INST: sdot    z0.s, z0.h, z0.h[0]
+// CHECK-ENCODING: [0x00,0xc8,0x80,0x44]
+// CHECK-ERROR: instruction requires: sme2 or sve2p1
+// CHECK-UNKNOWN: 4480c800 <unknown>
+
+sdot    z21.s, z10.h, z5.h[2]  // 01000100-10010101-11001001-01010101
+// CHECK-INST: sdot    z21.s, z10.h, z5.h[2]
+// CHECK-ENCODING: [0x55,0xc9,0x95,0x44]
+// CHECK-ERROR: instruction requires: sme2 or sve2p1
+// CHECK-UNKNOWN: 4495c955 <unknown>
+
+sdot    z23.s, z13.h, z0.h[1]  // 01000100-10001000-11001001-10110111
+// CHECK-INST: sdot    z23.s, z13.h, z0.h[1]
+// CHECK-ENCODING: [0xb7,0xc9,0x88,0x44]
+// CHECK-ERROR: instruction requires: sme2 or sve2p1
+// CHECK-UNKNOWN: 4488c9b7 <unknown>
+
+sdot    z31.s, z31.h, z7.h[3]  // 01000100-10011111-11001011-11111111
+// CHECK-INST: sdot    z31.s, z31.h, z7.h[3]
+// CHECK-ENCODING: [0xff,0xcb,0x9f,0x44]
+// CHECK-ERROR: instruction requires: sme2 or sve2p1
+// CHECK-UNKNOWN: 449fcbff <unknown>
+
+movprfx z23, z31
+sdot    z23.s, z13.h, z8.h  // 01000100-00001000-11001001-10110111
+// CHECK-INST:  movprfx z23, z31
+// CHECK-INST: sdot    z23.s, z13.h, z8.h
+// CHECK-ENCODING: [0xb7,0xc9,0x08,0x44]
+// CHECK-ERROR: instruction requires: sme2 or sve2p1
+// CHECK-UNKNOWN: 4408c9b7 <unknown>
+
+sdot    z0.s, z0.h, z0.h  // 01000100-00000000-11001000-00000000
+// CHECK-INST: sdot    z0.s, z0.h, z0.h
+// CHECK-ENCODING: [0x00,0xc8,0x00,0x44]
+// CHECK-ERROR: instruction requires: sme2 or sve2p1
+// CHECK-UNKNOWN: 4400c800 <unknown>
+
+sdot    z21.s, z10.h, z21.h  // 01000100-00010101-11001001-01010101
+// CHECK-INST: sdot    z21.s, z10.h, z21.h
+// CHECK-ENCODING: [0x55,0xc9,0x15,0x44]
+// CHECK-ERROR: instruction requires: sme2 or sve2p1
+// CHECK-UNKNOWN: 4415c955 <unknown>
+
+sdot    z23.s, z13.h, z8.h  // 01000100-00001000-11001001-10110111
+// CHECK-INST: sdot    z23.s, z13.h, z8.h
+// CHECK-ENCODING: [0xb7,0xc9,0x08,0x44]
+// CHECK-ERROR: instruction requires: sme2 or sve2p1
+// CHECK-UNKNOWN: 4408c9b7 <unknown>
+
+sdot    z31.s, z31.h, z31.h  // 01000100-00011111-11001011-11111111
+// CHECK-INST: sdot    z31.s, z31.h, z31.h
+// CHECK-ENCODING: [0xff,0xcb,0x1f,0x44]
+// CHECK-ERROR: instruction requires: sme2 or sve2p1
+// CHECK-UNKNOWN: 441fcbff <unknown>

llvm/test/MC/AArch64/SVE2p1/udot-diagnostics.s

+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 2>&1 < %s | FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid vector lane index
+
+udot z0.s, z0.h, z0.h[8]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 3].
+// CHECK-NEXT: udot z0.s, z0.h, z0.h[8]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+udot z0.s, z0.h, z0.h[-1]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 3].
+// CHECK-NEXT: udot z0.s, z0.h, z0.h[-1]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z31
+udot z0.s, z0.h, z0.h[8]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 3].
+// CHECK-NEXT: udot z0.s, z0.h, z0.h[8]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0, z31
+udot z0.s, z0.h, z0.h[-1]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 3].
+// CHECK-NEXT: udot z0.s, z0.h, z0.h[-1]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid vector suffix
+
+udot z0.h, z0.s, z0.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: udot z0.h, z0.s, z0.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+udot z0.h, z0.s, z0.s[1]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: udot z0.h, z0.s, z0.s[1]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:

llvm/test/MC/AArch64/SVE2p1/udot.s

+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sme2 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2 < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sme2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+movprfx z23, z31
+udot    z23.s, z13.h, z0.h[1]  // 01000100-10001000-11001101-10110111
+// CHECK-INST:  movprfx z23, z31
+// CHECK-INST: udot    z23.s, z13.h, z0.h[1]
+// CHECK-ENCODING: [0xb7,0xcd,0x88,0x44]
+// CHECK-ERROR: instruction requires: sme2 or sve2p1
+// CHECK-UNKNOWN: 4488cdb7 <unknown>
+
+udot    z0.s, z0.h, z0.h[0]  // 01000100-10000000-11001100-00000000
+// CHECK-INST: udot    z0.s, z0.h, z0.h[0]
+// CHECK-ENCODING: [0x00,0xcc,0x80,0x44]
+// CHECK-ERROR: instruction requires: sme2 or sve2p1
+// CHECK-UNKNOWN: 4480cc00 <unknown>
+
+udot    z21.s, z10.h, z5.h[2]  // 01000100-10010101-11001101-01010101
+// CHECK-INST: udot    z21.s, z10.h, z5.h[2]
+// CHECK-ENCODING: [0x55,0xcd,0x95,0x44]
+// CHECK-ERROR: instruction requires: sme2 or sve2p1
+// CHECK-UNKNOWN: 4495cd55 <unknown>
+
+udot    z23.s, z13.h, z0.h[1]  // 01000100-10001000-11001101-10110111
+// CHECK-INST: udot    z23.s, z13.h, z0.h[1]
+// CHECK-ENCODING: [0xb7,0xcd,0x88,0x44]
+// CHECK-ERROR: instruction requires: sme2 or sve2p1
+// CHECK-UNKNOWN: 4488cdb7 <unknown>
+
+udot    z31.s, z31.h, z7.h[3]  // 01000100-10011111-11001111-11111111
+// CHECK-INST: udot    z31.s, z31.h, z7.h[3]
+// CHECK-ENCODING: [0xff,0xcf,0x9f,0x44]
+// CHECK-ERROR: instruction requires: sme2 or sve2p1
+// CHECK-UNKNOWN: 449fcfff <unknown>
+
+movprfx z23, z31
+udot    z23.s, z13.h, z8.h  // 01000100-00001000-11001101-10110111
+// CHECK-INST:  movprfx z23, z31
+// CHECK-INST: udot    z23.s, z13.h, z8.h
+// CHECK-ENCODING: [0xb7,0xcd,0x08,0x44]
+// CHECK-ERROR: instruction requires: sme2 or sve2p1
+// CHECK-UNKNOWN: 4408cdb7 <unknown>
+
+udot    z0.s, z0.h, z0.h  // 01000100-00000000-11001100-00000000
+// CHECK-INST: udot    z0.s, z0.h, z0.h
+// CHECK-ENCODING: [0x00,0xcc,0x00,0x44]
+// CHECK-ERROR: instruction requires: sme2 or sve2p1
+// CHECK-UNKNOWN: 4400cc00 <unknown>
+
+udot    z21.s, z10.h, z21.h  // 01000100-00010101-11001101-01010101
+// CHECK-INST: udot    z21.s, z10.h, z21.h
+// CHECK-ENCODING: [0x55,0xcd,0x15,0x44]
+// CHECK-ERROR: instruction requires: sme2 or sve2p1
+// CHECK-UNKNOWN: 4415cd55 <unknown>
+
+udot    z23.s, z13.h, z8.h  // 01000100-00001000-11001101-10110111
+// CHECK-INST: udot    z23.s, z13.h, z8.h
+// CHECK-ENCODING: [0xb7,0xcd,0x08,0x44]
+// CHECK-ERROR: instruction requires: sme2 or sve2p1
+// CHECK-UNKNOWN: 4408cdb7 <unknown>
+
+udot    z31.s, z31.h, z31.h  // 01000100-00011111-11001111-11111111
+// CHECK-INST: udot    z31.s, z31.h, z31.h
+// CHECK-ENCODING: [0xff,0xcf,0x1f,0x44]
+// CHECK-ERROR: instruction requires: sme2 or sve2p1
+// CHECK-UNKNOWN: 441fcfff <unknown>