[AArch64][SVE2] Add the SVE2.1 ld1q gather & st1q scatter instructions

[david-arm]

Summary

This patch adds the assembly/disassembly for the following instructions:

ld1q : Gather load quadwords
st1q : Scatter store quadwords

The reference can be found here:
https://developer.arm.com/documentation/ddi0602/2022-09

[paulwalker-arm]

A request for some extra negative tests but otherwise looks good to me.

llvm/test/MC/AArch64/SVE2p1/ld1q-diagnostics.s

3

I think it's worth having negative tests for

ld1q {z0.q}, p0/z, [z0.d, sp]
ld1q {z0.q}, p0/z, [x0, z0.d]

llvm/lib/Target/AArch64/AArch64RegisterInfo.td

 def Z_s  : RegisterOperand<ZPR,  "printTypedVectorList<0,'s'>"> {
   let ParserMatchClass = ZPRVectorList<32, 1>;
 }
 
 def Z_d  : RegisterOperand<ZPR,  "printTypedVectorList<0,'d'>"> {
   let ParserMatchClass = ZPRVectorList<64, 1>;
 }
 
+def Z_q  : RegisterOperand<ZPR,  "printTypedVectorList<0,'q'>"> {
+  let ParserMatchClass = ZPRVectorList<128, 1>;
+}
+
 def ZZ_b  : RegisterOperand<ZPR2, "printTypedVectorList<0,'b'>"> {
   let ParserMatchClass = ZPRVectorList<8, 2>;
 }
 
 def ZZ_h  : RegisterOperand<ZPR2, "printTypedVectorList<0,'h'>"> {
   let ParserMatchClass = ZPRVectorList<16, 2>;
 }
 

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

let Predicates = [HasSVE] in {
   defm GLD1H_D    : sve_mem_64b_gld_vs2_64_unscaled<0b0110, "ld1h",    AArch64ld1_gather_z,    nxv2i16>;
   defm GLDFF1H_D  : sve_mem_64b_gld_vs2_64_unscaled<0b0111, "ldff1h",  AArch64ldff1_gather_z,  nxv2i16>;
   defm GLD1SW_D   : sve_mem_64b_gld_vs2_64_unscaled<0b1000, "ld1sw",   AArch64ld1s_gather_z,   nxv2i32>;
   defm GLDFF1SW_D : sve_mem_64b_gld_vs2_64_unscaled<0b1001, "ldff1sw", AArch64ldff1s_gather_z, nxv2i32>;
   defm GLD1W_D    : sve_mem_64b_gld_vs2_64_unscaled<0b1010, "ld1w",    AArch64ld1_gather_z,    nxv2i32>;
   defm GLDFF1W_D  : sve_mem_64b_gld_vs2_64_unscaled<0b1011, "ldff1w",  AArch64ldff1_gather_z,  nxv2i32>;
   defm GLD1D      : sve_mem_64b_gld_vs2_64_unscaled<0b1110, "ld1d",    AArch64ld1_gather_z,    nxv2i64>;
   defm GLDFF1D    : sve_mem_64b_gld_vs2_64_unscaled<0b1111, "ldff1d",  AArch64ldff1_gather_z,  nxv2i64>;
+  let Predicates = [HasSVE2p1] in {
+  defm GLD1Q      : sve_mem_128b_gld_64_unscaled<"ld1q">;
+  }
 
   // Gathers using scaled 64-bit offsets, e.g.
   //    ld1h z0.d, p0/z, [x0, z0.d, lsl #1]
   defm GLD1SH_D   : sve_mem_64b_gld_sv2_64_scaled<0b0100, "ld1sh",    AArch64ld1s_gather_scaled_z,   ZPR64ExtLSL16, nxv2i16>;
   defm GLDFF1SH_D : sve_mem_64b_gld_sv2_64_scaled<0b0101, "ldff1sh",  AArch64ldff1s_gather_scaled_z, ZPR64ExtLSL16, nxv2i16>;
   defm GLD1H_D    : sve_mem_64b_gld_sv2_64_scaled<0b0110, "ld1h",     AArch64ld1_gather_scaled_z,    ZPR64ExtLSL16, nxv2i16>;
   defm GLDFF1H_D  : sve_mem_64b_gld_sv2_64_scaled<0b0111, "ldff1h",   AArch64ldff1_gather_scaled_z,  ZPR64ExtLSL16, nxv2i16>;
   defm GLD1SW_D   : sve_mem_64b_gld_sv2_64_scaled<0b1000, "ld1sw",    AArch64ld1s_gather_scaled_z,   ZPR64ExtLSL32, nxv2i32>;
let Predicates = [HasSVE] in {
   defm SST1D   : sve_mem_64b_sst_vi_ptrs<0b110, "st1d", uimm5s8, AArch64st1_scatter_imm, nxv2i64>;
 
   // Scatters using unscaled 64-bit offsets, e.g.
   //    st1h z0.d, p0, [x0, z0.d]
   defm SST1B_D : sve_mem_sst_sv_64_unscaled<0b00, "st1b", AArch64st1_scatter, nxv2i8>;
   defm SST1H_D : sve_mem_sst_sv_64_unscaled<0b01, "st1h", AArch64st1_scatter, nxv2i16>;
   defm SST1W_D : sve_mem_sst_sv_64_unscaled<0b10, "st1w", AArch64st1_scatter, nxv2i32>;
   defm SST1D   : sve_mem_sst_sv_64_unscaled<0b11, "st1d", AArch64st1_scatter, nxv2i64>;
+  let Predicates = [HasSVE2p1] in {
+  defm SST1Q   : sve_mem_sst_128b_64_unscaled<"st1q">;
+  }
 
   // Scatters using scaled 64-bit offsets, e.g.
   //    st1h z0.d, p0, [x0, z0.d, lsl #1]
   defm SST1H_D : sve_mem_sst_sv_64_scaled<0b01, "st1h", AArch64st1_scatter_scaled, ZPR64ExtLSL16, nxv2i16>;
   defm SST1W_D : sve_mem_sst_sv_64_scaled<0b10, "st1w", AArch64st1_scatter_scaled, ZPR64ExtLSL32, nxv2i32>;
   defm SST1D   : sve_mem_sst_sv_64_scaled<0b11, "st1d", AArch64st1_scatter_scaled, ZPR64ExtLSL64, nxv2i64>;
 
   multiclass sve_masked_scatter_x2_scaled<ValueType Ty, SDPatternOperator Store, string Inst> {

llvm/lib/Target/AArch64/SVEInstrFormats.td

 
 
 multiclass sve2p1_int_while_rr_pair<string mnemonic, bits<3> opc> {
  def _B : sve2p1_int_while_rr_pair<mnemonic, 0b00, opc, PP_b_mul_r>;
  def _H : sve2p1_int_while_rr_pair<mnemonic, 0b01, opc, PP_h_mul_r>;
  def _S : sve2p1_int_while_rr_pair<mnemonic, 0b10, opc, PP_s_mul_r>;
  def _D : sve2p1_int_while_rr_pair<mnemonic, 0b11, opc, PP_d_mul_r>;
 }
+
+
+class sve_mem_128b_gld_64_unscaled<string mnemonic>
+    : I<(outs Z_q:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm),
+        mnemonic, "\t$Zt, $Pg/z, [$Zn, $Rm]",
+        "", []>, Sched<[]> {
+  bits<5> Zt;
+  bits<5> Zn;
+  bits<3> Pg;
+  bits<5> Rm;
+  let Inst{31-21} = 0b11000100000;
+  let Inst{20-16} = Rm;
+  let Inst{15-13} = 0b101;
+  let Inst{12-10} = Pg;
+  let Inst{9-5}   = Zn;
+  let Inst{4-0}   = Zt;
+
+  let mayLoad = 1;
+}
+
+
+multiclass sve_mem_128b_gld_64_unscaled<string mnemonic> {
+  def NAME : sve_mem_128b_gld_64_unscaled<mnemonic>;
+
+  def : InstAlias<mnemonic # " $Zt, $Pg/z, [$Zn]",
+                  (!cast<Instruction>(NAME) Z_q:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>;
+}
+
+class sve_mem_sst_128b_64_unscaled<string mnemonic>
+    : I<(outs ), (ins Z_q:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm),
+        mnemonic, "\t$Zt, $Pg, [$Zn, $Rm]",
+        "", []>, Sched<[]> {
+  bits<5> Zt;
+  bits<5> Zn;
+  bits<3> Pg;
+  bits<5> Rm;
+  let Inst{31-21} = 0b11100100001;
+  let Inst{20-16} = Rm;
+  let Inst{15-13} = 0b001;
+  let Inst{12-10} = Pg;
+  let Inst{9-5}   = Zn;
+  let Inst{4-0}   = Zt;
+
+  let mayStore = 1;
+}
+
+
+multiclass sve_mem_sst_128b_64_unscaled<string mnemonic> {
+  def NAME : sve_mem_sst_128b_64_unscaled<mnemonic>;
+
+  def : InstAlias<mnemonic # " $Zt, $Pg, [$Zn]",
+                  (!cast<Instruction>(NAME) Z_q:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>;
+}

llvm/test/MC/AArch64/SVE2p1/ld1q-diagnostics.s

+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 2>&1 < %s | FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid predicate register
+
+ld1q {z0.q}, p8/z, [z0.d, x0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: ld1q {z0.q}, p8/z, [z0.d, x0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ld1q {z23.q}, p2/m, [z3.d]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: ld1q {z23.q}, p2/m, [z3.d]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ld1q {z21.q}, p2.q, [z5.d]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: ld1q {z21.q}, p2.q, [z5.d]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid order of base & offset
+
+ld1q {z0.q}, p0/z, [x0, z0.d]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: ld1q {z0.q}, p0/z, [x0, z0.d]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid general purpose register
+
+ld1q {z0.q}, p0/z, [z0.d, sp]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: ld1q {z0.q}, p0/z, [z0.d, sp]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid suffixes
+
+ld1q {z0.q}, p0/z, [z2.s]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: ld1q {z0.q}, p0/z, [z2.s]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:

llvm/test/MC/AArch64/SVE2p1/ld1q.s

+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p1 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p1 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p1 < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p1 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
+ld1q    {z0.q}, p0/z, [z0.d, x0]  // 11000100-00000000-10100000-00000000
+// CHECK-INST: ld1q    { z0.q }, p0/z, [z0.d, x0]
+// CHECK-ENCODING: [0x00,0xa0,0x00,0xc4]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: c400a000 <unknown>
+
+ld1q    {z21.q}, p5/z, [z10.d, x21]  // 11000100-00010101-10110101-01010101
+// CHECK-INST: ld1q    { z21.q }, p5/z, [z10.d, x21]
+// CHECK-ENCODING: [0x55,0xb5,0x15,0xc4]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: c415b555 <unknown>
+
+ld1q    {z23.q}, p3/z, [z13.d, x8]  // 11000100-00001000-10101101-10110111
+// CHECK-INST: ld1q    { z23.q }, p3/z, [z13.d, x8]
+// CHECK-ENCODING: [0xb7,0xad,0x08,0xc4]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: c408adb7 <unknown>
+
+ld1q    {z31.q}, p7/z, [z31.d]  // 11000100-00011111-10111111-11111111
+// CHECK-INST: ld1q    { z31.q }, p7/z, [z31.d]
+// CHECK-ENCODING: [0xff,0xbf,0x1f,0xc4]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: c41fbfff <unknown>
+

llvm/test/MC/AArch64/SVE2p1/st1q-diagnostics.s

+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 2>&1 < %s | FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid predicate register
+
+st1q {z0.q}, p8, [z0.d, x0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: st1q {z0.q}, p8, [z0.d, x0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+st1q {z23.q}, p2/m, [z3.d]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: st1q {z23.q}, p2/m, [z3.d]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+st1q {z21.q}, p2.q, [z5.d]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: st1q {z21.q}, p2.q, [z5.d]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid order of base & offset
+
+st1q {z0.q}, p0, [x0, z0.d]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: st1q {z0.q}, p0, [x0, z0.d]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid general purpose register
+
+st1q {z0.q}, p0, [z0.d, sp]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: st1q {z0.q}, p0, [z0.d, sp]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid suffixes
+
+st1q {z0.q}, p0, [z2.s]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: st1q {z0.q}, p0, [z2.s]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:

llvm/test/MC/AArch64/SVE2p1/st1q.s

+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p1 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sve2p1 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p1 < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p1 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
+st1q    {z0.q}, p0, [z0.d, x0]  // 11100100-00100000-00100000-00000000
+// CHECK-INST: st1q    { z0.q }, p0, [z0.d, x0]
+// CHECK-ENCODING: [0x00,0x20,0x20,0xe4]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e4202000 <unknown>
+
+st1q    {z21.q}, p5, [z10.d, x21]  // 11100100-00110101-00110101-01010101
+// CHECK-INST: st1q    { z21.q }, p5, [z10.d, x21]
+// CHECK-ENCODING: [0x55,0x35,0x35,0xe4]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e4353555 <unknown>
+
+st1q    {z23.q}, p3, [z13.d, x8]  // 11100100-00101000-00101101-10110111
+// CHECK-INST: st1q    { z23.q }, p3, [z13.d, x8]
+// CHECK-ENCODING: [0xb7,0x2d,0x28,0xe4]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e4282db7 <unknown>
+
+st1q    {z31.q}, p7, [z31.d]  // 11100100-00111111-00111111-11111111
+// CHECK-INST: st1q    { z31.q }, p7, [z31.d]
+// CHECK-ENCODING: [0xff,0x3f,0x3f,0xe4]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e43f3fff <unknown>
+