[AArch64][SVE2] Add the SVE2.1 quadword variants of ld1w/ld1d/st1w/st1d

[david-arm]

Summary

This patch adds the assembly/disassembly for the following instructions:

st1w: Contiguous store words from vector (128-bit vector elements)
st1d: Contiguous store doublewords from vector (128-bit vector elements)
ld1w: Contiguous load unsigned words to vector (128-bit vector elements)
ld1d: Contiguous load unsigned doublewords to vector (128-bit vector elements)

The reference can be found here:
https://developer.arm.com/documentation/ddi0602/2022-09

[c-rhodes]

llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td

21 ↗

(On Diff #472582)

most of SVE is supported by N2, this should be [HasSVE2p1]

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

1299

As with previous comments I'm guessing you only using this so you don't get the alias, but I think we're ok having the alias.

llvm/lib/Target/AArch64/SVEInstrFormats.td

5782–5788

Do we need this class? It looks like the only difference is the support aliases, but I don't see a reason the quad case need to be more restrictive.

9227–9229

Given they're structurally identical what about extending sve_mem_cst_si to include an extra q bit. This will also mean the new instructions have a matching set of InstAliases. This also matches the path you've been able to take regarding the stores.

9256

Perhaps just extend sve_mem_cld_ss_base? I'd be tempted to extend sve_mem_cld_ss also, but that's up to you.

[david-arm]

llvm/lib/Target/AArch64/SVEInstrFormats.td

9227–9229

I assume you mean sve_mem_cld_si_base, not sve_mem_cst_si since they are stores? If you're referring to sve_mem_cld_si_base then they're not quite structurally the same since the dtype field is only bits 24-23 for quadwords. By bringing them together it does become a bit confusing because treating the quadword encoding group as having a 24-21 bit dtype field leads to exactly the same dtypes we have for LD1SB (1100) and LD1SH (1000).

Reusing the classes reduces code for sure, but it might make it more confusing. I'll give it a try anyway and see how it looks.

[david-arm]

• Attempted to reuse as many existing classes as possible.
• Added aliases for all new loads and stores that don't require the braces around the vector register, e.g. ld1w z3.q, ...
• Added tests for all additional aliases.

[david-arm]

llvm/lib/Target/AArch64/SVEInstrFormats.td

9227–9229

Hmm, the problem with reusing the classes is that it requires inverting the meaning of the nf bit. In sve_mem_cld_si_base the nf is bit 20, which is set to 0b0 for normal non-quadword loads, i.e. LD1W_IMM, however for sve_mem_128b_cld_si bit 20 is 0b1! It seems a bit odd to say that a faulting quadword load has nf=1. If you don't mind I'll keep the existing class as it is?

9256

Again, I think this doesn't really help because the quadword loads (sve_mem_128b_cld_ss) and other loads (sve_mem_cld_ss_base) have different bits 15-14:

sve_mem_128b_cld_ss: 0b10
sve_mem_cld_ss_base: 0b01

It seems to me like the two encoding groups are not close enough to be reused easily.

[paulwalker-arm]

llvm/lib/Target/AArch64/SVEInstrFormats.td

9227–9229

I think you're putting a bit too much stock into what these things are called but sure, I can see how there's a bit more going on here than normal so if this is your preference then yes this works for me. Thanks for investigating.

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

let Predicates = [HasSVEorSME] in {
   defm LD1SW_D_IMM : sve_mem_cld_si<0b0100, "ld1sw", Z_d, ZPR64>;
   defm LD1H_IMM    : sve_mem_cld_si<0b0101, "ld1h",  Z_h, ZPR16>;
   defm LD1H_S_IMM  : sve_mem_cld_si<0b0110, "ld1h",  Z_s, ZPR32>;
   defm LD1H_D_IMM  : sve_mem_cld_si<0b0111, "ld1h",  Z_d, ZPR64>;
   defm LD1SH_D_IMM : sve_mem_cld_si<0b1000, "ld1sh", Z_d, ZPR64>;
   defm LD1SH_S_IMM : sve_mem_cld_si<0b1001, "ld1sh", Z_s, ZPR32>;
   defm LD1W_IMM    : sve_mem_cld_si<0b1010, "ld1w",  Z_s, ZPR32>;
   defm LD1W_D_IMM  : sve_mem_cld_si<0b1011, "ld1w",  Z_d, ZPR64>;
+  let Predicates = [HasSVE2p1] in {
+  defm LD1W_Q_IMM  : sve_mem_128b_cld_si<0b10, "ld1w">;
+  }
   defm LD1SB_D_IMM : sve_mem_cld_si<0b1100, "ld1sb", Z_d, ZPR64>;
   defm LD1SB_S_IMM : sve_mem_cld_si<0b1101, "ld1sb", Z_s, ZPR32>;
   defm LD1SB_H_IMM : sve_mem_cld_si<0b1110, "ld1sb", Z_h, ZPR16>;
   defm LD1D_IMM    : sve_mem_cld_si<0b1111, "ld1d",  Z_d, ZPR64>;
+  let Predicates = [HasSVE2p1] in {
+  defm LD1D_Q_IMM  : sve_mem_128b_cld_si<0b11, "ld1d">;
+  }
 
   // LD1R loads (splat scalar to vector)
   defm LD1RB_IMM    : sve_mem_ld_dup<0b00, 0b00, "ld1rb",  Z_b, ZPR8,  uimm6s1>;
   defm LD1RB_H_IMM  : sve_mem_ld_dup<0b00, 0b01, "ld1rb",  Z_h, ZPR16, uimm6s1>;
   defm LD1RB_S_IMM  : sve_mem_ld_dup<0b00, 0b10, "ld1rb",  Z_s, ZPR32, uimm6s1>;
   defm LD1RB_D_IMM  : sve_mem_ld_dup<0b00, 0b11, "ld1rb",  Z_d, ZPR64, uimm6s1>;
   defm LD1RSW_IMM   : sve_mem_ld_dup<0b01, 0b00, "ld1rsw", Z_d, ZPR64, uimm6s4>;
   defm LD1RH_IMM    : sve_mem_ld_dup<0b01, 0b01, "ld1rh",  Z_h, ZPR16, uimm6s2>;
let Predicates = [HasSVEorSME] in {
   defm LD1SW_D : sve_mem_cld_ss<0b0100, "ld1sw", Z_d, ZPR64, GPR64NoXZRshifted32>;
   defm LD1H    : sve_mem_cld_ss<0b0101, "ld1h",  Z_h, ZPR16, GPR64NoXZRshifted16>;
   defm LD1H_S  : sve_mem_cld_ss<0b0110, "ld1h",  Z_s, ZPR32, GPR64NoXZRshifted16>;
   defm LD1H_D  : sve_mem_cld_ss<0b0111, "ld1h",  Z_d, ZPR64, GPR64NoXZRshifted16>;
   defm LD1SH_D : sve_mem_cld_ss<0b1000, "ld1sh", Z_d, ZPR64, GPR64NoXZRshifted16>;
   defm LD1SH_S : sve_mem_cld_ss<0b1001, "ld1sh", Z_s, ZPR32, GPR64NoXZRshifted16>;
   defm LD1W    : sve_mem_cld_ss<0b1010, "ld1w",  Z_s, ZPR32, GPR64NoXZRshifted32>;
   defm LD1W_D  : sve_mem_cld_ss<0b1011, "ld1w",  Z_d, ZPR64, GPR64NoXZRshifted32>;
+  let Predicates = [HasSVE2p1] in {
+  defm LD1W_Q  : sve_mem_128b_cld_ss<0b10, "ld1w", GPR64NoXZRshifted32>;
+  }
   defm LD1SB_D : sve_mem_cld_ss<0b1100, "ld1sb", Z_d, ZPR64, GPR64NoXZRshifted8>;
   defm LD1SB_S : sve_mem_cld_ss<0b1101, "ld1sb", Z_s, ZPR32, GPR64NoXZRshifted8>;
   defm LD1SB_H : sve_mem_cld_ss<0b1110, "ld1sb", Z_h, ZPR16, GPR64NoXZRshifted8>;
   defm LD1D    : sve_mem_cld_ss<0b1111, "ld1d",  Z_d, ZPR64, GPR64NoXZRshifted64>;
+  let Predicates = [HasSVE2p1] in {
+  defm LD1D_Q  : sve_mem_128b_cld_ss<0b11, "ld1d", GPR64NoXZRshifted64>;
+  }
 } // End HasSVEorSME
 
 let Predicates = [HasSVE] in {
   // non-faulting continuous load with reg+immediate
   defm LDNF1B_IMM    : sve_mem_cldnf_si<0b0000, "ldnf1b",  Z_b, ZPR8>;
   defm LDNF1B_H_IMM  : sve_mem_cldnf_si<0b0001, "ldnf1b",  Z_h, ZPR16>;
   defm LDNF1B_S_IMM  : sve_mem_cldnf_si<0b0010, "ldnf1b",  Z_s, ZPR32>;
   defm LDNF1B_D_IMM  : sve_mem_cldnf_si<0b0011, "ldnf1b",  Z_d, ZPR64>;
let Predicates = [HasSVEorSME] in {
   defm ST1B_H_IMM : sve_mem_cst_si<0b00, 0b01, "st1b", Z_h, ZPR16>;
   defm ST1B_S_IMM : sve_mem_cst_si<0b00, 0b10, "st1b", Z_s, ZPR32>;
   defm ST1B_D_IMM : sve_mem_cst_si<0b00, 0b11, "st1b", Z_d, ZPR64>;
   defm ST1H_IMM   : sve_mem_cst_si<0b01, 0b01, "st1h", Z_h, ZPR16>;
   defm ST1H_S_IMM : sve_mem_cst_si<0b01, 0b10, "st1h", Z_s, ZPR32>;
   defm ST1H_D_IMM : sve_mem_cst_si<0b01, 0b11, "st1h", Z_d, ZPR64>;
   defm ST1W_IMM   : sve_mem_cst_si<0b10, 0b10, "st1w", Z_s, ZPR32>;
   defm ST1W_D_IMM : sve_mem_cst_si<0b10, 0b11, "st1w", Z_d, ZPR64>;
+  let Predicates = [HasSVE2p1] in {
+  defm ST1W_Q_IMM : sve_mem_cst_si<0b10, 0b00, "st1w", Z_q, ZPR128>;
+  }
   defm ST1D_IMM   : sve_mem_cst_si<0b11, 0b11, "st1d", Z_d, ZPR64>;
+  let Predicates = [HasSVE2p1] in {
+  defm ST1D_Q_IMM : sve_mem_cst_si<0b11, 0b10, "st1d", Z_q, ZPR128>;
+  }
 
   // contiguous store with reg+reg addressing.
   defm ST1B   : sve_mem_cst_ss<0b0000, "st1b", Z_b, ZPR8,  GPR64NoXZRshifted8>;
   defm ST1B_H : sve_mem_cst_ss<0b0001, "st1b", Z_h, ZPR16, GPR64NoXZRshifted8>;
   defm ST1B_S : sve_mem_cst_ss<0b0010, "st1b", Z_s, ZPR32, GPR64NoXZRshifted8>;
   defm ST1B_D : sve_mem_cst_ss<0b0011, "st1b", Z_d, ZPR64, GPR64NoXZRshifted8>;
   defm ST1H   : sve_mem_cst_ss<0b0101, "st1h", Z_h, ZPR16, GPR64NoXZRshifted16>;
   defm ST1H_S : sve_mem_cst_ss<0b0110, "st1h", Z_s, ZPR32, GPR64NoXZRshifted16>;
   defm ST1H_D : sve_mem_cst_ss<0b0111, "st1h", Z_d, ZPR64, GPR64NoXZRshifted16>;
   defm ST1W   : sve_mem_cst_ss<0b1010, "st1w", Z_s, ZPR32, GPR64NoXZRshifted32>;
   defm ST1W_D : sve_mem_cst_ss<0b1011, "st1w", Z_d, ZPR64, GPR64NoXZRshifted32>;
+  let Predicates = [HasSVE2p1] in {
+  defm ST1W_Q : sve_mem_cst_ss<0b1000, "st1w", Z_q, ZPR128, GPR64NoXZRshifted32>;
+  }
   defm ST1D   : sve_mem_cst_ss<0b1111, "st1d", Z_d, ZPR64, GPR64NoXZRshifted64>;
+  let Predicates = [HasSVE2p1] in {
+  defm ST1D_Q : sve_mem_cst_ss<0b1110, "st1d", Z_q, ZPR128, GPR64NoXZRshifted64>;
+  }
 } // End HasSVEorSME
 
 let Predicates = [HasSVE] in {
   // Scatters using unpacked, unscaled 32-bit offsets, e.g.
   //    st1h z0.d, p0, [x0, z0.d, uxtw]
   defm SST1B_D : sve_mem_64b_sst_sv_32_unscaled<0b000, "st1b", AArch64st1_scatter_sxtw, AArch64st1_scatter_uxtw, ZPR64ExtSXTW8Only, ZPR64ExtUXTW8Only, nxv2i8>;
   defm SST1H_D : sve_mem_64b_sst_sv_32_unscaled<0b010, "st1h", AArch64st1_scatter_sxtw, AArch64st1_scatter_uxtw, ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i16>;
   defm SST1W_D : sve_mem_64b_sst_sv_32_unscaled<0b100, "st1w", AArch64st1_scatter_sxtw, AArch64st1_scatter_uxtw, ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i32>;

llvm/lib/Target/AArch64/SVEInstrFormats.td

multiclass sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm,
   def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]",
                  (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>;
   def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]",
                   (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>;
   def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]",
                   (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;
 }
 
 class sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList,
                      string asm, Operand immtype>
 : I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4),
   asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]",
   "",
   []>, Sched<[]> {
   bits<3> Pg;
   bits<5> Rn;
   bits<5> Zt;
   bits<4> imm4;
   let Inst{31-25} = 0b1110010;
   let Inst{24-23} = sz;
   let Inst{22-21} = nregs;
   let Inst{20}    = 1;
   let Inst{19-16} = imm4;

 
 
 multiclass sve_mem_sst_128b_64_unscaled<string mnemonic> {
   def NAME : sve_mem_sst_128b_64_unscaled<mnemonic>;
 
   def : InstAlias<mnemonic # " $Zt, $Pg, [$Zn]",
                   (!cast<Instruction>(NAME) Z_q:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>;
 }
+
+
+// SVE contiguous load (quadwords, scalar plus immediate)
+class sve_mem_128b_cld_si<bits<2> dtype, string mnemonic>
+    : I<(outs Z_q:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4),
+        mnemonic, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]",
+        "", []>, Sched<[]> {
+  bits<5> Zt;
+  bits<5> Rn;
+  bits<3> Pg;
+  bits<4> imm4;
+  let Inst{31-25} = 0b1010010;
+  let Inst{24-23} = dtype;
+  let Inst{22-20} = 0b001;
+  let Inst{19-16} = imm4;
+  let Inst{15-13} = 0b001;
+  let Inst{12-10} = Pg;
+  let Inst{9-5}   = Rn;
+  let Inst{4-0}   = Zt;
+
+  let mayLoad = 1;
+}
+
+multiclass sve_mem_128b_cld_si<bits<2> dtype, string mnemonic> {
+  def NAME : sve_mem_128b_cld_si<dtype, mnemonic>;
+
+  def : InstAlias<mnemonic # " $Zt, $Pg/z, [$Rn]",
+                  (!cast<Instruction>(NAME) Z_q:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;
+  def : InstAlias<mnemonic # " $Zt, $Pg/z, [$Rn]",
+                  (!cast<Instruction>(NAME) ZPR128:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>;
+  def : InstAlias<mnemonic # " $Zt, $Pg/z, [$Rn, $imm4, mul vl]",
+                  (!cast<Instruction>(NAME) ZPR128:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>;
+}
+
+
+// SVE contiguous load (quadwords, scalar plus scalar)
+class sve_mem_128b_cld_ss<bits<2> dtype, string mnemonic, RegisterOperand gprsh_ty>
+    : I<(outs Z_q:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprsh_ty:$Rm),
+        mnemonic, "\t$Zt, $Pg/z, [$Rn, $Rm]", "",
+        []>, Sched<[]> {
+  bits<5> Zt;
+  bits<5> Rn;
+  bits<3> Pg;
+  bits<5> Rm;
+  let Inst{31-25} = 0b1010010;
+  let Inst{24-23} = dtype;
+  let Inst{22-21} = 0b00;
+  let Inst{20-16} = Rm;
+  let Inst{15-13} = 0b100;
+  let Inst{12-10} = Pg;
+  let Inst{9-5}   = Rn;
+  let Inst{4-0}   = Zt;
+
+  let mayLoad = 1;
+}
+
+multiclass sve_mem_128b_cld_ss<bits<2> dtype, string mnemonic, RegisterOperand gprsh_ty> {
+  def NAME : sve_mem_128b_cld_ss<dtype, mnemonic, gprsh_ty>;
+
+  def : InstAlias<mnemonic # " $Zt, $Pg/z, [$Rn, $Rm]",
+                 (!cast<Instruction>(NAME) ZPR128:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprsh_ty:$Rm), 0>;
+}

llvm/test/MC/AArch64/SVE2p1/ld1d_q-diagnostics.s

+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 2>&1 < %s | FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid predicate register
+
+ld1d {z0.q}, p8/z, [x0, x0, lsl #3]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: ld1d {z0.q}, p8/z, [x0, x0, lsl #3]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ld1d {z23.q}, p2/m, [x13, #-8, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: ld1d {z23.q}, p2/m, [x13, #-8, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ld1d {z23.q}, p2.q, [x13, #-8, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: ld1d {z23.q}, p2.q, [x13, #-8, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid immediate range
+
+ld1d {z0.q}, p0/z, [x0, #-9, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: index must be an integer in range [-8, 7].
+// CHECK-NEXT: ld1d {z0.q}, p0/z, [x0, #-9, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ld1d {z3.q}, p0/z, [x0, #8, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: index must be an integer in range [-8, 7].
+// CHECK-NEXT: ld1d {z3.q}, p0/z, [x0, #8, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:

llvm/test/MC/AArch64/SVE2p1/ld1d_q.s

+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p1 < %s \
+// RUN:        | llvm-objdump -d --no-print-imm-hex --mattr=+sve2p1 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p1 < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p1 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
+ld1d    {z0.q}, p0/z, [x0, x0, lsl #3]  // 10100101-10000000-10000000-00000000
+// CHECK-INST: ld1d    { z0.q }, p0/z, [x0, x0, lsl #3]
+// CHECK-ENCODING: [0x00,0x80,0x80,0xa5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: a5808000 <unknown>
+
+ld1d    {z21.q}, p5/z, [x10, x21, lsl #3]  // 10100101-10010101-10010101-01010101
+// CHECK-INST: ld1d    { z21.q }, p5/z, [x10, x21, lsl #3]
+// CHECK-ENCODING: [0x55,0x95,0x95,0xa5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: a5959555 <unknown>
+
+ld1d    {z23.q}, p3/z, [x13, x8, lsl #3]  // 10100101-10001000-10001101-10110111
+// CHECK-INST: ld1d    { z23.q }, p3/z, [x13, x8, lsl #3]
+// CHECK-ENCODING: [0xb7,0x8d,0x88,0xa5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: a5888db7 <unknown>
+
+ld1d    z23.q, p3/z, [x13, x8, lsl #3]  // 10100101-10001000-10001101-10110111
+// CHECK-INST: ld1d    { z23.q }, p3/z, [x13, x8, lsl #3]
+// CHECK-ENCODING: [0xb7,0x8d,0x88,0xa5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: a5888db7 <unknown>
+
+ld1d    {z0.q}, p0/z, [x0]  // 10100101-10010000-00100000-00000000
+// CHECK-INST: ld1d    { z0.q }, p0/z, [x0]
+// CHECK-ENCODING: [0x00,0x20,0x90,0xa5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: a5902000 <unknown>
+
+ld1d    z0.q, p0/z, [x0]  // 10100101-10010000-00100000-00000000
+// CHECK-INST: ld1d    { z0.q }, p0/z, [x0]
+// CHECK-ENCODING: [0x00,0x20,0x90,0xa5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: a5902000 <unknown>
+
+ld1d    {z21.q}, p5/z, [x10, #5, mul vl]  // 10100101-10010101-00110101-01010101
+// CHECK-INST: ld1d    { z21.q }, p5/z, [x10, #5, mul vl]
+// CHECK-ENCODING: [0x55,0x35,0x95,0xa5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: a5953555 <unknown>
+
+ld1d    {z23.q}, p3/z, [x13, #-8, mul vl]  // 10100101-10011000-00101101-10110111
+// CHECK-INST: ld1d    { z23.q }, p3/z, [x13, #-8, mul vl]
+// CHECK-ENCODING: [0xb7,0x2d,0x98,0xa5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: a5982db7 <unknown>
+
+ld1d    {z31.q}, p7/z, [sp, #-1, mul vl]  // 10100101-10011111-00111111-11111111
+// CHECK-INST: ld1d    { z31.q }, p7/z, [sp, #-1, mul vl]
+// CHECK-ENCODING: [0xff,0x3f,0x9f,0xa5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: a59f3fff <unknown>
+
+ld1d    z31.q, p7/z, [sp, #-1, mul vl]  // 10100101-10011111-00111111-11111111
+// CHECK-INST: ld1d    { z31.q }, p7/z, [sp, #-1, mul vl]
+// CHECK-ENCODING: [0xff,0x3f,0x9f,0xa5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: a59f3fff <unknown>

llvm/test/MC/AArch64/SVE2p1/ld1w_q-diagnostics.s

+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 2>&1 < %s | FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid predicate register
+
+ld1w {z0.q}, p8/z, [x0, x0, lsl #3]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: ld1w {z0.q}, p8/z, [x0, x0, lsl #3]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ld1w {z23.q}, p2/m, [x13, #-8, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: ld1w {z23.q}, p2/m, [x13, #-8, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ld1w {z23.q}, p2.q, [x13, #-8, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: ld1w {z23.q}, p2.q, [x13, #-8, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid immediate range
+
+ld1w {z0.q}, p0/z, [x0, #-9, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: index must be an integer in range [-8, 7].
+// CHECK-NEXT: ld1w {z0.q}, p0/z, [x0, #-9, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ld1w {z3.q}, p0/z, [x0, #8, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: index must be an integer in range [-8, 7].
+// CHECK-NEXT: ld1w {z3.q}, p0/z, [x0, #8, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:

llvm/test/MC/AArch64/SVE2p1/ld1w_q.s

+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p1 < %s \
+// RUN:        | llvm-objdump -d --no-print-imm-hex --mattr=+sve2p1 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p1 < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p1 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
+ld1w    {z0.q}, p0/z, [x0, x0, lsl #2]  // 10100101-00000000-10000000-00000000
+// CHECK-INST: ld1w    { z0.q }, p0/z, [x0, x0, lsl #2]
+// CHECK-ENCODING: [0x00,0x80,0x00,0xa5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: a5008000 <unknown>
+
+ld1w    {z21.q}, p5/z, [x10, x21, lsl #2]  // 10100101-00010101-10010101-01010101
+// CHECK-INST: ld1w    { z21.q }, p5/z, [x10, x21, lsl #2]
+// CHECK-ENCODING: [0x55,0x95,0x15,0xa5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: a5159555 <unknown>
+
+ld1w    {z23.q}, p3/z, [x13, x8, lsl #2]  // 10100101-00001000-10001101-10110111
+// CHECK-INST: ld1w    { z23.q }, p3/z, [x13, x8, lsl #2]
+// CHECK-ENCODING: [0xb7,0x8d,0x08,0xa5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: a5088db7 <unknown>
+
+ld1w    z23.q, p3/z, [x13, x8, lsl #2]  // 10100101-00001000-10001101-10110111
+// CHECK-INST: ld1w    { z23.q }, p3/z, [x13, x8, lsl #2]
+// CHECK-ENCODING: [0xb7,0x8d,0x08,0xa5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: a5088db7 <unknown>
+
+ld1w    {z0.q}, p0/z, [x0]  // 10100101-00010000-00100000-00000000
+// CHECK-INST: ld1w    { z0.q }, p0/z, [x0]
+// CHECK-ENCODING: [0x00,0x20,0x10,0xa5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: a5102000 <unknown>
+
+ld1w    {z21.q}, p5/z, [x10, #5, mul vl]  // 10100101-00010101-00110101-01010101
+// CHECK-INST: ld1w    { z21.q }, p5/z, [x10, #5, mul vl]
+// CHECK-ENCODING: [0x55,0x35,0x15,0xa5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: a5153555 <unknown>
+
+ld1w    {z23.q}, p3/z, [x13, #-8, mul vl]  // 10100101-00011000-00101101-10110111
+// CHECK-INST: ld1w    { z23.q }, p3/z, [x13, #-8, mul vl]
+// CHECK-ENCODING: [0xb7,0x2d,0x18,0xa5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: a5182db7 <unknown>
+
+ld1w    {z31.q}, p7/z, [sp, #-1, mul vl]  // 10100101-00011111-00111111-11111111
+// CHECK-INST: ld1w    { z31.q }, p7/z, [sp, #-1, mul vl]
+// CHECK-ENCODING: [0xff,0x3f,0x1f,0xa5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: a51f3fff <unknown>
+

llvm/test/MC/AArch64/SVE2p1/st1d_q-diagnostics.s

+-26
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 2>&1 < %s | FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid predicate register
+
+st1d {z0.q}, p8, [x0, x0, lsl #3]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: st1d {z0.q}, p8, [x0, x0, lsl #3]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+st1d {z23.q}, p2/m, [x13, #-8, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: st1d {z23.q}, p2/m, [x13, #-8, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+st1d {z23.q}, p2.q, [x13, #-8, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: st1d {z23.q}, p2.q, [x13, #-8, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid immediate range
+
+st1d {z0.q}, p0, [x0, #-9, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: index must be an integer in range [-8, 7].
+// CHECK-NEXT: st1d {z0.q}, p0, [x0, #-9, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+st1d {z3.q}, p0, [x0, #8, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: index must be an integer in range [-8, 7].
+// CHECK-NEXT: st1d {z3.q}, p0, [x0, #8, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:

llvm/test/MC/AArch64/SVE2p1/st1d_q.s

+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p1 < %s \
+// RUN:        | llvm-objdump -d --no-print-imm-hex --mattr=+sve2p1 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p1 < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p1 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
+st1d    {z0.q}, p0, [x0, x0, lsl #3]  // 11100101-11000000-01000000-00000000
+// CHECK-INST: st1d    { z0.q }, p0, [x0, x0, lsl #3]
+// CHECK-ENCODING: [0x00,0x40,0xc0,0xe5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e5c04000 <unknown>
+
+st1d    {z21.q}, p5, [x10, x21, lsl #3]  // 11100101-11010101-01010101-01010101
+// CHECK-INST: st1d    { z21.q }, p5, [x10, x21, lsl #3]
+// CHECK-ENCODING: [0x55,0x55,0xd5,0xe5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e5d55555 <unknown>
+
+st1d    {z23.q}, p3, [x13, x8, lsl #3]  // 11100101-11001000-01001101-10110111
+// CHECK-INST: st1d    { z23.q }, p3, [x13, x8, lsl #3]
+// CHECK-ENCODING: [0xb7,0x4d,0xc8,0xe5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e5c84db7 <unknown>
+
+st1d    z23.q, p3, [x13, x8, lsl #3]  // 11100101-11001000-01001101-10110111
+// CHECK-INST: st1d    { z23.q }, p3, [x13, x8, lsl #3]
+// CHECK-ENCODING: [0xb7,0x4d,0xc8,0xe5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e5c84db7 <unknown>
+
+st1d    {z0.q}, p0, [x0]  // 11100101-11000000-11100000-00000000
+// CHECK-INST: st1d    { z0.q }, p0, [x0]
+// CHECK-ENCODING: [0x00,0xe0,0xc0,0xe5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e5c0e000 <unknown>
+
+st1d    z0.q, p0, [x0]  // 11100101-11000000-11100000-00000000
+// CHECK-INST: st1d    { z0.q }, p0, [x0]
+// CHECK-ENCODING: [0x00,0xe0,0xc0,0xe5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e5c0e000 <unknown>
+
+st1d    {z21.q}, p5, [x10, #5, mul vl]  // 11100101-11000101-11110101-01010101
+// CHECK-INST: st1d    { z21.q }, p5, [x10, #5, mul vl]
+// CHECK-ENCODING: [0x55,0xf5,0xc5,0xe5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e5c5f555 <unknown>
+
+st1d    {z23.q}, p3, [x13, #-8, mul vl]  // 11100101-11001000-11101101-10110111
+// CHECK-INST: st1d    { z23.q }, p3, [x13, #-8, mul vl]
+// CHECK-ENCODING: [0xb7,0xed,0xc8,0xe5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e5c8edb7 <unknown>
+
+st1d    {z31.q}, p7, [sp, #-1, mul vl]  // 11100101-11001111-11111111-11111111
+// CHECK-INST: st1d    { z31.q }, p7, [sp, #-1, mul vl]
+// CHECK-ENCODING: [0xff,0xff,0xcf,0xe5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e5cfffff <unknown>
+
+st1d    z31.q, p7, [sp, #-1, mul vl]  // 11100101-11001111-11111111-11111111
+// CHECK-INST: st1d    { z31.q }, p7, [sp, #-1, mul vl]
+// CHECK-ENCODING: [0xff,0xff,0xcf,0xe5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e5cfffff <unknown>
+

llvm/test/MC/AArch64/SVE2p1/st1w_q-diagnostics.s

+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 2>&1 < %s | FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid predicate register
+
+st1w {z0.q}, p8, [x0, x0, lsl #3]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: st1w {z0.q}, p8, [x0, x0, lsl #3]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+st1w {z23.q}, p2/m, [x13, #-8, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: st1w {z23.q}, p2/m, [x13, #-8, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+st1w {z23.q}, p2.q, [x13, #-8, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: st1w {z23.q}, p2.q, [x13, #-8, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid immediate range
+
+st1w {z0.q}, p0, [x0, #-9, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: index must be an integer in range [-8, 7].
+// CHECK-NEXT: st1w {z0.q}, p0, [x0, #-9, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+st1w {z3.q}, p0, [x0, #8, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: index must be an integer in range [-8, 7].
+// CHECK-NEXT: st1w {z3.q}, p0, [x0, #8, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:

llvm/test/MC/AArch64/SVE2p1/st1w_q.s

+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p1 < %s \
+// RUN:        | llvm-objdump -d --no-print-imm-hex --mattr=+sve2p1 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2p1 < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sve2p1 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
+st1w    {z0.q}, p0, [x0, x0, lsl #2]  // 11100101-00000000-01000000-00000000
+// CHECK-INST: st1w    { z0.q }, p0, [x0, x0, lsl #2]
+// CHECK-ENCODING: [0x00,0x40,0x00,0xe5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e5004000 <unknown>
+
+st1w    {z21.q}, p5, [x10, x21, lsl #2]  // 11100101-00010101-01010101-01010101
+// CHECK-INST: st1w    { z21.q }, p5, [x10, x21, lsl #2]
+// CHECK-ENCODING: [0x55,0x55,0x15,0xe5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e5155555 <unknown>
+
+st1w    {z23.q}, p3, [x13, x8, lsl #2]  // 11100101-00001000-01001101-10110111
+// CHECK-INST: st1w    { z23.q }, p3, [x13, x8, lsl #2]
+// CHECK-ENCODING: [0xb7,0x4d,0x08,0xe5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e5084db7 <unknown>
+
+st1w    z23.q, p3, [x13, x8, lsl #2]  // 11100101-00001000-01001101-10110111
+// CHECK-INST: st1w    { z23.q }, p3, [x13, x8, lsl #2]
+// CHECK-ENCODING: [0xb7,0x4d,0x08,0xe5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e5084db7 <unknown>
+
+st1w    {z0.q}, p0, [x0]  // 11100101-00000000-11100000-00000000
+// CHECK-INST: st1w    { z0.q }, p0, [x0]
+// CHECK-ENCODING: [0x00,0xe0,0x00,0xe5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e500e000 <unknown>
+
+st1w    z0.q, p0, [x0]  // 11100101-00000000-11100000-00000000
+// CHECK-INST: st1w    { z0.q }, p0, [x0]
+// CHECK-ENCODING: [0x00,0xe0,0x00,0xe5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e500e000 <unknown>
+
+st1w    {z21.q}, p5, [x10, #5, mul vl]  // 11100101-00000101-11110101-01010101
+// CHECK-INST: st1w    { z21.q }, p5, [x10, #5, mul vl]
+// CHECK-ENCODING: [0x55,0xf5,0x05,0xe5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e505f555 <unknown>
+
+st1w    {z23.q}, p3, [x13, #-8, mul vl]  // 11100101-00001000-11101101-10110111
+// CHECK-INST: st1w    { z23.q }, p3, [x13, #-8, mul vl]
+// CHECK-ENCODING: [0xb7,0xed,0x08,0xe5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e508edb7 <unknown>
+
+st1w    {z31.q}, p7, [sp, #-1, mul vl]  // 11100101-00001111-11111111-11111111
+// CHECK-INST: st1w    { z31.q }, p7, [sp, #-1, mul vl]
+// CHECK-ENCODING: [0xff,0xff,0x0f,0xe5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e50fffff <unknown>
+
+st1w    z31.q, p7, [sp, #-1, mul vl]  // 11100101-00001111-11111111-11111111
+// CHECK-INST: st1w    { z31.q }, p7, [sp, #-1, mul vl]
+// CHECK-ENCODING: [0xff,0xff,0x0f,0xe5]
+// CHECK-ERROR: instruction requires: sve2p1
+// CHECK-UNKNOWN: e50fffff <unknown>
+