[AArch64][SVE2] Add the SVE2.1 quadword structured load/store instructions

[david-arm]

Summary

This patch adds the assembly/disassembly for the following instructions:

ld2q : Contiguous load two-quadword structures to two vectors
ld3q : Contiguous load three-quadword structures to three vectors
ld4q : Contiguous load four-quadword structures to four vectors
st2q : Contiguous store two-quadword structures from two vectors
st3q : Contiguous store three-quadword structures to three vectors
st4q : Contiguous store four-quadword structures to four vectors

The reference can be found here:
https://developer.arm.com/documentation/ddi0602/2022-09

[david-arm]

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

1026–1042

I've tried to reuse the existing class by passing in an extra 'q' parameter, but the sz field changes meaning for quadword ops and in fact becomes nregs.

1462

It's difficult to reuse the existing sve_mem_est_si and sve_mem_est_ss classes because the sz/nregs fields are shifted, i.e.

sve_mem_est_si:

let Inst{24-23} = sz;
let Inst{22-21} = nregs;

sve_mem_128b_est_si:

let Inst{23-22} = nregs;
let Inst{21-20} = 0b00;

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

1462

Agreed. The new instructions sit within a different part of the encoding tables anyway and so having new instructions classes is the correct play.

llvm/lib/Target/AArch64/SVEInstrFormats.td

5813

Please add a comment for the encoding group.

5867

Please add a comment for the encoding group.

5921

typo?

7131–7133

You're being overly literal regarding the name of these parameters. They were likely picked to match the original encoding groups but looking at "SVE load multiple structures (scalar plus immediate)" they now just call {22-21} opc.

It's better for the variables to always be placed in the same part of the instruction. It doesn't really matter if the names don't exactly match. In this instance I think it'll be better to replace nregs and q with a 3-bit opc. You may as well keep sz even though it means something different for the quadword variants, because it matters not. However, if you don't like keeping sz then just make opc a 5-bit opcode instead.

7163–7164

As above.

[david-arm]

llvm/lib/Target/AArch64/SVEInstrFormats.td

7131–7133

The main reason I chose to write it like this is because otherwise the list of instructions in AArch64SVEInstrInfo.td just looks weird/inconsistent, i.e.

defm LD2D_IMM : sve_mem_eld_si<0b11, 0b001, ZZ_d,   "ld2d", simm4s2>;
defm LD3D_IMM : sve_mem_eld_si<0b11, 0b010, ZZZ_d,  "ld3d", simm4s3>;
defm LD4D_IMM : sve_mem_eld_si<0b11, 0b011, ZZZZ_d, "ld4d", simm4s4>;
let Predicates = [HasSVE2p1_or_HasSME2p1] in {
defm LD2Q_IMM : sve_mem_eld_si<0b01, 0b100, ZZ_q,   "ld2q", simm4s2>;
defm LD3Q_IMM : sve_mem_eld_si<0b10, 0b100, ZZZ_q,  "ld3q", simm4s3>;
defm LD4Q_IMM : sve_mem_eld_si<0b11, 0b100, ZZZZ_q, "ld4q", simm4s4>;
}

because the quadword variants don't follow the pattern of the ones above. I guess it's a choice between having a potentially confusing instruction class format, or a confusing list of instructions. I don't really mind though or feel strongly about it - happy to go with the latter!

[paulwalker-arm]

llvm/lib/Target/AArch64/SVEInstrFormats.td

7131–7133

In this instance my preference is simpler instruction classes. I don't find the above a confusing list of instructions, but then I know the bit patterns are arbitrary and so don't attach meaning to them :)

[david-arm]

• Refactored load classes.

[paulwalker-arm]

llvm/lib/Target/AArch64/SVEInstrFormats.td

7131–7133

Personally I would have kept it simple with Inst{22-20} = nregs, so that everything reads left to right, but there's nothing here I can't live with.

llvm/lib/Target/AArch64/AArch64RegisterInfo.td

 def ZZ_s  : RegisterOperand<ZPR2, "printTypedVectorList<0,'s'>"> {
   let ParserMatchClass = ZPRVectorList<32, 2>;
 }
 
 def ZZ_d  : RegisterOperand<ZPR2, "printTypedVectorList<0,'d'>"> {
   let ParserMatchClass = ZPRVectorList<64, 2>;
 }
 
+def ZZ_q  : RegisterOperand<ZPR2, "printTypedVectorList<0,'q'>"> {
+  let ParserMatchClass = ZPRVectorList<128, 2>;
+}
+
 def ZZZ_b  : RegisterOperand<ZPR3, "printTypedVectorList<0,'b'>"> {
   let ParserMatchClass = ZPRVectorList<8, 3>;
 }
 
 def ZZZ_h  : RegisterOperand<ZPR3, "printTypedVectorList<0,'h'>"> {
   let ParserMatchClass = ZPRVectorList<16, 3>;
 }
 
 def ZZZ_s  : RegisterOperand<ZPR3, "printTypedVectorList<0,'s'>"> {
   let ParserMatchClass = ZPRVectorList<32, 3>;
 }
 
 def ZZZ_d  : RegisterOperand<ZPR3, "printTypedVectorList<0,'d'>"> {
   let ParserMatchClass = ZPRVectorList<64, 3>;
 }
 
+def ZZZ_q  : RegisterOperand<ZPR3, "printTypedVectorList<0,'q'>"> {
+  let ParserMatchClass = ZPRVectorList<128, 3>;
+}
+
 def ZZZZ_b : RegisterOperand<ZPR4, "printTypedVectorList<0,'b'>"> {
   let ParserMatchClass = ZPRVectorList<8, 4>;
 }
 
 def ZZZZ_h : RegisterOperand<ZPR4, "printTypedVectorList<0,'h'>"> {
   let ParserMatchClass = ZPRVectorList<16, 4>;
 }
 
 def ZZZZ_s : RegisterOperand<ZPR4, "printTypedVectorList<0,'s'>"> {
   let ParserMatchClass = ZPRVectorList<32, 4>;
 }
 
 def ZZZZ_d : RegisterOperand<ZPR4, "printTypedVectorList<0,'d'>"> {
   let ParserMatchClass = ZPRVectorList<64, 4>;
 }
 
+def ZZZZ_q : RegisterOperand<ZPR4, "printTypedVectorList<0,'q'>"> {
+  let ParserMatchClass = ZPRVectorList<128, 4>;
+}
+
 // SME2 multiple-of-2 or 4 multi-vector operands
 def ZPR2Mul2 : RegisterClass<"AArch64", [untyped], 128, (add (decimate ZSeqPairs, 2))> {
   let Size = 256;
 }
 
 def ZPR4Mul4 : RegisterClass<"AArch64", [untyped], 128, (add (decimate ZSeqQuads, 4))> {
   let Size = 512;
 }

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

let Predicates = [HasSVE] in {
   defm LDFF1SB_D : sve_mem_cldff_ss<0b1100, "ldff1sb", Z_d, ZPR64, GPR64shifted8>;
   defm LDFF1SB_S : sve_mem_cldff_ss<0b1101, "ldff1sb", Z_s, ZPR32, GPR64shifted8>;
   defm LDFF1SB_H : sve_mem_cldff_ss<0b1110, "ldff1sb", Z_h, ZPR16, GPR64shifted8>;
   defm LDFF1D    : sve_mem_cldff_ss<0b1111, "ldff1d",  Z_d, ZPR64, GPR64shifted64>;
 } // End HasSVE
 
 let Predicates = [HasSVEorSME] in {
   // LD(2|3|4) structured loads with reg+immediate
-  defm LD2B_IMM : sve_mem_eld_si<0b00, 0b01, ZZ_b,   "ld2b", simm4s2>;
-  defm LD3B_IMM : sve_mem_eld_si<0b00, 0b10, ZZZ_b,  "ld3b", simm4s3>;
-  defm LD4B_IMM : sve_mem_eld_si<0b00, 0b11, ZZZZ_b, "ld4b", simm4s4>;
-  defm LD2H_IMM : sve_mem_eld_si<0b01, 0b01, ZZ_h,   "ld2h", simm4s2>;
-  defm LD3H_IMM : sve_mem_eld_si<0b01, 0b10, ZZZ_h,  "ld3h", simm4s3>;
-  defm LD4H_IMM : sve_mem_eld_si<0b01, 0b11, ZZZZ_h, "ld4h", simm4s4>;
-  defm LD2W_IMM : sve_mem_eld_si<0b10, 0b01, ZZ_s,   "ld2w", simm4s2>;
-  defm LD3W_IMM : sve_mem_eld_si<0b10, 0b10, ZZZ_s,  "ld3w", simm4s3>;
-  defm LD4W_IMM : sve_mem_eld_si<0b10, 0b11, ZZZZ_s, "ld4w", simm4s4>;
-  defm LD2D_IMM : sve_mem_eld_si<0b11, 0b01, ZZ_d,   "ld2d", simm4s2>;
-  defm LD3D_IMM : sve_mem_eld_si<0b11, 0b10, ZZZ_d,  "ld3d", simm4s3>;
-  defm LD4D_IMM : sve_mem_eld_si<0b11, 0b11, ZZZZ_d, "ld4d", simm4s4>;
+  defm LD2B_IMM : sve_mem_eld_si<0b00, 0b001, ZZ_b,   "ld2b", simm4s2>;
+  defm LD3B_IMM : sve_mem_eld_si<0b00, 0b010, ZZZ_b,  "ld3b", simm4s3>;
+  defm LD4B_IMM : sve_mem_eld_si<0b00, 0b011, ZZZZ_b, "ld4b", simm4s4>;
+  defm LD2H_IMM : sve_mem_eld_si<0b01, 0b001, ZZ_h,   "ld2h", simm4s2>;
+  defm LD3H_IMM : sve_mem_eld_si<0b01, 0b010, ZZZ_h,  "ld3h", simm4s3>;
+  defm LD4H_IMM : sve_mem_eld_si<0b01, 0b011, ZZZZ_h, "ld4h", simm4s4>;
+  defm LD2W_IMM : sve_mem_eld_si<0b10, 0b001, ZZ_s,   "ld2w", simm4s2>;
+  defm LD3W_IMM : sve_mem_eld_si<0b10, 0b010, ZZZ_s,  "ld3w", simm4s3>;
+  defm LD4W_IMM : sve_mem_eld_si<0b10, 0b011, ZZZZ_s, "ld4w", simm4s4>;
+  defm LD2D_IMM : sve_mem_eld_si<0b11, 0b001, ZZ_d,   "ld2d", simm4s2>;
+  defm LD3D_IMM : sve_mem_eld_si<0b11, 0b010, ZZZ_d,  "ld3d", simm4s3>;
+  defm LD4D_IMM : sve_mem_eld_si<0b11, 0b011, ZZZZ_d, "ld4d", simm4s4>;
+  let Predicates = [HasSVE2p1_or_HasSME2p1] in {
+  defm LD2Q_IMM : sve_mem_eld_si<0b01, 0b100, ZZ_q,   "ld2q", simm4s2>;
+  defm LD3Q_IMM : sve_mem_eld_si<0b10, 0b100, ZZZ_q,  "ld3q", simm4s3>;
+  defm LD4Q_IMM : sve_mem_eld_si<0b11, 0b100, ZZZZ_q, "ld4q", simm4s4>;
+  }
 
   // LD(2|3|4) structured loads (register + register)
-  def LD2B : sve_mem_eld_ss<0b00, 0b01, ZZ_b,   "ld2b", GPR64NoXZRshifted8>;
-  def LD3B : sve_mem_eld_ss<0b00, 0b10, ZZZ_b,  "ld3b", GPR64NoXZRshifted8>;
-  def LD4B : sve_mem_eld_ss<0b00, 0b11, ZZZZ_b, "ld4b", GPR64NoXZRshifted8>;
-  def LD2H : sve_mem_eld_ss<0b01, 0b01, ZZ_h,   "ld2h", GPR64NoXZRshifted16>;
-  def LD3H : sve_mem_eld_ss<0b01, 0b10, ZZZ_h,  "ld3h", GPR64NoXZRshifted16>;
-  def LD4H : sve_mem_eld_ss<0b01, 0b11, ZZZZ_h, "ld4h", GPR64NoXZRshifted16>;
-  def LD2W : sve_mem_eld_ss<0b10, 0b01, ZZ_s,   "ld2w", GPR64NoXZRshifted32>;
-  def LD3W : sve_mem_eld_ss<0b10, 0b10, ZZZ_s,  "ld3w", GPR64NoXZRshifted32>;
-  def LD4W : sve_mem_eld_ss<0b10, 0b11, ZZZZ_s, "ld4w", GPR64NoXZRshifted32>;
-  def LD2D : sve_mem_eld_ss<0b11, 0b01, ZZ_d,   "ld2d", GPR64NoXZRshifted64>;
-  def LD3D : sve_mem_eld_ss<0b11, 0b10, ZZZ_d,  "ld3d", GPR64NoXZRshifted64>;
-  def LD4D : sve_mem_eld_ss<0b11, 0b11, ZZZZ_d, "ld4d", GPR64NoXZRshifted64>;
+  def LD2B : sve_mem_eld_ss<0b00, 0b101, ZZ_b,   "ld2b", GPR64NoXZRshifted8>;
+  def LD3B : sve_mem_eld_ss<0b00, 0b110, ZZZ_b,  "ld3b", GPR64NoXZRshifted8>;
+  def LD4B : sve_mem_eld_ss<0b00, 0b111, ZZZZ_b, "ld4b", GPR64NoXZRshifted8>;
+  def LD2H : sve_mem_eld_ss<0b01, 0b101, ZZ_h,   "ld2h", GPR64NoXZRshifted16>;
+  def LD3H : sve_mem_eld_ss<0b01, 0b110, ZZZ_h,  "ld3h", GPR64NoXZRshifted16>;
+  def LD4H : sve_mem_eld_ss<0b01, 0b111, ZZZZ_h, "ld4h", GPR64NoXZRshifted16>;
+  def LD2W : sve_mem_eld_ss<0b10, 0b101, ZZ_s,   "ld2w", GPR64NoXZRshifted32>;
+  def LD3W : sve_mem_eld_ss<0b10, 0b110, ZZZ_s,  "ld3w", GPR64NoXZRshifted32>;
+  def LD4W : sve_mem_eld_ss<0b10, 0b111, ZZZZ_s, "ld4w", GPR64NoXZRshifted32>;
+  def LD2D : sve_mem_eld_ss<0b11, 0b101, ZZ_d,   "ld2d", GPR64NoXZRshifted64>;
+  def LD3D : sve_mem_eld_ss<0b11, 0b110, ZZZ_d,  "ld3d", GPR64NoXZRshifted64>;
+  def LD4D : sve_mem_eld_ss<0b11, 0b111, ZZZZ_d, "ld4d", GPR64NoXZRshifted64>;
+  let Predicates = [HasSVE2p1_or_HasSME2p1] in {
+  def LD2Q : sve_mem_eld_ss<0b01, 0b001, ZZ_q,   "ld2q", GPR64NoXZRshifted128>;
+  def LD3Q : sve_mem_eld_ss<0b10, 0b001, ZZZ_q,  "ld3q", GPR64NoXZRshifted128>;
+  def LD4Q : sve_mem_eld_ss<0b11, 0b001, ZZZZ_q, "ld4q", GPR64NoXZRshifted128>;
+  }
 } // End HasSVEorSME
 
 let Predicates = [HasSVE] in {
   // Gathers using unscaled 32-bit offsets, e.g.
   //    ld1h z0.s, p0/z, [x0, z0.s, uxtw]
   defm GLD1SB_S   : sve_mem_32b_gld_vs_32_unscaled<0b0000, "ld1sb",   AArch64ld1s_gather_sxtw_z,   AArch64ld1s_gather_uxtw_z,   ZPR32ExtSXTW8Only, ZPR32ExtUXTW8Only, nxv4i8>;
   defm GLDFF1SB_S : sve_mem_32b_gld_vs_32_unscaled<0b0001, "ldff1sb", AArch64ldff1s_gather_sxtw_z, AArch64ldff1s_gather_uxtw_z, ZPR32ExtSXTW8Only, ZPR32ExtUXTW8Only, nxv4i8>;
   defm GLD1B_S    : sve_mem_32b_gld_vs_32_unscaled<0b0010, "ld1b",    AArch64ld1_gather_sxtw_z,    AArch64ld1_gather_uxtw_z,    ZPR32ExtSXTW8Only, ZPR32ExtUXTW8Only, nxv4i8>;
let Predicates = [HasSVEorSME] in {
   defm ST3H_IMM : sve_mem_est_si<0b01, 0b10, ZZZ_h,  "st3h", simm4s3>;
   defm ST4H_IMM : sve_mem_est_si<0b01, 0b11, ZZZZ_h, "st4h", simm4s4>;
   defm ST2W_IMM : sve_mem_est_si<0b10, 0b01, ZZ_s,   "st2w", simm4s2>;
   defm ST3W_IMM : sve_mem_est_si<0b10, 0b10, ZZZ_s,  "st3w", simm4s3>;
   defm ST4W_IMM : sve_mem_est_si<0b10, 0b11, ZZZZ_s, "st4w", simm4s4>;
   defm ST2D_IMM : sve_mem_est_si<0b11, 0b01, ZZ_d,   "st2d", simm4s2>;
   defm ST3D_IMM : sve_mem_est_si<0b11, 0b10, ZZZ_d,  "st3d", simm4s3>;
   defm ST4D_IMM : sve_mem_est_si<0b11, 0b11, ZZZZ_d, "st4d", simm4s4>;
+  let Predicates = [HasSVE2p1_or_HasSME2p1] in {
+  defm ST2Q_IMM : sve_mem_128b_est_si<0b01, ZZ_q,    "st2q", simm4s2>;
+  defm ST3Q_IMM : sve_mem_128b_est_si<0b10, ZZZ_q,   "st3q", simm4s3>;
+  defm ST4Q_IMM : sve_mem_128b_est_si<0b11, ZZZZ_q,  "st4q", simm4s4>;
+  }
 
   // ST(2|3|4) structured stores (register + register)
   def ST2B : sve_mem_est_ss<0b00, 0b01, ZZ_b,   "st2b", GPR64NoXZRshifted8>;
   def ST3B : sve_mem_est_ss<0b00, 0b10, ZZZ_b,  "st3b", GPR64NoXZRshifted8>;
   def ST4B : sve_mem_est_ss<0b00, 0b11, ZZZZ_b, "st4b", GPR64NoXZRshifted8>;
   def ST2H : sve_mem_est_ss<0b01, 0b01, ZZ_h,   "st2h", GPR64NoXZRshifted16>;
   def ST3H : sve_mem_est_ss<0b01, 0b10, ZZZ_h,  "st3h", GPR64NoXZRshifted16>;
   def ST4H : sve_mem_est_ss<0b01, 0b11, ZZZZ_h, "st4h", GPR64NoXZRshifted16>;
   def ST2W : sve_mem_est_ss<0b10, 0b01, ZZ_s,   "st2w", GPR64NoXZRshifted32>;
   def ST3W : sve_mem_est_ss<0b10, 0b10, ZZZ_s,  "st3w", GPR64NoXZRshifted32>;
   def ST4W : sve_mem_est_ss<0b10, 0b11, ZZZZ_s, "st4w", GPR64NoXZRshifted32>;
   def ST2D : sve_mem_est_ss<0b11, 0b01, ZZ_d,   "st2d", GPR64NoXZRshifted64>;
   def ST3D : sve_mem_est_ss<0b11, 0b10, ZZZ_d,  "st3d", GPR64NoXZRshifted64>;
   def ST4D : sve_mem_est_ss<0b11, 0b11, ZZZZ_d, "st4d", GPR64NoXZRshifted64>;
-
+  let Predicates = [HasSVE2p1_or_HasSME2p1] in {
+  def ST2Q : sve_mem_128b_est_ss<0b01, ZZ_q,    "st2q", GPR64NoXZRshifted128>;
+  def ST3Q : sve_mem_128b_est_ss<0b10, ZZZ_q,   "st3q", GPR64NoXZRshifted128>;
+  def ST4Q : sve_mem_128b_est_ss<0b11, ZZZZ_q,  "st4q", GPR64NoXZRshifted128>;
+  }
   // Non-temporal contiguous stores (register + immediate)
   defm STNT1B_ZRI : sve_mem_cstnt_si<0b00, "stnt1b", Z_b, ZPR8>;
   defm STNT1H_ZRI : sve_mem_cstnt_si<0b01, "stnt1h", Z_h, ZPR16>;
   defm STNT1W_ZRI : sve_mem_cstnt_si<0b10, "stnt1w", Z_s, ZPR32>;
   defm STNT1D_ZRI : sve_mem_cstnt_si<0b11, "stnt1d", Z_d, ZPR64>;
 
   // Non-temporal contiguous stores (register + register)
   defm STNT1B_ZRR : sve_mem_cstnt_ss<0b00, "stnt1b", Z_b, ZPR8, GPR64NoXZRshifted8>;

llvm/lib/Target/AArch64/SVEInstrFormats.td

 multiclass sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList,
                           string asm, Operand immtype> {
   def NAME : sve_mem_est_si<sz, nregs, VecList, asm, immtype>;
 
   def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]",
                   (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;
 }
 
+
+// SVE store multiple structures (quadwords, scalar plus immediate)
+class sve_mem_128b_est_si<bits<2> nregs, RegisterOperand VecList,
+                          string asm, Operand immtype>
+    : I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4),
+        asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]",
+        "", []>, Sched<[]> {
+  bits<5> Zt;
+  bits<5> Rn;
+  bits<3> Pg;
+  bits<4> imm4;
+  let Inst{31-24} = 0b11100100;
+  let Inst{23-22} = nregs;
+  let Inst{21-20} = 0b00;
+  let Inst{19-16} = imm4;
+  let Inst{15-13} = 0b000;
+  let Inst{12-10} = Pg;
+  let Inst{9-5}   = Rn;
+  let Inst{4-0}   = Zt;
+
+  let mayStore = 1;
+}
+
+multiclass sve_mem_128b_est_si<bits<2> nregs, RegisterOperand VecList,
+                               string asm, Operand immtype> {
+  def NAME : sve_mem_128b_est_si<nregs, VecList, asm, immtype>;
+
+  def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]",
+                  (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;
+}
+
+
 class sve_mem_est_ss<bits<2> sz, bits<2> nregs, RegisterOperand VecList,
                      string asm, RegisterOperand gprty>
 : I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
   asm, "\t$Zt, $Pg, [$Rn, $Rm]",
   "",
   []>, Sched<[]> {
   bits<3> Pg;
   bits<5> Rm;
   bits<5> Rn;
   bits<5> Zt;
   let Inst{31-25} = 0b1110010;
   let Inst{24-23} = sz;
   let Inst{22-21} = nregs;
   let Inst{20-16} = Rm;
   let Inst{15-13} = 0b011;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
   let mayStore = 1;
 }
 
+
+// SVE store multiple structures (quadwords, scalar plus scalar)
+class sve_mem_128b_est_ss<bits<2> nregs, RegisterOperand VecList,
+                          string asm, RegisterOperand gprty>
+    : I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
+        asm, "\t$Zt, $Pg, [$Rn, $Rm]",
+        "", []>, Sched<[]> {
+  bits<5> Zt;
+  bits<5> Rn;
+  bits<3> Pg;
+  bits<5> Rm;
+  let Inst{31-24} = 0b11100100;
+  let Inst{23-22} = nregs;
+  let Inst{21}    = 0b1;
+  let Inst{20-16} = Rm;
+  let Inst{15-13} = 0b000;
+  let Inst{12-10} = Pg;
+  let Inst{9-5}   = Rn;
+  let Inst{4-0}   = Zt;
+
+  let mayStore = 1;
+}
+
+
 class sve_mem_cst_ss_base<bits<4> dtype, string asm,
                           RegisterOperand listty, RegisterOperand gprty>
 : I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
   asm, "\t$Zt, $Pg, [$Rn, $Rm]",
   "",
   []>, Sched<[]> {
   bits<3> Pg;
   bits<5> Rm;
multiclass sve_mem_cst_ss<bits<4> dtype, string asm,
                           RegisterOperand listty, ZPRRegOp zprty,
                           RegisterOperand gprty> {
   def NAME : sve_mem_cst_ss_base<dtype, asm, listty, gprty>;
 
   def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]",
                   (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>;
 }
 
 class sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand VecList>
 : I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4),
   asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]",
   "",
   []>, Sched<[]> {
   bits<3> Pg;
   bits<5> Rn;
   bits<5> Zt;
   bits<4> imm4;
def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), []>,
            PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm)>;
   }
 }
 
 multiclass sve_mem_cldnf_si<bits<4> dtype, string asm, RegisterOperand listty,
                             ZPRRegOp zprty>
 : sve_mem_cld_si_base<dtype, 1, asm, listty, zprty>;
 
-class sve_mem_eld_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList,
+class sve_mem_eld_si<bits<2> sz, bits<3> nregs, RegisterOperand VecList,
                      string asm, Operand immtype>
 : I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4),
   asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]",
   "",
   []>, Sched<[]> {
   bits<5> Zt;
   bits<3> Pg;
   bits<5> Rn;
   bits<4> imm4;
   let Inst{31-25} = 0b1010010;
   let Inst{24-23} = sz;
-  let Inst{22-21} = nregs;
-  let Inst{20}    = 0;
+  let Inst{22-21} = nregs{1-0};
+  let Inst{20}    = nregs{2};
   let Inst{19-16} = imm4;
   let Inst{15-13} = 0b111;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
   let mayLoad = 1;
 }
 
-multiclass sve_mem_eld_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList,
+multiclass sve_mem_eld_si<bits<2> sz, bits<3> nregs, RegisterOperand VecList,
                           string asm, Operand immtype> {
   def NAME : sve_mem_eld_si<sz, nregs, VecList, asm, immtype>;
 
   def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
                   (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;
 }
 
-class sve_mem_eld_ss<bits<2> sz, bits<2> nregs, RegisterOperand VecList,
+
+class sve_mem_eld_ss<bits<2> sz, bits<3> nregs, RegisterOperand VecList,
                      string asm, RegisterOperand gprty>
 : I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
   asm, "\t$Zt, $Pg/z, [$Rn, $Rm]",
   "",
   []>, Sched<[]> {
   bits<3> Pg;
   bits<5> Rm;
   bits<5> Rn;
   bits<5> Zt;
   let Inst{31-25} = 0b1010010;
   let Inst{24-23} = sz;
-  let Inst{22-21} = nregs;
+  let Inst{22-21} = nregs{1-0};
   let Inst{20-16} = Rm;
-  let Inst{15-13} = 0b110;
+  let Inst{15}    = 0b1;
+  let Inst{14}    = nregs{2};
+  let Inst{13}    = 0b0;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
   let mayLoad = 1;
 }
 
 //===----------------------------------------------------------------------===//

llvm/test/MC/AArch64/SVE2p1/ld2q-diagnostics.s

+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 2>&1 < %s | FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid predicate register
+
+ld2q {z0.q, z1.q}, p8/z, [z0.d, x0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: ld2q {z0.q, z1.q}, p8/z, [z0.d, x0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ld2q {z23.q, z24.q}, p2/m, [x0, x0, lsl #4]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: ld2q {z23.q, z24.q}, p2/m, [x0, x0, lsl #4]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ld2q {z21.q, z22.q}, p2.q, [x10, x21, lsl #4]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: ld2q {z21.q, z22.q}, p2.q, [x10, x21, lsl #4]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid immediate offset
+
+ld2q {z23.q, z24.q}, p3/z, [x13, #-17, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: index must be a multiple of 2 in range [-16, 14].
+// CHECK-NEXT: ld2q {z23.q, z24.q}, p3/z, [x13, #-17, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ld2q {z23.q, z24.q}, p3/z, [x13, #15, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: index must be a multiple of 2 in range [-16, 14].
+// CHECK-NEXT: ld2q {z23.q, z24.q}, p3/z, [x13, #15, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:

llvm/test/MC/AArch64/SVE2p1/ld2q.s

+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p1 < %s \
+// RUN:        | llvm-objdump -d --no-print-imm-hex --mattr=+sme2p1 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p1 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2p1,-sve2p1 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p1 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sme2p1 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+ld2q    {z0.q, z1.q}, p0/z, [x0, x0, lsl #4]  // 10100100-10100000-10000000-00000000
+// CHECK-INST: ld2q    { z0.q, z1.q }, p0/z, [x0, x0, lsl #4]
+// CHECK-ENCODING: [0x00,0x80,0xa0,0xa4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a4a08000 <unknown>
+
+ld2q    {z21.q, z22.q}, p5/z, [x10, x21, lsl #4]  // 10100100-10110101-10010101-01010101
+// CHECK-INST: ld2q    { z21.q, z22.q }, p5/z, [x10, x21, lsl #4]
+// CHECK-ENCODING: [0x55,0x95,0xb5,0xa4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a4b59555 <unknown>
+
+ld2q    {z23.q, z24.q}, p3/z, [x13, x8, lsl #4]  // 10100100-10101000-10001101-10110111
+// CHECK-INST: ld2q    { z23.q, z24.q }, p3/z, [x13, x8, lsl #4]
+// CHECK-ENCODING: [0xb7,0x8d,0xa8,0xa4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a4a88db7 <unknown>
+
+ld2q    {z0.q, z1.q}, p0/z, [x0]  // 10100100-10010000-11100000-00000000
+// CHECK-INST: ld2q    { z0.q, z1.q }, p0/z, [x0]
+// CHECK-ENCODING: [0x00,0xe0,0x90,0xa4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a490e000 <unknown>
+
+ld2q    {z21.q, z22.q}, p5/z, [x10, #10, mul vl]  // 10100100-10010101-11110101-01010101
+// CHECK-INST: ld2q    { z21.q, z22.q }, p5/z, [x10, #10, mul vl]
+// CHECK-ENCODING: [0x55,0xf5,0x95,0xa4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a495f555 <unknown>
+
+ld2q    {z23.q, z24.q}, p3/z, [x13, #-16, mul vl]  // 10100100-10011000-11101101-10110111
+// CHECK-INST: ld2q    { z23.q, z24.q }, p3/z, [x13, #-16, mul vl]
+// CHECK-ENCODING: [0xb7,0xed,0x98,0xa4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a498edb7 <unknown>
+
+ld2q    {z31.q, z0.q}, p7/z, [sp, #-2, mul vl]  // 10100100-10011111-11111111-11111111
+// CHECK-INST: ld2q    { z31.q, z0.q }, p7/z, [sp, #-2, mul vl]
+// CHECK-ENCODING: [0xff,0xff,0x9f,0xa4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a49fffff <unknown>

llvm/test/MC/AArch64/SVE2p1/ld3q-diagnostics.s

+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 2>&1 < %s | FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid predicate register
+
+ld3q {z0.q, z1.q, z2.q}, p8/z, [z0.d, x0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: ld3q {z0.q, z1.q, z2.q}, p8/z, [z0.d, x0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ld3q {z23.q, z24.q, z25.q}, p2/m, [x0, x0, lsl #4]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: ld3q {z23.q, z24.q, z25.q}, p2/m, [x0, x0, lsl #4]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ld3q {z21.q, z22.q, z23.q}, p2.q, [x10, x21, lsl #4]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: ld3q {z21.q, z22.q, z23.q}, p2.q, [x10, x21, lsl #4]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid immediate offset
+
+ld3q {z23.q, z24.q, z25.q}, p3/z, [x13, #-25, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: index must be a multiple of 3 in range [-24, 21].
+// CHECK-NEXT: ld3q {z23.q, z24.q, z25.q}, p3/z, [x13, #-25, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ld3q {z23.q, z24.q, z25.q}, p3/z, [x13, #22, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: index must be a multiple of 3 in range [-24, 21].
+// CHECK-NEXT: ld3q {z23.q, z24.q, z25.q}, p3/z, [x13, #22, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:

llvm/test/MC/AArch64/SVE2p1/ld3q.s

+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p1 < %s \
+// RUN:        | llvm-objdump -d --no-print-imm-hex --mattr=+sme2p1 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p1 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2p1,-sve2p1 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p1 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sme2p1 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+ld3q    {z0.q, z1.q, z2.q}, p0/z, [x0, x0, lsl #4]  // 10100101-00100000-10000000-00000000
+// CHECK-INST: ld3q    { z0.q - z2.q }, p0/z, [x0, x0, lsl #4]
+// CHECK-ENCODING: [0x00,0x80,0x20,0xa5]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a5208000 <unknown>
+
+ld3q    {z21.q, z22.q, z23.q}, p5/z, [x10, x21, lsl #4]  // 10100101-00110101-10010101-01010101
+// CHECK-INST: ld3q    { z21.q - z23.q }, p5/z, [x10, x21, lsl #4]
+// CHECK-ENCODING: [0x55,0x95,0x35,0xa5]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a5359555 <unknown>
+
+ld3q    {z23.q, z24.q, z25.q}, p3/z, [x13, x8, lsl #4]  // 10100101-00101000-10001101-10110111
+// CHECK-INST: ld3q    { z23.q - z25.q }, p3/z, [x13, x8, lsl #4]
+// CHECK-ENCODING: [0xb7,0x8d,0x28,0xa5]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a5288db7 <unknown>
+
+ld3q    {z0.q, z1.q, z2.q}, p0/z, [x0]  // 10100101-00010000-11100000-00000000
+// CHECK-INST: ld3q    { z0.q - z2.q }, p0/z, [x0]
+// CHECK-ENCODING: [0x00,0xe0,0x10,0xa5]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a510e000 <unknown>
+
+ld3q    {z21.q, z22.q, z23.q}, p5/z, [x10, #15, mul vl]  // 10100101-00010101-11110101-01010101
+// CHECK-INST: ld3q    { z21.q - z23.q }, p5/z, [x10, #15, mul vl]
+// CHECK-ENCODING: [0x55,0xf5,0x15,0xa5]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a515f555 <unknown>
+
+ld3q    {z23.q, z24.q, z25.q}, p3/z, [x13, #-24, mul vl]  // 10100101-00011000-11101101-10110111
+// CHECK-INST: ld3q    { z23.q - z25.q }, p3/z, [x13, #-24, mul vl]
+// CHECK-ENCODING: [0xb7,0xed,0x18,0xa5]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a518edb7 <unknown>
+
+ld3q    {z31.q, z0.q, z1.q}, p7/z, [sp, #-3, mul vl]  // 10100101-00011111-11111111-11111111
+// CHECK-INST: ld3q    { z31.q, z0.q, z1.q }, p7/z, [sp, #-3, mul vl]
+// CHECK-ENCODING: [0xff,0xff,0x1f,0xa5]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a51fffff <unknown>

llvm/test/MC/AArch64/SVE2p1/ld4q-diagnostics.s

+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 2>&1 < %s | FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid predicate register
+
+ld4q {z0.q, z1.q, z2.q, z3.q}, p8/z, [z0.d, x0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: ld4q {z0.q, z1.q, z2.q, z3.q}, p8/z, [z0.d, x0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ld4q {z23.q, z24.q, z25.q, z26.q}, p2/m, [x0, x0, lsl #4]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: ld4q {z23.q, z24.q, z25.q, z26.q}, p2/m, [x0, x0, lsl #4]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ld4q {z21.q, z22.q, z23.q, z24.q}, p2.q, [x10, x21, lsl #4]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: ld4q {z21.q, z22.q, z23.q, z24.q}, p2.q, [x10, x21, lsl #4]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid immediate offset
+
+ld4q {z23.q, z24.q, z25.q, z26.q}, p3/z, [x13, #-33, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: index must be a multiple of 4 in range [-32, 28].
+// CHECK-NEXT: ld4q {z23.q, z24.q, z25.q, z26.q}, p3/z, [x13, #-33, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+ld4q {z23.q, z24.q, z25.q, z26.q}, p3/z, [x13, #29, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: index must be a multiple of 4 in range [-32, 28].
+// CHECK-NEXT: ld4q {z23.q, z24.q, z25.q, z26.q}, p3/z, [x13, #29, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:

llvm/test/MC/AArch64/SVE2p1/ld4q.s

+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p1 < %s \
+// RUN:        | llvm-objdump -d --no-print-imm-hex --mattr=+sme2p1 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p1 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2p1,-sve2p1 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p1 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sme2p1 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+ld4q    {z0.q, z1.q, z2.q, z3.q}, p0/z, [x0, x0, lsl #4]  // 10100101-10100000-10000000-00000000
+// CHECK-INST: ld4q    { z0.q - z3.q }, p0/z, [x0, x0, lsl #4]
+// CHECK-ENCODING: [0x00,0x80,0xa0,0xa5]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a5a08000 <unknown>
+
+ld4q    {z21.q, z22.q, z23.q, z24.q}, p5/z, [x10, x21, lsl #4]  // 10100101-10110101-10010101-01010101
+// CHECK-INST: ld4q    { z21.q - z24.q }, p5/z, [x10, x21, lsl #4]
+// CHECK-ENCODING: [0x55,0x95,0xb5,0xa5]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a5b59555 <unknown>
+
+ld4q    {z23.q, z24.q, z25.q, z26.q}, p3/z, [x13, x8, lsl #4]  // 10100101-10101000-10001101-10110111
+// CHECK-INST: ld4q    { z23.q - z26.q }, p3/z, [x13, x8, lsl #4]
+// CHECK-ENCODING: [0xb7,0x8d,0xa8,0xa5]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a5a88db7 <unknown>
+
+ld4q    {z0.q, z1.q, z2.q, z3.q}, p0/z, [x0]  // 10100101-10010000-11100000-00000000
+// CHECK-INST: ld4q    { z0.q - z3.q }, p0/z, [x0]
+// CHECK-ENCODING: [0x00,0xe0,0x90,0xa5]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a590e000 <unknown>
+
+ld4q    {z21.q, z22.q, z23.q, z24.q}, p5/z, [x10, #20, mul vl]  // 10100101-10010101-11110101-01010101
+// CHECK-INST: ld4q    { z21.q - z24.q }, p5/z, [x10, #20, mul vl]
+// CHECK-ENCODING: [0x55,0xf5,0x95,0xa5]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a595f555 <unknown>
+
+ld4q    {z23.q, z24.q, z25.q, z26.q}, p3/z, [x13, #-32, mul vl]  // 10100101-10011000-11101101-10110111
+// CHECK-INST: ld4q    { z23.q - z26.q }, p3/z, [x13, #-32, mul vl]
+// CHECK-ENCODING: [0xb7,0xed,0x98,0xa5]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a598edb7 <unknown>
+
+ld4q    {z31.q, z0.q, z1.q, z2.q}, p7/z, [sp, #-4, mul vl]  // 10100101-10011111-11111111-11111111
+// CHECK-INST: ld4q    { z31.q, z0.q, z1.q, z2.q }, p7/z, [sp, #-4, mul vl]
+// CHECK-ENCODING: [0xff,0xff,0x9f,0xa5]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: a59fffff <unknown>

llvm/test/MC/AArch64/SVE2p1/st2q-diagnostics.s

+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 2>&1 < %s | FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid predicate register
+
+st2q {z0.q, z1.q}, p8, [z0.d, x0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: st2q {z0.q, z1.q}, p8, [z0.d, x0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+st2q {z23.q, z24.q}, p2/m, [x0, x0, lsl #4]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: st2q {z23.q, z24.q}, p2/m, [x0, x0, lsl #4]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+st2q {z21.q, z22.q}, p2.q, [x10, x21, lsl #4]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: st2q {z21.q, z22.q}, p2.q, [x10, x21, lsl #4]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid immediate offset
+
+st2q {z23.q, z24.q}, p3, [x13, #-17, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: index must be a multiple of 2 in range [-16, 14].
+// CHECK-NEXT: st2q {z23.q, z24.q}, p3, [x13, #-17, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+st2q {z23.q, z24.q}, p3, [x13, #15, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: index must be a multiple of 2 in range [-16, 14].
+// CHECK-NEXT: st2q {z23.q, z24.q}, p3, [x13, #15, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:

llvm/test/MC/AArch64/SVE2p1/st2q.s

+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p1 < %s \
+// RUN:        | llvm-objdump -d --no-print-imm-hex --mattr=+sme2p1 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p1 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2p1,-sve2p1 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p1 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sme2p1 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+st2q    {z0.q, z1.q}, p0, [x0, x0, lsl #4]  // 11100100-01100000-00000000-00000000
+// CHECK-INST: st2q    { z0.q, z1.q }, p0, [x0, x0, lsl #4]
+// CHECK-ENCODING: [0x00,0x00,0x60,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e4600000 <unknown>
+
+st2q    {z21.q, z22.q}, p5, [x10, x21, lsl #4]  // 11100100-01110101-00010101-01010101
+// CHECK-INST: st2q    { z21.q, z22.q }, p5, [x10, x21, lsl #4]
+// CHECK-ENCODING: [0x55,0x15,0x75,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e4751555 <unknown>
+
+st2q    {z23.q, z24.q}, p3, [x13, x8, lsl #4]  // 11100100-01101000-00001101-10110111
+// CHECK-INST: st2q    { z23.q, z24.q }, p3, [x13, x8, lsl #4]
+// CHECK-ENCODING: [0xb7,0x0d,0x68,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e4680db7 <unknown>
+
+st2q    {z0.q, z1.q}, p0, [x0]  // 11100100-01000000-00000000-00000000
+// CHECK-INST: st2q    { z0.q, z1.q }, p0, [x0]
+// CHECK-ENCODING: [0x00,0x00,0x40,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e4400000 <unknown>
+
+st2q    {z21.q, z22.q}, p5, [x10, #10, mul vl]  // 11100100-01000101-00010101-01010101
+// CHECK-INST: st2q    { z21.q, z22.q }, p5, [x10, #10, mul vl]
+// CHECK-ENCODING: [0x55,0x15,0x45,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e4451555 <unknown>
+
+st2q    {z23.q, z24.q}, p3, [x13, #-16, mul vl]  // 11100100-01001000-00001101-10110111
+// CHECK-INST: st2q    { z23.q, z24.q }, p3, [x13, #-16, mul vl]
+// CHECK-ENCODING: [0xb7,0x0d,0x48,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e4480db7 <unknown>
+
+st2q    {z31.q, z0.q}, p7, [sp, #-2, mul vl]  // 11100100-01001111-00011111-11111111
+// CHECK-INST: st2q    { z31.q, z0.q }, p7, [sp, #-2, mul vl]
+// CHECK-ENCODING: [0xff,0x1f,0x4f,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e44f1fff <unknown>
+

llvm/test/MC/AArch64/SVE2p1/st3q-diagnostics.s

+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p1 2>&1 < %s | FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid predicate register
+
+st3q {z0.q, z1.q, z2.q}, p8/z, [z0.d, x0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: st3q {z0.q, z1.q, z2.q}, p8/z, [z0.d, x0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+st3q {z23.q, z24.q, z25.q}, p2/m, [x0, x0, lsl #4]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: st3q {z23.q, z24.q, z25.q}, p2/m, [x0, x0, lsl #4]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+st3q {z21.q, z22.q, z23.q}, p2.q, [x10, x21, lsl #4]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: st3q {z21.q, z22.q, z23.q}, p2.q, [x10, x21, lsl #4]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid immediate offset
+
+st3q {z23.q, z24.q, z25.q}, p3, [x13, #-25, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: index must be a multiple of 3 in range [-24, 21].
+// CHECK-NEXT: st3q {z23.q, z24.q, z25.q}, p3, [x13, #-25, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+st3q {z23.q, z24.q, z25.q}, p3, [x13, #22, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: index must be a multiple of 3 in range [-24, 21].
+// CHECK-NEXT: st3q {z23.q, z24.q, z25.q}, p3, [x13, #22, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:

llvm/test/MC/AArch64/SVE2p1/st3q.s

+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p1 < %s \
+// RUN:        | llvm-objdump -d --no-print-imm-hex --mattr=+sme2p1 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p1 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2p1,-sve2p1 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p1 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sme2p1 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+st3q    {z0.q, z1.q, z2.q}, p0, [x0, x0, lsl #4]  // 11100100-10100000-00000000-00000000
+// CHECK-INST: st3q    { z0.q - z2.q }, p0, [x0, x0, lsl #4]
+// CHECK-ENCODING: [0x00,0x00,0xa0,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e4a00000 <unknown>
+
+st3q    {z21.q, z22.q, z23.q}, p5, [x10, x21, lsl #4]  // 11100100-10110101-00010101-01010101
+// CHECK-INST: st3q    { z21.q - z23.q }, p5, [x10, x21, lsl #4]
+// CHECK-ENCODING: [0x55,0x15,0xb5,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e4b51555 <unknown>
+
+st3q    {z23.q, z24.q, z25.q}, p3, [x13, x8, lsl #4]  // 11100100-10101000-00001101-10110111
+// CHECK-INST: st3q    { z23.q - z25.q }, p3, [x13, x8, lsl #4]
+// CHECK-ENCODING: [0xb7,0x0d,0xa8,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e4a80db7 <unknown>
+
+st3q    {z0.q, z1.q, z2.q}, p0, [x0]  // 11100100-10000000-00000000-00000000
+// CHECK-INST: st3q    { z0.q - z2.q }, p0, [x0]
+// CHECK-ENCODING: [0x00,0x00,0x80,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e4800000 <unknown>
+
+st3q    {z21.q, z22.q, z23.q}, p5, [x10, #15, mul vl]  // 11100100-10000101-00010101-01010101
+// CHECK-INST: st3q    { z21.q - z23.q }, p5, [x10, #15, mul vl]
+// CHECK-ENCODING: [0x55,0x15,0x85,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e4851555 <unknown>
+
+st3q    {z23.q, z24.q, z25.q}, p3, [x13, #-24, mul vl]  // 11100100-10001000-00001101-10110111
+// CHECK-INST: st3q    { z23.q - z25.q }, p3, [x13, #-24, mul vl]
+// CHECK-ENCODING: [0xb7,0x0d,0x88,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e4880db7 <unknown>
+
+st3q    {z31.q, z0.q, z1.q}, p7, [sp, #-3, mul vl]  // 11100100-10001111-00011111-11111111
+// CHECK-INST: st3q    { z31.q, z0.q, z1.q }, p7, [sp, #-3, mul vl]
+// CHECK-ENCODING: [0xff,0x1f,0x8f,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e48f1fff <unknown>

llvm/test/MC/AArch64/SVE2p1/st4q-diagnostics.s

+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p1 2>&1 < %s | FileCheck %s
+
+// --------------------------------------------------------------------------//
+// Invalid predicate register
+
+st4q {z0.q, z1.q, z2.q, z3.q}, p8/z, [z0.d, x0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: st4q {z0.q, z1.q, z2.q, z3.q}, p8/z, [z0.d, x0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+st4q {z23.q, z24.q, z25.q, z26.q}, p2/m, [x0, x0, lsl #4]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: st4q {z23.q, z24.q, z25.q, z26.q}, p2/m, [x0, x0, lsl #4]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+st4q {z21.q, z22.q, z23.q, z24.q}, p2.q, [x10, x21, lsl #4]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: st4q {z21.q, z22.q, z23.q, z24.q}, p2.q, [x10, x21, lsl #4]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// --------------------------------------------------------------------------//
+// Invalid immediate offset
+
+st4q {z23.q, z24.q, z25.q, z26.q}, p3, [x13, #-33, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: index must be a multiple of 4 in range [-32, 28].
+// CHECK-NEXT: st4q {z23.q, z24.q, z25.q, z26.q}, p3, [x13, #-33, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+st4q {z23.q, z24.q, z25.q, z26.q}, p3, [x13, #29, mul vl]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: index must be a multiple of 4 in range [-32, 28].
+// CHECK-NEXT: st4q {z23.q, z24.q, z25.q, z26.q}, p3, [x13, #29, mul vl]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:

llvm/test/MC/AArch64/SVE2p1/st4q.s

+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2p1 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p1 < %s \
+// RUN:        | llvm-objdump -d --no-print-imm-hex --mattr=+sme2p1 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p1 < %s \
+// RUN:        | llvm-objdump -d --mattr=-sme2p1,-sve2p1 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p1 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sme2p1 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+st4q    {z0.q, z1.q, z2.q, z3.q}, p0, [x0, x0, lsl #4]  // 11100100-11100000-00000000-00000000
+// CHECK-INST: st4q    { z0.q - z3.q }, p0, [x0, x0, lsl #4]
+// CHECK-ENCODING: [0x00,0x00,0xe0,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e4e00000 <unknown>
+
+st4q    {z21.q, z22.q, z23.q, z24.q}, p5, [x10, x21, lsl #4]  // 11100100-11110101-00010101-01010101
+// CHECK-INST: st4q    { z21.q - z24.q }, p5, [x10, x21, lsl #4]
+// CHECK-ENCODING: [0x55,0x15,0xf5,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e4f51555 <unknown>
+
+st4q    {z23.q, z24.q, z25.q, z26.q}, p3, [x13, x8, lsl #4]  // 11100100-11101000-00001101-10110111
+// CHECK-INST: st4q    { z23.q - z26.q }, p3, [x13, x8, lsl #4]
+// CHECK-ENCODING: [0xb7,0x0d,0xe8,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e4e80db7 <unknown>
+
+st4q    {z0.q, z1.q, z2.q, z3.q}, p0, [x0]  // 11100100-11000000-00000000-00000000
+// CHECK-INST: st4q    { z0.q - z3.q }, p0, [x0]
+// CHECK-ENCODING: [0x00,0x00,0xc0,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e4c00000 <unknown>
+
+st4q    {z21.q, z22.q, z23.q, z24.q}, p5, [x10, #20, mul vl]  // 11100100-11000101-00010101-01010101
+// CHECK-INST: st4q    { z21.q - z24.q }, p5, [x10, #20, mul vl]
+// CHECK-ENCODING: [0x55,0x15,0xc5,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e4c51555 <unknown>
+
+st4q    {z23.q, z24.q, z25.q, z26.q}, p3, [x13, #-32, mul vl]  // 11100100-11001000-00001101-10110111
+// CHECK-INST: st4q    { z23.q - z26.q }, p3, [x13, #-32, mul vl]
+// CHECK-ENCODING: [0xb7,0x0d,0xc8,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e4c80db7 <unknown>
+
+st4q    {z31.q, z0.q, z1.q, z2.q}, p7, [sp, #-4, mul vl]  // 11100100-11001111-00011111-11111111
+// CHECK-INST: st4q    { z31.q, z0.q, z1.q, z2.q }, p7, [sp, #-4, mul vl]
+// CHECK-ENCODING: [0xff,0x1f,0xcf,0xe4]
+// CHECK-ERROR: instruction requires: sme2p1 or sve2p1
+// CHECK-UNKNOWN: e4cf1fff <unknown>
+