[AArch64][SVE] Asm: Support for gather LD1/LDFF1 (scalar + vector (32bit elts, scaled)) load instructions.

[sdesmalen]

Summary

Patch [3/8] in series to add support for SVE's gather load instructions
that use scalar+vector addressing modes:

• Patch [1/8]: https://reviews.llvm.org/D45951
• Patch [2/8]: https://reviews.llvm.org/D45952
• Patch [3/8]: https://reviews.llvm.org/D45953
• Patch [4/8]: https://reviews.llvm.org/D45954
• Patch [5/8]: https://reviews.llvm.org/D45955
• Patch [6/8]: https://reviews.llvm.org/D45956
• Patch [7/8]: https://reviews.llvm.org/D45957
• Patch [8/8]: https://reviews.llvm.org/D45958

[fhahn]

lib/Target/AArch64/SVEInstrFormats.td

733

There seems to be a 1 line difference only between this class and sve_mem_32b_gld_vs added in D45952. I think it would be great if we could have one class for those similar cases if possible. It takes parameters already, so I think it would make sense to add a new one?

[sdesmalen]

Replaced by D46023

[sdesmalen]

lib/Target/AArch64/SVEInstrFormats.td

733

Thanks for the suggestion, I have merged these classes (also for 64b) into one, reducing the size of the diff. The change is made in D46023 .

lib/Target/AArch64/AArch64SVEInstrInfo.td

let Predicates = [HasSVE] in {
   defm GLDFF1B_S  : sve_mem_32b_gld_vs_32_unscaled<0b0011, "ldff1b",  ZPR32ExtSXTW8, ZPR32ExtUXTW8>;
   defm GLD1SH_S   : sve_mem_32b_gld_vs_32_unscaled<0b0100, "ld1sh",   ZPR32ExtSXTW8, ZPR32ExtUXTW8>;
   defm GLDFF1SH_S : sve_mem_32b_gld_vs_32_unscaled<0b0101, "ldff1sh", ZPR32ExtSXTW8, ZPR32ExtUXTW8>;
   defm GLD1H_S    : sve_mem_32b_gld_vs_32_unscaled<0b0110, "ld1h",    ZPR32ExtSXTW8, ZPR32ExtUXTW8>;
   defm GLDFF1H_S  : sve_mem_32b_gld_vs_32_unscaled<0b0111, "ldff1h",  ZPR32ExtSXTW8, ZPR32ExtUXTW8>;
   defm GLD1W      : sve_mem_32b_gld_vs_32_unscaled<0b1010, "ld1w",    ZPR32ExtSXTW8, ZPR32ExtUXTW8>;
   defm GLDFF1W    : sve_mem_32b_gld_vs_32_unscaled<0b1011, "ldff1w",  ZPR32ExtSXTW8, ZPR32ExtUXTW8>;
 
+  // Gathers using scaled 32-bit offsets, e.g.
+  //    ld1h z32.s, p0/z, [z0.s, uxtw #1]
+  defm GLD1SH_S   : sve_mem_32b_gld_sv_32_scaled<0b0100, "ld1sh",   ZPR32ExtSXTW16, ZPR32ExtUXTW16>;
+  defm GLDFF1SH_S : sve_mem_32b_gld_sv_32_scaled<0b0101, "ldff1sh", ZPR32ExtSXTW16, ZPR32ExtUXTW16>;
+  defm GLD1H_S    : sve_mem_32b_gld_sv_32_scaled<0b0110, "ld1h",    ZPR32ExtSXTW16, ZPR32ExtUXTW16>;
+  defm GLDFF1H_S  : sve_mem_32b_gld_sv_32_scaled<0b0111, "ldff1h",  ZPR32ExtSXTW16, ZPR32ExtUXTW16>;
+  defm GLD1W      : sve_mem_32b_gld_sv_32_scaled<0b1010, "ld1w",    ZPR32ExtSXTW32, ZPR32ExtUXTW32>;
+  defm GLDFF1W    : sve_mem_32b_gld_sv_32_scaled<0b1011, "ldff1w",  ZPR32ExtSXTW32, ZPR32ExtUXTW32>;
+
   // continuous store with immediates
   defm ST1B_IMM   : sve_mem_cst_si<0b00, 0b00, "st1b", Z_b, ZPR8>;
   defm ST1B_H_IMM : sve_mem_cst_si<0b00, 0b01, "st1b", Z_h, ZPR16>;
   defm ST1B_S_IMM : sve_mem_cst_si<0b00, 0b10, "st1b", Z_s, ZPR32>;
   defm ST1B_D_IMM : sve_mem_cst_si<0b00, 0b11, "st1b", Z_d, ZPR64>;
   defm ST1H_IMM   : sve_mem_cst_si<0b01, 0b01, "st1h", Z_h, ZPR16>;
   defm ST1H_S_IMM : sve_mem_cst_si<0b01, 0b10, "st1h", Z_s, ZPR32>;
   defm ST1H_D_IMM : sve_mem_cst_si<0b01, 0b11, "st1h", Z_d, ZPR64>;

lib/Target/AArch64/SVEInstrFormats.td

multiclass sve_mem_eld_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList,
   def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
                   (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;
 }
 
 //===----------------------------------------------------------------------===//
 // SVE Memory - 32-bit Gather and Unsized Contiguous Group
 //===----------------------------------------------------------------------===//
 
+class sve_mem_32b_gld_sv<bits<4> opc, bit xs, string asm,
+                         RegisterOperand zprext>
+: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm),
+  asm, "\t$Zt, $Pg/z, [$Rn, $Zm]",
+  "",
+  []>, Sched<[]> {
+  bits<3> Pg;
+  bits<5> Rn;
+  bits<5> Zm;
+  bits<5> Zt;
+  let Inst{31-25} = 0b1000010;
+  let Inst{24-23} = opc{3-2};
+  let Inst{22}    = xs;
+  let Inst{21}    = 0b1;
+  let Inst{20-16} = Zm;
+  let Inst{15}    = 0b0;
+  let Inst{14-13} = opc{1-0};
+  let Inst{12-10} = Pg;
+  let Inst{9-5}   = Rn;
+  let Inst{4-0}   = Zt;
+
+  let mayLoad = 1;
+  let Defs = !if(!eq(opc{0}, 1), [FFR], []);
+  let Uses = !if(!eq(opc{0}, 1), [FFR], []);
+}
+
+multiclass sve_mem_32b_gld_sv_32_scaled<bits<4> opc, string asm,
+                                        RegisterOperand sxtw_opnd,
+                                        RegisterOperand uxtw_opnd> {
+  def _UXTW_SCALED_REAL : sve_mem_32b_gld_sv<opc, 0, asm, uxtw_opnd>;
+  def _SXTW_SCALED_REAL : sve_mem_32b_gld_sv<opc, 1, asm, sxtw_opnd>;
+
+  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",
+                  (!cast<Instruction>(NAME # _UXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>;
+  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",
+                  (!cast<Instruction>(NAME # _SXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>;
+}
+
 class sve_mem_32b_gld_vs<bits<4> opc, bit xs, string asm,
                          RegisterOperand zprext>
 : I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm),
   asm, "\t$Zt, $Pg/z, [$Rn, $Zm]",
   "",
   []>, Sched<[]> {
   bits<3> Pg;
   bits<5> Rn;

test/MC/AArch64/SVE/ld1h.s

 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 00 40 80 84 <unknown>
 
 ld1h    { z0.s }, p0/z, [x0, z0.s, sxtw]
 // CHECK-INST: ld1h    { z0.s }, p0/z, [x0, z0.s, sxtw]
 // CHECK-ENCODING: [0x00,0x40,0xc0,0x84]
 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 00 40 c0 84 <unknown>
+
+ld1h    { z31.s }, p7/z, [sp, z31.s, uxtw #1]
+// CHECK-INST: ld1h    { z31.s }, p7/z, [sp, z31.s, uxtw #1]
+// CHECK-ENCODING: [0xff,0x5f,0xbf,0x84]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 5f bf 84 <unknown>
+
+ld1h    { z31.s }, p7/z, [sp, z31.s, sxtw #1]
+// CHECK-INST: ld1h    { z31.s }, p7/z, [sp, z31.s, sxtw #1]
+// CHECK-ENCODING: [0xff,0x5f,0xff,0x84]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 5f ff 84 <unknown>

test/MC/AArch64/SVE/ld1sh.s

 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 00 00 80 84 <unknown>
 
 ld1sh   { z0.s }, p0/z, [x0, z0.s, sxtw]
 // CHECK-INST: ld1sh   { z0.s }, p0/z, [x0, z0.s, sxtw]
 // CHECK-ENCODING: [0x00,0x00,0xc0,0x84]
 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 00 00 c0 84 <unknown>
+
+ld1sh   { z31.s }, p7/z, [sp, z31.s, uxtw #1]
+// CHECK-INST: ld1sh   { z31.s }, p7/z, [sp, z31.s, uxtw #1]
+// CHECK-ENCODING: [0xff,0x1f,0xbf,0x84]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 1f bf 84 <unknown>
+
+ld1sh   { z31.s }, p7/z, [sp, z31.s, sxtw #1]
+// CHECK-INST: ld1sh   { z31.s }, p7/z, [sp, z31.s, sxtw #1]
+// CHECK-ENCODING: [0xff,0x1f,0xff,0x84]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 1f ff 84 <unknown>

test/MC/AArch64/SVE/ld1w.s

 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 00 40 00 85 <unknown>
 
 ld1w    { z0.s }, p0/z, [x0, z0.s, sxtw]
 // CHECK-INST: ld1w    { z0.s }, p0/z, [x0, z0.s, sxtw]
 // CHECK-ENCODING: [0x00,0x40,0x40,0x85]
 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 00 40 40 85 <unknown>
+
+ld1w    { z31.s }, p7/z, [sp, z31.s, uxtw #2]
+// CHECK-INST: ld1w    { z31.s }, p7/z, [sp, z31.s, uxtw #2]
+// CHECK-ENCODING: [0xff,0x5f,0x3f,0x85]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 5f 3f 85 <unknown>
+
+ld1w    { z31.s }, p7/z, [sp, z31.s, sxtw #2]
+// CHECK-INST: ld1w    { z31.s }, p7/z, [sp, z31.s, sxtw #2]
+// CHECK-ENCODING: [0xff,0x5f,0x7f,0x85]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 5f 7f 85 <unknown>

test/MC/AArch64/SVE/ldff1h.s

 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 00 60 80 84 <unknown>
 
 ldff1h    { z0.s }, p0/z, [x0, z0.s, sxtw]
 // CHECK-INST: ldff1h    { z0.s }, p0/z, [x0, z0.s, sxtw]
 // CHECK-ENCODING: [0x00,0x60,0xc0,0x84]
 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 00 60 c0 84 <unknown>
+
+ldff1h  { z31.s }, p7/z, [sp, z31.s, uxtw #1]
+// CHECK-INST: ldff1h  { z31.s }, p7/z, [sp, z31.s, uxtw #1]
+// CHECK-ENCODING: [0xff,0x7f,0xbf,0x84]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 7f bf 84 <unknown>
+
+ldff1h  { z31.s }, p7/z, [sp, z31.s, sxtw #1]
+// CHECK-INST: ldff1h  { z31.s }, p7/z, [sp, z31.s, sxtw #1]
+// CHECK-ENCODING: [0xff,0x7f,0xff,0x84]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 7f ff 84 <unknown>

test/MC/AArch64/SVE/ldff1sh.s

 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 00 20 80 84 <unknown>
 
 ldff1sh   { z0.s }, p0/z, [x0, z0.s, sxtw]
 // CHECK-INST: ldff1sh   { z0.s }, p0/z, [x0, z0.s, sxtw]
 // CHECK-ENCODING: [0x00,0x20,0xc0,0x84]
 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 00 20 c0 84 <unknown>
+
+ldff1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1]
+// CHECK-INST: ldff1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1]
+// CHECK-ENCODING: [0xff,0x3f,0xbf,0x84]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 3f bf 84 <unknown>
+
+ldff1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1]
+// CHECK-INST: ldff1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1]
+// CHECK-ENCODING: [0xff,0x3f,0xff,0x84]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 3f ff 84 <unknown>

test/MC/AArch64/SVE/ldff1w.s

 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 00 60 00 85 <unknown>
 
 ldff1w  { z0.s }, p0/z, [x0, z0.s, sxtw]
 // CHECK-INST: ldff1w  { z0.s }, p0/z, [x0, z0.s, sxtw]
 // CHECK-ENCODING: [0x00,0x60,0x40,0x85]
 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 00 60 40 85 <unknown>
+
+ldff1w  { z31.s }, p7/z, [sp, z31.s, uxtw #2]
+// CHECK-INST: ldff1w  { z31.s }, p7/z, [sp, z31.s, uxtw #2]
+// CHECK-ENCODING: [0xff,0x7f,0x3f,0x85]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 7f 3f 85 <unknown>
+
+ldff1w  { z31.s }, p7/z, [sp, z31.s, sxtw #2]
+// CHECK-INST: ldff1w  { z31.s }, p7/z, [sp, z31.s, sxtw #2]
+// CHECK-ENCODING: [0xff,0x7f,0x7f,0x85]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 7f 7f 85 <unknown>