[AArch64][SVE] Asm: Support for gather LD1/LDFF1 (scalar + vector (packed 32bit elts, scaled)) load instructions.

[sdesmalen]

Summary

Patch [7/8] in series to add support for SVE's gather load instructions
that use scalar+vector addressing modes:

• Patch [1/8]: https://reviews.llvm.org/D45951
• Patch [2/8]: https://reviews.llvm.org/D45952
• Patch [3/8]: https://reviews.llvm.org/D45953
• Patch [4/8]: https://reviews.llvm.org/D45954
• Patch [5/8]: https://reviews.llvm.org/D45955
• Patch [6/8]: https://reviews.llvm.org/D45956
• Patch [7/8]: https://reviews.llvm.org/D45957
• Patch [8/8]: https://reviews.llvm.org/D45958

[sdesmalen]

Replaced by D46023

lib/Target/AArch64/AArch64SVEInstrInfo.td

let Predicates = [HasSVE] in {
   defm GLDFF1H_D  : sve_mem_64b_gld_vs_32_unscaled<0b0111, "ldff1h",  ZPR64ExtSXTW8, ZPR64ExtUXTW8>;
   defm GLD1SW_D   : sve_mem_64b_gld_vs_32_unscaled<0b1000, "ld1sw",   ZPR64ExtSXTW8, ZPR64ExtUXTW8>;
   defm GLDFF1SW_D : sve_mem_64b_gld_vs_32_unscaled<0b1001, "ldff1sw", ZPR64ExtSXTW8, ZPR64ExtUXTW8>;
   defm GLD1W_D    : sve_mem_64b_gld_vs_32_unscaled<0b1010, "ld1w",    ZPR64ExtSXTW8, ZPR64ExtUXTW8>;
   defm GLDFF1W_D  : sve_mem_64b_gld_vs_32_unscaled<0b1011, "ldff1w",  ZPR64ExtSXTW8, ZPR64ExtUXTW8>;
   defm GLD1D      : sve_mem_64b_gld_vs_32_unscaled<0b1110, "ld1d",    ZPR64ExtSXTW8, ZPR64ExtUXTW8>;
   defm GLDFF1D    : sve_mem_64b_gld_vs_32_unscaled<0b1111, "ldff1d",  ZPR64ExtSXTW8, ZPR64ExtUXTW8>;
 
+  // Gathers using scaled 32-bit offsets packed in 64-bits elements, e.g.
+  //    ld1h z32.d, p0/z, [x0, z0.d, uxtw #1]
+  defm GLD1SH_D   : sve_mem_64b_gld_sv_32_scaled<0b0100, "ld1sh",  ZPR64ExtSXTW16, ZPR64ExtUXTW16>;
+  defm GLDFF1SH_D : sve_mem_64b_gld_sv_32_scaled<0b0101, "ldff1sh",ZPR64ExtSXTW16, ZPR64ExtUXTW16>;
+  defm GLD1H_D    : sve_mem_64b_gld_sv_32_scaled<0b0110, "ld1h",   ZPR64ExtSXTW16, ZPR64ExtUXTW16>;
+  defm GLDFF1H_D  : sve_mem_64b_gld_sv_32_scaled<0b0111, "ldff1h", ZPR64ExtSXTW16, ZPR64ExtUXTW16>;
+  defm GLD1SW_D   : sve_mem_64b_gld_sv_32_scaled<0b1000, "ld1sw",  ZPR64ExtSXTW32, ZPR64ExtUXTW32>;
+  defm GLDFF1SW_D : sve_mem_64b_gld_sv_32_scaled<0b1001, "ldff1sw",ZPR64ExtSXTW32, ZPR64ExtUXTW32>;
+  defm GLD1W_D    : sve_mem_64b_gld_sv_32_scaled<0b1010, "ld1w",   ZPR64ExtSXTW32, ZPR64ExtUXTW32>;
+  defm GLDFF1W_D  : sve_mem_64b_gld_sv_32_scaled<0b1011, "ldff1w", ZPR64ExtSXTW32, ZPR64ExtUXTW32>;
+  defm GLD1D      : sve_mem_64b_gld_sv_32_scaled<0b1110, "ld1d",   ZPR64ExtSXTW64, ZPR64ExtUXTW64>;
+  defm GLDFF1D    : sve_mem_64b_gld_sv_32_scaled<0b1111, "ldff1d", ZPR64ExtSXTW64, ZPR64ExtUXTW64>;
+
   // continuous store with immediates
   defm ST1B_IMM   : sve_mem_cst_si<0b00, 0b00, "st1b", Z_b, ZPR8>;
   defm ST1B_H_IMM : sve_mem_cst_si<0b00, 0b01, "st1b", Z_h, ZPR16>;
   defm ST1B_S_IMM : sve_mem_cst_si<0b00, 0b10, "st1b", Z_s, ZPR32>;
   defm ST1B_D_IMM : sve_mem_cst_si<0b00, 0b11, "st1b", Z_d, ZPR64>;
   defm ST1H_IMM   : sve_mem_cst_si<0b01, 0b01, "st1h", Z_h, ZPR16>;
   defm ST1H_S_IMM : sve_mem_cst_si<0b01, 0b10, "st1h", Z_s, ZPR32>;
   defm ST1H_D_IMM : sve_mem_cst_si<0b01, 0b11, "st1h", Z_d, ZPR64>;

lib/Target/AArch64/SVEInstrFormats.td

def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",
                   (!cast<Instruction>(NAME # _SXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>;
 }
 
 
 //===----------------------------------------------------------------------===//
 // SVE Memory - 64-bit Gather Group
 //===----------------------------------------------------------------------===//
 
+class sve_mem_64b_gld_sv<bits<4> opc, bit xs, string asm,
+                         RegisterOperand zprext>
+: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm),
+  asm, "\t$Zt, $Pg/z, [$Rn, $Zm]",
+  "",
+  []>, Sched<[]> {
+  bits<3> Pg;
+  bits<5> Rn;
+  bits<5> Zm;
+  bits<5> Zt;
+  let Inst{31-25} = 0b1100010;
+  let Inst{24-23} = opc{3-2};
+  let Inst{22}    = xs;
+  let Inst{21}    = 0b1;
+  let Inst{20-16} = Zm;
+  let Inst{15}    = 0b0;
+  let Inst{14-13} = opc{1-0};
+  let Inst{12-10} = Pg;
+  let Inst{9-5}   = Rn;
+  let Inst{4-0}   = Zt;
+
+  let mayLoad = 1;
+  let Defs = !if(!eq(opc{0}, 1), [FFR], []);
+  let Uses = !if(!eq(opc{0}, 1), [FFR], []);
+}
+
+multiclass sve_mem_64b_gld_sv_32_scaled<bits<4> opc, string asm,
+                                        RegisterOperand sxtw_opnd,
+                                        RegisterOperand uxtw_opnd> {
+  def _UXTW_SCALED_REAL : sve_mem_64b_gld_sv<opc, 0, asm, uxtw_opnd>;
+  def _SXTW_SCALED_REAL : sve_mem_64b_gld_sv<opc, 1, asm, sxtw_opnd>;
+
+  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",
+                  (!cast<Instruction>(NAME # _UXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>;
+  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",
+                  (!cast<Instruction>(NAME # _SXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>;
+}
+
 class sve_mem_64b_gld_vs<bits<4> opc, bit xs, string asm,
                          RegisterOperand zprext>
 : I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm),
   asm, "\t$Zt, $Pg/z, [$Rn, $Zm]",
   "",
   []>, Sched<[]> {
   bits<3> Pg;
   bits<5> Rn;

test/MC/AArch64/SVE/ld1d.s

 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 55 55 95 c5 <unknown>
 
 ld1d    { z21.d }, p5/z, [x10, z21.d, sxtw]
 // CHECK-INST: ld1d    { z21.d }, p5/z, [x10, z21.d, sxtw]
 // CHECK-ENCODING: [0x55,0x55,0xd5,0xc5]
 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 55 55 d5 c5 <unknown>
+
+ld1d    { z0.d }, p0/z, [x0, z0.d, uxtw #3]
+// CHECK-INST: ld1d    { z0.d }, p0/z, [x0, z0.d, uxtw #3]
+// CHECK-ENCODING: [0x00,0x40,0xa0,0xc5]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 40 a0 c5 <unknown>
+
+ld1d    { z0.d }, p0/z, [x0, z0.d, sxtw #3]
+// CHECK-INST: ld1d    { z0.d }, p0/z, [x0, z0.d, sxtw #3]
+// CHECK-ENCODING: [0x00,0x40,0xe0,0xc5]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 40 e0 c5 <unknown>

test/MC/AArch64/SVE/ld1h.s

 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 55 55 95 c4 <unknown>
 
 ld1h    { z21.d }, p5/z, [x10, z21.d, sxtw]
 // CHECK-INST: ld1h    { z21.d }, p5/z, [x10, z21.d, sxtw]
 // CHECK-ENCODING: [0x55,0x55,0xd5,0xc4]
 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 55 55 d5 c4 <unknown>
+
+ld1h    { z0.d }, p0/z, [x0, z0.d, uxtw #1]
+// CHECK-INST: ld1h    { z0.d }, p0/z, [x0, z0.d, uxtw #1]
+// CHECK-ENCODING: [0x00,0x40,0xa0,0xc4]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 40 a0 c4 <unknown>
+
+ld1h    { z0.d }, p0/z, [x0, z0.d, sxtw #1]
+// CHECK-INST: ld1h    { z0.d }, p0/z, [x0, z0.d, sxtw #1]
+// CHECK-ENCODING: [0x00,0x40,0xe0,0xc4]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 40 e0 c4 <unknown>

test/MC/AArch64/SVE/ld1sh.s

 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 55 15 95 c4 <unknown>
 
 ld1sh   { z21.d }, p5/z, [x10, z21.d, sxtw]
 // CHECK-INST: ld1sh   { z21.d }, p5/z, [x10, z21.d, sxtw]
 // CHECK-ENCODING: [0x55,0x15,0xd5,0xc4]
 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 55 15 d5 c4 <unknown>
+
+ld1sh   { z0.d }, p0/z, [x0, z0.d, uxtw #1]
+// CHECK-INST: ld1sh   { z0.d }, p0/z, [x0, z0.d, uxtw #1]
+// CHECK-ENCODING: [0x00,0x00,0xa0,0xc4]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 00 a0 c4 <unknown>
+
+ld1sh   { z0.d }, p0/z, [x0, z0.d, sxtw #1]
+// CHECK-INST: ld1sh   { z0.d }, p0/z, [x0, z0.d, sxtw #1]
+// CHECK-ENCODING: [0x00,0x00,0xe0,0xc4]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 00 e0 c4 <unknown>

test/MC/AArch64/SVE/ld1sw.s

 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 55 15 15 c5 <unknown>
 
 ld1sw   { z21.d }, p5/z, [x10, z21.d, sxtw]
 // CHECK-INST: ld1sw   { z21.d }, p5/z, [x10, z21.d, sxtw]
 // CHECK-ENCODING: [0x55,0x15,0x55,0xc5]
 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 55 15 55 c5 <unknown>
+
+ld1sw   { z0.d }, p0/z, [x0, z0.d, uxtw #2]
+// CHECK-INST: ld1sw   { z0.d }, p0/z, [x0, z0.d, uxtw #2]
+// CHECK-ENCODING: [0x00,0x00,0x20,0xc5]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 00 20 c5 <unknown>
+
+ld1sw   { z0.d }, p0/z, [x0, z0.d, sxtw #2]
+// CHECK-INST: ld1sw   { z0.d }, p0/z, [x0, z0.d, sxtw #2]
+// CHECK-ENCODING: [0x00,0x00,0x60,0xc5]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 00 60 c5 <unknown>

test/MC/AArch64/SVE/ld1w.s

 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 55 55 15 c5 <unknown>
 
 ld1w    { z21.d }, p5/z, [x10, z21.d, sxtw]
 // CHECK-INST: ld1w    { z21.d }, p5/z, [x10, z21.d, sxtw]
 // CHECK-ENCODING: [0x55,0x55,0x55,0xc5]
 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 55 55 55 c5 <unknown>
+
+ld1w    { z0.d }, p0/z, [x0, z0.d, uxtw #2]
+// CHECK-INST: ld1w    { z0.d }, p0/z, [x0, z0.d, uxtw #2]
+// CHECK-ENCODING: [0x00,0x40,0x20,0xc5]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 40 20 c5 <unknown>
+
+ld1w    { z0.d }, p0/z, [x0, z0.d, sxtw #2]
+// CHECK-INST: ld1w    { z0.d }, p0/z, [x0, z0.d, sxtw #2]
+// CHECK-ENCODING: [0x00,0x40,0x60,0xc5]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 40 60 c5 <unknown>

test/MC/AArch64/SVE/ldff1d.s

 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 55 75 95 c5 <unknown>
 
 ldff1d  { z21.d }, p5/z, [x10, z21.d, sxtw]
 // CHECK-INST: ldff1d  { z21.d }, p5/z, [x10, z21.d, sxtw]
 // CHECK-ENCODING: [0x55,0x75,0xd5,0xc5]
 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 55 75 d5 c5 <unknown>
+
+ldff1d  { z0.d }, p0/z, [x0, z0.d, uxtw #3]
+// CHECK-INST: ldff1d  { z0.d }, p0/z, [x0, z0.d, uxtw #3]
+// CHECK-ENCODING: [0x00,0x60,0xa0,0xc5]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 60 a0 c5 <unknown>
+
+ldff1d  { z0.d }, p0/z, [x0, z0.d, sxtw #3]
+// CHECK-INST: ldff1d  { z0.d }, p0/z, [x0, z0.d, sxtw #3]
+// CHECK-ENCODING: [0x00,0x60,0xe0,0xc5]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 60 e0 c5 <unknown>

test/MC/AArch64/SVE/ldff1h.s

 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 55 75 95 c4 <unknown>
 
 ldff1h  { z21.d }, p5/z, [x10, z21.d, sxtw]
 // CHECK-INST: ldff1h  { z21.d }, p5/z, [x10, z21.d, sxtw]
 // CHECK-ENCODING: [0x55,0x75,0xd5,0xc4]
 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 55 75 d5 c4 <unknown>
+
+ldff1h  { z0.d }, p0/z, [x0, z0.d, uxtw #1]
+// CHECK-INST: ldff1h  { z0.d }, p0/z, [x0, z0.d, uxtw #1]
+// CHECK-ENCODING: [0x00,0x60,0xa0,0xc4]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 60 a0 c4 <unknown>
+
+ldff1h  { z0.d }, p0/z, [x0, z0.d, sxtw #1]
+// CHECK-INST: ldff1h  { z0.d }, p0/z, [x0, z0.d, sxtw #1]
+// CHECK-ENCODING: [0x00,0x60,0xe0,0xc4]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 60 e0 c4 <unknown>

test/MC/AArch64/SVE/ldff1sh.s

 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 55 35 95 c4 <unknown>
 
 ldff1sh { z21.d }, p5/z, [x10, z21.d, sxtw]
 // CHECK-INST: ldff1sh { z21.d }, p5/z, [x10, z21.d, sxtw]
 // CHECK-ENCODING: [0x55,0x35,0xd5,0xc4]
 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 55 35 d5 c4 <unknown>
+
+ldff1sh { z0.d }, p0/z, [x0, z0.d, uxtw #1]
+// CHECK-INST: ldff1sh { z0.d }, p0/z, [x0, z0.d, uxtw #1]
+// CHECK-ENCODING: [0x00,0x20,0xa0,0xc4]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 20 a0 c4 <unknown>
+
+ldff1sh { z0.d }, p0/z, [x0, z0.d, sxtw #1]
+// CHECK-INST: ldff1sh { z0.d }, p0/z, [x0, z0.d, sxtw #1]
+// CHECK-ENCODING: [0x00,0x20,0xe0,0xc4]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 20 e0 c4 <unknown>

test/MC/AArch64/SVE/ldff1sw.s

 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 55 35 15 c5 <unknown>
 
 ldff1sw { z21.d }, p5/z, [x10, z21.d, sxtw]
 // CHECK-INST: ldff1sw { z21.d }, p5/z, [x10, z21.d, sxtw]
 // CHECK-ENCODING: [0x55,0x35,0x55,0xc5]
 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 55 35 55 c5 <unknown>
+
+ldff1sw { z0.d }, p0/z, [x0, z0.d, uxtw #2]
+// CHECK-INST: ldff1sw { z0.d }, p0/z, [x0, z0.d, uxtw #2]
+// CHECK-ENCODING: [0x00,0x20,0x20,0xc5]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 20 20 c5 <unknown>
+
+ldff1sw { z0.d }, p0/z, [x0, z0.d, sxtw #2]
+// CHECK-INST: ldff1sw { z0.d }, p0/z, [x0, z0.d, sxtw #2]
+// CHECK-ENCODING: [0x00,0x20,0x60,0xc5]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 20 60 c5 <unknown>

test/MC/AArch64/SVE/ldff1w.s

 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 55 75 15 c5 <unknown>
 
 ldff1w  { z21.d }, p5/z, [x10, z21.d, sxtw]
 // CHECK-INST: ldff1w  { z21.d }, p5/z, [x10, z21.d, sxtw]
 // CHECK-ENCODING: [0x55,0x75,0x55,0xc5]
 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 55 75 55 c5 <unknown>
+
+ldff1w  { z0.d }, p0/z, [x0, z0.d, uxtw #2]
+// CHECK-INST: ldff1w  { z0.d }, p0/z, [x0, z0.d, uxtw #2]
+// CHECK-ENCODING: [0x00,0x60,0x20,0xc5]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 60 20 c5 <unknown>
+
+ldff1w  { z0.d }, p0/z, [x0, z0.d, sxtw #2]
+// CHECK-INST: ldff1w  { z0.d }, p0/z, [x0, z0.d, sxtw #2]
+// CHECK-ENCODING: [0x00,0x60,0x60,0xc5]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 60 60 c5 <unknown>