[AArch64][SVE] Asm: Support for FMUL (indexed)

[sdesmalen]

Summary

Unpredicated FP-multiply of SVE vector with a vector-element given by
vector[index], for example:

fmul z0.s, z1.s, z2.s[0]

which performs an unpredicated FP-multiply of all 32-bit elements in
'z1' with the first element from 'z2'.

This patch adds restricted register classes for SVE vectors:

ZPR_3b (only z0..z7 are allowed)  - for indexed vector of 16/32-bit elements.
ZPR_4b (only z0..z15 are allowed) - for indexed vector of 64-bit elements.

[fhahn]

LGTM, thanks!

lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp

509

nit: misaligned

517

nit: misaligned

lib/Target/AArch64/AArch64RegisterInfo.td

def ZPR : RegisterClass<"AArch64",
                         [nxv16i8, nxv8i16, nxv4i32, nxv2i64,
                          nxv2f16, nxv4f16, nxv8f16,
                          nxv1f32, nxv2f32, nxv4f32,
                          nxv1f64, nxv2f64],
                         128, (sequence "Z%u", 0, 31)> {
   let Size = 128;
 }
 
-class ZPRAsmOperand <string name, int Width>: AsmOperandClass {
+// SVE restricted 4 bit scalable vector register class
+def ZPR_4b : RegisterClass<"AArch64",
+                         [nxv16i8, nxv8i16, nxv4i32, nxv2i64,
+                          nxv2f16, nxv4f16, nxv8f16,
+                          nxv1f32, nxv2f32, nxv4f32,
+                          nxv1f64, nxv2f64],
+                         128, (sequence "Z%u", 0, 15)> {
+  let Size = 128;
+}
+
+// SVE restricted 3 bit scalable vector register class
+def ZPR_3b : RegisterClass<"AArch64",
+                         [nxv16i8, nxv8i16, nxv4i32, nxv2i64,
+                          nxv2f16, nxv4f16, nxv8f16,
+                          nxv1f32, nxv2f32, nxv4f32,
+                          nxv1f64, nxv2f64],
+                         128, (sequence "Z%u", 0, 7)> {
+  let Size = 128;
+}
+
+class ZPRAsmOperand<string name, int Width, string RegClassSuffix = "">
+    : AsmOperandClass {
   let Name = "SVE" # name # "Reg";
   let PredicateMethod = "isSVEDataVectorRegOfWidth<"
-                            # Width # ", AArch64::ZPRRegClassID>";
+                            # Width # ", AArch64::ZPR"
+                            # RegClassSuffix # "RegClassID>";
   let RenderMethod = "addRegOperands";
-  let DiagnosticType = "InvalidZPR" # Width;
+  let DiagnosticType = "InvalidZPR" # RegClassSuffix # Width;
   let ParserMethod = "tryParseSVEDataVector<false, "
                                # !if(!eq(Width, 0), "false", "true") # ">";
 }
 
 def ZPRAsmOpAny : ZPRAsmOperand<"VectorAny", 0>;
 def ZPRAsmOp8   : ZPRAsmOperand<"VectorB",   8>;
 def ZPRAsmOp16  : ZPRAsmOperand<"VectorH",   16>;
 def ZPRAsmOp32  : ZPRAsmOperand<"VectorS",   32>;
 def ZPRAsmOp64  : ZPRAsmOperand<"VectorD",   64>;
 def ZPRAsmOp128 : ZPRAsmOperand<"VectorQ",   128>;
 
 def ZPRAny  : ZPRRegOp<"",  ZPRAsmOpAny, ZPR>;
 def ZPR8    : ZPRRegOp<"b", ZPRAsmOp8,   ZPR>;
 def ZPR16   : ZPRRegOp<"h", ZPRAsmOp16,  ZPR>;
 def ZPR32   : ZPRRegOp<"s", ZPRAsmOp32,  ZPR>;
 def ZPR64   : ZPRRegOp<"d", ZPRAsmOp64,  ZPR>;
 def ZPR128  : ZPRRegOp<"q", ZPRAsmOp128, ZPR>;
 
+def ZPRAsmOp3b8   : ZPRAsmOperand<"Vector3bB", 8, "_3b">;
+def ZPRAsmOp3b16  : ZPRAsmOperand<"Vector3bH", 16, "_3b">;
+def ZPRAsmOp3b32  : ZPRAsmOperand<"Vector3bS", 32, "_3b">;
+
+def ZPR3b8  : ZPRRegOp<"b", ZPRAsmOp3b8,  ZPR_3b>;
+def ZPR3b16 : ZPRRegOp<"h", ZPRAsmOp3b16, ZPR_3b>;
+def ZPR3b32 : ZPRRegOp<"s", ZPRAsmOp3b32, ZPR_3b>;
+
+def ZPRAsmOp4b16  : ZPRAsmOperand<"Vector4bH", 16, "_4b">;
+def ZPRAsmOp4b32  : ZPRAsmOperand<"Vector4bS", 32, "_4b">;
+def ZPRAsmOp4b64  : ZPRAsmOperand<"Vector4bD", 64, "_4b">;
+
+def ZPR4b16 : ZPRRegOp<"h", ZPRAsmOp4b16, ZPR_4b>;
+def ZPR4b32 : ZPRRegOp<"s", ZPRAsmOp4b32, ZPR_4b>;
+def ZPR4b64 : ZPRRegOp<"d", ZPRAsmOp4b64, ZPR_4b>;
+
 class FPRasZPR<int Width> : AsmOperandClass{
   let Name = "FPR" # Width # "asZPR";
   let PredicateMethod = "isFPRasZPR<AArch64::FPR" # Width # "RegClassID>";
   let RenderMethod = "addFPRasZPRRegOperands<" # Width # ">";
 }
 
 class FPRasZPROperand<int Width> : RegisterOperand<ZPR> {
   let ParserMatchClass = FPRasZPR<Width>;

lib/Target/AArch64/AArch64SVEInstrInfo.td

let Predicates = [HasSVE] in {
   defm ORR_ZI : sve_int_log_imm<0b00, "orr", "orn">;
   defm EOR_ZI : sve_int_log_imm<0b01, "eor", "eon">;
   defm AND_ZI : sve_int_log_imm<0b10, "and", "bic">;
 
   defm FADD_ZPmI    : sve_fp_2op_i_p_zds<0b000, "fadd", sve_fpimm_half_one>;
   defm FMUL_ZPmI    : sve_fp_2op_i_p_zds<0b010, "fmul", sve_fpimm_half_two>;
   defm FMAX_ZPmI    : sve_fp_2op_i_p_zds<0b110, "fmax", sve_fpimm_zero_one>;
 
+
+  defm FMUL_ZZZI   : sve_fp_fmul_by_indexed_elem<"fmul">;
+
   // Splat immediate (unpredicated)
   defm DUP_ZI   : sve_int_dup_imm<"dup">;
   defm FDUP_ZI  : sve_int_dup_fpimm<"fdup">;
   defm DUPM_ZI : sve_int_dup_mask_imm<"dupm">;
 
   // Splat immediate (predicated)
   defm CPY_ZPmI  : sve_int_dup_imm_pred_merge<"cpy">;
   defm CPY_ZPzI  : sve_int_dup_imm_pred_zero<"cpy">;

lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp

return Kind == k_Register && Reg.Kind == RegKind::NeonVector &&
            AArch64MCRegisterClasses[AArch64::FPR128_loRegClassID].contains(
                Reg.RegNum);
   }
 
   template <unsigned Class> bool isSVEVectorReg() const {
     RegKind RK;
     switch (Class) {
     case AArch64::ZPRRegClassID:
+    case AArch64::ZPR_3bRegClassID:
+    case AArch64::ZPR_4bRegClassID:
       RK = RegKind::SVEDataVector;
       break;
     case AArch64::PPRRegClassID:
     case AArch64::PPR_3bRegClassID:
       RK = RegKind::SVEPredicateVector;
       break;
     default:
       llvm_unreachable("Unsupport register class");
bool AArch64AsmParser::showMatchError(SMLoc Loc, unsigned ErrCode,
   case Match_InvalidZPR0:
     return Error(Loc, "expected register without element width sufix");
   case Match_InvalidZPR8:
   case Match_InvalidZPR16:
   case Match_InvalidZPR32:
   case Match_InvalidZPR64:
   case Match_InvalidZPR128:
     return Error(Loc, "invalid element width");
+  case Match_InvalidZPR_3b8:
+    return Error(Loc, "Invalid restricted vector register, expected z0.b..z7.b");
+  case Match_InvalidZPR_3b16:
+    return Error(Loc, "Invalid restricted vector register, expected z0.h..z7.h");
+  case Match_InvalidZPR_3b32:
+    return Error(Loc, "Invalid restricted vector register, expected z0.s..z7.s");
+  case Match_InvalidZPR_4b16:
+    return Error(Loc, "Invalid restricted vector register, expected z0.h..z15.h");
+  case Match_InvalidZPR_4b32:
+    return Error(Loc, "Invalid restricted vector register, expected z0.s..z15.s");
+  case Match_InvalidZPR_4b64:
+    return Error(Loc, "Invalid restricted vector register, expected z0.d..z15.d");
   case Match_InvalidSVEPattern:
     return Error(Loc, "invalid predicate pattern");
   case Match_InvalidSVEPredicateAnyReg:
   case Match_InvalidSVEPredicateBReg:
   case Match_InvalidSVEPredicateHReg:
   case Match_InvalidSVEPredicateSReg:
   case Match_InvalidSVEPredicateDReg:
     return Error(Loc, "invalid predicate register.");
bool AArch64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
   case Match_InvalidZPR64LSL32:
   case Match_InvalidZPR64LSL64:
   case Match_InvalidZPR0:
   case Match_InvalidZPR8:
   case Match_InvalidZPR16:
   case Match_InvalidZPR32:
   case Match_InvalidZPR64:
   case Match_InvalidZPR128:
+  case Match_InvalidZPR_3b8:
+  case Match_InvalidZPR_3b16:
+  case Match_InvalidZPR_3b32:
+  case Match_InvalidZPR_4b16:
+  case Match_InvalidZPR_4b32:
+  case Match_InvalidZPR_4b64:
   case Match_InvalidSVEPredicateAnyReg:
   case Match_InvalidSVEPattern:
   case Match_InvalidSVEPredicateBReg:
   case Match_InvalidSVEPredicateHReg:
   case Match_InvalidSVEPredicateSReg:
   case Match_InvalidSVEPredicateDReg:
   case Match_InvalidSVEPredicate3bAnyReg:
   case Match_InvalidSVEPredicate3bBReg:

lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp

static DecodeStatus DecodeDDDRegisterClass(MCInst &Inst, unsigned RegNo,
                                            uint64_t Address,
                                            const void *Decoder);
 static DecodeStatus DecodeDDDDRegisterClass(MCInst &Inst, unsigned RegNo,
                                             uint64_t Address,
                                             const void *Decoder);
 static DecodeStatus DecodeZPRRegisterClass(MCInst &Inst, unsigned RegNo,
                                            uint64_t Address,
                                            const void *Decode);
+static DecodeStatus DecodeZPR_4bRegisterClass(MCInst &Inst, unsigned RegNo,
+                                              uint64_t Address,
+                                              const void *Decode);
+static DecodeStatus DecodeZPR_3bRegisterClass(MCInst &Inst, unsigned RegNo,
+                                              uint64_t Address,
+                                              const void *Decode);
 static DecodeStatus DecodeZPR2RegisterClass(MCInst &Inst, unsigned RegNo,
                                             uint64_t Address,
                                             const void *Decode);
 static DecodeStatus DecodeZPR3RegisterClass(MCInst &Inst, unsigned RegNo,
                                             uint64_t Address,
                                             const void *Decode);
 static DecodeStatus DecodeZPR4RegisterClass(MCInst &Inst, unsigned RegNo,
                                             uint64_t Address,
static DecodeStatus DecodeZPRRegisterClass(MCInst &Inst, unsigned RegNo,
   if (RegNo > 31)
     return Fail;
 
   unsigned Register = ZPRDecoderTable[RegNo];
   Inst.addOperand(MCOperand::createReg(Register));
   return Success;
 }
 
+static DecodeStatus DecodeZPR_4bRegisterClass(MCInst &Inst, unsigned RegNo,
+                                           uint64_t Address,
+                                           const void* Decoder) {
+  if (RegNo > 15)
+    return Fail;
+  return DecodeZPRRegisterClass(Inst, RegNo, Address, Decoder);
+}
+
+static DecodeStatus DecodeZPR_3bRegisterClass(MCInst &Inst, unsigned RegNo,
+                                           uint64_t Address,
+                                           const void* Decoder) {
+  if (RegNo > 7)
+    return Fail;
+  return DecodeZPRRegisterClass(Inst, RegNo, Address, Decoder);
+}
+
 static const unsigned ZZDecoderTable[] = {
   AArch64::Z0_Z1,   AArch64::Z1_Z2,   AArch64::Z2_Z3,   AArch64::Z3_Z4,
   AArch64::Z4_Z5,   AArch64::Z5_Z6,   AArch64::Z6_Z7,   AArch64::Z7_Z8,
   AArch64::Z8_Z9,   AArch64::Z9_Z10,  AArch64::Z10_Z11, AArch64::Z11_Z12,
   AArch64::Z12_Z13, AArch64::Z13_Z14, AArch64::Z14_Z15, AArch64::Z15_Z16,
   AArch64::Z16_Z17, AArch64::Z17_Z18, AArch64::Z18_Z19, AArch64::Z19_Z20,
   AArch64::Z20_Z21, AArch64::Z21_Z22, AArch64::Z22_Z23, AArch64::Z23_Z24,
   AArch64::Z24_Z25, AArch64::Z25_Z26, AArch64::Z26_Z27, AArch64::Z27_Z28,

lib/Target/AArch64/SVEInstrFormats.td

 }
 
 multiclass sve_fp_2op_i_p_zds<bits<3> opc, string asm, Operand imm_ty> {
   def _H : sve_fp_2op_i_p_zds<0b01, opc, asm, ZPR16, imm_ty>;
   def _S : sve_fp_2op_i_p_zds<0b10, opc, asm, ZPR32, imm_ty>;
   def _D : sve_fp_2op_i_p_zds<0b11, opc, asm, ZPR64, imm_ty>;
 }
 
+//===----------------------------------------------------------------------===//
+// SVE Floating Point Multiply - Indexed Group
+//===----------------------------------------------------------------------===//
+
+class sve_fp_fmul_by_indexed_elem<bits<2> sz, string asm, ZPRRegOp zprty,
+                                      ZPRRegOp zprty2, Operand itype>
+: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty2:$Zm, itype:$iop),
+  asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> {
+  bits<5> Zd;
+  bits<5> Zn;
+  let Inst{31-24} = 0b01100100;
+  let Inst{23-22} = sz;
+  let Inst{21}    = 0b1;
+  let Inst{15-10} = 0b001000;
+  let Inst{9-5}   = Zn;
+  let Inst{4-0}   = Zd;
+}
+
+multiclass sve_fp_fmul_by_indexed_elem<string asm> {
+  def _H : sve_fp_fmul_by_indexed_elem<{0, ?}, asm, ZPR16, ZPR3b16, VectorIndexH> {
+    bits<3> Zm;
+    bits<3> iop;
+    let Inst{22} = iop{2};
+    let Inst{20-19} = iop{1-0};
+    let Inst{18-16} = Zm;
+  }
+  def _S : sve_fp_fmul_by_indexed_elem<0b10, asm, ZPR32, ZPR3b32, VectorIndexS> {
+    bits<3> Zm;
+    bits<2> iop;
+    let Inst{20-19} = iop;
+    let Inst{18-16} = Zm;
+  }
+  def _D : sve_fp_fmul_by_indexed_elem<0b11, asm, ZPR64, ZPR4b64, VectorIndexD> {
+    bits<4> Zm;
+    bit iop;
+    let Inst{20} = iop;
+    let Inst{19-16} = Zm;
+  }
+}
 
 //===----------------------------------------------------------------------===//
 // SVE Stack Allocation Group
 //===----------------------------------------------------------------------===//
 
 class sve_int_arith_vl<bit opc, string asm>
 : I<(outs GPR64sp:$Rd), (ins GPR64sp:$Rn, simm6_32b:$imm6),
   asm, "\t$Rd, $Rn, $imm6",

test/MC/AArch64/SVE/fmul-diagnostics.s

 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid floating point constant, expected 0.5 or 2.0.
 // CHECK-NEXT: fmul z0.h, p0/m, z0.h, #2.0000000000000000000000001
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 fmul z0.h, p0/m, z0.h, #1.9999999999999999999999999
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid floating point constant, expected 0.5 or 2.0.
 // CHECK-NEXT: fmul z0.h, p0/m, z0.h, #1.9999999999999999999999999
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+
+// --------------------------------------------------------------------------//
+// Restricted ZPR range
+
+fmul z0.h, z0.h, z8.b[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected z0.h..z7.h
+// CHECK-NEXT: fmul z0.h, z0.h, z8.b[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmul z0.h, z0.h, z8.h[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected z0.h..z7.h
+// CHECK-NEXT: fmul z0.h, z0.h, z8.h[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmul z0.s, z0.s, z8.s[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected z0.s..z7.s
+// CHECK-NEXT: fmul z0.s, z0.s, z8.s[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmul z0.d, z0.d, z16.d[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected z0.d..z15.d
+// CHECK-NEXT: fmul z0.d, z0.d, z16.d[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+
+// --------------------------------------------------------------------------//
+// Index out of bounds
+
+fmul z0.h, z0.h, z0.h[-1]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 7].
+// CHECK-NEXT: fmul z0.h, z0.h, z0.h[-1]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmul z0.h, z0.h, z0.h[8]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 7].
+// CHECK-NEXT: fmul z0.h, z0.h, z0.h[8]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmul z0.s, z0.s, z0.s[-1]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 3].
+// CHECK-NEXT: fmul z0.s, z0.s, z0.s[-1]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmul z0.s, z0.s, z0.s[4]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 3].
+// CHECK-NEXT: fmul z0.s, z0.s, z0.s[4]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmul z0.d, z0.d, z0.d[-1]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 1].
+// CHECK-NEXT: fmul z0.d, z0.d, z0.d[-1]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+fmul z0.d, z0.d, z0.d[2]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 1].
+// CHECK-NEXT: fmul z0.d, z0.d, z0.d[2]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:

test/MC/AArch64/SVE/fmul.s

 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 3f 9c 9a 65 <unknown>
 
 fmul    z31.d, p7/m, z31.d, #2.0
 // CHECK-INST: fmul    z31.d, p7/m, z31.d, #2.0
 // CHECK-ENCODING: [0x3f,0x9c,0xda,0x65]
 // CHECK-ERROR: instruction requires: sve
 // CHECK-UNKNOWN: 3f 9c da 65 <unknown>
+
+fmul    z0.h, z0.h, z0.h[0]
+// CHECK-INST: fmul    z0.h, z0.h, z0.h[0]
+// CHECK-ENCODING: [0x00,0x20,0x20,0x64]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 20 20 64 <unknown>
+
+fmul    z0.s, z0.s, z0.s[0]
+// CHECK-INST: fmul    z0.s, z0.s, z0.s[0]
+// CHECK-ENCODING: [0x00,0x20,0xa0,0x64]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 20 a0 64 <unknown>
+
+fmul    z0.d, z0.d, z0.d[0]
+// CHECK-INST: fmul    z0.d, z0.d, z0.d[0]
+// CHECK-ENCODING: [0x00,0x20,0xe0,0x64]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 20 e0 64 <unknown>
+
+fmul    z31.h, z31.h, z7.h[7]
+// CHECK-INST: fmul    z31.h, z31.h, z7.h[7]
+// CHECK-ENCODING: [0xff,0x23,0x7f,0x64]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 23 7f 64 <unknown>
+
+fmul    z31.s, z31.s, z7.s[3]
+// CHECK-INST: fmul    z31.s, z31.s, z7.s[3]
+// CHECK-ENCODING: [0xff,0x23,0xbf,0x64]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 23 bf 64 <unknown>
+
+fmul    z31.d, z31.d, z15.d[1]
+// CHECK-INST: fmul    z31.d, z31.d, z15.d[1]
+// CHECK-ENCODING: [0xff,0x23,0xff,0x64]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 23 ff 64 <unknown>