diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -2171,10 +2171,10 @@ } // End HasBF16, HasSVE let Predicates = [HasBF16, HasSVEorSME] in { - defm BFMLALB_ZZZ : sve_bfloat_matmul_longvecl<0b0, 0b0, "bfmlalb", int_aarch64_sve_bfmlalb>; - defm BFMLALT_ZZZ : sve_bfloat_matmul_longvecl<0b1, 0b0, "bfmlalt", int_aarch64_sve_bfmlalt>; - defm BFMLALB_ZZZI : sve_bfloat_matmul_longvecl_idx<0b0, 0b0, "bfmlalb", int_aarch64_sve_bfmlalb_lane_v2>; - defm BFMLALT_ZZZI : sve_bfloat_matmul_longvecl_idx<0b1, 0b0, "bfmlalt", int_aarch64_sve_bfmlalt_lane_v2>; + defm BFMLALB_ZZZ : sve2_fp_mla_long<0b100, "bfmlalb", nxv4f32, nxv8bf16, int_aarch64_sve_bfmlalb>; + defm BFMLALT_ZZZ : sve2_fp_mla_long<0b101, "bfmlalt", nxv4f32, nxv8bf16, int_aarch64_sve_bfmlalt>; + defm BFMLALB_ZZZI : sve2_fp_mla_long_by_indexed_elem<0b100, "bfmlalb", nxv4f32, nxv8bf16, int_aarch64_sve_bfmlalb_lane_v2>; + defm BFMLALT_ZZZI : sve2_fp_mla_long_by_indexed_elem<0b101, "bfmlalt", nxv4f32, nxv8bf16, int_aarch64_sve_bfmlalt_lane_v2>; defm BFCVT_ZPmZ : sve_bfloat_convert<0b1, "bfcvt", int_aarch64_sve_fcvt_bf16f32>; defm BFCVTNT_ZPmZ : sve_bfloat_convert<0b0, "bfcvtnt", int_aarch64_sve_fcvtnt_bf16f32>; } // End HasBF16, HasSVEorSME @@ -3518,16 +3518,16 @@ defm FMINP_ZPmZZ : sve2_fp_pairwise_pred<0b111, "fminp", int_aarch64_sve_fminp>; // SVE2 floating-point multiply-add long (indexed) - defm FMLALB_ZZZI_SHH : sve2_fp_mla_long_by_indexed_elem<0b00, "fmlalb", int_aarch64_sve_fmlalb_lane>; - defm FMLALT_ZZZI_SHH : sve2_fp_mla_long_by_indexed_elem<0b01, "fmlalt", int_aarch64_sve_fmlalt_lane>; - defm FMLSLB_ZZZI_SHH : sve2_fp_mla_long_by_indexed_elem<0b10, "fmlslb", int_aarch64_sve_fmlslb_lane>; - defm FMLSLT_ZZZI_SHH : sve2_fp_mla_long_by_indexed_elem<0b11, "fmlslt", int_aarch64_sve_fmlslt_lane>; + defm FMLALB_ZZZI_SHH : sve2_fp_mla_long_by_indexed_elem<0b000, "fmlalb", nxv4f32, nxv8f16, int_aarch64_sve_fmlalb_lane>; + defm FMLALT_ZZZI_SHH : sve2_fp_mla_long_by_indexed_elem<0b001, "fmlalt", nxv4f32, nxv8f16, int_aarch64_sve_fmlalt_lane>; + defm FMLSLB_ZZZI_SHH : sve2_fp_mla_long_by_indexed_elem<0b010, "fmlslb", nxv4f32, nxv8f16, int_aarch64_sve_fmlslb_lane>; + defm FMLSLT_ZZZI_SHH : sve2_fp_mla_long_by_indexed_elem<0b011, "fmlslt", nxv4f32, nxv8f16, int_aarch64_sve_fmlslt_lane>; // SVE2 floating-point multiply-add long - defm FMLALB_ZZZ_SHH : sve2_fp_mla_long<0b00, "fmlalb", int_aarch64_sve_fmlalb>; - defm FMLALT_ZZZ_SHH : sve2_fp_mla_long<0b01, "fmlalt", int_aarch64_sve_fmlalt>; - defm FMLSLB_ZZZ_SHH : sve2_fp_mla_long<0b10, "fmlslb", int_aarch64_sve_fmlslb>; - defm FMLSLT_ZZZ_SHH : sve2_fp_mla_long<0b11, "fmlslt", int_aarch64_sve_fmlslt>; + defm FMLALB_ZZZ_SHH : sve2_fp_mla_long<0b000, "fmlalb", nxv4f32, nxv8f16, int_aarch64_sve_fmlalb>; + defm FMLALT_ZZZ_SHH : sve2_fp_mla_long<0b001, "fmlalt", nxv4f32, nxv8f16, int_aarch64_sve_fmlalt>; + defm FMLSLB_ZZZ_SHH : sve2_fp_mla_long<0b010, "fmlslb", nxv4f32, nxv8f16, int_aarch64_sve_fmlslb>; + defm FMLSLT_ZZZ_SHH : sve2_fp_mla_long<0b011, "fmlslt", nxv4f32, nxv8f16, int_aarch64_sve_fmlslt>; // SVE2 bitwise ternary operations defm EOR3_ZZZZ : sve2_int_bitwise_ternary_op<0b000, "eor3", AArch64eor3>; @@ -3655,10 +3655,10 @@ defm FCLAMP_ZZZ : sve2p1_fclamp<"fclamp">; def FDOT_ZZZ_S : sve_float_dot<0b0, "fdot">; def FDOT_ZZZI_S : sve_float_dot_indexed<0b0, "fdot">; -def BFMLSLB_ZZZ_S : sve_bfloat_matmul_longvecl<0b0, 0b1, "bfmlslb">; -def BFMLSLT_ZZZ_S : sve_bfloat_matmul_longvecl<0b1, 0b1, "bfmlslt">; -def BFMLSLB_ZZZI_S : sve_bfloat_matmul_longvecl_idx<0b0, 0b1, "bfmlslb">; -def BFMLSLT_ZZZI_S : sve_bfloat_matmul_longvecl_idx<0b1, 0b1, "bfmlslt">; +def BFMLSLB_ZZZ_S : sve2_fp_mla_long<0b110, "bfmlslb">; +def BFMLSLT_ZZZ_S : sve2_fp_mla_long<0b111, "bfmlslt">; +def BFMLSLB_ZZZI_S : sve2_fp_mla_long_by_indexed_elem<0b110, "bfmlslb">; +def BFMLSLT_ZZZI_S : sve2_fp_mla_long_by_indexed_elem<0b111, "bfmlslt">; def SDOT_ZZZ_HtoS : sve2p1_two_way_dot_vv<"sdot", 0b0>; def UDOT_ZZZ_HtoS : sve2p1_two_way_dot_vv<"udot", 0b1>; diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -2525,7 +2525,7 @@ // SVE2 Floating Point Widening Multiply-Add - Indexed Group //===----------------------------------------------------------------------===// -class sve2_fp_mla_long_by_indexed_elem opc, string asm> +class sve2_fp_mla_long_by_indexed_elem opc, string asm> : I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexH32b:$iop), asm, "\t$Zda, $Zn, $Zm$iop", @@ -2535,7 +2535,9 @@ bits<5> Zn; bits<3> Zm; bits<3> iop; - let Inst{31-21} = 0b01100100101; + let Inst{31-23} = 0b011001001; + let Inst{22} = opc{2}; + let Inst{21} = 0b1; let Inst{20-19} = iop{2-1}; let Inst{18-16} = Zm; let Inst{15-14} = 0b01; @@ -2551,17 +2553,18 @@ let ElementSize = ElementSizeNone; } -multiclass sve2_fp_mla_long_by_indexed_elem opc, string asm, +multiclass sve2_fp_mla_long_by_indexed_elem opc, string asm, + ValueType OutVT, ValueType InVT, SDPatternOperator op> { def NAME : sve2_fp_mla_long_by_indexed_elem; - def : SVE_4_Op_Imm_Pat(NAME)>; + def : SVE_4_Op_Imm_Pat(NAME)>; } //===----------------------------------------------------------------------===// // SVE2 Floating Point Widening Multiply-Add Group //===----------------------------------------------------------------------===// -class sve2_fp_mla_long opc, string asm> +class sve2_fp_mla_long opc, string asm> : I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), asm, "\t$Zda, $Zn, $Zm", "", @@ -2569,7 +2572,9 @@ bits<5> Zda; bits<5> Zn; bits<5> Zm; - let Inst{31-21} = 0b01100100101; + let Inst{31-23} = 0b011001001; + let Inst{22} = opc{2}; + let Inst{21} = 0b1; let Inst{20-16} = Zm; let Inst{15-14} = 0b10; let Inst{13} = opc{1}; @@ -2583,9 +2588,10 @@ let ElementSize = ElementSizeNone; } -multiclass sve2_fp_mla_long opc, string asm, SDPatternOperator op> { +multiclass sve2_fp_mla_long opc, string asm, ValueType OutVT, + ValueType InVT, SDPatternOperator op> { def NAME : sve2_fp_mla_long; - def : SVE_3_Op_Pat(NAME)>; + def : SVE_3_Op_Pat(NAME)>; } //===----------------------------------------------------------------------===// @@ -8369,47 +8375,6 @@ def : SVE_3_Op_Pat(NAME)>; } -class sve_bfloat_matmul_longvecl -: sve_bfloat_matmul { - let Inst{23} = 0b1; - let Inst{14} = 0b0; - let Inst{13} = sub; - let Inst{10} = BT; -} - -multiclass sve_bfloat_matmul_longvecl { - def NAME : sve_bfloat_matmul_longvecl; - def : SVE_3_Op_Pat(NAME)>; -} - -class sve_bfloat_matmul_longvecl_idx -: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexH32b:$iop), - asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> { - bits<5> Zda; - bits<5> Zn; - bits<3> Zm; - bits<3> iop; - let Inst{31-21} = 0b01100100111; - let Inst{20-19} = iop{2-1}; - let Inst{18-16} = Zm; - let Inst{15-14} = 0b01; - let Inst{13} = sub; - let Inst{12} = 0b0; - let Inst{11} = iop{0}; - let Inst{10} = BT; - let Inst{9-5} = Zn; - let Inst{4-0} = Zda; - - let Constraints = "$Zda = $_Zda"; - let DestructiveInstType = DestructiveOther; - let ElementSize = ElementSizeH; -} - -multiclass sve_bfloat_matmul_longvecl_idx { - def NAME : sve_bfloat_matmul_longvecl_idx; - def : SVE_4_Op_Imm_Pat(NAME)>; -} - class sve_bfloat_convert : I<(outs ZPR16:$Zd), (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn), asm, "\t$Zd, $Pg/m, $Zn", "", []>, Sched<[]> {