Index: llvm/include/llvm/IR/IntrinsicsAArch64.td =================================================================== --- llvm/include/llvm/IR/IntrinsicsAArch64.td +++ llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -3040,6 +3040,27 @@ } } + // + // Multi-vector multiply-add long long + // + + foreach ty = ["s", "u"] in { + foreach instr = ["mla", "mls"] in { + foreach za = ["za32", "za64"] in { + def int_aarch64_sme_ # ty # instr # _ # za # _single_vg4x1 : SME2_Matrix_ArrayVector_Single_Single_Intrinsic; + def int_aarch64_sme_ # ty # instr # _ # za # _single_vg4x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; + def int_aarch64_sme_ # ty # instr # _ # za # _single_vg4x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; + } + } + } + + def int_aarch64_sme_sumla_za32_single_vg4x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; + def int_aarch64_sme_sumla_za32_single_vg4x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; + + def int_aarch64_sme_usmla_za32_single_vg4x1 : SME2_Matrix_ArrayVector_Single_Single_Intrinsic; + def int_aarch64_sme_usmla_za32_single_vg4x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; + def int_aarch64_sme_usmla_za32_single_vg4x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; + // Multi-vector signed saturating doubling multiply high def int_aarch64_sve_sqdmulh_single_vgx2 : SME2_VG2_Multi_Single_Intrinsic; Index: llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td =================================================================== --- llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td +++ llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td @@ -515,51 +515,51 @@ def SMLALL_MZZI_BtoS : sme2_mla_ll_array_index_32b<"smlall", 0b000>; defm SMLALL_VG2_M2ZZI_BtoS : sme2_mla_ll_array_vg2_index_32b<"smlall", 0b000>; defm SMLALL_VG4_M4ZZI_BtoS : sme2_mla_ll_array_vg4_index_32b<"smlall", 0b000>; -def SMLALL_MZZ_BtoS : sme2_mla_ll_array_single<"smlall", 0b0000, MatrixOp32, ZPR8, ZPR4b8>; -defm SMLALL_VG2_M2ZZ_BtoS : sme2_mla_ll_array_vg24_single<"smlall", 0b00000, MatrixOp32, ZZ_b, ZPR4b8>; -defm SMLALL_VG4_M4ZZ_BtoS : sme2_mla_ll_array_vg24_single<"smlall", 0b01000, MatrixOp32, ZZZZ_b, ZPR4b8>; +defm SMLALL_MZZ_BtoS : sme2_mla_ll_array_single<"smlall", 0b0000, MatrixOp32, ZPR8, ZPR4b8, nxv16i8, int_aarch64_sme_smla_za32_single_vg4x1>; +defm SMLALL_VG2_M2ZZ_BtoS : sme2_mla_ll_array_vg2_single<"smlall", 0b00000, MatrixOp32, ZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_smla_za32_single_vg4x2>; +defm SMLALL_VG4_M4ZZ_BtoS : sme2_mla_ll_array_vg4_single<"smlall", 0b01000, MatrixOp32, ZZZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_smla_za32_single_vg4x4>; defm SMLALL_VG2_M2Z2Z_BtoS : sme2_mla_ll_array_vg2_multi<"smlall", 0b0000, MatrixOp32, ZZ_b_mul_r>; defm SMLALL_VG4_M4Z4Z_BtoS : sme2_mla_ll_array_vg4_multi<"smlall", 0b0000, MatrixOp32, ZZZZ_b_mul_r>; def USMLALL_MZZI_BtoS : sme2_mla_ll_array_index_32b<"usmlall", 0b001>; defm USMLALL_VG2_M2ZZI_BtoS : sme2_mla_ll_array_vg2_index_32b<"usmlall", 0b100>; defm USMLALL_VG4_M4ZZI_BtoS : sme2_mla_ll_array_vg4_index_32b<"usmlall", 0b100>; -def USMLALL_MZZ_BtoS : sme2_mla_ll_array_single<"usmlall", 0b0001, MatrixOp32, ZPR8, ZPR4b8>; -defm USMLALL_VG2_M2ZZ_BtoS : sme2_mla_ll_array_vg24_single<"usmlall", 0b00001, MatrixOp32, ZZ_b, ZPR4b8>; -defm USMLALL_VG4_M4ZZ_BtoS : sme2_mla_ll_array_vg24_single<"usmlall", 0b01001, MatrixOp32, ZZZZ_b, ZPR4b8>; +defm USMLALL_MZZ_BtoS : sme2_mla_ll_array_single<"usmlall", 0b0001, MatrixOp32, ZPR8, ZPR4b8, nxv16i8, int_aarch64_sme_usmla_za32_single_vg4x1>; +defm USMLALL_VG2_M2ZZ_BtoS : sme2_mla_ll_array_vg2_single<"usmlall", 0b00001, MatrixOp32, ZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_usmla_za32_single_vg4x2>; +defm USMLALL_VG4_M4ZZ_BtoS : sme2_mla_ll_array_vg4_single<"usmlall", 0b01001, MatrixOp32, ZZZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_usmla_za32_single_vg4x4>; defm USMLALL_VG2_M2Z2Z_BtoS : sme2_mla_ll_array_vg2_multi<"usmlall", 0b0001, MatrixOp32, ZZ_b_mul_r>; defm USMLALL_VG4_M4Z4Z_BtoS : sme2_mla_ll_array_vg4_multi<"usmlall", 0b0001, MatrixOp32, ZZZZ_b_mul_r>; def SMLSLL_MZZI_BtoS : sme2_mla_ll_array_index_32b<"smlsll", 0b010>; defm SMLSLL_VG2_M2ZZI_BtoS : sme2_mla_ll_array_vg2_index_32b<"smlsll", 0b001>; defm SMLSLL_VG4_M4ZZI_BtoS : sme2_mla_ll_array_vg4_index_32b<"smlsll", 0b001>; -def SMLSLL_MZZ_BtoS : sme2_mla_ll_array_single<"smlsll", 0b0010, MatrixOp32, ZPR8, ZPR4b8>; -defm SMLSLL_VG2_M2ZZ_BtoS : sme2_mla_ll_array_vg24_single<"smlsll", 0b00010, MatrixOp32, ZZ_b, ZPR4b8>; -defm SMLSLL_VG4_M4ZZ_BtoS : sme2_mla_ll_array_vg24_single<"smlsll", 0b01010, MatrixOp32, ZZZZ_b, ZPR4b8>; +defm SMLSLL_MZZ_BtoS : sme2_mla_ll_array_single<"smlsll", 0b0010, MatrixOp32, ZPR8, ZPR4b8, nxv16i8, int_aarch64_sme_smls_za32_single_vg4x1>; +defm SMLSLL_VG2_M2ZZ_BtoS : sme2_mla_ll_array_vg2_single<"smlsll", 0b00010, MatrixOp32, ZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_smls_za32_single_vg4x2>; +defm SMLSLL_VG4_M4ZZ_BtoS : sme2_mla_ll_array_vg4_single<"smlsll", 0b01010, MatrixOp32, ZZZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_smls_za32_single_vg4x4>; defm SMLSLL_VG2_M2Z2Z_BtoS : sme2_mla_ll_array_vg2_multi<"smlsll", 0b0010, MatrixOp32, ZZ_b_mul_r>; defm SMLSLL_VG4_M4Z4Z_BtoS : sme2_mla_ll_array_vg4_multi<"smlsll", 0b0010, MatrixOp32, ZZZZ_b_mul_r>; def UMLALL_MZZI_BtoS : sme2_mla_ll_array_index_32b<"umlall", 0b100>; defm UMLALL_VG2_M2ZZI_BtoS : sme2_mla_ll_array_vg2_index_32b<"umlall", 0b010>; defm UMLALL_VG4_M4ZZI_BtoS : sme2_mla_ll_array_vg4_index_32b<"umlall", 0b010>; -def UMLALL_MZZ_BtoS : sme2_mla_ll_array_single<"umlall", 0b0100, MatrixOp32, ZPR8, ZPR4b8>; -defm UMLALL_VG2_M2ZZ_BtoS : sme2_mla_ll_array_vg24_single<"umlall", 0b00100, MatrixOp32, ZZ_b, ZPR4b8>; -defm UMLALL_VG4_M4ZZ_BtoS : sme2_mla_ll_array_vg24_single<"umlall", 0b01100, MatrixOp32, ZZZZ_b, ZPR4b8>; +defm UMLALL_MZZ_BtoS : sme2_mla_ll_array_single<"umlall", 0b0100, MatrixOp32, ZPR8, ZPR4b8, nxv16i8, int_aarch64_sme_umla_za32_single_vg4x1>; +defm UMLALL_VG2_M2ZZ_BtoS : sme2_mla_ll_array_vg2_single<"umlall", 0b00100, MatrixOp32, ZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_umla_za32_single_vg4x2>; +defm UMLALL_VG4_M4ZZ_BtoS : sme2_mla_ll_array_vg4_single<"umlall", 0b01100, MatrixOp32, ZZZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_umla_za32_single_vg4x4>; defm UMLALL_VG2_M2Z2Z_BtoS : sme2_mla_ll_array_vg2_multi<"umlall", 0b0100, MatrixOp32, ZZ_b_mul_r>; defm UMLALL_VG4_M4Z4Z_BtoS : sme2_mla_ll_array_vg4_multi<"umlall", 0b0100, MatrixOp32, ZZZZ_b_mul_r>; def SUMLALL_MZZI_BtoS : sme2_mla_ll_array_index_32b<"sumlall", 0b101>; defm SUMLALL_VG2_M2ZZI_BtoS : sme2_mla_ll_array_vg2_index_32b<"sumlall", 0b110>; defm SUMLALL_VG4_M4ZZI_BtoS : sme2_mla_ll_array_vg4_index_32b<"sumlall", 0b110>; -defm SUMLALL_VG2_M2ZZ_BtoS : sme2_mla_ll_array_vg24_single<"sumlall", 0b00101, MatrixOp32, ZZ_b, ZPR4b8>; -defm SUMLALL_VG4_M4ZZ_BtoS : sme2_mla_ll_array_vg24_single<"sumlall", 0b01101, MatrixOp32, ZZZZ_b, ZPR4b8>; +defm SUMLALL_VG2_M2ZZ_BtoS : sme2_mla_ll_array_vg2_single<"sumlall", 0b00101, MatrixOp32, ZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_sumla_za32_single_vg4x2>; +defm SUMLALL_VG4_M4ZZ_BtoS : sme2_mla_ll_array_vg4_single<"sumlall", 0b01101, MatrixOp32, ZZZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_sumla_za32_single_vg4x4>; def UMLSLL_MZZI_BtoS : sme2_mla_ll_array_index_32b<"umlsll", 0b110>; defm UMLSLL_VG2_M2ZZI_BtoS : sme2_mla_ll_array_vg2_index_32b<"umlsll", 0b011>; defm UMLSLL_VG4_M4ZZI_BtoS : sme2_mla_ll_array_vg4_index_32b<"umlsll", 0b011>; -def UMLSLL_MZZ_BtoS : sme2_mla_ll_array_single<"umlsll", 0b0110, MatrixOp32, ZPR8, ZPR4b8>; -defm UMLSLL_VG2_M2ZZ_BtoS : sme2_mla_ll_array_vg24_single<"umlsll", 0b00110, MatrixOp32, ZZ_b, ZPR4b8>; -defm UMLSLL_VG4_M4ZZ_BtoS : sme2_mla_ll_array_vg24_single<"umlsll", 0b01110, MatrixOp32, ZZZZ_b, ZPR4b8>; +defm UMLSLL_MZZ_BtoS : sme2_mla_ll_array_single<"umlsll", 0b0110, MatrixOp32, ZPR8, ZPR4b8, nxv16i8, int_aarch64_sme_umls_za32_single_vg4x1>; +defm UMLSLL_VG2_M2ZZ_BtoS : sme2_mla_ll_array_vg2_single<"umlsll", 0b00110, MatrixOp32, ZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_umls_za32_single_vg4x2>; +defm UMLSLL_VG4_M4ZZ_BtoS : sme2_mla_ll_array_vg4_single<"umlsll", 0b01110, MatrixOp32, ZZZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_umls_za32_single_vg4x4>; defm UMLSLL_VG2_M2Z2Z_BtoS : sme2_mla_ll_array_vg2_multi<"umlsll", 0b0110, MatrixOp32, ZZ_b_mul_r>; defm UMLSLL_VG4_M4Z4Z_BtoS : sme2_mla_ll_array_vg4_multi<"umlsll", 0b0110, MatrixOp32, ZZZZ_b_mul_r>; @@ -742,36 +742,36 @@ def SMLALL_MZZI_HtoD : sme2_mla_ll_array_index_64b<"smlall", 0b00>; defm SMLALL_VG2_M2ZZI_HtoD : sme2_mla_ll_array_vg2_index_64b<"smlall", 0b00>; defm SMLALL_VG4_M4ZZI_HtoD : sme2_mla_ll_array_vg4_index_64b<"smlall", 0b00>; -def SMLALL_MZZ_HtoD : sme2_mla_ll_array_single<"smlall", 0b1000, MatrixOp64, ZPR16, ZPR4b16>; -defm SMLALL_VG2_M2ZZ_HtoD : sme2_mla_ll_array_vg24_single<"smlall", 0b10000, MatrixOp64, ZZ_h, ZPR4b16>; -defm SMLALL_VG4_M4ZZ_HtoD : sme2_mla_ll_array_vg24_single<"smlall", 0b11000, MatrixOp64, ZZZZ_h, ZPR4b16>; +defm SMLALL_MZZ_HtoD : sme2_mla_ll_array_single<"smlall", 0b1000, MatrixOp64, ZPR16, ZPR4b16, nxv8i16, int_aarch64_sme_smla_za64_single_vg4x1>; +defm SMLALL_VG2_M2ZZ_HtoD : sme2_mla_ll_array_vg2_single<"smlall", 0b10000, MatrixOp64, ZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_smla_za64_single_vg4x2>; +defm SMLALL_VG4_M4ZZ_HtoD : sme2_mla_ll_array_vg4_single<"smlall", 0b11000, MatrixOp64, ZZZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_smla_za64_single_vg4x4>; defm SMLALL_VG2_M2Z2Z_HtoD : sme2_mla_ll_array_vg2_multi<"smlall", 0b1000, MatrixOp64, ZZ_h_mul_r>; defm SMLALL_VG4_M4Z4Z_HtoD : sme2_mla_ll_array_vg4_multi<"smlall", 0b1000, MatrixOp64, ZZZZ_h_mul_r>; def SMLSLL_MZZI_HtoD : sme2_mla_ll_array_index_64b<"smlsll", 0b01>; defm SMLSLL_VG2_M2ZZI_HtoD : sme2_mla_ll_array_vg2_index_64b<"smlsll", 0b01>; defm SMLSLL_VG4_M4ZZI_HtoD : sme2_mla_ll_array_vg4_index_64b<"smlsll", 0b01>; -def SMLSLL_MZZ_HtoD : sme2_mla_ll_array_single<"smlsll", 0b1010, MatrixOp64, ZPR16, ZPR4b16>; -defm SMLSLL_VG2_M2ZZ_HtoD : sme2_mla_ll_array_vg24_single<"smlsll", 0b10010, MatrixOp64, ZZ_h, ZPR4b16>; -defm SMLSLL_VG4_M4ZZ_HtoD : sme2_mla_ll_array_vg24_single<"smlsll", 0b11010, MatrixOp64, ZZZZ_h, ZPR4b16>; +defm SMLSLL_MZZ_HtoD : sme2_mla_ll_array_single<"smlsll", 0b1010, MatrixOp64, ZPR16, ZPR4b16, nxv8i16, int_aarch64_sme_smls_za64_single_vg4x1>; +defm SMLSLL_VG2_M2ZZ_HtoD : sme2_mla_ll_array_vg2_single<"smlsll", 0b10010, MatrixOp64, ZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_smls_za64_single_vg4x2>; +defm SMLSLL_VG4_M4ZZ_HtoD : sme2_mla_ll_array_vg4_single<"smlsll", 0b11010, MatrixOp64, ZZZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_smls_za64_single_vg4x4>; defm SMLSLL_VG2_M2Z2Z_HtoD : sme2_mla_ll_array_vg2_multi<"smlsll", 0b1010, MatrixOp64, ZZ_h_mul_r>; defm SMLSLL_VG4_M4Z4Z_HtoD : sme2_mla_ll_array_vg4_multi<"smlsll", 0b1010, MatrixOp64, ZZZZ_h_mul_r>; def UMLALL_MZZI_HtoD : sme2_mla_ll_array_index_64b<"umlall", 0b10>; defm UMLALL_VG2_M2ZZI_HtoD : sme2_mla_ll_array_vg2_index_64b<"umlall", 0b10>; defm UMLALL_VG4_M4ZZI_HtoD : sme2_mla_ll_array_vg4_index_64b<"umlall", 0b10>; -def UMLALL_MZZ_HtoD : sme2_mla_ll_array_single<"umlall", 0b1100, MatrixOp64, ZPR16, ZPR4b16>; -defm UMLALL_VG2_M2ZZ_HtoD : sme2_mla_ll_array_vg24_single<"umlall", 0b10100, MatrixOp64, ZZ_h, ZPR4b16>; -defm UMLALL_VG4_M4ZZ_HtoD : sme2_mla_ll_array_vg24_single<"umlall", 0b11100, MatrixOp64, ZZZZ_h, ZPR4b16>; +defm UMLALL_MZZ_HtoD : sme2_mla_ll_array_single<"umlall", 0b1100, MatrixOp64, ZPR16, ZPR4b16, nxv8i16, int_aarch64_sme_umla_za64_single_vg4x1>; +defm UMLALL_VG2_M2ZZ_HtoD : sme2_mla_ll_array_vg2_single<"umlall", 0b10100, MatrixOp64, ZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_umla_za64_single_vg4x2>; +defm UMLALL_VG4_M4ZZ_HtoD : sme2_mla_ll_array_vg4_single<"umlall", 0b11100, MatrixOp64, ZZZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_umla_za64_single_vg4x4>; defm UMLALL_VG2_M2Z2Z_HtoD : sme2_mla_ll_array_vg2_multi<"umlall", 0b1100, MatrixOp64, ZZ_h_mul_r>; defm UMLALL_VG4_M4Z4Z_HtoD : sme2_mla_ll_array_vg4_multi<"umlall", 0b1100, MatrixOp64, ZZZZ_h_mul_r>; def UMLSLL_MZZI_HtoD : sme2_mla_ll_array_index_64b<"umlsll", 0b11>; defm UMLSLL_VG2_M2ZZI_HtoD : sme2_mla_ll_array_vg2_index_64b<"umlsll", 0b11>; defm UMLSLL_VG4_M4ZZI_HtoD : sme2_mla_ll_array_vg4_index_64b<"umlsll", 0b11>; -def UMLSLL_MZZ_HtoD : sme2_mla_ll_array_single<"umlsll", 0b1110, MatrixOp64, ZPR16, ZPR4b16>; -defm UMLSLL_VG2_M2ZZ_HtoD : sme2_mla_ll_array_vg24_single<"umlsll", 0b10110, MatrixOp64, ZZ_h, ZPR4b16>; -defm UMLSLL_VG4_M4ZZ_HtoD : sme2_mla_ll_array_vg24_single<"umlsll", 0b11110, MatrixOp64, ZZZZ_h, ZPR4b16>; +defm UMLSLL_MZZ_HtoD : sme2_mla_ll_array_single<"umlsll", 0b1110, MatrixOp64, ZPR16, ZPR4b16, nxv8i16, int_aarch64_sme_umls_za64_single_vg4x1>; +defm UMLSLL_VG2_M2ZZ_HtoD : sme2_mla_ll_array_vg2_single<"umlsll", 0b10110, MatrixOp64, ZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_umls_za64_single_vg4x2>; +defm UMLSLL_VG4_M4ZZ_HtoD : sme2_mla_ll_array_vg4_single<"umlsll", 0b11110, MatrixOp64, ZZZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_umls_za64_single_vg4x4>; defm UMLSLL_VG2_M2Z2Z_HtoD : sme2_mla_ll_array_vg2_multi<"umlsll", 0b1110, MatrixOp64, ZZ_h_mul_r>; defm UMLSLL_VG4_M4Z4Z_HtoD : sme2_mla_ll_array_vg4_multi<"umlsll", 0b1110, MatrixOp64, ZZZZ_h_mul_r>; } Index: llvm/lib/Target/AArch64/SMEInstrFormats.td =================================================================== --- llvm/lib/Target/AArch64/SMEInstrFormats.td +++ llvm/lib/Target/AArch64/SMEInstrFormats.td @@ -2714,6 +2714,16 @@ let Constraints = "$ZAda = $_ZAda"; } +multiclass sme2_mla_ll_array_single op, + MatrixOperand matrix_ty, ZPRRegOp vector_ty, + ZPRRegOp zpr_ty, ValueType vt, SDPatternOperator intrinsic> { + def NAME : sme2_mla_ll_array_single, SMEPseudo2Instr; + + def NAME # _PSEUDO : sme2_za_array_2op_multi_single_pseudo; + + def : SME2_ZA_TwoOp_Multi_Single_Pat; +} + class sme2_mla_ll_array_vg24_single op, MatrixOperand matrix_ty, RegisterOperand vector_ty, ZPRRegOp zpr_ty, string mnemonic> @@ -2749,12 +2759,33 @@ RegisterOperand multi_vector_ty, ZPRRegOp zpr_ty> { def NAME: sme2_mla_ll_array_vg24_single; + zpr_ty, mnemonic>, SMEPseudo2Instr; + + def NAME # _PSEUDO : sme2_za_array_2op_multi_single_pseudo; def : InstAlias(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>; } +multiclass sme2_mla_ll_array_vg2_single op, + MatrixOperand matrix_ty, + RegisterOperand multi_vector_ty, + ZPRRegOp zpr_ty, ValueType vt, SDPatternOperator intrinsic> { + + defm NAME: sme2_mla_ll_array_vg24_single; + + def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat; +} + +multiclass sme2_mla_ll_array_vg4_single op, + MatrixOperand matrix_ty, + RegisterOperand multi_vector_ty, + ZPRRegOp zpr_ty, ValueType vt, SDPatternOperator intrinsic> { + defm NAME: sme2_mla_ll_array_vg24_single; + + def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat; +} + // SME2 multiple vectors long long MLA two sources class sme2_mla_ll_array_vg2_multi op, MatrixOperand matrix_ty, Index: llvm/test/CodeGen/AArch64/sme2-intrinsics-mlall.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/AArch64/sme2-intrinsics-mlall.ll @@ -0,0 +1,528 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -mattr=+sme-i16i64 -verify-machineinstrs < %s | FileCheck %s + +; +; SMLALL +; + +; Single x1 + +define void @multi_vector_mul_add_single_long_vg4x1_s8(i32 %slice, %dummy, %zn, %zm) { +; CHECK-LABEL: multi_vector_mul_add_single_long_vg4x1_s8: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: smlall za.s[w8, 0:3], z1.b, z2.b +; CHECK-NEXT: smlall za.s[w8, 12:15], z1.b, z2.b +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.smla.za32.single.vg4x1.nxv16i8(i32 %slice, %zn, %zm) + %slice.12 = add i32 %slice, 12 + call void @llvm.aarch64.sme.smla.za32.single.vg4x1.nxv16i8(i32 %slice.12, %zn, %zm) + ret void +} + +define void @multi_vector_mul_add_single_long_vg4x1_s16(i32 %slice, %dummy, %zn, %zm) { +; CHECK-LABEL: multi_vector_mul_add_single_long_vg4x1_s16: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: smlall za.d[w8, 0:3], z1.h, z2.h +; CHECK-NEXT: smlall za.d[w8, 12:15], z1.h, z2.h +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.smla.za64.single.vg4x1.nxv8i16(i32 %slice, %zn, %zm) + %slice.12 = add i32 %slice, 12 + call void @llvm.aarch64.sme.smla.za64.single.vg4x1.nxv8i16(i32 %slice.12, %zn, %zm) + ret void +} + +; Single x2 + +define void @multi_vector_mul_add_single_long_vg4x2_s8(i32 %slice, %dummy, %zn0, %zn1, %zm) { +; CHECK-LABEL: multi_vector_mul_add_single_long_vg4x2_s8: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z1_z2 def $z1_z2 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z1_z2 def $z1_z2 +; CHECK-NEXT: smlall za.s[w8, 0:3, vgx2], { z1.b, z2.b }, z3.b +; CHECK-NEXT: smlall za.s[w8, 4:7, vgx2], { z1.b, z2.b }, z3.b +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.smla.za32.single.vg4x2.nxv16i8(i32 %slice, %zn0, %zn1, %zm) + %slice.4 = add i32 %slice, 4 + call void @llvm.aarch64.sme.smla.za32.single.vg4x2.nxv16i8(i32 %slice.4, %zn0, %zn1, %zm) + ret void +} + +define void @multi_vector_mul_add_single_long_vg4x2_s16(i32 %slice, %dummy, %zn0, %zn1, %zm) { +; CHECK-LABEL: multi_vector_mul_add_single_long_vg4x2_s16: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z1_z2 def $z1_z2 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z1_z2 def $z1_z2 +; CHECK-NEXT: smlall za.d[w8, 0:3, vgx2], { z1.h, z2.h }, z3.h +; CHECK-NEXT: smlall za.d[w8, 4:7, vgx2], { z1.h, z2.h }, z3.h +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.smla.za64.single.vg4x2.nxv8i16(i32 %slice, %zn0, %zn1, %zm) + %slice.4 = add i32 %slice, 4 + call void @llvm.aarch64.sme.smla.za64.single.vg4x2.nxv8i16(i32 %slice.4, %zn0, %zn1, %zm) + ret void +} + +; Single x4 + +define void @multi_vector_mul_add_single_long_vg4x4_s8(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm) { +; CHECK-LABEL: multi_vector_mul_add_single_long_vg4x4_s8: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: smlall za.s[w8, 0:3, vgx4], { z1.b - z4.b }, z5.b +; CHECK-NEXT: smlall za.s[w8, 4:7, vgx4], { z1.b - z4.b }, z5.b +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.smla.za32.single.vg4x4.nxv16i8(i32 %slice, %zn0, %zn1, %zn2, %zn3, %zm) + %slice.4 = add i32 %slice, 4 + call void @llvm.aarch64.sme.smla.za32.single.vg4x4.nxv16i8(i32 %slice.4, %zn0, %zn1, %zn2, %zn3, %zm) + ret void +} + +define void @multi_vector_mul_add_single_long_vg4x4_s16(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm) { +; CHECK-LABEL: multi_vector_mul_add_single_long_vg4x4_s16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: smlall za.d[w8, 0:3, vgx4], { z1.h - z4.h }, z5.h +; CHECK-NEXT: smlall za.d[w8, 4:7, vgx4], { z1.h - z4.h }, z5.h +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.smla.za64.single.vg4x4.nxv8i16(i32 %slice, %zn0, %zn1, %zn2, %zn3, %zm) + %slice.4 = add i32 %slice, 4 + call void @llvm.aarch64.sme.smla.za64.single.vg4x4.nxv8i16(i32 %slice.4, %zn0, %zn1, %zn2, %zn3, %zm) + ret void +} + +; UMLALL + +; Single x1 + +define void @multi_vector_mul_add_single_long_vg4x1_u8(i32 %slice, %dummy, %zn, %zm) { +; CHECK-LABEL: multi_vector_mul_add_single_long_vg4x1_u8: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: umlall za.s[w8, 0:3], z1.b, z2.b +; CHECK-NEXT: umlall za.s[w8, 12:15], z1.b, z2.b +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.umla.za32.single.vg4x1.nxv16i8(i32 %slice, %zn, %zm) + %slice.12 = add i32 %slice, 12 + call void @llvm.aarch64.sme.umla.za32.single.vg4x1.nxv16i8(i32 %slice.12, %zn, %zm) + ret void +} + +define void @multi_vector_mul_add_single_long_vg4x1_u16(i32 %slice, %dummy, %zn, %zm) { +; CHECK-LABEL: multi_vector_mul_add_single_long_vg4x1_u16: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: umlall za.d[w8, 0:3], z1.h, z2.h +; CHECK-NEXT: umlall za.d[w8, 12:15], z1.h, z2.h +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.umla.za64.single.vg4x1.nxv8i16(i32 %slice, %zn, %zm) + %slice.12 = add i32 %slice, 12 + call void @llvm.aarch64.sme.umla.za64.single.vg4x1.nxv8i16(i32 %slice.12, %zn, %zm) + ret void +} + +; Single x2 + +define void @multi_vector_mul_add_single_long_vg4x2_u8(i32 %slice, %dummy, %zn0, %zn1, %zm) { +; CHECK-LABEL: multi_vector_mul_add_single_long_vg4x2_u8: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z1_z2 def $z1_z2 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z1_z2 def $z1_z2 +; CHECK-NEXT: umlall za.s[w8, 0:3, vgx2], { z1.b, z2.b }, z3.b +; CHECK-NEXT: umlall za.s[w8, 4:7, vgx2], { z1.b, z2.b }, z3.b +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.umla.za32.single.vg4x2.nxv16i8(i32 %slice, %zn0, %zn1, %zm) + %slice.4 = add i32 %slice, 4 + call void @llvm.aarch64.sme.umla.za32.single.vg4x2.nxv16i8(i32 %slice.4, %zn0, %zn1, %zm) + ret void +} + +define void @multi_vector_mul_add_single_long_vg4x2_u16(i32 %slice, %dummy, %zn0, %zn1, %zm) { +; CHECK-LABEL: multi_vector_mul_add_single_long_vg4x2_u16: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z1_z2 def $z1_z2 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z1_z2 def $z1_z2 +; CHECK-NEXT: umlall za.d[w8, 0:3, vgx2], { z1.h, z2.h }, z3.h +; CHECK-NEXT: umlall za.d[w8, 4:7, vgx2], { z1.h, z2.h }, z3.h +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.umla.za64.single.vg4x2.nxv8i16(i32 %slice, %zn0, %zn1, %zm) + %slice.4 = add i32 %slice, 4 + call void @llvm.aarch64.sme.umla.za64.single.vg4x2.nxv8i16(i32 %slice.4, %zn0, %zn1, %zm) + ret void +} + +; Single x4 + +define void @multi_vector_mul_add_single_long_vg4x4_u8(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm) { +; CHECK-LABEL: multi_vector_mul_add_single_long_vg4x4_u8: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: umlall za.s[w8, 0:3, vgx4], { z1.b - z4.b }, z5.b +; CHECK-NEXT: umlall za.s[w8, 4:7, vgx4], { z1.b - z4.b }, z5.b +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.umla.za32.single.vg4x4.nxv16i8(i32 %slice, %zn0, %zn1, %zn2, %zn3, %zm) + %slice.4 = add i32 %slice, 4 + call void @llvm.aarch64.sme.umla.za32.single.vg4x4.nxv16i8(i32 %slice.4, %zn0, %zn1, %zn2, %zn3, %zm) + ret void +} + +define void @multi_vector_mul_add_single_long_vg4x4_u16(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm) { +; CHECK-LABEL: multi_vector_mul_add_single_long_vg4x4_u16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: umlall za.d[w8, 0:3, vgx4], { z1.h - z4.h }, z5.h +; CHECK-NEXT: umlall za.d[w8, 4:7, vgx4], { z1.h - z4.h }, z5.h +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.umla.za64.single.vg4x4.nxv8i16(i32 %slice, %zn0, %zn1, %zn2, %zn3, %zm) + %slice.4 = add i32 %slice, 4 + call void @llvm.aarch64.sme.umla.za64.single.vg4x4.nxv8i16(i32 %slice.4, %zn0, %zn1, %zn2, %zn3, %zm) + ret void +} + +; SMLSLL + +; Single x1 + +define void @multi_vector_mul_sub_single_long_vg4x1_s8(i32 %slice, %dummy, %zn, %zm) { +; CHECK-LABEL: multi_vector_mul_sub_single_long_vg4x1_s8: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: smlsll za.s[w8, 0:3], z1.b, z2.b +; CHECK-NEXT: smlsll za.s[w8, 12:15], z1.b, z2.b +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.smls.za32.single.vg4x1.nxv16i8(i32 %slice, %zn, %zm) + %slice.12 = add i32 %slice, 12 + call void @llvm.aarch64.sme.smls.za32.single.vg4x1.nxv16i8(i32 %slice.12, %zn, %zm) + ret void +} + +define void @multi_vector_mul_sub_single_long_vg4x1_s16(i32 %slice, %dummy, %zn, %zm) { +; CHECK-LABEL: multi_vector_mul_sub_single_long_vg4x1_s16: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: smlsll za.d[w8, 0:3], z1.h, z2.h +; CHECK-NEXT: smlsll za.d[w8, 12:15], z1.h, z2.h +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.smls.za64.single.vg4x1.nxv8i16(i32 %slice, %zn, %zm) + %slice.12 = add i32 %slice, 12 + call void @llvm.aarch64.sme.smls.za64.single.vg4x1.nxv8i16(i32 %slice.12, %zn, %zm) + ret void +} + +; Single x2 + +define void @multi_vector_mul_sub_single_long_vg4x2_s8(i32 %slice, %dummy, %zn0, %zn1, %zm) { +; CHECK-LABEL: multi_vector_mul_sub_single_long_vg4x2_s8: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z1_z2 def $z1_z2 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z1_z2 def $z1_z2 +; CHECK-NEXT: smlsll za.s[w8, 0:3, vgx2], { z1.b, z2.b }, z3.b +; CHECK-NEXT: smlsll za.s[w8, 4:7, vgx2], { z1.b, z2.b }, z3.b +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.smls.za32.single.vg4x2.nxv16i8(i32 %slice, %zn0, %zn1, %zm) + %slice.4 = add i32 %slice, 4 + call void @llvm.aarch64.sme.smls.za32.single.vg4x2.nxv16i8(i32 %slice.4, %zn0, %zn1, %zm) + ret void +} + +define void @multi_vector_mul_sub_single_long_vg4x2_s16(i32 %slice, %dummy, %zn0, %zn1, %zm) { +; CHECK-LABEL: multi_vector_mul_sub_single_long_vg4x2_s16: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z1_z2 def $z1_z2 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z1_z2 def $z1_z2 +; CHECK-NEXT: smlsll za.d[w8, 0:3, vgx2], { z1.h, z2.h }, z3.h +; CHECK-NEXT: smlsll za.d[w8, 4:7, vgx2], { z1.h, z2.h }, z3.h +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.smls.za64.single.vg4x2.nxv8i16(i32 %slice, %zn0, %zn1, %zm) + %slice.4 = add i32 %slice, 4 + call void @llvm.aarch64.sme.smls.za64.single.vg4x2.nxv8i16(i32 %slice.4, %zn0, %zn1, %zm) + ret void +} + +; Single x4 + +define void @multi_vector_mul_sub_single_long_vg4x4_s8(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm) { +; CHECK-LABEL: multi_vector_mul_sub_single_long_vg4x4_s8: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: smlsll za.s[w8, 0:3, vgx4], { z1.b - z4.b }, z5.b +; CHECK-NEXT: smlsll za.s[w8, 4:7, vgx4], { z1.b - z4.b }, z5.b +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.smls.za32.single.vg4x4.nxv16i8(i32 %slice, %zn0, %zn1, %zn2, %zn3, %zm) + %slice.4 = add i32 %slice, 4 + call void @llvm.aarch64.sme.smls.za32.single.vg4x4.nxv16i8(i32 %slice.4, %zn0, %zn1, %zn2, %zn3, %zm) + ret void +} + +define void @multi_vector_mul_sub_single_long_vg4x4_s16(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm) { +; CHECK-LABEL: multi_vector_mul_sub_single_long_vg4x4_s16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: smlsll za.d[w8, 0:3, vgx4], { z1.h - z4.h }, z5.h +; CHECK-NEXT: smlsll za.d[w8, 4:7, vgx4], { z1.h - z4.h }, z5.h +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.smls.za64.single.vg4x4.nxv8i16(i32 %slice, %zn0, %zn1, %zn2, %zn3, %zm) + %slice.4 = add i32 %slice, 4 + call void @llvm.aarch64.sme.smls.za64.single.vg4x4.nxv8i16(i32 %slice.4, %zn0, %zn1, %zn2, %zn3, %zm) + ret void +} + +; UMLSLL + +; Single x1 + +define void @multi_vector_mul_sub_single_long_vg4x1_u8(i32 %slice, %dummy, %zn, %zm) { +; CHECK-LABEL: multi_vector_mul_sub_single_long_vg4x1_u8: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: umlsll za.s[w8, 0:3], z1.b, z2.b +; CHECK-NEXT: umlsll za.s[w8, 12:15], z1.b, z2.b +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.umls.za32.single.vg4x1.nxv16i8(i32 %slice, %zn, %zm) + %slice.12 = add i32 %slice, 12 + call void @llvm.aarch64.sme.umls.za32.single.vg4x1.nxv16i8(i32 %slice.12, %zn, %zm) + ret void +} + +define void @multi_vector_mul_sub_single_long_vg4x1_u16(i32 %slice, %dummy, %zn, %zm) { +; CHECK-LABEL: multi_vector_mul_sub_single_long_vg4x1_u16: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: umlsll za.d[w8, 0:3], z1.h, z2.h +; CHECK-NEXT: umlsll za.d[w8, 12:15], z1.h, z2.h +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.umls.za64.single.vg4x1.nxv8i16(i32 %slice, %zn, %zm) + %slice.12 = add i32 %slice, 12 + call void @llvm.aarch64.sme.umls.za64.single.vg4x1.nxv8i16(i32 %slice.12, %zn, %zm) + ret void +} + +; Single x2 + +define void @multi_vector_mul_sub_single_long_vg4x2_u8(i32 %slice, %dummy, %zn0, %zn1, %zm) { +; CHECK-LABEL: multi_vector_mul_sub_single_long_vg4x2_u8: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z1_z2 def $z1_z2 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z1_z2 def $z1_z2 +; CHECK-NEXT: umlsll za.s[w8, 0:3, vgx2], { z1.b, z2.b }, z3.b +; CHECK-NEXT: umlsll za.s[w8, 4:7, vgx2], { z1.b, z2.b }, z3.b +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.umls.za32.single.vg4x2.nxv16i8(i32 %slice, %zn0, %zn1, %zm) + %slice.4 = add i32 %slice, 4 + call void @llvm.aarch64.sme.umls.za32.single.vg4x2.nxv16i8(i32 %slice.4, %zn0, %zn1, %zm) + ret void +} + +define void @multi_vector_mul_sub_single_long_vg4x2_u16(i32 %slice, %dummy, %zn0, %zn1, %zm) { +; CHECK-LABEL: multi_vector_mul_sub_single_long_vg4x2_u16: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z1_z2 def $z1_z2 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z1_z2 def $z1_z2 +; CHECK-NEXT: umlsll za.d[w8, 0:3, vgx2], { z1.h, z2.h }, z3.h +; CHECK-NEXT: umlsll za.d[w8, 4:7, vgx2], { z1.h, z2.h }, z3.h +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.umls.za64.single.vg4x2.nxv8i16(i32 %slice, %zn0, %zn1, %zm) + %slice.4 = add i32 %slice, 4 + call void @llvm.aarch64.sme.umls.za64.single.vg4x2.nxv8i16(i32 %slice.4, %zn0, %zn1, %zm) + ret void +} + +; Single x4 + +define void @multi_vector_mul_sub_single_long_vg4x4_u8(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm) { +; CHECK-LABEL: multi_vector_mul_sub_single_long_vg4x4_u8: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: umlsll za.s[w8, 0:3, vgx4], { z1.b - z4.b }, z5.b +; CHECK-NEXT: umlsll za.s[w8, 4:7, vgx4], { z1.b - z4.b }, z5.b +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.umls.za32.single.vg4x4.nxv16i8(i32 %slice, %zn0, %zn1, %zn2, %zn3, %zm) + %slice.4 = add i32 %slice, 4 + call void @llvm.aarch64.sme.umls.za32.single.vg4x4.nxv16i8(i32 %slice.4, %zn0, %zn1, %zn2, %zn3, %zm) + ret void +} + +define void @multi_vector_mul_sub_single_long_vg4x4_u16(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm) { +; CHECK-LABEL: multi_vector_mul_sub_single_long_vg4x4_u16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: umlsll za.d[w8, 0:3, vgx4], { z1.h - z4.h }, z5.h +; CHECK-NEXT: umlsll za.d[w8, 4:7, vgx4], { z1.h - z4.h }, z5.h +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.umls.za64.single.vg4x4.nxv8i16(i32 %slice, %zn0, %zn1, %zn2, %zn3, %zm) + %slice.4 = add i32 %slice, 4 + call void @llvm.aarch64.sme.umls.za64.single.vg4x4.nxv8i16(i32 %slice.4, %zn0, %zn1, %zn2, %zn3, %zm) + ret void +} + +; +; SUMLALL +; + +; Single x 2 + +define void @multi_vector_mul_add_single_signed_long_vg4x2_s8(i32 %slice, %dummy, %zn0, %zn1, %zm) { +; CHECK-LABEL: multi_vector_mul_add_single_signed_long_vg4x2_s8: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z1_z2 def $z1_z2 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z1_z2 def $z1_z2 +; CHECK-NEXT: sumlall za.s[w8, 0:3, vgx2], { z1.b, z2.b }, z3.b +; CHECK-NEXT: sumlall za.s[w8, 4:7, vgx2], { z1.b, z2.b }, z3.b +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.sumla.za32.single.vg4x2.nxv16i8(i32 %slice, %zn0, %zn1, %zm) + %slice.4 = add i32 %slice, 4 + call void @llvm.aarch64.sme.sumla.za32.single.vg4x2.nxv16i8(i32 %slice.4, %zn0, %zn1, %zm) + ret void +} + +; Single x 4 + +define void @multi_vector_mul_add_single_signed_long_vg4x4_s8(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm) { +; CHECK-LABEL: multi_vector_mul_add_single_signed_long_vg4x4_s8: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: sumlall za.s[w8, 0:3, vgx4], { z1.b - z4.b }, z5.b +; CHECK-NEXT: sumlall za.s[w8, 4:7, vgx4], { z1.b - z4.b }, z5.b +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.sumla.za32.single.vg4x4.nxv16i8(i32 %slice, %zn0, %zn1, %zn2, %zn3, %zm) + %slice.4 = add i32 %slice, 4 + call void @llvm.aarch64.sme.sumla.za32.single.vg4x4.nxv16i8(i32 %slice.4, %zn0, %zn1, %zn2, %zn3, %zm) + ret void +} + +; USMLALL + +; Single x1 + +define void @multi_vector_mul_add_single_unsigned_long_vg4x1_s8(i32 %slice, %dummy, %zn, %zm) { +; CHECK-LABEL: multi_vector_mul_add_single_unsigned_long_vg4x1_s8: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: usmlall za.s[w8, 0:3], z1.b, z2.b +; CHECK-NEXT: usmlall za.s[w8, 12:15], z1.b, z2.b +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.usmla.za32.single.vg4x1.nxv16i8(i32 %slice, %zn, %zm) + %slice.12 = add i32 %slice, 12 + call void @llvm.aarch64.sme.usmla.za32.single.vg4x1.nxv16i8(i32 %slice.12, %zn, %zm) + ret void +} + +; Single x 2 + +define void @multi_vector_mul_add_single_unsigned_long_vg4x2_s8(i32 %slice, %dummy, %zn0, %zn1, %zm) { +; CHECK-LABEL: multi_vector_mul_add_single_unsigned_long_vg4x2_s8: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z1_z2 def $z1_z2 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z1_z2 def $z1_z2 +; CHECK-NEXT: usmlall za.s[w8, 0:3, vgx2], { z1.b, z2.b }, z3.b +; CHECK-NEXT: usmlall za.s[w8, 4:7, vgx2], { z1.b, z2.b }, z3.b +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.usmla.za32.single.vg4x2.nxv16i8(i32 %slice, %zn0, %zn1, %zm) + %slice.4 = add i32 %slice, 4 + call void @llvm.aarch64.sme.usmla.za32.single.vg4x2.nxv16i8(i32 %slice.4, %zn0, %zn1, %zm) + ret void +} + +; Single x4 + +define void @multi_vector_mul_add_single_unsigned_long_vg4x4_s8(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm) { +; CHECK-LABEL: multi_vector_mul_add_single_unsigned_long_vg4x4_s8: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z1_z2_z3_z4 def $z1_z2_z3_z4 +; CHECK-NEXT: usmlall za.s[w8, 0:3, vgx4], { z1.b - z4.b }, z5.b +; CHECK-NEXT: usmlall za.s[w8, 4:7, vgx4], { z1.b - z4.b }, z5.b +; CHECK-NEXT: ret + call void @llvm.aarch64.sme.usmla.za32.single.vg4x4.nxv16i8(i32 %slice, %zn0, %zn1, %zn2, %zn3, %zm) + %slice.4 = add i32 %slice, 4 + call void @llvm.aarch64.sme.usmla.za32.single.vg4x4.nxv16i8(i32 %slice.4, %zn0, %zn1, %zn2, %zn3, %zm) + ret void +} + +declare void @llvm.aarch64.sme.smla.za32.single.vg4x1.nxv16i8(i32, , ) +declare void @llvm.aarch64.sme.smla.za32.single.vg4x2.nxv16i8(i32, , , ) +declare void @llvm.aarch64.sme.smla.za32.single.vg4x4.nxv16i8(i32, , , , , ) + +declare void @llvm.aarch64.sme.smla.za64.single.vg4x1.nxv8i16(i32, , ) +declare void @llvm.aarch64.sme.smla.za64.single.vg4x2.nxv8i16(i32, , , ) +declare void @llvm.aarch64.sme.smla.za64.single.vg4x4.nxv8i16(i32, , , , , ) + +declare void @llvm.aarch64.sme.umla.za32.single.vg4x1.nxv16i8(i32, , ) +declare void @llvm.aarch64.sme.umla.za32.single.vg4x2.nxv16i8(i32, , , ) +declare void @llvm.aarch64.sme.umla.za32.single.vg4x4.nxv16i8(i32, , , , , ) + +declare void @llvm.aarch64.sme.umla.za64.single.vg4x1.nxv8i16(i32, , ) +declare void @llvm.aarch64.sme.umla.za64.single.vg4x2.nxv8i16(i32, , , ) +declare void @llvm.aarch64.sme.umla.za64.single.vg4x4.nxv8i16(i32, , , , , ) + +declare void @llvm.aarch64.sme.smls.za32.single.vg4x1.nxv16i8(i32, , ) +declare void @llvm.aarch64.sme.smls.za32.single.vg4x2.nxv16i8(i32, , , ) +declare void @llvm.aarch64.sme.smls.za32.single.vg4x4.nxv16i8(i32, , , , , ) + +declare void @llvm.aarch64.sme.smls.za64.single.vg4x1.nxv8i16(i32, , ) +declare void @llvm.aarch64.sme.smls.za64.single.vg4x2.nxv8i16(i32, , , ) +declare void @llvm.aarch64.sme.smls.za64.single.vg4x4.nxv8i16(i32, , , , , ) + +declare void @llvm.aarch64.sme.umls.za32.single.vg4x1.nxv16i8(i32, , ) +declare void @llvm.aarch64.sme.umls.za32.single.vg4x2.nxv16i8(i32, , , ) +declare void @llvm.aarch64.sme.umls.za32.single.vg4x4.nxv16i8(i32, , , , , ) + +declare void @llvm.aarch64.sme.umls.za64.single.vg4x1.nxv8i16(i32, , ) +declare void @llvm.aarch64.sme.umls.za64.single.vg4x2.nxv8i16(i32, , , ) +declare void @llvm.aarch64.sme.umls.za64.single.vg4x4.nxv8i16(i32, , , , , ) + +declare void @llvm.aarch64.sme.sumla.za32.single.vg4x2.nxv16i8(i32, , , ) +declare void @llvm.aarch64.sme.sumla.za32.single.vg4x4.nxv16i8(i32, , , , , ) + +declare void @llvm.aarch64.sme.usmla.za32.single.vg4x1.nxv16i8(i32, , ) +declare void @llvm.aarch64.sme.usmla.za32.single.vg4x2.nxv16i8(i32, , , ) +declare void @llvm.aarch64.sme.usmla.za32.single.vg4x4.nxv16i8(i32, , , , , )