diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td --- a/llvm/include/llvm/IR/IntrinsicsAArch64.td +++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -1080,6 +1080,14 @@ llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; + class SVE2_3VectorArg_Indexed_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + LLVMSubdivide2VectorType<0>, + LLVMSubdivide2VectorType<0>, + llvm_i64_ty], + [IntrNoMem, ImmArg<3>]>; + // NOTE: There is no relationship between these intrinsics beyond an attempt // to reuse currently identical class definitions. class AdvSIMD_SVE_LOGB_Intrinsic : AdvSIMD_SVE_CNT_Intrinsic; @@ -1732,4 +1740,14 @@ def int_aarch64_sve_sqrshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; def int_aarch64_sve_sqrshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; + +def int_aarch64_sve_smlalb : SVE2_3VectorArg_Indexed_Intrinsic; +def int_aarch64_sve_smlalt : SVE2_3VectorArg_Indexed_Intrinsic; +def int_aarch64_sve_umlalb : SVE2_3VectorArg_Indexed_Intrinsic; +def int_aarch64_sve_umlalt : SVE2_3VectorArg_Indexed_Intrinsic; +def int_aarch64_sve_smlslb : SVE2_3VectorArg_Indexed_Intrinsic; +def int_aarch64_sve_smlslt : SVE2_3VectorArg_Indexed_Intrinsic; +def int_aarch64_sve_umlslb : SVE2_3VectorArg_Indexed_Intrinsic; +def int_aarch64_sve_umlslt : SVE2_3VectorArg_Indexed_Intrinsic; + } diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -1467,14 +1467,14 @@ defm SQDMULLT_ZZZI : sve2_int_mul_long_by_indexed_elem<0b101, "sqdmullt">; // SVE2 integer multiply-add long (indexed) - defm SMLALB_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1000, "smlalb">; - defm SMLALT_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1001, "smlalt">; - defm UMLALB_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1010, "umlalb">; - defm UMLALT_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1011, "umlalt">; - defm SMLSLB_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1100, "smlslb">; - defm SMLSLT_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1101, "smlslt">; - defm UMLSLB_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1110, "umlslb">; - defm UMLSLT_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1111, "umlslt">; + defm SMLALB_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1000, "smlalb", int_aarch64_sve_smlalb>; + defm SMLALT_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1001, "smlalt", int_aarch64_sve_smlalt>; + defm UMLALB_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1010, "umlalb", int_aarch64_sve_umlalb>; + defm UMLALT_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1011, "umlalt", int_aarch64_sve_umlalt>; + defm SMLSLB_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1100, "smlslb", int_aarch64_sve_smlslb>; + defm SMLSLT_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1101, "smlslt", int_aarch64_sve_smlslt>; + defm UMLSLB_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1110, "umlslb", int_aarch64_sve_umlslb>; + defm UMLSLT_ZZZI : sve2_int_mla_long_by_indexed_elem<0b1111, "umlslt", int_aarch64_sve_umlslt>; // SVE2 integer multiply-add long (vectors, unpredicated) defm SMLALB_ZZZ : sve2_int_mla_long<0b10000, "smlalb">; @@ -1487,10 +1487,10 @@ defm UMLSLT_ZZZ : sve2_int_mla_long<0b10111, "umlslt">; // SVE2 saturating multiply-add long (indexed) - defm SQDMLALB_ZZZI : sve2_int_mla_long_by_indexed_elem<0b0100, "sqdmlalb">; - defm SQDMLALT_ZZZI : sve2_int_mla_long_by_indexed_elem<0b0101, "sqdmlalt">; - defm SQDMLSLB_ZZZI : sve2_int_mla_long_by_indexed_elem<0b0110, "sqdmlslb">; - defm SQDMLSLT_ZZZI : sve2_int_mla_long_by_indexed_elem<0b0111, "sqdmlslt">; + defm SQDMLALB_ZZZI : sve2_int_mla_long_by_indexed_elem<0b0100, "sqdmlalb", null_frag>; + defm SQDMLALT_ZZZI : sve2_int_mla_long_by_indexed_elem<0b0101, "sqdmlalt", null_frag>; + defm SQDMLSLB_ZZZI : sve2_int_mla_long_by_indexed_elem<0b0110, "sqdmlslb", null_frag>; + defm SQDMLSLT_ZZZI : sve2_int_mla_long_by_indexed_elem<0b0111, "sqdmlslt", null_frag>; // SVE2 saturating multiply-add long (vectors, unpredicated) defm SQDMLALB_ZZZ : sve2_int_mla_long<0b11000, "sqdmlalb">; diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -2402,7 +2402,7 @@ // SVE2 Integer Multiply-Add Long - Indexed Group //===----------------------------------------------------------------------===// -multiclass sve2_int_mla_long_by_indexed_elem opc, string asm> { +multiclass sve2_int_mla_long_by_indexed_elem opc, string asm, SDPatternOperator op> { def _S : sve2_int_mla_by_indexed_elem<0b10, { opc{3}, 0b0, opc{2-1}, ?, opc{0} }, asm, ZPR32, ZPR16, ZPR3b16, VectorIndexH> { bits<3> Zm; @@ -2419,6 +2419,9 @@ let Inst{19-16} = Zm; let Inst{11} = iop{0}; } + + def : SVE_4_Op_Imm_Pat(NAME # _S)>; + def : SVE_4_Op_Imm_Pat(NAME # _D)>; } //===----------------------------------------------------------------------===// diff --git a/llvm/test/CodeGen/AArch64/sve2-mla-indexed.ll b/llvm/test/CodeGen/AArch64/sve2-mla-indexed.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve2-mla-indexed.ll @@ -0,0 +1,458 @@ +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s + +; +; SMLALB +; +define @smlalb_i32( %a, + %b, + %c) { +; CHECK-LABEL: smlalb_i32 +; CHECK: smlalb z0.s, z1.h, z2.h[1] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.smlalb.nxv4i32( %a, + %b, + %c, + i64 1) + ret %res +} + +define @smlalb_i32_2( %a, + %b, + %c) { +; CHECK-LABEL: smlalb_i32_2 +; CHECK: smlalb z0.s, z1.h, z2.h[7] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.smlalb.nxv4i32( %a, + %b, + %c, + i64 7) + ret %res +} + +define @smlalb_i64( %a, + %b, + %c) { +; CHECK-LABEL: smlalb_i64 +; CHECK: smlalb z0.d, z1.s, z2.s[0] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.smlalb.nxv2i64( %a, + %b, + %c, + i64 0) + ret %res +} + +define @smlalb_i64_2( %a, + %b, + %c) { +; CHECK-LABEL: smlalb_i64_2 +; CHECK: smlalb z0.d, z1.s, z2.s[3] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.smlalb.nxv2i64( %a, + %b, + %c, + i64 3) + ret %res +} + +; +; SMLALT +; +define @smlalt_i32( %a, + %b, + %c) { +; CHECK-LABEL: smlalt_i32 +; CHECK: smlalt z0.s, z1.h, z2.h[1] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.smlalt.nxv4i32( %a, + %b, + %c, + i64 1) + ret %res +} + +define @smlalt_i32_2( %a, + %b, + %c) { +; CHECK-LABEL: smlalt_i32_2 +; CHECK: smlalt z0.s, z1.h, z2.h[7] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.smlalt.nxv4i32( %a, + %b, + %c, + i64 7) + ret %res +} + +define @smlalt_i64( %a, + %b, + %c) { +; CHECK-LABEL: smlalt_i64 +; CHECK: smlalt z0.d, z1.s, z2.s[0] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.smlalt.nxv2i64( %a, + %b, + %c, + i64 0) + ret %res +} + +define @smlalt_i64_2( %a, + %b, + %c) { +; CHECK-LABEL: smlalt_i64_2 +; CHECK: smlalt z0.d, z1.s, z2.s[3] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.smlalt.nxv2i64( %a, + %b, + %c, + i64 3) + ret %res +} + +; +; UMLALB +; +define @umlalb_i32( %a, + %b, + %c) { +; CHECK-LABEL: umlalb_i32 +; CHECK: umlalb z0.s, z1.h, z2.h[1] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.umlalb.nxv4i32( %a, + %b, + %c, + i64 1) + ret %res +} + +define @umlalb_i32_2( %a, + %b, + %c) { +; CHECK-LABEL: umlalb_i32_2 +; CHECK: umlalb z0.s, z1.h, z2.h[7] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.umlalb.nxv4i32( %a, + %b, + %c, + i64 7) + ret %res +} + +define @umlalb_i64( %a, + %b, + %c) { +; CHECK-LABEL: umlalb_i64 +; CHECK: umlalb z0.d, z1.s, z2.s[0] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.umlalb.nxv2i64( %a, + %b, + %c, + i64 0) + ret %res +} + +define @umlalb_i64_2( %a, + %b, + %c) { +; CHECK-LABEL: umlalb_i64_2 +; CHECK: umlalb z0.d, z1.s, z2.s[3] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.umlalb.nxv2i64( %a, + %b, + %c, + i64 3) + ret %res +} + +; +; UMLALT +; +define @umlalt_i32( %a, + %b, + %c) { +; CHECK-LABEL: umlalt_i32 +; CHECK: umlalt z0.s, z1.h, z2.h[1] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.umlalt.nxv4i32( %a, + %b, + %c, + i64 1) + ret %res +} + +define @umlalt_i32_2( %a, + %b, + %c) { +; CHECK-LABEL: umlalt_i32_2 +; CHECK: umlalt z0.s, z1.h, z2.h[7] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.umlalt.nxv4i32( %a, + %b, + %c, + i64 7) + ret %res +} + +define @umlalt_i64( %a, + %b, + %c) { +; CHECK-LABEL: umlalt_i64 +; CHECK: umlalt z0.d, z1.s, z2.s[0] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.umlalt.nxv2i64( %a, + %b, + %c, + i64 0) + ret %res +} + +define @umlalt_i64_2( %a, + %b, + %c) { +; CHECK-LABEL: umlalt_i64_2 +; CHECK: umlalt z0.d, z1.s, z2.s[3] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.umlalt.nxv2i64( %a, + %b, + %c, + i64 3) + ret %res +} + +; +; SMLSLB +; +define @smlslb_i32( %a, + %b, + %c) { +; CHECK-LABEL: smlslb_i32 +; CHECK: smlslb z0.s, z1.h, z2.h[1] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.smlslb.nxv4i32( %a, + %b, + %c, + i64 1) + ret %res +} + +define @smlslb_i32_2( %a, + %b, + %c) { +; CHECK-LABEL: smlslb_i32_2 +; CHECK: smlslb z0.s, z1.h, z2.h[7] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.smlslb.nxv4i32( %a, + %b, + %c, + i64 7) + ret %res +} + +define @smlslb_i64( %a, + %b, + %c) { +; CHECK-LABEL: smlslb_i64 +; CHECK: smlslb z0.d, z1.s, z2.s[0] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.smlslb.nxv2i64( %a, + %b, + %c, + i64 0) + ret %res +} + +define @smlslb_i64_2( %a, + %b, + %c) { +; CHECK-LABEL: smlslb_i64_2 +; CHECK: smlslb z0.d, z1.s, z2.s[3] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.smlslb.nxv2i64( %a, + %b, + %c, + i64 3) + ret %res +} + +; +; SMLSLT +; +define @smlslt_i32( %a, + %b, + %c) { +; CHECK-LABEL: smlslt_i32 +; CHECK: smlslt z0.s, z1.h, z2.h[1] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.smlslt.nxv4i32( %a, + %b, + %c, + i64 1) + ret %res +} + +define @smlslt_i32_2( %a, + %b, + %c) { +; CHECK-LABEL: smlslt_i32_2 +; CHECK: smlslt z0.s, z1.h, z2.h[7] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.smlslt.nxv4i32( %a, + %b, + %c, + i64 7) + ret %res +} + +define @smlslt_i64( %a, + %b, + %c) { +; CHECK-LABEL: smlslt_i64 +; CHECK: smlslt z0.d, z1.s, z2.s[0] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.smlslt.nxv2i64( %a, + %b, + %c, + i64 0) + ret %res +} + +define @smlslt_i64_2( %a, + %b, + %c) { +; CHECK-LABEL: smlslt_i64_2 +; CHECK: smlslt z0.d, z1.s, z2.s[3] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.smlslt.nxv2i64( %a, + %b, + %c, + i64 3) + ret %res +} + +; +; UMLSLB +; +define @umlslb_i32( %a, + %b, + %c) { +; CHECK-LABEL: umlslb_i32 +; CHECK: umlslb z0.s, z1.h, z2.h[1] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.umlslb.nxv4i32( %a, + %b, + %c, + i64 1) + ret %res +} + +define @umlslb_i32_2( %a, + %b, + %c) { +; CHECK-LABEL: umlslb_i32_2 +; CHECK: umlslb z0.s, z1.h, z2.h[7] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.umlslb.nxv4i32( %a, + %b, + %c, + i64 7) + ret %res +} + +define @umlslb_i64( %a, + %b, + %c) { +; CHECK-LABEL: umlslb_i64 +; CHECK: umlslb z0.d, z1.s, z2.s[0] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.umlslb.nxv2i64( %a, + %b, + %c, + i64 0) + ret %res +} + +define @umlslb_i64_2( %a, + %b, + %c) { +; CHECK-LABEL: umlslb_i64_2 +; CHECK: umlslb z0.d, z1.s, z2.s[3] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.umlslb.nxv2i64( %a, + %b, + %c, + i64 3) + ret %res +} + +; +; UMLSLT +; +define @umlslt_i32( %a, + %b, + %c) { +; CHECK-LABEL: umlslt_i32 +; CHECK: umlslt z0.s, z1.h, z2.h[1] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.umlslt.nxv4i32( %a, + %b, + %c, + i64 1) + ret %res +} + +define @umlslt_i32_2( %a, + %b, + %c) { +; CHECK-LABEL: umlslt_i32_2 +; CHECK: umlslt z0.s, z1.h, z2.h[7] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.umlslt.nxv4i32( %a, + %b, + %c, + i64 7) + ret %res +} + +define @umlslt_i64( %a, + %b, + %c) { +; CHECK-LABEL: umlslt_i64 +; CHECK: umlslt z0.d, z1.s, z2.s[0] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.umlslt.nxv2i64( %a, + %b, + %c, + i64 0) + ret %res +} + +define @umlslt_i64_2( %a, + %b, + %c) { +; CHECK-LABEL: umlslt_i64_2 +; CHECK: umlslt z0.d, z1.s, z2.s[3] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.umlslt.nxv2i64( %a, + %b, + %c, + i64 3) + ret %res +} + +declare @llvm.aarch64.sve.smlalb.nxv4i32(, , , i64) +declare @llvm.aarch64.sve.smlalb.nxv2i64(, , , i64) +declare @llvm.aarch64.sve.smlalt.nxv4i32(, , , i64) +declare @llvm.aarch64.sve.smlalt.nxv2i64(, , , i64) +declare @llvm.aarch64.sve.umlalb.nxv4i32(, , , i64) +declare @llvm.aarch64.sve.umlalb.nxv2i64(, , , i64) +declare @llvm.aarch64.sve.umlalt.nxv4i32(, , , i64) +declare @llvm.aarch64.sve.umlalt.nxv2i64(, , , i64) +declare @llvm.aarch64.sve.smlslb.nxv4i32(, , , i64) +declare @llvm.aarch64.sve.smlslb.nxv2i64(, , , i64) +declare @llvm.aarch64.sve.smlslt.nxv4i32(, , , i64) +declare @llvm.aarch64.sve.smlslt.nxv2i64(, , , i64) +declare @llvm.aarch64.sve.umlslb.nxv4i32(, , , i64) +declare @llvm.aarch64.sve.umlslb.nxv2i64(, , , i64) +declare @llvm.aarch64.sve.umlslt.nxv4i32(, , , i64) +declare @llvm.aarch64.sve.umlslt.nxv2i64(, , , i64)