diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td --- a/clang/include/clang/Basic/arm_sve.td +++ b/clang/include/clang/Basic/arm_sve.td @@ -71,6 +71,7 @@ // R: scalar of 1/2 width element type (splat to vector type) // r: scalar of 1/4 width element type (splat to vector type) // @: unsigned scalar of 1/4 width element type (splat to vector type) +// ~: bfloat scalar (splat to vector type) // e: 1/2 width unsigned elements, 2x element count // b: 1/4 width unsigned elements, 4x element count // h: 1/2 width elements, 2x element count @@ -96,6 +97,7 @@ // O: svfloat16_t // M: svfloat32_t // N: svfloat64_t +// $: svbfloat16_t // J: Prefetch type (sv_prfop) // A: pointer to int8_t @@ -487,6 +489,20 @@ let ArchGuard = "defined(__ARM_FEATURE_SVE_MATMUL_FP64) && defined(__ARM_FEATURE_BF16_SCALAR_ARITHMETIC)" in { def SVLD1RO_BF : SInst<"svld1ro[_{2}]", "dPc", "b", MergeNone, "aarch64_sve_ld1ro">; } + +let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in { + def SVBFDOT : SInst<"svbfdot[_{0}]", "dd$$", "f", MergeNone, "aarch64_sve_bfdot">; + def SVBFMLALB : SInst<"svbfmlalb[_{0}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlalb">; + def SVBFMLALT : SInst<"svbfmlalt[_{0}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlalt">; + def SVBFMMLA : SInst<"svbfmmla[_{0}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmmla">; + def SVBFDOT_N : SInst<"svbfdot[_n_{0}]", "dd$~", "f", MergeNone, "aarch64_sve_bfdot">; + def SVBFMLAL_N : SInst<"svbfmlalb[_n_{0}]", "dd$~", "f", MergeNone, "aarch64_sve_bfmlalb">; + def SVBFMLALT_N : SInst<"svbfmlalt[_n_{0}]", "dd$~", "f", MergeNone, "aarch64_sve_bfmlalt">; + def SVBFDOT_LANE : SInst<"svbfdot_lane[_{d}]", "dd$$n", "f", MergeNone, "aarch64_sve_bfdot_lane", [], [ImmCheck<3, ImmCheck0_3>]>; + def SVBFMLALB_LANE : SInst<"svbfmlalb_lane[_{d}]", "dd$$n", "f", MergeNone, "aarch64_sve_bfmlalb_lane", [], [ImmCheck<3, ImmCheck0_7>]>; + def SVBFMLALT_LANE : SInst<"svbfmlalt_lane[_{d}]", "dd$$n", "f", MergeNone, "aarch64_sve_bfmlalt_lane", [], [ImmCheck<3, ImmCheck0_7>]>; +} + //////////////////////////////////////////////////////////////////////////////// // Stores @@ -1010,6 +1026,13 @@ defm SVFCVTZS_S32_F32 : SInstCvtMXZ<"svcvt_s32[_f32]", "ddPM", "dPM", "i", "aarch64_sve_fcvtzs", [IsOverloadCvt]>; defm SVFCVTZS_S64_F32 : SInstCvtMXZ<"svcvt_s64[_f32]", "ddPM", "dPM", "l", "aarch64_sve_fcvtzs_i64f32">; +let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in { + // svcvt_bf16_f32 + defm SVCVT_BF16_F32 : SInstCvtMXZ<"svcvt_bf16[_f32]", "ddPM", "dPM", "b", "aarch64_sve_cvt_bf16f32">; + // svcvtnt_bf16_f32 + defm SVCVTNT_BF16_F32 : SInstCvtMX<"svcvtnt_bf16[_f32]", "ddPM", "dPM", "b", "aarch64_sve_cvtnt_bf16f32">; +} + // svcvt_s##_f64 defm SVFCVTZS_S32_F64 : SInstCvtMXZ<"svcvt_s32[_f64]", "ttPd", "tPd", "d", "aarch64_sve_fcvtzs_i32f64">; defm SVFCVTZS_S64_F64 : SInstCvtMXZ<"svcvt_s64[_f64]", "ddPN", "dPN", "l", "aarch64_sve_fcvtzs", [IsOverloadCvt]>; diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfdot.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfdot.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfdot.c @@ -0,0 +1,54 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4 +#endif + +svfloat32_t test_bfdot_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfdot_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfdot.nxv4f32( %x, %y, %z) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfdot, _f32, , )(x, y, z); +} + +svfloat32_t test_bfdot_lane_0_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfdot_lane_0_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfdot.lane.nxv4f32( %x, %y, %z, i64 0) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfdot_lane, _f32, , )(x, y, z, 0); +} + +svfloat32_t test_bfdot_lane_1_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfdot_lane_1_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfdot.lane.nxv4f32( %x, %y, %z, i64 1) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfdot_lane, _f32, , )(x, y, z, 1); +} + +svfloat32_t test_bfdot_lane_2_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfdot_lane_2_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfdot.lane.nxv4f32( %x, %y, %z, i64 2) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfdot_lane, _f32, , )(x, y, z, 2); +} + +svfloat32_t test_bfdot_lane_3_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfdot_lane_3_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfdot.lane.nxv4f32( %x, %y, %z, i64 3) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfdot_lane, _f32, , )(x, y, z, 3); +} + +svfloat32_t test_bfdot_n_f32(svfloat32_t x, svbfloat16_t y, bfloat16_t z) { + // CHECK-LABEL: @test_bfdot_n_f32( + // CHECK: %[[SPLAT:.*]] = call @llvm.aarch64.sve.dup.x.nxv8bf16(bfloat %z) + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfdot.nxv4f32( %x, %y, %[[SPLAT]]) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfdot, _n_f32, , )(x, y, z); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmlalb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmlalb.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmlalb.c @@ -0,0 +1,82 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4 +#endif + +svfloat32_t test_svbfmlalb_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_svbfmlalb_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalb.nxv4f32( %x, %y, %z) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalb, _f32, , )(x, y, z); +} + +svfloat32_t test_bfmlalb_lane_0_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmlalb_lane_0_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalb.lane.nxv4f32( %x, %y, %z, i64 0) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalb_lane, _f32, , )(x, y, z, 0); +} + +svfloat32_t test_bfmlalb_lane_1_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmlalb_lane_1_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalb.lane.nxv4f32( %x, %y, %z, i64 1) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalb_lane, _f32, , )(x, y, z, 1); +} + +svfloat32_t test_bfmlalb_lane_2_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmlalb_lane_2_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalb.lane.nxv4f32( %x, %y, %z, i64 2) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalb_lane, _f32, , )(x, y, z, 2); +} + +svfloat32_t test_bfmlalb_lane_3_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmlalb_lane_3_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalb.lane.nxv4f32( %x, %y, %z, i64 3) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalb_lane, _f32, , )(x, y, z, 3); +} + +svfloat32_t test_bfmlalb_lane_4_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmlalb_lane_4_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalb.lane.nxv4f32( %x, %y, %z, i64 4) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalb_lane, _f32, , )(x, y, z, 4); +} + +svfloat32_t test_bfmlalb_lane_5_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmlalb_lane_5_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalb.lane.nxv4f32( %x, %y, %z, i64 5) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalb_lane, _f32, , )(x, y, z, 5); +} + +svfloat32_t test_bfmlalb_lane_6_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmlalb_lane_6_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalb.lane.nxv4f32( %x, %y, %z, i64 6) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalb_lane, _f32, , )(x, y, z, 6); +} + +svfloat32_t test_bfmlalb_lane_7_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmlalb_lane_7_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalb.lane.nxv4f32( %x, %y, %z, i64 7) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalb_lane, _f32, , )(x, y, z, 7); +} + +svfloat32_t test_bfmlalb_n_f32(svfloat32_t x, svbfloat16_t y, bfloat16_t z) { + // CHECK-LABEL: @test_bfmlalb_n_f32( + // CHECK: %[[SPLAT:.*]] = call @llvm.aarch64.sve.dup.x.nxv8bf16(bfloat %z) + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalb.nxv4f32( %x, %y, %[[SPLAT]]) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalb, _n_f32, , )(x, y, z); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmlalt.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmlalt.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmlalt.c @@ -0,0 +1,82 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4 +#endif + +svfloat32_t test_svbfmlalt_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_svbfmlalt_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalt.nxv4f32( %x, %y, %z) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalt, _f32, , )(x, y, z); +} + +svfloat32_t test_bfmlalt_lane_0_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmlalt_lane_0_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalt.lane.nxv4f32( %x, %y, %z, i64 0) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalt_lane, _f32, , )(x, y, z, 0); +} + +svfloat32_t test_bfmlalt_lane_1_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmlalt_lane_1_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalt.lane.nxv4f32( %x, %y, %z, i64 1) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalt_lane, _f32, , )(x, y, z, 1); +} + +svfloat32_t test_bfmlalt_lane_2_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmlalt_lane_2_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalt.lane.nxv4f32( %x, %y, %z, i64 2) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalt_lane, _f32, , )(x, y, z, 2); +} + +svfloat32_t test_bfmlalt_lane_3_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmlalt_lane_3_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalt.lane.nxv4f32( %x, %y, %z, i64 3) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalt_lane, _f32, , )(x, y, z, 3); +} + +svfloat32_t test_bfmlalt_lane_4_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmlalt_lane_4_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalt.lane.nxv4f32( %x, %y, %z, i64 4) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalt_lane, _f32, , )(x, y, z, 4); +} + +svfloat32_t test_bfmlalt_lane_5_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmlalt_lane_5_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalt.lane.nxv4f32( %x, %y, %z, i64 5) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalt_lane, _f32, , )(x, y, z, 5); +} + +svfloat32_t test_bfmlalt_lane_6_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmlalt_lane_6_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalt.lane.nxv4f32( %x, %y, %z, i64 6) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalt_lane, _f32, , )(x, y, z, 6); +} + +svfloat32_t test_bfmlalt_lane_7_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmlalt_lane_7_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalt.lane.nxv4f32( %x, %y, %z, i64 7) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalt_lane, _f32, , )(x, y, z, 7); +} + +svfloat32_t test_bfmlalt_n_f32(svfloat32_t x, svbfloat16_t y, bfloat16_t z) { + // CHECK-LABEL: @test_bfmlalt_n_f32( + // CHECK: %[[SPLAT:.*]] = call @llvm.aarch64.sve.dup.x.nxv8bf16(bfloat %z) + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalt.nxv4f32( %x, %y, %[[SPLAT]]) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalt, _n_f32, , )(x, y, z); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmmla.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmmla.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmmla.c @@ -0,0 +1,18 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4 +#endif + +svfloat32_t test_bfmmla_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmmla_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmmla.nxv4f32( %x, %y, %z) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmmla, _f32, , )(x, y, z); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvt-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvt-bfloat.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvt-bfloat.c @@ -0,0 +1,35 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svcvt_bf16_f32_x(svbool_t pg, svfloat32_t op) { + // CHECK-LABEL: test_svcvt_bf16_f32_x + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.cvt.bf16f32( undef, %[[PG]], %op) + // CHECK: ret %[[INTRINSIC]] + return SVE_ACLE_FUNC(svcvt_bf16, _f32, _x, )(pg, op); +} + +svbfloat16_t test_svcvt_bf16_f32_z(svbool_t pg, svfloat32_t op) { + // CHECK-LABEL: test_svcvt_bf16_f32_z + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.cvt.bf16f32( zeroinitializer, %[[PG]], %op) + // CHECK: ret %[[INTRINSIC]] + return SVE_ACLE_FUNC(svcvt_bf16, _f32, _z, )(pg, op); +} + +svbfloat16_t test_svcvt_bf16_f32_m(svbfloat16_t inactive, svbool_t pg, svfloat32_t op) { + // CHECK-LABEL: test_svcvt_bf16_f32_m + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.cvt.bf16f32( %inactive, %[[PG]], %op) + // CHECK: ret %[[INTRINSIC]] + return SVE_ACLE_FUNC(svcvt_bf16, _f32, _m, )(inactive, pg, op); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvtnt.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvtnt.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvtnt.c @@ -0,0 +1,27 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svcvtnt_bf16_f32_x(svbool_t pg, svfloat32_t op) { + // CHECK-LABEL: test_svcvtnt_bf16_f32_x + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.cvtnt.bf16f32( undef, %[[PG]], %op) + // CHECK: ret %[[INTRINSIC]] + return SVE_ACLE_FUNC(svcvtnt_bf16, _f32, _x, )(pg, op); +} + +svbfloat16_t test_svcvtnt_bf16_f32_m(svbfloat16_t inactive, svbool_t pg, svfloat32_t op) { + // CHECK-LABEL: test_svcvtnt_bf16_f32_m + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.cvtnt.bf16f32( %inactive, %[[PG]], %op) + // CHECK: ret %[[INTRINSIC]] + return SVE_ACLE_FUNC(svcvtnt_bf16, _f32, _m, )(inactive, pg, op); +} diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp --- a/clang/utils/TableGen/SveEmitter.cpp +++ b/clang/utils/TableGen/SveEmitter.cpp @@ -215,13 +215,13 @@ /// Return true if the intrinsic takes a splat operand. bool hasSplat() const { // These prototype modifiers are described in arm_sve.td. - return Proto.find_first_of("ajfrKLR@") != std::string::npos; + return Proto.find_first_of("ajfrKLR@~") != std::string::npos; } /// Return the parameter index of the splat operand. unsigned getSplatIdx() const { // These prototype modifiers are described in arm_sve.td. - auto Idx = Proto.find_first_of("ajfrKLR@"); + auto Idx = Proto.find_first_of("ajfrKLR@~"); assert(Idx != std::string::npos && Idx > 0 && "Prototype has no splat operand"); return Idx - 1; @@ -556,6 +556,12 @@ Bitwidth = ElementBitwidth; NumVectors = 0; break; + case '~': + ElementBitwidth = 16; + BFloat = true; + Float = false; + NumVectors = 0; + break; case 'R': ElementBitwidth /= 2; NumVectors = 0; @@ -688,6 +694,12 @@ Float = true; ElementBitwidth = 64; break; + case '$': + Predicate = false; + BFloat = true; + Float = false; + ElementBitwidth = 16; + break; case 'Q': Constant = true; Pointer = true; diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td --- a/llvm/include/llvm/IR/IntrinsicsAArch64.td +++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -1340,6 +1340,21 @@ [LLVMMatchType<0>, LLVMSubdivide4VectorType<0>, LLVMSubdivide4VectorType<0>], [IntrNoMem]>; +class SVE_bfloat + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + llvm_nxv8bf16_ty, + llvm_nxv8bf16_ty], + [IntrNoMem]>; + +class SVE_bfloat_index + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + llvm_nxv8bf16_ty, + llvm_nxv8bf16_ty, + llvm_i64_ty], + [IntrNoMem, ImmArg>]>; + // // Vector tuple creation intrinsics (ACLE) // @@ -1793,6 +1808,9 @@ def int_aarch64_sve_fcvtzs_i64f16 : Builtin_SVCVT<"svcvt_s64_f16_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>; def int_aarch64_sve_fcvtzs_i64f32 : Builtin_SVCVT<"svcvt_s64_f32_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; +def int_aarch64_sve_cvt_bf16f32 : Builtin_SVCVT<"svcvt_bf16_f32_m", llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>; +def int_aarch64_sve_cvtnt_bf16f32 : Builtin_SVCVT<"svcvtnt_bf16_f32_m", llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>; + def int_aarch64_sve_fcvtzu_i32f16 : Builtin_SVCVT<"svcvt_u32_f16_m", llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; def int_aarch64_sve_fcvtzu_i32f64 : Builtin_SVCVT<"svcvt_u32_f64_m", llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; def int_aarch64_sve_fcvtzu_i64f16 : Builtin_SVCVT<"svcvt_u64_f16_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>; @@ -2343,6 +2361,19 @@ // def int_aarch64_sve_fmmla : AdvSIMD_3VectorArg_Intrinsic; +// +// SVE ACLE: 7.2. BFloat16 extensions +// + +def int_aarch64_sve_bfdot : SVE_bfloat; +def int_aarch64_sve_bfmlalb : SVE_bfloat; +def int_aarch64_sve_bfmlalt : SVE_bfloat; + +def int_aarch64_sve_bfmmla : SVE_bfloat; + +def int_aarch64_sve_bfdot_lane : SVE_bfloat_index; +def int_aarch64_sve_bfmlalb_lane : SVE_bfloat_index; +def int_aarch64_sve_bfmlalt_lane : SVE_bfloat_index; } // diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -1239,15 +1239,15 @@ defm FSQRT_ZPmZ : sve_fp_2op_p_zd_HSD<0b01101, "fsqrt", int_aarch64_sve_fsqrt>; let Predicates = [HasBF16, HasSVE] in { - def BFDOT_ZZZ : sve_bfloat_dot<"bfdot">; - def BFDOT_ZZI : sve_bfloat_dot_indexed<"bfdot">; - def BFMMLA_ZZZ : sve_bfloat_matmul<"bfmmla">; - def BFMMLA_B_ZZZ : sve_bfloat_matmul_longvecl<0b0, "bfmlalb">; - def BFMMLA_T_ZZZ : sve_bfloat_matmul_longvecl<0b1, "bfmlalt">; - def BFMMLA_B_ZZI : sve_bfloat_matmul_longvecl_idx<0b0, "bfmlalb">; - def BFMMLA_T_ZZI : sve_bfloat_matmul_longvecl_idx<0b1, "bfmlalt">; - def BFCVT_ZPmZ : sve_bfloat_convert<0b1, "bfcvt">; - def BFCVTNT_ZPmZ : sve_bfloat_convert<0b0, "bfcvtnt">; + defm BFDOT_ZZZ : sve_bfloat_dot<"bfdot", int_aarch64_sve_bfdot>; + defm BFDOT_ZZI : sve_bfloat_dot_indexed<"bfdot", int_aarch64_sve_bfdot_lane>; + defm BFMMLA_ZZZ : sve_bfloat_matmul<"bfmmla", int_aarch64_sve_bfmmla>; + defm BFMMLA_B_ZZZ : sve_bfloat_matmul_longvecl<0b0, "bfmlalb", int_aarch64_sve_bfmlalb>; + defm BFMMLA_T_ZZZ : sve_bfloat_matmul_longvecl<0b1, "bfmlalt", int_aarch64_sve_bfmlalt>; + defm BFMMLA_B_ZZI : sve_bfloat_matmul_longvecl_idx<0b0, "bfmlalb", int_aarch64_sve_bfmlalb_lane>; + defm BFMMLA_T_ZZI : sve_bfloat_matmul_longvecl_idx<0b1, "bfmlalt", int_aarch64_sve_bfmlalt_lane>; + defm BFCVT_ZPmZ : sve_bfloat_convert<0b1, "bfcvt", int_aarch64_sve_cvt_bf16f32>; + defm BFCVTNT_ZPmZ : sve_bfloat_convert<0b0, "bfcvtnt", int_aarch64_sve_cvtnt_bf16f32>; } // InstAliases diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -7460,6 +7460,11 @@ let Inst{20-16} = Zm; } +multiclass sve_bfloat_dot { + def NAME : sve_bfloat_dot; + def : SVE_3_Op_Pat(NAME)>; +} + class sve_bfloat_dot_indexed : sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop", (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexS:$iop)> { @@ -7469,6 +7474,11 @@ let Inst{18-16} = Zm; } +multiclass sve_bfloat_dot_indexed { + def NAME : sve_bfloat_dot_indexed; + def : SVE_4_Op_Imm_Pat(NAME)>; +} + class sve_bfloat_matmul : I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { @@ -7486,6 +7496,11 @@ let ElementSize = ElementSizeH; } +multiclass sve_bfloat_matmul { + def NAME : sve_bfloat_matmul; + def : SVE_3_Op_Pat(NAME)>; +} + class sve_bfloat_matmul_longvecl : sve_bfloat_matmul { let Inst{23} = 0b1; @@ -7493,6 +7508,11 @@ let Inst{10} = BT; } +multiclass sve_bfloat_matmul_longvecl { + def NAME : sve_bfloat_matmul_longvecl; + def : SVE_3_Op_Pat(NAME)>; +} + class sve_bfloat_matmul_longvecl_idx : sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop", (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexH:$iop)> { @@ -7505,6 +7525,11 @@ let Inst{10} = BT; } +multiclass sve_bfloat_matmul_longvecl_idx { + def NAME : sve_bfloat_matmul_longvecl_idx; + def : SVE_4_Op_Imm_Pat(NAME)>; +} + class sve_bfloat_convert : I<(outs ZPR16:$Zd), (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn), asm, "\t$Zd, $Pg/m, $Zn", "", []>, Sched<[]> { @@ -7524,6 +7549,11 @@ let ElementSize = ElementSizeS; } +multiclass sve_bfloat_convert { + def NAME : sve_bfloat_convert; + def : SVE_3_Op_Pat(NAME)>; +} + //===----------------------------------------------------------------------===// // SVE Integer Matrix Multiply Group //===----------------------------------------------------------------------===// diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-bfloat.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-bfloat.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-bfloat.ll @@ -0,0 +1,243 @@ +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+bf16 -asm-verbose=0 < %s | FileCheck %s + +; +; BFDOT +; + +define @bfdot_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfdot_f32: +; CHECK-NEXT: bfdot z0.s, z1.h, z2.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfdot.nxv4f32( %a, %b, %c) + ret %out +} + +define @bfdot_lane_0_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfdot_lane_0_f32: +; CHECK-NEXT: bfdot z0.s, z1.h, z2.h[0] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfdot.lane.nxv4f32( %a, %b, %c, i64 0) + ret %out +} + +define @bfdot_lane_1_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfdot_lane_1_f32: +; CHECK-NEXT: bfdot z0.s, z1.h, z2.h[1] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfdot.lane.nxv4f32( %a, %b, %c, i64 1) + ret %out +} + +define @bfdot_lane_2_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfdot_lane_2_f32: +; CHECK-NEXT: bfdot z0.s, z1.h, z2.h[2] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfdot.lane.nxv4f32( %a, %b, %c, i64 2) + ret %out +} + +define @bfdot_lane_3_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfdot_lane_3_f32: +; CHECK-NEXT: bfdot z0.s, z1.h, z2.h[3] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfdot.lane.nxv4f32( %a, %b, %c, i64 3) + ret %out +} + +; +; BFMLALB +; + +define @bfmlalb_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalb_f32: +; CHECK-NEXT: bfmlalb z0.s, z1.h, z2.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalb.nxv4f32( %a, %b, %c) + ret %out +} + +define @bfmlalb_lane_0_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalb_lane_0_f32: +; CHECK-NEXT: bfmlalb z0.s, z1.h, z2.h[0] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalb.lane.nxv4f32( %a, %b, %c, i64 0) + ret %out +} + +define @bfmlalb_lane_1_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalb_lane_1_f32: +; CHECK-NEXT: bfmlalb z0.s, z1.h, z2.h[1] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalb.lane.nxv4f32( %a, %b, %c, i64 1) + ret %out +} + +define @bfmlalb_lane_2_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalb_lane_2_f32: +; CHECK-NEXT: bfmlalb z0.s, z1.h, z2.h[2] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalb.lane.nxv4f32( %a, %b, %c, i64 2) + ret %out +} + +define @bfmlalb_lane_3_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalb_lane_3_f32: +; CHECK-NEXT: bfmlalb z0.s, z1.h, z2.h[3] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalb.lane.nxv4f32( %a, %b, %c, i64 3) + ret %out +} + +define @bfmlalb_lane_4_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalb_lane_4_f32: +; CHECK-NEXT: bfmlalb z0.s, z1.h, z2.h[4] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalb.lane.nxv4f32( %a, %b, %c, i64 4) + ret %out +} + +define @bfmlalb_lane_5_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalb_lane_5_f32: +; CHECK-NEXT: bfmlalb z0.s, z1.h, z2.h[5] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalb.lane.nxv4f32( %a, %b, %c, i64 5) + ret %out +} + +define @bfmlalb_lane_6_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalb_lane_6_f32: +; CHECK-NEXT: bfmlalb z0.s, z1.h, z2.h[6] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalb.lane.nxv4f32( %a, %b, %c, i64 6) + ret %out +} + +define @bfmlalb_lane_7_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalb_lane_7_f32: +; CHECK-NEXT: bfmlalb z0.s, z1.h, z2.h[7] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalb.lane.nxv4f32( %a, %b, %c, i64 7) + ret %out +} + +; +; BFMLALT +; + +define @bfmlalt_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalt_f32: +; CHECK-NEXT: bfmlalt z0.s, z1.h, z2.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalt.nxv4f32( %a, %b, %c) + ret %out +} + +define @bfmlalt_lane_0_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalt_lane_0_f32: +; CHECK-NEXT: bfmlalt z0.s, z1.h, z2.h[0] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalt.lane.nxv4f32( %a, %b, %c, i64 0) + ret %out +} + +define @bfmlalt_lane_1_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalt_lane_1_f32: +; CHECK-NEXT: bfmlalt z0.s, z1.h, z2.h[1] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalt.lane.nxv4f32( %a, %b, %c, i64 1) + ret %out +} + +define @bfmlalt_lane_2_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalt_lane_2_f32: +; CHECK-NEXT: bfmlalt z0.s, z1.h, z2.h[2] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalt.lane.nxv4f32( %a, %b, %c, i64 2) + ret %out +} + +define @bfmlalt_lane_3_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalt_lane_3_f32: +; CHECK-NEXT: bfmlalt z0.s, z1.h, z2.h[3] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalt.lane.nxv4f32( %a, %b, %c, i64 3) + ret %out +} + +define @bfmlalt_lane_4_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalt_lane_4_f32: +; CHECK-NEXT: bfmlalt z0.s, z1.h, z2.h[4] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalt.lane.nxv4f32( %a, %b, %c, i64 4) + ret %out +} + +define @bfmlalt_lane_5_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalt_lane_5_f32: +; CHECK-NEXT: bfmlalt z0.s, z1.h, z2.h[5] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalt.lane.nxv4f32( %a, %b, %c, i64 5) + ret %out +} + +define @bfmlalt_lane_6_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalt_lane_6_f32: +; CHECK-NEXT: bfmlalt z0.s, z1.h, z2.h[6] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalt.lane.nxv4f32( %a, %b, %c, i64 6) + ret %out +} + +define @bfmlalt_lane_7_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalt_lane_7_f32: +; CHECK-NEXT: bfmlalt z0.s, z1.h, z2.h[7] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalt.lane.nxv4f32( %a, %b, %c, i64 7) + ret %out +} + +; +; BFMMLA +; + +define @bfmmla_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmmla_f32: +; CHECK-NEXT: bfmmla z0.s, z1.h, z2.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmmla.nxv4f32( %a, %b, %c) + ret %out +} + +; +; BFCVT +; + +define @cvt_bf16_f32( %a, %pg, %b) nounwind { +; CHECK-LABEL: cvt_bf16_f32: +; CHECK-NEXT: bfcvt z0.h, p0/m, z1.s +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.cvt.bf16f32( %a, %pg, %b) + ret %out +} + +; +; BFCVTNT +; + +define @cvtnt_bf16_f32( %a, %pg, %b) nounwind { +; CHECK-LABEL: cvtnt_bf16_f32: +; CHECK-NEXT: bfcvtnt z0.h, p0/m, z1.s +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.cvtnt.bf16f32( %a, %pg, %b) + ret %out +} + +declare @llvm.aarch64.sve.bfdot.nxv4f32(, , ) +declare @llvm.aarch64.sve.bfdot.lane.nxv4f32(, , , i64) +declare @llvm.aarch64.sve.bfmlalb.nxv4f32(, , ) +declare @llvm.aarch64.sve.bfmlalb.lane.nxv4f32(, , , i64) +declare @llvm.aarch64.sve.bfmlalt.nxv4f32(, , ) +declare @llvm.aarch64.sve.bfmlalt.lane.nxv4f32(, , , i64) +declare @llvm.aarch64.sve.bfmmla.nxv4f32(, , ) +declare @llvm.aarch64.sve.cvt.bf16f32(, , ) +declare @llvm.aarch64.sve.cvtnt.bf16f32(, , )