diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td --- a/clang/include/clang/Basic/arm_sve.td +++ b/clang/include/clang/Basic/arm_sve.td @@ -497,6 +497,20 @@ let ArchGuard = "defined(__ARM_FEATURE_SVE_MATMUL_FP64) && defined(__ARM_FEATURE_BF16_SCALAR_ARITHMETIC)" in { def SVLD1RO_BF : SInst<"svld1ro[_{2}]", "dPc", "b", MergeNone, "aarch64_sve_ld1ro">; } + +let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in { + def SVBFDOT : SInst<"svbfdot[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfdot", [IsOverloadNone]>; + def SVBFMLALB : SInst<"svbfmlalb[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmlalb", [IsOverloadNone]>; + def SVBFMLALT : SInst<"svbfmlalt[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmlalt", [IsOverloadNone]>; + def SVBFMMLA : SInst<"svbfmmla[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmmla", [IsOverloadNone]>; + def SVBFDOT_N : SInst<"svbfdot[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfdot", [IsOverloadNone]>; + def SVBFMLAL_N : SInst<"svbfmlalb[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfmlalb", [IsOverloadNone]>; + def SVBFMLALT_N : SInst<"svbfmlalt[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfmlalt", [IsOverloadNone]>; + def SVBFDOT_LANE : SInst<"svbfdot_lane[_{0}]", "MMddn", "b", MergeNone, "aarch64_sve_bfdot_lane", [IsOverloadNone], [ImmCheck<3, ImmCheck0_3>]>; + def SVBFMLALB_LANE : SInst<"svbfmlalb_lane[_{0}]", "MMddn", "b", MergeNone, "aarch64_sve_bfmlalb_lane", [IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>; + def SVBFMLALT_LANE : SInst<"svbfmlalt_lane[_{0}]", "MMddn", "b", MergeNone, "aarch64_sve_bfmlalt_lane", [IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>; +} + //////////////////////////////////////////////////////////////////////////////// // Stores @@ -1030,6 +1044,11 @@ defm SVFCVTZS_S32_F32 : SInstCvtMXZ<"svcvt_s32[_f32]", "ddPM", "dPM", "i", "aarch64_sve_fcvtzs", [IsOverloadCvt]>; defm SVFCVTZS_S64_F32 : SInstCvtMXZ<"svcvt_s64[_f32]", "ddPM", "dPM", "l", "aarch64_sve_fcvtzs_i64f32">; +let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in { + defm SVCVT_BF16_F32 : SInstCvtMXZ<"svcvt_bf16[_f32]", "ddPM", "dPM", "b", "aarch64_sve_fcvt_bf16f32">; + defm SVCVTNT_BF16_F32 : SInstCvtMX<"svcvtnt_bf16[_f32]", "ddPM", "dPM", "b", "aarch64_sve_fcvtnt_bf16f32">; +} + // svcvt_s##_f64 defm SVFCVTZS_S32_F64 : SInstCvtMXZ<"svcvt_s32[_f64]", "ttPd", "tPd", "d", "aarch64_sve_fcvtzs_i32f64">; defm SVFCVTZS_S64_F64 : SInstCvtMXZ<"svcvt_s64[_f64]", "ddPN", "dPN", "l", "aarch64_sve_fcvtzs", [IsOverloadCvt]>; diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfdot.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfdot.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfdot.c @@ -0,0 +1,40 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4 +#endif + +svfloat32_t test_bfdot_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfdot_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfdot( %x, %y, %z) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfdot, _f32, , )(x, y, z); +} + +svfloat32_t test_bfdot_lane_0_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfdot_lane_0_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfdot.lane( %x, %y, %z, i64 0) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfdot_lane, _f32, , )(x, y, z, 0); +} + +svfloat32_t test_bfdot_lane_3_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfdot_lane_3_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfdot.lane( %x, %y, %z, i64 3) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfdot_lane, _f32, , )(x, y, z, 3); +} + +svfloat32_t test_bfdot_n_f32(svfloat32_t x, svbfloat16_t y, bfloat16_t z) { + // CHECK-LABEL: @test_bfdot_n_f32( + // CHECK: %[[SPLAT:.*]] = call @llvm.aarch64.sve.dup.x.nxv8bf16(bfloat %z) + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfdot( %x, %y, %[[SPLAT]]) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfdot, _n_f32, , )(x, y, z); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmlalb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmlalb.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmlalb.c @@ -0,0 +1,40 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4 +#endif + +svfloat32_t test_svbfmlalb_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_svbfmlalb_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalb( %x, %y, %z) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalb, _f32, , )(x, y, z); +} + +svfloat32_t test_bfmlalb_lane_0_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmlalb_lane_0_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalb.lane( %x, %y, %z, i64 0) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalb_lane, _f32, , )(x, y, z, 0); +} + +svfloat32_t test_bfmlalb_lane_7_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmlalb_lane_7_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalb.lane( %x, %y, %z, i64 7) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalb_lane, _f32, , )(x, y, z, 7); +} + +svfloat32_t test_bfmlalb_n_f32(svfloat32_t x, svbfloat16_t y, bfloat16_t z) { + // CHECK-LABEL: @test_bfmlalb_n_f32( + // CHECK: %[[SPLAT:.*]] = call @llvm.aarch64.sve.dup.x.nxv8bf16(bfloat %z) + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalb( %x, %y, %[[SPLAT]]) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalb, _n_f32, , )(x, y, z); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmlalt.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmlalt.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmlalt.c @@ -0,0 +1,40 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4 +#endif + +svfloat32_t test_svbfmlalt_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_svbfmlalt_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalt( %x, %y, %z) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalt, _f32, , )(x, y, z); +} + +svfloat32_t test_bfmlalt_lane_0_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmlalt_lane_0_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalt.lane( %x, %y, %z, i64 0) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalt_lane, _f32, , )(x, y, z, 0); +} + +svfloat32_t test_bfmlalt_lane_7_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmlalt_lane_7_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalt.lane( %x, %y, %z, i64 7) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalt_lane, _f32, , )(x, y, z, 7); +} + +svfloat32_t test_bfmlalt_n_f32(svfloat32_t x, svbfloat16_t y, bfloat16_t z) { + // CHECK-LABEL: @test_bfmlalt_n_f32( + // CHECK: %[[SPLAT:.*]] = call @llvm.aarch64.sve.dup.x.nxv8bf16(bfloat %z) + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmlalt( %x, %y, %[[SPLAT]]) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmlalt, _n_f32, , )(x, y, z); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmmla.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmmla.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmmla.c @@ -0,0 +1,18 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4 +#endif + +svfloat32_t test_bfmmla_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) { + // CHECK-LABEL: @test_bfmmla_f32( + // CHECK: %[[RET:.*]] = call @llvm.aarch64.sve.bfmmla( %x, %y, %z) + // CHECK: ret %[[RET]] + return SVE_ACLE_FUNC(svbfmmla, _f32, , )(x, y, z); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvt-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvt-bfloat.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvt-bfloat.c @@ -0,0 +1,35 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svcvt_bf16_f32_x(svbool_t pg, svfloat32_t op) { + // CHECK-LABEL: test_svcvt_bf16_f32_x + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.fcvt.bf16f32( undef, %[[PG]], %op) + // CHECK: ret %[[INTRINSIC]] + return SVE_ACLE_FUNC(svcvt_bf16, _f32, _x, )(pg, op); +} + +svbfloat16_t test_svcvt_bf16_f32_z(svbool_t pg, svfloat32_t op) { + // CHECK-LABEL: test_svcvt_bf16_f32_z + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.fcvt.bf16f32( zeroinitializer, %[[PG]], %op) + // CHECK: ret %[[INTRINSIC]] + return SVE_ACLE_FUNC(svcvt_bf16, _f32, _z, )(pg, op); +} + +svbfloat16_t test_svcvt_bf16_f32_m(svbfloat16_t inactive, svbool_t pg, svfloat32_t op) { + // CHECK-LABEL: test_svcvt_bf16_f32_m + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.fcvt.bf16f32( %inactive, %[[PG]], %op) + // CHECK: ret %[[INTRINSIC]] + return SVE_ACLE_FUNC(svcvt_bf16, _f32, _m, )(inactive, pg, op); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvtnt.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvtnt.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvtnt.c @@ -0,0 +1,27 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_BF16 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svcvtnt_bf16_f32_x(svbool_t pg, svfloat32_t op) { + // CHECK-LABEL: test_svcvtnt_bf16_f32_x + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.fcvtnt.bf16f32( undef, %[[PG]], %op) + // CHECK: ret %[[INTRINSIC]] + return SVE_ACLE_FUNC(svcvtnt_bf16, _f32, _x, )(pg, op); +} + +svbfloat16_t test_svcvtnt_bf16_f32_m(svbfloat16_t inactive, svbool_t pg, svfloat32_t op) { + // CHECK-LABEL: test_svcvtnt_bf16_f32_m + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.fcvtnt.bf16f32( %inactive, %[[PG]], %op) + // CHECK: ret %[[INTRINSIC]] + return SVE_ACLE_FUNC(svcvtnt_bf16, _f32, _m, )(inactive, pg, op); +} diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp --- a/clang/utils/TableGen/SveEmitter.cpp +++ b/clang/utils/TableGen/SveEmitter.cpp @@ -92,8 +92,8 @@ bool isChar() const { return ElementBitwidth == 8; } bool isVoid() const { return Void & !Pointer; } bool isDefault() const { return DefaultType; } - bool isFloat() const { return Float; } - bool isBFloat() const { return BFloat; } + bool isFloat() const { return Float && !BFloat; } + bool isBFloat() const { return BFloat && !Float; } bool isFloatingPoint() const { return Float || BFloat; } bool isInteger() const { return !isFloatingPoint() && !Predicate; } bool isScalarPredicate() const { @@ -491,6 +491,7 @@ break; case 'b': BFloat = true; + Float = false; ElementBitwidth = 16; break; default: @@ -643,6 +644,7 @@ Predicate = false; Signed = false; Float = false; + BFloat = false; ElementBitwidth = Bitwidth = 64; NumVectors = 0; break; @@ -681,6 +683,7 @@ case 'M': Predicate = false; Float = true; + BFloat = false; ElementBitwidth = 32; break; case 'N': diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td --- a/llvm/include/llvm/IR/IntrinsicsAArch64.td +++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -1340,6 +1340,16 @@ [LLVMMatchType<0>, LLVMSubdivide4VectorType<0>, LLVMSubdivide4VectorType<0>], [IntrNoMem]>; +class SVE_4Vec_BF16 + : Intrinsic<[llvm_nxv4f32_ty], + [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty], + [IntrNoMem]>; + +class SVE_4Vec_BF16_Indexed + : Intrinsic<[llvm_nxv4f32_ty], + [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty, llvm_i64_ty], + [IntrNoMem, ImmArg>]>; + // // Vector tuple creation intrinsics (ACLE) // @@ -1793,6 +1803,9 @@ def int_aarch64_sve_fcvtzs_i64f16 : Builtin_SVCVT<"svcvt_s64_f16_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>; def int_aarch64_sve_fcvtzs_i64f32 : Builtin_SVCVT<"svcvt_s64_f32_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; +def int_aarch64_sve_fcvt_bf16f32 : Builtin_SVCVT<"svcvt_bf16_f32_m", llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>; +def int_aarch64_sve_fcvtnt_bf16f32 : Builtin_SVCVT<"svcvtnt_bf16_f32_m", llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>; + def int_aarch64_sve_fcvtzu_i32f16 : Builtin_SVCVT<"svcvt_u32_f16_m", llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; def int_aarch64_sve_fcvtzu_i32f64 : Builtin_SVCVT<"svcvt_u32_f64_m", llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; def int_aarch64_sve_fcvtzu_i64f16 : Builtin_SVCVT<"svcvt_u64_f16_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>; @@ -2343,6 +2356,19 @@ // def int_aarch64_sve_fmmla : AdvSIMD_3VectorArg_Intrinsic; +// +// SVE ACLE: 7.2. BFloat16 extensions +// + +def int_aarch64_sve_bfdot : SVE_4Vec_BF16; +def int_aarch64_sve_bfmlalb : SVE_4Vec_BF16; +def int_aarch64_sve_bfmlalt : SVE_4Vec_BF16; + +def int_aarch64_sve_bfmmla : SVE_4Vec_BF16; + +def int_aarch64_sve_bfdot_lane : SVE_4Vec_BF16_Indexed; +def int_aarch64_sve_bfmlalb_lane : SVE_4Vec_BF16_Indexed; +def int_aarch64_sve_bfmlalt_lane : SVE_4Vec_BF16_Indexed; } // diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -1239,15 +1239,15 @@ defm FSQRT_ZPmZ : sve_fp_2op_p_zd_HSD<0b01101, "fsqrt", int_aarch64_sve_fsqrt>; let Predicates = [HasBF16, HasSVE] in { - def BFDOT_ZZZ : sve_bfloat_dot<"bfdot">; - def BFDOT_ZZI : sve_bfloat_dot_indexed<"bfdot">; - def BFMMLA_ZZZ : sve_bfloat_matmul<"bfmmla">; - def BFMMLA_B_ZZZ : sve_bfloat_matmul_longvecl<0b0, "bfmlalb">; - def BFMMLA_T_ZZZ : sve_bfloat_matmul_longvecl<0b1, "bfmlalt">; - def BFMMLA_B_ZZI : sve_bfloat_matmul_longvecl_idx<0b0, "bfmlalb">; - def BFMMLA_T_ZZI : sve_bfloat_matmul_longvecl_idx<0b1, "bfmlalt">; - def BFCVT_ZPmZ : sve_bfloat_convert<0b1, "bfcvt">; - def BFCVTNT_ZPmZ : sve_bfloat_convert<0b0, "bfcvtnt">; + defm BFDOT_ZZZ : sve_bfloat_dot<"bfdot", int_aarch64_sve_bfdot>; + defm BFDOT_ZZI : sve_bfloat_dot_indexed<"bfdot", int_aarch64_sve_bfdot_lane>; + defm BFMMLA_ZZZ : sve_bfloat_matmul<"bfmmla", int_aarch64_sve_bfmmla>; + defm BFMMLA_B_ZZZ : sve_bfloat_matmul_longvecl<0b0, "bfmlalb", int_aarch64_sve_bfmlalb>; + defm BFMMLA_T_ZZZ : sve_bfloat_matmul_longvecl<0b1, "bfmlalt", int_aarch64_sve_bfmlalt>; + defm BFMMLA_B_ZZI : sve_bfloat_matmul_longvecl_idx<0b0, "bfmlalb", int_aarch64_sve_bfmlalb_lane>; + defm BFMMLA_T_ZZI : sve_bfloat_matmul_longvecl_idx<0b1, "bfmlalt", int_aarch64_sve_bfmlalt_lane>; + defm BFCVT_ZPmZ : sve_bfloat_convert<0b1, "bfcvt", int_aarch64_sve_fcvt_bf16f32>; + defm BFCVTNT_ZPmZ : sve_bfloat_convert<0b0, "bfcvtnt", int_aarch64_sve_fcvtnt_bf16f32>; } // InstAliases diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -7460,6 +7460,11 @@ let Inst{20-16} = Zm; } +multiclass sve_bfloat_dot { + def NAME : sve_bfloat_dot; + def : SVE_3_Op_Pat(NAME)>; +} + class sve_bfloat_dot_indexed : sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop", (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexS:$iop)> { @@ -7469,6 +7474,11 @@ let Inst{18-16} = Zm; } +multiclass sve_bfloat_dot_indexed { + def NAME : sve_bfloat_dot_indexed; + def : SVE_4_Op_Imm_Pat(NAME)>; +} + class sve_bfloat_matmul : I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { @@ -7486,6 +7496,11 @@ let ElementSize = ElementSizeH; } +multiclass sve_bfloat_matmul { + def NAME : sve_bfloat_matmul; + def : SVE_3_Op_Pat(NAME)>; +} + class sve_bfloat_matmul_longvecl : sve_bfloat_matmul { let Inst{23} = 0b1; @@ -7493,6 +7508,11 @@ let Inst{10} = BT; } +multiclass sve_bfloat_matmul_longvecl { + def NAME : sve_bfloat_matmul_longvecl; + def : SVE_3_Op_Pat(NAME)>; +} + class sve_bfloat_matmul_longvecl_idx : sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop", (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexH:$iop)> { @@ -7505,6 +7525,11 @@ let Inst{10} = BT; } +multiclass sve_bfloat_matmul_longvecl_idx { + def NAME : sve_bfloat_matmul_longvecl_idx; + def : SVE_4_Op_Imm_Pat(NAME)>; +} + class sve_bfloat_convert : I<(outs ZPR16:$Zd), (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn), asm, "\t$Zd, $Pg/m, $Zn", "", []>, Sched<[]> { @@ -7524,6 +7549,11 @@ let ElementSize = ElementSizeS; } +multiclass sve_bfloat_convert { + def NAME : sve_bfloat_convert; + def : SVE_3_Op_Pat(NAME)>; +} + //===----------------------------------------------------------------------===// // SVE Integer Matrix Multiply Group //===----------------------------------------------------------------------===// diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-bfloat.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-bfloat.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-bfloat.ll @@ -0,0 +1,243 @@ +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+bf16 -asm-verbose=0 < %s | FileCheck %s + +; +; BFDOT +; + +define @bfdot_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfdot_f32: +; CHECK-NEXT: bfdot z0.s, z1.h, z2.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfdot( %a, %b, %c) + ret %out +} + +define @bfdot_lane_0_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfdot_lane_0_f32: +; CHECK-NEXT: bfdot z0.s, z1.h, z2.h[0] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfdot.lane( %a, %b, %c, i64 0) + ret %out +} + +define @bfdot_lane_1_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfdot_lane_1_f32: +; CHECK-NEXT: bfdot z0.s, z1.h, z2.h[1] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfdot.lane( %a, %b, %c, i64 1) + ret %out +} + +define @bfdot_lane_2_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfdot_lane_2_f32: +; CHECK-NEXT: bfdot z0.s, z1.h, z2.h[2] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfdot.lane( %a, %b, %c, i64 2) + ret %out +} + +define @bfdot_lane_3_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfdot_lane_3_f32: +; CHECK-NEXT: bfdot z0.s, z1.h, z2.h[3] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfdot.lane( %a, %b, %c, i64 3) + ret %out +} + +; +; BFMLALB +; + +define @bfmlalb_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalb_f32: +; CHECK-NEXT: bfmlalb z0.s, z1.h, z2.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalb( %a, %b, %c) + ret %out +} + +define @bfmlalb_lane_0_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalb_lane_0_f32: +; CHECK-NEXT: bfmlalb z0.s, z1.h, z2.h[0] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalb.lane( %a, %b, %c, i64 0) + ret %out +} + +define @bfmlalb_lane_1_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalb_lane_1_f32: +; CHECK-NEXT: bfmlalb z0.s, z1.h, z2.h[1] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalb.lane( %a, %b, %c, i64 1) + ret %out +} + +define @bfmlalb_lane_2_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalb_lane_2_f32: +; CHECK-NEXT: bfmlalb z0.s, z1.h, z2.h[2] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalb.lane( %a, %b, %c, i64 2) + ret %out +} + +define @bfmlalb_lane_3_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalb_lane_3_f32: +; CHECK-NEXT: bfmlalb z0.s, z1.h, z2.h[3] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalb.lane( %a, %b, %c, i64 3) + ret %out +} + +define @bfmlalb_lane_4_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalb_lane_4_f32: +; CHECK-NEXT: bfmlalb z0.s, z1.h, z2.h[4] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalb.lane( %a, %b, %c, i64 4) + ret %out +} + +define @bfmlalb_lane_5_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalb_lane_5_f32: +; CHECK-NEXT: bfmlalb z0.s, z1.h, z2.h[5] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalb.lane( %a, %b, %c, i64 5) + ret %out +} + +define @bfmlalb_lane_6_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalb_lane_6_f32: +; CHECK-NEXT: bfmlalb z0.s, z1.h, z2.h[6] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalb.lane( %a, %b, %c, i64 6) + ret %out +} + +define @bfmlalb_lane_7_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalb_lane_7_f32: +; CHECK-NEXT: bfmlalb z0.s, z1.h, z2.h[7] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalb.lane( %a, %b, %c, i64 7) + ret %out +} + +; +; BFMLALT +; + +define @bfmlalt_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalt_f32: +; CHECK-NEXT: bfmlalt z0.s, z1.h, z2.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalt( %a, %b, %c) + ret %out +} + +define @bfmlalt_lane_0_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalt_lane_0_f32: +; CHECK-NEXT: bfmlalt z0.s, z1.h, z2.h[0] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalt.lane( %a, %b, %c, i64 0) + ret %out +} + +define @bfmlalt_lane_1_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalt_lane_1_f32: +; CHECK-NEXT: bfmlalt z0.s, z1.h, z2.h[1] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalt.lane( %a, %b, %c, i64 1) + ret %out +} + +define @bfmlalt_lane_2_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalt_lane_2_f32: +; CHECK-NEXT: bfmlalt z0.s, z1.h, z2.h[2] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalt.lane( %a, %b, %c, i64 2) + ret %out +} + +define @bfmlalt_lane_3_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalt_lane_3_f32: +; CHECK-NEXT: bfmlalt z0.s, z1.h, z2.h[3] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalt.lane( %a, %b, %c, i64 3) + ret %out +} + +define @bfmlalt_lane_4_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalt_lane_4_f32: +; CHECK-NEXT: bfmlalt z0.s, z1.h, z2.h[4] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalt.lane( %a, %b, %c, i64 4) + ret %out +} + +define @bfmlalt_lane_5_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalt_lane_5_f32: +; CHECK-NEXT: bfmlalt z0.s, z1.h, z2.h[5] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalt.lane( %a, %b, %c, i64 5) + ret %out +} + +define @bfmlalt_lane_6_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalt_lane_6_f32: +; CHECK-NEXT: bfmlalt z0.s, z1.h, z2.h[6] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalt.lane( %a, %b, %c, i64 6) + ret %out +} + +define @bfmlalt_lane_7_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmlalt_lane_7_f32: +; CHECK-NEXT: bfmlalt z0.s, z1.h, z2.h[7] +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmlalt.lane( %a, %b, %c, i64 7) + ret %out +} + +; +; BFMMLA +; + +define @bfmmla_f32( %a, %b, %c) nounwind { +; CHECK-LABEL: bfmmla_f32: +; CHECK-NEXT: bfmmla z0.s, z1.h, z2.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bfmmla( %a, %b, %c) + ret %out +} + +; +; BFCVT +; + +define @fcvt_bf16_f32( %a, %pg, %b) nounwind { +; CHECK-LABEL: fcvt_bf16_f32: +; CHECK-NEXT: bfcvt z0.h, p0/m, z1.s +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.fcvt.bf16f32( %a, %pg, %b) + ret %out +} + +; +; BFCVTNT +; + +define @fcvtnt_bf16_f32( %a, %pg, %b) nounwind { +; CHECK-LABEL: fcvtnt_bf16_f32: +; CHECK-NEXT: bfcvtnt z0.h, p0/m, z1.s +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.fcvtnt.bf16f32( %a, %pg, %b) + ret %out +} + +declare @llvm.aarch64.sve.bfdot(, , ) +declare @llvm.aarch64.sve.bfdot.lane(, , , i64) +declare @llvm.aarch64.sve.bfmlalb(, , ) +declare @llvm.aarch64.sve.bfmlalb.lane(, , , i64) +declare @llvm.aarch64.sve.bfmlalt(, , ) +declare @llvm.aarch64.sve.bfmlalt.lane(, , , i64) +declare @llvm.aarch64.sve.bfmmla(, , ) +declare @llvm.aarch64.sve.fcvt.bf16f32(, , ) +declare @llvm.aarch64.sve.fcvtnt.bf16f32(, , )