diff --git a/clang/include/clang/Basic/arm_neon.td b/clang/include/clang/Basic/arm_neon.td --- a/clang/include/clang/Basic/arm_neon.td +++ b/clang/include/clang/Basic/arm_neon.td @@ -190,28 +190,20 @@ def OP_SCALAR_QRDMLSH_LN : Op<(call "vqsub", $p0, (call "vqrdmulh", $p1, (call "vget_lane", $p2, $p3)))>; -multiclass ScalarGetSetLaneOpsF16 { - def _GET_LN : Op<(bitcast scalarTy, - (call "vget_lane", - (bitcast "int16x4_t", $p0), $p1))>; - def _GET_LNQ : Op<(bitcast scalarTy, - (call "vget_lane", - (bitcast "int16x8_t", $p0), $p1))>; - def _SET_LN : Op<(bitcast vectorTy4, - (call "vset_lane", - (bitcast "int16_t", $p0), - (bitcast "int16x4_t", $p1), $p2))>; - def _SET_LNQ : Op<(bitcast vectorTy8, - (call "vset_lane", - (bitcast "int16_t", $p0), - (bitcast "int16x8_t", $p1), $p2))>; -} - -defm OP_SCALAR_HALF: ScalarGetSetLaneOpsF16<"float16_t", - "float16x4_t", "float16x8_t">; -defm OP_SCALAR_BF16: ScalarGetSetLaneOpsF16<"bfloat16_t", - "bfloat16x4_t", "bfloat16x8_t">; +def OP_SCALAR_HALF_GET_LN : Op<(bitcast "float16_t", + (call "vget_lane", + (bitcast "int16x4_t", $p0), $p1))>; +def OP_SCALAR_HALF_GET_LNQ : Op<(bitcast "float16_t", + (call "vget_lane", + (bitcast "int16x8_t", $p0), $p1))>; +def OP_SCALAR_HALF_SET_LN : Op<(bitcast "float16x4_t", + (call "vset_lane", + (bitcast "int16_t", $p0), + (bitcast "int16x4_t", $p1), $p2))>; +def OP_SCALAR_HALF_SET_LNQ : Op<(bitcast "float16x8_t", + (call "vset_lane", + (bitcast "int16_t", $p0), + (bitcast "int16x8_t", $p1), $p2))>; def OP_DOT_LN : Op<(call "vdot", $p0, $p1, @@ -1918,10 +1910,12 @@ def VGET_HIGH_BF : NoTestOpInst<"vget_high", ".Q", "b", OP_HI>; def VGET_LOW_BF : NoTestOpInst<"vget_low", ".Q", "b", OP_LO>; - def VGET_LANE_BF : IOpInst<"vget_lane", "1.I", "b", OP_SCALAR_BF16_GET_LN>; - def VSET_LANE_BF : IOpInst<"vset_lane", ".1.I", "b", OP_SCALAR_BF16_SET_LN>; - def VGET_LANEQ_BF : IOpInst<"vget_lane", "1.I", "Qb", OP_SCALAR_BF16_GET_LNQ>; - def VSET_LANEQ_BF : IOpInst<"vset_lane", ".1.I", "Qb", OP_SCALAR_BF16_SET_LNQ>; + def VGET_LANE_BF : IInst<"vget_lane", "1.I", "bQb">; + def VSET_LANE_BF : IInst<"vset_lane", ".1.I", "bQb">; + def SCALAR_VDUP_LANE_BF : IInst<"vdup_lane", "1.I", "Sb">; + def SCALAR_VDUP_LANEQ_BF : IInst<"vdup_laneq", "1QI", "Sb"> { + let isLaneQ = 1; + } def VLD1_BF : WInst<"vld1", ".(c*!)", "bQb">; def VLD2_BF : WInst<"vld2", "2(c*!)", "bQb">; @@ -1957,18 +1951,6 @@ } -let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC) && !defined(__aarch64__)" in { - def SCALAR_VDUP_LANE_BF_A32 : IOpInst<"vduph_lane", "1.I", "b", OP_SCALAR_BF16_GET_LN>; - def SCALAR_VDUP_LANEQ_BF_A32 : IOpInst<"vduph_laneq", "1.I", "Hb", OP_SCALAR_BF16_GET_LNQ>; -} - -let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC) && defined(__aarch64__)" in { - def SCALAR_VDUP_LANE_BF_A64 : IInst<"vdup_lane", "1.I", "Sb">; - def SCALAR_VDUP_LANEQ_BF_A64 : IInst<"vdup_laneq", "1QI", "Sb"> { - let isLaneQ = 1; - } -} - let ArchGuard = "defined(__ARM_FEATURE_BF16) && !defined(__aarch64__)" in { let BigEndianSafe = 1 in { defm VREINTERPRET_BF : REINTERPRET_CROSS_TYPES< diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -6416,21 +6416,27 @@ default: break; case NEON::BI__builtin_neon_vget_lane_i8: case NEON::BI__builtin_neon_vget_lane_i16: + case NEON::BI__builtin_neon_vget_lane_bf16: case NEON::BI__builtin_neon_vget_lane_i32: case NEON::BI__builtin_neon_vget_lane_i64: case NEON::BI__builtin_neon_vget_lane_f32: case NEON::BI__builtin_neon_vgetq_lane_i8: case NEON::BI__builtin_neon_vgetq_lane_i16: + case NEON::BI__builtin_neon_vgetq_lane_bf16: case NEON::BI__builtin_neon_vgetq_lane_i32: case NEON::BI__builtin_neon_vgetq_lane_i64: case NEON::BI__builtin_neon_vgetq_lane_f32: + case NEON::BI__builtin_neon_vduph_lane_bf16: + case NEON::BI__builtin_neon_vduph_laneq_bf16: case NEON::BI__builtin_neon_vset_lane_i8: case NEON::BI__builtin_neon_vset_lane_i16: + case NEON::BI__builtin_neon_vset_lane_bf16: case NEON::BI__builtin_neon_vset_lane_i32: case NEON::BI__builtin_neon_vset_lane_i64: case NEON::BI__builtin_neon_vset_lane_f32: case NEON::BI__builtin_neon_vsetq_lane_i8: case NEON::BI__builtin_neon_vsetq_lane_i16: + case NEON::BI__builtin_neon_vsetq_lane_bf16: case NEON::BI__builtin_neon_vsetq_lane_i32: case NEON::BI__builtin_neon_vsetq_lane_i64: case NEON::BI__builtin_neon_vsetq_lane_f32: @@ -6876,12 +6882,16 @@ case NEON::BI__builtin_neon_vget_lane_i16: case NEON::BI__builtin_neon_vget_lane_i32: case NEON::BI__builtin_neon_vget_lane_i64: + case NEON::BI__builtin_neon_vget_lane_bf16: case NEON::BI__builtin_neon_vget_lane_f32: case NEON::BI__builtin_neon_vgetq_lane_i8: case NEON::BI__builtin_neon_vgetq_lane_i16: case NEON::BI__builtin_neon_vgetq_lane_i32: case NEON::BI__builtin_neon_vgetq_lane_i64: + case NEON::BI__builtin_neon_vgetq_lane_bf16: case NEON::BI__builtin_neon_vgetq_lane_f32: + case NEON::BI__builtin_neon_vduph_lane_bf16: + case NEON::BI__builtin_neon_vduph_laneq_bf16: return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane"); case NEON::BI__builtin_neon_vrndns_f32: { @@ -6894,11 +6904,13 @@ case NEON::BI__builtin_neon_vset_lane_i16: case NEON::BI__builtin_neon_vset_lane_i32: case NEON::BI__builtin_neon_vset_lane_i64: + case NEON::BI__builtin_neon_vset_lane_bf16: case NEON::BI__builtin_neon_vset_lane_f32: case NEON::BI__builtin_neon_vsetq_lane_i8: case NEON::BI__builtin_neon_vsetq_lane_i16: case NEON::BI__builtin_neon_vsetq_lane_i32: case NEON::BI__builtin_neon_vsetq_lane_i64: + case NEON::BI__builtin_neon_vsetq_lane_bf16: case NEON::BI__builtin_neon_vsetq_lane_f32: return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); @@ -9309,11 +9321,13 @@ case NEON::BI__builtin_neon_vset_lane_i16: case NEON::BI__builtin_neon_vset_lane_i32: case NEON::BI__builtin_neon_vset_lane_i64: + case NEON::BI__builtin_neon_vset_lane_bf16: case NEON::BI__builtin_neon_vset_lane_f32: case NEON::BI__builtin_neon_vsetq_lane_i8: case NEON::BI__builtin_neon_vsetq_lane_i16: case NEON::BI__builtin_neon_vsetq_lane_i32: case NEON::BI__builtin_neon_vsetq_lane_i64: + case NEON::BI__builtin_neon_vsetq_lane_bf16: case NEON::BI__builtin_neon_vsetq_lane_f32: Ops.push_back(EmitScalarExpr(E->getArg(2))); return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); @@ -9592,11 +9606,13 @@ : Intrinsic::aarch64_neon_sqsub; return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl"); } + case NEON::BI__builtin_neon_vget_lane_bf16: case NEON::BI__builtin_neon_vduph_lane_bf16: case NEON::BI__builtin_neon_vduph_lane_f16: { return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), "vget_lane"); } + case NEON::BI__builtin_neon_vgetq_lane_bf16: case NEON::BI__builtin_neon_vduph_laneq_bf16: case NEON::BI__builtin_neon_vduph_laneq_f16: { return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), diff --git a/clang/test/CodeGen/arm-bf16-getset-intrinsics.c b/clang/test/CodeGen/arm-bf16-getset-intrinsics.c --- a/clang/test/CodeGen/arm-bf16-getset-intrinsics.c +++ b/clang/test/CodeGen/arm-bf16-getset-intrinsics.c @@ -1,6 +1,8 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // RUN: %clang_cc1 -triple armv8.6a-arm-none-eabi -target-feature +neon -target-feature +bf16 -mfloat-abi hard \ // RUN: -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg -instcombine | FileCheck %s +// RUN: %clang_cc1 -triple armv8.6a-arm-none-eabi -target-feature +neon -target-feature +bf16 -mfloat-abi soft \ +// RUN: -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg -instcombine | FileCheck %s #include @@ -98,8 +100,8 @@ // CHECK-LABEL: @test_vget_lane_bf16( // CHECK-NEXT: entry: -// CHECK-NEXT: [[DOTCAST1:%.*]] = extractelement <4 x bfloat> [[V:%.*]], i32 1 -// CHECK-NEXT: ret bfloat [[DOTCAST1]] +// CHECK-NEXT: [[VGET_LANE:%.*]] = extractelement <4 x bfloat> [[V:%.*]], i32 1 +// CHECK-NEXT: ret bfloat [[VGET_LANE]] // bfloat16_t test_vget_lane_bf16(bfloat16x4_t v) { return vget_lane_bf16(v, 1); @@ -107,8 +109,8 @@ // CHECK-LABEL: @test_vgetq_lane_bf16( // CHECK-NEXT: entry: -// CHECK-NEXT: [[DOTCAST1:%.*]] = extractelement <8 x bfloat> [[V:%.*]], i32 7 -// CHECK-NEXT: ret bfloat [[DOTCAST1]] +// CHECK-NEXT: [[VGET_LANE:%.*]] = extractelement <8 x bfloat> [[V:%.*]], i32 7 +// CHECK-NEXT: ret bfloat [[VGET_LANE]] // bfloat16_t test_vgetq_lane_bf16(bfloat16x8_t v) { return vgetq_lane_bf16(v, 7); @@ -116,8 +118,8 @@ // CHECK-LABEL: @test_vset_lane_bf16( // CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x bfloat> [[V:%.*]], bfloat [[A:%.*]], i32 1 -// CHECK-NEXT: ret <4 x bfloat> [[TMP0]] +// CHECK-NEXT: [[VSET_LANE:%.*]] = insertelement <4 x bfloat> [[V:%.*]], bfloat [[A:%.*]], i32 1 +// CHECK-NEXT: ret <4 x bfloat> [[VSET_LANE]] // bfloat16x4_t test_vset_lane_bf16(bfloat16_t a, bfloat16x4_t v) { return vset_lane_bf16(a, v, 1); @@ -125,8 +127,8 @@ // CHECK-LABEL: @test_vsetq_lane_bf16( // CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = insertelement <8 x bfloat> [[V:%.*]], bfloat [[A:%.*]], i32 7 -// CHECK-NEXT: ret <8 x bfloat> [[TMP0]] +// CHECK-NEXT: [[VSET_LANE:%.*]] = insertelement <8 x bfloat> [[V:%.*]], bfloat [[A:%.*]], i32 7 +// CHECK-NEXT: ret <8 x bfloat> [[VSET_LANE]] // bfloat16x8_t test_vsetq_lane_bf16(bfloat16_t a, bfloat16x8_t v) { return vsetq_lane_bf16(a, v, 7); @@ -143,8 +145,8 @@ // CHECK-LABEL: @test_vduph_laneq_bf16( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VGETQ_LANE:%.*]] = extractelement <8 x bfloat> [[V:%.*]], i32 7 -// CHECK-NEXT: ret bfloat [[VGETQ_LANE]] +// CHECK-NEXT: [[VGET_LANE:%.*]] = extractelement <8 x bfloat> [[V:%.*]], i32 7 +// CHECK-NEXT: ret bfloat [[VGET_LANE]] // bfloat16_t test_vduph_laneq_bf16(bfloat16x8_t v) { return vduph_laneq_bf16(v, 7);