[AArch64][SVE] Enable bf16 vector.insert

[MattDevereau]

Summary

Allow passthrough bf16 registers for vector.insert

[peterwaller-arm]

Looks reasonable to me.

[paulwalker-arm]

Do the tests have to live in a new file instead of sitting alongside the other variants in sve-insert-vector.ll? I see nothing SVE2 related here.

[MattDevereau]

In D114858#3163710, @paulwalker-arm wrote:

Do the tests have to live in a new file instead of sitting alongside the other variants in sve-insert-vector.ll? I see nothing SVE2 related here.

I saw #if defined(__ARM_FEATURE_SVE2) && defined(__ARM_FEATURE_SVE_BF16) in another file so assumed bf16 was SVE2 only

[paulwalker-arm]

In D114858#3163966, @MattDevereau wrote:

In D114858#3163710, @paulwalker-arm wrote:

Do the tests have to live in a new file instead of sitting alongside the other variants in sve-insert-vector.ll? I see nothing SVE2 related here.

I saw #if defined(__ARM_FEATURE_SVE2) && defined(__ARM_FEATURE_SVE_BF16) in another file so assumed bf16 was SVE2 only

That will be to protect SVE2 specific BF16 instructions. SVE also has BF16 instructions, for example BFDOT. These tests only really care about the type as no BF16 related instructions are necessary so they can all sit within the same file. For a similar example see sve-vector-splat.ll.

[MattDevereau]

In D114858#3164008, @paulwalker-arm wrote:

In D114858#3163966, @MattDevereau wrote:

In D114858#3163710, @paulwalker-arm wrote:

Do the tests have to live in a new file instead of sitting alongside the other variants in sve-insert-vector.ll? I see nothing SVE2 related here.

I saw #if defined(__ARM_FEATURE_SVE2) && defined(__ARM_FEATURE_SVE_BF16) in another file so assumed bf16 was SVE2 only

That will be to protect SVE2 specific BF16 instructions. SVE also has BF16 instructions, for example BFDOT. These tests only really care about the type as no BF16 related instructions are necessary so they can all sit within the same file. For a similar example see sve-vector-splat.ll.

I realise that now since I ran some tests with sve+bf16 and they work fine. I'll move these back into the previous file :)

[MattDevereau]

moved tests into sve-insert-vector.ll

[MattDevereau]

removed stray #

[paulwalker-arm]

One minor issue to resolve before committing.

llvm/test/CodeGen/AArch64/sve-insert-vector.ll

469–470

Presumably this is bogus and should be removed?

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

for (auto VT : {MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16, MVT::nxv2f32,
       setOperationAction(ISD::SELECT_CC, VT, Expand);
     }
 
     for (auto VT : {MVT::nxv2bf16, MVT::nxv4bf16, MVT::nxv8bf16}) {
       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
       setOperationAction(ISD::MGATHER, VT, Custom);
       setOperationAction(ISD::MSCATTER, VT, Custom);
       setOperationAction(ISD::MLOAD, VT, Custom);
+      setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
     }
 
     setOperationAction(ISD::SPLAT_VECTOR, MVT::nxv8bf16, Custom);
 
     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom);
     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom);
 
     // NOTE: Currently this has to happen after computeRegisterProperties rather

llvm/test/CodeGen/AArch64/sve-insert-vector.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s --check-prefixes=CHECK
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -mattr=+bf16 < %s | FileCheck %s --check-prefixes=CHECK
 
 define <vscale x 2 x i64> @insert_v2i64_nxv2i64(<vscale x 2 x i64> %vec, <2 x i64> %subvec) nounwind {
 ; CHECK-LABEL: insert_v2i64_nxv2i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    st1d { z0.d }, p0, [sp]

 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ret
   %v0 = call <vscale x 12 x i32> @llvm.experimental.vector.insert.nxv4i32.nxv12i32(<vscale x 12 x i32> undef, <vscale x 4 x i32> %sv0, i64 0)
   %v1 = call <vscale x 12 x i32> @llvm.experimental.vector.insert.nxv4i32.nxv12i32(<vscale x 12 x i32> %v0, <vscale x 4 x i32> %sv1, i64 4)
   %v2 = call <vscale x 12 x i32> @llvm.experimental.vector.insert.nxv4i32.nxv12i32(<vscale x 12 x i32> %v1, <vscale x 4 x i32> %sv2, i64 8)
   ret <vscale x 12 x i32> %v2
 }
 
+define <vscale x 2 x bfloat> @insert_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %sv0, <vscale x 2 x bfloat> %sv1) nounwind {
+; CHECK-LABEL: insert_nxv2bf16_nxv2bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    z0.d, z1.d
+; CHECK-NEXT:    ret
+  %v0 = call <vscale x 2 x bfloat> @llvm.experimental.vector.insert.nxv2bf16.nxv2bf16(<vscale x 2 x bfloat> %sv0, <vscale x 2 x bfloat> %sv1, i64 0)
+  ret <vscale x 2 x bfloat> %v0
+}
+
+define <vscale x 4 x bfloat> @insert_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %sv0, <vscale x 4 x bfloat> %sv1) nounwind {
+; CHECK-LABEL: insert_nxv4bf16_nxv4bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    z0.d, z1.d
+; CHECK-NEXT:    ret
+  %v0 = call <vscale x 4 x bfloat> @llvm.experimental.vector.insert.nxv4bf16.nxv4bf16(<vscale x 4 x bfloat> %sv0, <vscale x 4 x bfloat> %sv1, i64 0)
+  ret <vscale x 4 x bfloat> %v0
+}
+
+define <vscale x 4 x bfloat> @insert_nxv4bf16_v4bf16(<vscale x 4 x bfloat> %sv0, <4 x bfloat> %v1) nounwind {
+; CHECK-LABEL: insert_nxv4bf16_v4bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str     x29, [sp, #-16]!                // 8-byte Folded Spill
+; CHECK-NEXT:    addvl   sp, sp, #-1
+; CHECK-NEXT:    ptrue   p0.s
+; CHECK-NEXT:    st1h    { z0.s }, p0, [sp, #1, mul vl]
+; CHECK-NEXT:    addpl   x8, sp, #4
+; CHECK-NEXT:    str     d1, [x8]
+; CHECK-NEXT:    ld1h    { z0.s }, p0/z, [sp, #1, mul vl]
+; CHECK-NEXT:    addvl   sp, sp, #1
+; CHECK-NEXT:    ldr     x29, [sp], #16                  // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %v0 = call <vscale x 4 x bfloat> @llvm.experimental.vector.insert.nxv4bf16.v4bf16(<vscale x 4 x bfloat> %sv0, <4 x bfloat> %v1, i64 0)
+  ret <vscale x 4 x bfloat> %v0
+}
+
+define <vscale x 8 x bfloat> @insert_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %sv0, <vscale x 8 x bfloat> %sv1) nounwind {
+; CHECK-LABEL: insert_nxv8bf16_nxv8bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    z0.d, z1.d
+; CHECK-NEXT:    ret
+  %v0 = call <vscale x 8 x bfloat> @llvm.experimental.vector.insert.nxv8bf16.nxv8bf16(<vscale x 8 x bfloat> %sv0, <vscale x 8 x bfloat> %sv1, i64 0)
+  ret <vscale x 8 x bfloat> %v0
+}
+
+define <vscale x 8 x bfloat> @insert_nxv8bf16_v8bf16(<vscale x 8 x bfloat> %sv0, <8 x bfloat> %v1) nounwind {
+; CHECK-LABEL: insert_nxv8bf16_v8bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str     x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl   sp, sp, #-1
+; CHECK-NEXT:    ptrue   p0.h
+; CHECK-NEXT:    st1h    { z0.h }, p0, [sp]
+; CHECK-NEXT:    str     q1, [sp]
+; CHECK-NEXT:    ld1h    { z0.h }, p0/z, [sp]
+; CHECK-NEXT:    addvl   sp, sp, #1
+; CHECK-NEXT:    ldr     x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %v0 = call <vscale x 8 x bfloat> @llvm.experimental.vector.insert.nxv8bf16.v8bf16(<vscale x 8 x bfloat> %sv0, <8 x bfloat> %v1, i64 0)
+  ret <vscale x 8 x bfloat> %v0
+}
+
 declare <vscale x 3 x i32> @llvm.experimental.vector.insert.nxv3i32.nxv2i32(<vscale x 3 x i32>, <vscale x 2 x i32>, i64)
 declare <vscale x 3 x float> @llvm.experimental.vector.insert.nxv3f32.nxv2f32(<vscale x 3 x float>, <vscale x 2 x float>, i64)
 declare <vscale x 6 x i32> @llvm.experimental.vector.insert.nxv6i32.nxv2i32(<vscale x 6 x i32>, <vscale x 2 x i32>, i64)
 declare <vscale x 6 x i32> @llvm.experimental.vector.insert.nxv6i32.nxv3i32(<vscale x 6 x i32>, <vscale x 3 x i32>, i64)
 declare <vscale x 12 x i32> @llvm.experimental.vector.insert.nxv4i32.nxv12i32(<vscale x 12 x i32>, <vscale x 4 x i32>, i64)
+declare <vscale x 8 x bfloat> @llvm.experimental.vector.insert.nxv8bf16.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, i64)
+declare <vscale x 8 x bfloat> @llvm.experimental.vector.insert.nxv8bf16.v8bf16(<vscale x 8 x bfloat>, <8 x bfloat>, i64)
+declare <vscale x 4 x bfloat> @llvm.experimental.vector.insert.nxv4bf16.nxv4bf16(<vscale x 4 x bfloat>, <vscale x 4 x bfloat>, i64)
+declare <vscale x 4 x bfloat> @llvm.experimental.vector.insert.nxv4bf16.v4bf16(<vscale x 4 x bfloat>, <4 x bfloat>, i64)
+declare <vscale x 2 x bfloat> @llvm.experimental.vector.insert.nxv2bf16.nxv2bf16(<vscale x 2 x bfloat>, <vscale x 2 x bfloat>, i64)