[X86] Enable intrinsics that convert float and bf16 data to each other

[skan]

Summary

Scalar version :
_mm_cvtsbh_ss , _mm_cvtness_sbh

Vector version:
_mm512_cvtpbh_ps , _mm256_cvtpbh_ps
_mm512_maskz_cvtpbh_ps , _mm256_maskz_cvtpbh_ps
_mm512_mask_cvtpbh_ps , _mm256_mask_cvtpbh_ps

[RKSimon]

lib/Headers/avx512bf16intrin.h

82

\headerfile <x86intrin.h>

84

Please can you write a better description?

96

\headerfile <x86intrin.h>

102

style/clang-format changes like these should be separated into their own patch

[skan]

rm clang-format fix in this patch

[skan]

lib/Headers/avx512bf16intrin.h

82

fixed in updated patch

96

fixed in updated patch

102

style changes are removed in the updated patch

[craig.topper]

lib/Headers/avx512bf16intrin.h

37

This needs to be a comment that's useful to user's of the compiler not compiler developers. It can't reference an implementation function inside the compiler

[skan]

lib/Headers/avx512bf16intrin.h

37

i can not think out a comment that 's needed by users since other comments has explained this function clearly, can i just remove this comment?

[craig.topper]

lib/Headers/avx512bf16intrin.h

37

Just write "This intrinsic does not correspond to a specific instruction,"

[skan]

make comments for _mm_cvtsbh_ss and _mm_cvtness_sbh better

[craig.topper]

LGTM

[RKSimon]

Yup, doxygen LGTM - cheers

lib/Headers/avx512bf16intrin.h

33

Single Float Data

[skan]

When target feature avx512bf16 is enabled, avx512vl is not enabled implicity for some reasons by now. But _mm_cvtness_sbh, _mm256_cvtpbh_ps , _mm256_maskz_cvtpbh_ps , _mm256_mask_cvtpbh_ps need avx512vl feature. So I move their definition from avx512bf16intrin to avx512vlbf16intrin.h, and move corresponding test cases from avx512bf16-builtins.c to avx512vlbf16-builtins.c.

And the test for _mm_cvtness_sbh is fixed in the new patch.

[skan]

avoid implicit type conversion for intrinsic parameter

[skan]

change mask paramater' s name from __M to __U in order to be consistent with other intrinsics in the file

[craig.topper]

LGTM

include/clang/Basic/BuiltinsX86.def

 TARGET_BUILTIN(__builtin_ia32_cvtw2mask256, "UsV16s", "ncV:256:", "avx512bw,avx512vl")
 TARGET_BUILTIN(__builtin_ia32_cvtsd2ss_round_mask, "V4fV4fV2dV4fUcIi", "ncV:128:", "avx512f")
 TARGET_BUILTIN(__builtin_ia32_cvtsi2ss32, "V4fV4fiIi", "ncV:128:", "avx512f")
 TARGET_BUILTIN(__builtin_ia32_cvtss2sd_round_mask, "V2dV2dV4fV2dUcIi", "ncV:128:", "avx512f")
 TARGET_BUILTIN(__builtin_ia32_cvtusi2ss32, "V4fV4fUiIi", "ncV:128:", "avx512f")
 TARGET_BUILTIN(__builtin_ia32_vpmultishiftqb512, "V64cV64cV64c", "ncV:512:", "avx512vbmi")
 TARGET_BUILTIN(__builtin_ia32_vpmultishiftqb128, "V16cV16cV16c", "ncV:128:", "avx512vbmi,avx512vl")
 TARGET_BUILTIN(__builtin_ia32_vpmultishiftqb256, "V32cV32cV32c", "ncV:256:", "avx512vbmi,avx512vl")
+
+// bf16 intrinsics
 TARGET_BUILTIN(__builtin_ia32_cvtne2ps2bf16_128, "V8sV4fV4f", "ncV:128:", "avx512bf16,avx512vl")
 TARGET_BUILTIN(__builtin_ia32_cvtne2ps2bf16_256, "V16sV8fV8f", "ncV:256:", "avx512bf16,avx512vl")
 TARGET_BUILTIN(__builtin_ia32_cvtne2ps2bf16_512, "V32sV16fV16f", "ncV:512:", "avx512bf16")
 TARGET_BUILTIN(__builtin_ia32_cvtneps2bf16_128_mask, "V8sV4fV8sUc", "ncV:128:", "avx512bf16,avx512vl")
 TARGET_BUILTIN(__builtin_ia32_cvtneps2bf16_256_mask, "V8sV8fV8sUc", "ncV:256:", "avx512bf16,avx512vl")
 TARGET_BUILTIN(__builtin_ia32_cvtneps2bf16_512_mask, "V16sV16fV16sUs", "ncV:512:", "avx512bf16")
 TARGET_BUILTIN(__builtin_ia32_dpbf16ps_128, "V4fV4fV4iV4i", "ncV:128:", "avx512bf16,avx512vl")
 TARGET_BUILTIN(__builtin_ia32_dpbf16ps_256, "V8fV8fV8iV8i", "ncV:256:", "avx512bf16,avx512vl")
 TARGET_BUILTIN(__builtin_ia32_dpbf16ps_512, "V16fV16fV16iV16i", "ncV:512:", "avx512bf16")
+TARGET_BUILTIN(__builtin_ia32_cvtsbf162ss_32, "fUs", "nc", "avx512bf16")
 
 // generic select intrinsics
 TARGET_BUILTIN(__builtin_ia32_selectb_128, "V16cUsV16cV16c", "ncV:128:", "avx512bw,avx512vl")
 TARGET_BUILTIN(__builtin_ia32_selectb_256, "V32cUiV32cV32c", "ncV:256:", "avx512bw,avx512vl")
 TARGET_BUILTIN(__builtin_ia32_selectb_512, "V64cULLiV64cV64c", "ncV:512:", "avx512bw")
 TARGET_BUILTIN(__builtin_ia32_selectw_128, "V8sUcV8sV8s", "ncV:128:", "avx512bw,avx512vl")
 TARGET_BUILTIN(__builtin_ia32_selectw_256, "V16sUsV16sV16s", "ncV:256:", "avx512bw,avx512vl")
 TARGET_BUILTIN(__builtin_ia32_selectw_512, "V32sUiV32sV32s", "ncV:512:", "avx512bw")

lib/CodeGen/CGBuiltin.cpp

 }
 
 Value *CodeGenFunction::EmitX86CpuIs(const CallExpr *E) {
   const Expr *CPUExpr = E->getArg(0)->IgnoreParenCasts();
   StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
   return EmitX86CpuIs(CPUStr);
 }
 
+// Convert a BF16 to a float.
+static Value *EmitX86CvtBF16ToFloatExpr(CodeGenFunction &CGF,
+                                        const CallExpr *E,
+                                        ArrayRef<Value *> Ops) {
+  llvm::Type *Int32Ty = CGF.Builder.getInt32Ty();
+  Value *ZeroExt = CGF.Builder.CreateZExt(Ops[0], Int32Ty);
+  Value *Shl = CGF.Builder.CreateShl(ZeroExt, 16);
+  llvm::Type *ResultType = CGF.ConvertType(E->getType());
+  Value *BitCast = CGF.Builder.CreateBitCast(Shl, ResultType);
+  return BitCast;
+}
+
 Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) {
 
   llvm::Type *Int32Ty = Builder.getInt32Ty();
 
   // Matching the struct layout from the compiler-rt/libgcc structure that is
   // filled in:
   // unsigned int __cpu_vendor;
   // unsigned int __cpu_type;

 
 // AVX512 bf16 intrinsics
   case X86::BI__builtin_ia32_cvtneps2bf16_128_mask: {
     Ops[2] = getMaskVecValue(*this, Ops[2],
                              Ops[0]->getType()->getVectorNumElements());
     Intrinsic::ID IID = Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128;
     return Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
   }
+  case X86::BI__builtin_ia32_cvtsbf162ss_32:
+    return EmitX86CvtBF16ToFloatExpr(*this, E, Ops);
 
   case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
   case X86::BI__builtin_ia32_cvtneps2bf16_512_mask: {
     Intrinsic::ID IID;
     switch (BuiltinID) {
     default: llvm_unreachable("Unsupported intrinsic!");
     case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
       IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_256;

lib/Headers/avx512bf16intrin.h

 #error "Never use <avx512bf16intrin.h> directly; include <immintrin.h> instead."
 #endif
 
 #ifndef __AVX512BF16INTRIN_H
 #define __AVX512BF16INTRIN_H
 
 typedef short __m512bh __attribute__((__vector_size__(64), __aligned__(64)));
 typedef short __m256bh __attribute__((__vector_size__(32), __aligned__(32)));
+typedef short __m128bh __attribute__((__vector_size__(16), __aligned__(16)));
+typedef unsigned short __bfloat16;
 
 #define __DEFAULT_FN_ATTRS512 \
   __attribute__((__always_inline__, __nodebug__, __target__("avx512bf16"), \
                  __min_vector_width__(512)))
+#define __DEFAULT_FN_ATTRS256                                                  \
+  __attribute__((__always_inline__, __nodebug__, __target__("avx512bf16"),     \
+                 __min_vector_width__(256)))
+#define __DEFAULT_FN_ATTRS128                                                  \
+  __attribute__((__always_inline__, __nodebug__, __target__("avx512bf16"),     \
+                 __min_vector_width__(128)))
+#define __DEFAULT_FN_ATTRS                                                     \
+  __attribute__((__always_inline__, __nodebug__, __target__("avx512bf16")))
+
+/// Convert One BF16 Data to One Single Data.
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the <c> EmitX86CvtBF16ToFloatExpr </c>
+/// function.
+///
+/// \param __A
+///    A bfloat data.
+/// \returns A float data whose sign field and exponent field keep unchanged,
+///    and fraction field is extended to 23 bits.
+static __inline__ float __DEFAULT_FN_ATTRS _mm_cvtsbh_ss(__bfloat16 __A) {
+  return __builtin_ia32_cvtsbf162ss_32(__A);
+}
+
+/// Convert One Single Float Data to One BF16 Data.
+///
+/// \headerfile <x86intrin.h>
+///
+/// \param __A
+///    A float data.
+/// \returns A bf16 data whose sign field and exponent field keep unchanged,
+///    and fraction field is truncated to 7 bits.
+static __inline__ __bfloat16 __DEFAULT_FN_ATTRS128 _mm_cvtness_sbh(float __A) {
+  __v4sf __V = {__A, 0, 0, 0};
+  __m128bh __R = _mm_cvtneps_pbh(__V);
+  return __R[0];
+}
 
 /// Convert Two Packed Single Data to One Packed BF16 Data.
 ///
 /// \headerfile <x86intrin.h>
 ///
 /// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
 ///
 /// \param __A
 ///    A 512-bit vector of [16 x float].
 /// \param __B
 ///    A 512-bit vector of [16 x float].
 /// \returns A 512-bit vector of [32 x bfloat] whose lower 256 bits come from
 ///    conversion of __B, and higher 256 bits come from conversion of __A.
 static __inline__ __m512bh __DEFAULT_FN_ATTRS512
 _mm512_cvtne2ps_pbh(__m512 __A, __m512 __B) {
   return (__m512bh)__builtin_ia32_cvtne2ps2bf16_512((__v16sf) __A,
                                                     (__v16sf) __B);
 }
 
 /// Convert Two Packed Single Data to One Packed BF16 Data.
 ///
 /// \headerfile <x86intrin.h>
 ///
 /// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
 ///
 /// \param __A
 ///    A 512-bit vector of [16 x float].
 /// \param __B
 ///    A 512-bit vector of [16 x float].
 /// \param __W
 ///    A 512-bit vector of [32 x bfloat].
 /// \param __U
 ///    A 32-bit mask value specifying what is chosen for each element.
 ///    A 1 means conversion of __A or __B. A 0 means element from __W.
 /// \returns A 512-bit vector of [32 x bfloat] whose lower 256 bits come from
 ///    conversion of __B, and higher 256 bits come from conversion of __A.
 static __inline__ __m512bh __DEFAULT_FN_ATTRS512
 _mm512_mask_cvtne2ps_pbh(__m512bh __W, __mmask32 __U, __m512 __A, __m512 __B) {
   return (__m512bh)__builtin_ia32_selectw_512((__mmask32)__U,
                                         (__v32hi)_mm512_cvtne2ps_pbh(__A, __B),
                                         (__v32hi)__W);
 }
 
 /// Convert Two Packed Single Data to One Packed BF16 Data.
 ///
 /// \headerfile <x86intrin.h>
 ///
 /// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
 ///
 /// \param __A

 ///  __A, __B and __D
 static __inline__ __m512 __DEFAULT_FN_ATTRS512
 _mm512_maskz_dpbf16_ps(__mmask16 __U, __m512 __D, __m512bh __A, __m512bh __B) {
   return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
                                        (__v16sf)_mm512_dpbf16_ps(__D, __A, __B),
                                        (__v16sf)_mm512_setzero_si512());
 }
 
+/// Convert Packed BF16 Data to Packed float Data.
+///
+/// \headerfile <x86intrin.h>
+///
+/// \param __A
+///    A 256-bit vector of [16 x bfloat].
+/// \returns A 512-bit vector of [16 x float] come from convertion of __A
+static __inline__ __m512 __DEFAULT_FN_ATTRS512 _mm512_cvtpbh_ps(__m256bh __A) {
+  return _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepi16_epi32(__A), 16));
+}
+
+/// Convert Packed BF16 Data to Packed float Data.
+///
+/// \headerfile <x86intrin.h>
+///
+/// \param __A
+///    A 128-bit vector of [8 x bfloat].
+/// \returns A 256-bit vector of [8 x float] come from convertion of __A
+static __inline__ __m256 __DEFAULT_FN_ATTRS256 _mm256_cvtpbh_ps(__m128bh __A) {
+  return _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepi16_epi32(__A), 16));
+}
+
+/// Convert Packed BF16 Data to Packed float Data using zeroing mask.
+///
+/// \headerfile <x86intrin.h>
+///
+/// \param __M
+///    A 16-bit mask. Elements are zeroed out when the corresponding mask
+///    bit is not set.
+/// \param __A
+///    A 256-bit vector of [16 x bfloat].
+/// \returns A 512-bit vector of [16 x float] come from convertion of __A
+static __inline__ __m512 __DEFAULT_FN_ATTRS512
+_mm512_maskz_cvtpbh_ps(__mmask16 __M, __m256bh __A) {
+  return _mm512_castsi512_ps(
+      _mm512_slli_epi32(_mm512_maskz_cvtepi16_epi32(__M, __A), 16));
+}
+
+/// Convert Packed BF16 Data to Packed float Data using zeroing mask.
+///
+/// \headerfile <x86intrin.h>
+///
+/// \param __M
+///    A 8-bit mask. Elements are zeroed out when the corresponding mask
+///    bit is not set.
+/// \param __A
+///    A 128-bit vector of [8 x bfloat].
+/// \returns A 256-bit vector of [8 x float] come from convertion of __A
+static __inline__ __m256 __DEFAULT_FN_ATTRS256
+_mm256_maskz_cvtpbh_ps(__mmask8 __M, __m128bh __A) {
+  return _mm256_castsi256_ps(
+      _mm256_slli_epi32(_mm256_maskz_cvtepi16_epi32(__M, __A), 16));
+}
+
+/// Convert Packed BF16 Data to Packed float Data using merging mask.
+///
+/// \headerfile <x86intrin.h>
+///
+/// \param __S
+///    A 512-bit vector of [16 x float]. Elements are copied from __S when
+///     the corresponding mask bit is not set.
+/// \param __M
+///    A 16-bit mask.
+/// \param __A
+///    A 256-bit vector of [16 x bfloat].
+/// \returns A 512-bit vector of [16 x float] come from convertion of __A
+static __inline__ __m512 __DEFAULT_FN_ATTRS512
+_mm512_mask_cvtpbh_ps(__m512 __S, __mmask16 __M, __m256bh __A) {
+  return _mm512_castsi512_ps(
+      _mm512_mask_slli_epi32(__S, __M, _mm512_cvtepi16_epi32(__A), 16));
+}
+
+/// Convert Packed BF16 Data to Packed float Data using merging mask.
+///
+/// \headerfile <x86intrin.h>
+///
+/// \param __S
+///    A 256-bit vector of [8 x float]. Elements are copied from __S when
+///     the corresponding mask bit is not set.
+/// \param __M
+///    A 8-bit mask. Elements are zeroed out when the corresponding mask
+///    bit is not set.
+/// \param __A
+///    A 128-bit vector of [8 x bfloat].
+/// \returns A 256-bit vector of [8 x float] come from convertion of __A
+static __inline__ __m256 __DEFAULT_FN_ATTRS256
+_mm256_mask_cvtpbh_ps(__m256 __S, __mmask8 __M, __m128bh __A) {
+  return _mm256_castsi256_ps(
+      _mm256_mask_slli_epi32(__S, __M, _mm256_cvtepi16_epi32(__A), 16));
+}
+
+#undef __DEFAULT_FN_ATTRS
+#undef __DEFAULT_FN_ATTRS128
+#undef __DEFAULT_FN_ATTRS256
 #undef __DEFAULT_FN_ATTRS512
 
 #endif

test/CodeGen/avx512bf16-builtins.c

 //  RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin \
 //  RUN:            -target-feature +avx512bf16 -emit-llvm -o - -Wall -Werror \
 //  RUN:            | FileCheck %s
 
 #include <immintrin.h>
 
-__m512bh test_mm512_cvtne2ps2bf16(__m512 A, __m512 B) {
-  // CHECK-LABEL: @test_mm512_cvtne2ps2bf16
+float test_mm_cvtsbh_ss(__bfloat16 A) {
+  // CHECK-LABEL: @test_mm_cvtsbh_ss
+  // CHECK: zext i16 %{{.*}} to i32
+  // CHECK: shl i32 %{{.*}}, 16
+  // CHECK: bitcast i32 %{{.*}} to float
+  // CHECK: ret float %{{.*}}
+  return _mm_cvtsbh_ss(A);
+}
+
+__bfloat16 test_mm_cvtness_sbh(float A) {
+  // CHECK-LABEL: @test_mm_cvtness_sbh
+  // CHECK: bitcast float %{{.*}} to i32
+  // CHECK: lshr i32 %{{.*}}, 16
+  // CHECK: trunc i32 %{{.*}} to i16
+  // CHECK: ret i16 %{{.*}}
+  return _mm_cvtness_sbh(A);
+}
+
+__m512bh test_mm512_cvtne2ps_pbh(__m512 A, __m512 B) {
+  // CHECK-LABEL: @test_mm512_cvtne2ps_pbh
   // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.512
   // CHECK: ret <32 x i16> %{{.*}}
   return _mm512_cvtne2ps_pbh(A, B);
 }
 
-__m512bh test_mm512_maskz_cvtne2ps2bf16(__m512 A, __m512 B, __mmask32 U) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtne2ps2bf16
+__m512bh test_mm512_maskz_cvtne2ps_pbh(__m512 A, __m512 B, __mmask32 U) {
+  // CHECK-LABEL: @test_mm512_maskz_cvtne2ps_pbh
   // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   // CHECK: ret <32 x i16> %{{.*}}
   return _mm512_maskz_cvtne2ps_pbh(U, A, B);
 }
 
-__m512bh test_mm512_mask_cvtne2ps2bf16(__m512bh C, __mmask32 U, __m512 A, __m512 B) {
-  // CHECK-LABEL: @test_mm512_mask_cvtne2ps2bf16
+__m512bh test_mm512_mask_cvtne2ps_pbh(__m512bh C, __mmask32 U, __m512 A, __m512 B) {
+  // CHECK-LABEL: @test_mm512_mask_cvtne2ps_pbh
   // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   // CHECK: ret <32 x i16> %{{.*}}
   return _mm512_mask_cvtne2ps_pbh(C, U, A, B);
 }
 
-__m256bh test_mm512_cvtneps2bf16(__m512 A) {
-  // CHECK-LABEL: @test_mm512_cvtneps2bf16
+__m256bh test_mm512_cvtneps_pbh(__m512 A) {
+  // CHECK-LABEL: @test_mm512_cvtneps_pbh
   // CHECK: @llvm.x86.avx512bf16.cvtneps2bf16.512
   // CHECK: ret <16 x i16> %{{.*}}
   return _mm512_cvtneps_pbh(A);
 }
 
-__m256bh test_mm512_mask_cvtneps2bf16(__m256bh C, __mmask16 U, __m512 A) {
-  // CHECK-LABEL: @test_mm512_mask_cvtneps2bf16
+__m256bh test_mm512_mask_cvtneps_pbh(__m256bh C, __mmask16 U, __m512 A) {
+  // CHECK-LABEL: @test_mm512_mask_cvtneps_pbh
   // CHECK: @llvm.x86.avx512bf16.cvtneps2bf16.512
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   // CHECK: ret <16 x i16> %{{.*}}
   return _mm512_mask_cvtneps_pbh(C, U, A);
 }
 
-__m256bh test_mm512_maskz_cvtneps2bf16(__m512 A, __mmask16 U) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtneps2bf16
+__m256bh test_mm512_maskz_cvtneps_pbh(__m512 A, __mmask16 U) {
+  // CHECK-LABEL: @test_mm512_maskz_cvtneps_pbh
   // CHECK: @llvm.x86.avx512bf16.cvtneps2bf16.512
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   // CHECK: ret <16 x i16> %{{.*}}
   return _mm512_maskz_cvtneps_pbh(U, A);
 }
 
 __m512 test_mm512_dpbf16_ps(__m512 D, __m512bh A, __m512bh B) {
   // CHECK-LABEL: @test_mm512_dpbf16_ps

 
 __m512 test_mm512_mask_dpbf16_ps(__m512 D, __m512bh A, __m512bh B, __mmask16 U) {
   // CHECK-LABEL: @test_mm512_mask_dpbf16_ps
   // CHECK: @llvm.x86.avx512bf16.dpbf16ps.512
   // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
   // CHECK: ret <16 x float> %{{.*}}
   return _mm512_mask_dpbf16_ps(D, U, A, B);
 }
+
+__m512 test_mm512_cvtpbh_ps(__m256bh A) {
+  // CHECK-LABEL: @test_mm512_cvtpbh_ps
+  // CHECK: sext <16 x i16> %{{.*}} to <16 x i32>
+  // CHECK: @llvm.x86.avx512.pslli.d.512
+  // CHECK: bitcast <8 x i64> %{{.*}} to <16 x float>
+  // CHECK: ret <16 x float> %{{.*}}
+  return _mm512_cvtpbh_ps(A);
+}
+
+__m256 test_mm256_cvtpbh_ps(__m128bh A) {
+  // CHECK-LABEL: @test_mm256_cvtpbh_ps
+  // CHECK: sext <8 x i16> %{{.*}} to <8 x i32>
+  // CHECK: @llvm.x86.avx2.pslli.d
+  // CHECK: bitcast <4 x i64> %{{.*}} to <8 x float>
+  // CHECK: ret <8 x float> %{{.*}}
+  return _mm256_cvtpbh_ps(A);
+}
+
+__m512 test_mm512_maskz_cvtpbh_ps(__mmask16 M, __m256bh A) {
+  // CHECK-LABEL: @test_mm512_maskz_cvtpbh_ps
+  // CHECK: sext <16 x i16> %{{.*}} to <16 x i32>
+  // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
+  // CHECK: @llvm.x86.avx512.pslli.d.512
+  // CHECK: bitcast <8 x i64> %{{.*}} to <16 x float>
+  // CHECK: ret <16 x float> %{{.*}}
+  return _mm512_maskz_cvtpbh_ps(M, A);
+}
+
+__m256 test_mm256_maskz_cvtpbh_ps(__mmask8 M, __m128bh A) {
+  // CHECK-LABEL: @test_mm256_maskz_cvtpbh_ps
+  // CHECK: sext <8 x i16> %{{.*}} to <8 x i32>
+  // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
+  // CHECK: @llvm.x86.avx2.pslli.d
+  // CHECK: bitcast <4 x i64> %{{.*}} to <8 x float>
+  // CHECK: ret <8 x float> %{{.*}}
+  return _mm256_maskz_cvtpbh_ps(M, A);
+}
+
+__m512 test_mm512_mask_cvtpbh_ps(__m512 S, __mmask16 M, __m256bh A) {
+  // CHECK-LABEL: @test_mm512_mask_cvtpbh_ps
+  // CHECK: sext <16 x i16> %{{.*}} to <16 x i32>
+  // CHECK: @llvm.x86.avx512.pslli.d.512
+  // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
+  // CHECK: bitcast <8 x i64> %{{.*}} to <16 x float>
+  // CHECK: ret <16 x float> %{{.*}}
+  return _mm512_mask_cvtpbh_ps(S, M, A);
+}
+
+__m256 test_mm256_mask_cvtpbh_ps(__m256 S, __mmask8 M, __m128bh A) {
+  // CHECK-LABEL: @test_mm256_mask_cvtpbh_ps
+  // CHECK: sext <8 x i16> %{{.*}} to <8 x i32>
+  // CHECK: @llvm.x86.avx2.pslli.d
+  // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
+  // CHECK: bitcast <4 x i64> %{{.*}} to <8 x float>
+  // CHECK: ret <8 x float> %{{.*}}
+  return _mm256_mask_cvtpbh_ps(S, M, A);
+}