[X86][AVX512] lowering broadcastm intrinsic - clang part

[jina.nahias]

Summary

This is the clang part, the llvm part: https://reviews.llvm.org/D38684

[RKSimon]

test/CodeGen/avx512cdintrin.c

106

Any reason why you can't use the actual insertion indices instead of regexps?

[jina.nahias]

test/CodeGen/avx512cdintrin.c

106

yes,
the cmpeq intrinsic returns a mask type.
if we wont send a mask type to the broadcastm intrinsic it would do a regular broadcast (not a mask broadcast).

[RKSimon]

LGTM

[jina.nahias]

commit in https://reviews.llvm.org/rL317456

include/clang/Basic/BuiltinsX86.def

 TARGET_BUILTIN(__builtin_ia32_cvtd2mask128, "UcV4i","","avx512dq,avx512vl")
 TARGET_BUILTIN(__builtin_ia32_cvtd2mask256, "UcV8i","","avx512dq,avx512vl")
 TARGET_BUILTIN(__builtin_ia32_cvtmask2d128, "V4iUc","","avx512dq,avx512vl")
 TARGET_BUILTIN(__builtin_ia32_cvtmask2d256, "V8iUc","","avx512dq,avx512vl")
 TARGET_BUILTIN(__builtin_ia32_cvtmask2q128, "V2LLiUc","","avx512dq,avx512vl")
 TARGET_BUILTIN(__builtin_ia32_cvtmask2q256, "V4LLiUc","","avx512dq,avx512vl")
 TARGET_BUILTIN(__builtin_ia32_cvtq2mask128, "UcV2LLi","","avx512dq,avx512vl")
 TARGET_BUILTIN(__builtin_ia32_cvtq2mask256, "UcV4LLi","","avx512dq,avx512vl")
-TARGET_BUILTIN(__builtin_ia32_broadcastmb512, "V8LLiUc","","avx512cd")
-TARGET_BUILTIN(__builtin_ia32_broadcastmw512, "V16iUs","","avx512cd")
-TARGET_BUILTIN(__builtin_ia32_broadcastmb128, "V2LLiUc","","avx512cd,avx512vl")
-TARGET_BUILTIN(__builtin_ia32_broadcastmb256, "V4LLiUc","","avx512cd,avx512vl")
-TARGET_BUILTIN(__builtin_ia32_broadcastmw128, "V4iUs","","avx512cd,avx512vl")
-TARGET_BUILTIN(__builtin_ia32_broadcastmw256, "V8iUs","","avx512cd,avx512vl")
 TARGET_BUILTIN(__builtin_ia32_pmovsdb512_mask, "V16cV16iV16cUs","","avx512f")
 TARGET_BUILTIN(__builtin_ia32_pmovsdb512mem_mask, "vV16c*V16iUs","","avx512f")
 TARGET_BUILTIN(__builtin_ia32_pmovswb512mem_mask, "vV32c*V32sUi","","avx512bw")
 TARGET_BUILTIN(__builtin_ia32_pmovsdw512_mask, "V16sV16iV16sUs","","avx512f")
 TARGET_BUILTIN(__builtin_ia32_pmovsdw512mem_mask, "vV16s*V16iUs","","avx512f")
 TARGET_BUILTIN(__builtin_ia32_pmovsqb512_mask, "V16cV8LLiV16cUc","","avx512f")
 TARGET_BUILTIN(__builtin_ia32_pmovsqb512mem_mask, "vV16c*V8LLiUc","","avx512f")
 TARGET_BUILTIN(__builtin_ia32_pmovsqd512_mask, "V8iV8LLiV8iUc","","avx512f")

lib/Headers/avx512cdintrin.h

_mm512_maskz_lzcnt_epi64 (__mmask8 __U, __m512i __A)
   return (__m512i) __builtin_ia32_vplzcntq_512_mask ((__v8di) __A,
              (__v8di) _mm512_setzero_si512 (),
              (__mmask8) __U);
 }
 
 static __inline__ __m512i __DEFAULT_FN_ATTRS
 _mm512_broadcastmb_epi64 (__mmask8 __A)
 {
-  return (__m512i) __builtin_ia32_broadcastmb512 (__A);
+  return (__m512i) _mm512_set1_epi64((long long) __A);
 }
 
 static __inline__ __m512i __DEFAULT_FN_ATTRS
 _mm512_broadcastmw_epi32 (__mmask16 __A)
 {
-  return (__m512i) __builtin_ia32_broadcastmw512 (__A);
+  return (__m512i) _mm512_set1_epi32((int) __A);
+
 }
 
 #undef __DEFAULT_FN_ATTRS
 
 #endif

lib/Headers/avx512vlcdintrin.h

 #define __AVX512VLCDINTRIN_H
 
 /* Define the default attributes for the functions in this file. */
 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("avx512vl,avx512cd")))
 
 
 static __inline__ __m128i __DEFAULT_FN_ATTRS
 _mm_broadcastmb_epi64 (__mmask8 __A)
-{
-  return (__m128i) __builtin_ia32_broadcastmb128 (__A);
+{ 
+  return (__m128i) _mm_set1_epi64x((long long) __A);
 }
 
 static __inline__ __m256i __DEFAULT_FN_ATTRS
 _mm256_broadcastmb_epi64 (__mmask8 __A)
 {
-  return (__m256i) __builtin_ia32_broadcastmb256 (__A);
+  return (__m256i) _mm256_set1_epi64x((long long)__A);
 }
 
 static __inline__ __m128i __DEFAULT_FN_ATTRS
 _mm_broadcastmw_epi32 (__mmask16 __A)
 {
-  return (__m128i) __builtin_ia32_broadcastmw128 (__A);
+  return (__m128i) _mm_set1_epi32((int)__A);
 }
 
 static __inline__ __m256i __DEFAULT_FN_ATTRS
 _mm256_broadcastmw_epi32 (__mmask16 __A)
 {
-  return (__m256i) __builtin_ia32_broadcastmw256 (__A);
+  return (__m256i) _mm256_set1_epi32((int)__A);
 }
 
 
 static __inline__ __m128i __DEFAULT_FN_ATTRS
 _mm_conflict_epi64 (__m128i __A)
 {
   return (__m128i) __builtin_ia32_vpconflictdi_128_mask ((__v2di) __A,
                (__v2di) _mm_undefined_si128 (),

test/CodeGen/avx512cdintrin.c

 }
 __m512i test_mm512_maskz_lzcnt_epi64(__mmask8 __U, __m512i __A) {
   // CHECK-LABEL: @test_mm512_maskz_lzcnt_epi64
   // CHECK: call <8 x i64> @llvm.ctlz.v8i64(<8 x i64> %{{.*}}, i1 false)
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_maskz_lzcnt_epi64(__U,__A); 
 }
 
-__m512i test_mm512_broadcastmb_epi64(__mmask8 __A) {
+__m512i test_mm512_broadcastmb_epi64(__m512i a, __m512i b) {
   // CHECK-LABEL: @test_mm512_broadcastmb_epi64
-  // CHECK: @llvm.x86.avx512.broadcastmb.512
-  return _mm512_broadcastmb_epi64(__A); 
+  // CHECK: icmp eq <8 x i64> %{{.*}}, %{{.*}}
+  // CHECK: zext i8 %{{.*}} to i64
+  // CHECK: insertelement <8 x i64> undef, i64 %{{.*}}, i32 0
+  // CHECK: insertelement <8 x i64> %{{.*}}, i64 %{{.*}}, i32 1
+  // CHECK: insertelement <8 x i64> %{{.*}}, i64 %{{.*}}, i32 2
+  // CHECK: insertelement <8 x i64> %{{.*}}, i64 %{{.*}}, i32 3
+  // CHECK: insertelement <8 x i64> %{{.*}}, i64 %{{.*}}, i32 4
+  // CHECK: insertelement <8 x i64> %{{.*}}, i64 %{{.*}}, i32 5
+  // CHECK: insertelement <8 x i64> %{{.*}}, i64 %{{.*}}, i32 6
+  // CHECK: insertelement <8 x i64> %{{.*}}, i64 %{{.*}}, i32 7
+  return _mm512_broadcastmb_epi64(_mm512_cmpeq_epu64_mask ( a, b)); 
 }
 
-__m512i test_mm512_broadcastmw_epi32(__mmask16 __A) {
+__m512i test_mm512_broadcastmw_epi32(__m512i a, __m512i b) {
   // CHECK-LABEL: @test_mm512_broadcastmw_epi32
-  // CHECK: @llvm.x86.avx512.broadcastmw.512
-  return _mm512_broadcastmw_epi32(__A); 
+  // CHECK: icmp eq <16 x i32> %{{.*}}, %{{.*}}
+  // CHECK: zext i16 %{{.*}} to i32
+  // CHECK: insertelement <16 x i32> undef, i32 %{{.*}}
+  // CHECK: insertelement <16 x i32> %{{.*}}, i32 %{{.*}}
+  // CHECK: insertelement <16 x i32> %{{.*}}, i32 %{{.*}}
+  // CHECK: insertelement <16 x i32> %{{.*}}, i32 %{{.*}}
+  // CHECK: insertelement <16 x i32> %{{.*}}, i32 %{{.*}}
+  // CHECK: insertelement <16 x i32> %{{.*}}, i32 %{{.*}}
+  // CHECK: insertelement <16 x i32> %{{.*}}, i32 %{{.*}}
+  // CHECK: insertelement <16 x i32> %{{.*}}, i32 %{{.*}}
+  // CHECK: insertelement <16 x i32> %{{.*}}, i32 %{{.*}}
+  // CHECK: insertelement <16 x i32> %{{.*}}, i32 %{{.*}}
+  // CHECK: insertelement <16 x i32> %{{.*}}, i32 %{{.*}}
+  // CHECK: insertelement <16 x i32> %{{.*}}, i32 %{{.*}}
+  // CHECK: insertelement <16 x i32> %{{.*}}, i32 %{{.*}}
+  // CHECK: insertelement <16 x i32> %{{.*}}, i32 %{{.*}}
+  // CHECK: insertelement <16 x i32> %{{.*}}, i32 %{{.*}}
+  // CHECK: insertelement <16 x i32> %{{.*}}, i32 %{{.*}}
+  return _mm512_broadcastmw_epi32(_mm512_cmpeq_epi32_mask ( a, b)); 
 }

test/CodeGen/avx512vlcd-builtins.c

 // RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vl -target-feature +avx512cd -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 
 #include <immintrin.h>
 
-__m128i test_mm_broadcastmb_epi64(__mmask8 __A) {
+__m128i test_mm_broadcastmb_epi64(__m128i a,__m128i b) {
   // CHECK-LABEL: @test_mm_broadcastmb_epi64
-  // CHECK: @llvm.x86.avx512.broadcastmb.128
-  return _mm_broadcastmb_epi64(__A); 
+  // CHECK: icmp eq <4 x i32> %{{.*}}, %{{.*}}
+  // CHECK: shufflevector <4 x i1> %{{.*}}, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  // CHECK: bitcast <8 x i1> %{{.*}} to i8
+  // CHECK: zext i8 %{{.*}} to i64
+  // CHECK: insertelement <2 x i64> undef, i64 %{{.*}}, i32 0
+  // CHECK: insertelement <2 x i64> %{{.*}}, i64 %{{.*}}, i32 1
+  return _mm_broadcastmb_epi64(_mm_cmpeq_epi32_mask (a, b)); 
 }
 
-__m256i test_mm256_broadcastmb_epi64(__mmask8 __A) {
+__m256i test_mm256_broadcastmb_epi64(__m256i a, __m256i b) {
   // CHECK-LABEL: @test_mm256_broadcastmb_epi64
-  // CHECK: @llvm.x86.avx512.broadcastmb.256
-  return _mm256_broadcastmb_epi64(__A); 
+  // CHECK: icmp eq <4 x i64> %{{.*}}, %{{.*}}
+  // CHECK: shufflevector <4 x i1> %{{.*}}, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  // CHECK: bitcast <8 x i1> %{{.*}} to i8
+  // CHECK: zext i8 %{{.*}} to i64
+  // CHECK: insertelement <4 x i64> undef, i64 %{{.*}}, i32 0
+  // CHECK: insertelement <4 x i64> %{{.*}}, i64 %{{.*}}, i32 1
+  // CHECK: insertelement <4 x i64> %{{.*}}, i64 %{{.*}}, i32 2
+  // CHECK: insertelement <4 x i64> %{{.*}}, i64 %{{.*}}, i32 3
+  return _mm256_broadcastmb_epi64(_mm256_cmpeq_epi64_mask ( a, b)); 
 }
 
-__m128i test_mm_broadcastmw_epi32(__mmask16 __A) {
+__m128i test_mm_broadcastmw_epi32(__m512i a, __m512i b) {
   // CHECK-LABEL: @test_mm_broadcastmw_epi32
-  // CHECK: @llvm.x86.avx512.broadcastmw.128
-  return _mm_broadcastmw_epi32(__A); 
+  // CHECK: icmp eq <16 x i32> %{{.*}}, %{{.*}}
+  // CHECK: bitcast <16 x i1> %{{.*}} to i16
+  // CHECK: zext i16 %{{.*}} to i32
+  // CHECK: insertelement <4 x i32> undef, i32 %{{.*}}, i32 0
+  // CHECK: insertelement <4 x i32> %{{.*}}, i32 %{{.*}}, i32 1
+  // CHECK: insertelement <4 x i32> %{{.*}}, i32 %{{.*}}, i32 2
+  // CHECK: insertelement <4 x i32> %{{.*}}, i32 %{{.*}}, i32 3
+  return _mm_broadcastmw_epi32(_mm512_cmpeq_epi32_mask ( a, b));
 }
 
-__m256i test_mm256_broadcastmw_epi32(__mmask16 __A) {
+__m256i test_mm256_broadcastmw_epi32(__m512i a, __m512i b) {
   // CHECK-LABEL: @test_mm256_broadcastmw_epi32
-  // CHECK: @llvm.x86.avx512.broadcastmw.256
-  return _mm256_broadcastmw_epi32(__A); 
+  // CHECK: icmp eq <16 x i32> %{{.*}}, %{{.*}}
+  // CHECK: bitcast <16 x i1> %{{.*}} to i16
+  // CHECK: zext i16 %{{.*}} to i32
+  // CHECK: insertelement <8 x i32> undef, i32 %{{.*}}, i32 0
+  // CHECK: insertelement <8 x i32> %{{.*}}, i32 %{{.*}}, i32 1
+  // CHECK: insertelement <8 x i32> %{{.*}}, i32 %{{.*}}, i32 2
+  // CHECK: insertelement <8 x i32> %{{.*}}, i32 %{{.*}}, i32 3
+  // CHECK: insertelement <8 x i32> %{{.*}}, i32 %{{.*}}, i32 4
+  // CHECK: insertelement <8 x i32> %{{.*}}, i32 %{{.*}}, i32 5
+  // CHECK: insertelement <8 x i32> %{{.*}}, i32 %{{.*}}, i32 6
+  // CHECK: insertelement <8 x i32> %{{.*}}, i32 %{{.*}}, i32 7
+  return _mm256_broadcastmw_epi32(_mm512_cmpeq_epi32_mask ( a, b)); 
 }
 
 __m128i test_mm_conflict_epi64(__m128i __A) {
   // CHECK-LABEL: @test_mm_conflict_epi64
   // CHECK: @llvm.x86.avx512.mask.conflict.q.128
   return _mm_conflict_epi64(__A); 
 }