[AVX512] Emit generic masked store intrinsics directly from clang instead of using x86 specific intrinsics.

[craig.topper]

Summary

This will allow us to remove the x86 specific intrinsics from the backend.

[delena]

lib/CodeGen/CGBuiltin.cpp

6304

What code do you receive at the end? There is no shuffle instruction in the architecture for mask vector.

test/CodeGen/avx512f-builtins.c

123

I suggest to remove %5, %6 from the test, you can put something like this:
CHECK: store <16 x i32> {{.*}}, align 1

[craig.topper]

lib/CodeGen/CGBuiltin.cpp

6304

That's not really a shuffle. It's an extract subvector, but the IR doesn't have a real instruction for that.

It's needed so we can go from i8 -> v8i1 -> v2i1/v4i1.

test/CodeGen/avx512f-builtins.c

123

I'll clean that up. I fixed most of them but looks like a missed a few.

[delena]

lib/CodeGen/CGBuiltin.cpp

6304

I understand. I just wanted to be sure that you receive only one "kmov %edi, %k1" at the end.

[craig.topper]

lib/CodeGen/CGBuiltin.cpp

6304

Yes, only one "kmov %edi, %k1" was generated.

[delena]

LGTM, After tests cleanup

[craig.topper]

Commited in r271246.

lib/CodeGen/CGBuiltin.cpp

Value *Result =
     llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size()));
 
   for (unsigned i = 0, e = Ops.size(); i != e; ++i)
     Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i));
 
   return Result;
 }
 
+static Value *EmitX86MaskedStore(CodeGenFunction &CGF,
+                                 SmallVectorImpl<Value *> &Ops,
+                                 unsigned Align) {
+  // Cast the pointer to right type.
+  Ops[0] = CGF.Builder.CreateBitCast(Ops[0],
+                               llvm::PointerType::getUnqual(Ops[1]->getType()));
+
+  // If the mask is all ones just emit a regular store.
+  if (const auto *C = dyn_cast<Constant>(Ops[2]))
+    if (C->isAllOnesValue())
+      return CGF.Builder.CreateAlignedStore(Ops[1], Ops[0], Align);
+
+  // Convert the mask from an integer type to a vector of i1.
+  unsigned NumElts = Ops[1]->getType()->getVectorNumElements();
+  llvm::VectorType *MaskTy = llvm::VectorType::get(CGF.Builder.getInt1Ty(),
+                         cast<IntegerType>(Ops[2]->getType())->getBitWidth());
+  Ops[2] = CGF.Builder.CreateBitCast(Ops[2], MaskTy);
+
+  // If we have less than 8 elements, then the starting mask was an i8 and
+  // we need to extract down to the right number of elements.
+  if (NumElts < 8) {
+    int Indices[4];
+    for (unsigned i = 0; i != NumElts; ++i)
+      Indices[i] = i;
+    Ops[2] = CGF.Builder.CreateShuffleVector(Ops[2], Ops[2],
+                                             makeArrayRef(Indices, NumElts),
+                                             "extract");
+  }
+
+  return CGF.Builder.CreateMaskedStore(Ops[1], Ops[0], Align, Ops[2]);
+}
+
 Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
                                            const CallExpr *E) {
   if (BuiltinID == X86::BI__builtin_ms_va_start ||
       BuiltinID == X86::BI__builtin_ms_va_end)
     return EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).getPointer(),
                           BuiltinID == X86::BI__builtin_ms_va_start);
   if (BuiltinID == X86::BI__builtin_ms_va_copy) {
     // Lower this manually. We can't reliably determine whether or not any

 #undef INTRINSIC_X86_XSAVE_ID
     Value *Mhi = Builder.CreateTrunc(
       Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty);
     Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty);
     Ops[1] = Mhi;
     Ops.push_back(Mlo);
     return Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
   }
+  case X86::BI__builtin_ia32_storedqudi128_mask:
+  case X86::BI__builtin_ia32_storedqusi128_mask:
+  case X86::BI__builtin_ia32_storedquhi128_mask:
+  case X86::BI__builtin_ia32_storedquqi128_mask:
+  case X86::BI__builtin_ia32_storeupd128_mask:
+  case X86::BI__builtin_ia32_storeups128_mask:
+  case X86::BI__builtin_ia32_storedqudi256_mask:
+  case X86::BI__builtin_ia32_storedqusi256_mask:
+  case X86::BI__builtin_ia32_storedquhi256_mask:
+  case X86::BI__builtin_ia32_storedquqi256_mask:
+  case X86::BI__builtin_ia32_storeupd256_mask:
+  case X86::BI__builtin_ia32_storeups256_mask:
+  case X86::BI__builtin_ia32_storedqudi512_mask:
+  case X86::BI__builtin_ia32_storedqusi512_mask:
+  case X86::BI__builtin_ia32_storedquhi512_mask:
+  case X86::BI__builtin_ia32_storedquqi512_mask:
+  case X86::BI__builtin_ia32_storeupd512_mask:
+  case X86::BI__builtin_ia32_storeups512_mask:
+    return EmitX86MaskedStore(*this, Ops, 1);
+
+  case X86::BI__builtin_ia32_movdqa32store128_mask:
+  case X86::BI__builtin_ia32_movdqa64store128_mask:
+  case X86::BI__builtin_ia32_storeaps128_mask:
+  case X86::BI__builtin_ia32_storeapd128_mask:
+  case X86::BI__builtin_ia32_movdqa32store256_mask:
+  case X86::BI__builtin_ia32_movdqa64store256_mask:
+  case X86::BI__builtin_ia32_storeaps256_mask:
+  case X86::BI__builtin_ia32_storeapd256_mask:
+  case X86::BI__builtin_ia32_movdqa32store512_mask:
+  case X86::BI__builtin_ia32_movdqa64store512_mask:
+  case X86::BI__builtin_ia32_storeaps512_mask:
+  case X86::BI__builtin_ia32_storeapd512_mask: {
+    unsigned Align =
+      getContext().getTypeAlignInChars(E->getArg(1)->getType()).getQuantity();
+    return EmitX86MaskedStore(*this, Ops, Align);
+  }
   case X86::BI__builtin_ia32_storehps:
   case X86::BI__builtin_ia32_storelps: {
     llvm::Type *PtrTy = llvm::PointerType::getUnqual(Int64Ty);
     llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2);
 
     // cast val v2i64
     Ops[1] = Builder.CreateBitCast(Ops[1], VecTy, "cast");
 

test/CodeGen/avx512bw-builtins.c

 
 __m512i test_mm512_maskz_loadu_epi8(__mmask64 __U, void const *__P) {
   // CHECK-LABEL: @test_mm512_maskz_loadu_epi8
   // CHECK: @llvm.x86.avx512.mask.loadu.b.512
   return _mm512_maskz_loadu_epi8(__U, __P); 
 }
 void test_mm512_mask_storeu_epi16(void *__P, __mmask32 __U, __m512i __A) {
   // CHECK-LABEL: @test_mm512_mask_storeu_epi16
-  // CHECK: @llvm.x86.avx512.mask.storeu.w.
+  // CHECK: @llvm.masked.store.v32i16(<32 x i16> %{{.*}}, <32 x i16>* %{{.*}}, i32 1, <32 x i1> %{{.*}})
   return _mm512_mask_storeu_epi16(__P, __U, __A); 
 }
 __mmask64 test_mm512_test_epi8_mask(__m512i __A, __m512i __B) {
   // CHECK-LABEL: @test_mm512_test_epi8_mask
   // CHECK: @llvm.x86.avx512.ptestm.b.512
   return _mm512_test_epi8_mask(__A, __B); 
 }
 
 void test_mm512_mask_storeu_epi8(void *__P, __mmask64 __U, __m512i __A) {
   // CHECK-LABEL: @test_mm512_mask_storeu_epi8
-  // CHECK: @llvm.x86.avx512.mask.storeu.b.
+  // CHECK: @llvm.masked.store.v64i8(<64 x i8> %{{.*}}, <64 x i8>* %{{.*}}, i32 1, <64 x i1> %{{.*}})
   return _mm512_mask_storeu_epi8(__P, __U, __A); 
 }
 __mmask64 test_mm512_mask_test_epi8_mask(__mmask64 __U, __m512i __A, __m512i __B) {
   // CHECK-LABEL: @test_mm512_mask_test_epi8_mask
   // CHECK: @llvm.x86.avx512.ptestm.b.512
   return _mm512_mask_test_epi8_mask(__U, __A, __B); 
 }
 

test/CodeGen/avx512f-builtins.c

__m512d test_mm512_mul_pd(__m512d a, __m512d b)
   // CHECK-LABEL: @test_mm512_mul_pd
   // CHECK: fmul <8 x double>
   return _mm512_mul_pd(a, b);
 }
 
 void test_mm512_storeu_si512 (void *__P, __m512i __A)
 {
   // CHECK-LABEL: @test_mm512_storeu_si512 
-  // CHECK: @llvm.x86.avx512.mask.storeu.d.512
+  // CHECK: store <16 x i32> %5, <16 x i32>* %6, align 1
+  // CHECK-NEXT: ret void
   _mm512_storeu_si512 ( __P,__A);
 }
 
 void test_mm512_storeu_ps(void *p, __m512 a)
 {
   // CHECK-LABEL: @test_mm512_storeu_ps
-  // CHECK: @llvm.x86.avx512.mask.storeu.ps.512
+  // CHECK: store <16 x float> %4, <16 x float>* %5, align 1
+  // CHECK-NEXT: ret void
   _mm512_storeu_ps(p, a);
 }
 
 void test_mm512_storeu_pd(void *p, __m512d a)
 {
   // CHECK-LABEL: @test_mm512_storeu_pd
-  // CHECK: @llvm.x86.avx512.mask.storeu.pd.512
+  // CHECK: store <8 x double> %4, <8 x double>* %5, align 1
+  // CHECK-NEXT: ret void
   _mm512_storeu_pd(p, a);
 }
 
 void test_mm512_mask_store_ps(void *p, __m512 a, __mmask16 m)
 {
   // CHECK-LABEL: @test_mm512_mask_store_ps
-  // CHECK: @llvm.x86.avx512.mask.store.ps.512
+  // CHECK: @llvm.masked.store.v16f32(<16 x float> %{{.*}}, <16 x float>* %{{.*}}, i32 64, <16 x i1> %{{.*}})
   _mm512_mask_store_ps(p, m, a);
 }
 
 void test_mm512_store_si512 (void *__P, __m512i __A)
 {
   // CHECK-LABEL: @test_mm512_store_si512 
   // CHECK: load <8 x i64>, <8 x i64>* %__A.addr.i, align 64
   // CHECK: [[SI512_3:%.+]] = load i8*, i8** %__P.addr.i, align 8
void test_mm512_store_pd(void *p, __m512d a)
   // CHECK-LABEL: @test_mm512_store_pd
   // CHECK: store <8 x double>
   _mm512_store_pd(p, a);
 }
 
 void test_mm512_mask_store_pd(void *p, __m512d a, __mmask8 m)
 {
   // CHECK-LABEL: @test_mm512_mask_store_pd
-  // CHECK: @llvm.x86.avx512.mask.store.pd.512
+  // CHECK: @llvm.masked.store.v8f64(<8 x double> %{{.*}}, <8 x double>* %{{.*}}, i32 64, <8 x i1> %{{.*}})
   _mm512_mask_store_pd(p, m, a);
 }
 
 void test_mm512_mask_storeu_epi32(void *__P, __mmask16 __U, __m512i __A) {
   // CHECK-LABEL: @test_mm512_mask_storeu_epi32
-  // CHECK: @llvm.x86.avx512.mask.storeu.d.512
+  // CHECK: @llvm.masked.store.v16i32(<16 x i32> %{{.*}}, <16 x i32>* %{{.*}}, i32 1, <16 x i1> %{{.*}})
   return _mm512_mask_storeu_epi32(__P, __U, __A); 
 }
 
 void test_mm512_mask_storeu_epi64(void *__P, __mmask8 __U, __m512i __A) {
   // CHECK-LABEL: @test_mm512_mask_storeu_epi64
-  // CHECK: @llvm.x86.avx512.mask.storeu.q.512
+  // CHECK: @llvm.masked.store.v8i64(<8 x i64> %{{.*}}, <8 x i64>* %{{.*}}, i32 1, <8 x i1> %{{.*}})
   return _mm512_mask_storeu_epi64(__P, __U, __A); 
 }
 
 __m512i test_mm512_loadu_si512 (void *__P)
 {
   // CHECK-LABEL: @test_mm512_loadu_si512 
   // CHECK: @llvm.x86.avx512.mask.loadu.d.512
   return _mm512_loadu_si512 ( __P);

 __m512i test_mm512_maskz_load_epi64(__mmask8 __U, void const *__P) {
   // CHECK-LABEL: @test_mm512_maskz_load_epi64
   // CHECK: @llvm.x86.avx512.mask.load.q.512
   return _mm512_maskz_load_epi64(__U, __P); 
 }
 
 void test_mm512_mask_store_epi32(void *__P, __mmask16 __U, __m512i __A) {
   // CHECK-LABEL: @test_mm512_mask_store_epi32
-  // CHECK: @llvm.x86.avx512.mask.store.d.512
+  // CHECK: @llvm.masked.store.v16i32(<16 x i32> %{{.*}}, <16 x i32>* %{{.*}}, i32 64, <16 x i1> %{{.*}})
   return _mm512_mask_store_epi32(__P, __U, __A); 
 }
 
 void test_mm512_mask_store_epi64(void *__P, __mmask8 __U, __m512i __A) {
   // CHECK-LABEL: @test_mm512_mask_store_epi64
-  // CHECK: @llvm.x86.avx512.mask.store.q.512
+  // CHECK: @llvm.masked.store.v8i64(<8 x i64> %{{.*}}, <8 x i64>* %{{.*}}, i32 64, <8 x i1> %{{.*}})
   return _mm512_mask_store_epi64(__P, __U, __A); 
 }
 
 __m512d test_mm512_movedup_pd(__m512d __A) {
   // CHECK-LABEL: @test_mm512_movedup_pd
   // CHECK: @llvm.x86.avx512.mask.movddup.512
   return _mm512_movedup_pd(__A); 
 }

test/CodeGen/avx512vl-builtins.c

 __m256i test_mm256_maskz_srav_epi64(__mmask8 __U, __m256i __X, __m256i __Y) {
   // CHECK-LABEL: @test_mm256_maskz_srav_epi64
   // CHECK: @llvm.x86.avx512.mask.psrav.q.256
   return _mm256_maskz_srav_epi64(__U, __X, __Y); 
 }
 
 void test_mm_mask_store_epi32(void *__P, __mmask8 __U, __m128i __A) {
   // CHECK-LABEL: @test_mm_mask_store_epi32
-  // CHECK: @llvm.x86.avx512.mask.store.d.128
+  // CHECK: @llvm.masked.store.v4i32(<4 x i32> %{{.*}}, <4 x i32>* %{{.}}, i32 16, <4 x i1> %{{.*}})
   return _mm_mask_store_epi32(__P, __U, __A); 
 }
 
 void test_mm256_mask_store_epi32(void *__P, __mmask8 __U, __m256i __A) {
   // CHECK-LABEL: @test_mm256_mask_store_epi32
-  // CHECK: @llvm.x86.avx512.mask.store.d.256
+  // CHECK: @llvm.masked.store.v8i32(<8 x i32> %{{.*}}, <8 x i32>* %{{.}}, i32 32, <8 x i1> %{{.*}})
   return _mm256_mask_store_epi32(__P, __U, __A); 
 }
 
 __m128i test_mm_mask_mov_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
   // CHECK-LABEL: @test_mm_mask_mov_epi32
   // CHECK: @llvm.x86.avx512.mask.mov
   return _mm_mask_mov_epi32(__W, __U, __A); 
 }

 __m256i test_mm256_maskz_load_epi64(__mmask8 __U, void const *__P) {
   // CHECK-LABEL: @test_mm256_maskz_load_epi64
   // CHECK: @llvm.x86.avx512.mask.load.q.256
   return _mm256_maskz_load_epi64(__U, __P); 
 }
 
 void test_mm_mask_store_epi64(void *__P, __mmask8 __U, __m128i __A) {
   // CHECK-LABEL: @test_mm_mask_store_epi64
-  // CHECK: @llvm.x86.avx512.mask.store.q.128
+  // CHECK: @llvm.masked.store.v2i64(<2 x i64> %{{.*}}, <2 x i64>* %{{.*}}, i32 16, <2 x i1> %{{.*}})
   return _mm_mask_store_epi64(__P, __U, __A); 
 }
 
 void test_mm256_mask_store_epi64(void *__P, __mmask8 __U, __m256i __A) {
   // CHECK-LABEL: @test_mm256_mask_store_epi64
-  // CHECK: @llvm.x86.avx512.mask.store.q.256
+  // CHECK: @llvm.masked.store.v4i64(<4 x i64> %{{.*}}, <4 x i64>* %{{.*}}, i32 32, <4 x i1> %{{.*}})
   return _mm256_mask_store_epi64(__P, __U, __A); 
 }
 
 __m128d test_mm_mask_movedup_pd(__m128d __W, __mmask8 __U, __m128d __A) {
   // CHECK-LABEL: @test_mm_mask_movedup_pd
   // CHECK: @llvm.x86.avx512.mask.movddup.128
   return _mm_mask_movedup_pd(__W, __U, __A); 
 }

 __m256 test_mm256_maskz_loadu_ps(__mmask8 __U, void const *__P) {
   // CHECK-LABEL: @test_mm256_maskz_loadu_ps
   // CHECK: @llvm.x86.avx512.mask.loadu.ps.256
   return _mm256_maskz_loadu_ps(__U, __P); 
 }
 
 void test_mm_mask_store_pd(void *__P, __mmask8 __U, __m128d __A) {
   // CHECK-LABEL: @test_mm_mask_store_pd
-  // CHECK: @llvm.x86.avx512.mask.store.pd.128
+  // CHECK: @llvm.masked.store.v2f64(<2 x double> %{{.*}}, <2 x double>* %{{.*}}, i32 16, <2 x i1> %{{.*}})
   return _mm_mask_store_pd(__P, __U, __A); 
 }
 
 void test_mm256_mask_store_pd(void *__P, __mmask8 __U, __m256d __A) {
   // CHECK-LABEL: @test_mm256_mask_store_pd
-  // CHECK: @llvm.x86.avx512.mask.store.pd.256
+  // CHECK: @llvm.masked.store.v4f64(<4 x double> %{{.*}}, <4 x double>* %{{.*}}, i32 32, <4 x i1> %{{.*}})
   return _mm256_mask_store_pd(__P, __U, __A); 
 }
 
 void test_mm_mask_store_ps(void *__P, __mmask8 __U, __m128 __A) {
   // CHECK-LABEL: @test_mm_mask_store_ps
-  // CHECK: @llvm.x86.avx512.mask.store.ps.128
+  // CHECK: @llvm.masked.store.v4f32(<4 x float> %{{.*}}, <4 x float>* %{{.*}}, i32 16, <4 x i1> %{{.*}})
   return _mm_mask_store_ps(__P, __U, __A); 
 }
 
 void test_mm256_mask_store_ps(void *__P, __mmask8 __U, __m256 __A) {
   // CHECK-LABEL: @test_mm256_mask_store_ps
-  // CHECK: @llvm.x86.avx512.mask.store.ps.256
+  // CHECK: @llvm.masked.store.v8f32(<8 x float> %{{.*}}, <8 x float>* %{{.*}}, i32 32, <8 x i1> %{{.*}})
   return _mm256_mask_store_ps(__P, __U, __A); 
 }
 
 void test_mm_mask_storeu_epi64(void *__P, __mmask8 __U, __m128i __A) {
   // CHECK-LABEL: @test_mm_mask_storeu_epi64
-  // CHECK: @llvm.x86.avx512.mask.storeu
+  // CHECK: @llvm.masked.store.v2i64(<2 x i64> %{{.*}}, <2 x i64>* %{{.*}}, i32 1, <2 x i1> %{{.*}})
   return _mm_mask_storeu_epi64(__P, __U, __A); 
 }
 
 void test_mm256_mask_storeu_epi64(void *__P, __mmask8 __U, __m256i __A) {
   // CHECK-LABEL: @test_mm256_mask_storeu_epi64
-  // CHECK: @llvm.x86.avx512.mask.storeu
+  // CHECK: @llvm.masked.store.v4i64(<4 x i64> %{{.*}}, <4 x i64>* %{{.*}}, i32 1, <4 x i1> %{{.*}})
   return _mm256_mask_storeu_epi64(__P, __U, __A); 
 }
 
 void test_mm_mask_storeu_epi32(void *__P, __mmask8 __U, __m128i __A) {
   // CHECK-LABEL: @test_mm_mask_storeu_epi32
-  // CHECK: @llvm.x86.avx512.mask.storeu
+  // CHECK: @llvm.masked.store.v4i32(<4 x i32> %{{.*}}, <4 x i32>* %{{.*}}, i32 1, <4 x i1> %{{.*}})
   return _mm_mask_storeu_epi32(__P, __U, __A); 
 }
 
 void test_mm256_mask_storeu_epi32(void *__P, __mmask8 __U, __m256i __A) {
   // CHECK-LABEL: @test_mm256_mask_storeu_epi32
-  // CHECK: @llvm.x86.avx512.mask.storeu
+  // CHECK: @llvm.masked.store.v8i32(<8 x i32> %{{.*}}, <8 x i32>* %{{.*}}, i32 1, <8 x i1> %{{.*}})
   return _mm256_mask_storeu_epi32(__P, __U, __A); 
 }
 
 void test_mm_mask_storeu_pd(void *__P, __mmask8 __U, __m128d __A) {
   // CHECK-LABEL: @test_mm_mask_storeu_pd
-  // CHECK: @llvm.x86.avx512.mask.storeu.pd.128
+  // CHECK: @llvm.masked.store.v2f64(<2 x double> %{{.*}}, <2 x double>* %{{.*}}, i32 1, <2 x i1> %{{.*}})
   return _mm_mask_storeu_pd(__P, __U, __A); 
 }
 
 void test_mm256_mask_storeu_pd(void *__P, __mmask8 __U, __m256d __A) {
   // CHECK-LABEL: @test_mm256_mask_storeu_pd
-  // CHECK: @llvm.x86.avx512.mask.storeu.pd.256
+  // CHECK: @llvm.masked.store.v4f64(<4 x double> %{{.*}}, <4 x double>* %{{.*}}, i32 1, <4 x i1> %{{.*}})
   return _mm256_mask_storeu_pd(__P, __U, __A); 
 }
 
 void test_mm_mask_storeu_ps(void *__P, __mmask8 __U, __m128 __A) {
   // CHECK-LABEL: @test_mm_mask_storeu_ps
-  // CHECK: @llvm.x86.avx512.mask.storeu.ps.128
+  // CHECK: @llvm.masked.store.v4f32(<4 x float> %{{.*}}, <4 x float>* %{{.*}}, i32 1, <4 x i1> %{{.*}})
   return _mm_mask_storeu_ps(__P, __U, __A); 
 }
 
 void test_mm256_mask_storeu_ps(void *__P, __mmask8 __U, __m256 __A) {
   // CHECK-LABEL: @test_mm256_mask_storeu_ps
-  // CHECK: @llvm.x86.avx512.mask.storeu.ps.256
+  // CHECK: @llvm.masked.store.v8f32(<8 x float> %{{.*}}, <8 x float>* %{{.*}}, i32 1, <8 x i1> %{{.*}})
   return _mm256_mask_storeu_ps(__P, __U, __A); 
 }
 
 __m128d test_mm_mask_unpackhi_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
   // CHECK-LABEL: @test_mm_mask_unpackhi_pd
   // CHECK: @llvm.x86.avx512.mask.unpckh.pd.128
   return _mm_mask_unpackhi_pd(__W, __U, __A, __B); 
 }

test/CodeGen/avx512vlbw-builtins.c

 __m256i test_mm256_maskz_loadu_epi8(__mmask32 __U, void const *__P) {
   // CHECK-LABEL: @test_mm256_maskz_loadu_epi8
   // CHECK: @llvm.x86.avx512.mask.loadu.b.256
   return _mm256_maskz_loadu_epi8(__U, __P); 
 }
 
 void test_mm_mask_storeu_epi16(void *__P, __mmask8 __U, __m128i __A) {
   // CHECK-LABEL: @test_mm_mask_storeu_epi16
-  // CHECK: @llvm.x86.avx512.mask.storeu.w.128
+  // CHECK: @llvm.masked.store.v8i16(<8 x i16> %{{.*}}, <8 x i16>* %{{.*}}, i32 1, <8 x i1> %{{.*}})
   return _mm_mask_storeu_epi16(__P, __U, __A); 
 }
 
 void test_mm256_mask_storeu_epi16(void *__P, __mmask16 __U, __m256i __A) {
   // CHECK-LABEL: @test_mm256_mask_storeu_epi16
-  // CHECK: @llvm.x86.avx512.mask.storeu.w.256
+  // CHECK: @llvm.masked.store.v16i16(<16 x i16> %{{.*}}, <16 x i16>* %{{.*}}, i32 1, <16 x i1> %{{.*}})
   return _mm256_mask_storeu_epi16(__P, __U, __A); 
 }
 
 void test_mm_mask_storeu_epi8(void *__P, __mmask16 __U, __m128i __A) {
   // CHECK-LABEL: @test_mm_mask_storeu_epi8
-  // CHECK: @llvm.x86.avx512.mask.storeu.b.128
+  // CHECK: @llvm.masked.store.v16i8(<16 x i8> %{{.*}}, <16 x i8>* %{{.*}}, i32 1, <16 x i1> %{{.*}})
   return _mm_mask_storeu_epi8(__P, __U, __A); 
 }
 
 void test_mm256_mask_storeu_epi8(void *__P, __mmask32 __U, __m256i __A) {
   // CHECK-LABEL: @test_mm256_mask_storeu_epi8
-  // CHECK: @llvm.x86.avx512.mask.storeu.b.256
+  // CHECK: @llvm.masked.store.v32i8(<32 x i8> %{{.*}}, <32 x i8>* %{{.*}}, i32 1, <32 x i1> %{{.*}})
   return _mm256_mask_storeu_epi8(__P, __U, __A); 
 }
 __mmask16 test_mm_test_epi8_mask(__m128i __A, __m128i __B) {
   // CHECK-LABEL: @test_mm_test_epi8_mask
   // CHECK: @llvm.x86.avx512.ptestm.b.128
   return _mm_test_epi8_mask(__A, __B); 
 }