[X86][AVX] Ensure vector non-temporal load/store intrinsics force pointer alignment (PR33830)

[RKSimon]

Summary

Clang specifies a max type alignment of 16 bytes on darwin targets (annoyingly in the driver not via cc1), meaning that the builtin nontemporal stores don't correctly align the loads/stores to 32 or 64 bytes when required, resulting in lowering to temporal unaligned loads/stores.

This patch casts the vectors to explicitly aligned types prior to the load/store to ensure that the require alignment is respected.

[craig.topper]

LGTM

cfe/trunk/lib/Headers/avx2intrin.h

 _mm256_xor_si256(__m256i __a, __m256i __b)
 {
   return (__m256i)((__v4du)__a ^ (__v4du)__b);
 }
 
 static __inline__ __m256i __DEFAULT_FN_ATTRS
 _mm256_stream_load_si256(__m256i const *__V)
 {
-  return (__m256i)__builtin_nontemporal_load((const __v4di *)__V);
+  typedef __v4di __v4di_aligned __attribute__((aligned(32)));
+  return (__m256i)__builtin_nontemporal_load((const __v4di_aligned *)__V);
 }
 
 static __inline__ __m128 __DEFAULT_FN_ATTRS
 _mm_broadcastss_ps(__m128 __X)
 {
   return (__m128)__builtin_shufflevector((__v4sf)__X, (__v4sf)__X, 0, 0, 0, 0);
 }
 

cfe/trunk/lib/Headers/avx512fintrin.h

 _mm512_kxor (__mmask16 __A, __mmask16 __B)
 {
   return (__mmask16) __builtin_ia32_kxorhi ((__mmask16) __A, (__mmask16) __B);
 }
 
 static __inline__ void __DEFAULT_FN_ATTRS
 _mm512_stream_si512 (__m512i * __P, __m512i __A)
 {
-  __builtin_nontemporal_store((__v8di)__A, (__v8di*)__P);
+  typedef __v8di __v8di_aligned __attribute__((aligned(64)));
+  __builtin_nontemporal_store((__v8di_aligned)__A, (__v8di_aligned*)__P);
 }
 
 static __inline__ __m512i __DEFAULT_FN_ATTRS
 _mm512_stream_load_si512 (void *__P)
 {
-  return (__m512i) __builtin_nontemporal_load((const __v8di *)__P);
+  typedef __v8di __v8di_aligned __attribute__((aligned(64)));
+  return (__m512i) __builtin_nontemporal_load((const __v8di_aligned *)__P);
 }
 
 static __inline__ void __DEFAULT_FN_ATTRS
 _mm512_stream_pd (double *__P, __m512d __A)
 {
-  __builtin_nontemporal_store((__v8df)__A, (__v8df*)__P);
+  typedef __v8df __v8df_aligned __attribute__((aligned(64)));
+  __builtin_nontemporal_store((__v8df_aligned)__A, (__v8df_aligned*)__P);
 }
 
 static __inline__ void __DEFAULT_FN_ATTRS
 _mm512_stream_ps (float *__P, __m512 __A)
 {
-  __builtin_nontemporal_store((__v16sf)__A, (__v16sf*)__P);
+  typedef __v16sf __v16sf_aligned __attribute__((aligned(64)));
+  __builtin_nontemporal_store((__v16sf_aligned)__A, (__v16sf_aligned*)__P);
 }
 
 static __inline__ __m512d __DEFAULT_FN_ATTRS
 _mm512_mask_compress_pd (__m512d __W, __mmask8 __U, __m512d __A)
 {
   return (__m512d) __builtin_ia32_compressdf512_mask ((__v8df) __A,
                   (__v8df) __W,
                   (__mmask8) __U);

cfe/trunk/lib/Headers/avxintrin.h

 /// \param __a
 ///    A pointer to a 32-byte aligned memory location that will receive the
 ///    integer values.
 /// \param __b
 ///    A 256-bit integer vector containing the values to be moved.
 static __inline void __DEFAULT_FN_ATTRS
 _mm256_stream_si256(__m256i *__a, __m256i __b)
 {
-  __builtin_nontemporal_store((__v4di)__b, (__v4di*)__a);
+  typedef __v4di __v4di_aligned __attribute__((aligned(32)));
+  __builtin_nontemporal_store((__v4di_aligned)__b, (__v4di_aligned*)__a);
 }
 
 /// \brief Moves double-precision values from a 256-bit vector of [4 x double]
 ///    to a 32-byte aligned memory location. To minimize caching, the data is
 ///    flagged as non-temporal (unlikely to be used again soon).
 ///
 /// \headerfile <x86intrin.h>
 ///
 /// This intrinsic corresponds to the <c> VMOVNTPD </c> instruction.
 ///
 /// \param __a
 ///    A pointer to a 32-byte aligned memory location that will receive the
 ///    double-precision floating-point values.
 /// \param __b
 ///    A 256-bit vector of [4 x double] containing the values to be moved.
 static __inline void __DEFAULT_FN_ATTRS
 _mm256_stream_pd(double *__a, __m256d __b)
 {
-  __builtin_nontemporal_store((__v4df)__b, (__v4df*)__a);
+  typedef __v4df __v4df_aligned __attribute__((aligned(32)));
+  __builtin_nontemporal_store((__v4df_aligned)__b, (__v4df_aligned*)__a);
 }
 
 /// \brief Moves single-precision floating point values from a 256-bit vector
 ///    of [8 x float] to a 32-byte aligned memory location. To minimize
 ///    caching, the data is flagged as non-temporal (unlikely to be used again
 ///    soon).
 ///
 /// \headerfile <x86intrin.h>
 ///
 /// This intrinsic corresponds to the <c> VMOVNTPS </c> instruction.
 ///
 /// \param __p
 ///    A pointer to a 32-byte aligned memory location that will receive the
 ///    single-precision floating point values.
 /// \param __a
 ///    A 256-bit vector of [8 x float] containing the values to be moved.
 static __inline void __DEFAULT_FN_ATTRS
 _mm256_stream_ps(float *__p, __m256 __a)
 {
-  __builtin_nontemporal_store((__v8sf)__a, (__v8sf*)__p);
+  typedef __v8sf __v8sf_aligned __attribute__((aligned(32)));
+  __builtin_nontemporal_store((__v8sf_aligned)__a, (__v8sf_aligned*)__p);
 }
 
 /* Create vectors */
 /// \brief Create a 256-bit vector of [4 x double] with undefined values.
 ///
 /// \headerfile <x86intrin.h>
 ///
 /// This intrinsic has no corresponding instruction.

cfe/trunk/test/CodeGen/x86-nontemporal.c

 // RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -target-feature +avx -target-feature +avx2 -target-feature +avx512f -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefix=CHECK
 // RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -target-feature +avx -target-feature +avx2 -target-feature +avx512f -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefix=CHECK
 
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -target-feature +avx -target-feature +avx2 -target-feature +avx512f -emit-llvm -o - -Wall -Werror -fmax-type-align=16 | FileCheck %s --check-prefix=CHECK16
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -target-feature +avx -target-feature +avx2 -target-feature +avx512f -fno-signed-char -emit-llvm -o - -Wall -Werror -fmax-type-align=16 | FileCheck %s --check-prefix=CHECK16
+// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -target-feature +avx -target-feature +avx2 -target-feature +avx512f -emit-llvm -o - -Wall -Werror -fmax-type-align=16 | FileCheck %s --check-prefix=CHECK
+// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -target-feature +avx -target-feature +avx2 -target-feature +avx512f -fno-signed-char -emit-llvm -o - -Wall -Werror -fmax-type-align=16 | FileCheck %s --check-prefix=CHECK
 
 #include <x86intrin.h>
 
-// FIXME: (PR33830) Tests showing failure to correctly align non-temporal load/stores on darwin targets where fmax-type-align is set to 16.
+// (PR33830) Tests ensure the correct alignment of non-temporal load/stores on darwin targets where fmax-type-align is set to 16.
 
 //
 // 128-bit vectors
 //
 
 void test_mm_stream_pd(double* A, __m128d B) {
   // CHECK-LABEL: test_mm_stream_pd
   // CHECK: store <2 x double> %{{.*}}, <2 x double>* %{{.*}}, align 16, !nontemporal

 
 //
 // 256-bit vectors
 //
 
 void test_mm256_stream_pd(double* A, __m256d B) {
   // CHECK-LABEL: test_mm256_stream_pd
   // CHECK: store <4 x double> %{{.*}}, <4 x double>* %{{.*}}, align 32, !nontemporal
-
-  // CHECK16-LABEL: test_mm256_stream_pd
-  // CHECK16: store <4 x double> %{{.*}}, <4 x double>* %{{.*}}, align 16, !nontemporal
   _mm256_stream_pd(A, B);
 }
 
 void test_mm256_stream_ps(float* A, __m256 B) {
   // CHECK-LABEL: test_mm256_stream_ps
   // CHECK: store <8 x float> %{{.*}}, <8 x float>* %{{.*}}, align 32, !nontemporal
-
-  // CHECK16-LABEL: test_mm256_stream_ps
-  // CHECK16: store <8 x float> %{{.*}}, <8 x float>* %{{.*}}, align 16, !nontemporal
   _mm256_stream_ps(A, B);
 }
 
 void test_mm256_stream_si256(__m256i* A, __m256i B) {
   // CHECK-LABEL: test_mm256_stream_si256
   // CHECK: store <4 x i64> %{{.*}}, <4 x i64>* %{{.*}}, align 32, !nontemporal
-
-  // CHECK16-LABEL: test_mm256_stream_si256
-  // CHECK16: store <4 x i64> %{{.*}}, <4 x i64>* %{{.*}}, align 16, !nontemporal
   _mm256_stream_si256(A, B);
 }
 
 __m256i test_mm256_stream_load_si256(__m256i const *A) {
   // CHECK-LABEL: test_mm256_stream_load_si256
   // CHECK: load <4 x i64>, <4 x i64>* %{{.*}}, align 32, !nontemporal
-
-  // CHECK16-LABEL: test_mm256_stream_load_si256
-  // CHECK16: load <4 x i64>, <4 x i64>* %{{.*}}, align 16, !nontemporal
   return _mm256_stream_load_si256(A);
 }
 
 //
 // 512-bit vectors
 //
 
 void test_mm512_stream_pd(double* A, __m512d B) {
   // CHECK-LABEL: test_mm512_stream_pd
   // CHECK: store <8 x double> %{{.*}}, <8 x double>* %{{.*}}, align 64, !nontemporal
-
-  // CHECK16-LABEL: test_mm512_stream_pd
-  // CHECK16: store <8 x double> %{{.*}}, <8 x double>* %{{.*}}, align 16, !nontemporal
   _mm512_stream_pd(A, B);
 }
 
 void test_mm512_stream_ps(float* A, __m512 B) {
   // CHECK-LABEL: test_mm512_stream_ps
   // CHECK: store <16 x float> %{{.*}}, <16 x float>* %{{.*}}, align 64, !nontemporal
-
-  // CHECK16-LABEL: test_mm512_stream_ps
-  // CHECK16: store <16 x float> %{{.*}}, <16 x float>* %{{.*}}, align 16, !nontemporal
   _mm512_stream_ps(A, B);
 }
 
 void test_mm512_stream_si512(__m512i* A, __m512i B) {
   // CHECK-LABEL: test_mm512_stream_si512
   // CHECK: store <8 x i64> %{{.*}}, <8 x i64>* %{{.*}}, align 64, !nontemporal
-
-  // CHECK16-LABEL: test_mm512_stream_si512
-  // CHECK16: store <8 x i64> %{{.*}}, <8 x i64>* %{{.*}}, align 16, !nontemporal
   _mm512_stream_si512(A, B);
 }
 
 __m512i test_mm512_stream_load_si512(void *A) {
   // CHECK-LABEL: test_mm512_stream_load_si512
   // CHECK: load <8 x i64>, <8 x i64>* %{{.*}}, align 64, !nontemporal
-
-  // CHECK16-LABEL: test_mm512_stream_load_si512
-  // CHECK16: load <8 x i64>, <8 x i64>* %{{.*}}, align 16, !nontemporal
   return _mm512_stream_load_si512(A);
 }