[x86][icelake][vpclmulqdq]

[coby]

Summary

added intrinsics support for vpclmulqdq instructions, matching a similar work on the backend (D40101)

[coby]

removing guards to allow better diags

[craig.topper]

LGTM

[craig.topper]

lib/Basic/Targets/X86.cpp

577 ↗

(On Diff #128139)

I think we should disable vpclmulqdq when pclmul is being disabled.

[coby]

lib/Basic/Targets/X86.cpp

577 ↗

(On Diff #128139)

agreed

[coby]

turning off pclmul forbids enabling vpclmulqdq

[craig.topper]

LGTM

cfe/trunk/include/clang/Basic/BuiltinsX86.def

 TARGET_BUILTIN(__builtin_ia32_vgf2p8affineqb_v64qi, "V64cV64cV64cIc", "", "avx512bw,gfni")
 TARGET_BUILTIN(__builtin_ia32_vgf2p8mulb_v16qi, "V16cV16cV16c", "", "gfni")
 TARGET_BUILTIN(__builtin_ia32_vgf2p8mulb_v32qi, "V32cV32cV32c", "", "avx,gfni")
 TARGET_BUILTIN(__builtin_ia32_vgf2p8mulb_v64qi, "V64cV64cV64c", "", "avx512bw,gfni")
 
 // CLMUL
 TARGET_BUILTIN(__builtin_ia32_pclmulqdq128, "V2LLiV2LLiV2LLiIc", "", "pclmul")
 
+// VPCLMULQDQ
+TARGET_BUILTIN(__builtin_ia32_pclmulqdq256, "V4LLiV4LLiV4LLiIc", "", "vpclmulqdq")
+TARGET_BUILTIN(__builtin_ia32_pclmulqdq512, "V8LLiV8LLiV8LLiIc", "", "avx512f,vpclmulqdq")
+
 // AVX
 TARGET_BUILTIN(__builtin_ia32_addsubpd256, "V4dV4dV4d", "", "avx")
 TARGET_BUILTIN(__builtin_ia32_addsubps256, "V8fV8fV8f", "", "avx")
 TARGET_BUILTIN(__builtin_ia32_haddpd256, "V4dV4dV4d", "", "avx")
 TARGET_BUILTIN(__builtin_ia32_hsubps256, "V8fV8fV8f", "", "avx")
 TARGET_BUILTIN(__builtin_ia32_hsubpd256, "V4dV4dV4d", "", "avx")
 TARGET_BUILTIN(__builtin_ia32_haddps256, "V8fV8fV8f", "", "avx")
 TARGET_BUILTIN(__builtin_ia32_maxpd256, "V4dV4dV4d", "", "avx")

cfe/trunk/include/clang/Driver/Options.td

 def msgx : Flag<["-"], "msgx">, Group<m_x86_Features_Group>;
 def mno_sgx : Flag<["-"], "mno-sgx">, Group<m_x86_Features_Group>;
 def msha : Flag<["-"], "msha">, Group<m_x86_Features_Group>;
 def mno_sha : Flag<["-"], "mno-sha">, Group<m_x86_Features_Group>;
 def mtbm : Flag<["-"], "mtbm">, Group<m_x86_Features_Group>;
 def mno_tbm : Flag<["-"], "mno-tbm">, Group<m_x86_Features_Group>;
 def mvaes : Flag<["-"], "mvaes">, Group<m_x86_Features_Group>;
 def mno_vaes : Flag<["-"], "mno-vaes">, Group<m_x86_Features_Group>;
+def mvpclmulqdq : Flag<["-"], "mvpclmulqdq">, Group<m_x86_Features_Group>;
+def mno_vpclmulqdq : Flag<["-"], "mno-vpclmulqdq">, Group<m_x86_Features_Group>;
 def mxop : Flag<["-"], "mxop">, Group<m_x86_Features_Group>;
 def mno_xop : Flag<["-"], "mno-xop">, Group<m_x86_Features_Group>;
 def mxsave : Flag<["-"], "mxsave">, Group<m_x86_Features_Group>;
 def mno_xsave : Flag<["-"], "mno-xsave">, Group<m_x86_Features_Group>;
 def mxsavec : Flag<["-"], "mxsavec">, Group<m_x86_Features_Group>;
 def mno_xsavec : Flag<["-"], "mno-xsavec">, Group<m_x86_Features_Group>;
 def mxsaveopt : Flag<["-"], "mxsaveopt">, Group<m_x86_Features_Group>;
 def mno_xsaveopt : Flag<["-"], "mno-xsaveopt">, Group<m_x86_Features_Group>;

cfe/trunk/lib/Basic/Targets/X86.h

enum MMX3DNowEnum {
     AMD3DNow,
     AMD3DNowAthlon
   } MMX3DNowLevel = NoMMX3DNow;
   enum XOPEnum { NoXOP, SSE4A, FMA4, XOP } XOPLevel = NoXOP;
 
   bool HasAES = false;
   bool HasVAES = false;
   bool HasPCLMUL = false;
+  bool HasVPCLMULQDQ = false;
   bool HasGFNI = false;
   bool HasLZCNT = false;
   bool HasRDRND = false;
   bool HasFSGSBASE = false;
   bool HasBMI = false;
   bool HasBMI2 = false;
   bool HasPOPCNT = false;
   bool HasRTM = false;

cfe/trunk/lib/Basic/Targets/X86.cpp

bool X86TargetInfo::initFeatureMap(
   case CK_K6:
   case CK_WinChipC6:
     setFeatureEnabledImpl(Features, "mmx", true);
     break;
 
   case CK_Icelake:
     setFeatureEnabledImpl(Features, "vaes", true);
     setFeatureEnabledImpl(Features, "gfni", true);
+    setFeatureEnabledImpl(Features, "vpclmulqdq", true);
     // TODO: Add icelake features here.
     LLVM_FALLTHROUGH;
   case CK_Cannonlake:
     setFeatureEnabledImpl(Features, "avx512ifma", true);
     setFeatureEnabledImpl(Features, "avx512vbmi", true);
     setFeatureEnabledImpl(Features, "sha", true);
     LLVM_FALLTHROUGH;
   case CK_SkylakeServer:
void X86TargetInfo::setSSELevel(llvm::StringMap<bool> &Features,
   case SSE41:
     Features["sse4.1"] = false;
     LLVM_FALLTHROUGH;
   case SSE42:
     Features["sse4.2"] = false;
     LLVM_FALLTHROUGH;
   case AVX:
     Features["fma"] = Features["avx"] = Features["f16c"] = Features["xsave"] =
-        Features["xsaveopt"] = Features["vaes"] = false;
+        Features["xsaveopt"] = Features["vaes"] = Features["vpclmulqdq"] = false;
     setXOPLevel(Features, FMA4, false);
     LLVM_FALLTHROUGH;
   case AVX2:
     Features["avx2"] = false;
     LLVM_FALLTHROUGH;
   case AVX512F:
     Features["avx512f"] = Features["avx512cd"] = Features["avx512er"] =
         Features["avx512pf"] = Features["avx512dq"] = Features["avx512bw"] =
void X86TargetInfo::setFeatureEnabledImpl(llvm::StringMap<bool> &Features,
   } else if (Name == "vaes") {
     if (Enabled) {
       setSSELevel(Features, AVX, Enabled);
       Features["aes"] = true;
     }
   } else if (Name == "pclmul") {
     if (Enabled)
       setSSELevel(Features, SSE2, Enabled);
+    else
+      Features["vpclmulqdq"] = false;
+  } else if (Name == "vpclmulqdq") {
+    if (Enabled) {
+      setSSELevel(Features, AVX, Enabled);
+      Features["pclmul"] = true;
+    }
   } else if (Name == "gfni") {
      if (Enabled)
       setSSELevel(Features, SSE2, Enabled);
   } else if (Name == "avx") {
     setSSELevel(Features, AVX, Enabled);
   } else if (Name == "avx2") {
     setSSELevel(Features, AVX2, Enabled);
   } else if (Name == "avx512f") {
if (Feature[0] != '+')
       continue;
 
     if (Feature == "+aes") {
       HasAES = true;
     } else if (Feature == "+vaes") {
       HasVAES = true;
     } else if (Feature == "+pclmul") {
       HasPCLMUL = true;
+    } else if (Feature == "+vpclmulqdq") {
+      HasVPCLMULQDQ = true;
     } else if (Feature == "+lzcnt") {
       HasLZCNT = true;
     } else if (Feature == "+rdrnd") {
       HasRDRND = true;
     } else if (Feature == "+fsgsbase") {
       HasFSGSBASE = true;
     } else if (Feature == "+bmi") {
       HasBMI = true;
if (HasAES)
     Builder.defineMacro("__AES__");
 
   if (HasVAES)
     Builder.defineMacro("__VAES__");
 
   if (HasPCLMUL)
     Builder.defineMacro("__PCLMUL__");
 
+  if (HasVPCLMULQDQ)
+    Builder.defineMacro("__VPCLMULQDQ__");
+
   if (HasLZCNT)
     Builder.defineMacro("__LZCNT__");
 
   if (HasRDRND)
     Builder.defineMacro("__RDRND__");
 
   if (HasFSGSBASE)
     Builder.defineMacro("__FSGSBASE__");
return llvm::StringSwitch<bool>(Name)
       .Case("sse3", true)
       .Case("ssse3", true)
       .Case("sse4", true)
       .Case("sse4.1", true)
       .Case("sse4.2", true)
       .Case("sse4a", true)
       .Case("tbm", true)
       .Case("vaes", true)
+      .Case("vpclmulqdq", true)
       .Case("x87", true)
       .Case("xop", true)
       .Case("xsave", true)
       .Case("xsavec", true)
       .Case("xsaves", true)
       .Case("xsaveopt", true)
       .Default(false);
 }
return llvm::StringSwitch<bool>(Feature)
       .Case("sse2", SSELevel >= SSE2)
       .Case("sse3", SSELevel >= SSE3)
       .Case("ssse3", SSELevel >= SSSE3)
       .Case("sse4.1", SSELevel >= SSE41)
       .Case("sse4.2", SSELevel >= SSE42)
       .Case("sse4a", XOPLevel >= SSE4A)
       .Case("tbm", HasTBM)
       .Case("vaes", HasVAES)
+      .Case("vpclmulqdq", HasVPCLMULQDQ)
       .Case("x86", true)
       .Case("x86_32", getTriple().getArch() == llvm::Triple::x86)
       .Case("x86_64", getTriple().getArch() == llvm::Triple::x86_64)
       .Case("xop", XOPLevel >= XOP)
       .Case("xsave", HasXSAVE)
       .Case("xsavec", HasXSAVEC)
       .Case("xsaves", HasXSAVES)
       .Case("xsaveopt", HasXSAVEOPT)

cfe/trunk/lib/Headers/CMakeLists.txt

set(files
   tbmintrin.h
   tgmath.h
   tmmintrin.h
   unwind.h
   vadefs.h
   vaesintrin.h
   varargs.h
   vecintrin.h
+  vpclmulqdqintrin.h
   wmmintrin.h
   __wmmintrin_aes.h
   __wmmintrin_pclmul.h
   x86intrin.h
   xmmintrin.h
   xopintrin.h
   xsavecintrin.h
   xsaveintrin.h

cfe/trunk/lib/Headers/immintrin.h

 ///    single-precision float values.
 static __inline __m256 __attribute__((__always_inline__, __nodebug__, __target__("f16c")))
 _mm256_cvtph_ps(__m128i __a)
 {
   return (__m256)__builtin_ia32_vcvtph2ps256((__v8hi)__a);
 }
 #endif /* __AVX2__ */
 
+#if !defined(_MSC_VER) || __has_feature(modules) || defined(__VPCLMULQDQ__)
+#include <vpclmulqdqintrin.h>
+#endif
+
 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__BMI__)
 #include <bmiintrin.h>
 #endif
 
 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__BMI2__)
 #include <bmi2intrin.h>
 #endif
 

cfe/trunk/lib/Headers/vpclmulqdqintrin.h

+/*===------------ vpclmulqdqintrin.h - VPCLMULQDQ intrinsics ---------------===
+ *
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ *===-----------------------------------------------------------------------===
+ */
+#ifndef __IMMINTRIN_H
+#error "Never use <vpclmulqdqintrin.h> directly; include <immintrin.h> instead."
+#endif
+
+#ifndef __VPCLMULQDQINTRIN_H
+#define __VPCLMULQDQINTRIN_H
+
+#define _mm256_clmulepi64_epi128(A, B, I) __extension__ ({    \
+  (__m256i)__builtin_ia32_pclmulqdq256((__v4di)(__m256i)(A),  \
+                                       (__v4di)(__m256i)(B),  \
+                                       (char)(I)); })
+
+#define _mm512_clmulepi64_epi128(A, B, I) __extension__ ({    \
+  (__m512i)__builtin_ia32_pclmulqdq512((__v8di)(__m512i)(A),  \
+                                       (__v8di)(__m512i)(B),  \
+                                       (char)(I)); })
+
+#endif // __VPCLMULQDQINTRIN_H
+

cfe/trunk/test/CodeGen/attr-target-x86.c

 // CHECK: narwhal{{.*}} #4
 // CHECK: bar{{.*}} #0
 // CHECK: qux{{.*}} #1
 // CHECK: qax{{.*}} #5
 // CHECK: qq{{.*}} #6
 // CHECK: lake{{.*}} #7
 // CHECK: #0 = {{.*}}"target-cpu"="i686" "target-features"="+x87"
 // CHECK: #1 = {{.*}}"target-cpu"="ivybridge" "target-features"="+aes,+avx,+cx16,+f16c,+fsgsbase,+fxsr,+mmx,+pclmul,+popcnt,+rdrnd,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt"
-// CHECK: #2 = {{.*}}"target-cpu"="i686" "target-features"="+x87,-aes,-avx,-avx2,-avx512bw,-avx512cd,-avx512dq,-avx512er,-avx512f,-avx512ifma,-avx512pf,-avx512vbmi,-avx512vl,-avx512vpopcntdq,-f16c,-fma,-fma4,-gfni,-pclmul,-sha,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-xop,-xsave,-xsaveopt"
+// CHECK: #2 = {{.*}}"target-cpu"="i686" "target-features"="+x87,-aes,-avx,-avx2,-avx512bw,-avx512cd,-avx512dq,-avx512er,-avx512f,-avx512ifma,-avx512pf,-avx512vbmi,-avx512vl,-avx512vpopcntdq,-f16c,-fma,-fma4,-gfni,-pclmul,-sha,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-xop,-xsave,-xsaveopt"
 // CHECK: #3 = {{.*}}"target-cpu"="i686" "target-features"="+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87"
-// CHECK: #4 = {{.*}}"target-cpu"="i686" "target-features"="+x87,-avx,-avx2,-avx512bw,-avx512cd,-avx512dq,-avx512er,-avx512f,-avx512ifma,-avx512pf,-avx512vbmi,-avx512vl,-avx512vpopcntdq,-f16c,-fma,-fma4,-sse4.1,-sse4.2,-vaes,-xop,-xsave,-xsaveopt"
+// CHECK: #4 = {{.*}}"target-cpu"="i686" "target-features"="+x87,-avx,-avx2,-avx512bw,-avx512cd,-avx512dq,-avx512er,-avx512f,-avx512ifma,-avx512pf,-avx512vbmi,-avx512vl,-avx512vpopcntdq,-f16c,-fma,-fma4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop,-xsave,-xsaveopt"
 // CHECK: #5 = {{.*}}"target-cpu"="ivybridge" "target-features"="+avx,+cx16,+f16c,+fsgsbase,+fxsr,+mmx,+pclmul,+popcnt,+rdrnd,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt,-aes,-vaes"
 // CHECK: #6 = {{.*}}"target-cpu"="i686" "target-features"="+x87,-3dnow,-3dnowa,-mmx"
 // CHECK: #7 = {{.*}}"target-cpu"="lakemont" "target-features"="+mmx"

cfe/trunk/test/CodeGen/vpclmulqdq-builtins.c

+// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +vpclmulqdq -emit-llvm -o - | FileCheck %s --check-prefix AVX
+// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -DAVX512 -target-feature +vpclmulqdq -target-feature +avx512f -emit-llvm -o - | FileCheck %s --check-prefixes AVX,AVX512
+
+#include <immintrin.h>
+
+__m256i test_mm256_clmulepi64_epi128(__m256i A, __m256i B) {
+  // AVX: @llvm.x86.pclmulqdq.256
+  return _mm256_clmulepi64_epi128(A, B, 0);
+}
+
+#ifdef AVX512
+__m512i test_mm512_clmulepi64_epi128(__m512i A, __m512i B) {
+  // AVX512: @llvm.x86.pclmulqdq.512
+  return _mm512_clmulepi64_epi128(A, B, 0);
+}
+#endif
+

cfe/trunk/test/Driver/x86-target-features.c

 
 // RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mvaes %s -### -o %t.o 2>&1 | FileCheck -check-prefix=VAES %s
 // RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mno-vaes %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-VAES %s
 // VAES: "-target-feature" "+vaes"
 // NO-VAES: "-target-feature" "-vaes"
 
 // RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mgfni %s -### -o %t.o 2>&1 | FileCheck -check-prefix=GFNI %s
 // RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mno-gfni %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-GFNI %s
+<<<<<<<
 // GFNI: "-target-feature" "+gfni"
 // NO-GFNI: "-target-feature" "-gfni
 
+=======
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mno-clzero %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-CLZERO %s
+// CLZERO: "-target-feature" "+clzero"
+// NO-CLZERO: "-target-feature" "-clzero"
+
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mvpclmulqdq %s -### -o %t.o 2>&1 | FileCheck -check-prefix=VPCLMULQDQ %s
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mno-vpclmulqdq %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-VPCLMULQDQ %s
+// VPCLMULQDQ: "-target-feature" "+vpclmulqdq"
+// NO-VPCLMULQDQ: "-target-feature" "-vpclmulqdq"
+>>>>>>>

cfe/trunk/test/Preprocessor/predefined-arch-macros.c

 // CHECK_ICL_M32: #define __SHA__ 1
 // CHECK_ICL_M32: #define __SSE2__ 1
 // CHECK_ICL_M32: #define __SSE3__ 1
 // CHECK_ICL_M32: #define __SSE4_1__ 1
 // CHECK_ICL_M32: #define __SSE4_2__ 1
 // CHECK_ICL_M32: #define __SSE__ 1
 // CHECK_ICL_M32: #define __SSSE3__ 1
 // CHECK_ICL_M32: #define __VAES__ 1
+// CHECK_ICL_M32: #define __VPCLMULQDQ__ 1
 // CHECK_ICL_M32: #define __XSAVEC__ 1
 // CHECK_ICL_M32: #define __XSAVEOPT__ 1
 // CHECK_ICL_M32: #define __XSAVES__ 1
 // CHECK_ICL_M32: #define __XSAVE__ 1
 // CHECK_ICL_M32: #define __corei7 1
 // CHECK_ICL_M32: #define __corei7__ 1
 // CHECK_ICL_M32: #define __i386 1
 // CHECK_ICL_M32: #define __i386__ 1

 // CHECK_ICL_M64: #define __SHA__ 1
 // CHECK_ICL_M64: #define __SSE2__ 1
 // CHECK_ICL_M64: #define __SSE3__ 1
 // CHECK_ICL_M64: #define __SSE4_1__ 1
 // CHECK_ICL_M64: #define __SSE4_2__ 1
 // CHECK_ICL_M64: #define __SSE__ 1
 // CHECK_ICL_M64: #define __SSSE3__ 1
 // CHECK_ICL_M64: #define __VAES__ 1
+// CHECK_ICL_M64: #define __VPCLMULQDQ__ 1
 // CHECK_ICL_M64: #define __XSAVEC__ 1
 // CHECK_ICL_M64: #define __XSAVEOPT__ 1
 // CHECK_ICL_M64: #define __XSAVES__ 1
 // CHECK_ICL_M64: #define __XSAVE__ 1
 // CHECK_ICL_M64: #define __amd64 1
 // CHECK_ICL_M64: #define __amd64__ 1
 // CHECK_ICL_M64: #define __corei7 1
 // CHECK_ICL_M64: #define __corei7__ 1

cfe/trunk/test/Preprocessor/x86_target_features.c

 // VAESNOAES-NOT: #define __AES__ 1
 // VAESNOAES-NOT: #define __VAES__ 1
 
 // RUN: %clang -target i386-unknown-unknown -march=atom -mgfni -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=GFNI %s
 
 // GFNI: #define __GFNI__ 1
 // GFNI: #define __SSE2__ 1
 
+// RUN: %clang -target i386-unknown-unknown -march=atom -mvpclmulqdq -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=VPCLMULQDQ %s
+
+// VPCLMULQDQ: #define __PCLMUL__ 1
+// VPCLMULQDQ: #define __VPCLMULQDQ__ 1
+
+// RUN: %clang -target i386-unknown-unknown -march=atom -mvpclmulqdq -mno-pclmul -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=VPCLMULQDQNOPCLMUL %s
+// VPCLMULQDQNOPCLMUL-NOT: #define __PCLMUL__ 1
+// VPCLMULQDQNOPCLMUL-NOT: #define __VPCLMULQDQ__ 1
+