[x86] wbnoinvd intrinsic

[GBuella]

Summary

The WBNOINVD instruction writes back all modified
cache lines in the processor’s internal cache to main memory
but does not invalidate (flush) the internal caches.

[craig.topper]

docs/ClangCommandLineReference.rst

2359

Did you manually add these? This file is normally generated by a tool and should be in alphabetical order.

lib/Basic/Targets/X86.cpp

136

Is this based on an old repo? Icelake features have been coded here since late December. But as I said in the llvm patch we probably can't enable this on all icelakes.

lib/Headers/ia32intrin.h

79 ↗

(On Diff #136065)

Can you separate wbinvd out of this patch? This has some Microsoft compatibility issues that need to be carefully checked. We seem to already have __wbinvd() defined in intrin.h but I'm not sure it does anything.

lib/Headers/wbnoinvdintrin.h

32

Use an inline function, not a macro.

[GBuella]

Added Ice Lake Server architecture.
Removed wbinvd related code.

[craig.topper]

LGTM

[GBuella]

Rebased the patch.

[craig.topper]

lib/Basic/Targets/X86.cpp

188

Not related to this patch, but does IcelakeServer have SGX? The adding icelake-server patch and removing sgx from skylakeserver intersected here.

[GBuella]

lib/Basic/Targets/X86.cpp

188

As far as I know both icelake-client & icelake-server has it.
https://software.intel.com/sites/default/files/managed/c5/15/architecture-instruction-set-extensions-programming-reference.pdf
says:
"ENCLV - Ice Lake Server and later ; Future Tremont and later"
enclv is an SGX leaf.

[craig.topper]

LGTM

docs/ClangCommandLineReference.rst

 
 .. option:: -mvpclmulqdq, -mno-vpclmulqdq
 
+.. option:: -mwbnoinvd, -mno-wbnoinvd
+
 .. option:: -mx87, -m80387, -mno-x87
 
 .. option:: -mxop, -mno-xop

include/clang/Basic/BuiltinsX86.def

 //CLWB
 TARGET_BUILTIN(__builtin_ia32_clwb, "vvC*", "", "clwb")
 
+//WBNOINVD
+TARGET_BUILTIN(__builtin_ia32_wbnoinvd, "v", "", "wbnoinvd")
+
 // ADX
 TARGET_BUILTIN(__builtin_ia32_addcarryx_u32, "UcUcUiUiUi*", "", "adx")
 TARGET_BUILTIN(__builtin_ia32_addcarry_u32, "UcUcUiUiUi*", "", "")

include/clang/Basic/X86Target.def

 
 /// \name Icelake Client
 /// Icelake client microarchitecture based processors.
-PROC(Icelake, "icelake", PROC_64_BIT)
+PROC(IcelakeClient, "icelake", PROC_64_BIT)
+
+/// \name Icelake Server
+/// Icelake server microarchitecture based processors.
+PROC(IcelakeServer, "icelakex", PROC_64_BIT)
 
 /// \name Knights Landing
 /// Knights Landing processor.

include/clang/Driver/Options.td

 def mno_clflushopt : Flag<["-"], "mno-clflushopt">, Group<m_x86_Features_Group>;
 def mclwb : Flag<["-"], "mclwb">, Group<m_x86_Features_Group>;
 def mno_clwb : Flag<["-"], "mno-clwb">, Group<m_x86_Features_Group>;
+def mwbnoinvd : Flag<["-"], "mwbnoinvd">, Group<m_x86_Features_Group>;
+def mno_wbnoinvd : Flag<["-"], "mno-wbnoinvd">, Group<m_x86_Features_Group>;
 def mclzero : Flag<["-"], "mclzero">, Group<m_x86_Features_Group>;
 def mno_clzero : Flag<["-"], "mno-clzero">, Group<m_x86_Features_Group>;
 def mcx16 : Flag<["-"], "mcx16">, Group<m_x86_Features_Group>;

lib/Basic/Targets/X86.h

   bool HasRetpoline = false;
   bool HasRetpolineExternalThunk = false;
   bool HasLAHFSAHF = false;
+  bool HasWBNOINVD = false;
 
 protected:
   /// \brief Enumeration of all of the X86 CPUs supported by Clang.

lib/Basic/Targets/X86.cpp

     setFeatureEnabledImpl(Features, "mmx", true);
     break;
 
-  case CK_Icelake:
+  case CK_IcelakeServer:
+    setFeatureEnabledImpl(Features, "wbnoinvd", true);
+    LLVM_FALLTHROUGH;
+  case CK_IcelakeClient:
     setFeatureEnabledImpl(Features, "vaes", true);
     setFeatureEnabledImpl(Features, "gfni", true);
     setFeatureEnabledImpl(Features, "vpclmulqdq", true);

       HasCLFLUSHOPT = true;
     } else if (Feature == "+clwb") {
       HasCLWB = true;
+    } else if (Feature == "+wbnoinvd") {
+      HasWBNOINVD = true;
     } else if (Feature == "+prefetchwt1") {
       HasPREFETCHWT1 = true;
     } else if (Feature == "+clzero") {

   case CK_SkylakeClient:
   case CK_SkylakeServer:
   case CK_Cannonlake:
-  case CK_Icelake:
+  case CK_IcelakeClient:
+  case CK_IcelakeServer:
     // FIXME: Historically, we defined this legacy name, it would be nice to
     // remove it at some point. We've never exposed fine-grained names for
     // recent primary x86 CPUs, and we should keep it that way.

     Builder.defineMacro("__CLFLUSHOPT__");
   if (HasCLWB)
     Builder.defineMacro("__CLWB__");
+  if (HasWBNOINVD)
+    Builder.defineMacro("__WBNOINVD__");
   if (HasMPX)
     Builder.defineMacro("__MPX__");
   if (HasSHSTK)

       .Case("tbm", true)
       .Case("vaes", true)
       .Case("vpclmulqdq", true)
+      .Case("wbnoinvd", true)
       .Case("x87", true)
       .Case("xop", true)
       .Case("xsave", true)

       .Case("tbm", HasTBM)
       .Case("vaes", HasVAES)
       .Case("vpclmulqdq", HasVPCLMULQDQ)
+      .Case("wbnoinvd", HasWBNOINVD)
       .Case("x86", true)
       .Case("x86_32", getTriple().getArch() == llvm::Triple::x86)
       .Case("x86_64", getTriple().getArch() == llvm::Triple::x86_64)

lib/Headers/CMakeLists.txt

   varargs.h
   vecintrin.h
   vpclmulqdqintrin.h
+  wbnoinvdintrin.h
   wmmintrin.h
   __wmmintrin_aes.h
   __wmmintrin_pclmul.h

lib/Headers/cpuid.h

 #define bit_3DNOWP      0x40000000
 #define bit_3DNOW       0x80000000
 
-/* Features in %ebx for leaf 0x80000001 */
+/* Features in %ebx for leaf 0x80000008 */
 #define bit_CLZERO      0x00000001
+#define bit_WBNOINVD    0x00000200
 
 
 #if __i386__

lib/Headers/wbnoinvdintrin.h

+/*===---- wbnoinvdintrin.h - wbnoinvd intrinsic-----------------------------===
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#ifndef __X86INTRIN_H
+#error "Never use <wbnoinvdintrin.h> directly; include <x86intrin.h> instead."
+#endif
+
+#ifndef __WBNOINVDINTRIN_H
+#define __WBNOINVDINTRIN_H
+
+static __inline__ void
+  __attribute__((__always_inline__, __nodebug__,  __target__("wbnoinvd")))
+_wbnoinvd (void)
+{
+  __builtin_ia32_wbnoinvd ();
+}
+
+#endif /* __WBNOINVDINTRIN_H */

lib/Headers/x86intrin.h

 #include <clzerointrin.h>
 #endif
 
+#if !defined(_MSC_VER) || __has_feature(modules) || defined(__WBNOINVD__)
+#include <wbnoinvdintrin.h>
+#endif
+
+
 #endif /* __X86INTRIN_H */

test/CodeGen/builtin-wbnoinvd.c

+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-apple-darwin -target-feature +wbnoinvd  -emit-llvm -o - -Wall -Werror | FileCheck %s
+
+#include <x86intrin.h>
+
+void test_wbnoinvd(void) {
+  //CHECK-LABEL: @test_wbnoinvd
+  //CHECK: @llvm.x86.wbnoinvd
+  _wbnoinvd();
+}

test/CodeGen/builtins-x86.c

-// RUN: %clang_cc1 -DUSE_64 -triple x86_64-unknown-unknown -target-feature +fxsr -target-feature +avx -target-feature +xsaveopt -target-feature +xsaves -target-feature +xsavec -target-feature +mwaitx -target-feature +clzero -target-feature +ibt -target-feature +shstk -emit-llvm -o %t %s
-// RUN: %clang_cc1 -DUSE_ALL -triple x86_64-unknown-unknown -target-feature +fxsr -target-feature +avx -target-feature +xsaveopt -target-feature +xsaves -target-feature +xsavec -target-feature +mwaitx -target-feature +ibt -target-feature +shstk -target-feature +clzero -fsyntax-only -o %t %s
+// RUN: %clang_cc1 -DUSE_64 -triple x86_64-unknown-unknown -target-feature +fxsr -target-feature +avx -target-feature +xsaveopt -target-feature +xsaves -target-feature +xsavec -target-feature +mwaitx -target-feature +clzero -target-feature +ibt -target-feature +shstk -target-feature +wbnoinvd -emit-llvm -o %t %s
+// RUN: %clang_cc1 -DUSE_ALL -triple x86_64-unknown-unknown -target-feature +fxsr -target-feature +avx -target-feature +xsaveopt -target-feature +xsaves -target-feature +xsavec -target-feature +mwaitx -target-feature +ibt -target-feature +shstk -target-feature +clzero -target-feature +wbnoinvd -fsyntax-only -o %t %s
 
 #ifdef USE_ALL
 #define USE_3DNOW

   tmp_i = __rdtsc();
   tmp_i = __builtin_ia32_rdtscp(&tmp_Ui);
   tmp_LLi = __builtin_ia32_rdpmc(tmp_i);
+  __builtin_ia32_wbnoinvd();
 #ifdef USE_64
   tmp_LLi = __builtin_ia32_cvtss2si64(tmp_V4f);
   tmp_LLi = __builtin_ia32_cvttss2si64(tmp_V4f);

test/Driver/x86-march.c

 // RUN:   | FileCheck %s -check-prefix=icelake
 // icelake: "-target-cpu" "icelake"
 //
+// RUN: %clang -target x86_64-unknown-unknown -c -### %s -march=icelakex 2>&1 \
+// RUN:   | FileCheck %s -check-prefix=icelakex
+// icelakex: "-target-cpu" "icelakex"
+//
 // RUN: %clang -target x86_64-unknown-unknown -c -### %s -march=lakemont 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=lakemont
 // lakemont: "-target-cpu" "lakemont"

test/Driver/x86-target-features.c

 // CLWB: "-target-feature" "+clwb"
 // NO-CLWB: "-target-feature" "-clwb"
 
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mwbnoinvd %s -### -o %t.o 2>&1 | FileCheck -check-prefix=WBNOINVD %s
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mno-wbnoinvd %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-WBNOINVD %s
+// WBNOINVD: "-target-feature" "+wbnoinvd"
+// NO-WBNOINVD: "-target-feature" "-wbnoinvd"
+
 // RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mmovbe %s -### -o %t.o 2>&1 | FileCheck -check-prefix=MOVBE %s
 // RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mno-movbe %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-MOVBE %s
 // MOVBE: "-target-feature" "+movbe"

test/Frontend/x86-target-cpu.c

 // RUN: %clang_cc1 -triple x86_64-unknown-unknown -target-cpu skx -verify %s
 // RUN: %clang_cc1 -triple x86_64-unknown-unknown -target-cpu cannonlake -verify %s
 // RUN: %clang_cc1 -triple x86_64-unknown-unknown -target-cpu icelake -verify %s
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -target-cpu icelakex -verify %s
 // RUN: %clang_cc1 -triple x86_64-unknown-unknown -target-cpu knl -verify %s
 // RUN: %clang_cc1 -triple x86_64-unknown-unknown -target-cpu knm -verify %s
 // RUN: %clang_cc1 -triple x86_64-unknown-unknown -target-cpu bonnell -verify %s

test/Preprocessor/predefined-arch-macros.c

 // CHECK_ICL_M32: #define __i386__ 1
 // CHECK_ICL_M32: #define __tune_corei7__ 1
 // CHECK_ICL_M32: #define i386 1
-//
+
+// RUN: %clang -march=icelakex -m32 -E -dM %s -o - 2>&1 \
+// RUN:     -target i386-unknown-linux \
+// RUN:   | FileCheck -match-full-lines %s -check-prefix=CHECK_ICX_M32
+// CHECK_ICX_M32: #define __AES__ 1
+// CHECK_ICX_M32: #define __AVX2__ 1
+// CHECK_ICX_M32: #define __AVX512BITALG__ 1
+// CHECK_ICX_M32: #define __AVX512BW__ 1
+// CHECK_ICX_M32: #define __AVX512CD__ 1
+// CHECK_ICX_M32: #define __AVX512DQ__ 1
+// CHECK_ICX_M32: #define __AVX512F__ 1
+// CHECK_ICX_M32: #define __AVX512IFMA__ 1
+// CHECK_ICX_M32: #define __AVX512VBMI2__ 1
+// CHECK_ICX_M32: #define __AVX512VBMI__ 1
+// CHECK_ICX_M32: #define __AVX512VL__ 1
+// CHECK_ICX_M32: #define __AVX512VNNI__ 1
+// CHECK_ICX_M32: #define __AVX512VPOPCNTDQ__ 1
+// CHECK_ICX_M32: #define __AVX__ 1
+// CHECK_ICX_M32: #define __BMI2__ 1
+// CHECK_ICX_M32: #define __BMI__ 1
+// CHECK_ICX_M32: #define __CLFLUSHOPT__ 1
+// CHECK_ICX_M32: #define __CLWB__ 1
+// CHECK_ICX_M32: #define __F16C__ 1
+// CHECK_ICX_M32: #define __FMA__ 1
+// CHECK_ICX_M32: #define __GFNI__ 1
+// CHECK_ICX_M32: #define __LZCNT__ 1
+// CHECK_ICX_M32: #define __MMX__ 1
+// CHECK_ICX_M32: #define __MPX__ 1
+// CHECK_ICX_M32: #define __PCLMUL__ 1
+// CHECK_ICX_M32: #define __PKU__ 1
+// CHECK_ICX_M32: #define __POPCNT__ 1
+// CHECK_ICX_M32: #define __PRFCHW__ 1
+// CHECK_ICX_M32: #define __RDPID__ 1
+// CHECK_ICX_M32: #define __RDRND__ 1
+// CHECK_ICX_M32: #define __RDSEED__ 1
+// CHECK_ICX_M32: #define __RTM__ 1
+// CHECK_ICX_M32: #define __SGX__ 1
+// CHECK_ICX_M32: #define __SHA__ 1
+// CHECK_ICX_M32: #define __SSE2__ 1
+// CHECK_ICX_M32: #define __SSE3__ 1
+// CHECK_ICX_M32: #define __SSE4_1__ 1
+// CHECK_ICX_M32: #define __SSE4_2__ 1
+// CHECK_ICX_M32: #define __SSE__ 1
+// CHECK_ICX_M32: #define __SSSE3__ 1
+// CHECK_ICX_M32: #define __VAES__ 1
+// CHECK_ICX_M32: #define __VPCLMULQDQ__ 1
+// CHECK_ICX_M32: #define __WBNOINVD__ 1
+// CHECK_ICX_M32: #define __XSAVEC__ 1
+// CHECK_ICX_M32: #define __XSAVEOPT__ 1
+// CHECK_ICX_M32: #define __XSAVES__ 1
+// CHECK_ICX_M32: #define __XSAVE__ 1
+// CHECK_ICX_M32: #define __corei7 1
+// CHECK_ICX_M32: #define __corei7__ 1
+// CHECK_ICX_M32: #define __i386 1
+// CHECK_ICX_M32: #define __i386__ 1
+// CHECK_ICX_M32: #define __tune_corei7__ 1
+// CHECK_ICX_M32: #define i386 1
+
 // RUN: %clang -march=icelake -m64 -E -dM %s -o - 2>&1 \
 // RUN:     -target i386-unknown-linux \
 // RUN:   | FileCheck -match-full-lines %s -check-prefix=CHECK_ICL_M64

 // CHECK_ICL_M64: #define __x86_64 1
 // CHECK_ICL_M64: #define __x86_64__ 1
 
+// RUN: %clang -march=icelakex -m64 -E -dM %s -o - 2>&1 \
+// RUN:     -target i386-unknown-linux \
+// RUN:   | FileCheck -match-full-lines %s -check-prefix=CHECK_ICX_M64
+// CHECK_ICX_M64: #define __AES__ 1
+// CHECK_ICX_M64: #define __AVX2__ 1
+// CHECK_ICX_M64: #define __AVX512BITALG__ 1
+// CHECK_ICX_M64: #define __AVX512BW__ 1
+// CHECK_ICX_M64: #define __AVX512CD__ 1
+// CHECK_ICX_M64: #define __AVX512DQ__ 1
+// CHECK_ICX_M64: #define __AVX512F__ 1
+// CHECK_ICX_M64: #define __AVX512IFMA__ 1
+// CHECK_ICX_M64: #define __AVX512VBMI2__ 1
+// CHECK_ICX_M64: #define __AVX512VBMI__ 1
+// CHECK_ICX_M64: #define __AVX512VL__ 1
+// CHECK_ICX_M64: #define __AVX512VNNI__ 1
+// CHECK_ICX_M64: #define __AVX512VPOPCNTDQ__ 1
+// CHECK_ICX_M64: #define __AVX__ 1
+// CHECK_ICX_M64: #define __BMI2__ 1
+// CHECK_ICX_M64: #define __BMI__ 1
+// CHECK_ICX_M64: #define __CLFLUSHOPT__ 1
+// CHECK_ICX_M64: #define __CLWB__ 1
+// CHECK_ICX_M64: #define __F16C__ 1
+// CHECK_ICX_M64: #define __FMA__ 1
+// CHECK_ICX_M64: #define __GFNI__ 1
+// CHECK_ICX_M64: #define __LZCNT__ 1
+// CHECK_ICX_M64: #define __MMX__ 1
+// CHECK_ICX_M64: #define __MPX__ 1
+// CHECK_ICX_M64: #define __PCLMUL__ 1
+// CHECK_ICX_M64: #define __PKU__ 1
+// CHECK_ICX_M64: #define __POPCNT__ 1
+// CHECK_ICX_M64: #define __PRFCHW__ 1
+// CHECK_ICX_M64: #define __RDPID__ 1
+// CHECK_ICX_M64: #define __RDRND__ 1
+// CHECK_ICX_M64: #define __RDSEED__ 1
+// CHECK_ICX_M64: #define __RTM__ 1
+// CHECK_ICX_M64: #define __SGX__ 1
+// CHECK_ICX_M64: #define __SHA__ 1
+// CHECK_ICX_M64: #define __SSE2__ 1
+// CHECK_ICX_M64: #define __SSE3__ 1
+// CHECK_ICX_M64: #define __SSE4_1__ 1
+// CHECK_ICX_M64: #define __SSE4_2__ 1
+// CHECK_ICX_M64: #define __SSE__ 1
+// CHECK_ICX_M64: #define __SSSE3__ 1
+// CHECK_ICX_M64: #define __VAES__ 1
+// CHECK_ICX_M64: #define __VPCLMULQDQ__ 1
+// CHECK_ICX_M64: #define __WBNOINVD__ 1
+// CHECK_ICX_M64: #define __XSAVEC__ 1
+// CHECK_ICX_M64: #define __XSAVEOPT__ 1
+// CHECK_ICX_M64: #define __XSAVES__ 1
+// CHECK_ICX_M64: #define __XSAVE__ 1
+// CHECK_ICX_M64: #define __amd64 1
+// CHECK_ICX_M64: #define __amd64__ 1
+// CHECK_ICX_M64: #define __corei7 1
+// CHECK_ICX_M64: #define __corei7__ 1
+// CHECK_ICX_M64: #define __tune_corei7__ 1
+// CHECK_ICX_M64: #define __x86_64 1
+// CHECK_ICX_M64: #define __x86_64__ 1
+
 // RUN: %clang -march=atom -m32 -E -dM %s -o - 2>&1 \
 // RUN:     -target i386-unknown-linux \
 // RUN:   | FileCheck -match-full-lines %s -check-prefix=CHECK_ATOM_M32