[X86] WaitPKG intrinsics

[craig.topper]

LGTM

[GBuella]

Using the modified LLVM intrinsics from D45253.

[craig.topper]

LGTM

lib/Headers/waitpkgintrin.h

41

Why does the intrinsic take size_t but then its truncated to 32 bits?

[GBuella]

lib/Headers/waitpkgintrin.h

41

When I replaced umwait32 & umwait64 with a single umwait LLVM intrinsic, I choose i32 as the type of the first argument.
The doc says:
bit[0] has a meaning, while bit[31:1] are reserved.
The doc doesn't suggest that the upper 32 bits might ever be used.
But I see, this might need some more discussion.

[GBuella]

Modified the intrinsic.

[craig.topper]

lib/Headers/waitpkgintrin.h

42

I think we should use "unsigned int" and "unsigned long long" explciitly. I think that's the prevailing behavior in the rest of our headers.

Also change the argument names to lower case. I think that's also pretty standard.

[craig.topper]

LGTM

include/clang/Basic/BuiltinsX86.def

 TARGET_BUILTIN(__builtin_ia32_selectpd_128, "V2dUcV2dV2d", "", "")
 TARGET_BUILTIN(__builtin_ia32_selectpd_256, "V4dUcV4dV4d", "", "")
 TARGET_BUILTIN(__builtin_ia32_selectpd_512, "V8dUcV8dV8d", "", "")
 
 // MONITORX/MWAITX
 TARGET_BUILTIN(__builtin_ia32_monitorx, "vv*UiUi", "", "mwaitx")
 TARGET_BUILTIN(__builtin_ia32_mwaitx, "vUiUiUi", "", "mwaitx")
 
+// WAITPKG
+TARGET_BUILTIN(__builtin_ia32_umonitor, "vv*", "", "waitpkg")
+TARGET_BUILTIN(__builtin_ia32_umwait, "vUiUiUi", "", "waitpkg")
+TARGET_BUILTIN(__builtin_ia32_tpause, "vUiUiUi", "", "waitpkg")
+
 // CLZERO
 TARGET_BUILTIN(__builtin_ia32_clzero, "vv*", "", "clzero")
 
 // MSVC
 TARGET_HEADER_BUILTIN(_BitScanForward, "UcUNi*UNi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_BitScanReverse, "UcUNi*UNi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 
 TARGET_HEADER_BUILTIN(_ReadWriteBarrier, "v", "nh", "intrin.h", ALL_MS_LANGUAGES, "")

include/clang/Driver/Options.td

 def msha : Flag<["-"], "msha">, Group<m_x86_Features_Group>;
 def mno_sha : Flag<["-"], "mno-sha">, Group<m_x86_Features_Group>;
 def mtbm : Flag<["-"], "mtbm">, Group<m_x86_Features_Group>;
 def mno_tbm : Flag<["-"], "mno-tbm">, Group<m_x86_Features_Group>;
 def mvaes : Flag<["-"], "mvaes">, Group<m_x86_Features_Group>;
 def mno_vaes : Flag<["-"], "mno-vaes">, Group<m_x86_Features_Group>;
 def mvpclmulqdq : Flag<["-"], "mvpclmulqdq">, Group<m_x86_Features_Group>;
 def mno_vpclmulqdq : Flag<["-"], "mno-vpclmulqdq">, Group<m_x86_Features_Group>;
+def mwaitpkg : Flag<["-"], "mwaitpkg">, Group<m_x86_Features_Group>;
+def mno_waitpkg : Flag<["-"], "mno-waitpkg">, Group<m_x86_Features_Group>;
 def mxop : Flag<["-"], "mxop">, Group<m_x86_Features_Group>;
 def mno_xop : Flag<["-"], "mno-xop">, Group<m_x86_Features_Group>;
 def mxsave : Flag<["-"], "mxsave">, Group<m_x86_Features_Group>;
 def mno_xsave : Flag<["-"], "mno-xsave">, Group<m_x86_Features_Group>;
 def mxsavec : Flag<["-"], "mxsavec">, Group<m_x86_Features_Group>;
 def mno_xsavec : Flag<["-"], "mno-xsavec">, Group<m_x86_Features_Group>;
 def mxsaveopt : Flag<["-"], "mxsaveopt">, Group<m_x86_Features_Group>;
 def mno_xsaveopt : Flag<["-"], "mno-xsaveopt">, Group<m_x86_Features_Group>;

lib/Basic/Targets/X86.h

class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
   bool HasCLFLUSHOPT = false;
   bool HasCLWB = false;
   bool HasMOVBE = false;
   bool HasPREFETCHWT1 = false;
   bool HasRDPID = false;
   bool HasRetpoline = false;
   bool HasRetpolineExternalThunk = false;
   bool HasLAHFSAHF = false;
+  bool HasWAITPKG = false;
 
 protected:
   /// \brief Enumeration of all of the X86 CPUs supported by Clang.
   ///
   /// Each enumeration represents a particular CPU supported by Clang. These
   /// loosely correspond to the options passed to '-march' or '-mtune' flags.
   enum CPUKind {
     CK_Generic,

lib/Basic/Targets/X86.cpp

for (const auto &Feature : Features) {
     } else if (Feature == "+rdpid") {
       HasRDPID = true;
     } else if (Feature == "+retpoline") {
       HasRetpoline = true;
     } else if (Feature == "+retpoline-external-thunk") {
       HasRetpolineExternalThunk = true;
     } else if (Feature == "+sahf") {
       HasLAHFSAHF = true;
+    } else if (Feature == "+waitpkg") {
+      HasWAITPKG = true;
     }
 
     X86SSEEnum Level = llvm::StringSwitch<X86SSEEnum>(Feature)
                            .Case("+avx512f", AVX512F)
                            .Case("+avx2", AVX2)
                            .Case("+avx", AVX)
                            .Case("+sse4.2", SSE42)
                            .Case("+sse4.1", SSE41)
void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
   if (HasSGX)
     Builder.defineMacro("__SGX__");
   if (HasPREFETCHWT1)
     Builder.defineMacro("__PREFETCHWT1__");
   if (HasCLZERO)
     Builder.defineMacro("__CLZERO__");
   if (HasRDPID)
     Builder.defineMacro("__RDPID__");
+  if (HasWAITPKG)
+    Builder.defineMacro("__WAITPKG__");
 
   // Each case falls through to the previous one here.
   switch (SSELevel) {
   case AVX512F:
     Builder.defineMacro("__AVX512F__");
     LLVM_FALLTHROUGH;
   case AVX2:
     Builder.defineMacro("__AVX2__");
return llvm::StringSwitch<bool>(Name)
       .Case("ssse3", true)
       .Case("sse4", true)
       .Case("sse4.1", true)
       .Case("sse4.2", true)
       .Case("sse4a", true)
       .Case("tbm", true)
       .Case("vaes", true)
       .Case("vpclmulqdq", true)
+      .Case("waitpkg", true)
       .Case("x87", true)
       .Case("xop", true)
       .Case("xsave", true)
       .Case("xsavec", true)
       .Case("xsaves", true)
       .Case("xsaveopt", true)
       .Default(false);
 }
return llvm::StringSwitch<bool>(Feature)
       .Case("sse3", SSELevel >= SSE3)
       .Case("ssse3", SSELevel >= SSSE3)
       .Case("sse4.1", SSELevel >= SSE41)
       .Case("sse4.2", SSELevel >= SSE42)
       .Case("sse4a", XOPLevel >= SSE4A)
       .Case("tbm", HasTBM)
       .Case("vaes", HasVAES)
       .Case("vpclmulqdq", HasVPCLMULQDQ)
+      .Case("waitpkg", HasWAITPKG)
       .Case("x86", true)
       .Case("x86_32", getTriple().getArch() == llvm::Triple::x86)
       .Case("x86_64", getTriple().getArch() == llvm::Triple::x86_64)
       .Case("xop", XOPLevel >= XOP)
       .Case("xsave", HasXSAVE)
       .Case("xsavec", HasXSAVEC)
       .Case("xsaves", HasXSAVES)
       .Case("xsaveopt", HasXSAVEOPT)

lib/Headers/CMakeLists.txt

set(files
   tgmath.h
   tmmintrin.h
   unwind.h
   vadefs.h
   vaesintrin.h
   varargs.h
   vecintrin.h
   vpclmulqdqintrin.h
+  waitpkgintrin.h
   wmmintrin.h
   __wmmintrin_aes.h
   __wmmintrin_pclmul.h
   x86intrin.h
   xmmintrin.h
   xopintrin.h
   xsavecintrin.h
   xsaveintrin.h

lib/Headers/cpuid.h

 #define bit_AVX512BW    0x40000000
 #define bit_AVX512VL    0x80000000
 
 /* Features in %ecx for leaf 7 sub-leaf 0 */
 #define bit_PREFTCHWT1       0x00000001
 #define bit_AVX512VBMI       0x00000002
 #define bit_PKU              0x00000004
 #define bit_OSPKE            0x00000010
+#define bit_WAITPKG          0x00000020
 #define bit_AVX512VBMI2      0x00000040
 #define bit_SHSTK            0x00000080
 #define bit_GFNI             0x00000100
 #define bit_VAES             0x00000200
 #define bit_VPCLMULQDQ       0x00000400
 #define bit_AVX512VNNI       0x00000800
 #define bit_AVX512BITALG     0x00001000
 #define bit_AVX512VPOPCNTDQ  0x00004000

lib/Headers/waitpkgintrin.h

+/*===----------------------- waitpkgintrin.h - WAITPKG --------------------===
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ *===-----------------------------------------------------------------------===
+ */
+#ifndef __X86INTRIN_H
+#error "Never use <waitpkgintrin.h> directly; include <x86intrin.h> instead."
+#endif
+
+#ifndef _WAITPKGINTRIN_H
+#define _WAITPKGINTRIN_H
+
+/* Define the default attributes for the functions in this file. */
+#define __DEFAULT_FN_ATTRS \
+  __attribute__((__always_inline__, __nodebug__,  __target__("waitpkg")))
+
+static __inline__ void __DEFAULT_FN_ATTRS
+_umonitor (void * __ADDRESS)
+{
+  __builtin_ia32_umonitor (__ADDRESS);
+}
+
+static __inline__ void __DEFAULT_FN_ATTRS
+_umwait (__SIZE_TYPE__  __CONTROL, __UINT64_TYPE__  __COUNTER)
+{
+  __builtin_ia32_umwait ((unsigned int)__CONTROL,
+                      (unsigned int)(__COUNTER >> 32), (unsigned int)__COUNTER);
+}
+
+static __inline__ void __DEFAULT_FN_ATTRS
+_tpause (__SIZE_TYPE__  __CONTROL, __UINT64_TYPE__  __COUNTER)
+{
+  __builtin_ia32_tpause ((unsigned int)__CONTROL,
+                      (unsigned int)(__COUNTER >> 32), (unsigned int)__COUNTER);
+}
+
+#undef __DEFAULT_FN_ATTRS
+
+#endif /* _WAITPKGINTRIN_H */

lib/Headers/x86intrin.h

 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__MWAITX__)
 #include <mwaitxintrin.h>
 #endif
 
 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__CLZERO__)
 #include <clzerointrin.h>
 #endif
 
+#if !defined(_MSC_VER) || __has_feature(modules) || defined(__WAITPKG__)
+#include <waitpkgintrin.h>
+#endif
+
 #endif /* __X86INTRIN_H */

test/CodeGen/waitpkg.c

+// RUN: %clang_cc1 %s -ffreestanding -triple x86_64-unknown-unknown -emit-llvm -target-feature +waitpkg -o - | FileCheck %s
+// RUN: %clang_cc1 %s -ffreestanding -triple i386-unknown-unknown -emit-llvm -target-feature +waitpkg -o - | FileCheck %s
+
+#include <x86intrin.h>
+
+#include <stddef.h>
+#include <stdint.h>
+
+void test_umonitor(void *address) {
+  //CHECK-LABEL: @test_umonitor
+  //CHECK: call void @llvm.x86.umonitor(i8* %{{.*}})
+  return _umonitor(address);
+}
+
+void test_umwait(size_t control, uint64_t counter) {
+  //CHECK-LABEL: @test_umwait
+  //CHECK: call void @llvm.x86.umwait(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
+  return _umwait(control, counter);
+}
+
+void test_tpause(size_t control, uint64_t counter) {
+  //CHECK-LABEL: @test_tpause
+  //CHECK: call void @llvm.x86.tpause(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
+  return _tpause(control, counter);
+}

test/Driver/x86-target-features.c

 // RUN: %clang -target i386-linux-gnu -mno-retpoline %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-RETPOLINE %s
 // RETPOLINE: "-target-feature" "+retpoline"
 // NO-RETPOLINE: "-target-feature" "-retpoline"
 
 // RUN: %clang -target i386-linux-gnu -mretpoline -mretpoline-external-thunk %s -### -o %t.o 2>&1 | FileCheck -check-prefix=RETPOLINE-EXTERNAL-THUNK %s
 // RUN: %clang -target i386-linux-gnu -mretpoline -mno-retpoline-external-thunk %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-RETPOLINE-EXTERNAL-THUNK %s
 // RETPOLINE-EXTERNAL-THUNK: "-target-feature" "+retpoline-external-thunk"
 // NO-RETPOLINE-EXTERNAL-THUNK: "-target-feature" "-retpoline-external-thunk"
+
+// RUN: %clang -target i386-linux-gnu -mwaitpkg %s -### -o %t.o 2>&1 | FileCheck -check-prefix=WAITPKG %s
+// RUN: %clang -target i386-linux-gnu -mno-waitpkg %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-WAITPKG %s
+// WAITPKG: "-target-feature" "+waitpkg"
+// NO-WAITPKG: "-target-feature" "-waitpkg"