[X86] directstore and movdir64b intrinsics

[craig.topper]

docs/ClangCommandLineReference.rst

2465 ↗

(On Diff #143630)

Please alphabetize this correctly.

include/clang/Basic/BuiltinsX86.def

1896 ↗

(On Diff #143630)

The u64 versions needs to go in BuiltinsX86_64.def. That will provide an extra safety check to prevent anyone from using the builtin without the intrin.h header on 32-bit hosts.

include/clang/Driver/Options.td

2634 ↗

(On Diff #143630)

Alphabetize

lib/Basic/Targets/X86.cpp

1304 ↗

(On Diff #143630)

Alphabetize

1379 ↗

(On Diff #143630)

Alphabetize

lib/Headers/movdirintrin.h

30 ↗

(On Diff #143630)

We aren't supposed to be checking the preprocessor defines in headers. We need to be able to put a target attribute on each function and control feature enabling at a function level.

35 ↗

(On Diff #143630)

Use 'unsigned int' instead of UINT32_TYPE

35 ↗

(On Diff #143630)

Lower case variable names

45 ↗

(On Diff #143630)

Use unsigned long long

[craig.topper]

lib/Headers/movdirintrin.h

39 ↗

(On Diff #143647)

Sorry, the X86_64 preprocessor check should have stayed. That can't be changed by a target attribute.

[craig.topper]

LGTM

cfe/trunk/docs/ClangCommandLineReference.rst

 .. option:: -mlwp, -mno-lwp
 
 .. option:: -mlzcnt, -mno-lzcnt
 
 .. option:: -mmmx, -mno-mmx
 
 .. option:: -mmovbe, -mno-movbe
 
+.. option:: -mmovdiri, -mno-movdiri
+
+.. option:: -mmovdir64b, -mno-movdir64b
+
 .. option:: -mmpx, -mno-mpx
 
 .. option:: -mmwaitx, -mno-mwaitx
 
 .. option:: -mpclmul, -mno-pclmul
 
 .. option:: -mpku, -mno-pku
 

cfe/trunk/include/clang/Basic/BuiltinsX86.def

 TARGET_BUILTIN(__builtin_ia32_tpause, "UcUiUiUi", "", "waitpkg")
 
 // CLZERO
 TARGET_BUILTIN(__builtin_ia32_clzero, "vv*", "", "clzero")
 
 // CLDEMOTE
 TARGET_BUILTIN(__builtin_ia32_cldemote, "vvC*", "", "cldemote")
 
+// Direct Move
+TARGET_BUILTIN(__builtin_ia32_directstore_u32, "vUi*Ui", "", "movdiri")
+TARGET_BUILTIN(__builtin_ia32_movdir64b, "vv*vC*", "", "movdir64b")
+
 // MSVC
 TARGET_HEADER_BUILTIN(_BitScanForward, "UcUNi*UNi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_BitScanReverse, "UcUNi*UNi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 
 TARGET_HEADER_BUILTIN(_ReadWriteBarrier, "v", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_ReadBarrier,      "v", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_WriteBarrier,     "v", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 

cfe/trunk/include/clang/Basic/BuiltinsX86_64.def

 TARGET_BUILTIN(__builtin_ia32_vcvttsd2si64, "LLiV2dIi","","avx512f")
 TARGET_BUILTIN(__builtin_ia32_vcvttsd2usi64, "ULLiV2dIi","","avx512f")
 TARGET_BUILTIN(__builtin_ia32_vcvttss2si64, "LLiV4fIi","","avx512f")
 TARGET_BUILTIN(__builtin_ia32_vcvttss2usi64, "ULLiV4fIi","","avx512f")
 TARGET_BUILTIN(__builtin_ia32_cvtsi2sd64, "V2dV2dLLiIi","","avx512f")
 TARGET_BUILTIN(__builtin_ia32_cvtsi2ss64, "V4fV4fLLiIi","","avx512f")
 TARGET_BUILTIN(__builtin_ia32_cvtusi2sd64, "V2dV2dULLiIi","","avx512f")
 TARGET_BUILTIN(__builtin_ia32_cvtusi2ss64, "V4fV4fULLiIi","","avx512f")
+TARGET_BUILTIN(__builtin_ia32_directstore_u64, "vULi*ULi", "", "movdiri")
 
 #undef BUILTIN
 #undef TARGET_BUILTIN
 #undef TARGET_HEADER_BUILTIN

cfe/trunk/include/clang/Driver/Options.td

 def mgfni : Flag<["-"], "mgfni">, Group<m_x86_Features_Group>;
 def mno_gfni : Flag<["-"], "mno-gfni">, Group<m_x86_Features_Group>;
 def mlwp : Flag<["-"], "mlwp">, Group<m_x86_Features_Group>;
 def mno_lwp : Flag<["-"], "mno-lwp">, Group<m_x86_Features_Group>;
 def mlzcnt : Flag<["-"], "mlzcnt">, Group<m_x86_Features_Group>;
 def mno_lzcnt : Flag<["-"], "mno-lzcnt">, Group<m_x86_Features_Group>;
 def mmovbe : Flag<["-"], "mmovbe">, Group<m_x86_Features_Group>;
 def mno_movbe : Flag<["-"], "mno-movbe">, Group<m_x86_Features_Group>;
+def mmovdiri : Flag<["-"], "mmovdiri">, Group<m_x86_Features_Group>;
+def mno_movdiri : Flag<["-"], "mno-movdiri">, Group<m_x86_Features_Group>;
+def mmovdir64b : Flag<["-"], "mmovdir64b">, Group<m_x86_Features_Group>;
+def mno_movdir64b : Flag<["-"], "mno-movdir64b">, Group<m_x86_Features_Group>;
 def mmpx : Flag<["-"], "mmpx">, Group<m_x86_Features_Group>;
 def mno_mpx : Flag<["-"], "mno-mpx">, Group<m_x86_Features_Group>;
 def mmwaitx : Flag<["-"], "mmwaitx">, Group<m_x86_Features_Group>;
 def mno_mwaitx : Flag<["-"], "mno-mwaitx">, Group<m_x86_Features_Group>;
 def mpku : Flag<["-"], "mpku">, Group<m_x86_Features_Group>;
 def mno_pku : Flag<["-"], "mno-pku">, Group<m_x86_Features_Group>;
 def mpclmul : Flag<["-"], "mpclmul">, Group<m_x86_Features_Group>;
 def mno_pclmul : Flag<["-"], "mno-pclmul">, Group<m_x86_Features_Group>;

cfe/trunk/lib/Basic/Targets/X86.h

class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
   bool HasMOVBE = false;
   bool HasPREFETCHWT1 = false;
   bool HasRDPID = false;
   bool HasRetpoline = false;
   bool HasRetpolineExternalThunk = false;
   bool HasLAHFSAHF = false;
   bool HasWBNOINVD = false;
   bool HasWAITPKG = false;
+  bool HasMOVDIRI = false;
+  bool HasMOVDIR64B = false;
 
 protected:
   /// \brief Enumeration of all of the X86 CPUs supported by Clang.
   ///
   /// Each enumeration represents a particular CPU supported by Clang. These
   /// loosely correspond to the options passed to '-march' or '-mtune' flags.
   enum CPUKind {
     CK_Generic,

cfe/trunk/lib/Basic/Targets/X86.cpp

bool X86TargetInfo::initFeatureMap(
   case CK_Pentium3:
   case CK_C3_2:
     setFeatureEnabledImpl(Features, "sse", true);
     setFeatureEnabledImpl(Features, "fxsr", true);
     break;
 
   case CK_Tremont:
     setFeatureEnabledImpl(Features, "cldemote", true);
+    setFeatureEnabledImpl(Features, "movdiri", true);
+    setFeatureEnabledImpl(Features, "movdir64b", true);
     setFeatureEnabledImpl(Features, "gfni", true);
     setFeatureEnabledImpl(Features, "waitpkg", true);
     LLVM_FALLTHROUGH;
   case CK_GoldmontPlus:
     setFeatureEnabledImpl(Features, "rdpid", true);
     setFeatureEnabledImpl(Features, "sgx", true);
     LLVM_FALLTHROUGH;
   case CK_Goldmont:
for (const auto &Feature : Features) {
     } else if (Feature == "+retpoline") {
       HasRetpoline = true;
     } else if (Feature == "+retpoline-external-thunk") {
       HasRetpolineExternalThunk = true;
     } else if (Feature == "+sahf") {
       HasLAHFSAHF = true;
     } else if (Feature == "+waitpkg") {
       HasWAITPKG = true;
+    } else if (Feature == "+movdiri") {
+      HasMOVDIRI = true;
+    } else if (Feature == "+movdir64b") {
+      HasMOVDIR64B = true;
     }
 
     X86SSEEnum Level = llvm::StringSwitch<X86SSEEnum>(Feature)
                            .Case("+avx512f", AVX512F)
                            .Case("+avx2", AVX2)
                            .Case("+avx", AVX)
                            .Case("+sse4.2", SSE42)
                            .Case("+sse4.1", SSE41)
void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
   if (HasCLZERO)
     Builder.defineMacro("__CLZERO__");
   if (HasRDPID)
     Builder.defineMacro("__RDPID__");
   if (HasCLDEMOTE)
     Builder.defineMacro("__CLDEMOTE__");
   if (HasWAITPKG)
     Builder.defineMacro("__WAITPKG__");
+  if (HasMOVDIRI)
+    Builder.defineMacro("__MOVDIRI__");
+  if (HasMOVDIR64B)
+    Builder.defineMacro("__MOVDIR64B__");
 
   // Each case falls through to the previous one here.
   switch (SSELevel) {
   case AVX512F:
     Builder.defineMacro("__AVX512F__");
     LLVM_FALLTHROUGH;
   case AVX2:
     Builder.defineMacro("__AVX2__");
return llvm::StringSwitch<bool>(Name)
       .Case("fma4", true)
       .Case("fsgsbase", true)
       .Case("fxsr", true)
       .Case("gfni", true)
       .Case("lwp", true)
       .Case("lzcnt", true)
       .Case("mmx", true)
       .Case("movbe", true)
+      .Case("movdiri", true)
+      .Case("movdir64b", true)
       .Case("mpx", true)
       .Case("mwaitx", true)
       .Case("pclmul", true)
       .Case("pku", true)
       .Case("popcnt", true)
       .Case("prefetchwt1", true)
       .Case("prfchw", true)
       .Case("rdpid", true)
return llvm::StringSwitch<bool>(Feature)
       .Case("gfni", HasGFNI)
       .Case("ibt", HasIBT)
       .Case("lwp", HasLWP)
       .Case("lzcnt", HasLZCNT)
       .Case("mm3dnow", MMX3DNowLevel >= AMD3DNow)
       .Case("mm3dnowa", MMX3DNowLevel >= AMD3DNowAthlon)
       .Case("mmx", MMX3DNowLevel >= MMX)
       .Case("movbe", HasMOVBE)
+      .Case("movdiri", HasMOVDIRI)
+      .Case("movdir64b", HasMOVDIR64B)
       .Case("mpx", HasMPX)
       .Case("mwaitx", HasMWAITX)
       .Case("pclmul", HasPCLMUL)
       .Case("pku", HasPKU)
       .Case("popcnt", HasPOPCNT)
       .Case("prefetchwt1", HasPREFETCHWT1)
       .Case("prfchw", HasPRFCHW)
       .Case("rdpid", HasRDPID)

cfe/trunk/lib/Headers/CMakeLists.txt

set(files
   iso646.h
   limits.h
   lwpintrin.h
   lzcntintrin.h
   mm3dnow.h
   mmintrin.h
   mm_malloc.h
   module.modulemap
+  movdirintrin.h
   msa.h
   mwaitxintrin.h
   nmmintrin.h
   opencl-c.h
   pkuintrin.h
   pmmintrin.h
   popcntintrin.h
   prfchwintrin.h

cfe/trunk/lib/Headers/cpuid.h

 #define bit_GFNI             0x00000100
 #define bit_VAES             0x00000200
 #define bit_VPCLMULQDQ       0x00000400
 #define bit_AVX512VNNI       0x00000800
 #define bit_AVX512BITALG     0x00001000
 #define bit_AVX512VPOPCNTDQ  0x00004000
 #define bit_RDPID            0x00400000
 #define bit_CLDEMOTE         0x02000000
+#define bit_MOVDIRI          0x08000000
+#define bit_MOVDIR64B        0x10000000
 
 /* Features in %edx for leaf 7 sub-leaf 0 */
 #define bit_AVX5124VNNIW  0x00000004
 #define bit_AVX5124FMAPS  0x00000008
 #define bit_IBT           0x00100000
 
 /* Features in %eax for leaf 13 sub-leaf 1 */
 #define bit_XSAVEOPT    0x00000001

cfe/trunk/lib/Headers/module.modulemap

explicit module intel {
     textual header "lzcntintrin.h"
     textual header "xopintrin.h"
     textual header "fma4intrin.h"
     textual header "mwaitxintrin.h"
     textual header "clzerointrin.h"
     textual header "wbnoinvdintrin.h"
     textual header "cldemoteintrin.h"
     textual header "waitpkgintrin.h"
+    textual header "movdirintrin.h"
 
     explicit module mm_malloc {
       requires !freestanding
       header "mm_malloc.h"
       export * // note: for <stdlib.h> dependency
     }
 
     explicit module cpuid {

cfe/trunk/lib/Headers/movdirintrin.h

+/*===------------------------- movdirintrin.h ------------------------------===
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ *===-----------------------------------------------------------------------===
+ */
+#ifndef __X86INTRIN_H
+#error "Never use <movdirintrin.h> directly; include <x86intrin.h> instead."
+#endif
+
+#ifndef _MOVDIRINTRIN_H
+#define _MOVDIRINTRIN_H
+
+// Move doubleword as direct store
+static __inline__ void
+__attribute__((__always_inline__, __nodebug__,  __target__("movdiri")))
+_directstoreu_u32 (void *__dst, unsigned int  __value)
+{
+  __builtin_ia32_directstore_u32((unsigned int *)__dst, (unsigned int)__value);
+}
+
+#ifdef __x86_64__
+
+// Move quadword as direct store
+static __inline__ void
+__attribute__((__always_inline__, __nodebug__,  __target__("movdiri")))
+_directstoreu_u64 (void *__dst, unsigned long __value)
+{
+  __builtin_ia32_directstore_u64((unsigned long *)__dst, __value);
+}
+
+#endif /* __x86_64__ */
+
+// Move 64 bytes as direct store
+static __inline__ void
+__attribute__((__always_inline__, __nodebug__,  __target__("movdir64b")))
+_movdir64b (void *__dst, const void *__src)
+{
+  __builtin_ia32_movdir64b(__dst, __src);
+}
+
+#endif /* _MOVDIRINTRIN_H */

cfe/trunk/lib/Headers/x86intrin.h

 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__CLDEMOTE__)
 #include <cldemoteintrin.h>
 #endif
 
 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__WAITPKG__)
 #include <waitpkgintrin.h>
 #endif
 
+#if !defined(_MSC_VER) || __has_feature(modules) || \
+  defined(__MOVDIRI__) || defined(__MOVDIR64B__)
+#include <movdirintrin.h>
+#endif
+
 #endif /* __X86INTRIN_H */

cfe/trunk/test/CodeGen/builtin-movdir.c

+// RUN: %clang_cc1 -ffreestanding -Wall -pedantic -triple x86_64-unknown-unknown -target-feature +movdiri -target-feature +movdir64b %s -emit-llvm -o - | FileCheck %s --check-prefix=X86_64 --check-prefix=CHECK
+// RUN: %clang_cc1 -ffreestanding -Wall -pedantic -triple i386-unknown-unknown -target-feature +movdiri -target-feature +movdir64b %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK
+
+#include <x86intrin.h>
+#include <stdint.h>
+
+void test_directstore32(void *dst, uint32_t value) {
+  // CHECK-LABEL: test_directstore32
+  // CHECK: call void @llvm.x86.directstore32
+  _directstoreu_u32(dst, value);
+}
+
+#ifdef __x86_64__
+
+void test_directstore64(void *dst, uint64_t value) {
+  // X86_64-LABEL: test_directstore64
+  // X86_64: call void @llvm.x86.directstore64
+  _directstoreu_u64(dst, value);
+}
+
+#endif
+
+void test_dir64b(void *dst, const void *src) {
+  // CHECK-LABEL: test_dir64b
+  // CHECK: call void @llvm.x86.movdir64b
+  _movdir64b(dst, src);
+}
+
+// CHECK: declare void @llvm.x86.directstore32(i8*, i32)
+// X86_64: declare void @llvm.x86.directstore64(i8*, i64)
+// CHECK: declare void @llvm.x86.movdir64b(i8*, i8*)

cfe/trunk/test/Driver/x86-target-features.c

 // RUN: %clang -target i386-linux-gnu -mretpoline -mno-retpoline-external-thunk %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-RETPOLINE-EXTERNAL-THUNK %s
 // RETPOLINE-EXTERNAL-THUNK: "-target-feature" "+retpoline-external-thunk"
 // NO-RETPOLINE-EXTERNAL-THUNK: "-target-feature" "-retpoline-external-thunk"
 
 // RUN: %clang -target i386-linux-gnu -mwaitpkg %s -### -o %t.o 2>&1 | FileCheck -check-prefix=WAITPKG %s
 // RUN: %clang -target i386-linux-gnu -mno-waitpkg %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-WAITPKG %s
 // WAITPKG: "-target-feature" "+waitpkg"
 // NO-WAITPKG: "-target-feature" "-waitpkg"
+
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mmovdiri %s -### -o %t.o 2>&1 | FileCheck -check-prefix=MOVDIRI %s
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mno-movdiri %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-MOVDIRI %s
+// MOVDIRI: "-target-feature" "+movdiri"
+// NO-MOVDIRI: "-target-feature" "-movdiri"
+
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mmovdir64b %s -### -o %t.o 2>&1 | FileCheck -check-prefix=MOVDIR64B %s
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mno-movdir64b %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-MOVDIR64B %s
+// MOVDIR64B: "-target-feature" "+movdir64b"
+// NO-MOVDIR64B: "-target-feature" "-movdir64b"

cfe/trunk/test/Preprocessor/predefined-arch-macros.c

 // RUN:   | FileCheck %s -check-prefix=CHECK_TRM_M32
 // CHECK_TRM_M32: #define __AES__ 1
 // CHECK_TRM_M32: #define __CLDEMOTE__ 1
 // CHECK_TRM_M32: #define __CLFLUSHOPT__ 1
 // CHECK_TRM_M32: #define __FSGSBASE__ 1
 // CHECK_TRM_M32: #define __FXSR__ 1
 // CHECK_TRM_M32: #define __GFNI__ 1
 // CHECK_TRM_M32: #define __MMX__ 1
+// CHECK_TRM_M32: #define __MOVDIR64B__ 1
+// CHECK_TRM_M32: #define __MOVDIRI__ 1
 // CHECK_TRM_M32: #define __MPX__ 1
 // CHECK_TRM_M32: #define __PCLMUL__ 1
 // CHECK_TRM_M32: #define __POPCNT__ 1
 // CHECK_TRM_M32: #define __PRFCHW__ 1
 // CHECK_TRM_M32: #define __RDPID__ 1
 // CHECK_TRM_M32: #define __RDRND__ 1
 // CHECK_TRM_M32: #define __RDSEED__ 1
 // CHECK_TRM_M32: #define __SGX__ 1

 // RUN:   | FileCheck %s -check-prefix=CHECK_TRM_M64
 // CHECK_TRM_M64: #define __AES__ 1
 // CHECK_TRM_M64: #define __CLDEMOTE__ 1
 // CHECK_TRM_M64: #define __CLFLUSHOPT__ 1
 // CHECK_TRM_M64: #define __FSGSBASE__ 1
 // CHECK_TRM_M64: #define __FXSR__ 1
 // CHECK_TRM_M64: #define __GFNI__ 1
 // CHECK_TRM_M64: #define __MMX__ 1
+// CHECK_TRM_M64: #define __MOVDIR64B__ 1
+// CHECK_TRM_M64: #define __MOVDIRI__ 1
 // CHECK_TRM_M64: #define __MPX__ 1
 // CHECK_TRM_M64: #define __PCLMUL__ 1
 // CHECK_TRM_M64: #define __POPCNT__ 1
 // CHECK_TRM_M64: #define __PRFCHW__ 1
 // CHECK_TRM_M64: #define __RDPID__ 1
 // CHECK_TRM_M64: #define __RDRND__ 1
 // CHECK_TRM_M64: #define __RDSEED__ 1
 // CHECK_TRM_M64: #define __SGX__ 1