[libc++][modules] Generates std.cppm.in.

[Mordante]

Summary

This takes the header restrictions into account instead of manually
duplicating this build information. This is a preparation to properly
support the libc++ disabled parts in the std module.

[Mordante]

Reworked the patch based on upstream changes and add to patch stack.

[ldionne]

libcxx/modules/std.cppm.in

1

Commit typo: instead of manually duplication => instead of manually duplicating

libcxx/utils/libcxx/header_information.py

153–156

195

200

libcxx/modules/std.cppm.in

 // -*- C++ -*-
 //===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
+// WARNING, this entire header is generated by
+// utils/generate_std_cppm_in.py
+// DO NOT MODIFY!
+
 module;
 
 #include <__config>
 
-// TODO MODULES This could be generated
-
 // The headers of Table 24: C++ library headers [tab:headers.cpp]
 // and the headers of Table 25: C++ headers for C library facilities [tab:headers.cpp.c]
 #include <algorithm>
 #include <any>
 #include <array>
-#include <atomic>
-#include <barrier>
 #include <bit>
 #include <bitset>
 #include <cassert>
 #include <cctype>
 #include <cerrno>
 #include <cfenv>
 #include <cfloat>
 #include <charconv>

 #include <deque>
 #include <exception>
 #include <execution>
 #include <expected>
 #include <filesystem>
 #include <format>
 #include <forward_list>
 #include <functional>
-#include <future>
 #include <initializer_list>
 #include <iosfwd>
 #include <iterator>
-#include <latch>
 #include <limits>
 #include <list>
 #include <map>
 #include <mdspan>
 #include <memory>
 #include <memory_resource>
 #include <mutex>
 #include <new>
 #include <numbers>
 #include <numeric>
 #include <optional>
 #include <print>
 #include <queue>
 #include <random>
 #include <ranges>
 #include <ratio>
 #include <scoped_allocator>
-#include <semaphore>
 #include <set>
-#include <shared_mutex>
 #include <source_location>
 #include <span>
 #include <stack>
 #include <stdexcept>
-#include <stop_token>
 #include <string>
 #include <string_view>
 #include <system_error>
-#include <thread>
 #include <tuple>
 #include <type_traits>
 #include <typeindex>
 #include <typeinfo>
 #include <unordered_map>
 #include <unordered_set>
 #include <utility>
 #include <valarray>
 #include <variant>
 #include <vector>
 #include <version>
 
 // *** Headers disabled by a feature ***
 
+#ifndef _LIBCPP_HAS_NO_ATOMIC_HEADER
+#  include <atomic>
+#endif // _LIBCPP_HAS_NO_ATOMIC_HEADER
+
 #ifndef _LIBCPP_HAS_NO_LOCALIZATION
 #  include <clocale>
 #  include <codecvt>
 #  include <fstream>
 #  include <iomanip>
 #  include <ios>
 #  include <iostream>
 #  include <istream>
 #  include <locale>
 #  include <ostream>
 #  include <regex>
 #  include <sstream>
 #  include <streambuf>
 #  include <strstream>
 #endif // _LIBCPP_HAS_NO_LOCALIZATION
+
+#ifndef _LIBCPP_HAS_NO_THREADS
+#  include <barrier>
+#  include <future>
+#  include <latch>
+#  include <semaphore>
+#  include <shared_mutex>
+#  include <stop_token>
+#  include <thread>
+#endif // _LIBCPP_HAS_NO_THREADS
+
 #ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
 #  include <cwchar>
 #  include <cwctype>
 #endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
 
 // *** Headers not yet available ***
-#if __has_include(<generator>)
-#  error "include <generator> unconditionally"
-#  include <generator>
-#endif //   __has_include(<generator>)
-#if __has_include(<hazard_pointer>)
-#  error "include <hazard_pointer> unconditionally"
-#  include <hazard_pointer>
-#endif //   __has_include(<hazard_pointer>)
-#if __has_include(<flat_map>)
-#  error "include <flat_map> unconditionally"
-#  include <flat_map>
-#endif // __has_include(<flat_map>)
-#if __has_include(<flat_set>)
-#  error "include <flat_set> unconditionally"
-#  include <flat_set>
-#endif //  __has_include(<flat_set>)
-#if __has_include(<rcu>)
-#  error "include <rcu> unconditionally"
-#  include <rcu>
-#endif //   __has_include(<rcu>)
-#if __has_include(<spanstream>)
-#  error "include <spanstream> unconditionally"
-#  include <spanstream>
-#endif //   __has_include(<spanstream>)
-#if __has_include(<stacktrace>)
-#  error "include <stacktrace> unconditionally"
-#  include <stacktrace>
-#endif //   __has_include(<stacktrace>)
-#if __has_include(<stdfloat>)
-#  error "include <stdfloat> unconditionally"
-#  include <stdfloat>
-#endif //   __has_include(<stdfloat>)
-#if __has_include(<syncstream>)
-#  error "include <syncstream> unconditionally"
-#  include <syncstream>
-#endif //   __has_include(<syncstream>)
-#if __has_include(<text_encoding>)
-#  error "include <text_encoding> unconditionally"
-#  include <text_encoding>
-#endif //   __has_include(<text_encoding>)
+# if __has_include(<flat_map>)
+#  error "update the header information for <flat_map> in libcxx/utils/generate_std_cppm_in.py"
+# endif //   __has_include(<flat_map>)
+# if __has_include(<flat_set>)
+#  error "update the header information for <flat_set> in libcxx/utils/generate_std_cppm_in.py"
+# endif //   __has_include(<flat_set>)
+# if __has_include(<generator>)
+#  error "update the header information for <generator> in libcxx/utils/generate_std_cppm_in.py"
+# endif //   __has_include(<generator>)
+# if __has_include(<hazard_pointer>)
+#  error "update the header information for <hazard_pointer> in libcxx/utils/generate_std_cppm_in.py"
+# endif //   __has_include(<hazard_pointer>)
+# if __has_include(<rcu>)
+#  error "update the header information for <rcu> in libcxx/utils/generate_std_cppm_in.py"
+# endif //   __has_include(<rcu>)
+# if __has_include(<spanstream>)
+#  error "update the header information for <spanstream> in libcxx/utils/generate_std_cppm_in.py"
+# endif //   __has_include(<spanstream>)
+# if __has_include(<stacktrace>)
+#  error "update the header information for <stacktrace> in libcxx/utils/generate_std_cppm_in.py"
+# endif //   __has_include(<stacktrace>)
+# if __has_include(<stdfloat>)
+#  error "update the header information for <stdfloat> in libcxx/utils/generate_std_cppm_in.py"
+# endif //   __has_include(<stdfloat>)
+# if __has_include(<syncstream>)
+#  error "update the header information for <syncstream> in libcxx/utils/generate_std_cppm_in.py"
+# endif //   __has_include(<syncstream>)
+# if __has_include(<text_encoding>)
+#  error "update the header information for <text_encoding> in libcxx/utils/generate_std_cppm_in.py"
+# endif //   __has_include(<text_encoding>)
 
 export module std;
 
 @LIBCXX_MODULE_STD_INCLUDE_SOURCES@

libcxx/test/libcxx/module_std.gen.py

 # to be one monolitic test. Since the test doesn't take very long it's
 # not a huge issue.
 
 # RUN: %{python} %s %{libcxx}/utils
 
 import sys
 
 sys.path.append(sys.argv[1])
-from libcxx.header_information import toplevel_headers
+from libcxx.header_information import module_headers
 
 BLOCKLIT = (
     ""  # block Lit from interpreting a RUN/XFAIL/etc inside the generation script
 )
 
-### Remove the headers that have no module associated with them
-
-# Note all C-headers using .h are filtered in the loop.
-
-# These headers are not available in C++23, but in older language Standards.
-toplevel_headers.remove("ccomplex")
-toplevel_headers.remove("ciso646")
-toplevel_headers.remove("cstdbool")
-toplevel_headers.remove("ctgmath")
-
 # Ignore several declarations found in the includes.
 #
 # Part of these items are bugs other are not yet implemented features.
 SkipDeclarations = dict()
 
 # See comment in the header.
 SkipDeclarations["cuchar"] = ["std::mbstate_t", "std::size_t"]
 

 // The GCC compiler flags are not always compatible with clang-tidy.
 // UNSUPPORTED{BLOCKLIT}: gcc
 
 // RUN{BLOCKLIT}: echo -n > %t.all_partitions
 """
 )
 
 # Validate all module parts.
-for header in toplevel_headers:
-    if header.endswith(".h"):  # Skip C compatibility headers
-        continue
-
+for header in module_headers:
     # Generate a module partition for the header module includes. This
     # makes it possible to verify that all headers export all their
     # named declarations.
     #
     # TODO MODULES This needs to take the header restrictions into account.
     print(
         f"// RUN{BLOCKLIT}: echo -e \""
         f"module;{nl}"

libcxx/utils/CMakeLists.txt

 add_custom_target(libcxx-generate-feature-test-macros
     COMMAND "${Python3_EXECUTABLE}" "${LIBCXX_SOURCE_DIR}/utils/generate_feature_test_macro_components.py"
     COMMENT "Generate the <version> header and tests for feature test macros.")
 
 add_custom_target(libcxx-generate-std-clang-module-header
   COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/generate_std_clang_module_header.py"
   COMMENT "Generate the <__std_clang_module> header")
 
+add_custom_target(libcxx-generate-std-cppm-in-file
+  COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/generate_std_cppm_in.py"
+  COMMENT "Generate the std.cppm.in file")
+
 add_custom_target(libcxx-generate-extended-grapheme-cluster-tables
     COMMAND
         "${Python3_EXECUTABLE}"
         "${LIBCXX_SOURCE_DIR}/utils/generate_extended_grapheme_cluster_table.py"
         "${LIBCXX_SOURCE_DIR}/include/__format/extended_grapheme_cluster_table.h"
     COMMENT "Generate the extended grapheme cluster header.")
 
 add_custom_target(libcxx-generate-extended-grapheme-cluster-tests
add_custom_target(libcxx-generate-iwyu-mapping
     COMMAND
         "${Python3_EXECUTABLE}"
         "${LIBCXX_SOURCE_DIR}/utils/generate_iwyu_mapping.py"
     COMMENT "Generate the mapping file for include-what-you-use")
 
 add_custom_target(libcxx-generate-files
     DEPENDS libcxx-generate-feature-test-macros
             libcxx-generate-std-clang-module-header
+            libcxx-generate-std-cppm-in-file
             libcxx-generate-extended-grapheme-cluster-tables
             libcxx-generate-extended-grapheme-cluster-tests
             libcxx-generate-escaped-output-table
             libcxx-generate-width-estimation-table
             libcxx-generate-iwyu-mapping
     COMMENT "Create all the auto-generated files in libc++ and its tests.")

libcxx/utils/generate_std_cppm_in.py

+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+import operator
+import os.path
+
+import libcxx.header_information
+
+
+header_include_requirements = libcxx.header_information.header_include_requirements
+
+
+always_available_headers = frozenset(
+    libcxx.header_information.module_headers
+).difference(*header_include_requirements.values())
+
+libcxx_module_directory = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "modules"
+)
+with open(
+    os.path.join(libcxx_module_directory, "std.cppm.in"), "w"
+) as std_module_cpp_in:
+    std_module_cpp_in.write(
+        """\
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// WARNING, this entire header is generated by
+// utils/generate_std_cppm_in.py
+// DO NOT MODIFY!
+
+module;
+
+#include <__config>
+
+// The headers of Table 24: C++ library headers [tab:headers.cpp]
+// and the headers of Table 25: C++ headers for C library facilities [tab:headers.cpp.c]
+"""
+    )
+    # Include the angle brackets in sorting so that <a.h> sorts before <a>
+    # like check-format wants.
+    for include in sorted([f"<{header}>" for header in always_available_headers]):
+        std_module_cpp_in.write(f"#include {include}\n")
+
+    std_module_cpp_in.write("\n// *** Headers disabled by a feature ***\n")
+    for requirements, headers in sorted(
+        header_include_requirements.items(), key=operator.itemgetter(0)
+    ):
+        std_module_cpp_in.write("\n")
+        if len(requirements) == 1:
+            std_module_cpp_in.write(f"#ifndef {requirements[0]}")
+        else:
+            std_module_cpp_in.write("#if")
+            for index, requirement in enumerate(requirements):
+                if index > 0:
+                    std_module_cpp_in.write(" &&")
+                std_module_cpp_in.write(f" !defined({requirement})")
+        std_module_cpp_in.write("\n")
+
+        for include in sorted([f"<{header}>" for header in headers]):
+            if include.endswith(".h>"):  # Skip C compatibility headers
+                continue
+            if (
+                include == "<experimental/regex>"
+            ):  # TODO MODULES remove when header this header is removed from libc++
+                continue
+            std_module_cpp_in.write(f"#  include {include}\n")
+
+        std_module_cpp_in.write(f"#endif // {requirements[0]}\n")
+
+    std_module_cpp_in.write("\n// *** Headers not yet available ***\n")
+    for include in sorted(
+        [f"<{header}>" for header in libcxx.header_information.headers_not_available]
+    ):
+        std_module_cpp_in.write(
+            f"""\
+# if __has_include({include})
+#  error "update the header information for {include} in libcxx/utils/generate_std_cppm_in.py"
+# endif //   __has_include({include})
+"""
+        )
+
+    std_module_cpp_in.write(
+        """
+export module std;
+
+@LIBCXX_MODULE_STD_INCLUDE_SOURCES@
+"""
+    )

libcxx/utils/libcxx/header_information.py

mandatory_inclusions = {
     "unordered_map": ["compare", "initializer_list"],
     "unordered_set": ["compare", "initializer_list"],
     "utility": ["compare", "initializer_list"],
     "valarray": ["initializer_list"],
     "variant": ["compare"],
     "vector": ["compare", "initializer_list"],
 }
 
+
+# These headers are not yet implemented in libc++
+#
+# These headers are required by the latest (draft) Standard but have not been
+# implemented yet. They are used in the generated module input. When the header
+# is added and this list is not updated the C++23 standard modules will fail to
+# build.
+headers_not_available = [
+    "flat_map",
+    "flat_set",
+    "generator",
+    "hazard_pointer",
+    "rcu",
+    "spanstream",
+    "stacktrace",
+    "stdfloat",
+    "syncstream",
+    "text_encoding",
+]
+
+
 def is_header(file):
     """Returns whether the given file is a header (i.e. not a directory or the modulemap file)."""
     return (
         not file.is_dir()
         and not file.name == "module.modulemap.in"
         and not file.name == "CMakeLists.txt"
         and file.name != "libcxx.imp"
     )
 
 libcxx_root = pathlib.Path(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
 include = pathlib.Path(os.path.join(libcxx_root, "include"))
 test = pathlib.Path(os.path.join(libcxx_root, "test"))
 assert libcxx_root.exists()
 
 toplevel_headers = sorted(
     p.relative_to(include).as_posix() for p in include.glob("[a-z]*") if is_header(p)
 )
 experimental_headers = sorted(
-    p.relative_to(include).as_posix() for p in include.glob("experimental/[a-z]*") if is_header(p)
+    p.relative_to(include).as_posix()
+    for p in include.glob("experimental/[a-z]*")
+    if is_header(p)
 )
 public_headers = toplevel_headers + experimental_headers
+
+# The headers used in the std and std.compat modules.
+module_headers = [
+    header
+    for header in toplevel_headers
+    if not header.endswith(".h")
+    # These headers have been removed in C++20 so never are part of a module.
+    and not header in ["ccomplex", "ciso646", "cstdbool", "ctgmath"]
+]