[libc++][modules] Generates std.cppm.in.

[Mordante]

Summary

This takes the header restrictions into account instead of manually
duplicating this build information. This is a preparation to properly
support the libc++ disabled parts in the std module.

[Mordante]

Reworked the patch based on upstream changes and add to patch stack.

[ldionne]

libcxx/modules/std.cppm.in

1

Commit typo: instead of manually duplication => instead of manually duplicating

libcxx/utils/libcxx/header_information.py

147–150

188

193

libcxx/modules/std.cppm.in

 // -*- C++ -*-
 //===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
+// WARNING, this entire header is generated by
+// utils/generate_std_cppm_in.py
+// DO NOT MODIFY!
+
 module;
 
 #include <__config>
 
-// TODO MODULES This could be generated
-
 // The headers of Table 24: C++ library headers [tab:headers.cpp]
 // and the headers of Table 25: C++ headers for C library facilities [tab:headers.cpp.c]
 #include <algorithm>
 #include <any>
 #include <array>
-#include <atomic>
-#include <barrier>
+#if !defined(_LIBCPP_HAS_NO_ATOMIC_HEADER)
+#  include <atomic>
+#endif
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+#  include <barrier>
+#endif
 #include <bit>
 #include <bitset>
 #include <cassert>
 #include <cctype>
 #include <cerrno>
 #include <cfenv>
 #include <cfloat>
 #include <charconv>
 #include <chrono>
 #include <cinttypes>
 #include <climits>
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <clocale>
+#endif
 #include <cmath>
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <codecvt>
+#endif
 #include <compare>
 #include <complex>
 #include <concepts>
 #include <condition_variable>
 #include <coroutine>
 #include <csetjmp>
 #include <csignal>
 #include <cstdarg>
 #include <cstddef>
 #include <cstdint>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
 #include <ctime>
 #include <cuchar>
+#if !defined(_LIBCPP_HAS_NO_WIDE_CHARACTERS)
+#  include <cwchar>
+#endif
+#if !defined(_LIBCPP_HAS_NO_WIDE_CHARACTERS)
+#  include <cwctype>
+#endif
 #include <deque>
 #include <exception>
 #include <execution>
 #include <expected>
 #include <filesystem>
 #include <format>
 #include <forward_list>
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <fstream>
+#endif
 #include <functional>
-#include <future>
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+#  include <future>
+#endif
 #include <initializer_list>
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <iomanip>
+#endif
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <ios>
+#endif
 #include <iosfwd>
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <iostream>
+#endif
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <istream>
+#endif
 #include <iterator>
-#include <latch>
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+#  include <latch>
+#endif
 #include <limits>
 #include <list>
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <locale>
+#endif
 #include <map>
 #include <mdspan>
 #include <memory>
 #include <memory_resource>
 #include <mutex>
 #include <new>
 #include <numbers>
 #include <numeric>
 #include <optional>
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <ostream>
+#endif
 #include <print>
 #include <queue>
 #include <random>
 #include <ranges>
 #include <ratio>
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <regex>
+#endif
 #include <scoped_allocator>
-#include <semaphore>
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+#  include <semaphore>
+#endif
 #include <set>
-#include <shared_mutex>
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+#  include <shared_mutex>
+#endif
 #include <source_location>
 #include <span>
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <sstream>
+#endif
 #include <stack>
 #include <stdexcept>
-#include <stop_token>
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+#  include <stop_token>
+#endif
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <streambuf>
+#endif
 #include <string>
 #include <string_view>
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  include <strstream>
+#endif
 #include <system_error>
-#include <thread>
+#if !defined(_LIBCPP_HAS_NO_THREADS)
+#  include <thread>
+#endif
 #include <tuple>
 #include <type_traits>
 #include <typeindex>
 #include <typeinfo>
 #include <unordered_map>
 #include <unordered_set>
 #include <utility>
 #include <valarray>
 #include <variant>
 #include <vector>
 #include <version>
 
-// *** Headers disabled by a feature ***
-
-#ifndef _LIBCPP_HAS_NO_LOCALIZATION
-#  include <clocale>
-#  include <codecvt>
-#  include <fstream>
-#  include <iomanip>
-#  include <ios>
-#  include <iostream>
-#  include <istream>
-#  include <locale>
-#  include <ostream>
-#  include <regex>
-#  include <sstream>
-#  include <streambuf>
-#  include <strstream>
-#endif // _LIBCPP_HAS_NO_LOCALIZATION
-#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
-#  include <cwchar>
-#  include <cwctype>
-#endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
-
 // *** Headers not yet available ***
-#if __has_include(<generator>)
-#  error "include <generator> unconditionally"
-#  include <generator>
-#endif //   __has_include(<generator>)
-#if __has_include(<hazard_pointer>)
-#  error "include <hazard_pointer> unconditionally"
-#  include <hazard_pointer>
-#endif //   __has_include(<hazard_pointer>)
 #if __has_include(<flat_map>)
-#  error "include <flat_map> unconditionally"
-#  include <flat_map>
-#endif // __has_include(<flat_map>)
+#  error "update the header information for <flat_map> in libcxx/utils/generate_std_cppm_in.py"
+#endif //   __has_include(<flat_map>)
 #if __has_include(<flat_set>)
-#  error "include <flat_set> unconditionally"
-#  include <flat_set>
-#endif //  __has_include(<flat_set>)
+#  error "update the header information for <flat_set> in libcxx/utils/generate_std_cppm_in.py"
+#endif //   __has_include(<flat_set>)
+#if __has_include(<generator>)
+#  error "update the header information for <generator> in libcxx/utils/generate_std_cppm_in.py"
+#endif //   __has_include(<generator>)
+#if __has_include(<hazard_pointer>)
+#  error "update the header information for <hazard_pointer> in libcxx/utils/generate_std_cppm_in.py"
+#endif //   __has_include(<hazard_pointer>)
 #if __has_include(<rcu>)
-#  error "include <rcu> unconditionally"
-#  include <rcu>
+#  error "update the header information for <rcu> in libcxx/utils/generate_std_cppm_in.py"
 #endif //   __has_include(<rcu>)
 #if __has_include(<spanstream>)
-#  error "include <spanstream> unconditionally"
-#  include <spanstream>
+#  error "update the header information for <spanstream> in libcxx/utils/generate_std_cppm_in.py"
 #endif //   __has_include(<spanstream>)
 #if __has_include(<stacktrace>)
-#  error "include <stacktrace> unconditionally"
-#  include <stacktrace>
+#  error "update the header information for <stacktrace> in libcxx/utils/generate_std_cppm_in.py"
 #endif //   __has_include(<stacktrace>)
 #if __has_include(<stdfloat>)
-#  error "include <stdfloat> unconditionally"
-#  include <stdfloat>
+#  error "update the header information for <stdfloat> in libcxx/utils/generate_std_cppm_in.py"
 #endif //   __has_include(<stdfloat>)
 #if __has_include(<syncstream>)
-#  error "include <syncstream> unconditionally"
-#  include <syncstream>
+#  error "update the header information for <syncstream> in libcxx/utils/generate_std_cppm_in.py"
 #endif //   __has_include(<syncstream>)
 #if __has_include(<text_encoding>)
-#  error "include <text_encoding> unconditionally"
-#  include <text_encoding>
+#  error "update the header information for <text_encoding> in libcxx/utils/generate_std_cppm_in.py"
 #endif //   __has_include(<text_encoding>)
 
 export module std;
 
 @LIBCXX_MODULE_STD_INCLUDE_SOURCES@

libcxx/test/libcxx/module_std.gen.py

 # to be one monolitic test. Since the test doesn't take very long it's
 # not a huge issue.
 
 # RUN: %{python} %s %{libcxx}/utils
 
 import sys
 
 sys.path.append(sys.argv[1])
-from libcxx.header_information import toplevel_headers
+from libcxx.header_information import module_headers
 
 BLOCKLIT = (
     ""  # block Lit from interpreting a RUN/XFAIL/etc inside the generation script
 )
 
-### Remove the headers that have no module associated with them
-
-# Note all C-headers using .h are filtered in the loop.
-
-# These headers are not available in C++23, but in older language Standards.
-toplevel_headers.remove("ccomplex")
-toplevel_headers.remove("ciso646")
-toplevel_headers.remove("cstdbool")
-toplevel_headers.remove("ctgmath")
-
 # Ignore several declarations found in the includes.
 #
 # Part of these items are bugs other are not yet implemented features.
 SkipDeclarations = dict()
 
 # See comment in the header.
 SkipDeclarations["cuchar"] = ["std::mbstate_t", "std::size_t"]
 

 // The GCC compiler flags are not always compatible with clang-tidy.
 // UNSUPPORTED{BLOCKLIT}: gcc
 
 // RUN{BLOCKLIT}: echo -n > %t.all_partitions
 """
 )
 
 # Validate all module parts.
-for header in toplevel_headers:
-    if header.endswith(".h"):  # Skip C compatibility headers
-        continue
-
+for header in module_headers:
     # Generate a module partition for the header module includes. This
     # makes it possible to verify that all headers export all their
     # named declarations.
     #
     # TODO MODULES This needs to take the header restrictions into account.
     print(
         f"// RUN{BLOCKLIT}: echo -e \""
         f"module;{nl}"

libcxx/utils/CMakeLists.txt

 add_custom_target(libcxx-generate-feature-test-macros
     COMMAND "${Python3_EXECUTABLE}" "${LIBCXX_SOURCE_DIR}/utils/generate_feature_test_macro_components.py"
     COMMENT "Generate the <version> header and tests for feature test macros.")
 
 add_custom_target(libcxx-generate-std-clang-module-header
   COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/generate_std_clang_module_header.py"
   COMMENT "Generate the <__std_clang_module> header")
 
+add_custom_target(libcxx-generate-std-cppm-in-file
+  COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/generate_std_cppm_in.py"
+  COMMENT "Generate the std.cppm.in file")
+
 add_custom_target(libcxx-generate-extended-grapheme-cluster-tables
     COMMAND
         "${Python3_EXECUTABLE}"
         "${LIBCXX_SOURCE_DIR}/utils/generate_extended_grapheme_cluster_table.py"
         "${LIBCXX_SOURCE_DIR}/include/__format/extended_grapheme_cluster_table.h"
     COMMENT "Generate the extended grapheme cluster header.")
 
 add_custom_target(libcxx-generate-extended-grapheme-cluster-tests
add_custom_target(libcxx-generate-iwyu-mapping
     COMMAND
         "${Python3_EXECUTABLE}"
         "${LIBCXX_SOURCE_DIR}/utils/generate_iwyu_mapping.py"
     COMMENT "Generate the mapping file for include-what-you-use")
 
 add_custom_target(libcxx-generate-files
     DEPENDS libcxx-generate-feature-test-macros
             libcxx-generate-std-clang-module-header
+            libcxx-generate-std-cppm-in-file
             libcxx-generate-extended-grapheme-cluster-tables
             libcxx-generate-extended-grapheme-cluster-tests
             libcxx-generate-escaped-output-table
             libcxx-generate-width-estimation-table
             libcxx-generate-iwyu-mapping
     COMMENT "Create all the auto-generated files in libc++ and its tests.")

libcxx/utils/generate_std_cppm_in.py

+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+import operator
+import os.path
+
+from libcxx.header_information import module_headers
+from libcxx.header_information import header_restrictions
+from libcxx.header_information import headers_not_available
+
+
+libcxx_module_directory = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "modules"
+)
+with open(
+    os.path.join(libcxx_module_directory, "std.cppm.in"), "w"
+) as std_module_cpp_in:
+    std_module_cpp_in.write(
+        """\
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// WARNING, this entire header is generated by
+// utils/generate_std_cppm_in.py
+// DO NOT MODIFY!
+
+module;
+
+#include <__config>
+
+// The headers of Table 24: C++ library headers [tab:headers.cpp]
+// and the headers of Table 25: C++ headers for C library facilities [tab:headers.cpp.c]
+"""
+    )
+    for header in module_headers:
+        if header in header_restrictions:
+            std_module_cpp_in.write(
+                f"""\
+#if {header_restrictions[header]}
+#  include <{header}>
+#endif
+"""
+            )
+        else:
+            std_module_cpp_in.write(f"#include <{header}>\n")
+
+    std_module_cpp_in.write("\n// *** Headers not yet available ***\n")
+    for header in sorted(headers_not_available):
+        std_module_cpp_in.write(
+            f"""\
+#if __has_include(<{header}>)
+#  error "update the header information for <{header}> in libcxx/utils/generate_std_cppm_in.py"
+#endif //   __has_include(<{header}>)
+"""
+        )
+
+    std_module_cpp_in.write(
+        """
+export module std;
+
+@LIBCXX_MODULE_STD_INCLUDE_SOURCES@
+"""
+    )

libcxx/utils/libcxx/header_information.py

mandatory_inclusions = {
     "unordered_map": ["compare", "initializer_list"],
     "unordered_set": ["compare", "initializer_list"],
     "utility": ["compare", "initializer_list"],
     "valarray": ["initializer_list"],
     "variant": ["compare"],
     "vector": ["compare", "initializer_list"],
 }
 
+
+# These headers are not yet implemented in libc++
+#
+# These headers are required by the latest (draft) Standard but have not been
+# implemented yet. They are used in the generated module input. The C++23 standard
+# modules will fail to build if a header is added but this list is not updated.
+headers_not_available = [
+    "flat_map",
+    "flat_set",
+    "generator",
+    "hazard_pointer",
+    "rcu",
+    "spanstream",
+    "stacktrace",
+    "stdfloat",
+    "syncstream",
+    "text_encoding",
+]
+
+
 def is_header(file):
     """Returns whether the given file is a header (i.e. not a directory or the modulemap file)."""
     return (
         not file.is_dir()
         and not file.name == "module.modulemap.in"
         and not file.name == "CMakeLists.txt"
         and file.name != "libcxx.imp"
     )
 
 libcxx_root = pathlib.Path(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
 include = pathlib.Path(os.path.join(libcxx_root, "include"))
 test = pathlib.Path(os.path.join(libcxx_root, "test"))
 assert libcxx_root.exists()
 
 toplevel_headers = sorted(
     p.relative_to(include).as_posix() for p in include.glob("[a-z]*") if is_header(p)
 )
 experimental_headers = sorted(
-    p.relative_to(include).as_posix() for p in include.glob("experimental/[a-z]*") if is_header(p)
+    p.relative_to(include).as_posix()
+    for p in include.glob("experimental/[a-z]*")
+    if is_header(p)
 )
 public_headers = toplevel_headers + experimental_headers
+
+# The headers used in the std and std.compat modules.
+#
+# This is the set of all C++23-and-later headers, excluding C compatibility headers.
+module_headers = [
+    header
+    for header in toplevel_headers
+    if not header.endswith(".h")
+    # These headers have been removed in C++20 so are never part of a module.
+    and not header in ["ccomplex", "ciso646", "cstdbool", "ctgmath"]
+]