[libc++][PSTL] Integrate the headers and add a CI job

[philnik]

Rebased

[philnik]

Try to fix CI

[philnik]

Rebased

[philnik]

• Rebased
• Try to fix CI

[philnik]

Rebased

[philnik]

Try to fix CI

[philnik]

Try to fix CI

[philnik]

Rebased

[philnik]

Rebased

[philnik]

Try to fix CIx

[ldionne]

libcxx/cmake/caches/With-pstl.cmake

2–3

These should be removed once we move to -fexperimental-library. Can you add a comment?

Actually, this whole CI job is going to go away once we move to -fexperimental-library for the PSTL.

libcxx/include/CMakeLists.txt

869

Can we rename all of those to __pstl/FOOOO immediately after this patch? I'd be fine with doing it before as well, both ways work.

954

Not needed anymore.

libcxx/include/pstl/internal/pstl_config.h

13

I would remove __pstl_config_site entirely and define _PSTL_PAR_BACKEND_SERIAL temporarily, like you do right now.

libcxx/utils/libcxx/test/features.py

212

[ldionne]

Actually, LGTM w/ comments addressed.

[ldionne]

libcxx/utils/libcxx/test/features.py

212

In retrospect, we probably do need to keep a with-pstl Lit feature temporarily until we enable it via -fexperimental-library. So I would keep it, but I would *also* introduce FIXME-PSTL-INTEGRATION cause those are semantically different.

[philnik]

Address comments

[philnik]

Try to fix CI

libcxx/cmake/caches/With-pstl.cmake

+set(LIBCXX_TEST_PARAMS "std=c++17" CACHE STRING "")
+set(LIBCXXABI_TEST_PARAMS "${LIBCXX_TEST_PARAMS}" CACHE STRING "")
+set(LIBCXX_ENABLE_PARALLEL_ALGORITHMS ON CACHE BOOL "")

libcxx/include/CMakeLists.txt

set(files
   __numeric/inner_product.h
   __numeric/iota.h
   __numeric/midpoint.h
   __numeric/partial_sum.h
   __numeric/reduce.h
   __numeric/transform_exclusive_scan.h
   __numeric/transform_inclusive_scan.h
   __numeric/transform_reduce.h
+  __pstl_algorithm
+  __pstl_execution
+  __pstl_memory
+  __pstl_numeric
   __random/bernoulli_distribution.h
   __random/binomial_distribution.h
   __random/cauchy_distribution.h
   __random/chi_squared_distribution.h
   __random/clamp_to_integral.h
   __random/default_random_engine.h
   __random/discard_block_engine.h
   __random/discrete_distribution.h
set(files
   memory
   memory_resource
   mutex
   new
   numbers
   numeric
   optional
   ostream
+  pstl/internal/algorithm_fwd.h
+  pstl/internal/algorithm_impl.h
+  pstl/internal/execution_defs.h
+  pstl/internal/execution_impl.h
+  pstl/internal/glue_algorithm_defs.h
+  pstl/internal/glue_algorithm_impl.h
+  pstl/internal/glue_execution_defs.h
+  pstl/internal/glue_memory_defs.h
+  pstl/internal/glue_memory_impl.h
+  pstl/internal/glue_numeric_defs.h
+  pstl/internal/glue_numeric_impl.h
+  pstl/internal/memory_impl.h
+  pstl/internal/numeric_fwd.h
+  pstl/internal/numeric_impl.h
+  pstl/internal/omp/parallel_for.h
+  pstl/internal/omp/parallel_for_each.h
+  pstl/internal/omp/parallel_invoke.h
+  pstl/internal/omp/parallel_merge.h
+  pstl/internal/omp/parallel_scan.h
+  pstl/internal/omp/parallel_stable_partial_sort.h
+  pstl/internal/omp/parallel_stable_sort.h
+  pstl/internal/omp/parallel_transform_reduce.h
+  pstl/internal/omp/parallel_transform_scan.h
+  pstl/internal/omp/util.h
+  pstl/internal/parallel_backend.h
+  pstl/internal/parallel_backend_omp.h
+  pstl/internal/parallel_backend_serial.h
+  pstl/internal/parallel_backend_tbb.h
+  pstl/internal/parallel_backend_utils.h
+  pstl/internal/parallel_impl.h
+  pstl/internal/pstl_config.h
+  pstl/internal/unseq_backend_simd.h
+  pstl/internal/utils.h
   queue
   random
   ranges
   ratio
   regex
   scoped_allocator
   semaphore
   set

 
 foreach(feature LIBCXX_ENABLE_FILESYSTEM LIBCXX_ENABLE_LOCALIZATION LIBCXX_ENABLE_FSTREAM LIBCXX_ENABLE_THREADS LIBCXX_ENABLE_WIDE_CHARACTERS)
   if (NOT ${${feature}})
     set(requires_${feature} "requires LIBCXX_CONFIGURED_WITHOUT_SUPPORT_FOR_THIS_HEADER")
   endif()
 endforeach()
 
 configure_file("__config_site.in" "${LIBCXX_GENERATED_INCLUDE_TARGET_DIR}/__config_site" @ONLY)
+configure_file("__pstl_config_site.in" "${LIBCXX_GENERATED_INCLUDE_TARGET_DIR}/__pstl_config_site" @ONLY)
 configure_file("module.modulemap.in" "${LIBCXX_GENERATED_INCLUDE_DIR}/module.modulemap" @ONLY)
 
 set(_all_includes "${LIBCXX_GENERATED_INCLUDE_TARGET_DIR}/__config_site"
                   "${LIBCXX_GENERATED_INCLUDE_DIR}/module.modulemap")
 foreach(f ${files})
   set(src "${CMAKE_CURRENT_SOURCE_DIR}/${f}")
   set(dst "${LIBCXX_GENERATED_INCLUDE_DIR}/${f}")
   add_custom_command(OUTPUT ${dst}

libcxx/include/pstl/internal/algorithm_impl.h

__brick_find_if(_RandomAccessIterator __first, _RandomAccessIterator __last, _Predicate __pred,
     typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _SizeType;
     return __unseq_backend::__simd_first(
         __first, _SizeType(0), __last - __first,
         [&__pred](_RandomAccessIterator __it, _SizeType __i) { return __pred(__it[__i]); });
 }
 
 template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
 _ForwardIterator
-__pattern_find_if(_Tag __tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
+__pattern_find_if(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
                   _Predicate __pred) noexcept
 {
     return __internal::__brick_find_if(__first, __last, __pred, typename _Tag::__is_vector{});
 }
 
 template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Predicate>
 _RandomAccessIterator
 __pattern_find_if(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,

libcxx/include/pstl/internal/pstl_config.h

 // -*- C++ -*-
 //===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef _PSTL_CONFIG_H
 #define _PSTL_CONFIG_H
 
 #include <__pstl_config_site>
 
+// TODO: Make this a proper configuration option
+#define _PSTL_PAR_BACKEND_SERIAL
+
 // The version is XYYZ, where X is major, YY is minor, and Z is patch (i.e. X.YY.Z)
 #define _PSTL_VERSION 16000
 #define _PSTL_VERSION_MAJOR (_PSTL_VERSION / 1000)
 #define _PSTL_VERSION_MINOR ((_PSTL_VERSION % 1000) / 10)
 #define _PSTL_VERSION_PATCH (_PSTL_VERSION % 10)
 
 #if !defined(_PSTL_PAR_BACKEND_SERIAL) && !defined(_PSTL_PAR_BACKEND_TBB) && !defined(_PSTL_PAR_BACKEND_OPENMP)
 #    error "A parallel backend must be specified"

libcxx/include/pstl/internal/utils.h

 // -*- C++ -*-
 //===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef _PSTL_UTILS_H
 #define _PSTL_UTILS_H
 
 #include <new>
 #include <iterator>
+#include <utility>
 
 _PSTL_HIDE_FROM_ABI_PUSH
 
 namespace __pstl
 {
 namespace __internal
 {
 
 template <typename _Fp>
 auto
 __except_handler(_Fp __f) -> decltype(__f())
 {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
     try
     {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
         return __f();
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
     }
     catch (const std::bad_alloc&)
     {
         throw; // re-throw bad_alloc according to the standard [algorithms.parallel.exceptions]
     }
     catch (...)
     {
         std::terminate(); // Good bye according to the standard [algorithms.parallel.exceptions]
     }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
 }
 
 template <typename _Fp>
 void
 __invoke_if(std::true_type, _Fp __f)
 {
     __f();
 }

libcxx/test/libcxx/clang_tidy.sh.cpp

 //===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 // REQUIRES: has-clang-tidy
 
+// XFAIL: with-pstl
+
 // The GCC compiler flags are not always compatible with clang-tidy.
 // UNSUPPORTED: gcc
 
 // TODO: run clang-tidy with modules enabled once they are supported
 // RUN: %{clang-tidy} %s --warnings-as-errors=* -header-filter=.* --checks='-*,libcpp-*' --load=%{test-tools}/clang_tidy_checks/libcxx-tidy.plugin -- %{compile_flags} -fno-modules
 // RUN: %{clang-tidy} %s --warnings-as-errors=* -header-filter=.* --config-file=%S/../../.clang-tidy -- -Wweak-vtables %{compile_flags} -fno-modules
 
 // Prevent <ext/hash_map> from generating deprecated warnings for this test.

libcxx/test/libcxx/min_max_macros.compile.pass.cpp

 //===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 // Test that headers are not tripped up by the surrounding code defining the
 // min() and max() macros.
 
+// XFAIL: with-pstl
+
 // Prevent <ext/hash_map> from generating deprecated warnings for this test.
 #if defined(__DEPRECATED)
 #    undef __DEPRECATED
 #endif
 
 #define TEST_MACROS() static_assert(min() == true && max() == true, "")
 #define min() true
 #define max() true

libcxx/test/libcxx/nasty_macros.compile.pass.cpp

 //===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 // Test that headers are not tripped up by the surrounding code defining various
 // alphabetic macros.
 
+// XFAIL: with-pstl
+
 // Prevent <ext/hash_map> from generating deprecated warnings for this test.
 #if defined(__DEPRECATED)
 #    undef __DEPRECATED
 #endif
 
 #define NASTY_MACRO This should not be expanded!!!
 
 // libc++ does not use single-letter names as a matter of principle.

libcxx/test/libcxx/transitive_includes.sh.cpp

 // properly include what they use. Note that we don't check which system (C) headers are
 // included transitively, because that is too unstable across platforms, and hence difficult
 // to test for.
 //
 // This is not meant to block libc++ from removing unused transitive includes
 // forever, however we do try to group removals for a couple of releases
 // to avoid breaking users at every release.
 
+// XFAIL: with-pstl
+
 // This test doesn't support being run when some headers are not available, since we
 // would need to add significant complexity to make that work.
 // UNSUPPORTED: no-localization, no-threads, no-wide-characters, no-filesystem, libcpp-has-no-incomplete-format
 
 // When built with modules, this test doesn't work because --trace-includes doesn't
 // report the stack of includes correctly.
 // UNSUPPORTED: modules-build
 

libcxx/utils/ci/buildkite-pipeline.yml

agents:
       queue: "libcxx-builders"
       os: "linux"
     retry:
       automatic:
         - exit_status: -1  # Agent was lost
           limit: 2
     timeout_in_minutes: 120
 
+  - label: "With PSTL support"
+    command: "libcxx/utils/ci/run-buildbot with-pstl"
+    artifact_paths:
+      - "**/test-results.xml"
+    env:
+      CC: "clang-${LLVM_HEAD_VERSION}"
+      CXX: "clang++-${LLVM_HEAD_VERSION}"
+    agents:
+      queue: "libcxx-builders"
+      os: "linux"
+    retry:
+      automatic:
+        - exit_status: -1  # Agent was lost
+          limit: 2
+    timeout_in_minutes: 120
+
   - label: "With LLVM's libunwind"
     command: "libcxx/utils/ci/run-buildbot generic-with_llvm_unwinder"
     artifact_paths:
       - "**/test-results.xml"
       - "**/*.abilist"
     env:
         CC: "clang-${LLVM_HEAD_VERSION}"
         CXX: "clang++-${LLVM_HEAD_VERSION}"

libcxx/utils/ci/run-buildbot

 #
 # Other miscellaneous jobs
 #
 generic-abi-unstable)
     clean
     generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Generic-abi-unstable.cmake"
     check-runtimes
 ;;
+with-pstl)
+    clean
+    generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/With-pstl.cmake"
+    check-runtimes
+;;
 apple-system)
     clean
 
     arch="$(uname -m)"
     xcrun --sdk macosx                                              \
         ${MONOREPO_ROOT}/libcxx/utils/ci/apple-install-libcxx.sh    \
             --llvm-root ${MONOREPO_ROOT}                            \
             --build-dir ${BUILD_DIR}                                \

libcxx/utils/libcxx/test/features.py

macros = {
   '_LIBCPP_ABI_VERSION': 'libcpp-abi-version',
   '_LIBCPP_HAS_NO_FILESYSTEM_LIBRARY': 'no-filesystem',
   '_LIBCPP_HAS_NO_RANDOM_DEVICE': 'no-random-device',
   '_LIBCPP_HAS_NO_LOCALIZATION': 'no-localization',
   '_LIBCPP_HAS_NO_FSTREAM': 'no-fstream',
   '_LIBCPP_HAS_NO_WIDE_CHARACTERS': 'no-wide-characters',
   '_LIBCPP_HAS_NO_UNICODE': 'libcpp-has-no-unicode',
   '_LIBCPP_ENABLE_DEBUG_MODE': 'libcpp-has-debug-mode',
+  '_LIBCPP_HAS_PARALLEL_ALGORITHMS': 'with-pstl',
 }
 for macro, feature in macros.items():
   DEFAULT_FEATURES.append(
     Feature(name=lambda cfg, m=macro, f=feature: f + ('={}'.format(compilerMacros(cfg)[m]) if compilerMacros(cfg)[m] else ''),
             when=lambda cfg, m=macro: m in compilerMacros(cfg))
   )