[libc++] Add more general death tests and make them work on more platforms

[philnik]

Summary

This version of death tests spawns processes instead of forking the current process. This makes it much easier to port the tests to more platforms, since a lot of platforms don't support forking since it's a very intrusive and expensive operation.

[philnik]

@mstorsjo @fsb4000 Could one of you help me out with spawning processes on windows? It should be fairly simple. You just have to return the output of stdout and stderr and the return value from run_test_process.

[philnik]

Try to fix CI

[ldionne]

libcxx/test/libcxx/algorithms/debug_less.inconsistent.pass.cpp

16–17

This seems unrelated, and it creates confusion in the patch. Can you please audit and only keep what's relevant to the patch?

[ldionne]

I'm ambivalent on this patch because it leaves us in a state where we have two frameworks for death tests and both are reasonably heavyweight. Instead, if we wanted to improve significantly on top of what we have today, I think we would need to implement support for these tests in the test harness (Lit).

We explored this a bit and it could look something like this:

#define TEST_EXPECTED_ERROR(expr, message)
  do {
    if (gather_tests) {
      print(f"{test_counter} [[[{message}]]]");
    } else if (test_counter-- == 0) {
      expr();
    }
  } while (false)

// From lit:
// 1. Run the test with no arguments.
// 2. Parse the output and build a mapping like this:
//      {test-number => expected-error-message}
// 3. Then you run the executable with each test number and you check the error message.

TEST_RUN_VERIFY_MAIN() {
  {
    typedef std::array<int, 0> C;
    C c = {};
    C const& cc = c;
    TEST_EXPECTED_ERROR(c[0], "cannot call array<T, 0>::operator[] on a zero-sized array");
    TEST_EXPECTED_ERROR(c[1], "cannot call array<T, 0>::operator[] on a zero-sized array");
    TEST_EXPECTED_ERROR(cc[0], "cannot call array<T, 0>::operator[] on a zero-sized array");
    TEST_EXPECTED_ERROR(cc[1], "cannot call array<T, 0>::operator[] on a zero-sized array");
  }
  {
    typedef std::array<const int, 0> C;
    C c = {{}};
    C const& cc = c;
    TEST_EXPECTED_ERROR(c[0], "cannot call array<T, 0>::operator[] on a zero-sized array");
    TEST_EXPECTED_ERROR(c[1], "cannot call array<T, 0>::operator[] on a zero-sized array");
    TEST_EXPECTED_ERROR(cc[0], "cannot call array<T, 0>::operator[] on a zero-sized array");
    TEST_EXPECTED_ERROR(cc[1], "cannot call array<T, 0>::operator[] on a zero-sized array");
  }

  return 0;
}

Unfortunately, the "From Lit" part ends up being quite complicated (we prototyped) and so it's not clear it's worth pursuing at this time. Barring that, I think we should instead improve the current death test framework to support intentional calls to std::terminate, which should be doable roughly with

diff --git a/libcxx/test/support/check_assertion.h b/libcxx/test/support/check_assertion.h
index 741dc4a78828..263413b50b62 100644
--- a/libcxx/test/support/check_assertion.h
+++ b/libcxx/test/support/check_assertion.h
@@ -111,7 +111,7 @@ inline AssertionInfoMatcher& GlobalMatcher() {
 
 struct DeathTest {
   enum ResultKind {
-    RK_DidNotDie, RK_MatchFound, RK_MatchFailure, RK_SetupFailure, RK_Unknown
+    RK_DidNotDie, RK_MatchFound, RK_MatchFailure, RK_SetupFailure, RK_TerminatedAsExpected, RK_Unknown
   };
 
   static const char* ResultKindToString(ResultKind RK) {
@@ -121,6 +121,7 @@ struct DeathTest {
     CASE(RK_DidNotDie);
     CASE(RK_SetupFailure);
     CASE(RK_MatchFound);
+    CASE(RK_TerminatedAsExpected);
     CASE(RK_Unknown);
     }
     return "not a result kind";
@@ -256,6 +257,11 @@ void std::__libcpp_verbose_abort(char const* format, ...) {
 template <class Func>
 inline bool ExpectDeath(const char* stmt, Func&& func, AssertionInfoMatcher Matcher) {
   DeathTest DT(Matcher);
+  set_terminate([] {
+    if (__expect_to_terminate__)
+      std::exit(RK_TerminatedAsExpected);
+    std::abort();
+  });
   DeathTest::ResultKind RK = DT.Run(func);
   auto OnFailure = [&](const char* msg) {
     std::fprintf(stderr, "EXPECT_DEATH( %s ) failed! (%s)\n\n", stmt, msg);
@@ -273,6 +279,8 @@ inline bool ExpectDeath(const char* stmt, Func&& func, AssertionInfoMatcher Matc
   switch (RK) {
   case DeathTest::RK_MatchFound:
     return true;
+  case DeathTest::RK_TerminatedAsExpected:
+    return true;
   case DeathTest::RK_SetupFailure:
     return OnFailure("child failed to setup test environment");
   case DeathTest::RK_Unknown:

libcxx/test/libcxx/containers/sequences/array/array.zero/assert.subscript.pass.cpp

20

Should this be TEST_TERMINATIONS? It's also problematic that the test is going to appear to work if you misuse the "test framework" like this. Could probably be detected by referencing something that TEST_TERMINATIONS defines from EXPECT_LIBCPP_ASSERT_FAILURE.

libcxx/test/support/terminating_test_helper.h

9

37

libcxx/test/libcxx/algorithms/alg.sorting/assert.min.max.pass.cpp

 // REQUIRES: has-unix-headers
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_ASSERTIONS=1
 
 #include <algorithm>
 #include <array>
 
-#include "check_assertion.h"
+#include "terminating_test_helper.h"
 
-int main(int, char**) {
+TEST_TERMINATIONS([] {
+  EXPECT_LIBCPP_ASSERT_FAILURE("initializer_list has to contain at least one element", [] {
     std::initializer_list<int> init_list{};
-  TEST_LIBCPP_ASSERT_FAILURE(std::ranges::minmax(init_list),
-                             "initializer_list has to contain at least one element");
+    (void)std::ranges::minmax(init_list);
+  });
 
-  TEST_LIBCPP_ASSERT_FAILURE(std::ranges::minmax(std::array<int, 0>{}),
-                             "range has to contain at least one element");
+  EXPECT_LIBCPP_ASSERT_FAILURE("range has to contain at least one element", [] {
+    (void)std::ranges::minmax(std::array<int, 0>{});
+  });
 
   return 0;
-}
+});

libcxx/test/libcxx/algorithms/alg.sorting/assert.sort.invalid_comparator.pass.cpp

 #include <memory>
 #include <ranges>
 #include <random>
 #include <set>
 #include <string>
 #include <vector>
 
 #include "bad_comparator_values.h"
-#include "check_assertion.h"
+#include "terminating_test_helper.h"
 
 void check_oob_sort_read() {
-    std::map<std::size_t, std::map<std::size_t, bool>> comparison_results; // terrible for performance, but really convenient
-    for (auto line : std::views::split(DATA, '\n') | std::views::filter([](auto const& line) { return !line.empty(); })) {
-        auto values = std::views::split(line, ' ');
-        auto it = values.begin();
-        std::size_t left = std::stol(std::string((*it).data(), (*it).size()));
-        it = std::next(it);
-        std::size_t right = std::stol(std::string((*it).data(), (*it).size()));
-        it = std::next(it);
-        bool result = static_cast<bool>(std::stol(std::string((*it).data(), (*it).size())));
-        comparison_results[left][right] = result;
-    }
-    auto predicate = [&](std::size_t* left, std::size_t* right) {
-        assert(left != nullptr && right != nullptr && "something is wrong with the test");
-        assert(comparison_results.contains(*left) && comparison_results[*left].contains(*right) && "malformed input data?");
-        return comparison_results[*left][*right];
-    };
+  std::map<std::size_t, std::map<std::size_t, bool>>
+      comparison_results; // terrible for performance, but really convenient
+    auto values                     = std::views::split(line, ' ');
+    auto it                         = values.begin();
+    std::size_t left                = std::stol(std::string((*it).data(), (*it).size()));
+    it                              = std::next(it);
+    std::size_t right               = std::stol(std::string((*it).data(), (*it).size()));
+    it                              = std::next(it);
+    bool result                     = static_cast<bool>(std::stol(std::string((*it).data(), (*it).size())));
 
-    std::vector<std::unique_ptr<std::size_t>> elements;
-    std::set<std::size_t*> valid_ptrs;
-    for (std::size_t i = 0; i != comparison_results.size(); ++i) {
-        elements.push_back(std::make_unique<std::size_t>(i));
-        valid_ptrs.insert(elements.back().get());
-    }
 
-    auto checked_predicate = [&](size_t* left, size_t* right) {
-        // If the pointers passed to the comparator are not in the set of pointers we
-        // set up above, then we're being passed garbage values from the algorithm
-        // because we're reading OOB.
-        assert(valid_ptrs.contains(left));
-        assert(valid_ptrs.contains(right));
-        return predicate(left, right);
-    };
 
-    // Check the classic sorting algorithms
-    {
-        std::vector<std::size_t*> copy;
-        for (auto const& e : elements)
-            copy.push_back(e.get());
-        TEST_LIBCPP_ASSERT_FAILURE(std::sort(copy.begin(), copy.end(), checked_predicate), "Would read out of bounds");
-    }
-    {
-        std::vector<std::size_t*> copy;
-        for (auto const& e : elements)
-            copy.push_back(e.get());
-        TEST_LIBCPP_ASSERT_FAILURE(std::stable_sort(copy.begin(), copy.end(), checked_predicate), "not a valid strict-weak ordering");
-    }
-    {
-        std::vector<std::size_t*> copy;
-        for (auto const& e : elements)
-            copy.push_back(e.get());
-        std::make_heap(copy.begin(), copy.end(), checked_predicate); // doesn't go OOB even with invalid comparator
-        TEST_LIBCPP_ASSERT_FAILURE(std::sort_heap(copy.begin(), copy.end(), checked_predicate), "not a valid strict-weak ordering");
-    }
-    {
-        std::vector<std::size_t*> copy;
-        for (auto const& e : elements)
-            copy.push_back(e.get());
-        TEST_LIBCPP_ASSERT_FAILURE(std::partial_sort(copy.begin(), copy.end(), copy.end(), checked_predicate), "not a valid strict-weak ordering");
-    }
-    {
-        std::vector<std::size_t*> copy;
-        for (auto const& e : elements)
-            copy.push_back(e.get());
-        std::vector<std::size_t*> results(copy.size(), nullptr);
-       TEST_LIBCPP_ASSERT_FAILURE(std::partial_sort_copy(copy.begin(), copy.end(), results.begin(), results.end(), checked_predicate), "not a valid strict-weak ordering");
-    }
-    {
-        std::vector<std::size_t*> copy;
-        for (auto const& e : elements)
-            copy.push_back(e.get());
-        std::nth_element(copy.begin(), copy.end(), copy.end(), checked_predicate); // doesn't go OOB even with invalid comparator
-    }
+  EXPECT_LIBCPP_ASSERT_FAILURE("Would read out of bounds", [&] {
+    std::sort(copy.begin(), copy.end(), checked_predicate);
+  });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("not a valid strict-weak ordering", [&] {
+    std::stable_sort(copy.begin(), copy.end(), checked_predicate);
+  });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("not a valid strict-weak ordering", [&] {
+    std::sort_heap(copy.begin(), copy.end(), checked_predicate);
+  });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("not a valid strict-weak ordering", [&] {
+
+    std::partial_sort(copy.begin(), copy.end(), copy.end(), checked_predicate);
+  });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("not a valid strict-weak ordering", [&] {
+
+    std::partial_sort_copy(copy.begin(), copy.end(), results.begin(), results.end(), checked_predicate);
+  });
+
+  EXPECT_SUCCESS([&] {
+    std::nth_element(
+        copy.begin(), copy.end(), copy.end(), checked_predicate); // doesn't go OOB even with invalid comparator
+  });
 
-    // Check the Ranges sorting algorithms
-    {
-        std::vector<std::size_t*> copy;
-        for (auto const& e : elements)
-            copy.push_back(e.get());
-        TEST_LIBCPP_ASSERT_FAILURE(std::ranges::sort(copy, checked_predicate), "Would read out of bounds");
-    }
-    {
-        std::vector<std::size_t*> copy;
-        for (auto const& e : elements)
-            copy.push_back(e.get());
-        TEST_LIBCPP_ASSERT_FAILURE(std::ranges::stable_sort(copy, checked_predicate), "not a valid strict-weak ordering");
-    }
-    {
-        std::vector<std::size_t*> copy;
-        for (auto const& e : elements)
-            copy.push_back(e.get());
-        std::ranges::make_heap(copy, checked_predicate); // doesn't go OOB even with invalid comparator
-        TEST_LIBCPP_ASSERT_FAILURE(std::ranges::sort_heap(copy, checked_predicate), "not a valid strict-weak ordering");
-    }
-    {
-        std::vector<std::size_t*> copy;
-        for (auto const& e : elements)
-            copy.push_back(e.get());
-        TEST_LIBCPP_ASSERT_FAILURE(std::ranges::partial_sort(copy, copy.end(), checked_predicate), "not a valid strict-weak ordering");
-    }
-    {
-        std::vector<std::size_t*> copy;
-        for (auto const& e : elements)
-            copy.push_back(e.get());
-        std::vector<std::size_t*> results(copy.size(), nullptr);
-        TEST_LIBCPP_ASSERT_FAILURE(std::ranges::partial_sort_copy(copy, results, checked_predicate), "not a valid strict-weak ordering");
-    }
-    {
-        std::vector<std::size_t*> copy;
-        for (auto const& e : elements)
-            copy.push_back(e.get());
-        std::ranges::nth_element(copy, copy.end(), checked_predicate); // doesn't go OOB even with invalid comparator
-    }
+  EXPECT_LIBCPP_ASSERT_FAILURE("Would read out of bounds", [&] {
+    std::ranges::sort(copy, checked_predicate);
+  });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("not a valid strict-weak ordering", [&] {
+    std::ranges::stable_sort(copy, checked_predicate);
+  });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("not a valid strict-weak ordering", [&] {
+    std::ranges::sort_heap(copy, checked_predicate);
+  });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("not a valid strict-weak ordering", [&] {
+
+    std::ranges::partial_sort(copy, copy.end(), checked_predicate);
+  });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("not a valid strict-weak ordering", [&] {
+
+    std::ranges::partial_sort_copy(copy, results, checked_predicate);
+  });
+
+  EXPECT_SUCCESS([&] {
+  });
 }
 
 struct FloatContainer {
   float value;
-  bool operator<(const FloatContainer& other) const {
-    return value < other.value;
-  }
+  bool operator<(const FloatContainer& other) const { return value < other.value; }
 };
 
 // Nans in floats do not satisfy strict weak ordering by breaking transitivity of equivalence.
 std::vector<FloatContainer> generate_float_data() {
-    std::vector<FloatContainer> floats(50);
-    for (int i = 0; i < 50; ++i) {
-        floats[i].value = static_cast<float>(i);
-    }
-    floats.push_back(FloatContainer{std::numeric_limits<float>::quiet_NaN()});
-    std::shuffle(floats.begin(), floats.end(), std::default_random_engine());
-    return floats;
 }
 
 void check_nan_floats() {
-    auto floats = generate_float_data();
-    TEST_LIBCPP_ASSERT_FAILURE(std::sort(floats.begin(), floats.end()), "not a valid strict-weak ordering");
-    floats = generate_float_data();
-    TEST_LIBCPP_ASSERT_FAILURE(std::stable_sort(floats.begin(), floats.end()), "not a valid strict-weak ordering");
-    floats = generate_float_data();
-    std::make_heap(floats.begin(), floats.end());
-    TEST_LIBCPP_ASSERT_FAILURE(std::sort_heap(floats.begin(), floats.end()), "not a valid strict-weak ordering");
-    TEST_LIBCPP_ASSERT_FAILURE(std::ranges::sort(generate_float_data(), std::less()), "not a valid strict-weak ordering");
-    TEST_LIBCPP_ASSERT_FAILURE(std::ranges::stable_sort(generate_float_data(), std::less()), "not a valid strict-weak ordering");
-    floats = generate_float_data();
-    std::ranges::make_heap(floats, std::less());
-    TEST_LIBCPP_ASSERT_FAILURE(std::ranges::sort_heap(floats, std::less()), "not a valid strict-weak ordering");
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("not a valid strict-weak ordering", [&] { std::sort(floats.begin(), floats.end()); });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("not a valid strict-weak ordering", [&] {
+    std::stable_sort(floats.begin(), floats.end());
+  });
+
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("not a valid strict-weak ordering", [&] {
+    std::sort_heap(floats.begin(), floats.end());
+  });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("not a valid strict-weak ordering", [&] {
+    std::ranges::sort(generate_float_data(), std::less());
+  });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("not a valid strict-weak ordering", [] {
+    std::ranges::stable_sort(generate_float_data(), std::less());
+  });
+
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("not a valid strict-weak ordering", [&] {
+    std::ranges::sort_heap(floats, std::less());
+  });
 }
 
 void check_irreflexive() {
-    std::vector<int> v(1);
-    TEST_LIBCPP_ASSERT_FAILURE(std::sort(v.begin(), v.end(), std::greater_equal<int>()), "not a valid strict-weak ordering");
-    TEST_LIBCPP_ASSERT_FAILURE(std::stable_sort(v.begin(), v.end(), std::greater_equal<int>()), "not a valid strict-weak ordering");
-    std::make_heap(v.begin(), v.end(), std::greater_equal<int>());
-    TEST_LIBCPP_ASSERT_FAILURE(std::sort_heap(v.begin(), v.end(), std::greater_equal<int>()), "not a valid strict-weak ordering");
-    TEST_LIBCPP_ASSERT_FAILURE(std::ranges::sort(v, std::greater_equal<int>()), "not a valid strict-weak ordering");
-    TEST_LIBCPP_ASSERT_FAILURE(std::ranges::stable_sort(v, std::greater_equal<int>()), "not a valid strict-weak ordering");
-    std::ranges::make_heap(v, std::greater_equal<int>());
-    TEST_LIBCPP_ASSERT_FAILURE(std::ranges::sort_heap(v, std::greater_equal<int>()), "not a valid strict-weak ordering");
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("not a valid strict-weak ordering", [&] {
+    std::sort(v.begin(), v.end(), std::greater_equal<int>());
+  });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("not a valid strict-weak ordering", [&] {
+    std::stable_sort(v.begin(), v.end(), std::greater_equal<int>());
+  });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("not a valid strict-weak ordering", [&] {
+    std::sort_heap(v.begin(), v.end(), std::greater_equal<int>());
+  });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("not a valid strict-weak ordering", [&] {
+    std::ranges::sort(v, std::greater_equal<int>());
+  });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("not a valid strict-weak ordering", [&] {
+    std::ranges::stable_sort(v, std::greater_equal<int>());
+  });
+  EXPECT_LIBCPP_ASSERT_FAILURE("not a valid strict-weak ordering", [&] {
+    std::ranges::sort_heap(v, std::greater_equal<int>());
+  });
 }
 
-int main(int, char**) {
-
-    check_oob_sort_read();
+TEST_TERMINATIONS([] {
 
-    check_nan_floats();
 
-    check_irreflexive();
 
-    return 0;
-}
+});

libcxx/test/libcxx/algorithms/debug_less.inconsistent.pass.cpp

 //===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 // <algorithm>
 
 // template <class _Compare> struct __debug_less
 
 // Make sure __debug_less asserts when the comparator is not consistent.
 
 // REQUIRES: has-unix-headers
-// UNSUPPORTED: !libcpp-has-debug-mode, c++03
+// REQUIRES: libcpp-has-debug-mode
+// UNSUPPORTED: c++03
 
 #include <algorithm>
 #include <iterator>
 
-#include "check_assertion.h"
+#include "terminating_test_helper.h"
 
 template <int ID>
 struct MyType {
     int value;
     explicit MyType(int xvalue = 0) : value(xvalue) {}
 };
 
 template <int ID1, int ID2>
 bool operator<(MyType<ID1> const& LHS, MyType<ID2> const& RHS) {
     return LHS.value < RHS.value;
 }
 
 template <class ValueType>
 struct BadComparator {
     bool operator()(ValueType const&, ValueType const&) const {
         return true;
     }
 };
 
-int main(int, char**) {
+TEST_TERMINATIONS([] {
     typedef MyType<0> MT0;
     MT0 one(1);
     MT0 two(2);
 
     BadComparator<MT0> c;
     std::__debug_less<BadComparator<MT0>> d(c);
 
-    TEST_LIBCPP_ASSERT_FAILURE(d(one, two), "Comparator does not induce a strict weak ordering");
+    EXPECT_LIBCPP_ASSERT_FAILURE("Comparator does not induce a strict weak ordering", [&] { d(one, two); });
 
     return 0;
-}
+});

libcxx/test/libcxx/algorithms/debug_less.pass.cpp

 // UNSUPPORTED: !libcpp-has-debug-mode, c++03
 
 #include <algorithm>
 #include <cassert>
 #include <functional>
 #include <iterator>
 
 #include "test_macros.h"
-#include "check_assertion.h"
 
 template <int ID>
 struct MyType {
     int value;
     explicit MyType(int xvalue = 0) : value(xvalue) {}
 };
 
 template <int ID1, int ID2>

libcxx/test/libcxx/algorithms/debug_three_way_comp.inconsistent.pass.cpp

 //===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 // <algorithm>
 
 // template <class _Comp> struct __debug_three_way_comp
 
 // Make sure __debug_three_way_comp asserts when the comparator is not consistent.
 
-// UNSUPPORTED: !libcpp-has-debug-mode, c++03, c++11, c++14, c++17
+// REQUIRES: libcpp-has-debug-mode
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 
 #include <algorithm>
 #include <iterator>
 
-#include "check_assertion.h"
+#include "terminating_test_helper.h"
 
 struct AlwaysLess {
   std::strong_ordering operator()(int, int) const { return std::strong_ordering::less; }
 };
 
 struct AlwaysGreater {
   std::strong_ordering operator()(int, int) const { return std::strong_ordering::greater; }
 };
 
 struct InconsistentEquals {
   std::strong_ordering operator()(int a, int) const {
     if (a == 0)
       return std::strong_ordering::equal;
     return std::strong_ordering::greater;
   }
 };
 
-int main(int, char**) {
+TEST_TERMINATIONS([] {
   int zero = 0;
   int one  = 1;
 
+  EXPECT_LIBCPP_ASSERT_FAILURE("Comparator does not induce a strict weak ordering", [&] {
     AlwaysLess alwaysLess;
     std::__debug_three_way_comp<AlwaysLess> debugAlwaysLess(alwaysLess);
-  TEST_LIBCPP_ASSERT_FAILURE(debugAlwaysLess(zero, one), "Comparator does not induce a strict weak ordering");
+    debugAlwaysLess(zero, one);
+  });
 
+  EXPECT_LIBCPP_ASSERT_FAILURE("Comparator does not induce a strict weak ordering", [&] {
     AlwaysGreater alwaysGreater;
     std::__debug_three_way_comp<AlwaysGreater> debugAlwaysGreater(alwaysGreater);
-  TEST_LIBCPP_ASSERT_FAILURE(debugAlwaysGreater(zero, one), "Comparator does not induce a strict weak ordering");
+    debugAlwaysGreater(zero, one);
+  });
 
+  EXPECT_LIBCPP_ASSERT_FAILURE("Comparator does not induce a strict weak ordering", [&] {
     InconsistentEquals inconsistentEquals;
     std::__debug_three_way_comp<InconsistentEquals> debugInconsistentEquals(inconsistentEquals);
-  TEST_LIBCPP_ASSERT_FAILURE(debugInconsistentEquals(zero, one), "Comparator does not induce a strict weak ordering");
+    debugInconsistentEquals(zero, one);
+  });
 
   return 0;
-}
+});

libcxx/test/libcxx/containers/sequences/array/array.zero/assert.back.pass.cpp

 // UNSUPPORTED: c++03
 // XFAIL: availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_ASSERTIONS=1
 
 // test that array<T, 0>::back() triggers an assertion
 
 #include <array>
 
-#include "check_assertion.h"
+#include "terminating_test_helper.h"
 
-int main(int, char**) {
-  {
-    typedef std::array<int, 0> C;
-    C c = {};
-    C const& cc = c;
-    TEST_LIBCPP_ASSERT_FAILURE(c.back(), "cannot call array<T, 0>::back() on a zero-sized array");
-    TEST_LIBCPP_ASSERT_FAILURE(cc.back(), "cannot call array<T, 0>::back() on a zero-sized array");
-  }
-  {
-    typedef std::array<const int, 0> C;
-    C c = {{}};
-    C const& cc = c;
-    TEST_LIBCPP_ASSERT_FAILURE(c.back(), "cannot call array<T, 0>::back() on a zero-sized array");
-    TEST_LIBCPP_ASSERT_FAILURE(cc.back(), "cannot call array<T, 0>::back() on a zero-sized array");
-  }
+TEST_TERMINATIONS([] {
+  EXPECT_LIBCPP_ASSERT_FAILURE("cannot call array<T, 0>::back() on a zero-sized array", [] {
+    std::array<int, 0> c = {};
+    c.back();
+  });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("cannot call array<T, 0>::back() on a zero-sized array", [] {
+    const std::array<int, 0> cc = {};
+    cc.back();
+  });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("cannot call array<T, 0>::back() on a zero-sized array", [] {
+    std::array<const int, 0> c = {};
+    c.back();
+  });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("cannot call array<T, 0>::back() on a zero-sized array", [] {
+    const std::array<const int, 0> cc = {};
+    cc.back();
+  });
 
   return 0;
-}
+});

libcxx/test/libcxx/containers/sequences/array/array.zero/assert.front.pass.cpp

 // UNSUPPORTED: c++03
 // XFAIL: availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_ASSERTIONS=1
 
 // test that array<T, 0>::back() triggers an assertion
 
 #include <array>
 
-#include "check_assertion.h"
+#include "terminating_test_helper.h"
 
-int main(int, char**) {
-  {
-    typedef std::array<int, 0> C;
-    C c = {};
-    C const& cc = c;
-    TEST_LIBCPP_ASSERT_FAILURE(c.front(), "cannot call array<T, 0>::front() on a zero-sized array");
-    TEST_LIBCPP_ASSERT_FAILURE(cc.front(), "cannot call array<T, 0>::front() on a zero-sized array");
-  }
-  {
-    typedef std::array<const int, 0> C;
-    C c = {{}};
-    C const& cc = c;
-    TEST_LIBCPP_ASSERT_FAILURE(c.front(), "cannot call array<T, 0>::front() on a zero-sized array");
-    TEST_LIBCPP_ASSERT_FAILURE(cc.front(), "cannot call array<T, 0>::front() on a zero-sized array");
-  }
+TEST_TERMINATIONS([] {
+  EXPECT_LIBCPP_ASSERT_FAILURE("cannot call array<T, 0>::front() on a zero-sized array", [] {
+    std::array<int, 0> c = {};
+    c.front();
+  });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("cannot call array<T, 0>::front() on a zero-sized array", [] {
+    const std::array<int, 0> cc = {};
+    cc.front();
+  });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("cannot call array<T, 0>::front() on a zero-sized array", [] {
+    std::array<const int, 0> c = {};
+    c.front();
+  });
+
+  EXPECT_LIBCPP_ASSERT_FAILURE("cannot call array<T, 0>::front() on a zero-sized array", [] {
+    const std::array<const int, 0> cc = {};
+    cc.front();
+  });
 
   return 0;
-}
+});

libcxx/test/libcxx/containers/sequences/array/array.zero/assert.subscript.pass.cpp

 // UNSUPPORTED: c++03
 // XFAIL: availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_ASSERTIONS=1
 
 // test that array<T, 0>::operator[] triggers an assertion
 
 #include <array>
 
-#include "check_assertion.h"
+#include "terminating_test_helper.h"
 
 int main(int, char**) {
   {
     typedef std::array<int, 0> C;
-    C c = {};
+    C c         = {};
     C const& cc = c;
-    TEST_LIBCPP_ASSERT_FAILURE(c[0], "cannot call array<T, 0>::operator[] on a zero-sized array");
-    TEST_LIBCPP_ASSERT_FAILURE(c[1], "cannot call array<T, 0>::operator[] on a zero-sized array");
-    TEST_LIBCPP_ASSERT_FAILURE(cc[0], "cannot call array<T, 0>::operator[] on a zero-sized array");
-    TEST_LIBCPP_ASSERT_FAILURE(cc[1], "cannot call array<T, 0>::operator[] on a zero-sized array");
+    EXPECT_LIBCPP_ASSERT_FAILURE("cannot call array<T, 0>::operator[] on a zero-sized array", [&] { c[0]; });
+    EXPECT_LIBCPP_ASSERT_FAILURE("cannot call array<T, 0>::operator[] on a zero-sized array", [&] { c[1]; });
+    EXPECT_LIBCPP_ASSERT_FAILURE("cannot call array<T, 0>::operator[] on a zero-sized array", [&] { cc[0]; });
+    EXPECT_LIBCPP_ASSERT_FAILURE("cannot call array<T, 0>::operator[] on a zero-sized array", [&] { cc[1]; });
   }
   {
     typedef std::array<const int, 0> C;
-    C c = {{}};
+    C c         = {{}};
     C const& cc = c;
-    TEST_LIBCPP_ASSERT_FAILURE(c[0], "cannot call array<T, 0>::operator[] on a zero-sized array");
-    TEST_LIBCPP_ASSERT_FAILURE(c[1], "cannot call array<T, 0>::operator[] on a zero-sized array");
-    TEST_LIBCPP_ASSERT_FAILURE(cc[0], "cannot call array<T, 0>::operator[] on a zero-sized array");
-    TEST_LIBCPP_ASSERT_FAILURE(cc[1], "cannot call array<T, 0>::operator[] on a zero-sized array");
+    EXPECT_LIBCPP_ASSERT_FAILURE("cannot call array<T, 0>::operator[] on a zero-sized array", [&] { c[0]; });
+    EXPECT_LIBCPP_ASSERT_FAILURE("cannot call array<T, 0>::operator[] on a zero-sized array", [&] { c[1]; });
+    EXPECT_LIBCPP_ASSERT_FAILURE("cannot call array<T, 0>::operator[] on a zero-sized array", [&] { cc[0]; });
+    EXPECT_LIBCPP_ASSERT_FAILURE("cannot call array<T, 0>::operator[] on a zero-sized array", [&] { cc[1]; });
   }
 
   return 0;
 }

libcxx/test/support/terminating_test_helper.h

+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TEST_TERMINATING_TEST_HELPER_H
+#define TEST_TERMINATING_TEST_HELPER_H
+
+#include <cassert>
+#include <cerrno>
+#include <cstdlib>
+#include <exception>
+#include <functional>
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include "test_macros.h"
+
+#if __has_include(<poll.h>) && __has_include(<spawn.h>) && __has_include(<unistd.h>) && __has_include(<sys/wait.h>)
+#  include <poll.h>
+#  include <spawn.h>
+#  include <sys/wait.h>
+#  include <unistd.h>
+#  define HAS_POSIX_SPAWN
+extern char** environ;
+#endif
+
+namespace terminate {
+
+enum class result : int {
+  success = 0,
+  std_terminate,
+  nondetermenistic_test_count,
+#ifdef _LIBCPP_VERSION
+  libcpp_assert,
+#endif
+};
+
+inline std::string to_string(result res) {
+  switch (res) {
+  case result::success:
+    return "unexpected success";
+  case result::std_terminate:
+    return "std::terminate";
+  case result::nondetermenistic_test_count:
+    return "nondeterministic test count";
+#ifdef _LIBCPP_VERSION
+  case result::libcpp_assert:
+    return "libc++ assertion";
+#endif
+  }
+  return "unknown";
+}
+
+struct process_output {
+  std::string std_out;
+  std::string std_err;
+};
+
+struct process_result {
+  result return_value;
+  process_output output;
+};
+
+#ifdef HAS_POSIX_SPAWN
+inline process_output get_child_output(int stdout_descriptor, int stderr_descriptor) {
+  process_output data;
+  char buffer[256];
+
+  std::array<pollfd, 2> polls   = {{{stdout_descriptor, POLLIN, 0}, {stderr_descriptor, POLLIN, 0}}};
+  std::array<bool, 2> read_done = {false, false};
+
+  int rval;
+  while ((rval = poll(polls.data(), polls.size(), /*timeout=*/-1)) > 0) {
+    if (!read_done[0] && polls[0].revents & POLLIN) {
+      ssize_t bytes_read = read(stdout_descriptor, buffer, sizeof(buffer));
+      if (bytes_read == 0)
+        read_done[0] = true;
+      data.std_out.append(buffer, bytes_read);
+    } else if (!read_done[1] && polls[1].revents & POLLIN) {
+      ssize_t bytes_read = read(stderr_descriptor, buffer, sizeof(buffer));
+      if (bytes_read == 0)
+        read_done[1] = true;
+      data.std_err.append(buffer, bytes_read);
+    } else {
+      break;
+    }
+  }
+
+  return data;
+}
+
+inline process_result run_test_process(char* bin_path, size_t test_number) {
+  process_result res;
+  int stdout_pipe[2];
+  int stderr_pipe[2];
+  assert(pipe(stdout_pipe) == 0);
+  assert(pipe(stderr_pipe) == 0);
+
+  posix_spawn_file_actions_t actions;
+
+  // redirect child output to our pipes
+  assert(posix_spawn_file_actions_init(&actions) == 0);
+  assert(posix_spawn_file_actions_addclose(&actions, stdout_pipe[0]) == 0);
+  assert(posix_spawn_file_actions_addclose(&actions, stderr_pipe[0]) == 0);
+  assert(posix_spawn_file_actions_adddup2(&actions, stdout_pipe[1], 1) == 0);
+  assert(posix_spawn_file_actions_adddup2(&actions, stderr_pipe[1], 2) == 0);
+  assert(posix_spawn_file_actions_addclose(&actions, stdout_pipe[1]) == 0);
+  assert(posix_spawn_file_actions_addclose(&actions, stderr_pipe[1]) == 0);
+
+  pid_t pid;
+  std::string test_number_str = std::to_string(test_number);
+  char* argv[]                = {bin_path, &test_number_str[0], nullptr};
+  assert(posix_spawn(&pid, bin_path, &actions, nullptr, argv, environ) == 0);
+
+  assert(close(stdout_pipe[1]) == 0);
+  assert(close(stderr_pipe[1]) == 0);
+
+  res.output = get_child_output(stdout_pipe[0], stderr_pipe[0]);
+
+  assert(close(stdout_pipe[0]) == 0);
+  assert(close(stderr_pipe[0]) == 0);
+
+  int status;
+  assert(waitpid(pid, &status, 0) != -1);
+
+  assert(posix_spawn_file_actions_destroy(&actions) == 0);
+
+  res.return_value = static_cast<result>(WEXITSTATUS(status));
+
+  return res;
+}
+#else
+#  error "No known way to spawn a process on this platform"
+#endif
+
+struct test_info {
+  std::string_view test_string;
+  std::function<bool(process_result)> check_result;
+};
+
+static size_t current_test = 0;
+static size_t test_to_run  = 0;
+static std::vector<terminate::test_info> test_infos;
+
+[[noreturn]] inline void exit(result res) { std::exit(static_cast<int>(res)); }
+
+[[noreturn]] inline void terminate_handler() { terminate::exit(result::std_terminate); }
+
+inline void run_tests(char* bin_path) {
+  for (size_t i = 0; i != current_test; ++i) {
+    auto result = terminate::run_test_process(bin_path, i);
+    if (!test_infos[i].check_result(result)) {
+      std::cerr << "Test #" << i << " ('" << test_infos[i].test_string << "') failed through "
+                << to_string(result.return_value) << ".\nstdout:\n"
+                << result.output.std_out << "\nstderr:\n"
+                << result.output.std_err << "\n";
+      std::exit(1);
+    }
+  }
+}
+
+inline bool result_is_expected_failure(process_result result) {
+  return result.return_value != result::success && result.return_value != result::nondetermenistic_test_count;
+}
+
+inline bool result_is_success(process_result result) { return result.return_value == result::success; }
+
+inline bool result_is_std_terminate(process_result result) { return result.return_value == result::std_terminate; }
+
+#ifdef _LIBCPP_VERSION
+inline bool result_is_libcpp_assert(const char* expected_string, process_result result) {
+  return result.return_value == result::libcpp_assert && result.output.std_err == expected_string;
+}
+#endif
+} // namespace terminate
+
+#ifdef _LIBCPP_VERSION
+void std::__libcpp_verbose_abort(char const* format, ...) {
+  // Extract information from the error message. This has to stay synchronized with
+  // how we format assertions in the library.
+  va_list list;
+  va_start(list, format);
+  [[maybe_unused]] const char* file       = va_arg(list, const char*);
+  [[maybe_unused]] int line               = va_arg(list, int);
+  [[maybe_unused]] const char* expression = va_arg(list, const char*);
+  [[maybe_unused]] const char* message    = va_arg(list, const char*);
+  std::cerr << message;
+  va_end(list);
+  terminate::exit(terminate::result::libcpp_assert);
+}
+#endif
+
+#define TEST_TERMINATIONS(...)                                                                                         \
+  int main(int argc, char** argv) {                                                                                    \
+    if (argc == 1) {                                                                                                   \
+      ++::terminate::current_test;                                                                                     \
+      __VA_ARGS__();                                                                                                   \
+      --::terminate::current_test;                                                                                     \
+      ::terminate::run_tests(argv[0]);                                                                                 \
+    } else {                                                                                                           \
+      assert(argc == 2);                                                                                               \
+      ::std::set_terminate(::terminate::terminate_handler);                                                            \
+      ::terminate::test_to_run = ::std::stoull(argv[1]);                                                               \
+      __VA_ARGS__();                                                                                                   \
+      ::terminate::exit(::terminate::result::nondetermenistic_test_count);                                             \
+    }                                                                                                                  \
+                                                                                                                       \
+    return 0;                                                                                                          \
+  }
+
+#define EXPECT_RESULT(result_check_function, ...)                                                                      \
+  do {                                                                                                                 \
+    ::terminate::test_infos.push_back({TEST_STRINGIZE(__VA_ARGS__), result_check_function});                           \
+    if (::terminate::current_test++ == ::terminate::test_to_run) {                                                     \
+      __VA_ARGS__();                                                                                                   \
+      ::terminate::exit(::terminate::result::success);                                                                 \
+    }                                                                                                                  \
+  } while (false)
+
+#define EXPECT_SUCCESS(...) EXPECT_RESULT(::terminate::result_is_success, __VA_ARGS__)
+#define EXPECT_TERMINATE(...) EXPECT_RESULT(::terminate::result_is_expected_failure, __VA_ARGS__)
+#define EXPECT_STD_TERMINATE(...) EXPECT_RESULT(::terminate::result_is_std_terminate, __VA_ARGS__)
+
+#ifdef _LIBCPP_VERSION
+#  define EXPECT_LIBCPP_ASSERT_FAILURE(expected_string, ...)                                                           \
+    EXPECT_RESULT(                                                                                                     \
+        [](::terminate::process_result res) { return ::terminate::result_is_libcpp_assert(expected_string, res); },    \
+        __VA_ARGS__)
+#else
+#  define EXPECT_LIBCPP_ASSERT_FAILURE(...)
+#endif
+
+#endif // TEST_TERMINATING_TEST_HELPER_H

libcxx/test/support/test_macros.h

 #   include <__config>
 #   define TEST_IMP_INCLUDED_HEADER
 # endif
 #endif
 #ifndef TEST_IMP_INCLUDED_HEADER
 #include <ciso646>
 #endif
 
-#define TEST_STRINGIZE_IMPL(x) #x
-#define TEST_STRINGIZE(x) TEST_STRINGIZE_IMPL(x)
+#define TEST_STRINGIZE_IMPL(...) #__VA_ARGS__
+#define TEST_STRINGIZE(...) TEST_STRINGIZE_IMPL(__VA_ARGS__)
 
 #define TEST_CONCAT1(X, Y) X##Y
 #define TEST_CONCAT(X, Y) TEST_CONCAT1(X, Y)
 
 #ifdef __has_feature
 #define TEST_HAS_FEATURE(X) __has_feature(X)
 #else
 #define TEST_HAS_FEATURE(X) 0