diff --git a/libcxx/test/libcxx/selftest/dsl/dsl.sh.py b/libcxx/test/libcxx/selftest/dsl/dsl.sh.py --- a/libcxx/test/libcxx/selftest/dsl/dsl.sh.py +++ b/libcxx/test/libcxx/selftest/dsl/dsl.sh.py @@ -240,22 +240,22 @@ self.assertEqual(dsl.programOutput(self.config, source), "STDOUT-OUTPUT") -class TestProgramSucceeds(SetupConfigs): +class TestRunSucceeds(SetupConfigs): """ - Tests for libcxx.test.dsl.programSucceeds + Tests for libcxx.test.dsl.runSucceeds """ def test_success(self): source = """ int main(int, char**) { return 0; } """ - self.assertTrue(dsl.programSucceeds(self.config, source)) + self.assertTrue(dsl.runSucceeds(self.config, source)) def test_failure(self): source = """ int main(int, char**) { return 1; } """ - self.assertFalse(dsl.programSucceeds(self.config, source)) + self.assertFalse(dsl.runSucceeds(self.config, source)) def test_compile_failure(self): source = """ @@ -263,9 +263,31 @@ """ self.assertRaises( dsl.ConfigurationCompilationError, - lambda: dsl.programSucceeds(self.config, source), + lambda: dsl.runSucceeds(self.config, source), ) +class TestCompileAndRunSucceeds(SetupConfigs): + """ + Tests for libcxx.test.dsl.compileAndRunSucceeds + """ + + def test_success(self): + source = """ + int main(int, char**) { return 0; } + """ + self.assertTrue(dsl.compileAndRunSucceeds(self.config, source)) + + def test_failure(self): + source = """ + int main(int, char**) { return 1; } + """ + self.assertFalse(dsl.compileAndRunSucceeds(self.config, source)) + + def test_compile_failure(self): + source = """ + this does not compile + """ + self.assertFalse(dsl.compileAndRunSucceeds(self.config, source)) class TestHasLocale(SetupConfigs): """ diff --git a/libcxx/test/libcxx/selftest/fuzz.cpp/compile-error.fuzz.cpp b/libcxx/test/libcxx/selftest/fuzz.cpp/compile-error.fuzz.cpp new file mode 100644 --- /dev/null +++ b/libcxx/test/libcxx/selftest/fuzz.cpp/compile-error.fuzz.cpp @@ -0,0 +1,19 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// XFAIL: * + +// Make sure the test DOES NOT pass if it fails at compile-time + +#include +#include + +struct Foo {}; +typedef Foo::x x; + +extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t*, std::size_t) { return 0; } diff --git a/libcxx/test/libcxx/selftest/fuzz.cpp/link-error.fuzz.cpp b/libcxx/test/libcxx/selftest/fuzz.cpp/link-error.fuzz.cpp new file mode 100644 --- /dev/null +++ b/libcxx/test/libcxx/selftest/fuzz.cpp/link-error.fuzz.cpp @@ -0,0 +1,22 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// XFAIL: * + +// Make sure the test DOES NOT pass if it fails at link-time + +#include +#include + +extern void this_is_an_undefined_symbol(); + +extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t*, std::size_t) { + this_is_an_undefined_symbol(); + + return 0; +} diff --git a/libcxx/test/libcxx/selftest/fuzz.cpp/run-error.fuzz.cpp b/libcxx/test/libcxx/selftest/fuzz.cpp/run-error.fuzz.cpp new file mode 100644 --- /dev/null +++ b/libcxx/test/libcxx/selftest/fuzz.cpp/run-error.fuzz.cpp @@ -0,0 +1,22 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// XFAIL: * + +// Make sure the test DOES NOT pass if it fails at runtime. + +#include +#include + +extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t*, std::size_t) { + volatile int* ptr = nullptr; + volatile int val = *ptr; + (void)val; + + return 0; +} diff --git a/libcxx/test/libcxx/selftest/fuzz.cpp/run-success.fuzz.cpp b/libcxx/test/libcxx/selftest/fuzz.cpp/run-success.fuzz.cpp new file mode 100644 --- /dev/null +++ b/libcxx/test/libcxx/selftest/fuzz.cpp/run-success.fuzz.cpp @@ -0,0 +1,14 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// Make sure the test passes pass if it succeeds at runtime. + +#include +#include + +extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t*, std::size_t) { return 0; } diff --git a/libcxx/test/libcxx/fuzzing/unique.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.unique/unique.fuzz.cpp rename from libcxx/test/libcxx/fuzzing/unique.pass.cpp rename to libcxx/test/std/algorithms/alg.modifying.operations/alg.unique/unique.fuzz.cpp --- a/libcxx/test/libcxx/fuzzing/unique.pass.cpp +++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.unique/unique.fuzz.cpp @@ -13,7 +13,7 @@ #include #include -#include "fuzz.h" +#include "fuzz_helper.h" extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *data, std::size_t size) { std::vector working(data, data + size); diff --git a/libcxx/test/libcxx/fuzzing/unique_copy.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.unique/unique_copy.fuzz.cpp rename from libcxx/test/libcxx/fuzzing/unique_copy.pass.cpp rename to libcxx/test/std/algorithms/alg.modifying.operations/alg.unique/unique_copy.fuzz.cpp --- a/libcxx/test/libcxx/fuzzing/unique_copy.pass.cpp +++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.unique/unique_copy.fuzz.cpp @@ -14,7 +14,7 @@ #include #include -#include "fuzz.h" +#include "fuzz_helper.h" extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *data, std::size_t size) { std::vector working(data, data + size); diff --git a/libcxx/test/libcxx/fuzzing/search.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/alg.search/search.fuzz.cpp rename from libcxx/test/libcxx/fuzzing/search.pass.cpp rename to libcxx/test/std/algorithms/alg.nonmodifying/alg.search/search.fuzz.cpp --- a/libcxx/test/libcxx/fuzzing/search.pass.cpp +++ b/libcxx/test/std/algorithms/alg.nonmodifying/alg.search/search.fuzz.cpp @@ -14,7 +14,7 @@ #include #include -#include "fuzz.h" +#include "fuzz_helper.h" extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *data, std::size_t size) { if (size < 2) diff --git a/libcxx/test/libcxx/fuzzing/make_heap.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/make.heap/make_heap.fuzz.cpp rename from libcxx/test/libcxx/fuzzing/make_heap.pass.cpp rename to libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/make.heap/make_heap.fuzz.cpp --- a/libcxx/test/libcxx/fuzzing/make_heap.pass.cpp +++ b/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/make.heap/make_heap.fuzz.cpp @@ -13,7 +13,7 @@ #include #include -#include "fuzz.h" +#include "fuzz_helper.h" extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *data, std::size_t size) { std::vector working(data, data + size); diff --git a/libcxx/test/libcxx/fuzzing/pop_heap.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/pop.heap/pop_heap.fuzz.cpp rename from libcxx/test/libcxx/fuzzing/pop_heap.pass.cpp rename to libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/pop.heap/pop_heap.fuzz.cpp --- a/libcxx/test/libcxx/fuzzing/pop_heap.pass.cpp +++ b/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/pop.heap/pop_heap.fuzz.cpp @@ -13,7 +13,7 @@ #include #include -#include "fuzz.h" +#include "fuzz_helper.h" extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *data, std::size_t size) { if (size < 2) diff --git a/libcxx/test/libcxx/fuzzing/push_heap.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/push.heap/push_heap.fuzz.cpp rename from libcxx/test/libcxx/fuzzing/push_heap.pass.cpp rename to libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/push.heap/push_heap.fuzz.cpp --- a/libcxx/test/libcxx/fuzzing/push_heap.pass.cpp +++ b/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/push.heap/push_heap.fuzz.cpp @@ -13,7 +13,7 @@ #include #include -#include "fuzz.h" +#include "fuzz_helper.h" extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *data, std::size_t size) { if (size < 2) diff --git a/libcxx/test/libcxx/fuzzing/nth_element.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.nth.element/nth_element.fuzz.cpp rename from libcxx/test/libcxx/fuzzing/nth_element.pass.cpp rename to libcxx/test/std/algorithms/alg.sorting/alg.nth.element/nth_element.fuzz.cpp --- a/libcxx/test/libcxx/fuzzing/nth_element.pass.cpp +++ b/libcxx/test/std/algorithms/alg.sorting/alg.nth.element/nth_element.fuzz.cpp @@ -13,7 +13,7 @@ #include #include -#include "fuzz.h" +#include "fuzz_helper.h" // Use the first element as a position into the data extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *data, std::size_t size) { diff --git a/libcxx/test/libcxx/fuzzing/partition.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.partitions/partition.fuzz.cpp rename from libcxx/test/libcxx/fuzzing/partition.pass.cpp rename to libcxx/test/std/algorithms/alg.sorting/alg.partitions/partition.fuzz.cpp --- a/libcxx/test/libcxx/fuzzing/partition.pass.cpp +++ b/libcxx/test/std/algorithms/alg.sorting/alg.partitions/partition.fuzz.cpp @@ -13,7 +13,7 @@ #include #include -#include "fuzz.h" +#include "fuzz_helper.h" extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *data, std::size_t size) { auto is_even = [](auto x) { return x % 2 == 0; }; diff --git a/libcxx/test/libcxx/fuzzing/partition_copy.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.partitions/partition_copy.fuzz.cpp rename from libcxx/test/libcxx/fuzzing/partition_copy.pass.cpp rename to libcxx/test/std/algorithms/alg.sorting/alg.partitions/partition_copy.fuzz.cpp --- a/libcxx/test/libcxx/fuzzing/partition_copy.pass.cpp +++ b/libcxx/test/std/algorithms/alg.sorting/alg.partitions/partition_copy.fuzz.cpp @@ -14,7 +14,7 @@ #include #include -#include "fuzz.h" +#include "fuzz_helper.h" extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *data, std::size_t size) { auto is_even = [](auto t) { diff --git a/libcxx/test/libcxx/fuzzing/stable_partition.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.partitions/stable_partition.fuzz.cpp rename from libcxx/test/libcxx/fuzzing/stable_partition.pass.cpp rename to libcxx/test/std/algorithms/alg.sorting/alg.partitions/stable_partition.fuzz.cpp --- a/libcxx/test/libcxx/fuzzing/stable_partition.pass.cpp +++ b/libcxx/test/std/algorithms/alg.sorting/alg.partitions/stable_partition.fuzz.cpp @@ -13,7 +13,7 @@ #include #include -#include "fuzz.h" +#include "fuzz_helper.h" extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *data, std::size_t size) { auto is_even = [](auto b) { return b.key % 2 == 0; }; diff --git a/libcxx/test/libcxx/fuzzing/partial_sort_copy.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.sort/partial.sort.copy/partial_sort_copy.fuzz.cpp rename from libcxx/test/libcxx/fuzzing/partial_sort_copy.pass.cpp rename to libcxx/test/std/algorithms/alg.sorting/alg.sort/partial.sort.copy/partial_sort_copy.fuzz.cpp --- a/libcxx/test/libcxx/fuzzing/partial_sort_copy.pass.cpp +++ b/libcxx/test/std/algorithms/alg.sorting/alg.sort/partial.sort.copy/partial_sort_copy.fuzz.cpp @@ -13,7 +13,7 @@ #include #include -#include "fuzz.h" +#include "fuzz_helper.h" // Use the first element as a count extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *data, std::size_t size) { diff --git a/libcxx/test/libcxx/fuzzing/partial_sort.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.sort/partial.sort/partial_sort.fuzz.cpp rename from libcxx/test/libcxx/fuzzing/partial_sort.pass.cpp rename to libcxx/test/std/algorithms/alg.sorting/alg.sort/partial.sort/partial_sort.fuzz.cpp --- a/libcxx/test/libcxx/fuzzing/partial_sort.pass.cpp +++ b/libcxx/test/std/algorithms/alg.sorting/alg.sort/partial.sort/partial_sort.fuzz.cpp @@ -13,7 +13,7 @@ #include #include -#include "fuzz.h" +#include "fuzz_helper.h" // Use the first element as a position into the data extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *data, std::size_t size) { diff --git a/libcxx/test/libcxx/fuzzing/sort.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.sort/sort/sort.fuzz.cpp rename from libcxx/test/libcxx/fuzzing/sort.pass.cpp rename to libcxx/test/std/algorithms/alg.sorting/alg.sort/sort/sort.fuzz.cpp --- a/libcxx/test/libcxx/fuzzing/sort.pass.cpp +++ b/libcxx/test/std/algorithms/alg.sorting/alg.sort/sort/sort.fuzz.cpp @@ -13,7 +13,7 @@ #include #include -#include "fuzz.h" +#include "fuzz_helper.h" extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *data, std::size_t size) { std::vector working(data, data + size); diff --git a/libcxx/test/libcxx/fuzzing/stable_sort.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.sort/stable.sort/stable_sort.fuzz.cpp rename from libcxx/test/libcxx/fuzzing/stable_sort.pass.cpp rename to libcxx/test/std/algorithms/alg.sorting/alg.sort/stable.sort/stable_sort.fuzz.cpp --- a/libcxx/test/libcxx/fuzzing/stable_sort.pass.cpp +++ b/libcxx/test/std/algorithms/alg.sorting/alg.sort/stable.sort/stable_sort.fuzz.cpp @@ -13,7 +13,7 @@ #include #include -#include "fuzz.h" +#include "fuzz_helper.h" extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *data, std::size_t size) { std::vector input; diff --git a/libcxx/test/libcxx/fuzzing/random.pass.cpp b/libcxx/test/std/numerics/rand/rand.dist/random.fuzz.cpp rename from libcxx/test/libcxx/fuzzing/random.pass.cpp rename to libcxx/test/std/numerics/rand/rand.dist/random.fuzz.cpp --- a/libcxx/test/libcxx/fuzzing/random.pass.cpp +++ b/libcxx/test/std/numerics/rand/rand.dist/random.fuzz.cpp @@ -8,6 +8,9 @@ // UNSUPPORTED: c++03, c++11 +// TODO: This seems to get into an infinite loop with -seed=1412493276. Find out what the problem is. +// UNSUPPORTED: stdlib=libc++ + #include #include #include @@ -17,7 +20,7 @@ #include #include -#include "fuzz.h" +#include "fuzz_helper.h" template std::vector GetValues(const std::uint8_t *data, std::size_t size) { diff --git a/libcxx/test/libcxx/fuzzing/regex.pass.cpp b/libcxx/test/std/re/regex.fuzz.cpp rename from libcxx/test/libcxx/fuzzing/regex.pass.cpp rename to libcxx/test/std/re/regex.fuzz.cpp --- a/libcxx/test/libcxx/fuzzing/regex.pass.cpp +++ b/libcxx/test/std/re/regex.fuzz.cpp @@ -10,12 +10,15 @@ // UNSUPPORTED: no-exceptions // UNSUPPORTED: no-localization +// TODO: regex crashes on a bunch of inputs. Fix them to allow the fuzzer to run at least 10 seconds without crashing. +// UNSUPPORTED: stdlib=libc++ + #include #include #include #include -#include "fuzz.h" +#include "fuzz_helper.h" template static int regex_test(const std::uint8_t *data, std::size_t size) { diff --git a/libcxx/test/libcxx/fuzzing/fuzz.h b/libcxx/test/support/fuzz_helper.h rename from libcxx/test/libcxx/fuzzing/fuzz.h rename to libcxx/test/support/fuzz_helper.h --- a/libcxx/test/libcxx/fuzzing/fuzz.h +++ b/libcxx/test/support/fuzz_helper.h @@ -109,37 +109,4 @@ return true; } -// When running inside OSS-Fuzz, we link against a fuzzing library that defines -// main() and calls LLVMFuzzerTestOneInput. -// -// Otherwise, when e.g. running the Lit tests, we define main() to run fuzzing -// tests on a few inputs. -#if !defined(LIBCPP_OSS_FUZZ) -extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t*, std::size_t); - -int main(int, char**) { - const char* test_cases[] = { - "", - "s", - "bac", - "bacasf", - "lkajseravea", - "adsfkajdsfjkas;lnc441324513,34535r34525234", - "b*c", - "ba?sf", - "lka*ea", - "adsf*kas;lnc441[0-9]1r34525234" - }; - - for (const char* tc : test_cases) { - const std::size_t size = std::strlen(tc); - const std::uint8_t* data = reinterpret_cast(tc); - int result = LLVMFuzzerTestOneInput(data, size); - assert(result == 0); - } - - return 0; -} -#endif // !LIBCPP_OSS_FUZZ - #endif // TEST_LIBCXX_FUZZING_FUZZ_H diff --git a/libcxx/utils/ci/buildkite-pipeline.yml b/libcxx/utils/ci/buildkite-pipeline.yml --- a/libcxx/utils/ci/buildkite-pipeline.yml +++ b/libcxx/utils/ci/buildkite-pipeline.yml @@ -95,6 +95,9 @@ - label: "GCC ${GCC_STABLE_VERSION} / C++latest" command: "libcxx/utils/ci/run-buildbot generic-gcc" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -113,6 +116,9 @@ - label: "C++26" command: "libcxx/utils/ci/run-buildbot generic-cxx26" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -131,6 +137,9 @@ - label: "Modular build" command: "libcxx/utils/ci/run-buildbot generic-modules" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -149,6 +158,9 @@ - label: "C++23 Module std" command: "libcxx/utils/ci/run-buildbot generic-module-std-cxx23" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -170,6 +182,9 @@ - label: "C++11" command: "libcxx/utils/ci/run-buildbot generic-cxx11" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -188,6 +203,9 @@ - label: "C++03" command: "libcxx/utils/ci/run-buildbot generic-cxx03" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -211,6 +229,9 @@ - label: "C++23" command: "libcxx/utils/ci/run-buildbot generic-cxx23" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -229,6 +250,9 @@ - label: "C++20" command: "libcxx/utils/ci/run-buildbot generic-cxx20" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -247,6 +271,9 @@ - label: "C++17" command: "libcxx/utils/ci/run-buildbot generic-cxx17" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -265,6 +292,9 @@ - label: "C++14" command: "libcxx/utils/ci/run-buildbot generic-cxx14" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -284,6 +314,9 @@ - label: "GCC ${GCC_STABLE_VERSION} / C++11" command: "libcxx/utils/ci/run-buildbot generic-gcc-cxx11" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -302,6 +335,9 @@ - label: "Clang 15" command: "libcxx/utils/ci/run-buildbot generic-cxx23" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -320,6 +356,9 @@ - label: "Clang 16" command: "libcxx/utils/ci/run-buildbot generic-cxx23" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -341,6 +380,9 @@ - label: "ASAN" command: "libcxx/utils/ci/run-buildbot generic-asan" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -359,6 +401,9 @@ - label: "TSAN" command: "libcxx/utils/ci/run-buildbot generic-tsan" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -377,6 +422,9 @@ - label: "UBSAN" command: "libcxx/utils/ci/run-buildbot generic-ubsan" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -395,6 +443,9 @@ - label: "MSAN" command: "libcxx/utils/ci/run-buildbot generic-msan" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -414,6 +465,9 @@ - label: "Bootstrapping build" command: "libcxx/utils/ci/run-buildbot bootstrapping-build" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" - "**/crash_diagnostics/*" @@ -435,6 +489,9 @@ - label: "Static libraries" command: "libcxx/utils/ci/run-buildbot generic-static" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -453,6 +510,9 @@ - label: "Shared library with merged ABI and unwinder libraries" command: "libcxx/utils/ci/run-buildbot generic-merged" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -471,6 +531,9 @@ - label: "Assertions enabled" command: "libcxx/utils/ci/run-buildbot generic-assertions" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -525,6 +588,9 @@ - label: "With LLVM's libunwind" command: "libcxx/utils/ci/run-buildbot generic-with_llvm_unwinder" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -543,6 +609,9 @@ - label: "Modular build with Local Submodule Visibility" command: "libcxx/utils/ci/run-buildbot generic-modules-lsv" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -563,6 +632,9 @@ - label: "No threads" command: "libcxx/utils/ci/run-buildbot generic-no-threads" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -581,6 +653,9 @@ - label: "No filesystem" command: "libcxx/utils/ci/run-buildbot generic-no-filesystem" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -599,6 +674,9 @@ - label: "No random device" command: "libcxx/utils/ci/run-buildbot generic-no-random_device" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -617,6 +695,9 @@ - label: "No locale" command: "libcxx/utils/ci/run-buildbot generic-no-localization" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -635,6 +716,9 @@ - label: "No Unicode" command: "libcxx/utils/ci/run-buildbot generic-no-unicode" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -653,6 +737,9 @@ - label: "No wide characters" command: "libcxx/utils/ci/run-buildbot generic-no-wide-characters" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -671,6 +758,9 @@ - label: "No experimental features" command: "libcxx/utils/ci/run-buildbot generic-no-experimental" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -689,6 +779,9 @@ - label: "No exceptions" command: "libcxx/utils/ci/run-buildbot generic-no-exceptions" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -707,6 +800,9 @@ - label: "Unstable ABI" command: "libcxx/utils/ci/run-buildbot generic-abi-unstable" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -726,6 +822,9 @@ - label: "Benchmarks" command: "libcxx/utils/ci/run-buildbot benchmarks" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -747,6 +846,9 @@ - label: "Clang-cl (DLL)" command: "bash libcxx/utils/ci/run-buildbot clang-cl-dll" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" agents: @@ -760,6 +862,9 @@ - label: "Clang-cl (Static)" command: "bash libcxx/utils/ci/run-buildbot clang-cl-static" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" agents: @@ -773,6 +878,9 @@ - label: "Clang-cl (no vcruntime exceptions)" command: "bash libcxx/utils/ci/run-buildbot clang-cl-no-vcruntime" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" agents: @@ -785,6 +893,9 @@ - label: "MinGW (DLL, x86_64)" command: "bash libcxx/utils/ci/run-buildbot mingw-dll" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" agents: @@ -798,6 +909,9 @@ - label: "MinGW (Static, x86_64)" command: "bash libcxx/utils/ci/run-buildbot mingw-static" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" agents: @@ -811,6 +925,9 @@ - label: "MinGW (DLL, i686)" command: "bash libcxx/utils/ci/run-buildbot mingw-dll-i686" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" agents: @@ -826,6 +943,9 @@ - label: "MacOS x86_64" command: "libcxx/utils/ci/run-buildbot generic-cxx20" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" agents: @@ -841,6 +961,9 @@ - label: "MacOS arm64" command: "libcxx/utils/ci/run-buildbot generic-cxx20" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" agents: @@ -856,6 +979,9 @@ - label: "MacOS with Modules" command: "libcxx/utils/ci/run-buildbot generic-modules" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" agents: @@ -871,6 +997,9 @@ - label: "Apple system" command: "libcxx/utils/ci/run-buildbot apple-system" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" agents: @@ -887,6 +1016,9 @@ - label: "Apple back-deployment macosx10.13" command: "libcxx/utils/ci/run-buildbot apple-system-backdeployment-10.13" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" agents: @@ -902,6 +1034,9 @@ - label: "Apple back-deployment macosx10.15" command: "libcxx/utils/ci/run-buildbot apple-system-backdeployment-10.15" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" agents: @@ -917,6 +1052,9 @@ - label: "Apple back-deployment macosx11.0 arm64" command: "libcxx/utils/ci/run-buildbot apple-system-backdeployment-11.0" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" agents: @@ -932,6 +1070,9 @@ - label: "Apple back-deployment with assertions enabled" command: "libcxx/utils/ci/run-buildbot apple-system-backdeployment-assertions-11.0" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" agents: @@ -948,6 +1089,9 @@ - label: "AArch64" command: "libcxx/utils/ci/run-buildbot aarch64" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" agents: @@ -962,6 +1106,9 @@ - label: "AArch64 -fno-exceptions" command: "libcxx/utils/ci/run-buildbot aarch64-no-exceptions" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" agents: @@ -976,6 +1123,9 @@ - label: "Armv8" command: "libcxx/utils/ci/run-buildbot armv8" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" agents: @@ -990,6 +1140,9 @@ - label: "Armv8 -fno-exceptions" command: "libcxx/utils/ci/run-buildbot armv8-no-exceptions" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" agents: @@ -1004,6 +1157,9 @@ - label: "Armv7" command: "libcxx/utils/ci/run-buildbot armv7" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" agents: @@ -1018,6 +1174,9 @@ - label: "Armv7 -fno-exceptions" command: "libcxx/utils/ci/run-buildbot armv7-no-exceptions" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" agents: @@ -1034,6 +1193,9 @@ - label: "AIX (32-bit)" command: "libcxx/utils/ci/run-buildbot aix" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -1052,6 +1214,9 @@ - label: "AIX (64-bit)" command: "libcxx/utils/ci/run-buildbot aix" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: @@ -1072,6 +1237,9 @@ - label: "FreeBSD 13 amd64" command: "libcxx/utils/ci/run-buildbot generic-cxx23" artifact_paths: + - "**/crash-" + - "**/oom-" + - "**/timeout-" - "**/test-results.xml" - "**/*.abilist" env: diff --git a/libcxx/utils/ci/oss-fuzz.sh b/libcxx/utils/ci/oss-fuzz.sh --- a/libcxx/utils/ci/oss-fuzz.sh +++ b/libcxx/utils/ci/oss-fuzz.sh @@ -19,12 +19,11 @@ -DCMAKE_INSTALL_PREFIX="${INSTALL}" cmake --build ${BUILD} --target install-cxx-headers -for test in libcxx/test/libcxx/fuzzing/*.pass.cpp; do +for test in $(find libcxx/test -name *.fuzz.cpp); do exe="$(basename ${test})" exe="${exe%.pass.cpp}" ${CXX} ${CXXFLAGS} \ - -std=c++14 \ - -DLIBCPP_OSS_FUZZ \ + -std=c++2b \ -D_LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS \ -nostdinc++ -cxx-isystem ${INSTALL}/include/c++/v1 \ -lpthread -ldl \ diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot --- a/libcxx/utils/ci/run-buildbot +++ b/libcxx/utils/ci/run-buildbot @@ -348,18 +348,23 @@ echo "--- Generating CMake" ${CMAKE} \ - -S "${MONOREPO_ROOT}/llvm" \ - -B "${BUILD_DIR}" \ - -GNinja -DCMAKE_MAKE_PROGRAM="${NINJA}" \ - -DCMAKE_CXX_COMPILER_LAUNCHER="ccache" \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ - -DLLVM_ENABLE_PROJECTS="clang" \ - -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ - -DLLVM_RUNTIME_TARGETS="$(${CXX} --print-target-triple)" \ - -DLLVM_TARGETS_TO_BUILD="host" \ - -DRUNTIMES_BUILD_ALLOW_DARWIN=ON \ - -DLLVM_ENABLE_ASSERTIONS=ON + -S "${MONOREPO_ROOT}/llvm" \ + -B "${BUILD_DIR}" \ + -GNinja -DCMAKE_MAKE_PROGRAM="${NINJA}" \ + -DCMAKE_CXX_COMPILER_LAUNCHER="ccache" \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_ENABLE_PROJECTS="clang" \ + -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \ + -DLLVM_RUNTIME_TARGETS="$(${CXX} --print-target-triple)" \ + -DLLVM_TARGETS_TO_BUILD="host" \ + -DCOMPILER_RT_CXX_LIBRARY=libcxx \ + -DRUNTIMES_BUILD_ALLOW_DARWIN=ON \ + + # TODO: This shouldn't be required but check-runtimes doesn't have the necessary dependency on compiler-rt + echo "+++ Build compiler-rt" + ${NINJA} -C "${BUILD_DIR}" compiler-rt echo "+++ Running the libc++ and libc++abi tests" ${NINJA} -C "${BUILD_DIR}" check-runtimes diff --git a/libcxx/utils/libcxx/test/dsl.py b/libcxx/utils/libcxx/test/dsl.py --- a/libcxx/utils/libcxx/test/dsl.py +++ b/libcxx/utils/libcxx/test/dsl.py @@ -148,9 +148,9 @@ @_memoizeExpensiveOperation( - lambda c, p, args=None: (c.substitutions, c.environment, p, args) + lambda c, p, args=None, additional_build_args=None: (c.substitutions, c.environment, p, args, additional_build_args) ) -def programOutput(config, program, args=None): +def programOutput(config, program, args=None, additional_build_args=None): """ Compiles a program for the test target, run it on the test target and return the output. @@ -161,10 +161,14 @@ """ if args is None: args = [] + + if additional_build_args is None: + additional_build_args = [] + with _makeConfigTest(config) as test: with open(test.getSourcePath(), "w") as source: source.write(program) - _, err, exitCode, _, buildcmd = _executeWithFakeConfig(test, ["%{build}"]) + _, err, exitCode, _, buildcmd = _executeWithFakeConfig(test, ["%{{build}} {}".format(" ".join(additional_build_args))]) if exitCode != 0: raise ConfigurationCompilationError( "Failed to build program, cmd:\n{}\nstderr is:\n{}".format( @@ -184,9 +188,9 @@ @_memoizeExpensiveOperation( - lambda c, p, args=None: (c.substitutions, c.environment, p, args) + lambda c, p, args=None, additional_build_args=None: (c.substitutions, c.environment, p, args, additional_build_args) ) -def programSucceeds(config, program, args=None): +def runSucceeds(config, program, args=None, additional_build_args=None): """ Compiles a program for the test target, run it on the test target and return whether it completed successfully. @@ -196,11 +200,25 @@ %{exec} does. """ try: - programOutput(config, program, args) + programOutput(config, program, args, additional_build_args) except ConfigurationRuntimeError: return False return True +def compileAndRunSucceeds(config, program, args=None, additional_build_args=None): + """ + Tries to compile a program for the test target, run it on the test target + and return whether it compiled and ran successfully. + + Note that execution of the program is done through the %{exec} substitution, + which means that the program may be run on a remote host depending on what + %{exec} does. + """ + try: + return runSucceeds(config, program, args, additional_build_args) + except ConfigurationCompilationError: + return False + @_memoizeExpensiveOperation(lambda c, f: (c.substitutions, c.environment, f)) def hasCompileFlag(config, flag): @@ -281,7 +299,7 @@ } #endif """ - return programSucceeds(config, program, args=[pipes.quote(l) for l in locales]) + return runSucceeds(config, program, args=[pipes.quote(l) for l in locales]) @_memoizeExpensiveOperation(lambda c, flags="": (c.substitutions, c.environment, flags)) diff --git a/libcxx/utils/libcxx/test/features.py b/libcxx/utils/libcxx/test/features.py --- a/libcxx/utils/libcxx/test/features.py +++ b/libcxx/utils/libcxx/test/features.py @@ -98,6 +98,17 @@ name="verify-support", when=lambda cfg: hasCompileFlag(cfg, "-Xclang -verify-ignore-unexpected"), ), + Feature( + name="fuzzer-support", + when=lambda cfg: compileAndRunSucceeds(cfg, """ + #include + #include + + extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *, std::size_t) { + return 0; + } + """, args=["-runs=1"], additional_build_args=["-fsanitize=fuzzer"]) + ), Feature( name="non-lockfree-atomics", when=lambda cfg: sourceBuilds( @@ -148,7 +159,7 @@ name="win32-broken-utf8-wchar-ctype", when=lambda cfg: not "_LIBCPP_HAS_NO_LOCALIZATION" in compilerMacros(cfg) and "_WIN32" in compilerMacros(cfg) - and not programSucceeds( + and not runSucceeds( cfg, """ #include @@ -165,7 +176,7 @@ Feature( name="win32-broken-printf-g-precision", when=lambda cfg: "_WIN32" in compilerMacros(cfg) - and not programSucceeds( + and not runSucceeds( cfg, """ #include @@ -183,7 +194,7 @@ Feature( name="glibc-old-ru_RU-decimal-point", when=lambda cfg: not "_LIBCPP_HAS_NO_LOCALIZATION" in compilerMacros(cfg) - and not programSucceeds( + and not runSucceeds( cfg, """ #include @@ -355,7 +366,7 @@ int main(int, char**) { return 0; } """, ) - and programSucceeds( + and runSucceeds( cfg, """ #include diff --git a/libcxx/utils/libcxx/test/format.py b/libcxx/utils/libcxx/test/format.py --- a/libcxx/utils/libcxx/test/format.py +++ b/libcxx/utils/libcxx/test/format.py @@ -218,6 +218,11 @@ automatically marked as UNSUPPORTED if the compiler does not support Clang-verify. + FOO.fuzz.cpp - Compiles with -fsanitize=fuzzer. This is disabled + if the compiler doesn't provide a libfuzzer, or + one that is incompatible with the standard library + being tested. + Substitution requirements =============================== @@ -287,6 +292,7 @@ "[.]gen[.][^.]+$", "[.]verify[.]cpp$", "[.]fail[.]cpp$", + "[.]fuzz[.]cpp$", ] sourcePath = testSuite.getSourcePath(pathInSuite) @@ -310,6 +316,7 @@ "-Xclang -verify -Xclang -verify-ignore-unexpected=note -ferror-limit=0" ) supportsVerify = "verify-support" in test.config.available_features + supportsFuzzer = "fuzzer-support" in test.config.available_features filename = test.path_in_suite[-1] if re.search("[.]sh[.][^.]+$", filename): @@ -362,6 +369,16 @@ "%dbg(EXECUTED AS) %{exec} %t.exe", ] return self._executeShTest(test, litConfig, steps) + elif filename.endswith(".fuzz.cpp"): + if not supportsFuzzer: + return lit.Test.Result( + lit.Test.UNSUPPORTED, + "Test {} requires support for libFuzzer, which isn't supported by the compiler".format(test.getFullName())) + steps = [ + "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -fsanitize=fuzzer -o %t.exe", + "%dbg(EXECUTED AS) %{exec} %t.exe -max_total_time=10 -timeout=10", + ] + return self._executeShTest(test, litConfig, steps) else: return lit.Test.Result( lit.Test.UNRESOLVED, "Unknown test suffix for '{}'".format(filename)