diff --git a/libcxx/test/configs/libcxx-trunk-bundled.cfg.in b/libcxx/test/configs/libcxx-trunk-bundled.cfg.in new file mode 100644 --- /dev/null +++ b/libcxx/test/configs/libcxx-trunk-bundled.cfg.in @@ -0,0 +1,57 @@ +@AUTO_GEN_COMMENT@ + +LIBCXX_ROOT = "@LIBCXX_SOURCE_DIR@" +INSTALL_ROOT = "@CMAKE_BINARY_DIR@" +COMPILER = "@CMAKE_CXX_COMPILER@" +EXEC_ROOT = "@LIBCXX_BINARY_DIR@" +CMAKE_OSX_SYSROOT = "@CMAKE_OSX_SYSROOT@"; + +import os +import pipes +import site +import sys +site.addsitedir(os.path.join(LIBCXX_ROOT, 'utils')) +import libcxx.test.features +import libcxx.test.format +import libcxx.test.newconfig +import libcxx.test.params + +# Configure basic properties of the test suite +config.name = 'libcxx-trunk-bundled' +config.target_triple = "@TARGET_TRIPLE@" +config.test_source_root = os.path.join(LIBCXX_ROOT, 'test') +config.test_format = libcxx.test.format.CxxStandardLibraryTest() +config.recursiveExpansionLimit = 10 +config.test_exec_root = EXEC_ROOT + +# Configure basic substitutions +bundlePy = os.path.join(LIBCXX_ROOT, 'utils', 'bundle.py') +config.substitutions.append(('%{cxx}', COMPILER)) +config.substitutions.append(('%{flags}', + '-isysroot {}'.format(CMAKE_OSX_SYSROOT) if CMAKE_OSX_SYSROOT else '' +)) +config.substitutions.append(('%{compile_flags}', + '-nostdinc++ -isystem {} -I {}'.format( + os.path.join(INSTALL_ROOT, 'include', 'c++', 'v1'), + os.path.join(LIBCXX_ROOT, 'test', 'support')) +)) +config.substitutions.append(('%{link_flags}', + '-nostdlib++ {} {} -latomic -lpthread'.format( + os.path.join(INSTALL_ROOT, 'lib', 'libc++.a'), + os.path.join(INSTALL_ROOT, 'lib', 'libc++abi.a')) +)) +config.substitutions.append(('%{exec}', + '{} {} %{{exec_args}} --bundle %t.bundle -- '.format( + pipes.quote(sys.executable), + pipes.quote(bundlePy)) +)) + +config.available_features.add("run-deferred") + +# Add parameters and features to the config +libcxx.test.newconfig.configure( + libcxx.test.params.DEFAULT_PARAMETERS, + libcxx.test.features.DEFAULT_FEATURES, + config, + lit_config +) diff --git a/libcxx/test/libcxx/debug/extern-templates.sh.cpp b/libcxx/test/libcxx/debug/extern-templates.sh.cpp --- a/libcxx/test/libcxx/debug/extern-templates.sh.cpp +++ b/libcxx/test/libcxx/debug/extern-templates.sh.cpp @@ -16,6 +16,7 @@ // XFAIL: LIBCXX-WINDOWS-FIXME +// BUILT_DEPENDENCIES: %t.lib // RUN: %{cxx} %{flags} %{compile_flags} %s %{link_flags} -fPIC -DTU1 -D_LIBCPP_DEBUG=1 -fvisibility=hidden -shared -o %t.lib // RUN: cd %T && %{cxx} %{flags} %{compile_flags} %s ./%basename_t.tmp.lib %{link_flags} -fPIC -DTU2 -D_LIBCPP_DEBUG=1 -fvisibility=hidden -o %t.exe // RUN: %{exec} %t.exe diff --git a/libcxx/test/libcxx/selftest/dsl/dsl.sh.py b/libcxx/test/libcxx/selftest/dsl/dsl.sh.py --- a/libcxx/test/libcxx/selftest/dsl/dsl.sh.py +++ b/libcxx/test/libcxx/selftest/dsl/dsl.sh.py @@ -6,6 +6,7 @@ # #===----------------------------------------------------------------------===## +# UNSUPPORTED: run-deferred # XFAIL: LIBCXX-WINDOWS-FIXME # Note: We prepend arguments with 'x' to avoid thinking there are too few diff --git a/libcxx/test/libcxx/selftest/pass.cpp/compile-error.pass.cpp b/libcxx/test/libcxx/selftest/pass.cpp/compile-error.pass.cpp --- a/libcxx/test/libcxx/selftest/pass.cpp/compile-error.pass.cpp +++ b/libcxx/test/libcxx/selftest/pass.cpp/compile-error.pass.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -// XFAIL: * +// XFAIL-BUILD: * // Make sure the test DOES NOT pass if it fails at compile-time diff --git a/libcxx/test/libcxx/selftest/pass.cpp/link-error.pass.cpp b/libcxx/test/libcxx/selftest/pass.cpp/link-error.pass.cpp --- a/libcxx/test/libcxx/selftest/pass.cpp/link-error.pass.cpp +++ b/libcxx/test/libcxx/selftest/pass.cpp/link-error.pass.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -// XFAIL: * +// XFAIL-BUILD: * // Make sure the test DOES NOT pass if it fails at link-time diff --git a/libcxx/test/libcxx/selftest/pass.cpp/werror.pass.cpp b/libcxx/test/libcxx/selftest/pass.cpp/werror.pass.cpp --- a/libcxx/test/libcxx/selftest/pass.cpp/werror.pass.cpp +++ b/libcxx/test/libcxx/selftest/pass.cpp/werror.pass.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -// XFAIL: * +// XFAIL-BUILD: * // Make sure that even a simple unused variable warning is treated as an // error in the test suite. This is to make sure the test suite always runs diff --git a/libcxx/test/libcxx/selftest/pass.mm/compile-error.pass.mm b/libcxx/test/libcxx/selftest/pass.mm/compile-error.pass.mm --- a/libcxx/test/libcxx/selftest/pass.mm/compile-error.pass.mm +++ b/libcxx/test/libcxx/selftest/pass.mm/compile-error.pass.mm @@ -8,7 +8,7 @@ // REQUIRES: objective-c++ -// XFAIL: * +// XFAIL-BUILD: * // Make sure the test DOES NOT pass if it fails at compile-time diff --git a/libcxx/test/libcxx/selftest/pass.mm/link-error.pass.mm b/libcxx/test/libcxx/selftest/pass.mm/link-error.pass.mm --- a/libcxx/test/libcxx/selftest/pass.mm/link-error.pass.mm +++ b/libcxx/test/libcxx/selftest/pass.mm/link-error.pass.mm @@ -8,7 +8,7 @@ // REQUIRES: objective-c++ -// XFAIL: * +// XFAIL-BUILD: * // Make sure the test DOES NOT pass if it fails at link-time diff --git a/libcxx/test/libcxx/strings/c.strings/version_cuchar.pass.cpp b/libcxx/test/libcxx/strings/c.strings/version_cuchar.pass.cpp --- a/libcxx/test/libcxx/strings/c.strings/version_cuchar.pass.cpp +++ b/libcxx/test/libcxx/strings/c.strings/version_cuchar.pass.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // -// XFAIL: * +// XFAIL-BUILD: * // Skip this test on windows. If built on top of the MSVC runtime, the // header actually does exist (although not provided by us). diff --git a/libcxx/test/std/input.output/filesystems/class.path/path.nonmember/path.io.unicode_bug.pass.cpp b/libcxx/test/std/input.output/filesystems/class.path/path.nonmember/path.io.unicode_bug.pass.cpp --- a/libcxx/test/std/input.output/filesystems/class.path/path.nonmember/path.io.unicode_bug.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/class.path/path.nonmember/path.io.unicode_bug.pass.cpp @@ -24,7 +24,7 @@ // TODO(EricWF) This test fails because "std::quoted" fails to compile // for char16_t and char32_t types. Combine with path.io.pass.cpp when this // passes. -// XFAIL: * +// XFAIL-BUILD: * #include "filesystem_include.h" #include diff --git a/libcxx/test/std/strings/c.strings/cuchar.pass.cpp b/libcxx/test/std/strings/c.strings/cuchar.pass.cpp --- a/libcxx/test/std/strings/c.strings/cuchar.pass.cpp +++ b/libcxx/test/std/strings/c.strings/cuchar.pass.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // -// XFAIL: libc++ +// XFAIL-BUILD: libc++ // Skip this test on windows. If built on top of the MSVC runtime, the // header actually does exist (although not provided by us). diff --git a/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/default.pass.cpp b/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/default.pass.cpp --- a/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/default.pass.cpp +++ b/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/default.pass.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14 -// XFAIL: libc++ +// XFAIL-BUILD: libc++ // diff --git a/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/hash.pass.cpp b/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/hash.pass.cpp --- a/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/hash.pass.cpp +++ b/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/hash.pass.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14 -// XFAIL: libc++ +// XFAIL-BUILD: libc++ // diff --git a/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/hash.pred.pass.cpp b/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/hash.pred.pass.cpp --- a/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/hash.pred.pass.cpp +++ b/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/hash.pred.pass.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14 -// XFAIL: libc++ +// XFAIL-BUILD: libc++ // diff --git a/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/pred.pass.cpp b/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/pred.pass.cpp --- a/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/pred.pass.cpp +++ b/libcxx/test/std/utilities/function.objects/func.search/func.search.bm/pred.pass.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14 -// XFAIL: libc++ +// XFAIL-BUILD: libc++ // diff --git a/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/default.pass.cpp b/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/default.pass.cpp --- a/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/default.pass.cpp +++ b/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/default.pass.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14 -// XFAIL: libc++ +// XFAIL-BUILD: libc++ // diff --git a/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/hash.pass.cpp b/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/hash.pass.cpp --- a/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/hash.pass.cpp +++ b/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/hash.pass.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14 -// XFAIL: libc++ +// XFAIL-BUILD: libc++ // diff --git a/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/hash.pred.pass.cpp b/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/hash.pred.pass.cpp --- a/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/hash.pred.pass.cpp +++ b/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/hash.pred.pass.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14 -// XFAIL: libc++ +// XFAIL-BUILD: libc++ // diff --git a/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/pred.pass.cpp b/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/pred.pass.cpp --- a/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/pred.pass.cpp +++ b/libcxx/test/std/utilities/function.objects/func.search/func.search.bmh/pred.pass.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14 -// XFAIL: libc++ +// XFAIL-BUILD: libc++ // diff --git a/libcxx/test/std/utilities/time/time.cal/time.cal.day/time.cal.day.nonmembers/streaming.pass.cpp b/libcxx/test/std/utilities/time/time.cal/time.cal.day/time.cal.day.nonmembers/streaming.pass.cpp --- a/libcxx/test/std/utilities/time/time.cal/time.cal.day/time.cal.day.nonmembers/streaming.pass.cpp +++ b/libcxx/test/std/utilities/time/time.cal/time.cal.day/time.cal.day.nonmembers/streaming.pass.cpp @@ -8,7 +8,7 @@ // UNSUPPORTED: c++03, c++11, c++14, c++17 // UNSUPPORTED: libcpp-has-no-localization -// XFAIL: * +// XFAIL-BUILD: * // // class day; diff --git a/libcxx/test/std/utilities/time/time.cal/time.cal.md/time.cal.md.nonmembers/streaming.pass.cpp b/libcxx/test/std/utilities/time/time.cal/time.cal.md/time.cal.md.nonmembers/streaming.pass.cpp --- a/libcxx/test/std/utilities/time/time.cal/time.cal.md/time.cal.md.nonmembers/streaming.pass.cpp +++ b/libcxx/test/std/utilities/time/time.cal/time.cal.md/time.cal.md.nonmembers/streaming.pass.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// XFAIL: * +// XFAIL-BUILD: * // // class month_day; diff --git a/libcxx/test/std/utilities/time/time.cal/time.cal.mdlast/streaming.pass.cpp b/libcxx/test/std/utilities/time/time.cal/time.cal.mdlast/streaming.pass.cpp --- a/libcxx/test/std/utilities/time/time.cal/time.cal.mdlast/streaming.pass.cpp +++ b/libcxx/test/std/utilities/time/time.cal/time.cal.mdlast/streaming.pass.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// XFAIL: * +// XFAIL-BUILD: * // // class month_day_last; diff --git a/libcxx/test/std/utilities/time/time.cal/time.cal.month/time.cal.month.nonmembers/streaming.pass.cpp b/libcxx/test/std/utilities/time/time.cal/time.cal.month/time.cal.month.nonmembers/streaming.pass.cpp --- a/libcxx/test/std/utilities/time/time.cal/time.cal.month/time.cal.month.nonmembers/streaming.pass.cpp +++ b/libcxx/test/std/utilities/time/time.cal/time.cal.month/time.cal.month.nonmembers/streaming.pass.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// XFAIL: * +// XFAIL-BUILD: * // // class month; diff --git a/libcxx/test/std/utilities/time/time.cal/time.cal.mwd/time.cal.mwd.nonmembers/streaming.pass.cpp b/libcxx/test/std/utilities/time/time.cal/time.cal.mwd/time.cal.mwd.nonmembers/streaming.pass.cpp --- a/libcxx/test/std/utilities/time/time.cal/time.cal.mwd/time.cal.mwd.nonmembers/streaming.pass.cpp +++ b/libcxx/test/std/utilities/time/time.cal/time.cal.mwd/time.cal.mwd.nonmembers/streaming.pass.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// XFAIL: * +// XFAIL-BUILD: * // // class month_weekday; diff --git a/libcxx/test/std/utilities/time/time.cal/time.cal.mwdlast/time.cal.mwdlast.nonmembers/streaming.pass.cpp b/libcxx/test/std/utilities/time/time.cal/time.cal.mwdlast/time.cal.mwdlast.nonmembers/streaming.pass.cpp --- a/libcxx/test/std/utilities/time/time.cal/time.cal.mwdlast/time.cal.mwdlast.nonmembers/streaming.pass.cpp +++ b/libcxx/test/std/utilities/time/time.cal/time.cal.mwdlast/time.cal.mwdlast.nonmembers/streaming.pass.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// XFAIL: * +// XFAIL-BUILD: * // // class month_weekday_last; diff --git a/libcxx/test/std/utilities/time/time.cal/time.cal.wdidx/time.cal.wdidx.nonmembers/streaming.pass.cpp b/libcxx/test/std/utilities/time/time.cal/time.cal.wdidx/time.cal.wdidx.nonmembers/streaming.pass.cpp --- a/libcxx/test/std/utilities/time/time.cal/time.cal.wdidx/time.cal.wdidx.nonmembers/streaming.pass.cpp +++ b/libcxx/test/std/utilities/time/time.cal/time.cal.wdidx/time.cal.wdidx.nonmembers/streaming.pass.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// XFAIL: * +// XFAIL-BUILD: * // // class weekday_indexed; diff --git a/libcxx/test/std/utilities/time/time.cal/time.cal.wdlast/time.cal.wdlast.nonmembers/streaming.pass.cpp b/libcxx/test/std/utilities/time/time.cal/time.cal.wdlast/time.cal.wdlast.nonmembers/streaming.pass.cpp --- a/libcxx/test/std/utilities/time/time.cal/time.cal.wdlast/time.cal.wdlast.nonmembers/streaming.pass.cpp +++ b/libcxx/test/std/utilities/time/time.cal/time.cal.wdlast/time.cal.wdlast.nonmembers/streaming.pass.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// XFAIL: * +// XFAIL-BUILD: * // // class weekday_last; diff --git a/libcxx/test/std/utilities/time/time.cal/time.cal.weekday/time.cal.weekday.nonmembers/streaming.pass.cpp b/libcxx/test/std/utilities/time/time.cal/time.cal.weekday/time.cal.weekday.nonmembers/streaming.pass.cpp --- a/libcxx/test/std/utilities/time/time.cal/time.cal.weekday/time.cal.weekday.nonmembers/streaming.pass.cpp +++ b/libcxx/test/std/utilities/time/time.cal/time.cal.weekday/time.cal.weekday.nonmembers/streaming.pass.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// XFAIL: * +// XFAIL-BUILD: * // // class weekday; diff --git a/libcxx/test/std/utilities/time/time.cal/time.cal.year/time.cal.year.nonmembers/streaming.pass.cpp b/libcxx/test/std/utilities/time/time.cal/time.cal.year/time.cal.year.nonmembers/streaming.pass.cpp --- a/libcxx/test/std/utilities/time/time.cal/time.cal.year/time.cal.year.nonmembers/streaming.pass.cpp +++ b/libcxx/test/std/utilities/time/time.cal/time.cal.year/time.cal.year.nonmembers/streaming.pass.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// XFAIL: * +// XFAIL-BUILD: * // // class year; diff --git a/libcxx/test/std/utilities/time/time.cal/time.cal.ym/time.cal.ym.nonmembers/streaming.pass.cpp b/libcxx/test/std/utilities/time/time.cal/time.cal.ym/time.cal.ym.nonmembers/streaming.pass.cpp --- a/libcxx/test/std/utilities/time/time.cal/time.cal.ym/time.cal.ym.nonmembers/streaming.pass.cpp +++ b/libcxx/test/std/utilities/time/time.cal/time.cal.ym/time.cal.ym.nonmembers/streaming.pass.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// XFAIL: * +// XFAIL-BUILD: * // // class year_month; diff --git a/libcxx/test/std/utilities/time/time.cal/time.cal.ymd/time.cal.ymd.nonmembers/streaming.pass.cpp b/libcxx/test/std/utilities/time/time.cal/time.cal.ymd/time.cal.ymd.nonmembers/streaming.pass.cpp --- a/libcxx/test/std/utilities/time/time.cal/time.cal.ymd/time.cal.ymd.nonmembers/streaming.pass.cpp +++ b/libcxx/test/std/utilities/time/time.cal/time.cal.ymd/time.cal.ymd.nonmembers/streaming.pass.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// XFAIL: * +// XFAIL-BUILD: * // // class year_month_day; diff --git a/libcxx/test/std/utilities/time/time.cal/time.cal.ymdlast/time.cal.ymdlast.nonmembers/streaming.pass.cpp b/libcxx/test/std/utilities/time/time.cal/time.cal.ymdlast/time.cal.ymdlast.nonmembers/streaming.pass.cpp --- a/libcxx/test/std/utilities/time/time.cal/time.cal.ymdlast/time.cal.ymdlast.nonmembers/streaming.pass.cpp +++ b/libcxx/test/std/utilities/time/time.cal/time.cal.ymdlast/time.cal.ymdlast.nonmembers/streaming.pass.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// XFAIL: * +// XFAIL-BUILD: * // // class year_month_day_last; diff --git a/libcxx/test/std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.nonmembers/streaming.pass.cpp b/libcxx/test/std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.nonmembers/streaming.pass.cpp --- a/libcxx/test/std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.nonmembers/streaming.pass.cpp +++ b/libcxx/test/std/utilities/time/time.cal/time.cal.ymwd/time.cal.ymwd.nonmembers/streaming.pass.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// XFAIL: * +// XFAIL-BUILD: * // // class year_month_weekday; diff --git a/libcxx/test/std/utilities/time/time.cal/time.cal.ymwdlast/time.cal.ymwdlast.nonmembers/streaming.pass.cpp b/libcxx/test/std/utilities/time/time.cal/time.cal.ymwdlast/time.cal.ymwdlast.nonmembers/streaming.pass.cpp --- a/libcxx/test/std/utilities/time/time.cal/time.cal.ymwdlast/time.cal.ymwdlast.nonmembers/streaming.pass.cpp +++ b/libcxx/test/std/utilities/time/time.cal/time.cal.ymwdlast/time.cal.ymwdlast.nonmembers/streaming.pass.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// XFAIL: * +// XFAIL-BUILD: * // // class year_month_weekday_last; diff --git a/libcxx/utils/bundle.py b/libcxx/utils/bundle.py new file mode 100644 --- /dev/null +++ b/libcxx/utils/bundle.py @@ -0,0 +1,175 @@ +# ===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===----------------------------------------------------------------------===## + +""" +Bundles an executable along with its dependencies into a directory suitable +for running an individual test at a later time. +""" + +import argparse +import base64 +from io import BytesIO, SEEK_SET +from pathlib import Path +import subprocess +import tarfile +from typing import Optional + + +class Bundler: + def __init__( + self, + xfail: bool, + dependencies: list[Path], + env: list[str], + codesign_identity: Optional[str], + ) -> None: + self.xfail = xfail + self.dependencies = dependencies + self.env = env + self.codesign_identity = codesign_identity + + @staticmethod + def is_test_exe(arg: str) -> bool: + # If an argument is a file that ends in `.tmp.exe`, assume it is the name of an + # executable generated by a test file. We call these test-executables below. + # This allows us to do custom processing like codesigning test-executables and + # changing their path when running from the bundle. It's also possible for there + # to be no such executable, for example in the case of a .sh.cpp test. + return arg.endswith(".tmp.exe") and Path(arg).exists() + + def sign_if_needed(self, path: Path) -> None: + assert self.codesign_identity is not None + if self.is_test_exe(str(path)): + subprocess.run( + ["xcrun", "codesign", "-f", "-s", self.codesign_identity, str(path)], + check=True, + env={}, + ) + + def sign_binaries(self) -> None: + if self.codesign_identity is None: + return + for dep in self.dependencies: + self.sign_if_needed(dep) + + def create_bundle(self, output: Path, command: list[str]) -> None: + raise NotImplementedError + + +class SelfExtractingTarballBundler(Bundler): + def create_bundle(self, output: Path, command: list[str]) -> None: + self.sign_binaries() + output.write_text("\n".join(self.create_script_lines(command))) + subprocess.run(["chmod", "+x", str(output)], check=True) + + def bundle_path(self, path: Path) -> Path: + return self.bundle_var / path.name + + @property + def bundle_var_name(self) -> str: + return "BUNDLE" + + @property + def bundle_var(self) -> Path: + return Path(f"${{{self.bundle_var_name}}}") + + def substitute_file_paths(self, arg: str) -> str: + for dep in self.dependencies: + if not dep.is_absolute: + continue + if str(dep) in arg: + arg = arg.replace(str(dep), str(self.bundle_path(dep))) + return f'"{arg}"' + + def create_script_lines(self, command: list[str]) -> None: + lines = [ + "#!/bin/sh", + "set -e", + "STATUS=0", + f'{self.bundle_var_name}="$(mktemp -d /tmp/libcxx.XXXXXXXXXX)"', + ] + + # Get the tarball by decoding the end of the shell script, which embeds the + # tarball at the end of itself. Then, untar the dependencies in the temporary + # directory, and remove the intermediate tarball that was extracted from the + # script in the process. + tar_path = self.bundle_var / ".temporary.tar" + lines += [ + f'tail -n +$[ $(grep -n "^BEGIN_TARBALL" "$0" | cut -d ":" -f 1) + 1 ] "${0}" | base64 --decode > "{tar_path}"', + f'tar -xf "{tar_path}" -C "{self.bundle_var}"', + f'rm "{tar_path}"', + ] + + # Make sure all test-executables in the bundle have 'execute' permissions on the + # host where the bundle is run. The host that compiled the test-executable might + # not have a notion of 'executable' permissions. + for exe in (d for d in self.dependencies if self.is_test_exe(str(d))): + lines.append(f"chmod +x {self.bundle_path(exe)}") + + # Execute the command in the temporary directory, with the correct environment. + # We tweak the command line to run it from the bundle by transforming the path + # of test-executables to their path in the temporary directory, where we know + # they have been untarred above. + lines.append(f'cd "{self.bundle_var}"') + + if self.env: + lines.append(f"export {' '.join(self.env)}") + + lines.append("set +e") + + lines.append(" ".join(self.substitute_file_paths(x) for x in command)) + + comparison = "-eq" if self.xfail else "-ne" + lines.append(f"if [ $? {comparison} 0 ]; then STATUS=1; fi") + + lines.append("set -e") + + # Make sure the temporary directory is removed when we're done. + lines.append(f'rm -r "{self.bundle_var}"') + + # The tarball itself is embedded at the end of the script so that it's + # sufficient to copy the script to get all dependencies. It's simpler than + # having to carry around a separate tarball. + lines += [ + "exit $STATUS", + "BEGIN_TARBALL", + self.create_base64_tar().decode("utf-8"), + ] + + return lines + + def create_base64_tar(self) -> bytes: + f = BytesIO() + with tarfile.open(fileobj=f, mode="w") as tarball: + for dep in self.dependencies: + if not dep.exists(): + raise RuntimeError(f"Missing test dependency: {dep}") + tarball.add(dep, arcname=dep.name) + f.seek(0, SEEK_SET) + return base64.b64encode(f.read()) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--bundle", type=Path, required=True) + parser.add_argument("--codesign_identity") + parser.add_argument("--env", nargs="*", default=[]) + parser.add_argument( + "--dependency", dest="dependencies", action="append", type=Path, default=[] + ) + parser.add_argument("--xfail", action="store_true") + parser.add_argument("command", nargs="+") + args = parser.parse_args() + + SelfExtractingTarballBundler( + args.xfail, args.dependencies, args.env, args.codesign_identity + ).create_bundle(args.bundle, args.command) + + +if __name__ == "__main__": + main() diff --git a/libcxx/utils/libcxx/test/features.py b/libcxx/utils/libcxx/test/features.py --- a/libcxx/utils/libcxx/test/features.py +++ b/libcxx/utils/libcxx/test/features.py @@ -213,4 +213,4 @@ Feature(name='__dummy_use_system_cxx_lib', when=lambda cfg: 'use_system_cxx_lib' in cfg.available_features, actions=_addSystemCxxLibDeclinations) -] +] \ No newline at end of file diff --git a/libcxx/utils/libcxx/test/format.py b/libcxx/utils/libcxx/test/format.py --- a/libcxx/utils/libcxx/test/format.py +++ b/libcxx/utils/libcxx/test/format.py @@ -6,12 +6,11 @@ # #===----------------------------------------------------------------------===## +from pathlib import Path import lit import lit.formats import os -import pipes import re -import shutil import subprocess def _supportsVerify(config): @@ -74,13 +73,21 @@ # Parse the test file, including custom directives additionalCompileFlags = [] fileDependencies = [] + builtDependencies = ["%t.exe"] + test.build_xfails = [] parsers = [ lit.TestRunner.IntegratedTestKeywordParser('FILE_DEPENDENCIES:', lit.TestRunner.ParserKind.LIST, initial_value=fileDependencies), + lit.TestRunner.IntegratedTestKeywordParser('BUILT_DEPENDENCIES:', + lit.TestRunner.ParserKind.LIST, + initial_value=builtDependencies), lit.TestRunner.IntegratedTestKeywordParser('ADDITIONAL_COMPILE_FLAGS:', lit.TestRunner.ParserKind.LIST, - initial_value=additionalCompileFlags) + initial_value=additionalCompileFlags), + lit.TestRunner.IntegratedTestKeywordParser('XFAIL-BUILD:', + lit.TestRunner.ParserKind.LIST, + initial_value=test.build_xfails), ] scriptInTest = lit.TestRunner.parseIntegratedTestScript(test, additional_parsers=parsers, @@ -88,13 +95,29 @@ if isinstance(scriptInTest, lit.Test.Result): return scriptInTest + # Copy any XFAIL-BUILD commands into the regular XFAIL set for the non-deferred case. + test.xfails.extend(test.build_xfails) + script = [] + exec_args = [] + if test.isExpectedToFail(): + exec_args.append("--xfail") + + # TODO: Add --test-exe flag to clean up bundler implementation. + exec_args.extend(lit.TestRunner.applySubstitutions( + (f"--dependency={d}" for d in builtDependencies), + substitutions, + recursion_limit=test.config.recursiveExpansionLimit, + )) + # For each file dependency in FILE_DEPENDENCIES, inject a command to copy # that file to the execution directory. Execute the copy from %S to allow # relative paths from the test directory. + test_dir = Path(test.getSourcePath()).parent for dep in fileDependencies: script += ['%dbg(SETUP) cd %S && cp {} %T'.format(dep)] + exec_args.append(f"--dependency={test_dir / dep}") script += preamble script += scriptInTest @@ -102,6 +125,8 @@ substitutions = [(s, x + ' ' + ' '.join(additionalCompileFlags)) if s == '%{compile_flags}' else (s, x) for (s, x) in substitutions] + substitutions.append(("%{exec_args}", " ".join(exec_args))) + # Perform substitutions in the script itself. script = lit.TestRunner.applySubstitutions(script, substitutions, recursion_limit=test.config.recursiveExpansionLimit) @@ -192,6 +217,7 @@ Equivalent to `%{exec} %t.exe`. This is intended to be used in conjunction with the %{build} substitution. """ + def getTestsInDirectory(self, testSuite, pathInSuite, litConfig, localConfig): SUPPORTED_SUFFIXES = ['[.]pass[.]cpp$', '[.]pass[.]mm$', '[.]compile[.]pass[.]cpp$', '[.]compile[.]fail[.]cpp$', @@ -269,7 +295,7 @@ "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe", "%dbg(EXECUTED AS) %{exec} %t.exe" ] - return self._executeShTest(test, litConfig, steps) + return self._executeShTest(test, litConfig, steps, run_deferrable=True) # This is like a .verify.cpp test when clang-verify is supported, # otherwise it's like a .compile.fail.cpp test. This is only provided # for backwards compatibility with the test suite. @@ -291,7 +317,7 @@ string = ' '.join(flags) config.substitutions = [(s, x + ' ' + string) if s == '%{compile_flags}' else (s, x) for (s, x) in config.substitutions] - def _executeShTest(self, test, litConfig, steps): + def _executeShTest(self, test, litConfig, steps, run_deferrable: bool = False): if test.config.unsupported: return lit.Test.Result(lit.Test.UNSUPPORTED, 'Test is unsupported') @@ -301,7 +327,11 @@ if litConfig.noExecute: return lit.Test.Result(lit.Test.XFAIL if test.isExpectedToFail() else lit.Test.PASS) - else: - _, tmpBase = _getTempPaths(test) - useExternalSh = False - return lit.TestRunner._runShTest(test, litConfig, useExternalSh, script, tmpBase) + + _, tmpBase = _getTempPaths(test) + result = lit.TestRunner._runShTest( + test, litConfig, useExternalSh=False, script=script, tmpBase=tmpBase + ) + if run_deferrable and "run-deferred" in test.config.available_features: + test.xfails = test.build_xfails + return result diff --git a/libcxx/utils/run_bundles.py b/libcxx/utils/run_bundles.py new file mode 100755 --- /dev/null +++ b/libcxx/utils/run_bundles.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +from argparse import ArgumentParser +import concurrent.futures +from concurrent.futures import ProcessPoolExecutor +from dataclasses import dataclass +import logging +from pathlib import Path +import subprocess +import sys + + +@dataclass(frozen=True) +class TestResult: + path: Path + passed: bool + output: str + + @property + def failed(self) -> bool: + return not self.passed + + def __str__(self) -> str: + status = "PASS" if self.passed else "FAIL" + output = f"\n{self.output}" if self.output else "" + return f"{status}: {self.path}{output}" + + +def run_bundle(path: Path) -> TestResult: + result = subprocess.run( + [str(path)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + try: + output = result.stdout.decode("utf-8") + except UnicodeDecodeError: + output = repr(result.stdout) + return TestResult(path, result.returncode == 0, output) + + +class BundleRunner: + def __init__(self, test_dir: Path) -> None: + self.test_dir = test_dir + + def run_all(self) -> bool: + failed = False + with ProcessPoolExecutor() as pool: + futures = [] + for bundle in self.test_dir.glob("**/*.bundle"): + logging.debug("Submitting work for %s", bundle) + futures.append(pool.submit(run_bundle, bundle)) + for future in concurrent.futures.as_completed(futures): + result = future.result() + if result.passed: + logging.info(result) + else: + failed = True + logging.error(result) + return failed + + +def main() -> None: + parser = ArgumentParser() + parser.add_argument("-v", "--verbose", action="count", default=0) + parser.add_argument("test_dir", type=Path) + args = parser.parse_args() + + if args.verbose >= 2: + level = logging.DEBUG + elif args.verbose == 1: + level = logging.INFO + else: + level = logging.WARNING + logging.basicConfig(level=level) + if BundleRunner(args.test_dir).run_all(): + sys.exit("Tests failed") + + +if __name__ == "__main__": + main()