diff --git a/libcxx/test/configs/libcxx-trunk-bundled.cfg.in b/libcxx/test/configs/libcxx-trunk-bundled.cfg.in new file mode 100644 --- /dev/null +++ b/libcxx/test/configs/libcxx-trunk-bundled.cfg.in @@ -0,0 +1,54 @@ +@AUTO_GEN_COMMENT@ + +LIBCXX_ROOT = "@LIBCXX_SOURCE_DIR@" +INSTALL_ROOT = "@CMAKE_BINARY_DIR@" +COMPILER = "@CMAKE_CXX_COMPILER@" +EXEC_ROOT = "@LIBCXX_BINARY_DIR@" +CMAKE_OSX_SYSROOT = "@CMAKE_OSX_SYSROOT@"; + +import os +import pipes +import site +import sys +site.addsitedir(os.path.join(LIBCXX_ROOT, 'utils')) +import libcxx.test.features +import libcxx.test.format +import libcxx.test.newconfig +import libcxx.test.params + +# Configure basic properties of the test suite +config.name = 'libcxx-trunk-bundled' +config.test_source_root = os.path.join(LIBCXX_ROOT, 'test') +config.test_format = libcxx.test.format.CxxStandardLibraryTest(xfail_deferred=True) +config.recursiveExpansionLimit = 10 +config.test_exec_root = EXEC_ROOT + +# Configure basic substitutions +bundlePy = os.path.join(LIBCXX_ROOT, 'utils', 'bundle.py') +config.substitutions.append(('%{cxx}', COMPILER)) +config.substitutions.append(('%{flags}', + '-isysroot {}'.format(CMAKE_OSX_SYSROOT) if CMAKE_OSX_SYSROOT else '' +)) +config.substitutions.append(('%{compile_flags}', + '-nostdinc++ -isystem {} -I {}'.format( + os.path.join(INSTALL_ROOT, 'include', 'c++', 'v1'), + os.path.join(LIBCXX_ROOT, 'test', 'support')) +)) +config.substitutions.append(('%{link_flags}', + '-nostdlib++ {} {} -lpthread'.format( + os.path.join(INSTALL_ROOT, 'lib', 'libc++.a'), + os.path.join(INSTALL_ROOT, 'lib', 'libc++abi.a')) +)) +config.substitutions.append(('%{exec}', + '{} {} %{{exec_args}} --bundle %t.bundle -- '.format( + pipes.quote(sys.executable), + pipes.quote(bundlePy)) +)) + +# Add parameters and features to the config +libcxx.test.newconfig.configure( + libcxx.test.params.DEFAULT_PARAMETERS, + libcxx.test.features.DEFAULT_FEATURES, + config, + lit_config +) diff --git a/libcxx/utils/bundle.py b/libcxx/utils/bundle.py new file mode 100644 --- /dev/null +++ b/libcxx/utils/bundle.py @@ -0,0 +1,167 @@ +# ===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===----------------------------------------------------------------------===## + +""" +Bundles an executable along with its dependencies into a directory suitable +for running an individual test at a later time. +""" + +import argparse +import base64 +from io import BytesIO, SEEK_SET +from pathlib import Path +import subprocess +import tarfile +from typing import Optional + + +class Bundler: + def __init__( + self, + xfail: bool, + env: list[str], + codesign_identity: Optional[str], + ) -> None: + self.xfail = xfail + self.env = env + self.codesign_identity = codesign_identity + + @staticmethod + def is_test_exe(arg: str) -> bool: + # If an argument is a file that ends in `.tmp.exe`, assume it is the name of an + # executable generated by a test file. We call these test-executables below. + # This allows us to do custom processing like codesigning test-executables and + # changing their path when running from the bundle. It's also possible for there + # to be no such executable, for example in the case of a .sh.cpp test. + return arg.endswith(".tmp.exe") and Path(arg).exists() + + def sign_if_needed(self, arg: str) -> None: + assert self.codesign_identity is not None + if self.is_test_exe(arg): + subprocess.run( + ["xcrun", "codesign", "-f", "-s", self.codesign_identity, arg], + check=True, + env={}, + ) + + def sign_inputs(self, inputs: list[str]) -> None: + if self.codesign_identity is None: + return + for arg in inputs: + self.sign_if_needed(arg) + + def create_bundle(self, output: Path, inputs: list[str]) -> None: + raise NotImplementedError + + +class SelfExtractingTarballBundler(Bundler): + def create_bundle(self, output: Path, inputs: list[str]) -> None: + self.sign_inputs(inputs) + output.write_text("\n".join(self.create_script_lines(inputs))) + subprocess.run(["chmod", "+x", str(output)], check=True) + + def bundle_path(self, arg: str) -> Path: + return self.bundle_var / Path(arg).name + + @property + def bundle_var_name(self) -> str: + return "BUNDLE" + + @property + def bundle_var(self) -> Path: + return Path(f"${{{self.bundle_var_name}}}") + + def create_script_lines(self, command: list[str]) -> None: + lines = [ + "#!/bin/sh", + "set -e", + "STATUS=0", + f'{self.bundle_var_name}="$(mktemp -d /tmp/libcxx.XXXXXXXXXX)"', + ] + + # Get the tarball by decoding the end of the shell script, which embeds the + # tarball at the end of itself. Then, untar the dependencies in the temporary + # directory, and remove the intermediate tarball that was extracted from the + # script in the process. + tar_path = self.bundle_var / ".temporary.tar" + lines += [ + f'tail -n +$[ $(grep -n "^BEGIN_TARBALL" "$0" | cut -d ":" -f 1) + 1 ] "${0}" | base64 --decode > "{tar_path}"', + f'tar -xf "{tar_path}" -C "{self.bundle_var}"', + f'rm "{tar_path}"', + ] + + # Make sure all test-executables in the bundle have 'execute' permissions on the + # host where the bundle is run. The host that compiled the test-executable might + # not have a notion of 'executable' permissions. + for exe in (p for p in command if self.is_test_exe(p)): + lines.append(f"chmod +x {self.bundle_path(exe)}") + + # Execute the command in the temporary directory, with the correct environment. + # We tweak the command line to run it from the bundle by transforming the path + # of test-executables to their path in the temporary directory, where we know + # they have been untarred above. + lines.append(f'cd "{self.bundle_var}"') + + if self.env: + lines.append(f"export {' '.join(self.env)}") + + lines.append("set +e") + + lines.append( + " ".join( + str(self.bundle_path(x)) if self.is_test_exe(x) else x for x in command + ) + ) + + comparison = "-eq" if self.xfail else "-ne" + lines.append(f"if [ $? {comparison} 0 ]; then STATUS=1; fi") + + lines.append("set -e") + + # Make sure the temporary directory is removed when we're done. + lines.append(f'rm -r "{self.bundle_var}"') + + # The tarball itself is embedded at the end of the script so that it's + # sufficient to copy the script to get all dependencies. It's simpler than + # having to carry around a separate tarball. + lines += [ + "exit $STATUS", + "BEGIN_TARBALL", + self.create_base64_tar(command).decode("utf-8"), + ] + + return lines + + def create_base64_tar(self, inputs: list[str]) -> bytes: + f = BytesIO() + with tarfile.open(fileobj=f, mode="w") as tarball: + # TODO: Add --dependency to %{exec_args} so we can make these explicit. + for arg in inputs: + path = Path(arg) + if path.exists(): + tarball.add(path, arcname=path.name) + f.seek(0, SEEK_SET) + return base64.b64encode(f.read()) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--bundle", type=Path, required=True) + parser.add_argument("--codesign_identity") + parser.add_argument("--env", nargs="*", default=[]) + parser.add_argument("--xfail", action="store_true") + parser.add_argument("command", nargs="+") + args = parser.parse_args() + + SelfExtractingTarballBundler( + args.xfail, args.env, args.codesign_identity + ).create_bundle(args.bundle, args.command) + + +if __name__ == "__main__": + main() diff --git a/libcxx/utils/libcxx/test/format.py b/libcxx/utils/libcxx/test/format.py --- a/libcxx/utils/libcxx/test/format.py +++ b/libcxx/utils/libcxx/test/format.py @@ -9,9 +9,7 @@ import lit import lit.formats import os -import pipes import re -import shutil import subprocess def _supportsVerify(config): @@ -102,6 +100,12 @@ substitutions = [(s, x + ' ' + ' '.join(additionalCompileFlags)) if s == '%{compile_flags}' else (s, x) for (s, x) in substitutions] + exec_args = [] + if test.isExpectedToFail(): + exec_args.append("--xfail") + + substitutions.append(("%{exec_args}", " ".join(exec_args))) + # Perform substitutions in the script itself. script = lit.TestRunner.applySubstitutions(script, substitutions, recursion_limit=test.config.recursiveExpansionLimit) @@ -192,6 +196,10 @@ Equivalent to `%{exec} %t.exe`. This is intended to be used in conjunction with the %{build} substitution. """ + + def __init__(self, xfail_deferred: bool = False) -> None: + self.xfail_deferred = xfail_deferred + def getTestsInDirectory(self, testSuite, pathInSuite, litConfig, localConfig): SUPPORTED_SUFFIXES = ['[.]pass[.]cpp$', '[.]pass[.]mm$', '[.]compile[.]pass[.]cpp$', '[.]compile[.]fail[.]cpp$', @@ -269,7 +277,7 @@ "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe", "%dbg(EXECUTED AS) %{exec} %t.exe" ] - return self._executeShTest(test, litConfig, steps) + return self._executeShTest(test, litConfig, steps, self.xfail_deferred) # This is like a .verify.cpp test when clang-verify is supported, # otherwise it's like a .compile.fail.cpp test. This is only provided # for backwards compatibility with the test suite. @@ -291,7 +299,7 @@ string = ' '.join(flags) config.substitutions = [(s, x + ' ' + string) if s == '%{compile_flags}' else (s, x) for (s, x) in config.substitutions] - def _executeShTest(self, test, litConfig, steps): + def _executeShTest(self, test, litConfig, steps, xfail_deferred: bool = False): if test.config.unsupported: return lit.Test.Result(lit.Test.UNSUPPORTED, 'Test is unsupported') @@ -301,7 +309,11 @@ if litConfig.noExecute: return lit.Test.Result(lit.Test.XFAIL if test.isExpectedToFail() else lit.Test.PASS) - else: - _, tmpBase = _getTempPaths(test) - useExternalSh = False - return lit.TestRunner._runShTest(test, litConfig, useExternalSh, script, tmpBase) + + _, tmpBase = _getTempPaths(test) + result = lit.TestRunner._runShTest( + test, litConfig, useExternalSh=False, script=script, tmpBase=tmpBase + ) + if xfail_deferred: + test.xfails = [] + return result