Index: docs/CommandGuide/lit.rst =================================================================== --- docs/CommandGuide/lit.rst +++ docs/CommandGuide/lit.rst @@ -358,6 +358,11 @@ **available_features** A set of features that can be used in `XFAIL`, `REQUIRES`, and `UNSUPPORTED` directives. + **setup_script** A script to run before any of the tests in this suite are + executed, or None. May be relative to the directory containing the + configuration file. Caution: this script will be run for this directory and + *once for every subdirectory.* + TEST DISCOVERY ~~~~~~~~~~~~~~ @@ -395,17 +400,18 @@ :program:`lit` provides various patterns that can be used with the RUN command. These are defined in TestRunner.py. The base set of substitutions are: - ========== ============== - Macro Substitution - ========== ============== - %s source path (path to the file currently being run) - %S source dir (directory of the file currently being run) - %p same as %S - %{pathsep} path separator - %t temporary file name unique to the test - %T temporary directory unique to the test - %% % - ========== ============== + ============== ============== + Macro Substitution + ============== ============== + %s source path (path to the file currently being run) + %S source dir (directory of the file currently being run) + %p same as %S + %{pathsep} path separator + %t temporary file name unique to the test + %T temporary directory unique to the test + %shared_output temporary directory shared by all tests in the same directory + %% % + ============== ============== Other substitutions are provided that are variations on this base set and further substitution patterns can be defined by each test module. See the Index: docs/TestingGuide.rst =================================================================== --- docs/TestingGuide.rst +++ docs/TestingGuide.rst @@ -292,6 +292,30 @@ putting the extra files in an ``Inputs/`` directory. This pattern is deprecated. + +Setup scripts +------------- + +Some tests may have common shared setup that would be redundant to perform in +each test. In this case you can use a local configuration file +(``lit.local.cfg``) to provide a setup script, using the following bit of +Python: + +.. code-block:: python + + config.setup_script = 'Inputs/base.sh' + +The path may be absolute or relative to the configuration file. This script +will be run before any tests in the directory are executed, with +``%shared_output`` as its current working directory. The same environment used +to execute tests in this directory will be used to run the script +(``config.environment``). + +Note that like other configuration, ``setup_script`` is inherited for tests in +subdirectories. This means the same script may be executed more than once, with +different working directories. + + Fragile tests ------------- @@ -464,6 +488,11 @@ Example: ``/home/user/llvm.build/test/MC/ELF/Output`` +``%shared_output`` + Path to a directory that can be shared across all test cases in the same + source directory. The directory may not exist unless you have a setup script, + described above. + ``%{pathsep}`` Expands to the path separator, i.e. ``:`` (or ``;`` on Windows). Index: utils/lit/lit/TestRunner.py =================================================================== --- utils/lit/lit/TestRunner.py +++ utils/lit/lit/TestRunner.py @@ -842,7 +842,9 @@ substitutions.extend(test.config.substitutions) tmpName = tmpBase + '.tmp' baseName = os.path.basename(tmpBase) - substitutions.extend([('%s', sourcepath), + sharedDir = os.path.join(os.path.dirname(tmpName), 'Shared') + substitutions.extend([('%shared_output', sharedDir), + ('%s', sourcepath), ('%S', sourcedir), ('%p', sourcedir), ('%{pathsep}', os.pathsep), @@ -853,6 +855,7 @@ # "%/[STpst]" should be normalized. substitutions.extend([ + ('%/shared_output', sharedDir.replace('\\', '/')), ('%/s', sourcepath.replace('\\', '/')), ('%/S', sourcedir.replace('\\', '/')), ('%/p', sourcedir.replace('\\', '/')), @@ -863,6 +866,7 @@ # "%:[STpst]" are paths without colons. if kIsWindows: substitutions.extend([ + ('%:shared_output', re.sub(r'^(.):', r'\1', sharedDir)), ('%:s', re.sub(r'^(.):', r'\1', sourcepath)), ('%:S', re.sub(r'^(.):', r'\1', sourcedir)), ('%:p', re.sub(r'^(.):', r'\1', sourcedir)), @@ -871,6 +875,7 @@ ]) else: substitutions.extend([ + ('%:shared_output', sharedDir), ('%:s', sourcepath), ('%:S', sourcedir), ('%:p', sourcedir), Index: utils/lit/lit/TestingConfig.py =================================================================== --- utils/lit/lit/TestingConfig.py +++ utils/lit/lit/TestingConfig.py @@ -100,13 +100,13 @@ 'unable to parse config file %r, traceback: %s' % ( path, traceback.format_exc())) - self.finish(litConfig) + self.finish(litConfig, path) def __init__(self, parent, name, suffixes, test_format, environment, substitutions, unsupported, test_exec_root, test_source_root, excludes, available_features, pipefail, limit_to_features = [], - is_early = False, parallelism_group = ""): + is_early = False, parallelism_group = "", setup_script = None): self.parent = parent self.name = str(name) self.suffixes = set(suffixes) @@ -126,8 +126,9 @@ # Whether the suite should be tested early in a given run. self.is_early = bool(is_early) self.parallelism_group = parallelism_group + self.setup_script = setup_script - def finish(self, litConfig): + def finish(self, litConfig, path): """finish() - Finish this config object, after loading is complete.""" self.name = str(self.name) @@ -143,6 +144,9 @@ # files. Should we distinguish them? self.test_source_root = str(self.test_source_root) self.excludes = set(self.excludes) + if self.setup_script is not None: + self.setup_script = os.path.join(os.path.dirname(path), + self.setup_script) @property def root(self): Index: utils/lit/lit/run.py =================================================================== --- utils/lit/lit/run.py +++ utils/lit/lit/run.py @@ -36,6 +36,48 @@ if self.failedCount == self.maxFailures: self.provider.cancel() +def _runSetup(test, litConfig): + if not test.config.setup_script: + raise + + # FIXME: This should be kept in sync with '%shared_output' in the default + # substitutions. + shared_dir = os.path.join(os.path.dirname(test.getExecPath()), + 'Output', 'Shared') + lit.util.mkdir_p(shared_dir) + + try: + out, err, exitCode = lit.util.executeCommand( + [test.config.setup_script], + cwd=shared_dir, + env=test.config.environment, + timeout=litConfig.maxIndividualTestTime) + timeoutInfo = None + except lit.util.ExecuteCommandTimeoutException as e: + out = e.out + err = e.err + exitCode = e.exitCode + timeoutInfo = e.msg + + if exitCode == 0: + return + + # Form the output log. + output = """Setup Script:\n--\n%s\n--\nExit Code: %d\n""" % ( + cmd, exitCode) + + if timeoutInfo is not None: + output += """Timeout: %s\n""" % (timeoutInfo,) + output += "\n" + + # Append the outputs, if present. + if out: + output += """Command Output (stdout):\n--\n%s\n--\n""" % (out,) + if err: + output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,) + + raise ValueError(output) + class Run(object): """ This class represents a concrete, configured testing run. @@ -62,6 +104,23 @@ if max_time: deadline = time.time() + max_time + # Handle any setup necessary for these tests. Set up outer directories + # before inner ones, but promise nothing else. + # FIXME: Parallelize this too! + seenDirs = set() + testsWithSetup = [] + for test in self.tests: + testDir = os.path.dirname(test.getExecPath()) + if testDir in seenDirs: + continue + seenDirs.add(testDir) + if not test.config.setup_script: + continue + testsWithSetup.append((test, testDir)) + testsWithSetup.sort(key=lambda pair: pair[1].count(os.sep)) + for test, _ in testsWithSetup: + _runSetup(test, self.lit_config) + # Start a process pool. Copy over the data shared between all test runs. # FIXME: Find a way to capture the worker process stderr. If the user # interrupts the workers before we make it into our task callback, they Index: utils/lit/tests/Inputs/setup-script/Inputs/base.sh =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/Inputs/base.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +touch base.txt +echo $(dirname $(dirname "$PWD")) >> "$SHARED_LOG_FILE" Index: utils/lit/tests/Inputs/setup-script/lit.cfg =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/lit.cfg @@ -0,0 +1,10 @@ +import lit.formats +import os + +config.name = 'shtest-shell' +config.suffixes = ['.txt'] +config.test_format = lit.formats.ShTest() +config.test_source_root = None +config.test_exec_root = None +config.setup_script = 'Inputs/base.sh' +config.environment['SHARED_LOG_FILE'] = os.environ.get('SHARED_LOG_FILE') Index: utils/lit/tests/Inputs/setup-script/subdir-custom/Inputs/custom.sh =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/subdir-custom/Inputs/custom.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +touch custom.txt +$(dirname "$0")/../../Inputs/base.sh Index: utils/lit/tests/Inputs/setup-script/subdir-custom/lit.local.cfg =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/subdir-custom/lit.local.cfg @@ -0,0 +1 @@ +config.setup_script = 'Inputs/custom.sh' Index: utils/lit/tests/Inputs/setup-script/subdir-custom/test.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/subdir-custom/test.txt @@ -0,0 +1,2 @@ +RUN: test -f %shared_output/base.txt +RUN: test -f %shared_output/custom.txt Index: utils/lit/tests/Inputs/setup-script/subdir-inherited/lit.local.cfg =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/subdir-inherited/lit.local.cfg @@ -0,0 +1 @@ +# No extra configuration, but we should still get the setup script run. Index: utils/lit/tests/Inputs/setup-script/subdir-inherited/test.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/subdir-inherited/test.txt @@ -0,0 +1 @@ +RUN: test -f %shared_output/base.txt Index: utils/lit/tests/Inputs/setup-script/subdir-no-local/test.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/subdir-no-local/test.txt @@ -0,0 +1 @@ +RUN: test -f %shared_output/base.txt Index: utils/lit/tests/Inputs/setup-script/subdir-no-setup/lit.local.cfg =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/subdir-no-setup/lit.local.cfg @@ -0,0 +1 @@ +config.setup_script = None Index: utils/lit/tests/Inputs/setup-script/subdir-no-setup/test.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/subdir-no-setup/test.txt @@ -0,0 +1 @@ +RUN: not test -d %shared_output \ No newline at end of file Index: utils/lit/tests/Inputs/setup-script/test.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/test.txt @@ -0,0 +1 @@ +RUN: test -f %shared_output/base.txt \ No newline at end of file Index: utils/lit/tests/setup-script.py =================================================================== --- /dev/null +++ utils/lit/tests/setup-script.py @@ -0,0 +1,31 @@ +# Test the 'setup_script' feature. + +# RUN: rm -rf %t %t.log.txt +# RUN: cp -r %{inputs}/setup-script %t +# RUN: env SHARED_LOG_FILE=%t.log.txt %{lit} -j 1 -v %t +# RUN: FileCheck %s < %t.log.txt +# RUN: FileCheck -check-prefix=NEGATIVE %s < %t.log.txt + +# RUN: rm -rf %t-shuffle %t-shuffle.log.txt +# RUN: cp -r %{inputs}/setup-script %t-shuffle +# RUN: env SHARED_LOG_FILE=%t-shuffle.log.txt %{lit} -j 1 -v --shuffle %t-shuffle +# RUN: FileCheck %s < %t-shuffle.log.txt +# RUN: FileCheck -check-prefix=NEGATIVE %s < %t-shuffle.log.txt + +# RUN: rm -rf %t-specific %t-specific.log.txt +# RUN: cp -r %{inputs}/setup-script %t-specific +# RUN: env SHARED_LOG_FILE=%t-specific.log.txt %{lit} -j 1 -v %t-specific/subdir-inherited +# RUN: FileCheck -check-prefix=CHECK-SPECIFIC %s < %t-specific.log.txt +# RUN: FileCheck -check-prefix=CHECK-SPECIFIC-NEGATIVE %s < %t-specific.log.txt + +# CHECK: setup-script.py.tmp{{(-shuffle)?}}{{$}} +# CHECK-DAG: setup-script.py.tmp{{(-shuffle)?}}/subdir-custom{{$}} +# CHECK-DAG: setup-script.py.tmp{{(-shuffle)?}}/subdir-inherited{{$}} +# CHECK-DAG: setup-script.py.tmp{{(-shuffle)?}}/subdir-no-local{{$}} +# NEGATIVE-NOT: subdir-no-setup + +# CHECK-SPECIFIC: setup-script.py.tmp-specific/subdir-inherited{{$}} +# CHECK-SPECIFIC-NEGATIVE-NOT: setup-script.py.tmp-specific{{$}} +# CHECK-SPECIFIC-NEGATIVE-NOT: subdir-custom +# CHECK-SPECIFIC-NEGATIVE-NOT: subdir-no-local +# CHECK-SPECIFIC-NEGATIVE-NOT: subdir-no-setup