Index: docs/CommandGuide/lit.rst =================================================================== --- docs/CommandGuide/lit.rst +++ docs/CommandGuide/lit.rst @@ -358,6 +358,11 @@ **available_features** A set of features that can be used in `XFAIL`, `REQUIRES`, and `UNSUPPORTED` directives. + **setup_script** A Python script to run before any of the tests in this suite + are executed, or None. May be relative to the directory containing the + configuration file. Caution: this script will be run for this directory and + *once for every subdirectory.* + TEST DISCOVERY ~~~~~~~~~~~~~~ @@ -395,17 +400,18 @@ :program:`lit` provides various patterns that can be used with the RUN command. These are defined in TestRunner.py. The base set of substitutions are: - ========== ============== - Macro Substitution - ========== ============== - %s source path (path to the file currently being run) - %S source dir (directory of the file currently being run) - %p same as %S - %{pathsep} path separator - %t temporary file name unique to the test - %T temporary directory unique to the test - %% % - ========== ============== + ============== ============== + Macro Substitution + ============== ============== + %s source path (path to the file currently being run) + %S source dir (directory of the file currently being run) + %p same as %S + %{pathsep} path separator + %t temporary file name unique to the test + %T temporary directory unique to the test + %shared_output temporary directory shared by all tests in the same directory + %% % + ============== ============== Other substitutions are provided that are variations on this base set and further substitution patterns can be defined by each test module. See the Index: docs/TestingGuide.rst =================================================================== --- docs/TestingGuide.rst +++ docs/TestingGuide.rst @@ -292,6 +292,32 @@ putting the extra files in an ``Inputs/`` directory. This pattern is deprecated. + +Setup scripts +------------- + +Some tests may have common shared setup that would be redundant to perform in +each test. In this case you can use a local configuration file +(``lit.local.cfg``) to provide a setup script, using the following bit of +Python: + +.. code-block:: python + + config.setup_script = 'Inputs/base.py' + +The path may be absolute or relative to the configuration file. This script +will be run before any tests in the directory are executed, with the following +globals defined: + +- ``config``: The testing configuration, like in a ``lit.local.cfg`` file +- ``lit_config``: Shared configuration for this lit execution +- ``shared_output``: The value of ``%shared_output`` for this directory + +Note that like other configuration, ``setup_script`` is inherited for tests in +subdirectories. This means the same script may be executed more than once, with +values of ``shared_output``. + + Fragile tests ------------- @@ -464,6 +490,11 @@ Example: ``/home/user/llvm.build/test/MC/ELF/Output`` +``%shared_output`` + Path to a directory that can be shared across all test cases in the same + source directory. (lit will not create this directory for you, but the + containing directory will exist.) + ``%{pathsep}`` Expands to the path separator, i.e. ``:`` (or ``;`` on Windows). Index: utils/lit/lit/TestRunner.py =================================================================== --- utils/lit/lit/TestRunner.py +++ utils/lit/lit/TestRunner.py @@ -842,7 +842,9 @@ substitutions.extend(test.config.substitutions) tmpName = tmpBase + '.tmp' baseName = os.path.basename(tmpBase) - substitutions.extend([('%s', sourcepath), + sharedDir = os.path.join(os.path.dirname(tmpName), 'Shared') + substitutions.extend([('%shared_output', sharedDir), + ('%s', sourcepath), ('%S', sourcedir), ('%p', sourcedir), ('%{pathsep}', os.pathsep), @@ -853,6 +855,7 @@ # "%/[STpst]" should be normalized. substitutions.extend([ + ('%/shared_output', sharedDir.replace('\\', '/')), ('%/s', sourcepath.replace('\\', '/')), ('%/S', sourcedir.replace('\\', '/')), ('%/p', sourcedir.replace('\\', '/')), @@ -863,6 +866,7 @@ # "%:[STpst]" are paths without colons. if kIsWindows: substitutions.extend([ + ('%:shared_output', re.sub(r'^(.):', r'\1', sharedDir)), ('%:s', re.sub(r'^(.):', r'\1', sourcepath)), ('%:S', re.sub(r'^(.):', r'\1', sourcedir)), ('%:p', re.sub(r'^(.):', r'\1', sourcedir)), @@ -871,6 +875,7 @@ ]) else: substitutions.extend([ + ('%:shared_output', sharedDir), ('%:s', sourcepath), ('%:S', sourcedir), ('%:p', sourcedir), Index: utils/lit/lit/TestingConfig.py =================================================================== --- utils/lit/lit/TestingConfig.py +++ utils/lit/lit/TestingConfig.py @@ -100,13 +100,13 @@ 'unable to parse config file %r, traceback: %s' % ( path, traceback.format_exc())) - self.finish(litConfig) + self.finish(litConfig, path) def __init__(self, parent, name, suffixes, test_format, environment, substitutions, unsupported, test_exec_root, test_source_root, excludes, available_features, pipefail, limit_to_features = [], - is_early = False, parallelism_group = ""): + is_early = False, parallelism_group = "", setup_script = None): self.parent = parent self.name = str(name) self.suffixes = set(suffixes) @@ -126,8 +126,9 @@ # Whether the suite should be tested early in a given run. self.is_early = bool(is_early) self.parallelism_group = parallelism_group + self.setup_script = setup_script - def finish(self, litConfig): + def finish(self, litConfig, path): """finish() - Finish this config object, after loading is complete.""" self.name = str(self.name) @@ -143,6 +144,9 @@ # files. Should we distinguish them? self.test_source_root = str(self.test_source_root) self.excludes = set(self.excludes) + if self.setup_script is not None: + self.setup_script = os.path.join(os.path.dirname(path), + self.setup_script) @property def root(self): Index: utils/lit/lit/run.py =================================================================== --- utils/lit/lit/run.py +++ utils/lit/lit/run.py @@ -36,6 +36,44 @@ if self.failedCount == self.maxFailures: self.provider.cancel() +def _runSetup(test, litConfig): + if not test.config.setup_script: + raise + setup_script = test.config.setup_script + + # FIXME: This should be kept in sync with '%shared_output' in the default + # substitutions. + shared_output = os.path.join(os.path.dirname(test.getExecPath()), + 'Output', 'Shared') + + # FIXME: Copied from TestingConfig.load_from_path + data = None + f = open(setup_script) + try: + data = f.read() + except: + litConfig.fatal('unable to load config file: %r' % (setup_script,)) + f.close() + + setup_script_globals = dict(globals()) + setup_script_globals['config'] = test.config + setup_script_globals['lit_config'] = litConfig + setup_script_globals['shared_output'] = shared_output + setup_script_globals['__file__'] = setup_script + try: + exec(compile(data, setup_script, 'exec'), setup_script_globals, None) + except SystemExit: + e = sys.exc_info()[1] + # We allow normal system exit inside a config file to just + # return control without error. + if e.args: + raise + except: + import traceback + litConfig.fatal( + 'unable to parse setup file %r, traceback: %s' % ( + setup_script, traceback.format_exc())) + class Run(object): """ This class represents a concrete, configured testing run. @@ -136,6 +174,23 @@ return True win32api.SetConsoleCtrlHandler(console_ctrl_handler, True) + # Handle any setup necessary for these tests. Set up outer directories + # before inner ones, but promise nothing else. + # FIXME: Parallelize this too! + seenDirs = set() + testsWithSetup = [] + for test in self.tests: + testDir = os.path.dirname(test.getExecPath()) + if testDir in seenDirs: + continue + seenDirs.add(testDir) + if not test.config.setup_script: + continue + testsWithSetup.append((test, testDir)) + testsWithSetup.sort(key=lambda pair: pair[1].count(os.sep)) + for test, _ in testsWithSetup: + _runSetup(test, self.lit_config) + # Save the display object on the runner so that we can update it from # our task completion callback. self.display = display Index: utils/lit/tests/Inputs/setup-script/Inputs/base.py =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/Inputs/base.py @@ -0,0 +1,20 @@ +import os + +def touch(fname, times=None): + with open(fname, 'a'): + os.utime(fname, times) + +# mkdir -p $shared_output, from https://stackoverflow.com/a/600612 +try: + os.makedirs(shared_output) +except OSError as e: + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + +touch(os.path.join(shared_output, 'base.txt')) +with open(os.environ['SHARED_LOG_FILE'], 'a') as shared_log_file: + shared_log_file.write(os.path.dirname(os.path.dirname(shared_output))) + shared_log_file.write('\n') + Index: utils/lit/tests/Inputs/setup-script/lit.cfg =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/lit.cfg @@ -0,0 +1,10 @@ +import lit.formats +import os + +config.name = 'shtest-shell' +config.suffixes = ['.txt'] +config.test_format = lit.formats.ShTest() +config.test_source_root = None +config.test_exec_root = None +config.setup_script = 'Inputs/base.py' +config.environment['SHARED_LOG_FILE'] = os.environ.get('SHARED_LOG_FILE') Index: utils/lit/tests/Inputs/setup-script/subdir-custom/Inputs/custom.py =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/subdir-custom/Inputs/custom.py @@ -0,0 +1,8 @@ +import os + +base = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), + 'Inputs', 'base.py') +with open(base) as base_file: + exec(compile(base_file.read(), base, 'exec')) + +touch(os.path.join(shared_output, 'custom.txt')) Index: utils/lit/tests/Inputs/setup-script/subdir-custom/lit.local.cfg =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/subdir-custom/lit.local.cfg @@ -0,0 +1 @@ +config.setup_script = 'Inputs/custom.py' Index: utils/lit/tests/Inputs/setup-script/subdir-custom/test.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/subdir-custom/test.txt @@ -0,0 +1,2 @@ +RUN: test -f %shared_output/base.txt +RUN: test -f %shared_output/custom.txt Index: utils/lit/tests/Inputs/setup-script/subdir-inherited/lit.local.cfg =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/subdir-inherited/lit.local.cfg @@ -0,0 +1 @@ +# No extra configuration, but we should still get the setup script run. Index: utils/lit/tests/Inputs/setup-script/subdir-inherited/test.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/subdir-inherited/test.txt @@ -0,0 +1 @@ +RUN: test -f %shared_output/base.txt Index: utils/lit/tests/Inputs/setup-script/subdir-no-local/test.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/subdir-no-local/test.txt @@ -0,0 +1 @@ +RUN: test -f %shared_output/base.txt Index: utils/lit/tests/Inputs/setup-script/subdir-no-setup/lit.local.cfg =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/subdir-no-setup/lit.local.cfg @@ -0,0 +1 @@ +config.setup_script = None Index: utils/lit/tests/Inputs/setup-script/subdir-no-setup/test.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/subdir-no-setup/test.txt @@ -0,0 +1 @@ +RUN: not test -d %shared_output \ No newline at end of file Index: utils/lit/tests/Inputs/setup-script/test.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/setup-script/test.txt @@ -0,0 +1 @@ +RUN: test -f %shared_output/base.txt \ No newline at end of file Index: utils/lit/tests/setup-script.py =================================================================== --- /dev/null +++ utils/lit/tests/setup-script.py @@ -0,0 +1,31 @@ +# Test the 'setup_script' feature. + +# RUN: rm -rf %t %t.log.txt +# RUN: cp -r %{inputs}/setup-script %t +# RUN: env SHARED_LOG_FILE=%t.log.txt %{lit} -j 1 -v %t +# RUN: FileCheck %s < %t.log.txt +# RUN: FileCheck -check-prefix=NEGATIVE %s < %t.log.txt + +# RUN: rm -rf %t-shuffle %t-shuffle.log.txt +# RUN: cp -r %{inputs}/setup-script %t-shuffle +# RUN: env SHARED_LOG_FILE=%t-shuffle.log.txt %{lit} -j 1 -v --shuffle %t-shuffle +# RUN: FileCheck %s < %t-shuffle.log.txt +# RUN: FileCheck -check-prefix=NEGATIVE %s < %t-shuffle.log.txt + +# RUN: rm -rf %t-specific %t-specific.log.txt +# RUN: cp -r %{inputs}/setup-script %t-specific +# RUN: env SHARED_LOG_FILE=%t-specific.log.txt %{lit} -j 1 -v %t-specific/subdir-inherited +# RUN: FileCheck -check-prefix=CHECK-SPECIFIC %s < %t-specific.log.txt +# RUN: FileCheck -check-prefix=CHECK-SPECIFIC-NEGATIVE %s < %t-specific.log.txt + +# CHECK: setup-script.py.tmp{{(-shuffle)?}}{{$}} +# CHECK-DAG: setup-script.py.tmp{{(-shuffle)?}}/subdir-custom{{$}} +# CHECK-DAG: setup-script.py.tmp{{(-shuffle)?}}/subdir-inherited{{$}} +# CHECK-DAG: setup-script.py.tmp{{(-shuffle)?}}/subdir-no-local{{$}} +# NEGATIVE-NOT: subdir-no-setup + +# CHECK-SPECIFIC: setup-script.py.tmp-specific/subdir-inherited{{$}} +# CHECK-SPECIFIC-NEGATIVE-NOT: setup-script.py.tmp-specific{{$}} +# CHECK-SPECIFIC-NEGATIVE-NOT: subdir-custom +# CHECK-SPECIFIC-NEGATIVE-NOT: subdir-no-local +# CHECK-SPECIFIC-NEGATIVE-NOT: subdir-no-setup