diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst --- a/llvm/docs/CommandGuide/lit.rst +++ b/llvm/docs/CommandGuide/lit.rst @@ -60,8 +60,9 @@ .. option:: --config-prefix=NAME - Search for :file:`{NAME}.cfg` and :file:`{NAME}.site.cfg` when searching for - test suites, instead of :file:`lit.cfg` and :file:`lit.site.cfg`. + Search for :file:`{NAME}.cfg`, :file:`{NAME}.site.cfg` and + :file:`{NAME}.site.multi.cfg` when searching for test suites, instead of + :file:`lit.cfg`, :file:`lit.site.cfg` and :file:`lit.site.multi.cfg`. .. option:: -D NAME[=VALUE], --param NAME[=VALUE] @@ -238,10 +239,25 @@ In the :program:`lit` model, every test must exist inside some *test suite*. :program:`lit` resolves the inputs specified on the command line to test suites -by searching upwards from the input path until it finds a :file:`lit.cfg` or -:file:`lit.site.cfg` file. These files serve as both a marker of test suites -and as configuration files which :program:`lit` loads in order to understand -how to find and run the tests inside the test suite. +by searching upwards from the input path until it finds a :file:`lit.cfg`, +:file:`lit.site.cfg` or :file:`lit.site.multi.cfg` file. These files serve as +both a marker of test suites and as configuration files which :program:`lit` +loads in order to understand how to find and run the tests inside the test suite. + +:file:`lit.site.multi.cfg` file is used when the test suite has multiple +configurations. The file is in JSON format with two key-value pairs. The format +looks like the example below. The sub-testsuite path could be either a absolute +filepath or a relative filepath to the path of the :file:`lit.site.multi.cfg` file. + +.. code-block:: json + + { + "name": "", + "test suites": [ + "", # or "" + "" # or "" + ] + } Once :program:`lit` has mapped the inputs into test suites it traverses the list of inputs adding tests for individual files and recursively searching for @@ -325,10 +341,10 @@ suite*. Test suites serve to define the format of the tests they contain, the logic for finding those tests, and any additional information to run the tests. -:program:`lit` identifies test suites as directories containing ``lit.cfg`` or -``lit.site.cfg`` files (see also :option:`--config-prefix`). Test suites are -initially discovered by recursively searching up the directory hierarchy for -all the input files passed on the command line. You can use +:program:`lit` identifies test suites as directories containing ``lit.cfg``, +``lit.site.cfg`` or ``lit.site.multi.cfg`` files (see also :option:`--config-prefix`). +Test suites are initially discovered by recursively searching up the directory +hierarchy for all the input files passed on the command line. You can use :option:`--show-suites` to display the discovered test suites at startup. Once a test suite is discovered, its config file is loaded. Config files diff --git a/llvm/utils/lit/lit/LitConfig.py b/llvm/utils/lit/lit/LitConfig.py --- a/llvm/utils/lit/lit/LitConfig.py +++ b/llvm/utils/lit/lit/LitConfig.py @@ -44,8 +44,10 @@ # Configuration files to look for when discovering test suites. self.config_prefix = config_prefix or 'lit' self.suffixes = ['cfg.py', 'cfg'] + self.multiconfig_suffixes = ['multi.cfg'] self.config_names = ['%s.%s' % (self.config_prefix,x) for x in self.suffixes] - self.site_config_names = ['%s.site.%s' % (self.config_prefix,x) for x in self.suffixes] + # Only exec side test suites could be multi-config'd. + self.site_config_names = ['%s.site.%s' % (self.config_prefix,x) for x in self.suffixes + self.multiconfig_suffixes] self.local_config_names = ['%s.local.%s' % (self.config_prefix,x) for x in self.suffixes] self.numErrors = 0 diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py --- a/llvm/utils/lit/lit/Test.py +++ b/llvm/utils/lit/lit/Test.py @@ -181,20 +181,13 @@ """TestSuite - Information on a group of tests. A test suite groups together a set of logically related tests. + This set of tests could be configured in different ways. """ - def __init__(self, name, source_root, exec_root, config): + def __init__(self, name, configs, is_multi_cfg): self.name = name - self.source_root = source_root - self.exec_root = exec_root - # The test suite configuration. - self.config = config - - def getSourcePath(self, components): - return os.path.join(self.source_root, *components) - - def getExecPath(self, components): - return os.path.join(self.exec_root, *components) + self.configs = configs + self.is_multi_cfg = is_multi_cfg class Test: """Test - Information on a single test instance.""" @@ -253,7 +246,7 @@ return self.result.code.isFailure def getFullName(self): - return self.suite.config.name + ' :: ' + '/'.join(self.path_in_suite) + return self.config.name + ' :: ' + '/'.join(self.path_in_suite) def getFilePath(self): if self.file_path: @@ -261,10 +254,10 @@ return self.getSourcePath() def getSourcePath(self): - return self.suite.getSourcePath(self.path_in_suite) + return self.config.getSourcePath(self.path_in_suite) def getExecPath(self): - return self.suite.getExecPath(self.path_in_suite) + return self.config.getExecPath(self.path_in_suite) def isExpectedToFail(self): """ @@ -278,7 +271,7 @@ """ features = self.config.available_features - triple = getattr(self.suite.config, 'target_triple', "") + triple = getattr(self.config, 'target_triple', "") # Check if any of the xfails match an available feature or the target. for item in self.xfails: @@ -351,7 +344,7 @@ """ features = self.config.available_features - triple = getattr(self.suite.config, 'target_triple', "") + triple = getattr(self.config, 'target_triple', "") try: return [item for item in self.unsupported @@ -367,4 +360,4 @@ This can be used for test suites with long running tests to maximize parallelism or where it is desirable to surface their failures early. """ - return self.suite.config.is_early + return self.config.is_early diff --git a/llvm/utils/lit/lit/TestingConfig.py b/llvm/utils/lit/lit/TestingConfig.py --- a/llvm/utils/lit/lit/TestingConfig.py +++ b/llvm/utils/lit/lit/TestingConfig.py @@ -157,6 +157,12 @@ self.test_source_root = str(self.test_source_root) self.excludes = set(self.excludes) + def getSourcePath(self, components): + return os.path.join(self.test_source_root, *components) + + def getExecPath(self, components): + return os.path.join(self.test_exec_root, *components) + @property def root(self): """root attribute - The root configuration for the test suite.""" diff --git a/llvm/utils/lit/lit/discovery.py b/llvm/utils/lit/lit/discovery.py --- a/llvm/utils/lit/lit/discovery.py +++ b/llvm/utils/lit/lit/discovery.py @@ -3,6 +3,7 @@ """ import copy +import json import os import sys @@ -22,8 +23,30 @@ cfgpath = chooseConfigFileFromDir(path, lit_config.config_names) return cfgpath -def getTestSuite(item, litConfig, cache): - """getTestSuite(item, litConfig, cache) -> (suite, relative_path) +def getTestSuiteConfigs(cfgpath, litConfig): + ts_name = '' + cfgpaths = [] + ts_multi = any( + cfgpath.endswith(s) for s in litConfig.multiconfig_suffixes) + if ts_multi: + # Read multi-cfg test suite + with open(cfgpath, 'r') as f: + data = json.load(f) + ts_name = data['name'] + for ln in data['test suites']: + ln = ln.strip() + if ln: + ln = os.path.normpath(ln) + if not os.path.isabs(ln): + ln = os.path.join(cfgpath, '..', ln) + cfgpaths.append(ln) + else: + cfgpaths.append(cfgpath) + return cfgpaths, ts_multi, ts_name + +def getTestSuite(item, litConfig, testSuiteCache, testConfigCache): + """getTestSuite(item, litConfig, testSuiteCache, + testConfigCache) -> (suite, relative_path) Find the test suite containing @arg item. @@ -58,22 +81,37 @@ if target: cfgpath = target - # We found a test suite, create a new config for it and load it. - if litConfig.debug: - litConfig.note('loading suite config %r' % cfgpath) - - cfg = TestingConfig.fromdefaults(litConfig) - cfg.load_from_path(cfgpath, litConfig) - source_root = os.path.realpath(cfg.test_source_root or path) - exec_root = os.path.realpath(cfg.test_exec_root or path) - return Test.TestSuite(cfg.name, source_root, exec_root, cfg), () + cfgpaths, ts_multi, ts_name = getTestSuiteConfigs(cfgpath, litConfig) + + cfgs = [] + for cp in cfgpaths: + cp = os.path.realpath(cp) + cp = os.path.normpath(cp) + cfg = testConfigCache.get(cp) + if cfg is None: + # We found a test suite config, create a TestingConfig and load it. + if litConfig.debug: + litConfig.note('loading suite config %r' % cp) + cfg = TestingConfig.fromdefaults(litConfig) + cfg.load_from_path(cp, litConfig) + cfg.test_exec_root = os.path.realpath(cfg.test_exec_root + or path) + cfg.test_source_root = os.path.realpath(cfg.test_source_root + or path) + testConfigCache[cp] = cfg + cfgs.append(cfg) + + if not ts_multi: + ts_name = cfg.name + + return Test.TestSuite(ts_name, cfgs, ts_multi), () def search(path): # Check for an already instantiated test suite. real_path = os.path.realpath(path) - res = cache.get(real_path) + res = testSuiteCache.get(real_path) if res is None: - cache[real_path] = res = search1(path) + testSuiteCache[real_path] = res = search1(path) return res # Canonicalize the path. @@ -92,17 +130,18 @@ ts, relative = search(item) return ts, tuple(relative + tuple(components)) -def getLocalConfig(ts, path_in_suite, litConfig, cache): +def getLocalConfig(testConfig, path_in_suite, litConfig, cache): def search1(path_in_suite): # Get the parent config. if not path_in_suite: - parent = ts.config + parent = testConfig else: parent = search(path_in_suite[:-1]) # Check if there is a local configuration file. - source_path = ts.getSourcePath(path_in_suite) - cfgpath = chooseConfigFileFromDir(source_path, litConfig.local_config_names) + source_path = testConfig.getSourcePath(path_in_suite) + cfgpath = chooseConfigFileFromDir(source_path, + litConfig.local_config_names) # If not, just reuse the parent config. if not cfgpath: @@ -114,10 +153,13 @@ if litConfig.debug: litConfig.note('loading local config %r' % cfgpath) config.load_from_path(cfgpath, litConfig) + # config without parent is a test suite config, not a local config + if config.parent is None: + config.parent = parent return config def search(path_in_suite): - key = (ts, path_in_suite) + key = (testConfig, path_in_suite) res = cache.get(key) if res is None: cache[key] = res = search1(path_in_suite) @@ -125,37 +167,43 @@ return search(path_in_suite) -def getTests(path, litConfig, testSuiteCache, localConfigCache): + +def getTests(path, litConfig, testSuiteCache, testConfigCache, + localConfigCache): # Find the test suite for this input and its relative path. - ts,path_in_suite = getTestSuite(path, litConfig, testSuiteCache) + ts,path_in_suite = getTestSuite(path, litConfig, testSuiteCache, + testConfigCache) if ts is None: litConfig.warning('unable to find test suite for %r' % path) return (),() - if litConfig.debug: - litConfig.note('resolved input %r to %r::%r' % (path, ts.name, - path_in_suite)) - - return ts, getTestsInSuite(ts, path_in_suite, litConfig, - testSuiteCache, localConfigCache) + tests = [] + for tc in ts.configs: + if litConfig.debug: + litConfig.note('resolved input %r to %r::%r' % + (path, tc.name, path_in_suite)) + tests.extend( + getTestsInSuite(ts, tc, path_in_suite, litConfig, testSuiteCache, + testConfigCache, localConfigCache)) + return tests -def getTestsInSuite(ts, path_in_suite, litConfig, - testSuiteCache, localConfigCache): +def getTestsInSuite(ts, tc, path_in_suite, litConfig, testSuiteCache, + testConfigCache, localConfigCache): # Check that the source path exists (errors here are reported by the # caller). - source_path = ts.getSourcePath(path_in_suite) + source_path = tc.getSourcePath(path_in_suite) if not os.path.exists(source_path): return # Check if the user named a test directly. if not os.path.isdir(source_path): - lc = getLocalConfig(ts, path_in_suite[:-1], litConfig, localConfigCache) + lc = getLocalConfig(tc, path_in_suite[:-1], litConfig, localConfigCache) yield Test.Test(ts, path_in_suite, lc) return # Otherwise we have a directory to search for tests, start by getting the # local configuration. - lc = getLocalConfig(ts, path_in_suite, litConfig, localConfigCache) + lc = getLocalConfig(tc, path_in_suite, litConfig, localConfigCache) # Search for tests. if lc.test_format is not None: @@ -177,13 +225,15 @@ # Check for nested test suites, first in the execpath in case there is a # site configuration and then in the source path. subpath = path_in_suite + (filename,) - file_execpath = ts.getExecPath(subpath) + file_execpath = tc.getExecPath(subpath) if dirContainsTestSuite(file_execpath, litConfig): sub_ts, subpath_in_suite = getTestSuite(file_execpath, litConfig, - testSuiteCache) + testSuiteCache, + testConfigCache) elif dirContainsTestSuite(file_sourcepath, litConfig): sub_ts, subpath_in_suite = getTestSuite(file_sourcepath, litConfig, - testSuiteCache) + testSuiteCache, + testConfigCache) else: sub_ts = None @@ -194,19 +244,21 @@ continue # Otherwise, load from the nested test suite, if present. - if sub_ts is not None: - subiter = getTestsInSuite(sub_ts, subpath_in_suite, litConfig, - testSuiteCache, localConfigCache) - else: - subiter = getTestsInSuite(ts, subpath, litConfig, testSuiteCache, + subts = ts if sub_ts is None else sub_ts + subpath = subpath if sub_ts is None else subpath_in_suite + for sub_tc in subts.configs: + subiter = getTestsInSuite(subts, sub_tc, subpath, litConfig, + testSuiteCache, testConfigCache, localConfigCache) + N = 0 + for res in subiter: + N += 1 + yield res + + if sub_ts and not N: + litConfig.warning('test suite %r contained no tests' % + sub_tc.name) - N = 0 - for res in subiter: - N += 1 - yield res - if sub_ts and not N: - litConfig.warning('test suite %r contained no tests' % sub_ts.name) def find_tests_for_inputs(lit_config, inputs): """ @@ -234,11 +286,13 @@ # Load the tests from the inputs. tests = [] test_suite_cache = {} + test_config_cache = {} # Avoid reloading the same test config. local_config_cache = {} for input in actual_inputs: prev = len(tests) - tests.extend(getTests(input, lit_config, - test_suite_cache, local_config_cache)[1]) + tests.extend( + getTests(input, lit_config, test_suite_cache, test_config_cache, + local_config_cache)) if prev == len(tests): lit_config.warning('input %r contained no tests' % input) diff --git a/llvm/utils/lit/lit/formats/base.py b/llvm/utils/lit/lit/formats/base.py --- a/llvm/utils/lit/lit/formats/base.py +++ b/llvm/utils/lit/lit/formats/base.py @@ -12,7 +12,7 @@ class FileBasedTest(TestFormat): def getTestsInDirectory(self, testSuite, path_in_suite, litConfig, localConfig): - source_path = testSuite.getSourcePath(path_in_suite) + source_path = localConfig.getSourcePath(path_in_suite) for filename in os.listdir(source_path): # Ignore dot files and excluded tests. if (filename.startswith('.') or @@ -52,7 +52,7 @@ litConfig, localConfig): dir = self.dir if dir is None: - dir = testSuite.getSourcePath(path_in_suite) + dir = localConfig.getSourcePath(path_in_suite) for dirname,subdirs,filenames in os.walk(dir): if not self.recursive: diff --git a/llvm/utils/lit/lit/formats/googletest.py b/llvm/utils/lit/lit/formats/googletest.py --- a/llvm/utils/lit/lit/formats/googletest.py +++ b/llvm/utils/lit/lit/formats/googletest.py @@ -89,7 +89,7 @@ def getTestsInDirectory(self, testSuite, path_in_suite, litConfig, localConfig): - source_path = testSuite.getSourcePath(path_in_suite) + source_path = localConfig.getSourcePath(path_in_suite) for subdir in self.test_sub_dirs: dir_path = os.path.join(source_path, subdir) if not os.path.isdir(dir_path): diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py --- a/llvm/utils/lit/lit/main.py +++ b/llvm/utils/lit/lit/main.py @@ -124,23 +124,30 @@ def print_discovered(tests, show_suites, show_tests): - tests.sort(key=lit.reports.by_suite_and_test_path) + tests.sort(key=lit.reports.by_suite_and_config_and_test_path) if show_suites: import itertools tests_by_suite = itertools.groupby(tests, lambda t: t.suite) print('-- Test Suites --') for suite, test_iter in tests_by_suite: - test_count = sum(1 for _ in test_iter) - print(' %s - %d tests' % (suite.name, test_count)) - print(' Source Root: %s' % suite.source_root) - print(' Exec Root : %s' % suite.exec_root) - features = ' '.join(sorted(suite.config.available_features)) - print(' Available Features: %s' % features) - substitutions = sorted(suite.config.substitutions) - substitutions = ('%s => %s' % (x, y) for (x, y) in substitutions) - substitutions = '\n'.ljust(30).join(substitutions) - print(' Available Substitutions: %s' % substitutions) + tests_by_config = itertools.groupby(test_iter, lambda t: lit.reports.getRoot(t.config)) + for _, test_iter_config in tests_by_config: + config = next(test_iter_config).config + test_count = sum(1 for _ in test_iter_config) + 1 + if suite.is_multi_cfg: + print(' %s :: %s - %d tests' % + (suite.name, config.name, test_count)) + else: + print(' %s - %d tests' % (config.name, test_count)) + print(' Source Root: %s' % config.test_source_root) + print(' Exec Root : %s' % config.test_exec_root) + features = ' '.join(sorted(config.available_features)) + print(' Available Features: %s' % features) + substitutions = sorted(config.substitutions) + substitutions = ('%s => %s' % (x, y) for (x, y) in substitutions) + substitutions = '\n'.ljust(30).join(substitutions) + print(' Available Substitutions: %s' % substitutions) if show_tests: print('-- Available Tests --') diff --git a/llvm/utils/lit/lit/reports.py b/llvm/utils/lit/lit/reports.py --- a/llvm/utils/lit/lit/reports.py +++ b/llvm/utils/lit/lit/reports.py @@ -5,11 +5,16 @@ import lit.Test +def getRoot(config): + if not config.parent: + return config + return getRoot(config.parent) -def by_suite_and_test_path(test): +def by_suite_and_config_and_test_path(test): # Suite names are not necessarily unique. Include object identity in sort # key to avoid mixing tests of different suites. - return (test.suite.name, id(test.suite), test.path_in_suite) + return (test.suite.name, id(test.suite), getRoot(test.config).name, + id(getRoot(test.config)), test.path_in_suite) class JsonReport(object): @@ -76,7 +81,7 @@ # TODO(yln): elapsed unused, put it somewhere? def write_results(self, tests, elapsed): assert not any(t.result.code in {lit.Test.EXCLUDED, lit.Test.SKIPPED} for t in tests) - tests.sort(key=by_suite_and_test_path) + tests.sort(key=by_suite_and_config_and_test_path) tests_by_suite = itertools.groupby(tests, lambda t: t.suite) with open(self.output_file, 'w') as file: @@ -90,7 +95,7 @@ skipped = sum(1 for t in tests if t.result.code in self.skipped_codes) failures = sum(1 for t in tests if t.isFailure()) - name = suite.config.name.replace('.', '-') + name = suite.name.replace('.', '-') # file.write(f'\n') file.write('\n'.format( name=quo(name), tests=len(tests), failures=failures, skipped=skipped)) @@ -100,8 +105,10 @@ def _write_test(self, file, test, suite_name): path = '/'.join(test.path_in_suite[:-1]).replace('.', '_') - # class_name = f'{suite_name}.{path or suite_name}' - class_name = suite_name + '.' + (path or suite_name) + class_name = suite_name + if test.suite.is_multi_cfg: + class_name += '.' + getRoot(test.config).name + class_name += '.' + (path or suite_name) name = test.path_in_suite[-1] time = test.result.elapsed or 0.0 # file.write(f' %t.out 2> %t.err +# RUN: FileCheck --check-prefix=CHECK-MULTI-OUT < %t.out %s +# RUN: FileCheck --check-prefix=CHECK-MULTI-ERR < %t.err %s +# +# CHECK-MULTI-OUT: -- Test Suites -- +# CHECK-MULTI-OUT: sub-suite - 2 tests +# CHECK-MULTI-OUT: Source Root: {{.*[/\\]discovery[/\\]subsuite$}} +# CHECK-MULTI-OUT: Exec Root : {{.*[/\\]discovery[/\\]subsuite$}} +# CHECK-MULTI-OUT: test-multi-cfg-test-suite :: exec-discovery-in-tree-suite - 1 tests +# CHECK-MULTI-OUT: Source Root: {{.*[/\\]exec-discovery-in-tree$}} +# CHECK-MULTI-OUT: Exec Root : {{.*[/\\]exec-discovery-in-tree[/\\]obj$}} +# CHECK-MULTI-OUT: test-multi-cfg-test-suite :: top-level-suite - 3 tests +# CHECK-MULTI-OUT: Source Root: {{.*[/\\]discovery$}} +# CHECK-MULTI-OUT: Exec Root : {{.*[/\\]exec-discovery$}} +# CHECK-MULTI-OUT: -- Available Tests -- +# CHECK-MULTI-OUT: sub-suite :: test-one +# CHECK-MULTI-OUT: sub-suite :: test-two +# CHECK-MULTI-OUT: exec-discovery-in-tree-suite :: test-one +# CHECK-MULTI-OUT: top-level-suite :: subdir/test-three +# CHECK-MULTI-OUT: top-level-suite :: test-one +# CHECK-MULTI-OUT: top-level-suite :: test-two +# +# CHECK-MULTI-ERR: loading suite config '{{.*(/|\\\\)exec-discovery(/|\\\\)lit.site.cfg}}' +# CHECK-MULTI-ERR-DAG: loading suite config '{{.*(/|\\\\)exec-discovery-in-tree(/|\\\\)obj(/|\\\\)lit.site.cfg}}' +# CHECK-MULTI-ERR-DAG: loading local config '{{.*(/|\\\\)discovery(/|\\\\)subdir(/|\\\\)lit.local.cfg}}' +# CHECK-MULTI-ERR-DAG: loading suite config '{{.*(/|\\\\)discovery(/|\\\\)subsuite(/|\\\\)lit.cfg}}' + # Check discovery when providing the special builtin 'config_map' # RUN: %{python} %{inputs}/config-map-discovery/driver.py \ # RUN: %{inputs}/config-map-discovery/main-config/lit.cfg \