diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst --- a/llvm/docs/CommandGuide/lit.rst +++ b/llvm/docs/CommandGuide/lit.rst @@ -154,6 +154,11 @@ suite take the most time to execute. Note that this option is most useful with ``-j 1``. +.. option:: --no-indirectly-run-check + + Do not error if a test would not be run if the user had specified the + containing directory instead of naming the test directly. + .. _selection-options: SELECTION OPTIONS @@ -372,6 +377,11 @@ **environment** A dictionary representing the environment to use when executing tests in the suite. + **standalone_tests** When true, mark a directory with tests expected to be run + standalone. Test discovery is disabled for that directory and + *--no-indirectly-run-check* is in effect. *lit.suffixes* and *lit.excludes* + must be empty when this variable is true. + **suffixes** For **lit** test formats which scan directories for tests, this variable is a list of suffixes to identify test files. Used by: *ShTest*. diff --git a/llvm/utils/lit/lit/TestingConfig.py b/llvm/utils/lit/lit/TestingConfig.py --- a/llvm/utils/lit/lit/TestingConfig.py +++ b/llvm/utils/lit/lit/TestingConfig.py @@ -60,7 +60,8 @@ test_source_root = None, excludes = [], available_features = available_features, - pipefail = True) + pipefail = True, + standalone_tests = False) def load_from_path(self, path, litConfig): """ @@ -105,7 +106,8 @@ environment, substitutions, unsupported, test_exec_root, test_source_root, excludes, available_features, pipefail, limit_to_features = [], - is_early = False, parallelism_group = None): + is_early = False, parallelism_group = None, + standalone_tests = False): self.parent = parent self.name = str(name) self.suffixes = set(suffixes) @@ -118,6 +120,7 @@ self.excludes = set(excludes) self.available_features = set(available_features) self.pipefail = pipefail + self.standalone_tests = standalone_tests # This list is used by TestRunner.py to restrict running only tests that # require one of the features in this list if this list is non-empty. # Configurations can set this list to restrict the set of tests to run. diff --git a/llvm/utils/lit/lit/discovery.py b/llvm/utils/lit/lit/discovery.py --- a/llvm/utils/lit/lit/discovery.py +++ b/llvm/utils/lit/lit/discovery.py @@ -160,8 +160,13 @@ # tests which are not executed. The check adds some performance # overhead which might be important if a large number of tests # are being run directly. - # --no-indirectly-run-check: skips this check. - if indirectlyRunCheck and lc.test_format is not None: + # This check can be disabled by using --no-indirectly-run-check or + # setting the standalone_tests variable in the suite's configuration. + if ( + indirectlyRunCheck + and lc.test_format is not None + and not lc.standalone_tests + ): found = False for res in lc.test_format.getTestsInDirectory(ts, test_dir_in_suite, litConfig, lc): @@ -171,6 +176,7 @@ if not found: litConfig.error( '%r would not be run indirectly: change name or LIT config' + '(e.g. suffixes or standalone_tests variables)' % test.getFullName()) yield test @@ -180,6 +186,15 @@ # local configuration. lc = getLocalConfig(ts, path_in_suite, litConfig, localConfigCache) + # Directory contains tests to be run standalone. Do not try to discover. + if lc.standalone_tests: + if lc.suffixes or lc.excludes: + litConfig.warning( + 'standalone_tests set in LIT config but suffixes or excludes' + ' are also set' + ) + return + # Search for tests. if lc.test_format is not None: for res in lc.test_format.getTestsInDirectory(ts, path_in_suite, diff --git a/llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/lit.cfg b/llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/lit.cfg new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/lit.cfg @@ -0,0 +1,5 @@ +import lit.formats +config.name = 'Standalone tests' +config.test_format = lit.formats.ShTest() +config.excludes = ['.test'] +config.standalone_tests = True diff --git a/llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/true.txt b/llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/true.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/true.txt @@ -0,0 +1 @@ +# RUN: true diff --git a/llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/lit.cfg b/llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/lit.cfg new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/lit.cfg @@ -0,0 +1,5 @@ +import lit.formats +config.name = 'Standalone tests' +config.test_format = lit.formats.ShTest() +config.suffixes = ['.txt'] +config.standalone_tests = True diff --git a/llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/true.txt b/llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/true.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/true.txt @@ -0,0 +1 @@ +# RUN: true diff --git a/llvm/utils/lit/tests/Inputs/standalone-tests/lit.cfg b/llvm/utils/lit/tests/Inputs/standalone-tests/lit.cfg new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/standalone-tests/lit.cfg @@ -0,0 +1,4 @@ +import lit.formats +config.name = 'Standalone tests' +config.test_format = lit.formats.ShTest() +config.standalone_tests = True diff --git a/llvm/utils/lit/tests/Inputs/standalone-tests/true.txt b/llvm/utils/lit/tests/Inputs/standalone-tests/true.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/standalone-tests/true.txt @@ -0,0 +1 @@ +# RUN: true diff --git a/llvm/utils/lit/tests/discovery.py b/llvm/utils/lit/tests/discovery.py --- a/llvm/utils/lit/tests/discovery.py +++ b/llvm/utils/lit/tests/discovery.py @@ -148,6 +148,36 @@ # RUN: %{lit} \ # RUN: %{inputs}/discovery/test.not-txt -j 1 --no-indirectly-run-check +# Check that a standalone test with no suffixes set is run without any errors. +# +# RUN: %{lit} %{inputs}/standalone-tests/true.txt -j 1 > %t.out +# RUN: FileCheck --check-prefix=CHECK-STANDALONE < %t.out %s +# +# CHECK-STANDALONE: PASS: Standalone tests :: true.txt + +# Check that an error is produced if suffixes variable is set for a suite with +# standalone tests. +# +# RUN: not %{lit} %{inputs}/standalone-tests-with-suffixes -j 1 2> %t.err +# RUN: FileCheck --check-prefixes=CHECK-STANDALONE-SUFFIXES,CHECK-STANDALONE-DISCOVERY < %t.err %s +# +# CHECK-STANDALONE-SUFFIXES: standalone_tests set {{.*}} but suffixes + +# Check that an error is produced if excludes variable is set for a suite with +# standalone tests. +# +# RUN: not %{lit} %{inputs}/standalone-tests-with-excludes -j 1 2> %t.err +# RUN: FileCheck --check-prefixes=CHECK-STANDALONE-EXCLUDES,CHECK-STANDALONE-DISCOVERY < %t.err %s +# +# CHECK-STANDALONE-EXCLUDES: standalone_tests set {{.*}} but {{.*}} excludes + +# Check that no discovery is done for testsuite with standalone tests. +# +# RUN: not %{lit} %{inputs}/standalone-tests -j 1 2>%t.err +# RUN: FileCheck --check-prefix=CHECK-STANDALONE-DISCOVERY < %t.err %s +# +# CHECK-STANDALONE-DISCOVERY: error: did not discover any tests for provided path(s) + # Check that we don't recurse infinitely when loading an site specific test # suite located inside the test source root. #