diff --git a/libcxx/utils/libcxx/test/format.py b/libcxx/utils/libcxx/test/format.py --- a/libcxx/utils/libcxx/test/format.py +++ b/libcxx/utils/libcxx/test/format.py @@ -182,7 +182,7 @@ return script -class CxxStandardLibraryTest(lit.formats.TestFormat): +class CxxStandardLibraryTest(lit.formats.FileBasedTest): """ Lit test format for the C++ Standard Library conformance test suite. @@ -278,7 +278,7 @@ in conjunction with the %{build} substitution. """ - def getTestsInDirectory(self, testSuite, pathInSuite, litConfig, localConfig): + def getTestsForPath(self, testSuite, pathInSuite, litConfig, localConfig): SUPPORTED_SUFFIXES = [ "[.]pass[.]cpp$", "[.]pass[.]mm$", @@ -293,22 +293,22 @@ "[.]verify[.]cpp$", "[.]fail[.]cpp$", ] + sourcePath = testSuite.getSourcePath(pathInSuite) - for filename in os.listdir(sourcePath): - # Ignore dot files and excluded tests. - if filename.startswith(".") or filename in localConfig.excludes: - continue - - filepath = os.path.join(sourcePath, filename) - if not os.path.isdir(filepath): - if any([re.search(ext, filename) for ext in SUPPORTED_SUFFIXES]): - # If this is a generated test, run the generation step and add - # as many Lit tests as necessary. - if re.search('[.]gen[.][^.]+$', filename): - for test in self._generateGenTest(testSuite, pathInSuite + (filename,), litConfig, localConfig): - yield test - else: - yield lit.Test.Test(testSuite, pathInSuite + (filename,), localConfig) + filename = os.path.basename(sourcePath) + + # Ignore dot files, excluded tests and tests with an unsupported suffix + hasSupportedSuffix = lambda f: any([re.search(ext, f) for ext in SUPPORTED_SUFFIXES]) + if filename.startswith(".") or filename in localConfig.excludes or not hasSupportedSuffix(filename): + return + + # If this is a generated test, run the generation step and add + # as many Lit tests as necessary. + if re.search('[.]gen[.][^.]+$', filename): + for test in self._generateGenTest(testSuite, pathInSuite, litConfig, localConfig): + yield test + else: + yield lit.Test.Test(testSuite, pathInSuite, localConfig) def execute(self, test, litConfig): VERIFY_FLAGS = ( diff --git a/llvm/utils/lit/examples/many-tests/ManyTests.py b/llvm/utils/lit/examples/many-tests/ManyTests.py --- a/llvm/utils/lit/examples/many-tests/ManyTests.py +++ b/llvm/utils/lit/examples/many-tests/ManyTests.py @@ -1,7 +1,7 @@ -from lit import Test +from lit import Test, TestFormat -class ManyTests(object): +class ManyTests(TestFormat): def __init__(self, N=10000): self.N = N diff --git a/llvm/utils/lit/lit/discovery.py b/llvm/utils/lit/lit/discovery.py --- a/llvm/utils/lit/lit/discovery.py +++ b/llvm/utils/lit/lit/discovery.py @@ -163,35 +163,43 @@ if not os.path.isdir(source_path): test_dir_in_suite = path_in_suite[:-1] lc = getLocalConfig(ts, test_dir_in_suite, litConfig, localConfigCache) - test = Test.Test(ts, path_in_suite, lc) - - # Issue a error if the specified test would not be run if - # the user had specified the containing directory instead of - # of naming the test directly. This helps to avoid writing - # tests which are not executed. The check adds some performance - # overhead which might be important if a large number of tests - # are being run directly. - # This check can be disabled by using --no-indirectly-run-check or - # setting the standalone_tests variable in the suite's configuration. - if ( - indirectlyRunCheck - and lc.test_format is not None - and not lc.standalone_tests - ): - found = False - for res in lc.test_format.getTestsInDirectory( - ts, test_dir_in_suite, litConfig, lc + + # TODO: Stop checking for indirectlyRunCheck and lc.standalone_tests here + # once we remove --no-indirectly-run-check, which is not needed anymore + # now that we error out when trying to run a test that wouldn't be + # discovered in the directory. + fallbackOnSingleTest = lc.test_format is None or not indirectlyRunCheck or lc.standalone_tests + tests = [Test.Test(ts, path_in_suite, lc)] if fallbackOnSingleTest else \ + lc.test_format.getTestsForPath(ts, path_in_suite, litConfig, lc) + + for test in tests: + # Issue a error if the specified test would not be run if + # the user had specified the containing directory instead of + # of naming the test directly. This helps to avoid writing + # tests which are not executed. The check adds some performance + # overhead which might be important if a large number of tests + # are being run directly. + # This check can be disabled by using --no-indirectly-run-check or + # setting the standalone_tests variable in the suite's configuration. + if ( + indirectlyRunCheck + and lc.test_format is not None + and not lc.standalone_tests ): - if test.getFullName() == res.getFullName(): - found = True - break - if not found: - litConfig.error( - "%r would not be run indirectly: change name or LIT config" - "(e.g. suffixes or standalone_tests variables)" % test.getFullName() - ) - - yield test + found = False + for res in lc.test_format.getTestsInDirectory( + ts, test_dir_in_suite, litConfig, lc + ): + if test.getFullName() == res.getFullName(): + found = True + break + if not found: + litConfig.error( + "%r would not be run indirectly: change name or LIT config" + "(e.g. suffixes or standalone_tests variables)" % test.getFullName() + ) + + yield test return # Otherwise we have a directory to search for tests, start by getting the diff --git a/llvm/utils/lit/lit/formats/base.py b/llvm/utils/lit/lit/formats/base.py --- a/llvm/utils/lit/lit/formats/base.py +++ b/llvm/utils/lit/lit/formats/base.py @@ -6,27 +6,42 @@ class TestFormat(object): - pass - + def getTestsForPath(self, testSuite, path_in_suite, litConfig, localConfig): + """ + Given the path to a test in the test suite, generates the Lit tests associated + to that path. There can be zero, one or more tests. For example, some testing + formats allow expanding a single path in the test suite into multiple Lit tests + (e.g. they are generated on the fly). + """ + yield lit.Test.Test(testSuite, path_in_suite, localConfig) ### class FileBasedTest(TestFormat): + def getTestsForPath(self, testSuite, path_in_suite, litConfig, localConfig): + """ + Expand each path in a test suite to a Lit test using that path and assuming + it is a file containing the test. File extensions excluded by the configuration + or not contained in the allowed extensions are ignored. + """ + filename = path_in_suite[-1] + + # Ignore dot files and excluded tests. + if filename.startswith(".") or filename in localConfig.excludes: + return + + base, ext = os.path.splitext(filename) + if ext in localConfig.suffixes: + yield lit.Test.Test(testSuite, path_in_suite, localConfig) + def getTestsInDirectory(self, testSuite, path_in_suite, litConfig, localConfig): source_path = testSuite.getSourcePath(path_in_suite) for filename in os.listdir(source_path): - # Ignore dot files and excluded tests. - if filename.startswith(".") or filename in localConfig.excludes: - continue - filepath = os.path.join(source_path, filename) if not os.path.isdir(filepath): - base, ext = os.path.splitext(filename) - if ext in localConfig.suffixes: - yield lit.Test.Test( - testSuite, path_in_suite + (filename,), localConfig - ) + for t in self.getTestsForPath(testSuite, path_in_suite + (filename,), litConfig, localConfig): + yield t ### diff --git a/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/custom_format.py b/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/custom_format.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/custom_format.py @@ -0,0 +1,12 @@ +import os +import lit.formats + +class CustomFormat(lit.formats.ShTest): + def getTestsForPath(self, testSuite, path_in_suite, litConfig, localConfig): + for sub in ['one.test', 'two.test']: + basePath = os.path.dirname(testSuite.getExecPath(path_in_suite)) + os.makedirs(basePath, exist_ok=True) + generatedFile = os.path.join(basePath, sub) + with open(generatedFile, 'w') as f: + f.write('RUN: true') + yield lit.Test.Test(testSuite, (generatedFile, ), localConfig) diff --git a/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/lit.cfg b/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/lit.cfg new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/lit.cfg @@ -0,0 +1,9 @@ +import sys, os +sys.path.append(os.path.dirname(__file__)) +from custom_format import CustomFormat + +config.name = "discovery-getTestsForPath-suite" +config.suffixes = [".test"] +config.test_format = CustomFormat() +config.test_source_root = None +config.test_exec_root = None diff --git a/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/x.test b/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/x.test new file mode 100644 diff --git a/llvm/utils/lit/tests/discovery.py b/llvm/utils/lit/tests/discovery.py --- a/llvm/utils/lit/tests/discovery.py +++ b/llvm/utils/lit/tests/discovery.py @@ -134,14 +134,15 @@ # CHECK-ASEXEC-DIRECT-TEST: -- Available Tests -- # CHECK-ASEXEC-DIRECT-TEST: top-level-suite :: subdir/test-three -# Check an error is emitted when the directly named test would not be run -# indirectly (e.g. when the directory containing the test is specified). +# Check that an error is emitted when the directly named test does not satisfy +# the test config's requirements. # # RUN: not %{lit} \ # RUN: %{inputs}/discovery/test.not-txt 2>%t.err -# RUN: FileCheck --check-prefix=CHECK-ERROR-INDIRECT-RUN-CHECK < %t.err %s +# RUN: FileCheck --check-prefix=CHECK-ERROR-INPUT-CONTAINED-NO-TESTS < %t.err %s # -# CHECK-ERROR-INDIRECT-RUN-CHECK: error: 'top-level-suite :: test.not-txt' would not be run indirectly +# CHECK-ERROR-INPUT-CONTAINED-NO-TESTS: warning: input 'Inputs/discovery/test.not-txt' contained no tests +# CHECK-ERROR-INPUT-CONTAINED-NO-TESTS: error: did not discover any tests for provided path(s) # Check that no error is emitted with --no-indirectly-run-check. # @@ -178,6 +179,15 @@ # # CHECK-STANDALONE-DISCOVERY: error: did not discover any tests for provided path(s) +# Check that a single file path can result in multiple tests being discovered if +# the test format implements those semantics. +# +# RUN: %{lit} %{inputs}/discovery-getTestsForPath/x.test > %t.out +# RUN: FileCheck --check-prefix=CHECK-getTestsForPath < %t.out %s +# +# CHECK-getTestsForPath: PASS: discovery-getTestsForPath-suite :: {{.+}}one.test +# CHECK-getTestsForPath: PASS: discovery-getTestsForPath-suite :: {{.+}}two.test + # Check that we don't recurse infinitely when loading an site specific test # suite located inside the test source root. #