Index: utils/lit/lit/formats/__init__.py =================================================================== --- utils/lit/lit/formats/__init__.py +++ utils/lit/lit/formats/__init__.py @@ -1,3 +1,8 @@ -from lit.formats.base import TestFormat # noqa: F401 +from lit.formats.base import ( # noqa: F401 + TestFormat, + FileBasedTest, + OneCommandPerFileTest +) + from lit.formats.googletest import GoogleTest # noqa: F401 from lit.formats.shtest import ShTest # noqa: F401 Index: utils/lit/lit/formats/base.py =================================================================== --- utils/lit/lit/formats/base.py +++ utils/lit/lit/formats/base.py @@ -1,50 +1,117 @@ -import abc +from __future__ import absolute_import +import os + +import lit.Test +import lit.util class TestFormat(object): - """Base class for test formats. - - A TestFormat encapsulates logic for finding and executing a certain type of - test. For example, a subclass FooTestFormat would contain the logic for - finding tests written in the 'Foo' format, and the logic for running a - single one. - - TestFormat is an Abstract Base Class (ABC). It uses the Python abc.ABCMeta - type and associated @abc.abstractmethod decorator. Together, these provide - subclass behaviour which is notionally similar to C++ pure virtual classes: - only subclasses which implement all abstract methods can be instantiated - (the implementation may come from an intermediate base). - - For details on ABCs, see: https://docs.python.org/2/library/abc.html. Note - that Python ABCs have extensive abilities beyond what is used here. For - TestFormat, we only care about enforcing that abstract methods are - implemented. - """ - - __metaclass__ = abc.ABCMeta - - @abc.abstractmethod - def getTestsInDirectory(self, testSuite, path_in_suite, litConfig, - localConfig): - """Finds tests of this format in the given directory. - - Args: - testSuite: a Test.TestSuite object. - path_in_suite: the subpath under testSuite to look for tests. - litConfig: the LitConfig for the test suite. - localConfig: a LitConfig with local specializations. - - Returns: - An iterable of Test.Test objects. - """ - - @abc.abstractmethod + pass + +### + +class FileBasedTest(TestFormat): + def getTestsInDirectory(self, testSuite, path_in_suite, + litConfig, localConfig): + source_path = testSuite.getSourcePath(path_in_suite) + for filename in os.listdir(source_path): + # Ignore dot files and excluded tests. + if (filename.startswith('.') or + filename in localConfig.excludes): + continue + + filepath = os.path.join(source_path, filename) + if not os.path.isdir(filepath): + base,ext = os.path.splitext(filename) + if ext in localConfig.suffixes: + yield lit.Test.Test(testSuite, path_in_suite + (filename,), + localConfig) + +### + +import re +import tempfile + +class OneCommandPerFileTest(TestFormat): + # FIXME: Refactor into generic test for running some command on a directory + # of inputs. + + def __init__(self, command, dir, recursive=False, + pattern=".*", useTempInput=False): + if isinstance(command, str): + self.command = [command] + else: + self.command = list(command) + if dir is not None: + dir = str(dir) + self.dir = dir + self.recursive = bool(recursive) + self.pattern = re.compile(pattern) + self.useTempInput = useTempInput + + def getTestsInDirectory(self, testSuite, path_in_suite, + litConfig, localConfig): + dir = self.dir + if dir is None: + dir = testSuite.getSourcePath(path_in_suite) + + for dirname,subdirs,filenames in os.walk(dir): + if not self.recursive: + subdirs[:] = [] + + subdirs[:] = [d for d in subdirs + if (d != '.svn' and + d not in localConfig.excludes)] + + for filename in filenames: + if (filename.startswith('.') or + not self.pattern.match(filename) or + filename in localConfig.excludes): + continue + + path = os.path.join(dirname,filename) + suffix = path[len(dir):] + if suffix.startswith(os.sep): + suffix = suffix[1:] + test = lit.Test.Test( + testSuite, path_in_suite + tuple(suffix.split(os.sep)), + localConfig) + # FIXME: Hack? + test.source_path = path + yield test + + def createTempInput(self, tmp, test): + raise NotImplementedError('This is an abstract method.') + def execute(self, test, litConfig): - """Runs the given 'test', which is of this format. + if test.config.unsupported: + return (lit.Test.UNSUPPORTED, 'Test is unsupported') + + cmd = list(self.command) + + # If using temp input, create a temporary file and hand it to the + # subclass. + if self.useTempInput: + tmp = tempfile.NamedTemporaryFile(suffix='.cpp') + self.createTempInput(tmp, test) + tmp.flush() + cmd.append(tmp.name) + elif hasattr(test, 'source_path'): + cmd.append(test.source_path) + else: + cmd.append(test.getSourcePath()) + + out, err, exitCode = lit.util.executeCommand(cmd) + + diags = out + err + if not exitCode and not diags.strip(): + return lit.Test.PASS,'' - Args: - test: a Test.Test object describing the test to run. - litConfig: the LitConfig for the test suite. + # Try to include some useful information. + report = """Command: %s\n""" % ' '.join(["'%s'" % a + for a in cmd]) + if self.useTempInput: + report += """Temporary File: %s\n""" % tmp.name + report += "--\n%s--\n""" % open(tmp.name).read() + report += """Output:\n--\n%s--""" % diags - Returns: - A tuple of (status:Test.ResultCode, message:str) - """ + return lit.Test.FAIL, report Index: utils/lit/lit/formats/shtest.py =================================================================== --- utils/lit/lit/formats/shtest.py +++ utils/lit/lit/formats/shtest.py @@ -1,13 +1,12 @@ from __future__ import absolute_import -import os - -import lit.Test import lit.TestRunner import lit.util -from .base import TestFormat -class ShTest(TestFormat): +from .base import FileBasedTest + + +class ShTest(FileBasedTest): """ShTest is a format with one file per test. This is the primary format for regression tests as described in the LLVM @@ -18,31 +17,9 @@ The ShTest files contain some number of shell-like command pipelines, along with assertions about what should be in the output. """ - - def __init__(self, execute_external = False): - """Initializer. - - The 'execute_external' argument controls whether lit uses its internal - logic for command pipelines, or passes the command to a shell - subprocess. - - Args: - execute_external: (optional) If true, use shell subprocesses instead - of lit's internal pipeline logic. - """ + def __init__(self, execute_external=False): self.execute_external = execute_external - def getTestsInDirectory(self, testSuite, path_in_suite, - litConfig, localConfig): - """Yields test files matching 'suffixes' from the localConfig.""" - file_matches = lit.util.listdir_files( - testSuite.getSourcePath(path_in_suite), - localConfig.suffixes, localConfig.excludes) - for filename in file_matches: - yield lit.Test.Test(testSuite, path_in_suite + (filename,), - localConfig) - def execute(self, test, litConfig): - """Interprets and runs the given test file, and returns the result.""" return lit.TestRunner.executeShTest(test, litConfig, self.execute_external)