Index: packages/Python/lldbsuite/test/configuration.py =================================================================== --- packages/Python/lldbsuite/test/configuration.py +++ packages/Python/lldbsuite/test/configuration.py @@ -101,6 +101,12 @@ # our test cases. regexp = None +# Sets of tests which are excluded at runtime +skip_files = None +skip_methods = None +xfail_files = None +xfail_methods = None + # By default, recorded session info for errored/failed test are dumped into its # own file under a session directory named after the timestamp of the test suite # run. Use '-s session-dir-name' to specify a specific dir name. Index: packages/Python/lldbsuite/test/dotest.py =================================================================== --- packages/Python/lldbsuite/test/dotest.py +++ packages/Python/lldbsuite/test/dotest.py @@ -202,6 +202,48 @@ sys.exit(0) +def parseExclusion(exclusion_file): + """Parse an exclusion file, of the following format, where + 'skip files', 'skip methods', 'xfail files', and 'xfail methods' + are the possible list heading values: + + skip files + + + + xfail methods + + """ + excl_type = None + case_type = None + + with open(exclusion_file) as f: + for line in f: + if not excl_type: + [excl_type, case_type] = line.split() + continue + + line = line.strip() + if not line: + excl_type = None + elif excl_type == 'skip' and case_type == 'files': + if not configuration.skip_files: + configuration.skip_files = [] + configuration.skip_files.append(line) + elif excl_type == 'skip' and case_type == 'methods': + if not configuration.skip_methods: + configuration.skip_methods = [] + configuration.skip_methods.append(line) + elif excl_type == 'xfail' and case_type == 'files': + if not configuration.xfail_files: + configuration.xfail_files = [] + configuration.xfail_files.append(line) + elif excl_type == 'xfail' and case_type == 'methods': + if not configuration.xfail_methods: + configuration.xfail_methods = [] + configuration.xfail_methods.append(line) + + def parseOptionsAndInitTestdirs(): """Initialize the list of directories containing our unittest scripts. @@ -331,6 +373,9 @@ if args.executable: lldbtest_config.lldbExec = os.path.realpath(args.executable) + if args.excluded: + parseExclusion(args.excluded) + if args.p: if args.p.startswith('-'): usage(parser) @@ -754,6 +799,12 @@ # We didn't match the regex, we're done. return + if configuration.skip_files: + import re + for file_regexp in configuration.skip_files: + if re.search(file_regexp, name): + return + # We found a match for our test. Add it to the suite. # Update the sys.path first. Index: packages/Python/lldbsuite/test/dotest_args.py =================================================================== --- packages/Python/lldbsuite/test/dotest_args.py +++ packages/Python/lldbsuite/test/dotest_args.py @@ -96,6 +96,9 @@ '-p', metavar='pattern', help='Specify a regexp filename pattern for inclusion in the test suite') + group.add_argument('--excluded', metavar='exclusion-file', help=textwrap.dedent( + '''Specify a file for tests to exclude. File should contain lists of regular expressions for test files or methods, + with each list under a matching header (xfail files, xfail methods, skip files, skip methods)''')) group.add_argument( '-G', '--category', Index: packages/Python/lldbsuite/test/test_result.py =================================================================== --- packages/Python/lldbsuite/test/test_result.py +++ packages/Python/lldbsuite/test/test_result.py @@ -18,6 +18,8 @@ # Third-party modules import unittest2 +from unittest2.util import strclass + # LLDB Modules from . import configuration from lldbsuite.test_event.event_builder import EventBuilder @@ -124,10 +126,23 @@ test, test._testMethodName).__func__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run" + def checkExclusion(self, exclusion_list, name): + if exclusion_list: + import re + for item in exclusion_list: + if re.search(item, name): + return True + return False + def startTest(self, test): if configuration.shouldSkipBecauseOfCategories( self.getCategoriesForTest(test)): self.hardMarkAsSkipped(test) + if self.checkExclusion( + configuration.skip_methods, + test._testMethodName): + self.hardMarkAsSkipped(test) + configuration.setCrashInfoHook( "%s at %s" % (str(test), inspect.getfile( @@ -145,6 +160,15 @@ EventBuilder.event_for_start(test)) def addSuccess(self, test): + if self.checkExclusion( + configuration.xfail_files, + strclass( + test.__class__)) or self.checkExclusion( + configuration.xfail_methods, + test._testMethodName): + self.addUnexpectedSuccess(test, None) + return + super(LLDBTestResult, self).addSuccess(test) if configuration.parsable: self.stream.write( @@ -214,6 +238,15 @@ test, err)) def addFailure(self, test, err): + if self.checkExclusion( + configuration.xfail_files, + strclass( + test.__class__)) or self.checkExclusion( + configuration.xfail_methods, + test._testMethodName): + self.addExpectedFailure(test, err, None) + return + configuration.sdir_has_content = True super(LLDBTestResult, self).addFailure(test, err) method = getattr(test, "markFailure", None)