[lldb][tests] Make it possible to expect failure for a whole category

[tatyana-krasnukha]

Summary

There already are decorators and "--excluded" option to mark test-cases/files
as expected to fail. However, when a new test file is added and it relates
to a feature that a target doesn't support, this requires either adding decorators
to that file or modifying the file provided as "--excluded" option value.

The purpose of this patch is to avoid any modifications in such cases.
E.g. if a target doesn't support "watchpoints" and passes "--xfail-category watchpoint"
to dotest, a testing job will not fail after a new watchpoint-related test file is added.

[teemperor]

This LGTM but I'm not the right person to sign this off. I guess Pavel and Jonas know this code better.

[JDevlieghere]

If I understand correctly this is the equivalent of skip-category but you want to XFAIL the tests instead of skip them? Seems reasonable to me :-)

[labath]

Seems reasonable, just make the option name consistent.

packages/Python/lldbsuite/test/dotest_args.py

85–88 ↗

(On Diff #235349)

Please use a consistent name for the two options.

[tatyana-krasnukha]

Increased number of lines of context.

Jonas, yes, it is.

Pavel, names are consistent (if I understand your comment correctly). The "--xfail-category" is singular like the "--skip-category".

[JDevlieghere]

Thanks!

lldb/packages/Python/lldbsuite/test/configuration.py

 suite = unittest2.TestSuite()
 
 # The list of categories we said we care about
 categories_list = None
 # set to true if we are going to use categories for cherry-picking test cases
 use_categories = False
 # Categories we want to skip
 skip_categories = ["darwin-log"]
+# Categories we expect to fail
+xfail_categories = []
 # use this to track per-category failures
 failures_per_category = {}
 
 # The path to LLDB.framework is optional.
 lldb_framework_path = None
 
 # Test suite repeat count.  Can be overwritten with '-# count'.
 count = 1

lldb/packages/Python/lldbsuite/test/dotest.py

         configuration.use_categories = True
     else:
         configuration.categories_list = []
 
     if args.skip_categories:
         configuration.skip_categories += test_categories.validate(
             args.skip_categories, False)
 
+    if args.xfail_categories:
+        configuration.xfail_categories += test_categories.validate(
+            args.xfail_categories, False)
+
     if args.E:
         os.environ['CFLAGS_EXTRAS'] = args.E
 
     if args.dwarf_version:
         configuration.dwarf_version = args.dwarf_version
         # We cannot modify CFLAGS_EXTRAS because they're used in test cases
         # that explicitly require no debug info.
         os.environ['CFLAGS'] = '-gdwarf-{}'.format(configuration.dwarf_version)

lldb/packages/Python/lldbsuite/test/dotest_args.py

group.add_argument(
         dest='categories_list',
         help=textwrap.dedent('''Specify categories of test cases of interest. Can be specified more than once.'''))
     group.add_argument(
         '--skip-category',
         metavar='category',
         action='append',
         dest='skip_categories',
         help=textwrap.dedent('''Specify categories of test cases to skip. Takes precedence over -G. Can be specified more than once.'''))
+    group.add_argument(
+        '--xfail-category',
+        metavar='category',
+        action='append',
+        dest='xfail_categories',
+        help=textwrap.dedent('''Specify categories of test cases that are expected to fail. Can be specified more than once.'''))
 
     # Configuration options
     group = parser.add_argument_group('Configuration options')
     group.add_argument(
         '--framework',
         metavar='framework-path',
         help='The path to LLDB.framework')
     group.add_argument(

lldb/packages/Python/lldbsuite/test/test_result.py

     def checkExclusion(self, exclusion_list, name):
         if exclusion_list:
             import re
             for item in exclusion_list:
                 if re.search(item, name):
                     return True
         return False
 
+    def checkCategoryExclusion(self, exclusion_list, test):
+        return not set(exclusion_list).isdisjoint(
+            self.getCategoriesForTest(test))
+
     def startTest(self, test):
         if configuration.shouldSkipBecauseOfCategories(
                 self.getCategoriesForTest(test)):
             self.hardMarkAsSkipped(test)
         if self.checkExclusion(
                 configuration.skip_tests, test.id()):
             self.hardMarkAsSkipped(test)
 
         self.counter += 1
         test.test_number = self.counter
         if self.showAll:
             self.stream.write(self.fmt % self.counter)
         super(LLDBTestResult, self).startTest(test)
         if self.results_formatter:
             self.results_formatter.handle_event(
                 EventBuilder.event_for_start(test))
 
     def addSuccess(self, test):
-        if self.checkExclusion(
-                configuration.xfail_tests, test.id()):
+        if (self.checkExclusion(
+                configuration.xfail_tests, test.id()) or
+            self.checkCategoryExclusion(
+                configuration.xfail_categories, test)):
             self.addUnexpectedSuccess(test, None)
             return
 
         super(LLDBTestResult, self).addSuccess(test)
         self.stream.write(
             "PASS: LLDB (%s) :: %s\n" %
             (self._config_string(test), str(test)))
         if self.results_formatter:

             "CLEANUP ERROR: LLDB (%s) :: %s\n" %
             (self._config_string(test), str(test)))
         if self.results_formatter:
             self.results_formatter.handle_event(
                 EventBuilder.event_for_cleanup_error(
                     test, err))
 
     def addFailure(self, test, err):
-        if self.checkExclusion(
-                configuration.xfail_tests, test.id()):
+        if (self.checkExclusion(
+                configuration.xfail_tests, test.id()) or
+            self.checkCategoryExclusion(
+                configuration.xfail_categories, test)):
             self.addExpectedFailure(test, err, None)
             return
 
         configuration.sdir_has_content = True
         super(LLDBTestResult, self).addFailure(test, err)
         method = getattr(test, "markFailure", None)
         if method:
             method()