[test] Add ability to expect timeouts

[labath]

Summary

This adds ability to mark test that do not complete due to hangs, crashes, etc., as "expected",
to avoid flagging the build red for a known problem. Functionally, this extends the scope of the
existing expectedFailureXXX decorators to cover these states as well. Once this is in, I will
start replacing the magic list of failing tests in dosep.py with our regular annotations which
should hopefully make code simpler.

[tfiala]

Looks good!

[tfiala]

Hey Pavel,

Change r255676 changed a few things that will require this patch to get updated. I think the main code change will be that ResultsFormatter._make_rerun_eligibility_key(self) has been renamed to ResultsFormatter._make_key(self). I folded the BasicResultsFormatter class into ResultsFormatter since any specialized results formatters still likely want to display the results summary output at the end. That key is now used in several places and they're not at all specific to rerun eligibility.

The rest of the differences are likely just line number changes as the new code here will just need to slide into different final spots in the now top-of-tree version of result_formatter.py.

Thanks, and sorry for the collision!

[labath]

No worries. I wanted to rename that function anyway, but then I
forgot, so it's good that you have reminded me. :)

lldb/trunk/packages/Python/lldbsuite/test/dosep.py

def getExpectedTimeouts(platform_name):
     host = sys.platform
     if platform_name is None:
         target = sys.platform
     else:
         m = re.search(r'remote-(\w+)', platform_name)
         target = m.group(1)
 
     expected_timeout = set()
-    expected_timeout.add("TestExpectedTimeout.py")
 
     if target.startswith("linux"):
         expected_timeout |= {
             "TestConnectRemote.py",
             "TestCreateAfterAttach.py",
             "TestEvents.py",
             "TestExitDuringStep.py",
 

lldb/trunk/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park

 """Tests that a timeout is detected by the testbot."""
 from __future__ import print_function
 
 import time
 
 import lldbsuite.test.lldbtest as lldbtest
 
 
 class ExpectedTimeoutTestCase(lldbtest.TestBase):
     """Forces test timeout."""
     mydir = lldbtest.TestBase.compute_mydir(__file__)
 
+    @lldbtest.expectedFailureAll()
     def test_buildbot_sees_expected_timeout(self):
         """Tests that expected timeout logic kicks in and is picked up."""
         while True:
             try:
                 time.sleep(1)
             except:
                 print("ignoring exception during sleep")

lldb/trunk/packages/Python/lldbsuite/test/lldbtest.py

 
 def expectedFailure(expected_fn, bugnumber=None):
     def expectedFailure_impl(func):
         @wraps(func)
         def wrapper(*args, **kwargs):
             from unittest2 import case
             self = args[0]
             if expected_fn(self):
+                if configuration.results_formatter_object is not None:
+                    # Mark this test as expected to fail.
+                    configuration.results_formatter_object.handle_event(
+                        EventBuilder.event_for_mark_test_expected_failure(self))
                 xfail_func = unittest2.expectedFailure(func)
                 xfail_func(*args, **kwargs)
             else:
                 func(*args, **kwargs)
         return wrapper
     # if bugnumber is not-callable(incluing None), that means decorator function is called with optional arguments
     # return decorator in this case, so it will be used to decorating original method
     if six.callable(bugnumber):

lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py

class EventBuilder(object):
 
     BASE_DICTIONARY = None
 
     # Test Event Types
     TYPE_JOB_RESULT = "job_result"
     TYPE_TEST_RESULT = "test_result"
     TYPE_TEST_START = "test_start"
     TYPE_MARK_TEST_RERUN_ELIGIBLE = "test_eligible_for_rerun"
+    TYPE_MARK_TEST_EXPECTED_FAILURE = "test_expected_failure"
     TYPE_SESSION_TERMINATE = "terminate"
 
     RESULT_TYPES = set([
         TYPE_JOB_RESULT,
         TYPE_TEST_RESULT
         ])
 
     # Test/Job Status Tags
def event_for_mark_test_rerun_eligible(test):
         be rerun.
         """
         event = EventBuilder._event_dictionary_common(
             test,
             EventBuilder.TYPE_MARK_TEST_RERUN_ELIGIBLE)
         return event
 
     @staticmethod
+    def event_for_mark_test_expected_failure(test):
+        """Creates an event that indicates the specified test is expected
+        to fail.
+
+        @param test the TestCase instance to which this pertains.
+
+        @return an event that specifies the given test is expected to fail.
+        """
+        event = EventBuilder._event_dictionary_common(
+            test,
+            EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE)
+        return event
+
+    @staticmethod
     def add_entries_to_all_events(entries_dict):
         """Specifies a dictionary of entries to add to all test events.
 
         This provides a mechanism for, say, a parallel test runner to
         indicate to each inferior dotest.py that it should add a
         worker index to each.
 
         Calling this method replaces all previous entries added
def __init__(self, out_file, options):
 
         # Keeps track of the test base filenames for tests that
         # are expected to timeout.  If a timeout occurs in any test
         # basename that matches this list, that result should be
         # converted into a non-issue.  We'll create an expected
         # timeout test status for this.
         self.expected_timeouts_by_basename = set()
 
+        # Tests which have reported that they are expecting to fail. These will
+        # be marked as expected failures even if they return a failing status,
+        # probably because they crashed or deadlocked.
+        self.expected_failures = set()
+
         # Keep track of rerun-eligible tests.
         # This is a set that contains tests saved as:
         # {test_filename}:{test_class}:{test_name}
         self.rerun_eligible_tests = set()
 
         # A dictionary of test files that had a failing
         # test, in the format of:
         # key = test path, value = array of test methods that need rerun
def _make_key(cls, result_event):
             component_count += 1
         if "test_name" in result_event:
             if component_count > 0:
                 key += "."
             key += result_event["test_name"]
             component_count += 1
         return key
 
+    def _mark_test_as_expected_failure(self, test_result_event):
+        key = self._make_key(test_result_event)
+        if key is not None:
+            self.expected_failures.add(key)
+        else:
+            sys.stderr.write(
+                "\nerror: test marked as expected failure but "
+                "failed to create key.\n")
+
     def _mark_test_for_rerun_eligibility(self, test_result_event):
         key = self._make_key(test_result_event)
         if key is not None:
             self.rerun_eligible_tests.add(key)
         else:
             sys.stderr.write(
                 "\nerror: test marked for re-run eligibility but "
                 "failed to create key.\n")
def _maybe_remap_expected_timeout(self, event):
 
         # Check if the timeout test's basename is in the expected timeout
         # list.  If so, convert to an expected timeout.
         basename = os.path.basename(event.get("test_filename", ""))
         if basename in self.expected_timeouts_by_basename:
             # Convert to an expected timeout.
             event["status"] = EventBuilder.STATUS_EXPECTED_TIMEOUT
 
+    def _maybe_remap_expected_failure(self, event):
+        if event is None:
+            return
+
+        key = self._make_key(event)
+        if key not in self.expected_failures:
+            return
+
+        status = event.get("status", None)
+        if status in EventBuilder.TESTRUN_ERROR_STATUS_VALUES:
+            event["status"] = EventBuilder.STATUS_EXPECTED_FAILURE
+        elif status == EventBuilder.STATUS_SUCCESS:
+            event["status"] = EventBuilder.STATUS_UNEXPECTED_SUCCESS
+
     def handle_event(self, test_event):
         """Handles the test event for collection into the formatter output.
 
         Derived classes may override this but should call down to this
         implementation first.
 
         @param test_event the test event as formatted by one of the
         event_for_* calls.
def handle_event(self, test_event):
                 # Possibly convert the job status (timeout, exceptional exit)
                 # to an appropriate test_result event.
                 self._maybe_remap_job_result_event(test_event)
                 event_type = test_event.get("event", "")
 
             # Remap timeouts to expected timeouts.
             if event_type in EventBuilder.RESULT_TYPES:
                 self._maybe_remap_expected_timeout(test_event)
+                self._maybe_remap_expected_failure(test_event)
                 event_type = test_event.get("event", "")
 
             if event_type == "terminate":
                 self.terminate_called = True
             elif event_type in EventBuilder.RESULT_TYPES:
                 # Keep track of event counts per test/job result status type.
                 # The only job (i.e. inferior process) results that make it
                 # here are ones that cannot be remapped to the most recently
def handle_event(self, test_event):
                 # out whether a process timeout or exceptional exit
                 # can be charged (i.e. assigned) to a test method.
                 worker_index = test_event.get("worker_index", None)
                 if worker_index is not None:
                     self.started_tests_by_worker[worker_index] = test_event
 
             elif event_type == EventBuilder.TYPE_MARK_TEST_RERUN_ELIGIBLE:
                 self._mark_test_for_rerun_eligibility(test_event)
+            elif event_type == EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE:
+                self._mark_test_as_expected_failure(test_event)
 
     def set_expected_timeouts_by_basename(self, basenames):
         """Specifies a list of test file basenames that are allowed to timeout
         without being called out as a timeout issue.
 
         These fall into a new status category called STATUS_EXPECTED_TIMEOUT.
         """
         if basenames is not None: