Treat cleanup errors separately from test failures

[zturner]

Summary

There is a certain class of test failures that arise as a result of errors that occur during cleanup. For the most part, these don't indicate test failures and so it's useful to distinguish them from test failures. This patch creates a new category of error, called a CleanupError, that is handled (and reported) independently of whether or not any individual tests failed.

This is especially important when there is a directory that contains multiple test suites, because previously a cleanup error on the first suite would automatically cause subsequent suites in the same directory to not run, as a result of exception propagation.

[zturner]

Upload full diffs with context.

[clayborg]

Looks good.

test/dotest.py

for ic in range(len(compilers) if iterCompilers else 1):
                 sdir_has_content = True
                 super(LLDBTestResult, self).addError(test, err)
                 method = getattr(test, "markError", None)
                 if method:
                     method()
                 if parsable:
                     self.stream.write("FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test)))
 
+            def addCleanupError(self, test, err):
+                global sdir_has_content
+                global parsable
+                sdir_has_content = True
+                super(LLDBTestResult, self).addCleanupError(test, err)
+                method = getattr(test, "markCleanupError", None)
+                if method:
+                    method()
+                if parsable:
+                    self.stream.write("CLEANUP ERROR: LLDB (%s) :: %s\n" % (self._config_string(test), str(test)))
+
             def addFailure(self, test, err):
                 global sdir_has_content
                 global failuresPerCategory
                 global parsable
                 sdir_has_content = True
                 super(LLDBTestResult, self).addFailure(test, err)
                 method = getattr(test, "markFailure", None)
                 if method:

test/lldbtest.py

def tearDownClass(cls):
         """
         Python unittest framework class teardown fixture.
         Do class-wide cleanup.
         """
 
         if doCleanup and not lldb.skip_build_and_cleanup:
             # First, let's do the platform-specific cleanup.
             module = builder_module()
-            if not module.cleanup():
-                raise Exception("Don't know how to do cleanup")
+            module.cleanup()
 
             # Subclass might have specific cleanup function defined.
             if getattr(cls, "classCleanup", None):
                 if traceAlways:
                     print >> sys.stderr, "Call class-specific cleanup function for class:", cls
                 try:
                     cls.classCleanup()
                 except:
def setUp(self):
         session_file = "{}.log".format(self.log_basename)
         unbuffered = 0 # 0 is the constant for unbuffered
         self.session = open(session_file, "w", unbuffered)
 
         # Optimistically set __errored__, __failed__, __expected__ to False
         # initially.  If the test errored/failed, the session info
         # (self.session) is then dumped into a session specific file for
         # diagnosis.
+        self.__cleanup_errored__ = False
         self.__errored__    = False
         self.__failed__     = False
         self.__expected__   = False
         # We are also interested in unexpected success.
         self.__unexpected__ = False
         # And skipped tests.
         self.__skipped__ = False
 
def tearDown(self):
         # In rare cases where there are multiple teardown cleanups added.
         if doCleanup and self.doTearDownCleanups:
             if self.dicts:
                 for dict in reversed(self.dicts):
                     self.cleanup(dictionary=dict)
 
         self.disableLogChannelsForCurrentTest()
 
-        # Decide whether to dump the session info.
-        self.dumpSessionInfo()
-
     # =========================================================
     # Various callbacks to allow introspection of test progress
     # =========================================================
 
     def markError(self):
         """Callback invoked when an error (unexpected exception) errored."""
         self.__errored__ = True
         with recording(self, False) as sbuf:
             # False because there's no need to write "ERROR" to the stderr twice.
             # Once by the Python unittest framework, and a second time by us.
             print >> sbuf, "ERROR"
 
+    def markCleanupError(self):
+        """Callback invoked when an error occurs while a test is cleaning up."""
+        self.__cleanup_errored__ = True
+        with recording(self, False) as sbuf:
+            # False because there's no need to write "CLEANUP_ERROR" to the stderr twice.
+            # Once by the Python unittest framework, and a second time by us.
+            print >> sbuf, "CLEANUP_ERROR"
+
     def markFailure(self):
         """Callback invoked when a failure (test assertion failure) occurred."""
         self.__failed__ = True
         with recording(self, False) as sbuf:
             # False because there's no need to write "FAIL" to the stderr twice.
             # Once by the Python unittest framework, and a second time by us.
             print >> sbuf, "FAIL"
 
def dumpSessionInfo(self):
         #
         # See http://docs.python.org/library/unittest.html#unittest.TestResult.
 
         # output tracebacks into session
         pairs = []
         if self.__errored__:
             pairs = lldb.test_result.errors
             prefix = 'Error'
+        if self.__cleanup_errored__:
+            pairs = lldb.test_result.cleanup_errors
+            prefix = 'CleanupError'
         elif self.__failed__:
             pairs = lldb.test_result.failures
             prefix = 'Failure'
         elif self.__expected__:
             pairs = lldb.test_result.expectedFailures
             prefix = 'ExpectedFailure'
         elif self.__skipped__:
             prefix = 'SkippedTest'

test/unittest2/case.py

def run(self, result=None):
                 except Exception:
                     result.addError(self, sys.exc_info())
                 else:
                     success = True
 
                 try:
                     self.tearDown()
                 except Exception:
-                    result.addError(self, sys.exc_info())
+                    result.addCleanupError(self, sys.exc_info())
                     success = False
 
+                self.dumpSessionInfo()
+
             cleanUpSuccess = self.doCleanups()
             success = success and cleanUpSuccess
             if success:
                 result.addSuccess(self)
         finally:
             result.stopTest(self)
             if orig_result is None:
                 stopTestRun = getattr(result, 'stopTestRun', None)

test/unittest2/result.py

class TestResult(unittest.TestResult):
     _previousTestClass = None
     _moduleSetUpFailed = False
     
     def __init__(self):
         self.failfast = False
         self.failures = []
         self.passes = []
         self.errors = []
+        self.cleanup_errors = []
         self.testsRun = 0
         self.skipped = []
         self.expectedFailures = []
         self.unexpectedSuccesses = []
         self.shouldStop = False
         self.buffer = False
         self._stdout_buffer = None
         self._stderr_buffer = None
class TestResult(unittest.TestResult):
     @failfast
     def addError(self, test, err):
         """Called when an error has occurred. 'err' is a tuple of values as
         returned by sys.exc_info().
         """
         self.errors.append((test, self._exc_info_to_string(err, test)))
         self._mirrorOutput = True
 
+    def addCleanupError(self, test, err):
+        """Called when an error has occurred during cleanup. 'err' is a tuple of
+        values as returned by sys.exc_info().
+        """
+        self.cleanup_errors.append((test, self._exc_info_to_string(err, test)))
+        self._mirrorOutput = True
+
     @failfast
     def addFailure(self, test, err):
         """Called when an error has occurred. 'err' is a tuple of values as
         returned by sys.exc_info()."""
         self.failures.append((test, self._exc_info_to_string(err, test)))
         self._mirrorOutput = True
 
     def addSuccess(self, test):