[test] Report error when inferior test processes exit with a non-zero code

[labath]

Summary

We've run into this problem when the test errored out so early (because it could not connect to
the remote device), that the code in D20193 did not catch the error. This resulted in the test
suite reporting success with 0 tests being run.

This patch makes sure that any non-zero exit code from the inferior process gets reported as an
error. Basically I expand the concept of "exceptional exits", which was previously being used for
signals to cover these cases as well.

[labath]

I think this also makes the code in D20193 obsolete. If this goes in, I can create a follow-up to remove that.

[labath]

I don't think the original version tried to make a nice error message. It got flagged as an error, and that was it. Now it will get flagged as exceptional exit.

In any case we will print out the stderr/stdout (I changed this from stderr-only, because in my case the only useful information about the error was in stdout), which should contain the exception that caused the non-zero exit, it's backtrace and everything.

[tfiala]

Looks fine here. Did either of you try this with an exceptional exit to make sure that still shows up right?

[labath]

I have now :)

[tfiala]

That's weird - it shows up as abandoned, but you did check it in, right Pavel?

[labath]

I also reverted it back, as it was causing some issues (if a specific test case in a test file failed, it still marked the whole file as failed). I'll need revisit this later...

[tfiala]

Ah yes okay, that orients what I heard earlier. Thanks!

lldb/trunk/packages/Python/lldbsuite/test/dosep.py

if worker_index_map is not None:
         global GET_WORKER_INDEX
         GET_WORKER_INDEX = get_worker_index_use_pid
 
 def report_test_failure(name, command, output, timeout):
     global output_lock
     with output_lock:
         if not (RESULTS_FORMATTER and RESULTS_FORMATTER.is_using_terminal()):
             print(file=sys.stderr)
-            print(output, file=sys.stderr)
             if timeout:
                 timeout_str = " (TIMEOUT)"
             else:
                 timeout_str = ""
             print("[%s FAILED]%s" % (name, timeout_str), file=sys.stderr)
             print("Command invoked: %s" % ' '.join(command), file=sys.stderr)
+            print("Command stderr:\n", output[1], file=sys.stderr)
+            print("Command stdout:\n", output[0], file=sys.stderr)
         update_progress(name)
 
 
 def report_test_pass(name, output):
     global output_lock
     with output_lock:
         update_progress(name)
 
def on_process_exited(self, command, output, was_timeout, exit_status):
 
         # Now parse the output.
         passes, failures, unexpected_successes = parse_test_results(output)
         if exit_status == 0:
             # stdout does not have any useful information from 'dotest.py',
             # only stderr does.
             report_test_pass(self.file_name, output[1])
         else:
-            report_test_failure(self.file_name, command, output[1], was_timeout)
+            report_test_failure(self.file_name, command, output, was_timeout)
 
         # Save off the results for the caller.
         self.results = (
             self.file_name,
             exit_status,
             passes,
             failures,
             unexpected_successes)

lldb/trunk/packages/Python/lldbsuite/test/test_runner/process_control.py

def soft_terminate_signals(self):
         @return a list of signal numbers that can be sent to soft terminate
         a process, or None if not applicable.
         """
         return None
 
     def is_exceptional_exit(self, popen_status):
         """Returns whether the program exit status is exceptional.
 
-        Returns whether the return code from a Popen process is exceptional
-        (e.g. signals on POSIX systems).
-
-        Derived classes should override this if they can detect exceptional
-        program exit.
+        Returns whether the return code from a Popen process is exceptional.
 
         @return True if the given popen_status represents an exceptional
         program exit; False otherwise.
         """
-        return False
+        return popen_status != 0
 
     def exceptional_exit_details(self, popen_status):
         """Returns the normalized exceptional exit code and a description.
 
         Given an exceptional exit code, returns the integral value of the
-        exception (e.g. signal number for POSIX) and a description (e.g.
-        signal name on POSIX) for the result.
-
-        Derived classes should override this if they can detect exceptional
-        program exit.
+        exception and a description for the result.
 
-        It is fine to not implement this so long as is_exceptional_exit()
-        always returns False.
+        Derived classes can override this if they want to want custom
+        exceptional exit code handling.
 
         @return (normalized exception code, symbolic exception description)
         """
-        raise Exception("exception_exit_details() called on unsupported class")
+        return (popen_status, "exit")
 
 
 class UnixProcessHelper(ProcessHelper):
     """Provides a ProcessHelper for Unix-like operating systems.
 
     This implementation supports anything that looks Posix-y
     (e.g. Darwin, Linux, *BSD, etc.)
     """
def was_soft_terminate(self, returncode, with_core):
             return returncode == -signal.SIGTERM
 
     def was_hard_terminate(self, returncode):
         return returncode == -signal.SIGKILL
 
     def soft_terminate_signals(self):
         return [signal.SIGQUIT, signal.SIGTERM]
 
-    def is_exceptional_exit(self, popen_status):
-        return popen_status < 0
-
     @classmethod
     def _signal_names_by_number(cls):
         return dict(
             (k, v) for v, k in reversed(sorted(signal.__dict__.items()))
             if v.startswith('SIG') and not v.startswith('SIG_'))
 
     def exceptional_exit_details(self, popen_status):
+        if popen_status >= 0:
+            return (popen_status, "exit")
         signo = -popen_status
         signal_names_by_number = self._signal_names_by_number()
         signal_name = signal_names_by_number.get(signo, "")
         return (signo, signal_name)
 
 class WindowsProcessHelper(ProcessHelper):
     """Provides a Windows implementation of the ProcessHelper class."""
     def __init__(self):