Change dosep to report pass/failure of individual test cases as well.

[zturner]

Summary

Each .py file can contain many test cases, and dosep currently reports only (# of files whose exit status was non-zero) / (total # of files examined).

This can skew the results if you have many .py files where only a small fraction of the test cases inside the .py file actually fail. With this patch we now print both ratios. The output after all tests have been run now looksl ike this:

[1/1] Testing LLDB (with a separate subprocess per test)
FAILED: <snipped/>
Ran 377 test suites (12 failed) (3.18%)
Ran 363 test cases (25 failed) (6.88%)
FAIL: LLDB (suite) :: Test-rdar-10449092.py (Windows zturner-win81 8 6.2.9200 AMD64 Intel64 Family 6 Model 45 Stepping 7, GenuineIntel)
...

[clayborg]

Looks like some windowns specific code was left in dosep.py:

import ctypes
from ctypes import wintypes

_OutputDebugString = ctypes.windll.kernel32.OutputDebugStringA
_OutputDebugString.argtypes = [wintypes.LPCSTR]
_OutputDebugString.restype = None

test/dosep.py

33–39

Doesn't this only need to be done on windows? Shouldn't you check for platform being windows?

[zturner]

Doh, sorry about that. Pretend you didn't see that, I'll get it out
tomorrow

[zturner]

Get rid of windows-specific debugging code that got left in.

[clayborg]

Looks good.

test/api/multithreaded/Makefile

 include $(LEVEL)/Makefile.rules
 
 clean::
-  rm -rf *.o *.d *.dSYM
+  rm $(wildcard -rf *.o *.d *.dSYM)

test/dosep.py

 # Status codes for running command with timeout.
 eTimedOut, ePassed, eFailed = 124, 0, 1
 
+def parse_test_results(output):
+    passes = 0
+    failures = 0
+    for result in output:
+        pass_count = re.search("^RESULT:.*([0-9]+) passes", result, re.MULTILINE)
+        fail_count = re.search("^RESULT:.*([0-9]+) failures", result, re.MULTILINE)
+        error_count = re.search("^RESULT:.*([0-9]+) errors", result, re.MULTILINE)
+        this_fail_count = 0
+        this_error_count = 0
+        if pass_count != None:
+            passes = passes + int(pass_count.group(1))
+        if fail_count != None:
+            failures = failures + int(fail_count.group(1))
+        if error_count != None:
+            failures = failures + int(error_count.group(1))
+        pass
+    return passes, failures
+
 def call_with_timeout(command, timeout):
+    process = None
     """Run command with a timeout if possible."""
+    if timeout_command and timeout != "0":
+        command = [timeout_command, timeout] + command
+    # Specifying a value for close_fds is unsupported on Windows when using subprocess.PIPE
     if os.name != "nt":
-        if timeout_command and timeout != "0":
-            return subprocess.call([timeout_command, timeout] + command,
-                                   stdin=subprocess.PIPE, close_fds=True)
-        return (ePassed if subprocess.call(command, stdin=subprocess.PIPE, close_fds=True) == 0
-                else eFailed)
+        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
     else:
-        if timeout_command and timeout != "0":
-            return subprocess.call([timeout_command, timeout] + command,
-                                   stdin=subprocess.PIPE)
-        return (ePassed if subprocess.call(command, stdin=subprocess.PIPE) == 0
-                else eFailed)
+        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    output = process.communicate()
+    exit_status = process.returncode
+    passes, failures = parse_test_results(output)
+    return exit_status, passes, failures
 
 def process_dir(root, files, test_root, dotest_argv):
     """Examine a directory for tests, and invoke any found within it."""
     timed_out = []
     failed = []
     passed = []
+    pass_sub_count = 0
+    fail_sub_count = 0
     for name in files:
         path = os.path.join(root, name)
 

 
         timeout = os.getenv("LLDB_%s_TIMEOUT" % timeout_name) or default_timeout
 
-        exit_status = call_with_timeout(command, timeout)
+        exit_status, pass_count, fail_count = call_with_timeout(command, timeout)
 
-        if ePassed == exit_status:
+        pass_sub_count = pass_sub_count + pass_count
+        fail_sub_count = fail_sub_count + fail_count
+
+        if exit_status == ePassed:
             passed.append(name)
         else:
             if eTimedOut == exit_status:
                 timed_out.append(name)
             failed.append(name)
-    return (timed_out, failed, passed)
+    return (timed_out, failed, passed, fail_sub_count, pass_sub_count)
 
 in_q = None
 out_q = None

     timed_out = []
     failed = []
     passed = []
+    fail_sub_count = 0
+    pass_sub_count = 0
 
     for test_result in test_results:
-        (dir_timed_out, dir_failed, dir_passed) = test_result
+        (dir_timed_out, dir_failed, dir_passed, dir_fail_sub_count, dir_pass_sub_count) = test_result
         timed_out += dir_timed_out
         failed += dir_failed
         passed += dir_passed
+        fail_sub_count = fail_sub_count + dir_fail_sub_count
+        pass_sub_count = pass_sub_count + dir_pass_sub_count
 
-    return (timed_out, failed, passed)
+    return (timed_out, failed, passed, fail_sub_count, pass_sub_count)
 
 def getExpectedTimeouts(platform_name):
     # returns a set of test filenames that might timeout

         num_threads = 1
 
     system_info = " ".join(platform.uname())
-    (timed_out, failed, passed) = walk_and_invoke(test_directory, test_subdir, dotest_argv,
-                                                  num_threads)
+    (timed_out, failed, passed, all_fails, all_passes) = walk_and_invoke(test_directory, test_subdir, dotest_argv, num_threads)
+
     timed_out = set(timed_out)
-    num_tests = len(failed) + len(passed)
+    num_test_files = len(failed) + len(passed)
+    num_tests = all_fails + all_passes
 
     # remove expected timeouts from failures
     expected_timeout = getExpectedTimeouts(dotest_options.lldb_platform_name)

             test_name = os.path.splitext(xtime)[0]
             touch(os.path.join(session_dir, "{}-{}".format(result, test_name)))
 
-    print "Ran %d tests." % num_tests
+    print "Ran %d test suites (%d failed) (%f%%)" % (num_test_files, len(failed), 100.0*len(failed)/num_test_files)
+    print "Ran %d test cases (%d failed) (%f%%)" % (num_tests, all_fails, 100.0*all_fails/num_tests)
     if len(failed) > 0:
         failed.sort()
-        print "Failing Tests (%d)" % len(failed)
         for f in failed:
             print "%s: LLDB (suite) :: %s (%s)" % (
                 "TIMEOUT" if f in timed_out else "FAIL", f, system_info

test/unittest2/result.py

     def __init__(self):
         self.failfast = False
         self.failures = []
+        self.passes = []
         self.errors = []
         self.testsRun = 0
         self.skipped = []

 
     def addSuccess(self, test):
         "Called when a test has completed successfully"
+        self.passes.append(test)
         pass
 
     def addSkip(self, test, reason):

test/unittest2/runner.py

                             (run, run != 1 and "s" or "", timeTaken))
         self.stream.writeln()
         
-        expectedFails = unexpectedSuccesses = skipped = 0
+        expectedFails = unexpectedSuccesses = skipped = passed = failed = errored = 0
         try:
             results = map(len, (result.expectedFailures,
                                 result.unexpectedSuccesses,
-                                result.skipped))
-            expectedFails, unexpectedSuccesses, skipped = results
+                                result.skipped,
+                                result.passes,
+                                result.failures,
+                                result.errors))
+            expectedFails, unexpectedSuccesses, skipped, passed, failed, errored = results
         except AttributeError:
             pass
         infos = []
+        infos.append("%d passes" % passed)
+        infos.append("%d failures" % failed)
+        infos.append("%d errors" % errored)
+        infos.append("%d skipped" % skipped)
+        infos.append("%d expected failures" % expectedFails)
+        infos.append("%d unexpected successes" % unexpectedSuccesses)
+        self.stream.write("RESULT: ")
         if not result.wasSuccessful():
             self.stream.write("FAILED")
-            failed, errored = map(len, (result.failures, result.errors))
-            if failed:
-                infos.append("failures=%d" % failed)
-            if errored:
-                infos.append("errors=%d" % errored)
         else:
-            self.stream.write("OK")
-        if skipped:
-            infos.append("skipped=%d" % skipped)
-        if expectedFails:
-            infos.append("expected failures=%d" % expectedFails)
-        if unexpectedSuccesses:
-            infos.append("unexpected successes=%d" % unexpectedSuccesses)
-        if infos:
-            self.stream.writeln(" (%s)" % (", ".join(infos),))
-        else:
-            self.stream.write("\n")
+            self.stream.write("PASSED")
+
+        self.stream.writeln(" (%s)" % (", ".join(infos),))
         return result