Print unexpected successes at the end of dosep.

[zturner]

Summary

New output looks like this:

FAIL: LLDB (suite) :: TestThreadStepOut.py (Windows zturner-win81 8 6.2.9200 AMD64 Intel64 Family 6 Model 45 Stepping 7, GenuineIntel)
FAIL: LLDB (suite) :: TestValueAPI.py (Windows zturner-win81 8 6.2.9200 AMD64 Intel64 Family 6 Model 45 Stepping 7, GenuineIntel)
FAIL: LLDB (suite) :: TestValueMD5Crash.py (Windows zturner-win81 8 6.2.9200 AMD64 Intel64 Family 6 Model 45 Stepping 7, GenuineIntel)
FAIL: LLDB (suite) :: TestVectorTypesFormatting.py (Windows zturner-win81 8 6.2.9200 AMD64 Intel64 Family 6 Model 45 Stepping 7, GenuineIntel)
Unexpected Successes (1)
UNEXPECTED SUCCESS: LLDB (suite) :: TestSettings.py (Windows zturner-win81 8 6.2.9200 AMD64 Intel64 Family 6 Model 45 Stepping 7, GenuineIntel)
[50856 refs]
ninja: build stopped: subcommand failed.

It is not smart enough to dig into the individual file (TestSettings.py in this example) and find out which test passed that was not expected. That can be done as future work if anyone is interested. The important thing though is that unexpected successes currently have zero visibility, so it is hard to tell when a change fixes existing broken stuff.

The hope is that surfacing this will lead to more tests being re-enabled.

[chaoren]

test/dosep.py

202

Don't need status here.

Nit: could you please wrap the if like the above?
Nit: xpasses reads like "expected passes" to me.

266

There's a line above this:

# result = (timed_out, failed, passed, fail_count, pass_count)

that needs to be updated.

(could you please include all context in the future? I recommend using arcanist)

500

I wonder if it'll be worthwhile to include the count (i.e., x unexpected successes out of y expected failures)

[zturner]

test/dosep.py

266

Arcanist didn't work well on windows last time I tried.

[zturner]

test/dosep.py

500

I thought about that. But if we're going to do that, might as well go all the way and print each one inside the test. Seems like a worthwhile thing to do, just didn't want to take that on quite yet.

[chaoren]

LGTM

test/dosep.py

with output_lock:
         test_counter.value += 1
         sys.stdout.flush()
         sys.stderr.flush()
 
 
 def parse_test_results(output):
     passes = 0
     failures = 0
+    unexpected_successes = 0
     for result in output:
         pass_count = re.search("^RESULT:.*([0-9]+) passes",
                                result, re.MULTILINE)
         fail_count = re.search("^RESULT:.*([0-9]+) failures",
                                result, re.MULTILINE)
         error_count = re.search("^RESULT:.*([0-9]+) errors",
                                 result, re.MULTILINE)
+        unexpected_success_count = re.search("^RESULT:.*([0-9]+) unexpected successes",
+                                             result, re.MULTILINE)
         this_fail_count = 0
         this_error_count = 0
         if pass_count is not None:
             passes = passes + int(pass_count.group(1))
         if fail_count is not None:
             failures = failures + int(fail_count.group(1))
+        if unexpected_success_count is not None:
+            unexpected_successes = unexpected_successes + int(unexpected_success_count.group(1))
         if error_count is not None:
             failures = failures + int(error_count.group(1))
         pass
-    return passes, failures
+    return passes, failures, unexpected_successes
 
 
 def call_with_timeout(command, timeout, name):
     """Run command with a timeout if possible."""
     """-s QUIT will create a coredump if they are enabled on your system"""
     process = None
     if timeout_command and timeout != "0":
         command = [timeout_command, '-s', 'QUIT', timeout] + command
     # Specifying a value for close_fds is unsupported on Windows when using
     # subprocess.PIPE
     if os.name != "nt":
         process = subprocess.Popen(command,
                                    stdin=subprocess.PIPE,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE,
                                    close_fds=True)
     else:
         process = subprocess.Popen(command,
                                    stdin=subprocess.PIPE,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
     output = process.communicate()
     exit_status = process.returncode
-    passes, failures = parse_test_results(output)
+    passes, failures, unexpected_successes = parse_test_results(output)
     if exit_status == 0:
         # stdout does not have any useful information from 'dotest.py',
         # only stderr does.
         report_test_pass(name, output[1])
     else:
         report_test_failure(name, command, output[1])
-    return name, exit_status, passes, failures
+    return name, exit_status, passes, failures, unexpected_successes
 
 
 def process_dir(root, files, test_root, dotest_argv):
     """Examine a directory for tests, and invoke any found within it."""
     results = []
     for name in files:
         script_file = os.path.join(test_root, "dotest.py")
         command = ([sys.executable, script_file] +
                    dotest_argv +
                    ["-p", name, root])
 
         timeout_name = os.path.basename(os.path.splitext(name)[0]).upper()
 
         timeout = (os.getenv("LLDB_%s_TIMEOUT" % timeout_name) or
                    getDefaultTimeout(dotest_options.lldb_platform_name))
 
         results.append(call_with_timeout(command, timeout, name))
 
-    # result = (name, status, passes, failures)
-    timed_out = [name for name, status, _, _ in results
+    # result = (name, status, passes, failures, unexpected_successes)
+    timed_out = [name for name, status, _, _, _ in results
                  if status == eTimedOut]
-    passed = [name for name, status, _, _ in results
+    passed = [name for name, status, _, _, _ in results
               if status == ePassed]
-    failed = [name for name, status, _, _ in results
+    failed = [name for name, status, _, _, _ in results
               if status != ePassed]
+    unexpected_passes = [name for name, _, _, _, unexpected_successes in results
+                         if unexpected_successes > 0]
+    
     pass_count = sum([result[2] for result in results])
     fail_count = sum([result[3] for result in results])
 
-    return (timed_out, passed, failed, pass_count, fail_count)
+    return (timed_out, passed, failed, unexpected_passes, pass_count, fail_count)
 
 in_q = None
 out_q = None
 
 
 def process_dir_worker(arg_tuple):
     """Worker thread main loop when in multithreaded mode.
     Takes one directory specification at a time and works on it."""
if num_threads > 1:
             num_threads,
             initializer=setup_global_variables,
             initargs=(output_lock, test_counter, total_tests, test_name_len,
                       dotest_options))
         test_results = pool.map(process_dir_worker, test_work_items)
     else:
         test_results = map(process_dir_worker, test_work_items)
 
-    # result = (timed_out, failed, passed, fail_count, pass_count)
+    # result = (timed_out, failed, passed, unexpected_successes, fail_count, pass_count)
     timed_out = sum([result[0] for result in test_results], [])
     passed = sum([result[1] for result in test_results], [])
     failed = sum([result[2] for result in test_results], [])
-    pass_count = sum([result[3] for result in test_results])
-    fail_count = sum([result[4] for result in test_results])
+    unexpected_successes = sum([result[3] for result in test_results], [])
+    pass_count = sum([result[4] for result in test_results])
+    fail_count = sum([result[5] for result in test_results])
 
-    return (timed_out, passed, failed, pass_count, fail_count)
+    return (timed_out, passed, failed, unexpected_successes, pass_count, fail_count)
 
 
 def getExpectedTimeouts(platform_name):
     # returns a set of test filenames that might timeout
     # are we running against a remote target?
     if platform_name is None:
         target = sys.platform
         remote = False
else:
         if num_threads_str:
             num_threads = int(num_threads_str)
         else:
             num_threads = multiprocessing.cpu_count()
     if num_threads < 1:
         num_threads = 1
 
     system_info = " ".join(platform.uname())
-    (timed_out, passed, failed, pass_count, fail_count) = walk_and_invoke(
+    (timed_out, passed, failed, unexpected_successes, pass_count, fail_count) = walk_and_invoke(
         test_directory, test_subdir, dotest_argv, num_threads)
 
     timed_out = set(timed_out)
     num_test_files = len(passed) + len(failed)
     num_test_cases = pass_count + fail_count
 
     # move core files into session dir
     cores = find('core.*', test_subdir)
if num_test_files > 0:
         sys.stdout.write(" (%d failed) (%f%%)" % (
             len(failed), 100.0 * len(failed) / num_test_files))
     print
     sys.stdout.write("Ran %d test cases" % num_test_cases)
     if num_test_cases > 0:
         sys.stdout.write(" (%d failed) (%f%%)" % (
             fail_count, 100.0 * fail_count / num_test_cases))
     print
+    exit_code = 0
+
     if len(failed) > 0:
         failed.sort()
         print "Failing Tests (%d)" % len(failed)
         for f in failed:
             print "%s: LLDB (suite) :: %s (%s)" % (
                 "TIMEOUT" if f in timed_out else "FAIL", f, system_info
             )
-        sys.exit(1)
-    sys.exit(0)
+        exit_code = 1
+
+    if len(unexpected_successes) > 0:
+        unexpected_successes.sort()
+        print "\nUnexpected Successes (%d)" % len(unexpected_successes)
+        for u in unexpected_successes:
+            print "UNEXPECTED SUCCESS: LLDB (suite) :: %s (%s)" % (u, system_info)
+
+    sys.exit(exit_code)
 
 if __name__ == '__main__':
     main()