Updated dosep.py to output progress and dump std{out,err} on test failure.

[chaoren]

I think it'll be a good idea if we just switched to using llvm-lit.

[zturner]

Agree, but that's a really huge undertaking. See this thread for a relevant
discussion

https://www.mail-archive.com/[e-mail]/msg06263.html

[chaoren]

• Display name of test suite that failed.

[chaoren]

• Accidentally printed stdout again instead of stderr.
• Passing lock and counter by argument to work on Windows.

[zturner]

Looks ok, did you get a chance to run on Windows to verify that it still
works?

[chaoren]

Yep, the last update was specifically made for it to work on Windows.

[zturner]

cool, lgtm then

lldb/trunk/test/dosep.py

 
 timeout_command = get_timeout_command()
 
 default_timeout = os.getenv("LLDB_TEST_TIMEOUT") or "10m"
 
 # Status codes for running command with timeout.
 eTimedOut, ePassed, eFailed = 124, 0, 1
 
+output_lock = None
+test_counter = None
+total_tests = None
+
+def setup_lock_and_counter(lock, counter, total):
+    global output_lock, test_counter, total_tests
+    output_lock = lock
+    test_counter = counter
+    total_tests = total
+
+def update_status(name = None, output = None):
+    global output_lock, test_counter, total_tests
+    with output_lock:
+        if output is not None:
+            print >> sys.stderr
+            print >> sys.stderr, 'Test suite %s failed' % name
+            print >> sys.stderr, 'stdout:\n' + output[0]
+            print >> sys.stderr, 'stderr:\n' + output[1]
+        sys.stderr.write("\r%*d out of %d test suites processed" %
+            (len(str(total_tests)), test_counter.value, total_tests))
+        test_counter.value += 1
+
 def parse_test_results(output):
     passes = 0
     failures = 0
     for result in output:
         pass_count = re.search("^RESULT:.*([0-9]+) passes", result, re.MULTILINE)
         fail_count = re.search("^RESULT:.*([0-9]+) failures", result, re.MULTILINE)
         error_count = re.search("^RESULT:.*([0-9]+) errors", result, re.MULTILINE)
         this_fail_count = 0
         this_error_count = 0
         if pass_count != None:
             passes = passes + int(pass_count.group(1))
         if fail_count != None:
             failures = failures + int(fail_count.group(1))
         if error_count != None:
             failures = failures + int(error_count.group(1))
         pass
     return passes, failures
 
-def call_with_timeout(command, timeout):
+def call_with_timeout(command, timeout, name):
     """Run command with a timeout if possible."""
     """-s QUIT will create a coredump if they are enabled on your system"""
     process = None
     if timeout_command and timeout != "0":
         command = [timeout_command, '-s', 'QUIT', timeout] + command
     # Specifying a value for close_fds is unsupported on Windows when using subprocess.PIPE
     if os.name != "nt":
         process = subprocess.Popen(command, stdin=subprocess.PIPE,
                                             stdout=subprocess.PIPE,
                                             stderr=subprocess.PIPE,
                                             close_fds=True)
     else:
         process = subprocess.Popen(command, stdin=subprocess.PIPE,
                                             stdout=subprocess.PIPE,
                                             stderr=subprocess.PIPE)
     output = process.communicate()
     exit_status = process.returncode
     passes, failures = parse_test_results(output)
+    update_status(name, output if failures > 0 else None)
     return exit_status, passes, failures
 
 def process_dir(root, files, test_root, dotest_argv):
     """Examine a directory for tests, and invoke any found within it."""
     timed_out = []
     failed = []
     passed = []
     pass_sub_count = 0
for name in files:
         command = ([sys.executable, script_file] +
                    dotest_argv +
                    ["-p", name, root])
 
         timeout_name = os.path.basename(os.path.splitext(name)[0]).upper()
 
         timeout = os.getenv("LLDB_%s_TIMEOUT" % timeout_name) or default_timeout
 
-        exit_status, pass_count, fail_count = call_with_timeout(command, timeout)
+        exit_status, pass_count, fail_count = call_with_timeout(command, timeout, name)
 
         pass_sub_count = pass_sub_count + pass_count
         fail_sub_count = fail_sub_count + fail_count
 
         if exit_status == ePassed:
             passed.append(name)
         else:
             if eTimedOut == exit_status:
def walk_and_invoke(test_directory, test_subdir, dotest_argv, num_threads):
     test_subdir - lldb/test/ or a subfolder with the tests we're interested in running
     """
 
     # Collect the test files that we'll run.
     test_work_items = []
     for root, dirs, files in os.walk(test_subdir, topdown=False):
         test_work_items.append((root, files, test_directory, dotest_argv))
 
+    global output_lock, test_counter, total_tests
+    output_lock = multiprocessing.Lock()
+    total_tests = len(test_work_items)
+    test_counter = multiprocessing.Value('i', 0)
+    print >> sys.stderr, "Testing: %d tests, %d threads" % (total_tests, num_threads)
+    update_status()
+
     # Run the items, either in a pool (for multicore speedup) or
     # calling each individually.
     if num_threads > 1:
-        pool = multiprocessing.Pool(num_threads)
+        pool = multiprocessing.Pool(num_threads,
+            initializer = setup_lock_and_counter,
+            initargs = (output_lock, test_counter, total_tests))
         test_results = pool.map(process_dir_worker, test_work_items)
     else:
         test_results = []
         for work_item in test_work_items:
             test_results.append(process_dir_worker(work_item))
 
     timed_out = []
     failed = []
for xtime in expected_timeout:
             result = "UnexpectedCompletion"
         else:
             result = None  # failed
 
         if result:
             test_name = os.path.splitext(xtime)[0]
             touch(os.path.join(session_dir, "{}-{}".format(result, test_name)))
 
+    print
     print "Ran %d test suites (%d failed) (%f%%)" % (num_test_files, len(failed), 100.0*len(failed)/num_test_files)
     print "Ran %d test cases (%d failed) (%f%%)" % (num_tests, all_fails, 100.0*all_fails/num_tests)
     if len(failed) > 0:
         failed.sort()
         print "Failing Tests (%d)" % len(failed)
         for f in failed:
             print "%s: LLDB (suite) :: %s (%s)" % (
                 "TIMEOUT" if f in timed_out else "FAIL", f, system_info
             )
         sys.exit(1)
     sys.exit(0)
 
 if __name__ == '__main__':
     main()