Configurable timeout for tests

[chaoren]

Summary

Each test will timeout after 5 minutes by default.
Override the default timeout of 5 minutes with LLDB_TEST_TIMEOUT.
E.g., LLDB_TEST_TIMEOUT=10m
Override the timeout for individual tests with LLDB_[TEST NAME]_TIMEOUT.
E.g., LLDB_TESTCONCURRENTEVENTS_TIMEOUT=2m
Set to "0" to run without timeout.

See http://reviews.llvm.org/D6364.

[chaoren]

Isolated the logic for finding the timeout command.

[zturner]

I know this got reverted last time because it was causing some tests to timeout. I don't know what was causing that hang, but does this fix it?

[chaoren]

Yes. When building using make instead of ninja, dotest requires stdin to be
piped. the stdin=subprocess.PIPE fixes it.

[ovyalov]

Minor comments.

test/dosep.py

43

s/timeout_command/timeout_command is not None

46

No need to have else statement here.

72

You may get basename without extension using sth like this - os.path.basename(os.path.splitext(name)[0])

72

Could you store timeout as a separate variable to simplify this expression?

166

You may make timed_out as set to speed up lookup here.

[chaoren]

Clearer wording and more reasonable example on docstirng.

[chaoren]

Cleaned up some messy code as per Oleksiy's suggestions.

[ovyalov]

LGTM

[vharron]

Greg are you okay with this?

[clayborg]

lgtm

[chaoren]

Updated docstring placement.

test/dosep.py

 
 """
 Run the test suite using a separate process for each test file.
+
+Each test will timeout after 5 minutes by default.
+
+Override the default timeout of 5 minutes with LLDB_TEST_TIMEOUT.
+E.g., LLDB_TEST_TIMEOUT=10m
+
+Override the timeout for individual tests with LLDB_[TEST NAME]_TIMEOUT.
+E.g., LLDB_TESTCONCURRENTEVENTS_TIMEOUT=2m
+
+Set to "0" to run without timeout.
 """
 
 import multiprocessing
 import os
 import platform
+import shlex
+import subprocess
 import sys
 
 from optparse import OptionParser
 
-# Command template of the invocation of the test driver.
-template = '%s %s/dotest.py %s -p %s %s'
+def get_timeout_command():
+    """Search for a suitable timeout command."""
+    if sys.platform.startswith("win32"):
+        return None
+    try:
+        subprocess.call("timeout")
+        return "timeout"
+    except OSError:
+        pass
+    try:
+        subprocess.call("gtimeout")
+        return "gtimeout"
+    except OSError:
+        pass
+    return None
+
+timeout_command = get_timeout_command()
+default_timeout = os.getenv("LLDB_TEST_TIMEOUT") or "5m"
+
+# Status codes for running command with timeout.
+eTimedOut, ePassed, eFailed = 124, 0, 1
+
+def call_with_timeout(command, timeout):
+    """Run command with a timeout if possible."""
+    if timeout_command:
+        return subprocess.call([timeout_command, timeout] + command,
+                               stdin=subprocess.PIPE)
+    return (ePassed if subprocess.call(command, stdin=subprocess.PIPE) == 0
+            else eFailed)
 
 def process_dir(root, files, test_root, dotest_options):
     """Examine a directory for tests, and invoke any found within it."""
+    timed_out = []
     failed = []
     passed = []
     for name in files:

         if os.path.islink(path):
             continue
 
-        command = template % (sys.executable, test_root, dotest_options if dotest_options else "", name, root)
-        if 0 != os.system(command):
-            failed.append(name)
-        else:
+        command = ([sys.executable, "%s/dotest.py" % test_root] +
+                   (shlex.split(dotest_options) if dotest_options else []) +
+                   ["-p", name, root])
+
+        timeout_name = os.path.basename(os.path.splitext(name)[0]).upper()
+
+        timeout = os.getenv("LLDB_%s_TIMEOUT" % timeout_name) or default_timeout
+
+        exit_status = call_with_timeout(command, timeout)
+
+        if ePassed == exit_status:
             passed.append(name)
-    return (failed, passed)
+        else:
+            if eTimedOut == exit_status:
+                timed_out.append(name)
+            failed.append(name)
+    return (timed_out, failed, passed)
 
 in_q = None
 out_q = None

         for work_item in test_work_items:
             test_results.append(process_dir_worker(work_item))
 
+    timed_out = []
     failed = []
     passed = []
 
     for test_result in test_results:
-        (dir_failed, dir_passed) = test_result
+        (dir_timed_out, dir_failed, dir_passed) = test_result
+        timed_out += dir_timed_out
         failed += dir_failed
         passed += dir_passed
 
-    return (failed, passed)
+    return (timed_out, failed, passed)
 
 def main():
     test_root = sys.path[0]

         num_threads = 1
 
     system_info = " ".join(platform.uname())
-    (failed, passed) = walk_and_invoke(test_root, dotest_options, num_threads)
+    (timed_out, failed, passed) = walk_and_invoke(test_root, dotest_options,
+                                                  num_threads)
+    timed_out = set(timed_out)
     num_tests = len(failed) + len(passed)
 
     print "Ran %d tests." % num_tests

         failed.sort()
         print "Failing Tests (%d)" % len(failed)
         for f in failed:
-          print "FAIL: LLDB (suite) :: %s (%s)" % (f, system_info)
+            print "%s: LLDB (suite) :: %s (%s)" % (
+                "TIMEOUT" if f in timed_out else "FAIL", f, system_info
+            )
         sys.exit(1)
     sys.exit(0)