dosep - force timeout processes to dump core when they timeout

[vharron]

Summary

move all core files to the session dir after all tests have completed

[clayborg]

Looks good.

lldb/trunk/test/dosep.py

 the environment variable LLDB_[TEST NAME]_TIMEOUT.
 
 E.g., export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=2m
 
 Set to "0" to run without time limit.
 
 E.g., export LLDB_TEST_TIMEOUT=0
 or    export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=0
+
+To collect core files for timed out tests, do the following before running dosep.py
+
+OSX
+ulimit -c unlimited
+sudo sysctl -w kern.corefile=core.%P
+
+Linux:
+ulimit -c unlimited
+echo core.%p | sudo tee /proc/sys/kernel/core_pattern
 """
 
 import multiprocessing
 import os
+import fnmatch
 import platform
 import re
 import dotest_args
 import shlex
 import subprocess
 import sys
 
 from optparse import OptionParser

 
 default_timeout = os.getenv("LLDB_TEST_TIMEOUT") or "10m"
 
 # Status codes for running command with timeout.
 eTimedOut, ePassed, eFailed = 124, 0, 1
 
 def call_with_timeout(command, timeout):
     """Run command with a timeout if possible."""
+    """-s QUIT will create a coredump if they are enabled on your system"""
     if os.name != "nt":
         if timeout_command and timeout != "0":
-            return subprocess.call([timeout_command, timeout] + command,
+            return subprocess.call([timeout_command, '-s', 'QUIT', timeout] + command,
                                    stdin=subprocess.PIPE, close_fds=True)
         return (ePassed if subprocess.call(command, stdin=subprocess.PIPE, close_fds=True) == 0
                 else eFailed)
     else:
         if timeout_command and timeout != "0":
-            return subprocess.call([timeout_command, timeout] + command,
+            return subprocess.call([timeout_command, '-s', 'QUIT', timeout] + command,
                                    stdin=subprocess.PIPE)
         return (ePassed if subprocess.call(command, stdin=subprocess.PIPE) == 0
                 else eFailed)
 
 def process_dir(root, files, test_root, dotest_argv):
     """Examine a directory for tests, and invoke any found within it."""
     timed_out = []
     failed = []
elif target.startswith("darwin"):
             "TestThreadSpecificBreakpoint.py", # times out on MBP Retina, Mid 2012
         }
     return expected_timeout
 
 def touch(fname, times=None):
     with open(fname, 'a'):
         os.utime(fname, times)
 
+def find(pattern, path):
+    result = []
+    for root, dirs, files in os.walk(path):
+        for name in files:
+            if fnmatch.fnmatch(name, pattern):
+                result.append(os.path.join(root, name))
+    return result
+
 def main():
     # We can't use sys.path[0] to determine the script directory
     # because it doesn't work under a debugger
     test_directory = os.path.dirname(os.path.realpath(__file__))
     parser = OptionParser(usage="""\
 Run lldb test suite using a separate process for each test file.
 
        Each test will run with a time limit of 10 minutes by default.
""")
     session_dir = os.path.join(os.getcwd(), dotest_options.s)
 
     # The root directory was specified on the command line
     if len(args) == 0:
         test_subdir = test_directory
     else:
         test_subdir = os.path.join(test_directory, args[0])
 
+    # clean core files in test tree from previous runs (Linux)
+    cores = find('core.*', test_subdir)
+    for core in cores:
+        os.unlink(core)
+
     if opts.num_threads:
         num_threads = opts.num_threads
     else:
         num_threads_str = os.environ.get("LLDB_TEST_THREADS")
         if num_threads_str:
             num_threads = int(num_threads_str)
         else:
             num_threads = multiprocessing.cpu_count()
     if num_threads < 1:
         num_threads = 1
 
     system_info = " ".join(platform.uname())
     (timed_out, failed, passed) = walk_and_invoke(test_directory, test_subdir, dotest_argv,
                                                   num_threads)
     timed_out = set(timed_out)
     num_tests = len(failed) + len(passed)
 
+    # move core files into session dir
+    cores = find('core.*', test_subdir)
+    for core in cores:
+        dst = core.replace(test_directory, "")[1:]
+        dst = dst.replace(os.path.sep, "-")
+        os.rename(core, os.path.join(session_dir, dst))
+
     # remove expected timeouts from failures
     expected_timeout = getExpectedTimeouts(dotest_options.lldb_platform_name)
     for xtime in expected_timeout:
         if xtime in timed_out:
             timed_out.remove(xtime)
             failed.remove(xtime)
             result = "ExpectedTimeout"
         elif xtime in passed: