implement timeout sample support for Linux

[tfiala]

Summary

This is the Linux counterpart to the recently-added macOS-side support for sampling a test process that times out, prior to killing it.

This implementation is based on the Linux perf tools. Each distribution will have a different way to install it. The hook is skipped if Linux perf is not available.

On Ubuntu 16.04, the requisite support can be retrieved with:

sudo apt-get install perf-tools-unstable

Like the macOS side, the content out the time spent at each call point per backtrace is specified in a file in the session directory of the format:
{TestName.py}-{pid}.sample

[tfiala]

packages/Python/lldbsuite/pre_kill_hook/linux.py

68 ↗

(On Diff #72382)

Haha woops - that should be:

raise Exception("failed to call 'perf record ..., error code: {}".format(returncode))

I'll fix that in the final.

[labath]

lgtm

[tfiala]

BTW, regarding this part:

On Ubuntu 16.04, the requisite support can be retrieved with:

sudo apt-get install perf-tools-unstable

This was incorrect. The perf tool was actually present even without the perf-tools-unstable on Ubuntu 16.04 x86_64. It is just that there are more tools for processing perf output available with that package, but this change doesn't require any of them. /usr/bin/perf comes from the linux-tools-common package on Ubuntu. I tried testing with and without that package installed.

lldb/trunk/packages/Python/lldbsuite/pre_kill_hook/linux.py

+"""Provides a pre-kill method to run on Linux.
+
+This timeout pre-kill method relies on the Linux perf-tools
+distribution.  The appropriate way to obtain this set of tools
+will depend on the Linux distribution.
+
+For Ubuntu 16.04, the invoke the following command:
+sudo apt-get install perf-tools-unstable
+"""
+from __future__ import print_function
+
+# system imports
+import os
+import subprocess
+import sys
+import tempfile
+
+
+def do_pre_kill(process_id, runner_context, output_stream, sample_time=3):
+    """Samples the given process id, and puts the output to output_stream.
+
+    @param process_id the local process to sample.
+
+    @param runner_context a dictionary of details about the architectures
+    and platform on which the given process is running.  Expected keys are
+    archs (array of architectures), platform_name, platform_url, and
+    platform_working_dir.
+
+    @param output_stream file-like object that should be used to write the
+    results of sampling.
+
+    @param sample_time specifies the time in seconds that should be captured.
+    """
+
+    # Validate args.
+    if runner_context is None:
+        raise Exception("runner_context argument is required")
+    if not isinstance(runner_context, dict):
+        raise Exception("runner_context argument must be a dictionary")
+
+    # We will try to run sample on the local host only if there is no URL
+    # to a remote.
+    if "platform_url" in runner_context and (
+            runner_context["platform_url"] is not None):
+        import pprint
+        sys.stderr.write(
+            "warning: skipping timeout pre-kill sample invocation because we "
+            "don't know how to run on a remote yet. runner_context={}\n"
+            .format(pprint.pformat(runner_context)))
+
+    # We're going to create a temp file, and immediately overwrite it with the
+    # following command.  This just ensures we don't have any races in
+    # creation of the temporary sample file.
+    fileno, filename = tempfile.mkstemp(suffix='perfdata')
+    os.close(fileno)
+    fileno = None
+
+    try:
+        with open(os.devnull, 'w') as devnull:
+            returncode = subprocess.call(['timeout', str(sample_time), 'perf',
+                                          'record', '-g', '-o', filename, '-p', str(process_id)],
+                                         stdout=devnull, stderr=devnull)
+        if returncode == 0 or returncode == 124:
+            # This is okay - this is the timeout return code, which is totally
+            # expected.
+            pass
+        else:
+            raise Exception("failed to call 'perf record .., error: {}".format(
+                returncode))
+
+        with open(os.devnull, 'w') as devnull:
+            output = subprocess.check_output(['perf', 'report', '--call-graph',
+                                              '--stdio', '-i', filename], stderr=devnull)
+        output_stream.write(output)
+    finally:
+        os.remove(filename)

lldb/trunk/packages/Python/lldbsuite/pre_kill_hook/tests/test_darwin.py

def __init__(self, methodName):
         self.child_work_queue = None
         self.verbose = False
 
     def tearDown(self):
         if self.verbose:
             print("parent: sending shut-down request to child")
         if self.process:
             self.child_work_queue.put("hello, child")
             self.process.join()
         if self.verbose:
             print("parent: child is fully shut down")
 
     def test_sample(self):
         # Ensure we're Darwin.
         if platform.system() != 'Darwin':
             self.skipTest("requires a Darwin-based OS")
 

lldb/trunk/packages/Python/lldbsuite/pre_kill_hook/tests/test_linux.py

+"""Test the pre-kill hook on Linux."""
+from __future__ import print_function
+
+# system imports
+from multiprocessing import Process, Queue
+import platform
+import re
+import subprocess
+from unittest import main, TestCase
+
+# third party
+from six import StringIO
+
+
+def do_child_thread():
+    import os
+    x = 0
+    while True:
+        x = x + 42 * os.getpid()
+    return x
+
+
+def do_child_process(child_work_queue, parent_work_queue, verbose):
+    import os
+
+    pid = os.getpid()
+    if verbose:
+        print("child: pid {} started, sending to parent".format(pid))
+    parent_work_queue.put(pid)
+
+    # Spin up a daemon thread to do some "work", which will show
+    # up in a sample of this process.
+    import threading
+    worker = threading.Thread(target=do_child_thread)
+    worker.daemon = True
+    worker.start()
+
+    if verbose:
+        print("child: waiting for shut-down request from parent")
+    child_work_queue.get()
+    if verbose:
+        print("child: received shut-down request.  Child exiting.")
+
+
+class LinuxPreKillTestCase(TestCase):
+
+    def __init__(self, methodName):
+        super(LinuxPreKillTestCase, self).__init__(methodName)
+        self.process = None
+        self.child_work_queue = None
+        self.verbose = False
+        # self.verbose = True
+
+    def tearDown(self):
+        if self.verbose:
+            print("parent: sending shut-down request to child")
+        if self.process:
+            self.child_work_queue.put("hello, child")
+            self.process.join()
+        if self.verbose:
+            print("parent: child is fully shut down")
+
+    def test_sample(self):
+        # Ensure we're Darwin.
+        if platform.system() != 'Linux':
+            self.skipTest("requires a Linux-based OS")
+
+        # Ensure we have the 'perf' tool.  If not, skip the test.
+        try:
+            perf_version = subprocess.check_output(["perf", "version"])
+            if perf_version is None or not (
+                    perf_version.startswith("perf version")):
+                raise Exception("The perf executable doesn't appear"
+                                " to be the Linux perf tools perf")
+        except Exception:
+            self.skipTest("requires the Linux perf tools 'perf' command")
+
+        # Start the child process.
+        self.child_work_queue = Queue()
+        parent_work_queue = Queue()
+        self.process = Process(target=do_child_process,
+                               args=(self.child_work_queue, parent_work_queue,
+                                     self.verbose))
+        if self.verbose:
+            print("parent: starting child")
+        self.process.start()
+
+        # Wait for the child to report its pid.  Then we know we're running.
+        if self.verbose:
+            print("parent: waiting for child to start")
+        child_pid = parent_work_queue.get()
+
+        # Sample the child process.
+        from linux import do_pre_kill
+        context_dict = {
+            "archs": [platform.machine()],
+            "platform_name": None,
+            "platform_url": None,
+            "platform_working_dir": None
+        }
+
+        if self.verbose:
+            print("parent: running pre-kill action on child")
+        output_io = StringIO()
+        do_pre_kill(child_pid, context_dict, output_io)
+        output = output_io.getvalue()
+
+        if self.verbose:
+            print("parent: do_pre_kill() wrote the following output:", output)
+        self.assertIsNotNone(output)
+
+        # We should have a samples count entry.
+        # Samples:
+        self.assertTrue("Samples:" in output, "should have found a 'Samples:' "
+                        "field in the sampled process output")
+
+        # We should see an event count entry
+        event_count_re = re.compile(r"Event count[^:]+:\s+(\d+)")
+        match = event_count_re.search(output)
+        self.assertIsNotNone(match, "should have found the event count entry "
+                             "in sample output")
+        if self.verbose:
+            print("cpu-clock events:", match.group(1))
+
+        # We should see some percentages in the file.
+        percentage_re = re.compile(r"\d+\.\d+%")
+        match = percentage_re.search(output)
+        self.assertIsNotNone(match, "should have found at least one percentage "
+                             "in the sample output")
+
+
+if __name__ == "__main__":
+    main()

lldb/trunk/packages/Python/lldbsuite/test/dosep.py

def on_timeout_pre_kill(self):
         # runner for our platform.
         module_name = "lldbsuite.pre_kill_hook." + platform.system().lower()
         import importlib
         try:
             module = importlib.import_module(module_name)
         except ImportError:
             # We don't have one for this platform.  Skip.
             sys.stderr.write("\nwarning: no timeout handler module: " +
-                             module_name)
+                             module_name + "\n")
             return
 
         # Try to run the pre-kill-hook method.
         try:
             # Run the pre-kill command.
             output_io = StringIO()
             module.do_pre_kill(self.pid, g_runner_context, output_io)
 
             # Write the output to a filename associated with the test file and
             # pid.
+            MAX_UNCOMPRESSED_BYTE_COUNT = 10 * 1024
+
+            content = output_io.getvalue()
+            compress_output = len(content) > MAX_UNCOMPRESSED_BYTE_COUNT
             basename = "{}-{}.sample".format(self.file_name, self.pid)
             sample_path = os.path.join(g_session_dir, basename)
+
+            if compress_output:
+                # Write compressed output into a .zip file.
+                from zipfile import ZipFile, ZIP_DEFLATED
+                zipfile = sample_path + ".zip"
+                with ZipFile(zipfile, "w", ZIP_DEFLATED) as sample_zip:
+                    sample_zip.writestr(basename, content)
+            else:
+                # Write raw output into a text file.
                 with open(sample_path, "w") as output_file:
-                output_file.write(output_io.getvalue())
+                    output_file.write(content)
         except Exception as e:
             sys.stderr.write("caught exception while running "
-                             "pre-kill action: {}".format(e))
+                             "pre-kill action: {}\n".format(e))
             return
 
     def is_exceptional_exit(self):
         """Returns whether the process returned a timeout.
 
         Not valid to call until after on_process_exited() completes.
 
         @return True if the exit is an exceptional exit (e.g. signal on

lldb/trunk/packages/Python/lldbsuite/test/test_runner/process_control.py

class UnixProcessHelper(ProcessHelper):
 
     def _kill_with_signal(self, popen_process, log_file, signum):
         # Validate we're ready to terminate this.
         if not self._validate_pre_terminate(popen_process, log_file):
             return
 
         # Choose kill mechanism based on whether we're targeting
         # a process group or just a process.
+        try:
             if popen_process.using_process_groups:
                 # if log_file:
                 #    log_file.write(
                 #        "sending signum {} to process group {} now\n".format(
                 #            signum, popen_process.pid))
                 os.killpg(popen_process.pid, signum)
             else:
                 # if log_file:
                 #    log_file.write(
                 #        "sending signum {} to process {} now\n".format(
                 #            signum, popen_process.pid))
                 os.kill(popen_process.pid, signum)
+        except OSError as error:
+            import errno
+            if error.errno == errno.ESRCH:
+                # This is okay - failed to find the process.  It may be that
+                # that the timeout pre-kill hook eliminated the process.  We'll
+                # ignore.
+                pass
+            else:
+                raise
 
     def soft_terminate(self, popen_process, log_file=None, want_core=True):
         # Choose signal based on desire for core file.
         if want_core:
             # SIGQUIT will generate core by default.  Can be caught.
             signum = signal.SIGQUIT
         else:
             # SIGTERM is the traditional nice way to kill a process.