Index: packages/Python/lldbsuite/support/optional_with.py =================================================================== --- /dev/null +++ packages/Python/lldbsuite/support/optional_with.py @@ -0,0 +1,56 @@ +# ==================================================================== +# Provides a with-style resource handler for optionally-None resources +# ==================================================================== + +class optional_with(object): + # pylint: disable=too-few-public-methods + # This is a wrapper - it is not meant to provide any extra methods. + """Provides a wrapper for objects supporting "with", allowing None. + + This lets a user use the "with object" syntax for resource usage + (e.g. locks) even when the wrapped with object is None. + + e.g. + + wrapped_lock = optional_with(thread.Lock()) + with wrapped_lock: + # Do something while the lock is obtained. + pass + + might_be_none = None + wrapped_none = optional_with(might_be_none) + with wrapped_none: + # This code here still works. + pass + + This prevents having to write code like this when + a lock is optional: + + if lock: + lock.acquire() + + try: + code_fragment_always_run() + finally: + if lock: + lock.release() + + And I'd posit it is safer, as it becomes impossible to + forget the try/finally using optional_with(), since + the with syntax can be used. + """ + def __init__(self, wrapped_object): + self.wrapped_object = wrapped_object + + def __enter__(self): + if self.wrapped_object is not None: + return self.wrapped_object.__enter__() + else: + return self + + def __exit__(self, the_type, value, traceback): + if self.wrapped_object is not None: + return self.wrapped_object.__exit__(the_type, value, traceback) + else: + # Don't suppress any exceptions + return False Index: packages/Python/lldbsuite/test/dosep.py =================================================================== --- packages/Python/lldbsuite/test/dosep.py +++ packages/Python/lldbsuite/test/dosep.py @@ -52,6 +52,7 @@ import lldbsuite import lldbsuite.support.seven as seven +from lldbsuite.support import optional_with from . import configuration from . import dotest_channels from . import dotest_args @@ -59,12 +60,7 @@ from .result_formatter import EventBuilder - -# Todo: Convert this folder layout to be relative-import friendly and -# don't hack up sys.path like this -sys.path.append(os.path.join(os.path.dirname(__file__), "test_runner", "lib")) -import lldb_utils -import process_control +from .test_runner import process_control # Status codes for running command with timeout. eTimedOut, ePassed, eFailed = 124, 0, 1 @@ -177,7 +173,7 @@ super(DoTestProcessDriver, self).__init__( soft_terminate_timeout=soft_terminate_timeout) self.output_file = output_file - self.output_lock = lldb_utils.OptionalWith(output_file_lock) + self.output_lock = optional_with.optional_with(output_file_lock) self.pid_events = pid_events self.results = None self.file_name = file_name Index: packages/Python/lldbsuite/test/test_runner/lib/lldb_utils.py =================================================================== --- packages/Python/lldbsuite/test/test_runner/lib/lldb_utils.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -The LLVM Compiler Infrastructure - -This file is distributed under the University of Illinois Open Source -License. See LICENSE.TXT for details. - -Provides classes used by the test results reporting infrastructure -within the LLDB test suite. - - -This module contains utilities used by the lldb test framwork. -""" - - -class OptionalWith(object): - # pylint: disable=too-few-public-methods - # This is a wrapper - it is not meant to provide any extra methods. - """Provides a wrapper for objects supporting "with", allowing None. - - This lets a user use the "with object" syntax for resource usage - (e.g. locks) even when the wrapped with object is None. - - e.g. - - wrapped_lock = OptionalWith(thread.Lock()) - with wrapped_lock: - # Do something while the lock is obtained. - pass - - might_be_none = None - wrapped_none = OptionalWith(might_be_none) - with wrapped_none: - # This code here still works. - pass - - This prevents having to write code like this when - a lock is optional: - - if lock: - lock.acquire() - - try: - code_fragament_always_run() - finally: - if lock: - lock.release() - - And I'd posit it is safer, as it becomes impossible to - forget the try/finally using OptionalWith(), since - the with syntax can be used. - """ - def __init__(self, wrapped_object): - self.wrapped_object = wrapped_object - - def __enter__(self): - if self.wrapped_object is not None: - return self.wrapped_object.__enter__() - else: - return self - - def __exit__(self, the_type, value, traceback): - if self.wrapped_object is not None: - return self.wrapped_object.__exit__(the_type, value, traceback) - else: - # Don't suppress any exceptions - return False Index: packages/Python/lldbsuite/test/test_runner/lib/process_control.py =================================================================== --- packages/Python/lldbsuite/test/test_runner/lib/process_control.py +++ /dev/null @@ -1,705 +0,0 @@ -""" -The LLVM Compiler Infrastructure - -This file is distributed under the University of Illinois Open Source -License. See LICENSE.TXT for details. - -Provides classes used by the test results reporting infrastructure -within the LLDB test suite. - - -This module provides process-management support for the LLDB test -running infrasructure. -""" - -# System imports -import os -import re -import signal -import subprocess -import sys -import threading - - -class CommunicatorThread(threading.Thread): - """Provides a thread class that communicates with a subprocess.""" - def __init__(self, process, event, output_file): - super(CommunicatorThread, self).__init__() - # Don't let this thread prevent shutdown. - self.daemon = True - self.process = process - self.pid = process.pid - self.event = event - self.output_file = output_file - self.output = None - - def run(self): - try: - # Communicate with the child process. - # This will not complete until the child process terminates. - self.output = self.process.communicate() - except Exception as exception: # pylint: disable=broad-except - if self.output_file: - self.output_file.write( - "exception while using communicate() for pid: {}\n".format( - exception)) - finally: - # Signal that the thread's run is complete. - self.event.set() - - -# Provides a regular expression for matching gtimeout-based durations. -TIMEOUT_REGEX = re.compile(r"(^\d+)([smhd])?$") - - -def timeout_to_seconds(timeout): - """Converts timeout/gtimeout timeout values into seconds. - - @param timeout a timeout in the form of xm representing x minutes. - - @return None if timeout is None, or the number of seconds as a float - if a valid timeout format was specified. - """ - if timeout is None: - return None - else: - match = TIMEOUT_REGEX.match(timeout) - if match: - value = float(match.group(1)) - units = match.group(2) - if units is None: - # default is seconds. No conversion necessary. - return value - elif units == 's': - # Seconds. No conversion necessary. - return value - elif units == 'm': - # Value is in minutes. - return 60.0 * value - elif units == 'h': - # Value is in hours. - return (60.0 * 60.0) * value - elif units == 'd': - # Value is in days. - return 24 * (60.0 * 60.0) * value - else: - raise Exception("unexpected units value '{}'".format(units)) - else: - raise Exception("could not parse TIMEOUT spec '{}'".format( - timeout)) - - -class ProcessHelper(object): - """Provides an interface for accessing process-related functionality. - - This class provides a factory method that gives the caller a - platform-specific implementation instance of the class. - - Clients of the class should stick to the methods provided in this - base class. - - @see ProcessHelper.process_helper() - """ - def __init__(self): - super(ProcessHelper, self).__init__() - - @classmethod - def process_helper(cls): - """Returns a platform-specific ProcessHelper instance. - @return a ProcessHelper instance that does the right thing for - the current platform. - """ - - # If you add a new platform, create an instance here and - # return it. - if os.name == "nt": - return WindowsProcessHelper() - else: - # For all POSIX-like systems. - return UnixProcessHelper() - - def create_piped_process(self, command, new_process_group=True): - # pylint: disable=no-self-use,unused-argument - # As expected. We want derived classes to implement this. - """Creates a subprocess.Popen-based class with I/O piped to the parent. - - @param command the command line list as would be passed to - subprocess.Popen(). Use the list form rather than the string form. - - @param new_process_group indicates if the caller wants the - process to be created in its own process group. Each OS handles - this concept differently. It provides a level of isolation and - can simplify or enable terminating the process tree properly. - - @return a subprocess.Popen-like object. - """ - raise Exception("derived class must implement") - - def supports_soft_terminate(self): - # pylint: disable=no-self-use - # As expected. We want derived classes to implement this. - """Indicates if the platform supports soft termination. - - Soft termination is the concept of a terminate mechanism that - allows the target process to shut down nicely, but with the - catch that the process might choose to ignore it. - - Platform supporter note: only mark soft terminate as supported - if the target process has some way to evade the soft terminate - request; otherwise, just support the hard terminate method. - - @return True if the platform supports a soft terminate mechanism. - """ - # By default, we do not support a soft terminate mechanism. - return False - - def soft_terminate(self, popen_process, log_file=None, want_core=True): - # pylint: disable=no-self-use,unused-argument - # As expected. We want derived classes to implement this. - """Attempts to terminate the process in a polite way. - - This terminate method is intended to give the child process a - chance to clean up and exit on its own, possibly with a request - to drop a core file or equivalent (i.e. [mini-]crashdump, crashlog, - etc.) If new_process_group was set in the process creation method - and the platform supports it, this terminate call will attempt to - kill the whole process tree rooted in this child process. - - @param popen_process the subprocess.Popen-like object returned - by one of the process-creation methods of this class. - - @param log_file file-like object used to emit error-related - logging info. May be None if no error-related info is desired. - - @param want_core True if the caller would like to get a core - dump (or the analogous crash report) from the terminated process. - """ - popen_process.terminate() - - def hard_terminate(self, popen_process, log_file=None): - # pylint: disable=no-self-use,unused-argument - # As expected. We want derived classes to implement this. - """Attempts to terminate the process immediately. - - This terminate method is intended to kill child process in - a manner in which the child process has no ability to block, - and also has no ability to clean up properly. If new_process_group - was specified when creating the process, and if the platform - implementation supports it, this will attempt to kill the - whole process tree rooted in the child process. - - @param popen_process the subprocess.Popen-like object returned - by one of the process-creation methods of this class. - - @param log_file file-like object used to emit error-related - logging info. May be None if no error-related info is desired. - """ - popen_process.kill() - - def was_soft_terminate(self, returncode, with_core): - # pylint: disable=no-self-use,unused-argument - # As expected. We want derived classes to implement this. - """Returns if Popen-like object returncode matches soft terminate. - - @param returncode the returncode from the Popen-like object that - terminated with a given return code. - - @param with_core indicates whether the returncode should match - a core-generating return signal. - - @return True when the returncode represents what the system would - issue when a soft_terminate() with the given with_core arg occurred; - False otherwise. - """ - if not self.supports_soft_terminate(): - # If we don't support soft termination on this platform, - # then this should always be False. - return False - else: - # Once a platform claims to support soft terminate, it - # needs to be able to identify it by overriding this method. - raise Exception("platform needs to implement") - - def was_hard_terminate(self, returncode): - # pylint: disable=no-self-use,unused-argument - # As expected. We want derived classes to implement this. - """Returns if Popen-like object returncode matches that of a hard - terminate attempt. - - @param returncode the returncode from the Popen-like object that - terminated with a given return code. - - @return True when the returncode represents what the system would - issue when a hard_terminate() occurred; False - otherwise. - """ - raise Exception("platform needs to implement") - - def soft_terminate_signals(self): - # pylint: disable=no-self-use - """Retrieve signal numbers that can be sent to soft terminate. - @return a list of signal numbers that can be sent to soft terminate - a process, or None if not applicable. - """ - return None - - def is_exceptional_exit(self, popen_status): - """Returns whether the program exit status is exceptional. - - Returns whether the return code from a Popen process is exceptional - (e.g. signals on POSIX systems). - - Derived classes should override this if they can detect exceptional - program exit. - - @return True if the given popen_status represents an exceptional - program exit; False otherwise. - """ - return False - - def exceptional_exit_details(self, popen_status): - """Returns the normalized exceptional exit code and a description. - - Given an exceptional exit code, returns the integral value of the - exception (e.g. signal number for POSIX) and a description (e.g. - signal name on POSIX) for the result. - - Derived classes should override this if they can detect exceptional - program exit. - - It is fine to not implement this so long as is_exceptional_exit() - always returns False. - - @return (normalized exception code, symbolic exception description) - """ - raise Exception("exception_exit_details() called on unsupported class") - - -class UnixProcessHelper(ProcessHelper): - """Provides a ProcessHelper for Unix-like operating systems. - - This implementation supports anything that looks Posix-y - (e.g. Darwin, Linux, *BSD, etc.) - """ - def __init__(self): - super(UnixProcessHelper, self).__init__() - - @classmethod - def _create_new_process_group(cls): - """Creates a new process group for the calling process.""" - os.setpgid(os.getpid(), os.getpid()) - - def create_piped_process(self, command, new_process_group=True): - # Determine what to run after the fork but before the exec. - if new_process_group: - preexec_func = self._create_new_process_group - else: - preexec_func = None - - # Create the process. - process = subprocess.Popen( - command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, # Elicits automatic byte -> string decoding in Py3 - close_fds=True, - preexec_fn=preexec_func) - - # Remember whether we're using process groups for this - # process. - process.using_process_groups = new_process_group - return process - - def supports_soft_terminate(self): - # POSIX does support a soft terminate via: - # * SIGTERM (no core requested) - # * SIGQUIT (core requested if enabled, see ulimit -c) - return True - - @classmethod - def _validate_pre_terminate(cls, popen_process, log_file): - # Validate args. - if popen_process is None: - raise ValueError("popen_process is None") - - # Ensure we have something that looks like a valid process. - if popen_process.pid < 1: - if log_file: - log_file.write("skipping soft_terminate(): no process id") - return False - - # We only do the process liveness check if we're not using - # process groups. With process groups, checking if the main - # inferior process is dead and short circuiting here is no - # good - children of it in the process group could still be - # alive, and they should be killed during a timeout. - if not popen_process.using_process_groups: - # Don't kill if it's already dead. - popen_process.poll() - if popen_process.returncode is not None: - # It has a returncode. It has already stopped. - if log_file: - log_file.write( - "requested to terminate pid {} but it has already " - "terminated, returncode {}".format( - popen_process.pid, popen_process.returncode)) - # Move along... - return False - - # Good to go. - return True - - def _kill_with_signal(self, popen_process, log_file, signum): - # Validate we're ready to terminate this. - if not self._validate_pre_terminate(popen_process, log_file): - return - - # Choose kill mechanism based on whether we're targeting - # a process group or just a process. - if popen_process.using_process_groups: - # if log_file: - # log_file.write( - # "sending signum {} to process group {} now\n".format( - # signum, popen_process.pid)) - os.killpg(popen_process.pid, signum) - else: - # if log_file: - # log_file.write( - # "sending signum {} to process {} now\n".format( - # signum, popen_process.pid)) - os.kill(popen_process.pid, signum) - - def soft_terminate(self, popen_process, log_file=None, want_core=True): - # Choose signal based on desire for core file. - if want_core: - # SIGQUIT will generate core by default. Can be caught. - signum = signal.SIGQUIT - else: - # SIGTERM is the traditional nice way to kill a process. - # Can be caught, doesn't generate a core. - signum = signal.SIGTERM - - self._kill_with_signal(popen_process, log_file, signum) - - def hard_terminate(self, popen_process, log_file=None): - self._kill_with_signal(popen_process, log_file, signal.SIGKILL) - - def was_soft_terminate(self, returncode, with_core): - if with_core: - return returncode == -signal.SIGQUIT - else: - return returncode == -signal.SIGTERM - - def was_hard_terminate(self, returncode): - return returncode == -signal.SIGKILL - - def soft_terminate_signals(self): - return [signal.SIGQUIT, signal.SIGTERM] - - def is_exceptional_exit(self, popen_status): - return popen_status < 0 - - @classmethod - def _signal_names_by_number(cls): - return dict( - (k, v) for v, k in reversed(sorted(signal.__dict__.items())) - if v.startswith('SIG') and not v.startswith('SIG_')) - - def exceptional_exit_details(self, popen_status): - signo = -popen_status - signal_names_by_number = self._signal_names_by_number() - signal_name = signal_names_by_number.get(signo, "") - return (signo, signal_name) - -class WindowsProcessHelper(ProcessHelper): - """Provides a Windows implementation of the ProcessHelper class.""" - def __init__(self): - super(WindowsProcessHelper, self).__init__() - - def create_piped_process(self, command, new_process_group=True): - if new_process_group: - # We need this flag if we want os.kill() to work on the subprocess. - creation_flags = subprocess.CREATE_NEW_PROCESS_GROUP - else: - creation_flags = 0 - - return subprocess.Popen( - command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, # Elicits automatic byte -> string decoding in Py3 - creationflags=creation_flags) - - def was_hard_terminate(self, returncode): - return returncode != 0 - - -class ProcessDriver(object): - """Drives a child process, notifies on important events, and can timeout. - - Clients are expected to derive from this class and override the - on_process_started and on_process_exited methods if they want to - hook either of those. - - This class supports timing out the child process in a platform-agnostic - way. The on_process_exited method is informed if the exit was natural - or if it was due to a timeout. - """ - def __init__(self, soft_terminate_timeout=10.0): - super(ProcessDriver, self).__init__() - self.process_helper = ProcessHelper.process_helper() - self.pid = None - # Create the synchronization event for notifying when the - # inferior dotest process is complete. - self.done_event = threading.Event() - self.io_thread = None - self.process = None - # Number of seconds to wait for the soft terminate to - # wrap up, before moving to more drastic measures. - # Might want this longer if core dumps are generated and - # take a long time to write out. - self.soft_terminate_timeout = soft_terminate_timeout - # Number of seconds to wait for the hard terminate to - # wrap up, before giving up on the io thread. This should - # be fast. - self.hard_terminate_timeout = 5.0 - self.returncode = None - - # ============================================= - # Methods for subclasses to override if desired. - # ============================================= - - def on_process_started(self): - pass - - def on_process_exited(self, command, output, was_timeout, exit_status): - pass - - def write(self, content): - # pylint: disable=no-self-use - # Intended - we want derived classes to be able to override - # this and use any self state they may contain. - sys.stdout.write(content) - - # ============================================================== - # Operations used to drive processes. Clients will want to call - # one of these. - # ============================================================== - - def run_command(self, command): - # Start up the child process and the thread that does the - # communication pump. - self._start_process_and_io_thread(command) - - # Wait indefinitely for the child process to finish - # communicating. This indicates it has closed stdout/stderr - # pipes and is done. - self.io_thread.join() - self.returncode = self.process.wait() - if self.returncode is None: - raise Exception( - "no exit status available for pid {} after the " - " inferior dotest.py should have completed".format( - self.process.pid)) - - # Notify of non-timeout exit. - self.on_process_exited( - command, - self.io_thread.output, - False, - self.returncode) - - def run_command_with_timeout(self, command, timeout, want_core): - # Figure out how many seconds our timeout description is requesting. - timeout_seconds = timeout_to_seconds(timeout) - - # Start up the child process and the thread that does the - # communication pump. - self._start_process_and_io_thread(command) - - self._wait_with_timeout(timeout_seconds, command, want_core) - - # ================ - # Internal details. - # ================ - - def _start_process_and_io_thread(self, command): - # Create the process. - self.process = self.process_helper.create_piped_process(command) - self.pid = self.process.pid - self.on_process_started() - - # Ensure the event is cleared that is used for signaling - # from the communication() thread when communication is - # complete (i.e. the inferior process has finished). - self.done_event.clear() - - self.io_thread = CommunicatorThread( - self.process, self.done_event, self.write) - self.io_thread.start() - - def _attempt_soft_kill(self, want_core): - # The inferior dotest timed out. Attempt to clean it - # with a non-drastic method (so it can clean up properly - # and/or generate a core dump). Often the OS can't guarantee - # that the process will really terminate after this. - self.process_helper.soft_terminate( - self.process, - want_core=want_core, - log_file=self) - - # Now wait up to a certain timeout period for the io thread - # to say that the communication ended. If that wraps up - # within our soft terminate timeout, we're all done here. - self.io_thread.join(self.soft_terminate_timeout) - if not self.io_thread.is_alive(): - # stdout/stderr were closed on the child process side. We - # should be able to wait and reap the child process here. - self.returncode = self.process.wait() - # We terminated, and the done_trying result is n/a - terminated = True - done_trying = None - else: - self.write("soft kill attempt of process {} timed out " - "after {} seconds\n".format( - self.process.pid, self.soft_terminate_timeout)) - terminated = False - done_trying = False - return terminated, done_trying - - def _attempt_hard_kill(self): - # Instruct the process to terminate and really force it to - # happen. Don't give the process a chance to ignore. - self.process_helper.hard_terminate( - self.process, - log_file=self) - - # Reap the child process. This should not hang as the - # hard_kill() mechanism is supposed to really kill it. - # Improvement option: - # If this does ever hang, convert to a self.process.poll() - # loop checking on self.process.returncode until it is not - # None or the timeout occurs. - self.returncode = self.process.wait() - - # Wait a few moments for the io thread to finish... - self.io_thread.join(self.hard_terminate_timeout) - if self.io_thread.is_alive(): - # ... but this is not critical if it doesn't end for some - # reason. - self.write( - "hard kill of process {} timed out after {} seconds waiting " - "for the io thread (ignoring)\n".format( - self.process.pid, self.hard_terminate_timeout)) - - # Set if it terminated. (Set up for optional improvement above). - terminated = self.returncode is not None - # Nothing else to try. - done_trying = True - - return terminated, done_trying - - def _attempt_termination(self, attempt_count, want_core): - if self.process_helper.supports_soft_terminate(): - # When soft termination is supported, we first try to stop - # the process with a soft terminate. Failing that, we try - # the hard terminate option. - if attempt_count == 1: - return self._attempt_soft_kill(want_core) - elif attempt_count == 2: - return self._attempt_hard_kill() - else: - # We don't have anything else to try. - terminated = self.returncode is not None - done_trying = True - return terminated, done_trying - else: - # We only try the hard terminate option when there - # is no soft terminate available. - if attempt_count == 1: - return self._attempt_hard_kill() - else: - # We don't have anything else to try. - terminated = self.returncode is not None - done_trying = True - return terminated, done_trying - - def _wait_with_timeout(self, timeout_seconds, command, want_core): - # Allow up to timeout seconds for the io thread to wrap up. - # If that completes, the child process should be done. - completed_normally = self.done_event.wait(timeout_seconds) - if completed_normally: - # Reap the child process here. - self.returncode = self.process.wait() - else: - # Prepare to stop the process - process_terminated = completed_normally - terminate_attempt_count = 0 - - # Try as many attempts as we support for trying to shut down - # the child process if it's not already shut down. - while not process_terminated: - terminate_attempt_count += 1 - # Attempt to terminate. - process_terminated, done_trying = self._attempt_termination( - terminate_attempt_count, want_core) - # Check if there's nothing more to try. - if done_trying: - # Break out of our termination attempt loop. - break - - # At this point, we're calling it good. The process - # finished gracefully, was shut down after one or more - # attempts, or we failed but gave it our best effort. - self.on_process_exited( - command, - self.io_thread.output, - not completed_normally, - self.returncode) - - -def patched_init(self, *args, **kwargs): - self.original_init(*args, **kwargs) - # Initialize our condition variable that protects wait()/poll(). - self.wait_condition = threading.Condition() - - -def patched_wait(self, *args, **kwargs): - self.wait_condition.acquire() - try: - result = self.original_wait(*args, **kwargs) - # The process finished. Signal the condition. - self.wait_condition.notify_all() - return result - finally: - self.wait_condition.release() - - -def patched_poll(self, *args, **kwargs): - self.wait_condition.acquire() - try: - result = self.original_poll(*args, **kwargs) - if self.returncode is not None: - # We did complete, and we have the return value. - # Signal the event to indicate we're done. - self.wait_condition.notify_all() - return result - finally: - self.wait_condition.release() - - -def patch_up_subprocess_popen(): - subprocess.Popen.original_init = subprocess.Popen.__init__ - subprocess.Popen.__init__ = patched_init - - subprocess.Popen.original_wait = subprocess.Popen.wait - subprocess.Popen.wait = patched_wait - - subprocess.Popen.original_poll = subprocess.Popen.poll - subprocess.Popen.poll = patched_poll - -# Replace key subprocess.Popen() threading-unprotected methods with -# threading-protected versions. -patch_up_subprocess_popen()