Index: utils/lit/lit/LitConfig.py =================================================================== --- utils/lit/lit/LitConfig.py +++ utils/lit/lit/LitConfig.py @@ -8,7 +8,8 @@ import lit.TestingConfig import lit.util -class LitConfig: +# LitConfig must be a new style class for properties to work +class LitConfig(object): """LitConfig - Configuration data for a 'lit' test runner instance, shared across all tests. @@ -21,7 +22,8 @@ def __init__(self, progname, path, quiet, useValgrind, valgrindLeakCheck, valgrindArgs, noExecute, debug, isWindows, - params, config_prefix = None): + params, config_prefix = None, + maxIndividualTestTime = 0): # The name of the test runner. self.progname = progname # The items to add to the PATH environment variable. @@ -57,6 +59,36 @@ self.valgrindArgs.append('--leak-check=no') self.valgrindArgs.extend(self.valgrindUserArgs) + self.maxIndividualTestTime = maxIndividualTestTime + + @property + def maxIndividualTestTime(self): + """ + Interface for getting maximum time to spend executing + a single test + """ + return self._maxIndividualTestTime + + @maxIndividualTestTime.setter + def maxIndividualTestTime(self, value): + """ + Interface for setting maximum time to spend executing + a single test + """ + self._maxIndividualTestTime = value + if self.maxIndividualTestTime > 0: + # The current implementation needs psutil to set + # a timeout per test. Check it's available. + # See lit.util.killProcessAndChildren() + try: + import psutil + except ImportError: + self.fatal("Setting a timeout per test requires the" + " Python psutil module but it could not be" + " found. Try installing it via pip or via" + " your operating system's package manager.") + elif self.maxIndividualTestTime < 0: + self.fatal('The timeout per test must be >= 0 seconds') def load_config(self, config, path): """load_config(config, path) - Load a config object from an alternate Index: utils/lit/lit/Test.py =================================================================== --- utils/lit/lit/Test.py +++ utils/lit/lit/Test.py @@ -33,6 +33,7 @@ XPASS = ResultCode('XPASS', True) UNRESOLVED = ResultCode('UNRESOLVED', True) UNSUPPORTED = ResultCode('UNSUPPORTED', False) +TIMEOUT = ResultCode('TIMEOUT', True) # Test metric values. Index: utils/lit/lit/TestRunner.py =================================================================== --- utils/lit/lit/TestRunner.py +++ utils/lit/lit/TestRunner.py @@ -3,6 +3,7 @@ import re import platform import tempfile +import threading import lit.ShUtil as ShUtil import lit.Test as Test @@ -14,6 +15,11 @@ self.command = command self.message = message +class TestTimeoutError(Exception): + def __init__(self, command, message): + self.command = command + self.message = message + kIsWindows = platform.system() == 'Windows' # Don't use close_fds on Windows. @@ -33,28 +39,126 @@ self.cwd = cwd self.env = dict(env) -def executeShCmd(cmd, shenv, results): +class TimeoutHelper(object): + """ + Object used to helper manage enforcing a timeout in + _executeShCmd(). It is passed through recursive calls + to collect processes that have been executed so that when + the timeout happens they can be killed. + """ + def __init__(self, timeout): + self.timeout = timeout + self._procs = [] + self._timeoutReached = False + self._doneKillPass = False + # This lock will be used to protect concurrent access + # to _procs and _doneKillPass + self._lock = None + self._timer = None + + def cancel(self): + if not self.active(): + return + self._timer.cancel() + + def active(self): + return self.timeout > 0 + + def addProcess(self, proc): + if not self.active(): + return + needToRunKill = False + with self._lock: + self._procs.append(proc) + # Avoid re-entering the lock by finding out if kill needs to be run + # again here but call it if necessary once we have left the lock. + # We could use a reentrant lock here instead but this code seems + # clearer to me. + needToRunKill = self._doneKillPass + + # The initial call to _kill() from the timer thread already happened so + # we need to call it again from this thread, otherwise this process + # will be left to run even though the timeout was already hit + if needToRunKill: + assert self.timeoutReached() + self._kill() + + def startTimer(self): + if not self.active(): + return + + # Do some late initialisation that's only needed + # if there is a timeout set + self._lock = threading.Lock() + self._timer = threading.Timer(self.timeout, self._handleTimeoutReached) + self._timer.start() + + def _handleTimeoutReached(self): + self._timeoutReached = True + self._kill() + + def timeoutReached(self): + return self._timeoutReached + + def _kill(self): + """ + This method may be called multiple times as we might get unlucky + and be in the middle of creating a new process in _executeShCmd() + which won't yet be in ``self._procs``. By locking here and in + addProcess() we should be able to kill processes launched after + the initial call to _kill() + """ + with self._lock: + for p in self._procs: + lit.util.killProcessAndChildren(p.pid) + # Empty the list and note that we've done a pass over the list + self._procs = [] # Python2 doesn't have list.clear() + self._doneKillPass = True + +def executeShCmd(cmd, shenv, results, timeout=0): + """ + Wrapper around _executeShCmd that handles + timeout + """ + # Use the helper even when no timeout is required to make + # other code simpler (i.e. avoid bunch of ``!= None`` checks) + timeoutHelper = TimeoutHelper(timeout) + if timeout > 0: + timeoutHelper.startTimer() + finalExitCode = _executeShCmd(cmd, shenv, results, timeoutHelper) + timeoutHelper.cancel() + if timeoutHelper.timeoutReached(): + raise TestTimeoutError(cmd, 'Reached test timeout of {} seconds'.format(timeout)) + return finalExitCode + + +def _executeShCmd(cmd, shenv, results, timeoutHelper): + if timeoutHelper.timeoutReached(): + # Prevent further recursion if the timeout has been hit + # as we should try avoid launching more processes. + return None + if isinstance(cmd, ShUtil.Seq): if cmd.op == ';': - res = executeShCmd(cmd.lhs, shenv, results) - return executeShCmd(cmd.rhs, shenv, results) + res = _executeShCmd(cmd.lhs, shenv, results, timeoutHelper) + return _executeShCmd(cmd.rhs, shenv, results, timeoutHelper) if cmd.op == '&': raise InternalShellError(cmd,"unsupported shell operator: '&'") if cmd.op == '||': - res = executeShCmd(cmd.lhs, shenv, results) + res = _executeShCmd(cmd.lhs, shenv, results, timeoutHelper) if res != 0: - res = executeShCmd(cmd.rhs, shenv, results) + res = _executeShCmd(cmd.rhs, shenv, results, timeoutHelper) return res if cmd.op == '&&': - res = executeShCmd(cmd.lhs, shenv, results) + res = _executeShCmd(cmd.lhs, shenv, results, timeoutHelper) if res is None: return res if res == 0: - res = executeShCmd(cmd.rhs, shenv, results) + res = _executeShCmd(cmd.rhs, shenv, results, timeoutHelper) return res raise ValueError('Unknown shell command: %r' % cmd.op) @@ -206,6 +310,8 @@ stderr = stderr, env = cmd_shenv.env, close_fds = kUseCloseFDs)) + # Let the helper know about this process + timeoutHelper.addProcess(procs[-1]) except OSError as e: raise InternalShellError(j, 'Could not create process ({}) due to {}'.format(executable, e)) @@ -311,7 +417,7 @@ results = [] try: shenv = ShellEnvironment(cwd, test.config.environment) - exitCode = executeShCmd(cmd, shenv, results) + exitCode = executeShCmd(cmd, shenv, results, timeout=litConfig.maxIndividualTestTime) except InternalShellError: e = sys.exc_info()[1] exitCode = 127 @@ -359,8 +465,12 @@ # run on clang with no real loss. command = litConfig.valgrindArgs + command - return lit.util.executeCommand(command, cwd=cwd, - env=test.config.environment) + try: + return lit.util.executeCommand(command, cwd=cwd, + env=test.config.environment, + timeout=litConfig.maxIndividualTestTime) + except lit.util.ExecuteCommandTimeoutException: + raise TestTimeoutError(command, 'Hit timeout of {} seconds'.format(litConfig.maxIndividualTestTime)) def parseIntegratedTestScriptCommands(source_path, keywords): """ @@ -564,12 +674,15 @@ lit.util.mkdir_p(os.path.dirname(tmpBase)) execdir = os.path.dirname(test.getExecPath()) - if useExternalSh: - res = executeScript(test, litConfig, tmpBase, script, execdir) - else: - res = executeScriptInternal(test, litConfig, tmpBase, script, execdir) - if isinstance(res, lit.Test.Result): - return res + try: + if useExternalSh: + res = executeScript(test, litConfig, tmpBase, script, execdir) + else: + res = executeScriptInternal(test, litConfig, tmpBase, script, execdir) + if isinstance(res, lit.Test.Result): + return res + except TestTimeoutError as e: + return lit.Test.Result(Test.TIMEOUT, e.message) out,err,exitCode = res if exitCode == 0: Index: utils/lit/lit/formats/googletest.py =================================================================== --- utils/lit/lit/formats/googletest.py +++ utils/lit/lit/formats/googletest.py @@ -109,8 +109,15 @@ if litConfig.noExecute: return lit.Test.PASS, '' - out, err, exitCode = lit.util.executeCommand( - cmd, env=test.config.environment) + try: + out, err, exitCode = lit.util.executeCommand( + cmd, env=test.config.environment, + timeout=litConfig.maxIndividualTestTime) + except lit.util.ExecuteCommandTimeoutException: + return (lit.Test.TIMEOUT, + 'Reached timeout of {} seconds'.format( + litConfig.maxIndividualTestTime) + ) if exitCode: return lit.Test.FAIL, out + err Index: utils/lit/lit/main.py =================================================================== --- utils/lit/lit/main.py +++ utils/lit/lit/main.py @@ -205,6 +205,10 @@ group.add_option("", "--xunit-xml-output", dest="xunit_output_file", help=("Write XUnit-compatible XML test reports to the" " specified file"), default=None) + group.add_option("", "--max-individual-test-time", dest="maxIndividualTestTime", + help="Maximum time to spend running a single test (in seconds)." + "0 means no time limit. [Default: 0]", + type=int, default=None) parser.add_option_group(group) group = OptionGroup(parser, "Test Selection") @@ -275,6 +279,14 @@ name,val = entry.split('=', 1) userParams[name] = val + # Decide what the requested maximum indvidual test time should be + if opts.maxIndividualTestTime != None: + maxIndividualTestTime = opts.maxIndividualTestTime + else: + # Default is zero + maxIndividualTestTime = 0 + + # Create the global config object. litConfig = lit.LitConfig.LitConfig( progname = os.path.basename(sys.argv[0]), @@ -287,12 +299,26 @@ debug = opts.debug, isWindows = isWindows, params = userParams, - config_prefix = opts.configPrefix) + config_prefix = opts.configPrefix, + maxIndividualTestTime = maxIndividualTestTime) # Perform test discovery. run = lit.run.Run(litConfig, lit.discovery.find_tests_for_inputs(litConfig, inputs)) + # After test discovery the configuration might have changed + # the maxIndividualTestTime. If we explicitly set this on the + # command line then override what was set in the test configuration + if opts.maxIndividualTestTime != None: + if opts.maxIndividualTestTime != litConfig.maxIndividualTestTime: + litConfig.warning(('The test suite configuration requested an individual' + ' test timeout of {0} seconds but a timeout of {1} seconds was' + ' requested on the command line. Forcing timeout to be {1}' + ' seconds') + .format(litConfig.maxIndividualTestTime, + opts.maxIndividualTestTime)) + litConfig.maxIndividualTestTime = opts.maxIndividualTestTime + if opts.showSuites or opts.showTests: # Aggregate the tests by suite. suitesAndTests = {} @@ -377,7 +403,7 @@ extra = ' of %d' % numTotalTests header = '-- Testing: %d%s tests, %d threads --'%(len(run.tests), extra, opts.numThreads) - + print('Using individual test timeout of {} seconds'.format(litConfig.maxIndividualTestTime)) progressBar = None if not opts.quiet: if opts.succinct and opts.useProgressBar: @@ -447,7 +473,8 @@ ('Unsupported Tests ', lit.Test.UNSUPPORTED), ('Unresolved Tests ', lit.Test.UNRESOLVED), ('Unexpected Passes ', lit.Test.XPASS), - ('Unexpected Failures', lit.Test.FAIL)): + ('Unexpected Failures', lit.Test.FAIL), + ('Individual timeouts', lit.Test.TIMEOUT)): if opts.quiet and not code.isFailure: continue N = len(byCode.get(code,[])) Index: utils/lit/lit/util.py =================================================================== --- utils/lit/lit/util.py +++ utils/lit/lit/util.py @@ -6,6 +6,7 @@ import signal import subprocess import sys +import threading def to_bytes(str): # Encode to UTF-8 to get binary data. @@ -155,17 +156,42 @@ pDigits, pfDigits, i*barH, pDigits, pfDigits, (i+1)*barH, '*'*w, ' '*(barW-w), cDigits, len(row), cDigits, len(items))) +class ExecuteCommandTimeoutException(Exception): + pass # Close extra file handles on UNIX (on Windows this cannot be done while # also redirecting input). kUseCloseFDs = not (platform.system() == 'Windows') -def executeCommand(command, cwd=None, env=None, input=None): +def executeCommand(command, cwd=None, env=None, input=None, timeout=0): p = subprocess.Popen(command, cwd=cwd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, close_fds=kUseCloseFDs) - out,err = p.communicate(input=input) - exitCode = p.wait() + timerObject = None + # FIXME: Because of the way nested function scopes work in Python 2.x we + # need to use a reference to a mutable object rather than a plain + # bool. In Python 3 we could use the "nonlocal" keyword but we need + # to support Python 2 as well. + hitTimeOut = [False] + try: + if timeout > 0: + def killProcess(): + # We may be invoking a shell so we need to kill the + # process and all its children. + hitTimeOut[0] = True + killProcessAndChildren(p.pid) + + timerObject = threading.Timer(timeout, killProcess) + timerObject.start() + + out,err = p.communicate(input=input) + exitCode = p.wait() + finally: + if timerObject != None: + timerObject.cancel() + + if hitTimeOut[0]: + raise ExecuteCommandTimeoutException() # Detect Ctrl-C in subprocess. if exitCode == -signal.SIGINT: @@ -193,3 +219,25 @@ sdk_path = out lit_config.note('using SDKROOT: %r' % sdk_path) config.environment['SDKROOT'] = sdk_path + +def killProcessAndChildren(pid): + """ + This function kills a process with ``pid`` and all its + running children (recursively). It is currently implemented + using the psutil module which provides a simple platform + neutral implementation. + + TODO: Reimplement this without using psutil so we can + remove our dependency on it. + """ + import psutil + try: + psutilProc = psutil.Process(pid) + for child in psutilProc.children(recursive=True): + try: + child.kill() + except psutil.NoSuchProcess: + pass + psutilProc.kill() + except psutil.NoSuchProcess: + pass Index: utils/lit/tests/Inputs/per_test_timeout/lit.cfg =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/per_test_timeout/lit.cfg @@ -0,0 +1,34 @@ +# -*- Python -*- +import os +import sys + +import lit.formats + +config.name = 'per_test_timeout' + +shellType = lit_config.params.get('external', '1') + +if shellType == '0': + lit_config.note('Using internal shell') + externalShell = False +else: + lit_config.note('Using external shell') + externalShell = True + +configSetTimeout = lit_config.params.get('set_timeout', '0') + +if configSetTimeout == '1': + # Try setting the max individual test time in the configuration + lit_config.maxIndividualTestTime = 1 + +# FIXME: GTest needs testing too +config.test_format = lit.formats.ShTest(execute_external=externalShell) +config.suffixes = ['.txt', '.py'] +config.test_source_root = os.path.dirname(__file__) +config.test_exec_root = config.test_source_root +config.target_triple = '(unused)' +src_root = os.path.join(config.test_source_root, '..') +config.environment['PYTHONPATH'] = src_root +config.substitutions.append(('%{python}', sys.executable)) +config.substitutions.append(('%{inputs}', config.test_source_root)) +config.substitutions.append(('%{lit}', sys.modules['__main__'].__file__)) Index: utils/lit/tests/Inputs/per_test_timeout/short.py =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/per_test_timeout/short.py @@ -0,0 +1,6 @@ +# RUN: %{python} %s +from __future__ import print_function + +import sys + +print("short program") Index: utils/lit/tests/Inputs/per_test_timeout/slow.py =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/per_test_timeout/slow.py @@ -0,0 +1,8 @@ +# RUN: %{python} %s +from __future__ import print_function + +import time +import sys + +print("slow program") +time.sleep(6) Index: utils/lit/tests/Inputs/per_test_timeout/timeout_external.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/per_test_timeout/timeout_external.txt @@ -0,0 +1,12 @@ +# RUN: not %{lit} \ +# RUN: %{inputs}/slow.py \ +# RUN: %{inputs}/short.py \ +# RUN: -j 1 -v --debug --max-individual-test-time 1 --param external=1 > %t.out 2> %t.err +# RUN: FileCheck --check-prefix=CHECK-OUT < %t.out %s +# RUN: FileCheck --check-prefix=CHECK-ERR < %t.err %s + +# CHECK-OUT: PASS: per_test_timeout :: short.py +# CHECK-OUT: TIMEOUT: per_test_timeout :: slow.py +# CHECK-OUT: Expected Passes{{ *}}: 1 +# CHECK-OUT: Individual timeouts{{ *}}: 1 +# CHECK-ERR: Using external shell Index: utils/lit/tests/Inputs/per_test_timeout/timeout_internal.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/per_test_timeout/timeout_internal.txt @@ -0,0 +1,12 @@ +# RUN: not %{lit} \ +# RUN: %{inputs}/slow.py \ +# RUN: %{inputs}/short.py \ +# RUN: -j 1 -v --debug --max-individual-test-time 1 --param external=0 > %t.out 2> %t.err +# RUN: FileCheck --check-prefix=CHECK-OUT < %t.out %s +# RUN: FileCheck --check-prefix=CHECK-ERR < %t.err %s + +# CHECK-OUT: PASS: per_test_timeout :: short.py +# CHECK-OUT: TIMEOUT: per_test_timeout :: slow.py +# CHECK-OUT: Expected Passes{{ *}}: 1 +# CHECK-OUT: Individual timeouts{{ *}}: 1 +# CHECK-ERR: Using internal shell Index: utils/lit/tests/Inputs/per_test_timeout/timeout_internal_set_in_config.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/per_test_timeout/timeout_internal_set_in_config.txt @@ -0,0 +1,12 @@ +# RUN: not %{lit} \ +# RUN: %{inputs}/slow.py \ +# RUN: %{inputs}/short.py \ +# RUN: -j 1 -v --debug --param external=0 --param set_timeout=1 > %t.out 2> %t.err +# RUN: FileCheck --check-prefix=CHECK-OUT < %t.out %s +# RUN: FileCheck --check-prefix=CHECK-ERR < %t.err %s + +# CHECK-OUT: PASS: per_test_timeout :: short.py +# CHECK-OUT: TIMEOUT: per_test_timeout :: slow.py +# CHECK-OUT: Expected Passes{{ *}}: 1 +# CHECK-OUT: Individual timeouts{{ *}}: 1 +# CHECK-ERR: Using internal shell