Index: test/CMakeLists.txt =================================================================== --- test/CMakeLists.txt +++ test/CMakeLists.txt @@ -62,16 +62,16 @@ add_python_test_target(check-lldb-single ${LLDB_SOURCE_DIR}/test/dotest.py - "${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS}" + "--no-multiprocess;${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS}" "Testing LLDB with args: ${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS}" ) -set(LLDB_DOSEP_ARGS -o;\"-q;${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS}\") +set(LLDB_DOTEST_ARGS -q;${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS}) # If tests crash cause LLDB to crash, or things are otherwise unstable, or if machine-parsable # output is desired (i.e. in continuous integration contexts) check-lldb-single is a better target. add_python_test_target(check-lldb - ${LLDB_SOURCE_DIR}/test/dosep.py - "${LLDB_DOSEP_ARGS}" - "Testing LLDB (with a separate subprocess per test)" + ${LLDB_SOURCE_DIR}/test/dotest.py + "${LLDB_DOTEST_ARGS}" + "Testing LLDB (parallel execution, with a separate subprocess per test)" ) Index: test/Makefile =================================================================== --- test/Makefile +++ test/Makefile @@ -30,4 +30,4 @@ #---------------------------------------------------------------------- check-local:: rm -rf lldb-test-traces - python $(PROJ_SRC_DIR)/dosep.py -o "--executable $(ToolDir)/lldb -q -s lldb-test-traces -u CXXFLAGS -u CFLAGS -C $(subst ccache,,$(CC))" + python $(PROJ_SRC_DIR)/dotest.py --executable $(ToolDir)/lldb -q -s lldb-test-traces -u CXXFLAGS -u CFLAGS -C $(subst ccache,,$(CC)) Index: test/dosep.py =================================================================== --- test/dosep.py +++ test/dosep.py @@ -38,7 +38,6 @@ import platform import re import dotest_args -import shlex import subprocess import sys @@ -131,8 +130,6 @@ result, re.MULTILINE) unexpected_success_count = re.search("^RESULT:.*([0-9]+) unexpected successes", result, re.MULTILINE) - this_fail_count = 0 - this_error_count = 0 if pass_count is not None: passes = passes + int(pass_count.group(1)) if fail_count is not None: @@ -183,7 +180,7 @@ script_file = os.path.join(test_root, "dotest.py") command = ([sys.executable, script_file] + dotest_argv + - ["-p", name, root]) + ["--inferior", "-p", name, root]) timeout_name = os.path.basename(os.path.splitext(name)[0]).upper() @@ -201,7 +198,7 @@ if status != ePassed] unexpected_passes = [name for name, _, _, _, unexpected_successes in results if unexpected_successes > 0] - + pass_count = sum([result[2] for result in results]) fail_count = sum([result[3] for result in results]) @@ -284,7 +281,6 @@ else: m = re.search('remote-(\w+)', platform_name) target = m.group(1) - remote = True expected_timeout = set() @@ -359,7 +355,27 @@ return result -def main(): +def main(print_details_on_success, num_threads, test_subdir): + """Run dotest.py in inferior mode in parallel. + + @param print_details_on_success the parsed value of the output-on-success + command line argument. When True, details of a successful dotest inferior + are printed even when everything succeeds. The normal behavior is to + not print any details when all the inferior tests pass. + + @param num_threads the parsed value of the num-threads command line + argument. + + @param test_subdir optionally specifies a subdir to limit testing + within. May be None if the entire test tree is to be used. This subdir + is assumed to be relative to the lldb/test root of the test hierarchy. + """ + + dotest_argv = sys.argv[1:] + + global output_on_success + output_on_success = print_details_on_success + # We can't use sys.path[0] to determine the script directory # because it doesn't work under a debugger test_directory = os.path.dirname(os.path.realpath(__file__)) @@ -383,37 +399,8 @@ E.g., export LLDB_TEST_TIMEOUT=0 or export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=0 """) - parser.add_option( - '-o', '--options', - type='string', action='store', - dest='dotest_options', - help="""The options passed to 'dotest.py' if specified.""") - - parser.add_option( - '-s', '--output-on-success', - action='store_true', - dest='output_on_success', - default=False, - help="""Print full output of 'dotest.py' even when it succeeds.""") - - parser.add_option( - '-t', '--threads', - type='int', - dest='num_threads', - help="""The number of threads to use when running tests separately.""") - - opts, args = parser.parse_args() - dotest_option_string = opts.dotest_options - - is_posix = (os.name == "posix") - dotest_argv = (shlex.split(dotest_option_string, posix=is_posix) - if dotest_option_string - else []) - parser = dotest_args.create_parser() global dotest_options - global output_on_success - output_on_success = opts.output_on_success dotest_options = dotest_args.parse_args(parser, dotest_argv) if not dotest_options.s: @@ -429,19 +416,17 @@ session_dir = os.path.join(os.getcwd(), dotest_options.s) # The root directory was specified on the command line - if len(args) == 0: - test_subdir = test_directory + if test_subdir and len(test_subdir) > 0: + test_subdir = os.path.join(test_directory, test_subdir) else: - test_subdir = os.path.join(test_directory, args[0]) + test_subdir = test_directory # clean core files in test tree from previous runs (Linux) cores = find('core.*', test_subdir) for core in cores: os.unlink(core) - if opts.num_threads: - num_threads = opts.num_threads - else: + if not num_threads: num_threads_str = os.environ.get("LLDB_TEST_THREADS") if num_threads_str: num_threads = int(num_threads_str) @@ -512,4 +497,8 @@ sys.exit(exit_code) if __name__ == '__main__': - main() + sys.stderr.write( + "error: dosep.py no longer supports being called directly. " + "Please call dotest.py directly. The dosep.py-specific arguments " + "have been added under the Parallel processing arguments.") + sys.exit(128) Index: test/dotest.py =================================================================== --- test/dotest.py +++ test/dotest.py @@ -29,8 +29,6 @@ import signal import subprocess import sys -import textwrap -import time import inspect import unittest2 import lldbtest_config @@ -245,6 +243,13 @@ lldb_platform_url = None lldb_platform_working_dir = None +# Parallel execution settings +is_inferior_test_runner = False +multiprocess_test_subdir = None +num_threads = None +output_on_success = False +no_multiprocess_test_runner = False + def usage(parser): parser.print_help() if verbose > 0: @@ -485,6 +490,11 @@ global lldb_platform_url global lldb_platform_working_dir global setCrashInfoHook + global is_inferior_test_runner + global multiprocess_test_subdir + global num_threads + global output_on_success + global no_multiprocess_test_runner do_help = False @@ -493,7 +503,7 @@ parser = dotest_args.create_parser() args = dotest_args.parse_args(parser, sys.argv[1:]) - + if args.unset_env_varnames: for env_var in args.unset_env_varnames: if env_var in os.environ: @@ -606,7 +616,7 @@ if args.d: sys.stdout.write("Suspending the process %d to wait for debugger to attach...\n" % os.getpid()) - sys.stdout.flush() + sys.stdout.flush() os.kill(os.getpid(), signal.SIGSTOP) if args.e: @@ -740,6 +750,21 @@ if dont_do_lldbmi_test and just_do_lldbmi_test: usage(parser) + if args.no_multiprocess: + no_multiprocess_test_runner = True + + if args.inferior: + is_inferior_test_runner = True + + if args.output_on_success: + output_on_success = True + + if args.num_threads: + num_threads = args.num_threads + + if args.test_subdir: + multiprocess_test_subdir = args.test_subdir + if args.lldb_platform_name: lldb_platform_name = args.lldb_platform_name if args.lldb_platform_url: @@ -1228,6 +1253,14 @@ if exitCode: sys.exit(exitCode) + +def isMultiprocessTestRunner(): + # We're not multiprocess when we're either explicitly + # the inferior (as specified by the multiprocess test + # runner) OR we've been told to skip using the multiprocess + # test runner + return not (is_inferior_test_runner or no_multiprocess_test_runner) + # On MacOS X, check to make sure that domain for com.apple.DebugSymbols defaults # does not exist before proceeding to running the test suite. if sys.platform.startswith("darwin"): @@ -1239,6 +1272,14 @@ # then, we walk the directory trees and collect the tests into our test suite. # parseOptionsAndInitTestdirs() + +# If we are running as the multiprocess test runner, kick off the +# multiprocess test runner here. +if isMultiprocessTestRunner(): + import dosep + dosep.main(output_on_success, num_threads, multiprocess_test_subdir) + raise "should never get here" + setupSysPath() setupCrashInfoHook() Index: test/dotest_args.py =================================================================== --- test/dotest_args.py +++ test/dotest_args.py @@ -107,6 +107,33 @@ group.set_defaults(disable_crash_dialog=True) group.set_defaults(hide_inferior_console=True) + group = parser.add_argument_group('Parallel execution options') + group.add_argument( + '--inferior', + action='store_true', + help=('specify this invocation is a multiprocess inferior, ' + 'used internally')) + group.add_argument( + '--no-multiprocess', + action='store_true', + help='skip running the multiprocess test runner') + group.add_argument( + '--output-on-success', + action='store_true', + help=('print full output of the dotest.py inferior, ' + 'even when all tests succeed')) + group.add_argument( + '--threads', + type=int, + dest='num_threads', + help=('The number of threads/processes to use when running tests ' + 'separately, defaults to the number of CPU cores available')) + parser.add_argument( + '--test-subdir', + action='store', + help='Specify a test subdirectory to use relative to the test root dir' + ) + # Remove the reference to our helper function del X Index: www/test.html =================================================================== --- www/test.html +++ www/test.html @@ -67,19 +67,30 @@

- Besides dotest.py, there is also dosep.py, which runs - multiple instances of dotest.py in parallel, thereby greatly - decreasing the time it takes to run the full testsuite. The number of concurrent - tests is controlled by the LLDB_TEST_THREADS environment variable or - the --threads command line parameter. The default value is the number - of CPUs on your system. To pass additional options to dotest.py, - specify those options as an -o argument to dosep.py. For - example, the command + The dotest.py script runs tests in parallel by default. + To disable the parallel test running feature, use the + --no-multiprocess flag. The number of + concurrent tests is controlled by + the LLDB_TEST_THREADS environment variable + or the --threads command line parameter. + The default value is the number of CPU cores on your + system.

- python dosep.py -o "--executable bin/lldb -C bin/clang"

- will specify the lldb and clang executables to test for each dotest invocation. - ninja check-lldb is wrapper around dosep.py. + The parallel test running feature will handle an + additional --test-subdir SUBDIR arg. When + specified, SUBDIR is relative to the root test directory + and will limit all parallel test running to that + sudirectory's tree of tests. +

+

+ The parallel test runner will run all tests within a + given directory serially, but will run multiple + directories concurrently. Thus, as a test writer, we + provide serialized test run semantics within a + directory. Note child directories are considered + entirely separate, so two child directories could be + running in parallel with a parent directory.

Running the test-suite remotely