Always use the multiprocess module

[• rafael]

Summary

This seems to work on freebsd these days. If someone could test on openbsd that would be awesome.

[chapuni]

llvm/utils/lit/lit/main.py

289 ↗

(On Diff #96974)

Could you leave them as obsolete and notify to a user, not just pruning?
I think they may be pruned after the next release.

[rnk]

The only complain I've heard about the new execution model is that if you quickly interrupt lit during Pool initialization you can get Python traceback spew from every worker. While the old code behaved slightly better in this area, fixing this seems separable from removing the old codepath.

Any idea who can test OpenBSD?

llvm/utils/lit/lit/run.py

240 ↗

(On Diff #96974)

You can get rid of this call and move the mp.Pool code into this method

243 ↗

(On Diff #96974)

This method should also be dead now.

[• rafael]

Any idea who can test OpenBSD?

Mark, can you give it a try?

Cheers,
Rafael

[kettenis]

In D31620#742592, @rafael wrote:

Any idea who can test OpenBSD?

Mark, can you give it a try?

Cheers,

Sure. Just running the "make lld/test" would be enough?

[kettenis]

In D31620#742606, @kettenis wrote:

In D31620#742592, @rafael wrote:

Any idea who can test OpenBSD?

Mark, can you give it a try?

Cheers,

Sure. Just running the "make lld/test" would be enough?

"make lld-test" works fine; trying "make check-all" now, but I'm not sure lldb is in a testable state on OpenBSD at this moment.

[• rafael]

Delete more dead code.

[• rafael]

r303280.

utils/lit/lit/main.py

debug_group.add_argument("--debug",
                       help="Enable debugging (for 'lit' development)",
                       action="store_true", default=False)
     debug_group.add_argument("--show-suites", dest="showSuites",
                       help="Show discovered test suites",
                       action="store_true", default=False)
     debug_group.add_argument("--show-tests", dest="showTests",
                       help="Show all discovered tests",
                       action="store_true", default=False)
-    debug_group.add_argument("--use-process-pool", dest="executionStrategy",
-                      help="Run tests in parallel with a process pool",
-                      action="store_const", const="PROCESS_POOL")
-    debug_group.add_argument("--use-processes", dest="executionStrategy",
-                      help="Run tests in parallel with processes (not threads)",
-                      action="store_const", const="PROCESSES")
-    debug_group.add_argument("--use-threads", dest="executionStrategy",
-                      help="Run tests in parallel with threads (not processes)",
-                      action="store_const", const="THREADS")
 
     opts = parser.parse_args()
     args = opts.test_paths
 
     if opts.show_version:
         print("lit %s" % (lit.__version__,))
         return
 
     if not args:
         parser.error('No inputs specified')
 
     if opts.numThreads is None:
         opts.numThreads = lit.util.detectCPUs()
 
-    if opts.executionStrategy is None:
-        opts.executionStrategy = 'PROCESS_POOL'
-
     if opts.maxFailures == 0:
         parser.error("Setting --max-failures to 0 does not have any effect.")
 
     inputs = args
 
     # Create the user defined parameters.
     userParams = dict(builtinParameters)
     for entry in opts.userParameters:
if not opts.quiet:
                 print(header)
                 progressBar = lit.ProgressBar.SimpleProgressBar('Testing: ')
         else:
             print(header)
 
     startTime = time.time()
     display = TestingProgressDisplay(opts, len(run.tests), progressBar)
     try:
-        run.execute_tests(display, opts.numThreads, opts.maxTime,
-                          opts.executionStrategy)
+        run.execute_tests(display, opts.numThreads, opts.maxTime)
     except KeyboardInterrupt:
         sys.exit(2)
     display.finish()
 
     testing_time = time.time() - startTime
     if not opts.quiet:
         print('Testing Time: %.2fs' % (testing_time,))
 

utils/lit/lit/run.py

 import os
 import sys
 import threading
 import time
 import traceback
 try:
     import Queue as queue
 except ImportError:
     import queue
 
 try:
     import win32api
 except ImportError:
     win32api = None
 
-try:
-    import multiprocessing
-except ImportError:
-    multiprocessing = None
-
 import lit.Test
 
 ###
 # Test Execution Implementation
 
 class LockedValue(object):
     def __init__(self, value):
         self.lock = threading.Lock()
class Run(object):
 
     def __init__(self, lit_config, tests):
         self.lit_config = lit_config
         self.tests = tests
 
     def execute_test(self, test):
         return execute_test(test, self.lit_config, self.parallelism_semaphores)
 
-    def execute_tests(self, display, jobs, max_time=None,
-                      execution_strategy=None):
+    def execute_tests(self, display, jobs, max_time=None):
         """
         execute_tests(display, jobs, [max_time])
 
         Execute each of the tests in the run, using up to jobs number of
         parallel tasks, and inform the display of each individual result. The
         provided tests should be a subset of the tests available in this run
         object.
 
         If max_time is non-None, it should be a time in seconds after which to
         stop executing tests.
 
         The display object will have its update method called with each test as
         it is completed. The calls are guaranteed to be locked with respect to
         one another, but are *not* guaranteed to be called on the same thread as
         this method was invoked on.
 
         Upon completion, each test in the run will have its result
         computed. Tests which were not actually executed (for any reason) will
         be given an UNRESOLVED result.
         """
-
-        if execution_strategy == 'PROCESS_POOL':
-            self.execute_tests_with_mp_pool(display, jobs, max_time)
-            return
-        # FIXME: Standardize on the PROCESS_POOL execution strategy and remove
-        # the other two strategies.
-
-        use_processes = execution_strategy == 'PROCESSES'
-
-        # Choose the appropriate parallel execution implementation.
-        consumer = None
-        if jobs != 1 and use_processes and multiprocessing:
-            try:
-                task_impl = multiprocessing.Process
-                queue_impl = multiprocessing.Queue
-                sem_impl = multiprocessing.Semaphore
-                canceled_flag =  multiprocessing.Value('i', 0)
-                consumer = MultiprocessResultsConsumer(self, display, jobs)
-            except:
-                # multiprocessing fails to initialize with certain OpenBSD and
-                # FreeBSD Python versions: http://bugs.python.org/issue3770
-                # Unfortunately the error raised also varies by platform.
-                self.lit_config.note('failed to initialize multiprocessing')
-                consumer = None
-        if not consumer:
-            task_impl = threading.Thread
-            queue_impl = queue.Queue
-            sem_impl = threading.Semaphore
-            canceled_flag = LockedValue(0)
-            consumer = ThreadResultsConsumer(display)
-
-        self.parallelism_semaphores = {k: sem_impl(v)
-            for k, v in self.lit_config.parallelism_groups.items()}
-
-        # Create the test provider.
-        provider = TestProvider(queue_impl, canceled_flag)
-        handleFailures(provider, consumer, self.lit_config.maxFailures)
-
-        # Putting tasks into the threading or multiprocessing Queue may block,
-        # so do it in a separate thread.
-        # https://docs.python.org/2/library/multiprocessing.html
-        # e.g: On Mac OS X, we will hang if we put 2^15 elements in the queue
-        # without taking any out.
-        queuer = task_impl(target=provider.queue_tests, args=(self.tests, jobs))
-        queuer.start()
-
-        # Install a console-control signal handler on Windows.
-        if win32api is not None:
-            def console_ctrl_handler(type):
-                provider.cancel()
-                return True
-            win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)
-
-        # Install a timeout handler, if requested.
-        if max_time is not None:
-            def timeout_handler():
-                provider.cancel()
-            timeout_timer = threading.Timer(max_time, timeout_handler)
-            timeout_timer.start()
-
-        # If not using multiple tasks, just run the tests directly.
-        if jobs == 1:
-            run_one_tester(self, provider, consumer)
-        else:
-            # Otherwise, execute the tests in parallel
-            self._execute_tests_in_parallel(task_impl, provider, consumer, jobs)
-
-        queuer.join()
-
-        # Cancel the timeout handler.
-        if max_time is not None:
-            timeout_timer.cancel()
-
-        # Update results for any tests which weren't run.
-        for test in self.tests:
-            if test.result is None:
-                test.setResult(lit.Test.Result(lit.Test.UNRESOLVED, '', 0.0))
-
-    def _execute_tests_in_parallel(self, task_impl, provider, consumer, jobs):
-        # Start all of the tasks.
-        tasks = [task_impl(target=run_one_tester,
-                           args=(self, provider, consumer))
-                 for i in range(jobs)]
-        for t in tasks:
-            t.start()
-
-        # Allow the consumer to handle results, if necessary.
-        consumer.handle_results()
-
-        # Wait for all the tasks to complete.
-        for t in tasks:
-            t.join()
-
-    def execute_tests_with_mp_pool(self, display, jobs, max_time=None):
         # Don't do anything if we aren't going to run any tests.
         if not self.tests or jobs == 0:
             return
 
         # Set up semaphores to limit parallelism of certain classes of tests.
         # For example, some ASan tests require lots of virtual memory and run
         # faster with less parallelism on OS X.
         self.parallelism_semaphores = \