[lit] Cleanly exit on user keyboard interrupt

[yln]

Summary

Graceful lit shutdown on user keyboard interrupt [Ctrl+C] was a
longstanding goal of mine. After a few refactorings this revision
finally enables it.

Printing of a helpful summary for interrupted runs (just as the one for
completed runs) will be tackled in future revisions.

I tested this on macOS and Linux.

[yln]

I tested this on macOS and Linux.

@rnk @jdenny
Can you try Windows? If it doesn't work we can put back the _install_win32_signal_handler part.

[serge-sans-paille]

@yln: that's a clean and short answer to the solution, the blogpost you're quoting makes everything super clear, thanks! LGTM if it works on Windows.

[jdenny]

I don't regularly work in windows. I'll try to find some time to check it out next week if no one else gets there first.

Thanks for working on this.

[rnk]

Worked well, this was the output I got:

$ ninja -j900 check-llvm
[1 processes, 0/2 @ ?/s : 0.000s ] ACTION //llvm/test:check-llvm(//llvm/utils/gn/build/toolchain:win)
-- Testing: 36824 tests, 32 workers --
Testing:  0
FAIL: LLVM-Unit :: CodeGen/./CodeGenTests.exe/MachineInstrBundleIteratorTest.CheckForBundles (2062 of 36824)
******************** TEST 'LLVM-Unit :: CodeGen/./CodeGenTests.exe/MachineInstrBundleIteratorTest.CheckForBundles' FAILED ********************
Note: Google Test filter = MachineInstrBundleIteratorTest.CheckForBundles
[==========] Running 1 test from 1 test case.
[----------] Global test environment set-up.
[----------] 1 test from MachineInstrBundleIteratorTest
[ RUN      ] MachineInstrBundleIteratorTest.CheckForBundles

********************
Testing:  0  interrupted by user, skipping remaining tests

Testing Time: 5.89s
  Expected Passes    : 1747
ninja: build stopped: interrupted by user.

The second time I tried, my interrupt got delivered to some random clang subprocesses, but after mashing Ctrl C enough it went through... Anyway, the new state is better than the old. Thanks a lot!

llvm/utils/lit/lit/main.py

if 'LIT_PRESERVES_TMP' not in os.environ:
         import tempfile
         tmp_dir = tempfile.mkdtemp(prefix="lit_tmp_")
         os.environ.update({
                 'TMPDIR': tmp_dir,
                 'TMP': tmp_dir,
                 'TEMP': tmp_dir,
                 'TEMPDIR': tmp_dir,
                 })
-    # FIXME: If Python does not exit cleanly, this directory will not be cleaned
-    # up. We should consider writing the lit pid into the temp directory,
-    # scanning for stale temp directories, and deleting temp directories whose
-    # lit process has died.
     try:
         run.execute()
     finally:
         if tmp_dir:
             try:
                 import shutil
                 shutil.rmtree(tmp_dir)
             except:

llvm/utils/lit/lit/run.py

class Run(object):
     def _execute(self, deadline):
         semaphores = {
             k: NopSemaphore() if v is None else
             multiprocessing.BoundedSemaphore(v) for k, v in
             self.lit_config.parallelism_groups.items()}
 
         self._increase_process_limit()
 
-        # Start a process pool. Copy over the data shared between all test runs.
-        # FIXME: Find a way to capture the worker process stderr. If the user
-        # interrupts the workers before we make it into our task callback, they
-        # will each raise a KeyboardInterrupt exception and print to stderr at
-        # the same time.
         pool = multiprocessing.Pool(self.workers, lit.worker.initialize,
                                     (self.lit_config, semaphores))
 
-        self._install_win32_signal_handler(pool)
-
         async_results = [
             pool.apply_async(lit.worker.execute, args=[test],
                              callback=self.progress_callback)
             for test in self.tests]
         pool.close()
 
         try:
             self._wait_for(async_results, deadline)
def _increase_process_limit(self):
             if soft_limit < desired_limit:
                 resource.setrlimit(NPROC, (desired_limit, hard_limit))
                 self.lit_config.note('Raised process limit from %d to %d' % \
                                         (soft_limit, desired_limit))
         except Exception as ex:
             # Warn, unless this is Windows, in which case this is expected.
             if os.name != 'nt':
                 self.lit_config.warning('Failed to raise process limit: %s' % ex)
-
-    def _install_win32_signal_handler(self, pool):
-        if lit.util.win32api is not None:
-            def console_ctrl_handler(type):
-                print('\nCtrl-C detected, terminating.')
-                pool.terminate()
-                pool.join()
-                lit.util.abort_now()
-                return True
-            lit.util.win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)

llvm/utils/lit/lit/util.py

else:
             for child in children_iterator:
                 try:
                     child.kill()
                 except psutil.NoSuchProcess:
                     pass
             psutilProc.kill()
         except psutil.NoSuchProcess:
             pass
-
-
-try:
-    import win32api
-except ImportError:
-    win32api = None
-
-def abort_now():
-    """Abort the current process without doing any exception teardown"""
-    sys.stdout.flush()
-    if win32api:
-        win32api.TerminateProcess(win32api.GetCurrentProcess(), 3)
-    else:
-        os.kill(0, 9)

llvm/utils/lit/lit/worker.py

 """
 The functions in this module are meant to run on a separate worker process.
 Exception: in single process mode _execute is called directly.
 
 For efficiency, we copy all data needed to execute all tests into each worker
 and store it in global variables. This reduces the cost of each task.
 """
+import signal
 import time
 import traceback
 
 import lit.Test
 import lit.util
 
 
 _lit_config = None
 _parallelism_semaphores = None
 
 
 def initialize(lit_config, parallelism_semaphores):
     """Copy data shared by all test executions into worker processes"""
     global _lit_config
     global _parallelism_semaphores
     _lit_config = lit_config
     _parallelism_semaphores = parallelism_semaphores
 
+    # We use the following strategy for dealing with Ctrl+C/KeyboardInterrupt in
+    # subprocesses created by the multiprocessing.Pool.
+    # https://noswap.com/blog/python-multiprocessing-keyboardinterrupt
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+
 
 def execute(test):
     """Run one test in a multiprocessing.Pool
 
     Side effects in this function and functions it calls are not visible in the
     main lit process.
 
     Arguments and results of this function are pickled, so they should be cheap
     to copy.
     """
-    try:
-        result = _execute_in_parallelism_group(test, _lit_config,
-                                               _parallelism_semaphores)
-        test.setResult(result)
-        return test
-    except KeyboardInterrupt:
-        # If a worker process gets an interrupt, abort it immediately.
-        lit.util.abort_now()
-    except:
-        traceback.print_exc()
 
 
 def _execute_in_parallelism_group(test, lit_config, parallelism_semaphores):
     pg = test.config.parallelism_group
     if callable(pg):
         pg = pg(test)
 
     if pg:
def _execute(test, lit_config):
     result.elapsed = time.time() - start
     return result
 
 
 def _execute_test_handle_errors(test, lit_config):
     try:
         result = test.config.test_format.execute(test, lit_config)
         return _adapt_result(result)
-    except KeyboardInterrupt:
-        raise
     except:
         if lit_config.debug:
             raise
         output = 'Exception during script execution:\n'
         output += traceback.format_exc()
         output += '\n'
         return lit.Test.Result(lit.Test.UNRESOLVED, output)