[lit] Set --single-process for single tests and --threads=1

[yln]

Summary

Automatically upgrade debugging experience (single process, no thread
pool) when:

1. we only run a single test
2. user specifies -j1

Fix --max-failures in single process mode. Option did not have an
effect in single process mode.

Add display feedback for single process mode. Adapted test.

Improve argument checking (require positive integers).

--single-process is now essentially an alias for -j1. Should we
remove it?

[rnk]

Looks good!

--single-process is now essentially an alias for -j1. Should we remove it?

I think so. I'd commit this, and then send a quick follow-up patch to remove --single-process.

llvm/utils/lit/lit/main.py

483

Maybe say "workers" instead of "threads" since lit has been using multiprocessing everywhere for a long time now.

[yln]

Okay, will do.
Thanks for the quick review! :)

[yln]

Follow-up commit for removing the command line option:
https://github.com/llvm/llvm-project/commit/3c76c09ebfa9fd8f55a6faed7792447b81d3b41b

llvm/utils/lit/lit/LitConfig.py

class LitConfig(object):
     The LitConfig object is also used to communicate with client configuration
     files, it is always passed in as the global variable 'lit' so that
     configuration files can access common functionality and internal components
     easily.
     """
 
     def __init__(self, progname, path, quiet,
                  useValgrind, valgrindLeakCheck, valgrindArgs,
-                 noExecute, debug, isWindows, singleProcess,
+                 noExecute, debug, isWindows,
                  params, config_prefix = None,
                  maxIndividualTestTime = 0,
                  maxFailures = None,
                  parallelism_groups = {},
                  echo_all_commands = False):
         # The name of the test runner.
         self.progname = progname
         # The items to add to the PATH environment variable.
         self.path = [str(p) for p in path]
         self.quiet = bool(quiet)
         self.useValgrind = bool(useValgrind)
         self.valgrindLeakCheck = bool(valgrindLeakCheck)
         self.valgrindUserArgs = list(valgrindArgs)
         self.noExecute = noExecute
         self.debug = debug
-        self.singleProcess = singleProcess
         self.isWindows = bool(isWindows)
         self.params = dict(params)
         self.bashPath = None
 
         # Configuration files to look for when discovering test suites.
         self.config_prefix = config_prefix or 'lit'
         self.suffixes = ['cfg.py', 'cfg']
         self.config_names = ['%s.%s' % (self.config_prefix,x) for x in self.suffixes]

llvm/utils/lit/lit/discovery.py

def load_test_suite(inputs):
 
     # Create the global config object.
     litConfig = LitConfig.LitConfig(progname = 'lit',
                                     path = [],
                                     quiet = False,
                                     useValgrind = False,
                                     valgrindLeakCheck = False,
                                     valgrindArgs = [],
-                                    singleProcess=False,
                                     noExecute = False,
                                     debug = False,
                                     isWindows = (platform.system()=='Windows'),
                                     params = {})
 
     # Perform test discovery.
     run = lit.run.Run(litConfig, find_tests_for_inputs(litConfig, inputs))
 
     # Return a unittest test suite which just runs the tests in order.
     return unittest.TestSuite([LitTestCase(test, run)
                                for test in run.tests])

llvm/utils/lit/lit/main.py

def main_with_tmp(builtinParameters):
 
     if opts.show_version:
         print("lit %s" % (lit.__version__,))
         return
 
     if not args:
         parser.error('No inputs specified')
 
-    if opts.numThreads is None:
-        opts.numThreads = lit.util.detectCPUs()
+    if opts.numThreads is not None and opts.numThreads <= 0:
+        parser.error("Option '--threads' or '-j' requires positive integer")
 
-    if opts.maxFailures == 0:
-        parser.error("Setting --max-failures to 0 does not have any effect.")
+    if opts.maxFailures is not None and opts.maxFailures <= 0:
+        parser.error("Option '--max-failures' requires positive integer")
 
     if opts.echoAllCommands:
         opts.showOutput = True
 
     inputs = args
 
     # Create the user defined parameters.
     userParams = dict(builtinParameters)
def main_with_tmp(builtinParameters):
     litConfig = lit.LitConfig.LitConfig(
         progname = os.path.basename(sys.argv[0]),
         path = opts.path,
         quiet = opts.quiet,
         useValgrind = opts.useValgrind,
         valgrindLeakCheck = opts.valgrindLeakCheck,
         valgrindArgs = opts.valgrindArgs,
         noExecute = opts.noExecute,
-        singleProcess = opts.singleProcess,
         debug = opts.debug,
         isWindows = isWindows,
         params = userParams,
         config_prefix = opts.configPrefix,
         maxIndividualTestTime = maxIndividualTestTime,
         maxFailures = opts.maxFailures,
         parallelism_groups = {},
         echo_all_commands = opts.echoAllCommands)
if (opts.numShards is not None) or (opts.runShard is not None):
                        (opts.runShard, opts.numShards,
                         len(run.tests), num_tests,
                         opts.numShards, opts.runShard, ix_preview))
 
     # Finally limit the number of tests, if desired.
     if opts.maxTests is not None:
         run.tests = run.tests[:opts.maxTests]
 
+    # Determine number of threads to use.
+    if opts.singleProcess:
+        opts.numThreads = 1
+    elif opts.numThreads is None:
+        opts.numThreads = lit.util.detectCPUs()
+
     # Don't create more threads than tests.
     opts.numThreads = min(len(run.tests), opts.numThreads)
 
     # Because some tests use threads internally, and at least on Linux each
     # of these threads counts toward the current process limit, try to
     # raise the (soft) process limit so that tests don't fail due to
     # resource exhaustion.
     try:
try:
 
         if max_procs_soft < desired_limit:
             resource.setrlimit(resource.RLIMIT_NPROC, (desired_limit, max_procs_hard))
             litConfig.note('raised the process limit from %d to %d' % \
                                (max_procs_soft, desired_limit))
     except:
         pass
 
-    extra = ''
-    if len(run.tests) != numTotalTests:
-        extra = ' of %d' % numTotalTests
-    header = '-- Testing: %d%s tests, %d threads --'%(len(run.tests), extra,
-                                                      opts.numThreads)
+    extra = (' of %d' % numTotalTests) if (len(run.tests) != numTotalTests) else ''
+    threads = 'single process' if (opts.numThreads == 1) else ('%d threads' % opts.numThreads)
+    header = '-- Testing: %d%s tests, %s --' % (len(run.tests), extra, threads)
     progressBar = None
     if not opts.quiet:
         if opts.succinct and opts.useProgressBar:
             try:
                 tc = lit.ProgressBar.TerminalController()
                 progressBar = lit.ProgressBar.ProgressBar(tc, header)
             except ValueError:
                 print(header)

llvm/utils/lit/lit/run.py

def execute_tests(self, display, jobs, max_time=None):
         one another, but are *not* guaranteed to be called on the same thread as
         this method was invoked on.
 
         Upon completion, each test in the run will have its result
         computed. Tests which were not actually executed (for any reason) will
         be given an UNRESOLVED result.
         """
         # Don't do anything if we aren't going to run any tests.
-        if not self.tests or jobs == 0:
+        if not self.tests:
             return
 
         # Save the display object on the runner so that we can update it from
         # our task completion callback.
         self.display = display
 
         self.failure_count = 0
         self.hit_max_failures = False
-        if self.lit_config.singleProcess:
+        if jobs == 1:
             for test_index, test in enumerate(self.tests):
                 lit.worker._execute_test(test, self.lit_config)
                 self.consume_test_result((test_index, test))
+                if self.hit_max_failures:
+                    break
         else:
             self.execute_tests_in_pool(jobs, max_time)
 
         # Mark any tests that weren't run as UNRESOLVED.
         for test in self.tests:
             if test.result is None:
                 test.setResult(lit.Test.Result(lit.Test.UNRESOLVED, '', 0.0))
 

llvm/utils/lit/lit/worker.py

 # The code in this file runs in a separate worker process.
-# Exception: if lit_config.singleProcess then _execute_test is called directly.
+# Exception: in single process mode _execute_test is called directly.
 import time
 import traceback
 
 import lit.util
 import lit.Test
 
 worker_lit_config = None
 worker_parallelism_semaphores = None

llvm/utils/lit/tests/max-failures.py

 # Check the behavior of --max-failures option.
 #
 # RUN: not %{lit} -j 1 -v %{inputs}/max-failures > %t.out
 # RUN: not %{lit} --max-failures=1 -j 1 -v %{inputs}/max-failures >> %t.out
 # RUN: not %{lit} --max-failures=2 -j 1 -v %{inputs}/max-failures >> %t.out
 # RUN: not %{lit} --max-failures=0 -j 1 -v %{inputs}/max-failures 2>> %t.out
 # RUN: FileCheck < %t.out %s
 #
 # END.
 
 # CHECK: Failing Tests (27)
 # CHECK: Failing Tests (1)
 # CHECK: Failing Tests (2)
-# CHECK: error: Setting --max-failures to 0 does not have any effect.
+# CHECK: error: Option '--max-failures' requires positive integer

llvm/utils/lit/tests/unit/TestRunner.py

def load_keyword_parser_lit_tests():
         TestIntegratedTestKeywordParser
         """
         # Create the global config object.
         lit_config = lit.LitConfig.LitConfig(progname='lit',
                                              path=[],
                                              quiet=False,
                                              useValgrind=False,
                                              valgrindLeakCheck=False,
-                                             singleProcess=False,
                                              valgrindArgs=[],
                                              noExecute=False,
                                              debug=False,
                                              isWindows=(
                                                platform.system() == 'Windows'),
                                              params={})
         TestIntegratedTestKeywordParser.litConfig = lit_config
         # Perform test discovery.