[dotest] Remove -q (quiet) flag.

[JDevlieghere]

Summary

I propose removing the -q (quiet) flag and changing the default behavior. Currently the flag serves two purposes that are somewhat contradictory, as illustrated by the difference between the argument name (quiet) and the configuration flag (parsable). On the one hand it reduces output, but on the other hand it prints more output, like the result of individual tests. My proposal is to guard the extra output behind the verbose flag and always print the individual test results.

[davide]

LGTM

lldb/docs/resources/test.rst

#. Configure the project to use this debug interpreter.
     #. If you want to enabled mixed mode debugging, check Enable native code debugging (this slows down debugging, so enable it only on an as-needed basis.)
 #. Set the command line for the test suite to run.
     #. Right click the project in solution explorer and choose the Debug tab.
     #. Enter the arguments to dotest.py.
     #. Example command options:
 
 ::
 
-   # quiet mode
-   -q
    --arch=i686
    # Path to debug lldb.exe
    --executable D:/src/llvmbuild/ninja/bin/lldb.exe
    # Directory to store log files
    -s D:/src/llvmbuild/ninja/lldb-test-traces
    -u CXXFLAGS -u CFLAGS
    # If a test crashes, show JIT debugging dialog.
    --enable-crash-dialog
    # Path to release clang.exe
    -C d:\src\llvmbuild\ninja_release\bin\clang.exe
    # Path to the particular test you want to debug.
    -p TestPaths.py
    # Root of test tree
    D:\src\llvm\tools\lldb\packages\Python\lldbsuite\test
 
 ::
 
-   -q --arch=i686 --executable D:/src/llvmbuild/ninja/bin/lldb.exe -s D:/src/llvmbuild/ninja/lldb-test-traces -u CXXFLAGS -u CFLAGS --enable-crash-dialog -C d:\src\llvmbuild\ninja_release\bin\clang.exe -p TestPaths.py D:\src\llvm\tools\lldb\packages\Python\lldbsuite\test --no-multiprocess
+   --arch=i686 --executable D:/src/llvmbuild/ninja/bin/lldb.exe -s D:/src/llvmbuild/ninja/lldb-test-traces -u CXXFLAGS -u CFLAGS --enable-crash-dialog -C d:\src\llvmbuild\ninja_release\bin\clang.exe -p TestPaths.py D:\src\llvm\tools\lldb\packages\Python\lldbsuite\test --no-multiprocess
 
 
 

lldb/lit/Suite/lit.cfg

shlibpath = os.path.pathsep.join(
      config.environment.get(shlibpath_var, '')))
   config.environment[shlibpath_var] = shlibpath
   break
 else:
   lit_config.warning("unable to inject shared library path on '{}'"
                      .format(platform.system()))
 
 # Build dotest command.
-dotest_cmd = [config.dotest_path, '-q']
+dotest_cmd = [config.dotest_path]
 dotest_cmd.extend(config.dotest_args_str.split(';'))
 
 # We don't want to force users passing arguments to lit to use `;` as a
 # separator. We use Python's simple lexical analyzer to turn the args into a
 # list.
 if config.dotest_lit_args_str:
   dotest_cmd.extend(shlex.split(config.dotest_lit_args_str))
 

lldb/packages/Python/lldbsuite/test/configuration.py

 # The arch might dictate some specific CFLAGS to be passed to the toolchain to build
 # the inferior programs.  The global variable cflags_extras provides a hook to do
 # just that.
 cflags_extras = ''
 
 # The filters (testclass.testmethod) used to admit tests into our test suite.
 filters = []
 
-# Parsable mode silences headers, and any other output this script might generate, and instead
-# prints machine-readable output similar to what clang tests produce.
-parsable = False
-
 # The regular expression pattern to match against eligible filenames as
 # our test cases.
 regexp = None
 
 # Sets of tests which are excluded at runtime
 skip_tests = None
 xfail_tests = None
 

lldb/packages/Python/lldbsuite/test/dotest.py

if args.set_env_vars:
             if len(parts) == 1:
                 os.environ[parts[0]] = ""
             else:
                 os.environ[parts[0]] = parts[1]
 
     if args.set_inferior_env_vars:
         lldbtest_config.inferior_env = ' '.join(args.set_inferior_env_vars)
 
-    # only print the args if being verbose (and parsable is off)
+    # Only print the args if being verbose.
     if args.v and not args.q:
         print(sys.argv)
 
     if args.h:
         do_help = True
 
     if args.compiler:
         configuration.compiler = os.path.realpath(args.compiler)
if args.excluded:
         for excl_file in args.excluded:
             parseExclusion(excl_file)
 
     if args.p:
         if args.p.startswith('-'):
             usage(parser)
         configuration.regexp = args.p
 
-    if args.q:
-        configuration.parsable = True
-
     if args.s:
         if args.s.startswith('-'):
             usage(parser)
         configuration.sdir_name = args.s
     configuration.session_file_format = args.session_file_format
 
     if args.t:
         os.environ['LLDB_COMMAND_TRACE'] = 'YES'
if not os.path.isdir(configuration.sdir_name):
             if exception.errno != errno.EEXIST:
                 raise
 
     #
     # Invoke the default TextTestRunner to run the test suite
     #
     checkCompiler()
 
-    if not configuration.parsable:
+    if configuration.verbose:
         print("compiler=%s" % configuration.compiler)
 
     # Iterating over all possible architecture and compiler combinations.
     os.environ["ARCH"] = configuration.arch
     os.environ["CC"] = configuration.compiler
     configString = "arch=%s compiler=%s" % (configuration.arch,
                                             configuration.compiler)
 
     # Translate ' ' to '-' for pathname component.
     if six.PY2:
         import string
         tbl = string.maketrans(' ', '-')
     else:
         tbl = str.maketrans(' ', '-')
     configPostfix = configString.translate(tbl)
 
     # Output the configuration.
-    if not configuration.parsable:
+    if configuration.verbose:
         sys.stderr.write("\nConfiguration: " + configString + "\n")
 
     # First, write out the number of collected test cases.
-    if not configuration.parsable:
+    if configuration.verbose:
         sys.stderr.write(configuration.separator + "\n")
         sys.stderr.write(
             "Collected %d test%s\n\n" %
             (configuration.suite.countTestCases(),
              configuration.suite.countTestCases() != 1 and "s" or ""))
 
-    if configuration.parsable:
-        v = 0
-    else:
-        v = configuration.verbose
-
     # Invoke the test runner.
     if configuration.count == 1:
         result = unittest2.TextTestRunner(
             stream=sys.stderr,
-            verbosity=v,
+            verbosity=configuration.verbose,
             resultclass=test_result.LLDBTestResult).run(
             configuration.suite)
     else:
         # We are invoking the same test suite more than once.  In this case,
         # mark __ignore_singleton__ flag as True so the signleton pattern is
         # not enforced.
         test_result.LLDBTestResult.__ignore_singleton__ = True
         for i in range(configuration.count):
 
             result = unittest2.TextTestRunner(
                 stream=sys.stderr,
-                verbosity=v,
+                verbosity=configuration.verbose,
                 resultclass=test_result.LLDBTestResult).run(
                 configuration.suite)
 
     configuration.failed = not result.wasSuccessful()
 
-    if configuration.sdir_has_content and not configuration.parsable:
+    if configuration.sdir_has_content and configuration.verbose:
         sys.stderr.write(
             "Session logs for test failures/errors/unexpected successes"
             " can be found in directory '%s'\n" %
             configuration.sdir_name)
 
     if configuration.useCategories and len(
             configuration.failuresPerCategory) > 0:
         sys.stderr.write("Failures per category:\n")

lldb/packages/Python/lldbsuite/test/dotest_args.py

group.add_argument(
         '--platform-working-dir',
         dest='lldb_platform_working_dir',
         metavar='platform-working-dir',
         help='The directory to use on the remote platform.')
 
     # Test-suite behaviour
     group = parser.add_argument_group('Runtime behaviour options')
     X('-d', 'Suspend the process after launch to wait indefinitely for a debugger to attach')
-    X('-q', "Don't print extra output from this script.")
     X('-t', 'Turn on tracing of lldb command and other detailed test executions')
     group.add_argument(
         '-u',
         dest='unset_env_varnames',
         metavar='variable',
         action='append',
         help='Specify an environment variable to unset before running the test cases. e.g., -u DYLD_INSERT_LIBRARIES -u MallocScribble')
     group.add_argument(

lldb/packages/Python/lldbsuite/test/test_result.py

class LLDBTestResult(unittest2.TextTestResult):
 
     def addSuccess(self, test):
         if self.checkExclusion(
                 configuration.xfail_tests, test.id()):
             self.addUnexpectedSuccess(test, None)
             return
 
         super(LLDBTestResult, self).addSuccess(test)
-        if configuration.parsable:
-            self.stream.write(
-                "PASS: LLDB (%s) :: %s\n" %
-                (self._config_string(test), str(test)))
         if self.results_formatter:
             self.results_formatter.handle_event(
                 EventBuilder.event_for_success(test))
 
     def _isBuildError(self, err_tuple):
         exception = err_tuple[1]
         return isinstance(exception, build_exception.BuildError)
 
def addError(self, test, err):
         if self._isBuildError(err):
             self._saveBuildErrorTuple(test, err)
         else:
             super(LLDBTestResult, self).addError(test, err)
 
         method = getattr(test, "markError", None)
         if method:
             method()
-        if configuration.parsable:
-            self.stream.write(
-                "FAIL: LLDB (%s) :: %s\n" %
-                (self._config_string(test), str(test)))
         if self.results_formatter:
             # Handle build errors as a separate event type
             if self._isBuildError(err):
                 error_event = EventBuilder.event_for_build_error(test, err)
             else:
                 error_event = EventBuilder.event_for_error(test, err)
             self.results_formatter.handle_event(error_event)
 
     def addCleanupError(self, test, err):
         configuration.sdir_has_content = True
         super(LLDBTestResult, self).addCleanupError(test, err)
         method = getattr(test, "markCleanupError", None)
         if method:
             method()
-        if configuration.parsable:
-            self.stream.write(
-                "CLEANUP ERROR: LLDB (%s) :: %s\n" %
-                (self._config_string(test), str(test)))
         if self.results_formatter:
             self.results_formatter.handle_event(
                 EventBuilder.event_for_cleanup_error(
                     test, err))
 
     def addFailure(self, test, err):
         if self.checkExclusion(
                 configuration.xfail_tests, test.id()):
             self.addExpectedFailure(test, err, None)
             return
 
         configuration.sdir_has_content = True
         super(LLDBTestResult, self).addFailure(test, err)
         method = getattr(test, "markFailure", None)
         if method:
             method()
-        if configuration.parsable:
-            self.stream.write(
-                "FAIL: LLDB (%s) :: %s\n" %
-                (self._config_string(test), str(test)))
         if configuration.useCategories:
             test_categories = self.getCategoriesForTest(test)
             for category in test_categories:
                 if category in configuration.failuresPerCategory:
                     configuration.failuresPerCategory[
                         category] = configuration.failuresPerCategory[category] + 1
                 else:
                     configuration.failuresPerCategory[category] = 1
         if self.results_formatter:
             self.results_formatter.handle_event(
                 EventBuilder.event_for_failure(test, err))
 
     def addExpectedFailure(self, test, err, bugnumber):
         configuration.sdir_has_content = True
         super(LLDBTestResult, self).addExpectedFailure(test, err, bugnumber)
         method = getattr(test, "markExpectedFailure", None)
         if method:
             method(err, bugnumber)
-        if configuration.parsable:
-            self.stream.write(
-                "XFAIL: LLDB (%s) :: %s\n" %
-                (self._config_string(test), str(test)))
         if self.results_formatter:
             self.results_formatter.handle_event(
                 EventBuilder.event_for_expected_failure(
                     test, err, bugnumber))
 
     def addSkip(self, test, reason):
         configuration.sdir_has_content = True
         super(LLDBTestResult, self).addSkip(test, reason)
         method = getattr(test, "markSkippedTest", None)
         if method:
             method()
-        if configuration.parsable:
-            self.stream.write(
-                "UNSUPPORTED: LLDB (%s) :: %s (%s) \n" %
-                (self._config_string(test), str(test), reason))
         if self.results_formatter:
             self.results_formatter.handle_event(
                 EventBuilder.event_for_skip(test, reason))
 
     def addUnexpectedSuccess(self, test, bugnumber):
         configuration.sdir_has_content = True
         super(LLDBTestResult, self).addUnexpectedSuccess(test, bugnumber)
         method = getattr(test, "markUnexpectedSuccess", None)
         if method:
             method(bugnumber)
-        if configuration.parsable:
-            self.stream.write(
-                "XPASS: LLDB (%s) :: %s\n" %
-                (self._config_string(test), str(test)))
         if self.results_formatter:
             self.results_formatter.handle_event(
                 EventBuilder.event_for_unexpected_success(
                     test, bugnumber))

lldb/utils/lldb-dotest/lldb-dotest.in

 #!/usr/bin/env python
 import subprocess
 import sys
 
 dotest_path = '@LLDB_SOURCE_DIR@/test/dotest.py'
 dotest_args_str = '@LLDB_DOTEST_ARGS@'
 
 if __name__ == '__main__':
     wrapper_args = sys.argv[1:]
     dotest_args = dotest_args_str.split(';')
     # Build dotest.py command.
-    cmd = [sys.executable, dotest_path, '-q']
+    cmd = [sys.executable, dotest_path]
     cmd.extend(dotest_args)
     cmd.extend(wrapper_args)
     # Invoke dotest.py and return exit code.
     print(' '.join(cmd))
     sys.exit(subprocess.call(cmd))