[LIT] add a -vv option to echo all executed commands

[george.karpenkov]

Summary

Debugging LIT scripts can be rather painful, as LIT directly does not specify which line has failed.
Rather, FileCheck is expected to report the failing location, but it can be often ambiguous if multiple commands are tested against the same prefix.
This change adds a -vv option, which echoes all output.
Then detecting the error becomes straightforward: last printed line is the failing one.

Of course, it could be desired to try to get failing line number directly from bash, but it involves excessive hacks on older bash versions (cf. https://stackoverflow.com/questions/24398691/how-to-get-the-real-line-number-of-a-failing-bash-command)

Furthermore, it would be good to support this option in the LIT's own shell, but I am not sure how could that easily be done.

[• ddunbar]

LGTM, with one inline request, thanks!

utils/lit/lit/main.py

207 ↗

(On Diff #106318)

Please make -vv imply -v, since this is generally how such options work.

[george.karpenkov]

utils/lit/lit/main.py

207 ↗

(On Diff #106318)

But it does, see two lines below:

if opts.echoAllCommands:
        opts.showOutput = True

llvm/trunk/docs/CommandGuide/lit.rst

 
  Show less output, for example don't show information on tests that pass.
 
 .. option:: -v, --verbose
 
  Show more information on test failures, for example the entire test output
  instead of just the test result.
 
+.. option:: -vv, --echo-all-commands
+
+ Echo all commands to stdout, as they are being executed.
+ This can be valuable for debugging test failures, as the last echoed command
+ will be the one which has failed.
+ This option implies ``--verbose``.
+
 .. option:: -a, --show-all
 
  Show more information about all tests, for example the entire test
  commandline and output.
 
 .. option:: --no-progress-bar
 
  Do not use curses based progress bar.

llvm/trunk/utils/lit/lit/LitConfig.py

class LitConfig(object):
     """
 
     def __init__(self, progname, path, quiet,
                  useValgrind, valgrindLeakCheck, valgrindArgs,
                  noExecute, debug, isWindows,
                  params, config_prefix = None,
                  maxIndividualTestTime = 0,
                  maxFailures = None,
-                 parallelism_groups = []):
+                 parallelism_groups = [],
+                 echo_all_commands = False):
         # The name of the test runner.
         self.progname = progname
         # The items to add to the PATH environment variable.
         self.path = [str(p) for p in path]
         self.quiet = bool(quiet)
         self.useValgrind = bool(useValgrind)
         self.valgrindLeakCheck = bool(valgrindLeakCheck)
         self.valgrindUserArgs = list(valgrindArgs)
def __init__(self, progname, path, quiet,
             else:
                 # The default is 'summary'.
                 self.valgrindArgs.append('--leak-check=no')
             self.valgrindArgs.extend(self.valgrindUserArgs)
 
         self.maxIndividualTestTime = maxIndividualTestTime
         self.maxFailures = maxFailures
         self.parallelism_groups = parallelism_groups
+        self.echo_all_commands = echo_all_commands
 
     @property
     def maxIndividualTestTime(self):
         """
             Interface for getting maximum time to spend executing
             a single test
         """
         return self._maxIndividualTestTime

llvm/trunk/utils/lit/lit/TestRunner.py

def executeScript(test, litConfig, tmpBase, commands, cwd):
     if litConfig.isWindows and not isWin32CMDEXE:
       mode += 'b'  # Avoid CRLFs when writing bash scripts.
     f = open(script, mode)
     if isWin32CMDEXE:
         f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
     else:
         if test.config.pipefail:
             f.write('set -o pipefail;')
+        if litConfig.echo_all_commands:
+            f.write('set -x;')
         f.write('{ ' + '; } &&\n{ '.join(commands) + '; }')
     f.write('\n')
     f.close()
 
     if isWin32CMDEXE:
         command = ['cmd','/c', script]
     else:
         if bashPath:

llvm/trunk/utils/lit/lit/main.py

format_group.add_argument("-q", "--quiet",
                      help="Suppress no error output",
                      action="store_true", default=False)
     format_group.add_argument("-s", "--succinct",
                      help="Reduce amount of output",
                      action="store_true", default=False)
     format_group.add_argument("-v", "--verbose", dest="showOutput",
                      help="Show test output for failures",
                      action="store_true", default=False)
+    format_group.add_argument("-vv", "--echo-all-commands",
+                     dest="echoAllCommands",
+                     action="store_true", default=False,
+                     help="Echo all commands as they are executed to stdout.\
+                     In case of failure, last command shown will be the\
+                     failing one.")
     format_group.add_argument("-a", "--show-all", dest="showAllOutput",
                      help="Display all commandlines and output",
                      action="store_true", default=False)
     format_group.add_argument("-o", "--output", dest="output_path",
                      help="Write test results to the provided path",
                      action="store", metavar="PATH")
     format_group.add_argument("--no-progress-bar", dest="useProgressBar",
                      help="Do not use curses based progress bar",
if not args:
         parser.error('No inputs specified')
 
     if opts.numThreads is None:
         opts.numThreads = lit.util.detectCPUs()
 
     if opts.maxFailures == 0:
         parser.error("Setting --max-failures to 0 does not have any effect.")
 
+    if opts.echoAllCommands:
+        opts.showOutput = True
+
     inputs = args
 
     # Create the user defined parameters.
     userParams = dict(builtinParameters)
     for entry in opts.userParameters:
         if '=' not in entry:
             name,val = entry,''
         else:
litConfig = lit.LitConfig.LitConfig(
         valgrindArgs = opts.valgrindArgs,
         noExecute = opts.noExecute,
         debug = opts.debug,
         isWindows = isWindows,
         params = userParams,
         config_prefix = opts.configPrefix,
         maxIndividualTestTime = maxIndividualTestTime,
         maxFailures = opts.maxFailures,
-        parallelism_groups = {})
+        parallelism_groups = {},
+        echo_all_commands = opts.echoAllCommands)
 
     # Perform test discovery.
     run = lit.run.Run(litConfig,
                       lit.discovery.find_tests_for_inputs(litConfig, inputs))
 
     # After test discovery the configuration might have changed
     # the maxIndividualTestTime. If we explicitly set this on the
     # command line then override what was set in the test configuration