[lit] Add a TAP-formatted output mode to lit

[ldrumm]

Summary

The Test Anything Protocol is a protocol to
allow test suites to communicate with a test harness. This allows lit
testsuites and the individual tests therein to be understood by
harnesses that understand the TAP protocol.

This adds the new command-line flag --display-format which accepts a
the named display format, defaulting to the informally specified
format currently displayed by lit.

[ldrumm]

Ping

[jhenderson]

It seems like this new feature should be discussed on llvm-dev via an RFC first, as there is probably a wider discussion to be had about customising output format.

llvm/utils/lit/lit/cl_arguments.py

def parse_args():
     format_group.add_argument("-o", "--output",
             type=lit.reports.JsonReport,
             help="Write test results to the provided path",
             metavar="PATH")
     format_group.add_argument("--no-progress-bar",
             dest="useProgressBar",
             help="Do not use curses based progress bar",
             action="store_false")
+    format_group.add_argument("--display-format",
+            dest="displayFormat",
+            help="How to display test progress and status",
+            choices=("lit", "tap"),
+            default="lit")
 
     # Note: this does not generate flags for user-defined result codes.
     success_codes = [c for c in lit.Test.ResultCode.all_codes()
                      if not c.isFailure]
     for code in success_codes:
         format_group.add_argument(
             "--show-{}".format(code.name.lower()),
             dest="shown_codes",

llvm/utils/lit/lit/display.py

if opts.succinct and opts.useProgressBar:
         import lit.ProgressBar
         try:
             tc = lit.ProgressBar.TerminalController()
             progress_bar = lit.ProgressBar.ProgressBar(tc, header)
             header = None
         except ValueError:
             progress_bar = lit.ProgressBar.SimpleProgressBar('Testing: ')
 
+    if opts.displayFormat == 'tap':
+        return TAPDisplay(opts, tests, total_tests, progress_bar)
+    assert opts.displayFormat == 'lit', \
+        'lit and tap are the only supported display formats'
     return Display(opts, tests, header, progress_bar)
 
 
 class NopDisplay(object):
     def print_header(self): pass
     def update(self, test): pass
     def clear(self, interrupted): pass
 
def update(self, test):
                 self.progress_bar.barColor = 'RED'
             percent = float(self.completed) / self.tests
             self.progress_bar.update(percent, test.getFullName())
 
     def clear(self, interrupted):
         if self.progress_bar:
             self.progress_bar.clear(interrupted)
 
-    def print_result(self, test):
-        # Show the test result line.
-        test_name = test.getFullName()
-        print('%s: %s (%d of %d)' % (test.result.code.name, test_name,
-                                     self.completed, self.tests))
+    def print_micro_results(self, test, file=sys.stdout, prefix=''):
+        items = sorted(test.result.microResults.items())
+        for micro_test_name, micro_test in items:
+            print("%s%s MICRO-TEST: %s" % (prefix, '*' * 3, micro_test_name))
 
-        # Show the test failure output, if requested.
-        if (test.isFailure() and self.opts.showOutput) or \
-           self.opts.showAllOutput:
-            if test.isFailure():
-                print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
-                                                  '*'*20))
-            out = test.result.output
-            # Encode/decode so that, when using Python 3.6.5 in Windows 10,
-            # print(out) doesn't raise UnicodeEncodeError if out contains
-            # special characters.  However, Python 2 might try to decode
-            # as part of the encode call if out is already encoded, so skip
-            # encoding if it raises UnicodeDecodeError.
-            if sys.stdout.encoding:
-                try:
-                    out = out.encode(encoding=sys.stdout.encoding,
-                                     errors="replace")
-                except UnicodeDecodeError:
-                    pass
-                # Python 2 can raise UnicodeDecodeError here too in cases
-                # where the stdout encoding is ASCII. Ignore decode errors
-                # in this case.
-                out = out.decode(encoding=sys.stdout.encoding, errors="ignore")
-            print(out)
-            print("*" * 20)
+            if not micro_test.metrics:
+                continue
+
+            sorted_metrics = sorted(micro_test.metrics.items())
+            for metric_name, value in sorted_metrics:
+                print('%s    %s:  %s ' % (prefix, metric_name, value.format()))
+
+    def print_result_metrics(self, test, file=sys.stdout, prefix=''):
+        print("%s%s TEST '%s' RESULTS %s" % (
+            prefix, '*' * 10, test.getFullName(), '*' * 10), file=file)
+        items = sorted(test.result.metrics.items())
+        for metric_name, value in items:
+            print('%s%s: %s ' % (prefix, metric_name, value.format()), file=file)
+        print("%s%s" % (prefix, "*" * 10), file=file)
+
+    def print_verbose_output(self, test, file=sys.stdout, prefix=''):
+            print("%s%s TEST '%s' FAILED %s" % (
+                prefix, '*' * 20, test.getFullName(), '*' * 20), file=file)
+        if file.encoding:
+                out = out.encode(encoding=file.encoding, errors="replace")
+            out = out.decode(encoding=file.encoding, errors="ignore")
+        if prefix:
+            out = ''.join(map(lambda ln: prefix + ln + '\n', out.splitlines()))
+        print(out, file=file)
+        print("%s%s" % (prefix, "*" * 20), file=file)
+
+    def print_result(self, test):
+        # Show the test result line.
+        test_name = test.getFullName()
+        print('%s: %s (%d of %d)' % (test.result.code.name, test_name,
+                                     self.completed, self.tests))
+
+        # Show the test failure output, if requested.
+        if (test.isFailure() and self.opts.showOutput) or \
+                self.opts.showAllOutput:
+            self.print_verbose_output(test)
 
         # Report test metrics, if present.
         if test.result.metrics:
-            print("%s TEST '%s' RESULTS %s" % ('*'*10, test.getFullName(),
-                                               '*'*10))
-            items = sorted(test.result.metrics.items())
-            for metric_name, value in items:
-                print('%s: %s ' % (metric_name, value.format()))
-            print("*" * 10)
+            self.print_result_metrics(test)
 
         # Report micro-tests, if present
         if test.result.microResults:
-            items = sorted(test.result.microResults.items())
-            for micro_test_name, micro_test in items:
-                print("%s MICRO-TEST: %s" %
-                         ('*'*3, micro_test_name))
+            self.print_micro_results(test)
 
-                if micro_test.metrics:
-                    sorted_metrics = sorted(micro_test.metrics.items())
-                    for metric_name, value in sorted_metrics:
-                        print('    %s:  %s ' % (metric_name, value.format()))
+        # Ensure the output is flushed.
+        sys.stdout.flush()
+
+
+class TAPDisplay(Display):
+    """
+    Test Anything Protocol output formatter. This display format prints a header
+    singled header line, followed by one line per test to stdout, following the
+    specification at http://testanything.org/
+    """
+    def __init__(self, opts, tests, total_tests, progress_bar):
+        super().__init__(opts, tests, '1..%d' % total_tests, progress_bar)
+
+    def print_result(self, test):
+        # Show the test result line.
+        test_name = test.getFullName()
+        num = self.completed
+        assert '#' not in test_name, "Test names cannot contain a # character"
+        assert not test_name.strip()[0].isdigit(), \
+                "Test names cannot start with a digit"
+        directive = ''
+        success = 'ok'
+        description = test_name
+        if test.isExpectedToFail():
+            directive = ' # TODO'
+        if test.result.code.name in ('UNSUPPORTED', 'UNRESOLVED'):
+            directive = ' # SKIP'
+
+        if test.isFailure() and not test.isExpectedToFail():
+            success = 'not ok'
+
+        print('%s %d - %s%s' % (success, num, description, directive))
+
+        # Show the test failure output, if requested.
+        if (test.isFailure() and self.opts.showOutput) or \
+                    self.opts.showAllOutput:
+            self.print_verbose_output(test, prefix='# ')
+
+        # Report test metrics, if present.
+        if test.result.metrics:
+            self.print_result_metrics(test, prefix='# ')
+
+        # Report micro-tests, if present
+        if test.result.microResults:
+            self.print_micro_results(test, prefix='# ')
 
         # Ensure the output is flushed.
         sys.stdout.flush()

llvm/utils/lit/lit/main.py

def main(builtin_params={}):
 
     start = time.time()
     run_tests(selected_tests, lit_config, opts, len(discovered_tests))
     elapsed = time.time() - start
 
     if opts.time_tests:
         print_histogram(discovered_tests)
 
+    if opts.displayFormat == 'lit':
         print_results(discovered_tests, elapsed, opts)
 
     for report in opts.reports:
-        report.write_results(tests_for_report, elapsed)
+        report.write_results(discovered_tests, elapsed)
 
     if lit_config.numErrors:
         sys.stderr.write('\n%d error(s) in tests\n' % lit_config.numErrors)
         sys.exit(2)
 
     if lit_config.numWarnings:
         sys.stderr.write('\n%d warning(s) in tests\n' % lit_config.numWarnings)
 

llvm/utils/lit/tests/test-output-tap.py

+# RUN: %{lit} -j 1 %{inputs}/show-result-codes --display-format=tap > %t.out || :
+# RUN: FileCheck --dump-input=fail < %t.out %s
+
+# CHECK: 1..4
+# CHECK-NEXT: not ok 1 - show-result-codes :: fail.txt
+# CHECK-NEXT: ok 2 - show-result-codes :: pass.txt
+# CHECK-NEXT: ok 3 - show-result-codes :: unsupported.txt # SKIP
+# CHECK-NEXT: ok 4 - show-result-codes :: xfail.txt # TODO