diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py --- a/llvm/utils/lit/lit/cl_arguments.py +++ b/llvm/utils/lit/lit/cl_arguments.py @@ -74,6 +74,11 @@ dest="useProgressBar", help="Do not use curses based progress bar", action="store_false") + format_group.add_argument("--display-format", + dest="displayFormat", + help="How to display test progress and status", + choices=("lit", "tap"), + default="lit") # Note: this does not generate flags for user-defined result codes. success_codes = [c for c in lit.Test.ResultCode.all_codes() diff --git a/llvm/utils/lit/lit/display.py b/llvm/utils/lit/lit/display.py --- a/llvm/utils/lit/lit/display.py +++ b/llvm/utils/lit/lit/display.py @@ -18,6 +18,10 @@ except ValueError: progress_bar = lit.ProgressBar.SimpleProgressBar('Testing: ') + if opts.displayFormat == 'tap': + return TAPDisplay(opts, tests, total_tests, progress_bar) + assert opts.displayFormat == 'lit', \ + 'lit and tap are the only supported display formats' return Display(opts, tests, header, progress_bar) @@ -62,6 +66,50 @@ if self.progress_bar: self.progress_bar.clear(interrupted) + def print_micro_results(self, test, file=sys.stdout, prefix=''): + items = sorted(test.result.microResults.items()) + for micro_test_name, micro_test in items: + print("%s%s MICRO-TEST: %s" % (prefix, '*' * 3, micro_test_name)) + + if not micro_test.metrics: + continue + + sorted_metrics = sorted(micro_test.metrics.items()) + for metric_name, value in sorted_metrics: + print('%s %s: %s ' % (prefix, metric_name, value.format())) + + def print_result_metrics(self, test, file=sys.stdout, prefix=''): + print("%s%s TEST '%s' RESULTS %s" % ( + prefix, '*' * 10, test.getFullName(), '*' * 10), file=file) + items = sorted(test.result.metrics.items()) + for metric_name, value in items: + print('%s%s: %s ' % (prefix, metric_name, value.format()), file=file) + print("%s%s" % (prefix, "*" * 10), file=file) + + def print_verbose_output(self, test, file=sys.stdout, prefix=''): + if test.isFailure(): + print("%s%s TEST '%s' FAILED %s" % ( + prefix, '*' * 20, test.getFullName(), '*' * 20), file=file) + out = test.result.output + # Encode/decode so that, when using Python 3.6.5 in Windows 10, + # print(out) doesn't raise UnicodeEncodeError if out contains + # special characters. However, Python 2 might try to decode + # as part of the encode call if out is already encoded, so skip + # encoding if it raises UnicodeDecodeError. + if file.encoding: + try: + out = out.encode(encoding=file.encoding, errors="replace") + except UnicodeDecodeError: + pass + # Python 2 can raise UnicodeDecodeError here too in cases + # where the stdout encoding is ASCII. Ignore decode errors + # in this case. + out = out.decode(encoding=file.encoding, errors="ignore") + if prefix: + out = ''.join(map(lambda ln: prefix + ln + '\n', out.splitlines())) + print(out, file=file) + print("%s%s" % (prefix, "*" * 20), file=file) + def print_result(self, test): # Show the test result line. test_name = test.getFullName() @@ -70,49 +118,62 @@ # Show the test failure output, if requested. if (test.isFailure() and self.opts.showOutput) or \ - self.opts.showAllOutput: - if test.isFailure(): - print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(), - '*'*20)) - out = test.result.output - # Encode/decode so that, when using Python 3.6.5 in Windows 10, - # print(out) doesn't raise UnicodeEncodeError if out contains - # special characters. However, Python 2 might try to decode - # as part of the encode call if out is already encoded, so skip - # encoding if it raises UnicodeDecodeError. - if sys.stdout.encoding: - try: - out = out.encode(encoding=sys.stdout.encoding, - errors="replace") - except UnicodeDecodeError: - pass - # Python 2 can raise UnicodeDecodeError here too in cases - # where the stdout encoding is ASCII. Ignore decode errors - # in this case. - out = out.decode(encoding=sys.stdout.encoding, errors="ignore") - print(out) - print("*" * 20) + self.opts.showAllOutput: + self.print_verbose_output(test) + + # Report test metrics, if present. + if test.result.metrics: + self.print_result_metrics(test) + + # Report micro-tests, if present + if test.result.microResults: + self.print_micro_results(test) + + # Ensure the output is flushed. + sys.stdout.flush() + + +class TAPDisplay(Display): + """ + Test Anything Protocol output formatter. This display format prints a header + singled header line, followed by one line per test to stdout, following the + specification at http://testanything.org/ + """ + def __init__(self, opts, tests, total_tests, progress_bar): + super().__init__(opts, tests, '1..%d' % total_tests, progress_bar) + + def print_result(self, test): + # Show the test result line. + test_name = test.getFullName() + num = self.completed + assert '#' not in test_name, "Test names cannot contain a # character" + assert not test_name.strip()[0].isdigit(), \ + "Test names cannot start with a digit" + directive = '' + success = 'ok' + description = test_name + if test.isExpectedToFail(): + directive = ' # TODO' + if test.result.code.name in ('UNSUPPORTED', 'UNRESOLVED'): + directive = ' # SKIP' + + if test.isFailure() and not test.isExpectedToFail(): + success = 'not ok' + + print('%s %d - %s%s' % (success, num, description, directive)) + + # Show the test failure output, if requested. + if (test.isFailure() and self.opts.showOutput) or \ + self.opts.showAllOutput: + self.print_verbose_output(test, prefix='# ') # Report test metrics, if present. if test.result.metrics: - print("%s TEST '%s' RESULTS %s" % ('*'*10, test.getFullName(), - '*'*10)) - items = sorted(test.result.metrics.items()) - for metric_name, value in items: - print('%s: %s ' % (metric_name, value.format())) - print("*" * 10) + self.print_result_metrics(test, prefix='# ') # Report micro-tests, if present if test.result.microResults: - items = sorted(test.result.microResults.items()) - for micro_test_name, micro_test in items: - print("%s MICRO-TEST: %s" % - ('*'*3, micro_test_name)) - - if micro_test.metrics: - sorted_metrics = sorted(micro_test.metrics.items()) - for metric_name, value in sorted_metrics: - print(' %s: %s ' % (metric_name, value.format())) + self.print_micro_results(test, prefix='# ') # Ensure the output is flushed. sys.stdout.flush() diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py --- a/llvm/utils/lit/lit/main.py +++ b/llvm/utils/lit/lit/main.py @@ -104,10 +104,11 @@ if opts.time_tests: print_histogram(discovered_tests) - print_results(discovered_tests, elapsed, opts) + if opts.displayFormat == 'lit': + print_results(discovered_tests, elapsed, opts) for report in opts.reports: - report.write_results(tests_for_report, elapsed) + report.write_results(discovered_tests, elapsed) if lit_config.numErrors: sys.stderr.write('\n%d error(s) in tests\n' % lit_config.numErrors) diff --git a/llvm/utils/lit/tests/test-output-tap.py b/llvm/utils/lit/tests/test-output-tap.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/test-output-tap.py @@ -0,0 +1,8 @@ +# RUN: %{lit} -j 1 %{inputs}/show-result-codes --display-format=tap > %t.out || : +# RUN: FileCheck --dump-input=fail < %t.out %s + +# CHECK: 1..4 +# CHECK-NEXT: not ok 1 - show-result-codes :: fail.txt +# CHECK-NEXT: ok 2 - show-result-codes :: pass.txt +# CHECK-NEXT: ok 3 - show-result-codes :: unsupported.txt # SKIP +# CHECK-NEXT: ok 4 - show-result-codes :: xfail.txt # TODO