diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py --- a/llvm/utils/lit/lit/cl_arguments.py +++ b/llvm/utils/lit/lit/cl_arguments.py @@ -115,6 +115,9 @@ execution_group.add_argument("--xunit-xml-output", type=lit.reports.XunitReport, help="Write XUnit-compatible XML test reports to the specified file") + execution_group.add_argument("--resultdb-output", + type=lit.reports.ResultDBReport, + help="Write LuCI ResuldDB compatible JSON to the specified file") execution_group.add_argument("--time-trace-output", type=lit.reports.TimeTraceReport, help="Write Chrome tracing compatible JSON to the specified file") @@ -229,7 +232,7 @@ else: opts.shard = None - opts.reports = filter(None, [opts.output, opts.xunit_xml_output, opts.time_trace_output]) + opts.reports = filter(None, [opts.output, opts.xunit_xml_output, opts.resultdb_output, opts.time_trace_output]) return opts diff --git a/llvm/utils/lit/lit/reports.py b/llvm/utils/lit/lit/reports.py --- a/llvm/utils/lit/lit/reports.py +++ b/llvm/utils/lit/lit/reports.py @@ -1,3 +1,5 @@ +import base64 +import datetime import itertools import json @@ -153,6 +155,90 @@ return 'Unsupported configuration' +def gen_resultdb_test_entry( + test_name, start_time, elapsed_time, test_output, result_code, is_expected +): + test_data = { + 'testId': test_name, + 'start_time': datetime.datetime.fromtimestamp(start_time).isoformat() + 'Z', + 'duration': '%.9fs' % elapsed_time, + 'summary_html': '

', + 'artifacts': { + 'artifact-content-in-request': { + 'contents': base64.b64encode(test_output.encode('utf-8')).decode( + 'utf-8' + ), + }, + }, + 'expected': is_expected, + } + if ( + result_code == lit.Test.PASS + or result_code == lit.Test.XPASS + or result_code == lit.Test.FLAKYPASS + ): + test_data['status'] = 'PASS' + elif result_code == lit.Test.FAIL or result_code == lit.Test.XFAIL: + test_data['status'] = 'FAIL' + elif ( + result_code == lit.Test.UNSUPPORTED + or result_code == lit.Test.SKIPPED + or result_code == lit.Test.EXCLUDED + ): + test_data['status'] = 'SKIP' + elif result_code == lit.Test.UNRESOLVED or result_code == lit.Test.TIMEOUT: + test_data['status'] = 'ABORT' + return test_data + + +class ResultDBReport(object): + def __init__(self, output_file): + self.output_file = output_file + + def write_results(self, tests, elapsed): + unexecuted_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED} + tests = [t for t in tests if t.result.code not in unexecuted_codes] + data = {} + data['__version__'] = lit.__versioninfo__ + data['elapsed'] = elapsed + # Encode the tests. + data['tests'] = tests_data = [] + for test in tests: + tests_data.append( + gen_resultdb_test_entry( + test_name=test.getFullName(), + start_time=test.result.start, + elapsed_time=test.result.elapsed, + test_output=test.result.output, + result_code=test.result.code, + is_expected=not test.result.code.isFailure, + ) + ) + if test.result.microResults: + for key, micro_test in test.result.microResults.items(): + # Expand parent test name with micro test name + parent_name = test.getFullName() + micro_full_name = parent_name + ':' + key + 'microres' + tests_data.append( + gen_resultdb_test_entry( + test_name=micro_full_name, + start_time=micro_test.start + if micro_test.start + else test.result.start, + elapsed_time=micro_test.elapsed + if micro_test.elapsed + else test.result.elapsed, + test_output=micro_test.output, + result_code=micro_test.code, + is_expected=not micro_test.code.isFailure, + ) + ) + + with open(self.output_file, 'w') as file: + json.dump(data, file, indent=2, sort_keys=True) + file.write('\n') + + class TimeTraceReport(object): def __init__(self, output_file): self.output_file = output_file diff --git a/llvm/utils/lit/tests/test-output-micro-resultdb.py b/llvm/utils/lit/tests/test-output-micro-resultdb.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/test-output-micro-resultdb.py @@ -0,0 +1,63 @@ +# RUN: %{lit} -j 1 -v %{inputs}/test-data-micro --resultdb-output %t.results.out +# RUN: FileCheck < %t.results.out %s +# RUN: rm %t.results.out + + +# CHECK: { +# CHECK: "__version__" +# CHECK: "elapsed" +# CHECK-NEXT: "tests": [ +# CHECK-NEXT: { +# CHECK-NEXT: "artifacts": { +# CHECK-NEXT: "artifact-content-in-request": { +# CHECK-NEXT: "contents": "VGVzdCBwYXNzZWQu" +# CHECK-NEXT: } +# CHECK-NEXT: }, +# CHECK-NEXT: "duration" +# CHECK-NEXT: "expected": true, +# CHECK-NEXT: "start_time" +# CHECK-NEXT: "status": "PASS", +# CHECK-NEXT: "summary_html": "

", +# CHECK-NEXT: "testId": "test-data-micro :: micro-tests.ini" +# CHECK-NEXT: }, +# CHECK-NEXT: { +# CHECK-NEXT: "artifacts": { +# CHECK-NEXT: "artifact-content-in-request": { +# CHECK-NEXT: "contents": "" +# CHECK-NEXT: } +# CHECK-NEXT: }, +# CHECK-NEXT: "duration" +# CHECK-NEXT: "expected": true, +# CHECK-NEXT: "start_time" +# CHECK-NEXT: "status": "PASS", +# CHECK-NEXT: "summary_html": "

", +# CHECK-NEXT: "testId": "test-data-micro :: micro-tests.ini:test0microres" +# CHECK-NEXT: }, +# CHECK-NEXT: { +# CHECK-NEXT: "artifacts": { +# CHECK-NEXT: "artifact-content-in-request": { +# CHECK-NEXT: "contents": "" +# CHECK-NEXT: } +# CHECK-NEXT: }, +# CHECK-NEXT: "duration" +# CHECK-NEXT: "expected": true, +# CHECK-NEXT: "start_time" +# CHECK-NEXT: "status": "PASS", +# CHECK-NEXT: "summary_html": "

", +# CHECK-NEXT: "testId": "test-data-micro :: micro-tests.ini:test1microres" +# CHECK-NEXT: }, +# CHECK-NEXT: { +# CHECK-NEXT: "artifacts": { +# CHECK-NEXT: "artifact-content-in-request": { +# CHECK-NEXT: "contents": "" +# CHECK-NEXT: } +# CHECK-NEXT: }, +# CHECK-NEXT: "duration" +# CHECK-NEXT: "expected": true, +# CHECK-NEXT: "start_time" +# CHECK-NEXT: "status": "PASS", +# CHECK-NEXT: "summary_html": "

", +# CHECK-NEXT: "testId": "test-data-micro :: micro-tests.ini:test2microres" +# CHECK-NEXT: } +# CHECK-NEXT: ] +# CHECK-NEXT: } diff --git a/llvm/utils/lit/tests/test-output-resultdb.py b/llvm/utils/lit/tests/test-output-resultdb.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/test-output-resultdb.py @@ -0,0 +1,22 @@ +# RUN: %{lit} -j 1 -v %{inputs}/test-data --resultdb-output %t.results.out > %t.out +# RUN: FileCheck < %t.results.out %s + +# CHECK: { +# CHECK: "__version__" +# CHECK: "elapsed" +# CHECK-NEXT: "tests": [ +# CHECK-NEXT: { +# CHECK-NEXT: "artifacts": { +# CHECK-NEXT: "artifact-content-in-request": { +# CHECK-NEXT: "contents": "VGVzdCBwYXNzZWQu" +# CHECK-NEXT: } +# CHECK-NEXT: }, +# CHECK-NEXT: "duration" +# CHECK-NEXT: "expected": true, +# CHECK-NEXT: "start_time" +# CHECK-NEXT: "status": "PASS", +# CHECK-NEXT: "summary_html": "

", +# CHECK-NEXT: "testId": "test-data :: metrics.ini" +# CHECK-NEXT: } +# CHECK-NEXT: ] +# CHECK-NEXT: }