[lit] Add option to output test result as resultdb json format.

[haowei]

Summary

This change adds an option --output-format=resultdb to allow llvm-lit generating JSON output for the test results, which can be better integrated with certain CI/CQ framework. This change does not alter the default behavior of llvm-lit when generating an output file.

[yln]

Hi Haowei,

Can you change this to be a new type of report instead of changing an existing one?
Look at TimeTraceReport for the recipe. Maybe --resultdb-output and ResultDbReport are appropriate names for parameter/class name?

This way the change would be purely additive and I then would be okay landing this as-is, i.e., without tests and documentation. It would then be "your problem" if we (upstream) break this functionality. This is only holds if my fellow reviewers think this okay as well, of course.

[haowei]

In D108238#2950560, @yln wrote:

Hi Haowei,

Can you change this to be a new type of report instead of changing an existing one?
Look at TimeTraceReport for the recipe. Maybe --resultdb-output and ResultDbReport are appropriate names for parameter/class name?

This way the change would be purely additive and I then would be okay landing this as-is, i.e., without tests and documentation. It would then be "your problem" if we (upstream) break this functionality. This is only holds if my fellow reviewers think this okay as well, of course.

I forgot to include the unit tests in the previous change, sorry about that. I was hoping to reuse the code instead of re-inventing the wheels but if adding a new type of report is what you strongly suggest, I can work on implementing that.

[haowei]

Reimplemented into a dedicated class.

[yln]

LGTM, I am happy with the patch: changes are now purely additive and include tests.

Please wait a week or so before landing to give others a chance to comment. Thanks! :)

llvm/utils/lit/lit/cl_arguments.py

execution_group.add_argument("--time-tests",
             action="store_true")
     execution_group.add_argument("--no-execute",
             dest="noExecute",
             help="Don't execute any tests (assume PASS)",
             action="store_true")
     execution_group.add_argument("--xunit-xml-output",
             type=lit.reports.XunitReport,
             help="Write XUnit-compatible XML test reports to the specified file")
+    execution_group.add_argument("--resultdb-output",
+            type=lit.reports.ResultDBReport,
+            help="Write LuCI ResuldDB compatible JSON to the specified file")
     execution_group.add_argument("--time-trace-output",
             type=lit.reports.TimeTraceReport,
             help="Write Chrome tracing compatible JSON to the specified file")
     execution_group.add_argument("--timeout",
             dest="maxIndividualTestTime",
             help="Maximum time to spend running a single test (in seconds). "
                  "0 means no time limit. [Default: 0]",
             type=_non_negative_int)
if opts.numShards or opts.runShard:
         if not opts.numShards or not opts.runShard:
             parser.error("--num-shards and --run-shard must be used together")
         if opts.runShard > opts.numShards:
             parser.error("--run-shard must be between 1 and --num-shards (inclusive)")
         opts.shard = (opts.runShard, opts.numShards)
     else:
         opts.shard = None
 
-    opts.reports = filter(None, [opts.output, opts.xunit_xml_output, opts.time_trace_output])
+    opts.reports = filter(None, [opts.output, opts.xunit_xml_output, opts.resultdb_output, opts.time_trace_output])
 
     return opts
 
 
 def _positive_int(arg):
     return _int(arg, 'positive', lambda i: i > 0)
 
 

llvm/utils/lit/lit/reports.py

+import base64
+import datetime
 import itertools
 import json
 
 from xml.sax.saxutils import quoteattr as quo
 
 import lit.Test
 
 
def _get_skip_reason(self, test):
 
         assert code == lit.Test.UNSUPPORTED
         features = test.getMissingRequiredFeatures()
         if features:
             return 'Missing required feature(s): ' + ', '.join(features)
         return 'Unsupported configuration'
 
 
+def gen_resultdb_test_entry(
+    test_name, start_time, elapsed_time, test_output, result_code, is_expected
+):
+    test_data = {
+        'testId': test_name,
+        'start_time': datetime.datetime.fromtimestamp(start_time).isoformat() + 'Z',
+        'duration': '%.9fs' % elapsed_time,
+        'summary_html': '<p><text-artifact artifact-id="artifact-content-in-request"></p>',
+        'artifacts': {
+            'artifact-content-in-request': {
+                'contents': base64.b64encode(test_output.encode('utf-8')).decode(
+                    'utf-8'
+                ),
+            },
+        },
+        'expected': is_expected,
+    }
+    if (
+        result_code == lit.Test.PASS
+        or result_code == lit.Test.XPASS
+        or result_code == lit.Test.FLAKYPASS
+    ):
+        test_data['status'] = 'PASS'
+    elif result_code == lit.Test.FAIL or result_code == lit.Test.XFAIL:
+        test_data['status'] = 'FAIL'
+    elif (
+        result_code == lit.Test.UNSUPPORTED
+        or result_code == lit.Test.SKIPPED
+        or result_code == lit.Test.EXCLUDED
+    ):
+        test_data['status'] = 'SKIP'
+    elif result_code == lit.Test.UNRESOLVED or result_code == lit.Test.TIMEOUT:
+        test_data['status'] = 'ABORT'
+    return test_data
+
+
+class ResultDBReport(object):
+    def __init__(self, output_file):
+        self.output_file = output_file
+
+    def write_results(self, tests, elapsed):
+        unexecuted_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED}
+        tests = [t for t in tests if t.result.code not in unexecuted_codes]
+        data = {}
+        data['__version__'] = lit.__versioninfo__
+        data['elapsed'] = elapsed
+        # Encode the tests.
+        data['tests'] = tests_data = []
+        for test in tests:
+            tests_data.append(
+                gen_resultdb_test_entry(
+                    test_name=test.getFullName(),
+                    start_time=test.result.start,
+                    elapsed_time=test.result.elapsed,
+                    test_output=test.result.output,
+                    result_code=test.result.code,
+                    is_expected=not test.result.code.isFailure,
+                )
+            )
+            if test.result.microResults:
+                for key, micro_test in test.result.microResults.items():
+                    # Expand parent test name with micro test name
+                    parent_name = test.getFullName()
+                    micro_full_name = parent_name + ':' + key + 'microres'
+                    tests_data.append(
+                        gen_resultdb_test_entry(
+                            test_name=micro_full_name,
+                            start_time=micro_test.start
+                            if micro_test.start
+                            else test.result.start,
+                            elapsed_time=micro_test.elapsed
+                            if micro_test.elapsed
+                            else test.result.elapsed,
+                            test_output=micro_test.output,
+                            result_code=micro_test.code,
+                            is_expected=not micro_test.code.isFailure,
+                        )
+                    )
+
+        with open(self.output_file, 'w') as file:
+            json.dump(data, file, indent=2, sort_keys=True)
+            file.write('\n')
+
+
 class TimeTraceReport(object):
     def __init__(self, output_file):
         self.output_file = output_file
         self.skipped_codes = {lit.Test.EXCLUDED,
                               lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
 
     def write_results(self, tests, elapsed):
         # Find when first test started so we can make start times relative.

llvm/utils/lit/tests/test-output-micro-resultdb.py

+# RUN: %{lit} -j 1 -v %{inputs}/test-data-micro --resultdb-output %t.results.out
+# RUN: FileCheck < %t.results.out %s
+# RUN: rm %t.results.out
+
+
+# CHECK: {
+# CHECK: "__version__"
+# CHECK: "elapsed"
+# CHECK-NEXT: "tests": [
+# CHECK-NEXT:    {
+# CHECK-NEXT:      "artifacts": {
+# CHECK-NEXT:        "artifact-content-in-request": {
+# CHECK-NEXT:          "contents": "VGVzdCBwYXNzZWQu"
+# CHECK-NEXT:        }
+# CHECK-NEXT:      },
+# CHECK-NEXT:      "duration"
+# CHECK-NEXT:      "expected": true,
+# CHECK-NEXT:      "start_time"
+# CHECK-NEXT:      "status": "PASS",
+# CHECK-NEXT:      "summary_html": "<p><text-artifact artifact-id=\"artifact-content-in-request\"></p>",
+# CHECK-NEXT:      "testId": "test-data-micro :: micro-tests.ini"
+# CHECK-NEXT:    },
+# CHECK-NEXT:    {
+# CHECK-NEXT:      "artifacts": {
+# CHECK-NEXT:        "artifact-content-in-request": {
+# CHECK-NEXT:          "contents": ""
+# CHECK-NEXT:        }
+# CHECK-NEXT:      },
+# CHECK-NEXT:      "duration"
+# CHECK-NEXT:      "expected": true,
+# CHECK-NEXT:      "start_time"
+# CHECK-NEXT:      "status": "PASS",
+# CHECK-NEXT:      "summary_html": "<p><text-artifact artifact-id=\"artifact-content-in-request\"></p>",
+# CHECK-NEXT:      "testId": "test-data-micro :: micro-tests.ini:test0microres"
+# CHECK-NEXT:    },
+# CHECK-NEXT:    {
+# CHECK-NEXT:      "artifacts": {
+# CHECK-NEXT:        "artifact-content-in-request": {
+# CHECK-NEXT:          "contents": ""
+# CHECK-NEXT:        }
+# CHECK-NEXT:      },
+# CHECK-NEXT:      "duration"
+# CHECK-NEXT:      "expected": true,
+# CHECK-NEXT:      "start_time"
+# CHECK-NEXT:      "status": "PASS",
+# CHECK-NEXT:      "summary_html": "<p><text-artifact artifact-id=\"artifact-content-in-request\"></p>",
+# CHECK-NEXT:      "testId": "test-data-micro :: micro-tests.ini:test1microres"
+# CHECK-NEXT:    },
+# CHECK-NEXT:    {
+# CHECK-NEXT:      "artifacts": {
+# CHECK-NEXT:        "artifact-content-in-request": {
+# CHECK-NEXT:          "contents": ""
+# CHECK-NEXT:        }
+# CHECK-NEXT:      },
+# CHECK-NEXT:      "duration"
+# CHECK-NEXT:      "expected": true,
+# CHECK-NEXT:      "start_time"
+# CHECK-NEXT:      "status": "PASS",
+# CHECK-NEXT:      "summary_html": "<p><text-artifact artifact-id=\"artifact-content-in-request\"></p>",
+# CHECK-NEXT:      "testId": "test-data-micro :: micro-tests.ini:test2microres"
+# CHECK-NEXT:    }
+# CHECK-NEXT: ]
+# CHECK-NEXT: }

llvm/utils/lit/tests/test-output-resultdb.py

+# RUN: %{lit} -j 1 -v %{inputs}/test-data --resultdb-output %t.results.out > %t.out
+# RUN: FileCheck < %t.results.out %s
+
+# CHECK: {
+# CHECK: "__version__"
+# CHECK: "elapsed"
+# CHECK-NEXT: "tests": [
+# CHECK-NEXT:   {
+# CHECK-NEXT:      "artifacts": {
+# CHECK-NEXT:        "artifact-content-in-request": {
+# CHECK-NEXT:          "contents": "VGVzdCBwYXNzZWQu"
+# CHECK-NEXT:        }
+# CHECK-NEXT:      },
+# CHECK-NEXT:      "duration"
+# CHECK-NEXT:      "expected": true,
+# CHECK-NEXT:      "start_time"
+# CHECK-NEXT:      "status": "PASS",
+# CHECK-NEXT:      "summary_html": "<p><text-artifact artifact-id=\"artifact-content-in-request\"></p>",
+# CHECK-NEXT:      "testId": "test-data :: metrics.ini"
+# CHECK-NEXT:    }
+# CHECK-NEXT: ]
+# CHECK-NEXT: }