Index: lnt/trunk/lnt/tests/test_suite.py =================================================================== --- lnt/trunk/lnt/tests/test_suite.py +++ lnt/trunk/lnt/tests/test_suite.py @@ -69,6 +69,25 @@ """ +CSV_REPORT_TEMPLATE = \ +"""Program;CC;CC_Time;CC_Hash;Exec;Exec_Time;Score +{%- for suite in suites -%} + {%- for test in suite.tests %} +{{ suite.name }}/{{ test.path }}/{{ test.name }}; + {%- if test.code == "NOEXE" -%} + fail;*;*; + {%- else -%} + pass;{{ test.metrics.compile_time if test.metrics }};{{ test.metrics.hash if test.metrics }}; + {%- endif -%} + {%- if test.code == "FAIL" or test.code == "NOEXE" -%} + fail;*;*; + {%- else -%} + pass;{{ test.metrics.exec_time if test.metrics }};{{ test.metrics.score if test.metrics }}; + {%- endif -%} + {% endfor %} +{%- endfor -%} +""" + # _importProfile imports a single profile. It must be at the top level (and # not within TestSuiteTest) so that multiprocessing can import it correctly. def _importProfile(name_filename): @@ -90,11 +109,7 @@ str) -def _lit_json_to_xunit_xml(json_reports): - # type: (list) -> str - """Take the lit report jason dicts and convert them - to an xunit xml report for CI to digest.""" - template_engine = jinja2.Template(XML_REPORT_TEMPLATE, autoescape=True) +def _lit_json_to_template(json_reports, template_engine): # For now, only show first runs report. json_report = json_reports[0] tests_by_suite = defaultdict(list) @@ -112,7 +127,8 @@ entry = {'name': test_name, 'path': '.'.join(path), 'time': time, - 'code': code} + 'code': code, + 'metrics': tests.get('metrics', None)} if code != "PASS": entry['output'] = output @@ -134,6 +150,23 @@ return str_template +def _lit_json_to_xunit_xml(json_reports): + # type: (list) -> str + """Take the lit report jason dicts and convert them + to an xunit xml report for CI to digest.""" + template_engine = jinja2.Template(XML_REPORT_TEMPLATE, autoescape=True) + return _lit_json_to_template(json_reports, template_engine) + + +def _lit_json_to_csv(json_reports): + # type: (list) -> str + """Take the lit report json dicts and convert them + to a csv report, similar to the old test-suite make-based + *.report.simple.csv files.""" + template_engine = jinja2.Template(CSV_REPORT_TEMPLATE, autoescape=True) + return _lit_json_to_template(json_reports, template_engine) + + class TestSuiteTest(BuiltinTest): def __init__(self): super(TestSuiteTest, self).__init__() @@ -477,6 +510,12 @@ with open(xml_report_path, 'w') as fd: fd.write(str_template) + csv_report_path = os.path.join(self._base_path, + 'test-results.csv') + str_template = _lit_json_to_csv(json_reports) + with open(csv_report_path, 'w') as fd: + fd.write(str_template) + return self.submit(report_path, self.opts, commit=True) def run(self, nick, compile=True, test=True): Index: lnt/trunk/tests/runtest/test_suite.py =================================================================== --- lnt/trunk/tests/runtest/test_suite.py +++ lnt/trunk/tests/runtest/test_suite.py @@ -2,7 +2,7 @@ # # RUN: rm -rf %t.SANDBOX %t.SANDBOX2 || true # -# Check a basic nt run. +# Check a basic test-suite run. # RUN: lnt runtest test-suite \ # RUN: --sandbox %t.SANDBOX \ # RUN: --no-timestamp \ @@ -16,6 +16,7 @@ # RUN: FileCheck --check-prefix CHECK-BASIC < %t.err %s # RUN: FileCheck --check-prefix CHECK-REPORT < %t.SANDBOX/build/report.json %s # RUN: FileCheck --check-prefix CHECK-XML < %t.SANDBOX/build/test-results.xunit.xml %s +# RUN: FileCheck --check-prefix CHECK-CSV < %t.SANDBOX/build/test-results.csv %s # CHECK-REPORT: "run_order": "154331" # CHECK-REPORT: "Name": "nts.{{[^.]+}}.compile" @@ -44,6 +45,9 @@ # CHECK-XML: # CHECK-XML: +# CHECK-CSV: Program;CC;CC_Time;CC_Hash;Exec;Exec_Time;Score +# CHECK-CSV-NEXT: foo//foo;pass;1.3;xyz;pass;1.4;1.5 + # Use the same sandbox again with --no-configure # RUN: lnt runtest test-suite \ # RUN: --sandbox %t.SANDBOX \