[LNT] Generate csv test report for test-suite

[kristof.beyls]

Summary

csv test reports are typically easier to quickly analyze than json or
xml reports. Therefore, also create a csv test report as part of 'lnt
runtest test-suite'; similar to how it's done for 'lnt runtest nt'.

Hopefully this is one of many steps to still take to be able to
deprecate 'lnt runtest nt'.

[kristof.beyls]

Correct test name generation for CSV output.

[MatzeB]

IMO lnt runtest should rather be lnt submit and not drive the whole build and run, but that discussion is for another day :)

That said with things as they are this LGTM. You may want to wait a bit if Chris has comments as he is probably a better reviewer for LNT code.

[cmatthews]

lnt/tests/test_suite.py

74

I have never seen the {%- syntax in Jinja2, are you using {%- over the standard {% for some reason?

[MatzeB]

lnt/tests/test_suite.py

74

That syntax removes the whitespace before the {% from what I remember :)

[cmatthews]

Ah, I see. LGTM!

[kristof.beyls]

That syntax removes the whitespace before the {% from what I remember :)

Indeed, see http://jinja.pocoo.org/docs/2.9/templates/#whitespace-control for the details.
It's necessary in this case since the white space matters in a csv file.

lnt/tests/test_suite.py

<testsuite name="{{ suite.name }}"
     {% endfor %}
     <system-out></system-out>
     <system-err></system-err>
 </testsuite>
 {% endfor %}
 </testsuites>
 """
 
+CSV_REPORT_TEMPLATE = \
+"""Program;CC;CC_Time;CC_Hash;Exec;Exec_Time;Score
+{%- for suite in suites -%}
+    {%- for test in suite.tests %}
+{{ suite.name }}/{{ test.path }}/{{ test.name }};
+        {%- if test.code == "NOEXE" -%}
+            fail;*;*;
+        {%- else -%}
+            pass;{{ test.metrics.compile_time if test.metrics }};{{ test.metrics.hash if test.metrics }};
+        {%- endif -%}
+        {%- if test.code == "FAIL" or test.code == "NOEXE" -%}
+            fail;*;*;
+        {%- else -%}
+            pass;{{ test.metrics.exec_time if test.metrics }};{{ test.metrics.score if test.metrics }};
+        {%- endif -%}
+    {% endfor %}
+{%- endfor -%}
+"""
+
 # _importProfile imports a single profile. It must be at the top level (and
 # not within TestSuiteTest) so that multiprocessing can import it correctly.
 def _importProfile(name_filename):
     name, filename = name_filename
 
     if not os.path.exists(filename):
         warning('Profile %s does not exist' % filename)
         return None
 
     pf = lnt.testing.profile.profile.Profile.fromFile(filename)
     if not pf:
         return None
 
     pf.upgrade()
     profilefile = pf.render()
     return lnt.testing.TestSamples(name + '.profile',
                                    [profilefile],
                                    {},
                                    str)
 
 
-def _lit_json_to_xunit_xml(json_reports):
-    # type: (list) -> str
-    """Take the lit report jason dicts and convert them
-    to an xunit xml report for CI to digest."""
-    template_engine = jinja2.Template(XML_REPORT_TEMPLATE, autoescape=True)
+def _lit_json_to_template(json_reports, template_engine):
     # For now, only show first runs report.
     json_report = json_reports[0]
     tests_by_suite = defaultdict(list)
     for tests in json_report['tests']:
         name = tests['name']
         code = tests['code']
         time = tests['elapsed']
         output = tests.get('output', 'No output collected for this test.')
 
         x = name.split("::")
         suite_name = x[1].strip().split("/")[0]
         test_name = x[1].strip().split("/")[-1]
         path = x[1].strip().split("/")[:-1]
 
         entry = {'name': test_name,
                  'path': '.'.join(path),
                  'time': time,
-                 'code': code}
+                 'code': code,
+                 'metrics': tests.get('metrics', None)}
         if code != "PASS":
             entry['output'] = output
 
         tests_by_suite[suite_name].append(entry)
     suites = []
     for id, suite in enumerate(tests_by_suite):
         tests = tests_by_suite[suite]
         entry = {'name': suite,
                  'id': id,
                  'tests': tests,
                  'timestamp': datetime.datetime.now().replace(microsecond=0).isoformat(),
                  'num_tests': len(tests),
                  'num_failures': len(
                      [x for x in tests if x['code'] == 'FAIL']),
                  'num_errors': len(
                      [x for x in tests if x['code'] == 'NOEXE'])}
         suites.append(entry)
     str_template = template_engine.render(suites=suites)
     return str_template
 
 
+def _lit_json_to_xunit_xml(json_reports):
+    # type: (list) -> str
+    """Take the lit report jason dicts and convert them
+    to an xunit xml report for CI to digest."""
+    template_engine = jinja2.Template(XML_REPORT_TEMPLATE, autoescape=True)
+    return _lit_json_to_template(json_reports, template_engine)
+
+
+def _lit_json_to_csv(json_reports):
+    # type: (list) -> str
+    """Take the lit report json dicts and convert them
+    to a csv report, similar to the old test-suite make-based
+    *.report.simple.csv files."""
+    template_engine = jinja2.Template(CSV_REPORT_TEMPLATE, autoescape=True)
+    return _lit_json_to_template(json_reports, template_engine)
+
+
 class TestSuiteTest(BuiltinTest):
     def __init__(self):
         super(TestSuiteTest, self).__init__()
         self.configured = False
         self.compiled = False
         self.trained = False
 
     def describe(self):
def run_test(self, name, args):
 
         xml_report_path = os.path.join(self._base_path,
                                        'test-results.xunit.xml')
 
         str_template = _lit_json_to_xunit_xml(json_reports)
         with open(xml_report_path, 'w') as fd:
             fd.write(str_template)
 
+        csv_report_path = os.path.join(self._base_path,
+                                       'test-results.csv')
+        str_template = _lit_json_to_csv(json_reports)
+        with open(csv_report_path, 'w') as fd:
+            fd.write(str_template)
+
         return self.submit(report_path, self.opts, commit=True)
 
     def run(self, nick, compile=True, test=True):
         path = self._base_path
 
         if not os.path.exists(path):
             mkdir_p(path)
 

tests/runtest/test_suite.py

 # Testing for the 'lnt runtest test-suite' module.
 #
 # RUN: rm -rf  %t.SANDBOX  %t.SANDBOX2 || true
 #
-# Check a basic nt run.
+# Check a basic test-suite run.
 # RUN: lnt runtest test-suite \
 # RUN:     --sandbox %t.SANDBOX \
 # RUN:     --no-timestamp \
 # RUN:     --test-suite %S/Inputs/test-suite-cmake \
 # RUN:     --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
 # RUN:     --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
 # RUN:     --use-make %S/Inputs/test-suite-cmake/fake-make \
 # RUN:     --use-lit %S/Inputs/test-suite-cmake/fake-lit \
 # RUN:     > %t.log 2> %t.err
 # RUN: FileCheck  --check-prefix CHECK-STDOUT < %t.log %s
 # RUN: FileCheck  --check-prefix CHECK-BASIC < %t.err %s
 # RUN: FileCheck  --check-prefix CHECK-REPORT < %t.SANDBOX/build/report.json %s
 # RUN: FileCheck  --check-prefix CHECK-XML < %t.SANDBOX/build/test-results.xunit.xml %s
+# RUN: FileCheck  --check-prefix CHECK-CSV < %t.SANDBOX/build/test-results.csv %s
 
 # CHECK-REPORT: "run_order": "154331"
 # CHECK-REPORT: "Name": "nts.{{[^.]+}}.compile"
 # CHECK-REPORT: "Name": "nts.{{[^.]+}}.compile.status"
 #
 # CHECK-STDOUT: Import succeeded.
 # CHECK-STDOUT: Added Machines: 1
 # CHECK-STDOUT: Added Runs    : 1

 # CHECK-XML:            errors="0"
 # CHECK-XML:            failures="0"
 # CHECK-XML:            timestamp="2
 # CHECK-XML:     <testcase classname=""
 # CHECK-XML:               name="foo" time="1.0">
 # CHECK-XML:     </testcase>
 # CHECK-XML: </testsuite>
 
+# CHECK-CSV: Program;CC;CC_Time;CC_Hash;Exec;Exec_Time;Score
+# CHECK-CSV-NEXT: foo//foo;pass;1.3;xyz;pass;1.4;1.5
+
 # Use the same sandbox again with --no-configure
 # RUN: lnt runtest test-suite \
 # RUN:     --sandbox %t.SANDBOX \
 # RUN:     --no-timestamp \
 # RUN:     --no-configure \
 # RUN:     --test-suite %S/Inputs/test-suite-cmake \
 # RUN:     --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
 # RUN:     --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \