[lit] Don't include tests skipped due to sharding in reports

[arichardson]

Summary

When running multiple shards, don't include skipped tests in the xunit
output since merging the files will result in duplicates.
In our CHERI Jenkins CI, I configured the libc++ tests to run using sharding
(since we are testing using a single-CPU QEMU). We then merge the generated
XUnit xml files to produce a final result, but if the individual XMLs
report tests excluded due to sharding each test is included N times in the
final result. This also makes it difficult to find the tests that were
skipped due to missing REQUIRES: etc.

[arichardson]

fix tests and remove debug output

[yln]

Thanks for working on this! I forgot to consider the interaction with sharding.

llvm/utils/lit/lit/main.py

105

Can we filter the list of tests here (if sharding is active)? Other report types are also affected by this.

[arichardson]

Simplify by moving logic into main.py

[yln]

Thanks! It would be super awesome to have a test for this, but I won't block on it.

llvm/utils/lit/lit/main.py

def main(builtin_params={}):
     elapsed = time.time() - start
 
     if opts.time_tests:
         print_histogram(discovered_tests)
 
     print_results(discovered_tests, elapsed, opts)
 
     for report in opts.reports:
-        report.write_results(discovered_tests, elapsed)
+        report.write_results(discovered_tests, elapsed, opts.runShard)
 
     if lit_config.numErrors:
         sys.stderr.write('\n%d error(s) in tests\n' % lit_config.numErrors)
         sys.exit(2)
 
     if lit_config.numWarnings:
         sys.stderr.write('\n%d warning(s) in tests\n' % lit_config.numWarnings)
 

llvm/utils/lit/lit/reports.py

def by_suite_and_test_path(test):
     # key to avoid mixing tests of different suites.
     return (test.suite.name, id(test.suite), test.path_in_suite)
 
 
 class JsonReport(object):
     def __init__(self, output_file):
         self.output_file = output_file
 
-    def write_results(self, tests, elapsed):
+    def write_results(self, tests, elapsed, shard):
         unexecuted_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED}
         tests = [t for t in tests if t.result.code not in unexecuted_codes]
         # Construct the data we will write.
         data = {}
         # Encode the current lit version as a schema version.
         data['__version__'] = lit.__versioninfo__
         data['elapsed'] = elapsed
         # FIXME: Record some information on the lit configuration used?

 
 
 class XunitReport(object):
     def __init__(self, output_file):
         self.output_file = output_file
         self.skipped_codes = {lit.Test.EXCLUDED,
                               lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
 
-    def write_results(self, tests, elapsed):
+    def write_results(self, tests, elapsed, shard):
         tests.sort(key=by_suite_and_test_path)
         tests_by_suite = itertools.groupby(tests, lambda t: t.suite)
 
         with open(self.output_file, 'w') as file:
             file.write('<?xml version="1.0" encoding="UTF-8"?>\n')
             file.write('<testsuites time="{time:.2f}">\n'.format(time=elapsed))
             for suite, test_iter in tests_by_suite:
-                self._write_testsuite(file, suite, list(test_iter))
+                self._write_testsuite(file, suite, list(test_iter), shard)
             file.write('</testsuites>\n')
 
-    def _write_testsuite(self, file, suite, tests):
+    def _write_testsuite(self, file, suite, tests, shard):
+        print("_write_testsuite:", shard)
+        # When running multiple shards, don't include skipped tests in the xunit output
+        # since merging the files will result in duplicates otherwise:
+        if shard is not None:
+            tests = [t for t in tests if t.result.code != lit.Test.EXCLUDED]
         skipped = sum(1 for t in tests if t.result.code in self.skipped_codes)
         failures = sum(1 for t in tests if t.isFailure())
 
         name = suite.config.name.replace('.', '-')
         # file.write(f'<testsuite name={quo(name)} tests="{len(tests)}" failures="{failures}" skipped="{skipped}">\n')
         file.write('<testsuite name={name} tests="{tests}" failures="{failures}" skipped="{skipped}">\n'.format(
             name=quo(name), tests=len(tests), failures=failures, skipped=skipped))
         for test in tests:
def _write_test(self, file, test, suite_name):
             file.write('>\n  <skipped message={reason}/>\n</testcase>\n'.format(
                 reason=quo(reason)))
         else:
             file.write('/>\n')
 
     def _get_skip_reason(self, test):
         code = test.result.code
         if code == lit.Test.EXCLUDED:
-            return 'Test not selected (--filter, --max-tests, --run-shard)'
+            return 'Test not selected (--filter, --max-tests)'
         if code == lit.Test.SKIPPED:
             return 'User interrupt'
 
         assert code == lit.Test.UNSUPPORTED
         features = test.getMissingRequiredFeatures()
         if features:
             return 'Missing required feature(s): ' + ', '.join(features)
         return 'Unsupported configuration'