diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py --- a/llvm/utils/lit/lit/main.py +++ b/llvm/utils/lit/lit/main.py @@ -252,29 +252,34 @@ lit.util.printHistogram(test_times, title='Tests') +def add_result_category(result_code, label): + assert isinstance(result_code, lit.Test.ResultCode) + category = (result_code, "%s Tests" % label, label) + result_codes.append(category) + + # Status code, summary label, group label -failure_codes = [ +result_codes = [ + # Passes + (lit.Test.SKIPPED, 'Skipped Tests', 'Skipped'), + (lit.Test.UNSUPPORTED, 'Unsupported Tests', 'Unsupported'), + (lit.Test.PASS, 'Expected Passes', ''), + (lit.Test.FLAKYPASS, 'Passes With Retry', ''), + (lit.Test.XFAIL, 'Expected Failures', 'Expected Failing'), + # Failures (lit.Test.UNRESOLVED, 'Unresolved Tests', 'Unresolved'), (lit.Test.TIMEOUT, 'Individual Timeouts', 'Timed Out'), (lit.Test.FAIL, 'Unexpected Failures', 'Failing'), (lit.Test.XPASS, 'Unexpected Passes', 'Unexpected Passing') ] -all_codes = [ - (lit.Test.SKIPPED, 'Skipped Tests', 'Skipped'), - (lit.Test.UNSUPPORTED, 'Unsupported Tests', 'Unsupported'), - (lit.Test.PASS, 'Expected Passes', ''), - (lit.Test.FLAKYPASS, 'Passes With Retry', ''), - (lit.Test.XFAIL, 'Expected Failures', 'Expected Failing'), -] + failure_codes - def print_results(tests, elapsed, opts): - tests_by_code = {code: [] for (code, _, _) in all_codes} + tests_by_code = {code: [] for (code, _, _) in result_codes} for test in tests: tests_by_code[test.result.code].append(test) - for (code, _, group_label) in all_codes: + for (code, _, group_label) in result_codes: print_group(code, group_label, tests_by_code[code], opts) print_summary(tests_by_code, opts.quiet, elapsed) @@ -300,7 +305,7 @@ if not quiet: print('\nTesting Time: %.2fs' % elapsed) - codes = failure_codes if quiet else all_codes + codes = [c for c in result_codes if not quiet or c.isFailure] groups = [(label, len(tests_by_code[code])) for code, label, _ in codes] groups = [(label, count) for label, count in groups if count] if not groups: diff --git a/llvm/utils/lit/tests/Inputs/custom-result-category/format.py b/llvm/utils/lit/tests/Inputs/custom-result-category/format.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/custom-result-category/format.py @@ -0,0 +1,18 @@ +import lit +import lit.formats + +CUSTOM_PASS = lit.Test.ResultCode('CUSTOM_PASS', False) +CUSTOM_FAILURE = lit.Test.ResultCode('CUSTOM_FAILURE', True) + +lit.main.add_result_category(CUSTOM_PASS, "My Passed") +lit.main.add_result_category(CUSTOM_FAILURE, "My Failed") + + +class MyFormat(lit.formats.ShTest): + def execute(self, test, lit_config): + result = super(MyFormat, self).execute(test, lit_config) + if result.code.isFailure: + result.code = CUSTOM_FAILURE + else: + result.code = CUSTOM_PASS + return result diff --git a/llvm/utils/lit/tests/Inputs/custom-result-category/lit.cfg b/llvm/utils/lit/tests/Inputs/custom-result-category/lit.cfg new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/custom-result-category/lit.cfg @@ -0,0 +1,10 @@ +import lit +import site +site.addsitedir(os.path.dirname(__file__)) +import format + +config.name = 'custom-result-category' +config.suffixes = ['.txt'] +config.test_format = format.MyFormat() +config.test_source_root = None +config.test_exec_root = None diff --git a/llvm/utils/lit/tests/Inputs/custom-result-category/test1.txt b/llvm/utils/lit/tests/Inputs/custom-result-category/test1.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/custom-result-category/test1.txt @@ -0,0 +1 @@ +# RUN: true diff --git a/llvm/utils/lit/tests/Inputs/custom-result-category/test2.txt b/llvm/utils/lit/tests/Inputs/custom-result-category/test2.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/custom-result-category/test2.txt @@ -0,0 +1 @@ +# RUN: false diff --git a/llvm/utils/lit/tests/Inputs/shtest-inject/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-inject/lit.cfg --- a/llvm/utils/lit/tests/Inputs/shtest-inject/lit.cfg +++ b/llvm/utils/lit/tests/Inputs/shtest-inject/lit.cfg @@ -3,7 +3,7 @@ preamble_commands = [ 'echo "THIS WAS"', 'echo "INJECTED"' -]; +] config.name = 'shtest-inject' config.suffixes = ['.txt'] diff --git a/llvm/utils/lit/tests/custom-result-category.py b/llvm/utils/lit/tests/custom-result-category.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/custom-result-category.py @@ -0,0 +1,13 @@ +# Test lit.main.add_result_category() extension API. + +# RUN: not %{lit} -j 1 %{inputs}/custom-result-category | FileCheck %s + +# CHECK: CUSTOM_PASS: custom-result-category :: test1.txt +# CHECK: CUSTOM_FAILURE: custom-result-category :: test2.txt + +# TODO(yln): Passing tests shouldn't be printed by default. +# CHECK: My Passed Tests (1) +# CHECK: My Failed Tests (1) + +# CHECK: My Passed Tests: 1 +# CHECK: My Failed Tests: 1