diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst --- a/llvm/docs/CommandGuide/lit.rst +++ b/llvm/docs/CommandGuide/lit.rst @@ -115,6 +115,11 @@ Show the names of tests that were expected to fail. +.. option:: --show + + Show the names of the specified tests. Choose from: + all, excluded, skipped, unsupported, pass, flakypass, xfail. + .. _execution-options: EXECUTION OPTIONS @@ -161,9 +166,6 @@ .. option:: --max-failures MAX_FAILURES Stop execution after the given number of failures. - An integer argument may be passed on the command line - or the environment vairable MAX_FAILURES may be set - prior to execution. .. option:: --max-tests=N diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py --- a/llvm/utils/lit/lit/cl_arguments.py +++ b/llvm/utils/lit/lit/cl_arguments.py @@ -71,6 +71,11 @@ format_group.add_argument("--show-xfail", help="Show tests that were expected to fail", action="store_true") + format_group.add_argument("--show", + nargs="+", + help="Show specified tests. Failing tests are always printed.", + choices=_result_code_choices(), + default=[]) execution_group = parser.add_argument_group("Test Execution") execution_group.add_argument("--path", @@ -187,17 +192,28 @@ else: opts.shard = None - opts.show_results = set() + opts.shown_codes = set() if opts.show_unsupported: - opts.show_results.add(lit.Test.UNSUPPORTED) + opts.shown_codes.add(lit.Test.UNSUPPORTED) if opts.show_xfail: - opts.show_results.add(lit.Test.XFAIL) + opts.shown_codes.add(lit.Test.XFAIL) + for code in opts.show: + if code == 'all': + opts.shown_codes.update(lit.Test.ResultCode.all_codes()) + else: + opts.shown_codes.add(lit.Test.ResultCode._instances[code.upper()]) opts.reports = filter(None, [opts.output, opts.xunit_xml_output]) return opts +def _result_code_choices(): + codes = [c.name.lower() for c in lit.Test.ResultCode.all_codes() + if not c.isFailure] + return ['all'] + codes + + def _positive_int(arg): return _int(arg, 'positive', lambda i: i > 0) diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py --- a/llvm/utils/lit/lit/main.py +++ b/llvm/utils/lit/lit/main.py @@ -265,15 +265,15 @@ tests_by_code[test.result.code].append(test) for code in lit.Test.ResultCode.all_codes(): - print_group(tests_by_code[code], code, opts.show_results) + print_group(tests_by_code[code], code, opts.shown_codes) print_summary(tests_by_code, opts.quiet, elapsed) -def print_group(tests, code, show_results): +def print_group(tests, code, shown_codes): if not tests: return - if not code.isFailure and code not in show_results: + if not code.isFailure and code not in shown_codes: return print('*' * 20) print('{} Tests ({}):'.format(code.label, len(tests))) diff --git a/llvm/utils/lit/tests/Inputs/show-tests/fail.txt b/llvm/utils/lit/tests/Inputs/show-tests/fail.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/show-tests/fail.txt @@ -0,0 +1 @@ +RUN: false diff --git a/llvm/utils/lit/tests/Inputs/show-tests/lit.cfg b/llvm/utils/lit/tests/Inputs/show-tests/lit.cfg new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/show-tests/lit.cfg @@ -0,0 +1,6 @@ +import lit.formats +config.name = 'show-tests' +config.suffixes = ['.txt'] +config.test_format = lit.formats.ShTest() +config.test_source_root = None +config.test_exec_root = None diff --git a/llvm/utils/lit/tests/Inputs/show-tests/pass.txt b/llvm/utils/lit/tests/Inputs/show-tests/pass.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/show-tests/pass.txt @@ -0,0 +1 @@ +RUN: true diff --git a/llvm/utils/lit/tests/Inputs/show-tests/unsupported.txt b/llvm/utils/lit/tests/Inputs/show-tests/unsupported.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/show-tests/unsupported.txt @@ -0,0 +1,2 @@ +REQUIRES: missing-feature +RUN: true diff --git a/llvm/utils/lit/tests/Inputs/show-tests/xfail.txt b/llvm/utils/lit/tests/Inputs/show-tests/xfail.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/show-tests/xfail.txt @@ -0,0 +1,2 @@ +XFAIL: * +RUN: false diff --git a/llvm/utils/lit/tests/show-tests.py b/llvm/utils/lit/tests/show-tests.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/show-tests.py @@ -0,0 +1,27 @@ +# Test the --show {all,unsupported,...} option. +# +# RUN: not %{lit} %{inputs}/show-tests --show all | FileCheck %s --check-prefix=ALL +# RUN: not %{lit} %{inputs}/show-tests | FileCheck %s --check-prefix=NONE +# RUN: not %{lit} %{inputs}/show-tests --show pass | FileCheck %s --check-prefix=ONE +# RUN: not %{lit} %{inputs}/show-tests --show pass xfail | FileCheck %s --check-prefix=MULTIPLE + +# ALL: Unsupported Tests (1) +# ALL: Passed Tests (1) +# ALL: Expectedly Failed Tests (1) +# ALL: Failed Tests (1) + +# Failing tests are always shown +# NONE-NOT: Unsupported Tests (1) +# NONE-NOT: Passed Tests (1) +# NONE-NOT: Expectedly Failed Tests (1) +# NONE: Failed Tests (1) + +# ONE-NOT: Unsupported Tests (1) +# ONE: Passed Tests (1) +# ONE-NOT: Expectedly Failed Tests (1) +# ONE: Failed Tests (1) + +# MULTIPLE-NOT: Unsupported Tests (1) +# MULTIPLE: Passed Tests (1) +# MULTIPLE: Expectedly Failed Tests (1) +# MULTIPLE: Failed Tests (1)