diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst --- a/llvm/docs/CommandGuide/lit.rst +++ b/llvm/docs/CommandGuide/lit.rst @@ -214,6 +214,21 @@ of this option, which is especially useful in environments where the call to ``lit`` is issued indirectly. +.. option:: --filter-out=REGEXP + + Filter out those tests whose name matches the regular expression specified in + ``REGEXP``. The environment variable ``LIT_FILTER_OUT`` can be also used in + place of this option, which is especially useful in environments where the + call to ``lit`` is issued indirectly. + +.. option:: --xfail=LIST + + Treat those tests whose name is in the semicolon separated list ``LIST`` as + ``XFAIL``. This can be helpful when one does not want to modify the test + suite. The environment variable ``LIT_XFAIL`` can be also used in place of + this option, which is especially useful in environments where the call to + ``lit`` is issued indirectly. + ADDITIONAL OPTIONS ------------------ diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py --- a/llvm/utils/lit/lit/cl_arguments.py +++ b/llvm/utils/lit/lit/cl_arguments.py @@ -158,6 +158,16 @@ type=_case_insensitive_regex, help="Only run tests with paths matching the given regular expression", default=os.environ.get("LIT_FILTER", ".*")) + selection_group.add_argument("--filter-out", + metavar="REGEX", + type=_case_insensitive_regex, + help="Filter out tests with paths matching the given regular expression", + default=os.environ.get("LIT_FILTER_OUT", "^$")) + selection_group.add_argument("--xfail", + metavar="LIST", + type=_semicolon_list, + help="XFAIL tests with paths in the semicolon separated list", + default=os.environ.get("LIT_XFAIL", "")) selection_group.add_argument("--num-shards", dest="numShards", metavar="M", @@ -242,6 +252,10 @@ raise _error("invalid regular expression: '{}', {}", arg, reason) +def _semicolon_list(arg): + return arg.split(';') + + def _error(desc, *args): msg = desc.format(*args) return argparse.ArgumentTypeError(msg) diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py --- a/llvm/utils/lit/lit/main.py +++ b/llvm/utils/lit/lit/main.py @@ -68,7 +68,9 @@ determine_order(discovered_tests, opts.order) selected_tests = [t for t in discovered_tests if - opts.filter.search(t.getFullName())] + opts.filter.search(t.getFullName()) and not + opts.filter_out.search(t.getFullName())] + if not selected_tests: sys.stderr.write('error: filter did not match any tests ' '(of %d discovered). ' % len(discovered_tests)) @@ -95,6 +97,8 @@ selected_tests = selected_tests[:opts.max_tests] + mark_xfail(discovered_tests, opts) + mark_excluded(discovered_tests, selected_tests) start = time.time() @@ -189,6 +193,11 @@ return selected_tests +def mark_xfail(selected_tests, opts): + for t in selected_tests: + if os.sep.join(t.path_in_suite) in opts.xfail: + t.xfails += '*' + def mark_excluded(discovered_tests, selected_tests): excluded_tests = set(discovered_tests) - set(selected_tests) result = lit.Test.Result(lit.Test.EXCLUDED) diff --git a/llvm/utils/lit/tests/Inputs/xfail-cl/false.txt b/llvm/utils/lit/tests/Inputs/xfail-cl/false.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/xfail-cl/false.txt @@ -0,0 +1 @@ +# RUN: false diff --git a/llvm/utils/lit/tests/Inputs/xfail-cl/false2.txt b/llvm/utils/lit/tests/Inputs/xfail-cl/false2.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/xfail-cl/false2.txt @@ -0,0 +1 @@ +# RUN: false diff --git a/llvm/utils/lit/tests/Inputs/xfail-cl/lit.cfg b/llvm/utils/lit/tests/Inputs/xfail-cl/lit.cfg new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/xfail-cl/lit.cfg @@ -0,0 +1,4 @@ +import lit.formats +config.name = 'top-level-suite' +config.suffixes = ['.txt'] +config.test_format = lit.formats.ShTest() diff --git a/llvm/utils/lit/tests/Inputs/xfail-cl/true.txt b/llvm/utils/lit/tests/Inputs/xfail-cl/true.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/xfail-cl/true.txt @@ -0,0 +1 @@ +# RUN: true diff --git a/llvm/utils/lit/tests/selecting.py b/llvm/utils/lit/tests/selecting.py --- a/llvm/utils/lit/tests/selecting.py +++ b/llvm/utils/lit/tests/selecting.py @@ -9,9 +9,12 @@ # CHECK-BAD-PATH: error: did not discover any tests for provided path(s) # Check that we exit with an error if we filter out all tests, but allow it with --allow-empty-runs. +# Check that we exit with an error if we skip all tests, but allow it with --allow-empty-runs. # # RUN: not %{lit} --filter 'nonexistent' %{inputs}/discovery 2>&1 | FileCheck --check-prefixes=CHECK-BAD-FILTER,CHECK-BAD-FILTER-ERROR %s # RUN: %{lit} --filter 'nonexistent' --allow-empty-runs %{inputs}/discovery 2>&1 | FileCheck --check-prefixes=CHECK-BAD-FILTER,CHECK-BAD-FILTER-ALLOW %s +# RUN: not %{lit} --filter-out '.*' %{inputs}/discovery 2>&1 | FileCheck --check-prefixes=CHECK-BAD-FILTER,CHECK-BAD-FILTER-ERROR %s +# RUN: %{lit} --filter-out '.*' --allow-empty-runs %{inputs}/discovery 2>&1 | FileCheck --check-prefixes=CHECK-BAD-FILTER,CHECK-BAD-FILTER-ALLOW %s # CHECK-BAD-FILTER: error: filter did not match any tests (of 5 discovered). # CHECK-BAD-FILTER-ERROR: Use '--allow-empty-runs' to suppress this error. # CHECK-BAD-FILTER-ALLOW: Suppressing error because '--allow-empty-runs' was specified. @@ -21,6 +24,9 @@ # RUN: %{lit} --filter 'o[a-z]e' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s # RUN: %{lit} --filter 'O[A-Z]E' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s # RUN: env LIT_FILTER='o[a-z]e' %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s +# RUN: %{lit} --filter-out 'test-t[a-z]' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s +# RUN: %{lit} --filter-out 'test-t[A-Z]' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s +# RUN: env LIT_FILTER_OUT='test-t[a-z]' %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s # CHECK-FILTER: Testing: 2 of 5 tests # CHECK-FILTER: Excluded: 3 diff --git a/llvm/utils/lit/tests/xfail-cl.py b/llvm/utils/lit/tests/xfail-cl.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/xfail-cl.py @@ -0,0 +1,9 @@ +# Check that regex-XFAILing works and can be configured via env var. +# +# RUN: %{lit} --xfail 'false.txt;false2.txt' %{inputs}/xfail-cl | FileCheck --check-prefix=CHECK-FILTER %s +# RUN: env LIT_XFAIL='false.txt;false2.txt' %{lit} %{inputs}/xfail-cl | FileCheck --check-prefix=CHECK-FILTER %s +# END: +# CHECK-FILTER: Testing: 3 tests, 3 workers +# CHECK-FILTER-DAG: {{XFAIL}}: top-level-suite :: false.txt +# CHECK-FILTER-DAG: {{XFAIL}}: top-level-suite :: false2.txt +# CHECK-FILTER-DAG: PASS: top-level-suite :: true.txt