diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py --- a/llvm/utils/lit/lit/Test.py +++ b/llvm/utils/lit/lit/Test.py @@ -36,6 +36,7 @@ UNRESOLVED = ResultCode('UNRESOLVED', True) UNSUPPORTED = ResultCode('UNSUPPORTED', False) TIMEOUT = ResultCode('TIMEOUT', True) +SKIPPED = ResultCode('SKIPPED', False) # Test metric values. diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py --- a/llvm/utils/lit/lit/main.py +++ b/llvm/utils/lit/lit/main.py @@ -89,12 +89,14 @@ run_tests(filtered_tests, lit_config, opts, len(discovered_tests)) elapsed = time.time() - start - executed_tests = [t for t in filtered_tests if t.result] + # TODO(yln): eventually, all functions below should act on discovered_tests + executed_tests = [ + t for t in filtered_tests if t.result.code != lit.Test.SKIPPED] if opts.time_tests: print_histogram(executed_tests) - print_results(executed_tests, elapsed, opts) + print_results(filtered_tests, elapsed, opts) if opts.output_path: #TODO(yln): pass in discovered_tests @@ -256,6 +258,7 @@ ] all_codes = [ + (lit.Test.SKIPPED, 'Skipped Tests', 'Skipped'), (lit.Test.UNSUPPORTED, 'Unsupported Tests', 'Unsupported'), (lit.Test.PASS, 'Expected Passes', ''), (lit.Test.FLAKYPASS, 'Passes With Retry', ''), @@ -277,11 +280,11 @@ def print_group(code, label, tests, opts): if not tests: return - if code == lit.Test.PASS: + # TODO(yln): FLAKYPASS? Make this more consistent! + if code in {lit.Test.SKIPPED, lit.Test.PASS}: return if (lit.Test.XFAIL == code and not opts.show_xfail) or \ - (lit.Test.UNSUPPORTED == code and not opts.show_unsupported) or \ - (lit.Test.UNRESOLVED == code and (opts.max_failures is not None)): + (lit.Test.UNSUPPORTED == code and not opts.show_unsupported): return print('*' * 20) print('%s Tests (%d):' % (label, len(tests))) diff --git a/llvm/utils/lit/lit/run.py b/llvm/utils/lit/lit/run.py --- a/llvm/utils/lit/lit/run.py +++ b/llvm/utils/lit/lit/run.py @@ -42,7 +42,7 @@ Upon completion, each test in the run will have its result computed. Tests which were not actually executed (for any reason) will - be given an UNRESOLVED result. + be marked SKIPPED. """ self.failures = 0 @@ -51,12 +51,13 @@ timeout = self.timeout or one_week deadline = time.time() + timeout - self._execute(deadline) - - # Mark any tests that weren't run as UNRESOLVED. - for test in self.tests: - if test.result is None: - test.setResult(lit.Test.Result(lit.Test.UNRESOLVED, '', 0.0)) + try: + self._execute(deadline) + finally: + skipped = lit.Test.Result(lit.Test.SKIPPED) + for test in self.tests: + if test.result is None: + test.setResult(skipped) def _execute(self, deadline): self._increase_process_limit() diff --git a/llvm/utils/lit/tests/Inputs/max-time/fast.txt b/llvm/utils/lit/tests/Inputs/max-time/fast.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/max-time/fast.txt @@ -0,0 +1 @@ +RUN: true diff --git a/llvm/utils/lit/tests/Inputs/max-time/lit.cfg b/llvm/utils/lit/tests/Inputs/max-time/lit.cfg new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/max-time/lit.cfg @@ -0,0 +1,6 @@ +import lit.formats +config.name = 'lit-time' +config.suffixes = ['.txt'] +config.test_format = lit.formats.ShTest() +config.test_source_root = None +config.test_exec_root = None diff --git a/llvm/utils/lit/tests/Inputs/max-time/slow.txt b/llvm/utils/lit/tests/Inputs/max-time/slow.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/max-time/slow.txt @@ -0,0 +1 @@ +RUN: sleep 5 diff --git a/llvm/utils/lit/tests/max-time.py b/llvm/utils/lit/tests/max-time.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/max-time.py @@ -0,0 +1,6 @@ +# Test overall lit timeout (--max-time). +# +# RUN: %{lit} %{inputs}/max-time --max-time=1 | FileCheck %s + +# CHECK: Skipped Tests : 1 +# CHECK: Expected Passes: 1