diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst --- a/llvm/docs/CommandGuide/lit.rst +++ b/llvm/docs/CommandGuide/lit.rst @@ -412,12 +412,16 @@ **root** The root configuration. This is the top-most :program:`lit` configuration in the project. - **is_early** Whether the test suite as a whole should be given a head start - before other test suites run. - - **early_tests** An explicit set of '/' separated test paths that should be - given a head start before other tests run. For example, the top five or so - slowest tests. See also: `--time-tests` + **start_phase** The phase of the test suite to start in. The default is zero + and tests are ordered from least to greatest. Therefore to have a simple head + start for all of the tests in the suite, set the value to -1. In general, it's + better to leave this value at zero and use `start_phases` instead. + + **start_phases** A dictionary of '/' separated test paths (or path prefixes) + where the value is an offset to be added to the `start_phase` of a test suite. + For example: the top five or so slowest tests might be put in the `-1` offset; + and fast/low-reward tests might be put in the `+1` offset. See also: + `--time-tests` **pipefail** Normally a test using a shell pipe fails if any of the commands on the pipe fail. If this is not desired, setting this variable to false diff --git a/llvm/test/Unit/lit.cfg.py b/llvm/test/Unit/lit.cfg.py --- a/llvm/test/Unit/lit.cfg.py +++ b/llvm/test/Unit/lit.cfg.py @@ -13,8 +13,8 @@ # suffixes: A list of file extensions to treat as test files. config.suffixes = [] -# is_early; Request to run this suite early. -config.is_early = True +# start_phase; Request to start this suite early. +config.start_phase = -1 # test_source_root: The root path where tests are located. # test_exec_root: The root path where tests should be run. diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py --- a/llvm/utils/lit/lit/Test.py +++ b/llvm/utils/lit/lit/Test.py @@ -396,14 +396,19 @@ identifiers = set(filter(BooleanExpression.isIdentifier, tokens)) return identifiers - def isEarlyTest(self): + def getTestPhase(self): """ - isEarlyTest() -> bool + getTestPhase() -> int - Check whether this test should be executed early in a particular run. - This can be used for test suites with long running tests to maximize - parallelism or where it is desirable to surface their failures early. + Get the phase that a particular test should start in. This can be used + for test suites with long running tests to maximize concurrency or where + it is desirable to surface their failures early. It can also be used to + run fast and low reward tests late. """ - if '/'.join(self.path_in_suite) in self.suite.config.early_tests: - return True - return self.suite.config.is_early + + base_phase = self.suite.config.start_phase + for depth in range(len(self.path_in_suite), 0, -1): + path = '/'.join(self.path_in_suite[:depth]) + if path in self.suite.config.start_phases: + return base_phase + self.suite.config.start_phases[path] + return base_phase diff --git a/llvm/utils/lit/lit/TestingConfig.py b/llvm/utils/lit/lit/TestingConfig.py --- a/llvm/utils/lit/lit/TestingConfig.py +++ b/llvm/utils/lit/lit/TestingConfig.py @@ -125,10 +125,9 @@ # require one of the features in this list if this list is non-empty. # Configurations can set this list to restrict the set of tests to run. self.limit_to_features = set(limit_to_features) - # Whether the suite should be tested early in a given run. - self.is_early = bool(is_early) - # List of tests to run early. - self.early_tests = {} + # Phased start ordering of tests or test prefixes. + self.start_phase = -1 if is_early else 0 + self.start_phases = {} self.parallelism_group = parallelism_group self._recursiveExpansionLimit = None diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py --- a/llvm/utils/lit/lit/cl_arguments.py +++ b/llvm/utils/lit/lit/cl_arguments.py @@ -9,7 +9,7 @@ class TestOrder(enum.Enum): - EARLY_TESTS_THEN_BY_NAME = enum.auto() + PHASE_THEN_BY_NAME = enum.auto() FAILING_FIRST = enum.auto() RANDOM = enum.auto() @@ -209,7 +209,7 @@ elif opts.incremental: opts.order = TestOrder.FAILING_FIRST else: - opts.order = TestOrder.EARLY_TESTS_THEN_BY_NAME + opts.order = TestOrder.PHASE_THEN_BY_NAME if opts.numShards or opts.runShard: if not opts.numShards or not opts.runShard: diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py --- a/llvm/utils/lit/lit/main.py +++ b/llvm/utils/lit/lit/main.py @@ -160,8 +160,8 @@ def determine_order(tests, order): from lit.cl_arguments import TestOrder - if order == TestOrder.EARLY_TESTS_THEN_BY_NAME: - tests.sort(key=lambda t: (not t.isEarlyTest(), t.getFullName())) + if order == TestOrder.PHASE_THEN_BY_NAME: + tests.sort(key=lambda t: (t.getTestPhase(), t.getFullName())) elif order == TestOrder.FAILING_FIRST: def by_mtime(test): return os.path.getmtime(test.getFilePath()) diff --git a/llvm/utils/lit/tests/Inputs/early-tests/lit.cfg b/llvm/utils/lit/tests/Inputs/early-tests/lit.cfg deleted file mode 100644 --- a/llvm/utils/lit/tests/Inputs/early-tests/lit.cfg +++ /dev/null @@ -1,7 +0,0 @@ -import lit.formats -config.name = 'early-tests' -config.suffixes = ['.txt'] -config.test_format = lit.formats.ShTest() -config.test_source_root = None -config.test_exec_root = None -config.early_tests = { "subdir/ccc.txt" } diff --git a/llvm/utils/lit/tests/Inputs/start-phases/lit.cfg b/llvm/utils/lit/tests/Inputs/start-phases/lit.cfg new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/start-phases/lit.cfg @@ -0,0 +1,15 @@ +import lit.formats +config.name = 'early-tests' +config.suffixes = ['.txt'] +config.test_format = lit.formats.ShTest() +config.test_source_root = None +config.test_exec_root = None +config.start_phases = { + "subdir" : -4, + "subdir/early.txt" : -3, + "test9.txt" : -2, + "test8.txt" : -1, + "test2.txt" : +1, + "test1.txt" : +2, + "subdir/late.txt" : +3, +} diff --git a/llvm/utils/lit/tests/Inputs/early-tests/aaa.txt b/llvm/utils/lit/tests/Inputs/start-phases/subdir/early.txt rename from llvm/utils/lit/tests/Inputs/early-tests/aaa.txt rename to llvm/utils/lit/tests/Inputs/start-phases/subdir/early.txt diff --git a/llvm/utils/lit/tests/Inputs/early-tests/bbb.txt b/llvm/utils/lit/tests/Inputs/start-phases/subdir/late.txt rename from llvm/utils/lit/tests/Inputs/early-tests/bbb.txt rename to llvm/utils/lit/tests/Inputs/start-phases/subdir/late.txt diff --git a/llvm/utils/lit/tests/Inputs/early-tests/subdir/ccc.txt b/llvm/utils/lit/tests/Inputs/start-phases/subdir/misc.txt rename from llvm/utils/lit/tests/Inputs/early-tests/subdir/ccc.txt rename to llvm/utils/lit/tests/Inputs/start-phases/subdir/misc.txt diff --git a/llvm/utils/lit/tests/Inputs/start-phases/test1.txt b/llvm/utils/lit/tests/Inputs/start-phases/test1.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/start-phases/test1.txt @@ -0,0 +1 @@ +# RUN: true diff --git a/llvm/utils/lit/tests/Inputs/start-phases/test2.txt b/llvm/utils/lit/tests/Inputs/start-phases/test2.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/start-phases/test2.txt @@ -0,0 +1 @@ +# RUN: true diff --git a/llvm/utils/lit/tests/Inputs/start-phases/test5.txt b/llvm/utils/lit/tests/Inputs/start-phases/test5.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/start-phases/test5.txt @@ -0,0 +1 @@ +# RUN: true diff --git a/llvm/utils/lit/tests/Inputs/start-phases/test8.txt b/llvm/utils/lit/tests/Inputs/start-phases/test8.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/start-phases/test8.txt @@ -0,0 +1 @@ +# RUN: true diff --git a/llvm/utils/lit/tests/Inputs/start-phases/test9.txt b/llvm/utils/lit/tests/Inputs/start-phases/test9.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/start-phases/test9.txt @@ -0,0 +1 @@ +# RUN: true diff --git a/llvm/utils/lit/tests/early-tests.py b/llvm/utils/lit/tests/early-tests.py deleted file mode 100644 --- a/llvm/utils/lit/tests/early-tests.py +++ /dev/null @@ -1,9 +0,0 @@ -## Check that we can run tests early. - -# RUN: %{lit} -j1 %{inputs}/early-tests | FileCheck %s - -# CHECK: -- Testing: 3 tests, 1 workers -- -# CHECK-NEXT: PASS: early-tests :: subdir/ccc.txt -# CHECK-NEXT: PASS: early-tests :: aaa.txt -# CHECK-NEXT: PASS: early-tests :: bbb.txt -# CHECK: Passed: 3 diff --git a/llvm/utils/lit/tests/start-phases.py b/llvm/utils/lit/tests/start-phases.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/start-phases.py @@ -0,0 +1,15 @@ +## Check that we can phase order the start of tests. + +# RUN: %{lit} -j1 %{inputs}/start-phases | FileCheck %s +# END. + +# CHECK: -- Testing: 8 tests, 1 workers -- +# CHECK-NEXT: PASS: early-tests :: subdir/misc.txt +# CHECK-NEXT: PASS: early-tests :: subdir/early.txt +# CHECK-NEXT: PASS: early-tests :: test9.txt +# CHECK-NEXT: PASS: early-tests :: test8.txt +# CHECK-NEXT: PASS: early-tests :: test5.txt +# CHECK-NEXT: PASS: early-tests :: test2.txt +# CHECK-NEXT: PASS: early-tests :: test1.txt +# CHECK-NEXT: PASS: early-tests :: subdir/late.txt +# CHECK: Passed: 8 diff --git a/mlir/test/Unit/lit.cfg.py b/mlir/test/Unit/lit.cfg.py --- a/mlir/test/Unit/lit.cfg.py +++ b/mlir/test/Unit/lit.cfg.py @@ -13,8 +13,8 @@ # suffixes: A list of file extensions to treat as test files. config.suffixes = [] -# is_early; Request to run this suite early. -config.is_early = True +# start_phase; Request to start this suite early. +config.start_phase = -1 # test_source_root: The root path where tests are located. # test_exec_root: The root path where tests should be run.