Index: llvm/utils/lit/lit/cl_arguments.py =================================================================== --- llvm/utils/lit/lit/cl_arguments.py +++ llvm/utils/lit/lit/cl_arguments.py @@ -11,6 +11,7 @@ class TestOrder(enum.Enum): DEFAULT = enum.auto() RANDOM = enum.auto() + LEXICAL = enum.auto() def parse_args(): @@ -150,9 +151,22 @@ metavar="N", help="Maximum time to spend testing (in seconds)", type=_positive_int) + selection_group.add_argument("--default-order", + dest="order", + help="Run tests in default order", + action="store_const", + const=TestOrder.DEFAULT, + default=TestOrder.DEFAULT) + selection_group.add_argument("--lexical-order", + dest="order", + help="Run tests in lexical order", + action="store_const", + const=TestOrder.LEXICAL) selection_group.add_argument("--shuffle", + dest="order", help="Run tests in random order", - action="store_true") + action="store_const", + const=TestOrder.RANDOM) selection_group.add_argument("-i", "--incremental", help="Run failed tests first (DEPRECATED: now always enabled)", action="store_true") @@ -215,11 +229,6 @@ if opts.incremental: print('WARNING: --incremental is deprecated. Failing tests now always run first.') - if opts.shuffle: - opts.order = TestOrder.RANDOM - else: - opts.order = TestOrder.DEFAULT - if opts.numShards or opts.runShard: if not opts.numShards or not opts.runShard: parser.error("--num-shards and --run-shard must be used together") Index: llvm/utils/lit/lit/main.py =================================================================== --- llvm/utils/lit/lit/main.py +++ llvm/utils/lit/lit/main.py @@ -169,6 +169,8 @@ if order == TestOrder.RANDOM: import random random.shuffle(tests) + elif order == TestOrder.LEXICAL: + tests.sort(key=lambda t: t.getFullName()) else: assert order == TestOrder.DEFAULT, 'Unknown TestOrder value' tests.sort(key=lambda t: (not t.previous_failure, -t.previous_elapsed, t.getFullName())) Index: llvm/utils/lit/tests/custom-result-category.py =================================================================== --- llvm/utils/lit/tests/custom-result-category.py +++ llvm/utils/lit/tests/custom-result-category.py @@ -1,9 +1,6 @@ # UNSUPPORTED: system-windows # Test lit.main.add_result_category() extension API. -# FIXME: this test depends on order of tests -# RUN: rm -f %{inputs}/custom-result-category/.lit_test_times.txt - # RUN: not %{lit} -j 1 %{inputs}/custom-result-category | FileCheck %s # CHECK: CUSTOM_PASS: custom-result-category :: test1.txt Index: llvm/utils/lit/tests/googletest-format.py =================================================================== --- llvm/utils/lit/tests/googletest-format.py +++ llvm/utils/lit/tests/googletest-format.py @@ -1,8 +1,5 @@ # Check the various features of the GoogleTest format. -# FIXME: this test depends on order of tests -# RUN: rm -f %{inputs}/googletest-format/.lit_test_times.txt - # RUN: not %{lit} -j 1 -v %{inputs}/googletest-format > %t.out # FIXME: Temporarily dump test output so we can debug failing tests on # buildbots. Index: llvm/utils/lit/tests/googletest-upstream-format.py =================================================================== --- llvm/utils/lit/tests/googletest-upstream-format.py +++ llvm/utils/lit/tests/googletest-upstream-format.py @@ -1,8 +1,5 @@ # Check the various features of the GoogleTest format. -# FIXME: this test depends on order of tests -# RUN: rm -f %{inputs}/googletest-upstream-format/.lit_test_times.txt - # RUN: not %{lit} -j 1 -v %{inputs}/googletest-upstream-format > %t.out # RUN: FileCheck < %t.out %s # Index: llvm/utils/lit/tests/lit.cfg =================================================================== --- llvm/utils/lit/tests/lit.cfg +++ llvm/utils/lit/tests/lit.cfg @@ -61,7 +61,7 @@ config.substitutions.append(('%{inputs}', os.path.join( config.test_source_root, 'Inputs'))) config.substitutions.append(('%{lit}', - "{env} %{{python}} {lit}".format( + "{env} %{{python}} {lit} --lexical-order".format( env="env -u FILECHECK_OPTS", lit=os.path.join(lit_path, 'lit.py')))) config.substitutions.append(('%{python}', '"%s"' % (sys.executable))) Index: llvm/utils/lit/tests/progress-bar.py =================================================================== --- llvm/utils/lit/tests/progress-bar.py +++ llvm/utils/lit/tests/progress-bar.py @@ -1,8 +1,5 @@ # Check the simple progress bar. -# FIXME: this test depends on order of tests -# RUN: rm -f %{inputs}/progress-bar/.lit_test_times.txt - # RUN: not %{lit} -j 1 -s %{inputs}/progress-bar > %t.out # RUN: FileCheck < %t.out %s # Index: llvm/utils/lit/tests/reorder.py =================================================================== --- llvm/utils/lit/tests/reorder.py +++ llvm/utils/lit/tests/reorder.py @@ -1,7 +1,7 @@ ## Check that we can reorder test runs. # RUN: cp %{inputs}/reorder/.lit_test_times.txt %{inputs}/reorder/.lit_test_times.txt.orig -# RUN: %{lit} -j1 %{inputs}/reorder > %t.out +# RUN: %{lit} --default-order -j1 %{inputs}/reorder > %t.out # RUN: cp %{inputs}/reorder/.lit_test_times.txt %{inputs}/reorder/.lit_test_times.txt.new # RUN: cp %{inputs}/reorder/.lit_test_times.txt.orig %{inputs}/reorder/.lit_test_times.txt # RUN: not diff %{inputs}/reorder/.lit_test_times.txt.new %{inputs}/reorder/.lit_test_times.txt.orig Index: llvm/utils/lit/tests/shtest-env.py =================================================================== --- llvm/utils/lit/tests/shtest-env.py +++ llvm/utils/lit/tests/shtest-env.py @@ -1,8 +1,5 @@ # Check the env command -# FIXME: this test depends on order of tests -# RUN: rm -f %{inputs}/shtest-env/.lit_test_times.txt - # RUN: not %{lit} -j 1 -a -v %{inputs}/shtest-env \ # RUN: | FileCheck -match-full-lines %s # Index: llvm/utils/lit/tests/shtest-format.py =================================================================== --- llvm/utils/lit/tests/shtest-format.py +++ llvm/utils/lit/tests/shtest-format.py @@ -1,8 +1,5 @@ # Check the various features of the ShTest format. -# FIXME: this test depends on order of tests -# RUN: rm -f %{inputs}/shtest-format/.lit_test_times.txt - # RUN: rm -f %t.xml # RUN: not %{lit} -j 1 -v %{inputs}/shtest-format --xunit-xml-output %t.xml > %t.out # RUN: FileCheck < %t.out %s Index: llvm/utils/lit/tests/shtest-keyword-parse-errors.py =================================================================== --- llvm/utils/lit/tests/shtest-keyword-parse-errors.py +++ llvm/utils/lit/tests/shtest-keyword-parse-errors.py @@ -1,6 +1,3 @@ -# FIXME: this test depends on order of tests -# RUN: rm -f %{inputs}/shtest-keyword-parse-errors/.lit_test_times.txt - # RUN: not %{lit} -j 1 -vv %{inputs}/shtest-keyword-parse-errors > %t.out # RUN: FileCheck -input-file %t.out %s # Index: llvm/utils/lit/tests/shtest-not.py =================================================================== --- llvm/utils/lit/tests/shtest-not.py +++ llvm/utils/lit/tests/shtest-not.py @@ -1,8 +1,5 @@ # Check the not command -# FIXME: this test depends on order of tests -# RUN: rm -f %{inputs}/shtest-not/.lit_test_times.txt - # RUN: not %{lit} -j 1 -a -v %{inputs}/shtest-not \ # RUN: | FileCheck -match-full-lines %s # Index: llvm/utils/lit/tests/shtest-run-at-line.py =================================================================== --- llvm/utils/lit/tests/shtest-run-at-line.py +++ llvm/utils/lit/tests/shtest-run-at-line.py @@ -1,9 +1,6 @@ # Check that -vv makes the line number of the failing RUN command clear. # (-v is actually sufficient in the case of the internal shell.) -# FIXME: this test depends on order of tests -# RUN: rm -f %{inputs}/shtest-run-at-line/.lit_test_times.txt - # RUN: not %{lit} -j 1 -vv %{inputs}/shtest-run-at-line > %t.out # RUN: FileCheck --input-file %t.out %s # Index: llvm/utils/lit/tests/shtest-shell.py =================================================================== --- llvm/utils/lit/tests/shtest-shell.py +++ llvm/utils/lit/tests/shtest-shell.py @@ -1,8 +1,5 @@ # Check the internal shell handling component of the ShTest format. -# FIXME: this test depends on order of tests -# RUN: rm -f %{inputs}/shtest-shell/.lit_test_times.txt - # RUN: not %{lit} -j 1 -v %{inputs}/shtest-shell > %t.out # FIXME: Temporarily dump test output so we can debug failing tests on # buildbots. @@ -11,8 +8,6 @@ # # Test again in non-UTF shell to catch potential errors with python 2 seen # on stdout-encoding.txt -# FIXME: lit's testing sets source_root == exec_root which complicates running lit more than once per test. -# RUN: rm -f %{inputs}/shtest-shell/.lit_test_times.txt # RUN: env PYTHONIOENCODING=ascii not %{lit} -j 1 -a %{inputs}/shtest-shell > %t.ascii.out # FIXME: Temporarily dump test output so we can debug failing tests on # buildbots.