diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst --- a/llvm/docs/CommandGuide/lit.rst +++ b/llvm/docs/CommandGuide/lit.rst @@ -224,6 +224,11 @@ List all of the discovered tests and exit. +.. option:: --repeat=N + + Run each test ``N`` times. Currently this is primarily useful for catching + failures in flaky tests. + EXIT STATUS ----------- diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py --- a/llvm/utils/lit/lit/Test.py +++ b/llvm/utils/lit/lit/Test.py @@ -244,6 +244,15 @@ # The test result, once complete. self.result = None + # The repeat index of this test, or None. + self.index = None + + def copyWithIndex(self, index): + import copy + res = copy.copy(self) + res.index = index + return res + def setResult(self, result): assert self.result is None, "result already set" assert isinstance(result, Result), "unexpected result type" diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py --- a/llvm/utils/lit/lit/TestRunner.py +++ b/llvm/utils/lit/lit/TestRunner.py @@ -1061,6 +1061,8 @@ execdir,execbase = os.path.split(execpath) tmpDir = os.path.join(execdir, 'Output') tmpBase = os.path.join(tmpDir, execbase) + if test.index is not None: + tmpBase += '_%d' % test.index return tmpDir, tmpBase def colonNormalizePath(path): diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py --- a/llvm/utils/lit/lit/cl_arguments.py +++ b/llvm/utils/lit/lit/cl_arguments.py @@ -163,6 +163,11 @@ debug_group.add_argument("--show-tests", help="Show all discovered tests and exit", action="store_true") + debug_group.add_argument("--repeat", + dest="repeatTests", + metavar="N", + help="Repeat tests N times", + type=_positive_int) debug_group.add_argument("--show-used-features", help="Show all features used in the test suite (in XFAIL, UNSUPPORTED and REQUIRES) and exit", action="store_true") diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py --- a/llvm/utils/lit/lit/main.py +++ b/llvm/utils/lit/lit/main.py @@ -66,6 +66,11 @@ determine_order(discovered_tests, opts.order) + if opts.repeatTests: + discovered_tests = [t.copyWithIndex(i) + for t in discovered_tests + for i in range(opts.repeatTests)] + selected_tests = [t for t in discovered_tests if opts.filter.search(t.getFullName())] if not selected_tests: