Revert: "[lit] Remove --repeat option, which wasn't that useful."

[jroelofs]

Summary

This reverts commit b647d5d21dd8fa22b6ff849563809b7763dcf57e.

Repeating tests this way isn't all that useful for gathering timing information, but it is quite useful for re-running flaky tests (for example when running a git-bisect script).

llvm/docs/CommandGuide/lit.rst

 .. option:: --show-suites
 
  List the discovered test suites and exit.
 
 .. option:: --show-tests
 
  List all of the discovered tests and exit.
 
+.. option:: --repeat=N
+
+ Run each test ``N`` times.  Currently this is primarily useful for catching
+ failures in flaky tests.
+
 EXIT STATUS
 -----------
 
 :program:`lit` will exit with an exit code of 1 if there are any FAIL or XPASS
 results.  Otherwise, it will exit with the status 0.  Other exit codes are used
 for non-test related failures (for example a user error or an internal program
 error).
 

llvm/utils/lit/lit/Test.py

def __init__(self, suite, path_in_suite, config, file_path = None):
 
         # An optional number of retries allowed before the test finally succeeds.
         # The test is run at most once plus the number of retries specified here.
         self.allowed_retries = getattr(config, 'test_retry_attempts', 0)
 
         # The test result, once complete.
         self.result = None
 
+        # The repeat index of this test, or None.
+        self.index = None
+
+    def copyWithIndex(self, index):
+        import copy
+        res = copy.copy(self)
+        res.index = index
+        return res
+
     def setResult(self, result):
         assert self.result is None, "result already set"
         assert isinstance(result, Result), "unexpected result type"
         try:
             expected_to_fail = self.isExpectedToFail()
         except ValueError as err:
             # Syntax error in an XFAIL line.
             result.code = UNRESOLVED

llvm/utils/lit/lit/TestRunner.py

 
 def getTempPaths(test):
     """Get the temporary location, this is always relative to the test suite
     root, not test source root."""
     execpath = test.getExecPath()
     execdir,execbase = os.path.split(execpath)
     tmpDir = os.path.join(execdir, 'Output')
     tmpBase = os.path.join(tmpDir, execbase)
+    if test.index is not None:
+        tmpBase += '_%d' % test.index
     return tmpDir, tmpBase
 
 def colonNormalizePath(path):
     if kIsWindows:
         return re.sub(r'^(.):', r'\1', path.replace('\\', '/'))
     else:
         assert path[0] == '/'
         return path[1:]

llvm/utils/lit/lit/cl_arguments.py

debug_group.add_argument("--debug",
             help="Enable debugging (for 'lit' development)",
             action="store_true")
     debug_group.add_argument("--show-suites",
             help="Show discovered test suites and exit",
             action="store_true")
     debug_group.add_argument("--show-tests",
             help="Show all discovered tests and exit",
             action="store_true")
+    debug_group.add_argument("--repeat",
+            dest="repeatTests",
+            metavar="N",
+            help="Repeat tests N times",
+            type=_positive_int)
     debug_group.add_argument("--show-used-features",
             help="Show all features used in the test suite (in XFAIL, UNSUPPORTED and REQUIRES) and exit",
             action="store_true")
 
     # LIT is special: environment variables override command line arguments.
     env_args = shlex.split(os.environ.get("LIT_OPTS", ""))
     args = sys.argv[1:] + env_args
     opts = parser.parse_args(args)

llvm/utils/lit/lit/main.py

if opts.maxIndividualTestTime is not None:  # `not None` is important (default: 0)
                 ' requested on the command line. Forcing timeout to be {1}'
                 ' seconds')
                 .format(lit_config.maxIndividualTestTime,
                         opts.maxIndividualTestTime))
             lit_config.maxIndividualTestTime = opts.maxIndividualTestTime
 
     determine_order(discovered_tests, opts.order)
 
+    if opts.repeatTests:
+        discovered_tests = [t.copyWithIndex(i)
+                            for t in discovered_tests
+                            for i in range(opts.repeatTests)]
+
     selected_tests = [t for t in discovered_tests if
                       opts.filter.search(t.getFullName())]
     if not selected_tests:
         sys.stderr.write('error: filter did not match any tests '
                          '(of %d discovered).  ' % len(discovered_tests))
         if opts.allow_empty_runs:
             sys.stderr.write("Suppressing error because '--allow-empty-runs' "
                              'was specified.\n')