Index: llvm/trunk/utils/lit/lit/LitConfig.py =================================================================== --- llvm/trunk/utils/lit/lit/LitConfig.py +++ llvm/trunk/utils/lit/lit/LitConfig.py @@ -24,7 +24,8 @@ noExecute, debug, isWindows, params, config_prefix = None, maxIndividualTestTime = 0, - maxFailures = None): + maxFailures = None, + parallelism_groups = []): # The name of the test runner. self.progname = progname # The items to add to the PATH environment variable. @@ -62,6 +63,7 @@ self.maxIndividualTestTime = maxIndividualTestTime self.maxFailures = maxFailures + self.parallelism_groups = parallelism_groups @property def maxIndividualTestTime(self): Index: llvm/trunk/utils/lit/lit/TestingConfig.py =================================================================== --- llvm/trunk/utils/lit/lit/TestingConfig.py +++ llvm/trunk/utils/lit/lit/TestingConfig.py @@ -106,7 +106,7 @@ environment, substitutions, unsupported, test_exec_root, test_source_root, excludes, available_features, pipefail, limit_to_features = [], - is_early = False): + is_early = False, parallelism_group = ""): self.parent = parent self.name = str(name) self.suffixes = set(suffixes) @@ -125,6 +125,7 @@ self.limit_to_features = set(limit_to_features) # Whether the suite should be tested early in a given run. self.is_early = bool(is_early) + self.parallelism_group = parallelism_group def finish(self, litConfig): """finish() - Finish this config object, after loading is complete.""" Index: llvm/trunk/utils/lit/lit/main.py =================================================================== --- llvm/trunk/utils/lit/lit/main.py +++ llvm/trunk/utils/lit/lit/main.py @@ -313,6 +313,13 @@ isWindows = platform.system() == 'Windows' + parallelism_groups = {} + if platform.system() == 'Darwin': + # Only run up to 3 64-bit sanitized processes simultaneously on Darwin. + # Using more scales badly and hogs the system due to inefficient handling + # of large mmap'd regions (terabytes) by the kernel. + parallelism_groups["darwin-64bit-sanitizer"] = 3 + # Create the global config object. litConfig = lit.LitConfig.LitConfig( progname = os.path.basename(sys.argv[0]), @@ -327,7 +334,8 @@ params = userParams, config_prefix = opts.configPrefix, maxIndividualTestTime = maxIndividualTestTime, - maxFailures = opts.maxFailures) + maxFailures = opts.maxFailures, + parallelism_groups = parallelism_groups) # Perform test discovery. run = lit.run.Run(litConfig, Index: llvm/trunk/utils/lit/lit/run.py =================================================================== --- llvm/trunk/utils/lit/lit/run.py +++ llvm/trunk/utils/lit/lit/run.py @@ -175,9 +175,14 @@ def __init__(self, lit_config, tests): self.lit_config = lit_config self.tests = tests + self.parallelism_semaphores = "" def execute_test(self, test): + pg = test.config.parallelism_group + if callable(pg): pg = pg(test) + result = None + if pg: self.parallelism_semaphores[pg].acquire() start_time = time.time() try: result = test.config.test_format.execute(test, self.lit_config) @@ -189,6 +194,8 @@ result = lit.Test.Result(code, output) elif not isinstance(result, lit.Test.Result): raise ValueError("unexpected result from test execution") + + result.elapsed = time.time() - start_time except KeyboardInterrupt: raise except: @@ -198,7 +205,8 @@ output += traceback.format_exc() output += '\n' result = lit.Test.Result(lit.Test.UNRESOLVED, output) - result.elapsed = time.time() - start_time + finally: + if pg: self.parallelism_semaphores[pg].release() test.setResult(result) @@ -231,6 +239,7 @@ try: task_impl = multiprocessing.Process queue_impl = multiprocessing.Queue + sem_impl = multiprocessing.Semaphore canceled_flag = multiprocessing.Value('i', 0) consumer = MultiprocessResultsConsumer(self, display, jobs) except: @@ -242,9 +251,13 @@ if not consumer: task_impl = threading.Thread queue_impl = queue.Queue + sem_impl = threading.Semaphore canceled_flag = LockedValue(0) consumer = ThreadResultsConsumer(display) + self.parallelism_semaphores = {k: sem_impl(v) + for k, v in self.lit_config.parallelism_groups.items()} + # Create the test provider. provider = TestProvider(queue_impl, canceled_flag) handleFailures(provider, consumer, self.lit_config.maxFailures)