Index: projects/compiler-rt/test/asan/Unit/lit.site.cfg.in =================================================================== --- projects/compiler-rt/test/asan/Unit/lit.site.cfg.in +++ projects/compiler-rt/test/asan/Unit/lit.site.cfg.in @@ -27,3 +27,6 @@ # Set LD_LIBRARY_PATH to pick dynamic runtime up properly. push_ld_library_path(config, config.compiler_rt_libdir) + +if config.host_os == 'Darwin': + config.parallelism_group = config.darwin_sanitizer_parallelism_group_func Index: projects/compiler-rt/test/asan/lit.cfg =================================================================== --- projects/compiler-rt/test/asan/lit.cfg +++ projects/compiler-rt/test/asan/lit.cfg @@ -241,3 +241,6 @@ # Only run the tests on supported OSs. if config.host_os not in ['Linux', 'Darwin', 'FreeBSD', 'Windows']: config.unsupported = True + +if config.host_os == 'Darwin' and config.target_arch in ["x86_64", "x86_64h"]: + config.parallelism_group = "darwin-64bit-sanitizer" Index: projects/compiler-rt/test/lit.common.cfg =================================================================== --- projects/compiler-rt/test/lit.common.cfg +++ projects/compiler-rt/test/lit.common.cfg @@ -216,3 +216,9 @@ # retries. We don't do this on otther platforms because it's slower. if platform.system() == 'Windows': config.test_retry_attempts = 2 + +# Only run up to 3 64-bit sanitized processes simultaneously on Darwin. +# Using more scales badly and hogs the system due to inefficient handling +# of large mmap'd regions (terabytes) by the kernel. +if platform.system() == 'Darwin': + lit_config.parallelism_groups["darwin-64bit-sanitizer"] = 3 Index: projects/compiler-rt/test/tsan/Unit/lit.site.cfg.in =================================================================== --- projects/compiler-rt/test/tsan/Unit/lit.site.cfg.in +++ projects/compiler-rt/test/tsan/Unit/lit.site.cfg.in @@ -11,3 +11,6 @@ # FIXME: De-hardcode this path. config.test_exec_root = "@COMPILER_RT_BINARY_DIR@/lib/tsan/tests" config.test_source_root = config.test_exec_root + +if config.host_os == 'Darwin': + config.parallelism_group = config.darwin_sanitizer_parallelism_group_func Index: projects/compiler-rt/test/tsan/lit.cfg =================================================================== --- projects/compiler-rt/test/tsan/lit.cfg +++ projects/compiler-rt/test/tsan/lit.cfg @@ -83,3 +83,6 @@ # because the test hangs. if config.target_arch != 'aarch64': config.available_features.add('stable-runtime') + +if config.host_os == 'Darwin' and config.target_arch in ["x86_64", "x86_64h"]: + config.parallelism_group = "darwin-64bit-sanitizer" Index: projects/compiler-rt/unittests/lit.common.unit.cfg =================================================================== --- projects/compiler-rt/unittests/lit.common.unit.cfg +++ projects/compiler-rt/unittests/lit.common.unit.cfg @@ -28,3 +28,13 @@ config.environment['TMP'] = os.environ['TMP'] if 'TEMP' in os.environ: config.environment['TEMP'] = os.environ['TEMP'] + +if config.host_os == 'Darwin': + # Only run up to 3 64-bit sanitized processes simultaneously on Darwin. + # Using more scales badly and hogs the system due to inefficient handling + # of large mmap'd regions (terabytes) by the kernel. + lit_config.parallelism_groups["darwin-64bit-sanitizer"] = 3 + + def darwin_sanitizer_parallelism_group_func(test): + return "darwin-64bit-sanitizer" if "x86_64" in test.file_path else "" + config.darwin_sanitizer_parallelism_group_func = darwin_sanitizer_parallelism_group_func Index: utils/lit/lit/LitConfig.py =================================================================== --- utils/lit/lit/LitConfig.py +++ utils/lit/lit/LitConfig.py @@ -24,7 +24,8 @@ noExecute, debug, isWindows, params, config_prefix = None, maxIndividualTestTime = 0, - maxFailures = None): + maxFailures = None, + parallelism_groups = []): # The name of the test runner. self.progname = progname # The items to add to the PATH environment variable. @@ -62,6 +63,7 @@ self.maxIndividualTestTime = maxIndividualTestTime self.maxFailures = maxFailures + self.parallelism_groups = parallelism_groups @property def maxIndividualTestTime(self): Index: utils/lit/lit/TestingConfig.py =================================================================== --- utils/lit/lit/TestingConfig.py +++ utils/lit/lit/TestingConfig.py @@ -106,7 +106,7 @@ environment, substitutions, unsupported, test_exec_root, test_source_root, excludes, available_features, pipefail, limit_to_features = [], - is_early = False): + is_early = False, parallelism_group = ""): self.parent = parent self.name = str(name) self.suffixes = set(suffixes) @@ -125,6 +125,7 @@ self.limit_to_features = set(limit_to_features) # Whether the suite should be tested early in a given run. self.is_early = bool(is_early) + self.parallelism_group = parallelism_group def finish(self, litConfig): """finish() - Finish this config object, after loading is complete.""" Index: utils/lit/lit/main.py =================================================================== --- utils/lit/lit/main.py +++ utils/lit/lit/main.py @@ -327,7 +327,8 @@ params = userParams, config_prefix = opts.configPrefix, maxIndividualTestTime = maxIndividualTestTime, - maxFailures = opts.maxFailures) + maxFailures = opts.maxFailures, + parallelism_groups = {}) # Perform test discovery. run = lit.run.Run(litConfig, Index: utils/lit/lit/run.py =================================================================== --- utils/lit/lit/run.py +++ utils/lit/lit/run.py @@ -177,9 +177,15 @@ self.tests = tests def execute_test(self, test): + pg = test.config.parallelism_group + if callable(pg): pg = pg(test) + result = None - start_time = time.time() + semaphore = None try: + if pg: semaphore = self.parallelism_semaphores[pg] + if semaphore: semaphore.acquire() + start_time = time.time() result = test.config.test_format.execute(test, self.lit_config) # Support deprecated result from execute() which returned the result @@ -189,6 +195,8 @@ result = lit.Test.Result(code, output) elif not isinstance(result, lit.Test.Result): raise ValueError("unexpected result from test execution") + + result.elapsed = time.time() - start_time except KeyboardInterrupt: raise except: @@ -198,7 +206,8 @@ output += traceback.format_exc() output += '\n' result = lit.Test.Result(lit.Test.UNRESOLVED, output) - result.elapsed = time.time() - start_time + finally: + if semaphore: semaphore.release() test.setResult(result) @@ -231,6 +240,7 @@ try: task_impl = multiprocessing.Process queue_impl = multiprocessing.Queue + sem_impl = multiprocessing.Semaphore canceled_flag = multiprocessing.Value('i', 0) consumer = MultiprocessResultsConsumer(self, display, jobs) except: @@ -242,9 +252,13 @@ if not consumer: task_impl = threading.Thread queue_impl = queue.Queue + sem_impl = threading.Semaphore canceled_flag = LockedValue(0) consumer = ThreadResultsConsumer(display) + self.parallelism_semaphores = {k: sem_impl(v) + for k, v in self.lit_config.parallelism_groups.items()} + # Create the test provider. provider = TestProvider(queue_impl, canceled_flag) handleFailures(provider, consumer, self.lit_config.maxFailures)