[Darwin][NFC] Refactor throttling of 64bit sanitizer tests on Darwin

[yln]

Summary

Underlying condition for throttling is "has large mmap'd regions" (i.e.,
shadow memory) and not sanitizers in general (e.g., UBSan does not need
to be throttled).

Rename parallelism group darwin-64bit-sanitizer to shadow-memory and
apply it unconditionally to all tests which require it. We can then have
all the Darwin throttling logic in one place in the commen lit config.

Throttle sanitizer_common unit tests. Configuration was previously
missing from sanitizer_common/Unit/lit.site.cfg.

[yln]

Close manually (accidentally used wrong revision ID in commit message).
https://github.com/llvm/llvm-project/commit/6eef7d05249fb88bf5b31289f4c27d75f50c1ae4

compiler-rt/test/asan/lit.cfg

else:
   config.substitutions.append(('%fPIC', '-fPIC'))
   config.substitutions.append(('%fPIE', '-fPIE'))
   config.substitutions.append(('%pie', '-pie'))
 
 # Only run the tests on supported OSs.
 if config.host_os not in ['Linux', 'Darwin', 'FreeBSD', 'SunOS', 'Windows', 'NetBSD']:
   config.unsupported = True
 
-if config.host_os == 'Darwin':
-  if config.target_arch in ["x86_64", "x86_64h"]:
-    config.parallelism_group = "darwin-64bit-sanitizer"
+config.parallelism_group = 'shadow-memory'

compiler-rt/test/fuzzer/lit.cfg

 
 default_asan_opts_str = ':'.join(config.default_sanitizer_opts)
 if default_asan_opts_str:
   config.environment['ASAN_OPTIONS'] = default_asan_opts_str
   default_asan_opts_str += ':'
 config.substitutions.append(('%env_asan_opts=',
                              'env ASAN_OPTIONS=' + default_asan_opts_str))
 
-if config.host_os == 'Darwin':
-  if config.target_arch in ["x86_64", "x86_64h"]:
-    config.parallelism_group = "darwin-64bit-sanitizer"
+config.parallelism_group = 'shadow-memory'

compiler-rt/test/lit.common.cfg

if re.search(r'ON', llvm_config_cmd.stdout.read().decode('ascii')):
   config.available_features.add('asserts')
 llvm_config_cmd.wait()
 
 # Sanitizer tests tend to be flaky on Windows due to PR24554, so add some
 # retries. We don't do this on otther platforms because it's slower.
 if platform.system() == 'Windows':
   config.test_retry_attempts = 2
 
-# Only run up to 3 64-bit sanitized processes simultaneously on Darwin.
-# Using more scales badly and hogs the system due to inefficient handling
-# of large mmap'd regions (terabytes) by the kernel.
-if platform.system() == 'Darwin':
-  lit_config.parallelism_groups["darwin-64bit-sanitizer"] = 3
+# No throttling on non-Darwin platforms.
+lit_config.parallelism_groups['shadow-memory'] = None
 
+if platform.system() == 'Darwin':
+  ios_device = config.apple_platform != 'osx' and not config.apple_platform.endswith('sim')
   # Force sequential execution when running tests on iOS devices.
-if config.host_os == 'Darwin' and config.apple_platform != "osx" and not config.apple_platform.endswith("sim"):
-  lit_config.warning("iOS device test cases being run sequentially")
-  lit_config.parallelism_groups["ios-device"] = 1
-  config.parallelism_group = "ios-device"
+  if ios_device:
+    lit_config.warning('Forcing sequential execution for iOS device tests')
+    lit_config.parallelism_groups['ios-device'] = 1
+    config.parallelism_group = 'ios-device'
+
+  # Only run up to 3 processes that require shadow memory simultaneously on
+  # 64-bit Darwin. Using more scales badly and hogs the system due to
+  # inefficient handling of large mmap'd regions (terabytes) by the kernel.
+  elif config.target_arch in ['x86_64', 'x86_64h']:
+    lit_config.warning('Throttling sanitizer tests that require shadow memory on Darwin 64bit')
+    lit_config.parallelism_groups['shadow-memory'] = 3
 
 # Multiple substitutions are necessary to support multiple shared objects used
 # at once.
 # Note that substitutions with numbers have to be defined first to avoid
 # being subsumed by substitutions with smaller postfix.
 for postfix in ["2", "1", ""]:
   if config.host_os == 'Darwin':
     config.substitutions.append( ("%ld_flags_rpath_exe" + postfix, '-Wl,-rpath,@executable_path/ %dynamiclib' + postfix) )

compiler-rt/test/sanitizer_common/Unit/lit.site.cfg.in

 @LIT_SITE_CFG_IN_HEADER@
 
 # Load common config for all compiler-rt unit tests.
 lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/unittests/lit.common.unit.configured")
 
 # Setup config name.
 config.name = 'SanitizerCommon-Unit'
 
 # Setup test source and exec root. For unit tests, we define
 # it as build directory with sanitizer_common tests.
 # FIXME: De-hardcode this path.
 config.test_exec_root = os.path.join("@COMPILER_RT_BINARY_DIR@", "lib",
                                      "sanitizer_common", "tests")
 config.test_source_root = config.test_exec_root
+
+if config.host_os == 'Darwin':
+  config.parallelism_group = config.darwin_sanitizer_parallelism_group_func

compiler-rt/test/sanitizer_common/lit.common.cfg

 config.substitutions.append( ('%env_tool_opts=',
                               'env ' + tool_options + '=' + default_tool_options_str))
 
 config.suffixes = ['.c', '.cc', '.cpp']
 
 if config.host_os not in ['Linux', 'Darwin', 'NetBSD', 'FreeBSD']:
   config.unsupported = True
 
-if config.host_os == 'Darwin':
-  if config.target_arch in ["x86_64", "x86_64h"]:
-    config.parallelism_group = "darwin-64bit-sanitizer"
+config.parallelism_group = 'shadow-memory'

compiler-rt/test/tsan/lit.cfg

 config.suffixes = ['.c', '.cc', '.cpp', '.m', '.mm']
 
 if config.host_os not in ['FreeBSD', 'Linux', 'Darwin', 'NetBSD']:
   config.unsupported = True
 
 if config.android:
   config.unsupported = True
 
-if config.host_os == 'Darwin':
-  if config.target_arch in ["x86_64", "x86_64h"]:
-    config.parallelism_group = "darwin-64bit-sanitizer"
+config.parallelism_group = 'shadow-memory'

compiler-rt/unittests/lit.common.unit.cfg

 # Propagate the temp directory. Windows requires this because it uses \Windows\
 # if none of these are present.
 if 'TMP' in os.environ:
     config.environment['TMP'] = os.environ['TMP']
 if 'TEMP' in os.environ:
     config.environment['TEMP'] = os.environ['TEMP']
 
 if config.host_os == 'Darwin':
-  # Only run up to 3 64-bit sanitized processes simultaneously on Darwin.
-  # Using more scales badly and hogs the system due to inefficient handling
-  # of large mmap'd regions (terabytes) by the kernel.
-  lit_config.parallelism_groups["darwin-64bit-sanitizer"] = 3
+  # Only run up to 3 processes that require shadow memory simultaneously on
+  # 64-bit Darwin. Using more scales badly and hogs the system due to
+  # inefficient handling of large mmap'd regions (terabytes) by the kernel.
+  lit_config.parallelism_groups["shadow-memory"] = 3
 
   # The test config gets pickled and sent to multiprocessing workers, and that
   # only works for code if it is stored at the top level of some module.
   # Therefore, we have to put the code in a .py file, add it to path, and import
   # it to store it in the config.
   import site
   site.addsitedir(os.path.dirname(__file__))
   import lit_unittest_cfg_utils
   config.darwin_sanitizer_parallelism_group_func = \
     lit_unittest_cfg_utils.darwin_sanitizer_parallelism_group_func

compiler-rt/unittests/lit_unittest_cfg_utils.py

-# Put all 64-bit sanitizer tests in the darwin-64bit-sanitizer parallelism
-# group. This will only run three of them concurrently.
+# Put all 64-bit tests in the shadow-memory parallelism group. We throttle those
+# in our common lit config (lit.common.unit.cfg).
 def darwin_sanitizer_parallelism_group_func(test):
-  return "darwin-64bit-sanitizer" if "x86_64" in test.file_path else ""
+  return "shadow-memory" if "x86_64" in test.file_path else None