[libcxx] Allow manually specifying locales for use in tests via CMake

[broadwaylamb]

Summary

When running tests on a remote machine, it doesn't make sense to
detect supported locales on the host machine.

This patch introduces a way to manually list the locales that are
supported by the target machine in the LIBCXX_TEST_AVAILABLE_LOCALES
CMake variable.

[vsk]

Looks good to me. Please wait for another +1, as I'm not a regular libcxx contributor.

[jroelofs]

My long term plan for this was to have a set of "remote" TargetInfo objects that mirror/wrap these "local" ones, and have the SSHExecutor sub them in as appropriate, but this seems like a pragmatic compromise.

LGTM

[broadwaylamb]

In D72720#1823146, @jroelofs wrote:

My long term plan for this was to have a set of "remote" TargetInfo objects that mirror/wrap these "local" ones, and have the SSHExecutor sub them in as appropriate, but this seems like a pragmatic compromise.

I went ahead and added LinuxRemoteTI in D72847. Can I ask you to take a look? If there's anything there that needs more work, or even if I got you completely wrong, please let me know.

If D72847 is approved, I'll close this review.

[broadwaylamb]

Closing this in favor of D72847

libcxx/test/CMakeLists.txt

 macro(pythonize_bool var)
   if (${var})
     set(${var} True)
   else()
     set(${var} False)
   endif()
 endmacro()
 
 set(LIBCXX_LIT_VARIANT "libcxx" CACHE STRING
     "Configuration variant to use for LIT.")
 
 set(LIBCXX_TEST_LINKER_FLAGS "" CACHE STRING
     "Additonal linker flags to pass when compiling the tests")
 set(LIBCXX_TEST_COMPILER_FLAGS "" CACHE STRING
     "Additonal linker flags to pass when compiling the tests")
 
+set(LIBCXX_TEST_AVAILABLE_LOCALES "" CACHE STRING
+    "Semicolon-separated list of locales supported by the system
+     the tests will be run on. If empty, the locales will be
+     inferred from the host system.
+     Setting this variable is useful if the tests will be run
+     on a remote machine. For the list of supported locales see 
+     `add_common_locales` in utils/libcxx/test/target_info.py")
+
 # The tests shouldn't link to any ABI library when it has been linked into
 # libc++ statically or via a linker script.
 if (LIBCXX_ENABLE_STATIC_ABI_LIBRARY OR LIBCXX_ENABLE_ABI_LINKER_SCRIPT)
   set(LIBCXX_CXX_ABI_LIBNAME "none")
 endif()
 
 # The tests shouldn't link to libunwind if we have a linker script which
 # already does so.

libcxx/test/lit.site.cfg.in

 config.use_libatomic            = @LIBCXX_HAVE_CXX_ATOMICS_WITH_LIB@
 config.debug_build              = @LIBCXX_DEBUG_BUILD@
 config.libcxxabi_shared         = @LIBCXX_LINK_TESTS_WITH_SHARED_LIBCXXABI@
 config.cxx_ext_threads          = @LIBCXX_BUILD_EXTERNAL_THREAD_LIBRARY@
 config.pstl_src_root            = "@ParallelSTL_SOURCE_DIR@" if @LIBCXX_ENABLE_PARALLEL_ALGORITHMS@ else None
 config.pstl_obj_root            = "@ParallelSTL_BINARY_DIR@" if @LIBCXX_ENABLE_PARALLEL_ALGORITHMS@ else None
 config.libcxx_gdb               = "@LIBCXX_GDB@"
 
+config.available_locales        = "@LIBCXX_TEST_AVAILABLE_LOCALES@"
+
 # Code signing
 config.llvm_codesign_identity   = "@LLVM_CODESIGNING_IDENTITY@"
 
 # Let the main config do the real work.
 config.loaded_site_config = True
 lit_config.load_config(config, "@LIBCXX_SOURCE_DIR@/test/lit.cfg")

libcxx/utils/libcxx/test/target_info.py

try:
         locale.setlocale(locale.LC_ALL, loc)
         return True
     except locale.Error:
         return False
     finally:
         locale.setlocale(locale.LC_ALL, default_locale)
 
 
-def add_common_locales(features, lit_config, is_windows=False):
+def add_common_locales(features, full_config, is_windows=False):
+    lit_config = full_config.lit_config
+
+    # Prefer locales that were specified manually.
+    # This can be the case if tests are being run on a remote board,
+    # when querying the host system locale doesn't make sense.
+    manually_specified_locales = full_config.get_lit_conf('available_locales')
+    
+    if manually_specified_locales:
+        manually_specified_locales = \
+            [loc.strip() for loc in manually_specified_locales.split(';')]
+        lit_config.note('Using manually specified locales: %s' \
+            % manually_specified_locales)
+        def locale_is_supported(loc_name):
+            return loc_name in manually_specified_locales
+    else:
+        locale_is_supported = test_locale
+    
     # A list of locales needed by the test-suite.
     # The list uses the canonical name for the locale used in the test-suite
     # TODO: On Linux ISO8859 *may* needs to hyphenated.
     locales = [
         ('en_US.UTF-8', 'English_United States.1252'),
         ('fr_FR.UTF-8', 'French_France.1252'),
         ('ru_RU.UTF-8', 'Russian_Russia.1251'),
         ('zh_CN.UTF-8', 'Chinese_China.936'),
         ('fr_CA.ISO8859-1', 'French_Canada.1252'),
         ('cs_CZ.ISO8859-2', 'Czech_Czech Republic.1250')
     ]
     for loc_id, windows_loc_name in locales:
         loc_name = windows_loc_name if is_windows else loc_id
-        if test_locale(loc_name):
+        if locale_is_supported(loc_name):
             features.add('locale.{0}'.format(loc_id))
         else:
             lit_config.warning('The locale {0} is not supported by '
                                'your platform. Some tests will be '
                                'unsupported.'.format(loc_name))
 
 
 class DarwinLocalTI(DefaultTargetInfo):
def get_platform(self):
         # /usr/lib/libc++.dylib.
         if name == 'macosx':
             version = self.get_macosx_version()
         else:
             version = self.get_sdk_version(name)
         return (True, name, version)
 
     def add_locale_features(self, features):
-        add_common_locales(features, self.full_config.lit_config)
+        add_common_locales(features, self.full_config)
 
     def add_cxx_compile_flags(self, flags):
         if self.full_config.use_deployment:
             _, name, _ = self.full_config.config.deployment
             cmd = ['xcrun', '--sdk', name, '--show-sdk-path']
         else:
             cmd = ['xcrun', '--show-sdk-path']
         out, err, exit_code = executeCommand(cmd)
def allow_cxxabi_link(self):
         return False
 
 
 class FreeBSDLocalTI(DefaultTargetInfo):
     def __init__(self, full_config):
         super(FreeBSDLocalTI, self).__init__(full_config)
 
     def add_locale_features(self, features):
-        add_common_locales(features, self.full_config.lit_config)
+        add_common_locales(features, self.full_config)
 
     def add_cxx_link_flags(self, flags):
         flags += ['-lc', '-lm', '-lpthread', '-lgcc_s', '-lcxxrt']
 
 
 class NetBSDLocalTI(DefaultTargetInfo):
     def __init__(self, full_config):
         super(NetBSDLocalTI, self).__init__(full_config)
 
     def add_locale_features(self, features):
-        add_common_locales(features, self.full_config.lit_config)
+        add_common_locales(features, self.full_config)
 
     def add_cxx_link_flags(self, flags):
         flags += ['-lc', '-lm', '-lpthread', '-lgcc_s', '-lc++abi',
                   '-lunwind']
 
 
 class LinuxLocalTI(DefaultTargetInfo):
     def __init__(self, full_config):
def platform_name(self):
         return name # Permitted to be None
 
     def platform_ver(self):
         _, ver, _ = self._distribution()
         ver = ver.lower().strip().replace(' ', '-')
         return ver # Permitted to be None.
 
     def add_locale_features(self, features):
-        add_common_locales(features, self.full_config.lit_config)
+        add_common_locales(features, self.full_config)
         # Some linux distributions have different locale data than others.
         # Insert the distributions name and name-version into the available
         # features to allow tests to XFAIL on them.
         name = self.platform_name()
         ver = self.platform_ver()
         if name:
             features.add(name)
         if name and ver:
def add_cxx_link_flags(self, flags):
             flags += ['-lpthread', '-lrt', '-lm', '-ldl']
 
 
 class WindowsLocalTI(DefaultTargetInfo):
     def __init__(self, full_config):
         super(WindowsLocalTI, self).__init__(full_config)
 
     def add_locale_features(self, features):
-        add_common_locales(features, self.full_config.lit_config,
+        add_common_locales(features, self.full_config,
                            is_windows=True)
 
     def use_lit_shell_default(self):
         # Default to the internal shell on Windows, as bash on Windows is
         # usually very slow.
         return True