diff --git a/libcxx/test/libcxx/selftest/dsl/dsl.sh.py b/libcxx/test/libcxx/selftest/dsl/dsl.sh.py new file mode 100644 --- /dev/null +++ b/libcxx/test/libcxx/selftest/dsl/dsl.sh.py @@ -0,0 +1,291 @@ +#===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===## +# RUN: %{python} %s '%S' '%T' '%{escaped_exec}' \ +# RUN: '%{escaped_cxx}' \ +# RUN: '%{escaped_flags}' \ +# RUN: '%{escaped_compile_flags}' \ +# RUN: '%{escaped_link_flags}' +# END. + +import base64 +import copy +import os +import platform +import subprocess +import sys +import unittest +from os.path import dirname + +# Allow importing 'lit' and the 'libcxx' module. Make sure we put the lit +# path first so we don't find any system-installed version. +monorepoRoot = dirname(dirname(dirname(dirname(dirname(dirname(__file__)))))) +sys.path = [os.path.join(monorepoRoot, 'libcxx', 'utils'), + os.path.join(monorepoRoot, 'llvm', 'utils', 'lit')] + sys.path +import libcxx.test.dsl as dsl +import lit.LitConfig + +# Steal some parameters from the config running this test so that we can +# bootstrap our own TestingConfig. +SOURCE_ROOT, EXEC_PATH, EXEC, CXX, FLAGS, COMPILE_FLAGS, LINK_FLAGS = sys.argv[1:] +sys.argv = sys.argv[:1] + +class SetupConfigs(unittest.TestCase): + """ + Base class for the tests below -- it creates a fake TestingConfig. + """ + def setUp(self): + """ + Create a fake TestingConfig that can be populated however we wish for + the purpose of running unit tests below. We pre-populate it with the + minimum required substitutions. + """ + self.litConfig = lit.LitConfig.LitConfig( + progname='lit', + path=[], + quiet=False, + useValgrind=False, + valgrindLeakCheck=False, + valgrindArgs=[], + noExecute=False, + debug=False, + isWindows=platform.system() == 'Windows', + params={}) + + self.config = lit.TestingConfig.TestingConfig.fromdefaults(self.litConfig) + self.config.test_source_root = SOURCE_ROOT + self.config.test_exec_root = EXEC_PATH + self.config.substitutions = [ + ('%{cxx}', base64.b64decode(CXX)), + ('%{flags}', base64.b64decode(FLAGS)), + ('%{compile_flags}', base64.b64decode(COMPILE_FLAGS)), + ('%{link_flags}', base64.b64decode(LINK_FLAGS)), + ('%{exec}', base64.b64decode(EXEC)) + ] + + def getSubstitution(self, substitution): + """ + Return a given substitution from the TestingConfig. It is an error if + there is no such substitution. + """ + found = [x for (s, x) in self.config.substitutions if s == substitution] + assert len(found) == 1 + return found[0] + + +class TestHasCompileFlag(SetupConfigs): + """ + Tests for libcxx.test.dsl.hasCompileFlag + """ + def test_no_flag_should_work(self): + self.assertTrue(dsl.hasCompileFlag(self.config, '')) + + def test_flag_exists(self): + self.assertTrue(dsl.hasCompileFlag(self.config, '-O1')) + + def test_nonexistent_flag(self): + self.assertFalse(dsl.hasCompileFlag(self.config, '-this_is_not_a_flag_any_compiler_has')) + + def test_multiple_flags(self): + self.assertTrue(dsl.hasCompileFlag(self.config, '-O1 -Dhello')) + + +class TestHasLocale(SetupConfigs): + """ + Tests for libcxx.test.dsl.hasLocale + """ + def test_doesnt_explode(self): + # It's really hard to test that a system has a given locale, so at least + # make sure we don't explode when we try to check it. + try: + dsl.hasLocale(self.config, 'en_US.UTF-8') + except subprocess.CalledProcessError: + self.fail("checking for hasLocale should not explode") + + def test_nonexistent_locale(self): + self.assertFalse(dsl.hasLocale(self.config, 'for_sure_this_is_not_an_existing_locale')) + + +class TestCompilerMacros(SetupConfigs): + """ + Tests for libcxx.test.dsl.compilerMacros + """ + def test_basic(self): + macros = dsl.compilerMacros(self.config) + self.assertIsInstance(macros, dict) + self.assertGreater(len(macros), 0) + for (k, v) in macros.items(): + self.assertIsInstance(k, str) + self.assertIsInstance(v, str) + + def test_no_flag(self): + macros = dsl.compilerMacros(self.config) + self.assertIn('__cplusplus', macros.keys()) + + def test_empty_flag(self): + macros = dsl.compilerMacros(self.config, '') + self.assertIn('__cplusplus', macros.keys()) + + def test_with_flag(self): + macros = dsl.compilerMacros(self.config, '-DFOO=3') + self.assertIn('__cplusplus', macros.keys()) + self.assertEqual(macros['FOO'], '3') + + def test_with_flags(self): + macros = dsl.compilerMacros(self.config, '-DFOO=3 -DBAR=hello') + self.assertIn('__cplusplus', macros.keys()) + self.assertEqual(macros['FOO'], '3') + self.assertEqual(macros['BAR'], 'hello') + + +class TestFeatureTestMacros(SetupConfigs): + """ + Tests for libcxx.test.dsl.featureTestMacros + """ + def test_basic(self): + macros = dsl.featureTestMacros(self.config) + self.assertIsInstance(macros, dict) + self.assertGreater(len(macros), 0) + for (k, v) in macros.items(): + self.assertIsInstance(k, str) + self.assertIsInstance(v, int) + + +class TestFeature(SetupConfigs): + """ + Tests for libcxx.test.dsl.Feature + """ + def test_trivial(self): + feature = dsl.Feature(name='name') + origSubstitutions = copy.deepcopy(self.config.substitutions) + self.assertTrue(feature.isSupported(self.config)) + feature.enableIn(self.config) + self.assertEqual(origSubstitutions, self.config.substitutions) + self.assertIn('name', self.config.available_features) + + def test_name_can_be_a_callable(self): + feature = dsl.Feature(name=lambda cfg: (self.assertIs(self.config, cfg), 'name')[1]) + assert feature.isSupported(self.config) + feature.enableIn(self.config) + self.assertIn('name', self.config.available_features) + + def test_adding_compile_flag(self): + feature = dsl.Feature(name='name', compileFlag='-foo') + origLinkFlags = copy.deepcopy(self.getSubstitution('%{link_flags}')) + assert feature.isSupported(self.config) + feature.enableIn(self.config) + self.assertIn('name', self.config.available_features) + self.assertIn('-foo', self.getSubstitution('%{compile_flags}')) + self.assertEqual(origLinkFlags, self.getSubstitution('%{link_flags}')) + + def test_adding_link_flag(self): + feature = dsl.Feature(name='name', linkFlag='-foo') + origCompileFlags = copy.deepcopy(self.getSubstitution('%{compile_flags}')) + assert feature.isSupported(self.config) + feature.enableIn(self.config) + self.assertIn('name', self.config.available_features) + self.assertIn('-foo', self.getSubstitution('%{link_flags}')) + self.assertEqual(origCompileFlags, self.getSubstitution('%{compile_flags}')) + + def test_adding_both_flags(self): + feature = dsl.Feature(name='name', compileFlag='-hello', linkFlag='-world') + assert feature.isSupported(self.config) + feature.enableIn(self.config) + self.assertIn('name', self.config.available_features) + + self.assertIn('-hello', self.getSubstitution('%{compile_flags}')) + self.assertNotIn('-world', self.getSubstitution('%{compile_flags}')) + + self.assertIn('-world', self.getSubstitution('%{link_flags}')) + self.assertNotIn('-hello', self.getSubstitution('%{link_flags}')) + + def test_unsupported_feature(self): + feature = dsl.Feature(name='name', when=lambda _: False) + self.assertFalse(feature.isSupported(self.config)) + # Also make sure we assert if we ever try to add it to a config + self.assertRaises(AssertionError, lambda: feature.enableIn(self.config)) + + def test_is_supported_gets_passed_the_config(self): + feature = dsl.Feature(name='name', when=lambda cfg: (self.assertIs(self.config, cfg), True)[1]) + self.assertTrue(feature.isSupported(self.config)) + + +class TestParameter(SetupConfigs): + """ + Tests for libcxx.test.dsl.Parameter + """ + def test_empty_name_should_blow_up(self): + self.assertRaises(ValueError, lambda: dsl.Parameter(name='', choices=['c++03'], type=str, help='', feature=lambda _: None)) + + def test_empty_choices_should_blow_up(self): + self.assertRaises(ValueError, lambda: dsl.Parameter(name='std', choices=[], type=str, help='', feature=lambda _: None)) + + def test_name_is_set_correctly(self): + param = dsl.Parameter(name='std', choices=['c++03'], type=str, help='', feature=lambda _: None) + self.assertEqual(param.name, 'std') + + def test_no_value_provided_on_command_line_and_no_default_value(self): + param = dsl.Parameter(name='std', choices=['c++03'], type=str, help='', feature=lambda _: None) + self.assertRaises(ValueError, lambda: param.getFeature(self.config, self.litConfig.params)) + + def test_no_value_provided_on_command_line_and_default_value(self): + param = dsl.Parameter(name='std', choices=['c++03'], type=str, help='', default='c++03', + feature=lambda std: dsl.Feature(name=std)) + param.getFeature(self.config, self.litConfig.params).enableIn(self.config) + self.assertIn('c++03', self.config.available_features) + + def test_value_provided_on_command_line_and_no_default_value(self): + self.litConfig.params['std'] = 'c++03' + param = dsl.Parameter(name='std', choices=['c++03'], type=str, help='', + feature=lambda std: dsl.Feature(name=std)) + param.getFeature(self.config, self.litConfig.params).enableIn(self.config) + self.assertIn('c++03', self.config.available_features) + + def test_value_provided_on_command_line_and_default_value(self): + self.litConfig.params['std'] = 'c++11' + param = dsl.Parameter(name='std', choices=['c++03', 'c++11'], type=str, default='c++03', help='', + feature=lambda std: dsl.Feature(name=std)) + param.getFeature(self.config, self.litConfig.params).enableIn(self.config) + self.assertIn('c++11', self.config.available_features) + self.assertNotIn('c++03', self.config.available_features) + + def test_feature_is_None(self): + self.litConfig.params['std'] = 'c++03' + param = dsl.Parameter(name='std', choices=['c++03'], type=str, help='', + feature=lambda _: None) + feature = param.getFeature(self.config, self.litConfig.params) + self.assertIsNone(feature) + + def test_boolean_value_parsed_from_trueish_string_parameter(self): + self.litConfig.params['enable_exceptions'] = "True" + param = dsl.Parameter(name='enable_exceptions', choices=[True, False], type=bool, help='', + feature=lambda exceptions: None if exceptions else ValueError()) + self.assertIsNone(param.getFeature(self.config, self.litConfig.params)) + + def test_boolean_value_from_true_boolean_parameter(self): + self.litConfig.params['enable_exceptions'] = True + param = dsl.Parameter(name='enable_exceptions', choices=[True, False], type=bool, help='', + feature=lambda exceptions: None if exceptions else ValueError()) + self.assertIsNone(param.getFeature(self.config, self.litConfig.params)) + + def test_boolean_value_parsed_from_falseish_string_parameter(self): + self.litConfig.params['enable_exceptions'] = "False" + param = dsl.Parameter(name='enable_exceptions', choices=[True, False], type=bool, help='', + feature=lambda exceptions: None if exceptions else dsl.Feature(name="-fno-exceptions")) + param.getFeature(self.config, self.litConfig.params).enableIn(self.config) + self.assertIn('-fno-exceptions', self.config.available_features) + + def test_boolean_value_from_false_boolean_parameter(self): + self.litConfig.params['enable_exceptions'] = False + param = dsl.Parameter(name='enable_exceptions', choices=[True, False], type=bool, help='', + feature=lambda exceptions: None if exceptions else dsl.Feature(name="-fno-exceptions")) + param.getFeature(self.config, self.litConfig.params).enableIn(self.config) + self.assertIn('-fno-exceptions', self.config.available_features) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/libcxx/test/libcxx/selftest/dsl/lit.local.cfg b/libcxx/test/libcxx/selftest/dsl/lit.local.cfg new file mode 100644 --- /dev/null +++ b/libcxx/test/libcxx/selftest/dsl/lit.local.cfg @@ -0,0 +1,18 @@ +# Since we try to pass substitutions as-is to some tests, we must "escape" +# them in case they contain other substitutions. Otherwise, the substitutions +# will be fully expanded when passed to the tests. For example, we want an +# %{exec} substitution that contains `--dependencies %{file_dependencies}` +# to be passed as-is, without substituting the file dependencies. This way, +# the test itself can use populate %{file_dependencies} as it sees fit, and +# %{exec} will respect it. +# +# To solve this problem, we add base64 encoded versions of substitutions just +# in this directory. We then base64-decode them from the tests when we need to. +# Another option would be to have a way to prevent expansion in Lit itself. +import base64 +escaped = [(k.replace('%{', '%{escaped_'), base64.b64encode(v)) for (k, v) in config.substitutions] +config.substitutions.extend(escaped) + +# The tests in this directory need to run Python +import sys +config.substitutions.append(('%{python}', sys.executable)) diff --git a/libcxx/utils/libcxx/test/dsl.py b/libcxx/utils/libcxx/test/dsl.py new file mode 100644 --- /dev/null +++ b/libcxx/utils/libcxx/test/dsl.py @@ -0,0 +1,291 @@ +#===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===## + +import distutils.util +import libcxx.test.newformat +import lit +import os +import pipes +import subprocess +import tempfile + +def _memoize(f): + cache = dict() + def memoized(x): + if x not in cache: + cache[x] = f(x) + return cache[x] + return memoized + +@_memoize +def _subprocess_call(command): + devNull = open(os.devnull, 'w') + return subprocess.call(command, shell=True, stdout=devNull, stderr=devNull) + +@_memoize +def _subprocess_check_output(command): + devNull = open(os.devnull, 'w') + return subprocess.check_output(command, shell=True, stderr=devNull) + +def _makeConfigTest(config): + sourceRoot = os.path.join(config.test_exec_root, '__config_src__') + execRoot = os.path.join(config.test_exec_root, '__config_exec__') + suite = lit.Test.TestSuite('__config__', sourceRoot, execRoot, config) + if not os.path.exists(sourceRoot): + os.makedirs(sourceRoot) + tmp = tempfile.NamedTemporaryFile(dir=sourceRoot, delete=False) + pathInSuite = [os.path.relpath(tmp.name, sourceRoot)] + class TestWrapper(lit.Test.Test): + def __enter__(self): return self + def __exit__(self, *args): os.remove(tmp.name) + return TestWrapper(suite, pathInSuite, config) + +def hasCompileFlag(config, flag): + """ + Return whether the compiler in the configuration supports a given compiler flag. + + This is done by executing the %{cxx} substitution with the given flag and + checking whether that succeeds. + """ + with _makeConfigTest(config) as test: + command = "%{{cxx}} -xc++ {} -Werror -fsyntax-only %{{flags}} %{{compile_flags}} {}".format(os.devnull, flag) + command = libcxx.test.newformat.parseScript(test, preamble=[command], fileDependencies=[])[0] + result = _subprocess_call(command) + return result == 0 + +def hasLocale(config, locale): + """ + Return whether the runtime execution environment supports a given locale. + + This is done by executing a program that tries to set the given locale using + %{exec} -- this means that the command may be executed on a remote host + depending on the %{exec} substitution. + """ + with _makeConfigTest(config) as test: + with open(test.getSourcePath(), 'w') as source: + source.write(""" + #include + int main(int, char** argv) {{ + if (::setlocale(LC_ALL, argv[1]) != NULL) return 0; + else return 1; + }} + """) + commands = [ + "mkdir -p %T", + "%{cxx} -xc++ %s %{flags} %{compile_flags} %{link_flags} -o %t.exe", + "%{{exec}} %t.exe {}".format(pipes.quote(locale)), + ] + commands = libcxx.test.newformat.parseScript(test, preamble=commands, fileDependencies=['%t.exe']) + result = _subprocess_call(' && '.join(commands)) + cleanup = libcxx.test.newformat.parseScript(test, preamble=['rm %t.exe'], fileDependencies=[])[0] + _subprocess_call(cleanup) + return result == 0 + +def compilerMacros(config, flags=''): + """ + Return a dictionary of predefined compiler macros. + + The keys are strings representing macros, and the values are strings + representing what each macro is defined to. + + If the optional `flags` argument (a string) is provided, these flags will + be added to the compiler invocation when generating the macros. + """ + with _makeConfigTest(config) as test: + command = "%{{cxx}} -xc++ {} -dM -E %{{flags}} %{{compile_flags}} {}".format(os.devnull, flags) + command = libcxx.test.newformat.parseScript(test, preamble=[command], fileDependencies=[])[0] + unparsed = _subprocess_check_output(command) + parsedMacros = dict() + for line in filter(None, map(str.strip, unparsed.split('\n'))): + assert line.startswith('#define ') + line = line[len('#define '):] + macro, _, value = line.partition(' ') + parsedMacros[macro] = value + return parsedMacros + +def featureTestMacros(config, flags=''): + """ + Return a dictionary of feature test macros. + + The keys are strings representing feature test macros, and the values are + integers representing the value of the macro. + """ + allMacros = compilerMacros(config, flags) + return {m: int(v.rstrip('LlUu')) for (m, v) in allMacros.items() if m.startswith('__cpp_')} + + +class Feature(object): + """ + Represents a Lit available feature that is enabled whenever it is supported. + + A feature like this informs the test suite about a capability of the compiler, + platform, etc. Unlike Parameters, it does not make sense to explicitly + control whether a Feature is enabled -- it should be enabled whenever it + is supported. + """ + def __init__(self, name, compileFlag=None, linkFlag=None, when=lambda _: True): + """ + Create a Lit feature for consumption by a test suite. + + - name + The name of the feature. This is what will end up in Lit's available + features if the feature is enabled. This can be either a string or a + callable, in which case it is passed the TestingConfig and should + generate a string representing the name of the feature. + + - compileFlag + An optional compile flag to add when this feature is added to a + TestingConfig. If provided, this must be a string representing a + compile flag that will be appended to the end of the %{compile_flags} + substitution of the TestingConfig. + + - linkFlag + An optional link flag to add when this feature is added to a + TestingConfig. If provided, this must be a string representing a + link flag that will be appended to the end of the %{link_flags} + substitution of the TestingConfig. + + - when + A callable that gets passed a TestingConfig and should return a + boolean representing whether the feature is supported in that + configuration. For example, this can use `hasCompileFlag` to + check whether the compiler supports the flag that the feature + represents. If omitted, the feature will always be considered + supported. + """ + self._name = name + self._compileFlag = compileFlag + self._linkFlag = linkFlag + self._isSupported = when + + def isSupported(self, config): + """ + Return whether the feature is supported by the given TestingConfig. + """ + return self._isSupported(config) + + def enableIn(self, config): + """ + Enable a feature in a TestingConfig. + + The name of the feature is added to the set of available features of + `config`, and any compile or link flags provided upon construction of + the Feature are added to the end of the corresponding substitution in + the config. + + It is an error to call `f.enableIn(cfg)` if the feature `f` is not + supported in that TestingConfig (i.e. if `not f.isSupported(cfg)`). + """ + assert self.isSupported(config), \ + "Trying to enable feature {} that is not supported in the given configuration".format(self._name) + + addTo = lambda subs, sub, flag: [(s, x + ' ' + flag) if s == sub else (s, x) for (s, x) in subs] + if self._compileFlag: + config.substitutions = addTo(config.substitutions, '%{compile_flags}', self._compileFlag) + if self._linkFlag: + config.substitutions = addTo(config.substitutions, '%{link_flags}', self._linkFlag) + + name = self._name(config) if callable(self._name) else self._name + config.available_features.add(name) + + +class Parameter(object): + """ + Represents a parameter of a Lit test suite. + + Parameters are used to customize the behavior of test suites in a user + controllable way, more specifically by passing `--param =` + when running Lit. Parameters have multiple possible values, and they can + have a default value when left unspecified. + + Parameters can have a Feature associated to them, in which case the Feature + is added to the TestingConfig if the parameter is enabled. It is an error if + the Parameter is enabled but the Feature associated to it is not supported, + for example trying to set the compilation standard to C++17 when `-std=c++17` + is not supported by the compiler. + + One important point is that Parameters customize the behavior of the test + suite in a bounded way, i.e. there should be a finite set of possible choices + for ``. While this may appear to be an aggressive restriction, this + is actually a very important constraint that ensures that the set of + configurations supported by a test suite is finite. Otherwise, a test + suite could have an unbounded number of supported configurations, and + nobody wants to be stuck maintaining that. If it's not possible for an + option to have a finite set of possible values (e.g. the path to the + compiler), it can be handled in the `lit.cfg`, but it shouldn't be + represented with a Parameter. + """ + def __init__(self, name, choices, type, help, feature, default=None): + """ + Create a Lit parameter to customize the behavior of a test suite. + + - name + The name of the parameter that can be used to set it on the command-line. + On the command-line, the parameter can be set using `--param =` + when running Lit. This must be non-empty. + + - choices + A non-empty set of possible values for this parameter. This must be + anything that can be iterated. It is an error if the parameter is + given a value that is not in that set, whether explicitly or through + a default value. + + - type + A callable that can be used to parse the value of the parameter given + on the command-line. As a special case, using the type `bool` also + allows parsing strings with boolean-like contents. + + - help + A string explaining the parameter, for documentation purposes. + TODO: We should be able to surface those from the Lit command-line. + + - feature + A callable that gets passed the parsed value of the parameter (either + the one passed on the command-line or the default one), and that returns + either None or a Feature. + + - default + An optional default value to use for the parameter when no value is + provided on the command-line. If the default value is a callable, it + is called with the TestingConfig and should return the default value + for the parameter. Whether the default value is computed or specified + directly, it must be in the 'choices' provided for that Parameter. + """ + self._name = name + if len(self._name) == 0: + raise ValueError("Parameter name must not be the empty string") + + self._choices = list(choices) # should be finite + if len(self._choices) == 0: + raise ValueError("Parameter '{}' must be given at least one possible value".format(self._name)) + + self._parse = lambda x: (distutils.util.strtobool(x) if type is bool and isinstance(x, str) + else type(x)) + self._help = help + self._feature = feature + self._default = default + + @property + def name(self): + """ + Return the name of the parameter. + + This is the name that can be used to set the parameter on the command-line + when running Lit. + """ + return self._name + + def getFeature(self, config, litParams): + param = litParams.get(self.name, None) + if param is None and self._default is None: + raise ValueError("Parameter {} doesn't have a default value, but it was not specified in the Lit parameters".format(self.name)) + getDefault = lambda: self._default(config) if callable(self._default) else self._default + value = self._parse(param) if param is not None else getDefault() + if value not in self._choices: + raise ValueError("Got value '{}' for parameter '{}', which is not in the provided set of possible choices: {}".format(value, self.name, self._choices)) + return self._feature(value)