Index: test/lit.cfg =================================================================== --- test/lit.cfg +++ test/lit.cfg @@ -18,6 +18,9 @@ import lit.formats import lit.util + + + class LibcxxTestFormat(lit.formats.FileBasedTest): """ Custom test format handler for use with the test format use by libc++. @@ -29,12 +32,22 @@ """ def __init__(self, cxx_under_test, use_verify_for_fail, - cpp_flags, ld_flags, exec_env): + cpp_flags, ld_flags, exec_env, + use_scan_build = None, + scan_build_output = None, + scan_build_args = [], + use_clang_tidy = None, + clang_tidy_args = []): self.cxx_under_test = cxx_under_test self.use_verify_for_fail = use_verify_for_fail self.cpp_flags = list(cpp_flags) self.ld_flags = list(ld_flags) self.exec_env = dict(exec_env) + self.use_scan_build = use_scan_build + self.scan_build_output = scan_build_output + self.scan_build_args = list(scan_build_args) + self.use_clang_tidy = use_clang_tidy + self.clang_tidy_args = list(clang_tidy_args) def execute_command(self, command, in_dir=None): kwargs = { @@ -144,11 +157,23 @@ exec_file.close() try: + if self.use_clang_tidy: + status,report = self._execute_clang_tidy_test(source_path) + if status != lit.Test.PASS: + return status,report + compile_cmd = [self.cxx_under_test, '-o', exec_path, source_path] + self.cpp_flags + self.ld_flags - cmd = compile_cmd + if self.use_scan_build: + cmd = [self.use_scan_build, '--status-bugs', + '-o', self.scan_build_output] + self.scan_build_args + else: + cmd = [] + cmd += compile_cmd out, err, exitCode = self.execute_command(cmd) - if exitCode != 0: + if exitCode != 0 or (self.use_scan_build and + (not os.path.exists(exec_path) + or os.path.getsize(exec_path) == 0)): report = """Command: %s\n""" % ' '.join(["'%s'" % a for a in cmd]) report += """Exit Code: %d\n""" % exitCode @@ -187,12 +212,29 @@ pass return lit.Test.PASS, "" + def _execute_clang_tidy_test(self, source_file): + cmd = [self.use_clang_tidy] + self.clang_tidy_args + [source_file] + cmd += ['--'] + self.cpp_flags + out, err, exitCode = self.execute_command(cmd) + if exitCode != 0 or out.strip(): + report = """Command: %s\n""" % ' '.join(["'%s'" % a + for a in cmd]) + report += """Exit Code: %d\n""" % exitCode + if out: + report += """Standard Output:\n--\n%s--""" % out + if err: + report += """Standard Error:\n--\n%s--""" % err + report += "\n\nCompilation failed unexpectedly!" + return lit.Test.FAIL, report + return lit.Test.PASS, "" + class Configuration(object): def __init__(self, lit_config, config): self.lit_config = lit_config self.config = config self.cxx = None + self.cxx_info = None self.src_root = None self.obj_root = None self.env = {} @@ -201,6 +243,11 @@ self.link_flags = [] self.use_system_lib = False self.use_clang_verify = False + self.use_scan_build = None + self.scan_build_output = None + self.scan_build_args = [] + self.use_clang_tidy = None + self.clang_tidy_args = [] if platform.system() not in ('Darwin', 'FreeBSD', 'Linux'): self.lit_config.fatal("unrecognized system") @@ -217,9 +264,20 @@ conf = self.get_lit_conf(name) if conf is None: return None - if conf.lower() in ('1', 'true'): + if conf.lower() in ('1', 'true', 'on'): return True - if conf.lower() in ('', '0', 'false'): + if conf.lower() in ('', '0', 'false', 'off'): + return False + self.lit_config.fatal( + "parameter '{}' should be true or false".format(name)) + + def get_lit_switch(self, name): + s = self.get_lit_conf(name) + if s is None: + return False + elif s.lower() in ('', '1', 'true', 'on'): + return True + elif s.lower() in ('0', 'false', 'off'): return False self.lit_config.fatal( "parameter '{}' should be true or false".format(name)) @@ -231,11 +289,13 @@ self.configure_obj_root() self.configure_use_system_lib() self.configure_use_clang_verify() - self.configure_env() self.configure_std_flag() self.configure_compile_flags() self.configure_link_flags() self.configure_sanitizer() + self.configure_warnings() + self.configure_scan_build() + self.configure_clang_tidy() self.configure_features() def get_test_format(self): @@ -244,7 +304,12 @@ self.use_clang_verify, cpp_flags=['-nostdinc++'] + self.compile_flags, ld_flags=['-nodefaultlibs'] + self.link_flags, - exec_env=self.env) + exec_env=self.env, + use_scan_build=self.use_scan_build, + scan_build_output=self.scan_build_output, + scan_build_args=self.scan_build_args, + use_clang_tidy=self.use_clang_tidy, + clang_tidy_args=self.clang_tidy_args) def configure_cxx(self): # Gather various compiler parameters. @@ -284,12 +349,8 @@ def configure_use_clang_verify(self): '''If set, run clang with -verify on failing tests.''' - self.use_clang_verify = self.get_lit_bool('use_clang_verify') - if self.use_clang_verify is None: - # TODO: Default this to True when using clang. - self.use_clang_verify = False - self.lit_config.note( - "inferred use_clang_verify as: %r" % self.use_clang_verify) + self.use_clang_verify = self.get_lit_switch('use_clang_verify') + def configure_features(self): additional_features = self.get_lit_conf('additional_features') @@ -364,54 +425,76 @@ self.compile_flags += ['-D_LIBCPP_HAS_NO_MONOTONIC_CLOCK'] def configure_compile_flags(self): + # Check if libcxx_headers is defined. If it is then use that as + # the location for the libcxx headers. + libcxx_headers = self.get_lit_conf('libcxx_headers', + self.src_root + '/include') # Configure extra compiler flags. - self.compile_flags += ['-I' + self.src_root + '/include', + self.compile_flags += ['-I' + libcxx_headers, '-I' + self.src_root + '/test/support'] if sys.platform == 'linux2': self.compile_flags += ['-D__STDC_FORMAT_MACROS', '-D__STDC_LIMIT_MACROS', '-D__STDC_CONSTANT_MACROS'] + # Add extra compile flags as specified on the command line. + extra_flags = self.get_lit_conf('compile_flags') + if extra_flags: + self.compile_flags += shlex.split(extra_flags) def configure_link_flags(self): - # Configure library search paths - abi_library_path = self.get_lit_conf('abi_library_path', '') - self.link_flags += ['-L' + self.obj_root + '/lib'] - if not self.use_system_lib: - self.link_flags += ['-Wl,-rpath', '-Wl,' + self.obj_root + '/lib'] + # Configure libc++ library + libcxx_library = self.get_lit_conf('libcxx_library') + # Ignore and warn if use_system_lib is true and libcxx_library is specified. + if self.use_system_lib and libcxx_library: + self.lit_config.warning( + 'Conflicting options detected: libcxx_library and use_system_lib') + libcxx_library = None + # Link to the specific libc++ library if specified. Otherwise link to -lc++ + # and add the build directory as a search path if use_system_lib is False. + if libcxx_library: + self.link_flags += [libcxx_library, '-Wl,-rpath', + '-Wl,' + os.path.dirname(libcxx_library)] + else: + self.link_flags += ['-lc++'] + if not self.use_system_lib: + self.link_flags += ['-L' + self.obj_root + '/lib', '-Wl,-rpath', + '-Wl,' + self.obj_root + '/lib'] + + # Add a search path for the ABI library path if specified. + abi_library_path = self.get_lit_conf('abi_library_path') if abi_library_path: self.link_flags += ['-L' + abi_library_path, '-Wl,-rpath', '-Wl,' + abi_library_path] - # Configure libraries - self.link_flags += ['-lc++'] - link_flags_str = self.get_lit_conf('link_flags') - if link_flags_str is None: - cxx_abi = self.get_lit_conf('cxx_abi', 'libcxxabi') - if cxx_abi == 'libstdc++': - self.link_flags += ['-lstdc++'] - elif cxx_abi == 'libsupc++': - self.link_flags += ['-lsupc++'] - elif cxx_abi == 'libcxxabi': - self.link_flags += ['-lc++abi'] - elif cxx_abi == 'libcxxrt': - self.link_flags += ['-lcxxrt'] - elif cxx_abi == 'none': - pass - else: - self.lit_config.fatal( - 'C++ ABI setting %s unsupported for tests' % cxx_abi) - - if sys.platform == 'darwin': - self.link_flags += ['-lSystem'] - elif sys.platform == 'linux2': - self.link_flags += ['-lgcc_eh', '-lc', '-lm', '-lpthread', - '-lrt', '-lgcc_s'] - elif sys.platform.startswith('freebsd'): - self.link_flags += ['-lc', '-lm', '-pthread', '-lgcc_s'] - else: - self.lit_config.fatal("unrecognized system: %r" % sys.platform) + + cxx_abi = self.get_lit_conf('cxx_abi', 'libcxxabi') + if cxx_abi == 'libstdc++': + self.link_flags += ['-lstdc++'] + elif cxx_abi == 'libsupc++': + self.link_flags += ['-lsupc++'] + elif cxx_abi == 'libcxxabi': + self.link_flags += ['-lc++abi'] + elif cxx_abi == 'libcxxrt': + self.link_flags += ['-lcxxrt'] + elif cxx_abi == 'none': + pass + else: + self.lit_config.fatal( + 'C++ ABI setting %s unsupported for tests' % cxx_abi) + + if sys.platform == 'darwin': + self.link_flags += ['-lSystem'] + elif sys.platform == 'linux2': + self.link_flags += ['-lgcc_eh', '-lc', '-lm', '-lpthread', + '-lrt', '-lgcc_s'] + elif sys.platform.startswith('freebsd'): + self.link_flags += ['-lc', '-lm', '-pthread', '-lgcc_s'] + else: + self.lit_config.fatal("unrecognized system: %r" % sys.platform) self.lit_config.note( "inferred link_flags as: %r" % self.link_flags) + + link_flags_str = self.get_lit_conf('link_flags') if link_flags_str: self.link_flags += shlex.split(link_flags_str) @@ -428,25 +511,74 @@ self.config.available_features.add(std) def configure_sanitizer(self): - san = self.get_lit_conf('llvm_use_sanitizer', '').strip() + san = self.get_lit_conf('use_sanitizer').strip() if san: - self.compile_flags += ['-fno-omit-frame-pointer'] + if san == 'None': + return + san_flags = [] + san_flags += ['-fno-omit-frame-pointer'] if san == 'Address': - self.compile_flags += ['-fsanitize=address'] + san_flags += ['-fsanitize=address'] self.config.available_features.add('asan') elif san == 'Memory' or san == 'MemoryWithOrigins': - self.compile_flags += ['-fsanitize=memory'] + san_flags += ['-fsanitize=memory'] if san == 'MemoryWithOrigins': - self.compile_flags += ['-fsanitize-memory-track-origins'] + san_flags += ['-fsanitize-memory-track-origins'] self.config.available_features.add('msan') elif san == 'Undefined': - self.compile_flags += ['-fsanitize=undefined', - '-fno-sanitize=vptr,function', - '-fno-sanitize-recover'] + san_flags += ['-fsanitize=undefined', + '-fno-sanitize=vptr,function', + '-fno-sanitize-recover'] self.config.available_features.add('ubsan') else: self.lit_config.fatal('unsupported value for ' 'libcxx_use_san: {0}'.format(san)) + self._configure_sanitizer_libraries(san_flags) + + def configure_warnings(self): + enable_warnings = self.get_lit_bool('enable_warnings') + if enable_warnings: + self.lit_config.note('Enabling warnings') + self.compile_flags += ['-Wall', '-Wextra', '-pedantic', + '-D_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER'] + + def configure_scan_build(self): + self.use_scan_build = self.get_lit_conf('use_scan_build') + if self.use_scan_build is None: + return + if self.use_scan_build == '': + self.use_scan_build = 'scan-build' + self.lit_config.note( + 'Using scan-build: %s' % self.use_scan_build) + output = self.get_lit_conf('scan_build_output') + if output is None: + output = tempfile.mkdtemp(suffix='', prefix='libcxx-scan-build-') + self.lit_config.note('Writing scan-build output to: %s' % output) + self.scan_build_output = output + args = self.get_lit_conf('scan_build_args') + if args: + args = shlex.split(args) + else: + args = ['-analyze-headers'] + self.lit_config.note('Using scan-build args: %r' % args) + self.scan_build_args = args + + def configure_clang_tidy(self): + self.use_clang_tidy = self.get_lit_conf('use_clang_tidy') + if self.use_clang_tidy is None: + return + if self.use_clang_tidy == '': + self.use_clang_tidy = 'clang-tidy' + self.lit_config.note( + 'Using clang-tidy: %s' % self.use_clang_tidy) + args = self.get_lit_conf('clang_tidy_args') + if args is not None: + self.clang_tidy_args = shlex.split(args) + else: + self.clang_tidy_args = ["-checks=-misc-*,clang-analyzer-*", + "-header-filter='*'"] + self.lit_config.note('Using clang-tidy args: %r' % self.clang_tidy_args) + def configure_triple(self): # Get or infer the target triple. @@ -459,13 +591,6 @@ self.lit_config.note( "inferred target_triple as: %r" % self.config.target_triple) - def configure_env(self): - # Configure extra linker parameters. - if sys.platform == 'darwin': - if not self.use_system_lib: - self.env['DYLD_LIBRARY_PATH'] = os.path.join(self.obj_root, - 'lib') - # name: The name of this test suite. config.name = 'libc++' Index: test/lit.site.cfg.in =================================================================== --- test/lit.site.cfg.in +++ test/lit.site.cfg.in @@ -6,8 +6,8 @@ config.python_executable = "@PYTHON_EXECUTABLE@" config.enable_shared = @LIBCXX_ENABLE_SHARED@ config.cxx_abi = "@LIBCXX_CXX_ABI_LIBNAME@" -config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@" config.abi_library_path = "@LIBCXX_CXX_ABI_LIBRARY_PATH@" +config.use_sanitizer = "@LLVM_USE_SANITIZER@" # Let the main config do the real work. lit_config.load_config(config, "@LIBCXX_SOURCE_DIR@/test/lit.cfg") Index: www/index.html =================================================================== --- www/index.html +++ www/index.html @@ -476,6 +476,7 @@
  • <type_traits>
  • Excellent notes by Marshall Clow
  • Status of debug mode
    • +
  • Using LIT on the command line
    • Index: www/lit_usage.html =================================================================== --- /dev/null +++ www/lit_usage.html @@ -0,0 +1,226 @@ + + + + + + Testing libc++ using LIT + + + + + + + + +
    + +

    Testing libc++ using LIT

    + +

    +libc++ uses LIT to configure and run it's tests. The primary way to run the +libc++ tests is by using make check-libcxx. However since libc++ +can be used in any number of possible configurations it is important to +customize the way LIT builds and runs the tests. This guide provides +information on how to use LIT directly to test libc++. +

    +

    +Documentation for LIT can be found +here. +

    + + +

    Getting Started

    + +

    +After building libc++ use the following commands before you start using LIT to +test. +

    + +

    +You can now run the libc++ tests by running: +

    + +

    +To only run a subsection of the tests use: +

    + + +

    Customization Options

    + +

    +libc++'s testsuite provides multiple options to configure the way the tests +are build and run. To use these options you pass them on the LIT command line +as --param NAME or --param NAME=VALUE. Some options +have default values specified during CMake's configuration. Passing the option +on the command line will override the default. +

    + +

    +

    libcxx_headers=<path/to/headers>

    +
    +Specify the libc++ headers that are tested. By default the headers in the source +tree are used. +
    +

    + +

    +

    libcxx_library=<path/to/libc++.so>

    +
    +Specify the libc++ library that is tested. By default the library in the build +directory is used. This option has no effect if use_system_lib is provided. +
    +

    + +

    +

    use_system_lib=<bool>

    +
    +Default: False
    +Enable or disable testing against the installed version of libc++ library. +Note: This does not use the installed headers. +
    +

    + +

    +

    compile_flags="<list-of-args>"

    +
    +Specify an additional list of compile flags to use. +Note: This options should not be used to change the standard version used. +
    +

    + +

    +

    link_flags="<list-of-args>"

    +
    +Specify an additional list of linker flags to use. +
    +

    + +

    +

    std=<standard version>

    +
    +Values: c++98, c++03, c++11, c++1z, c++14
    +Change the standard version used when building the tests. +
    +

    + +

    +

    use_sanitizer=<sanitizer name>

    +
    +Values: Memory, MemoryWithOrigins, Address, Undefined, None
    +Run the tests using the given sanitizer. If LLVM_USE_SANITIZER +was given when building libc++ then that sanitizer will be used by default. +If None is given then no sanitizer will be used. +
    +

    + +

    +

    enable_warnings[=<bool>]

    +
    +Default: False
    +Enable -Wall -Wextra -pedantic as well as disabling +the system header pragmas. +
    +

    + +

    +

    use_clang_verify[=<bool>]

    +
    +Default: False
    +Enable the use of clang verif on tests that expect a compilation error. +This verifies that the test fails to compile for the right reason. +
    +

    + +

    Scan Build Options

    + +

    +

    use_scan_build[=<path/to/scan-build>]

    +
    +When specified the tests are compiled through scan-build. The optional value is +used to specify the scan-build executable to use. If scan-build finds a defect +the test will fail. +
    +

    + +

    +

    scan_build_output=<path/to/scan-build/output>

    +
    +Specify a folder in which to store the scan-build output. If scan-build is +used without this option a temporary directory is created and reported at the +start of the LIT run. +
    +

    + +

    +

    scan_build_args="<list-of-args>"

    +
    +Default: -analyze-headers
    +Specify a list of args to be passed to scan-build. +
    +

    + +

    Clang Tidy Options

    + +

    +

    use_clang_tidy[=<path/to/clang-tidy>

    +
    +When specified the tests are run through clang-tidy as an extra step. The test +will fail if clang-tidy produces any output. The option value is used to specify +the clang-tidy executable to use. +
    +

    + + +

    +

    clang_tidy_args="<list-of-args>"

    +
    +Default: -checks=clang-analyzer-* -header-filter='*'
    +Specify a list of args to be passed to clang-tidy. +
    +

    + + +
    + +