Index: CMakeLists.txt =================================================================== --- CMakeLists.txt +++ CMakeLists.txt @@ -43,6 +43,7 @@ option(LIBCXX_ENABLE_CXX1Y "Enable -std=c++1y and use of c++1y language features if the compiler supports it." OFF) option(LIBCXX_ENABLE_SHARED "Build libc++ as a shared library." ON) option(LIBCXX_INSTALL_SUPPORT_HEADERS "Install libc++ support headers." ON) +option(LIBCXX_GENERATE_COVERAGE "Generate code coverage results from the tests." OFF) if (LIBCXX_BUILT_STANDALONE) set(LLVM_USE_SANITIZER "" CACHE STRING "Define the sanitizer used to build the library and tests") @@ -231,6 +232,11 @@ string(REPLACE ";" " " LIBCXX_CXX_FEATURE_FLAGS "${LIBCXX_CXX_FEATURE_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${LIBCXX_CXX_FEATURE_FLAGS}") +if (LIBCXX_GENERATE_COVERAGE) + include(LibcxxCodeCoverage) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${LIBCXX_CXX_COVERAGE_FLAGS}") +endif() + #=============================================================================== # Setup Source Code #=============================================================================== Index: cmake/Modules/LibcxxCodeCoverage.cmake =================================================================== --- /dev/null +++ cmake/Modules/LibcxxCodeCoverage.cmake @@ -0,0 +1,41 @@ + +find_program(LIBCXX_GCOV gcov) +if (NOT LIBCXX_GCOV) + message(FATAL_ERROR "Cannot find gcov...") +endif() + +find_program(LIBCXX_LCOV lcov) +find_program(LIBCXX_GENHTML genhtml) + +set(LIBCXX_CXX_COVERAGE_FLAGS "-g -O0 --coverage -fprofile-arcs -ftest-coverage") + +function(setup_lcov_test_target_coverage _coverage_dir) + if (NOT LIBCXX_LCOV) + message(FATAL_ERROR "Cannot find lcov...") + endif() + if (NOT LIBCXX_GENHTML) + message(FATAL_ERROR "Cannot find genhtml...") + endif() + + file(MAKE_DIRECTORY ${_coverage_dir}) + + add_custom_command(TARGET cxx + PRE_BUILD + COMMAND ${LIBCXX_LCOV} --directory . --directory ${_SRC_BUILD_DIR} --zerocounters + WORKING_DIRECTORY ${_coverage_dir} + COMMENT "Reseting code coverage information.") + + set(_SRC_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/../lib/CMakeFiles/cxx.dir/__/src/) + set(_CHMOD_END "{} \\$") + separate_arguments(_CHMOD_END) + add_custom_target(check-libcxx-coverage + COMMAND ${LIBCXX_LCOV} --directory . --directory ${_SRC_BUILD_DIR} --capture --output-file libcxx_coverage.info + COMMAND ${LIBCXX_LCOV} --remove libcxx_coverage.info 'test/*' 'test/*/*' '/usr/*' '/usr/include/*' --output-file libcxx_coverage.info + COMMAND ${LIBCXX_GENHTML} --demangle-cpp -o libcxx_coverage libcxx_coverage.info + COMMAND ${CMAKE_COMMAND} -E remove libcxx_coverage.info + COMMAND ${CMAKE_COMMAND} -E remove_directory ${_coverage_dir}/test + COMMAND find libcxx_coverage/ -type d -exec chmod 750 ${_CHMOD_END} + COMMAND find libcxx_coverage/ -type f -exec chmod 640 ${_CHMOD_END} + WORKING_DIRECTORY ${_coverage_dir} + COMMENT "Generating coverage results") +endfunction() Index: lib/CMakeLists.txt =================================================================== --- lib/CMakeLists.txt +++ lib/CMakeLists.txt @@ -43,7 +43,7 @@ append_if(libraries LIBCXX_HAS_RT_LIB rt) append_if(libraries LIBCXX_HAS_GCC_S_LIB gcc_s) -#if LIBCXX_CXX_ABI_LIBRARY_PATH is defined we want to add it to the search path. +# if LIBCXX_CXX_ABI_LIBRARY_PATH is defined we want to add it to the search path. if (DEFINED LIBCXX_CXX_ABI_LIBRARY_PATH) target_link_libraries(cxx "-L${LIBCXX_CXX_ABI_LIBRARY_PATH}") endif() @@ -51,7 +51,11 @@ # Setup flags. append_if(compile_flags LIBCXX_HAS_FPIC_FLAG -fPIC) -append_if(link_flags LIBCXX_HAS_NODEFAULTLIBS_FLAG -nodefaultlibs) +# FIXME: Currently -nodefaultlibs causes the tests not to link when using +# code coverage due to the order of the libraries passed to the linker. +if (NOT LIBCXX_GENERATE_COVERAGE) + append_if(link_flags LIBCXX_HAS_NODEFAULTLIBS_FLAG -nodefaultlibs) +endif() if ( APPLE ) if ( CMAKE_OSX_DEPLOYMENT_TARGET STREQUAL "10.6" ) Index: test/CMakeLists.txt =================================================================== --- test/CMakeLists.txt +++ test/CMakeLists.txt @@ -28,6 +28,10 @@ set(LIBCXX_BINARY_DIR ${CMAKE_BINARY_DIR}) set(PYTHON_EXECUTABLE ${PYTHON_EXECUTABLE}) pythonize_bool(LIBCXX_ENABLE_SHARED) + pythonize_bool(LIBCXX_GENERATE_COVERAGE) + if (LIBCXX_GENERATE_COVERAGE) + set(LIBCXX_COVERAGE_DIR "${CMAKE_CURRENT_BINARY_DIR}/coverage") + endif() set(AUTO_GEN_COMMENT "## Autogenerated by libcxx configuration.\n# Do not edit!") @@ -43,6 +47,11 @@ ${CMAKE_CURRENT_BINARY_DIR} DEPENDS cxx COMMENT "Running libcxx tests") + + if (LIBCXX_GENERATE_COVERAGE) + include(LibcxxCodeCoverage) + setup_lcov_test_target_coverage(${LIBCXX_COVERAGE_DIR}) + endif() else() message(WARNING "Could not find Python, no check target will be available!") endif() Index: test/lit.cfg =================================================================== --- test/lit.cfg +++ test/lit.cfg @@ -28,31 +28,53 @@ FOO.fail.cpp - Negative test case which is expected to fail compilation. """ - def __init__(self, cxx_under_test, use_verify_for_fail, - cpp_flags, ld_flags, exec_env): + def __init__(self, cxx_under_test, cpp_flags, ld_flags, exec_env, + test_root, + use_verify_for_fail = False, + generate_coverage = False, + coverage_flags=[], + coverage_root = None, + use_scan_build = None, + scan_build_output = None, + scan_build_args = [], + use_clang_tidy = None, + clang_tidy_args = []): self.cxx_under_test = cxx_under_test self.use_verify_for_fail = use_verify_for_fail self.cpp_flags = list(cpp_flags) self.ld_flags = list(ld_flags) self.exec_env = dict(exec_env) - - def execute_command(self, command, in_dir=None): - kwargs = { - 'stdin' :subprocess.PIPE, - 'stdout':subprocess.PIPE, - 'stderr':subprocess.PIPE, - } - if in_dir: - kwargs['cwd'] = in_dir - p = subprocess.Popen(command, **kwargs) - out,err = p.communicate() - exitCode = p.wait() - - # Detect Ctrl-C in subprocess. - if exitCode == -signal.SIGINT: - raise KeyboardInterrupt - - return out, err, exitCode + self.test_root = str(test_root) + self.generate_coverage = generate_coverage + self.coverage_flags = list(coverage_flags) + self.coverage_root = str(coverage_root) + self.use_scan_build = use_scan_build + self.scan_build_output = scan_build_output + self.scan_build_args = list(scan_build_args) + self.use_clang_tidy = use_clang_tidy + self.clang_tidy_args = list(clang_tidy_args) + + def execute_command_and_report(self, command, cwd=None, env=None, compile_cmd=None): + exec_env = dict(os.environ) + if env: + exec_env.extend(env) + out,err,exitCode = lit.util.executeCommand(command, cwd=cwd, env=exec_env) + report = '' + if compile_cmd: + report += """Compiled with: %s\n""" % \ + ' '.join(["'%s'" % a for a in compile_cmd]) + report += """Command: %s\n""" % \ + ' '.join(["'%s'" % a for a in command]) + if env: + report += """Environment: %s\n""" % \ + ' '.join(["'%s'='%s'" % (k,env[k]) for k in env]) + report += """Exit Code: %d\n""" % exitCode + if out: + report += """Standard Output:\n--\n%s--""" % out + if err: + report += """Standard Error:\n--\n%s--""" % err + report += '\n\n' + return report, exitCode, out, err def execute(self, test, lit_config): while True: @@ -63,22 +85,33 @@ raise time.sleep(0.1) + def _handle_metadata_line(self, ln, tag, out): + if tag not in ln: + return False + items = ln[ln.index(tag) + len(tag):].split(',') + items = [s.strip() for s in items if s.strip()] + out.extend(items) + return True + def _execute(self, test, lit_config): # Extract test metadata from the test file. requires = [] unsupported = [] + unsupported_xfail = [] + requires_xfail = [] use_verify = False with open(test.getSourcePath()) as f: for ln in f: - if 'XFAIL:' in ln: - items = ln[ln.index('XFAIL:') + 6:].split(',') - test.xfails.extend([s.strip() for s in items]) - elif 'REQUIRES:' in ln: - items = ln[ln.index('REQUIRES:') + 9:].split(',') - requires.extend([s.strip() for s in items]) - elif 'UNSUPPORTED:' in ln: - items = ln[ln.index('UNSUPPORTED:') + 12:].split(',') - unsupported.extend([s.strip() for s in items]) + if self._handle_metadata_line(ln, 'REQUIRES-XFAIL:', requires_xfail): + pass + elif self._handle_metadata_line(ln, 'UNSUPPORTED-XFAIL:', unsupported_xfail): + pass + elif self._handle_metadata_line(ln, 'XFAIL:', test.xfails): + pass + elif self._handle_metadata_line(ln, 'REQUIRES:', requires): + pass + elif self._handle_metadata_line(ln, 'UNSUPPORTED:', unsupported): + pass elif 'USE_VERIFY' in ln and self.use_verify_for_fail: use_verify = True elif not ln.strip().startswith("//") and ln.strip(): @@ -92,100 +125,144 @@ # introducing a dependency there. What we more ideally would like to do # is lift the "requires" handling to be a core lit framework feature. missing_required_features = [f for f in requires - if f not in test.config.available_features] + if f not in test.config.available_features + or f not in test.config.target_triple] if missing_required_features: return (lit.Test.UNSUPPORTED, "Test requires the following features: %s" % ( ', '.join(missing_required_features),)) unsupported_features = [f for f in unsupported - if f in test.config.available_features] + if f in test.config.available_features + or f in test.config.target_triple] if unsupported_features: return (lit.Test.UNSUPPORTED, "Test is unsupported with the following features: %s" % ( ', '.join(unsupported_features),)) - # Evaluate the test. - return self._evaluate_test(test, use_verify, lit_config) + # Compute list of features that make a test unsupported but also check + # that it fails + missing_required_xfail_features = [f for f in requires_xfail + if f not in test.config.available_features + and f not in test.config.target_triple] + unsupported_xfail_features = [f for f in unsupported_xfail + if f in test.config.available_features + or f in test.config.target_triple] - def _evaluate_test(self, test, use_verify, lit_config): + # Dispatch the test based on name and execute it. name = test.path_in_suite[-1] - source_path = test.getSourcePath() - source_dir = os.path.dirname(source_path) + if name.endswith('.pass.cpp'): + status,report,msg = self._evaluate_pass_test(test, lit_config) + elif name.endswith('.fail.cpp'): + status,report,msg = self._evaluate_fail_test(test, lit_config, use_verify) + else: + lit_config.fatal('Unrecognized test suffix: %s' % name) + + # If the test is REQUIRES-XFAIL on UNSUPPORTED-XFAIL translate the + # result of the test to reflect that. + if len(missing_required_xfail_features) != 0 or \ + len(unsupported_xfail_features) != 0: + if status == lit.Test.FAIL: + return lit.Test.PASS,"" + elif status == lit.Test.PASS: + report += "Test unexpectedly passed!" + return lit.Test.XPASS, report + else: + # TODO figure out what generates this + assert False + # Otherwise check for an error message and return the result + if msg: + report += msg + return status,report - # Check what kind of test this is. - assert name.endswith('.pass.cpp') or name.endswith('.fail.cpp') - expected_compile_fail = name.endswith('.fail.cpp') - # If this is a compile (failure) test, build it and check for failure. - if expected_compile_fail: - cmd = [self.cxx_under_test, '-c', + def _evaluate_fail_test(self, test, lit_config, use_verify): + source_path = test.getSourcePath() + cmd = [self.cxx_under_test, '-c', '-o', '/dev/null', source_path] + self.cpp_flags - expected_rc = 1 - if use_verify: - cmd += ['-Xclang', '-verify'] - expected_rc = 0 - out, err, rc = self.execute_command(cmd) - if rc == expected_rc: - return lit.Test.PASS, "" - else: - report = """Command: %s\n""" % ' '.join(["'%s'" % a - for a in cmd]) - report += """Exit Code: %d\n""" % rc - if out: - report += """Standard Output:\n--\n%s--""" % out - if err: - report += """Standard Error:\n--\n%s--""" % err - report += "\n\nExpected compilation to fail!" - return lit.Test.FAIL, report + expectedExitCode = 1 + if use_verify: + cmd += ['-Xclang', '-verify'] + expectedExitCode = 0 + report,exitCode,_,_ = self.execute_command_and_report(cmd) + if exitCode == expectedExitCode: + return lit.Test.PASS, report, "" else: - exec_file = tempfile.NamedTemporaryFile(suffix="exe", delete=False) - exec_path = exec_file.name - exec_file.close() + return lit.Test.FAIL, report, "Expected compilation to fail!" + + def _evaluate_pass_test(self, test, lit_config): + source_path = test.getSourcePath() + source_dir = os.path.dirname(source_path) + report = "" + + # If we are generating code coverage data run the compile command from + # the directory the .gcno and .gcda files should be put in. + build_cwd = None + if self.generate_coverage: + source_dir_str = str(os.path.realpath(source_dir)) + # Subtract the test prefix "/.../libcxx/test/" from the test file directory. + build_cwd = source_dir_str[len(str(self.test_root)) + 1:] + # Set build_cwd to "/.../build-libcxx/test/coverage/test/.../path-to-test-dir" + build_cwd = str(os.path.join(self.coverage_root, 'test', build_cwd)) + # Recursivly create the directory try: - compile_cmd = [self.cxx_under_test, '-o', exec_path, - source_path] + self.cpp_flags + self.ld_flags - cmd = compile_cmd - out, err, exitCode = self.execute_command(cmd) - if exitCode != 0: - report = """Command: %s\n""" % ' '.join(["'%s'" % a - for a in cmd]) - report += """Exit Code: %d\n""" % exitCode - if out: - report += """Standard Output:\n--\n%s--""" % out - if err: - report += """Standard Error:\n--\n%s--""" % err - report += "\n\nCompilation failed unexpectedly!" - return lit.Test.FAIL, report + os.makedirs(build_cwd) + except: + pass + exec_file = tempfile.NamedTemporaryFile(suffix="exe", delete=False) + exec_path = exec_file.name + exec_file.close() + + try: + if self.use_clang_tidy: + status,report,msg = self._execute_clang_tidy_test(source_path) + if status != lit.Test.PASS: + return status,report,msg + + compile_cmd = [self.cxx_under_test, '-o', exec_path, + source_path] + self.cpp_flags + self.ld_flags + if self.generate_coverage: + compile_cmd += self.coverage_flags + if self.use_scan_build: + cmd = [self.use_scan_build, '--status-bugs', + '-o', self.scan_build_output] + self.scan_build_args + else: cmd = [] - if self.exec_env: - cmd.append('env') - cmd.extend('%s=%s' % (name, value) - for name,value in self.exec_env.items()) - cmd.append(exec_path) - if lit_config.useValgrind: - cmd = lit_config.valgrindArgs + cmd - out, err, exitCode = self.execute_command(cmd, source_dir) - if exitCode != 0: - report = """Compiled With: %s\n""" % \ - ' '.join(["'%s'" % a for a in compile_cmd]) - report += """Command: %s\n""" % \ - ' '.join(["'%s'" % a for a in cmd]) - report += """Exit Code: %d\n""" % exitCode - if out: - report += """Standard Output:\n--\n%s--""" % out - if err: - report += """Standard Error:\n--\n%s--""" % err - report += "\n\nCompiled test failed unexpectedly!" - return lit.Test.FAIL, report - finally: - try: - os.remove(exec_path) - except: - pass - return lit.Test.PASS, "" + cmd += compile_cmd + report,exitCode,_,_ = self.execute_command_and_report(cmd, cwd=build_cwd) + if exitCode != 0 or (self.use_scan_build and + (not os.path.exists(exec_path) + or os.path.getsize(exec_path) == 0)): + return lit.Test.FAIL, report, "Compilation failed unexpectedly!" + + cmd = [] + cmd.append(exec_path) + if lit_config.useValgrind: + cmd = lit_config.valgrindArgs + cmd + report,exitCode,_,_ = self.execute_command_and_report(cmd, + cwd=source_dir, + env=self.exec_env, + compile_cmd=compile_cmd) + if exitCode != 0: + return lit.Test.FAIL, report, "Compiled test failed unexpectedly!" + + finally: + try: + os.remove(exec_path) + except: + pass + return lit.Test.PASS, report, "" + + + def _execute_clang_tidy_test(self, source_file): + cmd = [self.use_clang_tidy] + self.clang_tidy_args + [source_file] + cmd += ['--'] + self.cpp_flags + report,exitCode,out,_ = self.execute_command_and_report(cmd) + if exitCode != 0 or len(out) != 0: + return lit.Test.FAIL, report, "Compilation failed unexpectedly!" + return lit.Test.PASS, report, "" class Configuration(object): @@ -195,12 +272,21 @@ self.cxx = None self.src_root = None self.obj_root = None + self.test_root = None self.env = {} self.compile_flags = [] self.library_paths = [] self.link_flags = [] + self.generate_coverage = False + self.coverage_flags = [] + self.coverage_root = None self.use_system_lib = False self.use_clang_verify = False + self.use_scan_build = None + self.scan_build_output = None + self.scan_build_args = [] + self.use_clang_tidy = None + self.clang_tidy_args = [] if platform.system() not in ('Darwin', 'FreeBSD', 'Linux'): self.lit_config.fatal("unrecognized system") @@ -217,9 +303,20 @@ conf = self.get_lit_conf(name) if conf is None: return None - if conf.lower() in ('1', 'true'): + if conf.lower() in ('1', 'true', 'on'): return True - if conf.lower() in ('', '0', 'false'): + if conf.lower() in ('', '0', 'false', 'off'): + return False + self.lit_config.fatal( + "parameter '{}' should be true or false".format(name)) + + def get_lit_switch(self, name): + s = self.get_lit_conf(name) + if s is None: + return False + elif s.lower() in ('', '1', 'true', 'on'): + return True + elif s.lower() in ('0', 'false', 'off'): return False self.lit_config.fatal( "parameter '{}' should be true or false".format(name)) @@ -229,22 +326,38 @@ self.configure_triple() self.configure_src_root() self.configure_obj_root() + self.configure_test_root() self.configure_use_system_lib() self.configure_use_clang_verify() - self.configure_env() self.configure_std_flag() self.configure_compile_flags() self.configure_link_flags() self.configure_sanitizer() + self.configure_warnings() + self.configure_generate_coverage() + self.configure_scan_build() + self.configure_clang_tidy() self.configure_features() + self.lit_config.note('using compile_flags: %r' % self.compile_flags) + self.lit_config.note('using link_flags: %r' % self.link_flags) + self.lit_config.note('using available_features: %r' % self.config.available_features) def get_test_format(self): return LibcxxTestFormat( self.cxx, - self.use_clang_verify, cpp_flags=['-nostdinc++'] + self.compile_flags, ld_flags=['-nodefaultlibs'] + self.link_flags, - exec_env=self.env) + exec_env=self.env, + test_root=self.test_root, + use_verify_for_fail=self.use_clang_verify, + generate_coverage=self.generate_coverage, + coverage_flags=self.coverage_flags, + coverage_root=self.coverage_root, + use_scan_build=self.use_scan_build, + scan_build_output=self.scan_build_output, + scan_build_args=self.scan_build_args, + use_clang_tidy=self.use_clang_tidy, + clang_tidy_args=self.clang_tidy_args) def configure_cxx(self): # Gather various compiler parameters. @@ -257,11 +370,10 @@ self.config.environment['PATH']) if clangxx: self.cxx = clangxx - self.lit_config.note( - "inferred cxx_under_test as: %r" % self.cxx) if not self.cxx: self.lit_config.fatal('must specify user parameter cxx_under_test ' '(e.g., --param=cxx_under_test=clang++)') + self.lit_config.note('using cxx_under_test as: %r' % self.cxx) def configure_src_root(self): self.src_root = self.get_lit_conf( @@ -270,6 +382,9 @@ def configure_obj_root(self): self.obj_root = self.get_lit_conf('libcxx_obj_root', self.src_root) + def configure_test_root(self): + self.test_root = str(os.path.realpath(os.path.join(self.src_root, 'test'))) + def configure_use_system_lib(self): # This test suite supports testing against either the system library or # the locally built one; the former mode is useful for testing ABI @@ -279,17 +394,13 @@ if self.use_system_lib is None: # Default to testing against the locally built libc++ library. self.use_system_lib = False - self.lit_config.note( - "inferred use_system_lib as: %r" % self.use_system_lib) + else: + self.lit_config.note('using use_system_lib as: %r' % self.use_system_lib) def configure_use_clang_verify(self): '''If set, run clang with -verify on failing tests.''' - self.use_clang_verify = self.get_lit_bool('use_clang_verify') - if self.use_clang_verify is None: - # TODO: Default this to True when using clang. - self.use_clang_verify = False - self.lit_config.note( - "inferred use_clang_verify as: %r" % self.use_clang_verify) + self.use_clang_verify = self.get_lit_switch('use_clang_verify') + def configure_features(self): additional_features = self.get_lit_conf('additional_features') @@ -359,54 +470,74 @@ self.compile_flags += ['-D_LIBCPP_HAS_NO_MONOTONIC_CLOCK'] def configure_compile_flags(self): + # Check if libcxx_headers is defined. If it is then use that as + # the location for the libcxx headers. + libcxx_headers = self.get_lit_conf('libcxx_headers', + self.src_root + '/include') # Configure extra compiler flags. - self.compile_flags += ['-I' + self.src_root + '/include', + self.compile_flags += ['-I' + libcxx_headers, '-I' + self.src_root + '/test/support'] if sys.platform.startswith('linux'): self.compile_flags += ['-D__STDC_FORMAT_MACROS', '-D__STDC_LIMIT_MACROS', '-D__STDC_CONSTANT_MACROS'] + # Add extra compile flags as specified on the command line. + extra_flags = self.get_lit_conf('compile_flags') + if extra_flags: + self.compile_flags += shlex.split(extra_flags) + def configure_link_flags(self): - # Configure library search paths - abi_library_path = self.get_lit_conf('abi_library_path', '') - self.link_flags += ['-L' + self.obj_root + '/lib'] - if not self.use_system_lib: - self.link_flags += ['-Wl,-rpath', '-Wl,' + self.obj_root + '/lib'] + # Configure libc++ library + libcxx_library = self.get_lit_conf('libcxx_library') + # Ignore and warn if use_system_lib is true and libcxx_library is specified. + if self.use_system_lib and libcxx_library: + self.lit_config.warning( + 'Conflicting options detected: libcxx_library and use_system_lib') + libcxx_library = None + # Link to the specific libc++ library if specified. Otherwise link to -lc++ + # and add the build directory as a search path if use_system_lib is False. + if libcxx_library: + self.link_flags += [libcxx_library, '-Wl,-rpath', + '-Wl,' + os.path.dirname(libcxx_library)] + else: + self.link_flags += ['-lc++'] + if not self.use_system_lib: + self.link_flags += ['-L' + self.obj_root + '/lib', '-Wl,-rpath', + '-Wl,' + self.obj_root + '/lib'] + + # Add a search path for the ABI library path if specified. + abi_library_path = self.get_lit_conf('abi_library_path') if abi_library_path: self.link_flags += ['-L' + abi_library_path, '-Wl,-rpath', '-Wl,' + abi_library_path] - # Configure libraries - self.link_flags += ['-lc++'] - link_flags_str = self.get_lit_conf('link_flags') - if link_flags_str is None: - cxx_abi = self.get_lit_conf('cxx_abi', 'libcxxabi') - if cxx_abi == 'libstdc++': - self.link_flags += ['-lstdc++'] - elif cxx_abi == 'libsupc++': - self.link_flags += ['-lsupc++'] - elif cxx_abi == 'libcxxabi': - self.link_flags += ['-lc++abi'] - elif cxx_abi == 'libcxxrt': - self.link_flags += ['-lcxxrt'] - elif cxx_abi == 'none': - pass - else: - self.lit_config.fatal( - 'C++ ABI setting %s unsupported for tests' % cxx_abi) - - if sys.platform == 'darwin': - self.link_flags += ['-lSystem'] - elif sys.platform.startswith('linux'): - self.link_flags += ['-lgcc_eh', '-lc', '-lm', '-lpthread', - '-lrt', '-lgcc_s'] - elif sys.platform.startswith('freebsd'): - self.link_flags += ['-lc', '-lm', '-pthread', '-lgcc_s'] - else: - self.lit_config.fatal("unrecognized system: %r" % sys.platform) - self.lit_config.note( - "inferred link_flags as: %r" % self.link_flags) + cxx_abi = self.get_lit_conf('cxx_abi', 'libcxxabi') + if cxx_abi == 'libstdc++': + self.link_flags += ['-lstdc++'] + elif cxx_abi == 'libsupc++': + self.link_flags += ['-lsupc++'] + elif cxx_abi == 'libcxxabi': + self.link_flags += ['-lc++abi'] + elif cxx_abi == 'libcxxrt': + self.link_flags += ['-lcxxrt'] + elif cxx_abi == 'none': + pass + else: + self.lit_config.fatal( + 'C++ ABI setting %s unsupported for tests' % cxx_abi) + + if sys.platform == 'darwin': + self.link_flags += ['-lSystem'] + elif sys.platform.startswith('linux'): + self.link_flags += ['-lgcc_eh', '-lc', '-lm', '-lpthread', + '-lrt', '-lgcc_s'] + elif sys.platform.startswith('freebsd'): + self.link_flags += ['-lc', '-lm', '-pthread', '-lgcc_s'] + else: + self.lit_config.fatal("unrecognized system: %r" % sys.platform) + + link_flags_str = self.get_lit_conf('link_flags') if link_flags_str: self.link_flags += shlex.split(link_flags_str) @@ -423,8 +554,8 @@ self.config.available_features.add(std) def configure_sanitizer(self): - san = self.get_lit_conf('llvm_use_sanitizer', '').strip() - if san: + san = self.get_lit_conf('use_sanitizer').strip() + if san and san != 'None': self.compile_flags += ['-fno-omit-frame-pointer'] if sys.platform.startswith('linux'): self.link_flags += ['-ldl'] @@ -438,12 +569,67 @@ self.config.available_features.add('msan') elif san == 'Undefined': self.compile_flags += ['-fsanitize=undefined', - '-fno-sanitize=vptr,function', - '-fno-sanitize-recover'] + '-fno-sanitize=vptr,function', + '-fno-sanitize-recover'] self.config.available_features.add('ubsan') else: self.lit_config.fatal('unsupported value for ' - 'libcxx_use_san: {0}'.format(san)) + 'use_sanitizer: {0}'.format(san)) + + def configure_warnings(self): + enable_warnings = self.get_lit_bool('enable_warnings') + if enable_warnings: + self.lit_config.note('Enabling warnings') + self.compile_flags += ['-Wall', '-Wextra', '-pedantic', + '-D_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER'] + + def configure_generate_coverage(self): + self.generate_coverage = self.get_lit_bool('generate_coverage') + if self.generate_coverage: + self.coverage_flags = ['-g', '-O0', '--coverage', '-fprofile-arcs', '-ftest-coverage'] + self.coverage_root = self.get_lit_conf('coverage_root', None) + if not self.coverage_root: + self.lit_config.fatal("'%s' is not a valid value for coverage_root" % self.coverage_root) + self.lit_config.note('outputing code coverage to: %s' % self.coverage_root) + + def configure_scan_build(self): + self.use_scan_build = self.get_lit_conf('use_scan_build') + if self.use_scan_build is None: + return + if self.use_scan_build == '': + self.use_scan_build = 'scan-build' + self.lit_config.note( + 'Using scan-build: %s' % self.use_scan_build) + output = self.get_lit_conf('scan_build_output') + if output is None: + output = tempfile.mkdtemp(suffix='', prefix='libcxx-scan-build-') + self.lit_config.note('Writing scan-build output to: %s' % output) + self.scan_build_output = output + args = self.get_lit_conf('scan_build_args') + if args: + args = shlex.split(args) + else: + args = ['-analyze-headers'] + self.lit_config.note('Using scan-build args: %r' % args) + self.scan_build_args = args + + + def configure_clang_tidy(self): + self.use_clang_tidy = self.get_lit_conf('use_clang_tidy') + if self.use_clang_tidy is None: + return + if self.use_clang_tidy == '': + self.use_clang_tidy = 'clang-tidy' + self.lit_config.note( + 'Using clang-tidy: %s' % self.use_clang_tidy) + args = self.get_lit_conf('clang_tidy_args') + if args is not None: + self.clang_tidy_args = shlex.split(args) + else: + self.clang_tidy_args = ["-checks=-*,llvm*,clang-analyzer*,-clang-analyzer-alpha*", + "-header-filter='.*'"] + self.lit_config.note('Using clang-tidy args: %r' % self.clang_tidy_args) + def configure_triple(self): # Get or infer the target triple. @@ -452,7 +638,7 @@ # under test. if not self.config.target_triple: target_triple = lit.util.capture( - [self.cxx, '-dumpmachine']).strip() + [self.cxx, '-dumpmachine']).strip() # Drop sub-major version components from the triple, because the # current XFAIL handling expects exact matches for feature checks. # Example: x86_64-apple-darwin14.0.0 -> x86_64-apple-darwin14 @@ -467,15 +653,8 @@ target_triple.endswith('suse-linux'): target_triple += '-gnu' self.config.target_triple = target_triple - self.lit_config.note( - "inferred target_triple as: %r" % self.config.target_triple) - - def configure_env(self): - # Configure extra linker parameters. - if sys.platform == 'darwin': - if not self.use_system_lib: - self.env['DYLD_LIBRARY_PATH'] = os.path.join(self.obj_root, - 'lib') + self.lit_config.note( + "using target_triple: %r" % self.config.target_triple) # name: The name of this test suite. Index: test/lit.site.cfg.in =================================================================== --- test/lit.site.cfg.in +++ test/lit.site.cfg.in @@ -6,8 +6,10 @@ config.python_executable = "@PYTHON_EXECUTABLE@" config.enable_shared = @LIBCXX_ENABLE_SHARED@ config.cxx_abi = "@LIBCXX_CXX_ABI_LIBNAME@" -config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@" config.abi_library_path = "@LIBCXX_CXX_ABI_LIBRARY_PATH@" +config.use_sanitizer = "@LLVM_USE_SANITIZER@" +config.generate_coverage = "@LIBCXX_GENERATE_COVERAGE@" +config.coverage_root = "@LIBCXX_COVERAGE_DIR@" # Let the main config do the real work. lit_config.load_config(config, "@LIBCXX_SOURCE_DIR@/test/lit.cfg") Index: www/index.html =================================================================== --- www/index.html +++ www/index.html @@ -473,6 +473,7 @@
  • <type_traits>
  • Excellent notes by Marshall Clow
  • Status of debug mode
  • +
  • Using LIT on the command line
  • Index: www/lit_usage.html =================================================================== --- /dev/null +++ www/lit_usage.html @@ -0,0 +1,227 @@ + + + + + + Testing libc++ using LIT + + + + + + + + +
    + +

    Testing libc++ using LIT

    + +

    +libc++ uses LIT to configure and run it's tests. The primary way to run the +libc++ tests is by using make check-libcxx. However since libc++ +can be used in any number of possible configurations it is important to +customize the way LIT builds and runs the tests. This guide provides +information on how to use LIT directly to test libc++. +

    +

    +Documentation for LIT can be found +here. +

    + + +

    Getting Started

    + +

    +After building libc++ use the following commands before you start using LIT to +test. +

    + +

    +You can now run the libc++ tests by running: +

    + +

    +To only run a subsection of the tests use: +

    + + +

    Customization Options

    + +

    +libc++'s testsuite provides multiple options to configure the way the tests +are build and run. To use these options you pass them on the LIT command line +as --param NAME or --param NAME=VALUE. Some options +have default values specified during CMake's configuration. Passing the option +on the command line will override the default. +

    + +

    +

    libcxx_headers=<path/to/headers>

    +
    +Specify the libc++ headers that are tested. By default the headers in the source +tree are used. +
    +

    + +

    +

    libcxx_library=<path/to/libc++.so>

    +
    +Specify the libc++ library that is tested. By default the library in the build +directory is used. This option has no effect if use_system_lib is provided. +
    +

    + +

    +

    use_system_lib=<bool>

    +
    +Default: False
    +Enable or disable testing against the installed version of libc++ library. +Note: This does not use the installed headers. +
    +

    + +

    +

    compile_flags="<list-of-args>"

    +
    +Specify an additional list of compile flags to use. +Note: This options should not be used to change the standard version used. +
    +

    + +

    +

    link_flags="<list-of-args>"

    +
    +Specify an additional list of linker flags to use. +
    +

    + +

    +

    std=<standard version>

    +
    +Values: c++98, c++03, c++11, c++1z, c++14
    +Change the standard version used when building the tests. +
    +

    + +

    +

    use_sanitizer=<sanitizer name>

    +
    +Values: Memory, MemoryWithOrigins, Address, Undefined, None
    +Run the tests using the given sanitizer. If LLVM_USE_SANITIZER +was given when building libc++ then that sanitizer will be used by default. +If None is given then no sanitizer will be used. +
    +

    + +

    +

    enable_warnings[=<bool>]

    +
    +Default: False
    +Enable -Wall -Wextra -pedantic as well as disabling +the system header pragmas. +
    +

    + +

    +

    use_clang_verify[=<bool>]

    +
    +Default: False
    +Enable the use of clang verif on tests that expect a compilation error. +This verifies that the test fails to compile for the right reason. +
    +

    + +

    Scan Build Options

    + +

    +

    use_scan_build[=<path/to/scan-build>]

    +
    +When specified the tests are compiled through scan-build. The optional value is +used to specify the scan-build executable to use. If scan-build finds a defect +the test will fail. +
    +

    + +

    +

    scan_build_output=<path/to/scan-build/output>

    +
    +Specify a folder in which to store the scan-build output. If scan-build is +used without this option a temporary directory is created and reported at the +start of the LIT run. +
    +

    + +

    +

    scan_build_args="<list-of-args>"

    +
    +Default: -analyze-headers
    +Specify a list of args to be passed to scan-build. +
    +

    + +

    Clang Tidy Options

    + +

    +

    use_clang_tidy[=<path/to/clang-tidy>

    +
    +When specified the tests are run through clang-tidy as an extra step. The test +will fail if clang-tidy produces any output. The option value is used to specify +the clang-tidy executable to use. +
    +

    + + +

    +

    clang_tidy_args="<list-of-args>"

    +
    +Default: -checks=-*,llvm*,clang-analyzer*,-clang-analyzer-alpha* + -header-filter='.*'
    +Specify a list of args to be passed to clang-tidy. +
    +

    + + +
    + +