[libcxx] Add code coverage configuration to CMake and LIT.

[EricWF]

Summary

This patch adds configuration to CMake and LIT for running the libc++ test-suite to generate code coverage.

To use code coverage use following instructions.

• Find the clang resource dir using $CXX -print-search-dirs. Let <library-dir> be the first library search directory.
• cmake <regular-options> -DLIBCXX_GENERATE_COVERAGE=ON -DLIBCXX_COVERAGE_LIBRARY=<library-dir>/lib/<platform>/libclang_rt.profile.a <source>
• make cxx
• make check-libcxx
• make generate-libcxx-coverage

The reason I want this patch upstreamed is so I can setup a bot that generates code coverage and posts in online for every revision.

[danalbert]

CMakeLists.txt

69

Why is this needed? --coverage should handle this for us.

[EricWF]

CMakeLists.txt

69

--coverage doesn't link profileRT with -nodefaultlibs. Unfortunately this means we have to manually link it.

[danalbert]

CMakeLists.txt

69

That's disappointing, but makes sense.

We really need to add a -stdlib=none...

Couldn't cmake handle finding this directory by calling -print-search-dirs?

test/libcxx/test/config.py

601

But it doesn't need to be linked here?

I don't think we actually want --coverage for the tests.

[EricWF]

CMakeLists.txt

69

Couldn't cmake handle finding this directory by calling -print-search-dirs?

I had a CMake patch that did this, but it's ugly and tricky. There are a bunch of problems manually linking compiler-rt libraries. The format of the library path is <search-dir-output>/lib/<platform>/libclang_rt.profile-<arch>.a. So CMake would have to automagically figure out the platform name and the target arch. It's a lot easier for this one case to just make the user provide it manually.

test/libcxx/test/config.py

601

I haven't needed to manually link. It seems that profileRT is typically a static library so all the symbols get exposed via libc++.

I don't think we actually want --coverage for the tests.

Why not? we need to build the tests with code coverage so that header-only parts of libc++ get coverage stats. The coverage information for all files under libcxx/test are removed from the final output.

[danalbert]

LGTM.

CMakeLists.txt

69

Okay. If it gets to be enough of a PITA we'll fix it later, but this will do for now unless anyone else feels strongly about it.

test/libcxx/test/config.py

601

I haven't needed to manually link. It seems that profileRT is typically a static library so all the symbols get exposed via libc++.

That makes sense.

we need to build the tests with code coverage so that header-only parts of libc++ get coverage stats.

Right... was thinking of the more typical coverage case where most of the code isn't in headers.

CMakeLists.txt

    This option may only be used when LIBCXX_ENABLE_THREADS=OFF." ON)
 option(LIBCXX_INSTALL_HEADERS "Install the libc++ headers." ON)
 option(LIBCXX_INSTALL_SUPPORT_HEADERS "Install libc++ support headers." ON)
+option(LIBCXX_GENERATE_COVERAGE "Enable generating code coverage." OFF)
+set(LIBCXX_COVERAGE_LIBRARY "" CACHE STRING
+       "The Profile-rt library used to build with code coverage")
 option(LIBCXX_ENABLE_STATIC_ABI_LIBRARY "Statically link the ABI library" OFF)
 set(LIBCXX_SYSROOT "" CACHE STRING "Use alternate sysroot.")
 set(LIBCXX_GCC_TOOLCHAIN "" CACHE STRING "Use alternate GCC toolchain.")

 include(config-ix)
 # Configure ABI library
 include(HandleLibCXXABI)
+# Configure coverage options.
+if (LIBCXX_GENERATE_COVERAGE)
+  include(CodeCoverage)
+  set(CMAKE_BUILD_TYPE "COVERAGE" CACHE STRING "" FORCE)
+endif()
 
 #===============================================================================
 # Setup Compiler Flags

 
 append_if(LIBCXX_CXX_FLAGS LIBCXX_TARGET_TRIPLE
           "-target ${LIBCXX_TARGET_TRIPLE}")
+
 append_if(LIBCXX_CXX_FLAGS LIBCXX_SYSROOT "--sysroot ${LIBCXX_SYSROOT}")
 append_if(LIBCXX_CXX_FLAGS LIBCXX_GCC_TOOLCHAIN
           "-gcc-toolchain ${LIBCXX_GCC_TOOLCHAIN}")
 
+if (LLVM_USE_SANITIZER AND LIBCXX_GENERATE_COVERAGE)
+  message(FATAL_ERROR "LLVM_USE_SANITIZER cannot be used with LIBCXX_GENERATE_COVERAGE")
+endif()
+
 string(REPLACE ";" " " LIBCXX_CXX_FLAGS "${LIBCXX_CXX_FLAGS}")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${LIBCXX_CXX_FLAGS}")
 

cmake/Modules/CodeCoverage.cmake

+find_program(CODE_COVERAGE_LCOV lcov)
+if (NOT CODE_COVERAGE_LCOV)
+  message(FATAL_ERROR "Cannot find lcov...")
+endif()
+
+find_program(CODE_COVERAGE_GENHTML genhtml)
+if (NOT CODE_COVERAGE_GENHTML)
+  message(FATAL_ERROR "Cannot find genhtml...")
+endif()
+
+set(CMAKE_CXX_FLAGS_COVERAGE "-g -O0 --coverage")
+
+function(setup_lcov_test_target_coverage target_name output_dir capture_dirs source_dirs)
+  file(MAKE_DIRECTORY ${output_dir})
+
+  set(CAPTURE_DIRS "")
+  foreach(cdir ${capture_dirs})
+    list(APPEND CAPTURE_DIRS "-d;${cdir}")
+  endforeach()
+
+  set(EXTRACT_DIRS "")
+  foreach(sdir ${source_dirs})
+    list(APPEND EXTRACT_DIRS "'${sdir}/*'")
+  endforeach()
+
+  message(STATUS "Capture Directories: ${CAPTURE_DIRS}")
+  message(STATUS "Extract Directories: ${EXTRACT_DIRS}")
+
+  add_custom_target(generate-lib${target_name}-coverage
+        COMMAND ${CODE_COVERAGE_LCOV} --capture ${CAPTURE_DIRS} -o test_coverage.info
+        COMMAND ${CODE_COVERAGE_LCOV} --extract test_coverage.info ${EXTRACT_DIRS} -o test_coverage.info
+        COMMAND ${CODE_COVERAGE_GENHTML} --demangle-cpp test_coverage.info -o test_coverage
+        COMMAND ${CMAKE_COMMAND} -E remove test_coverage.info
+        WORKING_DIRECTORY ${output_dir}
+        COMMENT "Generating coverage results")
+endfunction()

lib/CMakeLists.txt

+set(LIBCXX_LIB_CMAKEFILES_DIR "${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}"  PARENT_SCOPE)
+
 # Get sources
 file(GLOB LIBCXX_SOURCES ../src/*.cpp)
 if(WIN32)

 append_if(libraries LIBCXX_HAS_RT_LIB rt)
 append_if(libraries LIBCXX_HAS_GCC_S_LIB gcc_s)
 
+if (LIBCXX_COVERAGE_LIBRARY)
+  target_link_libraries(cxx ${LIBCXX_COVERAGE_LIBRARY})
+endif()
 target_link_libraries(cxx ${libraries})
 
+
 # Setup flags.
 append_if(LIBCXX_COMPILE_FLAGS LIBCXX_HAS_FPIC_FLAG -fPIC)
 append_if(LIBCXX_LINK_FLAGS LIBCXX_HAS_NODEFAULTLIBS_FLAG -nodefaultlibs)

test/CMakeLists.txt

   pythonize_bool(LIBCXX_ENABLE_STDOUT)
   pythonize_bool(LIBCXX_ENABLE_THREADS)
   pythonize_bool(LIBCXX_ENABLE_MONOTONIC_CLOCK)
+  pythonize_bool(LIBCXX_GENERATE_COVERAGE)
+
   # The tests shouldn't link to any ABI library when it has been linked into
   # libc++ statically.
   if (LIBCXX_ENABLE_STATIC_ABI_LIBRARY)

     DEPENDS cxx
     COMMENT "Running libcxx tests"
     ${cmake_3_2_USES_TERMINAL})
+
+  if (LIBCXX_GENERATE_COVERAGE)
+    include(CodeCoverage)
+    set(output_dir "${CMAKE_CURRENT_BINARY_DIR}/coverage")
+    set(capture_dirs "${LIBCXX_LIB_CMAKEFILES_DIR}/cxx.dir/;${CMAKE_CURRENT_BINARY_DIR}")
+    set(extract_dirs "${LIBCXX_SOURCE_DIR}/include;${LIBCXX_SOURCE_DIR}/src")
+    setup_lcov_test_target_coverage("cxx" "${output_dir}" "${capture_dirs}" "${extract_dirs}")
+  endif()
 else()
   message(WARNING
           "LIT_EXECUTABLE not set, no check-libcxx target will be available!")

test/libcxx/test/config.py

         self.configure_debug_mode()
         self.configure_warnings()
         self.configure_sanitizer()
+        self.configure_coverage()
         self.configure_substitutions()
         self.configure_features()
 

                 self.lit_config.fatal('unsupported value for '
                                       'use_sanitizer: {0}'.format(san))
 
+    def configure_coverage(self):
+        self.generate_coverage = self.get_lit_bool('generate_coverage', False)
+        if self.generate_coverage:
+            self.cxx.flags += ['-g', '--coverage']
+            self.cxx.compile_flags += ['-O0']
+
     def configure_substitutions(self):
         sub = self.config.substitutions
         # Configure compiler substitions

test/lit.site.cfg.in

 config.target_triple            = "@LIBCXX_TARGET_TRIPLE@"
 config.sysroot                  = "@LIBCXX_SYSROOT@"
 config.gcc_toolchain            = "@LIBCXX_GCC_TOOLCHAIN@"
+config.generate_coverage        = "@LIBCXX_GENERATE_COVERAGE@"
 config.target_info              = "@LIBCXX_TARGET_INFO@"
 config.executor                 = "@LIBCXX_EXECUTOR@"