[WIP] moving libfuzzer to "runtimes"

[george.karpenkov]

Summary

Hi,

This is still work in progress, with a few TODO items left.
The main remaining issue that it is for now it is not clear how to nicely configure libFuzzer tests do depend on other runtimes (asan/ubsan).
I'll ask that question separately on llvm-dev.

Many of changes done in this diff can/should be moved to separate small patches, and they will be, but I wanted to show the overall picture as well.

Supporting self-contained run still remains to be done.

.gitignore

 
 #==============================================================================#
 # Directories to ignore (do not add trailing '/'s, they skip symlinks).
 #==============================================================================#
 # External projects that are tracked independently.
 projects/*
 !projects/*.*
 !projects/Makefile
-runtimes/*
 !runtimes/*.*
 # Clang, which is tracked independently.
 tools/clang
 # LLDB, which is tracked independently.
 tools/lldb
 # lld, which is tracked independently.
 tools/lld
 # llgo, which is tracked independently.

cmake/modules/LLVMExternalProjectUtils.cmake

CMAKE_ARGS ${${nameCanon}_CMAKE_ARGS}
                ${compiler_args}
                -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
                -DLLVM_BINARY_DIR=${PROJECT_BINARY_DIR}
                -DLLVM_CONFIG_PATH=$<TARGET_FILE:llvm-config>
                -DLLVM_ENABLE_WERROR=${LLVM_ENABLE_WERROR}
                -DPACKAGE_VERSION=${PACKAGE_VERSION}
                -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
                -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}
+               -DCMAKE_EXPORT_COMPILE_COMMANDS=1
                ${ARG_CMAKE_ARGS}
                ${PASSTHROUGH_VARIABLES}
     INSTALL_COMMAND ""
     STEP_TARGETS configure build
     BUILD_ALWAYS 1
     USES_TERMINAL_CONFIGURE 1
     USES_TERMINAL_BUILD 1
     USES_TERMINAL_INSTALL 1

lib/CMakeLists.txt

 add_subdirectory(ObjectYAML)
 add_subdirectory(Option)
 add_subdirectory(DebugInfo)
 add_subdirectory(ExecutionEngine)
 add_subdirectory(Target)
 add_subdirectory(AsmParser)
 add_subdirectory(LineEditor)
 add_subdirectory(ProfileData)
-add_subdirectory(Fuzzer)
 add_subdirectory(Passes)
 add_subdirectory(ToolDrivers)
 add_subdirectory(XRay)
 add_subdirectory(Testing)

lib/Fuzzer/test/fuzzer-customcrossoverandmutate.test

-RUN: LLVMFuzzer-CustomCrossOverAndMutateTest -seed=1 -runs=100000

runtimes/CMakeLists.txt

if(LLVM_INCLUDE_TESTS)
           count
           llvm-nm
           llvm-objdump
           llvm-xray
           not
           obj2yaml
           sancov
           sanstats
+          gtest_main
+          gtest
+          # TODO: we can not depend on runtimes, see
+          # https://reviews.llvm.org/D33048 for the discussion.
+          asan
+          ubsan
         )
       foreach(target ${test_targets} ${SUB_COMPONENT_CHECK_TARGETS})
         add_dependencies(${target} ${RUNTIMES_TEST_DEPENDS})
       endforeach()
     endif()
   endif()
 endif()

runtimes/libfuzzer/CMakeLists.txt

 include(CheckCXXSourceCompiles)
 
+add_custom_target(fuzzer)
+add_dependencies(fuzzer LLVMFuzzer LLVMFuzzerNoMain)
+
 if( APPLE )
   CHECK_CXX_SOURCE_COMPILES("
       static thread_local int blah;
       int main() {
         return 0;
       }
       " HAS_THREAD_LOCAL)
 
   if( NOT HAS_THREAD_LOCAL )
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Dthread_local=__thread")
   endif()
 endif()
 
+set(LIBFUZZER_FLAGS_BASE "${CMAKE_CXX_FLAGS}")
+
 if( LLVM_USE_SANITIZE_COVERAGE )
   if(NOT "${LLVM_USE_SANITIZER}" STREQUAL "Address")
     message(FATAL_ERROR
       "LibFuzzer and its tests require LLVM_USE_SANITIZER=Address and "
       "LLVM_USE_SANITIZE_COVERAGE=YES to be set."
       )
   endif()
-  set(LIBFUZZER_FLAGS_BASE "${CMAKE_CXX_FLAGS}")
 
   # Disable the coverage and sanitizer instrumentation for the fuzzer itself.
   set(CMAKE_CXX_FLAGS "${LIBFUZZER_FLAGS_BASE} -fno-sanitize-coverage=trace-pc-guard,edge,trace-cmp,indirect-calls,8bit-counters -Werror")
 endif()
 
+# TODO: move this function elsewhere.
+function(set_target_output_directories target output_dir)
+  # For RUNTIME_OUTPUT_DIRECTORY variable, Multi-configuration generators
+  # append a per-configuration subdirectory to the specified directory.
+  # To avoid the appended folder, the configuration specific variable must be
+  # set 'RUNTIME_OUTPUT_DIRECTORY_${CONF}':
+  # RUNTIME_OUTPUT_DIRECTORY_DEBUG, RUNTIME_OUTPUT_DIRECTORY_RELEASE, ...
+  if(CMAKE_CONFIGURATION_TYPES)
+    foreach(build_mode ${CMAKE_CONFIGURATION_TYPES})
+      string(TOUPPER "${build_mode}" CONFIG_SUFFIX)
+      set_target_properties("${target}" PROPERTIES
+          "ARCHIVE_OUTPUT_DIRECTORY_${CONFIG_SUFFIX}" ${output_dir}
+          "LIBRARY_OUTPUT_DIRECTORY_${CONFIG_SUFFIX}" ${output_dir}
+          "RUNTIME_OUTPUT_DIRECTORY_${CONFIG_SUFFIX}" ${output_dir})
+    endforeach()
+  else()
+    set_target_properties("${target}" PROPERTIES
+        ARCHIVE_OUTPUT_DIRECTORY ${output_dir}
+        LIBRARY_OUTPUT_DIRECTORY ${output_dir}
+        RUNTIME_OUTPUT_DIRECTORY ${output_dir})
+  endif()
+endfunction()
+
 # Compile libFuzzer if the compilation is specifically requested, OR
 # if the platform is known to be working.
 if ( LLVM_USE_SANITIZE_COVERAGE OR CMAKE_SYSTEM_NAME MATCHES "Darwin|Linux" )
   add_library(LLVMFuzzerNoMainObjects OBJECT
       FuzzerCrossOver.cpp
       FuzzerDriver.cpp
       FuzzerExtFunctionsDlsym.cpp
       FuzzerExtFunctionsDlsymWin.cpp
add_library(LLVMFuzzerNoMain STATIC
       $<TARGET_OBJECTS:LLVMFuzzerNoMainObjects>
       )
   target_link_libraries(LLVMFuzzerNoMain ${LLVM_PTHREAD_LIB})
   add_library(LLVMFuzzer STATIC
       FuzzerMain.cpp
       $<TARGET_OBJECTS:LLVMFuzzerNoMainObjects>
       )
   target_link_libraries(LLVMFuzzer ${LLVM_PTHREAD_LIB})
+
+
+  # TODO: Avoid duplication with compiler-rt/cmake/base-config-ix.cmake
+  if(NOT DEFINED OS_DIR)
+    string(TOLOWER ${CMAKE_SYSTEM_NAME} OS_DIR)
+  endif()
+  string(REGEX MATCH "[0-9]+\\.[0-9]+(\\.[0-9]+)?" CLANG_VERSION
+         ${PACKAGE_VERSION})
+
+  # TODO: avoid repetition, extract into a function.
+  set_target_output_directories(LLVMFuzzer
+    ${LLVM_LIBRARY_OUTPUT_INTDIR}/clang/${CLANG_VERSION}/lib/${OS_DIR})
+
+  set_target_output_directories(LLVMFuzzerNoMain
+    ${LLVM_LIBRARY_OUTPUT_INTDIR}/clang/${CLANG_VERSION}/lib/${OS_DIR})
 endif()
 
-if( LLVM_USE_SANITIZE_COVERAGE AND LLVM_INCLUDE_TESTS )
+if( LLVM_INCLUDE_TESTS )
   add_subdirectory(test)
 endif()

runtimes/libfuzzer/test/CMakeLists.txt

 # Build all these tests with -O0, otherwise optimizations may merge some
 # basic blocks and we'll fail to discover the targets.
 # We change the flags for every build type because we might be doing
 # a multi-configuration build (e.g. Xcode) where CMAKE_BUILD_TYPE doesn't
 # mean anything.
 set(variables_to_filter
   CMAKE_CXX_FLAGS_RELEASE
   CMAKE_CXX_FLAGS_DEBUG
   CMAKE_CXX_FLAGS_RELWITHDEBINFO
   CMAKE_CXX_FLAGS_MINSIZEREL
   LIBFUZZER_FLAGS_BASE
   )
 foreach (VARNAME ${variables_to_filter})
   string(REGEX REPLACE "([-/]O)[123s]" "\\10" ${VARNAME} "${${VARNAME}}")
 endforeach()
 
+# TODO: try enabling for all of libfuzzer.
+if(TARGET clang)
+  set(CMAKE_CXX_COMPILER "${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++")
+  set(CMAKE_C_COMPILER "${LLVM_RUNTIME_OUTPUT_INTDIR}/clang")
+endif()
+
+# Relax this restriction from HandleLLVMOptions.
+string(REPLACE "-Wl,-z,defs" "" CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS}")
+
+
 # Enable the coverage instrumentation (it is disabled for the Fuzzer lib).
-set(CMAKE_CXX_FLAGS "${LIBFUZZER_FLAGS_BASE} -fsanitize-coverage=trace-pc-guard,indirect-calls,trace-cmp,trace-div,trace-gep -gline-tables-only")
+set(CMAKE_CXX_FLAGS "${LIBFUZZER_FLAGS_BASE} -fsanitize=address -fsanitize-coverage=trace-pc-guard,indirect-calls,trace-cmp,trace-div,trace-gep -gline-tables-only -std=c++11")
 
 if(MSVC)
   # For tests use the CRT specified for release build
   # (asan doesn't support MDd and MTd)
   if ("${LLVM_USE_CRT_RELEASE}" STREQUAL "")
     set(CRT_FLAG " /MD ")
   else()
     set(CRT_FLAG " /${LLVM_USE_CRT_RELEASE} ")
function(add_libfuzzer_test name)
   endif()
   add_executable(LLVMFuzzer-${name}
     ${add_libfuzzer_test_SOURCES}
     )
   target_link_libraries(LLVMFuzzer-${name} LLVMFuzzer)
   # Place binary where llvm-lit expects to find it
   set_target_properties(LLVMFuzzer-${name}
     PROPERTIES RUNTIME_OUTPUT_DIRECTORY
-    "${CMAKE_BINARY_DIR}/lib/Fuzzer/test"
+    "${CMAKE_BINARY_DIR}/libfuzzer/test"
     )
   add_dependencies(TestBinaries LLVMFuzzer-${name})
 endfunction()
 
 ###############################################################################
 # Basic tests
 ###############################################################################
 

 
 target_link_libraries(LLVMFuzzer-DSOTest
   LLVMFuzzer-DSO1
   LLVMFuzzer-DSO2
   LLVMFuzzer
   )
 
 set_target_properties(LLVMFuzzer-DSOTest PROPERTIES RUNTIME_OUTPUT_DIRECTORY
-  "${CMAKE_BINARY_DIR}/lib/Fuzzer/test")
+  "${CMAKE_BINARY_DIR}/libfuzzer/test")
 
 if(MSVC)
   set_output_directory(LLVMFuzzer-DSO1
-    BINARY_DIR "${CMAKE_BINARY_DIR}/lib/Fuzzer/test"
-    LIBRARY_DIR "${CMAKE_BINARY_DIR}/lib/Fuzzer/test")
+    BINARY_DIR "${CMAKE_BINARY_DIR}/libfuzzer/test"
+    LIBRARY_DIR "${CMAKE_BINARY_DIR}/libfuzzer/test")
   set_output_directory(LLVMFuzzer-DSO2
-    BINARY_DIR "${CMAKE_BINARY_DIR}/lib/Fuzzer/test"
-    LIBRARY_DIR "${CMAKE_BINARY_DIR}/lib/Fuzzer/test")
+    BINARY_DIR "${CMAKE_BINARY_DIR}/libfuzzer/test"
+    LIBRARY_DIR "${CMAKE_BINARY_DIR}/libfuzzer/test")
 else(MSVC)
   set_output_directory(LLVMFuzzer-DSO1
-    LIBRARY_DIR "${CMAKE_BINARY_DIR}/lib/Fuzzer/lib")
+    LIBRARY_DIR "${CMAKE_BINARY_DIR}/libfuzzer/lib")
   set_output_directory(LLVMFuzzer-DSO2
-    LIBRARY_DIR "${CMAKE_BINARY_DIR}/lib/Fuzzer/lib")
+    LIBRARY_DIR "${CMAKE_BINARY_DIR}/libfuzzer/lib")
 endif()
 
 add_dependencies(TestBinaries LLVMFuzzer-DSOTest)
 
 ###############################################################################
 # Configure lit to run the tests
 #
 # Note this is done after declaring all tests so we can inform lit if any tests
configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/unit/lit.site.cfg.in
   ${CMAKE_CURRENT_BINARY_DIR}/unit/lit.site.cfg
   )
 
 add_lit_testsuite(check-fuzzer "Running Fuzzer tests"
     ${CMAKE_CURRENT_BINARY_DIR}
     DEPENDS TestBinaries
     )
-
-# Don't add dependencies on Windows. The linker step would fail on Windows,
-# since cmake will use link.exe for linking and won't include compiler-rt libs.
-if(NOT MSVC)
-  add_dependencies(check-fuzzer FileCheck sancov not)
-endif()

runtimes/libfuzzer/test/FuzzerUnittest.cpp

 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 
 // Avoid ODR violations (LibFuzzer is built without ASan and this test is built
 // with ASan) involving C++ standard library types when using libcxx.
 #define _LIBCPP_HAS_NO_ASAN
 
+// Do not attempt to use LLVM ostream from gtest.
+#define GTEST_NO_LLVM_RAW_OSTREAM 1
+
 #include "FuzzerCorpus.h"
 #include "FuzzerDictionary.h"
 #include "FuzzerInternal.h"
 #include "FuzzerMerge.h"
 #include "FuzzerMutate.h"
 #include "FuzzerRandom.h"
 #include "FuzzerTracePC.h"
 #include "gtest/gtest.h"

runtimes/libfuzzer/test/dump_coverage.test

+XFAIL: darwin
+
 RUN: rm -rf %t_workdir && mkdir -p %t_workdir
 RUN: env ASAN_OPTIONS=coverage_dir='"%t_workdir"' not LLVMFuzzer-NullDerefTest -dump_coverage=1 2>&1 | FileCheck %s
 RUN: sancov -covered-functions LLVMFuzzer-NullDerefTest* %t_workdir/*.sancov | FileCheck %s --check-prefix=SANCOV
 RUN: env ASAN_OPTIONS=coverage_dir='"%t_workdir"' LLVMFuzzer-DSOTest -dump_coverage=1 -runs=0 2>&1 | FileCheck %s --check-prefix=DSO
 RUN: env ASAN_OPTIONS=coverage_dir='"%t_workdir"' not LLVMFuzzer-NullDerefTest -dump_coverage=0 2>&1 | FileCheck %s --check-prefix=NOCOV
 
 CHECK: SanitizerCoverage: {{.*}}LLVMFuzzer-NullDerefTest.{{.*}}.sancov: {{.*}} PCs written
 SANCOV: LLVMFuzzerTestOneInput
 
 DSO: SanitizerCoverage: {{.*}}LLVMFuzzer-DSOTest.{{.*}}.sancov: {{.*}} PCs written
 DSO-DAG: SanitizerCoverage: {{.*}}LLVMFuzzer-DSO1.{{.*}}.sancov: {{.*}} PCs written
 DSO-DAG: SanitizerCoverage: {{.*}}LLVMFuzzer-DSO2.{{.*}}.sancov: {{.*}} PCs written
 
 NOCOV-NOT: SanitizerCoverage: {{.*}} PCs written

runtimes/libfuzzer/test/fuzzer-customcrossoverandmutate.test

+RUN: env ASAN_OPTIONS=detect_container_overflow=0 LLVMFuzzer-CustomCrossOverAndMutateTest -seed=1 -runs=100000

runtimes/libfuzzer/test/no-coverage/CMakeLists.txt

 # AFL Driver test
 ###############################################################################
 if(NOT MSVC)
   add_executable(AFLDriverTest
     ../AFLDriverTest.cpp ../../afl/afl_driver.cpp)
 
   set_target_properties(AFLDriverTest
       PROPERTIES RUNTIME_OUTPUT_DIRECTORY
-      "${CMAKE_BINARY_DIR}/lib/Fuzzer/test"
+      "${CMAKE_BINARY_DIR}/libfuzzer/test"
       )
 
   add_dependencies(TestBinaries AFLDriverTest)
 endif()

runtimes/libfuzzer/test/ubsan/CMakeLists.txt

 # These tests are instrumented with ubsan in non-recovery mode.
 
 set(CMAKE_CXX_FLAGS
-  "${LIBFUZZER_FLAGS_BASE} -fsanitize=undefined -fno-sanitize-recover=all")
+  "${LIBFUZZER_FLAGS_BASE} -fsanitize=undefined -fsanitize=address -fsanitize-coverage=trace-pc-guard,indirect-calls,trace-cmp,trace-div,trace-gep -fno-sanitize-recover=all")
 
 set(UbsanTests
   SignedIntOverflowTest
   )
 
 foreach(Test ${UbsanTests})
   add_libfuzzer_test(${Test}-Ubsan SOURCES ../${Test}.cpp)
 endforeach()