[libFuzzer][MSVC] Enable building libFuzzer with MSVC

[metzman]

Summary

Enable building libFuzzer with MSVC.

Make the following changes to allow this:

• Don't try to include <endian.h> in FuzzerSHA1.cpp. MSVC doesn't have this header, and WINDOWS is always little endian (even on ARM)

• Remove unneeded cast.

• Account for long being 32 bit on x86 Windows to avoid

warnings.

• Use /Ehsc to silence warnings.

• Don't use macros to redefine thread_local since MSVC

complains about it, and it isn't necessary in Windows builds.

[metzman]

• undo changes possibly affecting x86 Linux

[metzman]

Fix compile error

[metzman]

Matt or Vitaly can you please take a look?

This is change enables building libFuzzer with MSVC and fixes any outstanding issues and warnings.

cc @rnk

[vitalybuka]

LGTM

[rnk]

compiler-rt/trunk/lib/fuzzer/CMakeLists.txt

77

Are you sure we want to enable exceptional destructor cleanups in libfuzzer targets? I don't think it will hurt, but what warnings are we hitting? Does libfuzzer actually use try/catch? If so, yes, this makes sense.

[metzman]

compiler-rt/trunk/lib/fuzzer/CMakeLists.txt

77

Are you sure we want to enable exceptional destructor cleanups in libfuzzer targets?

Not particularly, I did it to silence the warnings.

but what warnings are we hitting?

We get about 30 of these warnings:

C:\Program Files (x86)\Microsoft Visual Studio\2017\Professional\VC\Tools\MSVC\14.14.26428\include\xlocale(315): warning C4530: C++ exception handler used, but unwind semantics are not enabled. Specify /EHsc

I think the problem is with FuzzerDefs.h which is included by almost every file.

Does libfuzzer actually use try/catch?

I think it doens't do so explicitly but I assumed it did implicitly by including a system header that uses try/catch.

Maybe we should just get rid of this change and ignore the warning.

[rnk]

I think that means you need to pass -D_HAS_EXCEPTIONS=0 to suppress try / catch usage in the Visual C++ STL headers. Usually /EHs-c- /D_HAS_EXCEPTIONS=0 go together as the canonical "disable exceptions" flags.

[metzman]

In D56510#1368480, @rnk wrote:

I think that means you need to pass -D_HAS_EXCEPTIONS=0 to suppress try / catch usage in the Visual C++ STL headers. Usually /EHs-c- /D_HAS_EXCEPTIONS=0 go together as the canonical "disable exceptions" flags.

OK. I opened https://reviews.llvm.org/D57119 to address this.

compiler-rt/trunk/cmake/config-ix.cmake

if(NOT SANITIZER_MIN_OSX_VERSION)
     if(SANITIZER_MIN_OSX_VERSION VERSION_GREATER "10.9")
       message(WARNING "macOS deployment target '${SANITIZER_MIN_OSX_VERSION}' is too new, setting to '10.9' instead.")
       set(SANITIZER_MIN_OSX_VERSION 10.9)
     endif()
   endif()
 
   # We're setting the flag manually for each target OS
   set(CMAKE_OSX_DEPLOYMENT_TARGET "")
-  
   set(DARWIN_COMMON_CFLAGS -stdlib=libc++)
   set(DARWIN_COMMON_LINK_FLAGS
     -stdlib=libc++
     -lc++
     -lc++abi)
-  
   check_linker_flag("-fapplication-extension" COMPILER_RT_HAS_APP_EXTENSION)
   if(COMPILER_RT_HAS_APP_EXTENSION)
     list(APPEND DARWIN_COMMON_LINK_FLAGS "-fapplication-extension")
   endif()
 
   set(DARWIN_osx_CFLAGS
     ${DARWIN_COMMON_CFLAGS}
     -mmacosx-version-min=${SANITIZER_MIN_OSX_VERSION})
   set(DARWIN_osx_LINK_FLAGS
     ${DARWIN_COMMON_LINK_FLAGS}
     -mmacosx-version-min=${SANITIZER_MIN_OSX_VERSION})
 
   if(DARWIN_osx_SYSROOT)
     list(APPEND DARWIN_osx_CFLAGS -isysroot ${DARWIN_osx_SYSROOT})
     list(APPEND DARWIN_osx_LINK_FLAGS -isysroot ${DARWIN_osx_SYSROOT})
   endif()
 
   # Figure out which arches to use for each OS
   darwin_get_toolchain_supported_archs(toolchain_arches)
   message(STATUS "Toolchain supported arches: ${toolchain_arches}")
-  
   if(NOT MACOSX_VERSION_MIN_FLAG)
     darwin_test_archs(osx
       DARWIN_osx_ARCHS
       ${toolchain_arches})
     message(STATUS "OSX supported arches: ${DARWIN_osx_ARCHS}")
     foreach(arch ${DARWIN_osx_ARCHS})
       list(APPEND COMPILER_RT_SUPPORTED_ARCH ${arch})
       set(CAN_TARGET_${arch} 1)

 if (COMPILER_RT_HAS_SANITIZER_COMMON AND XRAY_SUPPORTED_ARCH AND
     OS_NAME MATCHES "Darwin|Linux|FreeBSD|NetBSD|OpenBSD|Fuchsia")
   set(COMPILER_RT_HAS_XRAY TRUE)
 else()
   set(COMPILER_RT_HAS_XRAY FALSE)
 endif()
 
 if (COMPILER_RT_HAS_SANITIZER_COMMON AND FUZZER_SUPPORTED_ARCH AND
-    OS_NAME MATCHES "Android|Darwin|Linux|NetBSD|FreeBSD|OpenBSD|Fuchsia|Windows" AND
-    # TODO: Support builds with MSVC.
-    NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC" AND
-    NOT "${CMAKE_C_COMPILER_ID}" STREQUAL "MSVC")
+    OS_NAME MATCHES "Android|Darwin|Linux|NetBSD|FreeBSD|OpenBSD|Fuchsia|Windows")
   set(COMPILER_RT_HAS_FUZZER TRUE)
 else()
   set(COMPILER_RT_HAS_FUZZER FALSE)
 endif()
 
 if (COMPILER_RT_HAS_SANITIZER_COMMON AND SHADOWCALLSTACK_SUPPORTED_ARCH AND
     OS_NAME MATCHES "Linux|Android")
   set(COMPILER_RT_HAS_SHADOWCALLSTACK TRUE)
 else()
   set(COMPILER_RT_HAS_SHADOWCALLSTACK FALSE)
 endif()

compiler-rt/trunk/lib/fuzzer/CMakeLists.txt

 endif()
 
 append_list_if(COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG -fno-omit-frame-pointer LIBFUZZER_CFLAGS)
 
 if (CMAKE_CXX_FLAGS MATCHES "fsanitize-coverage")
   list(APPEND LIBFUZZER_CFLAGS -fno-sanitize-coverage=trace-pc-guard,edge,trace-cmp,indirect-calls,8bit-counters)
 endif()
 
+if(OS_NAME MATCHES "Windows")
+  # Silence warnings with /Ehsc and avoid an error by unecessarily defining
+  # thread_local when it isn't even used on Windows.
+  list(APPEND LIBFUZZER_CFLAGS /EHsc)
+else()
   if(NOT HAS_THREAD_LOCAL)
     list(APPEND LIBFUZZER_CFLAGS -Dthread_local=__thread)
   endif()
+endif()
 
 set(FUZZER_SUPPORTED_OS ${SANITIZER_COMMON_SUPPORTED_OS})
 
 add_compiler_rt_object_libraries(RTfuzzer
   OS ${FUZZER_SUPPORTED_OS}
   ARCHS ${FUZZER_SUPPORTED_ARCH}
   SOURCES ${LIBFUZZER_SOURCES}
   ADDITIONAL_HEADERS ${LIBFUZZER_HEADERS}

compiler-rt/trunk/lib/fuzzer/FuzzerBuiltinsMsvc.h

 #error "_BitScanReverse64 unavailable on this platform so MSVC is unsupported."
 #endif
 #include <intrin.h>
 #include <cstdint>
 #include <cstdlib>
 
 // __builtin_return_address() cannot be compiled with MSVC. Use the equivalent
 // from <intrin.h>
-#define GET_CALLER_PC() reinterpret_cast<uintptr_t>(_ReturnAddress())
+#define GET_CALLER_PC() _ReturnAddress()
 
 namespace fuzzer {
 
 inline uint8_t  Bswap(uint8_t x)  { return x; }
 // Use alternatives to __builtin functions from <stdlib.h> and <intrin.h> on
 // Windows since the builtins are not supported by MSVC.
 inline uint16_t Bswap(uint16_t x) { return _byteswap_ushort(x); }
 inline uint32_t Bswap(uint32_t x) { return _byteswap_ulong(x); }

compiler-rt/trunk/lib/fuzzer/FuzzerSHA1.cpp

 #include <sstream>
 #include <stdint.h>
 #include <string.h>
 
 namespace {  // Added for LibFuzzer
 
 #ifdef __BIG_ENDIAN__
 # define SHA_BIG_ENDIAN
-#elif defined __LITTLE_ENDIAN__
+// Windows is always little endian and MSVC doesn't have <endian.h>
+#elif defined __LITTLE_ENDIAN__ || LIBFUZZER_WINDOWS
 /* override */
 #elif defined __BYTE_ORDER
 # if __BYTE_ORDER__ ==  __ORDER_BIG_ENDIAN__
 # define SHA_BIG_ENDIAN
 # endif
 #else // ! defined __LITTLE_ENDIAN__
 # include <endian.h> // machine/endian.h
 # if __BYTE_ORDER__ ==  __ORDER_BIG_ENDIAN__

compiler-rt/trunk/lib/fuzzer/FuzzerValueBitMap.h

public:
   // Computes a hash function of Value and sets the corresponding bit.
   // Returns true if the bit was changed from 0 to 1.
   ATTRIBUTE_NO_SANITIZE_ALL
   inline bool AddValue(uintptr_t Value) {
     uintptr_t Idx = Value % kMapSizeInBits;
     uintptr_t WordIdx = Idx / kBitsInWord;
     uintptr_t BitIdx = Idx % kBitsInWord;
     uintptr_t Old = Map[WordIdx];
-    uintptr_t New = Old | (1UL << BitIdx);
+    uintptr_t New = Old | (1ULL << BitIdx);
     Map[WordIdx] = New;
     return New != Old;
   }
 
   ATTRIBUTE_NO_SANITIZE_ALL
   inline bool AddValueModPrime(uintptr_t Value) {
     return AddValue(Value % kMapPrimeMod);
   }
 
   inline bool Get(uintptr_t Idx) {
     assert(Idx < kMapSizeInBits);
     uintptr_t WordIdx = Idx / kBitsInWord;
     uintptr_t BitIdx = Idx % kBitsInWord;
-    return Map[WordIdx] & (1UL << BitIdx);
+    return Map[WordIdx] & (1ULL << BitIdx);
   }
 
   size_t SizeInBits() const { return kMapSizeInBits; }
 
   template <class Callback>
   ATTRIBUTE_NO_SANITIZE_ALL
   void ForEach(Callback CB) const {
     for (size_t i = 0; i < kMapSizeInWords; i++)