[llvm] Use hidden visibility when building for MinGW with Clang

[mstorsjo]

Summary

Since c5b3de6745c37dd991430b9b88ff97c35b6fc455 (git main,
August 11th), Clang does generate working hidden visibility
on MinGW targets. Using that reduces the number of exports from
a dylib build of LLVM significantly, which is vital for fitting
within the limit of 64k exported symbols from a DLL.

It's essential that if we set CMAKE_CXX_VISIBILITY_PRESET=hidden
(which passes -fvisibility=hidden on the command line), we also
must define LLVM_EXTERNAL_VISIBILITY consistently to override
it. (If there are mismatches, e.g. setting hidden visibility generally
but never overriding it back to default for the symbols that do need
to be exported, we'd get broken builds in such configurations.)

We don't want to be using attribute((visibility("hidden"))) on
MinGW with GCC, because GCC produces a warning about it. (GCC hasn't
warned about the command line options that set hidden visibility
though.) Clang has historically not warned about either of them, so
it is harmless to use the hidden visibility when building with older
Clang (so we don't need to detect the exact version of Clang/LLVM where
it has an effect).

This reduces the number of exported symbols for a dylib build of LLVM;
previously libLLVM exported around 64650 symbols (when the maximum is

65536. when the ARM, AArch64 and X86 targets were enabled. If enabling

more targets (or if building with e.g. assertions enabled), it would
exceed the limit. Now with visibility flags in use, the same build
with ARM, AArch64 and X86 ends up at around 35k exported symbols.

[mati865]

Thanks, confirmed the built compiler still works (bootstrapped it twice with itself) and with all architectures enabled we are at 35289 exported symbols.

llvm/cmake/modules/HandleLLVMOptions.cmake

if( LLVM_ENABLE_PIC )
   # clang with -O3 -fPIC generates code that SEGVs.
   # Both can be worked around by compiling with -O instead.
   if(${CMAKE_SYSTEM_NAME} STREQUAL "SunOS" AND LLVM_NATIVE_ARCH STREQUAL "Sparc")
     llvm_replace_compiler_option(CMAKE_CXX_FLAGS_RELEASE "-O[23]" "-O")
     llvm_replace_compiler_option(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O[23]" "-O")
   endif()
 endif()
 
-if(NOT WIN32 AND NOT CYGWIN AND NOT (${CMAKE_SYSTEM_NAME} MATCHES "AIX"))
-  # MinGW warns if -fvisibility-inlines-hidden is used.
+if((NOT (${CMAKE_SYSTEM_NAME} MATCHES "AIX")) AND
+   (NOT (WIN32 OR CYGWIN) OR (MINGW AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")))
+  # GCC for MinGW does nothing about -fvisibility-inlines-hidden, but warns
+  # about use of the attributes. As long as we don't use the attributes (to
+  # override the default) we shouldn't set the command line options either.
   # GCC on AIX warns if -fvisibility-inlines-hidden is used and Clang on AIX doesn't currently support visibility.
   check_cxx_compiler_flag("-fvisibility-inlines-hidden" SUPPORTS_FVISIBILITY_INLINES_HIDDEN_FLAG)
   append_if(SUPPORTS_FVISIBILITY_INLINES_HIDDEN_FLAG "-fvisibility-inlines-hidden" CMAKE_CXX_FLAGS)
 endif()
 
 if(CMAKE_SIZEOF_VOID_P EQUAL 8 AND MINGW)
   add_definitions( -D_FILE_OFFSET_BITS=64 )
 endif()

llvm/include/llvm/Support/Compiler.h

 /// into a shared library, then the class should be private to the library and
 /// not accessible from outside it.  Can also be used to mark variables and
 /// functions, making them private to any shared library they are linked into.
 /// On PE/COFF targets, library visibility is the default, so this isn't needed.
 ///
 /// LLVM_EXTERNAL_VISIBILITY - classes, functions, and variables marked with
 /// this attribute will be made public and visible outside of any shared library
 /// they are linked in to.
-#if __has_attribute(visibility) && !defined(__MINGW32__) &&                    \
-    !defined(__CYGWIN__) && !defined(_WIN32)
+#if __has_attribute(visibility) &&                                             \
+    (!(defined(_WIN32) || defined(__CYGWIN__)) ||                              \
+     (defined(__MINGW32__) && defined(__clang__)))
 #define LLVM_LIBRARY_VISIBILITY __attribute__ ((visibility("hidden")))
 #if defined(LLVM_BUILD_LLVM_DYLIB) || defined(LLVM_BUILD_SHARED_LIBS)
 #define LLVM_EXTERNAL_VISIBILITY __attribute__((visibility("default")))
 #else
 #define LLVM_EXTERNAL_VISIBILITY
 #endif
 #else
 #define LLVM_LIBRARY_VISIBILITY

llvm/lib/Target/CMakeLists.txt

add_llvm_component_library(LLVMTarget
   Core
   MC
   Support
   )
 
 # When building shared objects for each target there are some internal APIs
 # that are used across shared objects which we can't hide.
 if (NOT BUILD_SHARED_LIBS AND NOT APPLE AND
-    NOT MINGW AND
+    (NOT (WIN32 OR CYGWIN) OR (MINGW AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")) AND
     NOT (${CMAKE_SYSTEM_NAME} MATCHES "AIX") AND
     NOT DEFINED CMAKE_CXX_VISIBILITY_PRESET)
   # Set default visibility to hidden, so we don't export all the Target classes
   # in libLLVM.so.
   set(CMAKE_CXX_VISIBILITY_PRESET hidden)
 endif()
 
 foreach(t ${LLVM_TARGETS_TO_BUILD})