[mlir] Add MLIR-C dylib.

[stellaraccident]

Summary

Per discussion on discord and various feature requests across bindings (Haskell and Rust bindings authors have asked me directly), we should be building a link-ready MLIR-C dylib which exports the C API and can be used without linking to anything else.

This patch:

• Adds a new MLIR-C aggregate shared library (libMLIR-C.so), which is similar in name and function to libLLVM-C.so.
• It is guarded by the new CMake option MLIR_BUILD_MLIR_C_DYLIB, which has a similar purpose/name to the LLVM_BUILD_LLVM_C_DYLIB option.
• On all platforms, this will work with both static, BUILD_SHARED_LIBS, and libMLIR builds, if supported:
  • In static builds: libMLIR-C.so will export the CAPI symbols and statically link all dependencies into itself.
  • In BUILD_SHARED_LIBS: libMLIR-C.so will export the CAPI symbols and have dynamic dependencies on implementation shared libraries.
  • In libMLIR.so mode: same as static. libMLIR.so was not finished for actual linking use within the project. An eventual relayering so that libMLIR-C.so depends on libMLIR.so is possible but requires first re-engineering the latter to use the aggregate facility.
• On Linux, exported symbols are filtered to only the CAPI. On others (MacOS, Windows), all symbols are exported. A CMake status is printed unless if global visibility is hidden indicating that this has not yet been implemented. The library should still work, but it will be larger and more likely to conflict until fixed. Someone should look at lifting the corresponding support from libLLVM-C.so and adapting. Or, for special uses, just build with -DCMAKE_CXX_VISIBILITY_PRESET=hidden -DCMAKE_C_VISIBILITY_PRESET=hidden.
• Includes fixes to execution engine symbol export macros to enable default visibility. Without this, the advice to use hidden visibility would have resulted in test failures and unusable execution engine support libraries.

[stellaraccident]

Rebase.

[jpienaar]

Thanks!

mlir/lib/CAPI/CMakeLists.txt

31

OOC would this message be shown even if they had set this cmake option? (it appears so but not sure)

[stellaraccident]

Update when warning status prints.

[stellaraccident]

mlir/lib/CAPI/CMakeLists.txt

31

Ah, I meant to guard that better but forgot. Attempted to make it better scoped. It is really an advertisement for people not me to do some work on their platform :)

mlir/CMakeLists.txt

option(MLIR_INCLUDE_INTEGRATION_TESTS
        "Generate build targets for the MLIR integration tests.")
 
 set(MLIR_INSTALL_AGGREGATE_OBJECTS 1 CACHE BOOL
     "Installs object files needed for add_mlir_aggregate to work out of \
     tree. Package maintainers can disable this to exclude these assets if \
     not desired. Enabling this will result in object files being written \
     under lib/objects-{CMAKE_BUILD_TYPE}.")
 
+set(MLIR_BUILD_MLIR_C_DYLIB 0 CACHE BOOL "Builds libMLIR-C shared library.")
+
 #-------------------------------------------------------------------------------
 # Python Bindings Configuration
 # Requires:
 #   The pybind11 library can be found (set with -DPYBIND_DIR=...)
 #   The python executable is correct (set with -DPython3_EXECUTABLE=...)
 #-------------------------------------------------------------------------------
 
 set(MLIR_ENABLE_BINDINGS_PYTHON 0 CACHE BOOL

mlir/include/mlir/ExecutionEngine/CRunnerUtils.h

 #define MLIR_CRUNNERUTILS_EXPORT __declspec(dllexport)
 #define MLIR_CRUNNERUTILS_DEFINE_FUNCTIONS
 #else
 // We are using this library
 #define MLIR_CRUNNERUTILS_EXPORT __declspec(dllimport)
 #endif // mlir_c_runner_utils_EXPORTS
 #endif // MLIR_CRUNNERUTILS_EXPORT
 #else  // _WIN32
-#define MLIR_CRUNNERUTILS_EXPORT
+// Non-windows: use visibility attributes.
+#define MLIR_CRUNNERUTILS_EXPORT __attribute__((visibility("default")))
 #define MLIR_CRUNNERUTILS_DEFINE_FUNCTIONS
 #endif // _WIN32
 
 #include <array>
 #include <cassert>
 #include <cstdint>
 #include <initializer_list>
 

mlir/include/mlir/ExecutionEngine/RunnerUtils.h

 // We are building this library
 #define MLIR_RUNNERUTILS_EXPORT __declspec(dllexport)
 #else
 // We are using this library
 #define MLIR_RUNNERUTILS_EXPORT __declspec(dllimport)
 #endif // mlir_runner_utils_EXPORTS
 #endif // MLIR_RUNNERUTILS_EXPORT
 #else
-#define MLIR_RUNNERUTILS_EXPORT
+// Non-windows: use visibility attributes.
+#define MLIR_RUNNERUTILS_EXPORT __attribute__((visibility("default")))
 #endif // _WIN32
 
 #include <assert.h>
 #include <cmath>
 #include <iostream>
 
 #include "mlir/ExecutionEngine/CRunnerUtils.h"
 

mlir/lib/CAPI/CMakeLists.txt

+# For upstream, we accumulate all libraries into the MLIR_CAPI_LIBRARIES
+# property via a custom wrapper function. This is then used to create an
+# aggregate below.
+set_property(GLOBAL APPEND PROPERTY MLIR_CAPI_LIBRARIES)
+function(add_mlir_upstream_c_api_library name)
+  add_mlir_public_c_api_library(${name} ${ARGN})
+  set_property(GLOBAL APPEND PROPERTY MLIR_CAPI_LIBRARIES ${name})
+endfunction()
+
 add_subdirectory(Debug)
 add_subdirectory(Dialect)
 add_subdirectory(Conversion)
 add_subdirectory(ExecutionEngine)
 add_subdirectory(Interfaces)
 add_subdirectory(IR)
 add_subdirectory(Registration)
 add_subdirectory(Transforms)
+
+# Build the optional CAPI dylib.
+if(MLIR_BUILD_MLIR_C_DYLIB)
+  message(STATUS "Building MLIR-C dylib")
+  get_property(_capi_libraries GLOBAL PROPERTY MLIR_CAPI_LIBRARIES)
+  add_mlir_aggregate(MLIR-C
+    SHARED
+    EMBED_LIBS
+      ${_capi_libraries}
+  )
+  if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    target_link_options(MLIR-C PRIVATE "-Wl,-exclude-libs,ALL")
+  else()
+    if(NOT CMAKE_C_VISIBILITY_PRESET STREQUAL "hidden" OR NOT CMAKE_CXX_VISIBILITY_PRESET STREQUAL "hidden")
+      message(STATUS "MLIR-C on this platform exports all symbols. Recommend building with CMAKE_(C|CXX)_VISIBILITY_PRESET=hidden or implement filtering support.")
+    endif()
+  endif()
+endif()

mlir/lib/CAPI/Conversion/CMakeLists.txt

 get_property(conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS)
-add_mlir_public_c_api_library(MLIRCAPIConversion
+add_mlir_upstream_c_api_library(MLIRCAPIConversion
   Passes.cpp
 
   LINK_LIBS PUBLIC
   ${conversion_libs}
 )

mlir/lib/CAPI/Debug/CMakeLists.txt

-add_mlir_public_c_api_library(MLIRCAPIDebug
+add_mlir_upstream_c_api_library(MLIRCAPIDebug
   Debug.cpp
 
   LINK_LIBS PUBLIC
   MLIRSupport
 )

mlir/lib/CAPI/Dialect/CMakeLists.txt

-add_mlir_public_c_api_library(MLIRCAPIAsync
+add_mlir_upstream_c_api_library(MLIRCAPIAsync
   Async.cpp
   AsyncPasses.cpp
 
   PARTIAL_SOURCES_INTENDED
   DEPENDS
   MLIRAsyncPassIncGen
 
   LINK_LIBS PUBLIC
   MLIRCAPIIR
   MLIRAsync
   MLIRAsyncTransforms
   MLIRPass
 )
 
-add_mlir_public_c_api_library(MLIRCAPIGPU
+add_mlir_upstream_c_api_library(MLIRCAPIGPU
   GPU.cpp
   GPUPasses.cpp
 
   PARTIAL_SOURCES_INTENDED
   DEPENDS
   MLIRGPUPassIncGen
 
   LINK_LIBS PUBLIC
   MLIRCAPIIR
   MLIRGPUTransforms
   MLIRPass
 )
 
-add_mlir_public_c_api_library(MLIRCAPILLVM
+add_mlir_upstream_c_api_library(MLIRCAPILLVM
   LLVM.cpp
 
   PARTIAL_SOURCES_INTENDED
   LINK_LIBS PUBLIC
   MLIRCAPIIR
   MLIRLLVMIR
 )
 
-add_mlir_public_c_api_library(MLIRCAPILinalg
+add_mlir_upstream_c_api_library(MLIRCAPILinalg
   Linalg.cpp
   LinalgPasses.cpp
 
   PARTIAL_SOURCES_INTENDED
   DEPENDS
   MLIRLinalgPassIncGen
 
   LINK_LIBS PUBLIC
   MLIRCAPIIR
   MLIRLinalg
   MLIRPass
   MLIRLinalgTransforms
 )
 
-add_mlir_public_c_api_library(MLIRCAPISCF
+add_mlir_upstream_c_api_library(MLIRCAPISCF
   SCF.cpp
 
   PARTIAL_SOURCES_INTENDED
   LINK_LIBS PUBLIC
   MLIRCAPIIR
   MLIRSCF
 )
 
-add_mlir_public_c_api_library(MLIRCAPIShape
+add_mlir_upstream_c_api_library(MLIRCAPIShape
   Shape.cpp
 
   PARTIAL_SOURCES_INTENDED
   LINK_LIBS PUBLIC
   MLIRCAPIIR
   MLIRShape
 )
 
-add_mlir_public_c_api_library(MLIRCAPISparseTensor
+add_mlir_upstream_c_api_library(MLIRCAPISparseTensor
   SparseTensor.cpp
   SparseTensorPasses.cpp
 
   PARTIAL_SOURCES_INTENDED
   LINK_LIBS PUBLIC
   MLIRCAPIIR
   MLIRSparseTensor
   MLIRSparseTensorTransforms
 )
 
-add_mlir_public_c_api_library(MLIRCAPIStandard
+add_mlir_upstream_c_api_library(MLIRCAPIStandard
   Standard.cpp
 
   PARTIAL_SOURCES_INTENDED
   LINK_LIBS PUBLIC
   MLIRCAPIIR
   MLIRStandard
 )
 
-add_mlir_public_c_api_library(MLIRCAPITensor
+add_mlir_upstream_c_api_library(MLIRCAPITensor
   Tensor.cpp
 
   PARTIAL_SOURCES_INTENDED
   LINK_LIBS PUBLIC
   MLIRCAPIIR
   MLIRTensor
 )

mlir/lib/CAPI/ExecutionEngine/CMakeLists.txt

 # Main API shared library.
-add_mlir_public_c_api_library(MLIRCEXECUTIONENGINE
+add_mlir_upstream_c_api_library(MLIRCEXECUTIONENGINE
   ExecutionEngine.cpp
 
   LINK_LIBS PUBLIC
   MLIRExecutionEngine
   MLIRLLVMToLLVMIRTranslation
 )

mlir/lib/CAPI/IR/CMakeLists.txt

 # Main API shared library.
-add_mlir_public_c_api_library(MLIRCAPIIR
+add_mlir_upstream_c_api_library(MLIRCAPIIR
   AffineExpr.cpp
   AffineMap.cpp
   BuiltinAttributes.cpp
   BuiltinTypes.cpp
   Diagnostics.cpp
   DialectHandle.cpp
   IntegerSet.cpp
   IR.cpp

mlir/lib/CAPI/Interfaces/CMakeLists.txt

-add_mlir_public_c_api_library(MLIRCAPIInterfaces
+add_mlir_upstream_c_api_library(MLIRCAPIInterfaces
   Interfaces.cpp
 
   LINK_LIBS PUBLIC
   MLIRInferTypeOpInterface)

mlir/lib/CAPI/Registration/CMakeLists.txt

 # Dialect registration.
 get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
 get_property(translation_libs GLOBAL PROPERTY MLIR_TRANSLATION_LIBS)
 get_property(conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS)
-add_mlir_public_c_api_library(MLIRCAPIRegistration
+add_mlir_upstream_c_api_library(MLIRCAPIRegistration
   Registration.cpp
 
   LINK_LIBS PUBLIC
   MLIRCAPIIR
   MLIRLLVMToLLVMIRTranslation
   ${dialect_libs}
   ${translation_libs}
   ${conversion_libs}
 )

mlir/lib/CAPI/Transforms/CMakeLists.txt

-add_mlir_public_c_api_library(MLIRCAPITransforms
+add_mlir_upstream_c_api_library(MLIRCAPITransforms
   Passes.cpp
 
   LINK_LIBS PUBLIC
   MLIRTransforms
 )

mlir/test/CAPI/CMakeLists.txt

 function(_add_capi_test_executable name)
   cmake_parse_arguments(ARG
     ""
     ""
     "LINK_LIBS"
     ${ARGN})
   set(LLVM_LINK_COMPONENTS
     )
   add_llvm_executable(${name}
     PARTIAL_SOURCES_INTENDED
     ${ARG_UNPARSED_ARGUMENTS})
   llvm_update_compile_flags(${name})
-  target_link_libraries(${name}
+  if(MLIR_BUILD_MLIR_C_DYLIB)
+    target_link_libraries(${name} PRIVATE
+      MLIR-C)
+  else()
+    target_link_libraries(${name} PRIVATE
       ${ARG_LINK_LIBS})
+  endif()
 endfunction(_add_capi_test_executable)
 
 _add_capi_test_executable(mlir-capi-execution-engine-test
   execution_engine.c
 LINK_LIBS PRIVATE
   MLIRCAPIConversion
   MLIRCEXECUTIONENGINE
   MLIRCAPIRegistration