diff --git a/mlir/cmake/modules/AddMLIR.cmake b/mlir/cmake/modules/AddMLIR.cmake --- a/mlir/cmake/modules/AddMLIR.cmake +++ b/mlir/cmake/modules/AddMLIR.cmake @@ -54,7 +54,7 @@ # with large dependencies. function(add_mlir_library name) cmake_parse_arguments(ARG - "SHARED;INSTALL_WITH_TOOLCHAIN;EXCLUDE_FROM_LIBMLIR" + "SHARED;INSTALL_WITH_TOOLCHAIN;EXCLUDE_FROM_LIBMLIR;DISABLE_INSTALL" "" "ADDITIONAL_HEADERS;DEPENDS;LINK_COMPONENTS;LINK_LIBS" ${ARGN}) @@ -131,7 +131,9 @@ if(TARGET ${name}) target_link_libraries(${name} INTERFACE ${LLVM_COMMON_LIBS}) - add_mlir_library_install(${name}) + if(NOT ARG_DISABLE_INSTALL) + add_mlir_library_install(${name}) + endif() else() # Add empty "phony" target add_custom_target(${name}) diff --git a/mlir/cmake/modules/AddMLIRPython.cmake b/mlir/cmake/modules/AddMLIRPython.cmake --- a/mlir/cmake/modules/AddMLIRPython.cmake +++ b/mlir/cmake/modules/AddMLIRPython.cmake @@ -1,10 +1,404 @@ +################################################################################ +# Python modules +# MLIR's Python modules are both directly used by the core project and are +# available for use and embedding into external projects (in their own +# namespace and with their own deps). In order to facilitate this, python +# artifacts are split between declarations, which make a subset of +# things available to be built and "add", which in line with the normal LLVM +# nomenclature, adds libraries. +################################################################################ + +# Function: declare_mlir_python_sources +# Declares pure python sources as part of a named grouping that can be built +# later. +# Arguments: +# ROOT_DIR: Directory where the python namespace begins (defaults to +# CMAKE_CURRENT_SOURCE_DIR). For non-relocatable sources, this will +# typically just be the root of the python source tree (current directory). +# For relocatable sources, this will point deeper into the directory that +# can be relocated. For generated sources, can be relative to +# CMAKE_CURRENT_BINARY_DIR. Generated and non generated sources cannot be +# mixed. +# ADD_TO_PARENT: Adds this source grouping to a previously declared source +# grouping. Source groupings form a DAG. +# SOURCES: List of specific source files relative to ROOT_DIR to include. +# SOURCES_GLOB: List of glob patterns relative to ROOT_DIR to include. +function(declare_mlir_python_sources name) + cmake_parse_arguments(ARG + "" + "ROOT_DIR;ADD_TO_PARENT" + "SOURCES;SOURCES_GLOB" + ${ARGN}) + + if(NOT ARG_ROOT_DIR) + set(ARG_ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}") + endif() + + # Process the glob. + set(_glob_sources) + if(ARG_SOURCES_GLOB) + set(_glob_spec ${ARG_SOURCES_GLOB}) + list(TRANSFORM _glob_spec PREPEND "${ARG_ROOT_DIR}/") + file(GLOB_RECURSE _glob_sources + RELATIVE "${ARG_ROOT_DIR}" + ${_glob_spec} + ) + list(APPEND ARG_SOURCES ${_glob_sources}) + endif() + + # We create a custom target to carry properties and dependencies for + # generated sources. + add_custom_target(${name}) + set(_file_depends "${ARG_SOURCES}") + list(TRANSFORM _file_depends PREPEND "${ARG_ROOT_DIR}/") + set_target_properties(${name} PROPERTIES + PYTHON_SOURCES_TYPE pure + PYTHON_ROOT_DIR "${ARG_ROOT_DIR}" + PYTHON_SOURCES "${ARG_SOURCES}" + PYTHON_FILE_DEPENDS "${_file_depends}" + PYTHON_DEPENDS "" + ) + + # Add to parent. + if(ARG_ADD_TO_PARENT) + set_property(TARGET ${ARG_ADD_TO_PARENT} APPEND PROPERTY PYTHON_DEPENDS ${name}) + endif() +endfunction() + +# Function: declare_mlir_python_extension +# Declares a buildable python extension from C++ source files. The built +# module is considered a python source file and included as everything else. +# Arguments: +# MODULE_NAME: Local import name of the module (i.e. "_mlir"). +# ADD_TO_PARENT: Same as for declare_mlir_python_sources. +# SOURCES: C++ sources making up the module. +# PRIVATE_LINK_LIBS: List of libraries to link in privately to the module +# regardless of how it is included in the project (generally should be +# static libraries that can be included with hidden visibility). +# EMBED_CAPI_LINK_LIBS: Dependent CAPI libraries that this extension depends +# on. These will be collected for all extensions and put into an +# aggregate dylib that is linked against. +function(declare_mlir_python_extension name) + cmake_parse_arguments(ARG + "" + "MODULE_NAME;ADD_TO_PARENT" + "SOURCES;PRIVATE_LINK_LIBS;EMBED_CAPI_LINK_LIBS" + ${ARGN}) + + add_custom_target(${name}) + set_target_properties(${name} PROPERTIES + PYTHON_SOURCES_TYPE extension + PYTHON_EXTENSION_MODULE_NAME "${ARG_MODULE_NAME}" + PYTHON_CPP_SOURCES "${ARG_SOURCES}" + PYTHON_PRIVATE_LINK_LIBS "${ARG_PRIVATE_LINK_LIBS}" + PYTHON_EMBED_CAPI_LINK_LIBS "${ARG_EMBED_CAPI_LINK_LIBS}" + PYTHON_FILE_DEPENDS "" + PYTHON_DEPENDS "" + ) + + # Add to parent. + if(ARG_ADD_TO_PARENT) + set_property(TARGET ${ARG_ADD_TO_PARENT} APPEND PROPERTY PYTHON_DEPENDS ${name}) + endif() +endfunction() + +# Function: add_mlir_python_modules +# Adds python modules to a project, building them from a list of declared +# source groupings (see declare_mlir_python_sources and +# declare_mlir_python_extension). One of these must be called for each +# packaging root in use. +# Arguments: +# ROOT_PREFIX: The directory in the build tree to emit sources. This will +# typically be something like ${MY_BINARY_DIR}/python_packages/foobar +# for non-relocatable modules or a deeper directory tree for relocatable. +# INSTALL_PREFIX: Prefix into the install tree for installing the package. +# Typically mirrors the path above but without an absolute path. +# DECLARED_SOURCES: List of declared source groups to include. The entire +# DAG of source modules is included. +# COMMON_CAPI_LINK_LIBS: List of dylibs (typically one) to make every +# extension depend on (see mlir_python_add_common_capi_library). +function(add_mlir_python_modules name) + cmake_parse_arguments(ARG + "" + "ROOT_PREFIX;INSTALL_PREFIX;COMMON_CAPI_LINK_LIBS" + "DECLARED_SOURCES" + ${ARGN}) + # Helper to process an individual target. + function(_process_target modules_target sources_target) + get_target_property(_source_type ${sources_target} PYTHON_SOURCES_TYPE) + if(_source_type STREQUAL "pure") + # Pure python sources to link into the tree. + get_target_property(_python_root_dir ${sources_target} PYTHON_ROOT_DIR) + get_target_property(_python_sources ${sources_target} PYTHON_SOURCES) + foreach(_source_relative_path ${_python_sources}) + set(_src_path "${_python_root_dir}/${_source_relative_path}") + set(_dest_path "${ARG_ROOT_PREFIX}/${_source_relative_path}") + + get_filename_component(_dest_dir "${_dest_path}" DIRECTORY) + get_filename_component(_install_path "${ARG_INSTALL_PREFIX}/${_source_relative_path}" DIRECTORY) + + file(MAKE_DIRECTORY "${_dest_dir}") + add_custom_command( + TARGET ${modules_target} PRE_BUILD + COMMENT "Copying python source ${_src_path} -> ${_dest_path}" + DEPENDS "${_src_path}" + BYPRODUCTS "${_dest_path}" + COMMAND "${CMAKE_COMMAND}" -E create_symlink + "${_src_path}" "${_dest_path}" + ) + install( + FILES "${_src_path}" + DESTINATION "${_install_path}" + COMPONENT ${modules_target} + ) + endforeach() + elseif(_source_type STREQUAL "extension") + # Native CPP extension. + get_target_property(_module_name ${sources_target} PYTHON_EXTENSION_MODULE_NAME) + get_target_property(_cpp_sources ${sources_target} PYTHON_CPP_SOURCES) + get_target_property(_private_link_libs ${sources_target} PYTHON_PRIVATE_LINK_LIBS) + set(_extension_target "${name}.extension.${_module_name}.dso") + add_mlir_python_extension(${_extension_target} "${_module_name}" + INSTALL_COMPONENT ${modules_target} + INSTALL_DIR "${ARG_INSTALL_PREFIX}/_mlir_libs" + OUTPUT_DIRECTORY "${ARG_ROOT_PREFIX}/_mlir_libs" + SOURCES ${_cpp_sources} + LINK_LIBS PRIVATE + ${_private_link_libs} + ${ARG_COMMON_CAPI_LINK_LIBS} + ) + add_dependencies(${name} ${_extension_target}) + mlir_python_setup_extension_rpath(${_extension_target}) + else() + message(SEND_ERROR "Unrecognized source type '${_source_type}' for python source target ${sources_target}") + return() + endif() + endfunction() + + _flatten_mlir_python_targets(_flat_targets ${ARG_DECLARED_SOURCES}) + # Collect dependencies. + set(_depends) + foreach(sources_target ${_flat_targets}) + get_target_property(_local_depends ${sources_target} PYTHON_FILE_DEPENDS) + list(APPEND _depends ${_local_depends}) + endforeach() + + # Build the modules target. + add_custom_target(${name} DEPENDS ${_depends}) + foreach(sources_target ${_flat_targets}) + _process_target(${name} ${sources_target}) + endforeach() + + # Create an install target. + if (NOT LLVM_ENABLE_IDE) + add_llvm_install_targets( + install-${name} + DEPENDS ${name} + COMPONENT ${name}) + endif() +endfunction() + +# Function: declare_mlir_dialect_python_bindings +# Helper to generate source groups for dialects, including both static source +# files and a TD_FILE to generate wrappers. +# +# This will generate a source group named ${ADD_TO_PARENT}.${DIALECT_NAME}. +# +# Arguments: +# ROOT_DIR: Same as for declare_mlir_python_sources(). +# ADD_TO_PARENT: Same as for declare_mlir_python_sources(). Unique names +# for the subordinate source groups are derived from this. +# TD_FILE: Tablegen file to generate source for (relative to ROOT_DIR). +# DIALECT_NAME: Python name of the dialect. +# SOURCES: Same as declare_mlir_python_sources(). +# SOURCES_GLOB: Same as declare_mlir_python_sources(). +# DEPENDS: Additional dependency targets. +function(declare_mlir_dialect_python_bindings) + cmake_parse_arguments(ARG + "" + "ROOT_DIR;ADD_TO_PARENT;TD_FILE;DIALECT_NAME" + "SOURCES;SOURCES_GLOB;DEPENDS" + ${ARGN}) + # Sources. + set(_dialect_target "${ARG_ADD_TO_PARENT}.${ARG_DIALECT_NAME}") + declare_mlir_python_sources(${_dialect_target} + ROOT_DIR "${ARG_ROOT_DIR}" + ADD_TO_PARENT "${ARG_ADD_TO_PARENT}" + SOURCES "${ARG_SOURCES}" + SOURCES_GLOB "${ARG_SOURCES_GLOB}" + ) + + # Tablegen + if(ARG_TD_FILE) + set(tblgen_target "${ARG_ADD_TO}.${ARG_DIALECT_NAME}.tablegen") + set(td_file "${ARG_ROOT_DIR}/${ARG_TD_FILE}") + get_filename_component(relative_td_directory "${ARG_TD_FILE}" DIRECTORY) + set(dialect_filename "${relative_td_directory}/_${ARG_DIALECT_NAME}_ops_gen.py") + set(LLVM_TARGET_DEFINITIONS ${td_file}) + mlir_tablegen("${dialect_filename}" -gen-python-op-bindings + -bind-dialect=${ARG_DIALECT_NAME}) + add_public_tablegen_target(${tblgen_target}) + if(ARG_DEPENDS) + add_dependencies(${tblgen_target} ${ARG_DEPENDS}) + endif() + + # Generated. + declare_mlir_python_sources("${ARG_ADD_TO_PARENT}.${ARG_DIALECT_NAME}.ops_gen" + ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}" + ADD_TO_PARENT "${_dialect_target}" + SOURCES "${dialect_filename}" + ) + endif() +endfunction() + +# Function: mlir_python_setup_extension_rpath +# Sets RPATH properties on a target, assuming that it is being output to +# an _mlir_libs directory with all other libraries. For static linkage, +# the RPATH will just be the origin. If linking dynamically, then the LLVM +# library directory will be added. +# Arguments: +# RELATIVE_INSTALL_ROOT: If building dynamically, an RPATH entry will be +# added to the install tree lib/ directory by first traversing this +# path relative to the installation location. Typically a number of ".." +# entries, one for each level of the install path. +function(mlir_python_setup_extension_rpath target) + cmake_parse_arguments(ARG + "" + "RELATIVE_INSTALL_ROOT" + "" + ${ARGN}) + + # RPATH handling. + # For the build tree, include the LLVM lib directory and the current + # directory for RPATH searching. For install, just the current directory + # (assumes that needed dependencies have been installed). + if(NOT APPLE AND NOT UNIX) + return() + endif() + + set(_origin_prefix "\$ORIGIN") + if(APPLE) + set(_origin_prefix "@loader_path") + endif() + set_target_properties(${target} PROPERTIES + BUILD_WITH_INSTALL_RPATH OFF + BUILD_RPATH "${_origin_prefix}" + INSTALL_RPATH "${_origin_prefix}" + ) + + # For static builds, that is all that is needed: all dependencies will be in + # the one directory. For shared builds, then we also need to add the global + # lib directory. This will be absolute for the build tree and relative for + # install. + # When we have access to CMake >= 3.20, there is a helper to calculate this. + if(BUILD_SHARED_LIBS AND ARG_RELATIVE_INSTALL_ROOT) + get_filename_component(_real_lib_dir "${LLVM_LIBRARY_OUTPUT_INTDIR}" REALPATH) + set_property(TARGET ${target} APPEND PROPERTY + BUILD_RPATH "${_real_lib_dir}") + set_property(TARGET ${target} APPEND PROPERTY + INSTALL_RPATH "${_origin_prefix}/${ARG_RELATIVE_INSTALL_ROOT}/lib${LLVM_LIBDIR_SUFFIX}") + endif() +endfunction() + +# Function: add_mlir_python_common_capi_library +# Adds a shared library which embeds dependent CAPI libraries needed to link +# all extensions. +# Arguments: +# INSTALL_COMPONENT: Name of the install component. Typically same as the +# target name passed to add_mlir_python_modules(). +# INSTALL_DESTINATION: Prefix into the install tree in which to install the +# library. +# OUTPUT_DIRECTORY: Full path in the build tree in which to create the +# library. Typically, this will be the common _mlir_libs directory where +# all extensions are emitted. +# RELATIVE_INSTALL_ROOT: See mlir_python_setup_extension_rpath(). +# DECLARED_SOURCES: Source groups from which to discover dependent +# EMBED_CAPI_LINK_LIBS. +# EMBED_LIBS: Additional libraries to embed (must be built with OBJECTS and +# have an "obj.${name}" object library associated). +function(add_mlir_python_common_capi_library name) + cmake_parse_arguments(ARG + "" + "INSTALL_COMPONENT;INSTALL_DESTINATION;OUTPUT_DIRECTORY;RELATIVE_INSTALL_ROOT" + "DECLARED_SOURCES;EMBED_LIBS" + ${ARGN}) + # TODO: Upgrade to the aggregate utility in https://reviews.llvm.org/D106419 + # once ready. + + # Collect all explicit and transitive embed libs. + set(_embed_libs ${ARG_EMBED_LIBS}) + _flatten_mlir_python_targets(_all_source_targets ${ARG_DECLARED_SOURCES}) + foreach(t ${_all_source_targets}) + get_target_property(_local_embed_libs ${t} PYTHON_EMBED_CAPI_LINK_LIBS) + if(_local_embed_libs) + list(APPEND _embed_libs ${_local_embed_libs}) + endif() + endforeach() + list(REMOVE_DUPLICATES _embed_libs) + + foreach(lib ${_embed_libs}) + if(XCODE) + # Xcode doesn't support object libraries, so we have to trick it into + # linking the static libraries instead. + list(APPEND _deps "-force_load" ${lib}) + else() + list(APPEND _objects $) + endif() + # Accumulate transitive deps of each exported lib into _DEPS. + list(APPEND _deps $) + endforeach() + + add_mlir_library(${name} + PARTIAL_SOURCES_INTENDED + SHARED + DISABLE_INSTALL + ${_objects} + EXCLUDE_FROM_LIBMLIR + LINK_LIBS + ${_deps} + ) + if(MSVC) + set_property(TARGET ${name} PROPERTY WINDOWS_EXPORT_ALL_SYMBOLS ON) + endif() + set_target_properties(${name} PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${ARG_OUTPUT_DIRECTORY}" + BINARY_OUTPUT_DIRECTORY "${ARG_OUTPUT_DIRECTORY}" + ) + mlir_python_setup_extension_rpath(${name} + RELATIVE_INSTALL_ROOT "${ARG_RELATIVE_INSTALL_ROOT}" + ) + install(TARGETS ${name} + COMPONENT ${ARG_INSTALL_COMPONENT} + LIBRARY DESTINATION "${ARG_INSTALL_DESTINATION}" + RUNTIME DESTINATION "${ARG_INSTALL_DESTINATION}" + ) + +endfunction() + +function(_flatten_mlir_python_targets output_var) + set(_flattened) + foreach(t ${ARGN}) + get_target_property(_source_type ${t} PYTHON_SOURCES_TYPE) + get_target_property(_depends ${t} PYTHON_DEPENDS) + if(_source_type) + list(APPEND _flattened "${t}") + if(_depends) + _flatten_mlir_python_targets(_local_flattened ${_depends}) + list(APPEND _flattened ${_local_flattened}) + endif() + endif() + endforeach() + list(REMOVE_DUPLICATES _flattened) + set(${output_var} "${_flattened}" PARENT_SCOPE) +endfunction() + ################################################################################ # Build python extension ################################################################################ function(add_mlir_python_extension libname extname) cmake_parse_arguments(ARG "" - "INSTALL_DIR" + "INSTALL_COMPONENT;INSTALL_DIR;OUTPUT_DIRECTORY" "SOURCES;LINK_LIBS" ${ARGN}) if (ARG_UNPARSED_ARGUMENTS) @@ -14,6 +408,18 @@ message(FATAL_ERROR " Missing SOURCES argument to add_mlir_python_extension(${libname}, ...") endif() + # Build-time RPath layouts require to be a directory one up from the + # binary root. + # TODO: Don't reference the LLVM_BINARY_DIR here: the invariant is that + # the output directory must be at the same level of the lib directory + # where libMLIR.so is installed. This is presently not optimal from a + # project separation perspective and a discussion on how to better + # segment MLIR libraries needs to happen. + # TODO: Remove this when downstreams are moved off of it. + if(NOT ARG_OUTPUT_DIRECTORY) + set(ARG_OUTPUT_DIRECTORY ${LLVM_BINARY_DIR}/python) + endif() + # Normally on unix-like platforms, extensions are built as "MODULE" libraries # and do not explicitly link to the python shared object. This allows for # some greater deployment flexibility since the extension will bind to @@ -67,14 +473,7 @@ # Configure the output to match python expectations. set_target_properties( ${libname} PROPERTIES - # Build-time RPath layouts require to be a directory one up from the - # binary root. - # TODO: Don't reference the LLVM_BINARY_DIR here: the invariant is that - # the output directory must be at the same level of the lib directory - # where libMLIR.so is installed. This is presently not optimal from a - # project separation perspective and a discussion on how to better - # segment MLIR libraries needs to happen. - LIBRARY_OUTPUT_DIRECTORY ${LLVM_BINARY_DIR}/python + LIBRARY_OUTPUT_DIRECTORY ${ARG_OUTPUT_DIRECTORY} OUTPUT_NAME "${extname}" PREFIX "${PYTHON_MODULE_PREFIX}" SUFFIX "${PYTHON_MODULE_SUFFIX}${PYTHON_MODULE_EXTENSION}" @@ -85,7 +484,7 @@ # control where the .dll gets written. set_target_properties( ${libname} PROPERTIES - RUNTIME_OUTPUT_DIRECTORY ${LLVM_BINARY_DIR}/python + RUNTIME_OUTPUT_DIRECTORY ${ARG_OUTPUT_DIRECTORY} ) endif() @@ -111,53 +510,16 @@ $<$:LINKER:--exclude-libs,ALL> ) - llvm_setup_rpath(${libname}) - ################################################################################ # Install ################################################################################ if (ARG_INSTALL_DIR) install(TARGETS ${libname} - COMPONENT ${libname} + COMPONENT ${ARG_INSTALL_COMPONENT} LIBRARY DESTINATION ${ARG_INSTALL_DIR} ARCHIVE DESTINATION ${ARG_INSTALL_DIR} # NOTE: Even on DLL-platforms, extensions go in the lib directory tree. RUNTIME DESTINATION ${ARG_INSTALL_DIR} ) endif() - - if (NOT LLVM_ENABLE_IDE) - add_llvm_install_targets( - install-${libname} - DEPENDS ${libname} - COMPONENT ${libname}) - endif() - endfunction() - -function(add_mlir_dialect_python_bindings tblgen_target) - cmake_parse_arguments(ARG - "" - "TD_FILE;DIALECT_NAME" - "DEPENDS" - ${ARGN}) - - set(dialect_filename "_${ARG_DIALECT_NAME}_ops_gen.py") - set(LLVM_TARGET_DEFINITIONS ${ARG_TD_FILE}) - mlir_tablegen("${dialect_filename}" -gen-python-op-bindings - -bind-dialect=${ARG_DIALECT_NAME}) - add_public_tablegen_target( - ${tblgen_target}) - if(ARG_DEPENDS) - add_dependencies(${tblgen_target} ${ARG_DEPENDS}) - endif() - - add_custom_command( - TARGET ${tblgen_target} POST_BUILD - COMMENT "Copying generated python source \"dialects/${dialect_filename}\"" - BYPRODUCTS "${PROJECT_BINARY_DIR}/python/mlir/dialects/${dialect_filename}" - COMMAND "${CMAKE_COMMAND}" -E copy_if_different - "${CMAKE_CURRENT_BINARY_DIR}/${dialect_filename}" - "${PROJECT_BINARY_DIR}/python/mlir/dialects/${dialect_filename}") -endfunction() - diff --git a/mlir/docs/Bindings/Python.md b/mlir/docs/Bindings/Python.md --- a/mlir/docs/Bindings/Python.md +++ b/mlir/docs/Bindings/Python.md @@ -60,13 +60,19 @@ # Now run `cmake`, `ninja`, et al. ``` -For interactive use, it is sufficient to add the `python` directory in your -`build/` directory to the `PYTHONPATH`. Typically: +For interactive use, it is sufficient to add the +`tools/mlir/python_packages/mlir_core/` directory in your `build/` directory to +the `PYTHONPATH`. Typically: ```shell -export PYTHONPATH=$(cd build && pwd)/python +export PYTHONPATH=$(cd build && pwd)/tools/mlir/python_packages/mlir_core ``` +Note that if you have installed (i.e. via `ninja install`, et al), then +python packages for all enabled projects will be in your install tree under +`python_packages/` (i.e. `python_packages/mlir_core`). Official distributions +are built with a more specialized setup. + ## Design ### Use cases diff --git a/mlir/lib/Bindings/CMakeLists.txt b/mlir/lib/Bindings/CMakeLists.txt deleted file mode 100644 --- a/mlir/lib/Bindings/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -if(MLIR_ENABLE_BINDINGS_PYTHON) - add_subdirectory(Python) -endif() diff --git a/mlir/lib/Bindings/Python/CMakeLists.txt b/mlir/lib/Bindings/Python/CMakeLists.txt deleted file mode 100644 --- a/mlir/lib/Bindings/Python/CMakeLists.txt +++ /dev/null @@ -1,137 +0,0 @@ -include(AddMLIRPython) -add_custom_target(MLIRBindingsPythonExtension) - -################################################################################ -# All python extensions must link through one DSO which exports the CAPI, and -# this must have a globally unique name amongst all embeddors of the python -# library since it will effectively have global scope. -# -# The presence of this aggregate library is part of the long term plan, but its -# use needs to be made more flexible. -################################################################################ - -set(public_api_libs - MLIRCAPIConversion - MLIRCAPIDebug - MLIRCEXECUTIONENGINE - MLIRCAPIIR - MLIRCAPIRegistration - MLIRCAPITransforms - - # Dialects - MLIRCAPIAsync - MLIRCAPIGPU - MLIRCAPILinalg - MLIRCAPILLVM - MLIRCAPIShape - MLIRCAPISparseTensor - MLIRCAPIStandard - MLIRCAPISCF - MLIRCAPITensor -) - -foreach(lib ${public_api_libs}) - if(XCODE) - # Xcode doesn't support object libraries, so we have to trick it into - # linking the static libraries instead. - list(APPEND _DEPS "-force_load" ${lib}) - else() - list(APPEND _OBJECTS $) - endif() - # Accumulate transitive deps of each exported lib into _DEPS. - list(APPEND _DEPS $) -endforeach() - -add_mlir_library(MLIRPythonCAPI - PARTIAL_SOURCES_INTENDED - SHARED - ${_OBJECTS} - EXCLUDE_FROM_LIBMLIR - LINK_LIBS - ${_DEPS} -) -if(MSVC) - set_property(TARGET MLIRPythonCAPI PROPERTY WINDOWS_EXPORT_ALL_SYMBOLS ON) -endif() - -################################################################################ -# Build core python extension -################################################################################ -add_mlir_python_extension(MLIRCoreBindingsPythonExtension _mlir - INSTALL_DIR - python - SOURCES - DialectLinalg.cpp - DialectSparseTensor.cpp - MainModule.cpp - IRAffine.cpp - IRAttributes.cpp - IRCore.cpp - IRModule.cpp - IRTypes.cpp - PybindUtils.cpp - Pass.cpp - ExecutionEngine.cpp - LINK_LIBS PRIVATE - LLVMSupport - MLIRPythonCAPI -) -add_dependencies(MLIRBindingsPythonExtension MLIRCoreBindingsPythonExtension) - -add_subdirectory(Transforms) -add_subdirectory(Conversions) - -add_mlir_python_extension(MLIRAllPassesRegistrationBindingsPythonExtension _mlirAllPassesRegistration - INSTALL_DIR - python - SOURCES - AllPassesRegistration.cpp - LINK_LIBS PRIVATE - LLVMSupport - MLIRPythonCAPI -) -add_dependencies(MLIRBindingsPythonExtension MLIRAllPassesRegistrationBindingsPythonExtension) - -add_mlir_python_extension(MLIRAsyncPassesBindingsPythonExtension _mlirAsyncPasses - INSTALL_DIR - python - SOURCES - AsyncPasses.cpp - LINK_LIBS PRIVATE - LLVMSupport - MLIRPythonCAPI -) -add_dependencies(MLIRBindingsPythonExtension MLIRAsyncPassesBindingsPythonExtension) - -add_mlir_python_extension(MLIRSparseTensorPassesBindingsPythonExtension _mlirSparseTensorPasses - INSTALL_DIR - python - SOURCES - SparseTensorPasses.cpp - LINK_LIBS PRIVATE - LLVMSupport - MLIRPythonCAPI -) -add_dependencies(MLIRBindingsPythonExtension MLIRSparseTensorPassesBindingsPythonExtension) - -add_mlir_python_extension(MLIRGPUPassesBindingsPythonExtension _mlirGPUPasses - INSTALL_DIR - python - SOURCES - GPUPasses.cpp - LINK_LIBS PRIVATE - LLVMSupport - MLIRPythonCAPI -) -add_dependencies(MLIRBindingsPythonExtension MLIRGPUPassesBindingsPythonExtension) - -add_mlir_python_extension(MLIRLinalgPassesBindingsPythonExtension _mlirLinalgPasses - INSTALL_DIR - python - SOURCES - LinalgPasses.cpp - LINK_LIBS PRIVATE - LLVMSupport - MLIRPythonCAPI -) -add_dependencies(MLIRBindingsPythonExtension MLIRLinalgPassesBindingsPythonExtension) diff --git a/mlir/lib/Bindings/Python/Conversions/CMakeLists.txt b/mlir/lib/Bindings/Python/Conversions/CMakeLists.txt deleted file mode 100644 --- a/mlir/lib/Bindings/Python/Conversions/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -################################################################################ -# Build python extension -################################################################################ - -add_mlir_python_extension(MLIRConversionsBindingsPythonExtension _mlirConversions - INSTALL_DIR - python - SOURCES - Conversions.cpp - LINK_LIBS PRIVATE - MLIRPythonCAPI -) diff --git a/mlir/lib/Bindings/Python/Transforms/CMakeLists.txt b/mlir/lib/Bindings/Python/Transforms/CMakeLists.txt deleted file mode 100644 --- a/mlir/lib/Bindings/Python/Transforms/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -################################################################################ -# Build python extension -################################################################################ - -add_mlir_python_extension(MLIRTransformsBindingsPythonExtension _mlirTransforms - INSTALL_DIR - python - SOURCES - Transforms.cpp - LINK_LIBS PRIVATE - MLIRPythonCAPI -) diff --git a/mlir/lib/CMakeLists.txt b/mlir/lib/CMakeLists.txt --- a/mlir/lib/CMakeLists.txt +++ b/mlir/lib/CMakeLists.txt @@ -2,7 +2,6 @@ add_flag_if_supported("-Werror=global-constructors" WERROR_GLOBAL_CONSTRUCTOR) add_subdirectory(Analysis) -add_subdirectory(Bindings) add_subdirectory(Conversion) add_subdirectory(Dialect) add_subdirectory(ExecutionEngine) diff --git a/mlir/python/CMakeLists.txt b/mlir/python/CMakeLists.txt --- a/mlir/python/CMakeLists.txt +++ b/mlir/python/CMakeLists.txt @@ -1,49 +1,309 @@ +include(AddMLIRPython) + ################################################################################ -# Copy python source tree. +# Structural groupings. ################################################################################ -file(GLOB_RECURSE PY_SRC_FILES - RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" - "${CMAKE_CURRENT_SOURCE_DIR}/mlir/*.py") +declare_mlir_python_sources(MLIRPythonSources) +declare_mlir_python_sources(MLIRPythonSources.Dialects + ADD_TO_PARENT MLIRPythonSources) + +declare_mlir_python_sources(MLIRPythonTestSources) +declare_mlir_python_sources(MLIRPythonTestSources.Dialects + ADD_TO_PARENT MLIRPythonTestSources) -add_custom_target(MLIRBindingsPythonSources ALL - DEPENDS - ${PY_SRC_FILES} +################################################################################ +# Pure python sources and generated code +################################################################################ + +declare_mlir_python_sources(MLIRPythonSources.Core + ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir" + ADD_TO_PARENT MLIRPythonSources + SOURCES + _cext_loader.py + _dlloader.py + _mlir_libs/__init__.py + ir.py + passmanager.py + dialects/_ods_common.py ) -foreach(PY_SRC_FILE ${PY_SRC_FILES}) - set(PY_DEST_FILE "${PROJECT_BINARY_DIR}/python/${PY_SRC_FILE}") - get_filename_component(PY_DEST_DIR "${PY_DEST_FILE}" DIRECTORY) - file(MAKE_DIRECTORY "${PY_DEST_DIR}") - add_custom_command( - TARGET MLIRBindingsPythonSources PRE_BUILD - COMMENT "Copying python source ${PY_SRC_FILE} -> ${PY_DEST_FILE}" - DEPENDS "${PY_SRC_FILE}" - BYPRODUCTS "${PY_DEST_FILE}" - COMMAND "${CMAKE_COMMAND}" -E create_symlink - "${CMAKE_CURRENT_SOURCE_DIR}/${PY_SRC_FILE}" "${PY_DEST_FILE}" - ) -endforeach() +declare_mlir_python_sources(MLIRPythonSources.ExecutionEngine + ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir" + ADD_TO_PARENT MLIRPythonSources + SOURCES + execution_engine.py + SOURCES_GLOB + runtime/*.py +) + +declare_mlir_python_sources(MLIRPythonSources.Passes + ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir" + ADD_TO_PARENT MLIRPythonSources + SOURCES_GLOB + conversions/*.py + transforms/*.py +) + +################################################################################ +# Dialect bindings +################################################################################ + +declare_mlir_dialect_python_bindings( + ADD_TO_PARENT MLIRPythonSources.Dialects + ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir" + TD_FILE dialects/AsyncOps.td + SOURCES_GLOB dialects/async_dialect/*.py + DIALECT_NAME async_dialect) + +declare_mlir_dialect_python_bindings( + ADD_TO_PARENT MLIRPythonSources.Dialects + ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir" + TD_FILE dialects/BuiltinOps.td + SOURCES + dialects/builtin.py + dialects/_builtin_ops_ext.py + DIALECT_NAME builtin) + +declare_mlir_dialect_python_bindings( + ADD_TO_PARENT MLIRPythonSources.Dialects + ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir" + TD_FILE dialects/GPUOps.td + SOURCES_GLOB dialects/gpu/*.py + DIALECT_NAME gpu) + +declare_mlir_dialect_python_bindings( + ADD_TO_PARENT MLIRPythonSources.Dialects + ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir" + TD_FILE dialects/LinalgOps.td + SOURCES + dialects/_linalg_ops_ext.py + SOURCES_GLOB + dialects/linalg/*.py + DIALECT_NAME linalg + DEPENDS LinalgOdsGen) + +declare_mlir_dialect_python_bindings( + ADD_TO_PARENT MLIRPythonSources.Dialects + ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir" + TD_FILE dialects/MathOps.td + SOURCES dialects/math.py + DIALECT_NAME math) + +declare_mlir_dialect_python_bindings( + ADD_TO_PARENT MLIRPythonSources.Dialects + ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir" + TD_FILE dialects/MemRefOps.td + SOURCES dialects/memref.py + DIALECT_NAME memref) + +declare_mlir_dialect_python_bindings( + ADD_TO_PARENT MLIRPythonTestSources.Dialects + ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir" + TD_FILE dialects/PythonTest.td + SOURCES dialects/python_test.py + DIALECT_NAME python_test) + +declare_mlir_dialect_python_bindings( + ADD_TO_PARENT MLIRPythonSources.Dialects + ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir" + TD_FILE dialects/ShapeOps.td + SOURCES dialects/shape.py + DIALECT_NAME shape) + +declare_mlir_dialect_python_bindings( + ADD_TO_PARENT MLIRPythonSources.Dialects + ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir" + SOURCES dialects/sparse_tensor.py + DIALECT_NAME sparse_tensor) + +declare_mlir_dialect_python_bindings( + ADD_TO_PARENT MLIRPythonSources.Dialects + ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir" + TD_FILE dialects/StandardOps.td + SOURCES dialects/std.py + DIALECT_NAME std) + +declare_mlir_dialect_python_bindings( + ADD_TO_PARENT MLIRPythonSources.Dialects + ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir" + TD_FILE dialects/TensorOps.td + SOURCES dialects/tensor.py + DIALECT_NAME tensor) + +declare_mlir_dialect_python_bindings( + ADD_TO_PARENT MLIRPythonSources.Dialects + ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir" + TD_FILE dialects/TosaOps.td + SOURCES dialects/tosa.py + DIALECT_NAME tosa) + +declare_mlir_dialect_python_bindings( + ADD_TO_PARENT MLIRPythonSources.Dialects + ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir" + TD_FILE dialects/VectorOps.td + SOURCES dialects/vector.py + DIALECT_NAME vector) + +################################################################################ +# Python extensions. +# The sources for these are all in lib/Bindings/Python, but since they have to +# be rebuilt for each package and integrate with the source setup here, we +# just reference them here instead of having ordered, cross package target +# dependencies. +################################################################################ + +set(PYTHON_SOURCE_DIR "${MLIR_SOURCE_DIR}/lib/Bindings/Python") +declare_mlir_python_extension(MLIRPythonExtension.Core + MODULE_NAME _mlir + ADD_TO_PARENT MLIRPythonSources.Core + SOURCES + ${PYTHON_SOURCE_DIR}/DialectLinalg.cpp # TODO: Break this out. + ${PYTHON_SOURCE_DIR}/DialectSparseTensor.cpp # TODO: Break this out. + ${PYTHON_SOURCE_DIR}/MainModule.cpp + ${PYTHON_SOURCE_DIR}/IRAffine.cpp + ${PYTHON_SOURCE_DIR}/IRAttributes.cpp + ${PYTHON_SOURCE_DIR}/IRCore.cpp + ${PYTHON_SOURCE_DIR}/IRModule.cpp + ${PYTHON_SOURCE_DIR}/IRTypes.cpp + ${PYTHON_SOURCE_DIR}/PybindUtils.cpp + ${PYTHON_SOURCE_DIR}/Pass.cpp + ${PYTHON_SOURCE_DIR}/ExecutionEngine.cpp # TODO: Break this out. + PRIVATE_LINK_LIBS + LLVMSupport + EMBED_CAPI_LINK_LIBS + MLIRCAPIDebug + MLIRCAPIIR + MLIRCAPIRegistration # TODO: See about dis-aggregating + + # Dialects + MLIRCAPILinalg # TODO: Remove when above is removed. + MLIRCAPISparseTensor # TODO: Remove when above is removed. + MLIRCAPIStandard + + # Execution engine (remove once disaggregated). + MLIRCEXECUTIONENGINE +) + +declare_mlir_python_extension(MLIRPythonExtension.AllPassesRegistration + MODULE_NAME _mlirAllPassesRegistration + SOURCES + ${PYTHON_SOURCE_DIR}/AllPassesRegistration.cpp + PRIVATE_LINK_LIBS + LLVMSupport + EMBED_CAPI_LINK_LIBS + MLIRCAPIConversion + MLIRCAPITransforms +) + +declare_mlir_python_extension(MLIRPythonExtension.AsyncDialectPasses + MODULE_NAME _mlirAsyncPasses + ADD_TO_PARENT MLIRPythonSources.Dialects.async_dialect + SOURCES + ${PYTHON_SOURCE_DIR}/AsyncPasses.cpp + PRIVATE_LINK_LIBS + LLVMSupport + EMBED_CAPI_LINK_LIBS + MLIRCAPIAsync +) + +declare_mlir_python_extension(MLIRPythonExtension.Conversions + MODULE_NAME _mlirConversions + ADD_TO_PARENT MLIRPythonSources.Passes + SOURCES + ${PYTHON_SOURCE_DIR}/Conversions/Conversions.cpp + PRIVATE_LINK_LIBS + LLVMSupport + EMBED_CAPI_LINK_LIBS + MLIRCAPIConversion +) + +declare_mlir_python_extension(MLIRPythonExtension.GPUDialectPasses + MODULE_NAME _mlirGPUPasses + ADD_TO_PARENT MLIRPythonSources.Dialects.gpu + SOURCES + ${PYTHON_SOURCE_DIR}/GPUPasses.cpp + PRIVATE_LINK_LIBS + LLVMSupport + EMBED_CAPI_LINK_LIBS + MLIRCAPIGPU +) + +declare_mlir_python_extension(MLIRPythonExtension.LinalgPasses + MODULE_NAME _mlirLinalgPasses + ADD_TO_PARENT MLIRPythonSources.Dialects.linalg + SOURCES + ${PYTHON_SOURCE_DIR}/LinalgPasses.cpp + PRIVATE_LINK_LIBS + LLVMSupport + EMBED_CAPI_LINK_LIBS + MLIRCAPILinalg +) -# Note that we copy from the source tree just like for headers because -# it will not be polluted with py_cache runtime artifacts (from testing and -# such). -install( - DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/mlir - DESTINATION python - COMPONENT MLIRBindingsPythonSources - FILES_MATCHING PATTERN "*.py" +declare_mlir_python_extension(MLIRPythonExtension.SparseTensorDialectPasses + MODULE_NAME _mlirSparseTensorPasses + ADD_TO_PARENT MLIRPythonSources.Dialects.sparse_tensor + SOURCES + ${PYTHON_SOURCE_DIR}/SparseTensorPasses.cpp + PRIVATE_LINK_LIBS + LLVMSupport + EMBED_CAPI_LINK_LIBS + MLIRCAPISparseTensor ) -if (NOT LLVM_ENABLE_IDE) - add_llvm_install_targets( - install-MLIRBindingsPythonSources - DEPENDS MLIRBindingsPythonSources - COMPONENT MLIRBindingsPythonSources) -endif() +declare_mlir_python_extension(MLIRPythonExtension.Transforms + MODULE_NAME _mlirTransforms + ADD_TO_PARENT MLIRPythonSources.Passes + SOURCES + ${PYTHON_SOURCE_DIR}/Transforms/Transforms.cpp + PRIVATE_LINK_LIBS + LLVMSupport + EMBED_CAPI_LINK_LIBS + MLIRCAPITransforms +) ################################################################################ -# Generated sources. +# Common CAPI dependency DSO. +# All python extensions must link through one DSO which exports the CAPI, and +# this must have a globally unique name amongst all embeddors of the python +# library since it will effectively have global scope. +# +# The presence of this aggregate library is part of the long term plan, but its +# use needs to be made more flexible. +# +# TODO: Upgrade to the aggregate utility in https://reviews.llvm.org/D106419 +# once ready. ################################################################################ -add_subdirectory(mlir/dialects) +add_mlir_python_common_capi_library(MLIRPythonCAPI + INSTALL_COMPONENT MLIRPythonModules + INSTALL_DESTINATION python_packages/mlir_core/mlir/_mlir_libs + OUTPUT_DIRECTORY "${MLIR_BINARY_DIR}/python_packages/mlir_core/mlir/_mlir_libs" + RELATIVE_INSTALL_ROOT "../../../.." + DECLARED_SOURCES + MLIRPythonSources + MLIRPythonExtension.AllPassesRegistration +) + +################################################################################ +# The fully assembled package of modules. +# This must come last. +################################################################################ + +add_mlir_python_modules(MLIRPythonModules + ROOT_PREFIX "${MLIR_BINARY_DIR}/python_packages/mlir_core/mlir" + INSTALL_PREFIX "python_packages/mlir_core/mlir" + DECLARED_SOURCES + MLIRPythonSources + MLIRPythonExtension.AllPassesRegistration + COMMON_CAPI_LINK_LIBS + MLIRPythonCAPI + ) + + +add_mlir_python_modules(MLIRPythonTestModules + ROOT_PREFIX "${MLIR_BINARY_DIR}/python_packages/mlir_test/mlir" + INSTALL_PREFIX "python_packages/mlir_test/mlir" + DECLARED_SOURCES + MLIRPythonTestSources + ) diff --git a/mlir/python/mlir/_cext_loader.py b/mlir/python/mlir/_cext_loader.py --- a/mlir/python/mlir/_cext_loader.py +++ b/mlir/python/mlir/_cext_loader.py @@ -3,28 +3,27 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception """Common module for looking up and manipulating C-Extensions.""" -# Packaged installs have a top-level _mlir_libs package with symbols: -# load_extension(name): Loads a named extension module -# preload_dependency(public_name): Loads a shared-library/DLL into the -# namespace. TODO: Remove this in favor of a more robust mechanism. -# Conditionally switch based on whether we are in a package context. +# The normal layout is to have a nested _mlir_libs package that contains +# all native libraries and extensions. If that exists, use it, but also fallback +# to old behavior where extensions were at the top level as loose libraries. +# TODO: Remove the fallback once downstreams adapt. try: - import _mlir_libs + from ._mlir_libs import * + # TODO: Remove these aliases once everything migrates + _preload_dependency = preload_dependency + _load_extension = load_extension except ModuleNotFoundError: # Assume that we are in-tree. # The _dlloader takes care of platform specific setup before we try to # load a shared library. - from ._dlloader import preload_dependency as _preload_dependency + # TODO: Remove _dlloader once all consolidated on the _mlir_libs approach. + from ._dlloader import preload_dependency - def _load_extension(name): + def load_extension(name): import importlib return importlib.import_module(name) # i.e. '_mlir' at the top level -else: - # Packaged distribution. - _load_extension = _mlir_libs.load_extension - _preload_dependency = _mlir_libs.preload_dependency -_preload_dependency("MLIRPythonCAPI") +preload_dependency("MLIRPythonCAPI") # Expose the corresponding C-Extension module with a well-known name at this # top-level module. This allows relative imports like the following to @@ -32,7 +31,7 @@ # from .._cext_loader import _cext # This reduces coupling, allowing embedding of the python sources into another # project that can just vary based on this top-level loader module. -_cext = _load_extension("_mlir") +_cext = load_extension("_mlir") def _reexport_cext(cext_module_name, target_module_name): diff --git a/mlir/python/mlir/_mlir_libs/__init__.py b/mlir/python/mlir/_mlir_libs/__init__.py new file mode 100644 --- /dev/null +++ b/mlir/python/mlir/_mlir_libs/__init__.py @@ -0,0 +1,21 @@ +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import importlib +import os + +__all__ = [ + "load_extension", + "preload_dependency", +] + +_this_dir = os.path.dirname(__file__) + +def load_extension(name): + return importlib.import_module(f".{name}", __package__) + + +def preload_dependency(public_name): + # TODO: Implement this hook to pre-load DLLs with ctypes on Windows. + pass diff --git a/mlir/python/mlir/dialects/CMakeLists.txt b/mlir/python/mlir/dialects/CMakeLists.txt deleted file mode 100644 --- a/mlir/python/mlir/dialects/CMakeLists.txt +++ /dev/null @@ -1,86 +0,0 @@ -include(AddMLIRPython) - -################################################################################ -# Generate dialect-specific bindings. -################################################################################ - -add_mlir_dialect_python_bindings(MLIRBindingsPythonAsyncOps - TD_FILE AsyncOps.td - DIALECT_NAME async_dialect) -add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonAsyncOps) - -add_mlir_dialect_python_bindings(MLIRBindingsPythonBuiltinOps - TD_FILE BuiltinOps.td - DIALECT_NAME builtin) -add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonBuiltinOps) - -add_mlir_dialect_python_bindings(MLIRBindingsPythonGPUOps - TD_FILE GPUOps.td - DIALECT_NAME gpu) -add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonGPUOps) - -add_mlir_dialect_python_bindings(MLIRBindingsPythonLinalgOps - TD_FILE LinalgOps.td - DIALECT_NAME linalg - DEPENDS LinalgOdsGen) -add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonLinalgOps) - -add_mlir_dialect_python_bindings(MLIRBindingsPythonMathOps - TD_FILE MathOps.td - DIALECT_NAME math) -add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonMathOps) - -add_mlir_dialect_python_bindings(MLIRBindingsPythonMemRefOps - TD_FILE MemRefOps.td - DIALECT_NAME memref) -add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonMemRefOps) - -add_mlir_dialect_python_bindings(MLIRBindingsPythonShapeOps - TD_FILE ShapeOps.td - DIALECT_NAME shape) -add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonShapeOps) - -add_mlir_dialect_python_bindings(MLIRBindingsPythonStandardOps - TD_FILE StandardOps.td - DIALECT_NAME std) -add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonStandardOps) - -add_mlir_dialect_python_bindings(MLIRBindingsPythonTensorOps - TD_FILE TensorOps.td - DIALECT_NAME tensor) -add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonTensorOps) - -add_mlir_dialect_python_bindings(MLIRBindingsPythonTosaOps - TD_FILE TosaOps.td - DIALECT_NAME tosa) -add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonTosaOps) - -add_mlir_dialect_python_bindings(MLIRBindingsPythonVectorOps - TD_FILE VectorOps.td - DIALECT_NAME vector) -add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonVectorOps) - -################################################################################ -# Installation. -################################################################################ - -# Dialect sources are generated. Install separately. -# Note that __pycache__ directories may have been left by tests and other -# executions. And __init__.py is handled as a regular source file. -# TODO: Eliminate this glob install, instead adding INSTALL_COMPONENT to -# add_mlir_dialect_python_bindings and installing the precise file there. -install( - DIRECTORY ${PROJECT_BINARY_DIR}/python/mlir/dialects - DESTINATION python/mlir - COMPONENT MLIRBindingsPythonDialects - FILES_MATCHING PATTERN "_*_gen.py" - PATTERN "__pycache__" EXCLUDE - PATTERN "__init__.py" EXCLUDE -) - -if (NOT LLVM_ENABLE_IDE) - add_llvm_install_targets( - install-MLIRBindingsPythonDialects - DEPENDS MLIRBindingsPythonSources - COMPONENT MLIRBindingsPythonDialects) -endif() diff --git a/mlir/python/mlir/dialects/PythonTest.td b/mlir/python/mlir/dialects/PythonTest.td new file mode 100644 --- /dev/null +++ b/mlir/python/mlir/dialects/PythonTest.td @@ -0,0 +1,33 @@ +//===-- python_test_ops.td - Python test Op definitions ----*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef PYTHON_TEST_OPS +#define PYTHON_TEST_OPS + +include "mlir/Bindings/Python/Attributes.td" +include "mlir/IR/OpBase.td" + +def Python_Test_Dialect : Dialect { + let name = "python_test"; + let cppNamespace = "PythonTest"; +} +class TestOp traits = []> + : Op; + +def AttributedOp : TestOp<"attributed_op"> { + let arguments = (ins I32Attr:$mandatory_i32, + OptionalAttr:$optional_i32, + UnitAttr:$unit); +} + +def PropertyOp : TestOp<"property_op"> { + let arguments = (ins I32Attr:$property, + I32:$idx); +} + +#endif // PYTHON_TEST_OPS diff --git a/mlir/python/mlir/dialects/_builtin_ops_ext.py b/mlir/python/mlir/dialects/_builtin_ops_ext.py --- a/mlir/python/mlir/dialects/_builtin_ops_ext.py +++ b/mlir/python/mlir/dialects/_builtin_ops_ext.py @@ -2,11 +2,14 @@ # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -from typing import Optional, Sequence +try: + from typing import Optional, Sequence -import inspect + import inspect -from ..ir import * + from ..ir import * +except ImportError as e: + raise RuntimeError("Error loading imports from extension module") from e class ModuleOp: diff --git a/mlir/python/mlir/dialects/_linalg_ops_ext.py b/mlir/python/mlir/dialects/_linalg_ops_ext.py --- a/mlir/python/mlir/dialects/_linalg_ops_ext.py +++ b/mlir/python/mlir/dialects/_linalg_ops_ext.py @@ -2,12 +2,16 @@ # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -from typing import Optional, Sequence, Union -from ..ir import * -from ._ods_common import get_default_loc_context -# TODO: resolve name collision for Linalg functionality that is injected inside -# the _mlir.dialects.linalg directly via pybind. -from _mlir.dialects.linalg import fill_builtin_region +try: + from typing import Optional, Sequence, Union + from ..ir import * + from ._ods_common import get_default_loc_context + # TODO: resolve name collision for Linalg functionality that is injected inside + # the _mlir.dialects.linalg directly via pybind. + from .._cext_loader import _cext + fill_builtin_region = _cext.dialects.linalg.fill_builtin_region +except ImportError as e: + raise RuntimeError("Error loading imports from extension module") from e def isa(cls: Type, ty: Type): diff --git a/mlir/python/mlir/dialects/linalg/opdsl/lang/emitter.py b/mlir/python/mlir/dialects/linalg/opdsl/lang/emitter.py --- a/mlir/python/mlir/dialects/linalg/opdsl/lang/emitter.py +++ b/mlir/python/mlir/dialects/linalg/opdsl/lang/emitter.py @@ -10,7 +10,8 @@ from mlir.dialects import math # TODO: resolve name collision for Linalg functionality that is injected inside # the _mlir.dialects.linalg directly via pybind. -from _mlir.dialects.linalg import fill_builtin_region +from ....._cext_loader import _cext +fill_builtin_region = _cext.dialects.linalg.fill_builtin_region from .scalar_expr import * from .config import * diff --git a/mlir/test/CMakeLists.txt b/mlir/test/CMakeLists.txt --- a/mlir/test/CMakeLists.txt +++ b/mlir/test/CMakeLists.txt @@ -1,10 +1,6 @@ add_subdirectory(CAPI) add_subdirectory(lib) -if(MLIR_ENABLE_BINDINGS_PYTHON) - add_subdirectory(python) -endif() - llvm_canonicalize_cmake_booleans( MLIR_ENABLE_BINDINGS_PYTHON LLVM_BUILD_EXAMPLES @@ -124,11 +120,8 @@ if(MLIR_ENABLE_BINDINGS_PYTHON) list(APPEND MLIR_TEST_DEPENDS - MLIRBindingsPythonExtension - MLIRBindingsPythonSources - MLIRBindingsPythonTestOps - MLIRTransformsBindingsPythonExtension - MLIRConversionsBindingsPythonExtension + MLIRPythonModules + MLIRPythonTestModules ) endif() diff --git a/mlir/test/lit.cfg.py b/mlir/test/lit.cfg.py --- a/mlir/test/lit.cfg.py +++ b/mlir/test/lit.cfg.py @@ -103,13 +103,8 @@ # by copying/linking sources to build. if config.enable_bindings_python: llvm_config.with_environment('PYTHONPATH', [ - # TODO: Don't reference the llvm_obj_root here: the invariant is that - # the python/ must be at the same level of the lib directory - # where libMLIR.so is installed. This is presently not optimal from a - # project separation perspective and a discussion on how to better - # segment MLIR libraries needs to happen. See also - # lib/Bindings/Python/CMakeLists.txt for where this is set up. - os.path.join(config.llvm_obj_root, 'python'), + os.path.join(config.mlir_obj_root, 'python_packages', 'mlir_core'), + os.path.join(config.mlir_obj_root, 'python_packages', 'mlir_test'), ], append_path=True) if config.enable_assertions: diff --git a/mlir/test/python/CMakeLists.txt b/mlir/test/python/CMakeLists.txt deleted file mode 100644 --- a/mlir/test/python/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -include(AddMLIRPython) - -add_mlir_dialect_python_bindings(MLIRBindingsPythonTestOps - TD_FILE python_test_ops.td - DIALECT_NAME python_test) diff --git a/mlir/test/python/ir/operation.py b/mlir/test/python/ir/operation.py --- a/mlir/test/python/ir/operation.py +++ b/mlir/test/python/ir/operation.py @@ -533,12 +533,12 @@ # One of the custom ops should resolve to the default OpView. custom = module.body.operations[0] - # CHECK: <_mlir.ir.OpView object + # CHECK: OpView object print(repr(custom)) # Check again to make sure negative caching works. custom = module.body.operations[0] - # CHECK: <_mlir.ir.OpView object + # CHECK: OpView object print(repr(custom)) run(testKnownOpView)