diff --git a/mlir/CMakeLists.txt b/mlir/CMakeLists.txt --- a/mlir/CMakeLists.txt +++ b/mlir/CMakeLists.txt @@ -51,6 +51,39 @@ option(MLIR_INCLUDE_INTEGRATION_TESTS "Generate build targets for the MLIR integration tests.") +#------------------------------------------------------------------------------- +# Python Bindings Configuration +# Requires: +# The pybind11 library can be found (set with -DPYBIND_DIR=...) +# The python executable is correct (set with -DPYTHON_EXECUTABLE=...) +# +# Version locking +# --------------- +# By default, python extensions are version locked to specific Python libraries. +# This linking mode is somewhat more consistent across platforms and surfaces +# undefined symbols at link time (vs runtime). It is suitable for development +# workflows but can be disabled for more flexible deployment by +# setting -DMLIR_PYTHON_BINDINGS_VERSION_LOCKED=OFF +#------------------------------------------------------------------------------- + +option(MLIR_ENABLE_PYTHON_BINDINGS + "Enables building of Python bindings.") +option(MLIR_PYTHON_BINDINGS_VERSION_LOCKED + "Links to specific python libraries, resolving all symbols." + ON) + +if(MLIR_ENABLE_PYTHON_BINDINGS) + find_package(PythonInterp REQUIRED) + find_package(PythonLibs REQUIRED) + message(STATUS "Found python include dirs: ${PYTHON_INCLUDE_DIRS}") + message(STATUS "Found ppython libraries: ${PYTHON_LIBRARIES}") + find_package(pybind11 CONFIG REQUIRED) + message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIRS}") + message(STATUS "Python prefix = '${PYTHON_MODULE_PREFIX}', " + "suffix = '${PYTHON_MODULE_SUFFIX}', " + "extension = '${PYTHON_MODULE_EXTENSION}") +endif() + include_directories( "include") include_directories( ${MLIR_INCLUDE_DIR}) @@ -65,6 +98,8 @@ add_definitions(-DMLIR_INCLUDE_TESTS) add_subdirectory(unittests) add_subdirectory(test) + # Must come after test, since it extends it. + add_subdirectory(bindings_test) endif() if (MLIR_INCLUDE_INTEGRATION_TESTS) add_definitions(-DMLIR_INCLUDE_INTEGRATION_TESTS) diff --git a/mlir/bindings_test/CMakeLists.txt b/mlir/bindings_test/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/mlir/bindings_test/CMakeLists.txt @@ -0,0 +1,3 @@ +if(MLIR_ENABLE_PYTHON_BINDINGS) + add_subdirectory(Python) +endif() diff --git a/mlir/bindings_test/Python/CMakeLists.txt b/mlir/bindings_test/Python/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/mlir/bindings_test/Python/CMakeLists.txt @@ -0,0 +1,26 @@ +# Passed to lit.site.cfg.py.in to set up the path where to find the libraries. + +set(TEST_DEPENDS + MLIRBindingsPythonExtension + FileCheck count not + mlir-opt + ) + +configure_lit_site_cfg( + ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in + ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py + ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py + ) + +add_lit_testsuite(check-mlir-bindings-python "Running the MLIR Python bindings tests" + ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ${TEST_DEPENDS} + ) +set_target_properties(check-mlir-bindings-python PROPERTIES FOLDER "bindings_test/python") + +# Extend the check-mlir target if the python bindings are enabled. +add_dependencies(check-mlir check-mlir-bindings-python) + +add_lit_testsuites(MLIR_BINDINGS_PYTHON ${CMAKE_CURRENT_SOURCE_DIR} + DEPENDS ${TEST_DEPENDS} + ) diff --git a/mlir/bindings_test/Python/lit.cfg.py b/mlir/bindings_test/Python/lit.cfg.py new file mode 100644 --- /dev/null +++ b/mlir/bindings_test/Python/lit.cfg.py @@ -0,0 +1,58 @@ +# -*- Python -*- + +import os +import platform +import re +import subprocess +import tempfile + +import lit.formats +import lit.util + +from lit.llvm import llvm_config +from lit.llvm.subst import ToolSubst + +# Configuration file for the 'lit' integration test runner. + +# name: The name of this integration test suite. +config.name = 'MLIR_BINDINGS_PYTHON' + +config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell) + +# suffixes: A list of file extensions to treat as integration test files. +config.suffixes = ['.py'] + +# test_source_root: The root path where integration tests are located. +config.test_source_root = os.path.dirname(__file__) + +# test_exec_root: The root path where integration tests should be run. +config.test_exec_root = os.path.join(config.mlir_obj_root, 'bindings_test/Python') + +config.substitutions.append(('%PATH%', config.environment['PATH'])) +config.substitutions.append(('%shlibext', config.llvm_shlib_ext)) +config.substitutions.append(('%mlir_src_root', config.mlir_src_root)) +config.substitutions.append(('%PYTHON', config.python_executable)) + +llvm_config.with_system_environment(['HOME', 'INCLUDE', 'LIB', 'TMP', 'TEMP']) + +llvm_config.use_default_substitutions() + +# excludes: A list of directories to exclude from the integration testsuite. +config.excludes = ['CMakeLists.txt', 'README.txt', 'LICENSE.txt', + 'lit.cfg.py', 'lit.site.cfg.py'] + +# Tweak the PATH to include the tools dir. +llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True) + +# Add the python path for both the source and binary tree. +llvm_config.with_environment('PYTHONPATH', [ + os.path.join(config.mlir_src_root, "lib", "Bindings", "Python"), + os.path.join(config.mlir_obj_root, "lib", "Bindings", "Python"), +], append_path=True) + +tool_dirs = [config.mlir_tools_dir, config.llvm_tools_dir] +tools = [ + 'mlir-opt', +] + +llvm_config.add_tool_substitutions(tools, tool_dirs) diff --git a/mlir/bindings_test/Python/lit.site.cfg.py.in b/mlir/bindings_test/Python/lit.site.cfg.py.in new file mode 100644 --- /dev/null +++ b/mlir/bindings_test/Python/lit.site.cfg.py.in @@ -0,0 +1,51 @@ +@LIT_SITE_CFG_IN_HEADER@ + +import sys + +config.host_triple = "@LLVM_HOST_TRIPLE@" +config.target_triple = "@TARGET_TRIPLE@" +config.llvm_src_root = "@LLVM_SOURCE_DIR@" +config.llvm_obj_root = "@LLVM_BINARY_DIR@" +config.llvm_tools_dir = "@LLVM_TOOLS_DIR@" +config.llvm_lib_dir = "@LLVM_LIBRARY_DIR@" +config.llvm_shlib_dir = "@SHLIBDIR@" +config.llvm_shlib_ext = "@SHLIBEXT@" +config.llvm_exe_ext = "@EXEEXT@" +config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@" +config.python_executable = "@PYTHON_EXECUTABLE@" +config.gold_executable = "@GOLD_EXECUTABLE@" +config.ld64_executable = "@LD64_EXECUTABLE@" +config.enable_shared = @ENABLE_SHARED@ +config.enable_assertions = @ENABLE_ASSERTIONS@ +config.targets_to_build = "@TARGETS_TO_BUILD@" +config.native_target = "@LLVM_NATIVE_ARCH@" +config.llvm_bindings = "@LLVM_BINDINGS@".split(' ') +config.host_os = "@HOST_OS@" +config.host_cc = "@HOST_CC@" +config.host_cxx = "@HOST_CXX@" +config.host_cmake = "@CMAKE_COMMAND@" +# Note: ldflags can contain double-quoted paths, so must use single quotes here. +config.host_ldflags = '@HOST_LDFLAGS@' +config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@" +config.llvm_host_triple = '@LLVM_HOST_TRIPLE@' +config.host_arch = "@HOST_ARCH@" +config.mlir_src_root = "@MLIR_SOURCE_DIR@" +config.mlir_obj_root = "@MLIR_BINARY_DIR@" +config.mlir_tools_dir = "@MLIR_TOOLS_DIR@" +config.python_executable = "@PYTHON_EXECUTABLE@" + +# Support substitution of the tools_dir with user parameters. This is +# used when we can't determine the tool dir at configuration time. +try: + config.llvm_tools_dir = config.llvm_tools_dir % lit_config.params + config.llvm_shlib_dir = config.llvm_shlib_dir % lit_config.params +except KeyError: + e = sys.exc_info()[1] + key, = e.args + lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key)) + +import lit.llvm +lit.llvm.initialize(lit_config, config) + +# Let the main config do the real work. +lit_config.load_config(config, "@MLIR_SOURCE_DIR@/bindings_test/Python/lit.cfg.py") diff --git a/mlir/bindings_test/Python/smoke_test.py b/mlir/bindings_test/Python/smoke_test.py new file mode 100644 --- /dev/null +++ b/mlir/bindings_test/Python/smoke_test.py @@ -0,0 +1,6 @@ +# RUN: %PYTHON %s | FileCheck %s + +import mlir + +# CHECK: From the native module +print(mlir.get_test_value()) diff --git a/mlir/lib/Bindings/CMakeLists.txt b/mlir/lib/Bindings/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/mlir/lib/Bindings/CMakeLists.txt @@ -0,0 +1,3 @@ +if(MLIR_ENABLE_PYTHON_BINDINGS) + add_subdirectory(Python) +endif() diff --git a/mlir/lib/Bindings/Python/CMakeLists.txt b/mlir/lib/Bindings/Python/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/mlir/lib/Bindings/Python/CMakeLists.txt @@ -0,0 +1,69 @@ +# Normally on unix-like platforms, extensions are built as "MODULE" libraries +# and do not explicitly link to the python shared object. This allows for +# some greater deployment flexibility since the extension will bind to +# symbols in the python interpreter on load. However, it also keeps the +# linker from erroring on undefined symbols, leaving this to (usually obtuse) +# runtime errors. Building in "SHARED" mode with an explicit link to the +# python libraries allows us to build with the expectation of no undefined +# symbols, which is better for development. +if(MLIR_PYTHON_BINDINGS_VERSION_LOCKED) + set(PYEXT_LINK_MODE SHARED) + set(PYEXT_LIBADD ${PYTHON_LIBRARIES}) +else() + set(PYEXT_LINK_MODE MODULE) + set(PYEXT_LIBADD) +endif() + +# The actual extension library produces a shared-object or DLL and has +# sources that must be compiled in accordance with pybind11 needs (RTTI and +# exceptions). +add_library(MLIRBindingsPythonExtension ${PYEXT_LINK_MODE} + MainModule.cpp +) + +target_include_directories(MLIRBindingsPythonExtension PRIVATE + "${PYTHON_INCLUDE_DIRS}" + "${pybind11_INCLUDE_DIRS}") + +# The extension itself must be compiled with RTTI and exceptions enabled. +# Also, some warning classes triggered by pybind11 are disabled. +target_compile_options(MLIRBindingsPythonExtension PRIVATE + $<$,$,$>: + # Enable RTTI and exceptions. + -frtti -fexceptions + # Noisy pybind warnings + -Wno-unused-value + -Wno-covered-switch-default + > + $<$: + # Enable RTTI and exceptions. + /EHsc /GR> +) + +# Configure the output to match python expectations. +set_target_properties( + MLIRBindingsPythonExtension PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + OUTPUT_NAME "_mlir" + PREFIX "${PYTHON_MODULE_PREFIX}" + SUFFIX "${PYTHON_MODULE_SUFFIX}${PYTHON_MODULE_EXTENSION}" +) + +# pybind11 requires binding code to be compiled with -fvisibility=hidden +# For static linkage, better code can be generated if the entire project +# compiles that way, but that is not enforced here. Instead, include a linker +# script that explicitly hides anything but the PyInit_* symbols, allowing gc +# to take place. +set_target_properties( + MLIRBindingsPythonExtension PROPERTIES CXX_VISIBILITY_PRESET "hidden") +if(NOT MSVC) + set_target_properties(MLIRBindingsPythonExtension + PROPERTIES LINK_FLAGS + "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/unix_version.lds") +endif() + +target_link_libraries(MLIRBindingsPythonExtension + PRIVATE + + ${PYEXT_LIBADD} +) diff --git a/mlir/lib/Bindings/Python/MainModule.cpp b/mlir/lib/Bindings/Python/MainModule.cpp new file mode 100644 --- /dev/null +++ b/mlir/lib/Bindings/Python/MainModule.cpp @@ -0,0 +1,16 @@ +//===- MainModule.cpp - Main pybind module --------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include + +PYBIND11_MODULE(_mlir, m) { + m.doc() = "MLIR Python Native Extension"; + + m.def("get_test_value", + []() { return std::string("From the native module"); }); +} diff --git a/mlir/lib/Bindings/Python/mlir/__init__.py b/mlir/lib/Bindings/Python/mlir/__init__.py new file mode 100644 --- /dev/null +++ b/mlir/lib/Bindings/Python/mlir/__init__.py @@ -0,0 +1,11 @@ +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# Note that the only function of this module is currently to load the +# native module and re-export its symbols. In the future, this file is +# reserved as a trampoline to handle environment specific loading needs +# and arbitrate any one-time initialization needed in various shared-library +# scenarios. + +from _mlir import * diff --git a/mlir/lib/Bindings/Python/unix_version.lds b/mlir/lib/Bindings/Python/unix_version.lds new file mode 100644 --- /dev/null +++ b/mlir/lib/Bindings/Python/unix_version.lds @@ -0,0 +1,4 @@ +{ + global: PyInit__mlir; + local: *; +}; diff --git a/mlir/lib/CMakeLists.txt b/mlir/lib/CMakeLists.txt --- a/mlir/lib/CMakeLists.txt +++ b/mlir/lib/CMakeLists.txt @@ -2,6 +2,7 @@ add_flag_if_supported("-Werror=global-constructors" WERROR_GLOBAL_CONSTRUCTOR) add_subdirectory(Analysis) +add_subdirectory(Bindings) add_subdirectory(Conversion) add_subdirectory(Dialect) add_subdirectory(EDSC) @@ -15,4 +16,4 @@ add_subdirectory(TableGen) add_subdirectory(Target) add_subdirectory(Transforms) -add_subdirectory(Translation) \ No newline at end of file +add_subdirectory(Translation)