diff --git a/mlir/CMakeLists.txt b/mlir/CMakeLists.txt --- a/mlir/CMakeLists.txt +++ b/mlir/CMakeLists.txt @@ -134,6 +134,10 @@ "Prime the python detection by searching for a full 'Development' \ component first (temporary while diagnosing environment specific Python \ detection issues)") +set(MLIR_DETECT_PYTHON_SIMPLE_CONFIG 0 CACHE BOOL + "Uses a simplified, experimental mechanism for finding Python \ + development packages. Requires CMake >= 3.18.\ + See: https://github.com/llvm/llvm-project/issues/54906") if(MLIR_ENABLE_BINDINGS_PYTHON) include(MLIRDetectPythonEnv) mlir_configure_python_dev_packages() diff --git a/mlir/cmake/modules/CMakeLists.txt b/mlir/cmake/modules/CMakeLists.txt --- a/mlir/cmake/modules/CMakeLists.txt +++ b/mlir/cmake/modules/CMakeLists.txt @@ -45,7 +45,9 @@ # This should be removed in the future. file(COPY . DESTINATION ${mlir_cmake_builddir} - FILES_MATCHING PATTERN *.cmake + FILES_MATCHING + PATTERN *.cmake + PATTERN *.py PATTERN CMakeFiles EXCLUDE ) @@ -82,6 +84,8 @@ ${CMAKE_CURRENT_SOURCE_DIR}/AddMLIR.cmake ${CMAKE_CURRENT_SOURCE_DIR}/AddMLIRPython.cmake ${CMAKE_CURRENT_SOURCE_DIR}/MLIRDetectPythonEnv.cmake + ${CMAKE_CURRENT_SOURCE_DIR}/MLIRSimplePythonModuleConfig.cmake + ${CMAKE_CURRENT_SOURCE_DIR}/query_python_module_config.py DESTINATION ${MLIR_INSTALL_PACKAGE_DIR} COMPONENT mlir-cmake-exports) diff --git a/mlir/cmake/modules/MLIRDetectPythonEnv.cmake b/mlir/cmake/modules/MLIRDetectPythonEnv.cmake --- a/mlir/cmake/modules/MLIRDetectPythonEnv.cmake +++ b/mlir/cmake/modules/MLIRDetectPythonEnv.cmake @@ -9,33 +9,44 @@ "for full support. Detected current version: ${CMAKE_VERSION}") endif() - # After CMake 3.18, we are able to limit the scope of the search to just - # Development.Module. Searching for Development will fail in situations where - # the Python libraries are not available. When possible, limit to just - # Development.Module. - # See https://pybind11.readthedocs.io/en/stable/compiling.html#findpython-mode - if(CMAKE_VERSION VERSION_LESS "3.18.0") - message(WARNING - "This version of CMake is not compatible with statically built Python " - "installations. If Python fails to detect below this may apply to you. " - "Recommend upgrading to at least CMake 3.18. " - "Detected current version: ${CMAKE_VERSION}" - ) - set(_python_development_component Development) + if(MLIR_DETECT_PYTHON_SIMPLE_CONFIG + AND NOT TARGET Python3::Module + AND CMAKE_VERSION VERSION_GREATER_EQUAL "3.18.0") + # New/simplified Python development setup. + find_package(Python3 ${LLVM_MINIMUM_PYTHON_VERSION} + COMPONENTS Interpreter REQUIRED) + include(MLIRSimplePythonModuleConfig) else() - if(MLIR_DETECT_PYTHON_ENV_PRIME_SEARCH) - # Prime the search for python to see if there is a full development - # package. This seems to work around cmake bugs searching only for - # Development.Module in some environments. However, in other environments - # it may interfere with the subsequent search for Development.Module. - find_package(Python3 ${LLVM_MINIMUM_PYTHON_VERSION} - COMPONENTS Interpreter Development) + # Standard find_package approach, with all of our hacks added. + # After CMake 3.18, we are able to limit the scope of the search to just + # Development.Module. Searching for Development will fail in situations where + # the Python libraries are not available. When possible, limit to just + # Development.Module. + # See https://pybind11.readthedocs.io/en/stable/compiling.html#findpython-mode + if(CMAKE_VERSION VERSION_LESS "3.18.0") + message(WARNING + "This version of CMake is not compatible with statically built Python " + "installations. If Python fails to detect below this may apply to you. " + "Recommend upgrading to at least CMake 3.18. " + "Detected current version: ${CMAKE_VERSION}" + ) + set(_python_development_component Development) + else() + if(MLIR_DETECT_PYTHON_ENV_PRIME_SEARCH) + # Prime the search for python to see if there is a full development + # package. This seems to work around cmake bugs searching only for + # Development.Module in some environments. However, in other environments + # it may interfere with the subsequent search for Development.Module. + find_package(Python3 ${LLVM_MINIMUM_PYTHON_VERSION} + COMPONENTS Interpreter Development) + endif() + set(_python_development_component Development.Module) endif() - set(_python_development_component Development.Module) + find_package(Python3 ${LLVM_MINIMUM_PYTHON_VERSION} + COMPONENTS Interpreter ${_python_development_component} NumPy REQUIRED) + unset(_python_development_component) endif() - find_package(Python3 ${LLVM_MINIMUM_PYTHON_VERSION} - COMPONENTS Interpreter ${_python_development_component} NumPy REQUIRED) - unset(_python_development_component) + message(STATUS "Found python include dirs: ${Python3_INCLUDE_DIRS}") message(STATUS "Found python libraries: ${Python3_LIBRARIES}") message(STATUS "Found numpy v${Python3_NumPy_VERSION}: ${Python3_NumPy_INCLUDE_DIRS}") @@ -57,7 +68,7 @@ else() message(STATUS "Checking for pybind11 in python path...") execute_process( - COMMAND "${Python3_EXECUTABLE}" + COMMAND ${Python3_INTERPRETER_LAUNCHER} "${Python3_EXECUTABLE}" -c "import pybind11;print(pybind11.get_cmake_dir(), end='')" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} RESULT_VARIABLE STATUS diff --git a/mlir/cmake/modules/MLIRSimplePythonModuleConfig.cmake b/mlir/cmake/modules/MLIRSimplePythonModuleConfig.cmake new file mode 100644 --- /dev/null +++ b/mlir/cmake/modules/MLIRSimplePythonModuleConfig.cmake @@ -0,0 +1,103 @@ +# This is a prototype utility to auto-detect the Python3 development +# configuration, suitable for building MLIR Python. The aim of using this +# (vs the CMake FindPython3 builtin support) is robustness. We have found +# the CMake version to be extremely unreliable and it is terribly complicated +# and not clear how to debug or improve. This CMake module is being retained +# in a standalone state (versus commingled with the rest of the build) so +# that it can perhaps find a home upstream or inspire improvements. It should +# be usable by projects that depend on MLIR without any other MLIR +# entanglements. +# See: https://github.com/llvm/llvm-project/issues/54906 +# +# Usage: +# The top-level CMake file should find the Python3 interpreter with +# something like: +# find_package(Python3 3.6 COMPONENTS Interpreter) +# This can be influenced from the command-line with: +# -DPython3_EXECUTABLE=$(which python) +# +# If anything goes wrong with this, just run the query_python_module_config.py +# file (next to this one) with your Python interpreter. Paste the results +# to an issue for further help. + +function(_mlir_query_python_module_config) + if(NOT Python3_Interpreter_FOUND) + message(FATAL_ERROR "In order to configure Python for development, the interpreter must be configured first") + endif() + execute_process(COMMAND + ${Python3_INTERPRETER_LAUNCHER} + "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/query_python_module_config.py" + RESULT_VARIABLE _result + ERROR_VARIABLE _error_output + OUTPUT_VARIABLE _config_script + ) + if(NOT _result EQUAL 0) + message(SEND_ERROR "Failed to detect Python development settings:\n${_error_output}") + set(_mlir_python_module_config_script "" PARENT_SCOPE) + return() + endif() + set(_mlir_python_module_config_script "${_config_script}" PARENT_SCOPE) +endfunction() + +# Only use the simplified development mode if someone up-stack hasn't +# already managed to use the regular find_package approach. +if(Python3_Development.Module_FOUND OR Python3_Development_FOUND) + message(VERBOSE "Not using simple Python config: Already configured for development") +else() + _mlir_query_python_module_config() + cmake_language(EVAL CODE "${_mlir_python_module_config_script}") + if(NOT Python3_Development.Module_FOUND) + message(FATAL_ERROR "Something went wrong with MLIR Python auto-detection. Please report an issue.") + endif() + + # Imported Interface libraries. + # See: https://cmake.org/cmake/help/latest/module/FindPython3.html#imported-targets + add_library(Python3::Module INTERFACE IMPORTED) + set_property(TARGET Python3::Module + PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${Python3_INCLUDE_DIRS}") + if(Python3_LIBRARIES) + target_link_libraries(Python3::Module INTERFACE ${Python3_LIBRARIES}) + endif() + add_library(Python3::NumPy INTERFACE IMPORTED) + set_property(TARGET Python3::NumPy + PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${Python3_NumPy_INCLUDE_DIRS}") + target_link_libraries(Python3::NumPy INTERFACE Python3::Module) + + # Commands. + # See: https://cmake.org/cmake/help/latest/module/FindPython3.html#commands + function (Python3_add_library name) + cmake_parse_arguments( + PARSE_ARGV 1 ARG + "STATIC;SHARED;MODULE;WITH_SOABI" + "" + "") + if(ARG_STATIC) + set(_type STATIC) + elseif(ARG_SHARED) + set(_type SHARED) + elseif(ARG_MODULE) + set(_type MODULE) + else() + message(FATAL_ERROR "Expected STATIC|SHARED|MODULE") + endif() + + add_library(${name} ${_type} ${ARG_UNPARSED_ARGUMENTS}) + if(type STREQUAL MODULE) + target_link_libraries(${name} PRIVATE Python3::Module) + # Follow Python rules for module names. + set_property(TARGET ${name} PROPERTY PREFIX "") + if(CMAKE_SYSTEM_NAME STREQUAL "Windows") + set_property (TARGET ${name} PROPERTY SUFFIX ".pyd") + endif() + # Add SOABI. + if(ARG_WITH_SOABI) + get_property(_suffix TARGET ${name} PROPERTY SUFFIX) + if (NOT _suffix) + set(_suffix "${CMAKE_SHARED_MODULE_SUFFIX}") + endif() + set_property(TARGET ${name} PROPERTY SUFFIX ".${Python3_SOABI}${suffix}") + endif() + endif() + endfunction() +endif() diff --git a/mlir/cmake/modules/query_python_module_config.py b/mlir/cmake/modules/query_python_module_config.py new file mode 100644 --- /dev/null +++ b/mlir/cmake/modules/query_python_module_config.py @@ -0,0 +1,159 @@ +# Queries the Python configuration and emits CMake statements that +# populate the following variables: +# +# Python3_Development.Module_FOUND +# Python3_INCLUDE_DIRS +# Python3_LIBRARIES +# Python3_LIBRARY_DIRS +# Python3_LINK_OPTIONS +# Python3_NumPy_FOUND +# Python3_NumPy_INCLUDE_DIRS +# Python3_NumPy_VERSION +# Python3_SOABI +# +# Will return a failure code if a functioning Python development environment +# is not found. Outputs hints and diagnostics to stderr. + +import os +import platform +import sys +from typing import List + +# Windows only has extension building information in distutils.sysconfig. +# Specifically, the EXT_SUFFIX in the distutils version will be complete +# and version qualified, and it will be empty in sysconfig. Both will +# work on the same host, but the former is needed for being compatible +# across versions. +# Why? No one knows. +import distutils +import sysconfig +try: + from distutils import sysconfig as distutils_sysconfig +except ModuleNotFoundError: + distutils_sysconfig = sysconfig + + +def _sanitize_dirs(dirs: List[str]) -> List[str]: + """Remove duplicates and empties.""" + new_dirs = [] + for d in sorted(dirs): + if not d or d in new_dirs: continue + new_dirs.append(d) + return new_dirs + + +def _file_exists_in_dirs(filename: str, dirs: List[str]) -> bool: + """Determines if a file exists in a list of dirs.""" + for d in dirs: + if os.path.exists(os.path.join(d, filename)): return True + return False + + +def _indent_dirs(dirs: List[str]) -> str: + return " " + "\n ".join(dirs) + + +def _strip_shlib_suffix(s: str) -> str: + suffixes = [".pyd", ".so"] + for suffix in suffixes: + if s.endswith(suffix): + return s[:-len(suffix)] + return s + + +# See documentation for sysconfig.get_paths(). +config_paths = sysconfig.get_paths() + +# Include dirs. +include_dirs = [] +try: + include_dirs.append(config_paths["include"]) +except KeyError: + pass +try: + include_dirs.append(config_paths["platinclude"]) +except KeyError: + pass +include_dirs = _sanitize_dirs(include_dirs) +if not _file_exists_in_dirs("Python.h", include_dirs): + print( + f"ERROR: Could not find Python.h in the following include " + f"directories for the Python interpreter {sys.executable}. " + f"This most likely means that this Python instance does not have " + f"development files installed. Check your OS package manager " + f"for a corresponding dev package:\n" + f" {_indent_dirs(include_dirs)}", + file=sys.stderr) + sys.exit(1) +print(f"SET(Python3_INCLUDE_DIRS \"{';'.join(include_dirs)}\")") + +# Library dirs. +# It is fine for these to be empty. But if we have a library to link +# against, we will search them. +library_dirs = [] +try: + library_dirs.append(config_paths["stdlib"]) +except KeyError: + pass +try: + library_dirs.append(config_paths["platstdlib"]) +except KeyError: + pass +library_dirs = _sanitize_dirs(library_dirs) +print(f"SET(Python3_LIBRARY_DIRS \"{';'.join(library_dirs)}\")") + +# SOABI. +# CMake standardized on the SOABI, which is something like +# 'cpython-37m-x86_64-linux-gnu' and is the string before the extension +# module suffix. This is available directly in Posix-y Pythons. But on +# Windows, we only get an EXT_SUFFIX like '.cp38-win_amd64.pyd'. On Linux, +# this is something like '.cpython-37m-x86_64-linux-gnu.so'. In order to +# keep the code paths more common (and less likely to diverge between +# platforms), we use EXT_SUFFIX exclusively and munge it. +soabi = _strip_shlib_suffix(distutils_sysconfig.get_config_var("EXT_SUFFIX")) +if soabi.startswith("."): soabi = soabi[1:] +print(f"SET(Python3_SOABI \"{soabi}\")") + +# Libraries. +# For extensions, we only link against libraries on Windows. See: +# https://docs.python.org/3/extending/windows.html +# We have to link against a 'pythonXY.lib' (where X = major version, and +# Y = minor version). This will typically be in the '%prefix%\libs' +# dir, and the version suffix can be found in %py_version_nodot% in the +# regular sysconfig. +if platform.system() == "Windows": + install_prefix = sysconfig.get_config_var("prefix") + version_nodot = sysconfig.get_config_var("py_version_nodot") + if not install_prefix or not version_nodot: + print( + f"ERROR: Could not determine Windows Python install location for " + f"{sys.executable}", + file=sys.stderr) + sys.exit(1) + libs_dir = os.path.join(install_prefix, "libs") + import_lib = os.path.join(libs_dir, f"python{version_nodot}.lib") + if not os.path.exists(import_lib): + print( + f"ERROR: Could not find Python import library for " + f"{sys.executable}:\n {import_lib}", + file=sys.stderr) + sys.exit(1) + print(f"SET(Python3_LIBRARIES \"{import_lib}\")") +else: + print(f"SET(Python3_LIBRARIES \"\")") + +# Find NumPy. +try: + import numpy + print(f"SET(Python3_NumPy_INCLUDE_DIRS \"{numpy.get_include()}\")") + print(f"SET(Python3_NumPy_VERSION \"{numpy.__version__}\")") + print(f"SET(Python3_NumPy_FOUND TRUE)") +except ModuleNotFoundError: + print( + f"ERROR: Numpy not found. Recommend installing with:\n" + f" {sys.executable} -m pip install numpy", + file=sys.stderr) + sys.exit(1) + +# Marker that everything was found. +print(f"SET(Python3_Development.Module_FOUND TRUE)")