Move MLIR python sources to mlir/python.

[stellaraccident]

Summary

• NFC but has some fixes for CMake glitches discovered along the way (things not cleaning properly, co-mingled depends).
• Includes previously unsubmitted fix in D98681 and a TODO to fix it more appropriately in a smaller followup.

[stellaraccident]

Fix install.

[stellaraccident]

Rebase.

[mehdi_amini]

Seems mechanical, LGTM on the principle.

[stellaraccident]

In D101493#2727002, @mehdi_amini wrote:

Seems mechanical, LGTM on the principle.

Yes, mostly mechanical with CMake fixups that became apparent when things were separated.

[ftynse]

Tests seem to fail for me locally

: 'RUN: at line 1';   /usr/bin/python3.9 /home/zinenko/llvm-project/mlir/test/python/ir/value.py | /home/zinenko/llvm-project/build2/bin/FileCheck /home/zinenko/llvm-project/mlir/test/python/ir/value.py
--
Exit Code: 2

Command Output (stderr):
--
Traceback (most recent call last):
  File "/home/zinenko/llvm-project/build2/python/mlir/__init__.py", line 23, in <module>
    import _mlir_libs
ModuleNotFoundError: No module named '_mlir_libs'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/zinenko/llvm-project/build2/python/mlir/_cext_loader.py", line 12, in <module>
    import _mlir_libs
ModuleNotFoundError: No module named '_mlir_libs'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/zinenko/llvm-project/mlir/test/python/ir/value.py", line 4, in <module>
    from mlir.ir import *
  File "/home/zinenko/llvm-project/build2/python/mlir/__init__.py", line 28, in <module>
    from ._dlloader import preload_dependency as _preload_dependency
  File "/home/zinenko/llvm-project/build2/python/mlir/_dlloader.py", line 6, in <module>
    import platform
  File "/usr/lib/python3.9/platform.py", line 117, in <module>
    import re
  File "/usr/lib/python3.9/re.py", line 124, in <module>
    import enum
  File "/usr/lib/python3.9/enum.py", line 2, in <module>
    from types import MappingProxyType, DynamicClassAttribute
  File "/home/zinenko/llvm-project/mlir/test/python/ir/types.py", line 4, in <module>
    from mlir.ir import *
  File "/home/zinenko/llvm-project/build2/python/mlir/ir.py", line 6, in <module>
    from ._cext_loader import _reexport_cext
  File "/home/zinenko/llvm-project/build2/python/mlir/_cext_loader.py", line 17, in <module>
    from ._dlloader import preload_dependency as _preload_dependency
ImportError: cannot import name 'preload_dependency' from partially initialized module 'mlir._dlloader' (most likely due to a circular import) (/home/zinenko/llvm-project/build2/python/mlir/_dlloader.py)
FileCheck error: '<stdin>' is empty.
FileCheck command line:  /home/zinenko/llvm-project/build2/bin/FileCheck /home/zinenko/llvm-project/mlir/test/python/ir/value.py

--

********************
********************
Failed Tests (15):
  MLIR :: python/ir/affine_expr.py
  MLIR :: python/ir/affine_map.py
  MLIR :: python/ir/array_attributes.py
  MLIR :: python/ir/attributes.py
  MLIR :: python/ir/context_lifecycle.py
  MLIR :: python/ir/context_managers.py
  MLIR :: python/ir/debug.py
  MLIR :: python/ir/dialects.py
  MLIR :: python/ir/insertion_point.py
  MLIR :: python/ir/integer_set.py
  MLIR :: python/ir/location.py
  MLIR :: python/ir/module.py
  MLIR :: python/ir/operation.py
  MLIR :: python/ir/types.py
  MLIR :: python/ir/value.py

[stellaraccident]

In D101493#2727037, @ftynse wrote:

Tests seem to fail for me locally

: 'RUN: at line 1';   /usr/bin/python3.9 /home/zinenko/llvm-project/mlir/test/python/ir/value.py | /home/zinenko/llvm-project/build2/bin/FileCheck /home/zinenko/llvm-project/mlir/test/python/ir/value.py
--
Exit Code: 2

Command Output (stderr):
--
Traceback (most recent call last):
  File "/home/zinenko/llvm-project/build2/python/mlir/__init__.py", line 23, in <module>
    import _mlir_libs
ModuleNotFoundError: No module named '_mlir_libs'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/zinenko/llvm-project/build2/python/mlir/_cext_loader.py", line 12, in <module>
    import _mlir_libs
ModuleNotFoundError: No module named '_mlir_libs'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/zinenko/llvm-project/mlir/test/python/ir/value.py", line 4, in <module>
    from mlir.ir import *
  File "/home/zinenko/llvm-project/build2/python/mlir/__init__.py", line 28, in <module>
    from ._dlloader import preload_dependency as _preload_dependency
  File "/home/zinenko/llvm-project/build2/python/mlir/_dlloader.py", line 6, in <module>
    import platform
  File "/usr/lib/python3.9/platform.py", line 117, in <module>
    import re
  File "/usr/lib/python3.9/re.py", line 124, in <module>
    import enum
  File "/usr/lib/python3.9/enum.py", line 2, in <module>
    from types import MappingProxyType, DynamicClassAttribute
  File "/home/zinenko/llvm-project/mlir/test/python/ir/types.py", line 4, in <module>
    from mlir.ir import *
  File "/home/zinenko/llvm-project/build2/python/mlir/ir.py", line 6, in <module>
    from ._cext_loader import _reexport_cext
  File "/home/zinenko/llvm-project/build2/python/mlir/_cext_loader.py", line 17, in <module>
    from ._dlloader import preload_dependency as _preload_dependency
ImportError: cannot import name 'preload_dependency' from partially initialized module 'mlir._dlloader' (most likely due to a circular import) (/home/zinenko/llvm-project/build2/python/mlir/_dlloader.py)
FileCheck error: '<stdin>' is empty.
FileCheck command line:  /home/zinenko/llvm-project/build2/bin/FileCheck /home/zinenko/llvm-project/mlir/test/python/ir/value.py

--

********************
********************
Failed Tests (15):
  MLIR :: python/ir/affine_expr.py
  MLIR :: python/ir/affine_map.py
  MLIR :: python/ir/array_attributes.py
  MLIR :: python/ir/attributes.py
  MLIR :: python/ir/context_lifecycle.py
  MLIR :: python/ir/context_managers.py
  MLIR :: python/ir/debug.py
  MLIR :: python/ir/dialects.py
  MLIR :: python/ir/insertion_point.py
  MLIR :: python/ir/integer_set.py
  MLIR :: python/ir/location.py
  MLIR :: python/ir/module.py
  MLIR :: python/ir/operation.py
  MLIR :: python/ir/types.py
  MLIR :: python/ir/value.py

Tried to ping you on discord. Included here as well:

Regarding https://reviews.llvm.org/D101493, were you starting from a pre-existing build directory? (I had blown mine away and ninja check-mlir from scratch works)
Just verified again from a fresh build and tests pass.
Notably: this file should not exist (it was deleted quite some time ago): /home/zinenko/llvm-project/build2/python/mlir/__init__.py

[ftynse]

Tried to ping you on discord. Included here as well:

Sorry, I don't follow discord. Any synchronous discussion media basically shuts me off because of nine-hour time zone difference with most of the people. By the time I see the message, the discussion has usually ended or diverged and any reaction to previous messages would be largely off topic.

Regarding https://reviews.llvm.org/D101493, were you starting from a pre-existing build directory?

Yes, I was starting from a pre-existing build directory, but there seems to be another problem even with a clean directory.

FAIL: MLIR :: python/ir/value.py (766 of 954)
******************** TEST 'MLIR :: python/ir/value.py' FAILED ********************
Script:
--
: 'RUN: at line 1';   /usr/bin/python3.9 /home/zinenko/llvm-project/mlir/test/python/ir/value.py | /home/zinenko/llvm-project/build/bin/FileCheck /home/zinenko/llvm-project/mlir/test/python/ir/value.py
--
Exit Code: 1

Command Output (stderr):
--
Traceback (most recent call last):
  File "/home/zinenko/llvm-project/build/python/mlir/_cext_loader.py", line 12, in <module>
    import _mlir_libs
ModuleNotFoundError: No module named '_mlir_libs'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/zinenko/llvm-project/mlir/test/python/ir/value.py", line 4, in <module>
    from mlir.ir import *
  File "/home/zinenko/llvm-project/build/python/mlir/ir.py", line 6, in <module>
    from ._cext_loader import _reexport_cext
  File "/home/zinenko/llvm-project/build/python/mlir/_cext_loader.py", line 17, in <module>
    from ._dlloader import preload_dependency as _preload_dependency
  File "/home/zinenko/llvm-project/build/python/mlir/_dlloader.py", line 6, in <module>
    import platform
  File "/usr/lib/python3.9/platform.py", line 117, in <module>
    import re
  File "/usr/lib/python3.9/re.py", line 124, in <module>
    import enum
  File "/usr/lib/python3.9/enum.py", line 2, in <module>
    from types import MappingProxyType, DynamicClassAttribute
  File "/home/zinenko/llvm-project/mlir/test/python/ir/types.py", line 25, in <module>
    run(testParsePrint)
  File "/home/zinenko/llvm-project/mlir/test/python/ir/types.py", line 8, in run
    f()
  File "/home/zinenko/llvm-project/mlir/test/python/ir/types.py", line 15, in testParsePrint
    ctx = Context()
NameError: name 'Context' is not defined

[ftynse]

This particular problem seems to come from the name clash between the types.py test and the types.py distributed with Python. It's strange that you don't hit the problem.

[bondhugula]

I think you are missing an update to docs/Bindings/Python.md? (The paths there have to change.)

[stellaraccident]

Address comments and rebase.

[stellaraccident]

In D101493#2731131, @bondhugula wrote:

I think you are missing an update to docs/Bindings/Python.md? (The paths there have to change.)

Updated.

[stellaraccident]

In D101493#2728298, @ftynse wrote:

This particular problem seems to come from the name clash between the types.py test and the types.py distributed with Python. It's strange that you don't hit the problem.

Strange. In lieu of debugging, I just renamed that to builtin_types.py.

[stellaraccident]

Alex: Made a change that might appease your machine. Let me know.

[ftynse]

I tried that myself before with renaming (back) to ir_types and it worked, so it's okay.

[bondhugula]

Thanks for updating the docs!

mlir/CMakeLists.txt

else()
     message(WARNING "gtest not found, unittests will not be available")
   endif()
   add_subdirectory(test)
 endif()
 # Tools needs to come late to ensure that MLIR_ALL_LIBS is populated.
 # Generally things after this point may depend on MLIR_ALL_LIBS or libMLIR.so.
 add_subdirectory(tools)
 
+if(MLIR_BINDINGS_PYTHON_ENABLED)
+  # Python sources: built extensions come in via lib/Bindings/Python
+  add_subdirectory(python)
+endif()
+
 if( LLVM_INCLUDE_EXAMPLES )
   add_subdirectory(examples)
 endif()
 
 option(MLIR_INCLUDE_DOCS "Generate build targets for the MLIR docs."
   ${LLVM_INCLUDE_DOCS})
 if (MLIR_INCLUDE_DOCS)
   add_subdirectory(docs)

mlir/cmake/modules/AddMLIRPython.cmake

add_public_tablegen_target(
     ${tblgen_target})
   if(ARG_DEPENDS)
     add_dependencies(${tblgen_target} ${ARG_DEPENDS})
   endif()
 
   add_custom_command(
     TARGET ${tblgen_target} POST_BUILD
     COMMENT "Copying generated python source \"dialects/${dialect_filename}\""
+    BYPRODUCTS "${PROJECT_BINARY_DIR}/python/mlir/dialects/${dialect_filename}"
     COMMAND "${CMAKE_COMMAND}" -E copy_if_different
       "${CMAKE_CURRENT_BINARY_DIR}/${dialect_filename}"
       "${PROJECT_BINARY_DIR}/python/mlir/dialects/${dialect_filename}")
 endfunction()
 

mlir/cmake/modules/CMakeLists.txt

if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY)
   if(mlir_has_exports)
     install(EXPORT MLIRTargets DESTINATION ${MLIR_INSTALL_PACKAGE_DIR}
             COMPONENT mlir-cmake-exports)
   endif()
 
   install(FILES
     ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/MLIRConfig.cmake
     ${CMAKE_CURRENT_SOURCE_DIR}/AddMLIR.cmake
-    ${CMAKE_CURRENT_SOURCE_DIR}/AddMLIRPythonExtension.cmake
+    ${CMAKE_CURRENT_SOURCE_DIR}/AddMLIRPython.cmake
     ${CMAKE_CURRENT_SOURCE_DIR}/MLIRDetectPythonEnv.cmake
     DESTINATION ${MLIR_INSTALL_PACKAGE_DIR}
     COMPONENT mlir-cmake-exports)
 
   if(NOT LLVM_ENABLE_IDE)
     # Add a dummy target so this can be used with LLVM_DISTRIBUTION_COMPONENTS
     add_custom_target(mlir-cmake-exports)
     add_llvm_install_targets(install-mlir-cmake-exports
                              COMPONENT mlir-cmake-exports)
   endif()
 endif()

mlir/docs/Bindings/Python.md

 # MLIR Python Bindings
 
 Current status: Under development and not enabled by default
 
 ## Building
 
 ### Pre-requisites
 
 * A relatively recent Python3 installation
 * Installation of python dependencies as specified in
-  `mlir/lib/Bindings/Python/requirements.txt`
+  `mlir/python/requirements.txt`
 
 ### CMake variables
 
 * **`MLIR_BINDINGS_PYTHON_ENABLED`**`:BOOL`
 
   Enables building the Python bindings. Defaults to `OFF`.
 
 * **`Python3_EXECUTABLE`**:`STRING`

 # Linux specifically, this should be cross checked with minimum versions
 # here: https://github.com/pypa/manylinux
 # It is recommended to upgrade pip:
 python -m pip install --upgrade pip
 
 
 # Now the `python` command will resolve to your virtual environment and
 # packages will be installed there.
-python -m pip install -r mlir/lib/Bindings/Python/requirements.txt
+python -m pip install -r mlir/python/requirements.txt
 
 # Now run `cmake`, `ninja`, et al.
 ```
 
 For interactive use, it is sufficient to add the `python` directory in your
 `build/` directory to the `PYTHONPATH`. Typically:
 
 ```shell

mlir/lib/Bindings/Python/CMakeLists.txt

-include(AddMLIRPythonExtension)
+include(AddMLIRPython)
 add_custom_target(MLIRBindingsPythonExtension)
 
 ################################################################################
-# Copy python source tree.
-################################################################################
-
-file(GLOB_RECURSE PY_SRC_FILES
-  RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
-  "${CMAKE_CURRENT_SOURCE_DIR}/mlir/*.py")
-
-add_custom_target(MLIRBindingsPythonSources ALL
-  DEPENDS ${PY_SRC_FILES}
-)
-add_dependencies(MLIRBindingsPythonExtension MLIRBindingsPythonSources)
-
-foreach(PY_SRC_FILE ${PY_SRC_FILES})
-  set(PY_DEST_FILE "${PROJECT_BINARY_DIR}/python/${PY_SRC_FILE}")
-  get_filename_component(PY_DEST_DIR "${PY_DEST_FILE}" DIRECTORY)
-  file(MAKE_DIRECTORY "${PY_DEST_DIR}")
-  add_custom_command(
-    TARGET MLIRBindingsPythonSources PRE_BUILD
-    COMMENT "Copying python source ${PY_SRC_FILE} -> ${PY_DEST_FILE}"
-    DEPENDS "${PY_SRC_FILE}"
-    COMMAND "${CMAKE_COMMAND}" -E create_symlink
-        "${CMAKE_CURRENT_SOURCE_DIR}/${PY_SRC_FILE}" "${PY_DEST_FILE}"
-  )
-endforeach()
-
-################################################################################
-# Generate dialect-specific bindings.
-################################################################################
-
-add_mlir_dialect_python_bindings(MLIRBindingsPythonAsyncOps
-  TD_FILE AsyncOps.td
-  DIALECT_NAME async_dialect)
-add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonAsyncOps)
-
-add_mlir_dialect_python_bindings(MLIRBindingsPythonBuiltinOps
-  TD_FILE BuiltinOps.td
-  DIALECT_NAME builtin)
-add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonBuiltinOps)
-
-add_mlir_dialect_python_bindings(MLIRBindingsPythonGPUOps
-  TD_FILE GPUOps.td
-  DIALECT_NAME gpu)
-add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonGPUOps)
-
-add_mlir_dialect_python_bindings(MLIRBindingsPythonLinalgOps
-  TD_FILE LinalgOps.td
-  DIALECT_NAME linalg
-  DEPENDS LinalgOdsGen)
-add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonLinalgOps)
-
-add_mlir_dialect_python_bindings(MLIRBindingsPythonMemRefOps
-  TD_FILE MemRefOps.td
-  DIALECT_NAME memref)
-add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonMemRefOps)
-
-add_mlir_dialect_python_bindings(MLIRBindingsPythonShapeOps
-  TD_FILE ShapeOps.td
-  DIALECT_NAME shape)
-add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonShapeOps)
-
-add_mlir_dialect_python_bindings(MLIRBindingsPythonStandardOps
-  TD_FILE StandardOps.td
-  DIALECT_NAME std)
-add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonStandardOps)
-
-add_mlir_dialect_python_bindings(MLIRBindingsPythonTensorOps
-  TD_FILE TensorOps.td
-  DIALECT_NAME tensor)
-add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonTensorOps)
-
-################################################################################
 # Build core python extension
 ################################################################################
 add_mlir_python_extension(MLIRCoreBindingsPythonExtension _mlir
   INSTALL_DIR
     python
   SOURCES
     DialectLinalg.cpp
     MainModule.cpp
     IRAffine.cpp
     IRAttributes.cpp
     IRCore.cpp
     IRModule.cpp
     IRTypes.cpp
     PybindUtils.cpp
     Pass.cpp
     ExecutionEngine.cpp
 )
 add_dependencies(MLIRBindingsPythonExtension MLIRCoreBindingsPythonExtension)
 
-# Note that we copy from the source tree just like for headers because
-# it will not be polluted with py_cache runtime artifacts (from testing and
-# such).
-install(
-  DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/mlir
-  DESTINATION python
-  COMPONENT MLIRBindingsPythonSources
-  FILES_MATCHING PATTERN "*.py"
-)
-
-if (NOT LLVM_ENABLE_IDE)
-  add_llvm_install_targets(
-    install-MLIRBindingsPythonSources
-    DEPENDS MLIRBindingsPythonSources
-    COMPONENT MLIRBindingsPythonSources)
-endif()
-
-# Dialect sources are generated. Install separately.
-# Note that __pycache__ directories may have been left by tests and other
-# executions. And __init__.py is handled as a regular source file.
-install(
-  DIRECTORY ${PROJECT_BINARY_DIR}/python/mlir/dialects
-  DESTINATION python/mlir
-  COMPONENT MLIRBindingsPythonDialects
-  FILES_MATCHING PATTERN "*.py"
-  PATTERN "__pycache__" EXCLUDE
-  PATTERN "__init__.py" EXCLUDE
-)
-
-if (NOT LLVM_ENABLE_IDE)
-  add_llvm_install_targets(
-    install-MLIRBindingsPythonDialects
-    DEPENDS MLIRBindingsPythonSources
-    COMPONENT MLIRBindingsPythonDialects)
-endif()
-
 add_subdirectory(Transforms)
 add_subdirectory(Conversions)
 
 add_mlir_python_extension(MLIRAsyncPassesBindingsPythonExtension _mlirAsyncPasses
   INSTALL_DIR
     python
   SOURCES
     AsyncPasses.cpp

mlir/python/CMakeLists.txt

+################################################################################
+# Copy python source tree.
+################################################################################
+
+file(GLOB_RECURSE PY_SRC_FILES
+  RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
+  "${CMAKE_CURRENT_SOURCE_DIR}/mlir/*.py")
+
+add_custom_target(MLIRBindingsPythonSources ALL
+  DEPENDS
+    ${PY_SRC_FILES}
+)
+
+foreach(PY_SRC_FILE ${PY_SRC_FILES})
+  set(PY_DEST_FILE "${PROJECT_BINARY_DIR}/python/${PY_SRC_FILE}")
+  get_filename_component(PY_DEST_DIR "${PY_DEST_FILE}" DIRECTORY)
+  file(MAKE_DIRECTORY "${PY_DEST_DIR}")
+  add_custom_command(
+    TARGET MLIRBindingsPythonSources PRE_BUILD
+    COMMENT "Copying python source ${PY_SRC_FILE} -> ${PY_DEST_FILE}"
+    DEPENDS "${PY_SRC_FILE}"
+    BYPRODUCTS "${PY_DEST_FILE}"
+    COMMAND "${CMAKE_COMMAND}" -E create_symlink
+        "${CMAKE_CURRENT_SOURCE_DIR}/${PY_SRC_FILE}" "${PY_DEST_FILE}"
+  )
+endforeach()
+
+# Note that we copy from the source tree just like for headers because
+# it will not be polluted with py_cache runtime artifacts (from testing and
+# such).
+install(
+  DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/mlir
+  DESTINATION python
+  COMPONENT MLIRBindingsPythonSources
+  FILES_MATCHING PATTERN "*.py"
+)
+
+if (NOT LLVM_ENABLE_IDE)
+  add_llvm_install_targets(
+    install-MLIRBindingsPythonSources
+    DEPENDS MLIRBindingsPythonSources
+    COMPONENT MLIRBindingsPythonSources)
+endif()
+
+################################################################################
+# Generated sources.
+################################################################################
+
+add_subdirectory(mlir/dialects)

mlir/python/mlir/dialects/CMakeLists.txt

+include(AddMLIRPython)
+
+################################################################################
+# Generate dialect-specific bindings.
+################################################################################
+
+add_mlir_dialect_python_bindings(MLIRBindingsPythonAsyncOps
+  TD_FILE AsyncOps.td
+  DIALECT_NAME async_dialect)
+add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonAsyncOps)
+
+add_mlir_dialect_python_bindings(MLIRBindingsPythonBuiltinOps
+  TD_FILE BuiltinOps.td
+  DIALECT_NAME builtin)
+add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonBuiltinOps)
+
+add_mlir_dialect_python_bindings(MLIRBindingsPythonGPUOps
+  TD_FILE GPUOps.td
+  DIALECT_NAME gpu)
+add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonGPUOps)
+
+add_mlir_dialect_python_bindings(MLIRBindingsPythonLinalgOps
+  TD_FILE LinalgOps.td
+  DIALECT_NAME linalg
+  DEPENDS LinalgOdsGen)
+add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonLinalgOps)
+
+add_mlir_dialect_python_bindings(MLIRBindingsPythonMemRefOps
+  TD_FILE MemRefOps.td
+  DIALECT_NAME memref)
+add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonMemRefOps)
+
+add_mlir_dialect_python_bindings(MLIRBindingsPythonShapeOps
+  TD_FILE ShapeOps.td
+  DIALECT_NAME shape)
+add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonShapeOps)
+
+add_mlir_dialect_python_bindings(MLIRBindingsPythonStandardOps
+  TD_FILE StandardOps.td
+  DIALECT_NAME std)
+add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonStandardOps)
+
+add_mlir_dialect_python_bindings(MLIRBindingsPythonTensorOps
+  TD_FILE TensorOps.td
+  DIALECT_NAME tensor)
+add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonTensorOps)
+
+################################################################################
+# Installation.
+################################################################################
+
+# Dialect sources are generated. Install separately.
+# Note that __pycache__ directories may have been left by tests and other
+# executions. And __init__.py is handled as a regular source file.
+# TODO: Eliminate this glob install, instead adding INSTALL_COMPONENT to
+# add_mlir_dialect_python_bindings and installing the precise file there.
+install(
+  DIRECTORY ${PROJECT_BINARY_DIR}/python/mlir/dialects
+  DESTINATION python/mlir
+  COMPONENT MLIRBindingsPythonDialects
+  FILES_MATCHING PATTERN "_*_gen.py"
+  PATTERN "__pycache__" EXCLUDE
+  PATTERN "__init__.py" EXCLUDE
+)
+
+if (NOT LLVM_ENABLE_IDE)
+  add_llvm_install_targets(
+    install-MLIRBindingsPythonDialects
+    DEPENDS MLIRBindingsPythonSources
+    COMPONENT MLIRBindingsPythonDialects)
+endif()

mlir/test/Bindings/CMakeLists.txt

-if(MLIR_BINDINGS_PYTHON_ENABLED)
-  add_subdirectory(Python)
-endif()

mlir/test/Bindings/Python/.style.yapf

-[style]
-  based_on_style = google
-  column_limit = 80
-  indent_width = 2

mlir/test/CMakeLists.txt

-add_subdirectory(Bindings)
 add_subdirectory(CAPI)
 add_subdirectory(EDSC)
 add_subdirectory(SDBM)
 add_subdirectory(lib)
 
+if(MLIR_BINDINGS_PYTHON_ENABLED)
+  add_subdirectory(python)
+endif()
+
 llvm_canonicalize_cmake_booleans(
   MLIR_BINDINGS_PYTHON_ENABLED
   LLVM_BUILD_EXAMPLES
   MLIR_CUDA_CONVERSIONS_ENABLED
   MLIR_CUDA_RUNNER_ENABLED
   MLIR_ROCM_CONVERSIONS_ENABLED
   MLIR_ROCM_RUNNER_ENABLED
   MLIR_SPIRV_CPU_RUNNER_ENABLED
if(MLIR_VULKAN_RUNNER_ENABLED)
   list(APPEND MLIR_TEST_DEPENDS
     mlir-vulkan-runner
   )
 endif()
 
 if(MLIR_BINDINGS_PYTHON_ENABLED)
   list(APPEND MLIR_TEST_DEPENDS
     MLIRBindingsPythonExtension
+    MLIRBindingsPythonSources
     MLIRBindingsPythonTestOps
     MLIRTransformsBindingsPythonExtension
     MLIRConversionsBindingsPythonExtension
   )
 endif()
 
 add_lit_testsuite(check-mlir "Running the MLIR regression tests"
   ${CMAKE_CURRENT_BINARY_DIR}
   DEPENDS ${MLIR_TEST_DEPENDS}
   )
 set_target_properties(check-mlir PROPERTIES FOLDER "Tests")
 
 add_lit_testsuites(MLIR ${CMAKE_CURRENT_SOURCE_DIR}
   DEPENDS ${MLIR_TEST_DEPENDS}
 )

mlir/test/python/CMakeLists.txt

-include(AddMLIRPythonExtension)
+include(AddMLIRPython)
+
 add_mlir_dialect_python_bindings(MLIRBindingsPythonTestOps
   TD_FILE python_test_ops.td
   DIALECT_NAME python_test)