[mlir][python] Makes C++ extension code relocatable by way of a macro.

[stellaraccident]

Summary

• Resolves a TODO by making this configurable by downstreams.
• This seems to be the last thing allowing full use of the Python bindings as a library within another project (i.e. be embedding them).

[stellaraccident]

Since just resolving an NFC todo, please post review.

[stellaraccident]

We appear to have some dueling clang-format diffs. Ignoring.

mlir/include/mlir-c/Bindings/Python/Interop.h

 
 #include "mlir-c/AffineExpr.h"
 #include "mlir-c/AffineMap.h"
 #include "mlir-c/ExecutionEngine.h"
 #include "mlir-c/IR.h"
 #include "mlir-c/IntegerSet.h"
 #include "mlir-c/Pass.h"
 
-#define MLIR_PYTHON_CAPSULE_AFFINE_EXPR "mlir.ir.AffineExpr._CAPIPtr"
-#define MLIR_PYTHON_CAPSULE_AFFINE_MAP "mlir.ir.AffineMap._CAPIPtr"
-#define MLIR_PYTHON_CAPSULE_ATTRIBUTE "mlir.ir.Attribute._CAPIPtr"
-#define MLIR_PYTHON_CAPSULE_CONTEXT "mlir.ir.Context._CAPIPtr"
+// The 'mlir' Python package is relocatable and supports co-existing in multiple
+// projects. Each project must define its outer package prefix with this define
+// in order to provide proper isolation and local name resolution.
+// The default is for the upstream "import mlir" package layout.
+// Note that this prefix is internally stringified, allowing it to be passed
+// unquoted on the compiler command line without shell quote escaping issues.
+#ifndef MLIR_PYTHON_PACKAGE_PREFIX
+#define MLIR_PYTHON_PACKAGE_PREFIX mlir.
+#endif
+
+// Makes a fully-qualified name relative to the MLIR python package.
+#define MLIR_PYTHON_STRINGIZE(s) #s
+#define MLIR_PYTHON_STRINGIZE_ARG(arg) MLIR_PYTHON_STRINGIZE(arg)
+#define MAKE_MLIR_PYTHON_QUALNAME(local)                                       \
+  MLIR_PYTHON_STRINGIZE_ARG(MLIR_PYTHON_PACKAGE_PREFIX) local
+
+#define MLIR_PYTHON_CAPSULE_AFFINE_EXPR                                        \
+  MAKE_MLIR_PYTHON_QUALNAME("ir.AffineExpr._CAPIPtr")
+#define MLIR_PYTHON_CAPSULE_AFFINE_MAP                                         \
+  MAKE_MLIR_PYTHON_QUALNAME("ir.AffineMap._CAPIPtr")
+#define MLIR_PYTHON_CAPSULE_ATTRIBUTE                                          \
+  MAKE_MLIR_PYTHON_QUALNAME("ir.Attribute._CAPIPtr")
+#define MLIR_PYTHON_CAPSULE_CONTEXT                                            \
+  MAKE_MLIR_PYTHON_QUALNAME("ir.Context._CAPIPtr")
 #define MLIR_PYTHON_CAPSULE_EXECUTION_ENGINE                                   \
-  "mlir.execution_engine.ExecutionEngine._CAPIPtr"
-#define MLIR_PYTHON_CAPSULE_INTEGER_SET "mlir.ir.IntegerSet._CAPIPtr"
-#define MLIR_PYTHON_CAPSULE_LOCATION "mlir.ir.Location._CAPIPtr"
-#define MLIR_PYTHON_CAPSULE_MODULE "mlir.ir.Module._CAPIPtr"
-#define MLIR_PYTHON_CAPSULE_OPERATION "mlir.ir.Operation._CAPIPtr"
-#define MLIR_PYTHON_CAPSULE_TYPE "mlir.ir.Type._CAPIPtr"
-#define MLIR_PYTHON_CAPSULE_PASS_MANAGER "mlir.passmanager.PassManager._CAPIPtr"
-#define MLIR_PYTHON_CAPSULE_VALUE "mlir.ir.Value._CAPIPtr"
+  MAKE_MLIR_PYTHON_QUALNAME("execution_engine.ExecutionEngine._CAPIPtr")
+#define MLIR_PYTHON_CAPSULE_INTEGER_SET                                        \
+  MAKE_MLIR_PYTHON_QUALNAME("ir.IntegerSet._CAPIPtr")
+#define MLIR_PYTHON_CAPSULE_LOCATION                                           \
+  MAKE_MLIR_PYTHON_QUALNAME("ir.Location._CAPIPtr")
+#define MLIR_PYTHON_CAPSULE_MODULE                                             \
+  MAKE_MLIR_PYTHON_QUALNAME("ir.Module._CAPIPtr")
+#define MLIR_PYTHON_CAPSULE_OPERATION                                          \
+  MAKE_MLIR_PYTHON_QUALNAME("ir.Operation._CAPIPtr")
+#define MLIR_PYTHON_CAPSULE_TYPE MAKE_MLIR_PYTHON_QUALNAME("ir.Type._CAPIPtr")
+#define MLIR_PYTHON_CAPSULE_PASS_MANAGER                                       \
+  MAKE_MLIR_PYTHON_QUALNAME("passmanager.PassManager._CAPIPtr")
+#define MLIR_PYTHON_CAPSULE_VALUE MAKE_MLIR_PYTHON_QUALNAME("ir.Value._CAPIPtr")
 
 /** Attribute on MLIR Python objects that expose their C-API pointer.
  * This will be a type-specific capsule created as per one of the helpers
  * below.
  *
  * Ownership is not transferred by acquiring a capsule in this way: the
  * validity of the pointer wrapped by the capsule will be bounded by the
  * lifetime of the Python object that produced it. Only the name and pointer

mlir/include/mlir/Bindings/Python/PybindAdaptors.h

 #include "mlir-c/Bindings/Python/Interop.h"
 #include "mlir-c/IR.h"
 
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/Twine.h"
 
 namespace py = pybind11;
 
-// TODO: Move this to Interop.h and make it externally configurable/use it
-// consistently to locate the "import mlir" top-level.
-#define MLIR_PYTHON_PACKAGE_PREFIX "mlir."
-
 // Raw CAPI type casters need to be declared before use, so always include them
 // first.
 namespace pybind11 {
 namespace detail {
 
 template <typename T>
 struct type_caster<llvm::Optional<T>> : optional_caster<llvm::Optional<T>> {};
 
bool load(handle src, bool) {
     if (mlirAffineMapIsNull(value)) {
       return false;
     }
     return !mlirAffineMapIsNull(value);
   }
   static handle cast(MlirAffineMap v, return_value_policy, handle) {
     py::object capsule =
         py::reinterpret_steal<py::object>(mlirPythonAffineMapToCapsule(v));
-    return py::module::import(MLIR_PYTHON_PACKAGE_PREFIX "ir")
+    return py::module::import(MAKE_MLIR_PYTHON_QUALNAME("ir"))
         .attr("AffineMap")
         .attr(MLIR_PYTHON_CAPI_FACTORY_ATTR)(capsule)
         .release();
   }
 };
 
 /// Casts object <-> MlirAttribute.
 template <>
 struct type_caster<MlirAttribute> {
   PYBIND11_TYPE_CASTER(MlirAttribute, _("MlirAttribute"));
   bool load(handle src, bool) {
     py::object capsule = mlirApiObjectToCapsule(src);
     value = mlirPythonCapsuleToAttribute(capsule.ptr());
     if (mlirAttributeIsNull(value)) {
       return false;
     }
     return true;
   }
   static handle cast(MlirAttribute v, return_value_policy, handle) {
     py::object capsule =
         py::reinterpret_steal<py::object>(mlirPythonAttributeToCapsule(v));
-    return py::module::import(MLIR_PYTHON_PACKAGE_PREFIX "ir")
+    return py::module::import(MAKE_MLIR_PYTHON_QUALNAME("ir"))
         .attr("Attribute")
         .attr(MLIR_PYTHON_CAPI_FACTORY_ATTR)(capsule)
         .release();
   }
 };
 
 /// Casts object -> MlirContext.
 template <>
 struct type_caster<MlirContext> {
   PYBIND11_TYPE_CASTER(MlirContext, _("MlirContext"));
   bool load(handle src, bool) {
     if (src.is_none()) {
       // Gets the current thread-bound context.
       // TODO: This raises an error of "No current context" currently.
       // Update the implementation to pretty-print the helpful error that the
       // core implementations print in this case.
-      src = py::module::import(MLIR_PYTHON_PACKAGE_PREFIX "ir")
+      src = py::module::import(MAKE_MLIR_PYTHON_QUALNAME("ir"))
                 .attr("Context")
                 .attr("current");
     }
     py::object capsule = mlirApiObjectToCapsule(src);
     value = mlirPythonCapsuleToContext(capsule.ptr());
     if (mlirContextIsNull(value)) {
       return false;
     }
bool load(handle src, bool) {
     if (mlirLocationIsNull(value)) {
       return false;
     }
     return true;
   }
   static handle cast(MlirLocation v, return_value_policy, handle) {
     py::object capsule =
         py::reinterpret_steal<py::object>(mlirPythonLocationToCapsule(v));
-    return py::module::import(MLIR_PYTHON_PACKAGE_PREFIX "ir")
+    return py::module::import(MAKE_MLIR_PYTHON_QUALNAME("ir"))
         .attr("Location")
         .attr(MLIR_PYTHON_CAPI_FACTORY_ATTR)(capsule)
         .release();
   }
 };
 
 /// Casts object <-> MlirModule.
 template <>
 struct type_caster<MlirModule> {
   PYBIND11_TYPE_CASTER(MlirModule, _("MlirModule"));
   bool load(handle src, bool) {
     py::object capsule = mlirApiObjectToCapsule(src);
     value = mlirPythonCapsuleToModule(capsule.ptr());
     if (mlirModuleIsNull(value)) {
       return false;
     }
     return true;
   }
   static handle cast(MlirModule v, return_value_policy, handle) {
     py::object capsule =
         py::reinterpret_steal<py::object>(mlirPythonModuleToCapsule(v));
-    return py::module::import(MLIR_PYTHON_PACKAGE_PREFIX "ir")
+    return py::module::import(MAKE_MLIR_PYTHON_QUALNAME("ir"))
         .attr("Module")
         .attr(MLIR_PYTHON_CAPI_FACTORY_ATTR)(capsule)
         .release();
   };
 };
 
 /// Casts object <-> MlirOperation.
 template <>
 struct type_caster<MlirOperation> {
   PYBIND11_TYPE_CASTER(MlirOperation, _("MlirOperation"));
   bool load(handle src, bool) {
     py::object capsule = mlirApiObjectToCapsule(src);
     value = mlirPythonCapsuleToOperation(capsule.ptr());
     if (mlirOperationIsNull(value)) {
       return false;
     }
     return true;
   }
   static handle cast(MlirOperation v, return_value_policy, handle) {
     if (v.ptr == nullptr)
       return py::none();
     py::object capsule =
         py::reinterpret_steal<py::object>(mlirPythonOperationToCapsule(v));
-    return py::module::import(MLIR_PYTHON_PACKAGE_PREFIX "ir")
+    return py::module::import(MAKE_MLIR_PYTHON_QUALNAME("ir"))
         .attr("Operation")
         .attr(MLIR_PYTHON_CAPI_FACTORY_ATTR)(capsule)
         .release();
   };
 };
 
 /// Casts object -> MlirPassManager.
 template <>
bool load(handle src, bool) {
     if (mlirTypeIsNull(value)) {
       return false;
     }
     return true;
   }
   static handle cast(MlirType t, return_value_policy, handle) {
     py::object capsule =
         py::reinterpret_steal<py::object>(mlirPythonTypeToCapsule(t));
-    return py::module::import(MLIR_PYTHON_PACKAGE_PREFIX "ir")
+    return py::module::import(MAKE_MLIR_PYTHON_QUALNAME("ir"))
         .attr("Type")
         .attr(MLIR_PYTHON_CAPI_FACTORY_ATTR)(capsule)
         .release();
   }
 };
 
 } // namespace detail
 } // namespace pybind11
pure_subclass(py::handle scope, const char *derivedClassName,
     py::dict attributes;
 
     thisClass =
         metaclass(derivedClassName, py::make_tuple(superClass), attributes);
     scope.attr(derivedClassName) = thisClass;
   }
 
   template <typename Func, typename... Extra>
-  pure_subclass &def(const char *name, Func &&f, const Extra &...extra) {
+  pure_subclass &def(const char *name, Func &&f, const Extra &... extra) {
     py::cpp_function cf(
         std::forward<Func>(f), py::name(name), py::is_method(py::none()),
         py::sibling(py::getattr(thisClass, name, py::none())), extra...);
     thisClass.attr(cf.name()) = cf;
     return *this;
   }
 
   template <typename Func, typename... Extra>
   pure_subclass &def_property_readonly(const char *name, Func &&f,
-                                       const Extra &...extra) {
+                                       const Extra &... extra) {
     py::cpp_function cf(
         std::forward<Func>(f), py::name(name), py::is_method(py::none()),
         py::sibling(py::getattr(thisClass, name, py::none())), extra...);
     auto builtinProperty =
         py::reinterpret_borrow<py::object>((PyObject *)&PyProperty_Type);
     thisClass.attr(name) = builtinProperty(cf);
     return *this;
   }
 
   template <typename Func, typename... Extra>
   pure_subclass &def_staticmethod(const char *name, Func &&f,
-                                  const Extra &...extra) {
+                                  const Extra &... extra) {
     static_assert(!std::is_member_function_pointer<Func>::value,
                   "def_staticmethod(...) called with a non-static member "
                   "function pointer");
     py::cpp_function cf(
         std::forward<Func>(f), py::name(name), py::scope(thisClass),
         py::sibling(py::getattr(thisClass, name, py::none())), extra...);
     thisClass.attr(cf.name()) = py::staticmethod(cf);
     return *this;
   }
 
   template <typename Func, typename... Extra>
   pure_subclass &def_classmethod(const char *name, Func &&f,
-                                 const Extra &...extra) {
+                                 const Extra &... extra) {
     static_assert(!std::is_member_function_pointer<Func>::value,
                   "def_classmethod(...) called with a non-static member "
                   "function pointer");
     py::cpp_function cf(
         std::forward<Func>(f), py::name(name), py::scope(thisClass),
         py::sibling(py::getattr(thisClass, name, py::none())), extra...);
     thisClass.attr(cf.name()) =
         py::reinterpret_borrow<py::object>(PyClassMethod_New(cf.ptr()));

 public:
   using IsAFunctionTy = bool (*)(MlirAttribute);
 
   /// Subclasses by looking up the super-class dynamically.
   mlir_attribute_subclass(py::handle scope, const char *attrClassName,
                           IsAFunctionTy isaFunction)
       : mlir_attribute_subclass(
             scope, attrClassName, isaFunction,
-            py::module::import(MLIR_PYTHON_PACKAGE_PREFIX "ir")
+            py::module::import(MAKE_MLIR_PYTHON_QUALNAME("ir"))
                 .attr("Attribute")) {}
 
   /// Subclasses with a provided mlir.ir.Attribute super-class. This must
   /// be used if the subclass is being defined in the same extension module
   /// as the mlir.ir class (otherwise, it will trigger a recursive
   /// initialization).
   mlir_attribute_subclass(py::handle scope, const char *typeClassName,
                           IsAFunctionTy isaFunction, py::object superClass)

 public:
   using IsAFunctionTy = bool (*)(MlirType);
 
   /// Subclasses by looking up the super-class dynamically.
   mlir_type_subclass(py::handle scope, const char *typeClassName,
                      IsAFunctionTy isaFunction)
       : mlir_type_subclass(
             scope, typeClassName, isaFunction,
-            py::module::import(MLIR_PYTHON_PACKAGE_PREFIX "ir").attr("Type")) {}
+            py::module::import(MAKE_MLIR_PYTHON_QUALNAME("ir")).attr("Type")) {}
 
   /// Subclasses with a provided mlir.ir.Type super-class. This must
   /// be used if the subclass is being defined in the same extension module
   /// as the mlir.ir class (otherwise, it will trigger a recursive
   /// initialization).
   mlir_type_subclass(py::handle scope, const char *typeClassName,
                      IsAFunctionTy isaFunction, py::object superClass)
       : pure_subclass(scope, typeClassName, superClass) {