refactor(bindings/python): use declarative pymodule syntax

bindings/python/src/lib.rs

@@ -42,73 +42,102 @@ mod services;
 use pyo3_stub_gen::{define_stub_info_gatherer, derive::*};
 pub use services::*;
 
+// The `opendal` Python package is a mixed Python/native layout: the public
+// package is the hand-written `python/opendal/` (with its own `__init__.py`),
+// and this native module is built as the private submodule `opendal._opendal`
+// (`module-name` in pyproject.toml). The `#[pymodule] mod` name must therefore
+// be `_opendal` to match the compiled `opendal/_opendal.*.so` (its `PyInit`
+// symbol); `__init__.py` imports and re-exports from it as the public API.
 #[pymodule(gil_used = false)]
-fn _opendal(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
-    // Add version
-    m.add("__version__", env!("CARGO_PKG_VERSION"))?;
-
-    // Operator module
-    add_pymodule!(py, m, "operator", [Operator, AsyncOperator])?;
-
-    // File module
-    add_pymodule!(py, m, "file", [File, AsyncFile])?;
-
-    // Capability module
-    add_pymodule!(py, m, "capability", [Capability])?;
-
-    // Services module
-    add_pymodule!(py, m, "services", [PyScheme])?;
-
-    // Layers module
-    add_pymodule!(
-        py,
-        m,
-        "layers",
-        [
-            Layer,
-            CapabilityOverrideLayer,
-            RetryLayer,
-            ConcurrentLimitLayer,
-            MimeGuessLayer
-        ]
-    )?;
-
-    // Types module
-    add_pymodule!(
-        py,
-        m,
-        "types",
-        [Entry, EntryMode, Metadata, PresignedRequest]
-    )?;
-
-    m.add_class::<WriteOptions>()?;
-    m.add_class::<ReadOptions>()?;
-    m.add_class::<ListOptions>()?;
-    m.add_class::<StatOptions>()?;
-    m.add_class::<DeleteOptions>()?;
-
-    // Exceptions module
-    add_pyexceptions!(
-        py,
-        m,
-        "exceptions",
-        [
-            Error,
-            Unexpected,
-            Unsupported,
-            ConfigInvalid,
-            NotFound,
-            PermissionDenied,
-            IsADirectory,
-            NotADirectory,
-            AlreadyExists,
-            IsSameFile,
-            ConditionNotMatch,
-            RateLimited,
-            RangeNotSatisfied,
-        ]
-    )?;
-    Ok(())
+mod _opendal {
+    use pyo3::prelude::*;
+
+    #[pymodule]
+    mod operator {
+        #[pymodule_export]
+        use crate::{AsyncOperator, Operator};
+    }
+
+    #[pymodule]
+    mod file {
+        #[pymodule_export]
+        use crate::{AsyncFile, File};
+    }
+
+    #[pymodule]
+    mod capability {
+        #[pymodule_export]
+        use crate::Capability;
+    }
+
+    #[pymodule]
+    mod services {
+        #[pymodule_export]
+        use crate::PyScheme;
+    }
+
+    #[pymodule]
+    mod layers {
+        #[pymodule_export]
+        use crate::{
+            CapabilityOverrideLayer, ConcurrentLimitLayer, Layer, MimeGuessLayer, RetryLayer,
+        };
+    }
+
+    #[pymodule]
+    mod types {
+        #[pymodule_export]
+        use crate::{Entry, EntryMode, Metadata, PresignedRequest};
+    }
+
+    // Exceptions are `PyErr` subtypes, not `#[pyclass]`es, so they are added
+    // procedurally rather than via `#[pymodule_export]`.
+    #[pymodule]
+    mod exceptions {
+        #[pymodule_init]
+        fn init(m: &pyo3::Bound<'_, pyo3::types::PyModule>) -> pyo3::PyResult<()> {
+            use pyo3::prelude::*;
+
+            use crate::{
+                AlreadyExists, ConditionNotMatch, ConfigInvalid, Error, IsADirectory, IsSameFile,
+                NotADirectory, NotFound, PermissionDenied, RangeNotSatisfied, RateLimited,
+                Unexpected, Unsupported, add_exceptions,
+            };
+
+            add_exceptions!(
+                m,
+                [
+                    Error,
+                    Unexpected,
+                    Unsupported,
+                    ConfigInvalid,
+                    NotFound,
+                    PermissionDenied,
+                    IsADirectory,
+                    NotADirectory,
+                    AlreadyExists,
+                    IsSameFile,
+                    ConditionNotMatch,
+                    RateLimited,
+                    RangeNotSatisfied,
+                ]
+            )
+        }
+    }
+
+    // Option types live directly on the top-level `opendal` module.
+    #[pymodule_export]
+    use crate::{DeleteOptions, ListOptions, ReadOptions, StatOptions, WriteOptions};
+
+    #[pymodule_export]
+    #[allow(non_upper_case_globals)]
+    pub const __version__: &str = env!("CARGO_PKG_VERSION");
+
+    // Make the submodules importable as `opendal.<name>`.
+    #[pymodule_init]
+    fn init(m: &Bound<'_, PyModule>) -> PyResult<()> {
+        crate::register_submodules(m, "opendal")
+    }
 }
 
 define_stub_info_gatherer!(stub_info);

bindings/python/src/utils.rs

@@ -77,44 +77,37 @@ impl Buffer {
     }
 }
 
-/// Macro to create and register a PyO3 submodule with multiple classes.
+/// Recursively insert a module's nested `#[pymodule]` submodules into
+/// `sys.modules` under `parent_name` and qualify their `__name__`, so
+/// `from opendal.operator import ...` resolves.
 ///
-/// Example:
-/// ```rust
-/// add_pymodule!(py, m, "services", [PyScheme, PyOtherClass]);
-/// ```
-#[macro_export]
-macro_rules! add_pymodule {
-    ($py:expr, $parent:expr, $name:expr, [$($cls:ty),* $(,)?]) => {{
-        let sub_module = pyo3::types::PyModule::new($py, $name)?;
-        $(
-            sub_module.add_class::<$cls>()?;
-        )*
-        $parent.add_submodule(&sub_module)?;
-        $py.import("sys")?
-            .getattr("modules")?
-            .set_item(format!("opendal.{}", $name), &sub_module)?;
-        Ok::<_, pyo3::PyErr>(())
-    }};
+/// PyO3 attaches submodules as attributes but skips `sys.modules` (PyO3 #759);
+/// `parent_name` lets us use the public `opendal` name, not the `_opendal` lib.
+pub fn register_submodules(module: &Bound<'_, PyModule>, parent_name: &str) -> PyResult<()> {
+    let sys_modules = module.py().import("sys")?.getattr("modules")?;
+    for attr_name in module.index()? {
+        let attr_name: String = attr_name.extract()?;
+        let attr = module.getattr(&attr_name)?;
+        if let Ok(submodule) = attr.cast::<PyModule>() {
+            let qualified_name = format!("{parent_name}.{attr_name}");
+            submodule.setattr("__name__", &qualified_name)?;
+            sys_modules.set_item(&qualified_name, submodule)?;
+            register_submodules(submodule, &qualified_name)?;
+        }
+    }
+    Ok(())
 }
 
-/// Macro to create and register a PyO3 submodule containing exception types.
+/// Add exception types to a module by their Rust identifier.
 ///
-/// Example:
-/// ```rust
-/// add_pyexceptions!(py, m, "exceptions", [Error, Unexpected]);
-/// ```
+/// `create_exception!` types are `PyErr` subtypes, not `#[pyclass]`es, so they
+/// cannot be listed with `#[pymodule_export]`.
 #[macro_export]
-macro_rules! add_pyexceptions {
-    ($py:expr, $parent:expr, $name:expr, [$($exc:ty),* $(,)?]) => {{
-        let sub_module = pyo3::types::PyModule::new($py, $name)?;
+macro_rules! add_exceptions {
+    ($module:expr, [$($exc:ty),* $(,)?]) => {{
         $(
-            sub_module.add(stringify!($exc), $py.get_type::<$exc>())?;
+            $module.add(stringify!($exc), $module.py().get_type::<$exc>())?;
         )*
-        $parent.add_submodule(&sub_module)?;
-        $py.import("sys")?
-            .getattr("modules")?
-            .set_item(format!("opendal.{}", $name), &sub_module)?;
         Ok::<_, pyo3::PyErr>(())
     }};
 }