From d2a73822eff0da5c179dba176152a3308deee08f Mon Sep 17 00:00:00 2001 From: Andrej Orsula Date: Tue, 23 Jan 2024 22:31:22 +0100 Subject: [PATCH] Development towards 0.2.0 (#2) * Reimplement type mapping into enum Signed-off-by: Andrej Orsula * Add typy mapping for generated class bindings Signed-off-by: Andrej Orsula * Improve type mapping and use `call_method0()/1()` where appropriate Signed-off-by: Andrej Orsula * Add support for tuple and (frozen)set types Signed-off-by: Andrej Orsula * Update project status Signed-off-by: Andrej Orsula * Bump to 0.2.0 Signed-off-by: Andrej Orsula --------- Signed-off-by: Andrej Orsula --- Cargo.lock | 16 +- Cargo.toml | 8 +- README.md | 29 +- pyo3_bindgen/src/lib.rs | 22 +- pyo3_bindgen_engine/src/bindgen.rs | 25 +- pyo3_bindgen_engine/src/bindgen/attribute.rs | 30 +- pyo3_bindgen_engine/src/bindgen/class.rs | 153 +- pyo3_bindgen_engine/src/bindgen/function.rs | 156 +- pyo3_bindgen_engine/src/bindgen/module.rs | 299 +++- pyo3_bindgen_engine/src/build_utils.rs | 11 +- pyo3_bindgen_engine/src/types.rs | 1334 +++++++++++++----- pyo3_bindgen_engine/tests/bindgen.rs | 306 ++-- pyo3_bindgen_macros/src/lib.rs | 22 +- 13 files changed, 1661 insertions(+), 750 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f4ab40e..68a85bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -572,7 +572,7 @@ dependencies = [ [[package]] name = "pyo3_bindgen" -version = "0.1.1" +version = "0.2.0" dependencies = [ "pyo3_bindgen_engine", "pyo3_bindgen_macros", @@ -580,7 +580,7 @@ dependencies = [ [[package]] name = "pyo3_bindgen_cli" -version = "0.1.1" +version = "0.2.0" dependencies = [ "assert_cmd", "clap", @@ -592,7 +592,7 @@ dependencies = [ [[package]] name = "pyo3_bindgen_engine" -version = "0.1.1" +version = "0.2.0" dependencies = [ "criterion", "indoc", @@ -607,7 +607,7 @@ dependencies = [ [[package]] name = "pyo3_bindgen_macros" -version = "0.1.1" +version = "0.2.0" dependencies = [ "proc-macro2", "pyo3", @@ -656,9 +656,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.2" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", @@ -668,9 +668,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "3b7fa1134405e2ec9353fd416b17f8dacd46c473d7d3fd1cf202706a14eb792a" dependencies = [ "aho-corasick", "memchr", diff --git a/Cargo.toml b/Cargo.toml index bc13643..dcce94a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,12 +20,12 @@ license = "MIT OR Apache-2.0" readme = "README.md" repository = "https://github.com/AndrejOrsula/pyo3_bindgen" rust-version = "1.70" -version = "0.1.1" +version = "0.2.0" [workspace.dependencies] -pyo3_bindgen = { path = "pyo3_bindgen", version = "0.1.1" } -pyo3_bindgen_engine = { path = "pyo3_bindgen_engine", version = "0.1.1" } -pyo3_bindgen_macros = { path = "pyo3_bindgen_macros", version = "0.1.1" } +pyo3_bindgen = { path = "pyo3_bindgen", version = "0.2.0" } +pyo3_bindgen_engine = { path = "pyo3_bindgen_engine", version = "0.2.0" } +pyo3_bindgen_macros = { path = "pyo3_bindgen_macros", version = "0.2.0" } assert_cmd = { version = "2" } clap = { version = "4.4", features = ["derive"] } diff --git a/README.md b/README.md index da3b342..f536dc0 100644 --- a/README.md +++ b/README.md @@ -115,15 +115,8 @@ fn main() { Afterwards, include the generated bindings anywhere in your crate. ```rs -#[allow( - clippy::all, - non_camel_case_types, - non_snake_case, - non_upper_case_globals -)] -pub mod target_module { - include!(concat!(env!("OUT_DIR"), "/bindings.rs")); -} +include!(concat!(env!("OUT_DIR"), "/bindings.rs")); +pub use target_module::*; ``` ### Option 2: CLI tool @@ -155,26 +148,20 @@ pyo3_bindgen = { version = "0.1", features = ["macros"] } Then, you can call the `import_python!` macro anywhere in your crate. ```rs -#[allow( - clippy::all, - non_camel_case_types, - non_snake_case, - non_upper_case_globals -)] -pub mod target_module { - pyo3_bindgen::import_python!("target_module"); -} +pyo3_bindgen::import_python!("target_module"); +pub use target_module::*; ``` ## Status This project is in early development, and as such, the API of the generated bindings is not yet stable. -- Not all Python types are mapped to their Rust equivalents yet. Especially the support for mapping types of module-wide classes for which bindings are generated is also still missing. For this reason, a lot of boilerplate might be currently required when using the generated bindings (e.g. `let typed_value: target_module::Class = any_value.extract()?;`). -- The binding generation is primarily designed to be used inside build scripts or via procedural macros. Therefore, the performance of the codegen process is [benchmarked](./pyo3_bindgen_engine/benches/bindgen.rs) to understand the potential impact on build times. Surprisingly, even the initial unoptimized version of the engine is able to process the entire `numpy` 1.26.3 in ~300 ms on a *modern* laptop while generating 166k lines of formatted Rust code (line count includes documentation). Adding more features might increase this time, but there is also plenty of room for optimization in the current naive implementation. +- Not all Python types are mapped to their Rust equivalents yet. For this reason, some additional typecasting might be currently required when using the generated bindings (e.g. `let typed_value: target_module::Class = any_value.extract()?;`). +- The binding generation is primarily designed to be used inside build scripts or via procedural macros. Therefore, the performance of the codegen process is [benchmarked](./pyo3_bindgen_engine/benches/bindgen.rs) to understand the potential impact on build times. Although there is currently plenty of room for optimization in the current naive implementation, even the largest modules are processed in less than a second on a *modern* laptop. - The generation of bindings should never panic as long as the target Python module can be successfully imported. If it does, it is a bug resulting from an unexpected edge-case Python module structure or an unforeseen combination of enabled PyO3 features. +- However, the generated bindings might not directly compile in some specific cases. Currently, there are two known issue; bindings will contain duplicate function definitions if present in the original code, and function parameters might use the same name as a class defined in the same scope (allowed in Python but not in Rust). If you encounter any other issues, consider manually rewriting the problematic parts of the bindings. - Although implemented, the procedural macros might not work in all cases - especially when some PyO3 features are enabled. In most cases, PyO3 fails to import the target Python module when used from within a `proc_macro` crate. Therefore, it is recommended to use build scripts instead for now. -- The code will be refactored and cleaned up in the upcoming releases. The current implementation is a result of a very quick prototype that was built to test the feasibility of the idea. +- The code will be refactored and cleaned up in the upcoming releases. The current implementation is a result of a very quick prototype that was built to test the feasibility of the idea. For example, configurability of the generated bindings is planned (e.g. allowlist/ignorelist of attributes). Furthermore, automatic generation of dependent Python modules will be considered in order to provide a more complete typing experience. Please [report](https://github.com/AndrejOrsula/pyo3_bindgen/issues/new) any issues that you might encounter. Contributions are more than welcome! If you are looking for a place to start, consider searching for `TODO` comments in the codebase. diff --git a/pyo3_bindgen/src/lib.rs b/pyo3_bindgen/src/lib.rs index ac8e676..67ae92a 100644 --- a/pyo3_bindgen/src/lib.rs +++ b/pyo3_bindgen/src/lib.rs @@ -32,15 +32,8 @@ //! Afterwards, include the generated bindings anywhere in your crate. //! //! ```rs -//! #[allow( -//! clippy::all, -//! non_camel_case_types, -//! non_snake_case, -//! non_upper_case_globals -//! )] -//! pub mod target_module { -//! include!(concat!(env!("OUT_DIR"), "/bindings.rs")); -//! } +//! include!(concat!(env!("OUT_DIR"), "/bindings.rs")); +//! pub use target_module::*; //! ``` //! //! ### Option 2: CLI tool @@ -72,15 +65,8 @@ //! Then, you can call the `import_python!` macro anywhere in your crate. //! //! ```rs -//! #[allow( -//! clippy::all, -//! non_camel_case_types, -//! non_snake_case, -//! non_upper_case_globals -//! )] -//! pub mod target_module { -//! pyo3_bindgen::import_python!("target_module"); -//! } +//! pyo3_bindgen::import_python!("target_module"); +//! pub use target_module::*; //! ``` pub use pyo3_bindgen_engine::{ diff --git a/pyo3_bindgen_engine/src/bindgen.rs b/pyo3_bindgen_engine/src/bindgen.rs index 2542b4a..5c73d0f 100644 --- a/pyo3_bindgen_engine/src/bindgen.rs +++ b/pyo3_bindgen_engine/src/bindgen.rs @@ -10,6 +10,15 @@ pub use class::bind_class; pub use function::bind_function; pub use module::{bind_module, bind_reexport}; +// TODO: Refactor everything into a large configurable struct that keeps track of all the +// important information needed to properly generate the bindings +// - Use builder pattern for the configuration of the struct +// - Keep track of all the types/classes that have been generated +// - Keep track of all imports to understand where each type is coming from +// - Keep track of all the external types that are used as parameters/return types and consider generating bindings for them as well + +// TODO: Ensure there are no duplicate entries in the generated code + /// Generate Rust bindings to a Python module specified by its name. Generating bindings to /// submodules such as `os.path` is also supported as long as the module can be directly imported /// from the Python interpreter via `import os.path`. @@ -75,7 +84,21 @@ pub fn generate_bindings_for_module( py: pyo3::Python, module: &pyo3::types::PyModule, ) -> Result { - bind_module(py, module, module, &mut std::collections::HashSet::new()) + let all_types = module::collect_types_of_module( + py, + module, + module, + &mut std::collections::HashSet::new(), + &mut std::collections::HashSet::default(), + )?; + + bind_module( + py, + module, + module, + &mut std::collections::HashSet::new(), + &all_types, + ) } /// Generate Rust bindings to a Python module specified by its `source_code`. The module will be diff --git a/pyo3_bindgen_engine/src/bindgen/attribute.rs b/pyo3_bindgen_engine/src/bindgen/attribute.rs index fa8ab4d..57ca6bf 100644 --- a/pyo3_bindgen_engine/src/bindgen/attribute.rs +++ b/pyo3_bindgen_engine/src/bindgen/attribute.rs @@ -1,13 +1,15 @@ -use crate::types::map_attr_type; +use crate::types::Type; /// Generate Rust bindings to a Python attribute. The attribute can be a standalone /// attribute or a property of a class. -pub fn bind_attribute( +pub fn bind_attribute( py: pyo3::Python, - module_name: Option<&str>, + module_name: &str, + is_class: bool, name: &str, attr: &pyo3::PyAny, attr_type: &pyo3::PyAny, + all_types: &std::collections::HashSet, ) -> Result { let mut token_stream = proc_macro2::TokenStream::new(); @@ -72,17 +74,17 @@ pub fn bind_attribute( }; let setter_ident = quote::format_ident!("set_{}", name); - let getter_type = map_attr_type(getter_type, true)?; - let setter_type = map_attr_type(setter_type, false)?; + let getter_type = Type::try_from(getter_type)?.into_rs_owned(module_name, all_types); + let setter_type = Type::try_from(setter_type)?.into_rs_borrowed(module_name, all_types); - if let Some(module_name) = module_name { + if is_class { token_stream.extend(quote::quote! { #[doc = #getter_doc] pub fn #getter_ident<'py>( + &'py self, py: ::pyo3::marker::Python<'py>, ) -> ::pyo3::PyResult<#getter_type> { - py.import(::pyo3::intern!(py, #module_name))? - .getattr(::pyo3::intern!(py, #name))? + self.getattr(::pyo3::intern!(py, #name))? .extract() } }); @@ -90,11 +92,11 @@ pub fn bind_attribute( token_stream.extend(quote::quote! { #[doc = #setter_doc] pub fn #setter_ident<'py>( + &'py self, py: ::pyo3::marker::Python<'py>, value: #setter_type, ) -> ::pyo3::PyResult<()> { - py.import(::pyo3::intern!(py, #module_name))? - .setattr(::pyo3::intern!(py, #name), value)?; + self.setattr(::pyo3::intern!(py, #name), value)?; Ok(()) } }); @@ -103,10 +105,9 @@ pub fn bind_attribute( token_stream.extend(quote::quote! { #[doc = #getter_doc] pub fn #getter_ident<'py>( - &'py self, py: ::pyo3::marker::Python<'py>, ) -> ::pyo3::PyResult<#getter_type> { - self.as_ref(py) + py.import(::pyo3::intern!(py, #module_name))? .getattr(::pyo3::intern!(py, #name))? .extract() } @@ -115,12 +116,11 @@ pub fn bind_attribute( token_stream.extend(quote::quote! { #[doc = #setter_doc] pub fn #setter_ident<'py>( - &'py mut self, py: ::pyo3::marker::Python<'py>, value: #setter_type, ) -> ::pyo3::PyResult<()> { - self.as_ref(py) - .setattr(::pyo3::intern!(py, #name), value)?; + py.import(::pyo3::intern!(py, #module_name))? + .setattr(::pyo3::intern!(py, #name), value)?; Ok(()) } }); diff --git a/pyo3_bindgen_engine/src/bindgen/class.rs b/pyo3_bindgen_engine/src/bindgen/class.rs index 8980c4b..0b5ea46 100644 --- a/pyo3_bindgen_engine/src/bindgen/class.rs +++ b/pyo3_bindgen_engine/src/bindgen/class.rs @@ -2,17 +2,28 @@ use crate::bindgen::{bind_attribute, bind_function}; /// Generate Rust bindings to a Python class with all its methods and attributes (properties). /// This function will call itself recursively to generate bindings to all nested classes. -pub fn bind_class( +pub fn bind_class( py: pyo3::Python, root_module: &pyo3::types::PyModule, class: &pyo3::types::PyType, + all_types: &std::collections::HashSet, ) -> Result { let inspect = py.import("inspect")?; // Extract the names of the modules let root_module_name = root_module.name()?; - let full_class_name = class.name()?; - let class_name: &str = full_class_name.split('.').last().unwrap(); + let class_full_name = class.name()?; + let class_name = class_full_name.split('.').last().unwrap(); + let class_module_name = format!( + "{}{}{}", + class.getattr("__module__")?, + if class_full_name.contains('.') { + "." + } else { + "" + }, + class_full_name.trim_end_matches(&format!(".{class_name}")) + ); // Create the Rust class identifier (raw string if it is a keyword) let class_ident = if syn::parse_str::(class_name).is_ok() { @@ -104,7 +115,7 @@ pub fn bind_class( .getattr("__module__") .unwrap_or(pyo3::types::PyString::new(py, "")) .to_string() - .ne(full_class_name); + .ne(&class_module_name); let is_class = attr_type .is_subclass_of::() @@ -125,21 +136,53 @@ pub fn bind_class( debug_assert!(![is_class, is_function].iter().all(|&v| v)); if is_class && !is_reexport { - impl_token_stream.extend(bind_class(py, root_module, attr.downcast().unwrap())); + impl_token_stream.extend(bind_class( + py, + root_module, + attr.downcast().unwrap(), + all_types, + )); } else if is_function { fn_names.push(name.to_string()); - impl_token_stream.extend(bind_function(py, "", name, attr)); + impl_token_stream.extend(bind_function( + py, + &class_module_name, + name, + attr, + all_types, + )); } else if !name.starts_with('_') { - impl_token_stream.extend(bind_attribute(py, None, name, attr, attr_type)); + impl_token_stream.extend(bind_attribute( + py, + &class_module_name, + true, + name, + attr, + attr_type, + all_types, + )); } }); // Add new and call aliases (currently a reimplemented versions of the function) + // TODO: Call the Rust `self.__init__()` and `self.__call__()` functions directly instead of reimplementing it if fn_names.contains(&"__init__".to_string()) && !fn_names.contains(&"new".to_string()) { - impl_token_stream.extend(bind_function(py, "", "new", class.getattr("__init__")?)); + impl_token_stream.extend(bind_function( + py, + &class_module_name, + "new", + class.getattr("__init__")?, + all_types, + )); } if fn_names.contains(&"__call__".to_string()) && !fn_names.contains(&"call".to_string()) { - impl_token_stream.extend(bind_function(py, "", "call", class.getattr("__call__")?)); + impl_token_stream.extend(bind_function( + py, + &class_module_name, + "call", + class.getattr("__call__")?, + all_types, + )); } let mut doc = class.getattr("__doc__")?.to_string(); @@ -150,48 +193,62 @@ pub fn bind_class( Ok(quote::quote! { #[doc = #doc] #[repr(transparent)] - #[derive(Clone, Debug)] - pub struct #class_ident(pub ::pyo3::PyObject); - #[automatically_derived] - impl ::std::ops::Deref for #class_ident { - type Target = ::pyo3::PyObject; - fn deref(&self) -> &Self::Target { - &self.0 - } - } - #[automatically_derived] - impl ::std::ops::DerefMut for #class_ident { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } - } - #[automatically_derived] - impl<'py> ::pyo3::FromPyObject<'py> for #class_ident { - fn extract(value: &'py ::pyo3::PyAny) -> ::pyo3::PyResult { - Ok(Self(value.into())) - } - } - #[automatically_derived] - impl ::pyo3::ToPyObject for #class_ident { - fn to_object<'py>(&'py self, py: ::pyo3::Python<'py>) -> ::pyo3::PyObject { - self.as_ref(py).to_object(py) - } - } - #[automatically_derived] - impl From<::pyo3::PyObject> for #class_ident { - fn from(value: ::pyo3::PyObject) -> Self { - Self(value) - } - } - #[automatically_derived] - impl<'py> From<&'py ::pyo3::PyAny> for #class_ident { - fn from(value: &'py ::pyo3::PyAny) -> Self { - Self(value.into()) - } - } + pub struct #class_ident(::pyo3::PyAny); + // Note: Using these macros is probably not the best idea, but it makes possible wrapping around ::pyo3::PyAny instead of ::pyo3::PyObject, which improves usability + ::pyo3::pyobject_native_type_named!(#class_ident); + ::pyo3::pyobject_native_type_info!(#class_ident, ::pyo3::pyobject_native_static_type_object!(::pyo3::ffi::PyBaseObject_Type), ::std::option::Option::Some(#class_module_name)); + ::pyo3::pyobject_native_type_extract!(#class_ident); #[automatically_derived] impl #class_ident { #impl_token_stream } }) + + // Ok(quote::quote! { + // #[doc = #doc] + // #[repr(transparent)] + // #[derive(Clone, Debug)] + // pub struct #class_ident(pub ::pyo3::PyObject); + // #[automatically_derived] + // impl ::std::ops::Deref for #class_ident { + // type Target = ::pyo3::PyObject; + // fn deref(&self) -> &Self::Target { + // &self.0 + // } + // } + // #[automatically_derived] + // impl ::std::ops::DerefMut for #class_ident { + // fn deref_mut(&mut self) -> &mut Self::Target { + // &mut self.0 + // } + // } + // #[automatically_derived] + // impl<'py> ::pyo3::FromPyObject<'py> for #class_ident { + // fn extract(value: &'py ::pyo3::PyAny) -> ::pyo3::PyResult { + // Ok(Self(value.into())) + // } + // } + // #[automatically_derived] + // impl ::pyo3::ToPyObject for #class_ident { + // fn to_object<'py>(&'py self, py: ::pyo3::Python<'py>) -> ::pyo3::PyObject { + // self.as_ref(py).to_object(py) + // } + // } + // #[automatically_derived] + // impl From<::pyo3::PyObject> for #class_ident { + // fn from(value: ::pyo3::PyObject) -> Self { + // Self(value) + // } + // } + // #[automatically_derived] + // impl<'py> From<&'py ::pyo3::PyAny> for #class_ident { + // fn from(value: &'py ::pyo3::PyAny) -> Self { + // Self(value.into()) + // } + // } + // #[automatically_derived] + // impl #class_ident { + // #impl_token_stream + // } + // }) } diff --git a/pyo3_bindgen_engine/src/bindgen/function.rs b/pyo3_bindgen_engine/src/bindgen/function.rs index 825c51a..54ccfda 100644 --- a/pyo3_bindgen_engine/src/bindgen/function.rs +++ b/pyo3_bindgen_engine/src/bindgen/function.rs @@ -1,15 +1,16 @@ use itertools::Itertools; use pyo3::PyTypeInfo; -use crate::types::map_attr_type; +use crate::types::Type; /// Generate Rust bindings to a Python function. The function can be a standalone function or a /// method of a class. -pub fn bind_function( +pub fn bind_function( py: pyo3::Python, module_name: &str, name: &str, function: &pyo3::PyAny, + all_types: &std::collections::HashSet, ) -> Result { let inspect = py.import("inspect")?; @@ -57,7 +58,6 @@ pub fn bind_function( Some(param_default) }; // TODO: Turn into enum or process in-place - // TODO: Fully support positional-only parameters let param_kind = match param_kind.extract::().unwrap() { 0 => "POSITIONAL_ONLY", 1 => "POSITIONAL_OR_KEYWORD", @@ -69,7 +69,7 @@ pub fn bind_function( if param_name != "self" { match param_kind { - "POSITIONAL_ONLY" | "POSITIONAL_OR_KEYWORD" => { + "POSITIONAL_ONLY" => { positional_args_idents.push( if syn::parse_str::(¶m_name).is_ok() { quote::format_ident!("{}", param_name) @@ -78,7 +78,7 @@ pub fn bind_function( }, ); } - "KEYWORD_ONLY" => { + "KEYWORD_ONLY" | "POSITIONAL_OR_KEYWORD" => { keyword_args_idents.push( if syn::parse_str::(¶m_name).is_ok() { quote::format_ident!("{}", param_name) @@ -154,100 +154,92 @@ pub fn bind_function( .iter() .skip(usize::from(has_self_param)) .map(|(_, param_annotation, _, _)| { - map_attr_type(param_annotation.unwrap_or_else(|| pynone), false).unwrap() + Type::try_from(param_annotation.unwrap_or_else(|| pynone)) + .unwrap() + .into_rs_borrowed(module_name, all_types) }) .collect_vec(); - let return_annotation = map_attr_type(return_annotation.unwrap_or(pynone), true)?; + let return_annotation = + Type::try_from(return_annotation.unwrap_or(pynone))?.into_rs_owned(module_name, all_types); let mut doc = function.getattr("__doc__")?.to_string(); if doc == "None" { doc = String::new(); }; - // TODO: Use `call_method0` and `call_method1`` where appropriate - Ok(if has_self_param { - if let Some(var_keyword_ident) = var_keyword_ident { + let (maybe_ref_self, callable_object) = if has_self_param { + (quote::quote! { &'py self, }, quote::quote! { self }) + } else { + ( + quote::quote! {}, + quote::quote! { py.import(::pyo3::intern!(py, #module_name))? }, + ) + }; + + let has_positional_args = !positional_args_idents.is_empty(); + let set_args = match ( + positional_args_idents.len() > 1, + var_positional_ident.is_some(), + ) { + (true, _) => { quote::quote! { - #[doc = #doc] - pub fn #function_ident<'py>( - &'py mut self, - py: ::pyo3::marker::Python<'py>, - #(#param_idents: #param_types),* - ) -> ::pyo3::PyResult<#return_annotation> { - #[allow(unused_imports)] - use ::pyo3::IntoPy; - let __internal_args = ( - #({ - let #positional_args_idents: ::pyo3::PyObject = #positional_args_idents.into_py(py); - #positional_args_idents - },)* - ); - let __internal_kwargs = #var_keyword_ident; - #(__internal_kwargs.set_item(::pyo3::intern!(py, #keyword_args_names), #keyword_args_idents)?;)* - self.as_ref(py).call_method(::pyo3::intern!(py, #function_name), __internal_args, Some(__internal_kwargs))?.extract() - } + let __internal_args = ::pyo3::types::PyTuple::new( + py, + [#(::pyo3::IntoPy::<::pyo3::PyObject>::into_py(#positional_args_idents.to_owned(), py).as_ref(py),)*] + ); } - } else { + } + (false, true) => { + let var_positional_ident = var_positional_ident.unwrap(); quote::quote! { - #[doc = #doc] - pub fn #function_ident<'py>( - &'py mut self, - py: ::pyo3::marker::Python<'py>, - #(#param_idents: #param_types),* - ) -> ::pyo3::PyResult<#return_annotation> { - #[allow(unused_imports)] - use ::pyo3::IntoPy; - let __internal_args = ( - #({ - let #positional_args_idents: ::pyo3::PyObject = #positional_args_idents.into_py(py); - #positional_args_idents - },)* - ); - let __internal_kwargs = ::pyo3::types::PyDict::new(py); - #(__internal_kwargs.set_item(::pyo3::intern!(py, #keyword_args_names), #keyword_args_idents)?;)* - self.as_ref(py).call_method(::pyo3::intern!(py, #function_name), __internal_args, Some(__internal_kwargs))?.extract() - } + let __internal_args = #var_positional_ident; } } - } else if let Some(var_keyword_ident) = var_keyword_ident { - quote::quote! { - #[doc = #doc] - pub fn #function_ident<'py>( - py: ::pyo3::marker::Python<'py>, - #(#param_idents: #param_types),* - ) -> ::pyo3::PyResult<#return_annotation> { - #[allow(unused_imports)] - use ::pyo3::IntoPy; - let __internal_args = ( - #({ - let #positional_args_idents: ::pyo3::PyObject = #positional_args_idents.into_py(py); - #positional_args_idents - },)* - ); - let __internal_kwargs = #var_keyword_ident; - #(__internal_kwargs.set_item(::pyo3::intern!(py, #keyword_args_names), #keyword_args_idents)?;)* - py.import(::pyo3::intern!(py, #module_name))?.call_method(::pyo3::intern!(py, #function_name), __internal_args, Some(__internal_kwargs))?.extract() - } + (false, false) => { + quote::quote! { let __internal_args = (); } } + }; + + let has_kwargs = !keyword_args_idents.is_empty(); + let kwargs_initial = if let Some(var_keyword_ident) = var_keyword_ident { + quote::quote! { #var_keyword_ident } } else { - quote::quote! { + quote::quote! { ::pyo3::types::PyDict::new(py) } + }; + let set_kwargs = quote::quote! { + let __internal_kwargs = #kwargs_initial; + #(__internal_kwargs.set_item(::pyo3::intern!(py, #keyword_args_names), #keyword_args_idents)?;)* + }; + + let call_method = match (has_positional_args, has_kwargs) { + (_, true) => { + quote::quote! { + #set_args + #set_kwargs + #callable_object.call_method(::pyo3::intern!(py, #function_name), __internal_args, Some(__internal_kwargs))? + } + } + (true, false) => { + quote::quote! { + #set_args + #callable_object.call_method1(::pyo3::intern!(py, #function_name), __internal_args)? + } + } + (false, false) => { + quote::quote! { + #callable_object.call_method0(::pyo3::intern!(py, #function_name))? + } + } + }; + + Ok(quote::quote! { #[doc = #doc] pub fn #function_ident<'py>( - py: ::pyo3::marker::Python<'py>, - #(#param_idents: #param_types),* - ) -> ::pyo3::PyResult<#return_annotation> { - #[allow(unused_imports)] - use ::pyo3::IntoPy; - let __internal_args = ( - #({ - let #positional_args_idents: ::pyo3::PyObject = #positional_args_idents.into_py(py); - #positional_args_idents - },)* - ); - let __internal_kwargs = ::pyo3::types::PyDict::new(py); - #(__internal_kwargs.set_item(::pyo3::intern!(py, #keyword_args_names), #keyword_args_idents)?;)* - py.import(::pyo3::intern!(py, #module_name))?.call_method(::pyo3::intern!(py, #function_name), __internal_args, Some(__internal_kwargs))?.extract() - } + #maybe_ref_self + py: ::pyo3::marker::Python<'py>, + #(#param_idents: #param_types),* + ) -> ::pyo3::PyResult<#return_annotation> { + #call_method.extract() } }) } diff --git a/pyo3_bindgen_engine/src/bindgen/module.rs b/pyo3_bindgen_engine/src/bindgen/module.rs index fca4c8c..ccd0655 100644 --- a/pyo3_bindgen_engine/src/bindgen/module.rs +++ b/pyo3_bindgen_engine/src/bindgen/module.rs @@ -7,11 +7,12 @@ use crate::bindgen::{bind_attribute, bind_class, bind_function}; /// attributes of the Python module. During the first call, the `root_module` argument should be /// the same as the `module` argument and the `processed_modules` argument should be an empty /// `HashSet`. -pub fn bind_module( +pub fn bind_module( py: pyo3::Python, root_module: &pyo3::types::PyModule, module: &pyo3::types::PyModule, processed_modules: &mut std::collections::HashSet, + all_types: &std::collections::HashSet, ) -> Result { let inspect = py.import("inspect")?; @@ -89,12 +90,10 @@ pub fn bind_module( .is_subclass_of::() .unwrap_or(false) { - let is_submodule = attr + let is_part_of_package = attr .getattr("__package__") - // Note: full_module_name is used here for comparison on purpose. - // It unseres that submodules are created in the correct scopes. - .is_ok_and(|package| package.to_string().starts_with(full_module_name)); - is_submodule + .is_ok_and(|package| package.to_string().starts_with(root_module_name)); + is_part_of_package } else { true } @@ -131,9 +130,6 @@ pub fn bind_module( .is_true() .unwrap(); - // Make sure that only one of the three is true - debug_assert!(![is_module, is_class, is_function].iter().all(|&v| v)); - // Process hidden modules (shadowed by re-exported attributes of the same name) if (is_class || is_function) && is_reexport @@ -155,9 +151,9 @@ pub fn bind_module( == full_module_name { let content = if is_class { - bind_class(py, root_module, attr.downcast().unwrap()).unwrap() + bind_class(py, root_module, attr.downcast().unwrap(), all_types).unwrap() } else if is_function { - bind_function(py, full_module_name, name, attr).unwrap() + bind_function(py, full_module_name, name, attr, all_types).unwrap() } else { unreachable!() }; @@ -179,31 +175,57 @@ pub fn bind_module( } if is_module { - if processed_modules.insert(format!( - "{}.{}", - attr.getattr("__package__").unwrap(), - name - )) { - mod_token_stream.extend(bind_module( - py, - root_module, - attr.downcast().unwrap(), - processed_modules, + let is_submodule_of_current_module = attr + .getattr("__package__") + .is_ok_and(|package| package.to_string().starts_with(full_module_name)); + + if is_submodule_of_current_module { + if processed_modules.insert(format!( + "{}.{}", + attr.getattr("__package__").unwrap(), + name + )) { + mod_token_stream.extend(bind_module( + py, + root_module, + attr.downcast().unwrap(), + processed_modules, + all_types, + )); + } + } else { + mod_token_stream.extend(bind_reexport( + root_module_name, + full_module_name, + name, + attr, )); } } else if is_reexport { - mod_token_stream.extend(bind_reexport(full_module_name, name, attr)); + mod_token_stream.extend(bind_reexport( + root_module_name, + full_module_name, + name, + attr, + )); } else if is_class { - mod_token_stream.extend(bind_class(py, root_module, attr.downcast().unwrap())); + mod_token_stream.extend(bind_class( + py, + root_module, + attr.downcast().unwrap(), + all_types, + )); } else if is_function { - mod_token_stream.extend(bind_function(py, full_module_name, name, attr)); + mod_token_stream.extend(bind_function(py, full_module_name, name, attr, all_types)); } else { mod_token_stream.extend(bind_attribute( py, - Some(full_module_name), + full_module_name, + false, name, attr, attr_type, + all_types, )); } }); @@ -215,7 +237,17 @@ pub fn bind_module( Ok(if module_name == root_module_name { quote::quote! { - #mod_token_stream + #[doc = #doc] + #[allow( + clippy::all, + non_camel_case_types, + non_snake_case, + non_upper_case_globals, + unused + )] + mod #module_ident { + #mod_token_stream + } } } else { quote::quote! { @@ -231,11 +263,30 @@ pub fn bind_module( /// re-export attributes from submodules in the parent module. For example, `from os import path` /// makes the `os.path` submodule available in the current module as just `path`. pub fn bind_reexport( + root_module_name: &str, module_name: &str, name: &str, attr: &pyo3::PyAny, ) -> Result { - let attr_origin_module = attr.getattr("__module__")?.to_string(); + let full_attr_name = attr.getattr("__name__")?.to_string(); + let attr_name = if full_attr_name.contains('.') { + full_attr_name.split('.').last().unwrap() + } else { + full_attr_name.as_str() + }; + let is_module; + let attr_origin_module = if let Ok(module) = attr.getattr("__module__") { + is_module = false; + module.to_string() + } else { + is_module = true; + full_attr_name + .clone() + .split('.') + .take((full_attr_name.split('.').count() - 1).max(1)) + .join(".") + }; + let n_common_ancestors = module_name .split('.') .zip(attr_origin_module.split('.')) @@ -243,7 +294,10 @@ pub fn bind_reexport( .count(); let current_module_depth = module_name.split('.').count(); let reexport_path = if (current_module_depth - n_common_ancestors) > 0 { - std::iter::repeat("super".to_string()).take(current_module_depth - n_common_ancestors) + std::iter::repeat("super".to_string()).take( + current_module_depth - n_common_ancestors + + usize::from(is_module && !full_attr_name.contains('.')), + ) } else { std::iter::repeat("self".to_string()).take(1) }; @@ -260,7 +314,7 @@ pub fn bind_reexport( } }), ) - .chain(std::iter::once(name).map(|s| { + .chain(std::iter::once(attr_name).map(|s| { if syn::parse_str::(s).is_ok() { s.to_owned() } else { @@ -272,7 +326,188 @@ pub fn bind_reexport( // The path contains both ident and "::", combine into something that can be quoted let reexport_path = syn::parse_str::(&reexport_path).unwrap(); - Ok(quote::quote! { - pub use #reexport_path; - }) + let visibility = if attr_name == root_module_name { + quote::quote! {} + } else { + quote::quote! { + pub + } + }; + + if attr_name == name { + Ok(quote::quote! { + #visibility use #reexport_path; + }) + } else { + let name = if syn::parse_str::(name).is_ok() { + quote::format_ident!("{}", name) + } else { + quote::format_ident!("r#{}", name) + }; + Ok(quote::quote! { + #visibility use #reexport_path as #name; + }) + } +} + +pub fn collect_types_of_module( + py: pyo3::Python, + root_module: &pyo3::types::PyModule, + module: &pyo3::types::PyModule, + processed_modules: &mut std::collections::HashSet, + all_types: &mut std::collections::HashSet, +) -> Result, pyo3::PyErr> { + let inspect = py.import("inspect")?; + + // Extract the names of the modules + let root_module_name = root_module.name()?; + let full_module_name = module.name()?; + + // Iterate over all attributes of the module while updating the token stream + module + .dir() + .iter() + .map(|name| { + let name = name.str().unwrap().to_str().unwrap(); + let attr = module.getattr(name).unwrap(); + let attr_type = attr.get_type(); + (name, attr, attr_type) + }) + .filter(|&(_, _, attr_type)| { + // Skip builtin functions + !attr_type + .is_subclass_of::() + .unwrap_or(false) + }) + .filter(|&(name, _, _)| { + // Skip private attributes + !name.starts_with('_') || name == "__init__" || name == "__call__" + }) + .filter(|(_, attr, attr_type)| { + // Skip typing attributes + !attr + .getattr("__module__") + .is_ok_and(|module| module.to_string().contains("typing")) + && !attr_type.to_string().contains("typing") + }) + .filter(|(_, attr, _)| { + // Skip __future__ attributes + !attr + .getattr("__module__") + .is_ok_and(|module| module.to_string().contains("__future__")) + }) + .filter(|&(_, attr, _)| { + // Skip classes and functions that are not part of the package + // However, this should keep instances of classes and builtins even if they are builtins or from other packages + if let Ok(module) = attr.getattr("__module__") { + if module.to_string().starts_with(root_module_name) { + true + } else { + !(inspect + .call_method1("isclass", (attr,)) + .unwrap() + .is_true() + .unwrap() + || inspect + .call_method1("isfunction", (attr,)) + .unwrap() + .is_true() + .unwrap()) + } + } else { + true + } + }) + .filter(|&(_, attr, attr_type)| { + // Skip external modules + if attr_type + .is_subclass_of::() + .unwrap_or(false) + { + let is_part_of_package = attr + .getattr("__package__") + .is_ok_and(|package| package.to_string().starts_with(root_module_name)); + is_part_of_package + } else { + true + } + }) + .for_each(|(name, attr, attr_type)| { + let is_internal = attr + .getattr("__module__") + .unwrap_or(pyo3::types::PyString::new(py, "")) + .to_string() + .starts_with(root_module_name); + let is_reexport = is_internal + && attr + .getattr("__module__") + .unwrap_or(pyo3::types::PyString::new(py, "")) + .to_string() + .ne(full_module_name); + + let is_module = attr_type + .is_subclass_of::() + .unwrap_or(false); + + let is_class = attr_type + .is_subclass_of::() + .unwrap_or(false); + + // Process hidden modules (shadowed by re-exported attributes of the same name) + if is_class + && is_reexport + && attr + .getattr("__module__") + .unwrap() + .to_string() + .split('.') + .last() + .unwrap() + == name + && attr + .getattr("__module__") + .unwrap() + .to_string() + .split('.') + .take(full_module_name.split('.').count()) + .join(".") + == full_module_name + { + let full_class_name = + format!("{}.{}", full_module_name, attr.getattr("__name__").unwrap()); + all_types.insert(full_class_name.clone()); + let full_class_name = format!("{full_module_name}.{name}"); + all_types.insert(full_class_name.clone()); + } + + if is_module { + let is_submodule_of_current_module = attr + .getattr("__package__") + .is_ok_and(|package| package.to_string().starts_with(full_module_name)); + + if is_submodule_of_current_module + && processed_modules.insert(format!( + "{}.{}", + attr.getattr("__package__").unwrap(), + name + )) + { + let _ = collect_types_of_module( + py, + root_module, + attr.downcast().unwrap(), + processed_modules, + all_types, + ); + } + } else if is_class { + let full_class_name = + format!("{}.{}", full_module_name, attr.getattr("__name__").unwrap()); + all_types.insert(full_class_name.clone()); + let full_class_name = format!("{full_module_name}.{name}"); + all_types.insert(full_class_name.clone()); + } + }); + + Ok(all_types.clone()) } diff --git a/pyo3_bindgen_engine/src/build_utils.rs b/pyo3_bindgen_engine/src/build_utils.rs index a73814c..fd9c1b6 100644 --- a/pyo3_bindgen_engine/src/build_utils.rs +++ b/pyo3_bindgen_engine/src/build_utils.rs @@ -36,15 +36,8 @@ /// ```ignore /// // src/lib.rs /// -/// #[allow( -/// clippy::all, -/// non_camel_case_types, -/// non_snake_case, -/// non_upper_case_globals -/// )] -/// pub mod os { -/// include!(concat!(env!("OUT_DIR"), "/bindings.rs")); -/// } +/// include!(concat!(env!("OUT_DIR"), "/bindings.rs")); +/// pub use os::*; /// ``` // TODO: Add `println!("cargo:rerun-if-changed={}.py");` for all files of the target Python module pub fn build_bindings( diff --git a/pyo3_bindgen_engine/src/types.rs b/pyo3_bindgen_engine/src/types.rs index 3dcacc5..b36d862 100644 --- a/pyo3_bindgen_engine/src/types.rs +++ b/pyo3_bindgen_engine/src/types.rs @@ -1,406 +1,1098 @@ //! Module for handling Rust, Python and `PyO3` types. +// TODO: Remove allow once impl is finished +#![allow(unused)] -/// Map a Python type to a Rust type. +use itertools::Itertools; +use std::str::FromStr; + +/// Enum that maps Python types to Rust types. /// /// Note that this is not a complete mapping at the moment. The public API is /// subject to large changes. -/// -/// # Arguments -/// -/// * `attr_type` - The Python type to map (either a `PyType` or a `PyString`). -/// * `owned` - Whether the Rust type should be owned or not (e.g. `String` vs `&str`). -/// -/// # Returns -/// -/// The Rust type as a `TokenStream`. -// TODO: Support more complex type conversions -// TODO: Support module-wide classes/types for which bindings are generated -// TODO: Return `syn::Type` instead -// TODO: Refactor into something more elegant -pub fn map_attr_type( - attr_type: &pyo3::PyAny, - owned: bool, -) -> Result { - Ok( - if let Ok(attr_type) = attr_type.downcast::() { - match attr_type { - _string - if attr_type.is_subclass_of::()? - || attr_type.is_subclass_of::()? => - { - if owned { - quote::quote! { - ::std::string::String - } - } else { - quote::quote! { - &str - } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Type { + PyAny, + Unhandled(String), + Unknown, + + // Primitives + PyBool, + PyByteArray, + PyBytes, + PyFloat, + PyLong, + PyString, + + // Enums + Optional(Box), + Union(Vec), + PyNone, + + // Collections + PyDict { + t_key: Box, + t_value: Box, + }, + PyFrozenSet(Box), + PyList(Box), + PySet(Box), + PyTuple(Vec), + + // Additional types - std + IpV4Addr, + IpV6Addr, + Path, + // TODO: Map `PySlice` to `std::ops::Range` if possible + PySlice, + + // Additional types - num-complex + // TODO: Support conversion of `PyComplex`` to `num_complex::Complex` if enabled via `num-complex` feature + PyComplex, + + // Additional types - datetime + #[cfg(not(Py_LIMITED_API))] + PyDate, + #[cfg(not(Py_LIMITED_API))] + PyDateTime, + PyDelta, + #[cfg(not(Py_LIMITED_API))] + PyTime, + #[cfg(not(Py_LIMITED_API))] + PyTzInfo, + + // Python-specific types + PyCapsule, + PyCFunction, + #[cfg(not(Py_LIMITED_API))] + PyCode, + PyEllipsis, + #[cfg(all(not(Py_LIMITED_API), not(PyPy)))] + PyFrame, + PyFunction, + PyModule, + #[cfg(not(PyPy))] + PySuper, + PyTraceback, + PyType, +} + +impl TryFrom<&pyo3::types::PyAny> for Type { + type Error = pyo3::PyErr; + fn try_from(value: &pyo3::types::PyAny) -> Result { + Ok(match value { + t if t.is_instance_of::() => { + let t = t.downcast::()?; + Self::try_from(t)? + } + s if s.is_instance_of::() => { + let s = s.downcast::()?; + Self::from_str(s.to_str()?)? + } + typing if typing.get_type().getattr("__module__")?.to_string() == "typing" => { + Self::from_typing(typing)? + } + none if none.is_none() => Self::Unknown, + // Unknown | Handle as string if possible + _ => { + let value = value.to_string(); + match &value { + _class if value.starts_with("") => { + let value = value + .strip_prefix("") + .unwrap(); + Self::from_str(value)? } - } - _bytes if attr_type.is_subclass_of::()? => { - if owned { - quote::quote! { - Vec - } - } else { - quote::quote! { - &[u8] - } + _enum if value.starts_with("") => { + let value = value + .strip_prefix("") + .unwrap(); + Self::from_str(value)? } + _ => Self::from_str(&value)?, } - _bool if attr_type.is_subclass_of::()? => { - quote::quote! { - bool + } + }) + } +} + +impl TryFrom<&pyo3::types::PyType> for Type { + type Error = pyo3::PyErr; + fn try_from(value: &pyo3::types::PyType) -> Result { + Ok(match value { + // Primitives + t if t.is_subclass_of::()? => Self::PyBool, + t if t.is_subclass_of::()? => Self::PyByteArray, + t if t.is_subclass_of::()? => Self::PyBytes, + t if t.is_subclass_of::()? => Self::PyFloat, + t if t.is_subclass_of::()? => Self::PyLong, + t if t.is_subclass_of::()? => Self::PyString, + + // Collections + t if t.is_subclass_of::()? => Self::PyDict { + t_key: Box::new(Self::Unknown), + t_value: Box::new(Self::Unknown), + }, + t if t.is_subclass_of::()? => { + Self::PyFrozenSet(Box::new(Self::Unknown)) + } + t if t.is_subclass_of::()? => { + Self::PyList(Box::new(Self::Unknown)) + } + t if t.is_subclass_of::()? => Self::PySet(Box::new(Self::Unknown)), + t if t.is_subclass_of::()? => Self::PyTuple(vec![Self::Unknown]), + + // Additional types - std + t if t.is_subclass_of::()? => Self::PySlice, + + // Additional types - num-complex + t if t.is_subclass_of::()? => Self::PyComplex, + + // Additional types - datetime + #[cfg(not(Py_LIMITED_API))] + t if t.is_subclass_of::()? => Self::PyDate, + #[cfg(not(Py_LIMITED_API))] + t if t.is_subclass_of::()? => Self::PyDateTime, + #[cfg(not(Py_LIMITED_API))] + t if t.is_subclass_of::()? => Self::PyDelta, + #[cfg(not(Py_LIMITED_API))] + t if t.is_subclass_of::()? => Self::PyTime, + #[cfg(not(Py_LIMITED_API))] + t if t.is_subclass_of::()? => Self::PyTzInfo, + + // Python-specific types + t if t.is_subclass_of::()? => Self::PyCapsule, + t if t.is_subclass_of::()? => Self::PyCFunction, + #[cfg(not(Py_LIMITED_API))] + t if t.is_subclass_of::()? => Self::PyCode, + #[cfg(all(not(Py_LIMITED_API), not(PyPy)))] + t if t.is_subclass_of::()? => Self::PyFrame, + #[cfg(all(not(Py_LIMITED_API), not(PyPy)))] + t if t.is_subclass_of::()? => Self::PyFunction, + t if t.is_subclass_of::()? => Self::PyModule, + #[cfg(not(PyPy))] + t if t.is_subclass_of::()? => Self::PySuper, + t if t.is_subclass_of::()? => Self::PyTraceback, + t if t.is_subclass_of::()? => Self::PyType, + + // Unknown | Handle as string if possible + _ => { + let value = value.to_string(); + match &value { + _class if value.starts_with("") => { + let value = value + .strip_prefix("") + .unwrap(); + Self::from_str(value)? } - } - _int if attr_type.is_subclass_of::()? => { - quote::quote! { - i64 + _enum if value.starts_with("") => { + let value = value + .strip_prefix("") + .unwrap(); + Self::from_str(value)? } + _ => Self::Unhandled(value), } - _float if attr_type.is_subclass_of::()? => { - quote::quote! { - f64 - } + } + }) + } +} + +impl std::str::FromStr for Type { + type Err = pyo3::PyErr; + fn from_str(value: &str) -> Result { + Ok(match value { + "Any" => Self::PyAny, + + // Primitives + "bool" => Self::PyBool, + "bytearray" => Self::PyByteArray, + "bytes" => Self::PyBytes, + "float" => Self::PyFloat, + "int" => Self::PyLong, + "str" => Self::PyString, + + // Enums + optional + if optional.matches('|').count() == 1 && optional.matches("None").count() == 1 => + { + let t = optional + .split('|') + .map(str::trim) + .find(|x| *x != "None") + .unwrap(); + Self::Optional(Box::new(Self::from_str(t)?)) + } + r#union if r#union.contains('|') => { + let mut t_sequence = r#union + .split('|') + .map(|x| x.trim().to_string()) + .collect::>(); + ugly_hack_repair_complex_split_sequence(&mut t_sequence); + Self::Union( + t_sequence + .iter() + .map(|x| Self::from_str(x)) + .collect::, _>>()?, + ) + } + "None" | "NoneType" => Self::PyNone, + + // Collections + dict if dict.starts_with("dict[") && dict.ends_with(']') => { + let (key, value) = dict + .strip_prefix("dict[") + .unwrap() + .strip_suffix(']') + .unwrap() + .split_once(',') + .unwrap(); + let key = key.trim(); + let value = value.trim(); + Self::PyDict { + t_key: Box::new(Self::from_str(key)?), + t_value: Box::new(Self::from_str(value)?), } - // complex if attr_type.is_subclass_of::()? => { - // quote::quote! { - // todo!() - // } - // } - _list if attr_type.is_subclass_of::()? => { - quote::quote! { - Vec<&'py ::pyo3::types::PyAny> + } + "dict" | "Dict" => Self::PyDict { + t_key: Box::new(Self::Unknown), + t_value: Box::new(Self::Unknown), + }, + frozenset if frozenset.starts_with("frozenset[") && frozenset.ends_with(']') => { + let t = frozenset + .strip_prefix("frozenset[") + .unwrap() + .strip_suffix(']') + .unwrap(); + Self::PyFrozenSet(Box::new(Self::from_str(t)?)) + } + list if list.starts_with("list[") && list.ends_with(']') => { + let t = list + .strip_prefix("list[") + .unwrap() + .strip_suffix(']') + .unwrap(); + Self::PyList(Box::new(Self::from_str(t)?)) + } + "list" => Self::PyList(Box::new(Self::Unknown)), + sequence if sequence.starts_with("Sequence[") && sequence.ends_with(']') => { + let t = sequence + .strip_prefix("Sequence[") + .unwrap() + .strip_suffix(']') + .unwrap(); + Self::PyList(Box::new(Self::from_str(t)?)) + } + set if set.starts_with("set[") && set.ends_with(']') => { + let t = set.strip_prefix("set[").unwrap().strip_suffix(']').unwrap(); + Self::PySet(Box::new(Self::from_str(t)?)) + } + tuple if tuple.starts_with("tuple[") && tuple.ends_with(']') => { + let mut t_sequence = tuple + .strip_prefix("tuple[") + .unwrap() + .strip_suffix(']') + .unwrap() + .split(',') + .map(|x| x.trim().to_string()) + .collect::>(); + ugly_hack_repair_complex_split_sequence(&mut t_sequence); + Self::PyTuple( + t_sequence + .iter() + .map(|x| Self::from_str(x)) + .collect::, _>>()?, + ) + } + + // Additional types - std + "ipaddress.IPv4Address" => Self::IpV4Addr, + "ipaddress.IPv6Address" => Self::IpV6Addr, + "os.PathLike" | "pathlib.Path" => Self::Path, + "slice" => Self::PySlice, + + // Additional types - num-complex + "complex" => Self::PyComplex, + + // Additional types - datetime + #[cfg(not(Py_LIMITED_API))] + "datetime.date" => Self::PyDate, + #[cfg(not(Py_LIMITED_API))] + "datetime.datetime" => Self::PyDateTime, + "timedelta" => Self::PyDelta, + #[cfg(not(Py_LIMITED_API))] + "datetime.time" => Self::PyTime, + #[cfg(not(Py_LIMITED_API))] + "datetime.tzinfo" => Self::PyTzInfo, + + // Python-specific types + "capsule" => Self::PyCapsule, + "cfunction" => Self::PyCFunction, + #[cfg(not(Py_LIMITED_API))] + "code" => Self::PyCode, + "Ellipsis" | "..." => Self::PyEllipsis, + #[cfg(all(not(Py_LIMITED_API), not(PyPy)))] + "frame" => Self::PyFrame, + "function" => Self::PyFunction, + callable if callable.starts_with("Callable[") && callable.ends_with(']') => { + // TODO: Use callable types for something if useful + // let (args, return_value) = callable + // .strip_prefix("Callable[") + // .unwrap() + // .strip_suffix(']') + // .unwrap() + // .split_once(',') + // .unwrap(); + // let args = args + // .strip_prefix("[") + // .unwrap() + // .strip_suffix("]") + // .unwrap() + // .split(',') + // .map(|x| x.trim()) + // .collect::>(); + // let return_value = return_value.trim(); + Self::PyFunction + } + "Callable" | "callable" => Self::PyFunction, + "module" => Self::PyModule, + #[cfg(not(PyPy))] + "super" => Self::PySuper, + "traceback" => Self::PyTraceback, + typ if typ.starts_with("type[") && typ.ends_with(']') => { + // TODO: Use inner type for something if useful + // let t = typ + // .strip_prefix("type[") + // .unwrap() + // .strip_suffix(']') + // .unwrap(); + Self::PyType + } + + // typing + typing if typing.starts_with("typing.") => { + let s = typing.strip_prefix("typing.").unwrap(); + Self::from_str(s)? + } + + // collection.abc + collection if collection.starts_with("collection.abc.") => { + let s = collection.strip_prefix("collection.abc.").unwrap(); + Self::from_str(s)? + } + + unhandled => Self::Unhandled(unhandled.to_owned()), + }) + } +} + +impl Type { + pub fn from_typing(value: &pyo3::types::PyAny) -> pyo3::PyResult { + if let (Ok(t), Ok(t_inner)) = (value.getattr("__origin__"), value.getattr("__args__")) { + let t_inner = t_inner.downcast::()?; + + if t.is_instance_of::() { + let t = t.downcast::()?; + match Self::try_from(t)? { + Self::PyDict { .. } => { + let (t_key, t_value) = ( + Self::try_from(t_inner.get_item(0)?)?, + Self::try_from(t_inner.get_item(1)?)?, + ); + return Ok(Self::PyDict { + t_key: Box::new(t_key), + t_value: Box::new(t_value), + }); } - } - _dict if attr_type.is_subclass_of::()? => { - quote::quote! { - &'py ::pyo3::types::PyDict + Self::PyList(..) => { + let t_inner = Self::try_from(t_inner.get_item(0)?)?; + return Ok(Self::PyList(Box::new(t_inner))); } - } - _tuple if attr_type.is_subclass_of::()? => { - quote::quote! { - &'py ::pyo3::types::PyTuple + Self::PyTuple(..) => { + let t_sequence = t_inner + .iter() + .map(Self::try_from) + .collect::, _>>()?; + return Ok(Self::PyTuple(t_sequence)); } - } - _set if attr_type.is_subclass_of::()? => { - quote::quote! { - &'py ::pyo3::types::PySet + Self::PyType => { + // TODO: See if the inner type is useful for something here + return Ok(Self::PyType); } - } - _frozenset if attr_type.is_subclass_of::()? => { - quote::quote! { - &'py ::pyo3::types::PyFrozenSet + _ => { + // Noop - processed as string below + // eprintln!( + // "Warning: Unexpected type encountered: {value}\n \ + // Bindings could be improved by handling the type here \ + // Please report this as a bug. [scope: Type::from_typing()]", + // ); } } - _bytearray if attr_type.is_subclass_of::()? => { - if owned { - quote::quote! { - Vec - } - } else { - quote::quote! { - &[u8] + } + + let t = t.to_string(); + Ok(match &t { + _typing if t.starts_with("typing.") => { + let t = t.strip_prefix("typing.").unwrap(); + match t { + "Union" => { + let t_sequence = t_inner + .iter() + .map(Self::try_from) + .collect::, _>>()?; + + if t_sequence.len() == 2 && t_sequence.contains(&Self::PyNone) { + let t = t_sequence + .iter() + .find(|x| **x != Self::PyNone) + .unwrap() + .clone(); + Self::Optional(Box::new(t)) + } else { + Self::Union(t_sequence) + } } + _ => Self::Unhandled(value.to_string()), } } - _slice if attr_type.is_subclass_of::()? => { - quote::quote! { - &'py ::pyo3::types::PySlice + _collections if t.starts_with("") => { + let t = t + .strip_prefix("") + .unwrap(); + match t { + "Iterable" | "Sequence" => { + let t_inner = Self::try_from(t_inner.get_item(0)?)?; + Self::PyList(Box::new(t_inner)) + } + "Callable" => { + // TODO: Use callable types for something if useful (t_inner) + Self::PyFunction + } + _ => Self::Unhandled(value.to_string()), } } - _type if attr_type.is_subclass_of::()? => { - quote::quote! { - &'py ::pyo3::types::PyType + // Unknown | Handle the type as string if possible + _ => { + // TODO: Handle also the inner type here if possible + let t = t.to_string(); + match &t { + _class if t.starts_with("") => { + let t = t + .strip_prefix("") + .unwrap(); + Self::from_str(t)? + } + _enum if t.starts_with("") => { + let t = t + .strip_prefix("") + .unwrap(); + Self::from_str(t)? + } + _ => Self::from_str(&t)?, } } - _module if attr_type.is_subclass_of::()? => { - quote::quote! { - &'py ::pyo3::types::PyModule - } + }) + } else { + let value = value.to_string(); + Type::from_str(&value) + } + } + + #[must_use] + pub fn into_rs( + self, + owned: bool, + module_name: &str, + all_types: &std::collections::HashSet, + ) -> proc_macro2::TokenStream { + if owned { + self.into_rs_owned(module_name, all_types) + } else { + self.into_rs_borrowed(module_name, all_types) + } + } + + #[must_use] + pub fn into_rs_owned( + self, + module_name: &str, + all_types: &std::collections::HashSet, + ) -> proc_macro2::TokenStream { + match self { + Self::PyAny => { + quote::quote! {&'py ::pyo3::types::PyAny} + } + Self::Unhandled(..) => self.try_into_module_path(module_name, all_types), + + Self::Unknown => { + quote::quote! {&'py ::pyo3::types::PyAny} + } + + // Primitives + Self::PyBool => { + quote::quote! {bool} + } + Self::PyByteArray | Self::PyBytes => { + quote::quote! {Vec} + } + Self::PyFloat => { + quote::quote! {f64} + } + Self::PyLong => { + quote::quote! {i64} + } + Self::PyString => { + quote::quote! {::std::string::String} + } + + // Enums + Self::Optional(t) => { + let inner = t.into_rs_owned(module_name, all_types); + quote::quote! { + ::std::option::Option<#inner> } - // collections_abc_Buffer - // if attr_type.is_subclass_of::>()? => - // { - // quote::quote! { - // &'py ::pyo3::types::PyBuffer - // } - // } - #[cfg(not(Py_LIMITED_API))] - _datetime_datetime if attr_type.is_subclass_of::()? => { - quote::quote! { - &'py ::pyo3::types::PyDateTime - } + } + Self::Union(t_alternatives) => { + // TODO: Support Rust enum where possible + quote::quote! { + &'py ::pyo3::types::PyAny } - #[cfg(not(Py_LIMITED_API))] - _datetime_date if attr_type.is_subclass_of::()? => { - quote::quote! { - &'py ::pyo3::types::PyDate - } + } + Self::PyNone => { + // TODO: Not sure what to do with None + quote::quote! { + &'py ::pyo3::types::PyAny } - #[cfg(not(Py_LIMITED_API))] - _datetime_time if attr_type.is_subclass_of::()? => { + } + + // Collections + Self::PyDict { t_key, t_value } => { + if t_key.is_owned_hashable() { + let t_key = t_key.into_rs_owned(module_name, all_types); + let t_value = t_value.into_rs_owned(module_name, all_types); quote::quote! { - &'py ::pyo3::types::PyTime + ::std::collections::HashMap<#t_key, #t_value> } - } - #[cfg(not(Py_LIMITED_API))] - _datetime_tzinfo if attr_type.is_subclass_of::()? => { + } else { quote::quote! { - &'py ::pyo3::types::PyTzInfo + &'py ::pyo3::types::PyDict } } - #[cfg(not(Py_LIMITED_API))] - _timedelta if attr_type.is_subclass_of::()? => { + } + Self::PyFrozenSet(t) => { + if t.is_owned_hashable() { + let t = t.into_rs_owned(module_name, all_types); quote::quote! { - ::std::time::Duration + ::std::collections::HashSet<#t> } - } - _unknown => { + } else { quote::quote! { - &'py ::pyo3::types::PyAny + &'py ::pyo3::types::PyFrozenSet } } } - } else if let Ok(attr_type) = attr_type.downcast::() { - let attr_type = attr_type.to_str()?; - match attr_type { - "str" => { - if owned { - quote::quote! { - ::std::string::String - } - } else { - quote::quote! { - &str - } - } + Self::PyList(t) => { + let inner = t.into_rs_owned(module_name, all_types); + quote::quote! { + Vec<#inner> } - "bytes" => { - if owned { - quote::quote! { - Vec - } - } else { - quote::quote! { - &[u8] - } - } - } - "bool" => { - quote::quote! { - bool - } - } - "int" => { + } + Self::PySet(t) => { + if t.is_owned_hashable() { + let t = t.into_rs_owned(module_name, all_types); quote::quote! { - i64 + ::std::collections::HashSet<#t> } - } - "float" => { + } else { quote::quote! { - f64 + &'py ::pyo3::types::PySet } } - "complex" => { + } + Self::PyTuple(t_sequence) => { + if t_sequence.is_empty() + || (t_sequence.len() == 1 && t_sequence[0] == Self::Unknown) + { quote::quote! { - &'py ::pyo3::types::PyComplex + &'py ::pyo3::types::PyTuple } - } - list if list.starts_with("list[") && list.ends_with(']') => { - // TODO: Concrete type parsing for lists + } else { + let inner = t_sequence + .into_iter() + .map(|x| x.into_rs_owned(module_name, all_types)) + .collect::>(); quote::quote! { - Vec<&'py ::pyo3::types::PyAny> + (#(#inner),*) } } - dict if dict.starts_with("dict[") && dict.ends_with(']') => { - let (key, value) = dict - .strip_prefix("dict[") - .unwrap() - .strip_suffix(']') - .unwrap() - .split_once(',') - .unwrap(); - match (key, value) { - ("str", "Any") => { - quote::quote! { - ::std::collections::HashMap - } - } - ("str", "str") => { - quote::quote! { - ::std::collections::HashMap - } - } - ("str", "bool") => { - quote::quote! { - ::std::collections::HashMap - } - } - ("str", "int") => { - quote::quote! { - ::std::collections::HashMap - } - } - ("str", "float") => { - quote::quote! { - ::std::collections::HashMap - } - } - _unknown => { - quote::quote! { - &'py ::pyo3::types::PyDict - } - } - } - } - tuple if tuple.starts_with("tuple[") && tuple.ends_with(']') => { - // TODO: Concrete type parsing for tuple - quote::quote! { - &'py ::pyo3::types::PyTuple - } + } + + // Additional types - std + Self::IpV4Addr => { + quote::quote! {::std::net::IpV4Addr} + } + Self::IpV6Addr => { + quote::quote! {::std::net::IpV6Addr} + } + Self::Path => { + quote::quote! {::std::path::PathBuf} + } + Self::PySlice => { + quote::quote! {&'py ::pyo3::types::PySlice} + } + + // Additional types - num-complex + Self::PyComplex => { + quote::quote! {&'py ::pyo3::types::PyComplex} + } + + // Additional types - datetime + #[cfg(not(Py_LIMITED_API))] + Self::PyDate => { + quote::quote! {&'py ::pyo3::types::PyDate} + } + #[cfg(not(Py_LIMITED_API))] + Self::PyDateTime => { + quote::quote! {&'py ::pyo3::types::PyDateTime} + } + Self::PyDelta => { + quote::quote! {::std::time::Duration} + } + #[cfg(not(Py_LIMITED_API))] + Self::PyTime => { + quote::quote! {&'py ::pyo3::types::PyTime} + } + #[cfg(not(Py_LIMITED_API))] + Self::PyTzInfo => { + quote::quote! {&'py ::pyo3::types::PyTzInfo} + } + + // Python-specific types + Self::PyCapsule => { + quote::quote! {&'py ::pyo3::types::PyCapsule} + } + Self::PyCFunction => { + quote::quote! {&'py ::pyo3::types::PyCFunction} + } + #[cfg(not(Py_LIMITED_API))] + Self::PyCode => { + quote::quote! {&'py ::pyo3::types::PyCode} + } + Self::PyEllipsis => { + // TODO: Not sure what to do with ellipsis + quote::quote! { + &'py ::pyo3::types::PyAny } - set if set.starts_with("set[") && set.ends_with(']') => { - // TODO: Concrete type parsing for set - quote::quote! { - &'py ::pyo3::types::PySet - } + } + #[cfg(all(not(Py_LIMITED_API), not(PyPy)))] + Self::PyFrame => { + quote::quote! {&'py ::pyo3::types::PyFrame} + } + #[cfg(all(not(Py_LIMITED_API), not(PyPy)))] + Self::PyFunction => { + quote::quote! {&'py ::pyo3::types::PyFunction} + } + #[cfg(not(all(not(Py_LIMITED_API), not(PyPy))))] + Self::PyFunction => { + quote::quote! {&'py ::pyo3::types::PyAny} + } + Self::PyModule => { + quote::quote! {&'py ::pyo3::types::PyModule} + } + #[cfg(not(PyPy))] + Self::PySuper => { + quote::quote! {&'py ::pyo3::types::PySuper} + } + Self::PyTraceback => { + quote::quote! {&'py ::pyo3::types::PyTraceback} + } + Self::PyType => { + quote::quote! {&'py ::pyo3::types::PyType} + } + } + } + + #[must_use] + pub fn into_rs_borrowed( + self, + module_name: &str, + all_types: &std::collections::HashSet, + ) -> proc_macro2::TokenStream { + match self { + Self::PyAny => { + quote::quote! {&'py ::pyo3::types::PyAny} + } + Self::Unhandled(..) => self.try_into_module_path(module_name, all_types), + Self::Unknown => { + quote::quote! {&'py ::pyo3::types::PyAny} + } + + // Primitives + Self::PyBool => { + quote::quote! {bool} + } + Self::PyByteArray | Self::PyBytes => { + quote::quote! {&[u8]} + } + Self::PyFloat => { + quote::quote! {f64} + } + Self::PyLong => { + quote::quote! {i64} + } + Self::PyString => { + quote::quote! {&str} + } + + // Enums + Self::Optional(t) => { + let inner = t.into_rs_owned(module_name, all_types); + quote::quote! { + ::std::option::Option<#inner> } - frozenset if frozenset.starts_with("frozenset[") && frozenset.ends_with(']') => { - // TODO: Concrete type parsing for frozenset - quote::quote! { - &'py ::pyo3::types::PyFrozenSet - } + } + Self::Union(t_alternatives) => { + // TODO: Support Rust enum where possible + quote::quote! { + &'py ::pyo3::types::PyAny } - "bytearray" => { - if owned { - quote::quote! { - Vec - } - } else { - quote::quote! { - &[u8] - } - } + } + Self::PyNone => { + // TODO: Not sure what to do with None + quote::quote! { + &'py ::pyo3::types::PyAny } - "slice" => { + } + + // Collections + Self::PyDict { t_key, t_value } => { + if t_key.is_owned_hashable() { + let t_key = t_key.into_rs_owned(module_name, all_types); + let t_value = t_value.into_rs_owned(module_name, all_types); quote::quote! { - &'py ::pyo3::types::PySlice + &::std::collections::HashMap<#t_key, #t_value> } - } - "type" => { + } else { quote::quote! { - &'py ::pyo3::types::PyType + &'py ::pyo3::types::PyDict } } - "module" => { + } + Self::PyFrozenSet(t) => { + if t.is_owned_hashable() { + let t = t.into_rs_owned(module_name, all_types); quote::quote! { - &'py ::pyo3::types::PyModule + &::std::collections::HashSet<#t> } - } - // "collections.abc.Buffer" => { - // quote::quote! { - // todo!() - // } - // } - "datetime.datetime" => { + } else { quote::quote! { - &'py ::pyo3::types::PyDateTime + &'py ::pyo3::types::PyFrozenSet } } - "datetime.date" => { - quote::quote! { - &'py ::pyo3::types::PyDate - } + } + Self::PyList(t) => { + let inner = t.into_rs_owned(module_name, all_types); + quote::quote! { + &[#inner] } - "datetime.time" => { + } + Self::PySet(t) => { + if t.is_owned_hashable() { + let t = t.into_rs_owned(module_name, all_types); quote::quote! { - &'py ::pyo3::types::PyTime + &::std::collections::HashSet<#t> } - } - "datetime.tzinfo" => { + } else { quote::quote! { - &'py ::pyo3::types::PyTzInfo + &'py ::pyo3::types::PySet } } - "timedelta" => { + } + Self::PyTuple(t_sequence) => { + if t_sequence.is_empty() + || (t_sequence.len() == 1 && t_sequence[0] == Self::Unknown) + { quote::quote! { - ::std::time::Duration + &'py ::pyo3::types::PyTuple } - } - // "decimal.Decimal" => { - // quote::quote! { - // todo!() - // } - // } - "ipaddress.IPv4Address" => { + } else { + let inner = t_sequence + .into_iter() + .map(|x| x.into_rs_owned(module_name, all_types)) + .collect::>(); quote::quote! { - ::std::net::IpV4Addr + (#(#inner),*) } } - "ipaddress.IPv6Address" => { - quote::quote! { - ::std::net::IpV6Addr - } + } + + // Additional types - std + Self::IpV4Addr => { + quote::quote! {::std::net::IpV4Addr} + } + Self::IpV6Addr => { + quote::quote! {::std::net::IpV6Addr} + } + Self::Path => { + quote::quote! {::std::path::PathBuf} + } + Self::PySlice => { + quote::quote! {&'py ::pyo3::types::PySlice} + } + + // Additional types - num-complex + Self::PyComplex => { + quote::quote! {&'py ::pyo3::types::PyComplex} + } + + // Additional types - datetime + #[cfg(not(Py_LIMITED_API))] + Self::PyDate => { + quote::quote! {&'py ::pyo3::types::PyDate} + } + #[cfg(not(Py_LIMITED_API))] + Self::PyDateTime => { + quote::quote! {&'py ::pyo3::types::PyDateTime} + } + Self::PyDelta => { + quote::quote! {::std::time::Duration} + } + #[cfg(not(Py_LIMITED_API))] + Self::PyTime => { + quote::quote! {&'py ::pyo3::types::PyTime} + } + #[cfg(not(Py_LIMITED_API))] + Self::PyTzInfo => { + quote::quote! {&'py ::pyo3::types::PyTzInfo} + } + + // Python-specific types + Self::PyCapsule => { + quote::quote! {&'py ::pyo3::types::PyCapsule} + } + Self::PyCFunction => { + quote::quote! {&'py ::pyo3::types::PyCFunction} + } + #[cfg(not(Py_LIMITED_API))] + Self::PyCode => { + quote::quote! {&'py ::pyo3::types::PyCode} + } + Self::PyEllipsis => { + // TODO: Not sure what to do with ellipsis + quote::quote! { + &'py ::pyo3::types::PyAny } - "os.PathLike" | "pathlib.Path" => { - if owned { - quote::quote! { - ::std::path::PathBuf - } - } else { - quote::quote! { - &::std::path::Path - } - } + } + #[cfg(all(not(Py_LIMITED_API), not(PyPy)))] + Self::PyFrame => { + quote::quote! {&'py ::pyo3::types::PyFrame} + } + #[cfg(all(not(Py_LIMITED_API), not(PyPy)))] + Self::PyFunction => { + quote::quote! {&'py ::pyo3::types::PyFunction} + } + #[cfg(not(all(not(Py_LIMITED_API), not(PyPy))))] + Self::PyFunction => { + quote::quote! {&'py ::pyo3::types::PyAny} + } + Self::PyModule => { + quote::quote! {&'py ::pyo3::types::PyModule} + } + #[cfg(not(PyPy))] + Self::PySuper => { + quote::quote! {&'py ::pyo3::types::PySuper} + } + Self::PyTraceback => { + quote::quote! {&'py ::pyo3::types::PyTraceback} + } + Self::PyType => { + quote::quote! {&'py ::pyo3::types::PyType} + } + } + } + + fn try_into_module_path( + self, + module_name: &str, + all_types: &std::collections::HashSet, + ) -> proc_macro2::TokenStream { + let Self::Unhandled(value) = self else { + unreachable!() + }; + let module_root = if module_name.contains('.') { + module_name.split('.').next().unwrap() + } else { + module_name + }; + match value.as_str() { + // Ignorelist + "property" + | "member_descriptor" + | "method_descriptor" + | "getset_descriptor" + | "_collections._tuplegetter" + | "AsyncState" => { + quote::quote! {&'py ::pyo3::types::PyAny} + } + module_member_full if module_member_full.starts_with(module_root) => { + // Ignore unknown types + if !all_types.contains(module_member_full) { + return quote::quote! {&'py ::pyo3::types::PyAny}; } - optional if optional.ends_with(" | None") => { - let optional_type = optional.split_once('|').unwrap().0.trim_end(); - match optional_type { - "str" => { - if owned { - quote::quote! { - ::std::option::Option - } - } else { - quote::quote! { - ::std::option::Option<&str> + + let value_name = module_member_full.split('.').last().unwrap(); + + let n_common_ancestors = module_name + .split('.') + .zip(module_member_full.split('.')) + .take_while(|(a, b)| a == b) + .count(); + let current_module_depth = module_name.split('.').count(); + let reexport_path = if (current_module_depth - n_common_ancestors) > 0 { + std::iter::repeat("super".to_string()) + .take(current_module_depth - n_common_ancestors) + } else { + std::iter::repeat("self".to_string()).take(1) + }; + let reexport_path: String = reexport_path + .chain( + module_member_full + .split('.') + .skip(n_common_ancestors) + .map(|s| { + if syn::parse_str::(s).is_ok() { + s.to_owned() + } else { + format!("r#{s}") } - } - } - "bool" => { - quote::quote! { - ::std::option::Option - } - } - "int" => { - quote::quote! { - ::std::option::Option - } - } - "float" => { - quote::quote! { - ::std::option::Option - } - } - _unknown => { - quote::quote! { - ::std::option::Option<&'py ::pyo3::types::PyAny> - } - } + }), + ) + .join("::"); + + // The path contains both ident and "::", combine into something that can be quoted + let reexport_path = syn::parse_str::(&reexport_path).unwrap(); + quote::quote! { + &'py #reexport_path + } + } + _ => { + let value_without_brackets = value.split_once('[').unwrap_or((&value, "")).0; + let module_scopes = value_without_brackets.split('.'); + let n_module_scopes = module_scopes.clone().count(); + + // Approach: Find types without a module scope (no dot) and check if the type is local (or imported in the current module) + if !value_without_brackets.contains('.') { + if let Some(member) = all_types + .iter() + .filter(|member| { + member + .split('.') + .take(member.split('.').count() - 1) + .join(".") + == module_name + }) + .find(|&member| { + member.trim_start_matches(&format!("{module_name}.")) + == value_without_brackets + }) + { + return Self::Unhandled(member.to_owned()) + .try_into_module_path(module_name, all_types); } } - _unknown => { - quote::quote! { - &'py ::pyo3::types::PyAny + + // Approach: Find the shallowest match that contains the value + // TODO: Fix this! The matching might be wrong in many cases + let mut possible_matches = std::collections::HashSet::::default(); + for i in 0..n_module_scopes { + let module_member_scopes_end = module_scopes.clone().skip(i).join("."); + all_types + .iter() + .filter(|member| member.ends_with(&module_member_scopes_end)) + .for_each(|member| { + possible_matches.insert(member.to_owned()); + }); + if !possible_matches.is_empty() { + let shallowest_match = possible_matches + .iter() + .min_by(|m1, m2| m1.split('.').count().cmp(&m2.split('.').count())) + .unwrap(); + return Self::Unhandled(shallowest_match.to_owned()) + .try_into_module_path(module_name, all_types); } } + + // Unsupported + // TODO: Support more types + // dbg!(value); + quote::quote! {&'py ::pyo3::types::PyAny} } - } else { - quote::quote! { - &'py ::pyo3::types::PyAny + } + } + + fn is_owned_hashable(&self) -> bool { + matches!( + self, + Self::PyBool + | Self::IpV4Addr + | Self::IpV6Addr + | Self::Path + | Self::PyDelta + | Self::PyDict { .. } + | Self::PyFrozenSet(..) + | Self::PyLong + | Self::PySet(..) + | Self::PyString + ) + } +} + +// TODO: Replace this with something more sensible +fn ugly_hack_repair_complex_split_sequence(sequence: &mut Vec) { + let mut traversed_all_elements = false; + let mut start_index = 0; + 'outer: while !traversed_all_elements { + traversed_all_elements = true; + 'inner: for i in start_index..(sequence.len() - 1) { + let mut n_scopes = sequence[i].matches('[').count() - sequence[i].matches(']').count(); + if n_scopes == 0 { + continue; + } + for j in (i + 1)..sequence.len() { + n_scopes += sequence[j].matches('[').count(); + n_scopes -= sequence[j].matches(']').count(); + if n_scopes == 0 { + let mut new_element = sequence[i].clone(); + for relevant_element in sequence.iter().take(j + 1).skip(i + 1) { + new_element = format!("{new_element},{relevant_element}"); + } + + // Update sequence and remove the elements that were merged + sequence[i] = new_element; + sequence.drain((i + 1)..=j); + + if j < sequence.len() - 1 { + traversed_all_elements = false; + start_index = i; + break 'inner; + } else { + break 'outer; + } + } } - }, - ) + } + } } diff --git a/pyo3_bindgen_engine/tests/bindgen.rs b/pyo3_bindgen_engine/tests/bindgen.rs index 3a1ab26..befbc2c 100644 --- a/pyo3_bindgen_engine/tests/bindgen.rs +++ b/pyo3_bindgen_engine/tests/bindgen.rs @@ -42,20 +42,24 @@ test_bindgen! { "# rs:r#" - ///Getter for the `t_const_float` attribute - pub fn t_const_float<'py>(py: ::pyo3::marker::Python<'py>) -> ::pyo3::PyResult { - py.import(::pyo3::intern!(py, "t_mod_test_bindgen_attribute"))? - .getattr(::pyo3::intern!(py, "t_const_float"))? - .extract() - } - ///Setter for the `t_const_float` attribute - pub fn set_t_const_float<'py>( - py: ::pyo3::marker::Python<'py>, - value: f64, - ) -> ::pyo3::PyResult<()> { - py.import(::pyo3::intern!(py, "t_mod_test_bindgen_attribute"))? - .setattr(::pyo3::intern!(py, "t_const_float"), value)?; - Ok(()) + /// + #[allow(clippy::all, non_camel_case_types, non_snake_case, non_upper_case_globals, unused)] + mod t_mod_test_bindgen_attribute { + ///Getter for the `t_const_float` attribute + pub fn t_const_float<'py>(py: ::pyo3::marker::Python<'py>) -> ::pyo3::PyResult { + py.import(::pyo3::intern!(py, "t_mod_test_bindgen_attribute"))? + .getattr(::pyo3::intern!(py, "t_const_float"))? + .extract() + } + ///Setter for the `t_const_float` attribute + pub fn set_t_const_float<'py>( + py: ::pyo3::marker::Python<'py>, + value: f64, + ) -> ::pyo3::PyResult<()> { + py.import(::pyo3::intern!(py, "t_mod_test_bindgen_attribute"))? + .setattr(::pyo3::intern!(py, "t_const_float"), value)?; + Ok(()) + } } "# } @@ -70,27 +74,31 @@ test_bindgen! { "# rs:r#" - ///t_docs - pub fn t_fn<'py>( - py: ::pyo3::marker::Python<'py>, - t_arg1: &str, - ) -> ::pyo3::PyResult { - #[allow(unused_imports)] - use ::pyo3::IntoPy; - let __internal_args = ( - { - let t_arg1: ::pyo3::PyObject = t_arg1.into_py(py); - t_arg1 - }, - ); - let __internal_kwargs = ::pyo3::types::PyDict::new(py); - py.import(::pyo3::intern!(py, "t_mod_test_bindgen_function"))? - .call_method( - ::pyo3::intern!(py, "t_fn"), - __internal_args, - Some(__internal_kwargs), - )? - .extract() + /// + #[allow( + clippy::all, + non_camel_case_types, + non_snake_case, + non_upper_case_globals, + unused + )] + mod t_mod_test_bindgen_function { + ///t_docs + pub fn t_fn<'py>( + py: ::pyo3::marker::Python<'py>, + t_arg1: &str, + ) -> ::pyo3::PyResult { + let __internal_args = (); + let __internal_kwargs = ::pyo3::types::PyDict::new(py); + __internal_kwargs.set_item(::pyo3::intern!(py, "t_arg1"), t_arg1)?; + py.import(::pyo3::intern!(py, "t_mod_test_bindgen_function"))? + .call_method( + ::pyo3::intern!(py, "t_fn"), + __internal_args, + Some(__internal_kwargs), + )? + .extract() + } } "# } @@ -117,144 +125,96 @@ test_bindgen! { "# rs:r#" - ///t_docs - #[repr(transparent)] - #[derive(Clone, Debug)] - pub struct t_class(pub ::pyo3::PyObject); - #[automatically_derived] - impl ::std::ops::Deref for t_class { - type Target = ::pyo3::PyObject; - fn deref(&self) -> &Self::Target { - &self.0 - } - } - #[automatically_derived] - impl ::std::ops::DerefMut for t_class { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } - } - #[automatically_derived] - impl<'py> ::pyo3::FromPyObject<'py> for t_class { - fn extract(value: &'py ::pyo3::PyAny) -> ::pyo3::PyResult { - Ok(Self(value.into())) - } - } - #[automatically_derived] - impl ::pyo3::ToPyObject for t_class { - fn to_object<'py>(&'py self, py: ::pyo3::Python<'py>) -> ::pyo3::PyObject { - self.as_ref(py).to_object(py) - } - } - #[automatically_derived] - impl From<::pyo3::PyObject> for t_class { - fn from(value: ::pyo3::PyObject) -> Self { - Self(value) - } - } - #[automatically_derived] - impl<'py> From<&'py ::pyo3::PyAny> for t_class { - fn from(value: &'py ::pyo3::PyAny) -> Self { - Self(value.into()) - } - } - #[automatically_derived] - impl t_class { - ///t_docs_init - pub fn __init__<'py>( - &'py mut self, - py: ::pyo3::marker::Python<'py>, - t_arg1: &str, - t_arg2: &'py ::pyo3::types::PyAny, - ) -> ::pyo3::PyResult<&'py ::pyo3::types::PyAny> { - #[allow(unused_imports)] - use ::pyo3::IntoPy; - let __internal_args = ( - { - let t_arg1: ::pyo3::PyObject = t_arg1.into_py(py); - t_arg1 - }, - { - let t_arg2: ::pyo3::PyObject = t_arg2.into_py(py); - t_arg2 - }, - ); - let __internal_kwargs = ::pyo3::types::PyDict::new(py); - self.as_ref(py) - .call_method( - ::pyo3::intern!(py, "__init__"), - __internal_args, - Some(__internal_kwargs), - )? - .extract() - } - ///t_docs_method - pub fn t_method<'py>( - &'py mut self, - py: ::pyo3::marker::Python<'py>, - t_arg1: &'py ::pyo3::types::PyAny, - kwargs: &'py ::pyo3::types::PyDict, - ) -> ::pyo3::PyResult<&'py ::pyo3::types::PyAny> { - #[allow(unused_imports)] - use ::pyo3::IntoPy; - let __internal_args = ( - { - let t_arg1: ::pyo3::PyObject = t_arg1.into_py(py); - t_arg1 - }, - ); - let __internal_kwargs = kwargs; - self.as_ref(py) - .call_method( - ::pyo3::intern!(py, "t_method"), - __internal_args, - Some(__internal_kwargs), - )? - .extract() - } - ///Getter for the `t_prop` attribute - pub fn t_prop<'py>( - &'py self, - py: ::pyo3::marker::Python<'py>, - ) -> ::pyo3::PyResult { - self.as_ref(py).getattr(::pyo3::intern!(py, "t_prop"))?.extract() - } - ///Setter for the `t_prop` attribute - pub fn set_t_prop<'py>( - &'py mut self, - py: ::pyo3::marker::Python<'py>, - value: i64, - ) -> ::pyo3::PyResult<()> { - self.as_ref(py).setattr(::pyo3::intern!(py, "t_prop"), value)?; - Ok(()) - } - ///t_docs_init - pub fn new<'py>( - &'py mut self, - py: ::pyo3::marker::Python<'py>, - t_arg1: &str, - t_arg2: &'py ::pyo3::types::PyAny, - ) -> ::pyo3::PyResult<&'py ::pyo3::types::PyAny> { - #[allow(unused_imports)] - use ::pyo3::IntoPy; - let __internal_args = ( - { - let t_arg1: ::pyo3::PyObject = t_arg1.into_py(py); - t_arg1 - }, - { - let t_arg2: ::pyo3::PyObject = t_arg2.into_py(py); - t_arg2 - }, - ); - let __internal_kwargs = ::pyo3::types::PyDict::new(py); - self.as_ref(py) - .call_method( - ::pyo3::intern!(py, "__init__"), - __internal_args, - Some(__internal_kwargs), - )? - .extract() + /// + #[allow( + clippy::all, + non_camel_case_types, + non_snake_case, + non_upper_case_globals, + unused + )] + mod t_mod_test_bindgen_class { + ///t_docs + #[repr(transparent)] + pub struct t_class(::pyo3::PyAny); + ::pyo3::pyobject_native_type_named!(t_class); + ::pyo3::pyobject_native_type_info!( + t_class, + ::pyo3::pyobject_native_static_type_object!(::pyo3::ffi::PyBaseObject_Type), + ::std::option::Option::Some("t_mod_test_bindgen_classt_class") + ); + ::pyo3::pyobject_native_type_extract!(t_class); + #[automatically_derived] + impl t_class { + ///t_docs_init + pub fn __init__<'py>( + &'py self, + py: ::pyo3::marker::Python<'py>, + t_arg1: &str, + t_arg2: ::std::option::Option, + ) -> ::pyo3::PyResult<&'py ::pyo3::types::PyAny> { + let __internal_args = (); + let __internal_kwargs = ::pyo3::types::PyDict::new(py); + __internal_kwargs.set_item(::pyo3::intern!(py, "t_arg1"), t_arg1)?; + __internal_kwargs.set_item(::pyo3::intern!(py, "t_arg2"), t_arg2)?; + self.call_method( + ::pyo3::intern!(py, "__init__"), + __internal_args, + Some(__internal_kwargs), + )? + .extract() + } + ///t_docs_method + pub fn t_method<'py>( + &'py self, + py: ::pyo3::marker::Python<'py>, + t_arg1: &::std::collections::HashMap<::std::string::String, i64>, + kwargs: &'py ::pyo3::types::PyDict, + ) -> ::pyo3::PyResult<&'py ::pyo3::types::PyAny> { + let __internal_args = (); + let __internal_kwargs = kwargs; + __internal_kwargs.set_item(::pyo3::intern!(py, "t_arg1"), t_arg1)?; + self.call_method( + ::pyo3::intern!(py, "t_method"), + __internal_args, + Some(__internal_kwargs), + )? + .extract() + } + ///Getter for the `t_prop` attribute + pub fn t_prop<'py>( + &'py self, + py: ::pyo3::marker::Python<'py>, + ) -> ::pyo3::PyResult { + self.getattr(::pyo3::intern!(py, "t_prop"))?.extract() + } + ///Setter for the `t_prop` attribute + pub fn set_t_prop<'py>( + &'py self, + py: ::pyo3::marker::Python<'py>, + value: i64, + ) -> ::pyo3::PyResult<()> { + self.setattr(::pyo3::intern!(py, "t_prop"), value)?; + Ok(()) + } + ///t_docs_init + pub fn new<'py>( + &'py self, + py: ::pyo3::marker::Python<'py>, + t_arg1: &str, + t_arg2: ::std::option::Option, + ) -> ::pyo3::PyResult<&'py ::pyo3::types::PyAny> { + let __internal_args = (); + let __internal_kwargs = ::pyo3::types::PyDict::new(py); + __internal_kwargs.set_item(::pyo3::intern!(py, "t_arg1"), t_arg1)?; + __internal_kwargs.set_item(::pyo3::intern!(py, "t_arg2"), t_arg2)?; + self.call_method( + ::pyo3::intern!(py, "__init__"), + __internal_args, + Some(__internal_kwargs), + )? + .extract() + } } } "# diff --git a/pyo3_bindgen_macros/src/lib.rs b/pyo3_bindgen_macros/src/lib.rs index 61bd26f..5fad22a 100644 --- a/pyo3_bindgen_macros/src/lib.rs +++ b/pyo3_bindgen_macros/src/lib.rs @@ -14,25 +14,11 @@ mod parser; /// // use pyo3_bindgen::import_python; /// use pyo3_bindgen_macros::import_python; /// -/// #[allow( -/// clippy::all, -/// non_camel_case_types, -/// non_snake_case, -/// non_upper_case_globals -/// )] -/// pub mod sys { -/// import_python!("sys"); -/// } +/// import_python!("sys"); +/// pub use sys::*; /// -/// #[allow( -/// clippy::all, -/// non_camel_case_types, -/// non_snake_case, -/// non_upper_case_globals -/// )] -/// pub(crate) mod os_path { -/// import_python!("os.path"); -/// } +/// import_python!("os.path"); +/// pub use path::*; /// ``` #[proc_macro] pub fn import_python(input: proc_macro::TokenStream) -> proc_macro::TokenStream {