Basic introspection of #[derive(FromPyObject)]

Struct fields are not supported yet

Author: Tpt

pyo3-macros-backend/src/derive_attributes.rs

@@ -44,7 +44,7 @@ impl Parse for ContainerAttribute {
     }
 }
 
-#[derive(Default)]
+#[derive(Default, Clone)]
 pub struct ContainerAttributes {
     /// Treat the Container as a Wrapper, operate directly on its field
     pub transparent: Option<attributes::kw::transparent>,

pyo3-macros-backend/src/frompyobject.rs

@@ -1,5 +1,9 @@
 use crate::attributes::{DefaultAttribute, FromPyWithAttribute, RenamingRule};
 use crate::derive_attributes::{ContainerAttributes, FieldAttributes, FieldGetter};
+#[cfg(feature = "experimental-inspect")]
+use crate::introspection::ConcatenationBuilder;
+#[cfg(feature = "experimental-inspect")]
+use crate::utils::TypeExt;
 use crate::utils::{self, Ctx};
 use proc_macro2::TokenStream;
 use quote::{format_ident, quote, quote_spanned, ToTokens};
@@ -96,17 +100,29 @@ impl<'a> Enum<'a> {
             )
         )
     }
+
+    #[cfg(feature = "experimental-inspect")]
+    fn write_input_type(&self, builder: &mut ConcatenationBuilder, ctx: &Ctx) {
+        for (i, var) in self.variants.iter().enumerate() {
+            if i > 0 {
+                builder.push_str(" | ");
+            }
+            var.write_input_type(builder, ctx);
+        }
+    }
 }
 
 struct NamedStructField<'a> {
     ident: &'a syn::Ident,
     getter: Option<FieldGetter>,
     from_py_with: Option<FromPyWithAttribute>,
     default: Option<DefaultAttribute>,
+    ty: &'a syn::Type,
 }
 
 struct TupleStructField {
     from_py_with: Option<FromPyWithAttribute>,
+    ty: syn::Type,
 }
 
 /// Container Style
@@ -120,7 +136,8 @@ enum ContainerType<'a> {
     /// Newtype struct container, e.g. `#[transparent] struct Foo { a: String }`
     ///
     /// The field specified by the identifier is extracted directly from the object.
-    StructNewtype(&'a syn::Ident, Option<FromPyWithAttribute>),
+    #[cfg_attr(not(feature = "experimental-inspect"), allow(unused))]
+    StructNewtype(&'a syn::Ident, Option<FromPyWithAttribute>, &'a syn::Type),
     /// Tuple struct, e.g. `struct Foo(String)`.
     ///
     /// Variant contains a list of conversion methods for each of the fields that are directly
@@ -129,7 +146,8 @@ enum ContainerType<'a> {
     /// Tuple newtype, e.g. `#[transparent] struct Foo(String)`
     ///
     /// The wrapped field is directly extracted from the object.
-    TupleNewtype(Option<FromPyWithAttribute>),
+    #[cfg_attr(not(feature = "experimental-inspect"), allow(unused))]
+    TupleNewtype(Option<FromPyWithAttribute>, Box<syn::Type>),
 }
 
 /// Data container
@@ -168,6 +186,7 @@ impl<'a> Container<'a> {
                         );
                         Ok(TupleStructField {
                             from_py_with: attrs.from_py_with,
+                            ty: field.ty.clone(),
                         })
                     })
                     .collect::<Result<Vec<_>>>()?;
@@ -176,7 +195,7 @@ impl<'a> Container<'a> {
                     // Always treat a 1-length tuple struct as "transparent", even without the
                     // explicit annotation.
                     let field = tuple_fields.pop().unwrap();
-                    ContainerType::TupleNewtype(field.from_py_with)
+                    ContainerType::TupleNewtype(field.from_py_with, Box::new(field.ty))
                 } else if options.transparent.is_some() {
                     bail_spanned!(
                         fields.span() => "transparent structs and variants can only have 1 field"
@@ -216,6 +235,7 @@ impl<'a> Container<'a> {
                             getter: attrs.getter,
                             from_py_with: attrs.from_py_with,
                             default: attrs.default,
+                            ty: &field.ty,
                         })
                     })
                     .collect::<Result<Vec<_>>>()?;
@@ -237,7 +257,7 @@ impl<'a> Container<'a> {
                         field.getter.is_none(),
                         field.ident.span() => "`transparent` structs may not have a `getter` for the inner field"
                     );
-                    ContainerType::StructNewtype(field.ident, field.from_py_with)
+                    ContainerType::StructNewtype(field.ident, field.from_py_with, field.ty)
                 } else {
                     ContainerType::Struct(struct_fields)
                 }
@@ -274,10 +294,10 @@ impl<'a> Container<'a> {
     /// Build derivation body for a struct.
     fn build(&self, ctx: &Ctx) -> TokenStream {
         match &self.ty {
-            ContainerType::StructNewtype(ident, from_py_with) => {
+            ContainerType::StructNewtype(ident, from_py_with, _) => {
                 self.build_newtype_struct(Some(ident), from_py_with, ctx)
             }
-            ContainerType::TupleNewtype(from_py_with) => {
+            ContainerType::TupleNewtype(from_py_with, _) => {
                 self.build_newtype_struct(None, from_py_with, ctx)
             }
             ContainerType::Tuple(tups) => self.build_tuple_struct(tups, ctx),
@@ -438,6 +458,51 @@ impl<'a> Container<'a> {
 
         quote!(::std::result::Result::Ok(#self_ty{#fields}))
     }
+
+    #[cfg(feature = "experimental-inspect")]
+    fn write_input_type(&self, builder: &mut ConcatenationBuilder, ctx: &Ctx) {
+        match &self.ty {
+            ContainerType::StructNewtype(_, from_py_with, ty) => {
+                Self::write_field_input_type(from_py_with, ty, builder, ctx);
+            }
+            ContainerType::TupleNewtype(from_py_with, ty) => {
+                Self::write_field_input_type(from_py_with, ty, builder, ctx);
+            }
+            ContainerType::Tuple(tups) => {
+                builder.push_str("tuple[");
+                for (i, TupleStructField { from_py_with, ty }) in tups.iter().enumerate() {
+                    if i > 0 {
+                        builder.push_str(", ");
+                    }
+                    Self::write_field_input_type(from_py_with, ty, builder, ctx);
+                }
+                builder.push_str("]");
+            }
+            ContainerType::Struct(_) => {
+                // TODO: implement using a Protocol?
+                builder.push_str("_typeshed.Incomplete")
+            }
+        }
+    }
+
+    #[cfg(feature = "experimental-inspect")]
+    fn write_field_input_type(
+        from_py_with: &Option<FromPyWithAttribute>,
+        ty: &syn::Type,
+        builder: &mut ConcatenationBuilder,
+        ctx: &Ctx,
+    ) {
+        if from_py_with.is_some() {
+            // We don't know what from_py_with is doing
+            builder.push_str("_typeshed.Incomplete")
+        } else {
+            let ty = ty.clone().elide_lifetimes();
+            let pyo3_crate_path = &ctx.pyo3_path;
+            builder.push_tokens(
+                quote! { <#ty as #pyo3_crate_path::FromPyObject<'_>>::INPUT_TYPE.as_bytes() },
+            )
+        }
+    }
 }
 
 fn verify_and_get_lifetime(generics: &syn::Generics) -> Result<Option<&syn::LifetimeParam>> {
@@ -487,29 +552,64 @@ pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result<TokenStream> {
                 bail_spanned!(tokens.span() => "`transparent` or `annotation` is not supported \
                                                 at top level for enums");
             }
-            let en = Enum::new(en, &tokens.ident, options)?;
+            let en = Enum::new(en, &tokens.ident, options.clone())?;
             en.build(ctx)
         }
         syn::Data::Struct(st) => {
             if let Some(lit_str) = &options.annotation {
                 bail_spanned!(lit_str.span() => "`annotation` is unsupported for structs");
             }
             let ident = &tokens.ident;
-            let st = Container::new(&st.fields, parse_quote!(#ident), options)?;
+            let st = Container::new(&st.fields, parse_quote!(#ident), options.clone())?;
             st.build(ctx)
         }
         syn::Data::Union(_) => bail_spanned!(
             tokens.span() => "#[derive(FromPyObject)] is not supported for unions"
         ),
     };
 
+    #[cfg(feature = "experimental-inspect")]
+    let input_type = {
+        let mut builder = ConcatenationBuilder::default();
+        if tokens
+            .generics
+            .params
+            .iter()
+            .all(|p| matches!(p, syn::GenericParam::Lifetime(_)))
+        {
+            match &tokens.data {
+                syn::Data::Enum(en) => {
+                    Enum::new(en, &tokens.ident, options)?.write_input_type(&mut builder, ctx)
+                }
+                syn::Data::Struct(st) => {
+                    let ident = &tokens.ident;
+                    Container::new(&st.fields, parse_quote!(#ident), options.clone())?
+                        .write_input_type(&mut builder, ctx)
+                }
+                syn::Data::Union(_) => {
+                    // Not supported at this point
+                    builder.push_str("_typeshed.Incomplete")
+                }
+            }
+        } else {
+            // We don't know how to deal with generic parameters
+            // Blocked by https://github.com/rust-lang/rust/issues/76560
+            builder.push_str("_typeshed.Incomplete")
+        };
+        let input_type = builder.into_token_stream(&ctx.pyo3_path);
+        quote! { const INPUT_TYPE: &'static str = unsafe { ::std::str::from_utf8_unchecked(#input_type) }; }
+    };
+    #[cfg(not(feature = "experimental-inspect"))]
+    let input_type = quote! {};
+
     let ident = &tokens.ident;
     Ok(quote!(
         #[automatically_derived]
         impl #impl_generics #pyo3_path::FromPyObject<#lt_param> for #ident #ty_generics #where_clause {
             fn extract_bound(obj: &#pyo3_path::Bound<#lt_param, #pyo3_path::PyAny>) -> #pyo3_path::PyResult<Self>  {
                 #derives
             }
+            #input_type
         }
     ))
 }

pyo3-macros-backend/src/introspection.rs

@@ -464,13 +464,13 @@ impl<'a> From<IntrospectionNode<'a>> for AttributedIntrospectionNode<'a> {
 }
 
 #[derive(Default)]
-struct ConcatenationBuilder {
+pub struct ConcatenationBuilder {
     elements: Vec<ConcatenationBuilderElement>,
     current_string: String,
 }
 
 impl ConcatenationBuilder {
-    fn push_tokens(&mut self, token_stream: TokenStream) {
+    pub fn push_tokens(&mut self, token_stream: TokenStream) {
         if !self.current_string.is_empty() {
             self.elements.push(ConcatenationBuilderElement::String(take(
                 &mut self.current_string,
@@ -480,7 +480,7 @@ impl ConcatenationBuilder {
             .push(ConcatenationBuilderElement::TokenStream(token_stream));
     }
 
-    fn push_str(&mut self, value: &str) {
+    pub fn push_str(&mut self, value: &str) {
         self.current_string.push_str(value);
     }
 
@@ -502,7 +502,7 @@ impl ConcatenationBuilder {
         self.current_string.push('"');
     }
 
-    fn into_token_stream(self, pyo3_crate_path: &PyO3CratePath) -> TokenStream {
+    pub fn into_token_stream(self, pyo3_crate_path: &PyO3CratePath) -> TokenStream {
         let mut elements = self.elements;
         if !self.current_string.is_empty() {
             elements.push(ConcatenationBuilderElement::String(self.current_string));

pyo3-macros-backend/src/pyclass.rs

@@ -2373,6 +2373,13 @@ impl<'a> PyClassImplsBuilder<'a> {
             }
         };
 
+        let type_name = if cfg!(feature = "experimental-inspect") {
+            let full_name = get_class_python_module_and_name(cls, self.attr);
+            quote! { const TYPE_NAME: &'static str = #full_name; }
+        } else {
+            quote! {}
+        };
+
         Ok(quote! {
             #assertions
 
@@ -2393,6 +2400,8 @@ impl<'a> PyClassImplsBuilder<'a> {
                 type WeakRef = #weakref;
                 type BaseNativeType = #base_nativetype;
 
+                #type_name
+
                 fn items_iter() -> #pyo3_path::impl_::pyclass::PyClassItemsIter {
                     use #pyo3_path::impl_::pyclass::*;
                     let collector = PyClassImplCollector::<Self>::new();

pytests/src/pyclasses.rs

@@ -5,6 +5,7 @@ use pyo3::prelude::*;
 use pyo3::types::PyType;
 
 #[pyclass]
+#[derive(Clone, Default)]
 struct EmptyClass {}
 
 #[pymethods]
@@ -104,6 +105,7 @@ impl ClassWithDict {
 }
 
 #[pyclass]
+#[derive(Clone)]
 struct ClassWithDecorators {
     attr: usize,
 }
@@ -142,14 +144,32 @@ impl ClassWithDecorators {
     }
 }
 
+#[derive(FromPyObject, IntoPyObject)]
+enum AClass {
+    NewType(EmptyClass),
+    Tuple(EmptyClass, EmptyClass),
+    Struct {
+        f: EmptyClass,
+        #[pyo3(item(42))]
+        g: EmptyClass,
+        #[pyo3(default)]
+        h: EmptyClass,
+    },
+}
+
+#[pyfunction]
+fn map_a_class(cls: AClass) -> AClass {
+    cls
+}
+
 #[pymodule(gil_used = false)]
 pub mod pyclasses {
     #[cfg(any(Py_3_10, not(Py_LIMITED_API)))]
     #[pymodule_export]
     use super::ClassWithDict;
     #[pymodule_export]
     use super::{
-        AssertingBaseClass, ClassWithDecorators, ClassWithoutConstructor, EmptyClass, PyClassIter,
-        PyClassThreadIter,
+        map_a_class, AssertingBaseClass, ClassWithDecorators, ClassWithoutConstructor, EmptyClass,
+        PyClassIter, PyClassThreadIter,
     };
 }

pytests/stubs/pyclasses.pyi

@@ -1,3 +1,4 @@
+import _typeshed
 import typing
 
 class AssertingBaseClass:
@@ -34,3 +35,7 @@ class PyClassIter:
 class PyClassThreadIter:
     def __new__(cls, /) -> None: ...
     def __next__(self, /) -> int: ...
+
+def map_a_class(
+    cls: EmptyClass | tuple[EmptyClass, EmptyClass] | _typeshed.Incomplete,
+) -> typing.Any: ...

src/conversion.rs

@@ -407,6 +407,9 @@ impl<T> FromPyObject<'_> for T
 where
     T: PyClass + Clone,
 {
+    #[cfg(feature = "experimental-inspect")]
+    const INPUT_TYPE: &'static str = <T as crate::impl_::pyclass::PyClassImpl>::TYPE_NAME;
+
     fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult<Self> {
         let bound = obj.cast::<Self>()?;
         Ok(bound.try_borrow()?.clone())
@@ -417,6 +420,9 @@ impl<'py, T> FromPyObject<'py> for PyRef<'py, T>
 where
     T: PyClass,
 {
+    #[cfg(feature = "experimental-inspect")]
+    const INPUT_TYPE: &'static str = <T as crate::impl_::pyclass::PyClassImpl>::TYPE_NAME;
+
     fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult<Self> {
         obj.cast::<T>()?.try_borrow().map_err(Into::into)
     }
@@ -426,6 +432,9 @@ impl<'py, T> FromPyObject<'py> for PyRefMut<'py, T>
 where
     T: PyClass<Frozen = False>,
 {
+    #[cfg(feature = "experimental-inspect")]
+    const INPUT_TYPE: &'static str = <T as crate::impl_::pyclass::PyClassImpl>::TYPE_NAME;
+
     fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult<Self> {
         obj.cast::<T>()?.try_borrow_mut().map_err(Into::into)
     }

src/impl_/pyclass.rs

@@ -219,6 +219,9 @@ pub trait PyClassImpl: Sized + 'static {
     /// from the PyClassDocGenerator` type.
     const DOC: &'static CStr;
 
+    #[cfg(feature = "experimental-inspect")]
+    const TYPE_NAME: &'static str;
+
     fn items_iter() -> PyClassItemsIter;
 
     #[inline]