Introspection: properly tag modules as incomplete (#5207)

Author: Tpt

newsfragments/5207.added.md

@@ -0,0 +1 @@
+Type stubs: tag modules created using `#[pymodule]` or `#[pymodule_init]` functions as incomplete

pyo3-introspection/src/introspection.rs

@@ -45,11 +45,19 @@ fn parse_chunks(chunks: &[Chunk], main_module_name: &str) -> Result<Module> {
             name,
             members,
             consts,
+            incomplete,
             id: _,
         } = chunk
         {
             if name == main_module_name {
-                return convert_module(name, members, consts, &chunks_by_id, &chunks_by_parent);
+                return convert_module(
+                    name,
+                    members,
+                    consts,
+                    *incomplete,
+                    &chunks_by_id,
+                    &chunks_by_parent,
+                );
             }
         }
     }
@@ -60,6 +68,7 @@ fn convert_module(
     name: &str,
     members: &[String],
     consts: &[ConstChunk],
+    incomplete: bool,
     chunks_by_id: &HashMap<&str, &Chunk>,
     chunks_by_parent: &HashMap<&str, Vec<&Chunk>>,
 ) -> Result<Module> {
@@ -84,6 +93,7 @@ fn convert_module(
                 value: c.value.clone(),
             })
             .collect(),
+        incomplete,
     })
 }
 
@@ -102,12 +112,14 @@ fn convert_members(
                 name,
                 members,
                 consts,
+                incomplete,
                 id: _,
             } => {
                 modules.push(convert_module(
                     name,
                     members,
                     consts,
+                    *incomplete,
                     chunks_by_id,
                     chunks_by_parent,
                 )?);
@@ -375,6 +387,7 @@ enum Chunk {
         name: String,
         members: Vec<String>,
         consts: Vec<ConstChunk>,
+        incomplete: bool,
     },
     Class {
         id: String,

pyo3-introspection/src/model.rs

@@ -5,6 +5,7 @@ pub struct Module {
     pub classes: Vec<Class>,
     pub functions: Vec<Function>,
     pub consts: Vec<Const>,
+    pub incomplete: bool,
 }
 
 #[derive(Debug, Eq, PartialEq, Clone, Hash)]

pyo3-introspection/src/stubs.rs

@@ -1,4 +1,4 @@
-use crate::model::{Argument, Class, Const, Function, Module, VariableLengthArgument};
+use crate::model::{Argument, Arguments, Class, Const, Function, Module, VariableLengthArgument};
 use std::collections::{BTreeSet, HashMap};
 use std::path::{Path, PathBuf};
 
@@ -43,6 +43,31 @@ fn module_stubs(module: &Module) -> String {
     for function in &module.functions {
         elements.push(function_stubs(function, &mut modules_to_import));
     }
+
+    // We generate a __getattr__ method to tag incomplete stubs
+    // See https://typing.python.org/en/latest/guides/writing_stubs.html#incomplete-stubs
+    if module.incomplete && !module.functions.iter().any(|f| f.name == "__getattr__") {
+        elements.push(function_stubs(
+            &Function {
+                name: "__getattr__".into(),
+                decorators: Vec::new(),
+                arguments: Arguments {
+                    positional_only_arguments: Vec::new(),
+                    arguments: vec![Argument {
+                        name: "name".to_string(),
+                        default_value: None,
+                        annotation: Some("str".into()),
+                    }],
+                    vararg: None,
+                    keyword_only_arguments: Vec::new(),
+                    kwarg: None,
+                },
+                returns: Some("_typeshed.Incomplete".into()),
+            },
+            &mut modules_to_import,
+        ));
+    }
+
     let mut final_elements = Vec::new();
     for module_to_import in &modules_to_import {
         final_elements.push(format!("import {module_to_import}"));

pyo3-macros-backend/src/introspection.rs

@@ -25,6 +25,7 @@ use syn::{Attribute, Ident, ReturnType, Type, TypePath};
 
 static GLOBAL_COUNTER_FOR_UNIQUE_NAMES: AtomicUsize = AtomicUsize::new(0);
 
+#[allow(clippy::too_many_arguments)]
 pub fn module_introspection_code<'a>(
     pyo3_crate_path: &PyO3CratePath,
     name: &str,
@@ -33,6 +34,7 @@ pub fn module_introspection_code<'a>(
     consts: impl IntoIterator<Item = &'a Ident>,
     consts_values: impl IntoIterator<Item = &'a String>,
     consts_cfg_attrs: impl IntoIterator<Item = &'a Vec<Attribute>>,
+    incomplete: bool,
 ) -> TokenStream {
     IntrospectionNode::Map(
         [
@@ -74,6 +76,7 @@ pub fn module_introspection_code<'a>(
                         .collect(),
                 ),
             ),
+            ("incomplete", IntrospectionNode::Bool(incomplete)),
         ]
         .into(),
     )
@@ -309,6 +312,7 @@ fn argument_introspection_data<'a>(
 
 enum IntrospectionNode<'a> {
     String(Cow<'a, str>),
+    Bool(bool),
     IntrospectionId(Option<Cow<'a, Type>>),
     InputType { rust_type: Type, nullable: bool },
     OutputType { rust_type: Type },
@@ -342,6 +346,7 @@ impl IntrospectionNode<'_> {
             Self::String(string) => {
                 content.push_str_to_escape(&string);
             }
+            Self::Bool(value) => content.push_str(if value { "true" } else { "false" }),
             Self::IntrospectionId(ident) => {
                 content.push_str("\"");
                 content.push_tokens(if let Some(ident) = ident {

pyo3-macros-backend/src/module.rs

@@ -354,6 +354,7 @@ pub fn pymodule_module_impl(
         &module_consts,
         &module_consts_values,
         &module_consts_cfg_attrs,
+        pymodule_init.is_some(),
     );
     #[cfg(not(feature = "experimental-inspect"))]
     let introspection = quote! {};
@@ -438,7 +439,7 @@ pub fn pymodule_function_impl(
 
     #[cfg(feature = "experimental-inspect")]
     let introspection =
-        module_introspection_code(pyo3_path, &name.to_string(), &[], &[], &[], &[], &[]);
+        module_introspection_code(pyo3_path, &name.to_string(), &[], &[], &[], &[], &[], true);
     #[cfg(not(feature = "experimental-inspect"))]
     let introspection = quote! {};
     #[cfg(feature = "experimental-inspect")]

pytests/stubs/__init__.pyi

@@ -0,0 +1,3 @@
+import _typeshed
+
+def __getattr__(name: str) -> _typeshed.Incomplete: ...