[ty] Disallow Self in metaclass and static methods

crates/ty_python_semantic/resources/mdtest/annotations/self.md

@@ -796,20 +796,248 @@ class Foo:
         return Foo()
 
     @staticmethod
-    # TODO: The usage of `Self` here should be rejected because this is a static method
+    # error: [invalid-type-form] "`Self` cannot be used in a static method"
     def make() -> Self:
-        # error: [invalid-return-type]
         return Foo()
 
 class Bar(Generic[T]): ...
 
 # error: [invalid-type-form]
 class Baz(Bar[Self]): ...
+```
+
+## Self usage in static methods
+
+`Self` cannot be used anywhere in a static method, including parameters, return types, nested
+functions, and default argument values.
+
+```py
+from typing import Self
+
+class StaticMethodTests:
+    @staticmethod
+    # error: [invalid-type-form] "`Self` cannot be used in a static method"
+    def with_self_return() -> Self:
+        pass
+
+    @staticmethod
+    # error: [invalid-type-form] "`Self` cannot be used in a static method"
+    def with_self_param(x: Self) -> None:
+        pass
+
+    @staticmethod
+    def with_nested_function() -> None:
+        # `Self` in nested function inside static method is also invalid
+        # because `Self` binds to the outermost method (the static method).
+        # error: [invalid-type-form] "`Self` cannot be used in a static method"
+        def inner() -> Self:
+            pass
+
+    @staticmethod
+    # error: [invalid-type-form] "`Self` cannot be used in a static method"
+    def with_self_default(x: int = 0, y: "Self | None" = None) -> None:
+        pass
+```
+
+## Aliased staticmethod decorator
+
+Using an aliased `staticmethod` decorator should still be detected:
+
+```py
+from typing import Self
+
+sm = staticmethod
+
+class AliasedStaticMethod:
+    @sm
+    # error: [invalid-type-form] "`Self` cannot be used in a static method"
+    def aliased_static() -> Self:
+        pass
+```
+
+## `__new__` allows `Self`
+
+`__new__` is a static method even without an explicit `@staticmethod` decorator, but at runtime it
+is heavily special-cased by the interpreter to behave more like a classmethod. It always receives a
+`cls` parameter with type `type[Self]` and typically returns an object of type `Self`, so `Self` is
+permitted in `__new__`:
+
+```py
+from typing import Self
+
+class WithNew:
+    def __new__(cls) -> Self:
+        instance = object.__new__(cls)
+        return instance
+
+reveal_type(WithNew())  # revealed: WithNew
+
+class SubclassWithNew(WithNew):
+    def __new__(cls) -> Self:
+        return super().__new__(cls)
+
+reveal_type(SubclassWithNew())  # revealed: SubclassWithNew
+```
+
+## Stacked decorators with staticmethod
+
+When `@staticmethod` is stacked with other decorators, `Self` should still be invalid:
+
+```py
+from typing import Self, Callable
+
+def identity[**P, R](f: Callable[P, R]) -> Callable[P, R]:
+    return f
+
+class StackedDecorators:
+    @staticmethod
+    @identity
+    # error: [invalid-type-form] "`Self` cannot be used in a static method"
+    def static_then_identity() -> Self:
+        pass
+    # TODO: On Python <3.10, this should ideally be rejected, because `staticmethod` objects were not callable.
+    @identity
+    @staticmethod
+    # error: [invalid-type-form] "`Self` cannot be used in a static method"
+    def identity_then_static() -> Self:
+        pass
+```
+
+## Self usage in metaclasses
+
+The spec [prohibits the use of `Self` in metaclasses][spec], so we emit a diagnostic for this.
+
+```py
+from typing import Self
 
 class MyMetaclass(type):
-    # TODO: reject the Self usage. because self cannot be used within a metaclass.
+    # error: [invalid-type-form] "`Self` cannot be used in a metaclass"
+    registry: list[Self]
+
+    # error: [invalid-type-form] "`Self` cannot be used in a metaclass"
     def __new__(cls, name, bases, dct) -> Self:
         return cls(name, bases, dct)
+    # error: [invalid-type-form] "`Self` cannot be used in a metaclass"
+    def instance_method(self) -> Self:
+        return self
+
+    @classmethod
+    # error: [invalid-type-form] "`Self` cannot be used in a metaclass"
+    def metaclass_classmethod(cls) -> Self:
+        return cls("", (), {})
+    # Note: static methods in metaclasses get the static method error, not metaclass error
+    @staticmethod
+    # error: [invalid-type-form] "`Self` cannot be used in a static method"
+    def metaclass_staticmethod() -> Self:
+        pass
+```
+
+## Runtime use of `self` parameter in metaclass
+
+Using the `self` parameter as a runtime value should not be flagged, even in a metaclass. Only the
+literal `Self` type form should be disallowed.
+
+```py
+class AnnotableMeta(type):
+    def __or__(self, other):
+        return self  # No error: runtime use of `self`, not the `Self` type form
+```
+
+## Indirect metaclass inheritance
+
+Classes that inherit from `type` indirectly (through another metaclass) are also metaclasses:
+
+```py
+from typing import Self
+from abc import ABCMeta
+
+class IndirectMetaclass(ABCMeta):
+    # error: [invalid-type-form] "`Self` cannot be used in a metaclass"
+    def method(self) -> Self:
+        return self
+
+class MultiLevelMeta(IndirectMetaclass):
+    # error: [invalid-type-form] "`Self` cannot be used in a metaclass"
+    def another_method(self) -> Self:
+        return self
+```
+
+## Classes using a metaclass are not metaclasses
+
+A class that uses a metaclass (via `metaclass=...`) is _not_ itself a metaclass. `Self` should be
+valid in such classes:
+
+```py
+from typing import Self
+
+class SomeMeta(type):
+    pass
+
+class UsesMetaclass(metaclass=SomeMeta):
+    def method(self) -> Self:
+        reveal_type(self)  # revealed: Self@method
+        return self
+
+reveal_type(UsesMetaclass().method())  # revealed: UsesMetaclass
+
+class SubclassOfMetaclassUser(UsesMetaclass):
+    def another(self) -> Self:
+        return self
+
+reveal_type(SubclassOfMetaclassUser().another())  # revealed: SubclassOfMetaclassUser
+```
+
+## Nested class inside a metaclass
+
+A nested class inside a metaclass is _not_ a metaclass (unless it also inherits from `type`):
+
+```py
+from typing import Self
+
+class OuterMeta(type):
+    # error: [invalid-type-form] "`Self` cannot be used in a metaclass"
+    def meta_method(self) -> Self:
+        return self
+
+    class NestedRegularClass:
+        # This is fine - NestedRegularClass is not a metaclass
+        def method(self) -> Self:
+            reveal_type(self)  # revealed: Self@method
+            return self
+
+    class NestedMetaclass(type):
+        # error: [invalid-type-form] "`Self` cannot be used in a metaclass"
+        def nested_meta_method(self) -> Self:
+            return self
+```
+
+## `builtins.staticmethod`
+
+Using the fully qualified `builtins.staticmethod` should also be detected:
+
+```py
+from typing import Self
+import builtins
+
+class BuiltinsStaticMethod:
+    @builtins.staticmethod
+    # error: [invalid-type-form] "`Self` cannot be used in a static method"
+    def method() -> Self:
+        pass
+```
+
+## EnumMeta is a metaclass
+
+`enum.EnumMeta` (or `enum.EnumType` in Python 3.11+) is a metaclass, so `Self` should be invalid:
+
+```py
+from typing import Self
+from enum import EnumMeta
+
+class CustomEnumMeta(EnumMeta):
+    # error: [invalid-type-form] "`Self` cannot be used in a metaclass"
+    def custom_method(self) -> Self:
+        return self
 ```
 
 ## Explicit annotations override implicit `Self`
@@ -1006,3 +1234,5 @@ class Child(Base):
 reveal_type(Child.attr)  # revealed: Child
 reveal_type(Child().attr)  # revealed: Child
 ```
+
+[spec]: https://typing.python.org/en/latest/spec/generics.html#valid-locations-for-self

crates/ty_python_semantic/src/types.rs

@@ -6725,6 +6725,10 @@ enum InvalidTypeExpression<'db> {
     /// Type qualifiers that are invalid in type expressions,
     /// and which would require exactly one argument even if they appeared in an annotation expression
     TypeQualifierRequiresOneArgument(TypeQualifier),
+    /// `typing.Self` cannot be used in `@staticmethod` definitions.
+    TypingSelfInStaticMethod,
+    /// `typing.Self` cannot be used in metaclass definitions.
+    TypingSelfInMetaclass,
     /// Some types are always invalid in type expressions
     InvalidType(Type<'db>, ScopeId<'db>),
     InvalidBareParamSpec(TypeVarInstance<'db>),
@@ -6794,6 +6798,12 @@ impl<'db> InvalidTypeExpression<'db> {
                         "Type qualifier `{qualifier}` is not allowed in type expressions \
                         (only in annotation expressions, and only with exactly one argument)",
                     ),
+                    InvalidTypeExpression::TypingSelfInStaticMethod => {
+                        f.write_str("`Self` cannot be used in a static method")
+                    }
+                    InvalidTypeExpression::TypingSelfInMetaclass => {
+                        f.write_str("`Self` cannot be used in a metaclass")
+                    }
                     InvalidTypeExpression::InvalidType(Type::FunctionLiteral(function), _) => {
                         write!(
                             f,

crates/ty_python_semantic/src/types/infer.rs

@@ -49,7 +49,7 @@ use crate::semantic_index::expression::Expression;
 use crate::semantic_index::scope::ScopeId;
 use crate::semantic_index::{SemanticIndex, semantic_index};
 use crate::types::diagnostic::TypeCheckDiagnostics;
-use crate::types::function::FunctionType;
+use crate::types::function::{FunctionDecorators, FunctionType};
 use crate::types::generics::Specialization;
 use crate::types::unpacker::{UnpackResult, Unpacker};
 use crate::types::{
@@ -102,6 +102,64 @@ fn definition_cycle_initial<'db>(
     DefinitionInference::cycle_initial(definition.scope(db), Type::divergent(id))
 }
 
+/// Infer decorator expression types for a function definition.
+///
+/// This is a lightweight query that avoids the cycle risk of calling
+/// `infer_definition_types` when we need to check decorators while
+/// already inside definition inference (e.g. checking `Self` in a
+/// `@staticmethod`).
+#[salsa::tracked(returns(ref), heap_size=ruff_memory_usage::heap_size)]
+pub(crate) fn function_known_decorators<'db>(
+    db: &'db dyn Db,
+    definition: Definition<'db>,
+) -> FunctionDecoratorInference<'db> {
+    let file = definition.file(db);
+    let module = parsed_module(db, file).load(db);
+    let index = semantic_index(db, file);
+
+    TypeInferenceBuilder::new(db, InferenceRegion::Definition(definition), index, &module)
+        .finish_function_decorator_inference()
+}
+
+pub(crate) fn function_known_decorator_flags<'db>(
+    db: &'db dyn Db,
+    definition: Definition<'db>,
+) -> FunctionDecorators {
+    function_known_decorators(db, definition).known_decorators()
+}
+
+/// A compact inference result for function decorators.
+///
+/// Unlike [`DefinitionInference`], this stores only decorator expression types and
+/// diagnostics, plus precomputed known-decorator flags.
+#[derive(Debug, Eq, PartialEq, salsa::Update, get_size2::GetSize)]
+pub(crate) struct FunctionDecoratorInference<'db> {
+    expression_types: FxHashMap<ExpressionNodeKey, Type<'db>>,
+    known_decorators: FunctionDecorators,
+    diagnostics: TypeCheckDiagnostics,
+}
+
+impl<'db> FunctionDecoratorInference<'db> {
+    pub(crate) fn expression_type(
+        &self,
+        expression: impl Into<ExpressionNodeKey>,
+    ) -> Option<Type<'db>> {
+        self.expression_types.get(&expression.into()).copied()
+    }
+
+    pub(crate) fn expression_types(&self) -> &FxHashMap<ExpressionNodeKey, Type<'db>> {
+        &self.expression_types
+    }
+
+    pub(crate) fn known_decorators(&self) -> FunctionDecorators {
+        self.known_decorators
+    }
+
+    pub(crate) fn diagnostics(&self) -> &TypeCheckDiagnostics {
+        &self.diagnostics
+    }
+}
+
 /// Infer types for all deferred type expressions in a [`Definition`].
 ///
 /// Deferred expressions are type expressions (annotations, base classes, aliases...) in a stub