diff --git a/crates/ruff_benchmark/benches/ty_walltime.rs b/crates/ruff_benchmark/benches/ty_walltime.rs index 2f007db854f39a..61355a070498f1 100644 --- a/crates/ruff_benchmark/benches/ty_walltime.rs +++ b/crates/ruff_benchmark/benches/ty_walltime.rs @@ -171,7 +171,7 @@ static PANDAS: Benchmark = Benchmark::new( max_dep_date: "2025-06-17", python_version: PythonVersion::PY312, }, - 4500, + 5500, ); static PYDANTIC: Benchmark = Benchmark::new( @@ -202,7 +202,7 @@ static SYMPY: Benchmark = Benchmark::new( max_dep_date: "2025-06-17", python_version: PythonVersion::PY312, }, - 13600, + 13800, ); static TANJUN: Benchmark = Benchmark::new( diff --git a/crates/ty_python_semantic/resources/mdtest/call/constructor.md b/crates/ty_python_semantic/resources/mdtest/call/constructor.md index 35507ef5bb5e2b..1a524374d8333d 100644 --- a/crates/ty_python_semantic/resources/mdtest/call/constructor.md +++ b/crates/ty_python_semantic/resources/mdtest/call/constructor.md @@ -21,11 +21,6 @@ Since every class has `object` in its MRO, the default implementations are `obje `object`), no arguments are accepted and `TypeError` is raised if any are passed. - If `__new__` is defined but `__init__` is not, `object.__init__` will allow arbitrary arguments! -As of today there are a number of behaviors that we do not support: - -- `__new__` is assumed to return an instance of the class on which it is called -- User defined `__call__` on metaclass is ignored - ## Creating an instance of the `object` class itself Test the behavior of the `object` class itself. As implementation has to ignore `object` own methods @@ -261,6 +256,142 @@ class Box(Generic[T]): reveal_type(Box(1)) # revealed: Box[int] ``` +## `__new__` with method-level type variables mapping to class specialization + +When `__new__` has its own type parameters that map to the class's type parameter through the return +type, we should correctly infer the class specialization. + +```toml +[environment] +python-version = "3.12" +``` + +```py +class C[T]: + x: T + + def __new__[S](cls, x: S) -> "C[tuple[S, S]]": + return object.__new__(cls) + +reveal_type(C(1)) # revealed: C[tuple[int, int]] +reveal_type(C("hello")) # revealed: C[tuple[str, str]] +``` + +## `__new__` with arbitrary generic return types + +When `__new__` has method-level type variables in the return type that don't map to the class's type +parameters, the resolved return type should be used directly. + +```toml +[environment] +python-version = "3.12" +``` + +```py +class C: + def __new__[S](cls, x: S) -> S: + return x + +reveal_type(C("foo")) # revealed: Literal["foo"] +reveal_type(C(1)) # revealed: Literal[1] +``` + +## `__new__` returning non-instance generic containers + +```toml +[environment] +python-version = "3.12" +``` + +```py +class C: + def __new__[S](cls, x: S) -> list[S]: + return [x] + +reveal_type(C("foo")) # revealed: list[Literal["foo"]] +reveal_type(C(1)) # revealed: list[Literal[1]] +``` + +## Overloaded `__new__` with generic return types + +Overloaded `__new__` methods should correctly resolve to the matching overload and infer the class +specialization from the overload's return type. + +```py +from typing import Generic, Iterable, TypeVar, overload + +T = TypeVar("T") +T1 = TypeVar("T1") +T2 = TypeVar("T2") + +class MyZip(Generic[T]): + @overload + def __new__(cls) -> "MyZip[object]": ... + @overload + def __new__(cls, iter1: Iterable[T1], iter2: Iterable[T2]) -> "MyZip[tuple[T1, T2]]": ... + def __new__(cls, *args, **kwargs) -> "MyZip[object]": + raise NotImplementedError + +def check(a: tuple[int, ...], b: tuple[str, ...]) -> None: + reveal_type(MyZip(a, b)) # revealed: MyZip[tuple[int, str]] + reveal_type(MyZip()) # revealed: MyZip[object] +``` + +## Overloaded `__new__` with mixed instance and non-instance return types + +When `__new__` is overloaded and some overloads return the class instance type while others return a +different type, the decision of whether to skip `__init__` should be made per-overload based on +which overload matched at the call site. Non-instance-returning overloads use `__new__` directly, +while instance-returning overloads additionally validate `__init__`. + +```py +from typing import overload +from typing_extensions import Self + +class C: + @overload + def __new__(cls, x: int) -> int: ... + @overload + def __new__(cls, x: str) -> Self: ... + def __new__(cls, x: int | str) -> object: ... + def __init__(self, x: str) -> None: ... + +# The `int` overload is selected; its return type is not an instance of `C`, +# so `__init__` is skipped and the return type is `int`. +reveal_type(C(1)) # revealed: int + +# The `str -> Self` overload would return an instance of `C`, so `__init__` is +# also checked. `__init__` accepts `x: str`, so the call succeeds. +reveal_type(C("foo")) # revealed: C +``` + +## `__init__` incompatible with instance-returning `__new__` overloads + +When `__init__` parameters are incompatible with the arguments that would match instance-returning +`__new__` overloads, `__init__` errors are reported. The return type still comes from `__new__`. + +```py +from typing import overload +from typing_extensions import Self + +class D: + @overload + def __new__(cls, x: int) -> int: ... + @overload + def __new__(cls, x: str) -> Self: ... + def __new__(cls, x: int | str) -> object: ... + def __init__(self) -> None: ... + +# The `int` overload is selected; its return type is not an instance of `D`, +# so `__init__` is skipped and the return type is `int`. +reveal_type(D(1)) # revealed: int + +# The `str -> Self` overload returns an instance of `D`, so `__init__` is also +# checked. `__init__` takes no args, so it reports an error. +# error: [too-many-positional-arguments] +reveal_type(D("foo")) # revealed: D +``` + ## Constructor calls through `type[T]` with a bound TypeVar ```py diff --git a/crates/ty_python_semantic/resources/mdtest/classes.md b/crates/ty_python_semantic/resources/mdtest/classes.md index f825034324bbda..de1099b882854b 100644 --- a/crates/ty_python_semantic/resources/mdtest/classes.md +++ b/crates/ty_python_semantic/resources/mdtest/classes.md @@ -1,5 +1,102 @@ # Class definitions +## `__new__` return type + +Python's `__new__` method can return any type, not just an instance of the class. When `__new__` +returns a type that is not assignable to the class instance type, we use that return type directly. + +### `__new__` returning a different type + +```py +class ReturnsInt: + def __new__(cls) -> int: + return 42 + +reveal_type(ReturnsInt()) # revealed: int + +x: int = ReturnsInt() # OK +y: ReturnsInt = ReturnsInt() # error: [invalid-assignment] +``` + +### `__new__` returning a union type + +```py +class MaybeInt: + def __new__(cls, value: str) -> "int | MaybeInt": + try: + return int(value) + except ValueError: + return object.__new__(cls) + +reveal_type(MaybeInt("42")) # revealed: int | MaybeInt + +a: int | MaybeInt = MaybeInt("42") # OK +b: int = MaybeInt("42") # error: [invalid-assignment] +``` + +### `__new__` returning the class type + +When `__new__` returns the class type (or `Self`), the normal instance type is used. + +```py +class Normal: + def __new__(cls) -> "Normal": + return object.__new__(cls) + +reveal_type(Normal()) # revealed: Normal +``` + +### `__new__` with no return type annotation + +When `__new__` has no return type annotation, we fall back to the instance type. + +```py +class NoAnnotation: + def __new__(cls): + return object.__new__(cls) + +reveal_type(NoAnnotation()) # revealed: NoAnnotation +``` + +### `__new__` returning `Any` + +Per the spec, "an explicit return type of `Any` should be treated as a type that is not an instance +of the class being constructed." This means `__init__` is not called and the return type is `Any`. + +```py +from typing import Any + +class ReturnsAny: + def __new__(cls) -> Any: + return 42 + + def __init__(self, x: int) -> None: + pass + +# __init__ is skipped because `-> Any` is treated as non-instance per spec +reveal_type(ReturnsAny()) # revealed: Any +``` + +### `__new__` returning a union containing `Any` + +When `__new__` returns a union containing `Any`, `Any` is not a subtype of the instance type, so +`__init__` is skipped. + +```py +from typing import Any + +class MaybeAny: + def __new__(cls, value: int) -> "MaybeAny | Any": + if value > 0: + return object.__new__(cls) + return None + + def __init__(self, value: int) -> None: + pass + +reveal_type(MaybeAny(1)) # revealed: MaybeAny | Any +``` + ## Deferred resolution of bases ### Only the stringified name is deferred diff --git a/crates/ty_python_semantic/resources/mdtest/external/sqlmodel.md b/crates/ty_python_semantic/resources/mdtest/external/sqlmodel.md index 29a700148a8997..077b84d04712bd 100644 --- a/crates/ty_python_semantic/resources/mdtest/external/sqlmodel.md +++ b/crates/ty_python_semantic/resources/mdtest/external/sqlmodel.md @@ -19,11 +19,16 @@ class User(SQLModel): name: str user = User(id=1, name="John Doe") -reveal_type(user.id) # revealed: int -reveal_type(user.name) # revealed: str +# TODO: these should be `int` and `str` once we add pydantic model synthesis. +# Currently `Any` because `SQLModel.__new__` is annotated as `-> Any`, and the spec says +# "an explicit return type of `Any` should be treated as a type that is not an instance of +# the class being constructed." +reveal_type(user.id) # revealed: Any +reveal_type(user.name) # revealed: Any reveal_type(User.__init__) # revealed: (self: User, *, id: int, name: str) -> None -# error: [missing-argument] +# No `missing-argument` error here: `SQLModel.__new__` returns `Any`, so per the spec +# `__init__` is not evaluated and any arguments are accepted via `__new__`. User() ``` diff --git a/crates/ty_python_semantic/resources/mdtest/generics/legacy/classes.md b/crates/ty_python_semantic/resources/mdtest/generics/legacy/classes.md index c34ceb6e2ccd24..45ed91c96fd5f8 100644 --- a/crates/ty_python_semantic/resources/mdtest/generics/legacy/classes.md +++ b/crates/ty_python_semantic/resources/mdtest/generics/legacy/classes.md @@ -414,7 +414,7 @@ If either method comes from a generic base class, we don't currently use its inf to specialize the class. ```py -from typing_extensions import Generic, TypeVar +from typing_extensions import Generic, TypeVar, Self from ty_extensions import generic_context, into_callable T = TypeVar("T") @@ -422,7 +422,7 @@ U = TypeVar("U") V = TypeVar("V") class C(Generic[T, U]): - def __new__(cls, *args, **kwargs) -> "C[T, U]": + def __new__(cls, *args, **kwargs) -> Self: return object.__new__(cls) class D(C[V, int]): diff --git a/crates/ty_python_semantic/resources/mdtest/metaclass.md b/crates/ty_python_semantic/resources/mdtest/metaclass.md index 3c4762fe8fefec..c3aa019160051f 100644 --- a/crates/ty_python_semantic/resources/mdtest/metaclass.md +++ b/crates/ty_python_semantic/resources/mdtest/metaclass.md @@ -1,3 +1,329 @@ +## Custom `__call__` on metaclass + +When a metaclass defines a custom `__call__` method, it controls what happens when the class is +called. The return type and parameter types of the metaclass `__call__` are used instead of the +class's `__new__` and `__init__` methods. + +### Basic metaclass `__call__` + +```py +class Meta(type): + def __call__(cls, x: int, y: str) -> str: + return y + +class Foo(metaclass=Meta): ... + +reveal_type(Foo(1, "hello")) # revealed: str + +a: str = Foo(1, "hello") # OK +``` + +### Metaclass `__call__` with wrong arguments + +```py +class Meta(type): + def __call__(cls, x: int) -> int: + return x + +class Foo(metaclass=Meta): ... + +# error: [invalid-argument-type] +reveal_type(Foo("wrong")) # revealed: int +# error: [missing-argument] +reveal_type(Foo()) # revealed: int +# error: [too-many-positional-arguments] +reveal_type(Foo(1, 2)) # revealed: int +``` + +### Metaclass `__call__` takes precedence over `__init__` and `__new__` + +```py +class Meta(type): + def __call__(cls) -> str: + return "hello" + +class Foo(metaclass=Meta): + def __new__(cls, x: int) -> "Foo": + return object.__new__(cls) + + def __init__(self, x: int, y: int) -> None: + pass + +# The metaclass __call__ takes precedence, so no arguments are needed +# and the return type is str, not Foo. +reveal_type(Foo()) # revealed: str +``` + +### Metaclass `__call__` with TypeVar return type + +When the metaclass `__call__` returns a TypeVar bound to the class type, it's essentially a +pass-through to the normal constructor machinery. In this case, we should still check the `__new__` +and `__init__` signatures. + +```py +from typing import TypeVar + +T = TypeVar("T") + +class Meta(type): + def __call__(cls: type[T], *args, **kwargs) -> T: + return object.__new__(cls) + +class Foo(metaclass=Meta): + def __init__(self, x: int) -> None: + pass + +# The metaclass __call__ returns T (bound to Foo), so we check __init__ parameters. +Foo() # error: [missing-argument] +reveal_type(Foo(1)) # revealed: Foo +``` + +### Metaclass `__call__` with no return type annotation + +When the metaclass `__call__` has no return type annotation (returns `Unknown`), we should still +check the `__new__` and `__init__` signatures. + +```py +class Meta(type): + def __call__(cls, *args, **kwargs): + return object.__new__(cls) + +class Foo(metaclass=Meta): + def __init__(self, x: int) -> None: + pass + +# No return type annotation means we fall through to check __init__ parameters. +Foo() # error: [missing-argument] +reveal_type(Foo(1)) # revealed: Foo +``` + +### Metaclass `__call__` with specific parameters + +When the metaclass `__call__` has specific parameters (not just `*args, **kwargs`), we validate them +even when the return type is an instance type. Here both `__new__` and `__init__` accept anything, +so the errors must come from the metaclass `__call__`. + +```py +from typing import Any, TypeVar + +T = TypeVar("T") + +class Meta(type): + def __call__(cls: type[T], x: int) -> T: + return object.__new__(cls) + +class Foo(metaclass=Meta): + def __new__(cls, *args: Any, **kwargs: Any) -> "Foo": + return object.__new__(cls) + + def __init__(self, *args: Any, **kwargs: Any) -> None: + pass + +# The metaclass `__call__` requires exactly one `int` argument. +# error: [invalid-argument-type] +reveal_type(Foo("wrong")) # revealed: Foo +# error: [missing-argument] +reveal_type(Foo()) # revealed: Foo +# error: [too-many-positional-arguments] +reveal_type(Foo(1, 2)) # revealed: Foo +reveal_type(Foo(1)) # revealed: Foo +``` + +### Metaclass `__call__` returning the class instance type + +When the metaclass `__call__` returns the constructed class type (or a subclass), it's not +overriding normal construction. Per the spec, `__new__`/`__init__` should still be evaluated. + +```py +class Meta(type): + def __call__(cls, *args, **kwargs) -> "Foo": + return super().__call__(*args, **kwargs) + +class Foo(metaclass=Meta): + def __init__(self, x: int) -> None: + pass + +# The metaclass __call__ returns Foo, so we fall through to check __init__. +Foo() # error: [missing-argument] +Foo("wrong") # error: [invalid-argument-type] +reveal_type(Foo(1)) # revealed: Foo +``` + +### Metaclass `__call__` returning a specific class affects subclasses + +When a metaclass `__call__` returns a specific class (e.g., `-> Foo`), this is an instance type for +`Foo` itself, so `__init__` is checked. But for a subclass `Bar(Foo)`, the return type `Foo` is NOT +an instance of `Bar`, so the metaclass `__call__` is used directly and `Bar.__init__` is skipped. + +```py +from typing import Any + +class Meta(type): + def __call__(cls, *args: Any, **kwargs: Any) -> "Foo": + return super().__call__(*args, **kwargs) + +class Foo(metaclass=Meta): + def __init__(self, x: int) -> None: + pass + +class Bar(Foo): + def __init__(self, y: str) -> None: + pass + +# For Foo: return type `Foo` IS an instance of `Foo`, so `__init__` is checked. +Foo() # error: [missing-argument] +reveal_type(Foo(1)) # revealed: Foo + +# For Bar: return type `Foo` is NOT an instance of `Bar`, so `__init__` is +# skipped and the metaclass `__call__` (which accepts `*args, **kwargs`) is +# used directly. +reveal_type(Bar()) # revealed: Foo +reveal_type(Bar("hello")) # revealed: Foo +``` + +### Metaclass `__call__` returning bare `type` + +When the metaclass `__call__` is annotated as returning `type`, we use that return type. This is +stricter than mypy and pyright, which ignore the `-> type` annotation in this case. `__init__` is +skipped because the return type is not an instance of the class being constructed. + +```py +from typing import Any + +class Singleton(type): + _instances: dict["Singleton", object] = {} + + def __call__(cls, *args: Any, **kwargs: Any) -> type: + if cls not in cls._instances: + cls._instances[cls] = super().__call__(*args, **kwargs) + # error: [invalid-return-type] + return cls._instances[cls] + +class MyConfig(metaclass=Singleton): + def __init__(self, x: int) -> None: + pass + + def get(self, key: str) -> str: + return key + +# The metaclass `__call__` returns `type`, so that's what we infer. +# `__init__` is not checked: `MyConfig()` would violate `__init__(self, x: int)`, +# but we skip it because the return type is `type`, not `MyConfig`. +reveal_type(MyConfig()) # revealed: type + +# Instance methods are not available on `type`. +# error: [unresolved-attribute] +MyConfig().get("key") +``` + +### Metaclass `__call__` returning `Any` + +When a metaclass `__call__` returns `Any`, the return type is not an instance of the class being +constructed, so we use the metaclass `__call__` signature directly and skip `__new__`/`__init__` +validation. This is consistent with the treatment of `-> Any` on `__new__`, and matches pyright. + +```py +from typing import Any + +class Meta(type): + def __call__(cls, *args: Any, **kwargs: Any) -> Any: + return super().__call__(*args, **kwargs) + +class Foo(metaclass=Meta): + def __init__(self, x: int) -> None: + pass + +# The metaclass `__call__` accepts `(*args, **kwargs)` and returns `Any`, +# so we use that directly, skipping `__init__` validation. +reveal_type(Foo()) # revealed: Any +reveal_type(Foo("wrong")) # revealed: Any +``` + +### Overloaded metaclass `__call__` with mixed return types + +When a metaclass `__call__` is overloaded and some overloads return the class instance type while +others return a different type, non-instance-returning overloads use the metaclass `__call__` +directly, while instance-returning overloads are replaced by `__init__` validation. + +```py +from typing import Any, overload + +class Meta(type): + @overload + def __call__(cls, x: int) -> int: ... + @overload + def __call__(cls, x: str) -> "Foo": ... + def __call__(cls, x: int | str) -> Any: + return super().__call__(x) + +class Foo(metaclass=Meta): + def __init__(self, x: int) -> None: + pass + +# The `int` overload from the metaclass `__call__` is selected; its return type +# is not an instance of `Foo`, so it is used directly. +reveal_type(Foo(1)) # revealed: int + +# The `str -> Foo` metaclass overload matches and returns an instance, so `__init__` +# is also validated. `__init__` expects `x: int`, but got `str`. +Foo("hello") # error: [invalid-argument-type] + +# No overload matches. +Foo() # error: [no-matching-overload] +``` + +### Overloaded metaclass `__call__` returning only non-instance types + +When all overloads of a metaclass `__call__` return non-instance types, the metaclass fully +overrides `type.__call__` and `__init__` is not checked. + +```py +from typing import Any, overload + +class Meta(type): + @overload + def __call__(cls, x: int) -> int: ... + @overload + def __call__(cls, x: str) -> str: ... + def __call__(cls, x: int | str) -> Any: + return x + +class Bar(metaclass=Meta): + def __init__(self, x: int, y: int) -> None: + pass + +# `__init__` is not checked: it requires two `int` args, but we only pass one. +# No error is raised because the metaclass `__call__` controls construction. +reveal_type(Bar(1)) # revealed: int +reveal_type(Bar("hello")) # revealed: str +``` + +### Overloaded metaclass `__call__` with non-class return forms + +When all overloads return non-instance types that aren't simple class instances (e.g., `Callable`), +`__init__` should still be skipped. + +```py +from typing import Any, Callable, overload + +class Meta(type): + @overload + def __call__(cls, x: int) -> Callable[[], int]: ... + @overload + def __call__(cls, x: str) -> Callable[[], str]: ... + def __call__(cls, x: int | str) -> Any: + return lambda: x + +class Baz(metaclass=Meta): + def __init__(self, x: int, y: int) -> None: + pass + +# `__init__` is not checked: it requires two `int` args, but we only pass one. +# No error is raised because the metaclass `__call__` controls construction. +reveal_type(Baz(1)) # revealed: () -> int +reveal_type(Baz("hello")) # revealed: () -> str +``` + ## Default ```py diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index c925e1167944b9..ff5bee7fa0f874 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -4865,8 +4865,82 @@ impl<'db> Type<'db> { _ => self, }; - // As of now we do not model custom `__call__` on meta-classes, so the code below - // only deals with interplay between `__new__` and `__init__` methods. + // Check for a custom `__call__` on the metaclass (excluding `type.__call__`). + // Per the spec: if the return type is not an instance of the class being constructed + // (or a subclass thereof), use the metaclass `__call__` directly and skip `__new__`/`__init__`. + // If the return type is dynamic/contains typevars, fall through to evaluate `__new__`/`__init__`. + let metaclass_dunder_call = self_type.member_lookup_with_policy( + db, + "__call__".into(), + MemberLookupPolicy::NO_INSTANCE_FALLBACK + | MemberLookupPolicy::META_CLASS_NO_TYPE_FALLBACK, + ); + + let Some(constructor_instance_ty) = self_type.to_instance(db) else { + return fallback_bindings(); + }; + + let is_non_instance_overload = |sig: &Signature<'db>| { + !sig.return_ty.has_typevar(db) + && ConstructorReturnDisposition::of(db, sig.return_ty, class).is_not_instance() + }; + + // For overloaded metaclass `__call__` with mixed return types, we save the + // non-instance overloads and combine them with `__init__` after lookup. + // For instance-returning metaclass `__call__`, we save the bindings to include + // in the union with `__new__`/`__init__` so its parameters also get validated. + let mut metaclass_mixed_non_instance_sigs = None; + let mut metaclass_instance_bindings: Option<(Bindings<'db>, Type<'db>)> = None; + + if let Place::Defined(DefinedPlace { + ty: Type::BoundMethod(metaclass_call_method), + .. + }) = metaclass_dunder_call.place + { + let signature = metaclass_call_method.function(db).signature(db); + + let non_instance_sigs: SmallVec<[Signature<'db>; 4]> = signature + .overloads + .iter() + .filter(|sig| is_non_instance_overload(sig)) + .cloned() + .collect(); + + if !non_instance_sigs.is_empty() { + if non_instance_sigs.len() == signature.overloads.len() { + // All overloads return non-instance types: use metaclass `__call__` directly. + return Type::BoundMethod(metaclass_call_method).bindings(db); + } + + // Mixed: save ALL overloads with correct per-overload return types. + // Non-instance overloads keep their return type; instance-returning + // overloads get `constructor_instance_ty`. We defer `__init__` validation + // to call checking time (only when the matched overload is instance-returning). + let metaclass_self = metaclass_call_method.self_instance(db); + metaclass_mixed_non_instance_sigs = Some( + signature + .overloads + .iter() + .map(|sig| { + let mut bound = sig.bind_self(db, Some(metaclass_self)); + if !is_non_instance_overload(sig) { + bound.return_ty = constructor_instance_ty; + } + bound + }) + .collect::; 4]>>(), + ); + } else { + // All overloads return instance/uncertain types. Save the metaclass + // `__call__` bindings so we can include them in the union with + // `__new__`/`__init__` — this ensures the metaclass parameters are validated. + let metaclass_call_ty = Type::BoundMethod(metaclass_call_method); + metaclass_instance_bindings = + Some((metaclass_call_ty.bindings(db), metaclass_call_ty)); + } + } + + // The code below deals with interplay between `__new__` and `__init__` methods. // The logic is roughly as follows: // 1. If `__new__` is defined anywhere in the MRO (except for `object`, since it is always // present), we validate the constructor arguments against it. We then validate `__init__`, @@ -4877,10 +4951,6 @@ impl<'db> Type<'db> { // the way to `object` (single `self` argument call). This time it is correct to // fallback to `object.__init__`, since it will indeed check that no arguments are // passed. - // - // Note that we currently ignore `__new__` return type, since we do not yet support `Self` - // and most builtin classes use it as return type annotation. We always return the instance - // type. // Lookup `__new__` method in the MRO up to, but not including, `object`. Also, we must // avoid `__new__` on `type` since per descriptor protocol, if `__new__` is not defined on @@ -4892,16 +4962,10 @@ impl<'db> Type<'db> { // constructor-call bindings. let new_method = self_type.lookup_dunder_new(db); - let Some(constructor_instance_ty) = self_type.to_instance(db) else { - return fallback_bindings(); - }; - // Construct an instance type to look up `__init__`. We use `self_type` (possibly identity- // specialized) so the instance retains inferable class typevars during constructor checking. // TODO: we should use the actual return type of `__new__` to determine the instance type - let Some(lookup_init_ty) = self_type.to_instance(db) else { - return fallback_bindings(); - }; + let lookup_init_ty = constructor_instance_ty; // Lookup the `__init__` instance method in the MRO, excluding `object` initially; we only // fall back to `object.__init__` in the `__new__`-absent case (see rules above). @@ -4927,6 +4991,86 @@ impl<'db> Type<'db> { None => (None, false), }; + // Per the spec: "If the evaluated return type of `__new__` is not the class being + // constructed (or a subclass thereof), a type checker should assume that the `__init__` + // method will not be called." Also: "an explicit return type of `Any` should be treated + // as a type that is not an instance of the class being constructed." + // + // Use a structural class-based check instead of full assignability/subtyping to avoid + // deep recursive relation checks on large codebases (for example static-frame). + // + // For overloaded `__new__` with mixed return types (some instance, some not), we build + // combined overloads where non-instance overloads keep their return type and instance- + // returning overloads get `constructor_instance_ty` as their return type. + let mut new_mixed_non_instance_sigs = None; + + if let Some((ref new_bindings_inner, ref new_callable)) = new_bindings { + let func = match *new_callable { + Type::FunctionLiteral(func) => Some(func), + Type::BoundMethod(method) => Some(method.function(db)), + _ => None, + }; + if let Some(func) = func { + let enclosing_class = nearest_enclosing_class( + db, + semantic_index(db, func.file(db)), + func.definition(db).scope(db), + ); + let is_object_or_type = enclosing_class.is_some_and(|cls| { + matches!(cls.known(db), Some(KnownClass::Type | KnownClass::Object)) + }); + + if !is_object_or_type { + let signature = func.signature(db); + + let non_instance_sigs: SmallVec<[Signature<'db>; 4]> = signature + .overloads + .iter() + .filter(|sig| is_non_instance_overload(sig)) + .cloned() + .collect(); + + if !non_instance_sigs.is_empty() { + if non_instance_sigs.len() == signature.overloads.len() { + // All overloads return non-instance types: use `__new__` directly. + let new_return_ty = new_bindings_inner + .iter() + .flat_map(|cb| cb.overloads().iter()) + .map(|overload| overload.signature.return_ty) + .find(|return_ty| { + !return_ty.is_unknown() + && !return_ty.has_typevar(db) + && ConstructorReturnDisposition::of(db, *return_ty, class) + .is_not_instance() + }) + .unwrap_or(constructor_instance_ty); + let (new_bindings, _) = new_bindings.unwrap(); + return new_bindings + .with_generic_context(db, class_generic_context) + .with_constructor_instance_type(new_return_ty); + } + + // Mixed: save ALL overloads with correct per-overload return + // types. Non-instance overloads keep their return type; + // instance-returning overloads get `constructor_instance_ty`. + new_mixed_non_instance_sigs = Some( + signature + .overloads + .iter() + .map(|sig| { + let mut bound = sig.bind_self(db, Some(self_type)); + if !is_non_instance_overload(sig) { + bound.return_ty = constructor_instance_ty; + } + bound + }) + .collect::; 4]>>(), + ); + } + } + } + } + // Only fall back to `object.__init__` when `__new__` is absent. let init_bindings = match (&init_method_no_object.place, has_any_new) { ( @@ -4982,23 +5126,64 @@ impl<'db> Type<'db> { (Place::Undefined, true) => None, }; + // If `__new__` had mixed return types (some non-instance, some instance), keep all + // `__new__` overloads with correct per-overload return types, and attach `__init__` + // bindings as deferred conditional validation. At call checking time, `__init__` is + // only validated when the matched `__new__` overload is instance-returning. + if let Some(combined_sigs) = new_mixed_non_instance_sigs { + let mut bindings: Bindings<'db> = + CallableBinding::from_overloads(self_type, combined_sigs).into(); + + if let Some((init_bindings, _)) = init_bindings { + bindings.set_mixed_constructor_init(constructor_instance_ty, init_bindings); + } + + return bindings.with_generic_context(db, class_generic_context); + } + + // If the metaclass `__call__` had mixed return types (some non-instance, some instance), + // keep all overloads with per-overload return types, and attach `__init__` bindings as + // deferred conditional validation. At call checking time, `__init__` is only validated + // when the matched overload is instance-returning. + if let Some(combined_sigs) = metaclass_mixed_non_instance_sigs { + let mut bindings: Bindings<'db> = + CallableBinding::from_overloads(self, combined_sigs).into(); + + if let Some((init_bindings, _)) = init_bindings { + bindings.set_mixed_constructor_init(constructor_instance_ty, init_bindings); + } + + return bindings.with_generic_context(db, class_generic_context); + } + let bindings = if let Some(bindings) = missing_init_bindings { bindings } else { - match (new_bindings, init_bindings) { - (Some((new_bindings, new_callable)), Some((init_bindings, init_callable))) => { - let callable_type = UnionBuilder::new(db) - .add(new_callable) - .add(init_callable) - .build(); - // Use both `__new__` and `__init__` bindings so argument inference/checking - // happens under the combined constructor-call type context. - // In ty unions of callables are checked as "all must accept". - Bindings::from_union(callable_type, [new_bindings, init_bindings]) - } - (Some((new_bindings, _)), None) => new_bindings, - (None, Some((init_bindings, _))) => init_bindings, - (None, None) => return fallback_bindings(), + // Collect all bindings that must accept the constructor call. + // This may include `__new__`, `__init__`, and/or a metaclass `__call__`. + let mut all_bindings: SmallVec<[Bindings<'db>; 3]> = SmallVec::new(); + let mut callable_type_builder = UnionBuilder::new(db); + + if let Some((metaclass_bindings, metaclass_ty)) = metaclass_instance_bindings { + all_bindings.push(metaclass_bindings); + callable_type_builder = callable_type_builder.add(metaclass_ty); + } + if let Some((new_bindings, new_callable)) = new_bindings { + all_bindings.push(new_bindings); + callable_type_builder = callable_type_builder.add(new_callable); + } + if let Some((init_bindings, init_callable)) = init_bindings { + all_bindings.push(init_bindings); + callable_type_builder = callable_type_builder.add(init_callable); + } + + match all_bindings.len() { + 0 => return fallback_bindings(), + 1 => all_bindings.into_iter().next().unwrap(), + _ => { + let callable_type = callable_type_builder.build(); + Bindings::from_union(callable_type, all_bindings) + } } }; @@ -7149,6 +7334,98 @@ impl<'db> VarianceInferable<'db> for Type<'db> { } } +/// Whether a return type from `__new__` or metaclass `__call__` is an instance +/// of the class being constructed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ConstructorReturnDisposition { + /// The return type is definitely an instance of the constructed class. + Instance, + /// The return type is definitely *not* an instance of the constructed class. + NotInstance, + /// Cannot be determined (e.g., contains type variables or `Unknown`). + Uncertain, +} + +impl ConstructorReturnDisposition { + /// Determine whether `return_ty` is an instance of `constructor_class`. + fn of<'db>(db: &'db dyn Db, return_ty: Type<'db>, constructor_class: ClassType<'db>) -> Self { + match return_ty.resolve_type_alias(db) { + Type::Union(union) => { + let mut saw_uncertain = false; + for element in union.elements(db) { + match Self::of(db, *element, constructor_class) { + Self::NotInstance => return Self::NotInstance, + Self::Instance => {} + Self::Uncertain => saw_uncertain = true, + } + } + if saw_uncertain { + Self::Uncertain + } else { + Self::Instance + } + } + Type::Dynamic(DynamicType::Any) => Self::NotInstance, + Type::Dynamic(_) | Type::TypeVar(_) => Self::Uncertain, + Type::Never => Self::Instance, + Type::NominalInstance(instance) => { + // Check origin class identity first, since `is_subclass_of` returns false + // for Generic vs NonGeneric variants of the same class. + if instance.class(db).class_literal(db) == constructor_class.class_literal(db) + || instance.class(db).is_subclass_of(db, constructor_class) + { + Self::Instance + } else { + Self::NotInstance + } + } + Type::SubclassOf(subclass_of_ty) => match subclass_of_ty.subclass_of().into_class(db) { + Some(class) if class.is_subclass_of(db, constructor_class) => Self::Instance, + Some(_) => Self::NotInstance, + None => Self::Uncertain, + }, + ty => { + // Fall back to checking the meta type for remaining concrete + // types (e.g., `Callable`, `FunctionLiteral`, literals, etc.). + // If the return type's class is not a subclass of the + // constructor class, the return type can't be an instance. + let meta = ty.to_meta_type(db); + match meta { + Type::ClassLiteral(class_literal) => { + if class_literal + .default_specialization(db) + .is_subclass_of(db, constructor_class) + { + Self::Instance + } else { + Self::NotInstance + } + } + Type::GenericAlias(alias) => { + if ClassType::from(alias).is_subclass_of(db, constructor_class) { + Self::Instance + } else { + Self::NotInstance + } + } + Type::NominalInstance(instance) => { + if instance.class(db).is_subclass_of(db, constructor_class) { + Self::Instance + } else { + Self::NotInstance + } + } + _ => Self::Uncertain, + } + } + } + } + + const fn is_not_instance(self) -> bool { + matches!(self, Self::NotInstance) + } +} + #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, get_size2::GetSize)] pub enum PromoteLiteralsMode { On, diff --git a/crates/ty_python_semantic/src/types/call/bind.rs b/crates/ty_python_semantic/src/types/call/bind.rs index 1b61d52bdf73e7..9b7471d8c97743 100644 --- a/crates/ty_python_semantic/src/types/call/bind.rs +++ b/crates/ty_python_semantic/src/types/call/bind.rs @@ -56,6 +56,38 @@ use ruff_db::diagnostic::{Annotation, Diagnostic, SubDiagnostic, SubDiagnosticSe use ruff_python_ast::{self as ast, ArgOrKeyword, PythonVersion}; use ty_module_resolver::KnownModule; +/// Deferred `__init__` validation for constructors with mixed `__new__` overloads. +/// +/// When `__new__` is overloaded with a mix of instance-returning and non-instance-returning +/// overloads (e.g., one overload returns `int` and another returns `Self`), we can't validate +/// `__init__` at binding time because we don't yet know which `__new__` overload will match. +/// Instead, we store the `__init__` bindings here and validate them after overload resolution, +/// only when the matched `__new__` overload is instance-returning. +#[derive(Debug, Clone)] +struct MixedConstructorInit<'db> { + /// The expected instance type of the class being constructed, used to determine + /// whether the matched `__new__` overload is instance-returning. + /// + /// Instance-returning overloads have their `return_ty` explicitly set to this value + /// when building the mixed overload list (see `types.rs`), so equality comparison + /// is sufficient here. + instance_type: Type<'db>, + /// The `__init__` bindings to validate conditionally. + init_bindings: Bindings<'db>, +} + +impl<'db> MixedConstructorInit<'db> { + /// Returns `true` if any matched overload is instance-returning (i.e., its return type + /// equals `self.instance_type`). + fn is_instance_returning(&self, elements: &[CallableBinding<'db>]) -> bool { + elements.iter().any(|element| { + element + .matching_overloads() + .any(|(_, overload)| overload.return_ty == self.instance_type) + }) + } +} + /// Binding information for a possible union of callables. At a call site, the arguments must be /// compatible with _all_ of the types in the union for the call to be valid. /// @@ -80,6 +112,12 @@ pub(crate) struct Bindings<'db> { /// Whether each argument will be used as a value and/or a type form in this call. argument_forms: ArgumentForms, + + /// When `__new__` has overloads with mixed return types (some instance, some not), + /// this holds the `__init__` bindings for deferred validation. After overload resolution + /// determines which `__new__` overload matched, `__init__` is only checked if the matched + /// overload is instance-returning. See [`MixedConstructorInit`] for details. + mixed_constructor_init: Option>>, } impl<'db> Bindings<'db> { @@ -109,6 +147,7 @@ impl<'db> Bindings<'db> { constructor_instance_type: None, implicit_dunder_new_is_possibly_unbound, implicit_dunder_init_is_possibly_unbound, + mixed_constructor_init: None, } } @@ -154,9 +193,31 @@ impl<'db> Bindings<'db> { ); } } + if let Some(ref mut mixed_init) = self.mixed_constructor_init { + for binding in &mut mixed_init.init_bindings.elements { + for overload in &mut binding.overloads { + overload.signature.generic_context = GenericContext::merge_optional( + db, + overload.signature.generic_context, + Some(generic_context), + ); + } + } + } self } + pub(crate) fn set_mixed_constructor_init( + &mut self, + instance_type: Type<'db>, + init_bindings: Bindings<'db>, + ) { + self.mixed_constructor_init = Some(Box::new(MixedConstructorInit { + instance_type, + init_bindings, + })); + } + pub(crate) fn set_dunder_call_is_possibly_unbound(&mut self) { for binding in &mut self.elements { binding.dunder_call_is_possibly_unbound = true; @@ -195,6 +256,7 @@ impl<'db> Bindings<'db> { implicit_dunder_new_is_possibly_unbound: self.implicit_dunder_new_is_possibly_unbound, implicit_dunder_init_is_possibly_unbound: self.implicit_dunder_init_is_possibly_unbound, elements: self.elements.into_iter().map(f).collect(), + mixed_constructor_init: self.mixed_constructor_init, } } @@ -216,6 +278,12 @@ impl<'db> Bindings<'db> { for binding in &mut self.elements { binding.match_parameters(db, arguments, &mut argument_forms); } + if let Some(ref mut mixed_init) = self.mixed_constructor_init { + let mut init_forms = ArgumentForms::new(arguments.len()); + for binding in &mut mixed_init.init_bindings.elements { + binding.match_parameters(db, arguments, &mut init_forms); + } + } argument_forms.shrink_to_fit(); self.argument_forms = argument_forms; self @@ -271,6 +339,24 @@ impl<'db> Bindings<'db> { self.evaluate_known_cases(db, argument_types, dataclass_field_specifiers); + // For mixed `__new__` overloads: check `__init__` if matched overload is instance-returning. + let mut init_error = false; + if let Some(ref mut mixed_init) = self.mixed_constructor_init { + if mixed_init.is_instance_returning(&self.elements) + && mixed_init + .init_bindings + .check_types_impl( + db, + argument_types, + call_expression_tcx, + dataclass_field_specifiers, + ) + .is_err() + { + init_error = true; + } + } + // In order of precedence: // // - If every union element is Ok, then the union is too. @@ -300,6 +386,10 @@ impl<'db> Bindings<'db> { any_binding_error |= matches!(result, Err(CallErrorKind::BindingError)); all_not_callable &= matches!(result, Err(CallErrorKind::NotCallable)); } + if init_error { + any_binding_error = true; + all_ok = false; + } if all_ok { Ok(()) @@ -330,6 +420,57 @@ impl<'db> Bindings<'db> { // Constructor calls should combine `__new__`/`__init__` specializations instead of unioning. fn constructor_return_type(&self, db: &'db dyn Db) -> Option> { let constructor_instance_type = self.constructor_instance_type?; + + let class_literal = constructor_instance_type + .as_nominal_instance() + .and_then(|inst| inst.class(db).static_class_literal(db)) + .map(|(lit, _)| lit); + + // If any matched overload's signature return type, when resolved with the inferred + // specialization, is a non-instance type (e.g. `__new__[S] -> S` with `S` inferred as + // `str`), use that resolved type directly. This handles arbitrary `__new__` return + // types like `S`, `list[S]`, etc. + let constructor_class = constructor_instance_type + .as_nominal_instance() + .map(|inst| inst.class(db)); + for binding in &self.elements { + let Some((_, overload)) = binding.matching_overloads().next() else { + continue; + }; + let sig_return = overload.signature.return_ty; + if !sig_return.has_typevar(db) { + continue; + } + // Fast path: if the declared return is already a specialization of the + // constructed class, this overload is instance-returning and cannot trigger + // the non-instance early return. + if class_literal + .and_then(|lit| sig_return.specialization_of(db, lit)) + .is_some() + { + continue; + } + let Some(specialization) = overload.specialization else { + continue; + }; + let resolved = sig_return.apply_specialization(db, specialization); + if resolved.has_typevar(db) || resolved.is_unknown() { + continue; + } + // Check if the resolved type is an instance of the constructing class or a + // base class. If so, it's a normal instance-returning `__new__` and should be + // handled by the standard constructor return logic below. + let is_instance_return = resolved.as_nominal_instance().is_some_and(|inst| { + constructor_class.is_some_and(|cc| { + cc.class_literal(db) == inst.class(db).class_literal(db) + || cc.is_subclass_of(db, inst.class(db)) + }) + }); + if !is_instance_return { + return Some(resolved); + } + } + let Some(class_specialization) = constructor_instance_type.class_specialization(db) else { return Some(constructor_instance_type); }; @@ -342,10 +483,18 @@ impl<'db> Bindings<'db> { let Some((_, overload)) = binding.matching_overloads().next() else { continue; }; - let Some(specialization) = overload.specialization else { - continue; - }; - let Some(specialization) = specialization.restrict(db, class_context) else { + // Prefer extracting the class specialization from the resolved overload return type. + // Fall back to restricting the inferred specialization to class-level type + // variables. + let specialization = class_literal + // Fast path: use the already-resolved overload return type when possible. + .and_then(|lit| overload.return_ty.specialization_of(db, lit)) + .or_else(|| { + overload + .specialization + .and_then(|s| s.restrict(db, class_context)) + }); + let Some(specialization) = specialization else { continue; }; combined = Some(match combined { @@ -411,18 +560,24 @@ impl<'db> Bindings<'db> { // errors as normal. if let Some(binding) = self.single_element() { binding.report_diagnostics(context, node, None); - return; + } else { + for binding in self { + if binding.as_result().is_ok() { + continue; + } + let union_diag = UnionDiagnostic { + callable_type: self.callable_type(), + binding, + }; + binding.report_diagnostics(context, node, Some(&union_diag)); + } } - for binding in self { - if binding.as_result().is_ok() { - continue; + // Report `__init__` diagnostics for mixed constructor overloads + if let Some(ref mixed_init) = self.mixed_constructor_init { + if mixed_init.is_instance_returning(&self.elements) { + mixed_init.init_bindings.report_diagnostics(context, node); } - let union_diag = UnionDiagnostic { - callable_type: self.callable_type(), - binding, - }; - binding.report_diagnostics(context, node, Some(&union_diag)); } } @@ -1660,6 +1815,7 @@ impl<'db> From> for Bindings<'db> { constructor_instance_type: None, implicit_dunder_new_is_possibly_unbound: false, implicit_dunder_init_is_possibly_unbound: false, + mixed_constructor_init: None, } } } @@ -1685,6 +1841,7 @@ impl<'db> From> for Bindings<'db> { constructor_instance_type: None, implicit_dunder_new_is_possibly_unbound: false, implicit_dunder_init_is_possibly_unbound: false, + mixed_constructor_init: None, } } } @@ -3378,7 +3535,7 @@ impl<'a, 'db> ArgumentTypeChecker<'a, 'db> { return ty; } - let return_ty = self.constructor_instance_type.unwrap_or(self.return_ty); + let return_ty = self.return_ty; let mut combined_tcx = TypeContext::default(); let mut variance_in_return = TypeVarVariance::Bivariant; @@ -4024,9 +4181,27 @@ impl<'db> Binding<'db> { matcher.match_keyword_variadic(db, keywords_index, keywords_type); } // For constructor calls, return the constructed instance type (not `__init__`'s `None`). - self.return_ty = self - .constructor_instance_type - .unwrap_or(self.signature.return_ty); + // If `__new__` declares a return type that is a specialization of the class being + // constructed (e.g. `C[tuple[S, S]]`), prefer that over the generic constructor instance + // type (`C[T]`), so that method-level type variables are properly solved. + self.return_ty = if let Some(cit) = self.constructor_instance_type { + let sig_return_is_same_class = cit + .as_nominal_instance() + .and_then(|inst| inst.class(db).static_class_literal(db)) + .is_some_and(|(lit, _)| { + self.signature + .return_ty + .specialization_of(db, lit) + .is_some() + }); + if sig_return_is_same_class { + self.signature.return_ty + } else { + cit + } + } else { + self.signature.return_ty + }; self.parameter_tys = vec![None; parameters.len()].into_boxed_slice(); self.variadic_argument_matched_to_variadic_parameter = matcher.variadic_argument_matched_to_variadic_parameter; diff --git a/scripts/check_ecosystem.py b/scripts/check_ecosystem.py index 8bc2dbea550d0a..fcd1bafc3365d3 100755 --- a/scripts/check_ecosystem.py +++ b/scripts/check_ecosystem.py @@ -383,7 +383,7 @@ async def limited_parallelism(coroutine: T) -> T: errors = 0 for diff in diffs.values(): - if isinstance(diff, Exception): + if isinstance(diff, BaseException): errors += 1 else: total_removed += len(diff.removed) @@ -399,7 +399,7 @@ async def limited_parallelism(coroutine: T) -> T: print() for (org, repo), diff in diffs.items(): - if isinstance(diff, Exception): + if isinstance(diff, BaseException): changes = "error" print(f"
{repo} ({changes})") repo = repositories[(org, repo)]