[ty] Respect __new__ and metaclass __call__ return types

crates/ruff_benchmark/benches/ty_walltime.rs

@@ -171,7 +171,7 @@ static PANDAS: Benchmark = Benchmark::new(
         max_dep_date: "2025-06-17",
         python_version: PythonVersion::PY312,
     },
-    4500,
+    5500,
 );
 
 static PYDANTIC: Benchmark = Benchmark::new(
@@ -202,7 +202,7 @@ static SYMPY: Benchmark = Benchmark::new(
         max_dep_date: "2025-06-17",
         python_version: PythonVersion::PY312,
     },
-    13600,
+    13800,
 );
 
 static TANJUN: Benchmark = Benchmark::new(

crates/ty_python_semantic/resources/mdtest/call/constructor.md

@@ -21,11 +21,6 @@ Since every class has `object` in its MRO, the default implementations are `obje
     `object`), no arguments are accepted and `TypeError` is raised if any are passed.
 - If `__new__` is defined but `__init__` is not, `object.__init__` will allow arbitrary arguments!
 
-As of today there are a number of behaviors that we do not support:
-
-- `__new__` is assumed to return an instance of the class on which it is called
-- User defined `__call__` on metaclass is ignored
-
 ## Creating an instance of the `object` class itself
 
 Test the behavior of the `object` class itself. As implementation has to ignore `object` own methods
@@ -261,6 +256,142 @@ class Box(Generic[T]):
 reveal_type(Box(1))  # revealed: Box[int]
 ```
 
+## `__new__` with method-level type variables mapping to class specialization
+
+When `__new__` has its own type parameters that map to the class's type parameter through the return
+type, we should correctly infer the class specialization.
+
+```toml
+[environment]
+python-version = "3.12"
+```
+
+```py
+class C[T]:
+    x: T
+
+    def __new__[S](cls, x: S) -> "C[tuple[S, S]]":
+        return object.__new__(cls)
+
+reveal_type(C(1))  # revealed: C[tuple[int, int]]
+reveal_type(C("hello"))  # revealed: C[tuple[str, str]]
+```
+
+## `__new__` with arbitrary generic return types
+
+When `__new__` has method-level type variables in the return type that don't map to the class's type
+parameters, the resolved return type should be used directly.
+
+```toml
+[environment]
+python-version = "3.12"
+```
+
+```py
+class C:
+    def __new__[S](cls, x: S) -> S:
+        return x
+
+reveal_type(C("foo"))  # revealed: Literal["foo"]
+reveal_type(C(1))  # revealed: Literal[1]
+```
+
+## `__new__` returning non-instance generic containers
+
+```toml
+[environment]
+python-version = "3.12"
+```
+
+```py
+class C:
+    def __new__[S](cls, x: S) -> list[S]:
+        return [x]
+
+reveal_type(C("foo"))  # revealed: list[Literal["foo"]]
+reveal_type(C(1))  # revealed: list[Literal[1]]
+```
+
+## Overloaded `__new__` with generic return types
+
+Overloaded `__new__` methods should correctly resolve to the matching overload and infer the class
+specialization from the overload's return type.
+
+```py
+from typing import Generic, Iterable, TypeVar, overload
+
+T = TypeVar("T")
+T1 = TypeVar("T1")
+T2 = TypeVar("T2")
+
+class MyZip(Generic[T]):
+    @overload
+    def __new__(cls) -> "MyZip[object]": ...
+    @overload
+    def __new__(cls, iter1: Iterable[T1], iter2: Iterable[T2]) -> "MyZip[tuple[T1, T2]]": ...
+    def __new__(cls, *args, **kwargs) -> "MyZip[object]":
+        raise NotImplementedError
+
+def check(a: tuple[int, ...], b: tuple[str, ...]) -> None:
+    reveal_type(MyZip(a, b))  # revealed: MyZip[tuple[int, str]]
+    reveal_type(MyZip())  # revealed: MyZip[object]
+```
+
+## Overloaded `__new__` with mixed instance and non-instance return types
+
+When `__new__` is overloaded and some overloads return the class instance type while others return a
+different type, the decision of whether to skip `__init__` should be made per-overload based on
+which overload matched at the call site. Non-instance-returning overloads use `__new__` directly,
+while instance-returning overloads additionally validate `__init__`.
+
+```py
+from typing import overload
+from typing_extensions import Self
+
+class C:
+    @overload
+    def __new__(cls, x: int) -> int: ...
+    @overload
+    def __new__(cls, x: str) -> Self: ...
+    def __new__(cls, x: int | str) -> object: ...
+    def __init__(self, x: str) -> None: ...
+
+# The `int` overload is selected; its return type is not an instance of `C`,
+# so `__init__` is skipped and the return type is `int`.
+reveal_type(C(1))  # revealed: int
+
+# The `str -> Self` overload would return an instance of `C`, so `__init__` is
+# also checked. `__init__` accepts `x: str`, so the call succeeds.
+reveal_type(C("foo"))  # revealed: C
+```
+
+## `__init__` incompatible with instance-returning `__new__` overloads
+
+When `__init__` parameters are incompatible with the arguments that would match instance-returning
+`__new__` overloads, `__init__` errors are reported. The return type still comes from `__new__`.
+
+```py
+from typing import overload
+from typing_extensions import Self
+
+class D:
+    @overload
+    def __new__(cls, x: int) -> int: ...
+    @overload
+    def __new__(cls, x: str) -> Self: ...
+    def __new__(cls, x: int | str) -> object: ...
+    def __init__(self) -> None: ...
+
+# The `int` overload is selected; its return type is not an instance of `D`,
+# so `__init__` is skipped and the return type is `int`.
+reveal_type(D(1))  # revealed: int
+
+# The `str -> Self` overload returns an instance of `D`, so `__init__` is also
+# checked. `__init__` takes no args, so it reports an error.
+# error: [too-many-positional-arguments]
+reveal_type(D("foo"))  # revealed: D
+```
+
 ## Constructor calls through `type[T]` with a bound TypeVar
 
 ```py

crates/ty_python_semantic/resources/mdtest/classes.md

@@ -1,5 +1,102 @@
 # Class definitions
 
+## `__new__` return type
+
+Python's `__new__` method can return any type, not just an instance of the class. When `__new__`
+returns a type that is not assignable to the class instance type, we use that return type directly.
+
+### `__new__` returning a different type
+
+```py
+class ReturnsInt:
+    def __new__(cls) -> int:
+        return 42
+
+reveal_type(ReturnsInt())  # revealed: int
+
+x: int = ReturnsInt()  # OK
+y: ReturnsInt = ReturnsInt()  # error: [invalid-assignment]
+```
+
+### `__new__` returning a union type
+
+```py
+class MaybeInt:
+    def __new__(cls, value: str) -> "int | MaybeInt":
+        try:
+            return int(value)
+        except ValueError:
+            return object.__new__(cls)
+
+reveal_type(MaybeInt("42"))  # revealed: int | MaybeInt
+
+a: int | MaybeInt = MaybeInt("42")  # OK
+b: int = MaybeInt("42")  # error: [invalid-assignment]
+```
+
+### `__new__` returning the class type
+
+When `__new__` returns the class type (or `Self`), the normal instance type is used.
+
+```py
+class Normal:
+    def __new__(cls) -> "Normal":
+        return object.__new__(cls)
+
+reveal_type(Normal())  # revealed: Normal
+```
+
+### `__new__` with no return type annotation
+
+When `__new__` has no return type annotation, we fall back to the instance type.
+
+```py
+class NoAnnotation:
+    def __new__(cls):
+        return object.__new__(cls)
+
+reveal_type(NoAnnotation())  # revealed: NoAnnotation
+```
+
+### `__new__` returning `Any`
+
+Per the spec, "an explicit return type of `Any` should be treated as a type that is not an instance
+of the class being constructed." This means `__init__` is not called and the return type is `Any`.
+
+```py
+from typing import Any
+
+class ReturnsAny:
+    def __new__(cls) -> Any:
+        return 42
+
+    def __init__(self, x: int) -> None:
+        pass
+
+# __init__ is skipped because `-> Any` is treated as non-instance per spec
+reveal_type(ReturnsAny())  # revealed: Any
+```
+
+### `__new__` returning a union containing `Any`
+
+When `__new__` returns a union containing `Any`, `Any` is not a subtype of the instance type, so
+`__init__` is skipped.
+
+```py
+from typing import Any
+
+class MaybeAny:
+    def __new__(cls, value: int) -> "MaybeAny | Any":
+        if value > 0:
+            return object.__new__(cls)
+        return None
+
+    def __init__(self, value: int) -> None:
+        pass
+
+reveal_type(MaybeAny(1))  # revealed: MaybeAny | Any
+```
+
 ## Deferred resolution of bases
 
 ### Only the stringified name is deferred

crates/ty_python_semantic/resources/mdtest/external/sqlmodel.md

@@ -19,11 +19,16 @@ class User(SQLModel):
     name: str
 
 user = User(id=1, name="John Doe")
-reveal_type(user.id)  # revealed: int
-reveal_type(user.name)  # revealed: str
+# TODO: these should be `int` and `str` once we add pydantic model synthesis.
+# Currently `Any` because `SQLModel.__new__` is annotated as `-> Any`, and the spec says
+# "an explicit return type of `Any` should be treated as a type that is not an instance of
+# the class being constructed."
+reveal_type(user.id)  # revealed: Any
+reveal_type(user.name)  # revealed: Any
 
 reveal_type(User.__init__)  # revealed: (self: User, *, id: int, name: str) -> None
 
-# error: [missing-argument]
+# No `missing-argument` error here: `SQLModel.__new__` returns `Any`, so per the spec
+# `__init__` is not evaluated and any arguments are accepted via `__new__`.
 User()
 ```

crates/ty_python_semantic/resources/mdtest/generics/legacy/classes.md

@@ -414,15 +414,15 @@ If either method comes from a generic base class, we don't currently use its inf
 to specialize the class.
 
 ```py
-from typing_extensions import Generic, TypeVar
+from typing_extensions import Generic, TypeVar, Self
 from ty_extensions import generic_context, into_callable
 
 T = TypeVar("T")
 U = TypeVar("U")
 V = TypeVar("V")
 
 class C(Generic[T, U]):
-    def __new__(cls, *args, **kwargs) -> "C[T, U]":
+    def __new__(cls, *args, **kwargs) -> Self:
         return object.__new__(cls)
 
 class D(C[V, int]):