[ty] Limit recursion depth when displaying self-referential function types

crates/ty_python_semantic/resources/mdtest/ty_extensions.md

@@ -439,6 +439,36 @@ def foo(x: "TypeOf[foo]"):
     reveal_type(x)  # revealed: def foo(x: def foo(...)) -> Unknown
 ```
 
+## Deeply nested `TypeOf` chains
+
+Multiple redefinitions of a function with `TypeOf[foo]` as the return type create a chain of
+distinct function types. The display of such chains is truncated to prevent extremely long output:
+
+```py
+from ty_extensions import TypeOf
+
+def foo() -> TypeOf[foo]:  # error: [unresolved-reference]
+    return foo
+
+def foo() -> TypeOf[foo]:
+    return foo  # error: [invalid-return-type]
+
+def foo() -> TypeOf[foo]:
+    return foo  # error: [invalid-return-type]
+
+def foo() -> TypeOf[foo]:
+    return foo  # error: [invalid-return-type]
+
+def foo() -> TypeOf[foo]:
+    return foo  # error: [invalid-return-type]
+
+def foo() -> TypeOf[foo]:
+    return foo  # error: [invalid-return-type]
+
+# Truncated after 4 levels of function type nesting:
+reveal_type(foo)  # revealed: def foo() -> def foo() -> def foo() -> def foo() -> def foo(...)
+```
+
 ## `CallableTypeOf`
 
 The `CallableTypeOf` special form can be used to extract the `Callable` structural type inhabited by

crates/ty_python_semantic/src/types/display.rs

@@ -1468,8 +1468,13 @@ pub(crate) struct DisplayFunctionType<'db> {
 
 impl<'db> FmtDetailed<'db> for DisplayFunctionType<'db> {
     fn fmt_detailed(&self, f: &mut TypeWriter<'_, '_, 'db>) -> fmt::Result {
-        // Detect self-referential function types to prevent infinite recursion.
-        if self.settings.visited_function_types.contains(&self.ty) {
+        // Detect self-referential function types to prevent infinite recursion,
+        // and limit display depth for chains of different function types
+        // (e.g. multiple redefinitions with `TypeOf[foo]` return types).
+        const MAX_FUNCTION_TYPE_DISPLAY_DEPTH: usize = 4;
+        if self.settings.visited_function_types.contains(&self.ty)
+            || self.settings.visited_function_types.len() >= MAX_FUNCTION_TYPE_DISPLAY_DEPTH
+        {
             f.set_invalid_type_annotation();
             f.write_str("def ")?;
             write!(f, "{}", self.ty.name(self.db))?;