Type django.utils.* against Django 4.2 (#1575)

* Type all utils.* against Django 4.2

* fix: cached_property.name could be None

* fix: SessionMiddleware was not typed properly

* fix: conditional_escape should return SafeString, closes #1474

* chore: remove utils from allowlist

* chore: remove unrelated unused ignores from allowlist for CI

* chore: mark Protocols as type_check_only

* Update scripts/stubtest/allowlist.txt

Co-authored-by: Nikita Sobolev <[email protected]>

* Update django-stubs/utils/deprecation.pyi

Co-authored-by: Nikita Sobolev <[email protected]>

* fix: MultieValueDict should return Self

* fixes from MR review

* chore: close #1269

* chore: remove legacy Self import from _typeshed

* chore: move async with its sync counterpart

* chore: remove Trans from typings

* chore: fix tests

* Apply suggestions from code review

Co-authored-by: Nikita Sobolev <[email protected]>

* chore: use decorator syntax for cached_property

---------

Co-authored-by: Nikita Sobolev <[email protected]>

django-stubs/utils/autoreload.pyi

@@ -1,6 +1,7 @@
 import threading
 import types
 from collections.abc import Callable, Iterable, Iterator
+from logging import Logger
 from pathlib import Path
 from typing import Any
 
@@ -11,6 +12,7 @@ from typing_extensions import ParamSpec
 
 _P = ParamSpec("_P")
 
+logger: Logger
 autoreload_started: Signal
 file_changed: Signal
 DJANGO_AUTORELOAD_ENV: str

django-stubs/utils/connection.pyi

@@ -1,6 +1,8 @@
 from collections.abc import Iterator, Mapping, Sequence
 from typing import Any, Generic, TypeVar
 
+from django.utils.functional import cached_property
+
 class ConnectionProxy:
     def __init__(self, connections: Mapping[str, Any], alias: str) -> None: ...
     def __getattr__(self, item: str) -> Any: ...
@@ -17,13 +19,14 @@ class BaseConnectionHandler(Generic[_T]):
     settings_name: str | None
     exception_class: type[Exception]
     thread_critical: bool
-    def __init__(self, settings: Any | None = ...) -> None: ...
-    @property
+    @cached_property
     def settings(self) -> dict[str, Any]: ...
+    def __init__(self, settings: Any | None = ...) -> None: ...
     def configure_settings(self, settings: dict[str, Any] | None) -> dict[str, Any]: ...
     def create_connection(self, alias: str) -> _T: ...
     def __getitem__(self, alias: str) -> _T: ...
     def __setitem__(self, key: str, value: _T) -> None: ...
     def __delitem__(self, key: str) -> None: ...
     def __iter__(self) -> Iterator[str]: ...
-    def all(self) -> Sequence[_T]: ...
+    def all(self, initialized_only: bool = ...) -> Sequence[_T]: ...
+    def close_all(self) -> None: ...

django-stubs/utils/crypto.pyi

@@ -1,13 +1,14 @@
 from collections.abc import Callable
 from hmac import HMAC
 
-using_sysrandom: bool
 RANDOM_STRING_CHARS: str
 
+class InvalidAlgorithm(ValueError): ...
+
 def salted_hmac(
     key_salt: bytes | str, value: bytes | str, secret: bytes | str | None = ..., *, algorithm: str = ...
 ) -> HMAC: ...
-def get_random_string(length: int = ..., allowed_chars: str = ...) -> str: ...
+def get_random_string(length: int, allowed_chars: str = ...) -> str: ...
 def constant_time_compare(val1: bytes | str, val2: bytes | str) -> bool: ...
 def pbkdf2(
     password: bytes | str,

django-stubs/utils/datastructures.pyi

@@ -1,8 +1,8 @@
-from collections.abc import Collection, Iterable, Iterator, Mapping, MutableSet
-from typing import Any, Generic, Protocol, Tuple, TypeVar, overload  # noqa: Y022
+from collections.abc import Collection, Iterable, Iterator, Mapping, MutableMapping, MutableSet
+from typing import Any, Generic, NoReturn, Protocol, Tuple, TypeVar, overload  # noqa: Y022
 
-from _typeshed import Self
-from typing_extensions import TypeAlias
+from _typeshed import Incomplete
+from typing_extensions import Self, TypeAlias
 
 _K = TypeVar("_K")
 _V = TypeVar("_V")
@@ -39,13 +39,14 @@ class _IndexableCollection(Protocol[_I], Collection[_I]):
     @overload
     def __getitem__(self, index: int) -> _I: ...
     @overload
-    def __getitem__(self: Self, index: slice) -> Self: ...
+    def __getitem__(self, index: slice) -> Self: ...
 
 class OrderedSet(MutableSet[_K]):
     dict: dict[_K, None]
     def __init__(self, iterable: Iterable[_K] | None = ...) -> None: ...
     def __contains__(self, item: object) -> bool: ...
     def __iter__(self) -> Iterator[_K]: ...
+    def __reversed__(self) -> Iterator[_K]: ...
     def __bool__(self) -> bool: ...
     def __len__(self) -> int: ...
     def add(self, item: _K) -> None: ...
@@ -74,7 +75,7 @@ class MultiValueDict(dict[_K, _V]):
     def items(self) -> Iterator[tuple[_K, _V | list[object]]]: ...  # type: ignore
     def lists(self) -> Iterable[tuple[_K, list[_V]]]: ...
     def dict(self) -> dict[_K, _V | list[object]]: ...
-    def copy(self: Self) -> Self: ...
+    def copy(self) -> Self: ...
     def __getitem__(self, key: _K) -> _V | list[object]: ...  # type: ignore
     def __setitem__(self, key: _K, value: _V) -> None: ...
     # These overrides are needed to convince mypy that this isn't an abstract class
@@ -83,11 +84,13 @@ class MultiValueDict(dict[_K, _V]):
     def __iter__(self) -> Iterator[_K]: ...
     # Fake to make `values` work properly
     def values(self) -> Iterator[_V | list[object]]: ...  # type: ignore[override]
+    def __copy__(self) -> Self: ...
+    def __deepcopy__(self, memo: MutableMapping[int, Incomplete]) -> Self: ...
 
 class ImmutableList(tuple[_V, ...]):
     warning: str
-    def __new__(cls: type[Self], *args: Any, warning: str = ..., **kwargs: Any) -> Self: ...
-    def complain(self, *args: Any, **kwargs: Any) -> None: ...
+    def __new__(cls, *args: Any, warning: str = ..., **kwargs: Any) -> Self: ...
+    def complain(self, *args: Any, **kwargs: Any) -> NoReturn: ...
 
 class _ItemCallable(Protocol[_V]):
     """Don't mess with arguments when assigning in class body in stub"""

django-stubs/utils/datetime_safe.pyi

@@ -2,9 +2,13 @@ from datetime import date as real_date
 from datetime import datetime as real_datetime
 from datetime import time as real_time
 
-class date(real_date): ...
-class datetime(real_datetime): ...
-class time(real_time): ...
+class date(real_date):
+    def strftime(self, fmt: str) -> str: ...
+
+class datetime(real_datetime):
+    def strftime(self, fmt: str) -> str: ...
+    @classmethod
+    def combine(cls, date: real_date, time: real_time) -> datetime: ...  # type: ignore
 
 def new_date(d: real_date) -> date: ...
 def new_datetime(d: real_date | real_datetime) -> datetime: ...

django-stubs/utils/decorators.pyi

@@ -1,10 +1,7 @@
-from collections.abc import Awaitable, Callable, Iterable
-from typing import Protocol, TypeVar
+from collections.abc import Callable, Iterable
+from typing import TypeVar
 
-from django.http.request import HttpRequest
-from django.http.response import HttpResponseBase
 from django.utils.deprecation import MiddlewareMixin
-from django.utils.functional import classproperty as classproperty
 from django.views.generic.base import View
 
 _T = TypeVar("_T", bound=View | Callable)  # Any callable
@@ -16,10 +13,6 @@ def method_decorator(decorator: Callable | Iterable[Callable], name: str = ...)
 def decorator_from_middleware_with_args(middleware_class: type) -> Callable: ...
 def decorator_from_middleware(middleware_class: type) -> Callable: ...
 def make_middleware_decorator(middleware_class: type[MiddlewareMixin]) -> Callable: ...
-
-class AsyncGetResponseCallable(Protocol):
-    def __call__(self, __request: HttpRequest) -> Awaitable[HttpResponseBase]: ...
-
 def sync_and_async_middleware(func: _CallableType) -> _CallableType: ...
 def sync_only_middleware(func: _CallableType) -> _CallableType: ...
 def async_only_middleware(func: _CallableType) -> _CallableType: ...

django-stubs/utils/deprecation.pyi

@@ -1,14 +1,15 @@
-from collections.abc import Callable
+from collections.abc import Awaitable, Callable
 from typing import Any, Protocol
 
 from django.http.request import HttpRequest
 from django.http.response import HttpResponseBase
 from typing_extensions import TypeAlias
 
-class RemovedInDjango40Warning(DeprecationWarning): ...
-class RemovedInDjango41Warning(PendingDeprecationWarning): ...
+class RemovedInDjango50Warning(DeprecationWarning): ...
+class RemovedInDjango51Warning(PendingDeprecationWarning): ...
 
-RemovedInNextVersionWarning: TypeAlias = RemovedInDjango40Warning
+RemovedInNextVersionWarning: TypeAlias = RemovedInDjango50Warning
+RemovedAfterNextVersionWarning: TypeAlias = RemovedInDjango51Warning
 
 class warn_about_renamed_method:
     class_name: str
@@ -29,10 +30,17 @@ class DeprecationInstanceCheck(type):
     deprecation_warning: type[Warning]
     def __instancecheck__(self, instance: Any) -> bool: ...
 
-class GetResponseCallable(Protocol):
+class _GetResponseCallable(Protocol):
     def __call__(self, __request: HttpRequest) -> HttpResponseBase: ...
 
+class _AsyncGetResponseCallable(Protocol):
+    def __call__(self, __request: HttpRequest) -> Awaitable[HttpResponseBase]: ...
+
 class MiddlewareMixin:
-    get_response: GetResponseCallable
-    def __init__(self, get_response: GetResponseCallable = ...) -> None: ...
+    sync_capable: bool
+    async_capable: bool
+
+    get_response: _GetResponseCallable | _AsyncGetResponseCallable
+    def __init__(self, get_response: _GetResponseCallable | _AsyncGetResponseCallable) -> None: ...
     def __call__(self, request: HttpRequest) -> HttpResponseBase: ...
+    async def __acall__(self, request: HttpRequest) -> HttpResponseBase: ...

django-stubs/utils/encoding.pyi

@@ -26,9 +26,6 @@ def smart_str(s: _PT, encoding: str = ..., strings_only: Literal[True] = ..., er
 def smart_str(s: _S, encoding: str = ..., strings_only: bool = ..., errors: str = ...) -> _S: ...
 @overload
 def smart_str(s: Any, encoding: str = ..., strings_only: bool = ..., errors: str = ...) -> str: ...
-
-smart_text = smart_str  # Deprecated
-
 def is_protected_type(obj: Any) -> TypeGuard[_PT]: ...
 @overload
 def force_str(s: _S, encoding: str = ..., *, errors: str = ...) -> _S: ...
@@ -40,9 +37,6 @@ def force_str(s: _PT, encoding: str = ..., strings_only: Literal[True] = ..., er
 def force_str(s: _S, encoding: str = ..., strings_only: bool = ..., errors: str = ...) -> _S: ...
 @overload
 def force_str(s: Any, encoding: str = ..., strings_only: bool = ..., errors: str = ...) -> str: ...
-
-force_text = force_str  # Deprecated
-
 @overload
 def smart_bytes(s: _P, encoding: str = ..., strings_only: bool = ..., errors: str = ...) -> _P: ...
 @overload

django-stubs/utils/feedgenerator.pyi

@@ -64,6 +64,7 @@ class Enclosure:
 
 class RssFeed(SyndicationFeed):
     content_type: str
+    def rss_attributes(self) -> dict[str, str]: ...
     def write_items(self, handler: ContentHandler) -> None: ...
     def endChannelElement(self, handler: ContentHandler) -> None: ...
 

django-stubs/utils/formats.pyi

@@ -10,7 +10,7 @@ FORMAT_SETTINGS: frozenset[str]
 
 def reset_format_cache() -> None: ...
 def iter_format_modules(lang: str, format_module_path: list[str] | str | None = ...) -> Iterator[types.ModuleType]: ...
-def get_format_modules(lang: str | None = ..., reverse: bool = ...) -> list[types.ModuleType]: ...
+def get_format_modules(lang: str | None = ...) -> list[types.ModuleType]: ...
 def get_format(format_type: str, lang: str | None = ..., use_l10n: bool | None = ...) -> Any: ...
 
 get_format_lazy: Any
@@ -42,3 +42,4 @@ def localize_input(  # type: ignore
 @overload
 def localize_input(value: _T, default: str | None = ...) -> _T: ...
 def sanitize_separators(value: _T) -> _T: ...
+def sanitize_strftime_format(fmt: str) -> str: ...

django-stubs/utils/functional.pyi

@@ -9,12 +9,13 @@ _T = TypeVar("_T")
 
 class cached_property(Generic[_T]):
     func: Callable[..., _T]
-    name: str
-    def __init__(self, func: Callable[..., _T], name: str = ...) -> None: ...
+    name: str | None
+    def __init__(self, func: Callable[..., _T], name: str | None = ...) -> None: ...
     @overload
-    def __get__(self: Self, instance: None, cls: type[Any] = ...) -> Self: ...
+    def __get__(self: Self, instance: None, cls: type[Any] | None = ...) -> Self: ...
     @overload
-    def __get__(self, instance: object, cls: type[Any] = ...) -> _T: ...
+    def __get__(self, instance: object, cls: type[Any] | None = ...) -> _T: ...
+    def __set_name__(self, owner: type[Any], name: str) -> None: ...
 
 # Promise is only subclassed by a proxy class defined in the lazy function
 # so it makes sense for it to have all the methods available in that proxy class
@@ -63,6 +64,8 @@ class LazyObject:
     def __delattr__(self, name: str) -> None: ...
     def __reduce__(self) -> tuple[Callable, tuple[Model]]: ...
     def __copy__(self) -> LazyObject: ...
+    # TODO: Deepcopy can return a LazyObject or a wrapped object, but we'll need to make LazyObject generic first
+    def __deepcopy__(self, memo: dict[int, Any]) -> Any: ...
     __bytes__: Callable
     __bool__: Callable
     __dir__: Callable
@@ -74,12 +77,17 @@ class LazyObject:
     __iter__: Callable
     __len__: Callable
     __contains__: Callable
+    __gt__: Callable
+    __lt__: Callable
+    __add__: Callable
+    __str__: Callable[..., str]
 
 def unpickle_lazyobject(wrapped: Model) -> Model: ...
 
 class SimpleLazyObject(LazyObject):
     def __init__(self, func: Callable[[], Any]) -> None: ...
     def __copy__(self) -> SimpleLazyObject: ...
+    __radd__: Callable
 
 _PartitionMember = TypeVar("_PartitionMember")
 
@@ -92,7 +100,7 @@ _Get = TypeVar("_Get", covariant=True)
 class classproperty(Generic[_Get]):
     fget: Callable[[Any], _Get] | None
     def __init__(self, method: Callable[[Any], _Get] | None = ...) -> None: ...
-    def __get__(self, instance: Any | None, cls: type[Any] = ...) -> _Get: ...
+    def __get__(self, instance: Any | None, cls: type[Any] | None = ...) -> _Get: ...
     def getter(self, method: Callable[[Any], _Get]) -> classproperty[_Get]: ...
 
 class _Getter(Protocol[_Get]):

django-stubs/utils/html.pyi

@@ -4,19 +4,16 @@ from json import JSONEncoder
 from re import Pattern
 from typing import Any
 
-from django.utils.safestring import SafeString
-
-TRAILING_PUNCTUATION_CHARS: str
-WRAPPING_PUNCTUATION: list[tuple[str, str]]
-DOTS: list[str]
-word_split_re: Pattern[str]
-simple_url_re: Pattern[str]
-simple_url_2_re: Pattern[str]
+from _typeshed import Incomplete
+from django.utils.functional import SimpleLazyObject, _StrPromise
+from django.utils.safestring import SafeData, SafeString
 
 def escape(text: Any) -> SafeString: ...
 def escapejs(value: Any) -> SafeString: ...
 def json_script(value: Any, element_id: str | None = ..., encoder: type[JSONEncoder] | None = ...) -> SafeString: ...
-def conditional_escape(text: Any) -> str: ...
+
+# conditional_escape could use a protocol to be more precise, see https://github.com/typeddjango/django-stubs/issues/1474
+def conditional_escape(text: _StrPromise | SafeData) -> SafeString: ...
 def format_html(format_string: str, *args: Any, **kwargs: Any) -> SafeString: ...
 def format_html_join(sep: str, format_string: str, args_generator: Iterable[Iterable[Any]]) -> SafeString: ...
 def linebreaks(value: Any, autoescape: bool = ...) -> str: ...
@@ -35,3 +32,30 @@ def smart_urlquote(url: str) -> str: ...
 def urlize(text: str, trim_url_limit: int | None = ..., nofollow: bool = ..., autoescape: bool = ...) -> str: ...
 def avoid_wrapping(value: str) -> str: ...
 def html_safe(klass: type) -> type: ...
+
+class Urlizer:
+    trailing_punctuation_chars: str
+    wrapping_punctuation: Incomplete
+    word_split_re: Pattern[str] | SimpleLazyObject
+    simple_url_re: Pattern[str] | SimpleLazyObject
+    simple_url_2_re: Pattern[str] | SimpleLazyObject
+    mailto_template: str
+    url_template: str
+    def __call__(
+        self, text: Incomplete, trim_url_limit: Incomplete | None = ..., nofollow: bool = ..., autoescape: bool = ...
+    ) -> Incomplete: ...
+    def handle_word(
+        self,
+        word: Incomplete,
+        *,
+        safe_input: Incomplete,
+        trim_url_limit: Incomplete | None = ...,
+        nofollow: bool = ...,
+        autoescape: bool = ...,
+    ) -> Incomplete: ...
+    def trim_url(self, x: str, *, limit: int | None) -> Incomplete: ...
+    def trim_punctuation(self, word: str) -> tuple[str, str, str]: ...
+    @staticmethod
+    def is_email_simple(value: str) -> bool: ...
+
+urlizer: Urlizer

django-stubs/utils/http.pyi

@@ -9,10 +9,6 @@ ASCTIME_DATE: Pattern[str]
 RFC3986_GENDELIMS: str
 RFC3986_SUBDELIMS: str
 
-def urlquote(url: str, safe: str = ...) -> str: ...
-def urlquote_plus(url: str, safe: str = ...) -> str: ...
-def urlunquote(quoted_url: str) -> str: ...
-def urlunquote_plus(quoted_url: str) -> str: ...
 def urlencode(
     query: Mapping[str, str | bytes | int | Iterable[str | bytes | int]]
     | Iterable[tuple[str, str | bytes | int | Iterable[str | bytes | int]]]
@@ -32,15 +28,6 @@ def is_same_domain(host: str, pattern: str) -> bool: ...
 def url_has_allowed_host_and_scheme(
     url: str | None, allowed_hosts: str | Iterable[str] | None, require_https: bool = ...
 ) -> bool: ...
-def is_safe_url(url: str | None, allowed_hosts: str | Iterable[str] | None, require_https: bool = ...) -> bool: ...
-def parse_qsl(
-    qs: str,
-    keep_blank_values: bool = ...,
-    strict_parsing: bool = ...,
-    encoding: str = ...,
-    errors: str = ...,
-    max_num_fields: int | None = ...,
-    separator: str = ...,
-) -> list[tuple[str, str]]: ...
 def escape_leading_slashes(url: str) -> str: ...
 def content_disposition_header(as_attachment: bool, filename: str) -> str | None: ...
+def parse_header_parameters(line: str) -> tuple[str, dict[str, str]]: ...

django-stubs/utils/module_loading.pyi

@@ -1,5 +1,6 @@
 from typing import Any
 
+def cached_import(module_path: str, class_name: str) -> Any: ...
 def import_string(dotted_path: str) -> Any: ...
 def autodiscover_modules(*args: Any, **kwargs: Any) -> None: ...
 def module_has_submodule(package: Any, module_name: str) -> bool: ...