From 5db161ffbfd7d9cdd837410cd5d581c0ba597623 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 22 Mar 2024 12:55:06 -0700 Subject: [PATCH] Store typeshed patches in repo (#17054) This way, we don't have to do the two step rebase and merge + update hashes dance. We also get a cleaner commit history --- .pre-commit-config.yaml | 2 +- misc/sync-typeshed.py | 101 +++++----- ...e-of-LiteralString-in-builtins-13743.patch | 182 ++++++++++++++++++ ...ert-sum-literal-integer-change-13961.patch | 36 ++++ .../0001-Revert-typeshed-ctypes-change.patch | 32 +++ 5 files changed, 306 insertions(+), 47 deletions(-) create mode 100644 misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch create mode 100644 misc/typeshed_patches/0001-Revert-sum-literal-integer-change-13961.patch create mode 100644 misc/typeshed_patches/0001-Revert-typeshed-ctypes-change.patch diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4cfb8297a66a4..a7ff48051aad7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -exclude: '^(mypyc/external/)|(mypy/typeshed/)' # Exclude all vendored code from lints +exclude: '^(mypyc/external/)|(mypy/typeshed/)|misc/typeshed_patches' # Exclude all vendored code from lints repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 # must match test-requirements.txt diff --git a/misc/sync-typeshed.py b/misc/sync-typeshed.py index 2dc6e230df004..3101b4bfa72a6 100644 --- a/misc/sync-typeshed.py +++ b/misc/sync-typeshed.py @@ -11,6 +11,7 @@ import argparse import functools +import glob import os import re import shutil @@ -148,58 +149,66 @@ def main() -> None: if os.environ.get("GITHUB_TOKEN") is None: raise ValueError("GITHUB_TOKEN environment variable must be set") - branch_name = "mypybot/sync-typeshed" - subprocess.run(["git", "checkout", "-B", branch_name, "origin/master"], check=True) - - if not args.typeshed_dir: - # Clone typeshed repo if no directory given. - with tempfile.TemporaryDirectory() as tempdir: - print(f"Cloning typeshed in {tempdir}...") + with tempfile.TemporaryDirectory() as tmpdir: + # Stash patches before checking out a new branch + typeshed_patches = os.path.join("misc", "typeshed_patches") + tmp_patches = os.path.join(tmpdir, "typeshed_patches") + shutil.copytree(typeshed_patches, tmp_patches) + + branch_name = "mypybot/sync-typeshed" + subprocess.run(["git", "checkout", "-B", branch_name, "origin/master"], check=True) + + # Copy the stashed patches back + shutil.rmtree(typeshed_patches, ignore_errors=True) + shutil.copytree(tmp_patches, typeshed_patches) + if subprocess.run(["git", "diff", "--quiet", "--exit-code"], check=False).returncode != 0: + subprocess.run(["git", "commit", "-am", "Update typeshed patches"], check=True) + + if not args.typeshed_dir: + tmp_typeshed = os.path.join(tmpdir, "typeshed") + os.makedirs(tmp_typeshed) + # Clone typeshed repo if no directory given. + print(f"Cloning typeshed in {tmp_typeshed}...") subprocess.run( - ["git", "clone", "https://github.com/python/typeshed.git"], check=True, cwd=tempdir + ["git", "clone", "https://github.com/python/typeshed.git"], + check=True, + cwd=tmp_typeshed, ) - repo = os.path.join(tempdir, "typeshed") + repo = os.path.join(tmp_typeshed, "typeshed") commit = update_typeshed(repo, args.commit) - else: - commit = update_typeshed(args.typeshed_dir, args.commit) + else: + commit = update_typeshed(args.typeshed_dir, args.commit) - assert commit + assert commit - # Create a commit - message = textwrap.dedent( - f"""\ - Sync typeshed + # Create a commit + message = textwrap.dedent( + f"""\ + Sync typeshed - Source commit: - https://github.com/python/typeshed/commit/{commit} - """ - ) - subprocess.run(["git", "add", "--all", os.path.join("mypy", "typeshed")], check=True) - subprocess.run(["git", "commit", "-m", message], check=True) - print("Created typeshed sync commit.") - - commits_to_cherry_pick = [ - "5c00e362d", # LiteralString reverts - "44bc98bd5", # sum reverts - "61a490091", # ctypes reverts - ] - for commit in commits_to_cherry_pick: - try: - subprocess.run(["git", "cherry-pick", commit], check=True) - except subprocess.CalledProcessError: - if not sys.__stdin__.isatty(): - # We're in an automated context - raise - - # Allow the option to merge manually - print( - f"Commit {commit} failed to cherry pick." - " In a separate shell, please manually merge and continue cherry pick." - ) - rsp = input("Did you finish the cherry pick? [y/N]: ") - if rsp.lower() not in {"y", "yes"}: - raise - print(f"Cherry-picked {commit}.") + Source commit: + https://github.com/python/typeshed/commit/{commit} + """ + ) + subprocess.run(["git", "add", "--all", os.path.join("mypy", "typeshed")], check=True) + subprocess.run(["git", "commit", "-m", message], check=True) + print("Created typeshed sync commit.") + + patches = sorted(glob.glob(os.path.join(typeshed_patches, "*.patch"))) + for patch in patches: + cmd = ["git", "am", "--3way", patch] + try: + subprocess.run(cmd, check=True) + except subprocess.CalledProcessError as e: + raise RuntimeError( + f"\n\nFailed to apply patch {patch}\n" + "1. Resolve the conflict, `git add --update`, then run `git am --continue`\n" + "2. Run `git format-patch -1 -o misc/typeshed_patches ` " + "to update the patch file.\n" + "3. Re-run sync-typeshed.py" + ) from e + + print(f"Applied patch {patch}") if args.make_pr: subprocess.run(["git", "push", "--force", "origin", branch_name], check=True) diff --git a/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch b/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch new file mode 100644 index 0000000000000..6a0977dfc4891 --- /dev/null +++ b/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch @@ -0,0 +1,182 @@ +From 5c00e362d40aa26e0a22a740f05a52d05edf0f91 Mon Sep 17 00:00:00 2001 +From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> +Date: Mon, 26 Sep 2022 12:55:07 -0700 +Subject: [PATCH] Remove use of LiteralString in builtins (#13743) + +--- + mypy/typeshed/stdlib/builtins.pyi | 88 ------------------------------- + 1 file changed, 88 deletions(-) + +diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi +index b4765b26c..99919c64c 100644 +--- a/mypy/typeshed/stdlib/builtins.pyi ++++ b/mypy/typeshed/stdlib/builtins.pyi +@@ -61,7 +61,6 @@ from typing import ( # noqa: Y022 + from typing_extensions import ( # noqa: Y023 + Concatenate, + Literal, +- LiteralString, + ParamSpec, + Self, + TypeAlias, +@@ -434,31 +433,16 @@ class str(Sequence[str]): + def __new__(cls, object: object = ...) -> Self: ... + @overload + def __new__(cls, object: ReadableBuffer, encoding: str = ..., errors: str = ...) -> Self: ... +- @overload +- def capitalize(self: LiteralString) -> LiteralString: ... +- @overload + def capitalize(self) -> str: ... # type: ignore[misc] +- @overload +- def casefold(self: LiteralString) -> LiteralString: ... +- @overload + def casefold(self) -> str: ... # type: ignore[misc] +- @overload +- def center(self: LiteralString, width: SupportsIndex, fillchar: LiteralString = " ", /) -> LiteralString: ... +- @overload + def center(self, width: SupportsIndex, fillchar: str = " ", /) -> str: ... # type: ignore[misc] + def count(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def encode(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: ... + def endswith( + self, suffix: str | tuple[str, ...], start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + ) -> bool: ... +- @overload +- def expandtabs(self: LiteralString, tabsize: SupportsIndex = 8) -> LiteralString: ... +- @overload + def expandtabs(self, tabsize: SupportsIndex = 8) -> str: ... # type: ignore[misc] + def find(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... +- @overload +- def format(self: LiteralString, *args: LiteralString, **kwargs: LiteralString) -> LiteralString: ... +- @overload + def format(self, *args: object, **kwargs: object) -> str: ... + def format_map(self, map: _FormatMapMapping) -> str: ... + def index(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... +@@ -474,89 +458,32 @@ class str(Sequence[str]): + def isspace(self) -> bool: ... + def istitle(self) -> bool: ... + def isupper(self) -> bool: ... +- @overload +- def join(self: LiteralString, iterable: Iterable[LiteralString], /) -> LiteralString: ... +- @overload + def join(self, iterable: Iterable[str], /) -> str: ... # type: ignore[misc] +- @overload +- def ljust(self: LiteralString, width: SupportsIndex, fillchar: LiteralString = " ", /) -> LiteralString: ... +- @overload + def ljust(self, width: SupportsIndex, fillchar: str = " ", /) -> str: ... # type: ignore[misc] +- @overload +- def lower(self: LiteralString) -> LiteralString: ... +- @overload + def lower(self) -> str: ... # type: ignore[misc] +- @overload +- def lstrip(self: LiteralString, chars: LiteralString | None = None, /) -> LiteralString: ... +- @overload + def lstrip(self, chars: str | None = None, /) -> str: ... # type: ignore[misc] +- @overload +- def partition(self: LiteralString, sep: LiteralString, /) -> tuple[LiteralString, LiteralString, LiteralString]: ... +- @overload + def partition(self, sep: str, /) -> tuple[str, str, str]: ... # type: ignore[misc] +- @overload +- def replace(self: LiteralString, old: LiteralString, new: LiteralString, count: SupportsIndex = -1, /) -> LiteralString: ... +- @overload + def replace(self, old: str, new: str, count: SupportsIndex = -1, /) -> str: ... # type: ignore[misc] + if sys.version_info >= (3, 9): +- @overload +- def removeprefix(self: LiteralString, prefix: LiteralString, /) -> LiteralString: ... +- @overload + def removeprefix(self, prefix: str, /) -> str: ... # type: ignore[misc] +- @overload +- def removesuffix(self: LiteralString, suffix: LiteralString, /) -> LiteralString: ... +- @overload + def removesuffix(self, suffix: str, /) -> str: ... # type: ignore[misc] + + def rfind(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def rindex(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... +- @overload +- def rjust(self: LiteralString, width: SupportsIndex, fillchar: LiteralString = " ", /) -> LiteralString: ... +- @overload + def rjust(self, width: SupportsIndex, fillchar: str = " ", /) -> str: ... # type: ignore[misc] +- @overload +- def rpartition(self: LiteralString, sep: LiteralString, /) -> tuple[LiteralString, LiteralString, LiteralString]: ... +- @overload + def rpartition(self, sep: str, /) -> tuple[str, str, str]: ... # type: ignore[misc] +- @overload +- def rsplit(self: LiteralString, sep: LiteralString | None = None, maxsplit: SupportsIndex = -1) -> list[LiteralString]: ... +- @overload + def rsplit(self, sep: str | None = None, maxsplit: SupportsIndex = -1) -> list[str]: ... # type: ignore[misc] +- @overload +- def rstrip(self: LiteralString, chars: LiteralString | None = None, /) -> LiteralString: ... +- @overload + def rstrip(self, chars: str | None = None, /) -> str: ... # type: ignore[misc] +- @overload +- def split(self: LiteralString, sep: LiteralString | None = None, maxsplit: SupportsIndex = -1) -> list[LiteralString]: ... +- @overload + def split(self, sep: str | None = None, maxsplit: SupportsIndex = -1) -> list[str]: ... # type: ignore[misc] +- @overload +- def splitlines(self: LiteralString, keepends: bool = False) -> list[LiteralString]: ... +- @overload + def splitlines(self, keepends: bool = False) -> list[str]: ... # type: ignore[misc] + def startswith( + self, prefix: str | tuple[str, ...], start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + ) -> bool: ... +- @overload +- def strip(self: LiteralString, chars: LiteralString | None = None, /) -> LiteralString: ... +- @overload + def strip(self, chars: str | None = None, /) -> str: ... # type: ignore[misc] +- @overload +- def swapcase(self: LiteralString) -> LiteralString: ... +- @overload + def swapcase(self) -> str: ... # type: ignore[misc] +- @overload +- def title(self: LiteralString) -> LiteralString: ... +- @overload + def title(self) -> str: ... # type: ignore[misc] + def translate(self, table: _TranslateTable, /) -> str: ... +- @overload +- def upper(self: LiteralString) -> LiteralString: ... +- @overload + def upper(self) -> str: ... # type: ignore[misc] +- @overload +- def zfill(self: LiteralString, width: SupportsIndex, /) -> LiteralString: ... +- @overload + def zfill(self, width: SupportsIndex, /) -> str: ... # type: ignore[misc] + @staticmethod + @overload +@@ -567,9 +494,6 @@ class str(Sequence[str]): + @staticmethod + @overload + def maketrans(x: str, y: str, z: str, /) -> dict[int, int | None]: ... +- @overload +- def __add__(self: LiteralString, value: LiteralString, /) -> LiteralString: ... +- @overload + def __add__(self, value: str, /) -> str: ... # type: ignore[misc] + # Incompatible with Sequence.__contains__ + def __contains__(self, key: str, /) -> bool: ... # type: ignore[override] +@@ -578,25 +502,13 @@ class str(Sequence[str]): + def __getitem__(self, key: SupportsIndex | slice, /) -> str: ... + def __gt__(self, value: str, /) -> bool: ... + def __hash__(self) -> int: ... +- @overload +- def __iter__(self: LiteralString) -> Iterator[LiteralString]: ... +- @overload + def __iter__(self) -> Iterator[str]: ... # type: ignore[misc] + def __le__(self, value: str, /) -> bool: ... + def __len__(self) -> int: ... + def __lt__(self, value: str, /) -> bool: ... +- @overload +- def __mod__(self: LiteralString, value: LiteralString | tuple[LiteralString, ...], /) -> LiteralString: ... +- @overload + def __mod__(self, value: Any, /) -> str: ... +- @overload +- def __mul__(self: LiteralString, value: SupportsIndex, /) -> LiteralString: ... +- @overload + def __mul__(self, value: SupportsIndex, /) -> str: ... # type: ignore[misc] + def __ne__(self, value: object, /) -> bool: ... +- @overload +- def __rmul__(self: LiteralString, value: SupportsIndex, /) -> LiteralString: ... +- @overload + def __rmul__(self, value: SupportsIndex, /) -> str: ... # type: ignore[misc] + def __getnewargs__(self) -> tuple[str]: ... + +-- +2.39.3 (Apple Git-146) + diff --git a/misc/typeshed_patches/0001-Revert-sum-literal-integer-change-13961.patch b/misc/typeshed_patches/0001-Revert-sum-literal-integer-change-13961.patch new file mode 100644 index 0000000000000..044e672bfda5e --- /dev/null +++ b/misc/typeshed_patches/0001-Revert-sum-literal-integer-change-13961.patch @@ -0,0 +1,36 @@ +From 44bc98bd50e7170887f0740b53ed95a8eb04f00e Mon Sep 17 00:00:00 2001 +From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> +Date: Sat, 29 Oct 2022 12:47:21 -0700 +Subject: [PATCH] Revert sum literal integer change (#13961) + +This is allegedly causing large performance problems, see 13821 + +typeshed/8231 had zero hits on mypy_primer, so it's not the worst thing +to undo. Patching this in typeshed also feels weird, since there's a +more general soundness issue. If a typevar has a bound or constraint, we +might not want to solve it to a Literal. + +If we can confirm the performance regression or fix the unsoundness +within mypy, I might pursue upstreaming this in typeshed. + +(Reminder: add this to the sync_typeshed script once merged) +--- + mypy/typeshed/stdlib/builtins.pyi | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi +index 99919c64c..680cd5561 100644 +--- a/mypy/typeshed/stdlib/builtins.pyi ++++ b/mypy/typeshed/stdlib/builtins.pyi +@@ -1596,7 +1596,7 @@ _SupportsSumNoDefaultT = TypeVar("_SupportsSumNoDefaultT", bound=_SupportsSumWit + # without creating many false-positive errors (see #7578). + # Instead, we special-case the most common examples of this: bool and literal integers. + @overload +-def sum(iterable: Iterable[bool | _LiteralInteger], /, start: int = 0) -> int: ... # type: ignore[overload-overlap] ++def sum(iterable: Iterable[bool], /, start: int = 0) -> int: ... # type: ignore[overload-overlap] + @overload + def sum(iterable: Iterable[_SupportsSumNoDefaultT], /) -> _SupportsSumNoDefaultT | Literal[0]: ... + @overload +-- +2.39.3 (Apple Git-146) + diff --git a/misc/typeshed_patches/0001-Revert-typeshed-ctypes-change.patch b/misc/typeshed_patches/0001-Revert-typeshed-ctypes-change.patch new file mode 100644 index 0000000000000..27066bf3c25bb --- /dev/null +++ b/misc/typeshed_patches/0001-Revert-typeshed-ctypes-change.patch @@ -0,0 +1,32 @@ +From 61a490091d7c941780919660dc4fdfa88ae6474a Mon Sep 17 00:00:00 2001 +From: AlexWaygood +Date: Mon, 1 May 2023 20:34:55 +0100 +Subject: [PATCH] Revert typeshed ctypes change Since the plugin provides + superior type checking: + https://github.com/python/mypy/pull/13987#issuecomment-1310863427 A manual + cherry-pick of e437cdf. + +--- + mypy/typeshed/stdlib/_ctypes.pyi | 6 +----- + 1 file changed, 1 insertion(+), 5 deletions(-) + +diff --git a/mypy/typeshed/stdlib/_ctypes.pyi b/mypy/typeshed/stdlib/_ctypes.pyi +index 60bbc51d9..cf9cb81a4 100644 +--- a/mypy/typeshed/stdlib/_ctypes.pyi ++++ b/mypy/typeshed/stdlib/_ctypes.pyi +@@ -169,11 +169,7 @@ class Array(_CData, Generic[_CT]): + def _type_(self) -> type[_CT]: ... + @_type_.setter + def _type_(self, value: type[_CT]) -> None: ... +- # Note: only available if _CT == c_char +- @property +- def raw(self) -> bytes: ... +- @raw.setter +- def raw(self, value: ReadableBuffer) -> None: ... ++ raw: bytes # Note: only available if _CT == c_char + value: Any # Note: bytes if _CT == c_char, str if _CT == c_wchar, unavailable otherwise + # TODO These methods cannot be annotated correctly at the moment. + # All of these "Any"s stand for the array's element type, but it's not possible to use _CT +-- +2.39.3 (Apple Git-146) +