From 726fdfbf569c144310893440a40ee8ee05e6524e Mon Sep 17 00:00:00 2001
From: David Stansby <dstansby@gmail.com>
Date: Wed, 11 Sep 2024 07:25:11 +0100
Subject: [PATCH] Simplify mypy config for tests (#2156)

* Simplify mypy config for tests

* Add a list of test modules to expliclty ignore typing

* More test mypy fixes

* Remove old test_metadata file

* Fix ignoring v2 tests

* Fix name test
---
 .pre-commit-config.yaml                    |  2 +-
 pyproject.toml                             | 19 ++++++++++++++++-
 tests/{v3/test_metadata.py => __init__.py} |  0
 tests/v3/conftest.py                       | 24 +++++++++++-----------
 tests/v3/test_codec_entrypoints.py         |  9 ++++----
 tests/v3/test_common.py                    |  2 +-
 tests/v3/test_config.py                    | 17 +++++++--------
 tests/v3/test_group.py                     |  2 +-
 tests/v3/test_indexing.py                  | 15 +++++++-------
 tests/v3/test_metadata/__init__.py         |  0
 tests/v3/test_metadata/test_v3.py          |  2 +-
 tests/v3/test_properties.py                |  6 +++---
 tests/v3/test_store/__init__.py            |  0
 tests/v3/test_store/test_core.py           |  4 ++--
 tests/v3/test_store/test_remote.py         | 10 +++++----
 tests/v3/test_store/test_stateful_store.py |  4 ++--
 tests/v3/test_sync.py                      | 11 +++++-----
 tests/v3/test_v2.py                        |  5 ++---
 18 files changed, 75 insertions(+), 57 deletions(-)
 rename tests/{v3/test_metadata.py => __init__.py} (100%)
 create mode 100644 tests/v3/test_metadata/__init__.py
 create mode 100644 tests/v3/test_store/__init__.py

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 1f7373c028..2fd405b64d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -25,7 +25,7 @@ repos:
     rev: v1.11.2
     hooks:
       - id: mypy
-        files: src|tests/v3/test_(api|array|buffer).py
+        files: src|tests
         additional_dependencies:
           # Package dependencies
           - asciitree
diff --git a/pyproject.toml b/pyproject.toml
index d92b89c65e..49878b61f6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -225,6 +225,7 @@ python_version = "3.10"
 ignore_missing_imports = true
 namespace_packages = false
 
+
 strict = true
 warn_unreachable = true
 
@@ -236,6 +237,22 @@ module = [
 ]
 ignore_errors = true
 
+[[tool.mypy.overrides]]
+module = [
+    "tests.v2.*",
+    "tests.v3.package_with_entrypoint.*",
+    "tests.v3.test_codecs.*",
+    "tests.v3.test_metadata.*",
+    "tests.v3.test_store.*",
+    "tests.v3.test_config",
+    "tests.v3.test_group",
+    "tests.v3.test_indexing",
+    "tests.v3.test_properties",
+    "tests.v3.test_sync",
+    "tests.v3.test_v2",
+]
+ignore_errors = true
+
 [tool.pytest.ini_options]
 minversion = "7"
 testpaths = ["tests"]
@@ -262,6 +279,6 @@ markers = [
 
 [tool.repo-review]
 ignore = [
-	"PC111",  # fix Python code in documentation - enable later 
+	"PC111",  # fix Python code in documentation - enable later
 	"PC180",  # for JavaScript - not interested
 ]
diff --git a/tests/v3/test_metadata.py b/tests/__init__.py
similarity index 100%
rename from tests/v3/test_metadata.py
rename to tests/__init__.py
diff --git a/tests/v3/conftest.py b/tests/v3/conftest.py
index fcb5e3c867..b1308f058f 100644
--- a/tests/v3/conftest.py
+++ b/tests/v3/conftest.py
@@ -1,11 +1,20 @@
 from __future__ import annotations
 
+import pathlib
+from dataclasses import dataclass, field
 from typing import TYPE_CHECKING
 
+import numpy as np
+import numpy.typing as npt
+import pytest
+from hypothesis import HealthCheck, Verbosity, settings
+
 from zarr import AsyncGroup, config
+from zarr.store import LocalStore, MemoryStore, StorePath
+from zarr.store.remote import RemoteStore
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator
+    from collections.abc import Generator, Iterator
     from types import ModuleType
     from typing import Any, Literal
 
@@ -13,15 +22,6 @@
 
     from zarr.abc.store import Store
     from zarr.core.common import ChunkCoords, MemoryOrder, ZarrFormat
-import pathlib
-from dataclasses import dataclass, field
-
-import numpy as np
-import pytest
-from hypothesis import HealthCheck, Verbosity, settings
-
-from zarr.store import LocalStore, MemoryStore, StorePath
-from zarr.store.remote import RemoteStore
 
 
 async def parse_store(
@@ -102,7 +102,7 @@ def xp(request: pytest.FixtureRequest) -> Iterator[ModuleType]:
 
 
 @pytest.fixture(autouse=True)
-def reset_config():
+def reset_config() -> Generator[None, None, None]:
     config.reset()
     yield
     config.reset()
@@ -116,7 +116,7 @@ class ArrayRequest:
 
 
 @pytest.fixture
-def array_fixture(request: pytest.FixtureRequest) -> np.ndarray:
+def array_fixture(request: pytest.FixtureRequest) -> npt.NDArray[Any]:
     array_request: ArrayRequest = request.param
     return (
         np.arange(np.prod(array_request.shape))
diff --git a/tests/v3/test_codec_entrypoints.py b/tests/v3/test_codec_entrypoints.py
index 9e2932fdd5..95dae68762 100644
--- a/tests/v3/test_codec_entrypoints.py
+++ b/tests/v3/test_codec_entrypoints.py
@@ -1,5 +1,6 @@
 import os.path
 import sys
+from collections.abc import Generator
 
 import pytest
 
@@ -10,7 +11,7 @@
 
 
 @pytest.fixture()
-def set_path():
+def set_path() -> Generator[None, None, None]:
     sys.path.append(here)
     zarr.registry._collect_entrypoints()
     yield
@@ -23,14 +24,14 @@ def set_path():
 
 @pytest.mark.usefixtures("set_path")
 @pytest.mark.parametrize("codec_name", ["TestEntrypointCodec", "TestEntrypointGroup.Codec"])
-def test_entrypoint_codec(codec_name):
+def test_entrypoint_codec(codec_name: str) -> None:
     config.set({"codecs.test": "package_with_entrypoint." + codec_name})
     cls_test = zarr.registry.get_codec_class("test")
     assert cls_test.__qualname__ == codec_name
 
 
 @pytest.mark.usefixtures("set_path")
-def test_entrypoint_pipeline():
+def test_entrypoint_pipeline() -> None:
     config.set({"codec_pipeline.path": "package_with_entrypoint.TestEntrypointCodecPipeline"})
     cls = zarr.registry.get_pipeline_class()
     assert cls.__name__ == "TestEntrypointCodecPipeline"
@@ -38,7 +39,7 @@ def test_entrypoint_pipeline():
 
 @pytest.mark.usefixtures("set_path")
 @pytest.mark.parametrize("buffer_name", ["TestEntrypointBuffer", "TestEntrypointGroup.Buffer"])
-def test_entrypoint_buffer(buffer_name):
+def test_entrypoint_buffer(buffer_name: str) -> None:
     config.set(
         {
             "buffer": "package_with_entrypoint." + buffer_name,
diff --git a/tests/v3/test_common.py b/tests/v3/test_common.py
index 85c6d93d36..c28723d1a8 100644
--- a/tests/v3/test_common.py
+++ b/tests/v3/test_common.py
@@ -109,7 +109,7 @@ def test_parse_shapelike_valid(data: Iterable[int]) -> None:
 
 # todo: more dtypes
 @pytest.mark.parametrize("data", [("uint8", np.uint8), ("float64", np.float64)])
-def parse_dtype(data: tuple[str, np.dtype]) -> None:
+def parse_dtype(data: tuple[str, np.dtype[Any]]) -> None:
     unparsed, parsed = data
     assert parse_dtype(unparsed) == parsed
 
diff --git a/tests/v3/test_config.py b/tests/v3/test_config.py
index b3e04e51da..c0674ecbfd 100644
--- a/tests/v3/test_config.py
+++ b/tests/v3/test_config.py
@@ -10,7 +10,7 @@
 import zarr
 from zarr import Array, zeros
 from zarr.abc.codec import CodecInput, CodecOutput, CodecPipeline
-from zarr.abc.store import ByteSetter
+from zarr.abc.store import ByteSetter, Store
 from zarr.codecs import BatchedCodecPipeline, BloscCodec, BytesCodec, Crc32cCodec, ShardingCodec
 from zarr.core.array_spec import ArraySpec
 from zarr.core.buffer import NDBuffer
@@ -77,17 +77,18 @@ def test_config_defaults_can_be_overridden(key: str, old_val: Any, new_val: Any)
         assert config.get(key) == new_val
 
 
-def test_fully_qualified_name():
+def test_fully_qualified_name() -> None:
     class MockClass:
         pass
 
-    assert "v3.test_config.test_fully_qualified_name.<locals>.MockClass" == fully_qualified_name(
-        MockClass
+    assert (
+        fully_qualified_name(MockClass)
+        == "tests.v3.test_config.test_fully_qualified_name.<locals>.MockClass"
     )
 
 
 @pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"])
-def test_config_codec_pipeline_class(store):
+def test_config_codec_pipeline_class(store: Store) -> None:
     # has default value
     assert get_pipeline_class().__name__ != ""
 
@@ -138,7 +139,7 @@ class MockEnvCodecPipeline(CodecPipeline):
 
 
 @pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"])
-def test_config_codec_implementation(store):
+def test_config_codec_implementation(store: Store) -> None:
     # has default value
     assert fully_qualified_name(get_codec_class("blosc")) == config.defaults[0]["codecs"]["blosc"]
 
@@ -171,7 +172,7 @@ async def _encode_single(
 
 
 @pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"])
-def test_config_ndbuffer_implementation(store):
+def test_config_ndbuffer_implementation(store: Store) -> None:
     # has default value
     assert fully_qualified_name(get_ndbuffer_class()) == config.defaults[0]["ndbuffer"]
 
@@ -191,7 +192,7 @@ def test_config_ndbuffer_implementation(store):
     assert isinstance(got, TestNDArrayLike)
 
 
-def test_config_buffer_implementation():
+def test_config_buffer_implementation() -> None:
     # has default value
     assert fully_qualified_name(get_buffer_class()) == config.defaults[0]["buffer"]
 
diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py
index 6989b406da..64d4cbc867 100644
--- a/tests/v3/test_group.py
+++ b/tests/v3/test_group.py
@@ -681,7 +681,7 @@ async def test_asyncgroup_update_attributes(
     assert agroup_new_attributes.attrs == attributes_new
 
 
-async def test_group_members_async(store: LocalStore | MemoryStore):
+async def test_group_members_async(store: LocalStore | MemoryStore) -> None:
     group = AsyncGroup(
         GroupMetadata(),
         store_path=StorePath(store=store, path="root"),
diff --git a/tests/v3/test_indexing.py b/tests/v3/test_indexing.py
index 3230bbc675..0b08378726 100644
--- a/tests/v3/test_indexing.py
+++ b/tests/v3/test_indexing.py
@@ -25,12 +25,11 @@
 if TYPE_CHECKING:
     from collections.abc import Iterator
 
-    from zarr.abc.store import Store
     from zarr.core.common import ChunkCoords
 
 
 @pytest.fixture
-async def store() -> Iterator[Store]:
+async def store() -> Iterator[StorePath]:
     yield StorePath(await MemoryStore.open(mode="w"))
 
 
@@ -52,7 +51,7 @@ def zarr_array_from_numpy_array(
 
 class CountingDict(MemoryStore):
     @classmethod
-    async def open(cls):
+    async def open(cls) -> CountingDict:
         store = await super().open(mode="w")
         store.counter = Counter()
         return store
@@ -68,7 +67,7 @@ async def set(self, key, value, byte_range=None):
         return await super().set(key, value, byte_range)
 
 
-def test_normalize_integer_selection():
+def test_normalize_integer_selection() -> None:
     assert 1 == normalize_integer_selection(1, 100)
     assert 99 == normalize_integer_selection(-1, 100)
     with pytest.raises(IndexError):
@@ -79,7 +78,7 @@ def test_normalize_integer_selection():
         normalize_integer_selection(-1000, 100)
 
 
-def test_replace_ellipsis():
+def test_replace_ellipsis() -> None:
     # 1D, single item
     assert (0,) == replace_ellipsis(0, (100,))
 
@@ -258,7 +257,7 @@ def _test_get_basic_selection(a, z, selection):
 
 
 # noinspection PyStatementEffect
-def test_get_basic_selection_1d(store: StorePath):
+def test_get_basic_selection_1d(store: StorePath) -> None:
     # setup
     a = np.arange(1050, dtype=int)
     z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,))
@@ -328,7 +327,7 @@ def test_get_basic_selection_1d(store: StorePath):
 
 
 # noinspection PyStatementEffect
-def test_get_basic_selection_2d(store: StorePath):
+def test_get_basic_selection_2d(store: StorePath) -> None:
     # setup
     a = np.arange(10000, dtype=int).reshape(1000, 10)
     z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3))
@@ -349,7 +348,7 @@ def test_get_basic_selection_2d(store: StorePath):
     np.testing.assert_array_equal(z[fancy_selection], [0, 11])
 
 
-def test_fancy_indexing_fallback_on_get_setitem(store: StorePath):
+def test_fancy_indexing_fallback_on_get_setitem(store: StorePath) -> None:
     z = zarr_array_from_numpy_array(store, np.zeros((20, 20)))
     z[[1, 2, 3], [1, 2, 3]] = 1
     np.testing.assert_array_equal(
diff --git a/tests/v3/test_metadata/__init__.py b/tests/v3/test_metadata/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests/v3/test_metadata/test_v3.py b/tests/v3/test_metadata/test_v3.py
index bc43154a54..5e5d22f0b1 100644
--- a/tests/v3/test_metadata/test_v3.py
+++ b/tests/v3/test_metadata/test_v3.py
@@ -237,7 +237,7 @@ def test_metadata_to_dict(
 
 @pytest.mark.parametrize("fill_value", [-1, 0, 1, 2932897])
 @pytest.mark.parametrize("precision", ["ns", "D"])
-async def test_datetime_metadata(fill_value: int, precision: str):
+async def test_datetime_metadata(fill_value: int, precision: str) -> None:
     metadata_dict = {
         "zarr_format": 3,
         "node_type": "array",
diff --git a/tests/v3/test_properties.py b/tests/v3/test_properties.py
index c978187cf1..7a085c03b7 100644
--- a/tests/v3/test_properties.py
+++ b/tests/v3/test_properties.py
@@ -11,7 +11,7 @@
 
 
 @given(st.data())
-def test_roundtrip(data):
+def test_roundtrip(data: st.DataObject) -> None:
     nparray = data.draw(np_arrays)
     zarray = data.draw(arrays(arrays=st.just(nparray)))
     assert_array_equal(nparray, zarray[:])
@@ -23,7 +23,7 @@ def test_roundtrip(data):
 # Uncomment the next line to reproduce the original failure.
 # @reproduce_failure('6.111.2', b'AXicY2FgZGRAB/8/ndR2z7nkDZEDADWpBL4=')
 @pytest.mark.filterwarnings("ignore::RuntimeWarning")
-def test_basic_indexing(data):
+def test_basic_indexing(data: st.DataObject) -> None:
     zarray = data.draw(arrays())
     nparray = zarray[:]
     indexer = data.draw(basic_indices(shape=nparray.shape))
@@ -42,7 +42,7 @@ def test_basic_indexing(data):
 # Uncomment the next line to reproduce the original failure.
 # @reproduce_failure('6.111.2', b'AXicY2FgZGRAB/8/eLmF7qr/C5EDADZUBRM=')
 @pytest.mark.filterwarnings("ignore::RuntimeWarning")
-def test_vindex(data):
+def test_vindex(data: st.DataObject) -> None:
     zarray = data.draw(arrays())
     nparray = zarray[:]
 
diff --git a/tests/v3/test_store/__init__.py b/tests/v3/test_store/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests/v3/test_store/test_core.py b/tests/v3/test_store/test_core.py
index 23821acfa6..c65d91f9d0 100644
--- a/tests/v3/test_store/test_core.py
+++ b/tests/v3/test_store/test_core.py
@@ -7,7 +7,7 @@
 from zarr.store.memory import MemoryStore
 
 
-async def test_make_store_path(tmpdir) -> None:
+async def test_make_store_path(tmpdir: str) -> None:
     # None
     store_path = await make_store_path(None)
     assert isinstance(store_path.store, MemoryStore)
@@ -33,4 +33,4 @@ async def test_make_store_path(tmpdir) -> None:
     assert Path(store_path.store.root) == Path(tmpdir)
 
     with pytest.raises(TypeError):
-        await make_store_path(1)
+        await make_store_path(1)  # type: ignore[arg-type]
diff --git a/tests/v3/test_store/test_remote.py b/tests/v3/test_store/test_remote.py
index b09bf24e24..7495bec8e1 100644
--- a/tests/v3/test_store/test_remote.py
+++ b/tests/v3/test_store/test_remote.py
@@ -1,5 +1,7 @@
 import os
+from collections.abc import Generator
 
+import botocore.client
 import fsspec
 import pytest
 from upath import UPath
@@ -22,7 +24,7 @@
 
 
 @pytest.fixture(scope="module")
-def s3_base():
+def s3_base() -> Generator[None, None, None]:
     # writable local S3 system
 
     # This fixture is module-scoped, meaning that we can reuse the MotoServer across all tests
@@ -37,7 +39,7 @@ def s3_base():
     server.stop()
 
 
-def get_boto3_client():
+def get_boto3_client() -> botocore.client.BaseClient:
     from botocore.session import Session
 
     # NB: we use the sync botocore client for setup
@@ -46,7 +48,7 @@ def get_boto3_client():
 
 
 @pytest.fixture(autouse=True, scope="function")
-def s3(s3_base):
+def s3(s3_base) -> Generator[s3fs.S3FileSystem, None, None]:
     """
     Quoting Martin Durant:
     pytest-asyncio creates a new event loop for each async test.
@@ -81,7 +83,7 @@ async def alist(it):
     return out
 
 
-async def test_basic():
+async def test_basic() -> None:
     store = await RemoteStore.open(
         f"s3://{test_bucket_name}", mode="w", endpoint_url=endpoint_url, anon=False
     )
diff --git a/tests/v3/test_store/test_stateful_store.py b/tests/v3/test_store/test_stateful_store.py
index a8f51e96e6..d062f9235f 100644
--- a/tests/v3/test_store/test_stateful_store.py
+++ b/tests/v3/test_store/test_stateful_store.py
@@ -102,7 +102,7 @@ class ZarrStoreStateMachine(RuleBasedStateMachine):
         https://hypothesis.readthedocs.io/en/latest/stateful.html
     """
 
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__()
         self.model: dict[str, bytes] = {}
         self.store = SyncStoreWrapper(MemoryStore(mode="w"))
@@ -170,7 +170,7 @@ def delete(self, data: bytes) -> None:
         del self.model[key]
 
     @rule()
-    def clear(self):
+    def clear(self) -> None:
         assert not self.store.mode.readonly
         note("(clear)")
         self.store.clear()
diff --git a/tests/v3/test_sync.py b/tests/v3/test_sync.py
index a335f9b48c..22834747e7 100644
--- a/tests/v3/test_sync.py
+++ b/tests/v3/test_sync.py
@@ -8,11 +8,10 @@
 
 
 @pytest.fixture(params=[True, False])
-def sync_loop(request) -> asyncio.AbstractEventLoop | None:
+def sync_loop(request: pytest.FixtureRequest) -> asyncio.AbstractEventLoop | None:
     if request.param is True:
         return _get_loop()
-
-    if request.param is False:
+    else:
         return None
 
 
@@ -58,7 +57,7 @@ def foo() -> str:
         return "foo"
 
     with pytest.raises(TypeError):
-        sync(foo(), loop=sync_loop)
+        sync(foo(), loop=sync_loop)  # type: ignore[arg-type]
 
 
 @pytest.mark.filterwarnings("ignore:coroutine.*was never awaited")
@@ -82,7 +81,7 @@ def foo() -> str:
         return "foo"
 
     async def bar() -> str:
-        return sync(foo(), loop=sync_loop)
+        return sync(foo(), loop=sync_loop)  # type: ignore[arg-type]
 
     with pytest.raises(SyncError):
         sync(bar(), loop=sync_loop)
@@ -92,7 +91,7 @@ async def bar() -> str:
 def test_sync_raises_if_loop_is_invalid_type() -> None:
     foo = AsyncMock(return_value="foo")
     with pytest.raises(TypeError):
-        sync(foo(), loop=1)
+        sync(foo(), loop=1)  # type: ignore[arg-type]
     foo.assert_not_awaited()
 
 
diff --git a/tests/v3/test_v2.py b/tests/v3/test_v2.py
index 4f4dc5aed3..9ddde68e23 100644
--- a/tests/v3/test_v2.py
+++ b/tests/v3/test_v2.py
@@ -4,16 +4,15 @@
 import pytest
 
 from zarr import Array
-from zarr.abc.store import Store
 from zarr.store import MemoryStore, StorePath
 
 
 @pytest.fixture
-async def store() -> Iterator[Store]:
+async def store() -> Iterator[StorePath]:
     yield StorePath(await MemoryStore.open(mode="w"))
 
 
-def test_simple(store: Store):
+def test_simple(store: StorePath) -> None:
     data = np.arange(0, 256, dtype="uint16").reshape((16, 16))
 
     a = Array.create(