From fea4ec80b8fd89189e1dfb3a820119988059c577 Mon Sep 17 00:00:00 2001 From: FengYin Date: Thu, 21 Oct 2021 23:30:09 +1100 Subject: [PATCH 01/11] Support async cursor to_list. --- pytest_async_mongodb/plugin.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pytest_async_mongodb/plugin.py b/pytest_async_mongodb/plugin.py index 585b536..4194798 100644 --- a/pytest_async_mongodb/plugin.py +++ b/pytest_async_mongodb/plugin.py @@ -54,6 +54,14 @@ async def __anext__(self): except StopIteration: raise StopAsyncIteration() + async def to_list(self, length): + the_list = [] + try: + while length is None or len(the_list) < length: + the_list.append(next(self)) + finally: + return the_list + @wrapp_methods class AsyncCollection(mongomock.Collection): From ae7a1dadfd4b1a390b0519e4027dd22bda7fdb96 Mon Sep 17 00:00:00 2001 From: Feng-Yin Date: Thu, 21 Oct 2021 23:37:29 +1100 Subject: [PATCH 02/11] Update .gitignore ignore .vscode --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index b89c115..9a87d64 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ # Created by https://www.toptal.com/developers/gitignore/api/python,linux,visualstudiocode # Edit at https://www.toptal.com/developers/gitignore?templates=python,linux,visualstudiocode +### vscode ### +.vscode + ### Linux ### *~ From ac07b31faf955ff68acffe7f7934811504eb8fa5 Mon Sep 17 00:00:00 2001 From: Feng Yin Date: Sun, 13 Mar 2022 14:44:46 +1100 Subject: [PATCH 03/11] support bulk write and aggregate --- pytest_async_mongodb/plugin.py | 27 ++++++++++++++++++++++++++- tests/unit/test_plugin.py | 16 ++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/pytest_async_mongodb/plugin.py b/pytest_async_mongodb/plugin.py index 4194798..80a0c53 100644 --- a/pytest_async_mongodb/plugin.py +++ b/pytest_async_mongodb/plugin.py @@ -54,7 +54,26 @@ async def __anext__(self): except StopIteration: raise StopAsyncIteration() - async def to_list(self, length): + async def to_list(self, length=None): + the_list = [] + try: + while length is None or len(the_list) < length: + the_list.append(next(self)) + finally: + return the_list + + +class AsyncCommandCursor(mongomock.command_cursor.CommandCursor): + def __aiter__(self): + return self + + async def __anext__(self): + try: + return next(self) + except StopIteration: + raise StopAsyncIteration() + + async def to_list(self, length=None): the_list = [] try: while length is None or len(the_list) < length: @@ -87,6 +106,7 @@ class AsyncCollection(mongomock.Collection): "create_index", "ensure_index", "map_reduce", + "bulk_write", ] def find(self, *args, **kwargs) -> AsyncCursor: @@ -94,6 +114,11 @@ def find(self, *args, **kwargs) -> AsyncCursor: cursor.__class__ = AsyncCursor return cursor + def aggregate(self, *args, **kwargs) -> AsyncCommandCursor: + cursor = super().aggregate(*args, **kwargs) + cursor.__class__ = AsyncCommandCursor + return cursor + @wrapp_methods class AsyncDatabase(mongomock.Database): diff --git a/tests/unit/test_plugin.py b/tests/unit/test_plugin.py index 7e4613d..578bfbb 100644 --- a/tests/unit/test_plugin.py +++ b/tests/unit/test_plugin.py @@ -1,6 +1,7 @@ from bson import ObjectId from pytest_async_mongodb import plugin import pytest +from pymongo import InsertOne pytestmark = pytest.mark.asyncio @@ -165,3 +166,18 @@ async def test_find_sorted_with_filter(async_mongodb): "winner": "France", }, ] + + +async def test_bulk_write_and_to_list(async_mongodb): + await async_mongodb.championships.bulk_write( + [ + InsertOne({"_id": 1, "a": 22}), + InsertOne({"_id": 2, "a": 22}), + InsertOne({"_id": 3, "a": 33}), + ] + ) + result = async_mongodb.championships.find({"a": 22}) + docs = await result.to_list() + assert len(docs) == 2 + assert docs[0]["a"] == 22 + assert docs[1]["a"] == 22 From cb7e2b23c977879a7182dffd88f18f2f4fac64d8 Mon Sep 17 00:00:00 2001 From: Feng Yin Date: Tue, 12 Jul 2022 10:41:24 +1000 Subject: [PATCH 04/11] support mongomock 4.1.2 --- pytest_async_mongodb/plugin.py | 38 ++++++++++++++++++++++------------ requirements.txt | 3 ++- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/pytest_async_mongodb/plugin.py b/pytest_async_mongodb/plugin.py index 80a0c53..f69b09c 100644 --- a/pytest_async_mongodb/plugin.py +++ b/pytest_async_mongodb/plugin.py @@ -7,9 +7,18 @@ import mongomock import pytest import yaml +from packaging import version _cache = {} +try: + from pymongo import version as pymongo_version + + PYMONGO_VERSION = version.parse(pymongo_version) +except ImportError: + # Default Pymongo version if not present. + PYMONGO_VERSION = version.parse("4.0") + def pytest_addoption(parser): @@ -86,28 +95,31 @@ async def to_list(self, length=None): class AsyncCollection(mongomock.Collection): ASYNC_METHODS = [ + "bulk_write", + "count_documents", + "create_index", + "delete_one", + "delete_many", + "drop", + "estimated_document_count", "find_one", "find_one_and_delete", "find_one_and_replace", "find_one_and_update", - "find_and_modify", - "save", - "delete_one", - "delete_many", - "count", "insert_one", "insert_many", + "replace_one", "update_one", "update_many", - "replace_one", - "count_documents", - "estimated_document_count", - "drop", - "create_index", - "ensure_index", - "map_reduce", - "bulk_write", ] + if PYMONGO_VERSION < version.parse("4.0"): + ASYNC_METHODS += [ + "count", + "ensure_index", + "find_and_modify", + "map_reduce", + "save", + ] def find(self, *args, **kwargs) -> AsyncCursor: cursor = super().find(*args, **kwargs) diff --git a/requirements.txt b/requirements.txt index 978fc84..d4223d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ mongomock>=3.22.1 pyyaml>=5.1 pytest-asyncio>=0.11.0 pytest>=5.4 -pymongo>=3.10 \ No newline at end of file +pymongo>=3.10 +packaging>=21.3 From 29dbbb193346d2774b34d27be21e34a0c7e849e7 Mon Sep 17 00:00:00 2001 From: Feng-Yin Date: Wed, 13 Jul 2022 15:58:57 +1000 Subject: [PATCH 05/11] Cleanup the code Cleanup the code --- pytest.ini | 1 + pytest_async_mongodb/plugin.py | 98 ++++++++++++++++------------------ 2 files changed, 46 insertions(+), 53 deletions(-) diff --git a/pytest.ini b/pytest.ini index 46cdef2..ecec326 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,5 @@ [pytest] +asyncio_mode = auto async_mongodb_fixture_dir = tests/unit/fixtures ; This tests specifying exactly which fixtures to load diff --git a/pytest_async_mongodb/plugin.py b/pytest_async_mongodb/plugin.py index f69b09c..f967398 100644 --- a/pytest_async_mongodb/plugin.py +++ b/pytest_async_mongodb/plugin.py @@ -8,6 +8,7 @@ import pytest import yaml from packaging import version +import pytest_asyncio _cache = {} @@ -46,11 +47,13 @@ async def wrapped(*args, **kwargs): return wrapped -def wrapp_methods(cls): - for method_name in cls.ASYNC_METHODS: - method = getattr(cls, method_name) - setattr(cls, method_name, async_decorator(method)) - return cls +def async_wrap(obj): + # wrap all the public interfaces except the one has been re-defined in obj + for item in dir(obj._base_sync_obj): + if not item.startswith("_"): + member = getattr(obj._base_sync_obj, item) + if callable(member) and item not in dir(obj): + setattr(obj, item, async_decorator(member)) class AsyncCursor(mongomock.collection.Cursor): @@ -91,56 +94,36 @@ async def to_list(self, length=None): return the_list -@wrapp_methods -class AsyncCollection(mongomock.Collection): - - ASYNC_METHODS = [ - "bulk_write", - "count_documents", - "create_index", - "delete_one", - "delete_many", - "drop", - "estimated_document_count", - "find_one", - "find_one_and_delete", - "find_one_and_replace", - "find_one_and_update", - "insert_one", - "insert_many", - "replace_one", - "update_one", - "update_many", - ] - if PYMONGO_VERSION < version.parse("4.0"): - ASYNC_METHODS += [ - "count", - "ensure_index", - "find_and_modify", - "map_reduce", - "save", - ] +class AsyncCollection: + def __init__(self, mongomock_collection): + self._base_sync_obj = mongomock_collection + async_wrap(self) def find(self, *args, **kwargs) -> AsyncCursor: - cursor = super().find(*args, **kwargs) + cursor = self._base_sync_obj.find(*args, **kwargs) cursor.__class__ = AsyncCursor return cursor def aggregate(self, *args, **kwargs) -> AsyncCommandCursor: - cursor = super().aggregate(*args, **kwargs) + cursor = self._base_sync_obj.aggregate(*args, **kwargs) cursor.__class__ = AsyncCommandCursor return cursor -@wrapp_methods -class AsyncDatabase(mongomock.Database): +class AsyncDatabase: + def __init__(self, mongomock_db): + self._base_sync_obj = mongomock_db + async_wrap(self) - ASYNC_METHODS = ["list_collection_names"] + def __getattr__(self, attr): + return self[attr] + + def __getitem__(self, db_name): + return self.get_collection(db_name) def get_collection(self, *args, **kwargs) -> AsyncCollection: - collection = super().get_collection(*args, **kwargs) - collection.__class__ = AsyncCollection - return collection + collection = self._base_sync_obj.get_collection(*args, **kwargs) + return AsyncCollection(collection) class Session: @@ -151,29 +134,38 @@ async def __aexit__(self, exc_type, exc, tb): await asyncio.sleep(0) -class AsyncMockMongoClient(mongomock.MongoClient): +class AsyncMockMongoClient: + def __init__(self, mongomock_client): + self._base_sync_obj = mongomock_client + async_wrap(self) + + def __getattr__(self, attr): + return self[attr] + + def __getitem__(self, db_name): + return self.get_database(db_name) + def get_database(self, *args, **kwargs) -> AsyncDatabase: - db = super().get_database(*args, **kwargs) - db.__class__ = AsyncDatabase - return db + db = self._base_sync_obj.get_database(*args, **kwargs) + return AsyncDatabase(db) async def start_session(self, **kwargs): await asyncio.sleep(0) return Session() -@pytest.fixture(scope="function") -async def async_mongodb(pytestconfig): - client = AsyncMockMongoClient() +@pytest_asyncio.fixture(scope="function") +async def async_mongodb(event_loop, pytestconfig): + client = AsyncMockMongoClient(mongomock.MongoClient()) db = client["pytest"] await clean_database(db) await load_fixtures(db, pytestconfig) return db -@pytest.fixture(scope="function") -async def async_mongodb_client(pytestconfig): - client = AsyncMockMongoClient() +@pytest_asyncio.fixture(scope="function") +async def async_mongodb_client(event_loop, pytestconfig): + client = AsyncMockMongoClient(mongomock.MongoClient()) db = client["pytest"] await clean_database(db) await load_fixtures(db, pytestconfig) @@ -183,7 +175,7 @@ async def async_mongodb_client(pytestconfig): async def clean_database(db): collections = await db.list_collection_names() for name in collections: - db.drop_collection(name) + await db.drop_collection(name) async def load_fixtures(db, config): From 28fb605b1d0b654fcb818f4c3c18500624f1314b Mon Sep 17 00:00:00 2001 From: Feng-Yin Date: Wed, 13 Jul 2022 16:25:11 +1000 Subject: [PATCH 06/11] removed the unused code. --- pytest_async_mongodb/plugin.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pytest_async_mongodb/plugin.py b/pytest_async_mongodb/plugin.py index f967398..f70bd7a 100644 --- a/pytest_async_mongodb/plugin.py +++ b/pytest_async_mongodb/plugin.py @@ -7,19 +7,10 @@ import mongomock import pytest import yaml -from packaging import version import pytest_asyncio _cache = {} -try: - from pymongo import version as pymongo_version - - PYMONGO_VERSION = version.parse(pymongo_version) -except ImportError: - # Default Pymongo version if not present. - PYMONGO_VERSION = version.parse("4.0") - def pytest_addoption(parser): From 4c529c7af91aa5dc580585e091c32ff3741e8301 Mon Sep 17 00:00:00 2001 From: Feng-Yin Date: Wed, 13 Jul 2022 16:28:30 +1000 Subject: [PATCH 07/11] Removed unused requirements. --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d4223d2..4e201c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,3 @@ pyyaml>=5.1 pytest-asyncio>=0.11.0 pytest>=5.4 pymongo>=3.10 -packaging>=21.3 From c24a132c14df30536b63f9c63032dcec297810cc Mon Sep 17 00:00:00 2001 From: Feng-Yin Date: Wed, 13 Jul 2022 16:50:15 +1000 Subject: [PATCH 08/11] cleanup requirements and import --- pytest_async_mongodb/plugin.py | 1 - requirements.txt | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pytest_async_mongodb/plugin.py b/pytest_async_mongodb/plugin.py index f70bd7a..b7c82da 100644 --- a/pytest_async_mongodb/plugin.py +++ b/pytest_async_mongodb/plugin.py @@ -5,7 +5,6 @@ import json import codecs import mongomock -import pytest import yaml import pytest_asyncio diff --git a/requirements.txt b/requirements.txt index 4e201c8..3ce1cbb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ mongomock>=3.22.1 -pyyaml>=5.1 +pymongo>=3.10 pytest-asyncio>=0.11.0 pytest>=5.4 -pymongo>=3.10 +pyyaml>=5.1 From 39d00041764fd64ff0aeecb7c3c3c6e5ed640061 Mon Sep 17 00:00:00 2001 From: Feng-Yin Date: Wed, 13 Jul 2022 17:03:21 +1000 Subject: [PATCH 09/11] add testcase test_estimated_document_count --- tests/unit/test_plugin.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/test_plugin.py b/tests/unit/test_plugin.py index 578bfbb..63c80d4 100644 --- a/tests/unit/test_plugin.py +++ b/tests/unit/test_plugin.py @@ -181,3 +181,7 @@ async def test_bulk_write_and_to_list(async_mongodb): assert len(docs) == 2 assert docs[0]["a"] == 22 assert docs[1]["a"] == 22 + + +async def test_estimated_document_count(async_mongodb): + assert await async_mongodb.championships.estimated_document_count() == 4 From ac1180d600cc192f1ae4c6157f9dee15a070fa65 Mon Sep 17 00:00:00 2001 From: Feng-Yin Date: Wed, 13 Jul 2022 17:22:45 +1000 Subject: [PATCH 10/11] added testcase test_find_one_and_update --- tests/unit/test_plugin.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/test_plugin.py b/tests/unit/test_plugin.py index 63c80d4..8b6b00d 100644 --- a/tests/unit/test_plugin.py +++ b/tests/unit/test_plugin.py @@ -185,3 +185,14 @@ async def test_bulk_write_and_to_list(async_mongodb): async def test_estimated_document_count(async_mongodb): assert await async_mongodb.championships.estimated_document_count() == 4 + + +async def test_find_one_and_update(async_mongodb): + await async_mongodb.championships.find_one_and_update( + filter={"_id": ObjectId("608b0151a20cf0c679939f59")}, + update={"$set": {"year": 2022}}, + ) + doc = await async_mongodb.championships.find_one( + {"_id": ObjectId("608b0151a20cf0c679939f59")} + ) + assert doc["year"] == 2022 From 33d3944e109b658a28524212b3cb2c245997ea07 Mon Sep 17 00:00:00 2001 From: Feng-Yin Date: Thu, 14 Jul 2022 15:52:44 +1000 Subject: [PATCH 11/11] added chained ops testcase --- tests/unit/test_plugin.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_plugin.py b/tests/unit/test_plugin.py index 8b6b00d..bc59197 100644 --- a/tests/unit/test_plugin.py +++ b/tests/unit/test_plugin.py @@ -1,7 +1,7 @@ from bson import ObjectId from pytest_async_mongodb import plugin import pytest -from pymongo import InsertOne +from pymongo import InsertOne, DESCENDING pytestmark = pytest.mark.asyncio @@ -196,3 +196,25 @@ async def test_find_one_and_update(async_mongodb): {"_id": ObjectId("608b0151a20cf0c679939f59")} ) assert doc["year"] == 2022 + + +async def test_chained_operations(async_mongodb): + docs = ( + await async_mongodb.championships.find() + .sort("year", DESCENDING) + .skip(1) + .limit(2) + .to_list() + ) + assert len(docs) == 2 + assert docs[0]["year"] == 2014 + assert docs[1]["year"] == 2010 + docs = ( + await async_mongodb.championships.find() + .sort("year", DESCENDING) + .skip(3) + .limit(2) + .to_list() + ) + assert len(docs) == 1 + assert docs[0]["year"] == 2006