From 148523fa05876a4a3028484bed830fe817184156 Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Fri, 25 Aug 2023 14:25:32 -0400 Subject: [PATCH 01/31] MemoryStore: mongomock -> pymongo-inmemory --- setup.cfg | 5 ++ setup.py | 1 + src/maggma/stores/mongolike.py | 120 ++++++++++++--------------------- tests/stores/test_mongolike.py | 4 +- 4 files changed, 50 insertions(+), 80 deletions(-) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..0ddc78462 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,5 @@ +# TODO - this entire file can be removed once pymongo-inmemory supports pyproject.toml +# see https://github.com/kaizendorks/pymongo_inmemory/issues/81 +[pymongo_inmemory] +use_local_mongod = False +mongod_port = 27019 diff --git a/setup.py b/setup.py index 8f1617453..639ab5f97 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ "pydantic<2.0", "pydantic>=0.32.2", "pymongo>=4.2.0", + "pymongo-inmemory", "monty>=1.0.2", "mongomock>=3.10.0", "pydash>=4.1.0", diff --git a/src/maggma/stores/mongolike.py b/src/maggma/stores/mongolike.py index 6cd734b7b..563f3ad8f 100644 --- a/src/maggma/stores/mongolike.py +++ b/src/maggma/stores/mongolike.py @@ -5,7 +5,7 @@ """ import warnings -from itertools import chain, groupby +from itertools import chain from pathlib import Path from socket import socket @@ -18,15 +18,15 @@ from typing_extensions import Literal -import mongomock import orjson from monty.dev import requires from monty.io import zopen from monty.json import MSONable, jsanitize from monty.serialization import loadfn -from pydash import get, has, set_ +from pydash import has, set_ from pymongo import MongoClient, ReplaceOne, uri_parser from pymongo.errors import ConfigurationError, DocumentTooLarge, OperationFailure +from pymongo_inmemory import MongoClient as MemoryClient from sshtunnel import SSHTunnelForwarder from maggma.core import Sort, Store, StoreError @@ -139,10 +139,12 @@ def __init__( port: TCP port to connect to username: Username for the collection password: Password to connect with + ssh_tunnel: SSHTunnel instance to use for connection. safe_update: fail gracefully on DocumentTooLarge errors on update auth_source: The database to authenticate on. Defaults to the database name. default_sort: Default sort field and direction to use when querying. Can be used to ensure determinacy in query results. + mongoclient_kwargs: Dict of extra kwargs to pass to MongoClient() """ self.database = database self.collection_name = collection_name @@ -578,95 +580,57 @@ class MemoryStore(MongoStore): to a MongoStore """ - def __init__(self, collection_name: str = "memory_db", **kwargs): + def __init__( + self, + database: str = "mem", + collection_name: str = "memory_store", + host: str = "localhost", + port: int = 27019, # to avoid conflicts with localhost + safe_update: bool = False, + mongoclient_kwargs: Optional[Dict] = None, + default_sort: Optional[Dict[str, Union[Sort, int]]] = None, + **kwargs, + ): """ - Initializes the Memory Store Args: - collection_name: name for the collection in memory - """ - self.collection_name = collection_name - self.default_sort = None - self._coll = None - self.kwargs = kwargs - super(MongoStore, self).__init__(**kwargs) + database: The database name + collection_name: The collection name + host: Hostname for the database + port: TCP port to connect to + safe_update: fail gracefully on DocumentTooLarge errors on update + default_sort: Default sort field and direction to use when querying. + Can be used to ensure determinacy in query results. + mongoclient_kwargs: Dict of extra kwargs to pass to MongoClient() + """ + super().__init__( + database=database, + collection_name=collection_name, + host=host, + port=port, + safe_update=safe_update, + mongoclient_kwargs=mongoclient_kwargs, + default_sort=default_sort, + **kwargs, + ) def connect(self, force_reset: bool = False): """ Connect to the source data """ + conn: MemoryClient = MemoryClient( + host=self.host, + port=self.port, + **self.mongoclient_kwargs, + ) - if self._coll is None or force_reset: - self._coll = mongomock.MongoClient().db[self.name] # type: ignore - - def close(self): - """Close up all collections""" - self._coll.database.client.close() + db = conn[self.database] + self._coll = db[self.collection_name] # type: ignore @property def name(self): """Name for the store""" return f"mem://{self.collection_name}" - def __hash__(self): - """Hash for the store""" - return hash((self.name, self.last_updated_field)) - - def groupby( - self, - keys: Union[List[str], str], - criteria: Optional[Dict] = None, - properties: Union[Dict, List, None] = None, - sort: Optional[Dict[str, Union[Sort, int]]] = None, - skip: int = 0, - limit: int = 0, - ) -> Iterator[Tuple[Dict, List[Dict]]]: - """ - Simple grouping function that will group documents - by keys. - - Args: - keys: fields to group documents - criteria: PyMongo filter for documents to search in - properties: properties to return in grouped documents - sort: Dictionary of sort order for fields. Keys are field names and - values are 1 for ascending or -1 for descending. - skip: number documents to skip - limit: limit on total number of documents returned - - Returns: - generator returning tuples of (key, list of elements) - """ - keys = keys if isinstance(keys, list) else [keys] - - if properties is None: - properties = [] - if isinstance(properties, dict): - properties = list(properties.keys()) - - data = [ - doc for doc in self.query(properties=keys + properties, criteria=criteria) if all(has(doc, k) for k in keys) - ] - - def grouping_keys(doc): - return tuple(get(doc, k) for k in keys) - - for vals, group in groupby(sorted(data, key=grouping_keys), key=grouping_keys): - doc = {} # type: ignore - for k, v in zip(keys, vals): - set_(doc, k, v) - yield doc, list(group) - - def __eq__(self, other: object) -> bool: - """ - Check equality for MemoryStore - other: other MemoryStore to compare with - """ - if not isinstance(other, MemoryStore): - return False - - fields = ["collection_name", "last_updated_field"] - return all(getattr(self, f) == getattr(other, f) for f in fields) - class JSONStore(MemoryStore): """ diff --git a/tests/stores/test_mongolike.py b/tests/stores/test_mongolike.py index 68784a639..901e717ee 100644 --- a/tests/stores/test_mongolike.py +++ b/tests/stores/test_mongolike.py @@ -4,7 +4,6 @@ from pathlib import Path from unittest import mock -import mongomock.collection import orjson import pymongo.collection import pytest @@ -238,8 +237,9 @@ def test_mongostore_newer_in(mongostore): def test_memory_store_connect(): memorystore = MemoryStore() assert memorystore._coll is None + assert "mem:" in memorystore.name memorystore.connect() - assert isinstance(memorystore._collection, mongomock.collection.Collection) + assert isinstance(memorystore._collection, pymongo.collection.Collection) def test_groupby(memorystore): From 4ab8b4a975a2dc0989fb2e77b497db4a2ca5ff69 Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Fri, 25 Aug 2023 15:23:22 -0400 Subject: [PATCH 02/31] MemoryStorage: use local mongod when possible except CI --- .github/workflows/testing.yml | 1 + setup.cfg | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index c9296832a..ff53d8a06 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -70,6 +70,7 @@ jobs: env: CONTINUOUS_INTEGRATION: True MONGODB_SRV_URI: ${{ secrets.MONGODB_SRV_URI }} + PYMONGOIM__USE_LOCAL_MONGOD: False run: | pip install -e . pytest --cov=maggma --cov-report=xml diff --git a/setup.cfg b/setup.cfg index 0ddc78462..3948a7bf2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ # TODO - this entire file can be removed once pymongo-inmemory supports pyproject.toml # see https://github.com/kaizendorks/pymongo_inmemory/issues/81 [pymongo_inmemory] -use_local_mongod = False +use_local_mongod = True mongod_port = 27019 From 1587e2f2e6a40f55a9394bff7990ae1960f76bfb Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Fri, 25 Aug 2023 15:49:39 -0400 Subject: [PATCH 03/31] pin pymongo-inmemory to pymongo version --- .github/workflows/testing.yml | 1 + setup.cfg | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index ff53d8a06..d5c7e54d9 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -71,6 +71,7 @@ jobs: CONTINUOUS_INTEGRATION: True MONGODB_SRV_URI: ${{ secrets.MONGODB_SRV_URI }} PYMONGOIM__USE_LOCAL_MONGOD: False + PYMONGOIM__OPERATING_SYSTEM: ubuntu run: | pip install -e . pytest --cov=maggma --cov-report=xml diff --git a/setup.cfg b/setup.cfg index 3948a7bf2..b005e3ab7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,3 +3,4 @@ [pymongo_inmemory] use_local_mongod = True mongod_port = 27019 +mongo_version = 4.2 From a306cb38d37a9c394fdead23e3d5dc4a052b4afc Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Fri, 25 Aug 2023 16:25:18 -0400 Subject: [PATCH 04/31] try CI fix --- .github/workflows/testing.yml | 1 - setup.cfg | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index d5c7e54d9..2a1b75f7d 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -70,7 +70,6 @@ jobs: env: CONTINUOUS_INTEGRATION: True MONGODB_SRV_URI: ${{ secrets.MONGODB_SRV_URI }} - PYMONGOIM__USE_LOCAL_MONGOD: False PYMONGOIM__OPERATING_SYSTEM: ubuntu run: | pip install -e . diff --git a/setup.cfg b/setup.cfg index b005e3ab7..215621f28 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,5 @@ # TODO - this entire file can be removed once pymongo-inmemory supports pyproject.toml # see https://github.com/kaizendorks/pymongo_inmemory/issues/81 [pymongo_inmemory] -use_local_mongod = True mongod_port = 27019 mongo_version = 4.2 From ba2b1c544e30063382f3cb677379da59c442a129 Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Fri, 25 Aug 2023 16:49:57 -0400 Subject: [PATCH 05/31] CI try --- .github/workflows/testing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 2a1b75f7d..16f9c98d1 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -71,6 +71,7 @@ jobs: CONTINUOUS_INTEGRATION: True MONGODB_SRV_URI: ${{ secrets.MONGODB_SRV_URI }} PYMONGOIM__OPERATING_SYSTEM: ubuntu + PYMONGOIM__USE_LOCAL_MONGOD: True run: | pip install -e . pytest --cov=maggma --cov-report=xml From c00bd84e18399d671c46fee7caf01dbafe37091b Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Fri, 25 Aug 2023 16:50:36 -0400 Subject: [PATCH 06/31] CI mongo version --- .github/workflows/testing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 16f9c98d1..cc107a989 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -71,6 +71,7 @@ jobs: CONTINUOUS_INTEGRATION: True MONGODB_SRV_URI: ${{ secrets.MONGODB_SRV_URI }} PYMONGOIM__OPERATING_SYSTEM: ubuntu + PYMONGOIM__MONGO_VERSION: 4.2 PYMONGOIM__USE_LOCAL_MONGOD: True run: | pip install -e . From cb577369778c03faf5a2bc02005ecf17f83db5f4 Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Fri, 25 Aug 2023 17:00:39 -0400 Subject: [PATCH 07/31] CI fix --- .github/workflows/testing.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index cc107a989..bd5f7c113 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -72,7 +72,6 @@ jobs: MONGODB_SRV_URI: ${{ secrets.MONGODB_SRV_URI }} PYMONGOIM__OPERATING_SYSTEM: ubuntu PYMONGOIM__MONGO_VERSION: 4.2 - PYMONGOIM__USE_LOCAL_MONGOD: True run: | pip install -e . pytest --cov=maggma --cov-report=xml From fae402300b97e28466b2ed7be57996903d45e0b6 Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Fri, 25 Aug 2023 17:04:12 -0400 Subject: [PATCH 08/31] MemoryStore: use MongoDB v6 --- .github/workflows/testing.yml | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index bd5f7c113..ce1b921ad 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -71,7 +71,7 @@ jobs: CONTINUOUS_INTEGRATION: True MONGODB_SRV_URI: ${{ secrets.MONGODB_SRV_URI }} PYMONGOIM__OPERATING_SYSTEM: ubuntu - PYMONGOIM__MONGO_VERSION: 4.2 + PYMONGOIM__MONGO_VERSION: 6.0 run: | pip install -e . pytest --cov=maggma --cov-report=xml diff --git a/setup.cfg b/setup.cfg index 215621f28..a75e9baed 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,4 +2,4 @@ # see https://github.com/kaizendorks/pymongo_inmemory/issues/81 [pymongo_inmemory] mongod_port = 27019 -mongo_version = 4.2 +mongo_version = 6.0 From 09cd5cca5082b7f1b108f7efe87b868ab0323d5c Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Fri, 25 Aug 2023 17:11:55 -0400 Subject: [PATCH 09/31] MemoryStore: expand __eq__ testing --- tests/stores/test_mongolike.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/stores/test_mongolike.py b/tests/stores/test_mongolike.py index 901e717ee..4a1bc8ef1 100644 --- a/tests/stores/test_mongolike.py +++ b/tests/stores/test_mongolike.py @@ -547,6 +547,29 @@ def test_eq(mongostore, memorystore, jsonstore): assert mongostore != jsonstore assert memorystore != jsonstore + # test case courtesy of @sivonxay + + # two MemoryStore with the same collection name point to the same db in memory + store1 = MemoryStore() + store2 = MemoryStore() + store1.connect() + store2.connect() + assert store1 == store2 + store1.update([{"a": 1, "b": 2}, {"a": 2, "b": 3}], "a") + assert store2.count() == 2 + + # with different collection names, they do not + store1 = MemoryStore(collection_name="store1") + store2 = MemoryStore(collection_name="store2") + assert store1 != store2 + + store1.connect() + store2.connect() + + store1.update([{"a": 1, "b": 2}, {"a": 2, "b": 3}], "a") + assert store1.count() != store2.count() + assert store1 != store2 # Returns True + @pytest.mark.skipif( "mongodb+srv" not in os.environ.get("MONGODB_SRV_URI", ""), From cbfc58fa1570c8ae56e4802fbb942f4a047ccaa2 Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Fri, 25 Aug 2023 17:31:11 -0400 Subject: [PATCH 10/31] MemoryStore: drop collection on close() --- src/maggma/stores/file_store.py | 4 ++-- src/maggma/stores/mongolike.py | 10 ++++++++++ tests/stores/test_file_store.py | 17 +++++++++++------ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/maggma/stores/file_store.py b/src/maggma/stores/file_store.py index 393fdfdeb..44aef51cd 100644 --- a/src/maggma/stores/file_store.py +++ b/src/maggma/stores/file_store.py @@ -95,7 +95,7 @@ def __init__( self.json_name = json_name file_filters = file_filters if file_filters else ["*"] self.file_filters = re.compile("|".join(fnmatch.translate(p) for p in file_filters)) - self.collection_name = "file_store" + self.collection_name = str(self.path) self.key = "file_id" self.include_orphans = include_orphans self.read_only = read_only @@ -104,7 +104,7 @@ def __init__( self.metadata_store = JSONStore( paths=[str(self.path / self.json_name)], read_only=self.read_only, - collection_name=self.collection_name, + collection_name="metadata_" + self.collection_name, key=self.key, ) diff --git a/src/maggma/stores/mongolike.py b/src/maggma/stores/mongolike.py index 563f3ad8f..31e5e14c2 100644 --- a/src/maggma/stores/mongolike.py +++ b/src/maggma/stores/mongolike.py @@ -626,6 +626,16 @@ def connect(self, force_reset: bool = False): db = conn[self.database] self._coll = db[self.collection_name] # type: ignore + def close(self): + """ + Close up all collections. In contrast to MongoStore, for MemoryStore the close() + method actually DROPS (erases) the underlying collection, in keeping with the + idea that MemoryStore is supposed to exist in memory and not persist after closing. + """ + self._collection.drop() + self._collection.database.client.close() + self._coll = None + @property def name(self): """Name for the store""" diff --git a/tests/stores/test_file_store.py b/tests/stores/test_file_store.py index 4f1df1dfd..99633096f 100644 --- a/tests/stores/test_file_store.py +++ b/tests/stores/test_file_store.py @@ -71,15 +71,16 @@ def test_newer_in_on_local_update(test_dir): fs.connect() with open(test_dir / "calculation1" / "input.in", "w") as f: f.write("Ryan was here") + lu_fs1 = fs.last_updated + lutime_fs1 = fs.query_one({"path": {"$regex": "calculation1/input.in"}})["last_updated"] + fs.close() + assert fs._coll is None + fs2 = FileStore(test_dir, read_only=False) fs2.connect() - assert fs2.last_updated > fs.last_updated - assert ( - fs2.query_one({"path": {"$regex": "calculation1/input.in"}})["last_updated"] - > fs.query_one({"path": {"$regex": "calculation1/input.in"}})["last_updated"] - ) - assert len(fs.newer_in(fs2)) == 1 + assert fs2.last_updated > lu_fs1 + assert fs2.query_one({"path": {"$regex": "calculation1/input.in"}})["last_updated"] > lutime_fs1 def test_max_depth(test_dir): @@ -94,21 +95,25 @@ def test_max_depth(test_dir): fs = FileStore(test_dir, read_only=False) fs.connect() assert len(list(fs.query())) == 6 + fs.close() # 0 depth should parse 1 file fs = FileStore(test_dir, read_only=False, max_depth=0) fs.connect() assert len(list(fs.query())) == 1 + fs.close() # 1 depth should parse 5 files fs = FileStore(test_dir, read_only=False, max_depth=1) fs.connect() assert len(list(fs.query())) == 5 + fs.close() # 2 depth should parse 6 files fs = FileStore(test_dir, read_only=False, max_depth=2) fs.connect() assert len(list(fs.query())) == 6 + fs.close() def test_orphaned_metadata(test_dir): From 55963ffa3dcf6a5f90db76ba5a080df704905f6e Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Fri, 25 Aug 2023 17:42:22 -0400 Subject: [PATCH 11/31] MemoryStore: test cleanups --- src/maggma/stores/mongolike.py | 2 +- tests/stores/test_mongolike.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/maggma/stores/mongolike.py b/src/maggma/stores/mongolike.py index 31e5e14c2..f6038181f 100644 --- a/src/maggma/stores/mongolike.py +++ b/src/maggma/stores/mongolike.py @@ -702,7 +702,7 @@ def __init__( self.serialization_option = serialization_option self.serialization_default = serialization_default - super().__init__(**kwargs) + super().__init__(collection_name=str(self.paths[0]), **kwargs) def connect(self, force_reset=False): """ diff --git a/tests/stores/test_mongolike.py b/tests/stores/test_mongolike.py index 4a1bc8ef1..80cfe9437 100644 --- a/tests/stores/test_mongolike.py +++ b/tests/stores/test_mongolike.py @@ -568,7 +568,6 @@ def test_eq(mongostore, memorystore, jsonstore): store1.update([{"a": 1, "b": 2}, {"a": 2, "b": 3}], "a") assert store1.count() != store2.count() - assert store1 != store2 # Returns True @pytest.mark.skipif( From c7655bda72760f61d8605a7b213de8c0628cd90c Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Fri, 25 Aug 2023 18:26:46 -0400 Subject: [PATCH 12/31] MemoryStore: assure test files close() part 1 --- src/maggma/stores/file_store.py | 1 - tests/cli/test_init.py | 22 ++++++++++++---------- tests/stores/test_azure.py | 6 ++++++ tests/stores/test_mongolike.py | 3 ++- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/maggma/stores/file_store.py b/src/maggma/stores/file_store.py index 44aef51cd..35452e437 100644 --- a/src/maggma/stores/file_store.py +++ b/src/maggma/stores/file_store.py @@ -104,7 +104,6 @@ def __init__( self.metadata_store = JSONStore( paths=[str(self.path / self.json_name)], read_only=self.read_only, - collection_name="metadata_" + self.collection_name, key=self.key, ) diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py index fede31497..2c58e5e3a 100644 --- a/tests/cli/test_init.py +++ b/tests/cli/test_init.py @@ -30,6 +30,13 @@ def reporting_store(): store._collection.drop() +@pytest.fixture() +def memorystore(): + store = MemoryStore("temp") + yield store + store.close() + + def test_basic_run(): runner = CliRunner() result = runner.invoke(run, ["--help"]) @@ -40,8 +47,7 @@ def test_basic_run(): assert result.exit_code != 0 -def test_run_builder(mongostore): - memorystore = MemoryStore("temp") +def test_run_builder(mongostore, memorystore): builder = CopyBuilder(mongostore, memorystore) mongostore.update([{mongostore.key: i, mongostore.last_updated_field: datetime.utcnow()} for i in range(10)]) @@ -70,8 +76,7 @@ def test_run_builder(mongostore): assert "Update" not in result.output -def test_run_builder_chain(mongostore): - memorystore = MemoryStore("temp") +def test_run_builder_chain(mongostore, memorystore): builder1 = CopyBuilder(mongostore, memorystore) builder2 = CopyBuilder(mongostore, memorystore) @@ -101,8 +106,7 @@ def test_run_builder_chain(mongostore): assert "Update" not in result.output -def test_reporting(mongostore, reporting_store): - memorystore = MemoryStore("temp") +def test_reporting(mongostore, reporting_store, memorystore): builder = CopyBuilder(mongostore, memorystore) mongostore.update([{mongostore.key: i, mongostore.last_updated_field: datetime.utcnow()} for i in range(10)]) @@ -154,8 +158,7 @@ def test_python_notebook_source(): assert "Ended multiprocessing: DummyBuilder" in result.output -def test_memray_run_builder(mongostore): - memorystore = MemoryStore("temp") +def test_memray_run_builder(mongostore, memorystore): builder = CopyBuilder(mongostore, memorystore) mongostore.update([{mongostore.key: i, mongostore.last_updated_field: datetime.utcnow()} for i in range(10)]) @@ -184,8 +187,7 @@ def test_memray_run_builder(mongostore): assert "Update" not in result.output -def test_memray_user_output_dir(mongostore): - memorystore = MemoryStore("temp") +def test_memray_user_output_dir(mongostore, memorystore): builder = CopyBuilder(mongostore, memorystore) mongostore.update([{mongostore.key: i, mongostore.last_updated_field: datetime.utcnow()} for i in range(10)]) diff --git a/tests/stores/test_azure.py b/tests/stores/test_azure.py index a3aefc88c..d869e905f 100644 --- a/tests/stores/test_azure.py +++ b/tests/stores/test_azure.py @@ -74,6 +74,7 @@ def blobstore(): store.connect() yield store + store.close() @pytest.fixture() @@ -113,6 +114,7 @@ def blobstore_w_subdir(): store.connect() yield store + store.close() @pytest.fixture() @@ -141,6 +143,7 @@ def test_keys(): with pytest.raises(KeyError): store.update({"key2": "mp-2", "data": "1234"}) assert store.key == store.index.key == "key1" + index.close() def test_multi_update(blobstore_two_docs, blobstore_multi): @@ -275,6 +278,7 @@ def test_bad_import(mocker): index = MemoryStore("index") with pytest.raises(RuntimeError): AzureBlobStore(index, "bucket1") + index.close() def test_eq(mongostore, blobstore_two_docs): @@ -398,6 +402,7 @@ def test_no_container(): create_container=True, ) store.connect() + index.close() def test_name(blobstore): @@ -419,3 +424,4 @@ def test_no_login(): with pytest.raises(RuntimeError, match=r".*Could not instantiate BlobServiceClient.*"): store.connect() + index.close() diff --git a/tests/stores/test_mongolike.py b/tests/stores/test_mongolike.py index 80cfe9437..f0c431efc 100644 --- a/tests/stores/test_mongolike.py +++ b/tests/stores/test_mongolike.py @@ -36,7 +36,8 @@ def montystore(tmp_dir): def memorystore(): store = MemoryStore() store.connect() - return store + yield store + store.close() @pytest.fixture() From 837b4a742c0408e689dd97a0003ccfd061bb9e46 Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Fri, 25 Aug 2023 18:38:46 -0400 Subject: [PATCH 13/31] MemoryStore test updates part 2 --- tests/stores/test_aws.py | 6 ++++++ tests/stores/test_azure.py | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/stores/test_aws.py b/tests/stores/test_aws.py index 5f54379bb..162f621ea 100644 --- a/tests/stores/test_aws.py +++ b/tests/stores/test_aws.py @@ -46,6 +46,7 @@ def s3store(): ) yield store + store.close() @pytest.fixture() @@ -59,6 +60,7 @@ def s3store_w_subdir(): store.connect() yield store + store.close() @pytest.fixture() @@ -72,6 +74,7 @@ def s3store_multi(): store.connect() yield store + store.close() def test_keys(): @@ -304,6 +307,9 @@ def test_newer_in(s3store): assert len(old_store.newer_in(new_store.index)) == 2 assert len(new_store.newer_in(old_store.index)) == 0 + new_store.close() + old_store.close() + def test_additional_metadata(s3store): tic = datetime(2018, 4, 12, 16) diff --git a/tests/stores/test_azure.py b/tests/stores/test_azure.py index d869e905f..fd126a1fe 100644 --- a/tests/stores/test_azure.py +++ b/tests/stores/test_azure.py @@ -278,7 +278,6 @@ def test_bad_import(mocker): index = MemoryStore("index") with pytest.raises(RuntimeError): AzureBlobStore(index, "bucket1") - index.close() def test_eq(mongostore, blobstore_two_docs): @@ -366,6 +365,9 @@ def test_newer_in(blobstore): assert len(old_store.newer_in(new_store.index)) == 2 assert len(new_store.newer_in(old_store.index)) == 0 + old_store.close() + new_store.close() + def test_additional_metadata(blobstore_two_docs): tic = datetime(2018, 4, 12, 16) @@ -424,4 +426,3 @@ def test_no_login(): with pytest.raises(RuntimeError, match=r".*Could not instantiate BlobServiceClient.*"): store.connect() - index.close() From 2df7d6a81bd4c7cb06719d8483394caab05c9769 Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Fri, 25 Aug 2023 22:14:49 -0400 Subject: [PATCH 14/31] MemoryStore: unique per instance coll name --- src/maggma/stores/file_store.py | 4 ++-- src/maggma/stores/mongolike.py | 20 +++++++++++++++----- tests/stores/test_file_store.py | 24 ++++++------------------ tests/stores/test_mongolike.py | 31 +++++++++++++++++++------------ 4 files changed, 42 insertions(+), 37 deletions(-) diff --git a/src/maggma/stores/file_store.py b/src/maggma/stores/file_store.py index 35452e437..63bae4dac 100644 --- a/src/maggma/stores/file_store.py +++ b/src/maggma/stores/file_store.py @@ -95,7 +95,7 @@ def __init__( self.json_name = json_name file_filters = file_filters if file_filters else ["*"] self.file_filters = re.compile("|".join(fnmatch.translate(p) for p in file_filters)) - self.collection_name = str(self.path) + # self.collection_name = str(self.path) self.key = "file_id" self.include_orphans = include_orphans self.read_only = read_only @@ -110,7 +110,7 @@ def __init__( self.kwargs = kwargs super().__init__( - collection_name=self.collection_name, + # collection_name=self.collection_name, key=self.key, **self.kwargs, ) diff --git a/src/maggma/stores/mongolike.py b/src/maggma/stores/mongolike.py index f6038181f..75ebc7a47 100644 --- a/src/maggma/stores/mongolike.py +++ b/src/maggma/stores/mongolike.py @@ -5,6 +5,7 @@ """ import warnings +from datetime import datetime from itertools import chain from pathlib import Path from socket import socket @@ -583,7 +584,7 @@ class MemoryStore(MongoStore): def __init__( self, database: str = "mem", - collection_name: str = "memory_store", + collection_name: Optional[str] = None, host: str = "localhost", port: int = 27019, # to avoid conflicts with localhost safe_update: bool = False, @@ -594,7 +595,9 @@ def __init__( """ Args: database: The database name - collection_name: The collection name + collection_name: The collection name. If None (default) a unique collection name is set based + on the current date and time. This ensures that multiple Store instances can coexist without + overwriting one another. host: Hostname for the database port: TCP port to connect to safe_update: fail gracefully on DocumentTooLarge errors on update @@ -602,6 +605,8 @@ def __init__( Can be used to ensure determinacy in query results. mongoclient_kwargs: Dict of extra kwargs to pass to MongoClient() """ + if not collection_name: + collection_name = str(datetime.utcnow()) super().__init__( database=database, collection_name=collection_name, @@ -633,14 +638,19 @@ def close(self): idea that MemoryStore is supposed to exist in memory and not persist after closing. """ self._collection.drop() - self._collection.database.client.close() - self._coll = None + super().close() @property def name(self): """Name for the store""" return f"mem://{self.collection_name}" + def __del__(self): + """ + Ensure collection is dropped from memory on object destruction, even if .close() has not been called. + """ + self._collection.drop() + class JSONStore(MemoryStore): """ @@ -702,7 +712,7 @@ def __init__( self.serialization_option = serialization_option self.serialization_default = serialization_default - super().__init__(collection_name=str(self.paths[0]), **kwargs) + super().__init__(**kwargs) def connect(self, force_reset=False): """ diff --git a/tests/stores/test_file_store.py b/tests/stores/test_file_store.py index 99633096f..13db1c56b 100644 --- a/tests/stores/test_file_store.py +++ b/tests/stores/test_file_store.py @@ -71,16 +71,15 @@ def test_newer_in_on_local_update(test_dir): fs.connect() with open(test_dir / "calculation1" / "input.in", "w") as f: f.write("Ryan was here") - lu_fs1 = fs.last_updated - lutime_fs1 = fs.query_one({"path": {"$regex": "calculation1/input.in"}})["last_updated"] - fs.close() - assert fs._coll is None - fs2 = FileStore(test_dir, read_only=False) fs2.connect() - assert fs2.last_updated > lu_fs1 - assert fs2.query_one({"path": {"$regex": "calculation1/input.in"}})["last_updated"] > lutime_fs1 + assert fs2.last_updated > fs.last_updated + assert ( + fs2.query_one({"path": {"$regex": "calculation1/input.in"}})["last_updated"] + > fs.query_one({"path": {"$regex": "calculation1/input.in"}})["last_updated"] + ) + assert len(fs.newer_in(fs2)) == 1 def test_max_depth(test_dir): @@ -95,25 +94,21 @@ def test_max_depth(test_dir): fs = FileStore(test_dir, read_only=False) fs.connect() assert len(list(fs.query())) == 6 - fs.close() # 0 depth should parse 1 file fs = FileStore(test_dir, read_only=False, max_depth=0) fs.connect() assert len(list(fs.query())) == 1 - fs.close() # 1 depth should parse 5 files fs = FileStore(test_dir, read_only=False, max_depth=1) fs.connect() assert len(list(fs.query())) == 5 - fs.close() # 2 depth should parse 6 files fs = FileStore(test_dir, read_only=False, max_depth=2) fs.connect() assert len(list(fs.query())) == 6 - fs.close() def test_orphaned_metadata(test_dir): @@ -136,7 +131,6 @@ def test_orphaned_metadata(test_dir): assert len(list(fs.query({"tags": {"$exists": True}}))) == 6 # the orphan field should be populated for all documents assert len(list(fs.query({"orphan": {"$exists": True}}))) == 6 - fs.close() # re-init the store with a different max_depth parameter # this will result in orphaned metadata @@ -150,7 +144,6 @@ def test_orphaned_metadata(test_dir): assert len(list(fs.query({"file_id": {"$exists": True}}))) == 6 assert len(list(fs.query({"path_relative": {"$exists": True}}))) == 6 assert len(list(fs.query({"orphan": True}))) == 1 - fs.close() # re-init the store after renaming one of the files on disk # this will result in orphaned metadata @@ -165,7 +158,6 @@ def test_orphaned_metadata(test_dir): assert len(list(fs.query({"path": {"$exists": True}}))) == 6 # manually specifying orphan: True should still work assert len(list(fs.query({"orphan": True}))) == 1 - fs.close() def test_store_files_moved(test_dir): @@ -185,7 +177,6 @@ def test_store_files_moved(test_dir): assert len(list(fs.query({"orphan": False}))) == 6 original_file_ids = {f["file_id"] for f in fs.query()} original_paths = {f["path"] for f in fs.query()} - fs.close() # now copy the entire FileStore to a new directory and re-initialize copy_tree(test_dir, str(test_dir / "new_store_location")) @@ -309,7 +300,6 @@ def test_metadata(test_dir): item_from_store = next(iter(fs.query({"file_id": key}))) assert item_from_store.get("name", False) assert item_from_store.get("metadata", False) - fs.close() # only the updated item should have been written to the JSON, # and it should not contain any of the protected keys @@ -334,7 +324,6 @@ def test_metadata(test_dir): assert item_from_store["name"] == "input.in" assert item_from_store["parent"] == "calculation1" assert item_from_store.get("metadata") == {"experiment date": "2022-01-18"} - fs2.close() # make sure reconnecting with read_only=False doesn't remove metadata from the JSON fs3 = FileStore(test_dir, read_only=False) @@ -346,7 +335,6 @@ def test_metadata(test_dir): assert item_from_store["name"] == "input.in" assert item_from_store["parent"] == "calculation1" assert item_from_store.get("metadata") == {"experiment date": "2022-01-18"} - fs3.close() # test automatic metadata assignment def add_data_from_name(d): diff --git a/tests/stores/test_mongolike.py b/tests/stores/test_mongolike.py index f0c431efc..991acbf1d 100644 --- a/tests/stores/test_mongolike.py +++ b/tests/stores/test_mongolike.py @@ -36,8 +36,7 @@ def montystore(tmp_dir): def memorystore(): store = MemoryStore() store.connect() - yield store - store.close() + return store @pytest.fixture() @@ -441,7 +440,6 @@ def test_json_store_writeable(test_dir): assert jsonstore.count() == 2 jsonstore.update({"new": "hello", "task_id": 2}) assert jsonstore.count() == 3 - jsonstore.close() # repeat the above with the deprecated file_writable kwarg # if the .json does not exist, it should be created @@ -460,30 +458,30 @@ def test_json_store_writeable(test_dir): assert jsonstore.count() == 2 jsonstore.update({"new": "hello", "task_id": 2}) assert jsonstore.count() == 3 - jsonstore.close() + jsonstore = JSONStore("d.json", file_writable=True) jsonstore.connect() assert jsonstore.count() == 3 jsonstore.remove_docs({"a": 5}) assert jsonstore.count() == 2 - jsonstore.close() + jsonstore = JSONStore("d.json", file_writable=True) jsonstore.connect() assert jsonstore.count() == 2 - jsonstore.close() + with mock.patch("maggma.stores.JSONStore.update_json_file") as update_json_file_mock: jsonstore = JSONStore("d.json", file_writable=False) jsonstore.connect() jsonstore.update({"new": "hello", "task_id": 5}) assert jsonstore.count() == 3 - jsonstore.close() + update_json_file_mock.assert_not_called() with mock.patch("maggma.stores.JSONStore.update_json_file") as update_json_file_mock: jsonstore = JSONStore("d.json", file_writable=False) jsonstore.connect() jsonstore.remove_docs({"task_id": 5}) assert jsonstore.count() == 2 - jsonstore.close() + update_json_file_mock.assert_not_called() @@ -496,7 +494,6 @@ class SubFloat(float): jsonstore.connect() with pytest.raises(orjson.JSONEncodeError): jsonstore.update({"wrong_field": SubFloat(1.1), "task_id": 3}) - jsonstore.close() jsonstore = JSONStore( "a.json", @@ -506,7 +503,6 @@ class SubFloat(float): ) jsonstore.connect() jsonstore.update({"wrong_field": SubFloat(1.1), "task_id": 3}) - jsonstore.close() def test_jsonstore_last_updated(test_dir): @@ -551,8 +547,8 @@ def test_eq(mongostore, memorystore, jsonstore): # test case courtesy of @sivonxay # two MemoryStore with the same collection name point to the same db in memory - store1 = MemoryStore() - store2 = MemoryStore() + store1 = MemoryStore(collection_name="test_collection") + store2 = MemoryStore(collection_name="test_collection") store1.connect() store2.connect() assert store1 == store2 @@ -570,6 +566,17 @@ def test_eq(mongostore, memorystore, jsonstore): store1.update([{"a": 1, "b": 2}, {"a": 2, "b": 3}], "a") assert store1.count() != store2.count() + # same with default collection name, which is unique per-instance + store1 = MemoryStore(collection_name=None) + store2 = MemoryStore(collection_name=None) + assert store1 != store2 + + store1.connect() + store2.connect() + + store1.update([{"a": 1, "b": 2}, {"a": 2, "b": 3}], "a") + assert store1.count() != store2.count() + @pytest.mark.skipif( "mongodb+srv" not in os.environ.get("MONGODB_SRV_URI", ""), From 14e843be1c043f33ed3e6bd96f8ad2c3780af994 Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Fri, 25 Aug 2023 22:19:44 -0400 Subject: [PATCH 15/31] MemoryStore __del__ fix --- src/maggma/stores/mongolike.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/maggma/stores/mongolike.py b/src/maggma/stores/mongolike.py index 75ebc7a47..9ec1a8e8f 100644 --- a/src/maggma/stores/mongolike.py +++ b/src/maggma/stores/mongolike.py @@ -649,7 +649,8 @@ def __del__(self): """ Ensure collection is dropped from memory on object destruction, even if .close() has not been called. """ - self._collection.drop() + if self._coll is not None: + self._collection.drop() class JSONStore(MemoryStore): From c3f43f9f9fff6552b12709a48006205603beff16 Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Fri, 25 Aug 2023 22:38:25 -0400 Subject: [PATCH 16/31] MemoryStore fixes --- src/maggma/stores/mongolike.py | 16 ++++++++-------- tests/cli/test_init.py | 4 +--- tests/stores/test_aws.py | 3 --- tests/stores/test_azure.py | 5 ----- 4 files changed, 9 insertions(+), 19 deletions(-) diff --git a/src/maggma/stores/mongolike.py b/src/maggma/stores/mongolike.py index 9ec1a8e8f..02ce2d327 100644 --- a/src/maggma/stores/mongolike.py +++ b/src/maggma/stores/mongolike.py @@ -622,14 +622,14 @@ def connect(self, force_reset: bool = False): """ Connect to the source data """ - conn: MemoryClient = MemoryClient( - host=self.host, - port=self.port, - **self.mongoclient_kwargs, - ) - - db = conn[self.database] - self._coll = db[self.collection_name] # type: ignore + if self._coll is None or force_reset: + conn: MemoryClient = MemoryClient( + host=self.host, + port=self.port, + **self.mongoclient_kwargs, + ) + db = conn[self.database] + self._coll = db[self.collection_name] # type: ignore def close(self): """ diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py index 2c58e5e3a..101bef85f 100644 --- a/tests/cli/test_init.py +++ b/tests/cli/test_init.py @@ -32,9 +32,7 @@ def reporting_store(): @pytest.fixture() def memorystore(): - store = MemoryStore("temp") - yield store - store.close() + return MemoryStore("temp") def test_basic_run(): diff --git a/tests/stores/test_aws.py b/tests/stores/test_aws.py index 162f621ea..3172054a6 100644 --- a/tests/stores/test_aws.py +++ b/tests/stores/test_aws.py @@ -307,9 +307,6 @@ def test_newer_in(s3store): assert len(old_store.newer_in(new_store.index)) == 2 assert len(new_store.newer_in(old_store.index)) == 0 - new_store.close() - old_store.close() - def test_additional_metadata(s3store): tic = datetime(2018, 4, 12, 16) diff --git a/tests/stores/test_azure.py b/tests/stores/test_azure.py index fd126a1fe..b95984882 100644 --- a/tests/stores/test_azure.py +++ b/tests/stores/test_azure.py @@ -143,7 +143,6 @@ def test_keys(): with pytest.raises(KeyError): store.update({"key2": "mp-2", "data": "1234"}) assert store.key == store.index.key == "key1" - index.close() def test_multi_update(blobstore_two_docs, blobstore_multi): @@ -365,9 +364,6 @@ def test_newer_in(blobstore): assert len(old_store.newer_in(new_store.index)) == 2 assert len(new_store.newer_in(old_store.index)) == 0 - old_store.close() - new_store.close() - def test_additional_metadata(blobstore_two_docs): tic = datetime(2018, 4, 12, 16) @@ -404,7 +400,6 @@ def test_no_container(): create_container=True, ) store.connect() - index.close() def test_name(blobstore): From 1e8c2621a446d4cb8b1e908c492e000379c14db7 Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Fri, 25 Aug 2023 22:50:41 -0400 Subject: [PATCH 17/31] MemoryStore fixes --- src/maggma/stores/mongolike.py | 16 ++++++++-------- tests/builders/test_copy_builder.py | 6 +++++- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/maggma/stores/mongolike.py b/src/maggma/stores/mongolike.py index 02ce2d327..0f3e8013e 100644 --- a/src/maggma/stores/mongolike.py +++ b/src/maggma/stores/mongolike.py @@ -631,14 +631,14 @@ def connect(self, force_reset: bool = False): db = conn[self.database] self._coll = db[self.collection_name] # type: ignore - def close(self): - """ - Close up all collections. In contrast to MongoStore, for MemoryStore the close() - method actually DROPS (erases) the underlying collection, in keeping with the - idea that MemoryStore is supposed to exist in memory and not persist after closing. - """ - self._collection.drop() - super().close() + # def close(self): + # """ + # Close up all collections. In contrast to MongoStore, for MemoryStore the close() + # method actually DROPS (erases) the underlying collection, in keeping with the + # idea that MemoryStore is supposed to exist in memory and not persist after closing. + # """ + # self._collection.drop() + # super().close() @property def name(self): diff --git a/tests/builders/test_copy_builder.py b/tests/builders/test_copy_builder.py index 3018562d9..2d3d78013 100644 --- a/tests/builders/test_copy_builder.py +++ b/tests/builders/test_copy_builder.py @@ -101,7 +101,8 @@ def test_run(source, target, old_docs, new_docs): builder = CopyBuilder(source, target) builder.run() - builder.target.connect() + + target.connect() assert builder.target.query_one(criteria={"k": 0})["v"] == "new" assert builder.target.query_one(criteria={"k": 10})["v"] == "old" @@ -112,6 +113,8 @@ def test_query(source, target, old_docs, new_docs): source.update(old_docs) source.update(new_docs) builder.run() + + target.connect() all_docs = list(target.query(criteria={})) assert len(all_docs) == 14 assert min([d["k"] for d in all_docs]) == 6 @@ -127,6 +130,7 @@ def test_delete_orphans(source, target, old_docs, new_docs): source._collection.delete_many(deletion_criteria) builder.run() + target.connect() assert target._collection.count_documents(deletion_criteria) == 0 assert target.query_one(criteria={"k": 5})["v"] == "new" assert target.query_one(criteria={"k": 10})["v"] == "old" From 232ab894491da01b4e2391883065d2ecbfa8e574 Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Fri, 25 Aug 2023 22:52:02 -0400 Subject: [PATCH 18/31] fixes --- tests/builders/test_projection_builder.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/builders/test_projection_builder.py b/tests/builders/test_projection_builder.py index 4c44b4dfe..1224394ba 100644 --- a/tests/builders/test_projection_builder.py +++ b/tests/builders/test_projection_builder.py @@ -102,6 +102,8 @@ def test_update_targets(source1, source2, target): def test_run(source1, source2, target): builder = Projection_Builder(source_stores=[source1, source2], target_store=target) builder.run() + + target.connect() assert len(list(target.query())) == 15 assert target.query_one(criteria={"k": 0})["a"] == "a" assert target.query_one(criteria={"k": 0})["d"] == "d" @@ -117,4 +119,6 @@ def test_query(source1, source2, target): query_by_key=[0, 1, 2, 3, 4], ) builder.run() + + target.connect() assert len(list(target.query())) == 5 From e403bfddd6ba4d3d4fc51f00c08489dc3e6ae175 Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Fri, 25 Aug 2023 23:08:29 -0400 Subject: [PATCH 19/31] MontyStore: inherit from MongoStore --- src/maggma/stores/mongolike.py | 62 ++++++++++++++++++++++++++++++++-- tests/stores/test_aws.py | 3 -- tests/stores/test_mongolike.py | 5 ++- 3 files changed, 63 insertions(+), 7 deletions(-) diff --git a/src/maggma/stores/mongolike.py b/src/maggma/stores/mongolike.py index 0f3e8013e..cadb6154b 100644 --- a/src/maggma/stores/mongolike.py +++ b/src/maggma/stores/mongolike.py @@ -6,7 +6,7 @@ import warnings from datetime import datetime -from itertools import chain +from itertools import chain, groupby from pathlib import Path from socket import socket @@ -24,7 +24,7 @@ from monty.io import zopen from monty.json import MSONable, jsanitize from monty.serialization import loadfn -from pydash import has, set_ +from pydash import get, has, set_ from pymongo import MongoClient, ReplaceOne, uri_parser from pymongo.errors import ConfigurationError, DocumentTooLarge, OperationFailure from pymongo_inmemory import MongoClient as MemoryClient @@ -832,7 +832,7 @@ def __eq__(self, other: object) -> bool: "MontyStore requires MontyDB to be installed. See the MontyDB repository for more " "information: https://github.com/davidlatwe/montydb", ) -class MontyStore(MemoryStore): +class MontyStore(MongoStore): """ A MongoDB compatible store that uses on disk files for storage. @@ -973,6 +973,62 @@ def update(self, docs: Union[List[Dict], Dict], key: Union[List, str, None] = No self._collection.replace_one(search_doc, d, upsert=True) + def groupby( + self, + keys: Union[List[str], str], + criteria: Optional[Dict] = None, + properties: Union[Dict, List, None] = None, + sort: Optional[Dict[str, Union[Sort, int]]] = None, + skip: int = 0, + limit: int = 0, + ) -> Iterator[Tuple[Dict, List[Dict]]]: + """ + Simple grouping function that will group documents + by keys. + + Args: + keys: fields to group documents + criteria: PyMongo filter for documents to search in + properties: properties to return in grouped documents + sort: Dictionary of sort order for fields. Keys are field names and + values are 1 for ascending or -1 for descending. + skip: number documents to skip + limit: limit on total number of documents returned + + Returns: + generator returning tuples of (key, list of elements) + """ + keys = keys if isinstance(keys, list) else [keys] + + if properties is None: + properties = [] + if isinstance(properties, dict): + properties = list(properties.keys()) + + data = [ + doc for doc in self.query(properties=keys + properties, criteria=criteria) if all(has(doc, k) for k in keys) + ] + + def grouping_keys(doc): + return tuple(get(doc, k) for k in keys) + + for vals, group in groupby(sorted(data, key=grouping_keys), key=grouping_keys): + doc = {} # type: ignore + for k, v in zip(keys, vals): + set_(doc, k, v) + yield doc, list(group) + + # def __eq__(self, other: object) -> bool: + # """ + # Check equality for MemoryStore + # other: other MemoryStore to compare with + # """ + # if not isinstance(other, MemoryStore): + # return False + + # fields = ["collection_name", "last_updated_field"] + # return all(getattr(self, f) == getattr(other, f) for f in fields) + def _find_free_port(address="0.0.0.0"): s = socket() diff --git a/tests/stores/test_aws.py b/tests/stores/test_aws.py index 3172054a6..5f54379bb 100644 --- a/tests/stores/test_aws.py +++ b/tests/stores/test_aws.py @@ -46,7 +46,6 @@ def s3store(): ) yield store - store.close() @pytest.fixture() @@ -60,7 +59,6 @@ def s3store_w_subdir(): store.connect() yield store - store.close() @pytest.fixture() @@ -74,7 +72,6 @@ def s3store_multi(): store.connect() yield store - store.close() def test_keys(): diff --git a/tests/stores/test_mongolike.py b/tests/stores/test_mongolike.py index 991acbf1d..38fd6a836 100644 --- a/tests/stores/test_mongolike.py +++ b/tests/stores/test_mongolike.py @@ -535,14 +535,17 @@ def test_jsonstore_last_updated(test_dir): assert jsonstore.last_updated > start_time -def test_eq(mongostore, memorystore, jsonstore): +def test_eq(mongostore, memorystore, jsonstore, montystore): + assert montystore == montystore assert mongostore == mongostore assert memorystore == memorystore assert jsonstore == jsonstore assert mongostore != memorystore + assert mongostore != montystore assert mongostore != jsonstore assert memorystore != jsonstore + assert memorystore != montystore # test case courtesy of @sivonxay From acb98e46b5b8a1c4302befbb9941c7260c4c8872 Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Fri, 25 Aug 2023 23:33:19 -0400 Subject: [PATCH 20/31] test fixes and MontyStore __eq__ fixups --- src/maggma/stores/mongolike.py | 33 ++++++++++++--------------------- tests/cli/test_init.py | 12 ++++++------ tests/stores/test_mongolike.py | 4 ++-- 3 files changed, 20 insertions(+), 29 deletions(-) diff --git a/src/maggma/stores/mongolike.py b/src/maggma/stores/mongolike.py index cadb6154b..f15ad7bf6 100644 --- a/src/maggma/stores/mongolike.py +++ b/src/maggma/stores/mongolike.py @@ -506,7 +506,7 @@ def __eq__(self, other: object) -> bool: Check equality for MongoStore other: other mongostore to compare with """ - if not isinstance(other, MongoStore): + if not isinstance(other, self.__class__): return False fields = ["database", "collection_name", "host", "port", "last_updated_field"] @@ -631,19 +631,10 @@ def connect(self, force_reset: bool = False): db = conn[self.database] self._coll = db[self.collection_name] # type: ignore - # def close(self): - # """ - # Close up all collections. In contrast to MongoStore, for MemoryStore the close() - # method actually DROPS (erases) the underlying collection, in keeping with the - # idea that MemoryStore is supposed to exist in memory and not persist after closing. - # """ - # self._collection.drop() - # super().close() - @property def name(self): """Name for the store""" - return f"mem://{self.collection_name}" + return f"mem://{self.database}/{self.collection_name}" def __del__(self): """ @@ -820,7 +811,7 @@ def __eq__(self, other: object) -> bool: Args: other: other JSONStore to compare with """ - if not isinstance(other, JSONStore): + if not isinstance(other, self.__class__): return False fields = ["paths", "last_updated_field"] @@ -1018,16 +1009,16 @@ def grouping_keys(doc): set_(doc, k, v) yield doc, list(group) - # def __eq__(self, other: object) -> bool: - # """ - # Check equality for MemoryStore - # other: other MemoryStore to compare with - # """ - # if not isinstance(other, MemoryStore): - # return False + def __eq__(self, other: object) -> bool: + """ + Check equality for MontyStore + other: other Store to compare with + """ + if not isinstance(other, self.__class__): + return False - # fields = ["collection_name", "last_updated_field"] - # return all(getattr(self, f) == getattr(other, f) for f in fields) + fields = ["database_name", "collection_name", "last_updated_field"] + return all(getattr(self, f) == getattr(other, f) for f in fields) def _find_free_port(address="0.0.0.0"): diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py index 101bef85f..e5a7ca877 100644 --- a/tests/cli/test_init.py +++ b/tests/cli/test_init.py @@ -60,7 +60,7 @@ def test_run_builder(mongostore, memorystore): result = runner.invoke(run, ["-vvv", "--no_bars", "test_builder.json"]) assert result.exit_code == 0 - assert "Get" not in result.output + assert "Get " not in result.output assert "Update" not in result.output result = runner.invoke(run, ["-v", "-n", "2", "test_builder.json"]) @@ -70,7 +70,7 @@ def test_run_builder(mongostore, memorystore): result = runner.invoke(run, ["-vvv", "-n", "2", "--no_bars", "test_builder.json"]) assert result.exit_code == 0 - assert "Get" not in result.output + assert "Get " not in result.output assert "Update" not in result.output @@ -90,7 +90,7 @@ def test_run_builder_chain(mongostore, memorystore): result = runner.invoke(run, ["-vvv", "--no_bars", "test_builders.json"]) assert result.exit_code == 0 - assert "Get" not in result.output + assert "Get " not in result.output assert "Update" not in result.output result = runner.invoke(run, ["-v", "-n", "2", "test_builders.json"]) @@ -100,7 +100,7 @@ def test_run_builder_chain(mongostore, memorystore): result = runner.invoke(run, ["-vvv", "-n", "2", "--no_bars", "test_builders.json"]) assert result.exit_code == 0 - assert "Get" not in result.output + assert "Get " not in result.output assert "Update" not in result.output @@ -171,7 +171,7 @@ def test_memray_run_builder(mongostore, memorystore): result = runner.invoke(run, ["-vvv", "--no_bars", "--memray", "on", "test_builder.json"]) assert result.exit_code == 0 - assert "Get" not in result.output + assert "Get " not in result.output assert "Update" not in result.output result = runner.invoke(run, ["-v", "-n", "2", "--memray", "on", "test_builder.json"]) @@ -181,7 +181,7 @@ def test_memray_run_builder(mongostore, memorystore): result = runner.invoke(run, ["-vvv", "-n", "2", "--no_bars", "--memray", "on", "test_builder.json"]) assert result.exit_code == 0 - assert "Get" not in result.output + assert "Get " not in result.output assert "Update" not in result.output diff --git a/tests/stores/test_mongolike.py b/tests/stores/test_mongolike.py index 38fd6a836..9d7ad3b61 100644 --- a/tests/stores/test_mongolike.py +++ b/tests/stores/test_mongolike.py @@ -26,7 +26,7 @@ def mongostore(): @pytest.fixture() -def montystore(tmp_dir): +def montystore(): store = MontyStore("maggma_test") store.connect() return store @@ -279,7 +279,7 @@ def test_groupby(memorystore): # Monty store tests -def test_monty_store_connect(tmp_dir): +def test_monty_store_connect(): montystore = MontyStore(collection_name="my_collection") assert montystore._coll is None montystore.connect() From 03c2e3167744d0af1ff01cfbe50948ad614f00ad Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Sat, 26 Aug 2023 21:42:42 -0400 Subject: [PATCH 21/31] revert changes to FileStore and Azure tests --- tests/stores/test_azure.py | 2 -- tests/stores/test_file_store.py | 7 +++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/stores/test_azure.py b/tests/stores/test_azure.py index b95984882..a3aefc88c 100644 --- a/tests/stores/test_azure.py +++ b/tests/stores/test_azure.py @@ -74,7 +74,6 @@ def blobstore(): store.connect() yield store - store.close() @pytest.fixture() @@ -114,7 +113,6 @@ def blobstore_w_subdir(): store.connect() yield store - store.close() @pytest.fixture() diff --git a/tests/stores/test_file_store.py b/tests/stores/test_file_store.py index 13db1c56b..4f1df1dfd 100644 --- a/tests/stores/test_file_store.py +++ b/tests/stores/test_file_store.py @@ -131,6 +131,7 @@ def test_orphaned_metadata(test_dir): assert len(list(fs.query({"tags": {"$exists": True}}))) == 6 # the orphan field should be populated for all documents assert len(list(fs.query({"orphan": {"$exists": True}}))) == 6 + fs.close() # re-init the store with a different max_depth parameter # this will result in orphaned metadata @@ -144,6 +145,7 @@ def test_orphaned_metadata(test_dir): assert len(list(fs.query({"file_id": {"$exists": True}}))) == 6 assert len(list(fs.query({"path_relative": {"$exists": True}}))) == 6 assert len(list(fs.query({"orphan": True}))) == 1 + fs.close() # re-init the store after renaming one of the files on disk # this will result in orphaned metadata @@ -158,6 +160,7 @@ def test_orphaned_metadata(test_dir): assert len(list(fs.query({"path": {"$exists": True}}))) == 6 # manually specifying orphan: True should still work assert len(list(fs.query({"orphan": True}))) == 1 + fs.close() def test_store_files_moved(test_dir): @@ -177,6 +180,7 @@ def test_store_files_moved(test_dir): assert len(list(fs.query({"orphan": False}))) == 6 original_file_ids = {f["file_id"] for f in fs.query()} original_paths = {f["path"] for f in fs.query()} + fs.close() # now copy the entire FileStore to a new directory and re-initialize copy_tree(test_dir, str(test_dir / "new_store_location")) @@ -300,6 +304,7 @@ def test_metadata(test_dir): item_from_store = next(iter(fs.query({"file_id": key}))) assert item_from_store.get("name", False) assert item_from_store.get("metadata", False) + fs.close() # only the updated item should have been written to the JSON, # and it should not contain any of the protected keys @@ -324,6 +329,7 @@ def test_metadata(test_dir): assert item_from_store["name"] == "input.in" assert item_from_store["parent"] == "calculation1" assert item_from_store.get("metadata") == {"experiment date": "2022-01-18"} + fs2.close() # make sure reconnecting with read_only=False doesn't remove metadata from the JSON fs3 = FileStore(test_dir, read_only=False) @@ -335,6 +341,7 @@ def test_metadata(test_dir): assert item_from_store["name"] == "input.in" assert item_from_store["parent"] == "calculation1" assert item_from_store.get("metadata") == {"experiment date": "2022-01-18"} + fs3.close() # test automatic metadata assignment def add_data_from_name(d): From 671bc6f8f11f14f9dd7814023c99935e2cfb9f00 Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Sat, 26 Aug 2023 21:44:29 -0400 Subject: [PATCH 22/31] revert changes to some JsonStore test --- tests/stores/test_mongolike.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/stores/test_mongolike.py b/tests/stores/test_mongolike.py index 9d7ad3b61..76cad6d17 100644 --- a/tests/stores/test_mongolike.py +++ b/tests/stores/test_mongolike.py @@ -440,6 +440,7 @@ def test_json_store_writeable(test_dir): assert jsonstore.count() == 2 jsonstore.update({"new": "hello", "task_id": 2}) assert jsonstore.count() == 3 + jsonstore.close() # repeat the above with the deprecated file_writable kwarg # if the .json does not exist, it should be created @@ -458,30 +459,30 @@ def test_json_store_writeable(test_dir): assert jsonstore.count() == 2 jsonstore.update({"new": "hello", "task_id": 2}) assert jsonstore.count() == 3 - + jsonstore.close() jsonstore = JSONStore("d.json", file_writable=True) jsonstore.connect() assert jsonstore.count() == 3 jsonstore.remove_docs({"a": 5}) assert jsonstore.count() == 2 - + jsonstore.close() jsonstore = JSONStore("d.json", file_writable=True) jsonstore.connect() assert jsonstore.count() == 2 - + jsonstore.close() with mock.patch("maggma.stores.JSONStore.update_json_file") as update_json_file_mock: jsonstore = JSONStore("d.json", file_writable=False) jsonstore.connect() jsonstore.update({"new": "hello", "task_id": 5}) assert jsonstore.count() == 3 - + jsonstore.close() update_json_file_mock.assert_not_called() with mock.patch("maggma.stores.JSONStore.update_json_file") as update_json_file_mock: jsonstore = JSONStore("d.json", file_writable=False) jsonstore.connect() jsonstore.remove_docs({"task_id": 5}) assert jsonstore.count() == 2 - + jsonstore.close() update_json_file_mock.assert_not_called() @@ -494,6 +495,7 @@ class SubFloat(float): jsonstore.connect() with pytest.raises(orjson.JSONEncodeError): jsonstore.update({"wrong_field": SubFloat(1.1), "task_id": 3}) + jsonstore.close() jsonstore = JSONStore( "a.json", @@ -503,6 +505,7 @@ class SubFloat(float): ) jsonstore.connect() jsonstore.update({"wrong_field": SubFloat(1.1), "task_id": 3}) + jsonstore.close() def test_jsonstore_last_updated(test_dir): From 4e0e6b5e9f2713331cf462a5496ac8edcdf542f1 Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Sat, 26 Aug 2023 22:15:07 -0400 Subject: [PATCH 23/31] more changes --- src/maggma/stores/file_store.py | 2 -- src/maggma/stores/mongolike.py | 12 ++++++------ tests/stores/test_mongolike.py | 3 ++- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/maggma/stores/file_store.py b/src/maggma/stores/file_store.py index 63bae4dac..8baf19210 100644 --- a/src/maggma/stores/file_store.py +++ b/src/maggma/stores/file_store.py @@ -95,7 +95,6 @@ def __init__( self.json_name = json_name file_filters = file_filters if file_filters else ["*"] self.file_filters = re.compile("|".join(fnmatch.translate(p) for p in file_filters)) - # self.collection_name = str(self.path) self.key = "file_id" self.include_orphans = include_orphans self.read_only = read_only @@ -110,7 +109,6 @@ def __init__( self.kwargs = kwargs super().__init__( - # collection_name=self.collection_name, key=self.key, **self.kwargs, ) diff --git a/src/maggma/stores/mongolike.py b/src/maggma/stores/mongolike.py index f15ad7bf6..a4738a75e 100644 --- a/src/maggma/stores/mongolike.py +++ b/src/maggma/stores/mongolike.py @@ -636,12 +636,12 @@ def name(self): """Name for the store""" return f"mem://{self.database}/{self.collection_name}" - def __del__(self): - """ - Ensure collection is dropped from memory on object destruction, even if .close() has not been called. - """ - if self._coll is not None: - self._collection.drop() + # def __del__(self): + # """ + # Ensure collection is dropped from memory on object destruction, even if .close() has not been called. + # """ + # if self._coll is not None: + # self._collection.drop() class JSONStore(MemoryStore): diff --git a/tests/stores/test_mongolike.py b/tests/stores/test_mongolike.py index 76cad6d17..b5d27875e 100644 --- a/tests/stores/test_mongolike.py +++ b/tests/stores/test_mongolike.py @@ -29,7 +29,8 @@ def mongostore(): def montystore(): store = MontyStore("maggma_test") store.connect() - return store + yield store + store._collection.drop() @pytest.fixture() From 830d0a6cca4bd3457cddafd87b2685974e6ea2b4 Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Sat, 26 Aug 2023 22:30:28 -0400 Subject: [PATCH 24/31] test fixes --- tests/api/test_submission_resource.py | 3 ++- tests/stores/test_aws.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/api/test_submission_resource.py b/tests/api/test_submission_resource.py index aef7b73f4..5b5f48564 100644 --- a/tests/api/test_submission_resource.py +++ b/tests/api/test_submission_resource.py @@ -34,7 +34,8 @@ def owner_store(): store = MemoryStore("owners", key="name") store.connect() store.update([d.dict() for d in owners]) - return store + yield store + store._collection.drop() @pytest.fixture() diff --git a/tests/stores/test_aws.py b/tests/stores/test_aws.py index 5f54379bb..c3698618a 100644 --- a/tests/stores/test_aws.py +++ b/tests/stores/test_aws.py @@ -4,6 +4,7 @@ import boto3 import pytest from botocore.exceptions import ClientError +from maggma.core.store import StoreError from maggma.stores import MemoryStore, MongoStore, S3Store from moto import mock_s3 @@ -202,7 +203,7 @@ def objects_in_bucket(key): def test_close(s3store): list(s3store.query()) s3store.close() - with pytest.raises(AttributeError): + with pytest.raises(StoreError): list(s3store.query()) From d27558da552418b11e50a12ee8fdda5fd2f822de Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Sat, 26 Aug 2023 22:54:51 -0400 Subject: [PATCH 25/31] fixes --- tests/api/test_submission_resource.py | 3 +-- tests/stores/test_advanced_stores.py | 8 +++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/api/test_submission_resource.py b/tests/api/test_submission_resource.py index 5b5f48564..aef7b73f4 100644 --- a/tests/api/test_submission_resource.py +++ b/tests/api/test_submission_resource.py @@ -34,8 +34,7 @@ def owner_store(): store = MemoryStore("owners", key="name") store.connect() store.update([d.dict() for d in owners]) - yield store - store._collection.drop() + return store @pytest.fixture() diff --git a/tests/stores/test_advanced_stores.py b/tests/stores/test_advanced_stores.py index c0522a4fc..5db7a112f 100644 --- a/tests/stores/test_advanced_stores.py +++ b/tests/stores/test_advanced_stores.py @@ -284,7 +284,8 @@ def sandbox_store(): memstore = MemoryStore() store = SandboxStore(memstore, sandbox="test") store.connect() - return store + yield store + store._collection.drop() def test_sandbox_count(sandbox_store): @@ -312,10 +313,11 @@ def test_sandbox_distinct(sandbox_store): assert sandbox_store.distinct("a") == [1] sandbox_store._collection.insert_one({"a": 4, "d": 5, "e": 6, "sbxn": ["test"]}) - assert sandbox_store.distinct("a")[1] == 4 + assert sandbox_store.distinct("a") == [4, 1] sandbox_store._collection.insert_one({"a": 7, "d": 8, "e": 9, "sbxn": ["not_test"]}) - assert sandbox_store.distinct("a")[1] == 4 + print(sandbox_store.distinct("a")) + assert sandbox_store.distinct("a") == [4, 1] def test_sandbox_update(sandbox_store): From eac940637dadba843958a1f1f35a1008c029a4da Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Sat, 26 Aug 2023 23:04:38 -0400 Subject: [PATCH 26/31] fix --- tests/stores/test_advanced_stores.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/stores/test_advanced_stores.py b/tests/stores/test_advanced_stores.py index 5db7a112f..b80506c17 100644 --- a/tests/stores/test_advanced_stores.py +++ b/tests/stores/test_advanced_stores.py @@ -313,11 +313,11 @@ def test_sandbox_distinct(sandbox_store): assert sandbox_store.distinct("a") == [1] sandbox_store._collection.insert_one({"a": 4, "d": 5, "e": 6, "sbxn": ["test"]}) - assert sandbox_store.distinct("a") == [4, 1] + assert set(sandbox_store.distinct("a")) == {4, 1} sandbox_store._collection.insert_one({"a": 7, "d": 8, "e": 9, "sbxn": ["not_test"]}) print(sandbox_store.distinct("a")) - assert sandbox_store.distinct("a") == [4, 1] + assert set(sandbox_store.distinct("a")) == {4, 1} def test_sandbox_update(sandbox_store): From 421e4e9afdf8498166650b766e08507678fb9c5d Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Sat, 26 Aug 2023 23:25:55 -0400 Subject: [PATCH 27/31] rm print statement --- tests/stores/test_advanced_stores.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/stores/test_advanced_stores.py b/tests/stores/test_advanced_stores.py index b80506c17..abff3a97f 100644 --- a/tests/stores/test_advanced_stores.py +++ b/tests/stores/test_advanced_stores.py @@ -316,7 +316,6 @@ def test_sandbox_distinct(sandbox_store): assert set(sandbox_store.distinct("a")) == {4, 1} sandbox_store._collection.insert_one({"a": 7, "d": 8, "e": 9, "sbxn": ["not_test"]}) - print(sandbox_store.distinct("a")) assert set(sandbox_store.distinct("a")) == {4, 1} From 3cf4e3f133023a65e3683356656aad3e3285eab5 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 29 Aug 2023 14:46:50 -0700 Subject: [PATCH 28/31] Fix test_patch_submission --- tests/api/test_submission_resource.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/api/test_submission_resource.py b/tests/api/test_submission_resource.py index aef7b73f4..f484e931c 100644 --- a/tests/api/test_submission_resource.py +++ b/tests/api/test_submission_resource.py @@ -3,7 +3,7 @@ from random import randint import pytest -from fastapi import FastAPI +from fastapi import FastAPI, Query, Body from maggma.api.query_operator import PaginationQuery from maggma.api.query_operator.core import QueryOperator from maggma.api.resource import SubmissionResource @@ -49,7 +49,8 @@ def query(self, name): @pytest.fixture() def patch_query_op(): class PatchQuery(QueryOperator): - def query(self, name, update): + def query(self, name=Query(), update=Body()): + print("UPDATE:", update) return {"criteria": {"name": name}, "update": update} return PatchQuery() @@ -112,10 +113,10 @@ def test_submission_patch(owner_store, post_query_op, patch_query_op): app.include_router(endpoint.router) client = TestClient(app) - update = json.dumps({"last_updated": "2023-06-22T17:32:11.645713"}) + update = {"last_updated": "2023-06-22T17:32:11.645713"} assert client.get("/").status_code == 200 - assert client.patch(f"/?name=PersonAge9&update={update}").status_code == 200 + assert client.patch("/?name=PersonAge9", json=update).status_code == 200 def test_key_fields(owner_store, post_query_op): From b478ad595b635f988b68e2d35bdfeabc0a37481f Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Sun, 10 Sep 2023 10:23:59 -0400 Subject: [PATCH 29/31] bump pymongo-inmemory to 0.3.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 638fd614b..e27bc7e2e 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ "pydantic<2.0", "pydantic>=0.32.2", "pymongo>=4.2.0", - "pymongo-inmemory", + "pymongo-inmemory>=0.3.1", "monty>=1.0.2", "mongomock>=3.10.0", "pydash>=4.1.0", From 33a1fdf9777cac6561a251109a9d095caa60091d Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Sun, 10 Sep 2023 10:38:29 -0400 Subject: [PATCH 30/31] CI python version workaround --- .github/workflows/testing.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index d59a1b9b4..f46a33202 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -51,7 +51,9 @@ jobs: max-parallel: 6 matrix: os: [ubuntu-latest] - python-version: ["3.8", "3.9", "3.10", "3.11"] + # python-version: ["3.8", "3.9", "3.10", "3.11"] + # temporary workaround since pymongo-inmemory dropped 3.8 support + python-version: ["3.9", "3.10", "3.11"] runs-on: ${{ matrix.os }} From d448e4934d034f14c3810f3dc318dd50de080367 Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Wed, 15 May 2024 10:09:21 -0400 Subject: [PATCH 31/31] add pymongo-inmemory --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 30f5692ca..6be0ca5e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ dependencies = [ "pymongo>=4.2.0", "monty>=2023.9.25", "mongomock>=3.10.0", + "pymongo-inmemory>=0.4.1", "pydash>=4.1.0", "jsonschema>=3.1.1", "tqdm>=4.19.6",