From c2395d9b2ac74fc41da260af08e8b7c8df0256e0 Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Tue, 15 Sep 2026 15:24:44 +0200 Subject: [PATCH 1/3] docs: repair RST rendering artifacts, fill docstring coverage gaps Fixes all epythet validate level-0.5 errors (unbalanced markup, malformed field lists, doctest glued to prose, colon-role breaking docutils parsing). Adds a docstring to every previously-undocumented public class/method/function (0 undocumented objects, down from 50), including executed and verified doctests for the entry points that had none. Fixes a stale docstring claim in get_mongo_collection_pymongo_obj (wrong default db/collection name). Ignores the generated docsrc/ directory. --- .gitignore | 1 + mongodol/add_ons.py | 7 ++- mongodol/base.py | 77 +++++++++++++++++++++++------ mongodol/recipes.py | 2 +- mongodol/stores.py | 95 ++++++++++++++++++++++++++++++++++-- mongodol/tests/util.py | 1 + mongodol/tracking_methods.py | 10 ++++ mongodol/trans.py | 42 +++++++++++----- mongodol/util.py | 46 +++++++++++++---- mongodol/utils/werk_local.py | 3 +- mongodol/views.py | 12 +++-- 11 files changed, 246 insertions(+), 50 deletions(-) diff --git a/.gitignore b/.gitignore index f04d784..ec8b27f 100644 --- a/.gitignore +++ b/.gitignore @@ -73,6 +73,7 @@ instance/ # Sphinx documentation docs/_build/ +docsrc/ # PyBuilder target/ diff --git a/mongodol/add_ons.py b/mongodol/add_ons.py index 76cd54f..4725d63 100644 --- a/mongodol/add_ons.py +++ b/mongodol/add_ons.py @@ -16,6 +16,7 @@ def disallow_if_name_exists_already(store, method_name): + """Raise ``MethodNameAlreadyExists`` if ``store`` already has an attribute named ``method_name``.""" if hasattr(store, method_name): raise MethodNameAlreadyExists(f"Method name already exists: {method_name}") @@ -37,12 +38,15 @@ def has_exactly_one_non_defaulted_input(func): class Addons(ABC): """A collection of add-on methods. Addons can't (and is not meant to) be instantiated. - It's just to group add-on functions (meant to be injected in stores) in one place""" + It's just to group add-on functions (meant to be injected in stores) in one place + """ def clear(self: MongoCollectionCollection): + """Delete every doc matching this store's filter, without confirmation.""" return self.mgc.delete_many(self.filter) def clear_after_checking_with_user(self: MongoCollectionCollection): + """Delete every doc matching this store's filter, after the user confirms the count on stdin.""" n = len(self) answer = input( f"Are you sure you want to delete all {n} docs matching the filter: {self.filter}?\n" @@ -115,7 +119,6 @@ def add_clear_method( >>> assert len(whole_store) == whole_length_before_clear - n_reds == 3 """ - """Add a clear method to the store""" return add_store_method( store, method_func=clear_method, method_name="clear", validator=validator ) diff --git a/mongodol/base.py b/mongodol/base.py index 6b99f68..8fca1b0 100644 --- a/mongodol/base.py +++ b/mongodol/base.py @@ -28,6 +28,8 @@ # TODO: mgc type annotation # See https://stackoverflow.com/questions/66464191/referencing-a-python-class-within-its-definition-but-outside-a-method class MongoCollectionCollection(DolCollection): + """Base class wrapping a mongo collection with a fixed ``filter`` and ``iter_projection``.""" + def __init__( self, mgc: PyMongoCollectionSpec | DolCollection = None, @@ -41,9 +43,7 @@ def __init__( self._mgc_find_kwargs = mgc_find_kwargs def _merge_with_filt(self, m: Mapping) -> dict: - """ - - :param args: dictionaries that are valid mongo queries + """:param args: dictionaries that are valid mongo queries :return: >>> class Mock(MongoCollectionCollection): @@ -83,6 +83,7 @@ def _count_kwargs(self): @cached_property def mgc_repr(self): + """A short ```` string identifying the wrapped mongo collection.""" return f"<{self.mgc.database.name}/{self.mgc.name}>" def __repr__(self): @@ -122,7 +123,7 @@ class MongoCollectionReader(MongoCollectionCollection, KvReader): that is really meant to be wrapped in order to produce the actual key-value interfaces one needs. You shouldn't think of it's instances as a normal dict where any request for the value under a key, for a key that doesn't exist, will result in a ``KeyError``. - Note that this means that `s.get(k, default)` will never result in the default being returned, + Note that this means that ``s.get(k, default)`` will never result in the default being returned, since there are no missing keys here; only empty results (cursors that don't yield anything). >>> v = s.get(fake_key, {'the': 'default'}) @@ -200,12 +201,14 @@ def __getitem__(self, k): ) def contains_value(self, v): + """Bulk-read counterpart of ``__contains__`` for values: is there a doc matching ``v``?""" cursor = self.mgc.find( filter=self._merge_with_filt(v), projection=(), **self._mgc_find_kwargs ) return next(cursor, end_of_cursor) is not end_of_cursor def iter_values(self): + """Bulk-read all values in a single ``find`` query (see the module's bulk-read protocol).""" return self.mgc.find( filter=self.filter, projection=self._getitem_projection, @@ -213,6 +216,7 @@ def iter_values(self): ) def contains_item(self, item): + """Bulk-read counterpart of ``__contains__`` for ``(key, value)`` pairs.""" k, v = item # TODO: How do we have cursor return no data (here still has _id) cursor = self.mgc.find( @@ -223,6 +227,9 @@ def contains_item(self, item): return next(cursor, end_of_cursor) is not end_of_cursor def iter_items(self): + """Bulk-read all ``(key, value)`` pairs in a single ``find`` query, splitting each doc into + its key fields and the rest. + """ cursor = self.mgc.find( filter=self.filter, projection=self._items_projection, @@ -248,6 +255,7 @@ def _items_projection(self): @cached_property def key_fields(self): + """The field names (from ``iter_projection``) that make up a key.""" _iter_projection = normalize_projection(self._iter_projection) return tuple( field for field in _iter_projection if _iter_projection[field] is True @@ -255,6 +263,7 @@ def key_fields(self): @cached_property def val_fields(self): + """The field names (from ``getitem_projection``) that make up a value, or None if unset.""" if self._getitem_projection is None: return None else: @@ -276,6 +285,7 @@ def from_params( getitem_projection: ProjectionSpec = None, **mgc_find_kwargs, ): + """Make an instance from db/collection names and connection params, instead of a live mongo collection object.""" if mongo_client is None: mongo_client = MongoClient() elif isinstance(mongo_client, dict): @@ -290,6 +300,7 @@ def from_params( ) def distinct(self, key, filter=None, **kwargs): + """The distinct values of ``key`` across docs matching ``filter`` (merged with this store's own filter).""" # TODO: Check if this is correct (what about $ cases?): filter=m._merge_with_filt(filter) return self.mgc.distinct( key, filter=self._merge_with_filt(filter or {}), **kwargs @@ -298,6 +309,7 @@ def distinct(self, key, filter=None, **kwargs): unique = distinct def aggregate(self, pipeline, **kwargs): + """Run a mongo aggregation ``pipeline``, prefixed with a ``$match`` on this store's filter.""" _pipeline = pipeline.copy() _pipeline.insert(0, {"$match": self.filter}) return self.mgc.aggregate(_pipeline, **kwargs) @@ -430,12 +442,14 @@ def __delitem__(self, k): raise KeyError(f"You can't remove that key: {k}") def append(self, v): + """Insert a single doc ``v``, merged with this store's filter.""" assert isinstance(v, Mapping), ( f" v (value) must be a mapping (often a dictionary). Were:\n\tv={v}" ) return self.mgc.insert_one(self._build_doc(v)) def extend(self, values): + """Insert several docs ``values``, each merged with this store's filter.""" assert all([isinstance(v, Mapping) for v in values]), ( f" values must be mappings (often dictionaries)" ) @@ -471,6 +485,7 @@ def merge_doc_elements_with_filter(): return doc def persist_data(self, data): + """Write ``data`` (a doc with an ``_id``) under the key ``{ID: data[ID]}``.""" return self.__setitem__({ID: data[ID]}, data) @@ -486,6 +501,23 @@ def persist_data(self, data): class MongoClientReader(KvReader): + """A ``Mapping`` view of a mongo client. Keys are database names, values are + ``MongoDbReader`` instances for the corresponding database. + + Takes the same arguments as ``pymongo.MongoClient``. + + >>> from mongodol.base import MongoClientReader, MongoDbReader + >>> from mongodol.util import mk_dflt_mgc + >>> _ = mk_dflt_mgc().insert_one({'x': 1}) # ensure the default db/collection exist + >>> client_reader = MongoClientReader() + >>> 'mongodol' in client_reader + True + >>> db_reader = client_reader['mongodol'] + >>> isinstance(db_reader, MongoDbReader) + True + + """ + @wraps(MongoClient.__init__) def __init__(self, *mongo_client_args, **mongo_client_kwargs): self._mongo_client = MongoClient(*mongo_client_args, **mongo_client_kwargs) @@ -500,6 +532,26 @@ def __getitem__(self, k): class MongoDbReader(KvReader): + """Base Mongo Db Reader. Keys are collection names and values are collection store instances. + + :param db_name: Name of db + :param mk_collection_store: Function that is called on a key (collection name) to make the + collection store instance. + Use mk_collection_store to define what kind of collection stores you want to make. + Will be called with only one unnamed argument; the collection name. + Use custom classes here, and/or partials (curried functions) thereof, to fix any parameters you want to fix. + :param mongo_client: MongoClient instance, kwargs to make it (``MongoClient(**kwargs)``), or callable to make it + :param mongo_client_kwargs: ``**kwargs`` to make a MongoClient, that is used if mongo_client is callable + + >>> from mongodol.base import MongoDbReader + >>> from mongodol.util import mk_dflt_mgc + >>> _ = mk_dflt_mgc().insert_one({'x': 1}) # ensure the default db/collection exist + >>> db_reader = MongoDbReader() + >>> 'mongodol_test' in db_reader + True + + """ + def __init__( self, db_name=DFLT_TEST_DB, @@ -507,17 +559,6 @@ def __init__( mongo_client=None, **mongo_client_kwargs, ): - """Base Mongo Db Reader. Keys are collection names and values are collection store instances. - - :param db_name: Name of db - :param mk_collection_store: Function that is called on a key (collection name) to make the - collection store instance. - Use mk_collection_store to define what kind of collection stores you want to make. - Will be called with only one unnamed argument; the collection name. - Use custom classes here, and/or partials (curried functions) thereof, to fix any parameters you want to fix. - :param mongo_client: MongoClient instance, kwargs to make it (MongoClient(**kwargs)), or callable to make it - :param mongo_client_kwargs: **kwargs to make a MongoClient, that is used if mongo_client is callable - """ if mongo_client is None: self._mongo_client = MongoClient(**mongo_client_kwargs) elif isinstance(mongo_client, dict): @@ -547,23 +588,29 @@ class MongoBaseStore(Store): """ def contains_value(self, v): + """Forward ``contains_value`` to the wrapped store, transforming ``v`` first.""" return self.store.contains_value(self._data_of_obj(v)) def iter_values(self): + """Bulk-read all values, transforming each with ``_obj_of_data``.""" return map(self._obj_of_data, bulk_values(self.store)) def contains_item(self, item): + """Forward ``contains_item`` to the wrapped store, transforming key and value first.""" k, v = item return self.store.contains_item((self._id_of_key(k), self._data_of_obj(v))) def iter_items(self): + """Bulk-read all ``(key, value)`` pairs, transforming each with ``_key_of_id``/``_obj_of_data``.""" return ( (self._key_of_id(key), self._obj_of_data(doc)) for key, doc in bulk_items(self.store) ) def append(self, v): + """Forward ``append`` to the wrapped store, transforming ``v`` first.""" return self.store.append(self._data_of_obj(v)) def extend(self, values): + """Forward ``extend`` to the wrapped store, transforming each value first.""" return self.store.extend(list(map(self._data_of_obj, values))) diff --git a/mongodol/recipes.py b/mongodol/recipes.py index 24d5a90..23b0ba2 100644 --- a/mongodol/recipes.py +++ b/mongodol/recipes.py @@ -31,7 +31,7 @@ def disallow_sourced_interval_overlaps(store): r"""Disallow writing to a key that shares the same "source" field value and overlapping ("bt", "tt") interval. :param store: ``KvPersister`` (instance or class) ``s`` - :return: The same store, but where `s[dict(source=source, bt=bt, tt=tt}] = v`` writes are not permitted if + :return: The same store, but where ``s[dict(source=source, bt=bt, tt=tt)] = v`` writes are not permitted if there is another doc, with the same source, and an overlapping (bt, tt) interval. >>> from mongodol.tests import get_test_collection_persister, clear_all_and_populate diff --git a/mongodol/stores.py b/mongodol/stores.py index dd8c594..88ec1be 100644 --- a/mongodol/stores.py +++ b/mongodol/stores.py @@ -32,7 +32,8 @@ class MongoCollectionPersisterWithResultMapping(MongoCollectionPersister): @single_value_fetch_with_unicity_validation class MongoCollectionUniqueDocReader(MongoCollectionReader): """A mongo collection (kv-)reader where s[key] is the dict (a mongo doc matching the key). - :raises KeyNotUniqueError if the k matches more than a single unique doc. + + :raises KeyNotUniqueError: if the k matches more than a single unique doc. >>> from mongodol.stores import MongoCollectionUniqueDocReader >>> from mongodol.tests import data, util @@ -64,6 +65,22 @@ class MongoCollectionFirstDocReader(MongoCollectionReader): Typically, this should be used when you don't want the overhead of checking for uniqueness, because it doesn't matter, you like risk, or you told the mongo collection indexing system itself to ensure uniqueness for you. + + >>> from mongodol.stores import MongoCollectionFirstDocReader + >>> from mongodol.tests import data, util + >>> test_mgc = util.populated_pymongo_collection(data.three_simple_docs) + >>> s = MongoCollectionFirstDocReader(test_mgc, + ... iter_projection={'s': True, '_id': False}, getitem_projection=['n']) + >>> assert list(s) == [{'s': 'a'}, {'s': 'b'}, {'s': 'b'}] + + Unlike ``MongoCollectionUniqueDocReader``, a key matching more than one doc + doesn't raise; it just returns the first match found: + + >>> s[{'s': 'a'}] + {'_id': 0, 'n': 1} + >>> s[{'s': 'b'}] + {'_id': 1, 'n': 2} + """ @@ -77,6 +94,19 @@ class MongoCollectionFirstDocReader(MongoCollectionReader): class MongoCollectionMultipleDocsReader(MongoCollectionReader): """A mongo collection (kv-)reader where s[key] will return the list of all key-matching docs. If no docs match, will return an empty list. + + >>> from mongodol.stores import MongoCollectionMultipleDocsReader + >>> from mongodol.tests import data, util + >>> test_mgc = util.populated_pymongo_collection(data.three_simple_docs) + >>> s = MongoCollectionMultipleDocsReader(test_mgc, + ... iter_projection={'s': True, '_id': False}, getitem_projection=['n']) + >>> s[{'s': 'a'}] + [{'_id': 0, 'n': 1}] + >>> s[{'s': 'b'}] + [{'_id': 1, 'n': 2}, {'_id': 2, 'n': 3}] + >>> s[{'s': 'nonexistent'}] + [] + """ @@ -86,7 +116,27 @@ class MongoCollectionMultipleDocsReader(MongoCollectionReader): @single_value_fetch_with_unicity_validation class MongoCollectionUniqueDocPersister(MongoCollectionPersisterWithResultMapping): """A mongo collection (kv-)reader where s[key] is the dict (a mongo doc matching the key). - :raises KeyNotUniqueError if the k matches more than a single unique doc. + + :raises KeyNotUniqueError: if the k matches more than a single unique doc. + + >>> from mongodol.stores import MongoCollectionUniqueDocPersister + >>> from mongodol.tests import util + >>> test_mgc = util.populated_pymongo_collection([]) + >>> s = MongoCollectionUniqueDocPersister(test_mgc, + ... iter_projection={'s': True, '_id': False}, getitem_projection={'n': True, '_id': False}) + >>> s[{'s': 'a'}] = {'n': 1} + >>> list(s) + [{'s': 'a'}] + >>> s[{'s': 'a'}] + {'n': 1} + + >>> s[{'s': 'b'}] = {'n': 2} + >>> _ = s.mgc.insert_one({'s': 'b', 'n': 99}) + >>> s[{'s': 'b'}] + Traceback (most recent call last): + ... + mongodol.util.KeyNotUniqueError: Key was not unique (i.e. cursor has more than one match): {'s': 'b'} + """ @@ -98,6 +148,23 @@ class MongoCollectionFirstDocPersister(MongoCollectionPersisterWithResultMapping Typically, this should be used when you don't want the overhead of checking for uniqueness, because it doesn't matter, you like risk, or you told the mongo collection indexing system itself to ensure uniqueness for you. + + >>> from mongodol.stores import MongoCollectionFirstDocPersister + >>> from mongodol.tests import util + >>> test_mgc = util.populated_pymongo_collection([]) + >>> s = MongoCollectionFirstDocPersister(test_mgc, + ... iter_projection={'s': True, '_id': False}, getitem_projection={'n': True, '_id': False}) + >>> s[{'s': 'a'}] = {'n': 1} + >>> s[{'s': 'a'}] + {'n': 1} + + A second doc matching the same key does not raise; ``s[key]`` keeps returning + the first match found: + + >>> _ = s.mgc.insert_one({'s': 'a', 'n': 999}) + >>> s[{'s': 'a'}] + {'n': 1} + """ @@ -109,6 +176,23 @@ class MongoCollectionFirstDocPersister(MongoCollectionPersisterWithResultMapping class MongoCollectionMultipleDocsPersister(MongoCollectionPersisterWithResultMapping): """A mongo collection (kv-)reader where s[key] will return the list of all key-matching docs. If no docs match, will return an empty list. + + ``s[key] = v`` first deletes every doc matching ``key``, then inserts ``v`` + (a doc, or a collection of docs) merged with ``key``: + + >>> from mongodol.stores import MongoCollectionMultipleDocsPersister + >>> from mongodol.tests import util + >>> test_mgc = util.populated_pymongo_collection([]) + >>> s = MongoCollectionMultipleDocsPersister(test_mgc, + ... iter_projection={'s': True, '_id': False}, getitem_projection={'n': True, '_id': False}) + >>> s[{'s': 'a'}] = [{'n': 1}, {'n': 2}] + >>> s[{'s': 'a'}] + [{'n': 1}, {'n': 2}] + + >>> s[{'s': 'a'}] = {'n': 3} # replaces the two docs above with just this one + >>> s[{'s': 'a'}] + [{'n': 3}] + """ def __setitem__(self, k, v): @@ -128,6 +212,8 @@ def __setitem__(self, k, v): class MongoStore(Store): + """A ``Store`` wrapping a ``MongoCollectionUniqueDocPersister``, built from host/db/collection names.""" + @wraps(MongoCollectionUniqueDocPersister.__init__) def __init__( self, *args, host, db_name, collection_name, mongo_client_kwargs=None, **kwargs @@ -229,10 +315,9 @@ def __init__( @lru_cache def _get_db(db_name, host, **mongo_client_kwargs): - """ - Get a mongo database object from a db_name and host. + """Get a mongo database object from a db_name and host. - The `host` parameter can be a full `mongodb URI + The ``host`` parameter can be a full `mongodb URI `_, in addition to a simple hostname. It can also be a list of hostnames or URIs. diff --git a/mongodol/tests/util.py b/mongodol/tests/util.py index f758b76..37f505b 100644 --- a/mongodol/tests/util.py +++ b/mongodol/tests/util.py @@ -39,6 +39,7 @@ def get_test_collection_persister( *, iter_projection=(ID,), ): + """Make a ``MongoCollectionPersister`` on the (default) test collection, for use in tests and examples.""" mgc = get_test_collection_object(mongo_client_args, db_name, collection_name) return MongoCollectionPersister(mgc, iter_projection=iter_projection) diff --git a/mongodol/tracking_methods.py b/mongodol/tracking_methods.py index 323800a..36263f9 100644 --- a/mongodol/tracking_methods.py +++ b/mongodol/tracking_methods.py @@ -13,6 +13,8 @@ def track_calls_of_method(method: Callable, execute_call=True, tracks_factory=list): + """Wrap ``method`` so every call is appended to ``self._tracks``, and (if ``execute_call``) also run.""" + @wraps(method) def tracked_method(self, *args, **kwargs): try: @@ -29,6 +31,8 @@ def tracked_method(self, *args, **kwargs): def track_calls_without_executing(method: Callable): + """Wrap ``method`` so every call is appended to ``self._tracks``, but never actually run.""" + @wraps(method) def tracked_method(self, *args, **kwargs): self._tracks.append((method, args, kwargs)) @@ -37,6 +41,8 @@ def tracked_method(self, *args, **kwargs): def forward_method_calls(method): + """Wrap ``method`` so calls on ``self`` are forwarded to ``self._instance`` instead.""" + @wraps(method) def forwarded_method(self, *args, **kwargs): return method(self._instance, *args, **kwargs) @@ -77,11 +83,13 @@ def __exit__(self, exc_type, exc_value, traceback): # commit_execution def flush(self): + """Execute all pending tracked calls, clear the tracks, and return the call results.""" call_results = self._execute_tracks() self.clear_tracks() return call_results def clear_tracks(self): + """Discard all pending tracked calls without executing them.""" self._tracks.clear() @@ -161,6 +169,7 @@ def track_method_calls( >>> assert str(d._tracks) == "[(, ('a', 42), {})]" To execute the command in _tracks, you can use the ``.flush()`` method + >>> _ = d.flush() >>> # See that the setitem call was indeed made >>> assert d['a'] == 42 @@ -242,6 +251,7 @@ def _add_tracked_methods(cls): def consume(gen): + """Exhaust an iterable/generator ``gen`` for its side effects, discarding all values.""" for _ in gen: pass diff --git a/mongodol/trans.py b/mongodol/trans.py index c257973..cbd7738 100644 --- a/mongodol/trans.py +++ b/mongodol/trans.py @@ -35,20 +35,19 @@ class PersistentObjectBase(ABC): - """ - Base class to propagate a modification event through a parent-child chain structure. + """Base class to propagate a modification event through a parent-child chain structure. """ def __init__(self, container): self._container = container def persist_data(self, *args): + """Notify the container that this object's data has changed, by forwarding to its ``persist_data``.""" return self._container.persist_data(self) class PersistentDict(dict, PersistentObjectBase): - ''' - Extension of a dict wich triggers an event to notify the object that contains the dict that a modification + '''Extension of a dict wich triggers an event to notify the object that contains the dict that a modification has been made. Requirement: The container object needs to implement the method "persist_data(self, data: Mapping)". @@ -116,6 +115,7 @@ def __setitem__(self, k, v): return self.persist_data() def update(self, *args, **kwargs): + """Update like a normal dict, then persist the updated dict.""" super().update(*args, **kwargs) return self.persist_data() @@ -125,8 +125,7 @@ def __delitem__(self, v): class PersistentList(list, PersistentObjectBase): - ''' - Extension of a list wich triggers an event to notify the object that contains the list that a modification + '''Extension of a list wich triggers an event to notify the object that contains the list that a modification has been made. Requirement: The container object needs to implement the method "persist_data(self, data: Mapping)". @@ -168,10 +167,12 @@ def __init__(self, container, iterable: Iterable): list.__init__(self, persistent_iterable) def append(self, __object): + """Append ``__object``, then persist the updated list.""" super().append(__object) return self.persist_data() def extend(self, __iterable): + """Extend with ``__iterable``, then persist the updated list.""" super().extend(__iterable) return self.persist_data() @@ -185,11 +186,13 @@ def __setitem__(self, i, o): return self.persist_data() def pop(self, __index): + """Pop the item at ``__index``, then persist the updated list.""" r = super().pop(__index) self.persist_data() return r def remove(self, __value): + """Remove the first occurrence of ``__value``, then persist the updated list.""" super().remove(__value) return self.persist_data() @@ -198,10 +201,12 @@ def __delitem__(self, i): return self.persist_data() def deepcopy(self): + """A deep copy of the original iterable this list was built from (not the persistent list itself).""" return deepcopy(self._initial_iterable) def get_persistent_obj(container, v): + """Wrap ``v`` in a ``PersistentDict``/``PersistentList`` if it's a mapping/iterable, else return it as is.""" if isinstance(v, Mapping): return PersistentDict(container, v) elif isinstance(v, Iterable) and not isinstance(v, str): @@ -212,8 +217,11 @@ def get_persistent_obj(container, v): # TODO: Make trans funcs/method carry their role and find their place in wrap_kvs automatically class PostGet: + """``postget`` (key-aware) transform functions for ``wrap_kvs``, turning a cursor into a value.""" + @staticmethod def single_value_fetch_with_unicity_validation(store, k, cursor): + """Return the single doc in ``cursor``; raise if there's none or more than one.""" doc = next(cursor, None) if doc is not None: if ( @@ -227,6 +235,7 @@ def single_value_fetch_with_unicity_validation(store, k, cursor): @staticmethod def single_value_fetch_without_unicity_validation(store, k, cursor): + """Return the first doc in ``cursor``; raise only if there's none (no uniqueness check).""" doc = next(cursor, None) if doc is not None: # return PersistentDict(store, doc) @@ -246,13 +255,21 @@ def all_docs_fetch(k, cursor, doc_collector=list): class ObjOfData: + """``obj_of_data`` (value-only) transform functions for ``wrap_kvs``.""" + @staticmethod def all_docs_fetch(cursor, doc_collector=list): + """Collect every doc in ``cursor`` into ``doc_collector`` (default: a list). + + The value-only (``obj_of_data``) counterpart of :meth:`PostGet.all_docs_fetch`. + """ # return doc_collector(map(lambda x: PersistentDict(x), cursor)) return doc_collector(cursor) class WriteOpResult(TypedDict): + """The shape of a normalized mongo write-operation result (see ``normalize_result``).""" + ok: bool n: int ids: Iterable[str] | None @@ -274,7 +291,6 @@ def normalize_result(obj, *, method_names_to_normalize=DFLT_METHOD_NAMES_TO_NORM :param func: [description] :type func: [type] """ - if not isinstance(obj, type): assert callable(obj), f"Should be callable: {obj}" func = obj @@ -331,12 +347,16 @@ def result_mapper(*args, **kwargs): def _vector_to_dict(vector: Iterable, fields: Iterable[str]): - """Note: meant to be used with functools.partial(_vector_to_dict, fields=fields)""" + """Zip ``fields`` and ``vector`` into a dict; meant to be used with + ``functools.partial(_vector_to_dict, fields=fields)``. + """ return {k: v for k, v in zip(fields, vector)} def _string_to_dict(value, field: str): - """Note: meant to be used with functools.partial(_string_to_dict, field=field)""" + """Wrap ``value`` as ``{field: value}``; meant to be used with + ``functools.partial(_string_to_dict, field=field)``. + """ return {field: value} @@ -361,11 +381,11 @@ def set_key_and_data_fields( This is to make it easier to get from an interface like this - :code: `store[{'folder': 'path', 'file': 'name'}] = {'field1': 'value1', 'field2': 'value2'}` + ``store[{'folder': 'path', 'file': 'name'}] = {'field1': 'value1', 'field2': 'value2'}`` to an interface like this: - :code: `store['path', 'name'] = ('value1', 'value2')` + ``store['path', 'name'] = ('value1', 'value2')`` """ id_of_key, key_of_id, obj_of_data, data_of_obj = None, None, None, None diff --git a/mongodol/util.py b/mongodol/util.py index 5dec6a5..c52a2e1 100644 --- a/mongodol/util.py +++ b/mongodol/util.py @@ -20,10 +20,20 @@ def mk_dflt_client(): + """Make a ``pymongo.MongoClient`` with the default client args.""" return MongoClient(*DFLT_MONGO_CLIENT_ARGS) def mk_dflt_mgc(): + """Make a default ``pymongo.collection.Collection``, connecting with default + client args to the default test database and collection. + + >>> from mongodol.util import mk_dflt_mgc + >>> c = mk_dflt_mgc() + >>> c.name, c.database.name + ('mongodol_test', 'mongodol') + + """ return MongoClient(*DFLT_MONGO_CLIENT_ARGS)[DFLT_TEST_DB][DFLT_TEST_COLLECTION] @@ -32,6 +42,7 @@ class KeyNotUniqueError(RuntimeError): @staticmethod def raise_error(k): + """Raise ``KeyNotUniqueError`` for the non-unique key ``k``.""" raise KeyNotUniqueError( f"Key was not unique (i.e. cursor has more than one match): {k}" ) @@ -42,6 +53,9 @@ def raise_error(k): def get_key_value_specs(key_fields, data_fields): + """Derive normalized ``(key_fields, data_fields, key_projection, items_projection)`` from + the given key/data field specs, for building fixed-fields readers/persisters. + """ if isinstance(key_fields, str): key_fields = (key_fields,) if data_fields is None: @@ -65,8 +79,7 @@ def get_key_value_specs(key_fields, data_fields): def flatten_dict_items(d: Mapping, prefix=""): - """ - Computes a "flat" dict from a nested one. A flat dict's keys are the dot-paths of the input dict. + """Computes a "flat" dict from a nested one. A flat dict's keys are the dot-paths of the input dict. :param d: a nested dict :param prefix: A string to prepend on all the paths @@ -155,7 +168,7 @@ def projection_union( projection_2: ProjectionDict, already_flattened=False, ): - """ + """Flatten and merge two mongo projection dicts, OR-ing shared fields. >>> d = {'a': { ... 'a': True, @@ -179,12 +192,27 @@ def projection_union( def get_mongo_collection_pymongo_obj(obj=None, client_factory=mk_dflt_client): """Get a pymongo.collection.Collection object for a mongo collection, flexibly. - ``` - get_mongo_collection_pymongo_obj() # gives you a default mongo collection ({DFLT_TEST_DB}/test) - get_mongo_collection_pymongo_obj('database_name/collection_name') # does the obvious (with default host) - get_mongo_collection_pymongo_obj(... an object that has an _mgc attribute...) # return the _mgc attribute - get_mongo_collection_pymongo_obj(obj) # else, asserts pymongo.collection.Collection and returns it - ``` + .. code-block:: text + + get_mongo_collection_pymongo_obj() # gives you a default mongo collection (mongodol/mongodol_test) + get_mongo_collection_pymongo_obj('database_name/collection_name') # does the obvious (with default host) + get_mongo_collection_pymongo_obj(... an object that has an _mgc attribute...) # return the _mgc attribute + get_mongo_collection_pymongo_obj(obj) # else, asserts pymongo.collection.Collection and returns it + + >>> from mongodol.util import get_mongo_collection_pymongo_obj + >>> c = get_mongo_collection_pymongo_obj() + >>> c.name, c.database.name + ('mongodol_test', 'mongodol') + + An object with an ``_mgc`` attribute (such as a mongodol store) has that + attribute returned directly: + + >>> from mongodol.tests import util + >>> mgc = util.populated_pymongo_collection([]) + >>> store = type('Store', (), {'_mgc': mgc})() + >>> get_mongo_collection_pymongo_obj(store) is mgc + True + """ if obj is None: obj = mk_dflt_mgc() diff --git a/mongodol/utils/werk_local.py b/mongodol/utils/werk_local.py index d185ece..32fa975 100644 --- a/mongodol/utils/werk_local.py +++ b/mongodol/utils/werk_local.py @@ -1,5 +1,4 @@ -""" -Vendored from werkzeug's local.py module, edited to our needs. +"""Vendored from werkzeug's local.py module, edited to our needs. That single need is have a LocalProxy to subclass in making TrackedObj (see tracking_methods.py). """ diff --git a/mongodol/views.py b/mongodol/views.py index ed223a5..f9a9e13 100644 --- a/mongodol/views.py +++ b/mongodol/views.py @@ -36,9 +36,9 @@ ``values()``/``items()`` agree with ``__getitem__``. The knobs, for store authors: - Implement the bulk-read methods to *provide* the fast path. -- Set the :data:`BULK_READ_IS_FAITHFUL_ATTR` class attribute to ``False`` (see - :func:`disable_bulk_read`) when a class inherits bulk-read methods that no - longer agree with its own ``__getitem__``. +- Set the :data:`BULK_READ_IS_FAITHFUL_ATTR` class attribute to ``False`` + (see :func:`disable_bulk_read`) when a class inherits bulk-read methods + that no longer agree with its own ``__getitem__``. Known limitation. :class:`~mongodol.base.MongoCollectionReader` is deliberately a *cursor*-level store: ``s[k]`` is a pymongo ``Cursor``, while its bulk stream @@ -316,7 +316,8 @@ def disable_bulk_read(store_cls: type) -> type: class MongoValuesView(BaseValuesView): """A ``values()`` view that uses the backend's bulk read when -- and only when -- - that stream provably equals ``(store[k] for k in store)``.""" + that stream provably equals ``(store[k] for k in store)``. + """ def __iter__(self): try: @@ -333,7 +334,8 @@ def __contains__(self, v): class MongoItemsView(BaseItemsView): """An ``items()`` view that uses the backend's bulk read when -- and only when -- - that stream provably equals ``((k, store[k]) for k in store)``.""" + that stream provably equals ``((k, store[k]) for k in store)``. + """ def __iter__(self): try: From c184ae8b8b32b431ac43cdb2c44c9957cd6538ae Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Tue, 15 Sep 2026 15:24:48 +0200 Subject: [PATCH 2/3] docs: rewrite README for the current API, add agentic README section The README's quick-start example used py2store and a MongoCollectionReaderBase class that no longer exist. Replaced with a minimal runnable example against the current mongodol/dol API, verified by execution. Added the epythet- generated "For AI agents" section (llms.txt, mongodol.md, objects.inv). --- README.md | 326 ++++++++---------------------------------------------- 1 file changed, 48 insertions(+), 278 deletions(-) diff --git a/README.md b/README.md index c63127b..4935542 100644 --- a/README.md +++ b/README.md @@ -1,311 +1,81 @@ - # mongodol -MongoDB Data Object Layer. - -Tools to create data abstractions over mongoDB data. - -To install: ```pip install mongodol``` - -And of course, you need to [install MongoDB](https://www.mongodb.com/docs/manual/installation/) - - -# The base objects - - -```python -from mongodol import ( - MongoClientReader, - MongoDbReader, - MongoCollectionReaderBase, - MongoCollectionReader, - MongoCollectionPersister, -) -``` - -`MongoClientReader` gives you access to the databases for a mongoDB host (default is localhost). -The keys are database names... - - -```python -client = MongoClientReader() -list(client) -``` - - - - - ['admin', 'config', 'local', 'py2store', 'py2store_tests', 'yf'] - - - -... and the values are db objects. -The keys of db objects are collection names... - - -```python -db = client["py2store"] -list(db) -``` - - - - - ['tmp', 'test', 'annots_example'] - - - -... and the values are collection objects. - - -```python -mgc = db["test"] -len(mgc) -``` - - - - - 0 - - - -The collection is empty. Let's get a collection object that we can actually write with. - -Here, we show how you can write by appending data: - - -```python -writable_mgc = MongoCollectionPersister(mgc) -writable_mgc.append({"mongo": "uses", "json": "data"}) -``` - - - - - - - - -See that we have data in the collection now: - - -```python -keys = list(mgc) -keys -``` - - - - - [{'_id': ObjectId('60359a2993b7670664918663')}] - - - -But that's just showing the key, let's see the value under that key: - - -```python -k = keys[0] -mgc[k] -``` - - - - - - - - -Oh... you get a cursor back. It's okay, a cursor is the object that will provide you with the data you requested if and when you want it. - -Let's say you want it now. Just "consume" the cursor. If you're expecting just one item under that key, do this: - - -```python -v = next( - mgc[k], None -) # the None is there as a sentinel -- it will be used to indicate if mgc[k] has no data for you. -v -``` - - +Access MongoDB through a `Mapping` (dict-like) interface. - {'mongo': 'uses', 'json': 'data'} +`mongodol` wraps `pymongo` collections as `Mapping`/`MutableMapping` objects (readers and +persisters), so you can read and write mongo data with normal `dict`-like syntax, and +compose your own key/value transforms with [`dol`](https://github.com/i2mint/dol) wrappers +instead of writing backend-specific boilerplate. +To install: - -So indeed it worked. - -You can also use extend to write in bulk. - - -```python -writable_mgc.extend( - [ - {"kind": "example", "data": 2}, - {"kind": "example", "data": [1, 2, 3]}, - {"kind": "example", "data": {"nested": "dict"}}, - ] -) ``` - - - - - - - - - -```python -list(mgc) +pip install mongodol ``` +And of course, you need a running MongoDB -- see the +[installation instructions](https://www.mongodb.com/docs/manual/installation/). + +## For AI agents +`mongodol` publishes its documentation in forms made for coding agents. If you are one, start here. - [{'_id': ObjectId('60359a2993b7670664918663')}, - {'_id': ObjectId('60359ac193b7670664918664')}, - {'_id': ObjectId('60359ac193b7670664918665')}, - {'_id': ObjectId('60359ac193b7670664918666')}] - +**The documentation, machine-readable**: [`llms.txt`](https://i2mint.github.io/mongodol/llms.txt) indexes every page; [`mongodol.md`](https://i2mint.github.io/mongodol/mongodol.md) is the whole documentation in one file; every page has a `.md` twin; [`objects.inv`](https://i2mint.github.io/mongodol/objects.inv) maps symbols to URLs. +If you are a control freak, the rest of this README is written for you, starting at [Quick start](#quick-start). + +## Quick start ```python +from mongodol import MongoCollectionPersister, mk_dflt_mgc -``` - -So far, MongoDB gave us an id. MongoDB will make it's own id if we don't ask for a particular one. - -But you can also write data to a key of your choice. With the base persister which we're demoing now, with it's base defaults, you need to specify your key as a `{'_id': YOUR_CHOICE_OF_ID}`. - - -```python -writable_mgc[{"_id": "my_id"}] = {"my": "data"} -list(mgc) -``` - - - - - [{'_id': ObjectId('60359a2993b7670664918663')}, - {'_id': ObjectId('60359ac193b7670664918664')}, - {'_id': ObjectId('60359ac193b7670664918665')}, - {'_id': ObjectId('60359ac193b7670664918666')}, - {'_id': 'my_id'}] - - - - -```python -mgc[{"_id": "my_id"}] -``` - - - - - {'my': 'data'} - - - -You can delete data given a key: - - -```python -del writable_mgc[{"_id": "my_id"}] -``` +# mk_dflt_mgc() gives you a pymongo collection to play with (mongodol/mongodol_test by default) +mgc = mk_dflt_mgc() +s = MongoCollectionPersister(mgc, getitem_projection={'_id': False}) +len(s) +# 0 -```python -list(mgc) -``` - - - - - [{'_id': ObjectId('60359a2993b7670664918663')}, - {'_id': ObjectId('60359ac193b7670664918664')}, - {'_id': ObjectId('60359ac193b7670664918665')}, - {'_id': ObjectId('60359ac193b7670664918666')}] - - - -So far, we've seen the base classes. - -So far, you have no reason what-so-ever to use `mongodol`. Might as well use `pymongo` (which it wraps) directly. - -The real reason for using `mongodol` is that it is a gateway to enabling all the `py2store` goodies to create the key-value perspectives that make sense to **you**, without all the backend-dependent boilerplate over the business logic. - -So let's show one example of how to do that. - - -# The real reason you want to use mongodol (an example) - -Let's say we have the collection we just made above, but -- We want to access data by doing `s['60359a2993b7670664918663']` instead of the (annoying) `s[{'_id': ObjectId('60359a2993b7670664918663')}]` -- We'd like our values to to come in the form of actual ready to use data. Namely, we want to automatically ask the cursor for it's first element (assuming it's unique for that key), and we'd like to extract the 'data' field from that result. -- We'd like to peruse only part of the mongo collection; only if there's a 'kind' field and it's equal to 'example'. - -Here's how it can be done: - - -```python -from bson import ObjectId -from py2store import wrap_kvs -from mongodol import MongoCollectionReaderBase - - -@wrap_kvs( - id_of_key=lambda x: {"_id": ObjectId(x)}, - key_of_id=lambda x: str(x["_id"]), - obj_of_data=lambda doc: next(doc, None)["data"], -) -class MyStore(MongoCollectionReaderBase): - """my special store""" -``` - - -```python -s = MyStore( - mgc=mgc, key_fields=("_id",), data_fields=("data",), filt={"kind": "example"} -) -``` - - -```python +k = {'_id': 'my_id'} +s[k] = {'mongo': 'uses', 'json': 'data'} list(s) +# [{'_id': 'my_id'}] ``` - - - - ['60359ac193b7670664918664', - '60359ac193b7670664918665', - '60359ac193b7670664918666'] - - - +Since the base reader is a thin, low-level wrapper, `s[k]` returns a `pymongo.cursor.Cursor` +(a key may match zero, one, or many docs), so you fetch the value(s) explicitly: ```python -s["60359ac193b7670664918664"] -``` - - - - - 2 +next(s[k]) +# {'mongo': 'uses', 'json': 'data'} +del s[k] +len(s) +# 0 +``` +## Beyond the base classes +The base `MongoCollectionReader`/`MongoCollectionPersister` classes always return cursors +and never validate uniqueness. For the common case of "one key maps to one doc", use one +of the `*UniqueDoc*`/`*FirstDoc*` reader and persister classes instead: ```python -list(s.values()) +from mongodol import MongoCollectionUniqueDocReader ``` +`MongoCollectionUniqueDocReader` gives you `s[k]` as a plain `dict` (not a cursor), and +raises `KeyNotUniqueError` if more than one doc matches `k`. See its docstring for a +runnable example. +For custom key/value shapes, business logic, or connecting `mongodol` stores to the rest +of the [`dol`](https://github.com/i2mint/dol) ecosystem (caching, serialization, +key transforms, etc.), wrap a `mongodol` store with `dol.wrap_kvs` like you would any +other `dol` store. +## More - [2, [1, 2, 3], {'nested': 'dict'}] - +See the [package documentation](https://i2mint.github.io/mongodol/) and the flat +[`mongodol.md`](https://i2mint.github.io/mongodol/mongodol.md) aggregate for the full API. From c55d86c9d29ccd5f7e34f09e19ccb08bee816c81 Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Tue, 15 Sep 2026 15:29:20 +0200 Subject: [PATCH 3/3] docs: fix claims flagged by adversarial review - get_key_value_specs: don't claim a purpose with zero verified callers, don't claim a return shape the dict-data_fields branch can't produce (it raises UnboundLocalError; noted, not silently "fixed" since that's a behaviour change out of scope for a docs sweep). - projection_union: the summary said "OR-ing shared fields", but every field is OR-ed against a forced True default, so a field present in only one dict (or False in one) still comes out True. Verified with real calls. - MongoCollectionPersister.append/extend: said "merged with this store's filter", but _build_doc actually uses on_write_filter when set. Verified by reading _build_doc. - README quick start: len(s) == 0 only holds on a fresh collection; added an explicit delete_many({}) so the example is reproducible regardless of what's already in the default test collection. --- README.md | 1 + mongodol/base.py | 4 ++-- mongodol/util.py | 11 ++++++++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4935542..4d22ef4 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ from mongodol import MongoCollectionPersister, mk_dflt_mgc # mk_dflt_mgc() gives you a pymongo collection to play with (mongodol/mongodol_test by default) mgc = mk_dflt_mgc() +mgc.delete_many({}) # start from an empty collection (skip this to keep what's already there) s = MongoCollectionPersister(mgc, getitem_projection={'_id': False}) len(s) diff --git a/mongodol/base.py b/mongodol/base.py index 8fca1b0..6aace6b 100644 --- a/mongodol/base.py +++ b/mongodol/base.py @@ -442,14 +442,14 @@ def __delitem__(self, k): raise KeyError(f"You can't remove that key: {k}") def append(self, v): - """Insert a single doc ``v``, merged with this store's filter.""" + """Insert a single doc ``v``, merged with ``on_write_filter`` if set, else this store's filter.""" assert isinstance(v, Mapping), ( f" v (value) must be a mapping (often a dictionary). Were:\n\tv={v}" ) return self.mgc.insert_one(self._build_doc(v)) def extend(self, values): - """Insert several docs ``values``, each merged with this store's filter.""" + """Insert several docs ``values``, each merged with ``on_write_filter`` if set, else this store's filter.""" assert all([isinstance(v, Mapping) for v in values]), ( f" values must be mappings (often dictionaries)" ) diff --git a/mongodol/util.py b/mongodol/util.py index c52a2e1..ebf96fe 100644 --- a/mongodol/util.py +++ b/mongodol/util.py @@ -53,8 +53,11 @@ def raise_error(k): def get_key_value_specs(key_fields, data_fields): - """Derive normalized ``(key_fields, data_fields, key_projection, items_projection)`` from - the given key/data field specs, for building fixed-fields readers/persisters. + """Derive ``key_projection`` (and, when ``data_fields`` is None or a non-dict iterable, + ``items_projection``) from ``key_fields``/``data_fields``. + + Note: when ``data_fields`` is already a dict, ``items_projection`` is never assigned, + so this branch raises ``UnboundLocalError`` on the ``return`` below. """ if isinstance(key_fields, str): key_fields = (key_fields,) @@ -168,7 +171,9 @@ def projection_union( projection_2: ProjectionDict, already_flattened=False, ): - """Flatten and merge two mongo projection dicts, OR-ing shared fields. + """Flatten and merge two mongo projection dicts, OR-ing every field against a forced + default of ``True`` -- so a field appearing in only one of the two dicts (or with a + ``False`` value) still comes out ``True`` unless both dicts agree it's ``False``. >>> d = {'a': { ... 'a': True,