fix(backends): single-scroll bulk metadata fetch for Qdrant; bump scroll page size (#1796)

mempalace/backends/base.py

@@ -468,6 +468,39 @@ def effective_embedder_identity(self) -> Optional[EmbedderIdentity]:
         """
         return None
 
+    def get_all_metadata(self, where: Optional[dict] = None) -> list[dict]:
+        """Return every matching record's metadata in one logical pass (#1796).
+
+        Default implementation pages through :meth:`get` using
+        ``limit``/``offset`` -- correct for backends with a real server-side
+        cursor (e.g. Chroma's SQL OFFSET), and the same shape callers already
+        relied on before this method existed.
+
+        Backends whose ``get(limit=, offset=)`` is implemented by fully
+        materializing a result set and then Python-slicing it (no true
+        server-side cursor) MUST override this method to walk their native
+        cursor exactly once instead. Calling the default implementation on
+        such a backend is O(n^2) in collection size: each page re-walks the
+        entire collection just to discard everything outside the requested
+        slice. See issue #1796.
+        """
+        all_meta: list[dict] = []
+        offset = 0
+        page_size = 1000
+        while True:
+            kwargs: dict = {"include": ["metadatas"], "limit": page_size, "offset": offset}
+            if where:
+                kwargs["where"] = where
+            batch = self.get(**kwargs)
+            batch_meta = batch.metadatas if hasattr(batch, "metadatas") else batch.get("metadatas")
+            if not batch_meta:
+                break
+            all_meta.extend(batch_meta)
+            if len(batch_meta) < page_size:
+                break
+            offset += len(batch_meta)
+        return all_meta
+
     def maintenance_state(self) -> dict:
         """Return a structured snapshot of this collection's maintenance state.
 

mempalace/backends/qdrant.py

@@ -53,6 +53,20 @@
 _PAYLOAD_METADATA = "metadata"
 _POINT_NAMESPACE = uuid.UUID("c06c3fc7-5c14-4dc4-84c2-24a5f72d8dc1")
 _TOKEN_RE = re.compile(r"\w{2,}", re.UNICODE)
+# Page size for Qdrant's /points/scroll cursor. 4096 (up from the original
+# 256) cuts REST round-trips ~16x for any full-collection walk (#1796).
+# Qdrant's own docs suggest larger scroll batches are safe, and this is well
+# below typical REST payload-size limits for metadata-only (with_vector=False)
+# scrolls such as get_all_metadata().
+#
+# This constant also governs vector-bearing scrolls (with_vector=True), used
+# by _rows()/get() when embeddings are requested and by _query_local_exact()
+# for the $or/$contains local-filter query fallback. At 4096 rows per page,
+# high-dimensional embeddings make those particular responses tens of MB --
+# Qdrant handles it and round-trips still drop overall, but this is a real
+# trade-off, not a metadata-only optimization. (Noted in maintainer review
+# on #1832.)
+_SCROLL_PAGE_SIZE = 4096
 _SUPPORTED_OPERATORS = frozenset(
     {"$eq", "$ne", "$in", "$nin", "$and", "$or", "$contains", "$gt", "$gte", "$lt", "$lte"}
 )
@@ -480,7 +494,7 @@ def scroll_points(
         collection: str,
         *,
         qdrant_filter: Optional[dict] = None,
-        limit: int = 256,
+        limit: int = _SCROLL_PAGE_SIZE,
         offset: Any = None,
         with_vector: bool = False,
     ) -> tuple[list[dict], Any]:
@@ -732,7 +746,7 @@ def _scroll_all(
             points, offset = self._client.scroll_points(
                 self._remote_collection,
                 qdrant_filter=qdrant_filter,
-                limit=256,
+                limit=_SCROLL_PAGE_SIZE,
                 offset=offset,
                 with_vector=with_vector,
             )
@@ -1000,6 +1014,27 @@ def get(
             embeddings=[row["embedding"] or [] for row in rows] if spec.embeddings else None,
         )
 
+    def get_all_metadata(self, where: Optional[dict] = None) -> list[dict]:
+        """Return every matching record's metadata in one cursor pass (#1796).
+
+        Overrides the default offset-paginated implementation, which would
+        call self.get(limit=, offset=) in a loop -- and since self.get() is
+        backed by a full _scroll_all() materialization, each page of that
+        loop would re-walk the entire collection from the start just to
+        discard everything outside its slice (O(n^2) over collection size).
+
+        Delegates to self._rows(), the same single-scroll-plus-local-filter
+        helper that backs get()/delete(). With ids=None and
+        where_document=None, _rows() reduces to exactly one _scroll_all()
+        pass followed by an unconditional _matches_where() re-check on every
+        row -- the same filter logic get(), delete(), and lexical_search()
+        already use, so this can't independently drift from those call
+        sites. (Maintainer review on #1832: avoid duplicating the filter
+        dance inline.)
+        """
+        rows = self._rows(where=where)
+        return [row["metadata"] for row in rows]
+
     def delete(self, *, ids=None, where=None):
         _validate_where(where)
         if not self._remote_exists():

mempalace/mcp_server.py

@@ -851,7 +851,22 @@ def _safe_meta(meta):
 
 
 def _fetch_all_metadata(col, where=None):
-    """Paginate col.get() to avoid the 10K silent truncation limit."""
+    """Fetch every matching record's metadata via the backend's best strategy.
+
+    Delegates to BaseCollection.get_all_metadata() (#1796), which Chroma
+    satisfies with the same offset-paginated loop this function used to do
+    inline, and which Qdrant overrides with a single _scroll_all() pass.
+    Routing through one contract method means every backend gets its own
+    correct strategy without this caller needing to know which backend it's
+    talking to.
+    """
+    get_all = getattr(col, "get_all_metadata", None)
+    if callable(get_all):
+        return get_all(where=where)
+
+    # Defensive fallback for any collection object that predates the
+    # get_all_metadata() contract method (e.g. a third-party backend not yet
+    # updated). Preserves the exact previous behavior.
     total = col.count()
     all_meta = []
     offset = 0