Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions mempalace/backends/pgvector.py
Original file line number Diff line number Diff line change
Expand Up @@ -1058,6 +1058,21 @@ def count(self) -> int:
return 0
return self._client.count_rows(self._table)

def count_by_metadata_field(self, field: str) -> dict[str, int]:
"""Return ``{value: count}`` for a top-level metadata key, aggregated in
SQL. Lets callers (e.g. the status overview) tally by ``wing`` / ``room``
without streaming every drawer's metadata to the client — O(1) memory and
a single query instead of O(n). ``NULL`` values bucket under ``"unknown"``."""
self._ensure_open()
if not self._table_exists():
return {}
sql = (
f"SELECT metadata->>%s AS k, count(*) AS c "
f"FROM {_quote_identifier(self._table)} GROUP BY 1"
)
rows = self._client._execute(sql, [field], fetch=True)
return {(r[0] if r[0] is not None else "unknown"): int(r[1]) for r in rows}
Comment on lines +1069 to +1074

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

There is a correctness bug here when a collection contains both explicit "unknown" values and missing/null values for the metadata field. Because both map to the key "unknown" in the dict comprehension, one will overwrite the other, resulting in an incorrect count. Pushing the COALESCE aggregation to SQL solves this cleanly and correctly.

Suggested change
sql = (
f"SELECT metadata->>%s AS k, count(*) AS c "
f"FROM {_quote_identifier(self._table)} GROUP BY 1"
)
rows = self._client._execute(sql, [field], fetch=True)
return {(r[0] if r[0] is not None else "unknown"): int(r[1]) for r in rows}
sql = (
f"SELECT COALESCE(metadata->>%s, 'unknown') AS k, count(*) AS c "
f"FROM {_quote_identifier(self._table)} GROUP BY 1"
)
rows = self._client._execute(sql, [field], fetch=True)
return {r[0]: int(r[1]) for r in rows}


def lexical_search(self, *, query: str, n_results: int = 10, where: Optional[dict] = None):
_validate_where(where)
pushdown = None if _requires_local_filter(where) else where
Expand Down
24 changes: 17 additions & 7 deletions mempalace/mcp_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -1073,13 +1073,23 @@ def tool_status():
"backend": _selected_backend_name(),
}
try:
all_meta = _get_cached_metadata(col)
for m in all_meta:
m = m or {}
w = m.get("wing", "unknown")
r = m.get("room", "unknown")
wings[w] = wings.get(w, 0) + 1
rooms[r] = rooms.get(r, 0) + 1
grouper = getattr(col, "count_by_metadata_field", None)
if callable(grouper):
# Efficient path: backends that can aggregate server-side (e.g.
# pgvector via SQL GROUP BY) tally wing/room counts without
# streaming every drawer's metadata to the client. On a large
# palace the client-side fetch below is O(n) memory and can take
# minutes (or OOM); this is a single query.
wings.update(grouper("wing"))
rooms.update(grouper("room"))
else:
all_meta = _get_cached_metadata(col)
for m in all_meta:
m = m or {}
w = m.get("wing", "unknown")
r = m.get("room", "unknown")
wings[w] = wings.get(w, 0) + 1
rooms[r] = rooms.get(r, 0) + 1
except Exception as e:
logger.exception("tool_status metadata fetch failed")
result["error"] = str(e)
Expand Down