diff --git a/.claude-plugin/hooks/hooks.json b/.claude-plugin/hooks/hooks.json index c777287147..787f48376d 100644 --- a/.claude-plugin/hooks/hooks.json +++ b/.claude-plugin/hooks/hooks.json @@ -34,6 +34,17 @@ ] } ], + "SessionEnd": [ + { + "hooks": [ + { + "type": "command", + "command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks/mempal-session-end-hook.sh\"", + "timeout": 10 + } + ] + } + ], "PreCompact": [ { "hooks": [ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a38b261a1b..73854e5437 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.13"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} @@ -45,7 +45,7 @@ jobs: if: github.event_name == 'workflow_dispatch' runs-on: windows-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - uses: actions/setup-python@v6 with: python-version: "3.13" @@ -63,7 +63,7 @@ jobs: if: github.event_name == 'workflow_dispatch' runs-on: macos-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - uses: actions/setup-python@v6 with: python-version: "3.13" @@ -73,7 +73,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - uses: actions/setup-python@v6 with: python-version: "3.11" diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 9010baffe2..a0272543c8 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -23,7 +23,7 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 4ca318b20a..0f1432281a 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -22,14 +22,14 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 # Needed for the emulated linux/arm64 build on real pushes. - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 # Only authenticate + push for in-repo events. Fork PRs lack the # packages:write token, so they build (to validate the Dockerfile) but @@ -79,10 +79,10 @@ jobs: build-gpu: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Build GPU image (validation only — not published) uses: docker/build-push-action@v7 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7688ebe6d6..83cb5947a7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -73,7 +73,7 @@ jobs: echo "tag=$tag" >> "$GITHUB_OUTPUT" echo "Resolved tag: $tag" - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: # Fully-qualified refs/tags/ so an unqualified name can't resolve to a # same-named *branch* instead of the tag (checkout prefers branches). diff --git a/.github/workflows/version-guard.yml b/.github/workflows/version-guard.yml index 36155cb9da..98b5482952 100644 --- a/.github/workflows/version-guard.yml +++ b/.github/workflows/version-guard.yml @@ -16,7 +16,7 @@ jobs: check-versions: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Extract versions from all sources id: versions diff --git a/CHANGELOG.md b/CHANGELOG.md index bcd983f5c4..18b9d95f2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,68 +80,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), --- -## [Unreleased] — 2026-05-17 — *AGE-integration 6-phase plan: KnowledgeGraphAGE + write-through + palace structure + backfill + walk_palace MCP* - -Multi-project AGE-integration plan landed on this fork's `feat/age-kg-parity` branch ([PR #101](https://github.com/techempower-org/mempalace/pull/101)). Companion fork-PR on palace-daemon adds the read-side fusion endpoint ([techempower-org/palace-daemon#25](https://github.com/techempower-org/palace-daemon/pull/25)). The metaphor of *the AI walking into a palace and finding wings, rooms, and drawers* now corresponds to real Cypher traversal over a unified `Wing → Room → Drawer → MENTIONS → Entity` graph. - -### Added - -- **`KnowledgeGraphAGE` API parity with the SQLite `KnowledgeGraph`** ([`ff7187d`](https://github.com/techempower-org/mempalace/commit/ff7187d)). Adds 6 missing methods (`add_entity`, `invalidate`, `query_entity`, `query_relationship`, `timeline`, `seed_from_entity_facts`) on top of the existing `add_triple` / `query_triples` / `stats` / `clear`. AGE 1.6.0 Cypher dialect gaps documented + worked around: no `ON CREATE SET`, no multi-column `RETURN` inside `cypher()`, no list literals, no `SET` on edge properties inline. -- **Write-through KG middleware on `PostgresCollection`** ([`3321d83`](https://github.com/techempower-org/mempalace/commit/3321d83)). New `set_kg_writethrough(hook)` API; hook fires after every successful drawer write with `(drawer_id, document, metadata)`. New module `mempalace/kg_writethrough.py` provides `make_age_writethrough(kg, extractor)` factory + env-var-driven `make_writethrough_from_env()` + a builtin regex extractor fallback. Default behavior unchanged when no hook is registered. -- **Palace structure as native AGE nodes** ([`ff583c0`](https://github.com/techempower-org/mempalace/commit/ff583c0)). New module `mempalace/palace_graph_age.py` mirrors the SQL-aggregation pattern from `mempalace.palace_graph` into AGE: `populate_from_postgres(kg, dsn, table_name)` builds `Wing -[CONTAINS]-> Room -[CONTAINS]-> Drawer` + `Wing -[SHARED_VIA {via_room}]- Wing` tunnels. Idempotent via MERGE. -- **`backfill_age` — restartable AGE population from drawer table** ([`b3f0206`](https://github.com/techempower-org/mempalace/commit/b3f0206)). New module + CLI `mempalace-backfill-age` for one-shot postgres-drawers → postgres-AGE migration. Checkpoint table `mempalace_kg_backfill_state` makes re-runs safe. Smoke-tested on 5344-drawer test palace (58948 entities in 26 min); ~22h projected for production 274K palace. -- **`add_mention(drawer_id, entity_name)` on `KnowledgeGraphAGE`** (same `b3f0206`). Connects palace-structure to entity layer via `(Drawer)-[:MENTIONS]->(Entity)` edges. CREATE-ALWAYS semantics matches SQLite KG triples-table behavior; idempotency tracked externally via backfill checkpoint table. -- **`mempalace_walk_palace` MCP tool** ([`8022ecb`](https://github.com/techempower-org/mempalace/commit/8022ecb)). Agent-facing walk primitive: `start_wing="..."` walks down, `start_room="..."` enumerates across wings, `start_entity="..."` is inverse walk. Requires `MEMPALACE_BACKEND=postgres` + AGE populated. -- **`.gemini/config.yaml`** ([`c35c74e`](https://github.com/techempower-org/mempalace/commit/c35c74e)). Gemini Code Assist tuning — MEDIUM severity threshold, ignore patterns for benchmark JSONs and generated `FORK_CHANGELOG.md`. - -### Spike result that motivated the plan - -[AGE write-through bench on n=200 git-derived probes](https://github.com/techempower-org/multipass-structural-memory-eval/blob/feat/rlm-adapter/docs/benchmarks/2026-05-17-age-write-through-spike.md): - -| Mode | R@5 | Δ vs vector | -|---|---:|---:| -| vector_only (pgvector + MiniLM base) | 0.1850 | — | -| graph_only (AGE entity-overlap) | 0.2350 | **+5.0pp** | -| fusion (RRF combine) | 0.2750 | **+9.0pp** | - -### Cross-fork verification - -Independent same-day audit by [@nakata-app](https://github.com/nakata-app) on his AdaptMem fork found the same operational conclusion via a different angle (code-level audit found `KnowledgeGraphAGE` skeleton-only; state-level audit on the production palace-daemon found 2 placeholder vertices + 1 placeholder edge total). Cross-fork verification framing discussed at [MemPalace/mempalace/discussions/1384#discussioncomment-16951344](https://github.com/MemPalace/mempalace/discussions/1384#discussioncomment-16951344). - -### Fixed - -- **upstream/develop sync** ([`6058489`](https://github.com/techempower-org/mempalace/commit/6058489)). Merged 60 develop commits into fork main (last sync 2026-05-13 via PR #1487). 12 conflict files resolved across `cli.py`, `convo_miner.py`, `hooks_cli.py`, `mcp_server.py`, `miner.py`, `palace.py`, `palace_graph.py`, `searcher.py`, and three test files. Notable upstream changes brought in: Igor's #1519 `convo_miner` `min_chunk_size` validation, KG cache path canonicalization via `_canonicalize_kg_path` + `realpath` + `normcase` (4 commits), cold-start embedder diagnostics + opt-in warmup (#1495), tunnel hyphenated wing slug preservation (#1504), stratified palace state messages (#1498), bash 3.2 compat for hooks (#1440), entity-registry tmp cleanup on failure (#1408), and none-metadata MCP handler fix (#1445). Follow-up commit [`342a59f`](https://github.com/techempower-org/mempalace/commit/342a59f) drops shadow `chunk_*` properties on `MempalaceConfig`, scales hook timeout bounds in `tests/test_claude_plugin_hook_config.py` to milliseconds, and updates fork-side test assertions to match upstream's new `wing_` API shape (#1410). - ---- - -## [Unreleased] — 2026-05-14 / 2026-05-15 — *postgres cutover, hybrid retrieval, encoder-axis evidence* - -Fork-side work that landed after the v3.3.5 release. Nothing upstreamed yet; some of it is operator-flavored and lives only on the fork. - -### Added - -- **Postgres + pgvector + Apache AGE backend is now the production default** on this fork. Cutover from chromadb-on-disk happened over 2026-05-13 / 14; `main` now serves live traffic against a 273K-drawer palace on `disks:5433` (PG16 + pgvector 0.8.2 + AGE 1.6.0). Composes upstream [#665](https://github.com/MemPalace/mempalace/pull/665) (skuznetsov's `PostgresBackend` on the RFC 001 contract) plus the pgvector lazy-index race fix below. The chromadb backend still works behind `MEMPALACE_BACKEND=chroma`. Operator narrative: [`docs/operators/pgvector-cutover-runbook.md`](docs/operators/pgvector-cutover-runbook.md). -- **Hybrid retrieval** as `candidate_strategy="hybrid"` — vector candidates ∪ BM25 candidates (postgres `tsvector` + `pg_trgm` GIN for ILIKE identifier fallback) ∪ AGE graph-expanded candidates, hybrid-reranked. Exposed via [palace-daemon](https://github.com/techempower-org/palace-daemon)'s new `/search/keyword` and `/search/hybrid` HTTP endpoints. Search@5000 p50 125ms; recall@5 0.60 synthetic; filtered recall=1.00 on wing-scoped queries. -- **`symbol_header_prefix` keyword-only kwarg on `mempalace.miner.chunk_text`** for representation-axis experiments (AST-lite, encoder-domain adaptation with explicit symbol disambiguation). Backward-compatible — default `None` preserves existing behavior. Companion to discussion [#1384](https://github.com/MemPalace/mempalace/discussions/1384). -- **`scripts/derive_probes_from_git.py`** — deterministically derives n=200 retrieval probes from this repo's git log, filtering noise-prefix commits and picking a primary changed file per commit. Replaces the hand-curated n=20 set in [`scripts/chunk_strategy_ablation.py`](scripts/chunk_strategy_ablation.py) whose paired-bootstrap 95% CIs all overlapped zero. JSON snapshot at [`scripts/probes_v2_git_derived.json`](scripts/probes_v2_git_derived.json). `--probes ` flag added to the ablation harness so the larger set is wired in. -- **`scripts/verify_rrf_ftcode5k.py` + `scripts/verify_rrf_3way.py`** — local RRF reproduction against [adaptmem](https://github.com/nakata-app/adaptmem) FT-Code SentenceTransformer checkpoints (300 / 1000 / 5000). 3-way RRF on the n=200 probe set: default ONNX 0.4260 / FT-Code-1000 0.4229 / FT-Code-5000 0.3972 / **3-way fused 0.5101** (+0.0841 MRR vs best solo). Reproduces nakata-app's #1384 §4 inversion at 10× sample size — the encoder with worst solo MRR contributes the largest 2-way fusion lift. -- **`docs/benchmarks/2026-05-15-remaining-benches.json`** — captures 5 of 9 mempalace `tests/benchmarks/` suites that hadn't been re-run after the postgres cutover (`test_ingest_bench`, `test_knowledge_graph_bench`, `test_layers_bench`, `test_mcp_bench`, `test_memory_profile`). 64/64 passed in 1h 24m at `--bench-scale=small` against the production postgres palace. Companion to [`docs/benchmarks/2026-05-14-search-bench-hybrid-cutover.json`](docs/benchmarks/2026-05-14-search-bench-hybrid-cutover.json). - -### Fixed - -- **pgvector lazy-index race wedges the database** under concurrent writes. `PostgresBackend._maybe_create_vector_index` had a SELECT-then-`CREATE INDEX` race with a name-coupled existence check — three concurrent writers crossing the threshold held `ACCESS EXCLUSIVE` for 30+ minutes. Fixed at commit `4566f8a` with `pg_advisory_xact_lock(hashtext('vec_idx:'))` + `CREATE INDEX IF NOT EXISTS`. Operator follow-up at upstream [#665](https://github.com/MemPalace/mempalace/pull/665) plus recovery procedure documented in the runbook. -- **`docs/embedding.py`** doc comment warning custom-EF authors about the chromadb 1.5+ `EmbeddingFunction.embed_query` requirement. A bare class with `__call__` + `name()` passes `Collection.upsert` but raises `AttributeError` on `Collection.query(query_texts=...)` — mempalace's searcher catches that and silently falls back to BM25, making encoder swaps invisible. Subclassing `chromadb.api.types.EmbeddingFunction` inherits a default `embed_query` that delegates to `__call__`. We hit this debugging the RRF verifier; the comment is for downstream users implementing alternative encoders. - -### Cherry-picks from upstream PRs (in-flight, used early) - -- **[#1490](https://github.com/MemPalace/mempalace/pull/1490)** (open, @nakata-app) — `fix(benchmarks): honor --granularity across hybrid_v2/v3/v4; reject in palace/diary`. Three commits (`7ba6522`, `a8303ec`, `7766343`) cherry-picked. `build_palace_and_retrieve_hybrid_v{2,3,4}` accepted a `granularity` parameter but never branched on it, so `--granularity turn` and `--granularity session` produced bitwise-identical metrics. Per our fork-first convention; will deduplicate at upstream-merge time. - -### Documentation - -- Discussion [#1384](https://github.com/MemPalace/mempalace/discussions/1384) — chunking-strategy ablation × encoder thread with @nakata-app. Posted operator reproduction of their #1384 §4 RRF result on our n=200 probe set + flagged the EF.embed_query protocol gotcha. -- PR [#665](https://github.com/MemPalace/mempalace/pull/665) — third operator follow-up with cutover state, bench results, and the EF protocol note for anyone implementing alternative `BaseBackend` encoders. -## [Unreleased] — 2026-05-22 — *Upstream tunnels fixes (cherry-picked from MemPalace/develop)* - ## [3.3.6] — 2026-05-24 ### Features diff --git a/FORK_CHANGELOG.md b/FORK_CHANGELOG.md index ff27ee7d11..b92c13843a 100644 --- a/FORK_CHANGELOG.md +++ b/FORK_CHANGELOG.md @@ -18,6 +18,38 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). --- +## [2026-07-02] + + +### Changed + + +- **Sync upstream/develop through da5a48c (post-v3.5.0): remote MCP server w/ TLS + read-only, graph auto-population, Qdrant facets, list_drawers date filters, 213 commits** ([`TBD`](https://github.com/techempower-org/mempalace/commit/TBD)) + Merged 213 upstream commits (post-v3.5.0 ``da5a48c``). Notable + upstream additions: the turnkey secure remote MCP server with TLS and + a read-only server mode (#1877 / #1900), associative-graph + auto-population from mined sessions + ``cmd_hallways`` (#1895), + Qdrant server-side metadata facets (#1868), ``since``/``before`` + date filters on ``list_drawers`` (#1128 / #1891), authored-timestamp + preservation from transcripts (#1890), ``mine_palace_lock`` + re-entrancy for the HTTP transport (#1859), a pgvector metadata-only + fetch fix (#1892), SQLite magic-header ``detect()`` (#1893 / #1896), + FTS5 auto-heal (#1878), a host-root-logger fix (#1860 / #1885), + LaTeX extensions, and dependency bumps (ruff 0.15.20). + + ~50 conflicted files resolved by composing rather than choosing + sides: ``tool_list_drawers`` carries BOTH the upstream date filters + and the fork tag filters; ``tool_status`` keeps the fork's postgres + fast path (#267) and gains the upstream facets sweep; the HTTP + transport keeps host pinning and gains TLS + read-only; the merged + plugin hook config stays the fork's five-event ms-timeout shape. The + merged MCP tool surface stays at 39 tools (upstream's 34 plus fork + tools); all doc and manifest tool-count claims reconciled against the + live ``mcp_server.TOOLS`` count. + + *Files:* `mempalace/mcp_server.py`, `mempalace/searcher.py`, `mempalace/cli.py`, `mempalace/convo_miner.py`, `mempalace/embedding.py`, `mempalace/backends/base.py`, `mempalace/backends/pgvector.py`, `tests/conftest.py`, `tests/test_mcp_server.py`, `tests/test_backends.py` + + ## [2026-07-01] diff --git a/README.md b/README.md index fe1bb5b4b4..af8be74f4d 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ ## What this is -A verbatim-first local AI memory system. This fork tracks `upstream/develop` through the v3.5.0 sync (2026-06-26, commit `73e74bf`) and runs in production on a **409K+ drawer Postgres + pgvector + Apache AGE palace** behind [palace-daemon](https://github.com/techempower-org/palace-daemon). It carries fork-ahead commits that compose with — not replace — bensig's release direction; the v3.3.5 release (2026-05-10) includes our co-authored `_get_collection` retry-once via upstream #1377. 4830 tests pass on `main`. +A verbatim-first local AI memory system. This fork tracks `upstream/develop` through the post-v3.5.0 sync (2026-07-02, commit `da5a48c`) and runs in production on a **411K+ drawer Postgres + pgvector + Apache AGE palace** behind [palace-daemon](https://github.com/techempower-org/palace-daemon). It carries fork-ahead commits that compose with — not replace — bensig's release direction; the v3.3.5 release (2026-05-10) includes our co-authored `_get_collection` retry-once via upstream #1377. 4921 tests pass on `main`. The fork's architectural thinking — the four-layer memory model, the [verbatim-vs-derivative thesis](docs/research/verbatim-vs-derivative-axis.md), design principles, and the two-memory-layer pairing with Auto Dream — lives in [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md). The new things here are *what we've learned*, not just what we've fixed. @@ -237,99 +237,100 @@ The full enumeration of fork-ahead changes. The canonical source is [`docs/fork- | # | Description | Upstream PR | Fork commit | |---|---|---|---| -| 1 | Auto-query firing fixes: frozen turn counter, dead wing scoring, lowercase entities, turn-1 cadence | — | [`fad3e27`](https://github.com/techempower-org/mempalace/commit/fad3e27) | -| 2 | Auto-query: periodic depth signal, unknown-entity 0->1 bump, broader temporal patterns | — | [`864d7a4`](https://github.com/techempower-org/mempalace/commit/864d7a4) | -| 3 | Sync upstream/develop through v3.5.0 (73e74bf): MCP HTTP transport, source_file filter, checkpoint tool, SessionEnd hook, 185 commits | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 4 | Restore concurrent file mining via parallel-prepare/serial-write (regression from a dropped sync hunk); opt-in --workers | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 5 | Sync upstream/develop through v3.4.0 (2ec4bae): RFC-001 backend stack, diary checkpoints restored, 113 commits | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 6 | auto_wake: opt-in wake-on-demand for a sleeping palace-daemon host (wake command + /health poll + single retry) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 7 | AGE graph-walk: auto edge-endpoint indexes in backfill + bind anonymous RELATION targets (mempalace#335) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 8 | pluggable adaptmem_ft encoder backend selectable via MEMPALACE_EMBEDDING_MODEL (closes #308) | — | [`5fba6d8`](https://github.com/techempower-org/mempalace/commit/5fba6d8) | -| 9 | README.md landscape table — refresh upstream MemPalace star count from ~23K → ~53K (current 2026-05-28) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 10 | README.md + docs/ECOSYSTEM.md — soften 'engram-2 17% E2E QA' framing per the 2026-05-24 research doc's unsubstantiated finding (#319) | — | [`ddf00b4`](https://github.com/techempower-org/mempalace/commit/ddf00b4) | -| 11 | kg_llm_extractor rewrites AGE dollar-quote tag in triples so drawers indexing palace source code don't fail at add_triple (#313) | — | [`3fb9428`](https://github.com/techempower-org/mempalace/commit/3fb9428) | -| 12 | scripts/maintain-fork-changes.py + ship-prep step 1: resolve commit:HEAD placeholders and de-dup yaml entries (#316) | — | [`9060e09`](https://github.com/techempower-org/mempalace/commit/9060e09) | -| 13 | scripts/ship-prep.sh — one command bumps README test count and runs all three doc renderers (#312) | — | [`4677db8`](https://github.com/techempower-org/mempalace/commit/4677db8) | -| 14 | mempalace_search MCP input schema accepts fusion_mode (convex\|rrf) and forwards to search_memories (#302) | — | [`f753ec4`](https://github.com/techempower-org/mempalace/commit/f753ec4) | -| 15 | scripts/check-docs.sh finds pytest via main checkout when run from a worktree, fails hard instead of silently skipping test-count check (#311) | — | [`1d19a8b`](https://github.com/techempower-org/mempalace/commit/1d19a8b) | -| 16 | kg_triple_worker retries add_triple within-worker on transient psycopg errors instead of abandoning to lease-reclaim (#298) | — | [`36c0b02`](https://github.com/techempower-org/mempalace/commit/36c0b02) | -| 17 | mempalace_kg_stats returns structured backend-unavailable envelope on transient psycopg failures (#299) | — | [`8fd0b01`](https://github.com/techempower-org/mempalace/commit/8fd0b01) | -| 18 | mempalace why + tunnels — explain a drawer + inventory cross-wing tunnels (slice of #191) | — | [`fdcd0b4`](https://github.com/techempower-org/mempalace/commit/fdcd0b4) | -| 19 | RRF vs convex-blend rerank — A/B measurement on our corpus (#162) | — | [`ea5d567`](https://github.com/techempower-org/mempalace/commit/ea5d567) | -| 20 | KG triples gain SPOC context slot + worker auto-derives valid_from from drawer metadata (#161) | — | [`b87ce05`](https://github.com/techempower-org/mempalace/commit/b87ce05) | -| 21 | mempalace bulk-move — multi-drawer metadata relocation by source wing/room (#191) | — | [`1ca544b`](https://github.com/techempower-org/mempalace/commit/1ca544b) | -| 22 | mempalace move — fast direct-to-daemon single-drawer wing/room relocation (#191) | — | [`d007b6f`](https://github.com/techempower-org/mempalace/commit/d007b6f) | -| 23 | mempalace stats migrates to GET /stats REST + exposes graph/status sections (#191) | — | [`853bb25`](https://github.com/techempower-org/mempalace/commit/853bb25) | -| 24 | mempalace cypher — read-only Cypher query CLI (#191) | — | [`32a41b1`](https://github.com/techempower-org/mempalace/commit/32a41b1) | -| 25 | mempalace graph — fast direct-to-daemon KG structural snapshot (#191) | — | [`499f42d`](https://github.com/techempower-org/mempalace/commit/499f42d) | -| 26 | mempalace list — fast direct-to-daemon drawer browser (#191) | — | [`257137b`](https://github.com/techempower-org/mempalace/commit/257137b) | -| 27 | Recency decay weighting in search + mempalace prune --stale-days CLI (#158) | — | [`558d327`](https://github.com/techempower-org/mempalace/commit/558d327) | -| 28 | mempalace_rate_memory MCP tool + bounded rating signal in search ranking (#159) | — | [`583536c`](https://github.com/techempower-org/mempalace/commit/583536c) | -| 29 | Formalize wing/room derivation order; demote entity detector to last-resort hint (#157) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 30 | RRF fusion mode + convex-vs-RRF A/B harness (#162) | [#247](https://github.com/MemPalace/mempalace/pull/247) | [`6c9d10c`](https://github.com/techempower-org/mempalace/commit/6c9d10c) | -| 31 | mempalace stats: add ROOMS breakdown (drawer count by room) to the dashboard | — | [`1673465`](https://github.com/techempower-org/mempalace/commit/1673465) | -| 32 | Calibrated confidence field on search results + Brier-score eval column | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 33 | Evaluation doc: curated-authority vs auto-mined separation (#202) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 34 | Apply AGE statement_timeout in same transaction as cypher() (PR #228 follow-up) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 35 | LLM-based KG triple extraction: queue table, async worker, llama.cpp on familiar | — | [`59ac0bc`](https://github.com/techempower-org/mempalace/commit/59ac0bc) | -| 36 | Promote verbatim-vs-derivative essay from research/ to README (#170) | — | [`6a264d9`](https://github.com/techempower-org/mempalace/commit/6a264d9) | -| 37 | mempalace stats — palace analytics dashboard (#191) | — | [`6f994fb`](https://github.com/techempower-org/mempalace/commit/6f994fb) | -| 38 | CLI wiring: mempalace mine --source (#57) | — | [`5ed9fa7`](https://github.com/techempower-org/mempalace/commit/5ed9fa7) | -| 39 | Warp terminal source adapter (#62) | — | [`2e85585`](https://github.com/techempower-org/mempalace/commit/2e85585) | -| 40 | OpenCode adapter smoke test against real DB (#56) | — | [`a9ed72b`](https://github.com/techempower-org/mempalace/commit/a9ed72b) | -| 41 | Codex, Gemini, and Aider source adapters (#61, #59) | — | [`0c23165`](https://github.com/techempower-org/mempalace/commit/0c23165) | -| 42 | Filesystem + conversation source adapters (#63) | — | [`9a1facf`](https://github.com/techempower-org/mempalace/commit/9a1facf) | -| 43 | Widen auto-query signal patterns for natural recall phrases | — | [`33e780e`](https://github.com/techempower-org/mempalace/commit/33e780e) | -| 44 | Native rename_wing backend operation + CLI command (#154) | — | [`d045f83`](https://github.com/techempower-org/mempalace/commit/d045f83) | -| 45 | Standalone essay: the verbatim-vs-derivative axis (#47) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 46 | Research doc: uncertainty-aware retrieval analysis (#84) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 47 | Design doc: scope/collection filter on mempalace_search (#76) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 48 | Agent-shaped CLI surface — --json / --quiet for non-MCP integration | — | [`25ed900`](https://github.com/techempower-org/mempalace/commit/25ed900) | -| 49 | Design eval: multi-palace separation — curated vs auto-mined (#45) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 50 | Document .sh shim delegation to palace-daemon (counter-position to upstream #1069) | — | [`bf0a4d0`](https://github.com/techempower-org/mempalace/commit/bf0a4d0) | -| 51 | Honor ~/.mempalace/RETIRED marker — refuse default palace, surface retire message | — | [`798cf14`](https://github.com/techempower-org/mempalace/commit/798cf14) | -| 52 | Empty repo .opencode/opencode.json mcp block — disabled flag wasn't being respected | — | [`7133eee`](https://github.com/techempower-org/mempalace/commit/7133eee) | -| 53 | Drop \$comment from .opencode/opencode.json — schema rejects unknown root keys | — | [`637bb01`](https://github.com/techempower-org/mempalace/commit/637bb01) | -| 54 | Disable repo-level MCP entry by default + venv-python fallback | — | [`47018e5`](https://github.com/techempower-org/mempalace/commit/47018e5) | -| 55 | Stub resources/list + prompts/list so MCP clients stop ERROR-logging on connect | — | [`6ca0670`](https://github.com/techempower-org/mempalace/commit/6ca0670) | -| 56 | Bundled OpenCode live-capture plugin that bypasses option-K v1.2.1 bugs (filed upstream as #4, #5) | — | [`5522623`](https://github.com/techempower-org/mempalace/commit/5522623) | -| 57 | Documented OpenCode integration recipe (read-side MCP + push plugin + retrospective adapter) | — | [`60dc9e6`](https://github.com/techempower-org/mempalace/commit/60dc9e6) | -| 58 | .opencode/opencode.json — repo-root MCP config so opencode picks up mempalace automatically | [#1567](https://github.com/MemPalace/mempalace/pull/1567) (OPEN) | [`ba16b82`](https://github.com/techempower-org/mempalace/commit/ba16b82) | -| 59 | OpenCodeSourceAdapter (RFC 002) — retrospective ingest of OpenCode SQLite sessions | [#1484](https://github.com/MemPalace/mempalace/pull/1484) (OPEN) | [`2ffe652`](https://github.com/techempower-org/mempalace/commit/2ffe652) | -| 60 | mempalace_walk_palace MCP tool — agent walks the palace via AGE Cypher | — | [`8022ecb`](https://github.com/techempower-org/mempalace/commit/8022ecb) | -| 61 | Backfill AGE graph from existing drawer table — restartable, checkpointed | — | [`b3f0206`](https://github.com/techempower-org/mempalace/commit/b3f0206) | -| 62 | Wing/Room/Drawer hierarchy as native AGE nodes; Cypher MATCH walks palace structure | — | [`ff583c0`](https://github.com/techempower-org/mempalace/commit/ff583c0) | -| 63 | Write-through middleware on PostgresCollection — entities populate AGE on every drawer write | — | [`3321d83`](https://github.com/techempower-org/mempalace/commit/3321d83) | -| 64 | KnowledgeGraphAGE API parity with SQLite KG: add_entity, invalidate, query_entity, query_relationship, timeline, seed_from_entity_facts | — | [`ff7187d`](https://github.com/techempower-org/mempalace/commit/ff7187d) | -| 65 | Pending-writes journal + replay so daemon outages stop being silent | — | [`0c34464`](https://github.com/techempower-org/mempalace/commit/0c34464) | -| 66 | MCP server distinguishes 'backend unreachable' from 'no palace found' | — | [`0c34464`](https://github.com/techempower-org/mempalace/commit/0c34464) | -| 67 | Defense-in-depth metadata sanitizer at the chromadb-client chokepoint | — | [`f499814`](https://github.com/techempower-org/mempalace/commit/f499814) | -| 68 | Route Stop/PreCompact hooks through palace-daemon/clients/hook.py | — | [`42ded2e`](https://github.com/techempower-org/mempalace/commit/42ded2e) | -| 69 | KnowledgeGraphAGE skeleton — Apache AGE graph bootstrap over psycopg2 | — | [`a3ee623`](https://github.com/techempower-org/mempalace/commit/a3ee623) | -| 70 | README pivots to the four-layer model + Auto Dream as vindication of the verbatim-vs-derivative axis | — | [`55b36ca`](https://github.com/techempower-org/mempalace/commit/55b36ca) | -| 71 | CI: gate postgres-backend tests against a pgvector service container | — | [`da0bdbb`](https://github.com/techempower-org/mempalace/commit/da0bdbb) | -| 72 | PostgreSQL backend via #665 cherry-pick + fork-side adaptations + smoke tests | [#665](https://github.com/MemPalace/mempalace/pull/665) (OPEN) | [`5e90c72`](https://github.com/techempower-org/mempalace/commit/5e90c72) | -| 73 | daemon-route `mempalace status` / `search` / `mine` when PALACE_DAEMON_URL is set | — | [`22ef562`](https://github.com/techempower-org/mempalace/commit/22ef562) | -| 74 | daemon-route `mcp_server.py` via the `handle_request` JSON-RPC chokepoint | — | [`41359ba`](https://github.com/techempower-org/mempalace/commit/41359ba) | -| 75 | Preserve dashed project names in transcript-derived wings | [#10](https://github.com/MemPalace/mempalace/pull/10) | [`d76134d`](https://github.com/techempower-org/mempalace/commit/d76134d) | -| 76 | Drop wing_ prefix from transcript-derived wings to converge with operator mines | [#9](https://github.com/MemPalace/mempalace/pull/9) | [`86d4700`](https://github.com/techempower-org/mempalace/commit/86d4700) | -| 77 | Retire mempalace_session_recovery collection + read tool | [#8](https://github.com/MemPalace/mempalace/pull/8) | [`0b945e1`](https://github.com/techempower-org/mempalace/commit/0b945e1) | -| 78 | mempalace mined + purge --source-file (mining management surface) | [#7](https://github.com/MemPalace/mempalace/pull/7) | [`2e6ced9`](https://github.com/techempower-org/mempalace/commit/2e6ced9) | -| 79 | Drop hook-side checkpoint diary writes — verbatim-only architecture | [#6](https://github.com/MemPalace/mempalace/pull/6) | [`69768fc`](https://github.com/techempower-org/mempalace/commit/69768fc) | -| 80 | Restore transcript ingest via daemon /mine when PALACE_DAEMON_URL is set | [#2](https://github.com/MemPalace/mempalace/pull/2) | [`09d2ca6`](https://github.com/techempower-org/mempalace/commit/09d2ca6) | -| 81 | `hook_verbatim_mode` config flag preserves system tags + full tool I/O during transcript ingest | — | [`ef98961`](https://github.com/techempower-org/mempalace/commit/ef98961) | -| 82 | Retire the `kind=` filter — structural split made it inert | — | [`7ba28dc`](https://github.com/techempower-org/mempalace/commit/7ba28dc) | -| 83 | Hoist CLOSET_RANK_BOOSTS to module level + record VecRecall ablation finding | — | [`3cb03f3`](https://github.com/techempower-org/mempalace/commit/3cb03f3) | -| 84 | Strip embedded API key from .claude-plugin/ manifests; rely on env inheritance | — | [`9f91e18`](https://github.com/techempower-org/mempalace/commit/9f91e18) | -| 85 | Cherry-pick #1094 — coerce None metadatas at chromadb boundary | [#1094](https://github.com/MemPalace/mempalace/pull/1094) (OPEN) | [`43d728d`](https://github.com/techempower-org/mempalace/commit/43d728d) | -| 86 | Cherry-pick #1087 rewrite — collection.delete(where=) instead of nuke-and-rebuild | [#1087](https://github.com/MemPalace/mempalace/pull/1087) (OPEN) | [`366a9ad`](https://github.com/techempower-org/mempalace/commit/366a9ad) | -| 87 | Canonical YAML manifest + renderer for fork-ahead docs | — | [`5a01aec`](https://github.com/techempower-org/mempalace/commit/5a01aec) | -| 88 | Phase D migration + PreCompact recovery write | — | [`42817d7`](https://github.com/techempower-org/mempalace/commit/42817d7) | -| 89 | Surface drawer_id in search/diary/recovery payloads | — | [`9a8bb77`](https://github.com/techempower-org/mempalace/commit/9a8bb77) | -| 90 | Cherry-pick #1085 — batch ChromaDB inserts in miner (10–30× faster) | [#1085](https://github.com/MemPalace/mempalace/pull/1085) (CLOSED) | [`6be6fff`](https://github.com/techempower-org/mempalace/commit/6be6fff) | -| 91 | scripts/deploy.sh — one-command Syncthing-aware redeploy | — | [`8252025`](https://github.com/techempower-org/mempalace/commit/8252025) | -| 92 | Phases A–C of the checkpoint collection split | — | [`e266365`](https://github.com/techempower-org/mempalace/commit/e266365) | -| 93 | kind= filter on search_memories excludes Stop-hook checkpoints (transitional) | — | [`f9f5cc4`](https://github.com/techempower-org/mempalace/commit/f9f5cc4) | +| 1 | Sync upstream/develop through da5a48c (post-v3.5.0): remote MCP server w/ TLS + read-only, graph auto-population, Qdrant facets, list_drawers date filters, 213 commits | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 2 | Auto-query firing fixes: frozen turn counter, dead wing scoring, lowercase entities, turn-1 cadence | — | [`fad3e27`](https://github.com/techempower-org/mempalace/commit/fad3e27) | +| 3 | Auto-query: periodic depth signal, unknown-entity 0->1 bump, broader temporal patterns | — | [`864d7a4`](https://github.com/techempower-org/mempalace/commit/864d7a4) | +| 4 | Sync upstream/develop through v3.5.0 (73e74bf): MCP HTTP transport, source_file filter, checkpoint tool, SessionEnd hook, 185 commits | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 5 | Restore concurrent file mining via parallel-prepare/serial-write (regression from a dropped sync hunk); opt-in --workers | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 6 | Sync upstream/develop through v3.4.0 (2ec4bae): RFC-001 backend stack, diary checkpoints restored, 113 commits | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 7 | auto_wake: opt-in wake-on-demand for a sleeping palace-daemon host (wake command + /health poll + single retry) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 8 | AGE graph-walk: auto edge-endpoint indexes in backfill + bind anonymous RELATION targets (mempalace#335) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 9 | pluggable adaptmem_ft encoder backend selectable via MEMPALACE_EMBEDDING_MODEL (closes #308) | — | [`5fba6d8`](https://github.com/techempower-org/mempalace/commit/5fba6d8) | +| 10 | README.md landscape table — refresh upstream MemPalace star count from ~23K → ~53K (current 2026-05-28) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 11 | README.md + docs/ECOSYSTEM.md — soften 'engram-2 17% E2E QA' framing per the 2026-05-24 research doc's unsubstantiated finding (#319) | — | [`ddf00b4`](https://github.com/techempower-org/mempalace/commit/ddf00b4) | +| 12 | kg_llm_extractor rewrites AGE dollar-quote tag in triples so drawers indexing palace source code don't fail at add_triple (#313) | — | [`3fb9428`](https://github.com/techempower-org/mempalace/commit/3fb9428) | +| 13 | scripts/maintain-fork-changes.py + ship-prep step 1: resolve commit:HEAD placeholders and de-dup yaml entries (#316) | — | [`9060e09`](https://github.com/techempower-org/mempalace/commit/9060e09) | +| 14 | scripts/ship-prep.sh — one command bumps README test count and runs all three doc renderers (#312) | — | [`4677db8`](https://github.com/techempower-org/mempalace/commit/4677db8) | +| 15 | mempalace_search MCP input schema accepts fusion_mode (convex\|rrf) and forwards to search_memories (#302) | — | [`f753ec4`](https://github.com/techempower-org/mempalace/commit/f753ec4) | +| 16 | scripts/check-docs.sh finds pytest via main checkout when run from a worktree, fails hard instead of silently skipping test-count check (#311) | — | [`1d19a8b`](https://github.com/techempower-org/mempalace/commit/1d19a8b) | +| 17 | kg_triple_worker retries add_triple within-worker on transient psycopg errors instead of abandoning to lease-reclaim (#298) | — | [`36c0b02`](https://github.com/techempower-org/mempalace/commit/36c0b02) | +| 18 | mempalace_kg_stats returns structured backend-unavailable envelope on transient psycopg failures (#299) | — | [`8fd0b01`](https://github.com/techempower-org/mempalace/commit/8fd0b01) | +| 19 | mempalace why + tunnels — explain a drawer + inventory cross-wing tunnels (slice of #191) | — | [`fdcd0b4`](https://github.com/techempower-org/mempalace/commit/fdcd0b4) | +| 20 | RRF vs convex-blend rerank — A/B measurement on our corpus (#162) | — | [`ea5d567`](https://github.com/techempower-org/mempalace/commit/ea5d567) | +| 21 | KG triples gain SPOC context slot + worker auto-derives valid_from from drawer metadata (#161) | — | [`b87ce05`](https://github.com/techempower-org/mempalace/commit/b87ce05) | +| 22 | mempalace bulk-move — multi-drawer metadata relocation by source wing/room (#191) | — | [`1ca544b`](https://github.com/techempower-org/mempalace/commit/1ca544b) | +| 23 | mempalace move — fast direct-to-daemon single-drawer wing/room relocation (#191) | — | [`d007b6f`](https://github.com/techempower-org/mempalace/commit/d007b6f) | +| 24 | mempalace stats migrates to GET /stats REST + exposes graph/status sections (#191) | — | [`853bb25`](https://github.com/techempower-org/mempalace/commit/853bb25) | +| 25 | mempalace cypher — read-only Cypher query CLI (#191) | — | [`32a41b1`](https://github.com/techempower-org/mempalace/commit/32a41b1) | +| 26 | mempalace graph — fast direct-to-daemon KG structural snapshot (#191) | — | [`499f42d`](https://github.com/techempower-org/mempalace/commit/499f42d) | +| 27 | mempalace list — fast direct-to-daemon drawer browser (#191) | — | [`257137b`](https://github.com/techempower-org/mempalace/commit/257137b) | +| 28 | Recency decay weighting in search + mempalace prune --stale-days CLI (#158) | — | [`558d327`](https://github.com/techempower-org/mempalace/commit/558d327) | +| 29 | mempalace_rate_memory MCP tool + bounded rating signal in search ranking (#159) | — | [`583536c`](https://github.com/techempower-org/mempalace/commit/583536c) | +| 30 | Formalize wing/room derivation order; demote entity detector to last-resort hint (#157) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 31 | RRF fusion mode + convex-vs-RRF A/B harness (#162) | [#247](https://github.com/MemPalace/mempalace/pull/247) | [`6c9d10c`](https://github.com/techempower-org/mempalace/commit/6c9d10c) | +| 32 | mempalace stats: add ROOMS breakdown (drawer count by room) to the dashboard | — | [`1673465`](https://github.com/techempower-org/mempalace/commit/1673465) | +| 33 | Calibrated confidence field on search results + Brier-score eval column | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 34 | Evaluation doc: curated-authority vs auto-mined separation (#202) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 35 | Apply AGE statement_timeout in same transaction as cypher() (PR #228 follow-up) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 36 | LLM-based KG triple extraction: queue table, async worker, llama.cpp on familiar | — | [`59ac0bc`](https://github.com/techempower-org/mempalace/commit/59ac0bc) | +| 37 | Promote verbatim-vs-derivative essay from research/ to README (#170) | — | [`6a264d9`](https://github.com/techempower-org/mempalace/commit/6a264d9) | +| 38 | mempalace stats — palace analytics dashboard (#191) | — | [`6f994fb`](https://github.com/techempower-org/mempalace/commit/6f994fb) | +| 39 | CLI wiring: mempalace mine --source (#57) | — | [`5ed9fa7`](https://github.com/techempower-org/mempalace/commit/5ed9fa7) | +| 40 | Warp terminal source adapter (#62) | — | [`2e85585`](https://github.com/techempower-org/mempalace/commit/2e85585) | +| 41 | OpenCode adapter smoke test against real DB (#56) | — | [`a9ed72b`](https://github.com/techempower-org/mempalace/commit/a9ed72b) | +| 42 | Codex, Gemini, and Aider source adapters (#61, #59) | — | [`0c23165`](https://github.com/techempower-org/mempalace/commit/0c23165) | +| 43 | Filesystem + conversation source adapters (#63) | — | [`9a1facf`](https://github.com/techempower-org/mempalace/commit/9a1facf) | +| 44 | Widen auto-query signal patterns for natural recall phrases | — | [`33e780e`](https://github.com/techempower-org/mempalace/commit/33e780e) | +| 45 | Native rename_wing backend operation + CLI command (#154) | — | [`d045f83`](https://github.com/techempower-org/mempalace/commit/d045f83) | +| 46 | Standalone essay: the verbatim-vs-derivative axis (#47) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 47 | Research doc: uncertainty-aware retrieval analysis (#84) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 48 | Design doc: scope/collection filter on mempalace_search (#76) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 49 | Agent-shaped CLI surface — --json / --quiet for non-MCP integration | — | [`25ed900`](https://github.com/techempower-org/mempalace/commit/25ed900) | +| 50 | Design eval: multi-palace separation — curated vs auto-mined (#45) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 51 | Document .sh shim delegation to palace-daemon (counter-position to upstream #1069) | — | [`bf0a4d0`](https://github.com/techempower-org/mempalace/commit/bf0a4d0) | +| 52 | Honor ~/.mempalace/RETIRED marker — refuse default palace, surface retire message | — | [`798cf14`](https://github.com/techempower-org/mempalace/commit/798cf14) | +| 53 | Empty repo .opencode/opencode.json mcp block — disabled flag wasn't being respected | — | [`7133eee`](https://github.com/techempower-org/mempalace/commit/7133eee) | +| 54 | Drop \$comment from .opencode/opencode.json — schema rejects unknown root keys | — | [`637bb01`](https://github.com/techempower-org/mempalace/commit/637bb01) | +| 55 | Disable repo-level MCP entry by default + venv-python fallback | — | [`47018e5`](https://github.com/techempower-org/mempalace/commit/47018e5) | +| 56 | Stub resources/list + prompts/list so MCP clients stop ERROR-logging on connect | — | [`6ca0670`](https://github.com/techempower-org/mempalace/commit/6ca0670) | +| 57 | Bundled OpenCode live-capture plugin that bypasses option-K v1.2.1 bugs (filed upstream as #4, #5) | — | [`5522623`](https://github.com/techempower-org/mempalace/commit/5522623) | +| 58 | Documented OpenCode integration recipe (read-side MCP + push plugin + retrospective adapter) | — | [`60dc9e6`](https://github.com/techempower-org/mempalace/commit/60dc9e6) | +| 59 | .opencode/opencode.json — repo-root MCP config so opencode picks up mempalace automatically | [#1567](https://github.com/MemPalace/mempalace/pull/1567) (OPEN) | [`ba16b82`](https://github.com/techempower-org/mempalace/commit/ba16b82) | +| 60 | OpenCodeSourceAdapter (RFC 002) — retrospective ingest of OpenCode SQLite sessions | [#1484](https://github.com/MemPalace/mempalace/pull/1484) (OPEN) | [`2ffe652`](https://github.com/techempower-org/mempalace/commit/2ffe652) | +| 61 | mempalace_walk_palace MCP tool — agent walks the palace via AGE Cypher | — | [`8022ecb`](https://github.com/techempower-org/mempalace/commit/8022ecb) | +| 62 | Backfill AGE graph from existing drawer table — restartable, checkpointed | — | [`b3f0206`](https://github.com/techempower-org/mempalace/commit/b3f0206) | +| 63 | Wing/Room/Drawer hierarchy as native AGE nodes; Cypher MATCH walks palace structure | — | [`ff583c0`](https://github.com/techempower-org/mempalace/commit/ff583c0) | +| 64 | Write-through middleware on PostgresCollection — entities populate AGE on every drawer write | — | [`3321d83`](https://github.com/techempower-org/mempalace/commit/3321d83) | +| 65 | KnowledgeGraphAGE API parity with SQLite KG: add_entity, invalidate, query_entity, query_relationship, timeline, seed_from_entity_facts | — | [`ff7187d`](https://github.com/techempower-org/mempalace/commit/ff7187d) | +| 66 | Pending-writes journal + replay so daemon outages stop being silent | — | [`0c34464`](https://github.com/techempower-org/mempalace/commit/0c34464) | +| 67 | MCP server distinguishes 'backend unreachable' from 'no palace found' | — | [`0c34464`](https://github.com/techempower-org/mempalace/commit/0c34464) | +| 68 | Defense-in-depth metadata sanitizer at the chromadb-client chokepoint | — | [`f499814`](https://github.com/techempower-org/mempalace/commit/f499814) | +| 69 | Route Stop/PreCompact hooks through palace-daemon/clients/hook.py | — | [`42ded2e`](https://github.com/techempower-org/mempalace/commit/42ded2e) | +| 70 | KnowledgeGraphAGE skeleton — Apache AGE graph bootstrap over psycopg2 | — | [`a3ee623`](https://github.com/techempower-org/mempalace/commit/a3ee623) | +| 71 | README pivots to the four-layer model + Auto Dream as vindication of the verbatim-vs-derivative axis | — | [`55b36ca`](https://github.com/techempower-org/mempalace/commit/55b36ca) | +| 72 | CI: gate postgres-backend tests against a pgvector service container | — | [`da0bdbb`](https://github.com/techempower-org/mempalace/commit/da0bdbb) | +| 73 | PostgreSQL backend via #665 cherry-pick + fork-side adaptations + smoke tests | [#665](https://github.com/MemPalace/mempalace/pull/665) (OPEN) | [`5e90c72`](https://github.com/techempower-org/mempalace/commit/5e90c72) | +| 74 | daemon-route `mempalace status` / `search` / `mine` when PALACE_DAEMON_URL is set | — | [`22ef562`](https://github.com/techempower-org/mempalace/commit/22ef562) | +| 75 | daemon-route `mcp_server.py` via the `handle_request` JSON-RPC chokepoint | — | [`41359ba`](https://github.com/techempower-org/mempalace/commit/41359ba) | +| 76 | Preserve dashed project names in transcript-derived wings | [#10](https://github.com/MemPalace/mempalace/pull/10) | [`d76134d`](https://github.com/techempower-org/mempalace/commit/d76134d) | +| 77 | Drop wing_ prefix from transcript-derived wings to converge with operator mines | [#9](https://github.com/MemPalace/mempalace/pull/9) | [`86d4700`](https://github.com/techempower-org/mempalace/commit/86d4700) | +| 78 | Retire mempalace_session_recovery collection + read tool | [#8](https://github.com/MemPalace/mempalace/pull/8) | [`0b945e1`](https://github.com/techempower-org/mempalace/commit/0b945e1) | +| 79 | mempalace mined + purge --source-file (mining management surface) | [#7](https://github.com/MemPalace/mempalace/pull/7) | [`2e6ced9`](https://github.com/techempower-org/mempalace/commit/2e6ced9) | +| 80 | Drop hook-side checkpoint diary writes — verbatim-only architecture | [#6](https://github.com/MemPalace/mempalace/pull/6) | [`69768fc`](https://github.com/techempower-org/mempalace/commit/69768fc) | +| 81 | Restore transcript ingest via daemon /mine when PALACE_DAEMON_URL is set | [#2](https://github.com/MemPalace/mempalace/pull/2) | [`09d2ca6`](https://github.com/techempower-org/mempalace/commit/09d2ca6) | +| 82 | `hook_verbatim_mode` config flag preserves system tags + full tool I/O during transcript ingest | — | [`ef98961`](https://github.com/techempower-org/mempalace/commit/ef98961) | +| 83 | Retire the `kind=` filter — structural split made it inert | — | [`7ba28dc`](https://github.com/techempower-org/mempalace/commit/7ba28dc) | +| 84 | Hoist CLOSET_RANK_BOOSTS to module level + record VecRecall ablation finding | — | [`3cb03f3`](https://github.com/techempower-org/mempalace/commit/3cb03f3) | +| 85 | Strip embedded API key from .claude-plugin/ manifests; rely on env inheritance | — | [`9f91e18`](https://github.com/techempower-org/mempalace/commit/9f91e18) | +| 86 | Cherry-pick #1094 — coerce None metadatas at chromadb boundary | [#1094](https://github.com/MemPalace/mempalace/pull/1094) (OPEN) | [`43d728d`](https://github.com/techempower-org/mempalace/commit/43d728d) | +| 87 | Cherry-pick #1087 rewrite — collection.delete(where=) instead of nuke-and-rebuild | [#1087](https://github.com/MemPalace/mempalace/pull/1087) (OPEN) | [`366a9ad`](https://github.com/techempower-org/mempalace/commit/366a9ad) | +| 88 | Canonical YAML manifest + renderer for fork-ahead docs | — | [`5a01aec`](https://github.com/techempower-org/mempalace/commit/5a01aec) | +| 89 | Phase D migration + PreCompact recovery write | — | [`42817d7`](https://github.com/techempower-org/mempalace/commit/42817d7) | +| 90 | Surface drawer_id in search/diary/recovery payloads | — | [`9a8bb77`](https://github.com/techempower-org/mempalace/commit/9a8bb77) | +| 91 | Cherry-pick #1085 — batch ChromaDB inserts in miner (10–30× faster) | [#1085](https://github.com/MemPalace/mempalace/pull/1085) (CLOSED) | [`6be6fff`](https://github.com/techempower-org/mempalace/commit/6be6fff) | +| 92 | scripts/deploy.sh — one-command Syncthing-aware redeploy | — | [`8252025`](https://github.com/techempower-org/mempalace/commit/8252025) | +| 93 | Phases A–C of the checkpoint collection split | — | [`e266365`](https://github.com/techempower-org/mempalace/commit/e266365) | +| 94 | kind= filter on search_memories excludes Stop-hook checkpoints (transitional) | — | [`f9f5cc4`](https://github.com/techempower-org/mempalace/commit/f9f5cc4) | ### Recently merged into upstream diff --git a/deploy/docker-compose.server.yml b/deploy/docker-compose.server.yml new file mode 100644 index 0000000000..e1e14e80e5 --- /dev/null +++ b/deploy/docker-compose.server.yml @@ -0,0 +1,71 @@ +# MemPalace remote team server — MCP over HTTP, backed by a central Qdrant. +# +# One command stands up a shared memory service a whole team's AI clients +# connect to. Embeddings are still produced locally inside the mempalace +# container; only your own Qdrant ever receives the vectors and text. +# +# 1. cp deploy/server.env.example deploy/.env && edit deploy/.env +# (at minimum set MEMPALACE_MCP_HTTP_TOKEN to a long random secret) +# 2. docker compose -f deploy/docker-compose.server.yml --env-file deploy/.env up -d +# 3. connect a client (see the Remote / Team Server guide): +# claude mcp add --transport http mempalace http://YOUR_HOST:8765/mcp \ +# --header "Authorization: Bearer $MEMPALACE_MCP_HTTP_TOKEN" +# +# SECURITY: this exposes plaintext HTTP on :8765. For anything beyond a trusted +# private network, put a TLS-terminating reverse proxy in front (nginx/Caddy/ +# Traefik) and only expose the proxy. The bearer token is mandatory for the +# network-exposed (0.0.0.0) bind. + +services: + qdrant: + image: qdrant/qdrant:latest + restart: unless-stopped + volumes: + - qdrant-storage:/qdrant/storage + # Not published to the host: only the mempalace service reaches it over the + # internal compose network. Uncomment to inspect Qdrant directly. + # ports: + # - "6333:6333" + + mempalace: + image: ghcr.io/mempalace/mempalace:latest + restart: unless-stopped + depends_on: + - qdrant + command: + - serve + - --host + - "0.0.0.0" + - --port + - "8765" + - --backend + - qdrant + # Uncomment to expose recall without write access to most clients: + # - --read-only + environment: + # Required for the network-exposed bind. Set in deploy/.env. + MEMPALACE_MCP_HTTP_TOKEN: ${MEMPALACE_MCP_HTTP_TOKEN:?set MEMPALACE_MCP_HTTP_TOKEN in deploy/.env} + MEMPALACE_QDRANT_URL: http://qdrant:6333 + MEMPALACE_QDRANT_API_KEY: ${MEMPALACE_QDRANT_API_KEY:-} + # Set to cuda/dml/coreml on an accelerated host (see Dockerfile.gpu). + MEMPALACE_EMBEDDING_DEVICE: ${MEMPALACE_EMBEDDING_DEVICE:-auto} + ports: + - "8765:8765" + volumes: + - mempalace-data:/data + healthcheck: + # The image has no curl; use Python (always present). /healthz needs no auth. + # If you enable TLS on the server itself, switch this to https + ssl context. + test: + - CMD + - python + - -c + - "import urllib.request,sys; sys.exit(0) if urllib.request.urlopen('http://127.0.0.1:8765/healthz').read().strip()==b'ok' else sys.exit(1)" + interval: 30s + timeout: 5s + retries: 5 + start_period: 40s + +volumes: + qdrant-storage: + mempalace-data: diff --git a/deploy/mempalace-server.service b/deploy/mempalace-server.service new file mode 100644 index 0000000000..f5ff5b8943 --- /dev/null +++ b/deploy/mempalace-server.service @@ -0,0 +1,48 @@ +# MemPalace remote MCP server — systemd unit template. +# +# Install: +# sudo useradd --system --home /var/lib/mempalace --shell /usr/sbin/nologin mempalace +# sudo install -d -o mempalace -g mempalace -m 750 /var/lib/mempalace /etc/mempalace +# sudo cp deploy/server.env.example /etc/mempalace/server.env # then edit + chmod 600 +# sudo install -m 600 -o mempalace -g mempalace /etc/mempalace/server.env /etc/mempalace/server.env +# # install mempalace into a venv on PATH, or adjust ExecStart to its absolute path +# sudo cp deploy/mempalace-server.service /etc/systemd/system/ +# sudo systemctl daemon-reload && sudo systemctl enable --now mempalace-server +# +# This binds 0.0.0.0:8765 and requires MEMPALACE_MCP_HTTP_TOKEN (set in the +# EnvironmentFile). Front it with a TLS-terminating reverse proxy, or set +# MEMPALACE_MCP_TLS_CERT / _KEY in the EnvironmentFile for native TLS. + +[Unit] +Description=MemPalace remote MCP server +After=network-online.target +Wants=network-online.target + +[Service] +Type=exec +User=mempalace +Group=mempalace +EnvironmentFile=/etc/mempalace/server.env +ExecStart=mempalace serve --host 0.0.0.0 --port 8765 +Restart=on-failure +RestartSec=2 + +# --- Hardening --------------------------------------------------------------- +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +PrivateDevices=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +RestrictNamespaces=true +LockPersonality=true +MemoryDenyWriteExecute=false +# The palace and any local state live here; everything else is read-only. +ReadWritePaths=/var/lib/mempalace +StateDirectory=mempalace + +[Install] +WantedBy=multi-user.target diff --git a/deploy/server.env.example b/deploy/server.env.example new file mode 100644 index 0000000000..d6d309306d --- /dev/null +++ b/deploy/server.env.example @@ -0,0 +1,28 @@ +# MemPalace remote server environment. +# Copy to deploy/.env (compose) or /etc/mempalace/server.env (systemd) and edit. +# Keep this file readable only by the service account: chmod 600. + +# --- Required for a network-exposed (0.0.0.0) bind ------------------------------ +# Clients send: Authorization: Bearer . Generate a strong secret: +# openssl rand -hex 32 +MEMPALACE_MCP_HTTP_TOKEN= + +# --- Storage backend ------------------------------------------------------------ +# The team server should use a networked backend so the palace is shared. +MEMPALACE_BACKEND=qdrant +MEMPALACE_QDRANT_URL=http://qdrant:6333 +# MEMPALACE_QDRANT_API_KEY= + +# --- Embedding ------------------------------------------------------------------ +# auto | cpu | cuda | dml | coreml. Use cuda on a GPU host (needs the GPU image). +MEMPALACE_EMBEDDING_DEVICE=auto + +# --- Optional: native TLS (otherwise terminate TLS at a reverse proxy) ---------- +# Point these at a PEM cert/key the service account can read. When set, serve +# speaks HTTPS directly and clients connect to https://... +# MEMPALACE_MCP_TLS_CERT=/etc/mempalace/tls/cert.pem +# MEMPALACE_MCP_TLS_KEY=/etc/mempalace/tls/key.pem + +# --- Palace location (systemd / bare-metal) ------------------------------------- +# In Docker the palace lives on the mempalace-data volume by default. +# MEMPALACE_PALACE_PATH=/var/lib/mempalace/palace diff --git a/docs/authored-at.md b/docs/authored-at.md new file mode 100644 index 0000000000..6f4d72f94c --- /dev/null +++ b/docs/authored-at.md @@ -0,0 +1,45 @@ +# Authored date (`authored_at`) + +Conversation transcripts carry a per-line ISO-8601 `timestamp` (both Claude Code and +Codex JSONL). The miner records the most recent one per file as the drawer's +**`authored_at`** — when the content was actually written. + +This is distinct from the ingest date: + +| Field | Meaning | +|-------|---------| +| `filed_at` / result `created_at` | When the drawer was **mined** (written to the palace). A bulk re-mine collapses these to a single instant. | +| `authored_at` | When the underlying content was **written**, recovered from the transcript timestamps. Survives re-mining. | + +`authored_at` is surfaced in search results (and shown in the CLI `search` output), and is +used as a deterministic tie-break in hybrid ranking: candidates with identical scores order +with the more recently authored drawer first. Drawers without per-line timestamps (e.g. +markdown) fall back to `filed_at`. + +## Backfilling existing memory + +New mines populate `authored_at` automatically. Drawers mined before this feature only have +`filed_at`. Re-mining does **not** fix them — the scanner skips files already mined at the +current `NORMALIZE_VERSION`. Two options: + +1. **In-place backfill (recommended — no re-embedding).** `scripts/backfill_authored_at.py` + reads each convos drawer's source transcript and updates only the `authored_at` metadata. + Idempotent and safe to re-run; embeddings are untouched. + + ```bash + python scripts/backfill_authored_at.py \ + --palace ~/.mempalace/palace \ + --sessions ~/.claude --sessions ~/.codex # dry run + python scripts/backfill_authored_at.py \ + --palace ~/.mempalace/palace \ + --sessions ~/.claude --sessions ~/.codex --apply # write + ``` + + For the Docker MCP image, mount the volume and session dirs read-only — see the header of + `scripts/backfill_authored_at.py` for the exact `docker run` invocation. + + > Back up first: `tar czf palace-backup.tgz -C .` (or snapshot the + > `mempalace-data` volume). + +2. **Drop and recreate.** Delete the affected drawers and re-mine the transcripts; the fresh + mine stamps `authored_at`. Simpler, but re-embeds everything. diff --git a/docs/fork-changes.yaml b/docs/fork-changes.yaml index 2368d3fa8d..529d637510 100644 --- a/docs/fork-changes.yaml +++ b/docs/fork-changes.yaml @@ -24,6 +24,46 @@ entries: + - id: sync-upstream-368 + date: 2026-07-02 + bucket: Changed + commit: TBD + area: Reliability + summary: "Sync upstream/develop through da5a48c (post-v3.5.0): remote MCP server w/ TLS + read-only, graph auto-population, Qdrant facets, list_drawers date filters, 213 commits" + files: + - mempalace/mcp_server.py + - mempalace/searcher.py + - mempalace/cli.py + - mempalace/convo_miner.py + - mempalace/embedding.py + - mempalace/backends/base.py + - mempalace/backends/pgvector.py + - tests/conftest.py + - tests/test_mcp_server.py + - tests/test_backends.py + body: | + Merged 213 upstream commits (post-v3.5.0 ``da5a48c``). Notable + upstream additions: the turnkey secure remote MCP server with TLS and + a read-only server mode (#1877 / #1900), associative-graph + auto-population from mined sessions + ``cmd_hallways`` (#1895), + Qdrant server-side metadata facets (#1868), ``since``/``before`` + date filters on ``list_drawers`` (#1128 / #1891), authored-timestamp + preservation from transcripts (#1890), ``mine_palace_lock`` + re-entrancy for the HTTP transport (#1859), a pgvector metadata-only + fetch fix (#1892), SQLite magic-header ``detect()`` (#1893 / #1896), + FTS5 auto-heal (#1878), a host-root-logger fix (#1860 / #1885), + LaTeX extensions, and dependency bumps (ruff 0.15.20). + + ~50 conflicted files resolved by composing rather than choosing + sides: ``tool_list_drawers`` carries BOTH the upstream date filters + and the fork tag filters; ``tool_status`` keeps the fork's postgres + fast path (#267) and gains the upstream facets sweep; the HTTP + transport keeps host pinning and gains TLS + read-only; the merged + plugin hook config stays the fork's five-event ms-timeout shape. The + merged MCP tool surface stays at 39 tools (upstream's 34 plus fork + tools); all doc and manifest tool-count claims reconciled against the + live ``mcp_server.TOOLS`` count. + - id: auto-query-firing-fixes date: 2026-07-01 bucket: Fixed diff --git a/hooks/cursor/mempal_precompact_hook_cursor.sh b/hooks/cursor/mempal_precompact_hook_cursor.sh index f7e0feec33..dcb76e4be7 100755 --- a/hooks/cursor/mempal_precompact_hook_cursor.sh +++ b/hooks/cursor/mempal_precompact_hook_cursor.sh @@ -92,10 +92,10 @@ mempal_log "preCompact" "$MEMPAL_CONV_ID" \ # right before the irreversible compaction. The pending-save marker # below is the backstop: the next `stop` hook re-mines and nudges a # verbatim save regardless of whether this mine completed. -if "$MEMPAL_PYTHON_BIN" -c "import mempalace" >/dev/null 2>&1; then +if command -v mempalace >/dev/null 2>&1; then if mempal_is_valid_transcript "$MEMPAL_TRANSCRIPT" \ && [ -f "$MEMPAL_TRANSCRIPT" ]; then - "$MEMPAL_PYTHON_BIN" -m mempalace mine "$(dirname "$MEMPAL_TRANSCRIPT")" --mode convos \ + mempalace mine "$(dirname "$MEMPAL_TRANSCRIPT")" --mode convos \ >> "$MEMPAL_CURSOR_LOG" 2>&1 || \ mempal_log "preCompact" "$MEMPAL_CONV_ID" \ "WARN: mempalace mine convos returned non-zero" @@ -104,14 +104,14 @@ if "$MEMPAL_PYTHON_BIN" -c "import mempalace" >/dev/null 2>&1; then "skipping invalid transcript path: $MEMPAL_TRANSCRIPT" fi if [ -n "$MEMPAL_DIR" ] && [ -d "$MEMPAL_DIR" ]; then - "$MEMPAL_PYTHON_BIN" -m mempalace mine "$MEMPAL_DIR" --mode projects \ + mempalace mine "$MEMPAL_DIR" --mode projects \ >> "$MEMPAL_CURSOR_LOG" 2>&1 || \ mempal_log "preCompact" "$MEMPAL_CONV_ID" \ "WARN: mempalace mine projects returned non-zero" fi else mempal_log "preCompact" "$MEMPAL_CONV_ID" \ - "mempalace module not importable via $MEMPAL_PYTHON_BIN; skipping synchronous mine" + "mempalace CLI not on PATH; skipping synchronous mine" fi # ── Drop the pending-save marker ────────────────────────────────── diff --git a/hooks/cursor/mempal_save_hook_cursor.sh b/hooks/cursor/mempal_save_hook_cursor.sh index 12ba503028..19ed26a0c4 100755 --- a/hooks/cursor/mempal_save_hook_cursor.sh +++ b/hooks/cursor/mempal_save_hook_cursor.sh @@ -237,22 +237,22 @@ mempal_log "stop" "$MEMPAL_CONV_ID" "TRIGGERING SAVE at counter=$NEXT" # Cursor-configured timeout. `command -v mempalace` gates so a user # without the CLI on PATH (e.g. a fresh GUI-launched session) does # not see a noisy error. -if "$MEMPAL_PYTHON_BIN" -c "import mempalace" >/dev/null 2>&1; then +if command -v mempalace >/dev/null 2>&1; then if mempal_is_valid_transcript "$MEMPAL_TRANSCRIPT" \ && [ -f "$MEMPAL_TRANSCRIPT" ]; then - ( "$MEMPAL_PYTHON_BIN" -m mempalace mine "$(dirname "$MEMPAL_TRANSCRIPT")" --mode convos \ + ( mempalace mine "$(dirname "$MEMPAL_TRANSCRIPT")" --mode convos \ >> "$MEMPAL_CURSOR_LOG" 2>&1 ) & elif [ -n "$MEMPAL_TRANSCRIPT" ]; then mempal_log "stop" "$MEMPAL_CONV_ID" \ "skipping invalid transcript path: $MEMPAL_TRANSCRIPT" fi if [ -n "$MEMPAL_DIR" ] && [ -d "$MEMPAL_DIR" ]; then - ( "$MEMPAL_PYTHON_BIN" -m mempalace mine "$MEMPAL_DIR" --mode projects \ + ( mempalace mine "$MEMPAL_DIR" --mode projects \ >> "$MEMPAL_CURSOR_LOG" 2>&1 ) & fi else mempal_log "stop" "$MEMPAL_CONV_ID" \ - "mempalace module not importable via $MEMPAL_PYTHON_BIN; skipping background mine" + "mempalace CLI not on PATH; skipping background mine" fi # The followup is the load-bearing verbatim path for Cursor (see header), diff --git a/integrations/openclaw/SKILL.md b/integrations/openclaw/SKILL.md index 4ed4ba0262..5fd6d5cf61 100644 --- a/integrations/openclaw/SKILL.md +++ b/integrations/openclaw/SKILL.md @@ -1,7 +1,7 @@ --- name: mempalace description: "MemPalace — Local AI memory with 96.6% recall. Semantic search, temporal knowledge graph, palace architecture (wings/rooms/drawers). Free, no cloud, no API keys." -version: 3.3.0 +version: 3.5.0 homepage: https://github.com/MemPalace/mempalace user-invocable: true metadata: @@ -46,6 +46,10 @@ You have access to a local memory palace via MCP tools. The palace stores verbat ## Available Tools +Full MCP surface: 39 tools. Destructive or host-level tools are documented so +you know they exist, but use them only when the user explicitly asks or when a +tool-specific workflow below says to. + ### Search & Browse - `mempalace_search` — Semantic search across all memories. Always start here. - `query` (required): natural language search — keep it short, keywords or a question. Do NOT include system prompts or conversation context. @@ -58,6 +62,14 @@ You have access to a local memory palace via MCP tools. The palace stores verbat - `mempalace_status` — Palace overview: total drawers, wings, rooms, AAAK spec - `mempalace_list_wings` — All wings with drawer counts - `mempalace_list_rooms` — Rooms within a wing (optional wing filter) +- `mempalace_list_drawers` — Paginated drawer listing + - `wing`, `room`: optional filters + - `since`: only drawers filed on/after this ISO date/time + - `before`: only drawers filed before this ISO date/time + - `limit`: max results (default 20) + - `offset`: pagination offset (default 0) +- `mempalace_get_drawer` — Fetch a single drawer by ID. Returns full verbatim content and metadata. + - `drawer_id` (required) - `mempalace_get_taxonomy` — Full wing/room/count tree - `mempalace_get_aaak_spec` — Get AAAK compression dialect specification @@ -81,17 +93,58 @@ You have access to a local memory palace via MCP tools. The palace stores verbat - `mempalace_traverse` — Walk from a room, find connected ideas across wings - `start_room` (required): room to start from - `max_hops`: connection depth (default 2) -- `mempalace_find_tunnels` — Find rooms that bridge two wings - - `wing_a`, `wing_b` (required) +- `mempalace_find_tunnels` — Find rooms that bridge two wings via *implicit* overlap (rooms whose drawers naturally share content across wings — discovered, not declared) + - `wing_a`, `wing_b`: optional filters; omit both to scan all wing pairs +- `mempalace_create_tunnel` — Create an *explicit* cross-wing tunnel: a user/agent-declared link between two locations. Use when you notice content in one project relates to another (e.g. API design in `project_api` connects to schema in `project_database`). + - `source_wing`, `source_room`, `target_wing`, `target_room` (required) + - `label`: short description of the relationship + - `source_drawer_id`, `target_drawer_id`: anchor to specific drawers +- `mempalace_list_tunnels` — List all explicit tunnels, optionally filtered by wing + - `wing`: optional filter +- `mempalace_delete_tunnel` — Remove an explicit tunnel by ID + - `tunnel_id` (required) +- `mempalace_list_hallways` — List within-wing entity hallways (entity-to-entity co-occurrence links built at mine time) + - `wing`: optional filter +- `mempalace_delete_hallway` — Remove a hallway record by ID + - `hallway_id` (required) +- `mempalace_follow_tunnels` — From a room, follow explicit tunnels to connected drawers in other wings + - `wing`, `room` (required) - `mempalace_graph_stats` — Graph connectivity overview ### Write - `mempalace_add_drawer` — Store verbatim content into a wing/room - `wing`, `room`, `content` (required) - `source_file`: optional source reference + - `added_by`: optional filing agent label - Checks for duplicates automatically +- `mempalace_checkpoint` — Save a whole session in one call: dedup each item, file non-duplicates, then write one diary entry + - `items` (required): array of `{wing, room, content}`; content must be verbatim + - `diary`: optional `{agent_name, entry, topic?, wing?}`; entry should use AAAK format + - `dedup_threshold`: similarity threshold (default 0.9) +- `mempalace_update_drawer` — Update an existing drawer's content and/or move it to a different wing/room + - `drawer_id` (required) + - `content`, `wing`, `room`: at least one must be provided (no-op otherwise) - `mempalace_delete_drawer` — Remove a drawer by ID - `drawer_id` (required) + +### Ingest & Cleanup +- `mempalace_mine` — Mine a directory into the palace. Host-level ingest; call only when the user asks to import files. + - `source` (required): directory to mine + - `mode`: `projects` (default), `convos`, or `extract` + - `wing`: target wing (default: source directory name) + - `agent`: recorded on every drawer (default `mempalace`) + - `limit`: max files to process (0 = all) + - `dry_run`: preview without writing + - `extract`: convos extraction strategy (`exchange` default, or `general`) +- `mempalace_sync` — Prune drawers whose source files are gitignored, deleted, or moved. Use dry-run first. + - `project_dir`: optional project root scope + - `wing`: optional wing scope + - `apply`: actually delete; default is dry-run preview +- `mempalace_delete_by_source` — Bulk-delete drawers with one exact `source_file`. Destructive; use dry-run first. + - `source_file` (required): exact metadata value to remove + - `dry_run`: preview match count and sample (default true) + +### Diary & Session - `mempalace_diary_write` — Write a session diary entry - `agent_name` (required): your name/identifier - `entry` (required): what happened, what you learned, what matters @@ -99,6 +152,15 @@ You have access to a local memory palace via MCP tools. The palace stores verbat - `mempalace_diary_read` — Read recent diary entries - `agent_name` (required) - `last_n`: number of entries (default 10) +- `mempalace_memories_filed_away` — Acknowledge the latest silent auto-save checkpoint. + - Returns: how many messages were tucked into drawers since the last ack + - When to call: at the START of a session, to confirm prior-conversation persistence + +### System +- `mempalace_hook_settings` — Get or set auto-save hook behavior. Host-level setting; do not change silently. + - `silent_save`: true saves directly without MCP-level clutter + - `desktop_toast`: true shows a desktop notification when saves complete +- `mempalace_reconnect` — Force reconnect to the palace database after external writes or stale index state ## Setup diff --git a/mempalace/backends/base.py b/mempalace/backends/base.py index 303577af05..65537168fa 100644 --- a/mempalace/backends/base.py +++ b/mempalace/backends/base.py @@ -501,6 +501,15 @@ def get_all_metadata(self, where: Optional[dict] = None) -> list[dict]: offset += len(batch_meta) return all_meta + def facet_counts( + self, + field: str, + where: Optional[dict] = None, + limit: int = 1000, + ) -> dict[str, int]: + """Return counts for each distinct value of a metadata field.""" + raise UnsupportedCapabilityError("backend does not support facet_counts") + def maintenance_state(self) -> dict: """Return a structured snapshot of this collection's maintenance state. diff --git a/mempalace/backends/chroma.py b/mempalace/backends/chroma.py index ea36dc8216..7a201151fa 100644 --- a/mempalace/backends/chroma.py +++ b/mempalace/backends/chroma.py @@ -2456,7 +2456,26 @@ def health(self, palace: Optional[PalaceRef] = None) -> HealthStatus: @classmethod def detect(cls, path: str) -> bool: - return os.path.isfile(os.path.join(path, "chroma.sqlite3")) + """Return True when ``path`` looks like a chroma palace. + + Verifies the SQLite magic header rather than file presence alone. + Bare ``sqlite3.connect()`` against a missing path leaves a 0-byte + file behind (the SQLite header is written on the first statement, + not on connection), so file-presence alone treats those artifacts + as real chroma palaces and breaks multi-backend resolution. The + 16-byte ``SQLite format 3\\x00`` magic prefix is written as soon + as chromadb's ``PersistentClient`` does any work, so this check + accepts every real chroma palace while rejecting empty / garbage + files. See #1893. + """ + db_path = os.path.join(path, "chroma.sqlite3") + if not os.path.isfile(db_path): + return False + try: + with open(db_path, "rb") as f: + return f.read(16) == b"SQLite format 3\x00" + except OSError: + return False # ------------------------------------------------------------------ # Legacy (pre-RFC 001) surface — retained while callers migrate. diff --git a/mempalace/backends/pgvector.py b/mempalace/backends/pgvector.py index cfe3494507..b0632c8c55 100644 --- a/mempalace/backends/pgvector.py +++ b/mempalace/backends/pgvector.py @@ -674,13 +674,19 @@ def scroll_rows( *, where: Optional[dict] = None, with_embedding: bool = False, + with_document: bool = True, limit: Optional[int] = None, offset: Optional[int] = None, ) -> list[dict]: qi = _quote_identifier(table) params: list = [] where_sql = _where_to_sql(where, params) if where else "TRUE" - cols = "id, document, metadata" + # Project NULL into the document slot when the caller only needs + # metadata (e.g. mempalace_status's wing/room tally). Keeps the + # positional _row parser unchanged — document remains record[1] — + # while avoiding O(n × document_size) bytes over the wire on remote + # pgvector deployments. Follow-up to #1840. + cols = "id, document, metadata" if with_document else "id, NULL::text, metadata" if with_embedding: cols += ", embedding" sql = f"SELECT {cols} FROM {qi} WHERE {where_sql}" @@ -855,7 +861,15 @@ def _ensure_table(self, dimension: int) -> None: ) self._known_dimension = existing_dim or dimension - def _scroll(self, *, where=None, with_embedding=False, limit=None, offset=None) -> list[dict]: + def _scroll( + self, + *, + where=None, + with_embedding=False, + with_document=True, + limit=None, + offset=None, + ) -> list[dict]: self._ensure_open() if not self._table_exists(): if self._marker_exists(): @@ -865,10 +879,39 @@ def _scroll(self, *, where=None, with_embedding=False, limit=None, offset=None) self._table, where=where, with_embedding=with_embedding, + with_document=with_document, limit=limit, offset=offset, ) + def get_all_metadata(self, where=None) -> list[dict]: + """Single-pass metadata-only fetch — projects out the document column. + + The base implementation pages through ``get(include=["metadatas"])``, + which routes here via ``_scroll`` and (pre-this-override) always sent + the ``document`` text over the wire even when nothing consumed it. + For pgvector deployments where the client is remote (TLS over WAN), + that meant ``mempalace_status`` transferred O(n × document_size) + bytes per call, dominating wall time. With ``with_document=False`` + the SELECT replaces document with NULL, dropping the per-row payload + to id + metadata for every caller of this method. + + Filtered fetches still need the ``_matches_where`` post-filter for + non-pushdown semantics (array/object values where ``metadata @> ...`` + is broader than the exact match the caller asked for — same + correctness contract as #1840's filtered ``get`` path). Since that + post-filter only reads ``metadata``, we keep the single-scroll + + ``with_document=False`` fast path and just apply the filter locally + on the metadata dicts before returning. This extends the wire-byte + win to filtered callers as well. + """ + _validate_where(where) + pushdown = None if _requires_local_filter(where) else where + rows = self._scroll(where=pushdown, with_document=False) + if where is None: + return [row["metadata"] for row in rows] + return [row["metadata"] for row in rows if _matches_where(row["metadata"], where)] + def _rows( self, *, diff --git a/mempalace/backends/qdrant.py b/mempalace/backends/qdrant.py index 1e5b2b4317..c7b1550230 100644 --- a/mempalace/backends/qdrant.py +++ b/mempalace/backends/qdrant.py @@ -40,6 +40,7 @@ PalaceNotFoundError, PalaceRef, QueryResult, + UnsupportedCapabilityError, UnsupportedFilterError, _IncludeSpec, ) @@ -543,6 +544,38 @@ def count_points(self, collection: str) -> int: result = response.get("result") or {} return int(result.get("count") or 0) + def facet_counts( + self, + collection: str, + *, + field: str, + qdrant_filter: Optional[dict] = None, + limit: int = 1000, + ) -> dict[str, int]: + body: dict[str, Any] = { + "key": field, + "exact": True, + "limit": limit, + } + + if qdrant_filter: + body["filter"] = qdrant_filter + + response = self.request( + "POST", + f"/collections/{urlparse.quote(collection, safe='')}/facet", + body=body, + ) + + result = response.get("result") or {} + hits = result.get("hits") or [] + + return { + str(hit["value"]): int(hit.get("count") or 0) + for hit in hits + if hit.get("value") is not None + } + def delete_collection(self, collection: str) -> None: self.request("DELETE", f"/collections/{urlparse.quote(collection, safe='')}") @@ -1035,6 +1068,34 @@ def get_all_metadata(self, where: Optional[dict] = None) -> list[dict]: rows = self._rows(where=where) return [row["metadata"] for row in rows] + def facet_counts( + self, + field: str, + where: Optional[dict] = None, + limit: int = 1000, + ) -> dict[str, int]: + self._ensure_open() + # Validate the filter before the existence short-circuit so an + # unsupported local-only filter raises regardless of whether the + # collection has been materialized yet — matching the order used by + # get()/lexical_search() above (#1835 review). + _validate_where(where) + if _requires_local_filter(where): + raise UnsupportedCapabilityError("facet_counts does not support local-only filters") + if not self._remote_exists(): + if self._marker_exists(): + raise CollectionNotInitializedError(self._collection_name) + return {} + + q_filter = _qdrant_filter(where) + + return self._client.facet_counts( + self._remote_collection, + field=f"{_PAYLOAD_METADATA}.{field}", + qdrant_filter=q_filter, + limit=limit, + ) + def delete(self, *, ids=None, where=None): _validate_where(where) if not self._remote_exists(): @@ -1125,6 +1186,7 @@ class QdrantBackend(BaseBackend): "supports_embeddings_out", "supports_metadata_filters", "supports_lexical_search", + "supports_metadata_facets", "supports_namespace_isolation", "server_mode", } diff --git a/mempalace/backends/sqlite_exact.py b/mempalace/backends/sqlite_exact.py index 53f1cde559..fd29dfa6d0 100644 --- a/mempalace/backends/sqlite_exact.py +++ b/mempalace/backends/sqlite_exact.py @@ -1045,7 +1045,23 @@ def health(self, palace: Optional[PalaceRef] = None) -> HealthStatus: @classmethod def detect(cls, path: str) -> bool: - return os.path.isfile(os.path.join(path, _DB_FILENAME)) + """Return True when ``path`` looks like a sqlite_exact palace. + + Verifies the SQLite magic header rather than file presence alone, for + the same reason as :py:meth:`mempalace.backends.chroma.ChromaBackend.detect`: + bare ``sqlite3.connect()`` against a missing path leaves a 0-byte file + behind because the SQLite header is written on the first statement, + not on connection. The 16-byte ``SQLite format 3\\x00`` magic prefix + accepts every real palace while rejecting empty / garbage files. See #1893. + """ + db_path = os.path.join(path, _DB_FILENAME) + if not os.path.isfile(db_path): + return False + try: + with open(db_path, "rb") as f: + return f.read(16) == b"SQLite format 3\x00" + except OSError: + return False def create_collection(self, palace_path: str, collection_name: str) -> SQLiteExactCollection: return self.get_collection(palace_path, collection_name, create=True) diff --git a/mempalace/cli.py b/mempalace/cli.py index 6c3098cb14..2d5d7464e4 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -4554,6 +4554,21 @@ def _build_overlap_cypher(wing_a: str, wing_b: str, limit: int) -> str: ) +def cmd_hallways(args): + """List within-wing entity hallways (the auto-built associative graph).""" + from .hallways import list_hallways + + rows = list_hallways(getattr(args, "wing", None)) + if not rows: + print("No hallways yet — they are built from drawer entities when you mine.") + return + rows.sort(key=lambda h: h.get("co_occurrence_count", 0), reverse=True) + print(f" {len(rows)} hallway(s):") + for h in rows[: max(0, args.limit)]: + label = h.get("label") or f"{h.get('entity_a', '?')} <-> {h.get('entity_b', '?')}" + print(f" {label}") + + def _print_overlap_table(rows: list[dict], wing_a: str, wing_b: str) -> None: """Aligned columns: entity | A drawers | B drawers | total.""" if not rows: @@ -5199,6 +5214,7 @@ def cmd_repair(args): _rebuild_collection_via_temp, check_extraction_safety, index_read_recovery_guidance, + maybe_autoheal_fts5_index, maybe_repair_poisoned_max_seq_id_before_rebuild, print_sqlite_integrity_abort, sqlite_integrity_errors, @@ -5289,6 +5305,8 @@ def cmd_repair(args): # here so we can surface the clear recovery instructions and exit # cleanly before chromadb's compactor touches the disk. sqlite_errors = sqlite_integrity_errors(palace_path) + if sqlite_errors: + sqlite_errors = maybe_autoheal_fts5_index(palace_path, sqlite_errors) if sqlite_errors: print_sqlite_integrity_abort(palace_path, sqlite_errors) sys.exit(1) @@ -5453,6 +5471,154 @@ def cmd_mcp(args): print(f" {base_server_cmd} --palace /path/to/palace") +_SERVER_LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1", "[::1]"} +_SERVER_BIND_ALL_HOSTS = {"0.0.0.0", "::", "[::]"} + + +def _server_is_loopback(host: str) -> bool: + return (host or "").strip().lower() in _SERVER_LOOPBACK_HOSTS + + +def _server_token_path(palace_path: str) -> Path: + """Per-palace location for the auto-generated server bearer token. + + Distinct from the daemon's token dir; keyed by the canonical palace path so + one server per palace reuses a stable token across restarts. + """ + import hashlib + + canonical = os.path.abspath(os.path.realpath(os.path.expanduser(palace_path))) + key = hashlib.sha256(os.path.normcase(canonical).encode("utf-8")).hexdigest()[:24] + return Path.home() / ".mempalace" / "server" / key / "token" + + +def _load_or_create_server_token(palace_path: str) -> tuple[str, bool]: + """Return (token, created). Reuse an existing 0600 token or mint a new one.""" + import secrets + + token_path = _server_token_path(palace_path) + if token_path.exists(): + existing = token_path.read_text(encoding="utf-8").strip() + if existing: + return existing, False + token = secrets.token_urlsafe(32) + token_path.parent.mkdir(parents=True, exist_ok=True) + try: + os.chmod(str(token_path.parent), 0o700) + except OSError: + pass + # O_CREAT with 0600 so the token is never briefly world-readable on disk. + fd = os.open(str(token_path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as fh: + fh.write(token + "\n") + return token, True + + +def cmd_serve(args): + """Run a secure remote HTTP MCP server for a team to share one palace (#1877). + + A turnkey wrapper over ``mempalace-mcp --transport http``: it resolves a + bearer token (auto-generating a strong one for non-loopback binds), prints a + ready-to-paste client config, then execs the real server in the foreground so + Docker/systemd own the process lifecycle. The token is passed via the + environment, never argv, so it can't leak through ``ps``. + """ + host = args.host + port = int(args.port) + loopback = _server_is_loopback(host) + palace_path = ( + os.path.abspath(os.path.expanduser(args.palace)) + if args.palace + else MempalaceConfig().palace_path + ) + backend = _backend_arg(args) + + tls_cert = os.path.expanduser(args.tls_cert) if args.tls_cert else None + tls_key = os.path.expanduser(args.tls_key) if args.tls_key else None + if bool(tls_cert) != bool(tls_key): + print("mempalace: --tls-cert and --tls-key must be given together", file=sys.stderr) + sys.exit(2) + for label, path in (("--tls-cert", tls_cert), ("--tls-key", tls_key)): + if path and not os.path.isfile(path): + print(f"mempalace: {label} file not found: {path}", file=sys.stderr) + sys.exit(2) + scheme = "https" if tls_cert else "http" + + # Token resolution. Explicit flag > existing env > (non-loopback) auto-generated. + token = (args.token or os.environ.get("MEMPALACE_MCP_HTTP_TOKEN", "")).strip() + token_created = False + if not token and not loopback and not args.allow_insecure: + token, token_created = _load_or_create_server_token(palace_path) + + # Build the child environment. Token rides in the env (never argv) so it + # stays out of the process table. + env = dict(os.environ) + env["MEMPALACE_PALACE_PATH"] = palace_path + if backend: + env["MEMPALACE_BACKEND"] = str(backend).strip().lower() + if token: + env["MEMPALACE_MCP_HTTP_TOKEN"] = token + if args.allow_insecure: + env["MEMPALACE_MCP_HTTP_ALLOW_INSECURE_NO_TOKEN"] = "1" + + child = [ + sys.executable, + "-m", + "mempalace.mcp_server", + "--transport", + "http", + "--host", + host, + "--port", + str(port), + ] + if backend: + child += ["--backend", str(backend).strip().lower()] + child += ["--palace", palace_path] + if tls_cert: + child += ["--tls-cert", tls_cert, "--tls-key", tls_key] + if args.read_only: + child.append("--read-only") + + # Client-facing address: 0.0.0.0/:: means "all interfaces" — clients dial a + # real reachable host, so show a placeholder rather than the bind wildcard. + client_host = "YOUR_SERVER_HOST" if host.strip().lower() in _SERVER_BIND_ALL_HOSTS else host + url = f"{scheme}://{client_host}:{port}/mcp" + + print("Starting MemPalace remote MCP server") + print(f" palace : {palace_path}") + print(f" backend : {(backend or 'default').strip().lower() if backend else 'default'}") + print(f" bind : {host}:{port} ({'loopback' if loopback else 'network-exposed'})") + print(f" tls : {'on' if tls_cert else 'off (plaintext — terminate TLS at a proxy)'}") + print(f" read-only: {'yes' if args.read_only else 'no'}") + if token_created: + print("\n A new bearer token was generated and stored 0600 at:") + print(f" {_server_token_path(palace_path)}") + print(" Store it securely — clients need it to connect:") + print(f" {token}") + print("\nConnect a client:") + if token: + print( + f" claude mcp add --transport http mempalace {url} " + f'--header "Authorization: Bearer {token if token_created else "$MEMPALACE_MCP_HTTP_TOKEN"}"' + ) + else: + print(f" claude mcp add --transport http mempalace {url}") + print(f" curl {scheme}://{client_host}:{port}/healthz # liveness (no auth)\n") + sys.stdout.flush() + + # Foreground: hand the process to the real server so signals (SIGTERM from + # Docker/systemd) reach it directly. exec on POSIX; subprocess on Windows + # (no exec semantics) propagating the exit code. + if os.name == "posix": + os.execve(sys.executable, child, env) + else: + import subprocess + + completed = subprocess.run(child, env=env) + sys.exit(completed.returncode) + + def cmd_compress(args): """Compress drawers in a wing using AAAK Dialect.""" from .dialect import Dialect @@ -6198,19 +6364,6 @@ def main(): p_repair.add_argument( "--yes", action="store_true", help="Skip confirmation for destructive changes" ) - p_repair.add_argument( - "--mode", - choices=["rebuild", "legacy", "max-seq-id", "from-sqlite"], - default="legacy", - help=( - "rebuild/legacy: full-palace HNSW rebuild via extract + re-upsert (default; " - "rebuild and legacy are synonyms). " - "max-seq-id: un-poison max_seq_id rows corrupted by the legacy 0.6.x shim. " - "from-sqlite: rebuild by reading rows directly from chroma.sqlite3, " - "bypassing the chromadb client. Use when legacy mode bails because the " - "chromadb client cannot open the collection." - ), - ) p_repair.add_argument( "repair_action", nargs="?", @@ -6230,6 +6383,19 @@ def main(): "the palace really contains that count." ), ) + p_repair.add_argument( + "--mode", + choices=["rebuild", "legacy", "max-seq-id", "from-sqlite"], + default="legacy", + help=( + "rebuild/legacy: full-palace HNSW rebuild via extract + re-upsert (default; " + "rebuild and legacy are synonyms). " + "max-seq-id: un-poison max_seq_id rows corrupted by the legacy 0.6.x shim. " + "from-sqlite: rebuild by reading rows directly from chroma.sqlite3, " + "bypassing the chromadb client. Use when legacy mode bails because the " + "chromadb client cannot open the collection." + ), + ) p_repair.add_argument( "--source", default=None, @@ -6310,6 +6476,38 @@ def main(): help="Storage backend to include in the MCP startup command", ) + # serve — turnkey remote HTTP MCP server (#1877) + p_serve = sub.add_parser( + "serve", + help="Run a secure remote HTTP MCP server for a team to share one palace", + ) + p_serve.add_argument( + "--host", default="127.0.0.1", help="Bind address (use 0.0.0.0 for remote clients)" + ) + p_serve.add_argument("--port", type=int, default=8765, help="Bind port (default: 8765)") + p_serve.add_argument( + "--backend", default=None, help="Storage backend (default: config/env/detected)" + ) + p_serve.add_argument("--palace", default=None, help="Palace path (overrides config/env)") + p_serve.add_argument( + "--token", + default=None, + help="Bearer token clients must present. Default: reuse/auto-generate one for " + "non-loopback binds (stored 0600 under ~/.mempalace/server/).", + ) + p_serve.add_argument("--tls-cert", default=None, help="PEM certificate to enable TLS") + p_serve.add_argument("--tls-key", default=None, help="PEM private key matching --tls-cert") + p_serve.add_argument( + "--read-only", + action="store_true", + help="Expose recall only: mutating tools are hidden and refused", + ) + p_serve.add_argument( + "--allow-insecure", + action="store_true", + help="Permit a non-loopback bind with no token (only behind a trusted proxy)", + ) + # status # migrate p_migrate = sub.add_parser( @@ -6435,6 +6633,9 @@ def main(): ) p_migrate_wings.add_argument("--yes", action="store_true", help="Skip the confirmation prompt") + p_hallways = sub.add_parser("hallways", help="List entity hallways (associative graph)") + p_hallways.add_argument("--wing", default=None, help="Filter to one wing") + p_hallways.add_argument("--limit", type=int, default=50, help="Max hallways to show") p_status = sub.add_parser("status", help="Show what's been filed") p_status.add_argument( "--backend", @@ -6757,6 +6958,7 @@ def _nonneg_int(value: str) -> int: "sweep": cmd_sweep, "sync": cmd_sync, "mcp": cmd_mcp, + "serve": cmd_serve, "compress": cmd_compress, "wake-up": cmd_wakeup, "repair": cmd_repair, @@ -6768,6 +6970,7 @@ def _nonneg_int(value: str) -> int: "rename-wing": cmd_rename_wing, "rooms": cmd_rooms, "migrate-wings": cmd_migrate_wings, + "hallways": cmd_hallways, "status": cmd_status, "stats": cmd_stats, "tags": cmd_tags, diff --git a/mempalace/convo_miner.py b/mempalace/convo_miner.py index 2092c39499..e9238cfbda 100644 --- a/mempalace/convo_miner.py +++ b/mempalace/convo_miner.py @@ -10,6 +10,7 @@ import os import sys +import json import logging import stat from pathlib import Path @@ -20,6 +21,7 @@ from .collision_scan import assert_no_collisions from .ids import ID_RECIPE, make_convo_drawer_id, make_convo_sentinel_id from .normalize import normalize +from .entities import entities_metadata from .palace import ( NORMALIZE_VERSION, SKIP_DIRS, @@ -479,7 +481,42 @@ def scan_convos(convo_dir: str) -> list: # ============================================================================= -def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extract_mode): +def _extract_authored_at(filepath): + """Most-recent message timestamp in a transcript, used as the drawer's authored date. + + Both Claude Code and Codex JSONL transcripts carry a top-level ISO-8601 + ``timestamp`` on each line. We take the max so ``authored_at`` reflects when the + content was actually written, independent of when it was mined (``filed_at``). + This restores chronology: a session from days ago keeps its real date even when + re-mined today, instead of every drawer collapsing to ingest time. Returns None + for formats without per-line timestamps (e.g. plain ``.md``). + """ + path = Path(filepath) + if path.suffix != ".jsonl": + return None + latest = None + try: + with path.open(encoding="utf-8", errors="ignore") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + ts = json.loads(line).get("timestamp") + except (ValueError, TypeError, AttributeError): + continue + # ISO-8601 timestamps are strings; guard against a non-string + # ``timestamp`` so a malformed line can't raise TypeError on compare. + if isinstance(ts, str) and (latest is None or ts > latest): + latest = ts + except OSError: + return None + return latest + + +def _file_chunks_locked( + collection, source_file, chunks, wing, room, agent, extract_mode, authored_at=None +): """Lock the source file, purge stale drawers, and upsert fresh chunks. Combines the per-file serialization that prevents concurrent agents from @@ -543,6 +580,8 @@ def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extr "chunk_index": chunk["chunk_index"], "added_by": agent, "filed_at": filed_at, + "entities": entities_metadata(chunk["content"]), + "authored_at": authored_at if authored_at is not None else filed_at, "ingest_mode": "convos", "extract_mode": extract_mode, "normalize_version": NORMALIZE_VERSION, @@ -859,6 +898,22 @@ def mine_convos( ) +def _compute_hallways_for_wing_safe(wing, collection, drawers_filed): + """Auto-populate the associative graph from the entities just mined. + + Best-effort: hallway computation must never fail an otherwise-good mine, and is + skipped when nothing new was filed. + """ + if drawers_filed <= 0: + return + try: + from .hallways import compute_hallways_for_wing + + compute_hallways_for_wing(wing, col=collection) + except Exception as exc: + print(f" (hallways skipped: {exc})") + + def _mine_convos_impl( convo_dir: str, palace_path: str, @@ -997,7 +1052,14 @@ def _mine_convos_impl( # Lock + purge stale + file fresh chunks. Lock serializes concurrent # agents; purge removes pre-v2 drawers so the schema bump applies. drawers_added, room_delta, skipped = _file_chunks_locked( - collection, source_file, chunks, wing, room, agent, extract_mode + collection, + source_file, + chunks, + wing, + room, + agent, + extract_mode, + authored_at=_extract_authored_at(filepath), ) if skipped: files_skipped += 1 @@ -1012,6 +1074,10 @@ def _mine_convos_impl( break if not dry_run: + # Compute hallways before the FTS5 validation: the latter opens a direct sqlite + # connection to the Chroma DB, which can invalidate the live collection handle on + # some Chroma builds and make the hallway fetch fail. + _compute_hallways_for_wing_safe(wing, collection, total_drawers) _validate_palace_fts5_after_mine(palace_path) print(f"\n{'=' * 55}") diff --git a/mempalace/entities.py b/mempalace/entities.py new file mode 100644 index 0000000000..d2e5975e06 --- /dev/null +++ b/mempalace/entities.py @@ -0,0 +1,71 @@ +"""No-LLM structural entity extraction for the associative graph. + +Pulls deterministic, *structural* tokens from text — author-quoted code spans, URLs, +file paths, qualified identifiers, and CamelCase symbols — to populate the ``entities`` +drawer-metadata field that hallways/tunnels consume. Structural-only by design: no +wordlists, no NLP models, no domain vocabulary, so it stays language-neutral and +predictable, and biases to precision (only tokens that are unambiguously "a thing being +referred to") over recall. + +The output format matches what ``hallways._parse_entities`` expects: a ``;``-joined string. +""" + +import re + +# Author-quoted code spans are the highest-signal structural marker: `foo`, `obj.method()`. +_BACKTICK = re.compile(r"`([^`\n]{2,64})`") +# URLs. +_URL = re.compile(r"https?://[^\s)>\]}\"']+") +# Paths with a separator and a short extension: rag/foo.py, a/b/c.tsx. +_PATH = re.compile(r"\b[\w.-]+/[\w./-]*\.[A-Za-z][A-Za-z0-9]{0,4}\b") +# Qualified dotted identifiers; each segment starts with a letter and is >=2 chars, so +# "1.2.3", "e.g", and "i.e" are excluded: module.func, pkg.Class.method. +_QUALIFIED = re.compile(r"\b[A-Za-z][A-Za-z0-9_]+(?:\.[A-Za-z][A-Za-z0-9_]+)+\b") +# CamelCase with >=2 humps — strongly code-specific: ChromaBackend, MemoryStack. +_CAMEL = re.compile(r"\b[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]+)+\b") +# snake_case (must contain an underscore, so it can't match plain English): do_thing, +# _extract_authored_at. The optional leading/trailing `_?` matches dunder-style names +# whose underscore would otherwise fall outside the `\b` boundary (`_` is a word char). +_SNAKE = re.compile(r"\b_?[a-z][a-z0-9]*(?:_[a-z0-9]+)+_?\b") + +_MAX_ENTITIES = 24 +_MIN_LEN = 2 +_MAX_LEN = 64 + + +def _clean(token): + # `;` is the entities-metadata separator, so it must never survive inside an entity + # (e.g. a URL query string or a backtick span) or it would split the field. + return token.replace(";", " ").strip().strip("`.,:()[]{}<>\"'").strip() + + +def extract_structural_entities(text, max_entities=_MAX_ENTITIES): + """Return up to ``max_entities`` structural entities from ``text``. + + Deterministic and order-stable: entities are ranked by occurrence count (ties broken + by first appearance), deduplicated case-insensitively, preserving the first-seen + surface form. + """ + if not text: + return [] + counts = {} + order = {} + seq = 0 + for pattern in (_BACKTICK, _URL, _PATH, _QUALIFIED, _CAMEL, _SNAKE): + for match in pattern.finditer(text): + token = _clean(match.group(1) if pattern is _BACKTICK else match.group(0)) + if not (_MIN_LEN <= len(token) <= _MAX_LEN): + continue + key = token.lower() + if key not in counts: + counts[key] = 0 + order[key] = (seq, token) + seq += 1 + counts[key] += 1 + ranked = sorted(order, key=lambda k: (-counts[k], order[k][0])) + return [order[k][1] for k in ranked[:max_entities]] + + +def entities_metadata(text, max_entities=_MAX_ENTITIES): + """``;``-joined entity string for drawer metadata, or ``""`` when none are found.""" + return ";".join(extract_structural_entities(text, max_entities=max_entities)) diff --git a/mempalace/entity_detector.py b/mempalace/entity_detector.py index d79509c0b2..85b54e8fec 100644 --- a/mempalace/entity_detector.py +++ b/mempalace/entity_detector.py @@ -215,6 +215,8 @@ def _get_stopwords(languages: tuple) -> frozenset: ".md", ".rst", ".csv", + ".tex", + ".bib", } READABLE_EXTENSIONS = { diff --git a/mempalace/layers.py b/mempalace/layers.py index c4261fc27e..7cf8b522e9 100644 --- a/mempalace/layers.py +++ b/mempalace/layers.py @@ -352,6 +352,9 @@ def search(self, query: str, wing: str = None, room: str = None, n_results: int lines.append(f" {snippet}") if source: lines.append(f" src: {source}") + authored = (meta.get("authored_at") or "")[:10] + if authored: + lines.append(f" authored: {authored}") return "\n".join(lines) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 5f74fa4e31..aa598911cc 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -102,63 +102,127 @@ from .ids import ID_RECIPE, make_drawer_id_from_content # noqa: E402 -def _init_logging() -> None: - """Root-logger init: always stderr, optionally append to ``MEMPALACE_LOG_FILE``. +class _MempalaceLogFilter(logging.Filter): + """Pass only records emitted by mempalace's own loggers. + + Lets the ``MEMPALACE_LOG_FILE`` handler attach to an already-configured + root logger (a host app embedding the server, #1860) without copying the + host's — or a third-party library's — records into mempalace's diagnostic + file. mempalace loggers are ``mempalace`` / ``mempalace.*`` (the dotted + ``__name__`` family) plus the flat ``mempalace_mcp`` / + ``mempalace_format_miner`` / ``mempalace_hallways`` / ``mempalace_graph`` + loggers — every one is prefixed ``mempalace``. + """ + + def filter(self, record: logging.LogRecord) -> bool: + name = record.name + return name == "mempalace" or name.startswith(("mempalace.", "mempalace_")) + + +# Preserved across importlib.reload via globals(): a reload re-executes this +# module body, so a plain ``= False`` would reset the guard and let +# _init_logging() stack a duplicate file handler. globals().get keeps the prior +# True so the guard survives reload (#1885 review). +_logging_configured = globals().get("_logging_configured", False) + - Stderr-only is the default. When ``MEMPALACE_LOG_FILE`` is set, a - ``FileHandler`` is attached so MCP-client failures that the client - does not surface (e.g. the ``-32000`` cold-load timeout in #1495) - remain diagnosable from the file. +def _init_logging() -> None: + """Configure mempalace logging: stderr by default, optional file append. + + ``MEMPALACE_LOG_FILE``, when set, attaches a ``FileHandler`` so MCP-client + failures the client never surfaces (e.g. the ``-32000`` cold-load timeout + in #1495) stay diagnosable from the file. + + Root-logger ownership (#1860). The server must not hijack a host + application's logging, so the two cases are handled differently: + + * **Root unconfigured** (standalone ``mempalace-mcp``): own it — a stderr + handler (plus the optional file handler) via ``basicConfig`` at INFO. + The historical behaviour. + * **Root already configured** (an app imported ``mempalace.mcp_server`` + after setting up its own logging): leave the host's level, format, and + handlers untouched. Attach only the file handler, filtered to + mempalace's own records (`_MempalaceLogFilter`), so the host's logs do + not bleed into mempalace's file. With ``MEMPALACE_LOG_FILE`` unset the + root logger is not touched at all. + + Previously this called ``logging.basicConfig(..., force=True)``, which + reset root's handlers/level/format unconditionally and silently clobbered + any host app that had configured logging first (#1860). ``force`` existed + (#1495) only to stop ``basicConfig`` no-op'ing when handlers already + existed; the filtered additive handler preserves that diagnostic contract + without the collateral reset. + + The file handler is mempalace-filtered in both paths, so the file is a + clean mempalace-only stream. In the embedded path mempalace's records are + still subject to the host's root level — a host wanting INFO diagnostics in + the file should not raise root above INFO. The standalone path pins INFO. Failure modes: - * Invalid path (missing directory, no perms, Windows NUL byte) → - stderr-only with a warning. The env var must not become a new - server-start failure surface — that would defeat the diagnostic - goal. ``ValueError`` is included in the catch because Windows - raises it for paths with embedded NUL bytes, not ``OSError``. - * Root logger already configured (host app embedding the server, - transitive imports touching ``logging``) → ``force=True`` resets - the handlers so MEMPALACE_LOG_FILE's contract holds regardless - of what touched root logging first. Without ``force=True``, - ``basicConfig`` is a no-op when handlers exist and the env var - silently does nothing — exactly the diagnostic black hole #1495 - exists to close. - * Concurrent writers (multiple ``mempalace-mcp`` processes pointing - at the same path) interleave at the line level. The handler uses - append mode so nothing is overwritten, but operators running - Claude Code + Claude Desktop simultaneously should give each - process its own log path. - - ``delay=True`` is intentionally NOT set: deferring the open means an - invalid path raises at ``emit()`` time (unhandled), defeating the - fail-soft contract. With eager open the same error surfaces inside - ``FileHandler.__init__`` and lands in our ``except`` below. - - Module-level invocation: this function runs at import time, preserving - the side effect of the previous module-level ``logging.basicConfig`` - call. Callers that import ``mempalace.mcp_server`` for introspection - (``TOOLS`` dict, handler functions) inherit the reset; this matches - pre-PR behaviour and is intentional for an MCP entry-point module. + * Invalid path (missing directory, no perms, Windows NUL byte) → the file + handler is skipped with a warning naming ``MEMPALACE_LOG_FILE``; the + server still starts. ``ValueError`` is in the catch because Windows + raises it for embedded-NUL paths, not ``OSError``. + * Concurrent writers (multiple ``mempalace-mcp`` processes at one path) + interleave at the line level; append mode means nothing is overwritten, + but give each process its own path. + + ``delay=True`` is intentionally NOT set: deferring the open moves an + invalid-path error to ``emit()`` time (unhandled), defeating the fail-soft + contract. Eager open lands the same error in ``FileHandler.__init__`` and + our ``except`` below. + + Runs at import time (module-level call below) so importing the module for + introspection (``TOOLS`` dict, handler functions) configures logging once. """ - handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)] + global _logging_configured + if _logging_configured: + # Idempotent: a second call (e.g. importlib.reload) must not add a + # duplicate file handler in the embedded path. + return + _logging_configured = True + # MEMPALACE_LOG_FILE is operator-supplied and opt-in; this is a # local-first server (CLAUDE.md design principle), so no path # sanitization — the operator's process UID is the trust boundary. log_file = os.environ.get("MEMPALACE_LOG_FILE", "").strip() + file_handler: logging.Handler | None = None file_handler_error: Exception | None = None if log_file: try: - handlers.append(logging.FileHandler(log_file, mode="a", encoding="utf-8")) + file_handler = logging.FileHandler(log_file, mode="a", encoding="utf-8") + # Pin the format: the embedded path never calls basicConfig, so set + # it here instead of relying on logging's default formatter. The + # default already renders "%(message)s", but the explicit set makes + # both paths identical and independent of that default (#1885 review). + file_handler.setFormatter(logging.Formatter("%(message)s")) + # File is a mempalace-only diagnostic stream; keep host / library + # records out so it stays useful when the handler rides on a + # host-owned root logger (#1860). + file_handler.addFilter(_MempalaceLogFilter()) except (OSError, ValueError) as exc: # Fail-soft: see "Invalid path" failure mode above. Broad on # (OSError, ValueError) because Windows raises ValueError for # NUL-byte paths while POSIX uses OSError for missing-dir / EPERM. file_handler_error = exc - logging.basicConfig(level=logging.INFO, format="%(message)s", handlers=handlers, force=True) + + root = logging.getLogger() + if root.handlers: + # A host app (or a transitive import) already owns root logging. Do + # NOT reset it (#1860) — only add our filtered file handler, if any. + if file_handler is not None: + root.addHandler(file_handler) + else: + # Standalone server: own the unconfigured root logger as before. + handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)] + if file_handler is not None: + handlers.append(file_handler) + logging.basicConfig(level=logging.INFO, format="%(message)s", handlers=handlers) + if file_handler_error is not None: logging.getLogger("mempalace_mcp").warning( - "MEMPALACE_LOG_FILE=%r could not be opened (%s); using stderr only", + "MEMPALACE_LOG_FILE=%r could not be opened (%s); file logging disabled", log_file, file_handler_error, ) @@ -212,6 +276,23 @@ def _parse_args(): default=8765, help="HTTP port to bind when --transport=http (default: 8765)", ) + parser.add_argument( + "--tls-cert", + metavar="PATH", + help="PEM certificate to terminate TLS on the HTTP transport " + "(requires --tls-key; env MEMPALACE_MCP_TLS_CERT)", + ) + parser.add_argument( + "--tls-key", + metavar="PATH", + help="PEM private key matching --tls-cert (env MEMPALACE_MCP_TLS_KEY)", + ) + parser.add_argument( + "--read-only", + action="store_true", + help="Serve a read-only tool surface: the mutating tools are hidden from " + "tools/list and refused at dispatch (env MEMPALACE_MCP_READ_ONLY)", + ) args, unknown = parser.parse_known_args() if unknown: logger.debug("Ignoring unknown args: %s", unknown) @@ -232,6 +313,14 @@ def _parse_args(): _config = MempalaceConfig() +# Read-only server mode: when on, the mutating tools are hidden from tools/list +# and refused at dispatch (-32003). Resolved once at startup from --read-only or +# MEMPALACE_MCP_READ_ONLY. Computed inline (not via _truthy_env, defined below) +# so it is available to the request path regardless of import order. +_READ_ONLY = bool(getattr(_args, "read_only", False)) or os.environ.get( + "MEMPALACE_MCP_READ_ONLY", "" +).strip().lower() in {"1", "true", "yes", "on"} + _kg_by_path: dict = {} # KG instance cache; KnowledgeGraph or KnowledgeGraphAGE _kg_cache_lock = threading.Lock() _palace_flag_given: bool = bool(_args.palace) @@ -1442,6 +1531,15 @@ def _fetch_all_metadata(col, where=None): return all_meta +def _supports_metadata_facets(col) -> bool: + """Return True if the collection's backend implements metadata facets.""" + backend = getattr(col, "_backend", None) + if backend is None: + return False + capabilities = getattr(backend, "capabilities", None) + return isinstance(capabilities, (set, frozenset)) and "supports_metadata_facets" in capabilities + + _metadata_cache = None _metadata_cache_time = 0 _METADATA_CACHE_TTL = 5.0 # seconds @@ -1553,6 +1651,77 @@ def _sanitize_optional_source_file(value: str = None) -> str: return value +def _parse_date_filter(value: Optional[str] = None, field_name: str = "date") -> Optional[datetime]: + """Parse an optional ISO-8601 date/datetime filter bound (#1128). + + Accepts a date (``"2026-04-01"``), a naive timestamp + (``"2026-04-01T09:30:00"``), or one carrying a ``Z``/``+HH:MM`` offset. + Returns a naive ``datetime`` for wall-clock + comparison against drawer ``filed_at`` values, which are stored as naive + local ISO strings (``datetime.now().isoformat()``). Any timezone offset on + the input is dropped so an aware bound never raises a ``TypeError`` against + a naive ``filed_at``. Comparison is therefore wall-clock, which is what the + local-first single-machine model wants; an offset bound is matched on its + wall-clock fields, not its absolute instant, so a bound whose offset differs + from the zone ``filed_at`` was recorded in is matched by clock time. + The accepted grammar is a date, an ISO timestamp (optionally fractional), + and an optional ``Z``/``±HH:MM`` offset; other ISO 8601 forms (basic format, + week dates) are outside the contract and are rejected on the Python 3.9 floor + even where a newer ``fromisoformat`` would accept them. + Blank / whitespace-only means "no filter" (``None``). + Raises ``ValueError`` on an unparseable value so the caller can surface a + clear error, mirroring the wing/room sanitizers. + """ + if value is None: + return None + if not isinstance(value, str): + raise ValueError(f"{field_name} must be an ISO date string") + value = value.strip() + if not value: + return None + # datetime.fromisoformat before Python 3.11 rejects a trailing "Z" (Zulu), + # and appending "+00:00" would break a date-only value on 3.9/3.10 + # ("2026-04-01+00:00" is rejected there). Any offset is dropped below for + # wall-clock comparison anyway, so just strip a trailing Z/z; both date and + # date-time Zulu inputs then parse on the 3.9 floor. + iso = value[:-1] if value.endswith(("Z", "z")) else value + try: + parsed = datetime.fromisoformat(iso) + except ValueError as exc: + raise ValueError( + f"{field_name} must be an ISO date string " + f"(e.g. '2026-04-01' or '2026-04-01T09:30:00'), got {value!r}" + ) from exc + if parsed.tzinfo is not None: + parsed = parsed.replace(tzinfo=None) + return parsed + + +def _filed_at_in_window( + filed_at, since_dt: Optional[datetime], before_dt: Optional[datetime] +) -> bool: + """True if a drawer's ``filed_at`` falls in ``[since, before)`` (#1128). + + ``since`` is inclusive and ``before`` is exclusive, matching the issue spec. + Parsing (``Z``/offset normalization, tz drop) is delegated to + ``_parse_date_filter`` so a bound and a ``filed_at`` are compared + identically. A drawer whose ``filed_at`` is missing or unparseable cannot + be confirmed in-window, so it is EXCLUDED whenever a bound is active — a + date-filtered listing must never silently include rows of unknown age. + """ + try: + filed_dt = _parse_date_filter(filed_at, "filed_at") + except ValueError: + return False + if filed_dt is None: + return False + if since_dt is not None and filed_dt < since_dt: + return False + if before_dt is not None and filed_dt >= before_dt: + return False + return True + + # ==================== READ TOOLS ==================== @@ -1918,13 +2087,47 @@ def tool_status(): "backend": _selected_backend_name(), } try: - all_meta = _get_cached_metadata(col) - for m in all_meta: - m = m or {} - w = m.get("wing", "unknown") - r = m.get("room", "unknown") - wings[w] = wings.get(w, 0) + 1 - rooms[r] = rooms.get(r, 0) + 1 + if _supports_metadata_facets(col): + try: + temp_wings = col.facet_counts("wing") + wings.update(temp_wings) + try: + unknown_wings = count - sum(temp_wings.values()) + if unknown_wings > 0: + wings["unknown"] = wings.get("unknown", 0) + unknown_wings + except (TypeError, ValueError): + pass + + temp_rooms = col.facet_counts("room") + rooms.update(temp_rooms) + try: + unknown_rooms = count - sum(temp_rooms.values()) + if unknown_rooms > 0: + rooms["unknown"] = rooms.get("unknown", 0) + unknown_rooms + except (TypeError, ValueError): + pass + + except Exception as e: + logger.warning( + "Failed to fetch metadata facets, falling back to client-side loop: %s", e + ) + rooms.clear() + wings.clear() + all_meta = _get_cached_metadata(col) + for m in all_meta: + m = m or {} + w = m.get("wing", "unknown") + r = m.get("room", "unknown") + wings[w] = wings.get(w, 0) + 1 + rooms[r] = rooms.get(r, 0) + 1 + else: + all_meta = _get_cached_metadata(col) + for m in all_meta: + m = m or {} + w = m.get("wing", "unknown") + r = m.get("room", "unknown") + wings[w] = wings.get(w, 0) + 1 + rooms[r] = rooms.get(r, 0) + 1 except Exception as e: logger.exception("tool_status metadata fetch failed") result["error"] = str(e) @@ -1979,11 +2182,28 @@ def tool_list_wings(): wings = {} result = {"wings": wings} try: - all_meta = _get_cached_metadata(col) - for m in all_meta: - m = m or {} - w = m.get("wing", "unknown") - wings[w] = wings.get(w, 0) + 1 + try: + if not _supports_metadata_facets(col): + raise ValueError("facets not supported") + temp_wings = col.facet_counts("wing") + wings.update(temp_wings) + try: + unknown_wings = col.count() - sum(temp_wings.values()) + if unknown_wings > 0: + wings["unknown"] = wings.get("unknown", 0) + unknown_wings + except (TypeError, ValueError): + pass + except Exception as e: + if _supports_metadata_facets(col): + logger.warning( + "Failed to fetch metadata facets, falling back to client-side loop: %s", e + ) + wings.clear() + all_meta = _get_cached_metadata(col) + for m in all_meta: + m = m or {} + w = m.get("wing", "unknown") + wings[w] = wings.get(w, 0) + 1 except Exception as e: logger.exception("tool_list_wings metadata fetch failed") result["error"] = str(e) @@ -2011,13 +2231,34 @@ def tool_list_rooms(wing: str = None): return _collection_error_or_no_palace() rooms = {} result = {"wing": wing or "all", "rooms": rooms} + where = {"wing": wing} if wing else None try: - where = {"wing": wing} if wing else None - all_meta = _fetch_all_metadata(col, where=where) - for m in all_meta: - m = m or {} - r = m.get("room", "unknown") - rooms[r] = rooms.get(r, 0) + 1 + try: + if not _supports_metadata_facets(col): + raise ValueError("facets not supported") + temp_rooms = col.facet_counts("room", where=where) + rooms.update(temp_rooms) + try: + if wing: + wing_count = col.facet_counts("wing", where={"wing": wing}).get(wing, 0) + unknown_rooms = wing_count - sum(temp_rooms.values()) + else: + unknown_rooms = col.count() - sum(temp_rooms.values()) + if unknown_rooms > 0: + rooms["unknown"] = rooms.get("unknown", 0) + unknown_rooms + except (TypeError, ValueError): + pass + except Exception as e: + if _supports_metadata_facets(col): + logger.warning( + "Failed to fetch metadata facets, falling back to client-side loop: %s", e + ) + rooms.clear() + all_meta = _fetch_all_metadata(col, where=where) + for m in all_meta: + m = m or {} + r = m.get("room", "unknown") + rooms[r] = rooms.get(r, 0) + 1 except Exception as e: logger.exception("tool_list_rooms metadata fetch failed") result["error"] = str(e) @@ -2036,14 +2277,42 @@ def tool_get_taxonomy(): taxonomy = {} result = {"taxonomy": taxonomy} try: - all_meta = _get_cached_metadata(col) - for m in all_meta: - m = m or {} - w = m.get("wing", "unknown") - r = m.get("room", "unknown") - if w not in taxonomy: - taxonomy[w] = {} - taxonomy[w][r] = taxonomy[w].get(r, 0) + 1 + try: + if not _supports_metadata_facets(col): + raise ValueError("facets not supported") + from concurrent.futures import ThreadPoolExecutor + + wing_counts = col.facet_counts("wing") + wings = list(wing_counts.keys()) + temp_taxonomy = {} + with ThreadPoolExecutor(max_workers=max(1, min(8, len(wings)))) as executor: + futures = { + wing: executor.submit(col.facet_counts, "room", where={"wing": wing}) + for wing in wings + } + for wing, future in futures.items(): + room_counts = future.result() + try: + unknown_rooms = wing_counts[wing] - sum(room_counts.values()) + if unknown_rooms > 0: + room_counts["unknown"] = room_counts.get("unknown", 0) + unknown_rooms + except (TypeError, ValueError): + pass + temp_taxonomy[wing] = room_counts + taxonomy.update(temp_taxonomy) + except Exception as e: + if _supports_metadata_facets(col): + logger.warning( + "Failed to fetch metadata facets, falling back to client-side loop: %s", e + ) + all_meta = _get_cached_metadata(col) + for m in all_meta: + m = m or {} + w = m.get("wing", "unknown") + r = m.get("room", "unknown") + if w not in taxonomy: + taxonomy[w] = {} + taxonomy[w][r] = taxonomy[w].get(r, 0) + 1 except Exception as e: logger.exception("tool_get_taxonomy metadata fetch failed") result["error"] = str(e) @@ -3415,11 +3684,22 @@ def tool_get_drawer(drawer_id: str): def tool_list_drawers( wing: str = None, room: str = None, + since: str = None, + before: str = None, tags: list = None, limit: int = 20, offset: int = 0, ): - """List logical drawers with pagination. Optional wing/room/tag filter.""" + """List logical drawers with pagination. Optional wing/room/tag filter. + + Optional ``since`` / ``before`` filter by drawer ``filed_at`` (ISO date or + timestamp): ``since`` is inclusive, ``before`` is exclusive (#1128). A + drawer whose ``filed_at`` is missing or unparseable is excluded while a + date bound is active. The filter is applied in Python after the rows are + fetched — ChromaDB rejects string operands for ``$gte``/``$lt`` (1.5.7), + and ``filed_at`` is stored as an ISO string, so a server-side ``where`` + comparison is not available. + """ from .tags import extract_tags_from_metadata, normalise_tags limit = max(1, min(limit, _MAX_RESULTS)) @@ -3428,6 +3708,10 @@ def tool_list_drawers( try: wing = _sanitize_optional_name(wing, "wing") room = _resolve_room_alias(room) + since_dt = _parse_date_filter(since, "since") + before_dt = _parse_date_filter(before, "before") + if since_dt is not None and before_dt is not None and since_dt >= before_dt: + raise ValueError(f"since ({since!r}) must be earlier than before ({before!r})") except ValueError as e: return {"error": str(e)} normalised_tags = normalise_tags(tags) if tags else [] @@ -3452,6 +3736,14 @@ def tool_list_drawers( ids, documents, metadatas = _fetch_drawer_rows(col, where=where) drawers = _collapse_drawer_rows(ids, documents, metadatas) + + if since_dt is not None or before_dt is not None: + drawers = [ + d + for d in drawers + if _filed_at_in_window(d.get("metadata", {}).get("filed_at"), since_dt, before_dt) + ] + page = drawers[offset : offset + limit] # Surface the tag list on each entry without dropping upstream's # logical-drawer shape (wing/room/content_preview/metadata). @@ -3465,6 +3757,7 @@ def tool_list_drawers( "limit": limit, } except Exception as e: + logger.exception("tool_list_drawers failed") return {"error": str(e)} @@ -5171,7 +5464,7 @@ def tool_checkpoint(items, diary=None, dedup_threshold=0.9): "handler": tool_get_drawer, }, "mempalace_list_drawers": { - "description": "List drawers with pagination. Optional wing/room/tag filter. Returns IDs, wings, rooms, tags, content previews, and total matching count for pagination.", + "description": "List drawers with pagination. Optional wing/room/tag filter and since/before date filter on filed_at (since inclusive, before exclusive; drawers without a parseable filed_at are excluded when a date bound is set). Returns IDs, wings, rooms, tags, content previews, and total matching count for pagination.", "input_schema": { "type": "object", "properties": { @@ -5182,6 +5475,14 @@ def tool_checkpoint(items, diary=None, dedup_threshold=0.9): "items": {"type": "string"}, "description": "Only list drawers carrying ALL of these tags (optional)", }, + "since": { + "type": "string", + "description": "Only drawers filed on or after this ISO date/time, inclusive (e.g. '2026-04-01'). Optional.", + }, + "before": { + "type": "string", + "description": "Only drawers filed before this ISO date/time, exclusive (e.g. '2026-05-01'). Optional.", + }, "limit": { "type": "integer", "description": "Max results per page (default 20, max 100)", @@ -5291,7 +5592,10 @@ def tool_checkpoint(items, diary=None, dedup_threshold=0.9): "description": "Alias for 'entry' — accepted because add_drawer uses 'content'. Provide either 'entry' or 'content'; 'entry' wins if both are given.", }, }, - "required": ["agent_name", "entry"], + # 'entry' (or its alias 'content') is enforced at dispatch, not via a + # top-level anyOf: Anthropic rejects schemas with a top-level + # anyOf/oneOf/allOf and drops the whole tools array (400). + "required": ["agent_name"], }, "handler": tool_diary_write, }, @@ -5380,9 +5684,36 @@ def _internal_tool_error(req_id, tool_name: str, exc: BaseException = None) -> d } +def _mcp_read_only_refusal(req_id, tool_name: str): + """Refuse mutating tools when the server runs in read-only mode (#1877). + + Read-only is an operator-set server mode (``--read-only`` / + ``MEMPALACE_MCP_READ_ONLY``), distinct from the dynamic peer-writer lock: + it is an unconditional gate so a shared team server can expose recall + without write access. Enforced at dispatch, not merely hidden from + tools/list, so a client that calls a mutating tool by name is still refused. + """ + if not _READ_ONLY or tool_name not in _MUTATING_TOOLS: + return None + + return { + "jsonrpc": "2.0", + "id": req_id, + "error": { + "code": -32003, + "message": "Server is in read-only mode; this tool is disabled", + "data": {"tool": tool_name}, + }, + } + + def _mcp_tool_preflight_refusal(req_id, tool_name: str): """Run MCP request preflight gates outside handle_request complexity.""" + read_only_error = _mcp_read_only_refusal(req_id, tool_name) + if read_only_error is not None: + return read_only_error + sqlite_integrity_error = _mcp_sqlite_integrity_refusal(req_id, tool_name) if sqlite_integrity_error is not None: return sqlite_integrity_error @@ -5445,6 +5776,8 @@ def handle_request(request): # noqa: C901 — merged fork+upstream tool dispatc # Notifications (no id) never get a response per JSON-RPC spec return None elif method == "tools/list": + # In read-only mode, hide the mutating tools so clients don't advertise + # write capabilities they can't use (dispatch also refuses them, #1877). return { "jsonrpc": "2.0", "id": req_id, @@ -5452,6 +5785,7 @@ def handle_request(request): # noqa: C901 — merged fork+upstream tool dispatc "tools": [ {"name": n, "description": t["description"], "inputSchema": t["input_schema"]} for n, t in TOOLS.items() + if not (_READ_ONLY and n in _MUTATING_TOOLS) ] }, } @@ -5832,6 +6166,37 @@ def _json_rpc_parse_error(req_id=None): _HTTP_ALLOW_INSECURE_NO_TOKEN_ENV = "MEMPALACE_MCP_HTTP_ALLOW_INSECURE_NO_TOKEN" +def _resolve_tls_paths() -> tuple: + """Resolve the TLS cert/key from --tls-cert/--tls-key or env, or (None, None). + + Flags take precedence over ``MEMPALACE_MCP_TLS_CERT`` / ``MEMPALACE_MCP_TLS_KEY``. + Both must be given together; one without the other is a configuration error + (raised here, before any bind, so it fails loudly at startup). + """ + cert = ( + getattr(_args, "tls_cert", None) or os.environ.get("MEMPALACE_MCP_TLS_CERT", "") + ).strip() + key = (getattr(_args, "tls_key", None) or os.environ.get("MEMPALACE_MCP_TLS_KEY", "")).strip() + if bool(cert) != bool(key): + raise ValueError("TLS requires both --tls-cert and --tls-key (or the matching env vars)") + if not cert: + return None, None + for label, path in (("--tls-cert", cert), ("--tls-key", key)): + if not os.path.isfile(path): + raise ValueError(f"{label} file not found: {path!r}") + return cert, key + + +def _wrap_tls(sock, cert: str, key: str): + """Wrap a server socket in a TLS 1.2+ context. Raises on bad cert/key.""" + import ssl + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 + ctx.load_cert_chain(certfile=cert, keyfile=key) + return ctx.wrap_socket(sock, server_side=True) + + def _http_is_loopback(host: str) -> bool: """Whether ``host`` binds only to this machine.""" return (host or "").strip().lower() in _HTTP_LOOPBACK_HOSTS @@ -5897,6 +6262,11 @@ def _build_http_server(host: str, port: int): "when a trusted fronting layer provides access control." ) + # Resolve TLS before bind so a bad cert/key fails loudly rather than at the + # first request. TLS is transport encryption only — the bearer-token guard + # above still applies on a non-loopback bind. + tls_cert, tls_key = _resolve_tls_paths() + class _MCPHTTPServer(ThreadingHTTPServer): daemon_threads = True allow_reuse_address = True @@ -6019,6 +6389,10 @@ def do_POST(self): httpd.enforce_host_pin = _http_is_loopback(host) httpd.allowed_hosts = _http_allowed_host_values(host, bound_port) httpd.auth_token = auth_token + httpd.scheme = "http" + if tls_cert: + httpd.socket = _wrap_tls(httpd.socket, tls_cert, tls_key) + httpd.scheme = "https" return httpd @@ -6052,7 +6426,14 @@ def _serve_http(host: str, port: int) -> None: _HTTP_ALLOW_INSECURE_NO_TOKEN_ENV, ) with httpd: - logger.info("MemPalace MCP HTTP server listening on http://%s:%s/mcp", host, bound_port) + logger.info( + "MemPalace MCP HTTP server listening on %s://%s:%s/mcp%s%s", + getattr(httpd, "scheme", "http"), + host, + bound_port, + " (TLS)" if getattr(httpd, "scheme", "http") == "https" else "", + " (read-only)" if _READ_ONLY else "", + ) try: httpd.serve_forever(poll_interval=0.5) except KeyboardInterrupt: diff --git a/mempalace/miner.py b/mempalace/miner.py index 39c246f987..98ffa9ae33 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -133,6 +133,8 @@ def _read_text_no_follow(filepath: Path, root: Path) -> Optional[str]: ".csv", ".sql", ".toml", + ".tex", + ".bib", # C# / .NET ".cs", ".csproj", diff --git a/mempalace/palace.py b/mempalace/palace.py index ebdb83dedf..6e0893555a 100644 --- a/mempalace/palace.py +++ b/mempalace/palace.py @@ -1059,45 +1059,79 @@ def _validate_palace_fts5_after_mine(palace_path: str) -> None: raise MineValidationError(palace_path, errors) -# Per-thread record of palaces this thread already holds the lock for. Used by -# `mine_palace_lock` to short-circuit re-entrant acquisition from the same -# thread (e.g. miner.mine() acquires the outer lock then calls +# Process-wide record of palaces this PROCESS already holds the lock for. Used +# by `mine_palace_lock` to short-circuit re-entrant acquisition from the same +# process (e.g. miner.mine() acquires the outer lock then calls # ChromaCollection.upsert which now also tries to acquire). Without this guard # the inner call would block on its own outer flock (Linux fcntl locks are per -# open file description, so a same-thread second open of the lock file is a -# distinct lock and self-deadlocks). +# open file description, so a second open of the lock file from the same process +# is a distinct lock and self-conflicts / EWOULDBLOCKs). # -# The holder set is tagged with ``pid`` so that a forked child does NOT -# inherit re-entrant credit from its parent: the OS-level flock IS NOT -# inherited as a "we hold it" semantically — the child must reacquire — but -# Python's ``threading.local`` IS inherited across fork. The pid check -# clears stale state so a forked child correctly hits the fcntl path. -_palace_lock_holders = threading.local() - - -def _holder_state(): - """Return the per-thread (pid, keys) record, refreshing after fork.""" - keys = getattr(_palace_lock_holders, "keys", None) - pid = getattr(_palace_lock_holders, "pid", None) +# This MUST be process-wide, not thread-local: the MCP HTTP transport +# (ThreadingHTTPServer) acquires the long-lived writer-lease on one thread +# (`mcp_server._acquire_mcp_writer_lock`) but dispatches each write request on a +# different worker thread. A thread-local guard makes those handlers fail to see +# the process-held lease, re-acquire the flock, and self-conflict +# ("palace ... is held by PID "). flock is per-process and HTTP writes are +# serialized by `_HTTP_REQUEST_LOCK`, so the process is the correct re-entrancy +# boundary. +# +# The holder set is tagged with ``pid`` so that a forked child does NOT inherit +# re-entrant credit from its parent: the OS-level flock IS NOT inherited as a +# "we hold it" semantically — the child must reacquire. The pid check clears +# stale state so a forked child correctly hits the fcntl path. Access is guarded +# by ``_palace_lock_guard`` because the set is now shared across threads. +# +# Fork safety: ``_palace_lock_guard`` is a real ``threading.Lock``, so a child +# forked while another thread held it would inherit it locked (the holder thread +# does not exist in the child) and deadlock on the next acquire. An at-fork +# handler (registered below) replaces the guard with a fresh unlocked lock and +# clears state in the child, which must reacquire the flock anyway. +_palace_lock_guard = threading.Lock() +_palace_lock_pid = None +_palace_lock_keys = set() + + +def _reset_palace_lock_state_after_fork() -> None: + """Reset lock state in a forked child to avoid an inherited-locked deadlock.""" + global _palace_lock_guard, _palace_lock_pid, _palace_lock_keys + _palace_lock_guard = threading.Lock() + _palace_lock_keys = set() + _palace_lock_pid = os.getpid() + + +# Availability: Unix (no-op elsewhere — Windows has no fork()). +if hasattr(os, "register_at_fork"): + os.register_at_fork(after_in_child=_reset_palace_lock_state_after_fork) + + +def _holder_keys_locked(): + """Return the process-wide held-key set, refreshing after fork. + + Caller MUST hold ``_palace_lock_guard``. + """ + global _palace_lock_pid, _palace_lock_keys current_pid = os.getpid() - if keys is None or pid != current_pid: - keys = set() - _palace_lock_holders.keys = keys - _palace_lock_holders.pid = current_pid - return keys + if _palace_lock_pid != current_pid: + _palace_lock_keys = set() + _palace_lock_pid = current_pid + return _palace_lock_keys -def _held_by_this_thread(lock_key: str) -> bool: - """Return True if this thread already holds ``mine_palace_lock`` for ``lock_key``.""" - return lock_key in _holder_state() +def _held_by_this_process(lock_key: str) -> bool: + """Return True if this process already holds ``mine_palace_lock`` for ``lock_key``.""" + with _palace_lock_guard: + return lock_key in _holder_keys_locked() def _mark_held(lock_key: str) -> None: - _holder_state().add(lock_key) + with _palace_lock_guard: + _holder_keys_locked().add(lock_key) def _mark_released(lock_key: str) -> None: - _holder_state().discard(lock_key) + with _palace_lock_guard: + _holder_keys_locked().discard(lock_key) def _format_lock_holder(content: str) -> str: @@ -1176,11 +1210,13 @@ def mine_palace_lock(palace_path: str): raise MineAlreadyRunning so the caller can exit cleanly instead of piling up as a waiting worker. - Re-entrant: if the current thread already holds the lock for the same + Re-entrant: if the current process already holds the lock for the same palace, the context manager passes through without re-acquiring. This lets ChromaCollection write methods (which acquire the lock themselves to protect MCP/direct callers) compose with miner.mine() (which holds - the outer lock for the entire mine pipeline) without self-deadlock. + the outer lock for the entire mine pipeline) without self-deadlock, and + lets the threaded MCP HTTP transport write from a worker thread while the + long-lived writer-lease is held on another thread of the same process. """ lock_dir = os.path.join(os.path.expanduser("~"), ".mempalace", "locks") os.makedirs(lock_dir, exist_ok=True) @@ -1189,8 +1225,8 @@ def mine_palace_lock(palace_path: str): palace_key = hashlib.sha256(lock_key_source.encode()).hexdigest()[:16] lock_path = os.path.join(lock_dir, f"mine_palace_{palace_key}.lock") - if _held_by_this_thread(palace_key): - # Same thread already holds the lock for this palace — pass through. + if _held_by_this_process(palace_key): + # This process already holds the lock for this palace — pass through. yield return diff --git a/mempalace/repair.py b/mempalace/repair.py index 173d0a1658..09a545286d 100644 --- a/mempalace/repair.py +++ b/mempalace/repair.py @@ -682,6 +682,82 @@ def print_sqlite_integrity_abort(palace_path: str, errors: list[str]) -> None: print(" 6. Re-run `mempalace repair --yes`.") +# quick_check labels a corrupt FTS5 inverted index like: +# "malformed inverted index for FTS5 table main.embedding_fulltext_search" +# That specific failure is recoverable in place: the index is derived from the +# intact ``embedding_fulltext_search_content`` shadow table, so rebuilding it +# restores full-text search without touching any drawer rows. Concurrent +# killed-mid-write mines are the usual cause (#1596). +_FTS5_MALFORMED_RE = re.compile(r"malformed inverted index for FTS5 table", re.IGNORECASE) + + +def _errors_are_isolated_fts5(errors: list[str]) -> bool: + """True when every quick_check error is a malformed FTS5 inverted index. + + Only an isolated FTS5 failure is safe to auto-heal: the inverted index is + derived data that ``rebuild`` regenerates from the content shadow table. If + quick_check also reports page/row corruption, the data itself may be damaged + and rebuilding the index over it would mask real loss — that still aborts. + """ + return bool(errors) and all(_FTS5_MALFORMED_RE.search(e) for e in errors) + + +def maybe_autoheal_fts5_index(palace_path: str, errors: list[str], *, progress=print) -> list[str]: + """Rebuild a malformed FTS5 inverted index in place; return remaining errors. + + The repair preflight aborts when ``PRAGMA quick_check`` reports SQLite-layer + corruption. After concurrent killed-mid-write mines (#1596) the common + failure is an isolated ``malformed inverted index for FTS5 table``, which is + fully recoverable: the index rebuilds from the intact + ``embedding_fulltext_search_content`` table without touching drawer rows. + + When the errors are isolated to FTS5, rebuild the index under the palace + write lock (so a live mine cannot race the rebuild) and re-run quick_check. + Returns the remaining quick_check errors — empty when the heal succeeded. + Broader corruption, a lock held by another writer, or a rebuild failure + leaves ``errors`` unchanged so the caller still aborts with the banner. + """ + if not _errors_are_isolated_fts5(errors): + return errors + + sqlite_path = os.path.join(palace_path, "chroma.sqlite3") + if not os.path.exists(sqlite_path): + return errors + + # Lazy import: palace.py is heavier and importing it at module load would + # widen repair.py's import graph for callers that never hit this path. + from .palace import MineAlreadyRunning, mine_palace_lock + + progress( + "\n Isolated FTS5 inverted-index corruption detected; attempting an\n" + " in-place rebuild from the intact content table before aborting." + ) + try: + with mine_palace_lock(palace_path): + with closing(sqlite3.connect(sqlite_path, isolation_level=None)) as conn: + conn.execute( + "INSERT INTO embedding_fulltext_search" + "(embedding_fulltext_search) VALUES('rebuild')" + ) + conn.commit() + except MineAlreadyRunning as exc: + progress( + f" Skipped FTS5 rebuild: palace is being written by another process ({exc}). " + "Stop it and re-run." + ) + return errors + except Exception as exc: + progress(f" FTS5 rebuild failed (leaving palace untouched): {exc}") + return errors + + remaining = sqlite_integrity_errors(palace_path) + if remaining: + progress(" FTS5 rebuild did not clear quick_check; aborting for safety.") + else: + progress(" FTS5 index rebuilt from intact content; quick_check is clean.") + return remaining + + def index_read_recovery_guidance() -> str: """Recovery guidance for a failed drawer-index read in the legacy paths. @@ -927,6 +1003,8 @@ def rebuild_index( # corruption here lets us surface the clear recovery instructions and # exit cleanly before chromadb's compactor touches the disk. sqlite_errors = sqlite_integrity_errors(palace_path) + if sqlite_errors: + sqlite_errors = maybe_autoheal_fts5_index(palace_path, sqlite_errors, progress=progress) if sqlite_errors: print_sqlite_integrity_abort(palace_path, sqlite_errors) return diff --git a/mempalace/searcher.py b/mempalace/searcher.py index 8e706bb683..3b1d93fcd1 100644 --- a/mempalace/searcher.py +++ b/mempalace/searcher.py @@ -252,7 +252,18 @@ def _hybrid_rank( r["bm25_score"] = round(raw, 3) scored.append((vector_weight * vec_sim + bm25_weight * effective_norm, r)) - scored.sort(key=lambda pair: pair[0], reverse=True) + # Break exact score ties toward the more recently authored drawer so equal-score + # candidates rank chronologically instead of in arbitrary backend order. ISO-8601 + # ``authored_at`` strings sort chronologically; missing dates sort oldest. + # authored_at lives at the top level on the search_memories path and nested under + # "metadata" on the candidate-union path; check both so the tie-break works for each. + scored.sort( + key=lambda pair: ( + pair[0], + pair[1].get("authored_at") or pair[1].get("metadata", {}).get("authored_at") or "", + ), + reverse=True, + ) results[:] = [r for _, r in scored] return results @@ -1098,6 +1109,7 @@ def _metadata_filter_sql(row_id_expr: str) -> tuple[str, list[str]]: "source_file": Path(full_source).name if full_source else "?", "source_path": full_source, "created_at": meta.get("filed_at", "unknown"), + "authored_at": meta.get("authored_at", meta.get("filed_at", "unknown")), # No vector distance available in BM25-only mode. "similarity": None, "distance": None, @@ -1720,33 +1732,36 @@ def _merge_bm25_union_candidates( n_results=n_results * 3, where=where or None, ) - bm25_extra = [] - for hit in lexical.hits: - meta = hit.metadata or {} - full_source = meta.get("source_file", "") or "" - bm25_extra.append( - { - "text": hit.document or "", - "wing": meta.get("wing", "unknown"), - "room": meta.get("room", "unknown"), - "source_file": Path(full_source).name if full_source else "?", - "created_at": meta.get("filed_at", "unknown"), - "similarity": None, - "distance": None, - "effective_distance": None, - "closet_boost": 0.0, - "matched_via": "bm25_backend", - "bm25_score": round(float(hit.score), 3), - "_source_file_full": full_source, - "_chunk_index": meta.get("chunk_index"), - } - ) except UnsupportedCapabilityError: raise except Exception: logger.debug("candidate_strategy=union: lexical fetch failed", exc_info=True) return + bm25_extra = [] + for hit in lexical.hits: + meta = hit.metadata or {} + full_source = meta.get("source_file", "") or "" + bm25_extra.append( + { + "text": hit.document or "", + "wing": meta.get("wing", "unknown"), + "room": meta.get("room", "unknown"), + "source_file": Path(full_source).name if full_source else "?", + "source_path": full_source, + "created_at": meta.get("filed_at", "unknown"), + "authored_at": meta.get("authored_at", meta.get("filed_at", "unknown")), + "similarity": None, + "distance": None, + "effective_distance": None, + "closet_boost": 0.0, + "matched_via": "bm25_backend", + "bm25_score": round(float(hit.score), 3), + "_source_file_full": full_source, + "_chunk_index": meta.get("chunk_index"), + } + ) + def _dedup_key(entry: dict): full = entry.get("_source_file_full") ci = entry.get("_chunk_index") @@ -2467,6 +2482,7 @@ def search_memories( # noqa: C901 — fork-only fallback orchestration; complex "source_file": Path(source).name if source else "?", "source_path": source, "created_at": meta.get("filed_at", "unknown"), + "authored_at": meta.get("authored_at", meta.get("filed_at", "unknown")), "similarity": round(_distance_to_similarity(effective_dist, metric), 3), "distance": round(dist, 4), "effective_distance": round(effective_dist, 4), diff --git a/pyproject.toml b/pyproject.toml index 5bc0358da0..bc5ad283cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,7 +96,7 @@ dev = [ # (wired only into the test-windows job in ci.yml, scoped to the specific # error). Local/Linux/macOS runs never rerun, so real failures stay loud. "pytest-rerunfailures>=12.0", - "ruff==0.15.18", + "ruff==0.15.20", "psutil>=5.9", "hypothesis>=6.0", "pre-commit>=3.0", @@ -161,7 +161,7 @@ dev = [ # (wired only into the test-windows job in ci.yml, scoped to the specific # error). Local/Linux/macOS runs never rerun, so real failures stay loud. "pytest-rerunfailures>=12.0", - "ruff==0.15.18", + "ruff==0.15.20", "psutil>=5.9", "hypothesis>=6.0", "pre-commit>=3.0", diff --git a/scripts/backfill_authored_at.py b/scripts/backfill_authored_at.py new file mode 100644 index 0000000000..e3b1db2c84 --- /dev/null +++ b/scripts/backfill_authored_at.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Backfill ``authored_at`` onto existing conversation drawers. + +New mines stamp ``authored_at`` automatically (see ``convo_miner._extract_authored_at``), +but drawers mined before that change only have ``filed_at`` (ingest time). Re-mining does +NOT fix them: the scanner skips files already mined at the current ``NORMALIZE_VERSION``. + +This migration updates the affected drawers IN PLACE — metadata only, embeddings are left +untouched, so there is no re-embedding cost. It is idempotent (drawers already correct are +skipped) and safe to re-run. It only touches ``ingest_mode == "convos"`` drawers; markdown +drawers have no per-line timestamps and keep their ``filed_at`` fallback. + +Drawers whose source transcript is no longer on disk are left as-is (they keep falling back +to ``filed_at``), so point ``--sessions`` at the directories that still hold your ``.jsonl`` +transcripts (e.g. ``~/.claude`` and ``~/.codex``). + +Usage (dry-run prints what would change; pass --apply to write): + + python scripts/backfill_authored_at.py \ + --palace ~/.mempalace/palace \ + --sessions ~/.claude --sessions ~/.codex [--apply] + +In Docker (the MCP image), mount the volume and your session dirs read-only: + + docker run --rm \ + -v mempalace-data:/data \ + -v ~/.claude:/sessions/claude:ro -v ~/.codex:/sessions/codex:ro \ + -v "$PWD/scripts/backfill_authored_at.py:/tmp/backfill.py:ro" \ + --entrypoint /app/.venv/bin/python mempalace:local \ + /tmp/backfill.py --palace /data/.mempalace/palace \ + --sessions /sessions/claude --sessions /sessions/codex --apply +""" + +import argparse +import glob +import os + +import chromadb + +from mempalace.convo_miner import _extract_authored_at + +COLLECTION = "mempalace_drawers" +PAGE = 2000 +BATCH = 1000 + + +def _index_sessions(session_dirs): + """Map ``basename.jsonl -> realpath`` for every transcript under the given dirs.""" + index = {} + for root in session_dirs: + for f in glob.glob(os.path.join(os.path.expanduser(root), "**", "*.jsonl"), recursive=True): + index.setdefault(os.path.basename(f), f) + return index + + +def backfill_authored_at(collection, session_dirs, apply=False): + """Stamp ``authored_at`` on convos drawers from their source transcript timestamps. + + Returns a stats dict: ``scanned``, ``updated``, ``resolved_files``, ``unresolved_files``. + """ + index = _index_sessions(session_dirs) + cache = {} + unresolved = set() + pending_ids, pending_metas = [], [] + scanned = updated = 0 + + def flush(): + nonlocal pending_ids, pending_metas, updated + if pending_ids and apply: + collection.update(ids=pending_ids, metadatas=pending_metas) + updated += len(pending_ids) + pending_ids, pending_metas = [], [] + + offset = 0 + while True: + res = collection.get( + where={"ingest_mode": "convos"}, include=["metadatas"], limit=PAGE, offset=offset + ) + ids = res["ids"] + if not ids: + break + for drawer_id, meta in zip(ids, res["metadatas"]): + scanned += 1 + basename = os.path.basename(meta.get("source_file") or "") + if basename in cache: + authored = cache[basename] + else: + path = index.get(basename) + authored = _extract_authored_at(path) if path else None + cache[basename] = authored + if path is None and basename: + unresolved.add(basename) + if authored and meta.get("authored_at") != authored: + new_meta = dict(meta) + new_meta["authored_at"] = authored + pending_ids.append(drawer_id) + pending_metas.append(new_meta) + if len(pending_ids) >= BATCH: + flush() + offset += len(ids) + flush() + return { + "scanned": scanned, + "updated": updated, + "resolved_files": sum(1 for v in cache.values() if v), + "unresolved_files": len(unresolved), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--palace", required=True, help="Path to the ChromaDB palace dir") + parser.add_argument( + "--sessions", + action="append", + default=[], + required=True, + help="Directory holding .jsonl transcripts (repeatable)", + ) + parser.add_argument( + "--apply", + action="store_true", + help="Write changes (default is a dry run that only reports counts)", + ) + args = parser.parse_args() + + client = chromadb.PersistentClient(path=os.path.expanduser(args.palace)) + collection = client.get_collection(COLLECTION) + stats = backfill_authored_at(collection, args.sessions, apply=args.apply) + mode = "APPLIED" if args.apply else "DRY-RUN (use --apply to write)" + print( + f"{mode}: scanned={stats['scanned']} updated={stats['updated']} " + f"resolved_files={stats['resolved_files']} unresolved_files={stats['unresolved_files']}" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/_chroma_palace_helper.py b/tests/_chroma_palace_helper.py new file mode 100644 index 0000000000..b55806e055 --- /dev/null +++ b/tests/_chroma_palace_helper.py @@ -0,0 +1,57 @@ +"""Shared helpers: create minimal valid palace-marker SQLite files for tests. + +Many tests want to stand up "a chroma / sqlite_exact palace" cheaply — +historically they did this with ``(path / ".sqlite3").touch()`` or +``write_bytes(b"")``, relying on the backend ``detect()`` methods' old +``os.path.isfile()`` semantics. Post-#1893, both ``ChromaBackend.detect()`` +and ``SQLiteExactBackend.detect()`` require a valid SQLite magic header, so +the empty stand-in no longer registers. These helpers create the minimum +required to make detection fire without standing up a full palace. + +This module is intentionally not a ``test_*`` file: it ships utilities, not +tests. +""" + +import sqlite3 +from pathlib import Path +from typing import Union + + +def _write_minimal_sqlite_file(db_path: Path) -> None: + """Write a valid SQLite magic header at ``db_path``. + + Writing any statement is sufficient to land the 16-byte + ``SQLite format 3\\x00`` magic prefix that the backend ``detect()`` + methods check. + """ + + conn = sqlite3.connect(db_path) + try: + conn.execute("CREATE TABLE _detect_smoke(x)") + conn.commit() + finally: + conn.close() + + +def make_minimal_chroma_sqlite(palace_path: Union[Path, str]) -> Path: + """Create ``/chroma.sqlite3`` with a valid SQLite header. + + Returns the path to the file. Backs + :py:meth:`mempalace.backends.chroma.ChromaBackend.detect`. + """ + + db_path = Path(palace_path) / "chroma.sqlite3" + _write_minimal_sqlite_file(db_path) + return db_path + + +def make_minimal_sqlite_exact_sqlite(palace_path: Union[Path, str]) -> Path: + """Create ``/sqlite_exact.sqlite3`` with a valid SQLite header. + + Returns the path to the file. Backs + :py:meth:`mempalace.backends.sqlite_exact.SQLiteExactBackend.detect`. + """ + + db_path = Path(palace_path) / "sqlite_exact.sqlite3" + _write_minimal_sqlite_file(db_path) + return db_path diff --git a/tests/conftest.py b/tests/conftest.py index ed4a2485ed..29d9fec656 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -89,6 +89,17 @@ def _clear_cache(): if hasattr(mcp_server, "_kg_by_path"): mcp_server._kg_by_path.clear() + # Close (not just dereference) the cached chromadb client so its + # rust-side file handles are released; on Windows a bare deref + # leaves them locked and leaks across the session (#1128). + cached_client = getattr(mcp_server, "_client_cache", None) + if cached_client is not None: + close = getattr(cached_client, "close", None) + if callable(close): + try: + close() + except Exception: + pass mcp_server._client_cache = None mcp_server._collection_cache = None if hasattr(mcp_server, "_collection_cache_backend"): @@ -141,11 +152,40 @@ def _clear_cache(): except (ImportError, AttributeError): pass + _close_backend_palace_clients() + _clear_cache() yield _clear_cache() +def _close_backend_palace_clients(): + """Release chromadb clients opened through the backend layer. + + Many tests reach the store via palace.get_collection() (sweep, repair, + CLI, ...), which caches one PersistentClient per palace_path on the + long-lived backend singleton and never closes it. chromadb frees the + rust-side SQLite/HNSW file handles only on client.close(); on POSIX the + open handles are harmless, but on Windows they stay locked and accumulate + across the session until a later test's HNSW segment write fails + (#1128 Windows CI). close_palace() closes the client and drops the + handle without marking the backend closed, so it stays reusable. + """ + try: + from mempalace import palace as _palace + + backend = getattr(_palace, "_DEFAULT_BACKEND", None) + clients = getattr(backend, "_clients", None) + if clients: + for path in list(clients): + try: + backend.close_palace(path) + except Exception: + pass + except (ImportError, AttributeError): + pass + + @pytest.fixture(scope="session", autouse=True) def _isolate_home(): """Ensure HOME points to a temp dir for the entire test session. @@ -198,7 +238,11 @@ def collection(palace_path): col = client.get_or_create_collection("mempalace_drawers", metadata={"hnsw:space": "cosine"}) yield col client.delete_collection("mempalace_drawers") - del client + # close() (not a bare dereference) releases chromadb's rust-side SQLite/HNSW + # file handles. On Windows a mere `del` leaves them locked, so the temp + # palace cannot be removed and handles leak across the whole test session + # until a later test's HNSW write fails (#1128 Windows CI). + client.close() @pytest.fixture diff --git a/tests/test_backends.py b/tests/test_backends.py index bb50def85b..2130c0f6b6 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -625,12 +625,44 @@ def get_collection(self, *, palace, collection_name, create, options=None): assert calls[0][3] == {"dsn": "postgresql://example"} -def test_chroma_detect_matches_palace_with_chroma_sqlite(tmp_path): - (tmp_path / "chroma.sqlite3").write_bytes(b"") +def test_chroma_detect_matches_palace_with_sqlite_header(tmp_path): + """A real SQLite database at ``/chroma.sqlite3`` registers as chroma. + + Uses ``sqlite3.connect`` + a write so the SQLite magic header is actually + on disk — the only thing detection looks at. + """ + db_path = tmp_path / "chroma.sqlite3" + conn = sqlite3.connect(db_path) + conn.execute("CREATE TABLE detect_smoke(x)") + conn.commit() + conn.close() assert ChromaBackend.detect(str(tmp_path)) is True assert ChromaBackend.detect(str(tmp_path.parent)) is False +def test_chroma_detect_rejects_empty_chroma_sqlite(tmp_path): + """A 0-byte ``chroma.sqlite3`` is not a chroma palace (closes #1893). + + Bare ``sqlite3.connect()`` against a missing path leaves a 0-byte file + behind because the SQLite header is written on the first statement, not + on connect. Detection must reject that artifact so it cannot trip + ``BackendMismatchError`` against a real non-chroma backend marker in the + same directory. + """ + (tmp_path / "chroma.sqlite3").write_bytes(b"") + assert ChromaBackend.detect(str(tmp_path)) is False + + +def test_chroma_detect_rejects_non_sqlite_file(tmp_path): + """A non-SQLite file at the ``chroma.sqlite3`` path is not chroma. + + Defends against partial writes / garbage content / anything that lands at + the canonical path but isn't actually a SQLite database. + """ + (tmp_path / "chroma.sqlite3").write_bytes(b"not a sqlite file" * 4) + assert ChromaBackend.detect(str(tmp_path)) is False + + def test_chroma_lexical_search_uses_sqlite_fts_not_full_collection_scan(tmp_path): db_path = tmp_path / "chroma.sqlite3" conn = sqlite3.connect(db_path) diff --git a/tests/test_backfill_authored_at.py b/tests/test_backfill_authored_at.py new file mode 100644 index 0000000000..10d0855f64 --- /dev/null +++ b/tests/test_backfill_authored_at.py @@ -0,0 +1,86 @@ +"""Integration tests for the authored_at backfill migration (scripts/).""" + +import importlib.util +import uuid +from pathlib import Path + +import chromadb + +# The migration ships as a script, not a package module; load it directly. +_SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "backfill_authored_at.py" +_spec = importlib.util.spec_from_file_location("backfill_authored_at", _SCRIPT) +backfill_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(backfill_mod) + + +def _collection(): + # Unique name per call: EphemeralClient shares one in-memory instance across the + # process, so a fixed collection name would leak drawers between tests. + client = chromadb.EphemeralClient() + return client.create_collection(f"drawers_{uuid.uuid4().hex}") + + +def _add(col, drawer_id, source_file, authored_at=None): + meta = {"ingest_mode": "convos", "source_file": source_file, "filed_at": "2026-06-27T00:00:00"} + if authored_at is not None: + meta["authored_at"] = authored_at + col.add(ids=[drawer_id], documents=["hello"], metadatas=[meta], embeddings=[[0.1, 0.2, 0.3]]) + + +def _transcript(dir_path, name, *timestamps): + dir_path.mkdir(parents=True, exist_ok=True) + f = dir_path / name + f.write_text("".join(f'{{"timestamp": "{ts}"}}\n' for ts in timestamps)) + return f + + +def test_backfill_sets_latest_timestamp(tmp_path): + sessions = tmp_path / "claude" + _transcript(sessions, "abc.jsonl", "2026-06-10T08:00:00.000Z", "2026-06-12T09:00:00.000Z") + col = _collection() + # Stored source_file uses an old mount prefix; resolution is by basename. + _add(col, "d1", "/old/mount/abc.jsonl") + + stats = backfill_mod.backfill_authored_at(col, [str(sessions)], apply=True) + + assert stats["scanned"] == 1 + assert stats["updated"] == 1 + got = col.get(ids=["d1"], include=["metadatas"])["metadatas"][0] + assert got["authored_at"] == "2026-06-12T09:00:00.000Z" + + +def test_dry_run_writes_nothing(tmp_path): + sessions = tmp_path / "claude" + _transcript(sessions, "abc.jsonl", "2026-06-12T09:00:00.000Z") + col = _collection() + _add(col, "d1", "/old/mount/abc.jsonl") + + stats = backfill_mod.backfill_authored_at(col, [str(sessions)], apply=False) + + assert stats["updated"] == 1 # would update + assert "authored_at" not in col.get(ids=["d1"], include=["metadatas"])["metadatas"][0] + + +def test_idempotent_second_run_updates_nothing(tmp_path): + sessions = tmp_path / "claude" + _transcript(sessions, "abc.jsonl", "2026-06-12T09:00:00.000Z") + col = _collection() + _add(col, "d1", "/old/mount/abc.jsonl") + + backfill_mod.backfill_authored_at(col, [str(sessions)], apply=True) + stats2 = backfill_mod.backfill_authored_at(col, [str(sessions)], apply=True) + + assert stats2["updated"] == 0 + + +def test_unresolved_transcript_is_left_alone(tmp_path): + sessions = tmp_path / "claude" + sessions.mkdir() + col = _collection() + _add(col, "d1", "/old/mount/missing.jsonl") # no file on disk + + stats = backfill_mod.backfill_authored_at(col, [str(sessions)], apply=True) + + assert stats["updated"] == 0 + assert stats["unresolved_files"] == 1 + assert "authored_at" not in col.get(ids=["d1"], include=["metadatas"])["metadatas"][0] diff --git a/tests/test_cli_hallways.py b/tests/test_cli_hallways.py new file mode 100644 index 0000000000..76739d2852 --- /dev/null +++ b/tests/test_cli_hallways.py @@ -0,0 +1,59 @@ +"""Tests for the `hallways` CLI command.""" + +from argparse import Namespace + +import mempalace.hallways as hallways_mod +from mempalace.cli import cmd_hallways + + +def test_lists_sorted_by_count(monkeypatch, capsys): + rows = [ + { + "entity_a": "C", + "entity_b": "D", + "co_occurrence_count": 1, + "wing": "w", + "label": "C <-> D (x1)", + }, + { + "entity_a": "A", + "entity_b": "B", + "co_occurrence_count": 3, + "wing": "w", + "label": "A <-> B (x3)", + }, + ] + monkeypatch.setattr(hallways_mod, "list_hallways", lambda wing=None: list(rows)) + cmd_hallways(Namespace(wing=None, limit=50)) + out = capsys.readouterr().out + assert "2 hallway(s)" in out + assert "A <-> B (x3)" in out + # Highest co-occurrence first. + assert out.index("A <-> B") < out.index("C <-> D") + + +def test_respects_limit(monkeypatch, capsys): + rows = [ + {"entity_a": f"E{i}", "entity_b": "X", "co_occurrence_count": i, "label": f"E{i} <-> X"} + for i in range(5) + ] + monkeypatch.setattr(hallways_mod, "list_hallways", lambda wing=None: list(rows)) + cmd_hallways(Namespace(wing=None, limit=2)) + assert capsys.readouterr().out.count("<->") == 2 + + +def test_negative_limit_shows_nothing_not_tail(monkeypatch, capsys): + rows = [ + {"entity_a": f"E{i}", "entity_b": "X", "co_occurrence_count": i, "label": f"E{i} <-> X"} + for i in range(5) + ] + monkeypatch.setattr(hallways_mod, "list_hallways", lambda wing=None: list(rows)) + cmd_hallways(Namespace(wing=None, limit=-2)) + # A negative limit must not slice from the end (which would print all-but-2). + assert capsys.readouterr().out.count("<->") == 0 + + +def test_empty_message(monkeypatch, capsys): + monkeypatch.setattr(hallways_mod, "list_hallways", lambda wing=None: []) + cmd_hallways(Namespace(wing="x", limit=50)) + assert "No hallways yet" in capsys.readouterr().out diff --git a/tests/test_convo_miner_unit.py b/tests/test_convo_miner_unit.py index d1fd215c6b..eecb73c95a 100644 --- a/tests/test_convo_miner_unit.py +++ b/tests/test_convo_miner_unit.py @@ -8,6 +8,7 @@ from mempalace.convo_miner import ( CHUNK_SIZE, _emit_bounded, + _extract_authored_at, _file_chunks_locked, chunk_exchanges, detect_convo_room, @@ -487,3 +488,100 @@ def upsert(self, documents, ids, metadatas): assert dict(room_counts) == {} assert skipped is False assert col.batch_sizes == [2, 2, 1] + + def test_populates_entities_metadata(self, monkeypatch): + import mempalace.convo_miner as convo_miner + + class FakeCol: + def __init__(self): + self.metas = [] + + def delete(self, *args, **kwargs): + pass + + def get(self, ids=None, include=None, **kwargs): + return {"ids": [], "metadatas": []} + + def upsert(self, documents, ids, metadatas): + self.metas.extend(metadatas) + + chunks = [ + { + "content": "We changed `MemoryStack` in rag/foo.py via do_thing_now().", + "chunk_index": 0, + } + ] + col = FakeCol() + monkeypatch.setattr( + convo_miner, "file_already_mined", lambda collection, source_file, **kwargs: False + ) + monkeypatch.setattr(convo_miner, "mine_lock", lambda source_file: contextlib.nullcontext()) + monkeypatch.setattr(convo_miner, "_detect_hall_cached", lambda content: "conversations") + + _file_chunks_locked(col, "chat.txt", chunks, "wing", "general", "agent", "exchange") + + entities = col.metas[0]["entities"].split(";") + assert "MemoryStack" in entities + assert "rag/foo.py" in entities + assert "do_thing_now" in entities + + +class TestExtractAuthoredAt: + """authored_at = max per-line ``timestamp`` in a transcript (real authored date, + independent of mine time). Both Claude Code and Codex JSONL carry a top-level + ISO-8601 ``timestamp`` per line.""" + + def test_returns_latest_timestamp(self, tmp_path): + f = tmp_path / "session.jsonl" + f.write_text( + '{"type": "user", "timestamp": "2026-06-21T10:00:00.000Z"}\n' + '{"type": "assistant", "timestamp": "2026-06-23T14:30:00.000Z"}\n' + '{"type": "user", "timestamp": "2026-06-22T09:00:00.000Z"}\n' + ) + assert _extract_authored_at(f) == "2026-06-23T14:30:00.000Z" + + def test_ignores_lines_without_timestamp(self, tmp_path): + f = tmp_path / "session.jsonl" + f.write_text( + '{"type": "summary", "summary": "x"}\n' + '{"type": "assistant", "timestamp": "2026-06-23T14:30:00.000Z"}\n' + ) + assert _extract_authored_at(f) == "2026-06-23T14:30:00.000Z" + + def test_tolerates_blank_and_malformed_lines(self, tmp_path): + f = tmp_path / "session.jsonl" + f.write_text( + "\n" + "not json\n" + "[1, 2, 3]\n" # valid JSON, but no .get() + '{"timestamp": "2026-06-25T00:00:00.000Z"}\n' + ) + assert _extract_authored_at(f) == "2026-06-25T00:00:00.000Z" + + def test_none_for_non_jsonl(self, tmp_path): + f = tmp_path / "notes.md" + f.write_text("# heading\n") + assert _extract_authored_at(f) is None + + def test_none_when_no_timestamps(self, tmp_path): + f = tmp_path / "session.jsonl" + f.write_text('{"type": "user", "content": "hi"}\n') + assert _extract_authored_at(f) is None + + def test_none_for_missing_file(self, tmp_path): + assert _extract_authored_at(tmp_path / "absent.jsonl") is None + + def test_non_string_timestamp_does_not_crash(self, tmp_path): + # A non-string timestamp must be skipped, not raise TypeError on compare. + f = tmp_path / "session.jsonl" + f.write_text( + '{"type": "user", "timestamp": 1234567890}\n' + '{"type": "assistant", "timestamp": {"nested": true}}\n' + '{"type": "user", "timestamp": "2026-06-24T00:00:00.000Z"}\n' + ) + assert _extract_authored_at(f) == "2026-06-24T00:00:00.000Z" + + def test_only_non_string_timestamps_returns_none(self, tmp_path): + f = tmp_path / "session.jsonl" + f.write_text('{"timestamp": 1}\n{"timestamp": false}\n') + assert _extract_authored_at(f) is None diff --git a/tests/test_daemon.py b/tests/test_daemon.py index d8d1127f99..bf5ede3e07 100644 --- a/tests/test_daemon.py +++ b/tests/test_daemon.py @@ -5,6 +5,8 @@ import pytest +from _chroma_palace_helper import make_minimal_chroma_sqlite + from mempalace import daemon from mempalace import service @@ -728,7 +730,7 @@ def test_run_sync_structured_errors_on_sync_failures(tmp_path, monkeypatch): palace = tmp_path / "palace" palace.mkdir() - (palace / "chroma.sqlite3").touch() + make_minimal_chroma_sqlite(palace) def _raise(exc): def fn(**kw): diff --git a/tests/test_entities.py b/tests/test_entities.py new file mode 100644 index 0000000000..c365fb62fb --- /dev/null +++ b/tests/test_entities.py @@ -0,0 +1,69 @@ +"""Tests for no-LLM structural entity extraction.""" + +from mempalace.entities import entities_metadata, extract_structural_entities + + +def test_extracts_code_symbols_paths_urls(): + text = ( + "We patched `_extract_authored_at` in rag/convo_miner.py so MemoryStack and " + "ChromaBackend agree. See module.func and pkg.Class.method, plus do_thing_now. " + "Ref https://github.com/MemPalace/mempalace/pull/1890 for details." + ) + ents = set(extract_structural_entities(text)) + assert "_extract_authored_at" in ents + assert "rag/convo_miner.py" in ents + assert "MemoryStack" in ents + assert "ChromaBackend" in ents + assert "module.func" in ents + assert "pkg.Class.method" in ents + assert "do_thing_now" in ents + assert any(e.startswith("https://github.com/MemPalace") for e in ents) + + +def test_excludes_prose_noise(): + text = "This is a normal sentence, e.g. with i.e. abbreviations and version 1.2.3 here." + ents = extract_structural_entities(text) + # No plain prose words, no "e.g"/"i.e", no bare version numbers. + assert ents == [] + + +def test_ranked_by_frequency_then_order(): + text = "alpha_one alpha_one alpha_one beta_two beta_two gamma_three" + ents = extract_structural_entities(text) + assert ents[:3] == ["alpha_one", "beta_two", "gamma_three"] + + +def test_dedup_case_insensitive_keeps_first_form(): + text = "`MemoryStack` and memorystack and MEMORYSTACK" + ents = extract_structural_entities(text) + assert ents.count("MemoryStack") == 1 + assert ents == ["MemoryStack"] + + +def test_respects_max_entities(): + text = " ".join(f"sym_{i}_x" for i in range(50)) + assert len(extract_structural_entities(text, max_entities=10)) == 10 + + +def test_extracts_leading_underscore_snake_in_plain_text(): + # Not in backticks — must still be caught by the snake-case pattern. + ents = extract_structural_entities("we called _extract_authored_at and _do_thing here") + assert "_extract_authored_at" in ents + assert "_do_thing" in ents + + +def test_semicolon_in_entity_does_not_corrupt_metadata(): + # A backtick span containing ';' must not split the ;-joined metadata field. + md = entities_metadata("see `a(); b()` and TwoThing") + parts = md.split(";") + # Every part is a whole entity — no fragment is a bare separator artifact. + assert all(p.strip() for p in parts) + assert "TwoThing" in parts + + +def test_metadata_is_semicolon_joined(): + text = "`one_thing` and TwoThing" + md = entities_metadata(text) + assert md == "one_thing;TwoThing" + assert entities_metadata("") == "" + assert entities_metadata("just plain prose with nothing structural") == "" diff --git a/tests/test_entity_detector.py b/tests/test_entity_detector.py index cc7483122d..77868f5948 100644 --- a/tests/test_entity_detector.py +++ b/tests/test_entity_detector.py @@ -531,6 +531,24 @@ def test_scan_for_detection_skips_git_dir(tmp_path): assert not any(".git" in f for f in file_strs) +def test_scan_for_detection_includes_latex_prose(tmp_path): + # .tex and .bib are prose-heavy (author names, abstracts, citations) and + # belong in the preferred PROSE_EXTENSIONS bucket alongside .md / .rst, + # not the code-file fallback. .bib in particular is almost entirely + # author names — high entity density per byte. + (tmp_path / "paper.tex").write_text( + "\\documentclass{article}\\author{Leslie Lamport}\\begin{document}Body.\\end{document}" + ) + (tmp_path / "refs.bib").write_text( + "@article{l86, author={Leslie Lamport}, title={LaTeX}, year={1986}}" + ) + (tmp_path / "code.py").write_text("import os") + files = scan_for_detection(str(tmp_path)) + extensions = {os.path.splitext(str(f))[1] for f in files} + assert ".tex" in extensions + assert ".bib" in extensions + + # ── module-level constants ────────────────────────────────────────────── @@ -543,6 +561,8 @@ def test_stopwords_contains_common_words(): def test_prose_extensions(): assert ".txt" in PROSE_EXTENSIONS assert ".md" in PROSE_EXTENSIONS + assert ".tex" in PROSE_EXTENSIONS + assert ".bib" in PROSE_EXTENSIONS # ── _print_entity_list ───────────────────────────────────────────────── diff --git a/tests/test_hybrid_search.py b/tests/test_hybrid_search.py index 35aa579349..98c11aebb7 100644 --- a/tests/test_hybrid_search.py +++ b/tests/test_hybrid_search.py @@ -12,7 +12,7 @@ get_collection, upsert_closet_lines, ) -from mempalace.searcher import search_memories +from mempalace.searcher import _hybrid_rank, search_memories def _seed_drawers(palace_path): @@ -173,3 +173,34 @@ def test_source_file_filter_overrides_closet_boost_for_other_source(self, tmp_pa ids = [h["source_file"] for h in result["results"]] assert "fixture_D1.md" not in ids assert set(ids) <= {"fixture_D4.md"} + + +def test_hybrid_rank_breaks_score_ties_by_authored_at(): + """Identical-content hits get identical vector + BM25 scores; the tie must break + toward the more recently authored drawer, not arbitrary backend order.""" + older = { + "text": "alpha beta gamma", + "distance": 0.2, + "metadata": {"authored_at": "2026-06-21T10:00:00.000Z"}, + } + newer = { + "text": "alpha beta gamma", + "distance": 0.2, + "metadata": {"authored_at": "2026-06-27T10:00:00.000Z"}, + } + # Input order puts the older drawer first; the tiebreak should reorder it. + results = [older, newer] + _hybrid_rank(results, "alpha beta gamma") + assert results[0]["metadata"]["authored_at"] == "2026-06-27T10:00:00.000Z" + assert results[1]["metadata"]["authored_at"] == "2026-06-21T10:00:00.000Z" + + +def test_hybrid_rank_tiebreak_handles_top_level_authored_at(): + """The search_memories path puts authored_at at the top level (no `metadata` + nesting); the tie-break must read it there too.""" + older = {"text": "alpha beta gamma", "distance": 0.2, "authored_at": "2026-06-21T10:00:00.000Z"} + newer = {"text": "alpha beta gamma", "distance": 0.2, "authored_at": "2026-06-27T10:00:00.000Z"} + results = [older, newer] + _hybrid_rank(results, "alpha beta gamma") + assert results[0]["authored_at"] == "2026-06-27T10:00:00.000Z" + assert results[1]["authored_at"] == "2026-06-21T10:00:00.000Z" diff --git a/tests/test_mcp_http_transport.py b/tests/test_mcp_http_transport.py index 82121b752b..80b091582d 100644 --- a/tests/test_mcp_http_transport.py +++ b/tests/test_mcp_http_transport.py @@ -197,6 +197,123 @@ def test_bearer_token_enforced_when_configured(monkeypatch): thread.join(timeout=5) +def test_read_only_hides_and_refuses_mutating_tools(http_server, monkeypatch): + """Read-only mode (#1877): mutating tools are hidden from tools/list AND + refused at dispatch with -32003, while read tools still work.""" + monkeypatch.setattr(mcp, "_READ_ONLY", True) + port, _ = http_server + + status, body = _post(port, "/mcp", {"jsonrpc": "2.0", "id": 1, "method": "tools/list"}) + assert status == 200 + names = {t["name"] for t in json.loads(body)["result"]["tools"]} + assert "mempalace_search" in names # read tool stays + assert "mempalace_add_drawer" not in names # mutating tool hidden + assert names.isdisjoint(mcp._MUTATING_TOOLS) + + status, body = _post( + port, + "/mcp", + { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "mempalace_add_drawer", "arguments": {"content": "x"}}, + }, + ) + assert status == 200 + assert json.loads(body)["error"]["code"] == -32003 + + +def test_read_only_off_exposes_mutating_tools(http_server): + """Sanity: without read-only, mutating tools are present (guards the test above).""" + port, _ = http_server + status, body = _post(port, "/mcp", {"jsonrpc": "2.0", "id": 1, "method": "tools/list"}) + names = {t["name"] for t in json.loads(body)["result"]["tools"]} + assert "mempalace_add_drawer" in names + + +def _make_self_signed_cert(tmp_path): + """Write a throwaway self-signed cert/key via openssl; skip if unavailable.""" + import shutil + import subprocess + + if shutil.which("openssl") is None: + pytest.skip("openssl not available to generate a test certificate") + cert = tmp_path / "cert.pem" + key = tmp_path / "key.pem" + subprocess.run( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-keyout", + str(key), + "-out", + str(cert), + "-days", + "1", + "-nodes", + "-subj", + "/CN=localhost", + ], + check=True, + capture_output=True, + ) + return cert, key + + +def test_tls_serves_https(tmp_path, monkeypatch): + """With --tls-cert/--tls-key (via env), the server speaks TLS: a plain HTTP + client cannot read it, and an HTTPS client trusting the cert can.""" + import ssl + + cert, key = _make_self_signed_cert(tmp_path) + monkeypatch.setenv("MEMPALACE_MCP_TLS_CERT", str(cert)) + monkeypatch.setenv("MEMPALACE_MCP_TLS_KEY", str(key)) + + httpd = mcp._build_http_server("127.0.0.1", 0) + assert getattr(httpd, "scheme", "http") == "https" + port = httpd.server_address[1] + thread = threading.Thread( + target=httpd.serve_forever, kwargs={"poll_interval": 0.05}, daemon=True + ) + thread.start() + try: + # Full verification on: trust the self-signed cert as the CA and dial + # "localhost" (the cert CN, resolves to 127.0.0.1) so hostname checking + # passes without being disabled. + ctx = ssl.create_default_context(cafile=str(cert)) + conn = http.client.HTTPSConnection("localhost", port, context=ctx, timeout=5) + try: + conn.request("GET", "/healthz") + resp = conn.getresponse() + assert resp.status == 200 + assert resp.read() == b"ok\n" + finally: + conn.close() + + # A plaintext HTTP client must NOT be able to talk to the TLS socket. + with pytest.raises(Exception): + plain = http.client.HTTPConnection("127.0.0.1", port, timeout=5) + plain.request("GET", "/healthz") + plain.getresponse() + finally: + httpd.shutdown() + httpd.server_close() + thread.join(timeout=5) + + +def test_tls_requires_both_cert_and_key(tmp_path, monkeypatch): + """A cert without a key (or vice versa) is a startup error, not a silent skip.""" + cert, _key = _make_self_signed_cert(tmp_path) + monkeypatch.setenv("MEMPALACE_MCP_TLS_CERT", str(cert)) + monkeypatch.delenv("MEMPALACE_MCP_TLS_KEY", raising=False) + with pytest.raises(ValueError, match="both"): + mcp._build_http_server("127.0.0.1", 0) + + def test_loopback_and_origin_helpers(): assert mcp._http_is_loopback("127.0.0.1") assert mcp._http_is_loopback("localhost") diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 122f19f881..708b342217 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -203,13 +203,16 @@ def _make_fake_palace(tmp_path): """Create just enough on disk for ``_maybe_eager_warmup_embedder``'s fresh-install pre-check to pass (``chroma.sqlite3`` exists). - Returns the palace dir as a string. The file is empty — production - code must not read its bytes during pre-check; only its existence - gates whether warmup proceeds to the chromadb client open. + Returns the palace dir as a string. The file carries a real SQLite + header (but no chromadb schema) so backend detection's magic-header + check (#1893) accepts it; warmup must still gate on the pre-check + before any chromadb client open. """ + from _chroma_palace_helper import make_minimal_chroma_sqlite + palace = tmp_path / "palace" palace.mkdir() - (palace / "chroma.sqlite3").touch() + make_minimal_chroma_sqlite(palace) return str(palace) @staticmethod @@ -458,6 +461,143 @@ def test_log_file_invalid_path_failure_surfaces_before_first_log_record(self, tm f"stderr={result.stderr!r}" ) + def test_host_root_logger_config_survives_import(self, tmp_path): + """#1860: importing the server must NOT clobber a host app's root + logger. ``_init_logging`` previously called + ``logging.basicConfig(force=True)`` at import, resetting root's + level, format, and handlers — silently overriding any app that + configured logging before importing ``mempalace.mcp_server``.""" + marker = tmp_path / "rootstate.txt" + extra = ( + "import logging, pathlib\n" + # Host app configures logging BEFORE importing mempalace. + "logging.basicConfig(level=logging.DEBUG, " + "format='HOST %(levelname)s %(message)s')\n" + "_sentinel = logging.NullHandler()\n" + "logging.getLogger().addHandler(_sentinel)\n" + "from mempalace import mcp_server # noqa: F401 — triggers _init_logging()\n" + "_root = logging.getLogger()\n" + "_fmt = next((h.formatter._fmt for h in _root.handlers " + "if h.formatter is not None), None)\n" + f"pathlib.Path({str(marker)!r}).write_text(\n" + " f'level={logging.getLevelName(_root.level)}|'\n" + " f'sentinel={_sentinel in _root.handlers}|'\n" + " f'nhandlers={len(_root.handlers)}|'\n" + " f'fmt={_fmt!r}'\n" + ")\n" + "raise SystemExit(0)\n" + ) + result = self._run_main({"MEMPALACE_LOG_FILE": None}, extra_code=extra) + assert result.returncode == 0, f"stderr={result.stderr!r}" + state = marker.read_text() + # Root logger must remain exactly as the host configured it. + assert "level=DEBUG" in state, state + assert "sentinel=True" in state, state + # MEMPALACE_LOG_FILE unset + host owns root → mempalace adds no handler. + assert "nhandlers=2" in state, state + assert "fmt='HOST %(levelname)s %(message)s'" in state, state + + def test_log_file_with_host_root_captures_mempalace_only(self, tmp_path): + """#1860 + #1495: when a host app owns the root logger and + MEMPALACE_LOG_FILE is set, the file still captures mempalace's own + records — including the dotted ``mempalace.*`` family (the cold-load + path) — but NOT the host's. Proves the additive, mempalace-filtered + file handler: a naive 'reset root' or 'single dedicated logger' fix + would either leak host logs into the file or drop the dotted family.""" + log_path = tmp_path / "mcp.log" + extra = ( + "import logging\n" + # Host owns root logging before the import. + "logging.basicConfig(level=logging.DEBUG, format='%(message)s')\n" + "from mempalace import mcp_server # noqa: F401 — triggers _init_logging()\n" + "logging.getLogger('host.app').warning('HOST-ONLY-LINE-xyz')\n" + "logging.getLogger('mempalace.embedding').info('MEMPALACE-DOTTED-LINE-xyz')\n" + "logging.getLogger('mempalace_mcp').info('MEMPALACE-FLAT-LINE-xyz')\n" + "logging.shutdown()\n" + "raise SystemExit(0)\n" + ) + result = self._run_main({"MEMPALACE_LOG_FILE": str(log_path)}, extra_code=extra) + assert result.returncode == 0, f"stderr={result.stderr!r}" + assert log_path.exists(), f"log file missing; stderr={result.stderr!r}" + body = log_path.read_text(encoding="utf-8") + assert "MEMPALACE-DOTTED-LINE-xyz" in body, body + assert "MEMPALACE-FLAT-LINE-xyz" in body, body + assert "HOST-ONLY-LINE-xyz" not in body, body + # Format is "%(message)s" in the embedded path too: the line is the bare + # message with no "LEVEL:name:" prefix (the file handler sets its own + # formatter, independent of basicConfig which never runs here). + assert any(line == "MEMPALACE-FLAT-LINE-xyz" for line in body.splitlines()), body + + def test_embedded_host_warning_root_gates_mempalace_info(self, tmp_path): + """Documents the intentional embedded-mode level-gating tradeoff: when + a host owns root at WARNING, mempalace INFO heartbeats do NOT reach + MEMPALACE_LOG_FILE (the file handler rides on the host-gated root), but + WARNING/ERROR cold-load failure diagnostics still do. #1860 never + raises the host's level; #1495's motivating case is a standalone launch + (root empty -> INFO pinned) and is unaffected.""" + log_path = tmp_path / "mcp.log" + extra = ( + "import logging\n" + "logging.basicConfig(level=logging.WARNING, format='%(message)s')\n" + "from mempalace import mcp_server # noqa: F401 — triggers _init_logging()\n" + "logging.getLogger('mempalace_mcp').info('INFO-HEARTBEAT-xyz')\n" + "logging.getLogger('mempalace_mcp').warning('WARN-DIAG-xyz')\n" + "logging.shutdown()\n" + "raise SystemExit(0)\n" + ) + result = self._run_main({"MEMPALACE_LOG_FILE": str(log_path)}, extra_code=extra) + assert result.returncode == 0, f"stderr={result.stderr!r}" + body = log_path.read_text(encoding="utf-8") + assert "WARN-DIAG-xyz" in body, body + assert "INFO-HEARTBEAT-xyz" not in body, body + + def test_standalone_log_file_excludes_third_party_records(self, tmp_path): + """The MEMPALACE_LOG_FILE stream is mempalace-only in standalone mode + too: third-party library records reaching the root logger are kept out + of the file by ``_MempalaceLogFilter`` (the file stays a clean + mempalace diagnostic stream).""" + log_path = tmp_path / "mcp.log" + extra = ( + "import logging\n" + "from mempalace import mcp_server # noqa: F401 — standalone: root starts empty\n" + "logging.getLogger('chromadb.fake').warning('THIRDPARTY-LINE-xyz')\n" + "logging.getLogger('mempalace.embedding').info('MEMPALACE-STD-LINE-xyz')\n" + "logging.shutdown()\n" + "raise SystemExit(0)\n" + ) + result = self._run_main({"MEMPALACE_LOG_FILE": str(log_path)}, extra_code=extra) + assert result.returncode == 0, f"stderr={result.stderr!r}" + body = log_path.read_text(encoding="utf-8") + assert "MEMPALACE-STD-LINE-xyz" in body, body + assert "THIRDPARTY-LINE-xyz" not in body, body + + def test_reload_does_not_duplicate_file_handler(self, tmp_path): + """#1885 review: the idempotency guard must survive ``importlib.reload``, + not only a direct second call. A reload re-executes the module body; the + guard flag is restored from ``globals()`` so ``_init_logging`` early-exits + and does not stack a second ``FileHandler`` on root.""" + log_path = tmp_path / "mcp.log" + marker = tmp_path / "counts.txt" + extra = ( + "import logging, importlib, pathlib\n" + "from mempalace import mcp_server\n" + "def _nfile():\n" + " return sum(\n" + " isinstance(h, logging.FileHandler)\n" + " for h in logging.getLogger().handlers\n" + " )\n" + "_before = _nfile()\n" + "importlib.reload(mcp_server)\n" + "_after = _nfile()\n" + f"pathlib.Path({str(marker)!r}).write_text(f'{{_before}},{{_after}}')\n" + "raise SystemExit(0)\n" + ) + result = self._run_main({"MEMPALACE_LOG_FILE": str(log_path)}, extra_code=extra) + assert result.returncode == 0, f"stderr={result.stderr!r}" + before, after = marker.read_text().split(",") + assert before == "1", f"expected one file handler after import, got {before}" + assert after == "1", f"reload duplicated the file handler: {before}->{after}" + # ── Protocol Layer ────────────────────────────────────────────────────── @@ -2296,6 +2436,141 @@ def test_add_drawer_boundary_exact_chunk_size_stays_single( assert result["chunks"] == 1 assert "chunk_ids" not in result + def test_list_drawers_since_filter_inclusive( + self, monkeypatch, config, palace_path, seeded_collection, kg + ): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + # seeded filed_at values: 2026-01-01..2026-01-04; since is inclusive. + result = tool_list_drawers(since="2026-01-03") + assert result["total"] == 2 + assert result["count"] == 2 + filed = sorted(d["metadata"]["filed_at"] for d in result["drawers"]) + assert filed == ["2026-01-03T00:00:00", "2026-01-04T00:00:00"] + + def test_list_drawers_before_filter_exclusive( + self, monkeypatch, config, palace_path, seeded_collection, kg + ): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + # before is exclusive: 2026-01-03 keeps only 01 and 02. + result = tool_list_drawers(before="2026-01-03") + assert result["total"] == 2 + filed = sorted(d["metadata"]["filed_at"] for d in result["drawers"]) + assert filed == ["2026-01-01T00:00:00", "2026-01-02T00:00:00"] + + def test_list_drawers_since_and_before_window( + self, monkeypatch, config, palace_path, seeded_collection, kg + ): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + # [since, before): 02 and 03 kept, 01 below, 04 at/above the bound. + result = tool_list_drawers(since="2026-01-02", before="2026-01-04") + assert result["total"] == 2 + filed = sorted(d["metadata"]["filed_at"] for d in result["drawers"]) + assert filed == ["2026-01-02T00:00:00", "2026-01-03T00:00:00"] + + def test_list_drawers_date_window_single_day( + self, monkeypatch, config, palace_path, seeded_collection, kg + ): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + # since inclusive + before exclusive isolates exactly 2026-01-02. + result = tool_list_drawers(since="2026-01-02", before="2026-01-03") + assert result["total"] == 1 + assert result["drawers"][0]["metadata"]["filed_at"] == "2026-01-02T00:00:00" + + def test_list_drawers_date_filter_combines_with_wing( + self, monkeypatch, config, palace_path, seeded_collection, kg + ): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + # project wing = 01,02,03; since 2026-01-02 narrows to 02,03. + result = tool_list_drawers(wing="project", since="2026-01-02") + assert result["total"] == 2 + assert all(d["wing"] == "project" for d in result["drawers"]) + + def test_list_drawers_no_date_filter_unchanged( + self, monkeypatch, config, palace_path, seeded_collection, kg + ): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + # Omitting since/before leaves the full set (regression guard). + assert tool_list_drawers()["total"] == 4 + + def test_list_drawers_rejects_invalid_since( + self, monkeypatch, config, palace_path, seeded_collection, kg + ): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + result = tool_list_drawers(since="not-a-date") + assert "error" in result + assert "since" in result["error"] + + def test_list_drawers_rejects_invalid_before( + self, monkeypatch, config, palace_path, seeded_collection, kg + ): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + result = tool_list_drawers(before="2026-99-99") + assert "error" in result + assert "before" in result["error"] + + def test_list_drawers_rejects_inverted_window( + self, monkeypatch, config, palace_path, seeded_collection, kg + ): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + # since must be earlier than before; inverted bounds are a clear error, + # not a silently empty result. + result = tool_list_drawers(since="2026-06-01", before="2026-01-01") + assert "error" in result + assert "since" in result["error"] + assert "before" in result["error"] + + def test_list_drawers_excludes_undated_drawer_when_filtered( + self, monkeypatch, config, palace_path, seeded_collection, kg + ): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + # A drawer with no filed_at is present unfiltered but excluded once a + # date bound is active (its age cannot be confirmed in-window). + seeded_collection.add( + ids=["drawer_no_filed_at"], + documents=["A drawer without a filed_at timestamp."], + metadatas=[{"wing": "project", "room": "backend"}], + ) + assert tool_list_drawers()["total"] == 5 + filtered = tool_list_drawers(since="2026-01-01") + ids = [d["drawer_id"] for d in filtered["drawers"]] + assert "drawer_no_filed_at" not in ids + assert filtered["total"] == 4 + + def test_list_drawers_date_filter_paginates_on_filtered_total( + self, monkeypatch, config, palace_path, seeded_collection, kg + ): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + # window [01-01, 01-04) keeps 01, 02, 03; pagination runs on that + # filtered total, not the grand total of 4. + page1 = tool_list_drawers(since="2026-01-01", before="2026-01-04", limit=2, offset=0) + page2 = tool_list_drawers(since="2026-01-01", before="2026-01-04", limit=2, offset=2) + assert page1["total"] == 3 + assert page1["count"] == 2 + assert page2["total"] == 3 + assert page2["count"] == 1 + def test_add_drawer_chunked_logical_id_fetches_deletes_and_lists_as_one( monkeypatch, config, palace_path, kg @@ -4931,3 +5206,243 @@ def test_sqlite_integrity_refusal_handles_none_palace_path(monkeypatch): assert result["error"]["data"]["palace"] == "" assert result["error"]["data"]["sqlite_path"] == "" assert result["error"]["data"]["tool"] == "mempalace_kg_add" + + +class TestMetadataFacets: + def test_tool_status_uses_metadata_facets(self, monkeypatch): + from unittest.mock import MagicMock + import mempalace.mcp_server as mcp + + monkeypatch.setattr(mcp, "_sqlite_taxonomy", lambda: None) + monkeypatch.setattr(mcp, "_supports_metadata_facets", lambda _: True) + + col = MagicMock() + col.count.return_value = 5 + col.facet_counts.side_effect = [ + {"wing_a": 2, "wing_b": 3}, + {"room_x": 4, "room_y": 1}, + ] + monkeypatch.setattr(mcp, "_get_collection", lambda create=False: col) + result = mcp.tool_status() + + assert result["wings"] == { + "wing_a": 2, + "wing_b": 3, + } + + assert result["rooms"] == { + "room_x": 4, + "room_y": 1, + } + assert col.facet_counts.call_count == 2 + + def test_tool_list_wings_uses_metadata_facets(self, monkeypatch): + from unittest.mock import MagicMock + import mempalace.mcp_server as mcp + + monkeypatch.setattr(mcp, "_sqlite_taxonomy", lambda: None) + monkeypatch.setattr(mcp, "_supports_metadata_facets", lambda _: True) + + col = MagicMock() + col.facet_counts.return_value = { + "wing_a": 5, + "wing_b": 2, + } + monkeypatch.setattr(mcp, "_get_collection", lambda: col) + result = mcp.tool_list_wings() + + assert result == { + "wings": { + "wing_a": 5, + "wing_b": 2, + } + } + col.facet_counts.assert_called_once_with("wing") + + def test_tool_list_rooms_uses_metadata_facets(self, monkeypatch): + from unittest.mock import MagicMock + + import mempalace.mcp_server as mcp + + monkeypatch.setattr(mcp, "_sqlite_taxonomy", lambda: None) + monkeypatch.setattr(mcp, "_supports_metadata_facets", lambda _: True) + + col = MagicMock() + + col.facet_counts.return_value = { + "room1": 7, + "room2": 3, + } + + monkeypatch.setattr(mcp, "_get_collection", lambda: col) + + result = mcp.tool_list_rooms("engineering") + + assert result["rooms"] == { + "room1": 7, + "room2": 3, + } + + from unittest.mock import call + + assert col.facet_counts.call_args_list == [ + call("room", where={"wing": "engineering"}), + call("wing", where={"wing": "engineering"}), + ] + + def test_tool_get_taxonomy_uses_metadata_facets(self, monkeypatch): + from unittest.mock import MagicMock, call + import mempalace.mcp_server as mcp + + monkeypatch.setattr(mcp, "_sqlite_taxonomy", lambda: None) + monkeypatch.setattr(mcp, "_supports_metadata_facets", lambda _: True) + + col = MagicMock() + + def facet_counts_mock(field, where=None): + if field == "wing": + return {"wing_a": 2, "wing_b": 1} + if field == "room" and where == {"wing": "wing_a"}: + return {"room1": 2} + if field == "room" and where == {"wing": "wing_b"}: + return {"room2": 1} + return {} + + col.facet_counts.side_effect = facet_counts_mock + + monkeypatch.setattr(mcp, "_get_collection", lambda: col) + + result = mcp.tool_get_taxonomy() + assert col.facet_counts.call_args_list[0] == call("wing") + # Per-wing room facets run concurrently (ThreadPoolExecutor), so order is + # non-deterministic. Compare order-independently without a set() — a + # ``call`` carrying a dict kwarg is unhashable, so membership (==) is used. + room_calls = col.facet_counts.call_args_list[1:] + assert len(room_calls) == 2 + assert call("room", where={"wing": "wing_a"}) in room_calls + assert call("room", where={"wing": "wing_b"}) in room_calls + + assert result["taxonomy"] == { + "wing_a": { + "room1": 2, + }, + "wing_b": { + "room2": 1, + }, + } + + +class TestListDrawersDateFilters: + """Unit tests for the #1128 date-filter helpers in mcp_server.""" + + def test_parse_date_filter_none_and_blank(self): + from mempalace.mcp_server import _parse_date_filter + + assert _parse_date_filter(None, "since") is None + assert _parse_date_filter(" ", "since") is None + + def test_parse_date_filter_date_only(self): + from datetime import datetime + + from mempalace.mcp_server import _parse_date_filter + + assert _parse_date_filter("2026-04-01", "since") == datetime(2026, 4, 1) + + def test_parse_date_filter_full_timestamp(self): + from datetime import datetime + + from mempalace.mcp_server import _parse_date_filter + + assert _parse_date_filter("2026-04-01T09:30:00", "since") == datetime(2026, 4, 1, 9, 30) + + def test_parse_date_filter_drops_timezone(self): + from datetime import datetime + + from mempalace.mcp_server import _parse_date_filter + + # tz offset dropped -> naive wall-clock, never raises vs naive filed_at. + parsed = _parse_date_filter("2026-04-01T09:30:00+02:00", "since") + assert parsed == datetime(2026, 4, 1, 9, 30) + assert parsed.tzinfo is None + + def test_parse_date_filter_rejects_garbage(self): + import pytest + + from mempalace.mcp_server import _parse_date_filter + + with pytest.raises(ValueError, match="since"): + _parse_date_filter("not-a-date", "since") + + def test_parse_date_filter_rejects_impossible_date(self): + import pytest + + from mempalace.mcp_server import _parse_date_filter + + with pytest.raises(ValueError): + _parse_date_filter("2026-13-40", "before") + + def test_filed_at_in_window_since_inclusive(self): + from datetime import datetime + + from mempalace.mcp_server import _filed_at_in_window + + since = datetime(2026, 1, 2) + assert _filed_at_in_window("2026-01-02T00:00:00", since, None) is True + assert _filed_at_in_window("2026-01-01T23:59:59", since, None) is False + + def test_filed_at_in_window_before_exclusive(self): + from datetime import datetime + + from mempalace.mcp_server import _filed_at_in_window + + before = datetime(2026, 1, 3) + assert _filed_at_in_window("2026-01-02T23:59:59", None, before) is True + assert _filed_at_in_window("2026-01-03T00:00:00", None, before) is False + + def test_filed_at_in_window_missing_or_malformed_excluded(self): + from datetime import datetime + + from mempalace.mcp_server import _filed_at_in_window + + since = datetime(2026, 1, 1) + assert _filed_at_in_window(None, since, None) is False + assert _filed_at_in_window("", since, None) is False + assert _filed_at_in_window("garbage", since, None) is False + assert _filed_at_in_window(12345, since, None) is False + + def test_filed_at_in_window_tz_aware_wall_clock(self): + from datetime import datetime + + from mempalace.mcp_server import _filed_at_in_window + + # tz dropped on both sides -> wall-clock compare, no TypeError raised. + since = datetime(2026, 1, 2) + assert _filed_at_in_window("2026-01-02T08:00:00+05:00", since, None) is True + + def test_parse_date_filter_accepts_zulu_suffix(self): + from datetime import datetime + + from mempalace.mcp_server import _parse_date_filter + + # "Z" is not accepted by datetime.fromisoformat before 3.11; the helper + # strips it so Zulu inputs parse on the 3.9 floor, tz then dropped. + parsed = _parse_date_filter("2026-04-01T09:30:00Z", "since") + assert parsed == datetime(2026, 4, 1, 9, 30) + assert parsed.tzinfo is None + + # Date-only with a Zulu suffix must also parse on 3.9/3.10 (appending + # "+00:00" would have raised there; stripping Z does not). + parsed_date = _parse_date_filter("2026-04-01Z", "since") + assert parsed_date == datetime(2026, 4, 1) + assert parsed_date.tzinfo is None + + # Lowercase z is tolerated too. + assert _parse_date_filter("2026-04-01t09:30:00z", "since") == datetime(2026, 4, 1, 9, 30) + + def test_filed_at_in_window_accepts_zulu_filed_at(self): + from datetime import datetime + + from mempalace.mcp_server import _filed_at_in_window + + since = datetime(2026, 1, 2) + assert _filed_at_in_window("2026-01-02T08:00:00Z", since, None) is True diff --git a/tests/test_miner.py b/tests/test_miner.py index 0404cb0582..6e2e18d33f 100644 --- a/tests/test_miner.py +++ b/tests/test_miner.py @@ -538,6 +538,20 @@ def test_scan_project_includes_kotlin_files(): ] +def test_scan_project_includes_latex_files(): + with tempfile.TemporaryDirectory() as tmpdir: + project_root = Path(tmpdir).resolve() + write_file( + project_root / "main.tex", + "\\documentclass{article}\n\\begin{document}\nHello, world.\n\\end{document}\n" * 20, + ) + write_file( + project_root / "refs.bib", + "@article{lamport1986, author={Leslie Lamport}, title={LaTeX}, year={1986}}\n" * 20, + ) + assert scanned_files(project_root) == ["main.tex", "refs.bib"] + + def test_scan_project_respects_gitignore(): tmpdir = tempfile.mkdtemp() try: diff --git a/tests/test_palace.py b/tests/test_palace.py index acd1b492b5..426924a896 100644 --- a/tests/test_palace.py +++ b/tests/test_palace.py @@ -2,6 +2,8 @@ import chromadb +from _chroma_palace_helper import make_minimal_chroma_sqlite + from mempalace.backends import CollectionNotInitializedError, PalaceNotFoundError from mempalace.palace import _open_collection_or_explain, get_collection @@ -94,7 +96,7 @@ def test_open_collection_or_explain_state_e_unexpected_error(tmp_path, monkeypat emit, lines = _capture() palace = tmp_path / "palace" palace.mkdir() - (palace / "chroma.sqlite3").touch() # pass the isfile guard + make_minimal_chroma_sqlite(palace) # pass the isfile guard def boom(*args, **kwargs): raise RuntimeError("disk on fire") @@ -125,7 +127,7 @@ def test_open_collection_or_explain_propagates_palace_not_found_from_backend(tmp emit, lines = _capture() palace = tmp_path / "palace" palace.mkdir() - (palace / "chroma.sqlite3").touch() + make_minimal_chroma_sqlite(palace) def raise_pnf(*args, **kwargs): raise PalaceNotFoundError(str(palace)) @@ -151,7 +153,7 @@ def test_open_collection_or_explain_reraises_backend_closed_error(tmp_path, monk palace = tmp_path / "palace" palace.mkdir() - (palace / "chroma.sqlite3").touch() + make_minimal_chroma_sqlite(palace) def raise_closed(*args, **kwargs): raise BackendClosedError("ChromaBackend has been closed") @@ -171,7 +173,7 @@ def test_open_collection_or_explain_distinguishes_collection_subclass(tmp_path, emit, lines = _capture() palace = tmp_path / "palace" palace.mkdir() - (palace / "chroma.sqlite3").touch() + make_minimal_chroma_sqlite(palace) def raise_cnie(*args, **kwargs): raise CollectionNotInitializedError(str(palace)) diff --git a/tests/test_palace_locks.py b/tests/test_palace_locks.py index b4d2fbc259..e38d4a5014 100644 --- a/tests/test_palace_locks.py +++ b/tests/test_palace_locks.py @@ -11,6 +11,7 @@ import multiprocessing import os +import threading import time import sys @@ -68,6 +69,45 @@ def _hold_lock(palace_path: str, ready_flag: str, release_flag: str) -> int: # --------------------------------------------------------------------------- +def test_mine_palace_lock_reentrant_across_threads_same_process(tmp_path): + """Process-wide re-entrancy: a second acquisition from a *different thread* + of the same process passes through instead of self-conflicting. + + Regression for the MCP HTTP transport (ThreadingHTTPServer): the writer + lease is acquired on one thread (mcp_server._acquire_mcp_writer_lock) but + write requests are dispatched on other worker threads. With the old + thread-local re-entrancy those handlers re-acquired the process-held flock + and raised MineAlreadyRunning ("palace ... is held by PID "). + Re-entrancy is now process-wide, so same-process cross-thread acquisition is + a pass-through. + """ + palace = str(tmp_path / "palace") + os.makedirs(palace, exist_ok=True) + + outer = mine_palace_lock(palace) + outer.__enter__() # main thread holds the lease, like the MCP writer-lease + try: + result: dict = {} + + def worker(): + try: + with mine_palace_lock(palace): + result["acquired"] = True + except MineAlreadyRunning as exc: # pragma: no cover - failure path + result["error"] = str(exc) + + t = threading.Thread(target=worker) + t.start() + t.join(timeout=5) + + assert not t.is_alive(), "worker thread hung acquiring the palace lock" + assert result.get("acquired") is True, ( + f"cross-thread same-process acquisition should pass through, got: {result}" + ) + finally: + outer.__exit__(None, None, None) + + def test_single_acquire_succeeds(tmp_path, monkeypatch): monkeypatch.setenv("HOME", str(tmp_path)) with mine_palace_lock(str(tmp_path / "palace")): diff --git a/tests/test_pgvector_backend.py b/tests/test_pgvector_backend.py index f2c591942b..9505a0bba4 100644 --- a/tests/test_pgvector_backend.py +++ b/tests/test_pgvector_backend.py @@ -93,8 +93,19 @@ def query_rows(self, table, *, vector, limit, where, with_embedding): out.append(item) return out - def scroll_rows(self, table, *, where=None, with_embedding=False, limit=None, offset=None): - self.scroll_calls.append({"where": where, "limit": limit, "offset": offset}) + def scroll_rows( + self, + table, + *, + where=None, + with_embedding=False, + with_document=True, + limit=None, + offset=None, + ): + self.scroll_calls.append( + {"where": where, "limit": limit, "offset": offset, "with_document": with_document} + ) rows = self._filtered(table, where) if limit is not None or offset: # Mirror the real backend: ORDER BY id, then LIMIT/OFFSET. @@ -108,7 +119,9 @@ def scroll_rows(self, table, *, where=None, with_embedding=False, limit=None, of out.append( { "id": row["id"], - "document": row["document"], + # Match the real backend: NULL document becomes empty string + # via the SELECT NULL::text projection when with_document=False. + "document": row["document"] if with_document else "", "metadata": row.get("metadata") or {}, "embedding": row.get("embedding") if with_embedding else None, "distance": None, @@ -390,7 +403,7 @@ def test_pgvector_get_unfiltered_page_pushes_limit_offset(tmp_path, fake_pgvecto # An unfiltered page is pushed to SQL as LIMIT/OFFSET instead of fetching # the whole table and slicing in Python (the O(rows x pages) path). - assert client.scroll_calls == [{"where": None, "limit": 2, "offset": 1}] + assert client.scroll_calls == [{"where": None, "limit": 2, "offset": 1, "with_document": True}] # ORDER BY id, then OFFSET 1 LIMIT 2 -> b, c. assert page.ids == ["b", "c"] @@ -410,7 +423,9 @@ def test_pgvector_get_filtered_page_stays_on_full_scan(tmp_path, fake_pgvector): # A filtered get keeps the full-scan path (no LIMIT/OFFSET pushed) so the # exact _matches_where re-filter runs before pagination. - assert client.scroll_calls == [{"where": {"wing": "x"}, "limit": None, "offset": None}] + assert client.scroll_calls == [ + {"where": {"wing": "x"}, "limit": None, "offset": None, "with_document": True} + ] assert page.ids == ["c"] @@ -427,13 +442,17 @@ def test_pgvector_get_offset_only_and_limit_only_push(tmp_path, fake_pgvector): # offset-only (limit=None) is pushed. client.scroll_calls.clear() page = col.get(offset=2, include=["metadatas"]) - assert client.scroll_calls == [{"where": None, "limit": None, "offset": 2}] + assert client.scroll_calls == [ + {"where": None, "limit": None, "offset": 2, "with_document": True} + ] assert page.ids == ["c", "d"] # limit-only (offset=None) is pushed. client.scroll_calls.clear() page = col.get(limit=2, include=["metadatas"]) - assert client.scroll_calls == [{"where": None, "limit": 2, "offset": None}] + assert client.scroll_calls == [ + {"where": None, "limit": 2, "offset": None, "with_document": True} + ] assert page.ids == ["a", "b"] @@ -451,7 +470,9 @@ def test_pgvector_get_negative_bounds_use_python_slice(tmp_path, fake_pgvector): # A negative offset must not reach SQL (OFFSET -1 would error); it falls # through to the unchanged full-scan + Python-slice path. page = col.get(offset=-1, include=["metadatas"]) - assert client.scroll_calls == [{"where": None, "limit": None, "offset": None}] + assert client.scroll_calls == [ + {"where": None, "limit": None, "offset": None, "with_document": True} + ] assert page.ids == ["c"] @@ -473,6 +494,73 @@ def test_pgvector_get_pages_tile_without_overlap(tmp_path, fake_pgvector): assert p1 + p2 + p3 == ["a", "b", "c", "d", "e"] +def test_pgvector_get_all_metadata_skips_document_column(tmp_path, fake_pgvector): + """The metadata-only fast path must NOT pull document text over the wire. + + Default base ``get_all_metadata`` pages through ``get(include=["metadatas"])``, + which used to route here via scroll_rows with documents always selected — the + "separate follow-up" #1840 flagged. This override calls scroll_rows with + with_document=False so the SELECT projects NULL into the document slot, + dropping per-row payload for remote (TLS over WAN) clients where status + otherwise dominates wall time. + """ + _backend, col = _collection(tmp_path) + col.add( + ids=["a", "b", "c"], + documents=["doc_a", "doc_b", "doc_c"], + metadatas=[ + {"wing": "p", "room": "backend"}, + {"wing": "p", "room": "frontend"}, + {"wing": "q", "room": "backend"}, + ], + embeddings=[[1, 0], [0, 1], [0.5, 0.5]], + ) + client = fake_pgvector.instances[0] + client.scroll_calls.clear() + + metas = col.get_all_metadata() + + # Exactly one scroll, with_document=False (no document text on the wire). + assert client.scroll_calls == [ + {"where": None, "limit": None, "offset": None, "with_document": False} + ] + # Returns just the metadata dicts (full set, any order — sort by wing+room for stability). + metas_sorted = sorted(metas, key=lambda m: (m["wing"], m["room"])) + assert metas_sorted == [ + {"wing": "p", "room": "backend"}, + {"wing": "p", "room": "frontend"}, + {"wing": "q", "room": "backend"}, + ] + + +def test_pgvector_get_all_metadata_filtered_uses_fast_path(tmp_path, fake_pgvector): + """Filtered get_all_metadata uses the single-pass metadata-only fast path. + + ``_matches_where`` only reads ``metadata``, so we keep ``with_document=False`` + and apply the post-filter locally on the metadata dicts. SQL pushdown still + happens when the filter is pushdownable; the local ``_matches_where`` re-runs + for array/object semantics #1840's filtered path required. + """ + _backend, col = _collection(tmp_path) + col.add( + ids=["a", "b", "c"], + documents=["doc_a", "doc_b", "doc_c"], + metadatas=[{"wing": "x"}, {"wing": "y"}, {"wing": "x"}], + embeddings=[[1, 0], [0, 1], [0.5, 0.5]], + ) + client = fake_pgvector.instances[0] + client.scroll_calls.clear() + + metas = col.get_all_metadata(where={"wing": "x"}) + + # Exactly one scroll with with_document=False — pushdown forwards the + # equality filter to SQL; no document text on the wire. + assert client.scroll_calls == [ + {"where": {"wing": "x"}, "limit": None, "offset": None, "with_document": False} + ] + assert sorted(metas, key=lambda m: m["wing"]) == [{"wing": "x"}, {"wing": "x"}] + + def test_pgvector_delete_by_where_pushdown_and_local(tmp_path, fake_pgvector): _backend, col = _collection(tmp_path) col.add( diff --git a/tests/test_qdrant_backend.py b/tests/test_qdrant_backend.py index 07e211345d..d099baf9b7 100644 --- a/tests/test_qdrant_backend.py +++ b/tests/test_qdrant_backend.py @@ -5,6 +5,7 @@ import pytest from _backend_conformance import assert_partition_isolation +from _chroma_palace_helper import make_minimal_chroma_sqlite from mempalace.backends import ( BackendError, @@ -12,6 +13,7 @@ CollectionNotInitializedError, DimensionMismatchError, PalaceRef, + UnsupportedCapabilityError, available_backends, ) from mempalace.backends.qdrant import QdrantBackend @@ -82,6 +84,7 @@ def __init__(self, _config): self.query_calls = [] self.scroll_calls = [] self.created_indexes = [] + self.facet_calls = [] _FakeQdrantClient.instances.append(self) def request(self, *_args, **_kwargs): @@ -176,6 +179,33 @@ def count_points(self, collection): def delete_collection(self, collection): self.collections.pop(collection, None) + def facet_counts( + self, + collection, + *, + field, + qdrant_filter=None, + limit=1000, + ): + self.facet_calls.append((field, qdrant_filter)) + + counts = {} + + points = list(self.collections.get(collection, {"points": {}})["points"].values()) + + points = [point for point in points if _fake_match_filter(point, qdrant_filter)] + + for point in points: + metadata = point["payload"].get("metadata", {}) + actual_field = field.split(".", 1)[-1] if field.startswith("metadata.") else field + value = metadata.get(actual_field) + + if value is None: + continue + counts[value] = counts.get(value, 0) + 1 + + return counts + @pytest.fixture def fake_qdrant(monkeypatch): @@ -362,7 +392,7 @@ def test_qdrant_marker_participates_in_backend_mismatch(tmp_path, monkeypatch, f backend, col = _collection(tmp_path) col.upsert(ids=["a"], documents=["one"], metadatas=[{}], embeddings=[[1, 0]]) backend.close() - (tmp_path / "chroma.sqlite3").write_bytes(b"") + make_minimal_chroma_sqlite(tmp_path) monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "chroma") with pytest.raises(BackendMismatchError): @@ -527,3 +557,112 @@ def test_qdrant_live_rest_roundtrip_when_enabled(tmp_path): except Exception: pass backend.close() + + +def test_qdrant_facet_counts(tmp_path, fake_qdrant): + _, collection = _collection(tmp_path) + collection.upsert( + ids=["1", "2", "3", "4"], + documents=["a", "b", "c", "d"], + metadatas=[ + {"wing": "alpha"}, + {"wing": "alpha"}, + {"wing": "beta"}, + {"wing": "gamma"}, + ], + embeddings=[ + [1, 0], + [1, 0], + [1, 0], + [1, 0], + ], + ) + assert collection.facet_counts("wing") == { + "alpha": 2, + "beta": 1, + "gamma": 1, + } + + +def test_qdrant_facet_counts_where(tmp_path, fake_qdrant): + _, collection = _collection(tmp_path) + collection.upsert( + ids=["1", "2", "3"], + documents=["a", "b", "c"], + metadatas=[ + {"wing": "engineering", "room": "backend"}, + {"wing": "engineering", "room": "frontend"}, + {"wing": "design", "room": "ux"}, + ], + embeddings=[ + [1, 0], + [1, 0], + [1, 0], + ], + ) + assert collection.facet_counts( + "room", + where={"wing": "engineering"}, + ) == { + "backend": 1, + "frontend": 1, + } + + +def test_qdrant_facet_counts_rejects_local_filters(tmp_path, fake_qdrant): + _, collection = _collection(tmp_path) + with pytest.raises(UnsupportedCapabilityError): + collection.facet_counts( + "room", + where={ + "$or": [ + {"wing": "a"}, + {"wing": "b"}, + ] + }, + ) + + +def test_qdrant_facet_counts_passes_filter(tmp_path, fake_qdrant): + _, collection = _collection(tmp_path) + collection.upsert( + ids=["1"], + documents=["doc"], + metadatas=[{"wing": "engineering", "room": "backend"}], + embeddings=[[1, 0]], + ) + collection.facet_counts( + "room", + where={"wing": "engineering"}, + ) + client = fake_qdrant.instances[0] + assert len(client.facet_calls) == 1 + field, qfilter = client.facet_calls[0] + assert field == "metadata.room" + assert qfilter == { + "must": [ + { + "key": "metadata.wing", + "match": {"value": "engineering"}, + } + ] + } + + +def test_qdrant_facet_counts_ignores_missing_metadata(tmp_path, fake_qdrant): + _, collection = _collection(tmp_path) + collection.upsert( + ids=["1", "2"], + documents=["a", "b"], + metadatas=[ + {"wing": "alpha"}, + {}, + ], + embeddings=[ + [1, 0], + [1, 0], + ], + ) + assert collection.facet_counts("wing") == { + "alpha": 1, + } diff --git a/tests/test_repair.py b/tests/test_repair.py index 95c5638636..c1f4db4803 100644 --- a/tests/test_repair.py +++ b/tests/test_repair.py @@ -7,6 +7,8 @@ import pytest +from _chroma_palace_helper import make_minimal_chroma_sqlite + from mempalace import repair @@ -639,7 +641,7 @@ def test_status_returns_empty_when_db_present_no_drawers(tmp_path, capsys): 'uninitialized' (#1498). Mocks sqlite_drawer_count to assert the return-shape contract; see the real-disk sibling below for the no-chromadb-client invariant.""" - (tmp_path / "chroma.sqlite3").touch() + make_minimal_chroma_sqlite(tmp_path) with patch("mempalace.repair.sqlite_drawer_count", return_value=0): result = repair.status(palace_path=str(tmp_path)) @@ -678,7 +680,7 @@ def test_status_falls_through_to_capacity_when_sqlite_count_unreadable(tmp_path) """When sqlite_drawer_count returns None (schema drift / locked file), repair.status must fall through to hnsw_capacity_status instead of short-circuiting on 'empty' (#1498).""" - (tmp_path / "chroma.sqlite3").touch() + make_minimal_chroma_sqlite(tmp_path) with ( patch("mempalace.repair.sqlite_drawer_count", return_value=None), patch("mempalace.repair.hnsw_capacity_status") as capacity_status, @@ -714,7 +716,7 @@ def test_status_default_uses_configured_drawer_collection(tmp_path): # Provide the on-disk preconditions the stratified state helper (#1498) # checks before reaching the capacity probe: chroma.sqlite3 file exists # and sqlite_drawer_count returns a positive number (palace not empty). - (tmp_path / "chroma.sqlite3").touch() + make_minimal_chroma_sqlite(tmp_path) with ( patch("mempalace.repair._drawers_collection_name", return_value="custom_drawers"), patch("mempalace.repair.sqlite_drawer_count", return_value=1), @@ -2032,6 +2034,114 @@ def test_vacuum_and_rebuild_fts5_missing_sqlite(tmp_path): repair._vacuum_and_rebuild_fts5(str(tmp_path)) # no file — must not raise +# ── FTS5 inverted-index auto-heal (#1596) ───────────────────────────── + + +def _make_fts5_palace(tmp_path, *, corrupt: bool) -> str: + """Build a palace whose embedding_fulltext_search index is optionally + corrupted to the malformed-inverted-index quick_check state #1596 hits.""" + sqlite_path = tmp_path / "chroma.sqlite3" + with closing(sqlite3.connect(str(sqlite_path))) as conn: + conn.execute( + "CREATE VIRTUAL TABLE embedding_fulltext_search" + " USING fts5(string_value, tokenize='unicode61')" + ) + for i in range(200): + conn.execute( + "INSERT INTO embedding_fulltext_search(string_value) VALUES(?)", + (f"alpha beta gamma row{i} delta epsilon",), + ) + conn.commit() + if corrupt: + # Zero the last index segment leaf: quick_check then reports + # "malformed inverted index" while the content table stays intact. + conn.execute( + "UPDATE embedding_fulltext_search_data SET block=zeroblob(length(block)) " + "WHERE id=(SELECT max(id) FROM embedding_fulltext_search_data)" + ) + conn.commit() + return str(tmp_path) + + +def test_errors_are_isolated_fts5_classification(): + fts = "malformed inverted index for FTS5 table main.embedding_fulltext_search" + page = "Page 4 of B-tree 12345: database disk image is malformed" + assert repair._errors_are_isolated_fts5([fts]) + assert repair._errors_are_isolated_fts5([fts, fts]) + assert not repair._errors_are_isolated_fts5([]) + assert not repair._errors_are_isolated_fts5([page]) + # Any non-FTS5 error in the set means the data itself may be damaged. + assert not repair._errors_are_isolated_fts5([fts, page]) + + +def test_maybe_autoheal_fts5_index_heals_isolated_corruption(tmp_path): + palace = _make_fts5_palace(tmp_path, corrupt=True) + errors = repair.sqlite_integrity_errors(palace) + assert errors and repair._errors_are_isolated_fts5(errors) + + remaining = repair.maybe_autoheal_fts5_index(palace, errors, progress=lambda *_: None) + + assert remaining == [] + # quick_check is clean and full-text search works again. + assert repair.sqlite_integrity_errors(palace) == [] + with closing(sqlite3.connect(str(tmp_path / "chroma.sqlite3"))) as conn: + hits = conn.execute( + "SELECT count(*) FROM embedding_fulltext_search " + "WHERE embedding_fulltext_search MATCH 'gamma'" + ).fetchone()[0] + assert hits == 200 + + +def test_maybe_autoheal_fts5_index_leaves_non_fts5_errors_untouched(tmp_path): + palace = _make_fts5_palace(tmp_path, corrupt=False) + page_errors = ["Page 4 of B-tree 12345: database disk image is malformed"] + + # Not isolated FTS5: returned unchanged and the rebuild is never attempted. + with patch("mempalace.palace.mine_palace_lock") as lock: + remaining = repair.maybe_autoheal_fts5_index(palace, page_errors, progress=lambda *_: None) + assert remaining == page_errors + lock.assert_not_called() + + +def test_maybe_autoheal_fts5_index_skips_when_palace_is_being_mined(tmp_path): + from mempalace.palace import MineAlreadyRunning + + palace = _make_fts5_palace(tmp_path, corrupt=True) + errors = repair.sqlite_integrity_errors(palace) + + def _raise(_path): + raise MineAlreadyRunning("held by pid 999") + + # A live mine holds the lock: do not race the rebuild — surface and abort. + with patch("mempalace.palace.mine_palace_lock", side_effect=_raise): + remaining = repair.maybe_autoheal_fts5_index(palace, errors, progress=lambda *_: None) + + assert remaining == errors + # The FTS index is still corrupt because we refused to rebuild under contention. + assert repair.sqlite_integrity_errors(palace) == errors + + +def test_rebuild_index_preflight_autoheals_isolated_fts5_then_proceeds(tmp_path, monkeypatch): + """The preflight no longer hard-aborts on isolated FTS5 corruption (#1596): + it rebuilds the index, then continues into the rebuild path.""" + palace = _make_fts5_palace(tmp_path, corrupt=True) + + called = {} + + def _fake_max_seq(_palace_path, **_kwargs): + # Reached only if the preflight did NOT abort — record and stop early + # so the test doesn't need a real chromadb collection. + called["reached"] = True + return {"stopped": True} + + monkeypatch.setattr(repair, "maybe_repair_poisoned_max_seq_id_before_rebuild", _fake_max_seq) + + repair.rebuild_index(palace_path=palace, progress=lambda *_: None) + + assert called.get("reached") is True + assert repair.sqlite_integrity_errors(palace) == [] + + @patch("mempalace.repair.shutil") @patch("mempalace.repair.ChromaBackend") def test_rebuild_index_calls_vacuum(mock_backend_cls, mock_shutil, tmp_path): diff --git a/tests/test_searcher.py b/tests/test_searcher.py index cf70396f32..79f223ed01 100644 --- a/tests/test_searcher.py +++ b/tests/test_searcher.py @@ -9,6 +9,8 @@ import pytest +from _chroma_palace_helper import make_minimal_chroma_sqlite + from mempalace.searcher import SearchError, build_where_filter, search, search_memories @@ -427,7 +429,7 @@ def fake_palace_path(tmp_path): backend instead of raising on State A / State B.""" p = tmp_path / "palace" p.mkdir() - (p / "chroma.sqlite3").touch() + make_minimal_chroma_sqlite(p) return str(p) diff --git a/tests/test_serve.py b/tests/test_serve.py new file mode 100644 index 0000000000..79122927ce --- /dev/null +++ b/tests/test_serve.py @@ -0,0 +1,160 @@ +"""Tests for the turnkey `mempalace serve` command (#1877). + +These exercise the wrapper's security-relevant behavior — token autogeneration +and 0600 persistence, the secure-by-default non-loopback gate, and that the +bearer token is passed via the environment (never argv, so it can't leak via +``ps``) — without binding a real socket. ``cmd_serve`` ends by exec'ing the real +server; we intercept ``os.execve`` to capture the child invocation instead. +""" + +import argparse +import os +import stat + +import pytest + +from mempalace import cli + + +class _ExecCalled(Exception): + """Raised by the patched os.execve to stop cmd_serve at the exec boundary.""" + + +@pytest.fixture +def isolated_home(tmp_path, monkeypatch): + """Point ~ at a temp dir so server token state never touches the real home.""" + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) # Windows + # Don't inherit a token from the ambient environment. + monkeypatch.delenv("MEMPALACE_MCP_HTTP_TOKEN", raising=False) + return tmp_path + + +@pytest.fixture +def capture_exec(monkeypatch): + """Capture the child argv/env cmd_serve would launch, instead of running it. + + cmd_serve takes the os.execve branch on POSIX and the subprocess.run branch + on Windows. Patch both (rather than forcing os.name, which breaks + Path.home() on Windows) so the test is platform-agnostic. + """ + import subprocess + + captured = {} + + def _capture(argv, env): + captured["argv"] = argv + captured["env"] = env + raise _ExecCalled() + + monkeypatch.setattr(cli.os, "execve", lambda path, argv, env: _capture(argv, env)) + monkeypatch.setattr(subprocess, "run", lambda argv, env=None, **kw: _capture(argv, env)) + return captured + + +def _serve_args(tmp_path, **over): + base = dict( + host="127.0.0.1", + port=8765, + backend=None, + global_backend=None, + palace=str(tmp_path / "palace"), + token=None, + tls_cert=None, + tls_key=None, + read_only=False, + allow_insecure=False, + ) + base.update(over) + return argparse.Namespace(**base) + + +def test_token_helper_creates_0600_and_reuses(isolated_home): + palace = str(isolated_home / "palace") + token1, created1 = cli._load_or_create_server_token(palace) + assert created1 is True + assert token1 + + path = cli._server_token_path(palace) + assert path.exists() + if os.name == "posix": + # POSIX permission bits aren't meaningful on Windows (files report 0o666). + mode = stat.S_IMODE(path.stat().st_mode) + assert mode == 0o600, oct(mode) + dir_mode = stat.S_IMODE(path.parent.stat().st_mode) + assert dir_mode == 0o700, oct(dir_mode) + + token2, created2 = cli._load_or_create_server_token(palace) + assert created2 is False + assert token2 == token1 # stable across restarts + + +def test_loopback_serve_needs_no_token(isolated_home, capture_exec): + with pytest.raises(_ExecCalled): + cli.cmd_serve(_serve_args(isolated_home, host="127.0.0.1")) + env = capture_exec["env"] + assert "MEMPALACE_MCP_HTTP_TOKEN" not in env + assert "MEMPALACE_MCP_HTTP_ALLOW_INSECURE_NO_TOKEN" not in env + # No token persisted for a loopback bind. + assert not cli._server_token_path(str(isolated_home / "palace")).exists() + + +def test_non_loopback_autogenerates_token_in_env_not_argv(isolated_home, capture_exec): + with pytest.raises(_ExecCalled): + cli.cmd_serve(_serve_args(isolated_home, host="0.0.0.0")) + env = capture_exec["env"] + argv = capture_exec["argv"] + token = env.get("MEMPALACE_MCP_HTTP_TOKEN") + assert token, "a token must be generated for a network-exposed bind" + # Security: the token rides in the env, never on the command line. + assert all(token not in part for part in argv) + assert "--token" not in argv + # And it was persisted for reuse on the next start (0600 on POSIX). + path = cli._server_token_path(str(isolated_home / "palace")) + assert path.exists() + if os.name == "posix": + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + + +def test_allow_insecure_skips_token_and_sets_escape_hatch(isolated_home, capture_exec): + with pytest.raises(_ExecCalled): + cli.cmd_serve(_serve_args(isolated_home, host="0.0.0.0", allow_insecure=True)) + env = capture_exec["env"] + assert env.get("MEMPALACE_MCP_HTTP_ALLOW_INSECURE_NO_TOKEN") == "1" + assert "MEMPALACE_MCP_HTTP_TOKEN" not in env + assert not cli._server_token_path(str(isolated_home / "palace")).exists() + + +def test_read_only_flag_forwarded_to_child(isolated_home, capture_exec): + with pytest.raises(_ExecCalled): + cli.cmd_serve(_serve_args(isolated_home, read_only=True)) + assert "--read-only" in capture_exec["argv"] + + +def test_explicit_token_is_used_and_not_in_argv(isolated_home, capture_exec): + with pytest.raises(_ExecCalled): + cli.cmd_serve(_serve_args(isolated_home, host="0.0.0.0", token="my-secret-token")) + env = capture_exec["env"] + argv = capture_exec["argv"] + assert env["MEMPALACE_MCP_HTTP_TOKEN"] == "my-secret-token" + assert all("my-secret-token" not in part for part in argv) + # An explicitly-provided token is not persisted to the server token file. + assert not cli._server_token_path(str(isolated_home / "palace")).exists() + + +def test_tls_paths_forwarded_and_validated(isolated_home, capture_exec, tmp_path): + cert = tmp_path / "cert.pem" + key = tmp_path / "key.pem" + cert.write_text("x") + key.write_text("x") + with pytest.raises(_ExecCalled): + cli.cmd_serve(_serve_args(isolated_home, tls_cert=str(cert), tls_key=str(key))) + argv = capture_exec["argv"] + assert "--tls-cert" in argv and "--tls-key" in argv + + +def test_tls_requires_both_cert_and_key(isolated_home, capture_exec, tmp_path): + cert = tmp_path / "cert.pem" + cert.write_text("x") + with pytest.raises(SystemExit): + cli.cmd_serve(_serve_args(isolated_home, tls_cert=str(cert), tls_key=None)) diff --git a/tests/test_sqlite_exact_backend.py b/tests/test_sqlite_exact_backend.py index 796930559d..f6836d4a71 100644 --- a/tests/test_sqlite_exact_backend.py +++ b/tests/test_sqlite_exact_backend.py @@ -4,6 +4,8 @@ import pytest +from _chroma_palace_helper import make_minimal_chroma_sqlite, make_minimal_sqlite_exact_sqlite + import mempalace.backends.sqlite_exact as sqlite_exact_module from mempalace.backends import ( BackendMismatchError, @@ -410,7 +412,7 @@ def test_palace_wrapper_embeds_for_sqlite_exact(tmp_path, monkeypatch): def test_backend_mismatch_protection(tmp_path, monkeypatch): from mempalace.palace import get_collection - (tmp_path / "chroma.sqlite3").write_bytes(b"") + make_minimal_chroma_sqlite(tmp_path) monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "sqlite_exact") with pytest.raises(BackendMismatchError): @@ -420,14 +422,46 @@ def test_backend_mismatch_protection(tmp_path, monkeypatch): def test_mixed_backend_artifacts_are_rejected_even_when_chroma_selected(tmp_path, monkeypatch): from mempalace.palace import resolve_backend_name - (tmp_path / "chroma.sqlite3").write_bytes(b"") - (tmp_path / "sqlite_exact.sqlite3").write_bytes(b"") + make_minimal_chroma_sqlite(tmp_path) + make_minimal_sqlite_exact_sqlite(tmp_path) monkeypatch.setenv("MEMPALACE_BACKEND_EXPLICIT", "chroma") with pytest.raises(BackendMismatchError): resolve_backend_name(str(tmp_path)) +def test_sqlite_exact_detect_matches_palace_with_sqlite_header(tmp_path): + """A real SQLite database at ``/sqlite_exact.sqlite3`` registers + as sqlite_exact. Mirrors the chroma analog at + ``test_chroma_detect_matches_palace_with_sqlite_header``. + """ + make_minimal_sqlite_exact_sqlite(tmp_path) + assert SQLiteExactBackend.detect(str(tmp_path)) is True + assert SQLiteExactBackend.detect(str(tmp_path.parent)) is False + + +def test_sqlite_exact_detect_rejects_empty_sqlite_exact_sqlite(tmp_path): + """A 0-byte ``sqlite_exact.sqlite3`` is not a sqlite_exact palace (#1893). + + Same root cause as the chroma side: bare ``sqlite3.connect()`` against + a missing path leaves a 0-byte file behind because the SQLite header is + written on the first statement, not on connect. Detection must reject + that artifact so it cannot trip ``BackendMismatchError`` against a real + non-sqlite_exact backend marker in the same directory. + """ + (tmp_path / "sqlite_exact.sqlite3").write_bytes(b"") + assert SQLiteExactBackend.detect(str(tmp_path)) is False + + +def test_sqlite_exact_detect_rejects_non_sqlite_file(tmp_path): + """A non-SQLite file at the ``sqlite_exact.sqlite3`` path is not + sqlite_exact. Defends against partial writes / garbage content / anything + that lands at the canonical path but isn't actually a SQLite database. + """ + (tmp_path / "sqlite_exact.sqlite3").write_bytes(b"not a sqlite file" * 4) + assert SQLiteExactBackend.detect(str(tmp_path)) is False + + def test_sqlite_exact_exact_ranking_uses_cosine(tmp_path): _backend, col = _collection(tmp_path) halfway = [0.5, math.sqrt(0.75)] diff --git a/tests/test_sync.py b/tests/test_sync.py index 148bdc61c8..50fba19286 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -11,6 +11,8 @@ import chromadb import pytest +from _chroma_palace_helper import make_minimal_chroma_sqlite + def _seed_drawers(palace_path, repo_path, deleted_path, elsewhere_path): """Populate the drawers collection with 6 entries covering all buckets.""" @@ -1446,7 +1448,7 @@ def test_dry_run_renders_full_report(self, monkeypatch, tmp_dir, capsys): os.makedirs(palace) # Satisfy run_sync's detect_backend_for_path guard without spinning up # the real Chroma/embedder stack (which would disturb sys.stdout). - Path(palace, "chroma.sqlite3").touch() + make_minimal_chroma_sqlite(palace) monkeypatch.setattr( sync_module, "sync_palace", @@ -1472,7 +1474,7 @@ def test_apply_renders_removed_counts(self, monkeypatch, tmp_dir, capsys): palace = os.path.join(tmp_dir, "palace") os.makedirs(palace) - Path(palace, "chroma.sqlite3").touch() + make_minimal_chroma_sqlite(palace) monkeypatch.setattr( sync_module, "sync_palace", diff --git a/website/.vitepress/api-sidebar.json b/website/.vitepress/api-sidebar.json index 35caa5a0ce..cfc64fc354 100644 --- a/website/.vitepress/api-sidebar.json +++ b/website/.vitepress/api-sidebar.json @@ -83,6 +83,10 @@ "text": "mempalace.embedding", "link": "/reference/python-api/embedding" }, + { + "text": "mempalace.entities", + "link": "/reference/python-api/entities" + }, { "text": "mempalace.entity_detector", "link": "/reference/python-api/entity_detector" diff --git a/website/.vitepress/config.mts b/website/.vitepress/config.mts index a0a40ee832..7ad3a8044c 100644 --- a/website/.vitepress/config.mts +++ b/website/.vitepress/config.mts @@ -98,6 +98,7 @@ export default withMermaid( { text: 'Auto-Save Hooks', link: '/guide/hooks' }, { text: 'Cursor IDE Hooks', link: '/guide/cursor-hooks' }, { text: 'Configuration', link: '/guide/configuration' }, + { text: 'Remote / Team Server', link: '/guide/remote-server' }, ], }, ], diff --git a/website/.vitepress/theme/style.css b/website/.vitepress/theme/style.css index 0d231901e7..b95ba484db 100644 --- a/website/.vitepress/theme/style.css +++ b/website/.vitepress/theme/style.css @@ -165,8 +165,26 @@ /* ── Tables ─────────────────────────────────────────────────────────── */ .vp-doc table { + display: block; + /* Keep VitePress's horizontal scroll for wide tables; `overflow: hidden` + here would clip columns that don't fit the content column instead. */ + overflow-x: auto; border-radius: 8px; - overflow: hidden; +} + +/* Slightly denser cells so comparison tables fit the content column without a + horizontal scrollbar (VitePress default is 8px 16px). */ +.vp-doc td, +.vp-doc th { + padding: 8px 12px; +} + +/* Break only inline-code tokens that genuinely can't fit their column (long + description strings), while leaving natural column sizing intact so short + identifiers like `palace_path` stay on one line. `overflow-x: auto` above is + the safety net for any table still wider than the content column. */ +.vp-doc td code { + overflow-wrap: break-word; } .vp-doc th { diff --git a/website/guide/configuration.md b/website/guide/configuration.md index 05b6da20fb..3d4efdb5fb 100644 --- a/website/guide/configuration.md +++ b/website/guide/configuration.md @@ -20,6 +20,75 @@ Located at `~/.mempalace/config.json`: | `people_map` | `{}` | Entity name → AAAK code mappings | | `max_backups` | `10` | How many timestamped palace backups to keep before the oldest are pruned. Applies to `mempalace migrate` (`.pre-migrate.*`) and `mempalace repair max-seq-id` (`chroma.sqlite3.max-seq-id-backup-*`), which each write a full copy every run. Set to `0` to keep every backup (e.g. when an external retention policy manages cleanup). | +## Storage backends + +ChromaDB is the default and needs no configuration. MemPalace also ships a +pluggable backend contract, exercised across deliberately different substrates +(an embedded store, an exact-cosine local store, a REST store, and a SQL/JSONB +store) so the contract is never accidentally shaped around one vendor. Every +non-default backend is opt-in. + +| Backend | Mode | Install | Namespaces | Lexical | +| ------- | ---- | ------- | :--------: | :-----: | +| `chroma` _(default)_ | Local (embedded) | bundled | – | ✓ | +| `sqlite_exact` | Local (exact) | bundled | – | ✓ | +| `qdrant` | Server (REST) | bundled | ✓ | ✓ | +| `pgvector` | Server (Postgres) | `mempalace[pgvector]` | ✓ | ✓ | + + +Select a backend with `--backend ` on any `mempalace` / `mempalace-mcp` +command, `MEMPALACE_BACKEND=` in the environment, or `"backend": ""` +in `config.json`. + +::: warning Verbatim data leaves your machine on opt-in +When a server-mode backend points anywhere other than your own local or trusted +self-hosted service, MemPalace sends and stores verbatim drawer text and +metadata there. That is an explicit, deliberate backend choice — never the +default. +::: + +Server-mode backends isolate tenants by namespace and write a local marker file +(`_backend.json`) in the palace directory, guarding against silently +opening a palace against the wrong server. + +### ChromaDB + +The default. Local, embedded, no service to run. Drawers are stored at +[`palace_path`](#global-config); there are no connection settings to configure. + +### SQLite exact + +Local and built-in (no extra to install). Runs exact cosine over every row — no +ANN index — so it is the reference for exact-vector correctness checks and small +palaces. Select with `--backend sqlite_exact`; it has no connection settings. + +### Qdrant + +A networked REST backend. No driver to install — the client uses the Python +standard library — so you only need a [Qdrant](https://qdrant.tech/) instance +you control. + +| Variable | Default | Description | +| -------- | ------- | ----------- | +| `MEMPALACE_QDRANT_URL` | `http://localhost:6333` | Qdrant REST endpoint | +| `MEMPALACE_QDRANT_API_KEY` | _(none)_ | Sent as the `api-key` header when set | +| `MEMPALACE_QDRANT_NAMESPACE` | _(none)_ | Collection namespace prefix (tenant isolation) | +| `MEMPALACE_QDRANT_TIMEOUT` | `10.0` | REST request timeout, in seconds | + +### Postgres + pgvector + +A networked SQL/JSONB backend. Install the driver with +`pip install mempalace[pgvector]`; the server must have the `vector` extension +available. + +| Variable | Default | Description | +| -------- | ------- | ----------- | +| `MEMPALACE_PGVECTOR_DSN` | `postgresql://localhost:5432/mempalace` | Postgres connection string | +| `MEMPALACE_PGVECTOR_NAMESPACE` | _(none)_ | Schema namespace (tenant isolation) | + +For an end-to-end deployment that puts a server-mode backend behind the MCP +server, see [Remote / Team Server](/guide/remote-server). + ## Project Config Generated by `mempalace init` in your project directory: @@ -86,3 +155,4 @@ python -m mempalace.mcp_server --palace /custom/palace | `MEMPALACE_PALACE_PATH` | Override palace path (same as `--palace`) | | `MEMPAL_DIR` | Directory for auto-mining in hooks | | `MEMPALACE_MAX_BACKUPS` | Override `max_backups` retention count (`0` disables pruning) | +| `MEMPALACE_BACKEND` | Select the storage backend (default `chroma`) — see [Storage backends](#storage-backends) for each backend's connection variables | diff --git a/website/guide/mcp-integration.md b/website/guide/mcp-integration.md index 56ae551906..099e369178 100644 --- a/website/guide/mcp-integration.md +++ b/website/guide/mcp-integration.md @@ -34,7 +34,7 @@ Claude calls `mempalace_search` automatically, gets verbatim results, and answer ## CLI-only mode (`mcp_mode`) -If you want to run the palace **without** loading the 34-tool MCP surface — to save context window, run hooks + skills only, or drive everything from the CLI — set `mcp_mode` to `cli-only`. +If you want to run the palace **without** loading the 39-tool MCP surface — to save context window, run hooks + skills only, or drive everything from the CLI — set `mcp_mode` to `cli-only`. **Config file** (default location `~/.mempalace/config.json`): @@ -50,7 +50,7 @@ Or per-process via env (takes precedence over the config file): PALACE_MCP_MODE=cli-only ``` -Valid values: `"all"` (default — full 34-tool surface) and `"cli-only"`. Anything else — typo, missing config, garbled JSON — **fails open** to `"all"`, so a config bug never silently disables the tools. +Valid values: `"all"` (default — full 39-tool surface) and `"cli-only"`. Anything else — typo, missing config, garbled JSON — **fails open** to `"all"`, so a config bug never silently disables the tools. ### What `cli-only` does diff --git a/website/guide/remote-server.md b/website/guide/remote-server.md new file mode 100644 index 0000000000..f1071db20f --- /dev/null +++ b/website/guide/remote-server.md @@ -0,0 +1,192 @@ +# Remote / Team Server + +Run MemPalace as a **central memory service** that a whole team connects to: +one host stores the palace, does the embedding (optionally on a GPU), and +serves MCP over HTTP. Every teammate's AI reads and writes the same shared +memory instead of a palace on each laptop. + +This is built from three pieces that already ship in MemPalace: + +- the **HTTP transport** for the MCP server (`mempalace-mcp --transport http`), +- a **networked storage backend** ([Qdrant](https://qdrant.tech/) or + [Postgres + pgvector](/guide/configuration)), +- optional **GPU embedding** on the server. + +::: warning This is a deliberate step away from single-machine local-first +By default MemPalace keeps everything on your own machine. A central server is +still **your** infrastructure — no third-party API, no telemetry, nothing +phones home — but your verbatim memory now lives on a server you operate and +travels over your network. Run every component (Qdrant, the MCP host) on +hardware you control, put it on a private network or VPN, and treat the +bearer token and TLS setup below as mandatory, not optional. Embeddings are +still produced locally on the server by MemPalace; only your own storage +backend ever receives the vectors and text. +::: + +## Architecture + +``` + Teammate A ─┐ + Teammate B ─┤ MCP over HTTP ┌─ mempalace-mcp --transport http + Teammate C ─┴──(bearer token, TLS)─▶│ (one host: embedding + GPU) + └─────────────┬─────────────── + │ vectors + verbatim text + ▼ + Qdrant / pgvector + (central storage) +``` + +## 1. Central storage + +Pick a networked backend so all clients share one palace. **Qdrant** needs no +extra Python package — MemPalace talks to its REST API directly. + +Run Qdrant (Docker shown; use a managed/self-hosted instance you control): + +```bash +docker run -d --name qdrant -p 6333:6333 \ + -v "$HOME/qdrant_storage:/qdrant/storage" \ + qdrant/qdrant +``` + +Point MemPalace at it on the server host: + +```bash +export MEMPALACE_BACKEND=qdrant +export MEMPALACE_QDRANT_URL=http://localhost:6333 +export MEMPALACE_QDRANT_API_KEY=your-qdrant-api-key # if your Qdrant requires one +``` + +| Variable | Default | Purpose | +|---|---|---| +| `MEMPALACE_BACKEND` | `chroma` | Set to `qdrant` (or `pgvector`) to select the backend | +| `MEMPALACE_QDRANT_URL` | `http://localhost:6333` | Qdrant REST endpoint | +| `MEMPALACE_QDRANT_API_KEY` | _(none)_ | Sent as the `api-key` header when set | +| `MEMPALACE_QDRANT_NAMESPACE` | _(none)_ | Optional collection namespace prefix | +| `MEMPALACE_QDRANT_TIMEOUT` | backend default | REST request timeout (seconds) | + +The backend can also be set with `--backend qdrant` on any `mempalace` / +`mempalace-mcp` command, or with `"backend": "qdrant"` in `config.json`. + +Prefer Postgres? Install `pip install mempalace[pgvector]`, point +`MEMPALACE_BACKEND=pgvector` at a database with the `vector` extension, and +the rest of this guide applies unchanged. + +## 2. GPU embedding (optional) + +Embedding is the heaviest step; running it on the server's GPU keeps recall +fast for everyone. Install one acceleration extra and select the device: + +```bash +pip install mempalace[gpu] # NVIDIA CUDA (onnxruntime-gpu) +export MEMPALACE_EMBEDDING_DEVICE=cuda +``` + +Other targets: `mempalace[dml]` + `MEMPALACE_EMBEDDING_DEVICE=dml` (DirectML, +Windows AMD/Intel/NVIDIA), `mempalace[coreml]` + `=coreml` (Apple Neural +Engine), or `=auto` to pick the best available provider. CPU is the default +and needs no extra. + +## 3. Serve MCP over HTTP + +One command — `mempalace serve` — runs the server with secure defaults. On a +network-exposed (`0.0.0.0`) bind it **auto-generates a strong bearer token** +(stored `0600` under `~/.mempalace/server/`, printed once), prints a +ready-to-paste client config, and runs in the foreground so Docker/systemd own +the lifecycle. + +```bash +mempalace serve --host 0.0.0.0 --port 8765 --backend qdrant +``` + +Output includes the token and the exact client command. Useful flags: + +| Flag | Default | Purpose | +|---|---|---| +| `--host` | `127.0.0.1` | Bind address (`0.0.0.0` to accept remote clients) | +| `--port` | `8765` | Listen port | +| `--backend` | config/env | Storage backend (e.g. `qdrant`) | +| `--tls-cert` / `--tls-key` | _(none)_ | PEM cert + key to terminate **TLS natively** (server speaks `https`) | +| `--read-only` | off | Expose recall only — the mutating tools are hidden and refused | +| `--token` | auto | Use a specific bearer token instead of the generated one | +| `--allow-insecure` | off | Permit a non-loopback bind with no token (only behind a trusted proxy) | + +The token always travels via the environment, never the command line, so it +can't leak through `ps`. Binding to a non-loopback host with no token and no +`--allow-insecure` refuses to start. The server also guards against +DNS-rebinding with a `Host` allowlist and an `Origin` loopback check, and +serializes concurrent writes — so multiple teammates can write to the shared +palace at once over HTTP. + +::: tip TLS +Pass `--tls-cert`/`--tls-key` to terminate TLS in the server itself +(`https://…`). Otherwise the server is plaintext and you should front it with a +TLS-terminating reverse proxy (nginx/Caddy/Traefik) — never expose plaintext +`/mcp` beyond a trusted private network. +::: + +The underlying server is `mempalace-mcp --transport http` (the same flags exist +there if you'd rather wire the token/TLS yourself); `mempalace serve` is the +turnkey wrapper over it. + +## 4. Connect a client + +Point each teammate's MCP client at the server's `/mcp` endpoint with the +shared token. For Claude Code: + +```bash +claude mcp add --transport http mempalace https://memory.example.com/mcp \ + --header "Authorization: Bearer $MEMPALACE_MCP_HTTP_TOKEN" +``` + +Other MCP clients use the same two ingredients — the `…/mcp` URL and an +`Authorization: Bearer ` header. Verify connectivity from any host: + +```bash +curl https://memory.example.com/healthz # -> ok +``` + +Once connected, all of MemPalace's [MCP tools](/guide/mcp-integration) operate +against the shared palace — searches and saved memories are visible to the +whole team. + +## Operating notes + +- **Mining** still happens via the CLI (`mempalace mine …`) on the server host + against the same backend, so the central palace stays populated. +- **One writer-lease per process**: a single `mempalace-mcp --transport http` + process safely handles concurrent reads and writes. Don't point two server + processes at the same backend collection. +- **Health checks**: `GET /healthz` returns `200 ok` without a token, so it + works as a load-balancer/Kubernetes liveness probe. +- **Backups** are now your storage backend's responsibility (Qdrant snapshots + / Postgres backups) rather than a single laptop's palace directory. + +## One-command deployments + +The repo ships ready-to-edit deployment files under +[`deploy/`](https://github.com/MemPalace/mempalace/tree/main/deploy): + +**Docker Compose (server + Qdrant):** + +```bash +cp deploy/server.env.example deploy/.env # set MEMPALACE_MCP_HTTP_TOKEN +docker compose -f deploy/docker-compose.server.yml --env-file deploy/.env up -d +``` + +This brings up a Qdrant container and a MemPalace server running +`serve --host 0.0.0.0 --backend qdrant`, with a `/healthz` healthcheck and +persistent volumes. Embeddings stay local to the MemPalace container. + +**systemd:** + +`deploy/mempalace-server.service` is a hardened unit template +(`NoNewPrivileges`, `ProtectSystem=strict`, dedicated user) that runs +`mempalace serve` with its config from `/etc/mempalace/server.env`. Install +steps are in the file's header comment. + +## See also + +- [MCP Integration](/guide/mcp-integration) — the tools clients get once connected +- [Configuration](/guide/configuration) — config file, identity, environment variables +- [Local Models](/guide/local-models) — keeping embedding and any LLM assist local diff --git a/website/public/llms-full.txt b/website/public/llms-full.txt index af51d80025..e2ecddfae3 100644 --- a/website/public/llms-full.txt +++ b/website/public/llms-full.txt @@ -46,7 +46,7 @@ Files included, in order: ## What this is -A verbatim-first local AI memory system. This fork tracks `upstream/develop` through the v3.5.0 sync (2026-06-26, commit `73e74bf`) and runs in production on a **409K+ drawer Postgres + pgvector + Apache AGE palace** behind [palace-daemon](https://github.com/techempower-org/palace-daemon). It carries fork-ahead commits that compose with — not replace — bensig's release direction; the v3.3.5 release (2026-05-10) includes our co-authored `_get_collection` retry-once via upstream #1377. 4830 tests pass on `main`. +A verbatim-first local AI memory system. This fork tracks `upstream/develop` through the post-v3.5.0 sync (2026-07-02, commit `da5a48c`) and runs in production on a **411K+ drawer Postgres + pgvector + Apache AGE palace** behind [palace-daemon](https://github.com/techempower-org/palace-daemon). It carries fork-ahead commits that compose with — not replace — bensig's release direction; the v3.3.5 release (2026-05-10) includes our co-authored `_get_collection` retry-once via upstream #1377. 4921 tests pass on `main`. The fork's architectural thinking — the four-layer memory model, the [verbatim-vs-derivative thesis](docs/research/verbatim-vs-derivative-axis.md), design principles, and the two-memory-layer pairing with Auto Dream — lives in [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md). The new things here are *what we've learned*, not just what we've fixed. @@ -256,99 +256,100 @@ The full enumeration of fork-ahead changes. The canonical source is [`docs/fork- | # | Description | Upstream PR | Fork commit | |---|---|---|---| -| 1 | Auto-query firing fixes: frozen turn counter, dead wing scoring, lowercase entities, turn-1 cadence | — | [`fad3e27`](https://github.com/techempower-org/mempalace/commit/fad3e27) | -| 2 | Auto-query: periodic depth signal, unknown-entity 0->1 bump, broader temporal patterns | — | [`864d7a4`](https://github.com/techempower-org/mempalace/commit/864d7a4) | -| 3 | Sync upstream/develop through v3.5.0 (73e74bf): MCP HTTP transport, source_file filter, checkpoint tool, SessionEnd hook, 185 commits | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 4 | Restore concurrent file mining via parallel-prepare/serial-write (regression from a dropped sync hunk); opt-in --workers | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 5 | Sync upstream/develop through v3.4.0 (2ec4bae): RFC-001 backend stack, diary checkpoints restored, 113 commits | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 6 | auto_wake: opt-in wake-on-demand for a sleeping palace-daemon host (wake command + /health poll + single retry) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 7 | AGE graph-walk: auto edge-endpoint indexes in backfill + bind anonymous RELATION targets (mempalace#335) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 8 | pluggable adaptmem_ft encoder backend selectable via MEMPALACE_EMBEDDING_MODEL (closes #308) | — | [`5fba6d8`](https://github.com/techempower-org/mempalace/commit/5fba6d8) | -| 9 | README.md landscape table — refresh upstream MemPalace star count from ~23K → ~53K (current 2026-05-28) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 10 | README.md + docs/ECOSYSTEM.md — soften 'engram-2 17% E2E QA' framing per the 2026-05-24 research doc's unsubstantiated finding (#319) | — | [`ddf00b4`](https://github.com/techempower-org/mempalace/commit/ddf00b4) | -| 11 | kg_llm_extractor rewrites AGE dollar-quote tag in triples so drawers indexing palace source code don't fail at add_triple (#313) | — | [`3fb9428`](https://github.com/techempower-org/mempalace/commit/3fb9428) | -| 12 | scripts/maintain-fork-changes.py + ship-prep step 1: resolve commit:HEAD placeholders and de-dup yaml entries (#316) | — | [`9060e09`](https://github.com/techempower-org/mempalace/commit/9060e09) | -| 13 | scripts/ship-prep.sh — one command bumps README test count and runs all three doc renderers (#312) | — | [`4677db8`](https://github.com/techempower-org/mempalace/commit/4677db8) | -| 14 | mempalace_search MCP input schema accepts fusion_mode (convex\|rrf) and forwards to search_memories (#302) | — | [`f753ec4`](https://github.com/techempower-org/mempalace/commit/f753ec4) | -| 15 | scripts/check-docs.sh finds pytest via main checkout when run from a worktree, fails hard instead of silently skipping test-count check (#311) | — | [`1d19a8b`](https://github.com/techempower-org/mempalace/commit/1d19a8b) | -| 16 | kg_triple_worker retries add_triple within-worker on transient psycopg errors instead of abandoning to lease-reclaim (#298) | — | [`36c0b02`](https://github.com/techempower-org/mempalace/commit/36c0b02) | -| 17 | mempalace_kg_stats returns structured backend-unavailable envelope on transient psycopg failures (#299) | — | [`8fd0b01`](https://github.com/techempower-org/mempalace/commit/8fd0b01) | -| 18 | mempalace why + tunnels — explain a drawer + inventory cross-wing tunnels (slice of #191) | — | [`fdcd0b4`](https://github.com/techempower-org/mempalace/commit/fdcd0b4) | -| 19 | RRF vs convex-blend rerank — A/B measurement on our corpus (#162) | — | [`ea5d567`](https://github.com/techempower-org/mempalace/commit/ea5d567) | -| 20 | KG triples gain SPOC context slot + worker auto-derives valid_from from drawer metadata (#161) | — | [`b87ce05`](https://github.com/techempower-org/mempalace/commit/b87ce05) | -| 21 | mempalace bulk-move — multi-drawer metadata relocation by source wing/room (#191) | — | [`1ca544b`](https://github.com/techempower-org/mempalace/commit/1ca544b) | -| 22 | mempalace move — fast direct-to-daemon single-drawer wing/room relocation (#191) | — | [`d007b6f`](https://github.com/techempower-org/mempalace/commit/d007b6f) | -| 23 | mempalace stats migrates to GET /stats REST + exposes graph/status sections (#191) | — | [`853bb25`](https://github.com/techempower-org/mempalace/commit/853bb25) | -| 24 | mempalace cypher — read-only Cypher query CLI (#191) | — | [`32a41b1`](https://github.com/techempower-org/mempalace/commit/32a41b1) | -| 25 | mempalace graph — fast direct-to-daemon KG structural snapshot (#191) | — | [`499f42d`](https://github.com/techempower-org/mempalace/commit/499f42d) | -| 26 | mempalace list — fast direct-to-daemon drawer browser (#191) | — | [`257137b`](https://github.com/techempower-org/mempalace/commit/257137b) | -| 27 | Recency decay weighting in search + mempalace prune --stale-days CLI (#158) | — | [`558d327`](https://github.com/techempower-org/mempalace/commit/558d327) | -| 28 | mempalace_rate_memory MCP tool + bounded rating signal in search ranking (#159) | — | [`583536c`](https://github.com/techempower-org/mempalace/commit/583536c) | -| 29 | Formalize wing/room derivation order; demote entity detector to last-resort hint (#157) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 30 | RRF fusion mode + convex-vs-RRF A/B harness (#162) | [#247](https://github.com/MemPalace/mempalace/pull/247) | [`6c9d10c`](https://github.com/techempower-org/mempalace/commit/6c9d10c) | -| 31 | mempalace stats: add ROOMS breakdown (drawer count by room) to the dashboard | — | [`1673465`](https://github.com/techempower-org/mempalace/commit/1673465) | -| 32 | Calibrated confidence field on search results + Brier-score eval column | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 33 | Evaluation doc: curated-authority vs auto-mined separation (#202) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 34 | Apply AGE statement_timeout in same transaction as cypher() (PR #228 follow-up) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 35 | LLM-based KG triple extraction: queue table, async worker, llama.cpp on familiar | — | [`59ac0bc`](https://github.com/techempower-org/mempalace/commit/59ac0bc) | -| 36 | Promote verbatim-vs-derivative essay from research/ to README (#170) | — | [`6a264d9`](https://github.com/techempower-org/mempalace/commit/6a264d9) | -| 37 | mempalace stats — palace analytics dashboard (#191) | — | [`6f994fb`](https://github.com/techempower-org/mempalace/commit/6f994fb) | -| 38 | CLI wiring: mempalace mine --source (#57) | — | [`5ed9fa7`](https://github.com/techempower-org/mempalace/commit/5ed9fa7) | -| 39 | Warp terminal source adapter (#62) | — | [`2e85585`](https://github.com/techempower-org/mempalace/commit/2e85585) | -| 40 | OpenCode adapter smoke test against real DB (#56) | — | [`a9ed72b`](https://github.com/techempower-org/mempalace/commit/a9ed72b) | -| 41 | Codex, Gemini, and Aider source adapters (#61, #59) | — | [`0c23165`](https://github.com/techempower-org/mempalace/commit/0c23165) | -| 42 | Filesystem + conversation source adapters (#63) | — | [`9a1facf`](https://github.com/techempower-org/mempalace/commit/9a1facf) | -| 43 | Widen auto-query signal patterns for natural recall phrases | — | [`33e780e`](https://github.com/techempower-org/mempalace/commit/33e780e) | -| 44 | Native rename_wing backend operation + CLI command (#154) | — | [`d045f83`](https://github.com/techempower-org/mempalace/commit/d045f83) | -| 45 | Standalone essay: the verbatim-vs-derivative axis (#47) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 46 | Research doc: uncertainty-aware retrieval analysis (#84) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 47 | Design doc: scope/collection filter on mempalace_search (#76) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 48 | Agent-shaped CLI surface — --json / --quiet for non-MCP integration | — | [`25ed900`](https://github.com/techempower-org/mempalace/commit/25ed900) | -| 49 | Design eval: multi-palace separation — curated vs auto-mined (#45) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | -| 50 | Document .sh shim delegation to palace-daemon (counter-position to upstream #1069) | — | [`bf0a4d0`](https://github.com/techempower-org/mempalace/commit/bf0a4d0) | -| 51 | Honor ~/.mempalace/RETIRED marker — refuse default palace, surface retire message | — | [`798cf14`](https://github.com/techempower-org/mempalace/commit/798cf14) | -| 52 | Empty repo .opencode/opencode.json mcp block — disabled flag wasn't being respected | — | [`7133eee`](https://github.com/techempower-org/mempalace/commit/7133eee) | -| 53 | Drop \$comment from .opencode/opencode.json — schema rejects unknown root keys | — | [`637bb01`](https://github.com/techempower-org/mempalace/commit/637bb01) | -| 54 | Disable repo-level MCP entry by default + venv-python fallback | — | [`47018e5`](https://github.com/techempower-org/mempalace/commit/47018e5) | -| 55 | Stub resources/list + prompts/list so MCP clients stop ERROR-logging on connect | — | [`6ca0670`](https://github.com/techempower-org/mempalace/commit/6ca0670) | -| 56 | Bundled OpenCode live-capture plugin that bypasses option-K v1.2.1 bugs (filed upstream as #4, #5) | — | [`5522623`](https://github.com/techempower-org/mempalace/commit/5522623) | -| 57 | Documented OpenCode integration recipe (read-side MCP + push plugin + retrospective adapter) | — | [`60dc9e6`](https://github.com/techempower-org/mempalace/commit/60dc9e6) | -| 58 | .opencode/opencode.json — repo-root MCP config so opencode picks up mempalace automatically | [#1567](https://github.com/MemPalace/mempalace/pull/1567) (OPEN) | [`ba16b82`](https://github.com/techempower-org/mempalace/commit/ba16b82) | -| 59 | OpenCodeSourceAdapter (RFC 002) — retrospective ingest of OpenCode SQLite sessions | [#1484](https://github.com/MemPalace/mempalace/pull/1484) (OPEN) | [`2ffe652`](https://github.com/techempower-org/mempalace/commit/2ffe652) | -| 60 | mempalace_walk_palace MCP tool — agent walks the palace via AGE Cypher | — | [`8022ecb`](https://github.com/techempower-org/mempalace/commit/8022ecb) | -| 61 | Backfill AGE graph from existing drawer table — restartable, checkpointed | — | [`b3f0206`](https://github.com/techempower-org/mempalace/commit/b3f0206) | -| 62 | Wing/Room/Drawer hierarchy as native AGE nodes; Cypher MATCH walks palace structure | — | [`ff583c0`](https://github.com/techempower-org/mempalace/commit/ff583c0) | -| 63 | Write-through middleware on PostgresCollection — entities populate AGE on every drawer write | — | [`3321d83`](https://github.com/techempower-org/mempalace/commit/3321d83) | -| 64 | KnowledgeGraphAGE API parity with SQLite KG: add_entity, invalidate, query_entity, query_relationship, timeline, seed_from_entity_facts | — | [`ff7187d`](https://github.com/techempower-org/mempalace/commit/ff7187d) | -| 65 | Pending-writes journal + replay so daemon outages stop being silent | — | [`0c34464`](https://github.com/techempower-org/mempalace/commit/0c34464) | -| 66 | MCP server distinguishes 'backend unreachable' from 'no palace found' | — | [`0c34464`](https://github.com/techempower-org/mempalace/commit/0c34464) | -| 67 | Defense-in-depth metadata sanitizer at the chromadb-client chokepoint | — | [`f499814`](https://github.com/techempower-org/mempalace/commit/f499814) | -| 68 | Route Stop/PreCompact hooks through palace-daemon/clients/hook.py | — | [`42ded2e`](https://github.com/techempower-org/mempalace/commit/42ded2e) | -| 69 | KnowledgeGraphAGE skeleton — Apache AGE graph bootstrap over psycopg2 | — | [`a3ee623`](https://github.com/techempower-org/mempalace/commit/a3ee623) | -| 70 | README pivots to the four-layer model + Auto Dream as vindication of the verbatim-vs-derivative axis | — | [`55b36ca`](https://github.com/techempower-org/mempalace/commit/55b36ca) | -| 71 | CI: gate postgres-backend tests against a pgvector service container | — | [`da0bdbb`](https://github.com/techempower-org/mempalace/commit/da0bdbb) | -| 72 | PostgreSQL backend via #665 cherry-pick + fork-side adaptations + smoke tests | [#665](https://github.com/MemPalace/mempalace/pull/665) (OPEN) | [`5e90c72`](https://github.com/techempower-org/mempalace/commit/5e90c72) | -| 73 | daemon-route `mempalace status` / `search` / `mine` when PALACE_DAEMON_URL is set | — | [`22ef562`](https://github.com/techempower-org/mempalace/commit/22ef562) | -| 74 | daemon-route `mcp_server.py` via the `handle_request` JSON-RPC chokepoint | — | [`41359ba`](https://github.com/techempower-org/mempalace/commit/41359ba) | -| 75 | Preserve dashed project names in transcript-derived wings | [#10](https://github.com/MemPalace/mempalace/pull/10) | [`d76134d`](https://github.com/techempower-org/mempalace/commit/d76134d) | -| 76 | Drop wing_ prefix from transcript-derived wings to converge with operator mines | [#9](https://github.com/MemPalace/mempalace/pull/9) | [`86d4700`](https://github.com/techempower-org/mempalace/commit/86d4700) | -| 77 | Retire mempalace_session_recovery collection + read tool | [#8](https://github.com/MemPalace/mempalace/pull/8) | [`0b945e1`](https://github.com/techempower-org/mempalace/commit/0b945e1) | -| 78 | mempalace mined + purge --source-file (mining management surface) | [#7](https://github.com/MemPalace/mempalace/pull/7) | [`2e6ced9`](https://github.com/techempower-org/mempalace/commit/2e6ced9) | -| 79 | Drop hook-side checkpoint diary writes — verbatim-only architecture | [#6](https://github.com/MemPalace/mempalace/pull/6) | [`69768fc`](https://github.com/techempower-org/mempalace/commit/69768fc) | -| 80 | Restore transcript ingest via daemon /mine when PALACE_DAEMON_URL is set | [#2](https://github.com/MemPalace/mempalace/pull/2) | [`09d2ca6`](https://github.com/techempower-org/mempalace/commit/09d2ca6) | -| 81 | `hook_verbatim_mode` config flag preserves system tags + full tool I/O during transcript ingest | — | [`ef98961`](https://github.com/techempower-org/mempalace/commit/ef98961) | -| 82 | Retire the `kind=` filter — structural split made it inert | — | [`7ba28dc`](https://github.com/techempower-org/mempalace/commit/7ba28dc) | -| 83 | Hoist CLOSET_RANK_BOOSTS to module level + record VecRecall ablation finding | — | [`3cb03f3`](https://github.com/techempower-org/mempalace/commit/3cb03f3) | -| 84 | Strip embedded API key from .claude-plugin/ manifests; rely on env inheritance | — | [`9f91e18`](https://github.com/techempower-org/mempalace/commit/9f91e18) | -| 85 | Cherry-pick #1094 — coerce None metadatas at chromadb boundary | [#1094](https://github.com/MemPalace/mempalace/pull/1094) (OPEN) | [`43d728d`](https://github.com/techempower-org/mempalace/commit/43d728d) | -| 86 | Cherry-pick #1087 rewrite — collection.delete(where=) instead of nuke-and-rebuild | [#1087](https://github.com/MemPalace/mempalace/pull/1087) (OPEN) | [`366a9ad`](https://github.com/techempower-org/mempalace/commit/366a9ad) | -| 87 | Canonical YAML manifest + renderer for fork-ahead docs | — | [`5a01aec`](https://github.com/techempower-org/mempalace/commit/5a01aec) | -| 88 | Phase D migration + PreCompact recovery write | — | [`42817d7`](https://github.com/techempower-org/mempalace/commit/42817d7) | -| 89 | Surface drawer_id in search/diary/recovery payloads | — | [`9a8bb77`](https://github.com/techempower-org/mempalace/commit/9a8bb77) | -| 90 | Cherry-pick #1085 — batch ChromaDB inserts in miner (10–30× faster) | [#1085](https://github.com/MemPalace/mempalace/pull/1085) (CLOSED) | [`6be6fff`](https://github.com/techempower-org/mempalace/commit/6be6fff) | -| 91 | scripts/deploy.sh — one-command Syncthing-aware redeploy | — | [`8252025`](https://github.com/techempower-org/mempalace/commit/8252025) | -| 92 | Phases A–C of the checkpoint collection split | — | [`e266365`](https://github.com/techempower-org/mempalace/commit/e266365) | -| 93 | kind= filter on search_memories excludes Stop-hook checkpoints (transitional) | — | [`f9f5cc4`](https://github.com/techempower-org/mempalace/commit/f9f5cc4) | +| 1 | Sync upstream/develop through da5a48c (post-v3.5.0): remote MCP server w/ TLS + read-only, graph auto-population, Qdrant facets, list_drawers date filters, 213 commits | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 2 | Auto-query firing fixes: frozen turn counter, dead wing scoring, lowercase entities, turn-1 cadence | — | [`fad3e27`](https://github.com/techempower-org/mempalace/commit/fad3e27) | +| 3 | Auto-query: periodic depth signal, unknown-entity 0->1 bump, broader temporal patterns | — | [`864d7a4`](https://github.com/techempower-org/mempalace/commit/864d7a4) | +| 4 | Sync upstream/develop through v3.5.0 (73e74bf): MCP HTTP transport, source_file filter, checkpoint tool, SessionEnd hook, 185 commits | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 5 | Restore concurrent file mining via parallel-prepare/serial-write (regression from a dropped sync hunk); opt-in --workers | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 6 | Sync upstream/develop through v3.4.0 (2ec4bae): RFC-001 backend stack, diary checkpoints restored, 113 commits | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 7 | auto_wake: opt-in wake-on-demand for a sleeping palace-daemon host (wake command + /health poll + single retry) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 8 | AGE graph-walk: auto edge-endpoint indexes in backfill + bind anonymous RELATION targets (mempalace#335) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 9 | pluggable adaptmem_ft encoder backend selectable via MEMPALACE_EMBEDDING_MODEL (closes #308) | — | [`5fba6d8`](https://github.com/techempower-org/mempalace/commit/5fba6d8) | +| 10 | README.md landscape table — refresh upstream MemPalace star count from ~23K → ~53K (current 2026-05-28) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 11 | README.md + docs/ECOSYSTEM.md — soften 'engram-2 17% E2E QA' framing per the 2026-05-24 research doc's unsubstantiated finding (#319) | — | [`ddf00b4`](https://github.com/techempower-org/mempalace/commit/ddf00b4) | +| 12 | kg_llm_extractor rewrites AGE dollar-quote tag in triples so drawers indexing palace source code don't fail at add_triple (#313) | — | [`3fb9428`](https://github.com/techempower-org/mempalace/commit/3fb9428) | +| 13 | scripts/maintain-fork-changes.py + ship-prep step 1: resolve commit:HEAD placeholders and de-dup yaml entries (#316) | — | [`9060e09`](https://github.com/techempower-org/mempalace/commit/9060e09) | +| 14 | scripts/ship-prep.sh — one command bumps README test count and runs all three doc renderers (#312) | — | [`4677db8`](https://github.com/techempower-org/mempalace/commit/4677db8) | +| 15 | mempalace_search MCP input schema accepts fusion_mode (convex\|rrf) and forwards to search_memories (#302) | — | [`f753ec4`](https://github.com/techempower-org/mempalace/commit/f753ec4) | +| 16 | scripts/check-docs.sh finds pytest via main checkout when run from a worktree, fails hard instead of silently skipping test-count check (#311) | — | [`1d19a8b`](https://github.com/techempower-org/mempalace/commit/1d19a8b) | +| 17 | kg_triple_worker retries add_triple within-worker on transient psycopg errors instead of abandoning to lease-reclaim (#298) | — | [`36c0b02`](https://github.com/techempower-org/mempalace/commit/36c0b02) | +| 18 | mempalace_kg_stats returns structured backend-unavailable envelope on transient psycopg failures (#299) | — | [`8fd0b01`](https://github.com/techempower-org/mempalace/commit/8fd0b01) | +| 19 | mempalace why + tunnels — explain a drawer + inventory cross-wing tunnels (slice of #191) | — | [`fdcd0b4`](https://github.com/techempower-org/mempalace/commit/fdcd0b4) | +| 20 | RRF vs convex-blend rerank — A/B measurement on our corpus (#162) | — | [`ea5d567`](https://github.com/techempower-org/mempalace/commit/ea5d567) | +| 21 | KG triples gain SPOC context slot + worker auto-derives valid_from from drawer metadata (#161) | — | [`b87ce05`](https://github.com/techempower-org/mempalace/commit/b87ce05) | +| 22 | mempalace bulk-move — multi-drawer metadata relocation by source wing/room (#191) | — | [`1ca544b`](https://github.com/techempower-org/mempalace/commit/1ca544b) | +| 23 | mempalace move — fast direct-to-daemon single-drawer wing/room relocation (#191) | — | [`d007b6f`](https://github.com/techempower-org/mempalace/commit/d007b6f) | +| 24 | mempalace stats migrates to GET /stats REST + exposes graph/status sections (#191) | — | [`853bb25`](https://github.com/techempower-org/mempalace/commit/853bb25) | +| 25 | mempalace cypher — read-only Cypher query CLI (#191) | — | [`32a41b1`](https://github.com/techempower-org/mempalace/commit/32a41b1) | +| 26 | mempalace graph — fast direct-to-daemon KG structural snapshot (#191) | — | [`499f42d`](https://github.com/techempower-org/mempalace/commit/499f42d) | +| 27 | mempalace list — fast direct-to-daemon drawer browser (#191) | — | [`257137b`](https://github.com/techempower-org/mempalace/commit/257137b) | +| 28 | Recency decay weighting in search + mempalace prune --stale-days CLI (#158) | — | [`558d327`](https://github.com/techempower-org/mempalace/commit/558d327) | +| 29 | mempalace_rate_memory MCP tool + bounded rating signal in search ranking (#159) | — | [`583536c`](https://github.com/techempower-org/mempalace/commit/583536c) | +| 30 | Formalize wing/room derivation order; demote entity detector to last-resort hint (#157) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 31 | RRF fusion mode + convex-vs-RRF A/B harness (#162) | [#247](https://github.com/MemPalace/mempalace/pull/247) | [`6c9d10c`](https://github.com/techempower-org/mempalace/commit/6c9d10c) | +| 32 | mempalace stats: add ROOMS breakdown (drawer count by room) to the dashboard | — | [`1673465`](https://github.com/techempower-org/mempalace/commit/1673465) | +| 33 | Calibrated confidence field on search results + Brier-score eval column | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 34 | Evaluation doc: curated-authority vs auto-mined separation (#202) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 35 | Apply AGE statement_timeout in same transaction as cypher() (PR #228 follow-up) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 36 | LLM-based KG triple extraction: queue table, async worker, llama.cpp on familiar | — | [`59ac0bc`](https://github.com/techempower-org/mempalace/commit/59ac0bc) | +| 37 | Promote verbatim-vs-derivative essay from research/ to README (#170) | — | [`6a264d9`](https://github.com/techempower-org/mempalace/commit/6a264d9) | +| 38 | mempalace stats — palace analytics dashboard (#191) | — | [`6f994fb`](https://github.com/techempower-org/mempalace/commit/6f994fb) | +| 39 | CLI wiring: mempalace mine --source (#57) | — | [`5ed9fa7`](https://github.com/techempower-org/mempalace/commit/5ed9fa7) | +| 40 | Warp terminal source adapter (#62) | — | [`2e85585`](https://github.com/techempower-org/mempalace/commit/2e85585) | +| 41 | OpenCode adapter smoke test against real DB (#56) | — | [`a9ed72b`](https://github.com/techempower-org/mempalace/commit/a9ed72b) | +| 42 | Codex, Gemini, and Aider source adapters (#61, #59) | — | [`0c23165`](https://github.com/techempower-org/mempalace/commit/0c23165) | +| 43 | Filesystem + conversation source adapters (#63) | — | [`9a1facf`](https://github.com/techempower-org/mempalace/commit/9a1facf) | +| 44 | Widen auto-query signal patterns for natural recall phrases | — | [`33e780e`](https://github.com/techempower-org/mempalace/commit/33e780e) | +| 45 | Native rename_wing backend operation + CLI command (#154) | — | [`d045f83`](https://github.com/techempower-org/mempalace/commit/d045f83) | +| 46 | Standalone essay: the verbatim-vs-derivative axis (#47) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 47 | Research doc: uncertainty-aware retrieval analysis (#84) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 48 | Design doc: scope/collection filter on mempalace_search (#76) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 49 | Agent-shaped CLI surface — --json / --quiet for non-MCP integration | — | [`25ed900`](https://github.com/techempower-org/mempalace/commit/25ed900) | +| 50 | Design eval: multi-palace separation — curated vs auto-mined (#45) | — | [`TBD`](https://github.com/techempower-org/mempalace/commit/TBD) | +| 51 | Document .sh shim delegation to palace-daemon (counter-position to upstream #1069) | — | [`bf0a4d0`](https://github.com/techempower-org/mempalace/commit/bf0a4d0) | +| 52 | Honor ~/.mempalace/RETIRED marker — refuse default palace, surface retire message | — | [`798cf14`](https://github.com/techempower-org/mempalace/commit/798cf14) | +| 53 | Empty repo .opencode/opencode.json mcp block — disabled flag wasn't being respected | — | [`7133eee`](https://github.com/techempower-org/mempalace/commit/7133eee) | +| 54 | Drop \$comment from .opencode/opencode.json — schema rejects unknown root keys | — | [`637bb01`](https://github.com/techempower-org/mempalace/commit/637bb01) | +| 55 | Disable repo-level MCP entry by default + venv-python fallback | — | [`47018e5`](https://github.com/techempower-org/mempalace/commit/47018e5) | +| 56 | Stub resources/list + prompts/list so MCP clients stop ERROR-logging on connect | — | [`6ca0670`](https://github.com/techempower-org/mempalace/commit/6ca0670) | +| 57 | Bundled OpenCode live-capture plugin that bypasses option-K v1.2.1 bugs (filed upstream as #4, #5) | — | [`5522623`](https://github.com/techempower-org/mempalace/commit/5522623) | +| 58 | Documented OpenCode integration recipe (read-side MCP + push plugin + retrospective adapter) | — | [`60dc9e6`](https://github.com/techempower-org/mempalace/commit/60dc9e6) | +| 59 | .opencode/opencode.json — repo-root MCP config so opencode picks up mempalace automatically | [#1567](https://github.com/MemPalace/mempalace/pull/1567) (OPEN) | [`ba16b82`](https://github.com/techempower-org/mempalace/commit/ba16b82) | +| 60 | OpenCodeSourceAdapter (RFC 002) — retrospective ingest of OpenCode SQLite sessions | [#1484](https://github.com/MemPalace/mempalace/pull/1484) (OPEN) | [`2ffe652`](https://github.com/techempower-org/mempalace/commit/2ffe652) | +| 61 | mempalace_walk_palace MCP tool — agent walks the palace via AGE Cypher | — | [`8022ecb`](https://github.com/techempower-org/mempalace/commit/8022ecb) | +| 62 | Backfill AGE graph from existing drawer table — restartable, checkpointed | — | [`b3f0206`](https://github.com/techempower-org/mempalace/commit/b3f0206) | +| 63 | Wing/Room/Drawer hierarchy as native AGE nodes; Cypher MATCH walks palace structure | — | [`ff583c0`](https://github.com/techempower-org/mempalace/commit/ff583c0) | +| 64 | Write-through middleware on PostgresCollection — entities populate AGE on every drawer write | — | [`3321d83`](https://github.com/techempower-org/mempalace/commit/3321d83) | +| 65 | KnowledgeGraphAGE API parity with SQLite KG: add_entity, invalidate, query_entity, query_relationship, timeline, seed_from_entity_facts | — | [`ff7187d`](https://github.com/techempower-org/mempalace/commit/ff7187d) | +| 66 | Pending-writes journal + replay so daemon outages stop being silent | — | [`0c34464`](https://github.com/techempower-org/mempalace/commit/0c34464) | +| 67 | MCP server distinguishes 'backend unreachable' from 'no palace found' | — | [`0c34464`](https://github.com/techempower-org/mempalace/commit/0c34464) | +| 68 | Defense-in-depth metadata sanitizer at the chromadb-client chokepoint | — | [`f499814`](https://github.com/techempower-org/mempalace/commit/f499814) | +| 69 | Route Stop/PreCompact hooks through palace-daemon/clients/hook.py | — | [`42ded2e`](https://github.com/techempower-org/mempalace/commit/42ded2e) | +| 70 | KnowledgeGraphAGE skeleton — Apache AGE graph bootstrap over psycopg2 | — | [`a3ee623`](https://github.com/techempower-org/mempalace/commit/a3ee623) | +| 71 | README pivots to the four-layer model + Auto Dream as vindication of the verbatim-vs-derivative axis | — | [`55b36ca`](https://github.com/techempower-org/mempalace/commit/55b36ca) | +| 72 | CI: gate postgres-backend tests against a pgvector service container | — | [`da0bdbb`](https://github.com/techempower-org/mempalace/commit/da0bdbb) | +| 73 | PostgreSQL backend via #665 cherry-pick + fork-side adaptations + smoke tests | [#665](https://github.com/MemPalace/mempalace/pull/665) (OPEN) | [`5e90c72`](https://github.com/techempower-org/mempalace/commit/5e90c72) | +| 74 | daemon-route `mempalace status` / `search` / `mine` when PALACE_DAEMON_URL is set | — | [`22ef562`](https://github.com/techempower-org/mempalace/commit/22ef562) | +| 75 | daemon-route `mcp_server.py` via the `handle_request` JSON-RPC chokepoint | — | [`41359ba`](https://github.com/techempower-org/mempalace/commit/41359ba) | +| 76 | Preserve dashed project names in transcript-derived wings | [#10](https://github.com/MemPalace/mempalace/pull/10) | [`d76134d`](https://github.com/techempower-org/mempalace/commit/d76134d) | +| 77 | Drop wing_ prefix from transcript-derived wings to converge with operator mines | [#9](https://github.com/MemPalace/mempalace/pull/9) | [`86d4700`](https://github.com/techempower-org/mempalace/commit/86d4700) | +| 78 | Retire mempalace_session_recovery collection + read tool | [#8](https://github.com/MemPalace/mempalace/pull/8) | [`0b945e1`](https://github.com/techempower-org/mempalace/commit/0b945e1) | +| 79 | mempalace mined + purge --source-file (mining management surface) | [#7](https://github.com/MemPalace/mempalace/pull/7) | [`2e6ced9`](https://github.com/techempower-org/mempalace/commit/2e6ced9) | +| 80 | Drop hook-side checkpoint diary writes — verbatim-only architecture | [#6](https://github.com/MemPalace/mempalace/pull/6) | [`69768fc`](https://github.com/techempower-org/mempalace/commit/69768fc) | +| 81 | Restore transcript ingest via daemon /mine when PALACE_DAEMON_URL is set | [#2](https://github.com/MemPalace/mempalace/pull/2) | [`09d2ca6`](https://github.com/techempower-org/mempalace/commit/09d2ca6) | +| 82 | `hook_verbatim_mode` config flag preserves system tags + full tool I/O during transcript ingest | — | [`ef98961`](https://github.com/techempower-org/mempalace/commit/ef98961) | +| 83 | Retire the `kind=` filter — structural split made it inert | — | [`7ba28dc`](https://github.com/techempower-org/mempalace/commit/7ba28dc) | +| 84 | Hoist CLOSET_RANK_BOOSTS to module level + record VecRecall ablation finding | — | [`3cb03f3`](https://github.com/techempower-org/mempalace/commit/3cb03f3) | +| 85 | Strip embedded API key from .claude-plugin/ manifests; rely on env inheritance | — | [`9f91e18`](https://github.com/techempower-org/mempalace/commit/9f91e18) | +| 86 | Cherry-pick #1094 — coerce None metadatas at chromadb boundary | [#1094](https://github.com/MemPalace/mempalace/pull/1094) (OPEN) | [`43d728d`](https://github.com/techempower-org/mempalace/commit/43d728d) | +| 87 | Cherry-pick #1087 rewrite — collection.delete(where=) instead of nuke-and-rebuild | [#1087](https://github.com/MemPalace/mempalace/pull/1087) (OPEN) | [`366a9ad`](https://github.com/techempower-org/mempalace/commit/366a9ad) | +| 88 | Canonical YAML manifest + renderer for fork-ahead docs | — | [`5a01aec`](https://github.com/techempower-org/mempalace/commit/5a01aec) | +| 89 | Phase D migration + PreCompact recovery write | — | [`42817d7`](https://github.com/techempower-org/mempalace/commit/42817d7) | +| 90 | Surface drawer_id in search/diary/recovery payloads | — | [`9a8bb77`](https://github.com/techempower-org/mempalace/commit/9a8bb77) | +| 91 | Cherry-pick #1085 — batch ChromaDB inserts in miner (10–30× faster) | [#1085](https://github.com/MemPalace/mempalace/pull/1085) (CLOSED) | [`6be6fff`](https://github.com/techempower-org/mempalace/commit/6be6fff) | +| 92 | scripts/deploy.sh — one-command Syncthing-aware redeploy | — | [`8252025`](https://github.com/techempower-org/mempalace/commit/8252025) | +| 93 | Phases A–C of the checkpoint collection split | — | [`e266365`](https://github.com/techempower-org/mempalace/commit/e266365) | +| 94 | kind= filter on search_memories excludes Stop-hook checkpoints (transitional) | — | [`f9f5cc4`](https://github.com/techempower-org/mempalace/commit/f9f5cc4) | ### Recently merged into upstream diff --git a/website/reference/python-api/backends/base.md b/website/reference/python-api/backends/base.md index f339bc7752..2901d98d58 100644 --- a/website/reference/python-api/backends/base.md +++ b/website/reference/python-api/backends/base.md @@ -340,6 +340,14 @@ such a backend is O(n^2) in collection size: each page re-walks the entire collection just to discard everything outside the requested slice. See issue #1796. +#### `facet_counts` + +```python +def facet_counts(self, field: str, where: Optional[dict] = None, limit: int = 1000) -> dict[str, int] +``` + +Return counts for each distinct value of a metadata field. + #### `maintenance_state` ```python diff --git a/website/reference/python-api/backends/chroma.md b/website/reference/python-api/backends/chroma.md index 78a35704e3..ce10ab7917 100644 --- a/website/reference/python-api/backends/chroma.md +++ b/website/reference/python-api/backends/chroma.md @@ -215,6 +215,18 @@ def health(self, palace: Optional[PalaceRef] = None) -> HealthStatus def detect(cls, path: str) -> bool ``` +Return True when ``path`` looks like a chroma palace. + +Verifies the SQLite magic header rather than file presence alone. +Bare ``sqlite3.connect()`` against a missing path leaves a 0-byte +file behind (the SQLite header is written on the first statement, +not on connection), so file-presence alone treats those artifacts +as real chroma palaces and breaks multi-backend resolution. The +16-byte ``SQLite format 3\x00`` magic prefix is written as soon +as chromadb's ``PersistentClient`` does any work, so this check +accepts every real chroma palace while rejecting empty / garbage +files. See #1893. + #### `get_or_create_collection` ```python diff --git a/website/reference/python-api/backends/pgvector.md b/website/reference/python-api/backends/pgvector.md index 192b02a232..d50e29c83a 100644 --- a/website/reference/python-api/backends/pgvector.md +++ b/website/reference/python-api/backends/pgvector.md @@ -47,6 +47,32 @@ def get_stored_embedder_identity(self) def set_embedder_identity(self, identity) -> None ``` +#### `get_all_metadata` + +```python +def get_all_metadata(self, where = None) -> list[dict] +``` + +Single-pass metadata-only fetch — projects out the document column. + +The base implementation pages through ``get(include=["metadatas"])``, +which routes here via ``_scroll`` and (pre-this-override) always sent +the ``document`` text over the wire even when nothing consumed it. +For pgvector deployments where the client is remote (TLS over WAN), +that meant ``mempalace_status`` transferred O(n × document_size) +bytes per call, dominating wall time. With ``with_document=False`` +the SELECT replaces document with NULL, dropping the per-row payload +to id + metadata for every caller of this method. + +Filtered fetches still need the ``_matches_where`` post-filter for +non-pushdown semantics (array/object values where ``metadata @> ...`` +is broader than the exact match the caller asked for — same +correctness contract as #1840's filtered ``get`` path). Since that +post-filter only reads ``metadata``, we keep the single-scroll + +``with_document=False`` fast path and just apply the filter locally +on the metadata dicts before returning. This extends the wire-byte +win to filtered callers as well. + #### `add` ```python diff --git a/website/reference/python-api/backends/qdrant.md b/website/reference/python-api/backends/qdrant.md index 3cc500752d..da0a9b5db5 100644 --- a/website/reference/python-api/backends/qdrant.md +++ b/website/reference/python-api/backends/qdrant.md @@ -84,6 +84,12 @@ already use, so this can't independently drift from those call sites. (Maintainer review on #1832: avoid duplicating the filter dance inline.) +#### `facet_counts` + +```python +def facet_counts(self, field: str, where: Optional[dict] = None, limit: int = 1000) -> dict[str, int] +``` + #### `delete` ```python diff --git a/website/reference/python-api/backends/sqlite_exact.md b/website/reference/python-api/backends/sqlite_exact.md index c5376d7f9e..277e5d8757 100644 --- a/website/reference/python-api/backends/sqlite_exact.md +++ b/website/reference/python-api/backends/sqlite_exact.md @@ -140,6 +140,15 @@ def health(self, palace: Optional[PalaceRef] = None) -> HealthStatus def detect(cls, path: str) -> bool ``` +Return True when ``path`` looks like a sqlite_exact palace. + +Verifies the SQLite magic header rather than file presence alone, for +the same reason as :py:meth:`mempalace.backends.chroma.ChromaBackend.detect`: +bare ``sqlite3.connect()`` against a missing path leaves a 0-byte file +behind because the SQLite header is written on the first statement, +not on connection. The 16-byte ``SQLite format 3\x00`` magic prefix +accepts every real palace while rejecting empty / garbage files. See #1893. + #### `create_collection` ```python diff --git a/website/reference/python-api/cli.md b/website/reference/python-api/cli.md index 78f063ffde..015eb2ee4f 100644 --- a/website/reference/python-api/cli.md +++ b/website/reference/python-api/cli.md @@ -366,6 +366,14 @@ as a CLI verb. Supports ``--wing`` / ``--room`` scoping and a Daemon unreachable → exit 1; inner-error envelope → exit 2. +### `cmd_hallways` + +```python +def cmd_hallways(args) +``` + +List within-wing entity hallways (the auto-built associative graph). + ### `cmd_overlap` ```python @@ -485,6 +493,20 @@ def cmd_mcp(args) Show how to wire MemPalace into MCP-capable hosts. +### `cmd_serve` + +```python +def cmd_serve(args) +``` + +Run a secure remote HTTP MCP server for a team to share one palace (#1877). + +A turnkey wrapper over ``mempalace-mcp --transport http``: it resolves a +bearer token (auto-generating a strong one for non-loopback binds), prints a +ready-to-paste client config, then execs the real server in the foreground so +Docker/systemd own the process lifecycle. The token is passed via the +environment, never argv, so it can't leak through ``ps``. + ### `cmd_compress` ```python diff --git a/website/reference/python-api/entities.md b/website/reference/python-api/entities.md new file mode 100644 index 0000000000..a85055e231 --- /dev/null +++ b/website/reference/python-api/entities.md @@ -0,0 +1,36 @@ +# `mempalace.entities` + +Source: [`mempalace/entities.py`](https://github.com/techempower-org/mempalace/blob/main/mempalace/entities.py) + +No-LLM structural entity extraction for the associative graph. + +Pulls deterministic, *structural* tokens from text — author-quoted code spans, URLs, +file paths, qualified identifiers, and CamelCase symbols — to populate the ``entities`` +drawer-metadata field that hallways/tunnels consume. Structural-only by design: no +wordlists, no NLP models, no domain vocabulary, so it stays language-neutral and +predictable, and biases to precision (only tokens that are unambiguously "a thing being +referred to") over recall. + +The output format matches what ``hallways._parse_entities`` expects: a ``;``-joined string. + +## Functions + +### `extract_structural_entities` + +```python +def extract_structural_entities(text, max_entities = _MAX_ENTITIES) +``` + +Return up to ``max_entities`` structural entities from ``text``. + +Deterministic and order-stable: entities are ranked by occurrence count (ties broken +by first appearance), deduplicated case-insensitively, preserving the first-seen +surface form. + +### `entities_metadata` + +```python +def entities_metadata(text, max_entities = _MAX_ENTITIES) +``` + +``;``-joined entity string for drawer metadata, or ``""`` when none are found. diff --git a/website/reference/python-api/index.md b/website/reference/python-api/index.md index b5ce38599e..0cdecbf9a1 100644 --- a/website/reference/python-api/index.md +++ b/website/reference/python-api/index.md @@ -29,6 +29,7 @@ For task-oriented overviews of the main interfaces (search, memory stack, knowle - [`mempalace.diary_ingest`](./diary_ingest) — diary_ingest.py — Ingest daily summary files into the palace. - [`mempalace.dynamics`](./dynamics) — dynamics.py — Living-connection math for halls + tunnels. - [`mempalace.embedding`](./embedding) — Embedding function factory with hardware acceleration. +- [`mempalace.entities`](./entities) — No-LLM structural entity extraction for the associative graph. - [`mempalace.entity_detector`](./entity_detector) — entity_detector.py — Auto-detect people and projects from file content. - [`mempalace.entity_registry`](./entity_registry) — entity_registry.py — Persistent personal entity registry for MemPalace. - [`mempalace.exporter`](./exporter) — exporter.py — Export the palace as a browsable folder of markdown files. diff --git a/website/reference/python-api/mcp_server.md b/website/reference/python-api/mcp_server.md index 7986c357a8..0ef6960805 100644 --- a/website/reference/python-api/mcp_server.md +++ b/website/reference/python-api/mcp_server.md @@ -299,11 +299,19 @@ Fetch a single logical drawer by ID. Returns full content and metadata. ### `tool_list_drawers` ```python -def tool_list_drawers(wing: str = None, room: str = None, tags: list = None, limit: int = 20, offset: int = 0) +def tool_list_drawers(wing: str = None, room: str = None, since: str = None, before: str = None, tags: list = None, limit: int = 20, offset: int = 0) ``` List logical drawers with pagination. Optional wing/room/tag filter. +Optional ``since`` / ``before`` filter by drawer ``filed_at`` (ISO date or +timestamp): ``since`` is inclusive, ``before`` is exclusive (#1128). A +drawer whose ``filed_at`` is missing or unparseable is excluded while a +date bound is active. The filter is applied in Python after the rows are +fetched — ChromaDB rejects string operands for ``$gte``/``$lt`` (1.5.7), +and ``filed_at`` is stored as an ISO string, so a server-side ``where`` +comparison is not available. + ### `tool_update_drawer` ```python diff --git a/website/reference/python-api/palace.md b/website/reference/python-api/palace.md index 8c5890d754..2139a52d9f 100644 --- a/website/reference/python-api/palace.md +++ b/website/reference/python-api/palace.md @@ -187,11 +187,13 @@ Non-blocking: if another `mine` is already writing to this palace, raise MineAlreadyRunning so the caller can exit cleanly instead of piling up as a waiting worker. -Re-entrant: if the current thread already holds the lock for the same +Re-entrant: if the current process already holds the lock for the same palace, the context manager passes through without re-acquiring. This lets ChromaCollection write methods (which acquire the lock themselves to protect MCP/direct callers) compose with miner.mine() (which holds -the outer lock for the entire mine pipeline) without self-deadlock. +the outer lock for the entire mine pipeline) without self-deadlock, and +lets the threaded MCP HTTP transport write from a worker thread while the +long-lived writer-lease is held on another thread of the same process. ### `file_already_mined` diff --git a/website/reference/python-api/repair.md b/website/reference/python-api/repair.md index 0e2a78da8e..2a38eeda20 100644 --- a/website/reference/python-api/repair.md +++ b/website/reference/python-api/repair.md @@ -165,6 +165,26 @@ def print_sqlite_integrity_abort(palace_path: str, errors: list[str]) -> None Print a clear repair abort message for SQLite-layer corruption. +### `maybe_autoheal_fts5_index` + +```python +def maybe_autoheal_fts5_index(palace_path: str, errors: list[str], *, progress = print) -> list[str] +``` + +Rebuild a malformed FTS5 inverted index in place; return remaining errors. + +The repair preflight aborts when ``PRAGMA quick_check`` reports SQLite-layer +corruption. After concurrent killed-mid-write mines (#1596) the common +failure is an isolated ``malformed inverted index for FTS5 table``, which is +fully recoverable: the index rebuilds from the intact +``embedding_fulltext_search_content`` table without touching drawer rows. + +When the errors are isolated to FTS5, rebuild the index under the palace +write lock (so a live mine cannot race the rebuild) and re-run quick_check. +Returns the remaining quick_check errors — empty when the heal succeeded. +Broader corruption, a lock held by another writer, or a rebuild failure +leaves ``errors`` unchanged so the caller still aborts with the banner. + ### `index_read_recovery_guidance` ```python