diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 9d3d468a1..08410cd6a 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ "name": "mempalace", "source": "./.claude-plugin", "description": "AI memory system — mine projects and conversations into a searchable palace. 36 MCP tools, auto-save hooks, guided setup.", - "version": "3.7.0+oc.2", + "version": "3.7.1+oc.1", "author": { "name": "milla-jovovich" } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index b6a7c764f..8300b8cf9 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "mempalace", - "version": "3.7.0+oc.2", + "version": "3.7.1+oc.1", "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 36 MCP tools, auto-save hooks, and guided setup.", "author": { "name": "milla-jovovich" diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 114c1aeb2..a348f57c5 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "mempalace", - "version": "3.7.0+oc.2", + "version": "3.7.1+oc.1", "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 36 MCP tools, auto-save hooks, and guided setup.", "author": { "name": "milla-jovovich" diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d0c5db0b..0a2e58d64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,60 +23,87 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), --- +## [3.7.1] — 2026-08-12 + +Post-3.7.0 integrity patch: ingest no longer hangs on non-regular files, incomplete mines can be retried instead of permanently skipped, chromadb reconnect no longer rewinds the HNSW index, and MCP releases the writer lease on SIGTERM/SIGHUP. + +### Bug Fixes + +- **Ingest commands no longer hang on a named pipe.** `os.walk` and `glob` list a FIFO as an ordinary filename and MemPalace decides what to read from the suffix, so a pipe called `notes.md` in a mined directory wedged `mine` in the kernel forever: opening a FIFO for reading waits for a writer that never arrives, and the `S_ISREG` refusal written on the next line could never run. `mine --mode convos`, `sweep`, `init`, `compress` and `split` blocked the same way through their own readers. The four affected opens now pass `O_NONBLOCK`, which makes the existing type check reachable — a pipe is refused on its mode, with or without a live writer — and the discovery walks drop non-regular entries with a `SKIP: (not a regular file)` line, so the readers that use a plain `open()` never see one. Regular files read back byte-identical; the one case where the flag is not inert, a reader breaking a write lease, re-checks the file type and retries without it rather than dropping the file. `mine --mode extract` was already immune through its zero-size gate. (#2221) +- **Project re-mine no longer silently skips a partial or interrupted file.** Four related gaps in `process_file`: (1) multi-batch upserts now stamp every drawer with `chunk_total` so `file_already_mined` can tell "N of N committed" from "crashed after batch 1"; (2) `source_mtime` comes from the same `fstat` as the content read, so an append between read and a later re-stat cannot permanently hide the new tail; (3) a failed stale-drawer purge aborts the file instead of half-overwriting; (4) closets are purged even when the re-mine ends with zero drawers. A mid-file upsert failure also deletes the partial drawers and closets for that source before re-raising, so the next mine retries instead of treating the incomplete set as complete. (#2088, #2122, #2151) +- **Conversation mine completeness matches the project path.** Convo drawers now stamp `chunk_total`; a mid-batch upsert failure deletes that source's partial drawers before re-raising; `prefetch_mined_set` omits incomplete groups so the bulk "already filed" skip cannot permanently strand missing exchanges from an interrupted transcript mine. (#2183) +- **Stale chromadb System cache is cleared on palace reconnect.** After a peer or rebuild changes `chroma.sqlite3` on disk, both `mcp_server._get_client` and `ChromaBackend._client` drop chromadb's path-keyed `SharedSystemClient` cache before reopening — otherwise the stale in-memory HNSW segment is reused and can persist an outdated index over the peer's writes (index count going backwards). (#2002, #2028, #2026, #2032) +- **MCP releases the palace writer lease on SIGTERM/SIGHUP.** The lease was only released via `atexit`, which CPython skips on those signals' default disposition. SSH disconnect (SIGHUP) and container/systemd stop (SIGTERM) therefore left `mine_palace_*.lock` naming a dead PID until a contender's liveness check reclaimed it. `main()` now installs handlers that exit through `sys.exit`, so the existing `atexit` release path runs. (#2205) + +--- + ## [3.7.0] — 2026-08-11 ### Features -- **Embeddings via any OpenAI-compatible `/v1/embeddings` endpoint.** New `embedding_model: "openai-compat"` option computes embeddings on a server (LM Studio, llama.cpp, vLLM, Ollama's OpenAI shim, or a self-hosted endpoint) instead of a local ONNX model — useful for larger/multilingual embedders such as Qwen3-Embedding, or GPU offload. New `OpenAICompatEmbeddingFunction` in [`mempalace/embedding.py`](mempalace/embedding.py) speaks the standard `/v1/embeddings` protocol over stdlib `urllib` (no new dependency), batches requests, re-sorts the response by `index`, and L2-normalizes for the cosine collection. Endpoint settings are resolved by `MempalaceConfig` as a single source of truth — `embedding_api_url` / `embedding_api_model` / `embedding_api_key` in `config.json`, each overridable via the matching `MEMPALACE_EMBEDDING_API_*` env var. The embedding function's `name()` encodes the model id so changing it forces `mempalace repair rebuild-index` (different vector space). Mirrors the existing `openai-compat` LLM provider naming; stays local when the endpoint is on your machine/LAN. (#1559) -- **`mempalace_search` / `mempalace search` — `since`/`before` date window.** Semantic search is poor at temporal queries ("what did we discuss this week?" scores ~0.35 even when matching drawers exist), so the search surfaces now accept the same `[since, before)` window `list_drawers` gained in #1128: inclusive/exclusive ISO bounds compared wall-clock against each drawer's `filed_at`, undated drawers excluded while a bound is active. The window applies on every candidate path (vector, `candidate_strategy="union"`, and the BM25-only fallback); the vector candidate pool widens under an active window (ChromaDB cannot range-compare string metadata server-side), and a full pool is flagged via `date_filter_pool_truncated` instead of passing silently. Shared parsing lives in the new `mempalace.date_window` module, also backing the `list_drawers` filter. (#463) -- **Hermes agent memory provider (core).** In-package Hermes `MemoryProvider` files live turns through the shared `file_conversation_exchange()` path (same metadata as convo mining), with a background worker so the agent loop never blocks on Chroma. Wake-up L1 uses the long-lived collection only — a second PersistentClient is not opened, avoiding local SQLite races. Install/backfill/docs remain a stacked follow-up. (#1915, #2215) -- **Explicit RFC 002 source adapters on `mempalace mine`.** `mempalace mine --source ` resolves adapters through the registry, holds the palace writer lease for the full ingest, and keeps dry-runs inert (no real backend/KG open). Legacy `--mode` paths are unchanged. (#2068, #2062) -- **MCP refuses writes when the served library is no longer the one that started.** Detects mempalace (and chromadb when that backend is active) version drift or uninstall after upgrade without restarting the server; opt-out via `MEMPALACE_MCP_ALLOW_STALE_LIBRARY`. (#2081, #899) -- **Hook write-routing through the daemon.** Background hook saves and mines can honor the shared write-routing policy so multi-session setups serialize mutations through one local owner instead of racing the palace. (#2030, #1963) +- **Agent logstream coordination (RFC 003).** Append-only event layer for multi-agent work: durable task packets, wait/ack handoffs, patch and file artifacts, and live tailing over the MCP HTTP hub (`GET /logstream/stream` SSE). MCP tools include `mempalace_event_append` / `list` / `wait` / `ack`, artifact put/get, and patch submit; CLI `mempalace logstream` mirrors the core verbs. Events and artifacts stay local, verbatim, and separate from the drawer palace (`logstream.sqlite3`). Shared-brain / multi-machine agent fleets no longer need a human to relay status between hosts. (#2162, phases 1–5) + +- **Logstream multi-master sync foundation (RFC 004 step 0).** Estate-level logstream replication hooks so coordinated agents can share the coordination layer across palace replicas — the storage step for a replicated shared brain. (#2162, logsync) + +- **OpenAI-compatible embeddings.** Opt-in `embedding_model: "openai-compat"` talks to any `/v1/embeddings` endpoint (LM Studio, llama.cpp, vLLM, Ollama's OpenAI shim, or a self-hosted server) over stdlib `urllib` — no new dependency. Config and env vars set URL, model, and key; the model id is part of the embedder name so a model change forces a reindex. Default MiniLM/ONNX path is unchanged. (#1671, #1559) + +- **Search date window.** `mempalace_search` and `mempalace search` accept `since` / `before` (`[since, before)` on `filed_at`), shared with `list_drawers` via `mempalace.date_window`. Undated drawers are excluded while a bound is active; the vector candidate pool widens under a filter and reports truncation when the pool is full. (#2000, #463) + +- **Hermes memory provider (core).** In-package Hermes `MemoryProvider` files live turns through `file_conversation_exchange()` so metadata matches convo mining, with a background worker so the agent loop never blocks. Install/backfill/docs remain a stacked follow-up. (#1915, #2215) + +- **RFC 002 source adapters on mine.** `mempalace mine --source ` resolves registered adapters, holds the palace writer lease for the full ingest, and keeps dry-runs inert. Legacy `--mode` paths are unchanged. (#2068, #2062) + +- **Hook write-routing through the daemon.** Background hook saves and mines can honor the shared write-routing policy so multi-session setups serialize mutations through one local owner. (#2030, #1963) ### Performance -- **EmbeddingGemma groups documents by size before sub-batching.** The tokenizer pads every row of a sub-batch to the longest sequence in it, so arrival order decided the bill: one long verbatim message dragged a whole sub-batch up to its own length. Measured over 43,157 `sweep` drawers from 160 Claude Code transcripts, padded token slots drop 39.7% and the quadratic attention term 45.0%. Vectors move by at most one float32 ULP (1.2e-07 absolute, cosine 0.99999992), which is reduction-order rounding and not a change of meaning. Applies to `embedding_model: embeddinggemma` only; the default MiniLM embedder pads to a fixed width and was never affected. (#2104) -- **HNSW capacity probes are cached** and invalidated by palace file signature, so repeated MCP status/taxonomy paths no longer re-scan native segment files on every call. (#2051, #1471) -- **`chunk_text` line numbering is O(N)** via incremental tallies, fixing multi-second hangs on large sources. (#2054, #2055) +- **EmbeddingGemma groups documents by size before sub-batching**, cutting padded-token work on long sweeps without changing vector meaning. Applies only to `embedding_model: embeddinggemma`. (#2104) + +- **HNSW capacity probes are cached** and invalidated by palace file signature, so repeated status/taxonomy paths no longer re-scan native segments every call. (#2051, #1471) + +- **`chunk_text` line numbering is O(N)**, fixing multi-second hangs on large sources. (#2054, #2055) ### Bug Fixes -- **Orphaned per-source-file mine locks are reaped instead of accumulating forever.** `_cleanup_mine_lock_file` reclaims a lock correctly on the happy path, but only for the specific lock its own `mine_lock` context manager just released — a process killed abruptly (SIGKILL, force-quit, host crash) never reaches that cleanup, and nothing else revisited the file afterward. One long-lived installation was found with 5,636 stale entries in `~/.mempalace/locks/`, the oldest several months old, none held by any live process. `mine_lock` now opportunistically reaps locks older than an hour via the same nonblocking-flock-reacquire safety check `_cleanup_mine_lock_file` already uses, throttled to once per 15 minutes so it costs nothing on the common path. `mine_palace_*.lock` (the newer per-palace lock) is untouched — it has its own lifecycle and holder tracking. -- **Windows MCP stdout capture falls back cleanly** when fd-level redirection is unavailable, and fails closed if protocol stdout cannot be restored after a successful redirect. (#2211, #2210) -- **Claude Code `subagents/` transcripts are skipped by default** when mining conversations (98%+ noise on typical workspaces); `--include-subagents` opts back in. (#1330, #1217) -- **Worktree transcript cwd no longer mints a throwaway wing** — wing is derived from the project root above `.claude/worktrees/`. (#2206) -- **Markdown emphasis/bold no longer scores as emotional content** in the general extractor. (#2199, #2197) -- **Encoding hardening on Windows locales:** pin `encoding=utf-8` on config/dialect opens; safer mojibake repair that does not destroy clean Portuguese/Vietnamese/Turkish prose; repair-encoding CLI reconfigures stdio like other entry points; remaining non-ASCII CLI symbols replaced for GBK consoles. (#2098, #2208, #2194, #1104, #1034, #2193) -- **`rebuild_index` holds the palace writer lease** for the full snapshot→rebuild/swap cycle so concurrent writers cannot recreate HNSW divergence mid-repair. (#2195) -- **`MEMPALACE_PALACE_PATH` is restored after each service entrypoint** so multi-call processes do not leak the first call's palace into the second. (#2192) -- **systemd unit uses `Restart=always`** and documents `MEMPALACE_MCP_IDLE_HOURS=0` so the idle watchdog does not leave a dedicated server down after a clean exit. (#2203, #2204) -- **Mining works again on the default Chroma backend.** Once Chroma began declaring `requires_explicit_embeddings`, every write started routing through `EmbeddingCollection`, whose `_embed_texts` built rows with `list(ndarray)` — that unpacks into `np.float32` *scalars*, which chromadb rejects outright (`Expected embeddings to be a list of floats or ints, a list of lists, a numpy array, or a list of numpy arrays`). `mine`, and every other write against a default palace, aborted on the first drawer. Vectors now convert to real Python floats. The suite was structurally blind to this: conftest's autouse embedding fixture replaces `_embed_texts` itself for every module outside `test_embedding` / `test_embeddinggemma`, so the defective function was never executed under test — the regression tests therefore live in `test_embedding.py`, where that stub does not apply. (#2187) -- **ChatGPT data exports are parsed instead of stored as raw JSON.** A real `conversations.json` is a top-level array of conversations, which no parser claimed, so `mine --mode convos` chunked the raw JSON and lost every speaker turn while reporting success. Each conversation now normalizes to its own transcript, as Claude.ai privacy exports already do, so per-conversation dedup survives a re-export. The ChatGPT parser also type-checks its nested shapes, so an unrelated array carrying a `mapping` key is declined instead of raising. (#2160) -- **Local backends enforce process-lifetime single-writer ownership.** File-backed and unknown backends require one writer owner for the full process lifetime (daemon holds the lease until workers exit; writable MCP HTTP acquires ownership before bind and refuses startup when blocked). Read-only MCP may coexist; `sqlite_exact` opens genuine query-only/immutable readers; remote Milvus/Zilliz remain multi-process. Addresses multi-writer SQLite/WAL corruption from MCP HTTP + daemon + mine topologies. (#2079, #2045) -- **Chroma HNSW write defaults match chromadb** (`batch_size=100` / `sync_threshold=1000`) instead of the old 2/2 bloat guard that rewrote segments thousands of times on large mines. (#2107, #2106) -- **Repair and recovery are safer under contention.** `repair --mode from-sqlite` takes the mine-lock before archiving; rebuilds preserve a verified temp collection when the live swap fails; sparse drawers with zero `embedding_metadata` rows are no longer dropped; truncated ID pagination fails loud instead of pretending success. (#2109, #2086, #2087) -- **`repair --mode from-sqlite --dry-run` is a true preview.** It no longer archives or re-embeds; it prints per-collection would-be counts from SQLite ground truth and exits without touching the palace. Unreadable counts fail closed instead of inventing zeros. (#2133, #2095, #1654) -- **`repair --dry-run` is a true preview in the default (legacy) mode too.** That path ignored the flag entirely and ran the real rebuild — deleting any existing `.backup`, copying the palace over it, and re-filing the drawers collection. It now prints a read-only plan and exits without opening a chromadb client, which is itself a write to `chroma.sqlite3`. The plan names the live-collection delete the rebuild performs, warns when an existing backup would be destroyed, and reports the truncation guard as disabled when `--confirm-truncation-ok` is set. An isolated FTS5 inverted-index error is reported as auto-healable instead of raising the manual-recovery abort a real run never reaches, unreadable counts fail closed with a non-zero exit, and the `--dry-run` help no longer claims to be `--mode max-seq-id` only. (#2144) -- **`repair` and `migrate` survive a socket or a named pipe in the palace directory.** Both take a whole-directory `shutil.copytree` backup before they overwrite the palace, and `copytree` cannot duplicate a Unix domain socket left beside `chroma.sqlite3`, nor a named pipe: it copied everything else and then raised `shutil.Error`, so the command died at the backup step before any rebuild ran, and re-running only repeated it. Neither entry carries palace data, so both are now skipped and named in the output while the backup completes. Device nodes are skipped too, because the copy dereferences them instead of failing on them. Every other copy failure still aborts the command, including a symlink whose target cannot be read: no errno separates a deleted target from one on a volume that is not mounted, so an entry that may have held data is left for the copy to fail on. (#2207) -- **HNSW divergence is preflighted before remaining `col.count()` crash sites** across mine, dedup, migrate, repair, and palace helpers. (#2093) -- **Re-mine and conversation ingest no longer lose or duplicate drawers.** Content-hash dedup prevents duplicate LLM conversation drawers; sweeper drawers are excluded from convo extract-mode purge scope and failed purges abort; search returns round-trippable `drawer_id` values for `get_drawer`. (#2050, #2125, #2089, #2090, #2044, #2080) -- **MCP and daemon lifecycle harden multi-agent use.** Read-only mode refuses config and checkpoint-ack tools that rewrite host state; stdio MCP exits on stdin EOF/broken pipe so orphaned sessions release locks; daemon jobs refused the palace lock are deferred instead of failed permanently. (#2126, #2103, #2101, #2072, #2029, #2014) -- **Entity-candidate extraction no longer hangs on long ASCII runs** (base64, minified blobs) while preserving CJK/non-ASCII text. (#2127, #2065, #2063) -- **Mining windowing rejects `chunk_overlap` above half the chunk size**, stopping infinite chunk_text loops. (#2056, #2058) -- **`docker-compose.yml` is valid again.** The `environment:` key was declared with only comments beneath it, which YAML parses as null, so Compose rejected the whole file (`services.mcp.environment must be a mapping`) — every documented Compose command failed before starting. The key is commented out along with its examples, which now use mapping syntax so uncommenting them yields a valid block. (#2188) +- **Mining works again on the default Chroma backend.** Explicit-embedding writes no longer hand Chroma `np.float32` scalars that `normalize_embeddings` rejects. (#2187) + +- **ChatGPT data exports are parsed as conversations**, not stored as raw JSON arrays. (#2160) + +- **Local backends enforce process-lifetime single-writer ownership.** File-backed palaces require one writer owner for the process lifetime; read-only MCP may coexist; remote backends remain multi-process. (#2079, #2045) + +- **MCP refuses writes when the served library drifts** (mempalace, and chromadb when that backend is active) after an upgrade without restart. Opt out with `MEMPALACE_MCP_ALLOW_STALE_LIBRARY`. (#2081, #899) + +- **Chroma HNSW write defaults match chromadb** (`batch_size=100` / `sync_threshold=1000`), retiring the old 2/2 bloat guard. (#2107, #2106) + +- **Repair and recovery are safer under contention.** Mine-lock before archive, preserve temp collections on failed swap, fail loud on truncated pagination; dry-run is a true preview in both legacy and from-sqlite modes; backups skip sockets/pipes/device nodes so a live palace socket no longer aborts repair/migrate. (#2109, #2086, #2087, #2133, #2144, #2207, #2212) + +- **`rebuild_index` holds the palace writer lease** for the full snapshot→rebuild/swap cycle. (#2195) + +- **Orphaned per-source mine locks are reaped** (age + nonblocking flock), throttled to once per 15 minutes; palace-level locks are untouched. (#2200) + +- **HNSW divergence is preflighted** before remaining `count()` crash sites across mine, dedup, migrate, repair, and palace helpers. (#2093) + +- **Re-mine and conversation ingest no longer lose or duplicate drawers.** Content-hash dedup, sweeper purge scope, round-trippable `drawer_id` on search; Claude Code `subagents/` skipped by default (`--include-subagents` to opt in). (#2050, #2125, #2090, #1330, #1217) + +- **MCP and daemon lifecycle harden multi-agent use.** Read-only refuses config/checkpoint-ack host mutations; stdio exits on EOF; daemon defers lock-refused jobs; Windows stdout capture falls back or fails closed if the protocol stream cannot be restored. (#2126, #2072, #2029, #2211, #2210) + +- **Encoding and Windows locale hardening.** Pin UTF-8 on config/dialect opens; mojibake repair no longer destroys clean Portuguese/Vietnamese/Turkish prose; repair-encoding CLI reconfigures stdio; non-ASCII CLI symbols replaced for GBK consoles. (#2098, #2208, #2194, #1104, #1034, #2193) + +- **Small correctness fixes.** Entity-candidate ReDoS guard; reject pathological `chunk_overlap`; worktree cwd maps to the project wing; markdown emphasis is not emotion; service entrypoints restore `MEMPALACE_PALACE_PATH`; systemd `Restart=always` with idle watchdog docs; valid `docker-compose.yml` environment key. (#2127, #2056, #2206, #2199, #2192, #2203, #2204, #2188) ### Documentation -- Operator write-routing / single-writer recovery notes in `docs/write-routing-policy.md`. (#2079) -- Remote-server guide wording for read-only tools that change host state. (#2126) -- **The README's Docker section leads with the published image.** It previously documented only `docker build`, even though `ghcr.io/mempalace/mempalace` ships multi-arch — and a clone builds `develop`, not the release, so a build and a pull could differ silently. It now covers what the omissions actually cost: the MCP client config mounts a transcripts directory (without one the server starts and every mine finds nothing), the first embedding call downloads ~80 MB into `/data` and looks like a hang, bind mounts keep host ownership against the image's uid 1000 so a `0700` directory fails with a bare `PermissionError` on Linux (and `--user` is the wrong fix — `/data` is mode 700 owned by that uid), mining sources can be mounted read-only, and the GPU image is x86_64-only. (#2196) +- Agent logstream concept page, coordination protocol, shared-brain fleet guide, and RFC 003/004. (#2162) +- Operator write-routing / single-writer recovery notes. (#2079) +- Remote-server idle watchdog and read-only semantics. (#2126, #2204) +- README Docker section leads with the published image and real mount/permission pitfalls. (#2196) +- MX3 public-shim example and CONTRIBUTING Discussions cleanup. (#597, #555) ### Internal -- **The Docker workflow runs the image before publishing it.** It previously built both images without ever starting a container or parsing a Compose file, so a green run only proved the Dockerfile compiled — which is how two defects that break the first documented command shipped past it. `scripts/docker-smoke.sh` now validates both Compose files, checks entrypoint dispatch, mines a mounted directory, asserts the drawer reads back verbatim from a separate container, and drives a real MCP stdio handshake; publication is gated on it. The script runs the same way locally: `scripts/docker-smoke.sh `. (#2189) +- Docker publish is gated on a real smoke script (Compose parse, mine, MCP handshake). (#2189) +- Embedding empty-batch / plain-sequence regression tests; HNSW defaults assertions. (#2191, #2159) ---- ## [3.6.0] — 2026-07-14 @@ -698,7 +725,8 @@ Initial public release. --- -[Unreleased]: https://github.com/MemPalace/mempalace/compare/v3.7.0...HEAD +[Unreleased]: https://github.com/MemPalace/mempalace/compare/v3.7.1...HEAD +[3.7.1]: https://github.com/MemPalace/mempalace/compare/v3.7.0...v3.7.1 [3.7.0]: https://github.com/MemPalace/mempalace/compare/v3.6.0...v3.7.0 [3.6.0]: https://github.com/MemPalace/mempalace/compare/v3.5.0...v3.6.0 [3.5.0]: https://github.com/MemPalace/mempalace/compare/v3.4.1...v3.5.0 diff --git a/README.md b/README.md index 759df192f..3639571a3 100644 --- a/README.md +++ b/README.md @@ -315,7 +315,7 @@ PRs welcome. See [CONTRIBUTING.md](CONTRIBUTING.md). MIT — see [LICENSE](LICENSE). -[version-shield]: https://img.shields.io/badge/version-3.7.0+oc.2-4dc9f6?style=flat-square&labelColor=0a0e14 +[version-shield]: https://img.shields.io/badge/version-3.7.1+oc.1-4dc9f6?style=flat-square&labelColor=0a0e14 [release-link]: https://github.com/MemPalace/mempalace/releases [python-shield]: https://img.shields.io/badge/python-3.9+-7dd8f8?style=flat-square&labelColor=0a0e14&logo=python&logoColor=7dd8f8 [python-link]: https://www.python.org/ diff --git a/integrations/openclaw/SKILL.md b/integrations/openclaw/SKILL.md index 7d8a5a032..81fdb193c 100644 --- a/integrations/openclaw/SKILL.md +++ b/integrations/openclaw/SKILL.md @@ -1,7 +1,7 @@ --- name: mempalace description: "MemPalace — Local AI memory with 96.6% recall. Semantic search, temporal knowledge graph, palace architecture (wings/rooms/drawers). Free, no cloud, no API keys." -version: 3.7.0 +version: 3.7.1 homepage: https://github.com/MemPalace/mempalace user-invocable: true metadata: diff --git a/mempalace/backends/chroma.py b/mempalace/backends/chroma.py index 246cd0b58..2a0e1ad37 100644 --- a/mempalace/backends/chroma.py +++ b/mempalace/backends/chroma.py @@ -1548,6 +1548,33 @@ def _close_client(client) -> None: logger.debug("client.close() unavailable or failed", exc_info=True) +def _clear_chroma_system_cache() -> None: + """Drop chromadb's process-global ``SharedSystemClient`` cache. + + chromadb caches its ``System`` (and the live HNSW segment) keyed by path. + A bare ``chromadb.PersistentClient(path=...)`` reopen reuses that cached + System, so after a peer/rebuild has changed ``chroma.sqlite3`` on disk we + would rebuild against the stale in-memory segment and persist an outdated + index over the on-disk changes -- the same data-loss class as #2002, + reached via :meth:`ChromaBackend._client` instead of + ``mcp_server._get_client``. This mirrors the reset already performed by + ``mcp_server._force_chroma_cache_reset`` and ``repair._close_chroma_handles``. + + The clear is process-global (it evicts every palace's cached System, not + just this path); chromadb exposes no per-path eviction. It only fires on the + inode/mtime-change branch of ``_client``, never the steady-state hot path, + so the redundant rebuild cost is bounded to genuine external-change reopens. + """ + try: + from chromadb.api.client import SharedSystemClient + + clear = getattr(SharedSystemClient, "clear_system_cache", None) + if callable(clear): + clear() + except Exception: + logger.debug("Failed to clear chromadb SharedSystemClient cache", exc_info=True) + + class ChromaCollection(BaseCollection): """Thin adapter translating ChromaDB dict returns into typed results. @@ -1568,9 +1595,14 @@ class ChromaCollection(BaseCollection): directly without going through ``ChromaBackend``. """ - def __init__(self, collection, palace_path: Optional[str] = None): + def __init__(self, collection, palace_path: Optional[str] = None, after_write=None): self._collection = collection self._palace_path = palace_path + self._after_write = after_write + + def _record_write(self) -> None: + if self._after_write is not None: + self._after_write() @contextlib.contextmanager def _write_lock(self): @@ -1662,6 +1694,7 @@ def add(self, *, documents, ids, metadatas=None, embeddings=None): kwargs["embeddings"] = embeddings with self._write_lock(): self._collection.add(**kwargs) + self._record_write() def upsert(self, *, documents, ids, metadatas=None, embeddings=None): kwargs: dict[str, Any] = { @@ -1675,6 +1708,7 @@ def upsert(self, *, documents, ids, metadatas=None, embeddings=None): kwargs["embeddings"] = embeddings with self._write_lock(): self._collection.upsert(**kwargs) + self._record_write() def update( self, @@ -1695,6 +1729,7 @@ def update( kwargs["embeddings"] = embeddings with self._write_lock(): self._collection.update(**kwargs) + self._record_write() # ------------------------------------------------------------------ # Reads @@ -1840,6 +1875,7 @@ def delete(self, *, ids=None, where=None): kwargs["where"] = where with self._write_lock(): self._collection.delete(**kwargs) + self._record_write() def count(self): return self._collection.count() @@ -2292,6 +2328,19 @@ def _client(self, palace_path: str): or (mtime_appeared and palace_path in self._freshness) ): ChromaBackend._quarantined_paths.discard(palace_path) + # Release the old client's SQLite and HNSW handles before + # clearing chromadb's global system cache. Replacing the dict + # entry alone retains one loaded HNSW client per peer write. + _close_client(self._clients.pop(palace_path, None)) + cached = None + # #2028: the same external change means chromadb's path-keyed + # System cache is now stale. Reconstructing PersistentClient + # below would reuse the cached System (and its in-memory HNSW + # segment), so drop the shared cache first -- otherwise the + # rebuilt client persists an outdated index over the on-disk + # change. Gated on genuine external change (not first open) so + # cold opens never pay the global-evict cost. + _clear_chroma_system_cache() ChromaBackend._prepare_palace_for_open(palace_path) cached = chromadb.PersistentClient(path=palace_path) self._clients[palace_path] = cached @@ -2441,7 +2490,16 @@ def get_collection( raise ValueError(explanation) from e raise _pin_hnsw_threads(collection) - return ChromaCollection(collection, palace_path=palace_path) + # Collection creation and migration can write chroma.sqlite3 before + # the returned wrapper has a chance to run its after-write callback. + self._freshness[palace_path] = self._db_stat(palace_path) + return ChromaCollection( + collection, + palace_path=palace_path, + after_write=lambda: self._freshness.__setitem__( + palace_path, self._db_stat(palace_path) + ), + ) def close_palace(self, palace) -> None: """Drop cached handles for ``palace`` and release its SQLite file lock. @@ -2515,7 +2573,13 @@ def create_collection( metadata=_hnsw_creation_metadata({"hnsw_space": hnsw_space}), **ef_kwargs, ) - return ChromaCollection(collection, palace_path=palace_path) + return ChromaCollection( + collection, + palace_path=palace_path, + after_write=lambda: self._freshness.__setitem__( + palace_path, self._db_stat(palace_path) + ), + ) def _normalize_get_collection_args(args, kwargs): diff --git a/mempalace/cli.py b/mempalace/cli.py index 1fc9997d4..b28e3b56a 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -131,6 +131,13 @@ def _gather_origin_samples(project_dir) -> list: if total_chars >= _PASS_ZERO_TOTAL_CAP: break try: + # ``scan_for_detection`` picks candidates by extension, so a FIFO + # named ``notes.md`` reaches this loop; opening one for reading + # blocks until a writer appears. ``is_file()`` stats instead. + # It belongs inside the try: it raises PermissionError on an + # unreadable directory, which the open below used to absorb. + if not filepath.is_file(): + continue with open(filepath, encoding="utf-8", errors="replace") as f: content = f.read(_PASS_ZERO_PER_FILE_CAP) except OSError: @@ -263,9 +270,17 @@ def _ensure_mempalace_files_gitignored(project_dir) -> bool: if not (project_path / ".git").exists(): return False gitignore = project_path / ".gitignore" + # ``exists()`` is true for a FIFO, and both the read below and the append + # at the end of this function would block in the kernel on one. Decide by + # type instead: an absent file still yields "" as before, a regular one + # is read, and anything else is left untouched. + if gitignore.exists() and not gitignore.is_file(): + return False # Force UTF-8: Windows defaults to GBK and chokes on non-ASCII .gitignore # comments, killing auto-init even though the file is valid UTF-8. - existing = gitignore.read_text(encoding="utf-8", errors="replace") if gitignore.exists() else "" + existing = ( + gitignore.read_text(encoding="utf-8", errors="replace") if gitignore.is_file() else "" + ) existing_lines = {line.strip() for line in existing.splitlines()} missing = [p for p in _MEMPALACE_PROJECT_FILES if p not in existing_lines] if not missing: @@ -420,9 +435,19 @@ def cmd_init(args): if confirmed["people"] or confirmed["projects"] or confirmed.get("topics"): project_path = Path(args.dir).expanduser().resolve() entities_path = project_path / "entities.json" - with open(entities_path, "w", encoding="utf-8") as f: - json.dump(confirmed, f, indent=2, ensure_ascii=False) - print(f" Entities saved: {entities_path}") + # Opening a pre-existing FIFO for writing blocks in the kernel + # until a reader appears. Only a regular file is a valid target + # for the per-project audit trail; the global registry merge + # below is unaffected either way. + if entities_path.exists() and not entities_path.is_file(): + print( + f" ! Not writing entities: {entities_path} is not a regular file", + file=sys.stderr, + ) + else: + with open(entities_path, "w", encoding="utf-8") as f: + json.dump(confirmed, f, indent=2, ensure_ascii=False) + print(f" Entities saved: {entities_path}") from .config import normalize_wing_name from .miner import add_to_known_entities @@ -438,7 +463,14 @@ def cmd_init(args): print(" No entities detected -- proceeding with directory-based rooms.") # Pass 2: detect rooms from folder structure - detect_rooms_local(project_dir=args.dir, yes=getattr(args, "yes", False)) + try: + detect_rooms_local(project_dir=args.dir, yes=getattr(args, "yes", False)) + except OSError as exc: + # Writing mempalace.yaml is the point of init; a target it cannot + # write (a pre-existing pipe, a full disk) is a hard failure, and a + # message beats the traceback this used to produce. + print(f"\n ERROR: {exc}", file=sys.stderr) + sys.exit(1) cfg.init() backend = _backend_arg(args) if backend: @@ -2219,12 +2251,15 @@ def cmd_compress(args): # Load dialect (with optional entity config) config_path = args.config if not config_path: + # ``isfile`` rather than ``exists``: the latter is true for a FIFO, + # and ``Dialect.from_config`` opens whatever it is handed, which + # blocks in the kernel on a pipe named entities.json in the cwd. for candidate in ["entities.json", os.path.join(palace_path, "entities.json")]: - if os.path.exists(candidate): + if os.path.isfile(candidate): config_path = candidate break - if config_path and os.path.exists(config_path): + if config_path and os.path.isfile(config_path): dialect = Dialect.from_config(config_path) print(f" Loaded entity config: {config_path}") else: diff --git a/mempalace/convo_miner.py b/mempalace/convo_miner.py index 0dd6b7d37..f0368e761 100644 --- a/mempalace/convo_miner.py +++ b/mempalace/convo_miner.py @@ -10,6 +10,7 @@ from __future__ import annotations +import errno import os import sys import json @@ -200,10 +201,15 @@ def _path_within_root(path: Path, root: Path) -> bool: def _is_regular_source_file(filepath: Path, root: Path) -> bool: if not _path_within_root(filepath, root): return False - flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) fd = -1 try: - fd = os.open(filepath, flags) + try: + fd = os.open(filepath, flags) + except OSError as exc: + if exc.errno != errno.EAGAIN or not stat.S_ISREG(os.lstat(filepath).st_mode): + raise + fd = os.open(filepath, flags & ~getattr(os, "O_NONBLOCK", 0)) st = os.fstat(fd) return stat.S_ISREG(st.st_mode) and st.st_size <= MAX_FILE_SIZE except OSError: @@ -864,6 +870,10 @@ def scan_convos(convo_dir: str, include_subagents: bool = False) -> list: ) continue if not _is_regular_source_file(filepath, convo_path): + print( + f" SKIP: {filepath.name} (not a regular file)", + file=sys.stderr, + ) continue files.append(filepath) return files @@ -1020,6 +1030,7 @@ def _file_chunks_locked( normalized_chunk_size, expected_chunk_count, ) + written_ids: list[str] = [] chunk_iterator = iter(chunks) if target_needs_upsert else iter(()) while batch := list(islice(chunk_iterator, DRAWER_UPSERT_BATCH_SIZE)): batch_docs: list = [] @@ -1066,6 +1077,7 @@ def _file_chunks_locked( "extract_mode": extract_mode, "normalize_version": NORMALIZE_VERSION, "id_recipe": ID_RECIPE, + "chunk_total": expected_chunk_count, } if source_mtime is not None: meta["source_mtime"] = source_mtime @@ -1114,8 +1126,15 @@ def _file_chunks_locked( metadatas=batch_metas, ) drawers_added += len(batch_docs) + written_ids.extend(batch_ids) except Exception as e: if "already exists" not in str(e).lower(): + # Preserve a previously committed normalized generation, + # but never leave this attempt's successful earlier batches + # looking complete after a later batch fails (#2183). + cleanup_ids = list(dict.fromkeys([*written_ids, *batch_ids])) + if cleanup_ids: + collection.delete(ids=cleanup_ids) raise if normalized is not None: diff --git a/mempalace/entity_detector.py b/mempalace/entity_detector.py index 3854f6d5e..53216e1ba 100644 --- a/mempalace/entity_detector.py +++ b/mempalace/entity_detector.py @@ -628,6 +628,13 @@ def detect_entities( if files_read >= max_files: break try: + # Decide by file type before opening: ``scan_for_detection`` + # picks candidates by extension, so a FIFO named ``notes.md`` + # reaches this loop and a blocking open of one waits in the + # kernel for a writer that may never come. ``is_file()`` stats + # instead of opening and never blocks. + if not Path(filepath).is_file(): + continue with open(filepath, encoding="utf-8", errors="replace") as f: content = f.read(MAX_BYTES_PER_FILE) all_text.append(content) diff --git a/mempalace/hook_shell.py b/mempalace/hook_shell.py index 2f2f32ebc..bfeea8ace 100644 --- a/mempalace/hook_shell.py +++ b/mempalace/hook_shell.py @@ -9,6 +9,7 @@ from __future__ import annotations import json +import os import re import sys @@ -71,9 +72,18 @@ def count_human_messages(path: str) -> int: Claude transcripts are UTF-8. Windows Python defaults to cp1252 in many environments, so the encoding must be explicit. Invalid bytes are ignored to match the hooks' fail-soft behavior. + + A path that exists but is not a regular file counts zero rather than + being opened: opening a FIFO for reading blocks in the kernel until a + writer appears, and this function has no timeout. A path that does not + exist still raises from the ``open`` below, as before. + ``mempal_save_hook.sh`` screens with ``[ -f ]``, which is false for a + pipe, so the guard here covers callers that do not. """ count = 0 + if os.path.exists(path) and not os.path.isfile(path): + return count with open(path, encoding="utf-8", errors="ignore") as fh: for line in fh: try: diff --git a/mempalace/llm_refine.py b/mempalace/llm_refine.py index e3afe6b8e..9dadcb652 100644 --- a/mempalace/llm_refine.py +++ b/mempalace/llm_refine.py @@ -479,6 +479,13 @@ def collect_corpus_text( chunks: list[str] = [] for p in selected: try: + # ``_walk_prose`` selects by suffix and the stat above reads only + # st_mtime, so a FIFO named ``notes.md`` reaches this loop and a + # blocking open of one waits for a writer that may never come. + # Inside the try: is_file() raises on an unreadable directory, + # which the open below already absorbed. + if not p.is_file(): + continue with open(p, encoding="utf-8", errors="replace") as f: chunks.append(f.read(max_bytes_per_file)) except OSError: diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 1438ef862..f944d61dd 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -1374,10 +1374,7 @@ def _get_client(): inode_changed = current_inode != 0 and current_inode != _palace_db_inode mtime_changed = current_mtime != 0.0 and abs(current_mtime - _palace_db_mtime) > 0.01 - replacing_cached_client = _client_cache is not None and (inode_changed or mtime_changed) if _client_cache is None or inode_changed or mtime_changed: - if replacing_cached_client: - _force_chroma_cache_reset() # Run the HNSW capacity probe BEFORE chromadb opens the segment -- # if the index is severely undersized, segment load can segfault # the whole MCP server (#1222). The probe is pure sqlite + @@ -1385,6 +1382,13 @@ def _get_client(): _refresh_vector_disabled_flag() if inode_changed or mtime_changed: ChromaBackend._quarantined_paths.discard(_config.palace_path) + # #2002: a peer process changed chroma.sqlite3 on disk. chromadb + # caches its System (and the live HNSW segment) keyed by path, so + # make_client() below would hand back the STALE segment, which then + # persists its outdated index over the peer's writes, driving the + # persisted count backwards. Drop chromadb's shared cache first so + # make_client() rebuilds the segment from the on-disk state. + _force_chroma_cache_reset() _client_cache = ChromaBackend.make_client(_config.palace_path) _collection_cache = None _collection_cache_backend = None @@ -7367,6 +7371,7 @@ def _http_record_request(httpd, handler, status: int) -> None: def _record_sdk_http_request(httpd, scope: dict, headers: dict, status: int) -> None: """Record the same status/client metadata for the SDK ASGI transport.""" + status = int(status) now = time.time() peer = (scope.get("client") or ("", 0))[0] forwarded_for = headers.get("x-forwarded-for", "").split(",", 1)[0].strip() @@ -8377,14 +8382,19 @@ async def __call__(self, scope, receive, send): _record_sdk_http_request(state, scope, headers, 401) return + recorded = False + async def record_send(message): - nonlocal status + nonlocal recorded, status if message["type"] == "http.response.start": status = message["status"] + if scope["type"] == "http" and not recorded: + _record_sdk_http_request(state, scope, headers, status) + recorded = True await send(message) await self.wrapped(scope, receive, record_send) - if scope["type"] == "http": + if scope["type"] == "http" and not recorded: _record_sdk_http_request(state, scope, headers, status) return _BearerMiddleware(app), state @@ -8799,6 +8809,35 @@ def _warmup_with_lock(): _release_mcp_writer_lock() +def _install_shutdown_signal_handlers() -> None: + """Route terminal signals through ``sys.exit`` so ``atexit`` runs. + + The palace writer lease is released by an ``atexit`` callback registered + when the lock is acquired. CPython's default disposition for SIGTERM and + SIGHUP is immediate termination, which skips ``atexit`` and leaves + ``mine_palace_*.lock`` naming a dead PID until a contender's liveness + check reclaims it (#2205). Calling ``sys.exit(0)`` from the handler + unwinds the synchronous stdio/http loop and runs the existing release + path. SIGHUP is Unix-only (SSH session disconnect); Windows only gets + SIGTERM. Handlers are best-effort — signal registration only works from + the main thread and is a no-op when the platform omits the signal. + """ + import signal + + def _shutdown_handler(signum, frame): # noqa: ARG001 + raise SystemExit(0) + + for name in ("SIGTERM", "SIGHUP"): + sig = getattr(signal, name, None) + if sig is None: + continue + try: + signal.signal(sig, _shutdown_handler) + except (ValueError, OSError): + # Not in the main thread, or the platform rejects the install. + pass + + def main(): """MCP server entry point for the ``mempalace-mcp`` console script. @@ -8821,6 +8860,8 @@ def main(): # extend the protection to children. os.environ.pop("PYTHONPATH", None) + _install_shutdown_signal_handlers() + if _args.transport == "http": _run_http_loop() else: diff --git a/mempalace/miner.py b/mempalace/miner.py index dbb8271c1..07a664ea9 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -7,6 +7,7 @@ Stores verbatim chunks as drawers. No summaries. Ever. """ +import errno import os import re import sys @@ -58,19 +59,45 @@ def _path_within_root(path: Path, root: Path) -> bool: return False -def _read_text_no_follow(filepath: Path, root: Path) -> Optional[str]: +def _read_text_no_follow(filepath: Path, root: Path) -> Optional[tuple[str, float]]: + """Read ``filepath`` and return ``(content, mtime)`` from the SAME + ``fstat()`` call that validated the file, so callers never need a + separate, later ``os.path.getmtime()`` that could observe a file + modified in between (see #22: a stale re-stat lets appended content + be silently and permanently skipped).""" if not _path_within_root(filepath, root): return None - flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + # O_NONBLOCK is what makes the S_ISREG check below reachable. Opening a + # FIFO for reading parks in the kernel until a writer shows up, so + # without it the fstat never runs and a named pipe carrying a + # READABLE_EXTENSIONS suffix wedges the mine forever. With it the open + # returns immediately and the *file type* decides — no errno guesswork, + # and a FIFO that does have a live writer is rejected just the same. + # Linux open(2): "this flag has no effect for regular files and block + # devices". POSIX leaves it unspecified outside FIFOs and special files, + # and one Linux case is not a no-op — see the EAGAIN branch below. + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) fd = -1 try: - fd = os.open(filepath, flags) + try: + fd = os.open(filepath, flags) + except OSError as exc: + # A reader that breaks a write lease gets EAGAIN when it passes + # O_NONBLOCK, where a blocking open waits out lease-break-time + # and succeeds. The kernel grants leases on regular files only + # (F_SETLEASE on a pipe gives ENXIO), so re-check the type and + # then read it the way this code did before the flag existed; + # dropping it would silently lose a file that used to be mined. + if exc.errno != errno.EAGAIN or not stat.S_ISREG(os.lstat(filepath).st_mode): + raise + fd = os.open(filepath, flags & ~getattr(os, "O_NONBLOCK", 0)) st = os.fstat(fd) if not stat.S_ISREG(st.st_mode) or st.st_size > MAX_FILE_SIZE: return None + mtime = st.st_mtime with os.fdopen(fd, "r", encoding="utf-8", errors="replace") as f: fd = -1 - return f.read() + return f.read(), mtime except OSError: return None finally: @@ -487,10 +514,14 @@ def load_config(project_dir: str) -> dict: resolved_project_dir = Path(project_dir).expanduser().resolve() config_path = resolved_project_dir / "mempalace.yaml" - if not config_path.exists(): + # ``is_file()`` rather than ``exists()``: the latter is true for a FIFO, + # and the ``open`` at the end of this function would then block in the + # kernel until a writer appears. A config that is not a regular file is + # treated as absent, which lands on the auto-detected defaults below. + if not config_path.is_file(): # Fallback to legacy name legacy_path = resolved_project_dir / "mempal.yaml" - if legacy_path.exists(): + if legacy_path.is_file(): config_path = legacy_path else: from .config import normalize_wing_name @@ -1369,6 +1400,7 @@ def _build_drawer_metadata( line_start: Optional[int] = None, line_end: Optional[int] = None, content_date: Optional[str] = None, + chunk_total: Optional[int] = None, source_root: Optional[str] = None, ) -> dict: """Build the metadata dict for one drawer without upserting. @@ -1385,6 +1417,14 @@ def _build_drawer_metadata( (legacy callers, pre-Tier-6a drawers), the keys are absent from the returned dict and downstream code falls back to ``filed_at`` for the date and the 3-segment closet pointer format. + + ``chunk_total`` — the total number of chunks this mining pass expects + to write for ``source_file`` (see #21). Every chunk of the same pass + carries the same value so ``file_already_mined`` can tell "N of N + batches committed" from "crashed after batch 1 of N", instead of + treating any surviving drawer with a matching mtime as proof the file + is fully mined. ``None`` for legacy callers (e.g. ``add_drawer``, + which is inherently a single atomic write with no partial-batch risk). """ metadata = { "wing": wing, @@ -1407,6 +1447,8 @@ def _build_drawer_metadata( metadata["line_end"] = line_end if content_date: metadata["content_date"] = content_date + if chunk_total is not None: + metadata["chunk_total"] = chunk_total metadata["hall"] = detect_hall(content) entities = _extract_entities_for_metadata(content) if entities: @@ -1444,6 +1486,21 @@ def add_drawer( # ============================================================================= +def _route_project_chunks(chunks: list[dict], subject_router: SubjectRouter) -> dict[str, int]: + routed_counts: dict[str, int] = defaultdict(int) + for batch_start in range(0, len(chunks), DRAWER_UPSERT_BATCH_SIZE): + batch = chunks[batch_start : batch_start + DRAWER_UPSERT_BATCH_SIZE] + routes = subject_router.route_many([chunk["content"] for chunk in batch]) + if len(routes) != len(batch): + raise RuntimeError("subject router did not classify every project chunk") + for chunk, route in zip(batch, routes): + chunk["room"] = route.room + chunk["subject_route"] = route.method + chunk["subject_score"] = route.score + routed_counts[route.room] += 1 + return routed_counts + + def process_file( filepath: Path, project_path: Path, @@ -1475,9 +1532,10 @@ def process_file( if not dry_run and file_already_mined(collection, source_file, check_mtime=True): return 0, "general", None - content = _read_text_no_follow(filepath, project_path) - if content is None: + read_result = _read_text_no_follow(filepath, project_path) + if read_result is None: return 0, "general", None + content, read_mtime = read_result content = content.strip() if len(content) < effective_min: @@ -1507,19 +1565,9 @@ def process_file( ) return 0, room, "chunk_cap" + routed_counts = defaultdict(int) if subject_router is not None: - for batch_start in range(0, len(chunks), DRAWER_UPSERT_BATCH_SIZE): - batch = chunks[batch_start : batch_start + DRAWER_UPSERT_BATCH_SIZE] - routes = subject_router.route_many([chunk["content"] for chunk in batch]) - if len(routes) != len(batch): - raise RuntimeError("subject router did not classify every project chunk") - for chunk, route in zip(batch, routes): - chunk["room"] = route.room - chunk["subject_route"] = route.method - chunk["subject_score"] = route.score - routed_counts = defaultdict(int) - for chunk in chunks: - routed_counts[chunk["room"]] += 1 + routed_counts = _route_project_chunks(chunks, subject_router) room = max(routed_counts, key=routed_counts.get) if dry_run: @@ -1548,19 +1596,34 @@ def process_file( # hnswlib's thread-unsafe updatePoint path and can segfault on macOS ARM # with chromadb 0.6.3) into a clean delete+insert, bypassing the update # path entirely. + # + # A failed purge must abort this file's mine attempt rather than fall + # through to upsert: proceeding would either leave stale tail entries + # as permanent orphans (old chunk count > new) or silently overwrite + # only the overlapping chunk_index positions (not a real re-mine) -- + # see #23. Returning here (without touching source_mtime/chunk_total) + # leaves the old drawers' stored mtime untouched, so the next mine + # still sees a mismatch against the current on-disk mtime and retries. try: collection.delete(where={"source_file": source_file}) - except Exception: + except Exception as exc: + print( + f" ! [skip] {filepath.name[:50]:50} stale-drawer purge failed " + f"({exc!r}); leaving existing drawers untouched, will retry " + f"on the next mine", + file=sys.stderr, + ) logger.debug("Stale-drawer purge failed for %s", source_file, exc_info=True) + return 0, room, None - # Batch chunks into bounded upserts so the embedding model sees many - # chunks per forward pass without building one huge Chroma/SQLite - # request for pathological files. A bad chunk can fail its sub-batch; - # that is the deliberate trade-off for amortizing embedding overhead. - try: - source_mtime = os.path.getmtime(source_file) - except OSError: - source_mtime = None + # source_mtime is the mtime paired with the content actually read + # above (from _read_text_no_follow's own fstat), not a fresh re-stat + # here -- see #22. Re-statting separately can observe a file that was + # appended to between the read and this point, stamping drawers with + # an mtime that doesn't match what was actually chunked; the next + # mine's freshness check then sees stored-mtime == current-disk-mtime + # and silently, permanently skips the appended tail. + source_mtime = read_mtime # Tier 6a content-date: extract once per file (not per chunk) and # share across all chunks. Reads filename / frontmatter / content / @@ -1575,119 +1638,148 @@ def process_file( # in production and the 4-segment pointer form lives only in tests. # Per PR #1584 review (Igor, 2026-05-22). all_metas: list = [] - for batch_start in range(0, len(chunks), DRAWER_UPSERT_BATCH_SIZE): - batch_docs: list = [] - batch_ids: list = [] - batch_metas: list = [] - for chunk in chunks[batch_start : batch_start + DRAWER_UPSERT_BATCH_SIZE]: - chunk_room = chunk.get("room", room) - drawer_id = make_drawer_id_from_chunk( - wing, chunk_room, source_file, chunk["chunk_index"] + try: + for batch_start in range(0, len(chunks), DRAWER_UPSERT_BATCH_SIZE): + batch_docs: list = [] + batch_ids: list = [] + batch_metas: list = [] + for chunk in chunks[batch_start : batch_start + DRAWER_UPSERT_BATCH_SIZE]: + chunk_room = chunk.get("room", room) + drawer_id = make_drawer_id_from_chunk( + wing, chunk_room, source_file, chunk["chunk_index"] + ) + batch_docs.append(chunk["content"]) + batch_ids.append(drawer_id) + metadata = _build_drawer_metadata( + wing, + chunk_room, + source_file, + chunk["chunk_index"], + agent, + chunk["content"], + source_mtime, + line_start=chunk.get("line_start"), + line_end=chunk.get("line_end"), + content_date=file_content_date, + chunk_total=len(chunks), + source_root=str(project_path.resolve()), + ) + if subject_router is not None: + metadata.update( + { + "subject_policy": subject_router.fingerprint, + "subject_route": chunk["subject_route"], + "subject_score": float(chunk["subject_score"]), + } + ) + batch_metas.append(metadata) + assert_no_collisions(list(zip(batch_ids, batch_metas)), collection) + collection.upsert( + documents=batch_docs, + ids=batch_ids, + metadatas=batch_metas, ) - batch_docs.append(chunk["content"]) - batch_ids.append(drawer_id) - metadata = _build_drawer_metadata( - wing, - chunk_room, + drawers_added += len(batch_docs) + all_metas.extend(batch_metas) + except Exception: + # A successful earlier batch has the source's current mtime (and + # often chunk_total). Leaving those drawers behind would make the + # next run skip this incomplete rebuild when chunk_total is absent + # on legacy rows, and would leave partial content searchable until + # the next mine. The source lock prevents this cleanup from + # deleting another miner's work for the same file. (#2122) + try: + collection.delete(where={"source_file": source_file}) + except Exception: + logger.warning( + "Failed to clean partial drawers after upsert error for %s", source_file, - chunk["chunk_index"], - agent, - chunk["content"], - source_mtime, - line_start=chunk.get("line_start"), - line_end=chunk.get("line_end"), - content_date=file_content_date, - source_root=str(project_path.resolve()), + exc_info=True, ) - if subject_router is not None: - metadata.update( - { - "subject_policy": subject_router.fingerprint, - "subject_route": chunk["subject_route"], - "subject_score": float(chunk["subject_score"]), - } - ) - batch_metas.append(metadata) - assert_no_collisions(list(zip(batch_ids, batch_metas)), collection) - collection.upsert( - documents=batch_docs, - ids=batch_ids, - metadatas=batch_metas, - ) - drawers_added += len(batch_docs) - all_metas.extend(batch_metas) + if closets_col: + purge_file_closets(closets_col, source_file) + raise # Build closet — the searchable index pointing to these drawers. - # Purge first: a re-mine (mtime change or normalize_version bump) must - # fully replace the prior closets, not append to them. - if closets_col and drawers_added > 0: + # Purge unconditionally: the old drawers this closet pointed at were + # already deleted above regardless of how many chunks survived this + # pass's own length filter, so a re-mine that ends up with zero filed + # drawers must still end up with zero closets, not stale ones + # dangling on deleted drawer IDs (see #24). Only the closet + # rebuild itself is conditional on there being new drawers to point at. + if closets_col: purge_file_closets(closets_col, source_file) - if subject_router is None: - drawer_ids = [ - make_drawer_id_from_chunk(wing, room, source_file, chunk["chunk_index"]) - for chunk in chunks - ] - closet_lines = build_closet_lines( - source_file, - drawer_ids, - content, - wing, - room, - drawer_metas=all_metas, - ) - closet_id_base = ( - f"closet_{wing}_{room}_{hashlib.sha256(source_file.encode()).hexdigest()[:24]}" - ) - closet_meta = { - "wing": wing, - "room": room, - "source_file": source_file, - "drawer_count": len(chunks), - "filed_at": datetime.now().isoformat(), - "normalize_version": NORMALIZE_VERSION, - } - entities = _extract_entities_for_metadata(content) - if entities: - closet_meta["entities"] = entities - upsert_closet_lines(closets_col, closet_id_base, closet_lines, closet_meta) + if closets_col and drawers_added > 0: + if subject_router is not None: + room_chunks: dict[str, list[tuple[dict, dict]]] = defaultdict(list) + for chunk, metadata in zip(chunks, all_metas): + room_chunks[metadata["room"]].append((chunk, metadata)) + for closet_room, grouped in room_chunks.items(): + grouped_chunks = [item[0] for item in grouped] + grouped_metas = [item[1] for item in grouped] + drawer_ids = [ + make_drawer_id_from_chunk( + wing, closet_room, source_file, chunk["chunk_index"] + ) + for chunk in grouped_chunks + ] + room_content = "\n\n".join(chunk["content"] for chunk in grouped_chunks) + closet_lines = build_closet_lines( + source_file, + drawer_ids, + room_content, + wing, + closet_room, + drawer_metas=grouped_metas, + ) + closet_id_base = ( + f"closet_{wing}_{closet_room}_" + f"{hashlib.sha256(source_file.encode()).hexdigest()[:24]}" + ) + closet_meta = { + "wing": wing, + "room": closet_room, + "source_file": source_file, + "drawer_count": len(grouped_chunks), + "filed_at": datetime.now().isoformat(), + "normalize_version": NORMALIZE_VERSION, + "subject_policy": subject_router.fingerprint, + } + entities = _extract_entities_for_metadata(room_content) + if entities: + closet_meta["entities"] = entities + upsert_closet_lines(closets_col, closet_id_base, closet_lines, closet_meta) return drawers_added, room, None - room_chunks: dict[str, list[tuple[dict, dict]]] = defaultdict(list) - for chunk, metadata in zip(chunks, all_metas): - room_chunks[metadata["room"]].append((chunk, metadata)) - for closet_room, grouped in room_chunks.items(): - grouped_chunks = [item[0] for item in grouped] - grouped_metas = [item[1] for item in grouped] - drawer_ids = [ - make_drawer_id_from_chunk(wing, closet_room, source_file, chunk["chunk_index"]) - for chunk in grouped_chunks - ] - room_content = "\n\n".join(chunk["content"] for chunk in grouped_chunks) - closet_lines = build_closet_lines( - source_file, - drawer_ids, - room_content, - wing, - closet_room, - drawer_metas=grouped_metas, - ) - closet_id_base = ( - f"closet_{wing}_{closet_room}_" - f"{hashlib.sha256(source_file.encode()).hexdigest()[:24]}" - ) - entities = _extract_entities_for_metadata(room_content) - closet_meta = { - "wing": wing, - "room": closet_room, - "source_file": source_file, - "drawer_count": len(grouped_chunks), - "filed_at": datetime.now().isoformat(), - "normalize_version": NORMALIZE_VERSION, - } - if subject_router is not None: - closet_meta["subject_policy"] = subject_router.fingerprint - if entities: - closet_meta["entities"] = entities - upsert_closet_lines(closets_col, closet_id_base, closet_lines, closet_meta) + drawer_ids = [ + make_drawer_id_from_chunk(wing, room, source_file, c["chunk_index"]) for c in chunks + ] + # Pass drawer_metas so build_closet_lines can emit the Tier 6a + # 4-segment pointer (``topic|entities|YYYY-MM-DD:Lstart-Lend|→ids``) + # when line_start / line_end / content_date are present. Falls + # back to the legacy 3-segment form automatically when not. + closet_lines = build_closet_lines( + source_file, + drawer_ids, + content, + wing, + room, + drawer_metas=all_metas, + ) + closet_id_base = ( + f"closet_{wing}_{room}_{hashlib.sha256(source_file.encode()).hexdigest()[:24]}" + ) + entities = _extract_entities_for_metadata(content) + closet_meta = { + "wing": wing, + "room": room, + "source_file": source_file, + "drawer_count": drawers_added, + "filed_at": datetime.now().isoformat(), + "normalize_version": NORMALIZE_VERSION, + } + if entities: + closet_meta["entities"] = entities + upsert_closet_lines(closets_col, closet_id_base, closet_lines, closet_meta) return drawers_added, room, None @@ -1785,7 +1877,19 @@ def scan_project( # match the SKIP: (symlink) line above; silent drops at this # gate were the original #923 complaint. try: - file_size = filepath.stat().st_size + file_stat = filepath.stat() + # Reject anything that is not a regular file before it can + # reach a reader. os.walk lists FIFOs, sockets and device + # nodes as plain filenames and the extension filter above + # decides by name, so ``notes.md`` can be a named pipe. + # stat() itself never blocks on one; opening it can. + if not stat.S_ISREG(file_stat.st_mode): + print( + f" SKIP: {filepath.name} (not a regular file)", + file=sys.stderr, + ) + continue + file_size = file_stat.st_size if file_size > MAX_FILE_SIZE: print( f" SKIP: {filepath.name} ({file_size / (1024 * 1024):.1f} MB)" diff --git a/mempalace/normalize.py b/mempalace/normalize.py index 9503aeb0a..943feec82 100644 --- a/mempalace/normalize.py +++ b/mempalace/normalize.py @@ -19,6 +19,7 @@ No API key. No internet. Everything local. """ +import errno import json import os import re @@ -120,17 +121,29 @@ def _read_transcript_file(filepath: str) -> str: and normalize_conversations() both need: no symlinks, regular files only, size-capped, BOM-tolerant. """ - flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + # O_NONBLOCK keeps the "not a regular file" check below reachable: a + # blocking open of a FIFO waits in the kernel for a writer, so the + # S_ISREG test never runs. See ``miner._read_text_no_follow``, including + # why the EAGAIN branch re-checks the type and retries without the flag. + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) if os.path.islink(filepath): raise IOError(f"Could not read {filepath}: symlinked files are skipped") fd = -1 try: - fd = os.open(filepath, flags) + try: + fd = os.open(filepath, flags) + except OSError as exc: + if exc.errno != errno.EAGAIN or not stat.S_ISREG(os.lstat(filepath).st_mode): + raise + fd = os.open(filepath, flags & ~getattr(os, "O_NONBLOCK", 0)) file_stat = os.fstat(fd) if not stat.S_ISREG(file_stat.st_mode): - raise IOError(f"Could not read {filepath}: not a regular file") + # Text stays prefix-free: this raise is inside the ``try``, so the + # ``except OSError`` below composes "Could not read : ...". + raise IOError("not a regular file") if file_stat.st_size > 500 * 1024 * 1024: # 500 MB safety limit - raise IOError(f"File too large ({file_stat.st_size // (1024 * 1024)} MB): {filepath}") + # Prefix-free for the same reason as the branch above. + raise IOError(f"file too large ({file_stat.st_size // (1024 * 1024)} MB)") with os.fdopen(fd, "r", encoding="utf-8-sig", errors="replace") as f: fd = -1 return f.read() diff --git a/mempalace/palace.py b/mempalace/palace.py index 44d9cdb16..255d7518b 100644 --- a/mempalace/palace.py +++ b/mempalace/palace.py @@ -1455,6 +1455,15 @@ def file_already_mined( that extraction mode so exchange-mode and general-mode drawers can coexist for the same source transcript. Legacy drawers without extract_mode are treated as exchange-mode drawers. + + A drawer whose metadata carries ``chunk_total`` (see #21) is only + counted toward a match once its stored_mtime group has accumulated at + least that many drawers -- guarding against a mid-file crash between + upsert batches, where the surviving drawers share the current mtime + (the file itself was never touched) but are short of the full set. A + drawer with no ``chunk_total`` (legacy rows, or a single-shot + ``add_drawer()`` call with no partial-batch risk) is trusted on its own, + exactly as before. """ try: # Under the additive-mining model, a single ``source_file`` can have @@ -1471,6 +1480,9 @@ def file_already_mined( # first matching group regardless of ordering. current_mtime = os.path.getmtime(source_file) if check_mtime else None offset = 0 + # Tracks, per matching stored_mtime group, how many drawers have + # been seen so far toward that group's own chunk_total (#21). + group_counts: dict = {} while True: results = collection.get( where={"source_file": source_file}, @@ -1496,7 +1508,16 @@ def file_already_mined( stored_mtime = meta.get("source_mtime") if stored_mtime is None: continue - if abs(float(stored_mtime) - current_mtime) < 0.001: + if abs(float(stored_mtime) - current_mtime) >= 0.001: + continue + chunk_total = meta.get("chunk_total") + if chunk_total is None: + # No completion marker on this drawer — can't verify + # completeness for its group, trust the match as before. + return True + seen = group_counts.get(stored_mtime, 0) + 1 + group_counts[stored_mtime] = seen + if seen >= chunk_total: return True if not ids: break @@ -1529,12 +1550,21 @@ def prefetch_mined_set( When extract_mode is set, mirrors file_already_mined(..., extract_mode=...) so conversation mines skip per extraction mode rather than per source file. + Completeness mirrors :func:`file_already_mined`'s ``chunk_total`` rule + (#2183): a source that only has a mid-file partial (surviving drawers + share the current mtime but are short of ``chunk_total``) is **omitted** + from the result so the bulk skip path re-mines instead of permanently + stranding the missing exchanges. Drawers with no ``chunk_total`` + (legacy rows, registry sentinels) are trusted on their own, as before. + The convo miner walks thousands of transcript files; per-file `collection.get(where={"source_file": X})` costs ~2s on a 150k-drawer palace, making a 2000-file sweep take >1h of pure skip-checking. This helper drops that to a single paginated scan plus O(1) lookups. """ - mined: dict[str, Optional[float]] = {} + # Per source_file: per stored_mtime group → count + optional chunk_total. + # A source is only "mined" once some group is complete. + groups: dict[str, dict] = {} try: total = collection.count() offset = 0 @@ -1549,14 +1579,37 @@ def prefetch_mined_set( continue # Same default as file_already_mined: missing version == 1 version = meta.get("normalize_version", 1) - if version >= NORMALIZE_VERSION: - stored_mtime = meta.get("source_mtime") - mined[src] = float(stored_mtime) if stored_mtime is not None else None + if version < NORMALIZE_VERSION: + continue + stored_mtime = meta.get("source_mtime") + mtime_key = float(stored_mtime) if stored_mtime is not None else None + entry = groups.setdefault(src, {}).setdefault( + mtime_key, {"count": 0, "chunk_total": None} + ) + entry["count"] += 1 + chunk_total = meta.get("chunk_total") + if chunk_total is not None: + try: + entry["chunk_total"] = int(chunk_total) + except (TypeError, ValueError): + pass if not batch["ids"]: break offset += len(batch["ids"]) except Exception: - logger.warning("prefetch_mined_set: partial fetch, %d files loaded", len(mined)) + logger.warning("prefetch_mined_set: partial fetch, %d source groups loaded", len(groups)) + + mined: dict[str, Optional[float]] = {} + for src, by_mtime in groups.items(): + for mtime_key, entry in by_mtime.items(): + chunk_total = entry["chunk_total"] + if chunk_total is None: + # Legacy / registry: no completion marker — trust membership. + mined[src] = mtime_key + break + if entry["count"] >= chunk_total: + mined[src] = mtime_key + break return mined diff --git a/mempalace/project_scanner.py b/mempalace/project_scanner.py index d92b8167b..dd77c4524 100644 --- a/mempalace/project_scanner.py +++ b/mempalace/project_scanner.py @@ -188,6 +188,12 @@ def _parse_pom(path: Path) -> Optional[str]: def _parse_gradle_root_project_name(path: Path) -> Optional[str]: + # ``_parse_gradle`` reaches this with a SIBLING path it constructs itself + # (``build.gradle`` next to ``settings.gradle``), which the manifest walk + # never vetted. Opening a FIFO for reading blocks in the kernel until a + # writer appears; ``is_file()`` stats instead and never blocks. + if not path.is_file(): + return None try: text = path.read_text(encoding="utf-8", errors="replace") except OSError: @@ -421,7 +427,13 @@ def _collect_manifest_names(repo_root: Path) -> list[tuple[str, str, Path]]: parser = MANIFEST_PARSERS.get(fname) if not parser: continue - name = parser(dirpath / fname) + manifest_path = dirpath / fname + # Every parser below opens the path. A FIFO named + # ``package.json`` would park that open in the kernel until a + # writer appears; ``is_file()`` stats instead and never blocks. + if not manifest_path.is_file(): + continue + name = parser(manifest_path) if name: found.append((fname, name, dirpath)) return sorted(found, key=lambda entry: _manifest_sort_key(entry, repo_root)) diff --git a/mempalace/repair.py b/mempalace/repair.py index 902c4f7b0..d8048ee97 100644 --- a/mempalace/repair.py +++ b/mempalace/repair.py @@ -30,6 +30,7 @@ """ import argparse +import errno import os import shutil import sqlite3 @@ -62,10 +63,29 @@ def _no_follow_flag() -> int: return getattr(os, "O_NOFOLLOW", 0) +def _non_blocking_flag() -> int: + """Return O_NONBLOCK, or 0 where the platform has no such flag (Windows). + + Without it the ``S_ISREG`` refusal in ``_open_regular_file_no_follow`` + is unreachable for a FIFO: opening one for reading blocks in the kernel + until a writer appears, so ``repair`` would wedge instead of refusing. + """ + return getattr(os, "O_NONBLOCK", 0) + + def _open_regular_file_no_follow(path: str) -> int: if os.path.islink(path): raise RuntimeError(f"Refusing symlinked file: {path}") - fd = os.open(path, os.O_RDONLY | _no_follow_flag()) + flags = os.O_RDONLY | _no_follow_flag() | _non_blocking_flag() + try: + fd = os.open(path, flags) + except OSError as exc: + # EAGAIN here is a write-lease break, which the kernel grants on + # regular files only, so re-check the type and open the way this + # helper did before the flag existed. Anything else propagates. + if exc.errno != errno.EAGAIN or not stat.S_ISREG(os.lstat(path).st_mode): + raise + fd = os.open(path, flags & ~_non_blocking_flag()) try: st = os.fstat(fd) if not stat.S_ISREG(st.st_mode): diff --git a/mempalace/room_detector_local.py b/mempalace/room_detector_local.py index f754f463a..1c88f2655 100644 --- a/mempalace/room_detector_local.py +++ b/mempalace/room_detector_local.py @@ -292,6 +292,11 @@ def save_config(project_dir: str, project_name: str, rooms: list): ], } config_path = Path(project_dir).expanduser().resolve() / "mempalace.yaml" + # Opening a pre-existing FIFO for writing blocks in the kernel until a + # reader appears. Only a regular file is a valid config target; refuse + # loudly rather than park ``init`` with no output. + if config_path.exists() and not config_path.is_file(): + raise OSError(f"Refusing to write config: {config_path} is not a regular file") with open(config_path, "w") as f: yaml.dump(config, f, default_flow_style=False, sort_keys=False) diff --git a/mempalace/split_mega_files.py b/mempalace/split_mega_files.py index 8ce0c859b..d876c7722 100644 --- a/mempalace/split_mega_files.py +++ b/mempalace/split_mega_files.py @@ -26,6 +26,7 @@ import json import os import re +import stat from pathlib import Path HOME = Path.home() @@ -272,7 +273,14 @@ def main(): mega_files = [] max_scan_size = 500 * 1024 * 1024 # 500 MB for f in files: - if f.stat().st_size > max_scan_size: + file_stat = f.stat() + # ``glob`` lists a FIFO named ``x.txt`` like any other match, and + # read_text() on one blocks in the kernel until a writer appears. + # stat() never blocks, so the type decides before the open does. + if not stat.S_ISREG(file_stat.st_mode): + print(f" SKIP: {f.name} (not a regular file)") + continue + if file_stat.st_size > max_scan_size: print(f" SKIP: {f.name} exceeds {max_scan_size // (1024 * 1024)} MB limit") continue lines = f.read_text(errors="replace").splitlines(keepends=True) diff --git a/mempalace/sweeper.py b/mempalace/sweeper.py index d036b6e67..715e41d2a 100644 --- a/mempalace/sweeper.py +++ b/mempalace/sweeper.py @@ -40,6 +40,7 @@ import json import logging +import stat import sys from datetime import datetime from pathlib import Path @@ -101,7 +102,16 @@ def parse_claude_jsonl(path: str) -> Iterator[dict]: queue-operation, last-prompt) are filtered out. Malformed lines are skipped silently — data quality is the transcript writer's problem, not ours. + + Raises ``OSError`` when ``path`` is not a regular file. ``rglob`` in + ``sweep_directory`` lists a FIFO named ``session.jsonl`` like any + other match, and opening one for reading blocks in the kernel until a + writer appears — an unbounded hang for the whole sweep. ``stat`` never + blocks on one, and it raises for a missing path exactly as ``open`` + did before, so callers see the same error for the same mistake. """ + if not stat.S_ISREG(Path(path).stat().st_mode): + raise OSError(f"Refusing non-regular file: {path}") with open(path, "r", encoding="utf-8", errors="replace") as f: for line in f: line = line.strip() @@ -337,6 +347,20 @@ def sweep_directory(dir_path: str, palace_path: str) -> dict: failures: list[dict] = [] for f in files: + # A non-regular match is not a sweep failure, it is nothing to sweep. + # ``rglob`` lists a FIFO or a symlink to /dev/null like any other + # ``*.jsonl``; report it the way ``miner.scan_project`` reports one + # and leave ``failures`` (and the exit status) for real errors. + # ``parse_claude_jsonl`` still refuses one, for callers arriving by + # another route. + try: + regular = stat.S_ISREG(f.stat().st_mode) + except OSError as exc: + print(f" SKIP: {f.name} (stat error: {exc.strerror or exc})", file=sys.stderr) + continue + if not regular: + print(f" SKIP: {f.name} (not a regular file)", file=sys.stderr) + continue try: result = sweep(str(f), palace_path, source_label=str(f)) except Exception as exc: diff --git a/mempalace/version.py b/mempalace/version.py index 757be79e8..e335ac20b 100644 --- a/mempalace/version.py +++ b/mempalace/version.py @@ -1,3 +1,3 @@ """Single source of truth for the MemPalace package version.""" -__version__ = "3.7.0+oc.2" +__version__ = "3.7.1+oc.1" diff --git a/pyproject.toml b/pyproject.toml index b1c721bb5..4ce48e37c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "mempalace" -version = "3.7.0+oc.2" +version = "3.7.1+oc.1" description = "Give your AI a memory — mine projects and conversations into a searchable palace. No API key required." readme = "README.md" requires-python = ">=3.10" @@ -126,7 +126,7 @@ multilingual = [] # Binary-format extraction for `mempalace mine --mode extract`. Per-format # transformer routing: striprtf for .rtf, MarkItDown for .pdf/.docx/.pptx/ # .xlsx/.epub. MarkItDown requires Python ≥ 3.10 (env marker ensures it -# only installs where supported; other platforms still get RTF coverage). +# only installs where supported; Python 3.9 users still get RTF coverage). extract = [ "striprtf>=0.0.27", # Per PR #1555 review (Igor): bare ``markitdown`` is enough to import diff --git a/tests/test_backends.py b/tests/test_backends.py index cc0de716a..ee419eb83 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -1860,6 +1860,92 @@ class DummyClient: ] +def test_chroma_backend_resets_system_cache_on_inode_change(tmp_path, monkeypatch): + """#2028: ``_client`` must drop chromadb's path-keyed ``SharedSystemClient`` + cache *before* reconstructing ``PersistentClient`` on an inode/mtime change. + + chromadb caches its ``System`` (and live HNSW segment) keyed by path, so a + bare reopen reuses the stale segment and persists an outdated index over a + peer/rebuild's on-disk changes -- the #2002 data-loss class reached via + ``_client`` instead of ``mcp_server._get_client``. The reset must fire only + on a genuine external change (not first open) and must precede the reopen. + """ + palace = tmp_path / "palace" + palace.mkdir() + (palace / "chroma.sqlite3").write_text("") + + events = [] + + # Neutralize the on-disk HNSW pre-checks so the test exercises only the + # cache-reset / client-rebuild ordering. + for _name in ( + "_fix_missing_collection_type", + "_fix_blob_seq_ids", + "quarantine_invalid_hnsw_metadata", + "quarantine_stale_hnsw", + ): + monkeypatch.setattr(f"mempalace.backends.chroma.{_name}", lambda path, *a, **k: []) + + monkeypatch.setattr(ChromaBackend, "_quarantined_paths", set()) + + class DummyClient: + def close(self): + events.append(("close", None)) + + def _record_open(path): + events.append(("open", path)) + return DummyClient() + + monkeypatch.setattr("mempalace.backends.chroma.chromadb.PersistentClient", _record_open) + + from chromadb.api.client import SharedSystemClient + + def _record_clear(*args, **kwargs): + events.append(("clear", None)) + + monkeypatch.setattr(SharedSystemClient, "clear_system_cache", _record_clear) + + backend = ChromaBackend() + # ``_db_stat`` is called twice per ``_client`` call (freshness check, then + # re-stat after reopen). Same inode on the first call (first open, no prior + # freshness -> no reset), changed inode on the second (external change). + stats = iter([(1, 1.0), (1, 1.0), (2, 2.0), (2, 2.0)]) + monkeypatch.setattr(backend, "_db_stat", lambda path: next(stats)) + + backend._client(str(palace)) # first open: no external change -> no clear + backend._client(str(palace)) # inode 1 -> 2: clear, then reopen + + assert events == [ + ("open", str(palace)), # first open, no cache reset + ("close", None), # release the previous HNSW client before eviction + ("clear", None), # #2028: reset fires on the inode change... + ("open", str(palace)), # ...strictly before the PersistentClient reopen + ], events + + +def test_chroma_backend_does_not_treat_its_own_write_as_external(tmp_path): + """A local write refreshes the cached DB signature. + + Otherwise the next operation closes the client that backs every live + collection wrapper, producing ``RustBindingsAPI has no bindings`` while + ordinary mine and re-mine flows are still using those wrappers. + """ + palace = tmp_path / "palace" + backend = ChromaBackend() + reference = PalaceRef(id=str(palace), local_path=str(palace)) + collection = backend.get_collection( + palace=reference, + collection_name="mempalace_drawers", + create=True, + ) + cached = backend._clients[str(palace)] + + collection.upsert(ids=["one"], documents=["one"], metadatas=[{"wing": "test"}]) + + assert backend._client(str(palace)) is cached + assert collection.count() == 1 + + def test_explain_ef_mismatch_recognizes_chromadb_conflict(): """When ChromaDB rejects a collection read due to an EF-name mismatch (user changed MEMPALACE_EMBEDDING_MODEL on an existing palace), the diff --git a/tests/test_convo_miner.py b/tests/test_convo_miner.py index 3a069e97b..b9bf8c9ce 100644 --- a/tests/test_convo_miner.py +++ b/tests/test_convo_miner.py @@ -14,7 +14,12 @@ _resolve_wing, mine_convos, ) -from mempalace.palace import MineAlreadyRunning, file_already_mined, prefetch_mined_set +from mempalace.palace import ( + NORMALIZE_VERSION, + MineAlreadyRunning, + file_already_mined, + prefetch_mined_set, +) def test_convo_mining(): @@ -767,6 +772,71 @@ def test_prefetch_mined_set_none_for_drawer_without_stored_mtime(): shutil.rmtree(tmpdir, ignore_errors=True) +def test_prefetch_mined_set_omits_incomplete_chunk_total_group(): + """Mid-file partials with chunk_total must not bulk-skip the source (#2183).""" + tmpdir = tempfile.mkdtemp() + try: + palace_path = os.path.join(tmpdir, "palace") + client = chromadb.PersistentClient(path=palace_path) + col = client.get_or_create_collection("mempalace_drawers") + mtime = 1_700_000_000.0 + source = "/fake/session.jsonl" + # Only 2 of 3 expected chunks landed before a crash. + col.upsert( + ids=["d0", "d1"], + documents=["chunk 0", "chunk 1"], + metadatas=[ + { + "wing": "test", + "room": "general", + "source_file": source, + "chunk_index": 0, + "extract_mode": "exchange", + "normalize_version": NORMALIZE_VERSION, + "source_mtime": mtime, + "chunk_total": 3, + }, + { + "wing": "test", + "room": "general", + "source_file": source, + "chunk_index": 1, + "extract_mode": "exchange", + "normalize_version": NORMALIZE_VERSION, + "source_mtime": mtime, + "chunk_total": 3, + }, + ], + ) + mined = prefetch_mined_set(col, extract_mode="exchange") + assert source not in mined, ( + "prefetch_mined_set treated 2/3 chunks as fully filed — the bulk " + "skip path would permanently strand the missing exchange (#2183)" + ) + + col.upsert( + ids=["d2"], + documents=["chunk 2"], + metadatas=[ + { + "wing": "test", + "room": "general", + "source_file": source, + "chunk_index": 2, + "extract_mode": "exchange", + "normalize_version": NORMALIZE_VERSION, + "source_mtime": mtime, + "chunk_total": 3, + } + ], + ) + mined = prefetch_mined_set(col, extract_mode="exchange") + assert source in mined + assert abs(mined[source] - mtime) < 0.001 + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + def test_mine_convos_reprocesses_legacy_drawer_without_stored_mtime(capsys): """A file mined before source_mtime was tracked (simulated: drawer written directly, no source_mtime field) must be re-mined on the next diff --git a/tests/test_convo_miner_unit.py b/tests/test_convo_miner_unit.py index d6bb4c54f..3545c791a 100644 --- a/tests/test_convo_miner_unit.py +++ b/tests/test_convo_miner_unit.py @@ -718,6 +718,107 @@ def upsert(self, documents, ids, metadatas): assert drawers == 0 assert skipped is True + def test_stamps_chunk_total_for_completion_check(self, monkeypatch): + """Every convo drawer of one pass must carry chunk_total (#2183).""" + import mempalace.convo_miner as convo_miner + + class FakeCol: + def __init__(self): + self.metas = [] + + def delete(self, *args, **kwargs): + pass + + def get(self, ids=None, include=None, **kwargs): + return {"ids": [], "metadatas": []} + + def upsert(self, documents, ids, metadatas): + self.metas.extend(metadatas) + + chunks = [{"content": f"chunk {i} " * 20, "chunk_index": i} for i in range(5)] + col = FakeCol() + monkeypatch.setattr(convo_miner, "DRAWER_UPSERT_BATCH_SIZE", 2) + monkeypatch.setattr( + convo_miner, "file_already_mined", lambda collection, source_file, **kwargs: False + ) + monkeypatch.setattr(convo_miner, "mine_lock", lambda source_file: contextlib.nullcontext()) + monkeypatch.setattr(convo_miner, "_detect_hall_cached", lambda content: "conversations") + + _file_chunks_locked(col, "chat.txt", chunks, "wing", "general", "agent", "exchange") + + assert len(col.metas) == 5 + assert all(m.get("chunk_total") == 5 for m in col.metas), ( + "not every convo chunk carries the pass's chunk_total — a mid-file " + "crash would leave mtime-stamped partials that skip forever (#2183)" + ) + + def test_cleans_partial_drawers_after_batch_upsert_failure(self, monkeypatch, tmp_path): + """A failed later batch must not leave mtime-stamped partials (#2183).""" + import mempalace.convo_miner as convo_miner + + class FailingCol: + def __init__(self): + self.records = [] + self.upsert_calls = 0 + self.deleted_ids = [] + + def get(self, where=None, limit=None, offset=0, include=None, ids=None, **kwargs): + if ids is not None: + return {"ids": [], "metadatas": []} + records = self.records + if where and "source_file" in where: + records = [ + r + for r in records + if r["metadata"].get("source_file") == where["source_file"] + ] + page = records[offset : offset + (limit or len(records))] + return { + "ids": [r["id"] for r in page], + "metadatas": [r["metadata"] for r in page], + } + + def delete(self, ids=None, where=None, **kwargs): + if ids: + self.deleted_ids.extend(ids) + id_set = set(ids) + self.records = [r for r in self.records if r["id"] not in id_set] + return + if where and "source_file" in where: + src = where["source_file"] + self.records = [ + r for r in self.records if r["metadata"].get("source_file") != src + ] + + def upsert(self, documents, ids, metadatas): + self.upsert_calls += 1 + if self.upsert_calls == 2: + raise RuntimeError("simulated second-batch failure") + self.records.extend( + {"id": drawer_id, "metadata": metadata} + for drawer_id, metadata in zip(ids, metadatas) + ) + + source = tmp_path / "chat.txt" + source.write_text("content\n", encoding="utf-8") + chunks = [{"content": f"chunk {i} " * 20, "chunk_index": i} for i in range(3)] + col = FailingCol() + monkeypatch.setattr(convo_miner, "DRAWER_UPSERT_BATCH_SIZE", 2) + monkeypatch.setattr( + convo_miner, "file_already_mined", lambda collection, source_file, **kwargs: False + ) + monkeypatch.setattr(convo_miner, "mine_lock", lambda source_file: contextlib.nullcontext()) + monkeypatch.setattr(convo_miner, "_detect_hall_cached", lambda content: "conversations") + + with pytest.raises(RuntimeError, match="second-batch failure"): + _file_chunks_locked(col, str(source), chunks, "wing", "general", "agent", "exchange") + + assert col.records == [], ( + "partial convo drawers survived a mid-file upsert failure — the " + "next mine would skip this incomplete file forever (#2183)" + ) + assert col.deleted_ids, "cleanup did not delete the partial drawer ids" + class TestSourceFileDeleteIds: """#104: the sweeper writes drawers with no extract_mode at all diff --git a/tests/test_hybrid_search.py b/tests/test_hybrid_search.py index e60f77cb0..5b30e63b5 100644 --- a/tests/test_hybrid_search.py +++ b/tests/test_hybrid_search.py @@ -8,6 +8,7 @@ """ from mempalace.palace import ( + get_backend_for_palace, get_closets_collection, get_collection, upsert_closet_lines, @@ -15,6 +16,29 @@ from mempalace.searcher import _hybrid_rank, search_memories +def _close_palace(palace_path: str) -> None: + """Release chromadb client handles so the next open rebuilds from disk. + + Windows CI intermittently returns zero hybrid hits right after a fast + seed write (same class of flake as "Nothing found on disk" on tiny + closet collections). Closing the cached client forces the next + ``search_memories`` open to re-read segments that have been flushed. + """ + try: + get_backend_for_palace(palace_path).close_palace(palace_path) + except Exception: + pass + + +def _search(query: str, palace: str, **kwargs): + """Search, retrying once after a client reopen if results are empty.""" + result = search_memories(query, palace, **kwargs) + if result.get("results"): + return result + _close_palace(palace) + return search_memories(query, palace, **kwargs) + + def _seed_drawers(palace_path): """Insert 4 short drawers with deterministic content.""" col = get_collection(palace_path, create=True) @@ -33,6 +57,7 @@ def _seed_drawers(palace_path): {"wing": "backend", "room": "queue", "source_file": "fixture_D4.md"}, ], ) + _close_palace(palace_path) def _seed_strong_closet_for(palace_path, drawer_id, source_file, topics): @@ -65,6 +90,7 @@ def _seed_strong_closet_for(palace_path, drawer_id, source_file, topics): } ], ) + _close_palace(palace_path) # ── core invariant: closets can only HELP, never HIDE ───────────────────── @@ -75,7 +101,7 @@ def test_no_closets_degrades_to_direct_drawer_search(self, tmp_path): palace = str(tmp_path / "palace") _seed_drawers(palace) # No closets created. - result = search_memories("Kafka rebalance timeout", palace, n_results=3) + result = _search("Kafka rebalance timeout", palace, n_results=3) ids = [h["source_file"] for h in result["results"]] assert ids, "should return results" assert "fixture_D4.md" in ids, "direct drawer search alone should surface the Kafka drawer" @@ -92,7 +118,7 @@ def test_weak_closets_do_not_hide_direct_drawer_hits(self, tmp_path): source_file="fixture_D3.md", topics=["Kafka queue tuning", "consumer rebalance config"], ) - result = search_memories("Kafka consumer rebalance timeout", palace, n_results=5) + result = _search("Kafka consumer rebalance timeout", palace, n_results=5) ids = [h["source_file"] for h in result["results"]] assert "fixture_D4.md" in ids, ( "D4 must appear — direct drawer search alone would rank it first. " @@ -110,8 +136,9 @@ def test_closet_boost_lifts_matching_drawer(self, tmp_path): source_file="fixture_D1.md", topics=["JWT auth tokens", "session expiry", "authentication service"], ) - result = search_memories("JWT auth tokens expiry", palace, n_results=3) + result = _search("JWT auth tokens expiry", palace, n_results=3) ids = [h["source_file"] for h in result["results"]] + assert ids, f"expected hybrid hits after seeding drawers+closets; got {result!r}" assert ids[0] == "fixture_D1.md" top = result["results"][0] assert top["matched_via"] == "drawer+closet" @@ -131,7 +158,7 @@ def test_closet_preview_exposed_when_boosted(self, tmp_path): source_file="fixture_D1.md", topics=["JWT auth tokens", "session expiry", "authentication service"], ) - result = search_memories("JWT auth tokens expiry", palace, n_results=2) + result = _search("JWT auth tokens expiry", palace, n_results=2) top = result["results"][0] assert top["source_file"] == "fixture_D1.md" assert top["matched_via"] == "drawer+closet" @@ -142,7 +169,7 @@ def test_drawer_only_hits_have_no_closet_preview(self, tmp_path): palace = str(tmp_path / "palace") _seed_drawers(palace) # No closets - result = search_memories("TanStack Query", palace, n_results=2) + result = _search("TanStack Query", palace, n_results=2) assert result["results"] for h in result["results"]: assert h["matched_via"] == "drawer" @@ -157,7 +184,7 @@ class TestSourceFileFilter: def test_source_file_filter_excludes_other_sources(self, tmp_path): palace = str(tmp_path / "palace") _seed_drawers(palace) - result = search_memories( + result = _search( "Kafka consumer rebalance timeout", palace, n_results=5, @@ -179,7 +206,7 @@ def test_source_file_filter_overrides_closet_boost_for_other_source(self, tmp_pa source_file="fixture_D1.md", topics=["Kafka queue tuning", "consumer rebalance config"], ) - result = search_memories( + result = _search( "Kafka consumer rebalance", palace, n_results=5, diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 2724207ce..7bf3c5e50 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -77,6 +77,40 @@ def test_mcp_main_strips_leaked_pythonpath_from_env(): assert "ENV_AFTER: None" in result.stderr, f"MCP server did not strip PYTHONPATH: {diag}" +def test_install_shutdown_signal_handlers_routes_term_to_system_exit(): + """SIGTERM/SIGHUP must raise SystemExit so atexit can release the lease (#2205).""" + import signal + + from mempalace import mcp_server + + previous = {} + for name in ("SIGTERM", "SIGHUP"): + sig = getattr(signal, name, None) + if sig is None: + continue + previous[sig] = signal.getsignal(sig) + + try: + mcp_server._install_shutdown_signal_handlers() + term = signal.SIGTERM + handler = signal.getsignal(term) + assert callable(handler) + with pytest.raises(SystemExit) as exc_info: + handler(term, None) + assert exc_info.value.code == 0 + + sighup = getattr(signal, "SIGHUP", None) + if sighup is not None: + hup_handler = signal.getsignal(sighup) + assert callable(hup_handler) + with pytest.raises(SystemExit) as exc_info: + hup_handler(sighup, None) + assert exc_info.value.code == 0 + finally: + for sig, old in previous.items(): + signal.signal(sig, old) + + def _patch_mcp_server(monkeypatch, config, kg): """Patch the mcp_server module globals to use test fixtures.""" from mempalace import mcp_server @@ -605,150 +639,6 @@ def test_reload_does_not_duplicate_file_handler(self, tmp_path): class TestHandleRequest: - def test_official_sdk_owns_protocol_server(self): - """The protocol entrypoint is the official low-level MCP Server.""" - from mcp.server import Server - from mempalace import mcp_server - - assert isinstance(mcp_server._MCP_SDK_SERVER, Server) - - def test_official_sdk_refreshes_idle_clock_for_every_stdio_message(self, monkeypatch): - """SDK-owned ping/initialize/notifications pass through activity middleware.""" - import anyio - from mempalace import mcp_server - - monkeypatch.setattr(mcp_server.time, "monotonic", lambda: 123.0) - monkeypatch.setattr(mcp_server, "_last_request_time", 0.0) - assert mcp_server._sdk_refresh_idle_activity in mcp_server._MCP_SDK_SERVER.middleware - - async def exercise(): - async def call_next(ctx): - return ctx - - marker = object() - result = await mcp_server._sdk_refresh_idle_activity(marker, call_next) - assert result is marker - - anyio.run(exercise) - assert mcp_server._last_request_time == 123.0 - - def test_official_sdk_stdio_handshake_and_tools_list(self): - """Exercise the real SDK client/server handshake over stdio.""" - import anyio - from mcp import ClientSession - from mcp.client.stdio import StdioServerParameters, stdio_client - - async def exercise(): - server = StdioServerParameters( - command=sys.executable, - args=["-m", "mempalace.mcp_server"], - env=os.environ.copy(), - ) - async with stdio_client(server) as (read_stream, write_stream): - async with ClientSession(read_stream, write_stream) as session: - initialized = await session.initialize() - assert initialized.server_info.name == "mempalace" - tools = await session.list_tools() - assert "mempalace_search" in {tool.name for tool in tools.tools} - - anyio.run(exercise) - - def test_official_sdk_read_only_hides_and_refuses_writes(self, monkeypatch): - import anyio - import mcp.types as mcp_types - from mcp.shared.exceptions import MCPError - from mempalace import mcp_server - - monkeypatch.setattr(mcp_server, "_READ_ONLY", True) - - async def exercise(): - listed = await mcp_server._sdk_list_tools(None, None) - names = {tool.name for tool in listed.tools} - assert "mempalace_add_drawer" not in names - params = mcp_types.CallToolRequestParams( - name="mempalace_add_drawer", arguments={"content": "x"} - ) - with pytest.raises(MCPError) as exc_info: - await mcp_server._sdk_call_http_tool(None, params) - assert exc_info.value.error.code == -32003 - - anyio.run(exercise) - - def test_official_sdk_stdio_callbacks_proxy_live_hub(self, monkeypatch): - import socket - import threading - import time - - import anyio - import mcp.types as mcp_types - import uvicorn - from mempalace import mcp_server - - app, state = mcp_server._build_sdk_http_app("127.0.0.1", 0) - sock = socket.socket() - sock.bind(("127.0.0.1", 0)) - sock.listen() - port = sock.getsockname()[1] - state.server_address = ("127.0.0.1", port) - server = uvicorn.Server(uvicorn.Config(app, log_config=None, log_level="warning")) - thread = threading.Thread(target=server.run, kwargs={"sockets": [sock]}, daemon=True) - thread.start() - for _ in range(100): - if server.started: - break - time.sleep(0.02) - monkeypatch.setattr( - mcp_server, - "_hub_proxy_target", - lambda: (f"http://127.0.0.1:{port}", {}), - ) - - async def exercise(): - listed = await mcp_server._sdk_list_stdio_tools(None, None) - assert "mempalace_get_aaak_spec" in {tool.name for tool in listed.tools} - result = await mcp_server._sdk_call_stdio_tool( - None, - mcp_types.CallToolRequestParams(name="mempalace_get_aaak_spec", arguments={}), - ) - assert "AAAK" in result.content[0].text - - try: - anyio.run(exercise) - finally: - server.should_exit = True - thread.join(timeout=5) - - def test_official_sdk_tool_results_errors_and_concurrency(self): - import anyio - import mcp.types as mcp_types - from mcp.shared.exceptions import MCPError - from mempalace import mcp_server - - async def exercise(): - results = [] - - async def call_spec(): - result = await mcp_server._sdk_call_http_tool( - None, - mcp_types.CallToolRequestParams(name="mempalace_get_aaak_spec", arguments={}), - ) - results.append(result.content[0].text) - - async with anyio.create_task_group() as group: - group.start_soon(call_spec) - group.start_soon(call_spec) - assert len(results) == 2 - assert all("AAAK" in result for result in results) - with pytest.raises(MCPError) as exc_info: - await mcp_server._sdk_call_http_tool( - None, - mcp_types.CallToolRequestParams(name="does_not_exist", arguments={}), - ) - assert exc_info.value.code == -32601 - assert exc_info.value.message == "Unknown tool: does_not_exist" - - anyio.run(exercise) - def test_initialize(self): from mempalace.mcp_server import handle_request @@ -3419,19 +3309,18 @@ def test_dry_run_reports_count_without_deleting(self, monkeypatch, config, palac def test_dry_run_reports_closet_match_count(self, monkeypatch, config, palace_path, kg): """Dry run surfaces the closet blast radius (#1722) without deleting.""" self._seed(monkeypatch, config, palace_path, kg) - closets_col = self._seed_closets(palace_path) + self._seed_closets(palace_path) from mempalace.mcp_server import tool_delete_by_source + from mempalace.palace import get_closets_collection result = tool_delete_by_source("results_mempal_hybrid_v4_session_1.jsonl") assert result["dry_run"] is True assert result["closet_match_count"] == 2 - # The server drains stale Chroma handles after an mtime-triggered - # reconnect, so verify through a fresh collection handle. - del closets_col - from mempalace.palace import get_closets_collection - - fresh_closets = get_closets_collection(palace_path) - assert len(fresh_closets.get(include=[])["ids"]) == 3 + # Re-acquire: the staleness reconnect drops chromadb's path-keyed System + # cache (#2002), so a handle taken before the call is dead by now. + closets_col = get_closets_collection(palace_path, create=False) + # Nothing removed — all three closets still present. + assert len(closets_col.get(include=[])["ids"]) == 3 def test_commit_deletes_only_matching_source(self, monkeypatch, config, palace_path, kg): self._seed(monkeypatch, config, palace_path, kg) @@ -3448,20 +3337,19 @@ def test_commit_purges_matching_closets(self, monkeypatch, config, palace_path, """Deleting by source purges the matching closets too, so the AAAK index keeps no stale pointers at the now-deleted drawers (#1722).""" self._seed(monkeypatch, config, palace_path, kg) - closets_col = self._seed_closets(palace_path) + self._seed_closets(palace_path) from mempalace.mcp_server import tool_delete_by_source + from mempalace.palace import get_closets_collection result = tool_delete_by_source("results_mempal_hybrid_v4_session_1.jsonl", dry_run=False) assert result["success"] is True assert result["deleted"] == 2 assert result["closets_deleted"] == 2 + # Re-acquire: the staleness reconnect drops chromadb's path-keyed System + # cache (#2002), so a handle taken before the call is dead by now. + closets_col = get_closets_collection(palace_path, create=False) # The two benchmark closets are gone; the real-client closet survives. - # Reconnect intentionally invalidates pre-write Chroma handles. - del closets_col - from mempalace.palace import get_closets_collection - - fresh_closets = get_closets_collection(palace_path) - remaining = fresh_closets.get(include=["metadatas"]) + remaining = closets_col.get(include=["metadatas"]) sources = {m["source_file"] for m in remaining["metadatas"]} assert sources == {"notes/clients.md"} @@ -4924,33 +4812,50 @@ def spy_prepare(path): "_get_client should call _prepare_palace_for_open on reconnect" ) - def test_get_client_closes_cached_chroma_client_before_mtime_reconnect( + def test_get_client_resets_chroma_system_cache_on_reconnect( self, monkeypatch, config, palace_path, kg ): - """An in-place database update must not retain the old HNSW client.""" + """``_get_client`` must clear chromadb's path-keyed System/HNSW cache + (via ``_force_chroma_cache_reset``) *before* calling ``make_client`` on an + inode/mtime reconnect. Otherwise chromadb hands back the stale in-memory + HNSW segment, which persists its outdated index over a peer writer's + on-disk changes, driving the persisted count backwards (#2002).""" _patch_mcp_server(monkeypatch, config, kg) from mempalace import mcp_server + from mempalace.backends.chroma import ChromaBackend - make_minimal_chroma_sqlite(config.palace_path) - old_client = MagicMock() - new_client = MagicMock() - monkeypatch.setattr(mcp_server, "_client_cache", old_client) - monkeypatch.setattr(mcp_server, "_collection_cache", MagicMock()) - monkeypatch.setattr(mcp_server, "_palace_db_inode", 0) - monkeypatch.setattr(mcp_server, "_palace_db_mtime", 1.0) - monkeypatch.setattr(mcp_server, "_refresh_vector_disabled_flag", lambda: None) - monkeypatch.setattr(mcp_server.ChromaBackend, "make_client", lambda _path: new_client) + _client, _col = _get_collection(palace_path, create=True) + del _client + + # Prime the cache. + mcp_server._get_collection() - reset_calls = [] + # Simulate a peer writer touching chroma.sqlite3 on disk. + old_mtime = mcp_server._palace_db_mtime + monkeypatch.setattr(mcp_server, "_palace_db_mtime", old_mtime - 10.0) + + order: list[str] = [] + real_reset = mcp_server._force_chroma_cache_reset + real_make = ChromaBackend.make_client + + def spy_reset(): + order.append("reset") + real_reset() - def reset(): - reset_calls.append(True) - mcp_server._client_cache = None + @staticmethod + def spy_make(path): + order.append("make_client") + return real_make(path) + + monkeypatch.setattr(mcp_server, "_force_chroma_cache_reset", spy_reset) + monkeypatch.setattr(ChromaBackend, "make_client", spy_make) - monkeypatch.setattr(mcp_server, "_force_chroma_cache_reset", reset) + mcp_server._get_client() - assert mcp_server._get_client() is new_client - assert reset_calls == [True] + assert order == ["reset", "make_client"], ( + "_get_client must reset chromadb's system cache BEFORE reopening the " + "client on a staleness reconnect (#2002)" + ) def test_call_kg_retries_after_concurrent_close(self, monkeypatch): """A KG closed mid-handler must trigger a one-shot retry with a fresh diff --git a/tests/test_miner.py b/tests/test_miner.py index 83efc7c9b..037cd63d2 100644 --- a/tests/test_miner.py +++ b/tests/test_miner.py @@ -1243,6 +1243,354 @@ def upsert(self, documents, ids, metadatas): assert col.batch_sizes == [2, 2, 1] +def test_process_file_stamps_chunk_total_for_completion_check(tmp_path, monkeypatch): + """Every chunk across every batch of one mining pass must carry the + same ``chunk_total`` so ``file_already_mined`` can tell a complete + multi-batch mine from one that crashed partway through (#21).""" + from mempalace import miner + + class FakeCol: + def __init__(self): + self.metadatas: list = [] + + def get(self, *args, **kwargs): + return {"ids": []} + + def delete(self, *args, **kwargs): + pass + + def upsert(self, documents, ids, metadatas): + self.metadatas.extend(metadatas) + + source = tmp_path / "src.py" + source.write_text("print('hello')\n" * 20, encoding="utf-8") + chunks = [{"content": f"chunk {i} " * 20, "chunk_index": i} for i in range(5)] + col = FakeCol() + monkeypatch.setattr(miner, "DRAWER_UPSERT_BATCH_SIZE", 2) + monkeypatch.setattr(miner, "chunk_text", lambda content, source_file, **kwargs: chunks) + monkeypatch.setattr(miner, "detect_hall", lambda content: "code") + monkeypatch.setattr(miner, "_extract_entities_for_metadata", lambda content: "") + + miner.process_file( + source, + tmp_path, + col, + "wing", + [{"name": "general", "description": "General"}], + "agent", + False, + ) + + assert len(col.metadatas) == 5 + assert all(m["chunk_total"] == 5 for m in col.metadatas), ( + "not every chunk carries the pass's chunk_total — " + "file_already_mined can't verify completeness without it on every row" + ) + + +def test_process_file_stamps_metadata_with_read_time_mtime_not_a_later_restat( + tmp_path, monkeypatch +): + """The stored source_mtime must be the one paired with the content + that was actually read and chunked, not a fresh os.path.getmtime() + call later in the function (#22). Otherwise a file appended to + between the read and the old re-stat point gets stamped with an + mtime that matches its (now newer) on-disk state, so the next + mine's freshness check thinks nothing changed and the appended tail + is silently, permanently skipped.""" + from mempalace import miner + + class FakeCol: + def __init__(self): + self.metadatas: list = [] + + def get(self, *args, **kwargs): + return {"ids": []} + + def delete(self, *args, **kwargs): + pass + + def upsert(self, documents, ids, metadatas): + self.metadatas.extend(metadatas) + + source = tmp_path / "src.py" + source.write_text("print('hello')\n" * 20, encoding="utf-8") + + read_time_mtime = 1_700_000_000.0 + later_disk_mtime = 1_700_000_999.0 # simulates an append landing after the read + + monkeypatch.setattr( + miner, + "_read_text_no_follow", + lambda filepath, root: ("print('hello')\n" * 20, read_time_mtime), + ) + monkeypatch.setattr(os.path, "getmtime", lambda path: later_disk_mtime) + chunks = [{"content": "chunk 0 " * 20, "chunk_index": 0}] + col = FakeCol() + monkeypatch.setattr(miner, "chunk_text", lambda content, source_file, **kwargs: chunks) + monkeypatch.setattr(miner, "detect_hall", lambda content: "code") + monkeypatch.setattr(miner, "_extract_entities_for_metadata", lambda content: "") + + miner.process_file( + source, + tmp_path, + col, + "wing", + [{"name": "general", "description": "General"}], + "agent", + False, + ) + + assert len(col.metadatas) == 1 + assert col.metadatas[0]["source_mtime"] == read_time_mtime, ( + "drawer was stamped with a re-stat'd mtime instead of the one " + "paired with the content actually read/chunked" + ) + + +def test_process_file_aborts_when_stale_drawer_purge_fails(tmp_path, monkeypatch): + """A failed stale-drawer purge must abort this file's mine attempt, + not silently proceed to upsert on top of it (#23). Proceeding either + orphans old tail entries beyond the new chunk count, or overwrites + only the overlapping chunk_index positions — not a real re-mine — + with zero operator-visible signal unless DEBUG logging happens to be + enabled.""" + from mempalace import miner + + class FailingPurgeCol: + def __init__(self): + self.upsert_called = False + + def get(self, *args, **kwargs): + return {"ids": []} + + def delete(self, *args, **kwargs): + raise RuntimeError("simulated transient backend error") + + def upsert(self, documents, ids, metadatas): + self.upsert_called = True + + source = tmp_path / "src.py" + source.write_text("print('hello')\n" * 20, encoding="utf-8") + chunks = [{"content": f"chunk {i} " * 20, "chunk_index": i} for i in range(3)] + col = FailingPurgeCol() + monkeypatch.setattr(miner, "chunk_text", lambda content, source_file, **kwargs: chunks) + monkeypatch.setattr(miner, "detect_hall", lambda content: "code") + monkeypatch.setattr(miner, "_extract_entities_for_metadata", lambda content: "") + + drawers, room, skip_reason = miner.process_file( + source, + tmp_path, + col, + "wing", + [{"name": "general", "description": "General"}], + "agent", + False, + ) + + assert col.upsert_called is False, ( + "process_file inserted new chunks even though the stale-drawer " + "purge raised — old and new rows can now coexist as duplicates/orphans" + ) + assert drawers == 0 + + +def test_process_file_purges_closets_even_when_all_chunks_filtered_out(tmp_path, monkeypatch): + """Old closets must be purged whenever the old drawers were deleted, + even if the new content ends up producing zero filed drawers (#24). + Otherwise the stale closet entries point at drawer IDs that were + just deleted, permanently misdirecting search until the file + changes again in a way that produces at least one filed chunk.""" + from mempalace import miner + + class FakeCol: + def get(self, *args, **kwargs): + return {"ids": []} + + def delete(self, *args, **kwargs): + pass + + def upsert(self, documents, ids, metadatas): + raise AssertionError("no chunks should be upserted in this scenario") + + purged: list = [] + + source = tmp_path / "src.py" + source.write_text("x" * 200, encoding="utf-8") + col = FakeCol() + # Content passes the file-level min-length gate, but every individual + # chunk gets filtered out downstream (e.g. each fragment falls below + # min_chunk_size after boundary-splitting) -- modeled directly here. + monkeypatch.setattr(miner, "chunk_text", lambda content, source_file, **kwargs: []) + monkeypatch.setattr( + miner, "purge_file_closets", lambda closets_col, source_file: purged.append(source_file) + ) + monkeypatch.setattr( + miner, + "upsert_closet_lines", + lambda *a, **kw: pytest.fail("should not rebuild closets with zero drawers"), + ) + + drawers, room, skip_reason = miner.process_file( + source, + tmp_path, + col, + "wing", + [{"name": "general", "description": "General"}], + "agent", + False, + closets_col=object(), + ) + + assert drawers == 0 + assert purged == [str(source)], ( + "old closets for this source_file were left dangling — they point " + "at drawer IDs that collection.delete() already removed" + ) + + +def test_file_already_mined_detects_incomplete_multi_batch_remine(): + """A crash between upsert batches must not be mistaken for 'fully + mined' (#21). process_file stamps every chunk's metadata with + chunk_total (the total chunks expected for this pass). If killed + after batch 1 commits but before a later batch, the surviving + drawers share the current on-disk mtime (the file itself was never + touched) but their count is short of chunk_total — file_already_mined + must detect that and report False so the file gets fully re-mined, + not silently skipped forever.""" + tmpdir = tempfile.mkdtemp() + try: + palace_path = os.path.join(tmpdir, "palace") + os.makedirs(palace_path) + client = chromadb.PersistentClient(path=palace_path) + col = client.get_or_create_collection("mempalace_drawers") + + test_file = os.path.join(tmpdir, "big.md") + with open(test_file, "w") as f: + f.write("content") + mtime = os.path.getmtime(test_file) + + # Simulate a crash after only 2 of 3 expected chunks committed. + col.add( + ids=["d0", "d1"], + documents=["chunk 0", "chunk 1"], + metadatas=[ + { + "source_file": test_file, + "source_mtime": mtime, + "normalize_version": NORMALIZE_VERSION, + "chunk_total": 3, + }, + { + "source_file": test_file, + "source_mtime": mtime, + "normalize_version": NORMALIZE_VERSION, + "chunk_total": 3, + }, + ], + ) + + assert file_already_mined(col, test_file, check_mtime=True) is False, ( + "2 of 3 expected chunks were treated as a complete mine — the " + "missing chunk is now permanently unreachable since the file's " + "on-disk mtime never changes again" + ) + + # The 3rd batch lands (mine resumes/retries and completes the set). + col.add( + ids=["d2"], + documents=["chunk 2"], + metadatas=[ + { + "source_file": test_file, + "source_mtime": mtime, + "normalize_version": NORMALIZE_VERSION, + "chunk_total": 3, + } + ], + ) + assert file_already_mined(col, test_file, check_mtime=True) is True + finally: + del col, client + shutil.rmtree(tmpdir, ignore_errors=True) + + +def test_process_file_cleans_partial_drawers_after_a_batch_upsert_failure(tmp_path, monkeypatch): + """A failed later batch must not leave mtime-stamped drawers that skip retry (#2122).""" + from mempalace import miner + + class FailingCollection: + def __init__(self): + self.records = [] + self.upsert_calls = 0 + self.deleted_sources = [] + + def get(self, where=None, limit=None, offset=0, include=None): + records = self.records + if where and "source_file" in where: + records = [ + record + for record in records + if record["metadata"]["source_file"] == where["source_file"] + ] + page = records[offset : offset + (limit or len(records))] + return { + "ids": [record["id"] for record in page], + "metadatas": [record["metadata"] for record in page], + } + + def delete(self, where=None): + source_file = where.get("source_file") if where else None + self.deleted_sources.append(source_file) + self.records = [ + record + for record in self.records + if record["metadata"]["source_file"] != source_file + ] + + def upsert(self, documents, ids, metadatas): + self.upsert_calls += 1 + if self.upsert_calls == 2: + raise RuntimeError("simulated second-batch failure") + self.records.extend( + {"id": drawer_id, "metadata": metadata} + for drawer_id, metadata in zip(ids, metadatas) + ) + + class FakeClosets: + def __init__(self): + self.deleted_sources = [] + + def delete(self, where=None): + self.deleted_sources.append(where.get("source_file")) + + source = tmp_path / "src.py" + source.write_text("print('hello')\n" * 20, encoding="utf-8") + chunks = [{"content": f"chunk {index} " * 20, "chunk_index": index} for index in range(3)] + collection = FailingCollection() + closets = FakeClosets() + monkeypatch.setattr(miner, "DRAWER_UPSERT_BATCH_SIZE", 2) + monkeypatch.setattr(miner, "chunk_text", lambda content, source_file, **kwargs: chunks) + monkeypatch.setattr(miner, "assert_no_collisions", lambda *args, **kwargs: None) + + with pytest.raises(RuntimeError, match="second-batch failure"): + miner.process_file( + source, + tmp_path, + collection, + "wing", + [{"name": "general", "description": "General"}], + "agent", + False, + closets_col=closets, + ) + + assert collection.deleted_sources == [str(source), str(source)] + assert collection.records == [] + assert closets.deleted_sources == [str(source)] + assert file_already_mined(collection, str(source), check_mtime=True) is False + + # ── normalize_version schema gate ─────────────────────────────────────── # # When the normalization pipeline changes shape (e.g., strip_noise lands), diff --git a/tests/test_non_regular_file_guards.py b/tests/test_non_regular_file_guards.py new file mode 100644 index 000000000..e982d9da7 --- /dev/null +++ b/tests/test_non_regular_file_guards.py @@ -0,0 +1,722 @@ +"""Non-regular files must never wedge an ingest command. + +``os.walk``/``rglob`` list a FIFO, a socket and a device node as ordinary +filenames, and MemPalace decides what to read by extension. Opening a FIFO +for reading parks in the kernel until a writer appears, so a named pipe +called ``notes.md`` sitting in a mined directory used to hang ``mine``, +``sweep`` and ``init`` forever — no output, no error, no progress. + +Every check here is wrapped in :func:`hard_timeout`. A regression must turn +this file red; it must not hang the suite (an unbounded blocking open would +otherwise stall pytest itself, which reports as "still running", not as a +failure). +""" + +import argparse +import errno +import hashlib +import os +import signal +import socket +import stat as stat_module +import threading +from contextlib import contextmanager +from pathlib import Path +from unittest.mock import patch + +import pytest +import yaml + +from mempalace.cli import ( + _ensure_mempalace_files_gitignored, + _gather_origin_samples, + cmd_compress, + cmd_init, +) +from mempalace.convo_miner import _is_regular_source_file, scan_convos +from mempalace.entity_detector import detect_entities +from mempalace.format_miner import ExtractionStatus, extract_text +from mempalace.hook_shell import count_human_messages +from mempalace.llm_refine import collect_corpus_text +from mempalace.miner import _read_text_no_follow, load_config, mine, scan_project +from mempalace.normalize import _read_transcript_file +from mempalace.project_scanner import _collect_manifest_names +from mempalace.repair import _copy_file_no_follow, _open_regular_file_no_follow +from mempalace.room_detector_local import detect_rooms_local +from mempalace.split_mega_files import main as split_main +from mempalace.sweeper import parse_claude_jsonl, sweep_directory + +# ``os.mkfifo`` and ``SIGALRM`` are both POSIX-only. Windows has no FIFO in +# the filesystem namespace at all (its named pipes live under \\.\pipe\ and +# no directory walk can reach them), so there is nothing to reproduce there. +posix_only = pytest.mark.skipif( + not hasattr(os, "mkfifo") or not hasattr(signal, "SIGALRM"), + reason="requires POSIX FIFOs and SIGALRM", +) + +TIMEOUT_SECONDS = 10.0 + + +class Blocked(BaseException): + """Raised when a call under test blocks past the deadline. + + Deliberately derived from ``BaseException`` rather than ``Exception``. + Two separate handlers would otherwise eat the deadline and leave a + reverted fix looking green while it blocked for the full timeout: + + * ``except OSError`` guards every read site here, and ``TimeoutError`` + *is* an ``OSError`` — that one alone cost four vacuous passes. + * ``except Exception`` guards the paths the end-to-end tests cross, + including ``sweeper.sweep_directory`` and four sites inside + ``miner._mine_impl``, so a plain ``Exception`` subclass would be + swallowed there just as thoroughly. + """ + + +@contextmanager +def hard_timeout(seconds: float, what: str): + """Fail the test instead of blocking forever. + + ``signal.setitimer`` fires SIGALRM even while the interpreter sits in a + blocking ``open(2)``; the handler raises, and PEP 475 propagates that + exception rather than restarting the syscall. Without this, reverting + the fix would hang pytest rather than fail it. + """ + + def _fire(signum, frame): + raise Blocked(f"{what} blocked for more than {seconds}s") + + previous = signal.signal(signal.SIGALRM, _fire) + signal.setitimer(signal.ITIMER_REAL, seconds) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous) + + +def make_fifo(directory: Path, name: str) -> Path: + path = directory / name + os.mkfifo(path) + return path + + +def write_regular(directory: Path, name: str, content: str) -> Path: + path = directory / name + path.write_text(content, encoding="utf-8") + return path + + +# ───────────────────────────────────────────────────────────────────────── +# The four os.open read sites: the type check must be reachable +# ───────────────────────────────────────────────────────────────────────── + + +@posix_only +def test_read_text_no_follow_rejects_fifo(tmp_path): + fifo = make_fifo(tmp_path, "notes.md") + with hard_timeout(TIMEOUT_SECONDS, "_read_text_no_follow on a FIFO"): + assert _read_text_no_follow(fifo, tmp_path) is None + + +@posix_only +def test_is_regular_source_file_rejects_fifo(tmp_path): + fifo = make_fifo(tmp_path, "session.jsonl") + with hard_timeout(TIMEOUT_SECONDS, "_is_regular_source_file on a FIFO"): + assert _is_regular_source_file(fifo, tmp_path) is False + + +@posix_only +def test_read_transcript_file_rejects_fifo(tmp_path): + fifo = make_fifo(tmp_path, "session.jsonl") + with hard_timeout(TIMEOUT_SECONDS, "_read_transcript_file on a FIFO"): + with pytest.raises(IOError) as excinfo: + _read_transcript_file(str(fifo)) + message = str(excinfo.value) + assert "not a regular file" in message + # The path belongs in the message exactly once — the raise inside the + # try block is prefix-free so the wrapper composes it. + assert message.count(str(fifo)) == 1 + + +@posix_only +def test_open_regular_file_no_follow_rejects_fifo(tmp_path): + fifo = make_fifo(tmp_path, "chroma.sqlite3") + with hard_timeout(TIMEOUT_SECONDS, "_open_regular_file_no_follow on a FIFO"): + with pytest.raises(RuntimeError, match="Refusing non-regular file"): + _open_regular_file_no_follow(str(fifo)) + + +@posix_only +def test_copy_file_no_follow_rejects_fifo_source(tmp_path): + fifo = make_fifo(tmp_path, "chroma.sqlite3") + with hard_timeout(TIMEOUT_SECONDS, "_copy_file_no_follow from a FIFO"): + with pytest.raises(RuntimeError, match="Refusing non-regular file"): + _copy_file_no_follow(str(fifo), str(tmp_path / "backup.sqlite3")) + assert not (tmp_path / "backup.sqlite3").exists() + + +@posix_only +def test_read_text_no_follow_rejects_fifo_that_has_a_live_writer(tmp_path): + """The verdict comes from the file type, not from "nobody is writing". + + With a writer attached the open succeeds even without ``O_NONBLOCK``, so + this is not the hang regression — it pins the gate itself. Anyone who + "fixes" the hang by swallowing an errno instead of checking ``S_ISREG`` + turns this red, and a pipe whose content would otherwise be mined as a + verbatim drawer stays out of the palace. + """ + fifo = make_fifo(tmp_path, "notes.md") + writer_attached = threading.Event() + release_writer = threading.Event() + failures = [] + + def _hold_write_end(): + try: + fd = os.open(fifo, os.O_WRONLY) # blocks until a reader opens + except OSError as exc: # pragma: no cover - only on a broken setup + failures.append(exc) + writer_attached.set() + return + writer_attached.set() + release_writer.wait(TIMEOUT_SECONDS) + os.close(fd) + + thread = threading.Thread(target=_hold_write_end, daemon=True) + thread.start() + # Opening our own read end is what lets the writer's open(2) return. + reader_fd = os.open(fifo, os.O_RDONLY | os.O_NONBLOCK) + try: + assert writer_attached.wait(TIMEOUT_SECONDS), "writer never attached" + assert not failures, failures + with hard_timeout(TIMEOUT_SECONDS, "_read_text_no_follow on a written FIFO"): + assert _read_text_no_follow(fifo, tmp_path) is None + finally: + release_writer.set() + os.close(reader_fd) + thread.join(TIMEOUT_SECONDS) + + +# ───────────────────────────────────────────────────────────────────────── +# O_NONBLOCK must not change what a regular file reads back +# ───────────────────────────────────────────────────────────────────────── + + +@posix_only +def test_read_text_no_follow_still_reads_a_large_regular_file_whole(tmp_path): + """POSIX and Linux open(2) both say O_NONBLOCK has no effect on regular + files. This pins that: 2 MB is many buffered reads, and a short read or + an ``EAGAIN`` would truncate a drawer silently. + """ + payload = "the quick brown fox jumps over the lazy dog\n" * 50_000 + regular = write_regular(tmp_path, "big.md", payload) + with hard_timeout(TIMEOUT_SECONDS, "_read_text_no_follow on a 2 MB file"): + result = _read_text_no_follow(regular, tmp_path) + assert result is not None + content, mtime = result + assert content == payload + assert mtime == os.path.getmtime(regular) + + +@posix_only +def test_copy_file_no_follow_still_copies_a_large_regular_file_byte_for_byte(tmp_path): + """The palace backup path reads through the same non-blocking fd.""" + payload = os.urandom(2 * 1024 * 1024) + src = tmp_path / "chroma.sqlite3" + src.write_bytes(payload) + dst = tmp_path / "chroma.sqlite3.backup" + with hard_timeout(TIMEOUT_SECONDS, "_copy_file_no_follow of a 2 MB file"): + _copy_file_no_follow(str(src), str(dst)) + assert hashlib.sha256(dst.read_bytes()).hexdigest() == hashlib.sha256(payload).hexdigest() + + +# ───────────────────────────────────────────────────────────────────────── +# Discovery walks must not hand a non-regular file to a reader +# ───────────────────────────────────────────────────────────────────────── + + +@posix_only +def test_scan_project_skips_fifo_and_keeps_regular_files(tmp_path, capsys): + make_fifo(tmp_path, "notes.md") + write_regular(tmp_path, "real.md", "# Real\n") + with hard_timeout(TIMEOUT_SECONDS, "scan_project over a FIFO"): + found = scan_project(str(tmp_path)) + assert [path.name for path in found] == ["real.md"] + assert "SKIP: notes.md (not a regular file)" in capsys.readouterr().err + + +@posix_only +def test_scan_project_skips_unix_socket(tmp_path, capsys): + """Sockets fail the open with ENXIO rather than blocking, but they are + not readable either — the walk drops them at the same gate. + """ + # Bind through a short-lived cwd rather than the absolute path: AF_UNIX + # caps sun_path at 104 bytes on macOS (108 on Linux), and pytest's + # tmp_path under the macOS runner's /var/folders/... TMPDIR overruns it. + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + previous_cwd = os.getcwd() + try: + os.chdir(tmp_path) + sock.bind("mcp.md") + write_regular(tmp_path, "real.md", "# Real\n") + with hard_timeout(TIMEOUT_SECONDS, "scan_project over a socket"): + found = scan_project(str(tmp_path)) + finally: + os.chdir(previous_cwd) + sock.close() + assert (tmp_path / "mcp.md").is_socket() + assert [path.name for path in found] == ["real.md"] + assert "SKIP: mcp.md (not a regular file)" in capsys.readouterr().err + + +@posix_only +def test_scan_convos_skips_fifo_and_keeps_regular_files(tmp_path, capsys): + make_fifo(tmp_path, "piped.jsonl") + write_regular(tmp_path, "real.jsonl", '{"type": "user"}\n') + with hard_timeout(TIMEOUT_SECONDS, "scan_convos over a FIFO"): + found = scan_convos(str(tmp_path)) + assert [path.name for path in found] == ["real.jsonl"] + assert "SKIP: piped.jsonl (not a regular file)" in capsys.readouterr().err + + +@posix_only +def test_parse_claude_jsonl_refuses_fifo(tmp_path): + fifo = make_fifo(tmp_path, "session.jsonl") + with hard_timeout(TIMEOUT_SECONDS, "parse_claude_jsonl on a FIFO"): + with pytest.raises(OSError, match="Refusing non-regular file"): + list(parse_claude_jsonl(str(fifo))) + + +def test_parse_claude_jsonl_still_raises_for_a_missing_path(tmp_path): + """The type gate stats first, so a missing path must keep failing the + way the plain ``open`` used to. + """ + with pytest.raises(FileNotFoundError): + list(parse_claude_jsonl(str(tmp_path / "nope.jsonl"))) + + +def test_parse_claude_jsonl_still_reads_a_regular_transcript(tmp_path): + path = write_regular( + tmp_path, + "session.jsonl", + '{"type": "user", "sessionId": "s1", "uuid": "u1", ' + '"timestamp": "2026-01-01T00:00:00Z", ' + '"message": {"role": "user", "content": "hello"}}\n', + ) + records = list(parse_claude_jsonl(str(path))) + assert [record["content"] for record in records] == ["hello"] + + +@posix_only +def test_detect_entities_ignores_a_fifo_candidate(tmp_path): + """A FIFO must be invisible: same result as if it were never there, and + it must not consume the ``max_files`` budget. + """ + regular = write_regular( + tmp_path, + "people.md", + "Met Sarah Connor and John Connor at the office today.\n" * 5, + ) + fifo = make_fifo(tmp_path, "notes.md") + baseline = detect_entities([regular]) + with hard_timeout(TIMEOUT_SECONDS, "detect_entities over a FIFO"): + with_fifo = detect_entities([fifo, regular]) + assert with_fifo == baseline + + +@posix_only +def test_gather_origin_samples_ignores_a_fifo_candidate(tmp_path): + """``init``'s corpus-origin pass reads the same candidate list as + ``detect_entities`` and must drop a FIFO at the same gate. + """ + write_regular(tmp_path, "real.md", "# Real\n\nSome prose about the project.\n") + make_fifo(tmp_path, "notes.md") + with hard_timeout(TIMEOUT_SECONDS, "_gather_origin_samples over a FIFO"): + samples = _gather_origin_samples(str(tmp_path)) + assert len(samples) == 1 + assert "Some prose about the project." in samples[0] + + +@posix_only +def test_split_mega_files_skips_fifo(tmp_path, capsys, monkeypatch): + make_fifo(tmp_path, "piped.txt") + # Two detectable sessions, so the regular file is reported rather than + # dropped for having nothing to split. Shape copied from + # tests/test_split_mega_files.py::test_find_session_boundaries_two_sessions. + session = "Claude Code v1.0\ncontent\n" + "\n" * 5 + write_regular(tmp_path, "real.txt", session * 2) + monkeypatch.setattr("sys.argv", ["mempalace split", "--source", str(tmp_path), "--dry-run"]) + with hard_timeout(TIMEOUT_SECONDS, "split_mega_files.main over a FIFO"): + split_main() + out = capsys.readouterr().out + assert "SKIP: piped.txt (not a regular file)" in out + # The gate must drop the pipe and nothing else: a version that skipped + # every entry would satisfy the SKIP assertion above on its own. + assert "SKIP: real.txt" not in out + assert "real.txt" in out + + +@posix_only +def test_format_miner_extract_text_does_not_block_on_fifo(tmp_path): + """``mine --mode extract`` was already immune — its zero-size gate fires + first, because a FIFO stats as 0 bytes. Pinned so a future reshuffle of + those checks cannot reintroduce the hang here. + """ + fifo = make_fifo(tmp_path, "doc.pdf") + with hard_timeout(TIMEOUT_SECONDS, "extract_text on a FIFO"): + text, status = extract_text(fifo) + assert text is None + assert status is ExtractionStatus.SKIP_EMPTY + + +@posix_only +def test_collect_manifest_names_ignores_a_fifo_manifest(tmp_path): + real_repo = tmp_path / "real" + real_repo.mkdir() + (real_repo / "package.json").write_text('{"name": "real-project"}', encoding="utf-8") + piped = tmp_path / "piped" + piped.mkdir() + os.mkfifo(piped / "package.json") + + with hard_timeout(TIMEOUT_SECONDS, "_collect_manifest_names over a FIFO manifest"): + found = _collect_manifest_names(tmp_path) + assert [entry[1] for entry in found] == ["real-project"] + + +# ───────────────────────────────────────────────────────────────────────── +# Fixed-name reads: `exists()` is not a type check +# +# The sites above are fed by a directory walk. These are read by a name the +# code already knows, behind an `exists()` guard — which is true for a FIFO, +# so the open right after it blocks anyway. +# ───────────────────────────────────────────────────────────────────────── + + +@posix_only +def test_load_config_treats_a_fifo_yaml_as_absent(tmp_path): + """`mempalace.yaml` is the first thing `mine` reads.""" + make_fifo(tmp_path, "mempalace.yaml") + with hard_timeout(TIMEOUT_SECONDS, "load_config on a FIFO mempalace.yaml"): + config = load_config(str(tmp_path)) + assert [room["name"] for room in config["rooms"]] == ["general"] + + +@posix_only +def test_load_config_still_reads_a_regular_yaml(tmp_path): + (tmp_path / "mempalace.yaml").write_text( + yaml.dump({"wing": "realwing", "rooms": [{"name": "docs", "description": "d"}]}), + encoding="utf-8", + ) + config = load_config(str(tmp_path)) + assert config["wing"] == "realwing" + assert [room["name"] for room in config["rooms"]] == ["docs"] + + +@posix_only +def test_ensure_gitignore_leaves_a_fifo_gitignore_alone(tmp_path): + (tmp_path / ".git").mkdir() + make_fifo(tmp_path, ".gitignore") + with hard_timeout(TIMEOUT_SECONDS, "_ensure_mempalace_files_gitignored on a FIFO"): + assert _ensure_mempalace_files_gitignored(str(tmp_path)) is False + + +def test_ensure_gitignore_still_appends_to_a_regular_gitignore(tmp_path): + (tmp_path / ".git").mkdir() + (tmp_path / ".gitignore").write_text("*.pyc\n", encoding="utf-8") + assert _ensure_mempalace_files_gitignored(str(tmp_path)) is True + written = (tmp_path / ".gitignore").read_text(encoding="utf-8") + assert "mempalace.yaml" in written and "entities.json" in written + + +@posix_only +def test_cmd_compress_ignores_a_fifo_entities_json(tmp_path, monkeypatch, capsys): + """`compress` picks up `./entities.json` when no --config is given.""" + monkeypatch.chdir(tmp_path) + make_fifo(tmp_path, "entities.json") + args = argparse.Namespace(palace=None, wing=None, dry_run=False, config=None) + with patch("mempalace.cli.MempalaceConfig") as mock_config_cls: + mock_config_cls.return_value.palace_path = str(tmp_path / "nonexistent") + with hard_timeout(TIMEOUT_SECONDS, "cmd_compress with a FIFO entities.json"): + with pytest.raises(SystemExit): + cmd_compress(args) + out = capsys.readouterr().out + assert "No palace found" in out + assert "Loaded entity config" not in out + + +@posix_only +def test_cmd_compress_fifo_entities_json_does_not_shadow_the_palace_copy( + tmp_path, monkeypatch, capsys +): + """The candidate loop must not stop at a pipe. + + An `entities.json` FIFO in the cwd would otherwise be picked as the + config and then rejected by the load guard, hiding a perfectly good + `/entities.json` behind it. + """ + monkeypatch.chdir(tmp_path) + make_fifo(tmp_path, "entities.json") + palace = tmp_path / "palace" + palace.mkdir() + (palace / "entities.json").write_text('{"entities": {"Alice": "ALC"}}', encoding="utf-8") + args = argparse.Namespace(palace=None, wing=None, dry_run=False, config=None) + with patch("mempalace.cli.MempalaceConfig") as mock_config_cls: + mock_config_cls.return_value.palace_path = str(palace) + with hard_timeout(TIMEOUT_SECONDS, "cmd_compress candidate loop"): + with pytest.raises(SystemExit): + cmd_compress(args) + assert f"Loaded entity config: {palace / 'entities.json'}" in capsys.readouterr().out + + +@posix_only +def test_parse_gradle_ignores_a_fifo_sibling_settings_file(tmp_path): + """`build.gradle` is a regular file and clears the manifest gate; the + parser then reads the SIBLING `settings.gradle`, which no walk vetted. + """ + repo = tmp_path / "repo" + repo.mkdir() + (repo / "build.gradle").write_text("plugins { id 'java' }\n", encoding="utf-8") + make_fifo(repo, "settings.gradle") + with hard_timeout(TIMEOUT_SECONDS, "_collect_manifest_names with a FIFO settings.gradle"): + found = _collect_manifest_names(tmp_path) + assert [entry[1] for entry in found] == ["repo"] + + +@posix_only +def test_count_human_messages_ignores_a_fifo_transcript(tmp_path): + fifo = make_fifo(tmp_path, "transcript.jsonl") + with hard_timeout(TIMEOUT_SECONDS, "count_human_messages on a FIFO"): + assert count_human_messages(str(fifo)) == 0 + + +def test_count_human_messages_still_raises_for_a_missing_path(tmp_path): + """The type gate is narrowed to paths that exist, so a missing transcript + keeps failing the way the plain ``open`` did. + """ + with pytest.raises(FileNotFoundError): + count_human_messages(str(tmp_path / "nope.jsonl")) + + +def test_count_human_messages_still_counts_a_regular_transcript(tmp_path): + path = write_regular( + tmp_path, + "transcript.jsonl", + '{"message": {"role": "user", "content": "one"}}\n' + '{"message": {"role": "assistant", "content": "two"}}\n' + '{"message": {"role": "user", "content": "three"}}\n', + ) + assert count_human_messages(str(path)) == 2 + + +@posix_only +def test_cmd_init_refuses_to_write_entities_over_a_fifo(tmp_path, capsys): + """`init` writes `/entities.json`; opening a pre-existing FIFO + for writing blocks until a reader appears. + """ + make_fifo(tmp_path, "entities.json") + args = argparse.Namespace(dir=str(tmp_path), yes=True, no_llm=True) + detected = {"people": [{"name": "Alice"}], "projects": [], "topics": [], "uncertain": []} + confirmed = {"people": ["Alice"], "projects": [], "topics": []} + with ( + patch("mempalace.cli.MempalaceConfig"), + patch("mempalace.project_scanner.discover_entities", return_value=detected), + patch("mempalace.entity_detector.confirm_entities", return_value=confirmed), + patch("mempalace.room_detector_local.detect_rooms_local"), + patch("mempalace.cli._run_pass_zero", return_value=None), + patch("mempalace.cli._maybe_run_mine_after_init"), + ): + with hard_timeout(TIMEOUT_SECONDS, "cmd_init with a FIFO entities.json"): + cmd_init(args) + captured = capsys.readouterr() + assert "is not a regular file" in captured.err + assert "Entities saved" not in captured.out + + +@posix_only +def test_detect_rooms_local_refuses_a_fifo_config(tmp_path): + """`init` writes `mempalace.yaml`; a write open on a pipe blocks until a + reader appears, which parked `init` with no output at all. + """ + write_regular(tmp_path, "README.md", "note\n") + make_fifo(tmp_path, "mempalace.yaml") + with hard_timeout(TIMEOUT_SECONDS, "detect_rooms_local over a FIFO config"): + with pytest.raises(OSError, match="not a regular file"): + detect_rooms_local(project_dir=str(tmp_path), yes=True) + + +@posix_only +def test_collect_corpus_text_ignores_a_fifo(tmp_path): + """LLM refinement walks prose by suffix and stats only for mtime.""" + write_regular(tmp_path, "a.md", "real prose\n") + make_fifo(tmp_path, "notes.md") + with hard_timeout(TIMEOUT_SECONDS, "collect_corpus_text over a FIFO"): + text = collect_corpus_text(str(tmp_path)) + assert text == "real prose\n" + + +@posix_only +def test_sweep_directory_skips_a_fifo_without_booking_a_failure(tmp_path, capsys): + """A pipe is nothing to sweep, not a sweep failure. + + Booking it as a failure flips the command's exit status to 2 through + ``cli.cmd_sweep``, which breaks any script gating on it — and the same + input on a directory walk is a benign ``SKIP`` in ``mine``. + """ + convos = tmp_path / "convos" + convos.mkdir() + write_regular( + convos, + "real.jsonl", + '{"type": "user", "sessionId": "s1", "uuid": "u1", ' + '"timestamp": "2026-01-01T00:00:00Z", ' + '"message": {"role": "user", "content": "hello"}}\n', + ) + make_fifo(convos, "piped.jsonl") + with hard_timeout(TIMEOUT_SECONDS, "sweep_directory over a FIFO"): + result = sweep_directory(str(convos), str(tmp_path / "palace")) + # ``failures`` is what ``cli.cmd_sweep`` turns into ``sys.exit(2)``. + assert result["failures"] == [] + # ``files_attempted`` counts discovery, per its docstring, so the pipe + # stays in it; ``files_succeeded`` counts what was actually swept. + assert result["files_attempted"] == 2 + assert result["files_succeeded"] == 1 + assert result["drawers_added"] == 1 + assert "SKIP: piped.jsonl (not a regular file)" in capsys.readouterr().err + + +# ───────────────────────────────────────────────────────────────────────── +# O_NONBLOCK must not drop a regular file the blocking open would have read +# ───────────────────────────────────────────────────────────────────────── + + +@posix_only +def test_read_text_no_follow_retries_when_a_lease_break_returns_eagain(tmp_path, monkeypatch): + """A write lease is the one case where the flag changes `open` itself. + + Breaking a lease with ``O_NONBLOCK`` fails ``EAGAIN`` immediately, where + a blocking open waits out ``lease-break-time`` and succeeds. Left alone + that turns into a silently dropped file. The kernel grants leases on + regular files only, so the retry is authorised by the file *type*; the + errno only decides whether to look again. + + ``EAGAIN`` is injected rather than staged with a real lease so the test + costs milliseconds instead of the 45 s default lease-break-time. + """ + payload = "PAYLOAD THAT MUST STILL BE MINED\n" * 20 + target = write_regular(tmp_path, "notes.md", payload) + real_open = os.open + calls = {"n": 0} + + def _fake_open(path, flags, *args, **kwargs): + calls["n"] += 1 + if calls["n"] == 1: + assert flags & os.O_NONBLOCK, "first attempt should carry the flag" + raise OSError(errno.EAGAIN, os.strerror(errno.EAGAIN), str(path)) + assert not flags & os.O_NONBLOCK, "retry should drop the flag" + return real_open(path, flags, *args, **kwargs) + + monkeypatch.setattr("mempalace.miner.os.open", _fake_open) + with hard_timeout(TIMEOUT_SECONDS, "_read_text_no_follow under an injected EAGAIN"): + result = _read_text_no_follow(target, tmp_path) + assert result is not None + content, mtime = result + assert content == payload + assert mtime == os.path.getmtime(target) + assert calls["n"] == 2 + + +@posix_only +def test_read_text_no_follow_does_not_retry_eagain_on_a_fifo(tmp_path, monkeypatch): + """The retry is gated on the type, so a pipe never gets a blocking open. + + The stand-in raises if it is ever called without the flag, so a retry + that trusted the errno alone would fail this test rather than hang it. + """ + fifo = make_fifo(tmp_path, "notes.md") + + def _fake_open(path, flags, *args, **kwargs): + if flags & os.O_NONBLOCK: + raise OSError(errno.EAGAIN, os.strerror(errno.EAGAIN), str(path)) + raise AssertionError("must not retry a blocking open on a FIFO") + + monkeypatch.setattr("mempalace.miner.os.open", _fake_open) + with hard_timeout(TIMEOUT_SECONDS, "_read_text_no_follow EAGAIN on a FIFO"): + assert _read_text_no_follow(fifo, tmp_path) is None + + +@posix_only +def test_gather_origin_samples_survives_an_unreadable_directory(tmp_path): + """The type gate must not turn a skipped file into a crash. + + ``Path.is_file()`` raises ``PermissionError`` on a directory without + ``x``; the ``open`` it replaced was already inside the ``try`` that + absorbs exactly that, so the gate has to sit there too. + """ + write_regular(tmp_path, "real.md", "# Real\n\nsome prose here\n") + walled = tmp_path / "walled" + walled.mkdir() + write_regular(walled, "notes.md", "y" * 200) + os.chmod(walled, 0o444) + try: + with hard_timeout(TIMEOUT_SECONDS, "_gather_origin_samples over an unreadable dir"): + samples = _gather_origin_samples(str(tmp_path)) + finally: + os.chmod(walled, 0o755) + assert len(samples) == 1 + assert "some prose here" in samples[0] + + +def test_read_transcript_file_size_message_names_the_path_once(tmp_path): + """Both refusal branches compose the path through the same wrapper.""" + big = write_regular(tmp_path, "huge.jsonl", "not actually huge") + + class _HugeStat: + st_mode = stat_module.S_IFREG | 0o644 + st_size = 600 * 1024 * 1024 + + with patch("mempalace.normalize.os.fstat", return_value=_HugeStat()): + with pytest.raises(IOError) as excinfo: + _read_transcript_file(str(big)) + message = str(excinfo.value) + assert "too large" in message.lower() + assert message.count(str(big)) == 1 + + +# ───────────────────────────────────────────────────────────────────────── +# End to end through the real miner +# ───────────────────────────────────────────────────────────────────────── + + +@posix_only +def test_mine_completes_with_a_fifo_in_the_corpus(tmp_path): + """The original report: ``mempalace mine `` never returned.""" + project_root = tmp_path / "corpus" + project_root.mkdir() + (project_root / "mempalace.yaml").write_text( + yaml.dump( + { + "wing": "fifo_repro", + "rooms": [{"name": "general", "description": "General"}], + } + ), + encoding="utf-8", + ) + write_regular( + project_root, + "real.md", + "# Real note\n\n" + "The quick brown fox jumps over the lazy dog. " * 40, + ) + make_fifo(project_root, "notes.md") + + palace_path = tmp_path / "palace" + with hard_timeout(TIMEOUT_SECONDS, "mine() over a corpus holding a FIFO"): + mine(str(project_root), str(palace_path)) + + import chromadb + + collection = chromadb.PersistentClient(path=str(palace_path)).get_collection( + "mempalace_drawers" + ) + stored = collection.get(include=["metadatas"]) + sources = {Path(meta["source_file"]).name for meta in stored["metadatas"]} + assert sources == {"real.md"} diff --git a/tests/test_normalized_conversations.py b/tests/test_normalized_conversations.py index 855e99c28..237c0c3de 100644 --- a/tests/test_normalized_conversations.py +++ b/tests/test_normalized_conversations.py @@ -3,6 +3,7 @@ import hashlib import importlib import json +import os import pytest @@ -27,7 +28,7 @@ def _write_pair( "> teh exact user text must stay teh same\n" "The assistant answer also stays byte-for-byte intact.\n" ) - transcript_path.write_text(transcript_text) + transcript_path.write_bytes(transcript_text.encode("utf-8")) sidecar = { "schema": "mempalace-normalized-conversation/v1", "room": room, @@ -41,7 +42,7 @@ def _write_pair( "hermes_source": "telegram", } sidecar_path = tmp_path / "session.md.meta.json" - sidecar_path.write_text(json.dumps(sidecar)) + sidecar_path.write_bytes(json.dumps(sidecar).encode("utf-8")) return transcript_path, sidecar_path, transcript_text, sidecar @@ -854,7 +855,8 @@ def test_coverage_registry_advances_atomically_and_is_private(tmp_path): assert committed["wing"] == "wing_amber" assert normalized.read_applied_coverage(palace)["wing_amber"] == committed - assert registry_path.stat().st_mode & 0o777 == 0o600 + if os.name != "nt": + assert registry_path.stat().st_mode & 0o777 == 0o600 assert "telegram" in registry_path.read_text() before = registry_path.stat().st_mtime_ns assert normalized.commit_applied_coverage(palace, _coverage_receipt()) == committed diff --git a/tests/test_repair.py b/tests/test_repair.py index 2179e8443..9338553d2 100644 --- a/tests/test_repair.py +++ b/tests/test_repair.py @@ -1976,7 +1976,9 @@ def _seed_palace(palace_path, collection_name, rows): ``rows`` is a list of ``(id, document, metadata)`` tuples. """ - from mempalace.backends.chroma import ChromaBackend + import gc + + from mempalace.backends.chroma import ChromaBackend, _clear_chroma_system_cache backend = ChromaBackend() try: @@ -1991,7 +1993,15 @@ def _seed_palace(palace_path, collection_name, rows): # caller proceeds. Without this, an in-place rebuild on Windows # fails with WinError 32 on data_level0.bin during the archive # rename (cf. PR #1310 test-windows job). + # + # Also drop the process-global SharedSystemClient cache: closing the + # backend releases our PersistentClient handle, but chromadb can keep + # the path-keyed System alive and on Windows that blocks renaming the + # palace directory (WinError 5 Access is denied). Seen on PR #2228 + # ``test_rebuild_from_sqlite_raises_on_upsert_failure``. backend.close() + _clear_chroma_system_cache() + gc.collect() def test_extract_via_sqlite_returns_all_rows_with_metadata(tmp_path): diff --git a/uv.lock b/uv.lock index 0d2aa76d0..440e832f2 100644 --- a/uv.lock +++ b/uv.lock @@ -722,9 +722,9 @@ name = "faiss-cpu" version = "1.14.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform != 'win32'" }, - { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and sys_platform != 'win32'" }, - { name = "packaging", marker = "sys_platform != 'win32'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "packaging" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/83/b0/48c083d01b7b68c463c1d56507147a9d733f791e1c469a77215a872a9fb5/faiss_cpu-1.14.3-cp310-abi3-macosx_14_0_arm64.whl", hash = "sha256:a9369863290a3f0e033757e4c10577b6ef7431f1cede394dabd0a137e4e2ed45", size = 4768290, upload-time = "2026-06-13T02:19:03.427Z" }, @@ -1492,7 +1492,7 @@ wheels = [ [[package]] name = "mempalace" -version = "3.7.0+oc.2" +version = "3.7.1+oc.1" source = { editable = "." } dependencies = [ { name = "chromadb" }, @@ -1603,12 +1603,12 @@ name = "milvus-lite" version = "3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "faiss-cpu", marker = "sys_platform != 'win32'" }, - { name = "grpcio", marker = "sys_platform != 'win32'" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform != 'win32'" }, - { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and sys_platform != 'win32'" }, - { name = "pyarrow", marker = "sys_platform != 'win32'" }, - { name = "tomli", marker = "python_full_version < '3.11' and sys_platform != 'win32'" }, + { name = "faiss-cpu" }, + { name = "grpcio" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "pyarrow" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a3/9a/d80d260e6fe1246818a8ef782c374ba9c6ca46ca3b987c14eabe914ef805/milvus_lite-3.0.tar.gz", hash = "sha256:2c35d0d046b1faae3402cde1fb73d65f51ee8c6aba65f54de1dda46f7bb18b9b", size = 589749, upload-time = "2026-05-13T07:14:05.827Z" } wheels = [