Repository navigation
fix(state): serialize state.db schema surgery across processes #69609
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
ernst-bablick
wants to merge
1
commit into
NousResearch:main
Choose a base branch
from
ernst-bablick:fix/state-cross-process-repair-lock
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -12,7 +12,11 @@ | |
| cannot be handled at the FTS-rebuild layer. These tests verify the | ||
| sqlite_master surgery path recovers the canonical data and self-heals on open. | ||
| """ | ||
| import contextlib | ||
| import json | ||
| import sqlite3 | ||
| import subprocess | ||
| import sys | ||
| import uuid | ||
| from pathlib import Path | ||
|
|
||
|
|
@@ -616,3 +620,163 @@ def test_select_cached_agent_history_prefers_longer_live_transcript(): | |
| # No live transcript / not a list → no-op. | ||
| assert _select_cached_agent_history(persisted, None) is persisted | ||
| assert _select_cached_agent_history(persisted, "nope") is persisted | ||
|
|
||
|
|
||
| # --------------------------------------------------------------------------- | ||
| # Cross-process serialisation of the schema surgery | ||
| # --------------------------------------------------------------------------- | ||
| # A normal host runs several independent processes against one state.db: the | ||
| # gateway service, the Desktop app's own `hermes serve` backend, interactive | ||
| # CLI sessions and the TUI slash worker. `_repair_attempt_lock` is a | ||
| # threading.Lock and covers none of that, so two of them hitting a malformed | ||
| # DB at once each ran the full writable_schema surgery + VACUUM on a private | ||
| # connection — one repairing while the other was mid-surgery. | ||
|
|
||
|
|
||
| _HOLD_LOCK_SCRIPT = """ | ||
| import sys, time, fcntl, pathlib | ||
| sys.path.insert(0, {root!r}) | ||
| lock_path = pathlib.Path({lock!r}) | ||
| handle = lock_path.open("a+b") | ||
| fcntl.flock(handle.fileno(), fcntl.LOCK_EX) | ||
| print("locked", flush=True) | ||
| time.sleep({hold}) | ||
| """ | ||
|
|
||
|
|
||
| @contextlib.contextmanager | ||
| def _lock_held_by_other_process(db_path: Path, hold_seconds: float = 30.0): | ||
| """Hold the repair flock for *db_path* in a real child process.""" | ||
| script = _HOLD_LOCK_SCRIPT.format( | ||
| root=str(Path(hermes_state.__file__).parent), | ||
| lock=str(db_path.with_name(db_path.name + ".repair.lock")), | ||
| hold=hold_seconds, | ||
| ) | ||
| proc = subprocess.Popen( | ||
| [sys.executable, "-c", script], | ||
| stdout=subprocess.PIPE, text=True, | ||
| ) | ||
| try: | ||
| # Wait for the child to actually own the lock before yielding. | ||
| assert proc.stdout.readline().strip() == "locked" | ||
| yield | ||
| finally: | ||
| proc.kill() | ||
| proc.wait(timeout=10) | ||
|
|
||
|
|
||
| @pytest.mark.skipif(sys.platform == "win32", reason="POSIX flock test") | ||
| def test_repair_skips_surgery_while_another_process_holds_the_lock( | ||
| tmp_path, monkeypatch | ||
| ): | ||
| """The losing process must NOT run writable_schema surgery in parallel.""" | ||
| db_path = tmp_path / "state.db" | ||
| _build_healthy_db(db_path) | ||
| _corrupt_duplicate_fts(db_path) | ||
| monkeypatch.setattr(hermes_state, "_REPAIR_LOCK_TIMEOUT_SECONDS", 0.5) | ||
|
|
||
| with _lock_held_by_other_process(db_path): | ||
| report = repair_state_db_schema(db_path) | ||
|
|
||
| assert report["repaired"] is False | ||
| assert "repair lock" in (report["error"] or "") | ||
| # No surgery ran: no backup was taken and the DB is still malformed. | ||
| assert report["backup_path"] is None | ||
| assert not list(tmp_path.glob("state.db.malformed-backup-*")) | ||
| assert hermes_state._db_opens_cleanly(db_path) is not None | ||
|
|
||
|
|
||
| @pytest.mark.skipif(sys.platform == "win32", reason="POSIX flock test") | ||
| def test_repair_reports_success_when_the_holder_already_healed_the_db( | ||
| tmp_path, monkeypatch | ||
| ): | ||
| """Timing out against a healthy DB is a success, not an error.""" | ||
| db_path = tmp_path / "state.db" | ||
| _build_healthy_db(db_path) | ||
| monkeypatch.setattr(hermes_state, "_REPAIR_LOCK_TIMEOUT_SECONDS", 0.5) | ||
|
|
||
| with _lock_held_by_other_process(db_path): | ||
| report = repair_state_db_schema(db_path) | ||
|
|
||
| assert report["repaired"] is True | ||
| assert report["strategy"] == "repaired_by_other_process" | ||
|
|
||
|
|
||
| _REPAIR_SCRIPT = """ | ||
| import sys, json | ||
| sys.path.insert(0, {root!r}) | ||
| from hermes_state import repair_state_db_schema | ||
| print(json.dumps(repair_state_db_schema({db!r})), flush=True) | ||
| """ | ||
|
|
||
|
|
||
| @pytest.mark.skipif(sys.platform == "win32", reason="POSIX flock test") | ||
| def test_two_processes_repairing_at_once_perform_surgery_once(tmp_path): | ||
| """Concurrent repairers serialise; the loser sees a healed DB and stops. | ||
|
|
||
| Without the cross-process lock both processes back up and operate on | ||
| sqlite_master, i.e. one runs surgery on a database the other is | ||
| simultaneously rewriting. The backup count is the observable proxy for | ||
| "how many processes entered the critical section". | ||
| """ | ||
| db_path = tmp_path / "state.db" | ||
| _build_healthy_db(db_path) | ||
| _corrupt_duplicate_fts(db_path) | ||
|
|
||
| script = _REPAIR_SCRIPT.format( | ||
| root=str(Path(hermes_state.__file__).parent), db=str(db_path) | ||
| ) | ||
| procs = [ | ||
| subprocess.Popen( | ||
| [sys.executable, "-c", script], | ||
| stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, | ||
| ) | ||
| for _ in range(2) | ||
| ] | ||
| reports = [] | ||
| for proc in procs: | ||
| out, err = proc.communicate(timeout=120) | ||
| assert proc.returncode == 0, err | ||
| reports.append(json.loads(out.strip().splitlines()[-1])) | ||
|
|
||
| assert all(r["repaired"] for r in reports), reports | ||
| # Exactly one process did the work; the other found the DB already healthy. | ||
| strategies = sorted(r["strategy"] for r in reports) | ||
| assert "already_healthy" in strategies or "repaired_by_other_process" in strategies | ||
| assert len(list(tmp_path.glob("state.db.malformed-backup-*"))) == 1 | ||
|
|
||
| conn = sqlite3.connect(str(db_path)) | ||
| try: | ||
| assert conn.execute("PRAGMA integrity_check").fetchone()[0] == "ok" | ||
| assert conn.execute("SELECT COUNT(*) FROM messages").fetchone()[0] == 10 | ||
| finally: | ||
| conn.close() | ||
|
|
||
|
|
||
| def test_schema_surgery_bumps_the_schema_cookie(tmp_path): | ||
| """Live connections in other processes must be told to reload the schema. | ||
|
|
||
| Editing sqlite_master under writable_schema=ON does not bump the cookie | ||
| that every other connection checks before running a prepared statement, | ||
| so they keep compiling against objects the surgery just deleted. | ||
| """ | ||
| db_path = tmp_path / "state.db" | ||
| _build_healthy_db(db_path) | ||
| _corrupt_duplicate_fts(db_path) | ||
|
|
||
| probe = sqlite3.connect(str(db_path)) | ||
| try: | ||
| probe.execute("PRAGMA writable_schema=ON") | ||
| before = probe.execute("PRAGMA schema_version").fetchone()[0] | ||
| finally: | ||
| probe.close() | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This closes the only probe before repair, so the test only observes a persisted schema-version change through a fresh connection. Keep a connection alive across repair and execute a previously prepared schema-dependent operation to cover the claimed cache invalidation. |
||
|
|
||
| report = repair_state_db_schema(db_path) | ||
| assert report["repaired"] is True | ||
|
|
||
| probe = sqlite3.connect(str(db_path)) | ||
| try: | ||
| after = probe.execute("PRAGMA schema_version").fetchone()[0] | ||
| finally: | ||
| probe.close() | ||
| assert after != before | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This must not yield
True: when the lock file cannot be opened, this permits the same uncoordinated schema surgery the lock is meant to prevent. Please fail closed (yield False/ return a repair error), consistent with the quarantine lock's refusal behavior on current main.