From 1e7e50ccd31328a96bcf26dd800d426fcce0acb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 25 Mar 2026 18:45:34 +0100 Subject: [PATCH 1/3] feat(claude-code): full-session retain mode with document upsert and configurable tags Switch default retain behavior from per-turn chunks to full-session upsert. Each session is now retained as a single document (document_id = session_id) that gets updated on every Stop event, instead of creating fragmented documents with timestamp-suffixed IDs. New config options: - retainMode: "full-session" (default) or "chunked" (legacy) - retainTags: list with template variable support ({session_id}, {bank_id}, {timestamp}) - retainMetadata: extra metadata dict merged with built-in fields, supports templates --- .../claude-code/scripts/lib/client.py | 3 + .../claude-code/scripts/lib/config.py | 4 + .../claude-code/scripts/retain.py | 52 +++++++++-- .../claude-code/settings.json | 3 + .../claude-code/tests/test_hooks.py | 86 ++++++++++++++++++- 5 files changed, 137 insertions(+), 11 deletions(-) diff --git a/hindsight-integrations/claude-code/scripts/lib/client.py b/hindsight-integrations/claude-code/scripts/lib/client.py index f7e5125f33..58b7bc4c2e 100644 --- a/hindsight-integrations/claude-code/scripts/lib/client.py +++ b/hindsight-integrations/claude-code/scripts/lib/client.py @@ -105,6 +105,7 @@ def retain( document_id: str = "conversation", context: Optional[str] = None, metadata: Optional[dict] = None, + tags: Optional[list] = None, timeout: int = 15, ) -> dict: """Retain content into a bank's memory. @@ -121,6 +122,8 @@ def retain( } if context: item["context"] = context + if tags: + item["tags"] = tags body = { "items": [item], "async": True, diff --git a/hindsight-integrations/claude-code/scripts/lib/config.py b/hindsight-integrations/claude-code/scripts/lib/config.py index 96aae69381..0270288570 100644 --- a/hindsight-integrations/claude-code/scripts/lib/config.py +++ b/hindsight-integrations/claude-code/scripts/lib/config.py @@ -25,10 +25,13 @@ "recallTopK": None, # Retain "autoRetain": True, + "retainMode": "full-session", "retainRoles": ["user", "assistant"], "retainEveryNTurns": 10, "retainOverlapTurns": 2, "retainContext": "claude-code", + "retainTags": [], + "retainMetadata": {}, # Connection "hindsightApiUrl": None, "hindsightApiToken": None, @@ -60,6 +63,7 @@ "HINDSIGHT_AGENT_NAME": ("agentName", str), "HINDSIGHT_AUTO_RECALL": ("autoRecall", bool), "HINDSIGHT_AUTO_RETAIN": ("autoRetain", bool), + "HINDSIGHT_RETAIN_MODE": ("retainMode", str), "HINDSIGHT_RECALL_BUDGET": ("recallBudget", str), "HINDSIGHT_RECALL_MAX_TOKENS": ("recallMaxTokens", int), "HINDSIGHT_RECALL_MAX_QUERY_CHARS": ("recallMaxQueryChars", int), diff --git a/hindsight-integrations/claude-code/scripts/retain.py b/hindsight-integrations/claude-code/scripts/retain.py index 05570f82ee..df38576fb1 100755 --- a/hindsight-integrations/claude-code/scripts/retain.py +++ b/hindsight-integrations/claude-code/scripts/retain.py @@ -96,12 +96,13 @@ def main(): debug_log(config, f"Read {len(all_messages)} messages from transcript") - # Chunked retention logic — port of Openclaw's retainEveryNTurns + sliding window + # Retention mode: full session (default) or chunked (legacy) + retain_mode = config.get("retainMode", "full-session") retain_every_n = max(1, config.get("retainEveryNTurns", 1)) retain_full_window = False messages_to_retain = all_messages - if retain_every_n > 1: + if retain_mode == "chunked" and retain_every_n > 1: turn_count = increment_turn_count(session_id) if turn_count % retain_every_n != 0: next_at = ((turn_count // retain_every_n) + 1) * retain_every_n @@ -118,6 +119,10 @@ def main(): f"Turn {turn_count}: chunked retain firing " f"(window: {window_turns} turns, {len(messages_to_retain)} messages)", ) + else: + # Full session mode: retain all messages, always as full window + retain_full_window = True + debug_log(config, f"Full session retain: {len(all_messages)} messages") # Format transcript retain_roles = config.get("retainRoles", ["user", "assistant"]) @@ -148,12 +153,44 @@ def _dbg(*a): bank_id = derive_bank_id(hook_input, config) ensure_bank_mission(client, bank_id, config, debug_fn=_dbg) - # Unique document ID — mirrors Openclaw: {sessionKey}-{timestamp} - document_id = f"{session_id}-{int(time.time() * 1000)}" + # Document ID: use session_id so the same session always upserts the same document. + # In chunked mode, append timestamp to create distinct documents per chunk. + if retain_mode == "chunked" and retain_every_n > 1: + document_id = f"{session_id}-{int(time.time() * 1000)}" + else: + document_id = session_id + + # Resolve template variables in tags and metadata. + # Supported variables: {session_id}, {bank_id}, {timestamp} + template_vars = { + "session_id": session_id, + "bank_id": bank_id, + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + } + + def _resolve_template(value: str) -> str: + for k, v in template_vars.items(): + value = value.replace(f"{{{k}}}", v) + return value + + # Tags from config with template resolution + raw_tags = config.get("retainTags", []) + tags = [_resolve_template(t) for t in raw_tags] if raw_tags else None + + # Metadata: merge built-in defaults with user-configured extras + metadata = { + "retained_at": template_vars["timestamp"], + "message_count": str(message_count), + "session_id": session_id, + } + for k, v in config.get("retainMetadata", {}).items(): + metadata[k] = _resolve_template(str(v)) debug_log( config, f"Retaining to bank '{bank_id}', doc '{document_id}', {message_count} messages, {len(transcript)} chars" ) + if tags: + debug_log(config, f"Tags: {tags}") # POST to Hindsight retain API try: @@ -162,11 +199,8 @@ def _dbg(*a): content=transcript, document_id=document_id, context=config.get("retainContext", "claude-code"), - metadata={ - "retained_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), - "message_count": str(message_count), - "session_id": session_id, - }, + metadata=metadata, + tags=tags, timeout=15, ) debug_log(config, f"Retain response: {json.dumps(response)[:200]}") diff --git a/hindsight-integrations/claude-code/settings.json b/hindsight-integrations/claude-code/settings.json index 4af325c624..6d3d288ad4 100644 --- a/hindsight-integrations/claude-code/settings.json +++ b/hindsight-integrations/claude-code/settings.json @@ -5,6 +5,7 @@ "retainMission": "Extract technical decisions, architectural choices, user preferences, project context, and people/tool relationships. Ignore routine greetings and transient operational details.", "autoRecall": true, "autoRetain": true, + "retainMode": "full-session", "recallBudget": "mid", "recallMaxTokens": 1024, "recallTypes": ["world", "experience"], @@ -16,6 +17,8 @@ "retainRoles": ["user", "assistant"], "retainEveryNTurns": 10, "retainOverlapTurns": 2, + "retainTags": ["{session_id}"], + "retainMetadata": {}, "retainContext": "claude-code", "hindsightApiToken": null, "apiPort": 9077, diff --git a/hindsight-integrations/claude-code/tests/test_hooks.py b/hindsight-integrations/claude-code/tests/test_hooks.py index 74d8f93c77..848d49e601 100644 --- a/hindsight-integrations/claude-code/tests/test_hooks.py +++ b/hindsight-integrations/claude-code/tests/test_hooks.py @@ -25,7 +25,7 @@ # --------------------------------------------------------------------------- -def _run_hook(module_name, hook_input, monkeypatch, tmp_path, urlopen_side_effect=None, extra_env=None): +def _run_hook(module_name, hook_input, monkeypatch, tmp_path, urlopen_side_effect=None, extra_env=None, extra_settings=None): """Import and run a hook script's main() with mocked stdin/stdout/HTTP.""" # Isolated plugin dirs monkeypatch.setenv("CLAUDE_PLUGIN_ROOT", str(tmp_path / "plugin_root")) @@ -43,6 +43,8 @@ def _run_hook(module_name, hook_input, monkeypatch, tmp_path, urlopen_side_effec # Write a minimal settings.json enabling fast retains settings = {"autoRecall": True, "autoRetain": True, "retainEveryNTurns": 1, "hindsightApiUrl": "http://fake:9077"} + if extra_settings: + settings.update(extra_settings) (tmp_path / "plugin_root" / "settings.json").write_text(json.dumps(settings)) stdin_data = io.StringIO(json.dumps(hook_input)) @@ -251,13 +253,90 @@ def capture(req, timeout=None): assert "old memories" not in content assert "actual question" in content + def test_retain_tags_with_template_variables(self, monkeypatch, tmp_path): + """retainTags config should resolve template variables like {session_id}.""" + messages = [{"role": "user", "content": "hello"}, {"role": "assistant", "content": "world"}] + transcript = make_transcript_file(tmp_path, messages) + hook_input = make_hook_input(transcript_path=transcript, session_id="sess-tag-test") + captured = {} + + def capture(req, timeout=None): + if "/memories" in req.full_url and "/recall" not in req.full_url: + captured["body"] = json.loads(req.data.decode()) + return FakeHTTPResponse({}) + + _run_hook( + "retain", hook_input, monkeypatch, tmp_path, + urlopen_side_effect=capture, + extra_settings={"retainTags": ["{session_id}", "claude-code", "custom-tag"]}, + ) + + assert "body" in captured, "retain API was not called" + item = captured["body"]["items"][0] + assert item["tags"] == ["sess-tag-test", "claude-code", "custom-tag"] + + def test_retain_custom_metadata(self, monkeypatch, tmp_path): + """retainMetadata config should be merged with built-in metadata.""" + messages = [{"role": "user", "content": "hello"}, {"role": "assistant", "content": "world"}] + transcript = make_transcript_file(tmp_path, messages) + hook_input = make_hook_input(transcript_path=transcript, session_id="sess-meta-test") + captured = {} + + def capture(req, timeout=None): + if "/memories" in req.full_url and "/recall" not in req.full_url: + captured["body"] = json.loads(req.data.decode()) + return FakeHTTPResponse({}) + + _run_hook( + "retain", hook_input, monkeypatch, tmp_path, + urlopen_side_effect=capture, + extra_settings={"retainMetadata": {"project": "my-project", "session": "{session_id}"}}, + ) + + assert "body" in captured, "retain API was not called" + meta = captured["body"]["items"][0]["metadata"] + # Built-in metadata + assert meta["session_id"] == "sess-meta-test" + assert "retained_at" in meta + # Custom metadata with template resolution + assert meta["project"] == "my-project" + assert meta["session"] == "sess-meta-test" + + def test_full_session_uses_session_id_as_document_id(self, monkeypatch, tmp_path): + """In full-session mode, document_id should be the session_id (for upsert).""" + messages = [ + {"role": "user", "content": "first question"}, + {"role": "assistant", "content": "first answer"}, + {"role": "user", "content": "second question"}, + {"role": "assistant", "content": "second answer"}, + ] + transcript = make_transcript_file(tmp_path, messages) + hook_input = make_hook_input(transcript_path=transcript, session_id="sess-full-123") + captured = {} + + def capture(req, timeout=None): + if "/memories" in req.full_url and "/recall" not in req.full_url: + captured["body"] = json.loads(req.data.decode()) + return FakeHTTPResponse({}) + + _run_hook("retain", hook_input, monkeypatch, tmp_path, urlopen_side_effect=capture) + + assert "body" in captured, "retain API was not called" + item = captured["body"]["items"][0] + # document_id should be just the session_id, no timestamp suffix + assert item["document_id"] == "sess-full-123" + # Should contain ALL messages, not just the last turn + assert "first question" in item["content"] + assert "second question" in item["content"] + def test_chunked_retain_skips_below_threshold(self, monkeypatch, tmp_path): - """With retainEveryNTurns=5, first call should be skipped.""" + """With retainEveryNTurns=5 and retainMode=chunked, first call should be skipped.""" (tmp_path / "plugin_root").mkdir(exist_ok=True) (tmp_path / "plugin_data").mkdir(exist_ok=True) settings = { "autoRetain": True, "autoRecall": True, + "retainMode": "chunked", "retainEveryNTurns": 5, "hindsightApiUrl": "http://fake:9077", } @@ -279,6 +358,9 @@ def capture(req, timeout=None): monkeypatch.delenv(k, raising=False) monkeypatch.setenv("CLAUDE_PLUGIN_ROOT", str(tmp_path / "plugin_root")) monkeypatch.setenv("CLAUDE_PLUGIN_DATA", str(tmp_path / "plugin_data")) + monkeypatch.setenv("HINDSIGHT_RETAIN_MODE", "chunked") + # Ensure retainEveryNTurns isn't overridden by user config (~/.hindsight/claude-code.json) + monkeypatch.setattr("lib.config.USER_CONFIG_PATH", str(tmp_path / "nonexistent_user_config.json")) stdin_data = io.StringIO(json.dumps(hook_input)) scripts_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "scripts")) From d1b96c5e736836c8ea0ade591774c8bdbad8af89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 26 Mar 2026 11:29:47 +0100 Subject: [PATCH 2/3] fix(claude-code): respect retainEveryNTurns in full-session mode The turn-count gating was only applied in chunked mode, meaning full-session mode would re-ingest the entire transcript on every single Stop event. Now retainEveryNTurns gates both modes. Also fix test isolation: resolve ~/.hindsight/claude-code.json at call time (not module load) so HOME override in tests works correctly. --- .../claude-code/scripts/lib/config.py | 6 +-- .../claude-code/scripts/retain.py | 7 +-- .../claude-code/tests/test_hooks.py | 49 +++++++++++++++++-- 3 files changed, 52 insertions(+), 10 deletions(-) diff --git a/hindsight-integrations/claude-code/scripts/lib/config.py b/hindsight-integrations/claude-code/scripts/lib/config.py index 0270288570..1978771d59 100644 --- a/hindsight-integrations/claude-code/scripts/lib/config.py +++ b/hindsight-integrations/claude-code/scripts/lib/config.py @@ -104,9 +104,6 @@ def _load_settings_file(path: str, config: dict) -> None: debug_log(config, f"Failed to load {path}: {e}") -USER_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".hindsight", "claude-code.json") - - def load_config() -> dict: """Load plugin configuration from settings.json + env overrides. @@ -129,7 +126,8 @@ def load_config() -> dict: _load_settings_file(os.path.join(plugin_root, "settings.json"), config) # 2. User config — stable, version-independent, matches openclaw convention - _load_settings_file(USER_CONFIG_PATH, config) + user_config_path = os.path.join(os.path.expanduser("~"), ".hindsight", "claude-code.json") + _load_settings_file(user_config_path, config) # Apply environment variable overrides for env_name, (key, typ) in ENV_OVERRIDES.items(): diff --git a/hindsight-integrations/claude-code/scripts/retain.py b/hindsight-integrations/claude-code/scripts/retain.py index df38576fb1..77819663ce 100755 --- a/hindsight-integrations/claude-code/scripts/retain.py +++ b/hindsight-integrations/claude-code/scripts/retain.py @@ -102,13 +102,15 @@ def main(): retain_full_window = False messages_to_retain = all_messages - if retain_mode == "chunked" and retain_every_n > 1: + # Respect retainEveryNTurns in both modes + if retain_every_n > 1: turn_count = increment_turn_count(session_id) if turn_count % retain_every_n != 0: next_at = ((turn_count // retain_every_n) + 1) * retain_every_n debug_log(config, f"Turn {turn_count}/{retain_every_n}, skipping retain (next at turn {next_at})") return + if retain_mode == "chunked" and retain_every_n > 1: # Sliding window: N turns + configured overlap overlap_turns = config.get("retainOverlapTurns", 0) window_turns = retain_every_n + overlap_turns @@ -116,8 +118,7 @@ def main(): retain_full_window = True debug_log( config, - f"Turn {turn_count}: chunked retain firing " - f"(window: {window_turns} turns, {len(messages_to_retain)} messages)", + f"Chunked retain firing (window: {window_turns} turns, {len(messages_to_retain)} messages)", ) else: # Full session mode: retain all messages, always as full window diff --git a/hindsight-integrations/claude-code/tests/test_hooks.py b/hindsight-integrations/claude-code/tests/test_hooks.py index 848d49e601..760b068143 100644 --- a/hindsight-integrations/claude-code/tests/test_hooks.py +++ b/hindsight-integrations/claude-code/tests/test_hooks.py @@ -33,10 +33,11 @@ def _run_hook(module_name, hook_input, monkeypatch, tmp_path, urlopen_side_effec (tmp_path / "plugin_root").mkdir(exist_ok=True) (tmp_path / "plugin_data").mkdir(exist_ok=True) - # Strip real HINDSIGHT_* env vars + # Strip real HINDSIGHT_* env vars and neutralize user config (~/.hindsight/claude-code.json) for k in list(os.environ): if k.startswith("HINDSIGHT_"): monkeypatch.delenv(k, raising=False) + monkeypatch.setenv("HOME", str(tmp_path)) for k, v in (extra_env or {}).items(): monkeypatch.setenv(k, v) @@ -329,6 +330,49 @@ def capture(req, timeout=None): assert "first question" in item["content"] assert "second question" in item["content"] + def test_full_session_respects_retain_every_n_turns(self, monkeypatch, tmp_path): + """In full-session mode, retainEveryNTurns should still gate when retain fires.""" + messages = [{"role": "user", "content": "hello"}, {"role": "assistant", "content": "world"}] + transcript = make_transcript_file(tmp_path, messages) + hook_input = make_hook_input(transcript_path=transcript, session_id="sess-throttle") + captured = {} + + def capture(req, timeout=None): + if "/memories" in req.full_url and "/recall" not in req.full_url: + captured["called"] = True + captured["body"] = json.loads(req.data.decode()) + return FakeHTTPResponse({}) + + # retainEveryNTurns=3 in full-session mode — first 2 calls should be skipped + _run_hook( + "retain", hook_input, monkeypatch, tmp_path, + urlopen_side_effect=capture, + extra_settings={"retainEveryNTurns": 3}, + ) + # Turn 1 of 3 — should NOT retain + assert "called" not in captured + + # Turn 2 — still skip + captured.clear() + _run_hook( + "retain", hook_input, monkeypatch, tmp_path, + urlopen_side_effect=capture, + extra_settings={"retainEveryNTurns": 3}, + ) + assert "called" not in captured + + # Turn 3 — should fire, with full session content and session_id as doc ID + captured.clear() + _run_hook( + "retain", hook_input, monkeypatch, tmp_path, + urlopen_side_effect=capture, + extra_settings={"retainEveryNTurns": 3}, + ) + assert "called" in captured, "retain should fire on turn 3" + item = captured["body"]["items"][0] + assert item["document_id"] == "sess-throttle" # full-session uses session_id + assert "hello" in item["content"] + def test_chunked_retain_skips_below_threshold(self, monkeypatch, tmp_path): """With retainEveryNTurns=5 and retainMode=chunked, first call should be skipped.""" (tmp_path / "plugin_root").mkdir(exist_ok=True) @@ -359,8 +403,7 @@ def capture(req, timeout=None): monkeypatch.setenv("CLAUDE_PLUGIN_ROOT", str(tmp_path / "plugin_root")) monkeypatch.setenv("CLAUDE_PLUGIN_DATA", str(tmp_path / "plugin_data")) monkeypatch.setenv("HINDSIGHT_RETAIN_MODE", "chunked") - # Ensure retainEveryNTurns isn't overridden by user config (~/.hindsight/claude-code.json) - monkeypatch.setattr("lib.config.USER_CONFIG_PATH", str(tmp_path / "nonexistent_user_config.json")) + monkeypatch.setenv("HOME", str(tmp_path)) stdin_data = io.StringIO(json.dumps(hook_input)) scripts_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "scripts")) From 87dc621631f9635e1730f72b6e048cd100266953 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 26 Mar 2026 11:36:33 +0100 Subject: [PATCH 3/3] fix(claude-code): fix config tests after USER_CONFIG_PATH removal Update tests to use HOME env var override instead of monkeypatching the removed USER_CONFIG_PATH constant. Add autouse fixture to TestLoadConfig to isolate all config tests from real user config and HINDSIGHT_* env vars. --- .../claude-code/tests/test_config.py | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/hindsight-integrations/claude-code/tests/test_config.py b/hindsight-integrations/claude-code/tests/test_config.py index fa3fd04878..368fd3a796 100644 --- a/hindsight-integrations/claude-code/tests/test_config.py +++ b/hindsight-integrations/claude-code/tests/test_config.py @@ -28,6 +28,14 @@ def test_str_passthrough(self): class TestLoadConfig: + @pytest.fixture(autouse=True) + def _isolate_config(self, tmp_path, monkeypatch): + """Isolate from real user config and env vars.""" + monkeypatch.setenv("HOME", str(tmp_path)) + for k in list(os.environ): + if k.startswith("HINDSIGHT_"): + monkeypatch.delenv(k, raising=False) + def test_defaults_applied_when_no_settings_file(self, tmp_path, monkeypatch): monkeypatch.setenv("CLAUDE_PLUGIN_ROOT", str(tmp_path)) # No settings.json in tmp_path @@ -95,27 +103,26 @@ def test_user_config_overrides_plugin_settings(self, tmp_path, monkeypatch): user_cfg.write_text(json.dumps({"recallBudget": "high"})) monkeypatch.setenv("CLAUDE_PLUGIN_ROOT", str(plugin_root)) - import lib.config as cfg_mod - monkeypatch.setattr(cfg_mod, "USER_CONFIG_PATH", str(user_cfg)) + monkeypatch.setenv("HOME", str(tmp_path)) cfg = load_config() assert cfg["recallBudget"] == "high" def test_user_config_missing_falls_back_gracefully(self, tmp_path, monkeypatch): monkeypatch.setenv("CLAUDE_PLUGIN_ROOT", str(tmp_path)) - import lib.config as cfg_mod - monkeypatch.setattr(cfg_mod, "USER_CONFIG_PATH", str(tmp_path / "nonexistent.json")) + # HOME points to tmp_path where no .hindsight/claude-code.json exists + monkeypatch.setenv("HOME", str(tmp_path)) cfg = load_config() assert cfg["recallBudget"] == "mid" # default def test_env_var_wins_over_user_config(self, tmp_path, monkeypatch): plugin_root = tmp_path / "plugin" plugin_root.mkdir() - user_cfg = tmp_path / "claude-code.json" - user_cfg.write_text(json.dumps({"recallBudget": "low"})) + user_cfg_dir = tmp_path / ".hindsight" + user_cfg_dir.mkdir() + (user_cfg_dir / "claude-code.json").write_text(json.dumps({"recallBudget": "low"})) monkeypatch.setenv("CLAUDE_PLUGIN_ROOT", str(plugin_root)) + monkeypatch.setenv("HOME", str(tmp_path)) monkeypatch.setenv("HINDSIGHT_RECALL_BUDGET", "high") - import lib.config as cfg_mod - monkeypatch.setattr(cfg_mod, "USER_CONFIG_PATH", str(user_cfg)) cfg = load_config() assert cfg["recallBudget"] == "high"